Disk buffer for batches VictoriaLogs cannot take, lossless Docker positions, non-blocking reverse DNS

- Batches that fail go to /data/spool (SPOOL_MAX_MB, 1 GiB by default) and
  are sent again oldest first; retries no longer block the store loop and
  follow the shutdown context.
- Docker and host logs wait for room in a full queue instead of being
  dropped; the Docker position only moves once a line is stored or spooled.
- Reverse DNS no longer holds up the syslog listeners, with an LRU cache
  and a cap on concurrent lookups.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
cedricandClaude Opus 5.5 committed 2026-10-03 16:26:18 +02:00
1 parent 3466a29692
commit 3504263992
14 files changed
+778 -84

No files matched your search

+11 -6
View File
@@ -61,8 +61,8 @@ type DockerManager struct {
cfg dockerConfig
hostName string
containers []DockerContainer
followers map[string]*follower // container ID -> running follower
checkpoint map[string]time.Time // container ID -> timestamp of the last line read
followers map[string]*follower // container ID -> running follower
checkpoint map[string]time.Time // container ID -> timestamp of the last line stored
dirty bool
connected bool
lastErr string
@@ -389,16 +389,19 @@ func (m *DockerManager) follow(ctx context.Context, c DockerContainer) {
}
}
func sleepCtx(ctx context.Context, d time.Duration) {
// sleepCtx waits for d, or returns false if the context ends first.
func sleepCtx(ctx context.Context, d time.Duration) bool {
t := time.NewTimer(d)
defer t.Stop()
select {
case <-ctx.Done():
return false
case <-t.C:
return true
}
}
// since returns where to resume reading: just after the last line read, or
// since returns where to resume reading: just after the last line stored, or
// DOCKER_BACKFILL ago for a container seen for the first time.
func (m *DockerManager) since(id string) time.Time {
m.mu.Lock()
@@ -530,8 +533,6 @@ func (m *DockerManager) emit(c DockerContainer, stream string, line []byte) {
} else if t, err := time.Parse(time.RFC3339Nano, s); err == nil {
ts, s = t, ""
}
defer m.setCheckpoint(c.ID, ts)
s = ansiRe.ReplaceAllString(s, "")
if !utf8.ValidString(s) {
s = strings.ToValidUTF8(s, string(utf8.RuneError))
@@ -575,6 +576,10 @@ func (m *DockerManager) emit(c DockerContainer, stream string, line []byte) {
"compose_service": c.Service,
"stream": stream,
},
// Docker keeps the logs: wait for room in the queue rather than drop the
// line, and resume after it only once it is stored.
Wait: true,
Done: func() { m.setCheckpoint(c.ID, ts) },
})
}