Disk buffer for batches VictoriaLogs cannot take, lossless Docker positions, non-blocking reverse DNS
- Batches that fail go to /data/spool (SPOOL_MAX_MB, 1 GiB by default) and are sent again oldest first; retries no longer block the store loop and follow the shutdown context. - Docker and host logs wait for room in a full queue instead of being dropped; the Docker position only moves once a line is stored or spooled. - Reverse DNS no longer holds up the syslog listeners, with an LRU cache and a cap on concurrent lookups. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
3466a29692
commit
3504263992
14 files changed
+778
-84
No files matched your search
@@ -61,8 +61,8 @@ type DockerManager struct {
|
||||
cfg dockerConfig
|
||||
hostName string
|
||||
containers []DockerContainer
|
||||
followers map[string]*follower // container ID -> running follower
|
||||
checkpoint map[string]time.Time // container ID -> timestamp of the last line read
|
||||
followers map[string]*follower // container ID -> running follower
|
||||
checkpoint map[string]time.Time // container ID -> timestamp of the last line stored
|
||||
dirty bool
|
||||
connected bool
|
||||
lastErr string
|
||||
@@ -389,16 +389,19 @@ func (m *DockerManager) follow(ctx context.Context, c DockerContainer) {
|
||||
}
|
||||
}
|
||||
|
||||
func sleepCtx(ctx context.Context, d time.Duration) {
|
||||
// sleepCtx waits for d, or returns false if the context ends first.
|
||||
func sleepCtx(ctx context.Context, d time.Duration) bool {
|
||||
t := time.NewTimer(d)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-t.C:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// since returns where to resume reading: just after the last line read, or
|
||||
// since returns where to resume reading: just after the last line stored, or
|
||||
// DOCKER_BACKFILL ago for a container seen for the first time.
|
||||
func (m *DockerManager) since(id string) time.Time {
|
||||
m.mu.Lock()
|
||||
@@ -530,8 +533,6 @@ func (m *DockerManager) emit(c DockerContainer, stream string, line []byte) {
|
||||
} else if t, err := time.Parse(time.RFC3339Nano, s); err == nil {
|
||||
ts, s = t, ""
|
||||
}
|
||||
defer m.setCheckpoint(c.ID, ts)
|
||||
|
||||
s = ansiRe.ReplaceAllString(s, "")
|
||||
if !utf8.ValidString(s) {
|
||||
s = strings.ToValidUTF8(s, string(utf8.RuneError))
|
||||
@@ -575,6 +576,10 @@ func (m *DockerManager) emit(c DockerContainer, stream string, line []byte) {
|
||||
"compose_service": c.Service,
|
||||
"stream": stream,
|
||||
},
|
||||
// Docker keeps the logs: wait for room in the queue rather than drop the
|
||||
// line, and resume after it only once it is stored.
|
||||
Wait: true,
|
||||
Done: func() { m.setCheckpoint(c.ID, ts) },
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user