Log files source: follow the logs of Traefik, nginx and other programs

Files source (source_type=file), declared in Settings > Log sources > Log
files (/data/files.json, 50 at most): name (= app), path as mounted in the
container (file or glob, rotated copies .1/.gz left out, 200 files per
glob), format, optional host, switch.

- formats: plain text (level detected, leading timestamp read and removed),
  JSON lines (msg/time/level keys, numeric pino levels, other keys as
  fields, nested ones dotted), logfmt, access log CLF (nginx, Apache,
  Traefik CLF with router/backend/duration), Traefik access log in JSON
  (summary "GET host/path 200 · 12.5 ms · router · client"), regular
  expression with named groups time/level/msg/host/app/status; 5xx = err,
  4xx = warning; fields named like LogStream's own renamed field_<name>
- indented lines (text) and non-matching lines (regex) added to the
  previous entry, which waits 2 s for them
- inode + offset per file in /data/files-state.json, rotation (end of
  .1 read first), copytruncate, file renamed within the glob; last
  FILES_BACKFILL (1h) of dated lines read when turned on, new files read
  from their start
- status per file (files followed, entries read, not found / permission)

UI: rows with switch, name, path, format, host (regex field when needed),
status line; green edge, tint and FILE badge in the list, "Files" in the
source filter, HTTP fields first in the details, FR/EN. docker-compose,
.env.example and README (EN/FR): mounts, formats, FILES_BACKFILL.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
cedricandClaude Opus 5.5 committed 2026-10-05 10:27:54 +02:00
1 parent e87186e476
commit 087d9e7b30
14 files changed
+1934 -9

No files matched your search

+24
View File
@@ -25,6 +25,7 @@ type API struct {
host *HostLogs
dmesg *Dmesg
audit *Audit
files *Files
selfLog *SelfLog
dbstats dbStatsCache
}
@@ -56,6 +57,8 @@ func (a *API) Routes(mux *http.ServeMux) {
mux.HandleFunc("PUT /api/dmesg", a.dmesgConfigure)
mux.HandleFunc("GET /api/audit", a.auditStatus)
mux.HandleFunc("PUT /api/audit", a.auditConfigure)
mux.HandleFunc("GET /api/files", a.filesStatus)
mux.HandleFunc("PUT /api/files", a.filesConfigure)
mux.HandleFunc("GET /api/selflog", a.selfLogStatus)
mux.HandleFunc("PUT /api/selflog", a.selfLogConfigure)
}
@@ -80,6 +83,27 @@ func (a *API) auditConfigure(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, a.audit.Status())
}
// GET /api/files: the log files of the "Files" source (Settings > Sources).
func (a *API) filesStatus(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, a.files.Status())
}
// PUT /api/files {"sources": [{"id", "name", "path", "format", "regex", "host", "enabled"}]}
// (no id: a new file).
func (a *API) filesConfigure(w http.ResponseWriter, r *http.Request) {
var cfg filesConfig
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10)).Decode(&cfg); err != nil {
writeErr(w, http.StatusBadRequest, err)
return
}
if err := a.files.Configure(cfg); err != nil {
writeErr(w, http.StatusBadRequest, err)
return
}
log.Printf("file sources changed from %s: %d files", r.RemoteAddr, len(cfg.Sources))
writeJSON(w, http.StatusOK, a.files.Status())
}
// GET /api/selflog: state of the LogStream internal logs (Settings > Sources).
func (a *API) selfLogStatus(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, a.selfLog.Status())