Log files: find the history by date, count from the first poll

A file turned on was read from its start to find its last hour: with a
Traefik access log that holds months of lines (never rotated by
Traefik), LogStream spent minutes on old lines that were all left out,
and new lines waited behind them. Entries sent during the first poll
were also left out of the count shown in Settings.

- start of the FILES_BACKFILL history found by bisection on the dates
  of the lines; files without dates: only new lines
- up to 16 reads of 4 MB per file and per poll
- status "behind" (bytes left to read), shown as "Reading the history:
  N MB left"
- the status of a file exists before its first entries are counted

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
cedricandClaude Opus 5.5 committed 2026-10-05 12:42:09 +02:00
1 parent 194edee6de
commit 1a3b3ee23a
5 files changed
+140 -16

No files matched your search

+6 -1
View File
@@ -101,6 +101,7 @@ const I18N = {
fileDraft: 'Fill in the name and the path to save this file.',
fileOk: (r) => `Followed: ${r} entries since startup.`,
fileDetected: (f) => ` Format detected: ${f}.`,
fileBehind: (mb) => ` Reading the history: ${mb} MB left.`,
probeFound: (f) => `File found, format detected: ${f}.`,
probeSample: 'Last line: ',
file_empty: 'The file is empty for now: its format will be detected from its first lines.',
@@ -332,6 +333,7 @@ const I18N = {
fileDraft: 'Renseignez le nom et le chemin pour enregistrer ce fichier.',
fileOk: (r) => `Suivi : ${r} entrées depuis le démarrage.`,
fileDetected: (f) => ` Format détecté : ${f}.`,
fileBehind: (mb) => ` Lecture de l\'historique : ${mb} Mo restants.`,
probeFound: (f) => `Fichier trouvé, format détecté : ${f}.`,
probeSample: 'Dernière ligne : ',
file_empty: 'Le fichier est vide pour l\'instant : son format sera détecté à partir de ses premières lignes.',
@@ -2279,7 +2281,10 @@ function fileStatus(s) {
if (s.draft) return [t('fileDraft'), 'muted'];
if (!s.enabled) return [t('fileOff'), 'muted'];
if (s.code) return [t('file_' + s.code) + (s.code === 'not_found' ? '' : s.error || ''), 'bad'];
if (s.open) return [t('fileOk', fmtNum(s.read)) + (s.format === 'auto' && s.detected ? t('fileDetected', t('fmt_' + s.detected)) : ''), 'good'];
if (s.open) {
return [t('fileOk', fmtNum(s.read)) + (s.format === 'auto' && s.detected ? t('fileDetected', t('fmt_' + s.detected)) : '')
+ (s.behind > 1 << 20 ? t('fileBehind', fmtNum(Math.round(s.behind / (1 << 20)))) : ''), 'good'];
}
return [t('hostWaiting'), 'muted'];
}