diff --git a/README.fr.md b/README.fr.md index 0424019..9022abc 100644 --- a/README.fr.md +++ b/README.fr.md @@ -261,7 +261,11 @@ couleur, est mémorisé par navigateur. européen de polices respectueux de la vie privée ; sans accès à internet, la police du système est utilisée. Les ligatures sont désactivées pour que `->` ou `!=` s'affichent tels quels. Le réglage le plus dense est Très petite + Compacte + Inconsolata Condensed. -- **Données** : « Supprimer tous les logs » efface définitivement tous les logs stockés (il faut +- **Données** : chiffres de la base lus dans VictoriaLogs (actualisés au plus toutes les 30 s) : + lignes stockées, taille sur disque (index compris), taille brute et taux de compression, + période couverte avec la rétention, lignes des dernières 24 h et de la dernière heure, hôtes + et applications distincts, espace disque libre. Les tailles sont en Ko/Mo/Go (KB/MB/GB en + anglais). « Supprimer tous les logs » efface définitivement tous les logs stockés (il faut taper `PURGE` pour confirmer). Les tags et les paramètres sont conservés. VictoriaLogs doit être lancé avec `-delete.enable` (déjà présent dans `docker-compose.yml`). La fonction est désactivée par défaut : mettez `ALLOW_PURGE=true` pour l'autoriser. Tout administrateur peut diff --git a/README.md b/README.md index 3e2063b..c9ba357 100644 --- a/README.md +++ b/README.md @@ -237,7 +237,10 @@ remembered per browser. [Bunny Fonts](https://fonts.bunny.net), a privacy-friendly European font service; without internet access, the system font is used. Ligatures are disabled so `->` or `!=` show as typed. The densest setting is Tiny + Compact + Inconsolata Condensed. -- **Data**: "Delete all logs" permanently erases every stored log (you must type +- **Data**: database figures read from VictoriaLogs (refreshed at most every 30 s): stored + lines, size on disk (index included), raw size and compression ratio, period covered with + the retention, lines of the last 24 h and last hour, distinct hosts and apps, free disk + space. Sizes use KB/MB/GB (Ko/Mo/Go in French). "Delete all logs" permanently erases every stored log (you must type `PURGE` to confirm). Tags and settings are kept. VictoriaLogs needs `-delete.enable` (already set in `docker-compose.yml`). The feature is off by default: set `ALLOW_PURGE=true` to allow it. Any admin can then purge: turn on [authentication](#authentication) if the UI diff --git a/api.go b/api.go index bd48da6..e034298 100644 --- a/api.go +++ b/api.go @@ -23,6 +23,7 @@ type API struct { docker *DockerManager // nil when DOCKER_LOGS is off syslog *SyslogServer host *HostLogs + dbstats dbStatsCache } func (a *API) Routes(mux *http.ServeMux) { @@ -32,6 +33,7 @@ func (a *API) Routes(mux *http.ServeMux) { mux.HandleFunc("GET /api/facets", a.facets) mux.HandleFunc("GET /api/stream", a.stream) mux.HandleFunc("GET /api/stats", a.stats) + mux.HandleFunc("GET /api/dbstats", a.dbStats) mux.HandleFunc("GET /api/tags", a.listTags) mux.HandleFunc("POST /api/tags", a.createTag) mux.HandleFunc("POST /api/tags/reset", a.resetTags) @@ -314,6 +316,17 @@ func (a *API) stats(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, s) } +// GET /api/dbstats[?refresh=1]: size and content of the VictoriaLogs base +// (Settings > Data), cached for 30 s. +func (a *API) dbStats(w http.ResponseWriter, r *http.Request) { + st, err := a.dbstats.get(r.Context(), a.store, r.URL.Query().Get("refresh") == "1") + if err != nil { + writeErr(w, http.StatusBadGateway, err) + return + } + writeJSON(w, http.StatusOK, st) +} + func (a *API) listTags(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, a.tags.List()) } diff --git a/dbstats.go b/dbstats.go new file mode 100644 index 0000000..27e288d --- /dev/null +++ b/dbstats.go @@ -0,0 +1,167 @@ +package main + +import ( + "bufio" + "context" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "sync" + "time" +) + +// DBStats describes what VictoriaLogs holds (Settings > Data). Sizes are in +// bytes; a field VictoriaLogs did not give stays at its zero value. +type DBStats struct { + Rows int64 `json:"rows"` // stored log lines + DiskBytes int64 `json:"diskBytes"` // compressed data + index, on disk + IndexBytes int64 `json:"indexBytes"` // index part of DiskBytes + RawBytes int64 `json:"rawBytes"` // data before compression + FreeBytes int64 `json:"freeBytes"` // free space on the VictoriaLogs volume + Days int64 `json:"days"` // per-day partitions + Retention string `json:"retention"` // -retentionPeriod, e.g. "30d" + Oldest string `json:"oldest,omitempty"` + Newest string `json:"newest,omitempty"` + Last1h int64 `json:"last1h"` + Last24h int64 `json:"last24h"` + Hosts int64 `json:"hosts"` + Apps int64 `json:"apps"` + QueryError string `json:"queryError,omitempty"` // the LogsQL part failed, the metrics are still valid + Updated string `json:"updated"` +} + +// dbStatsCache keeps the last answer for a while: the LogsQL part reads the +// whole base, which is too heavy to repeat each time the tab is opened. +type dbStatsCache struct { + mu sync.Mutex + at time.Time + last *DBStats +} + +const dbStatsTTL = 30 * time.Second + +func (c *dbStatsCache) get(ctx context.Context, s *Store, force bool) (*DBStats, error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.last != nil && !force && time.Since(c.at) < dbStatsTTL { + return c.last, nil + } + st, err := s.DBStats(ctx) + if err != nil { + return nil, err + } + c.last, c.at = st, time.Now() + return st, nil +} + +// DBStats reads the storage metrics of VictoriaLogs (/metrics), then counts +// with LogsQL what the metrics do not tell (period, recent volume, sources). +func (s *Store) DBStats(ctx context.Context) (*DBStats, error) { + st, err := s.storageMetrics(ctx) + if err != nil { + return nil, err + } + st.Updated = time.Now().UTC().Format(time.RFC3339) + if st.Rows == 0 { + return st, nil + } + + qctx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + rows, err := s.Query(qctx, "* | stats min(_time) oldest, max(_time) newest, count_uniq(host) hosts, count_uniq(app) apps") + if err == nil && len(rows) > 0 { + r := rows[0] + st.Oldest, _ = r["oldest"].(string) + st.Newest, _ = r["newest"].(string) + st.Hosts, st.Apps = toInt(r["hosts"]), toInt(r["apps"]) + rows, err = s.Query(qctx, "_time:24h | stats count() last24h, count() if (_time:1h) last1h") + if err == nil && len(rows) > 0 { + st.Last24h, st.Last1h = toInt(rows[0]["last24h"]), toInt(rows[0]["last1h"]) + } + } + if err != nil { + st.QueryError = err.Error() + } + return st, nil +} + +func (s *Store) storageMetrics(ctx context.Context) (*DBStats, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.base+"/metrics", nil) + if err != nil { + return nil, err + } + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + msg, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + return nil, fmt.Errorf("VictoriaLogs /metrics: HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(msg))) + } + return parseStorageMetrics(resp.Body) +} + +// parseStorageMetrics picks the storage figures out of the Prometheus text +// format of VictoriaLogs' /metrics (app/vlstorage). +func parseStorageMetrics(r io.Reader) (*DBStats, error) { + st := &DBStats{} + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 64*1024), 1<<20) + for sc.Scan() { + line := sc.Text() + if line == "" || line[0] == '#' { + continue + } + name, labels, value := splitMetric(line) + switch name { + case "vl_storage_rows": + st.Rows += int64(value) + case "vl_data_size_bytes": + st.DiskBytes += int64(value) + if labels["type"] == "indexdb" { + st.IndexBytes += int64(value) + } + case "vl_uncompressed_data_size_bytes": + st.RawBytes += int64(value) + case "vl_free_disk_space_bytes": + st.FreeBytes = int64(value) + case "vl_partitions": + st.Days = int64(value) + case "flag": + if labels["name"] == "retentionPeriod" { + st.Retention = labels["value"] + } + } + } + return st, sc.Err() +} + +// splitMetric splits `name{a="x",b="y"} 12` into its parts. Label values with +// escaped quotes are not expected in the metrics read here. +func splitMetric(line string) (string, map[string]string, float64) { + sp := strings.LastIndexByte(line, ' ') + if sp < 0 { + return "", nil, 0 + } + value, _ := strconv.ParseFloat(line[sp+1:], 64) + head := line[:sp] + name, rest, ok := strings.Cut(head, "{") + if !ok { + return head, nil, value + } + labels := map[string]string{} + rest = strings.TrimSuffix(rest, "}") + for rest != "" { + k, v, ok := strings.Cut(rest, `="`) + if !ok { + break + } + val, after, _ := strings.Cut(v, `"`) + labels[strings.TrimSpace(k)] = val + rest = strings.TrimPrefix(after, ",") + } + return name, labels, value +} diff --git a/dbstats_test.go b/dbstats_test.go new file mode 100644 index 0000000..5d6d6b5 --- /dev/null +++ b/dbstats_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// Excerpt of VictoriaLogs v1.52 /metrics. +const vlMetrics = `# a comment +vl_free_disk_space_bytes{path="/vlogs"} 52428800000 +vl_storage_rows{type="storage/inmemory"} 120 +vl_storage_rows{type="storage/small"} 3000 +vl_storage_rows{type="storage/big"} 1000000 +vl_partitions 12 +vl_data_size_bytes{type="indexdb"} 2048 +vl_data_size_bytes{type="storage"} 1048576 +vl_compressed_data_size_bytes{type="storage/big"} 1000000 +vl_uncompressed_data_size_bytes{type="storage/inmemory"} 4096 +vl_uncompressed_data_size_bytes{type="storage/small"} 100000 +vl_uncompressed_data_size_bytes{type="storage/big"} 20000000 +flag{name="retentionPeriod", value="30d", is_set="true"} 1 +flag{name="httpListenAddr", value=":9428", is_set="true"} 1 +` + +func TestParseStorageMetrics(t *testing.T) { + st, err := parseStorageMetrics(strings.NewReader(vlMetrics)) + if err != nil { + t.Fatal(err) + } + want := DBStats{Rows: 1003120, DiskBytes: 1050624, IndexBytes: 2048, RawBytes: 20104096, FreeBytes: 52428800000, Days: 12, Retention: "30d"} + if *st != want { + t.Fatalf("got %+v\nwant %+v", *st, want) + } +} + +// statsVL answers /metrics and the LogsQL queries of DBStats. +type statsVL struct{ queries int } + +func (v *statsVL) RoundTrip(r *http.Request) (*http.Response, error) { + body := vlMetrics + if r.URL.Path == "/select/logsql/query" { + v.queries++ + _ = r.ParseForm() + if strings.HasPrefix(r.PostForm.Get("query"), "_time:24h") { + body = `{"last24h":"5000","last1h":"300"}` + "\n" + } else { + body = `{"oldest":"2026-09-21T08:00:00Z","newest":"2026-10-03T15:00:00Z","hosts":"4","apps":"17"}` + "\n" + } + } + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(body)), Header: http.Header{}}, nil +} + +func TestDBStatsHandler(t *testing.T) { + vl := &statsVL{} + store := NewStore("http://vl", 10, 10, time.Second, nil) + store.client.Transport = vl + store.streamClient.Transport = vl + a := &API{store: store} + + get := func(url string) DBStats { + rec := httptest.NewRecorder() + a.dbStats(rec, httptest.NewRequest("GET", url, nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status %d: %s", rec.Code, rec.Body) + } + var st DBStats + if err := json.Unmarshal(rec.Body.Bytes(), &st); err != nil { + t.Fatal(err) + } + return st + } + st := get("/api/dbstats") + if st.Rows != 1003120 || st.Oldest != "2026-09-21T08:00:00Z" || st.Hosts != 4 || st.Apps != 17 || st.Last24h != 5000 || st.Last1h != 300 || st.QueryError != "" { + t.Fatalf("unexpected stats: %+v", st) + } + get("/api/dbstats") + if vl.queries != 2 { + t.Fatalf("second call within the TTL should be cached, got %d queries", vl.queries) + } + get("/api/dbstats?refresh=1") + if vl.queries != 4 { + t.Fatalf("refresh=1 should query again, got %d queries", vl.queries) + } +} diff --git a/web/app.js b/web/app.js index 6b2cecf..704b0ff 100644 --- a/web/app.js +++ b/web/app.js @@ -126,6 +126,16 @@ const I18N = { fontHelp: 'Free fonts. The built-in ones (Inconsolata Condensed, the narrowest, Iosevka and Ubuntu Mono) are served by LogStream itself and work offline; they are narrow, so more text fits on each line. The others are loaded by your browser from Bunny Fonts, a privacy-friendly European font service; without internet access, the system font is used.', previewMsgs: ['error: link down on eth1, carrier lost', 'warning: disk /dev/sda temperature 58°C', 'backup finished: ok (12.4 GB in 00:14:32)'], dangerZone: 'Danger zone', + dbTitle: 'Database', dbRefresh: 'Refresh', dbLoading: 'Loading…', + dbUnits: ['B', 'KB', 'MB', 'GB', 'TB'], + dbRows: 'Stored lines', dbDisk: 'Size on disk', dbIndex: (x) => `including ${x} of index`, + dbRaw: 'Raw size', dbRatio: (x) => `compressed ×${x}`, + dbPeriod: 'Period', dbFromTo: (p) => `${p.from} → ${p.to}`, dbDays: (n) => `${n} day${n > 1 ? 's' : ''}`, + dbRetention: (r) => `retention ${r}`, + dbRecent: 'Last 24 h', dbLastHour: (n) => `${n} in the last hour`, + dbSources: 'Sources', dbHostsApps: (p) => `${p.h} host${p.hn > 1 ? 's' : ''} · ${p.a} app${p.an > 1 ? 's' : ''}`, + dbFree: 'Free disk space', dbEmpty: 'The database is empty.', + dbQueryErr: 'Some figures could not be computed: ', purgeHelp: 'Permanently delete every stored log to start from scratch. Color tags and settings are kept.', purgeBtn: 'Delete all logs…', purgePrompt: 'This permanently deletes ALL stored logs.\n\nType PURGE to confirm:', @@ -285,6 +295,16 @@ const I18N = { fontHelp: 'Polices libres. Les polices intégrées (Inconsolata Condensed, la plus étroite, Iosevka et Ubuntu Mono) sont servies par LogStream lui-même et fonctionnent hors ligne ; elles sont étroites, donc chaque ligne affiche plus de texte. Les autres sont chargées par votre navigateur depuis Bunny Fonts, un service européen respectueux de la vie privée ; sans accès à internet, la police du système est utilisée.', previewMsgs: ['error: link down on eth1, carrier lost', 'warning: disk /dev/sda temperature 58°C', 'backup finished: ok (12.4 GB in 00:14:32)'], dangerZone: 'Zone de danger', + dbTitle: 'Base de données', dbRefresh: 'Actualiser', dbLoading: 'Chargement…', + dbUnits: ['o', 'Ko', 'Mo', 'Go', 'To'], + dbRows: 'Lignes stockées', dbDisk: 'Taille sur disque', dbIndex: (x) => `dont ${x} d'index`, + dbRaw: 'Taille brute', dbRatio: (x) => `compression ×${x}`, + dbPeriod: 'Période', dbFromTo: (p) => `${p.from} → ${p.to}`, dbDays: (n) => `${n} jour${n > 1 ? 's' : ''}`, + dbRetention: (r) => `rétention ${r}`, + dbRecent: 'Dernières 24 h', dbLastHour: (n) => `${n} sur la dernière heure`, + dbSources: 'Sources', dbHostsApps: (p) => `${p.h} hôte${p.hn > 1 ? 's' : ''} · ${p.a} application${p.an > 1 ? 's' : ''}`, + dbFree: 'Espace disque libre', dbEmpty: 'La base est vide.', + dbQueryErr: 'Certains chiffres n\'ont pas pu être calculés : ', purgeHelp: 'Supprime définitivement tous les logs stockés pour repartir de zéro. Les tags de couleur et les réglages sont conservés.', purgeBtn: 'Supprimer tous les logs…', purgePrompt: 'Cette action supprime définitivement TOUS les logs stockés.\n\nTapez PURGE pour confirmer :', @@ -509,6 +529,7 @@ function setLang(next) { renderPresets(); renderTimeSettings(); renderPurge(); + renderDbStats(); renderInterface(); renderSyslog(); renderHost(); @@ -2010,6 +2031,69 @@ setInterval(() => { } }, 4000); +/* ================= Database statistics ================= */ + +// 1536 -> "1,5 Ko" / "1.5 KB" (binary multiples, the usual reading for disk sizes). +function fmtBytes(n) { + const units = t('dbUnits'); + let i = 0; + n = Number(n || 0); + while (n >= 1024 && i < units.length - 1) { n /= 1024; i++; } + const digits = i === 0 || n >= 100 ? 0 : 1; + return `${n.toLocaleString(t('locale'), { maximumFractionDigits: digits })} ${units[i]}`; +} + +const db = { stats: null, error: null, loading: false }; + +function renderDbStats() { + const el = $('#dbStats'); + const s = db.stats; + if (!s) { + el.innerHTML = `
${esc(db.error || t('dbLoading'))}
`; + return; + } + const row = (k, v, sub) => `
${esc(k)}
${esc(v)}${sub ? ` ${esc(sub)}` : ''}
`; + const day = (x) => fmtFull(x).replace(/[.,]\d{3}(?=\D*$)/, ''); + let html = row(t('dbRows'), fmtNum(s.rows)); + html += row(t('dbDisk'), fmtBytes(s.diskBytes), s.indexBytes ? t('dbIndex', fmtBytes(s.indexBytes)) : ''); + if (s.rawBytes) { + const ratio = s.diskBytes ? (s.rawBytes / s.diskBytes).toLocaleString(t('locale'), { maximumFractionDigits: 1 }) : ''; + html += row(t('dbRaw'), fmtBytes(s.rawBytes), ratio ? t('dbRatio', ratio) : ''); + } + if (s.oldest && s.newest) { + const sub = [s.days ? t('dbDays', s.days) : '', s.retention ? t('dbRetention', s.retention) : ''].filter(Boolean).join(' · '); + html += row(t('dbPeriod'), t('dbFromTo', { from: day(s.oldest), to: day(s.newest) }), sub); + } else if (s.retention) { + html += row(t('dbPeriod'), '—', t('dbRetention', s.retention)); + } + if (s.rows) { + html += row(t('dbRecent'), fmtNum(s.last24h), t('dbLastHour', fmtNum(s.last1h))); + html += row(t('dbSources'), t('dbHostsApps', { h: fmtNum(s.hosts), hn: s.hosts, a: fmtNum(s.apps), an: s.apps })); + } + if (s.freeBytes) html += row(t('dbFree'), fmtBytes(s.freeBytes)); + if (!s.rows) html += `
${esc(t('dbEmpty'))}
`; + if (s.queryError) html += `
${esc(t('dbQueryErr') + s.queryError)}
`; + el.innerHTML = html; +} + +async function loadDbStats(refresh = false) { + if (db.loading) return; + db.loading = true; + $('#dbRefresh').disabled = true; + try { + db.stats = await api('/api/dbstats' + (refresh ? '?refresh=1' : '')); + db.error = null; + } catch (e) { + db.stats = null; + db.error = e.message; + } finally { + db.loading = false; + $('#dbRefresh').disabled = false; + } + renderDbStats(); +} +$('#dbRefresh').addEventListener('click', () => loadDbStats(true)); + /* ================= Purge ================= */ const purge = { allowed: true, running: 0, error: null, code: null, wasRunning: false, timer: null }; @@ -2039,6 +2123,7 @@ async function loadPurgeStatus() { refresh(); loadFacets(); loadStats(); + loadDbStats(true); } purge.running = s.running || 0; } catch { /* shown on the next attempt */ } @@ -2237,6 +2322,8 @@ $('#settingsBtn').addEventListener('click', () => { loadSyslog(); loadHost(); loadDocker(); + renderDbStats(); + loadDbStats(); showSettingsTab(store.get('settingsTab', 'locale')); $('#settingsDlg').showModal(); }); diff --git a/web/index.html b/web/index.html index ce65663..e4c1ec5 100644 --- a/web/index.html +++ b/web/index.html @@ -339,6 +339,11 @@