diff --git a/.env.example b/.env.example index 40286ca..8fa42db 100644 --- a/.env.example +++ b/.env.example @@ -13,3 +13,5 @@ RDNS=on DNS_SERVER= # Allow "Delete all logs" in Settings (true/false) ALLOW_PURGE=true +# Maximum number of rows in a CSV export +EXPORT_MAX=100000 diff --git a/README.md b/README.md index b58221d..ed061a1 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,23 @@ Severity badges `err`/`crit` and `warning` use the colors of the `error` and `wa Shortcuts: `/` focuses the search box, `Esc` clears it. Clicking a row shows all its fields. +## CSV export + +The **Export** button (next to the log count) downloads every stored log matching the +current filters (search, time range, severity, host, app), newest first, up to `EXPORT_MAX` +rows (100,000 by default), not only the rows on screen. Two variants: + +- **CSV**: comma separated, UTF-8. +- **CSV for Excel**: semicolon separated with a UTF-8 BOM, so French Excel opens it directly + with accents. Cells starting with `=`, `+`, `-` or `@` are prefixed with `'` so that a + crafted log message cannot run as a formula. + +Columns: `received`, `message_time` (both as `YYYY-MM-DD HH:MM:SS.mmm` in the time zone +chosen in Settings), `severity`, `facility`, `host`, `host_ip`, `app`, `pid`, `source_ip`, +`proto`, `message`. In LogsQL mode, only the filter part is supported (no `| pipes`). + +From the command line: `curl -o logs.csv 'localhost:8080/api/export.csv?q=error&range=24h&tz=Europe/Paris'`. + ## Settings The gear icon opens the settings, organized in tabs. Everything except color tags is @@ -118,6 +135,7 @@ are only known by your router or a local DNS (Pi-hole, AdGuard, Unbound…), set | `RDNS` | `on` | resolve IP hosts to DNS names | | `DNS_SERVER` | empty | DNS server for reverse lookups (`ip` or `ip:port`) | | `ALLOW_PURGE` | `true` | allow "Delete all logs" in Settings | +| `EXPORT_MAX` | `100000` | maximum number of rows in a CSV export | | `TZ` | `Europe/Paris` | time zone for RFC 3164 timestamps (which carry none) | | `BATCH_SIZE`, `FLUSH_MS`, `QUEUE_SIZE` | `1000`, `1000`, `100000` | ingestion tuning | @@ -144,6 +162,7 @@ are only known by your router or a local DNS (Pi-hole, AdGuard, Unbound…), set | `query.go` | turns UI filters into LogsQL; live-view filter | | `hub.go` | pushes new messages to browsers (SSE) | | `rdns.go` | cached reverse DNS lookups | +| `export.go` | streamed CSV export | | `tags.go` | color tag storage | | `api.go` | `/api/*` HTTP routes | | `web/` | UI (HTML, CSS, plain JavaScript, no build step), embedded in the binary; translations live in `web/app.js` (`I18N`) | diff --git a/api.go b/api.go index 3fe5dbd..8b3fc77 100644 --- a/api.go +++ b/api.go @@ -18,6 +18,7 @@ type API struct { tags *TagStore rdns *ReverseDNS allowPurge bool + exportMax int } func (a *API) Routes(mux *http.ServeMux) { @@ -34,6 +35,7 @@ func (a *API) Routes(mux *http.ServeMux) { mux.HandleFunc("DELETE /api/tags/{id}", a.deleteTag) mux.HandleFunc("GET /api/purge", a.purgeStatus) mux.HandleFunc("POST /api/purge", a.purge) + mux.HandleFunc("GET /api/export.csv", a.exportCSV) } func writeJSON(w http.ResponseWriter, status int, v any) { @@ -261,6 +263,7 @@ func (a *API) stream(w http.ResponseWriter, r *http.Request) { func (a *API) stats(w http.ResponseWriter, r *http.Request) { s := a.store.Stats() s["clients"] = a.hub.Count() + s["exportMax"] = a.exportMax writeJSON(w, http.StatusOK, s) } diff --git a/docker-compose.yml b/docker-compose.yml index 50ee92b..8c55c02 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,7 @@ services: RDNS: ${RDNS:-on} # replace IP hosts with their DNS name (PTR) DNS_SERVER: ${DNS_SERVER:-} # e.g. 192.168.1.1 to query your LAN DNS; empty = system resolver ALLOW_PURGE: ${ALLOW_PURGE:-true} + EXPORT_MAX: ${EXPORT_MAX:-100000} volumes: - logstream-data:/data # tags.json diff --git a/export.go b/export.go new file mode 100644 index 0000000..0132e68 --- /dev/null +++ b/export.go @@ -0,0 +1,146 @@ +package main + +import ( + "encoding/csv" + "fmt" + "log" + "net/http" + "strconv" + "strings" + "time" +) + +// Columns of the CSV export: stored field -> column name. +var exportColumns = []struct{ field, name string }{ + {"received", "received"}, + {"msg_time", "message_time"}, + {"severity", "severity"}, + {"facility", "facility"}, + {"host", "host"}, + {"host_ip", "host_ip"}, + {"app", "app"}, + {"procid", "pid"}, + {"source", "source_ip"}, + {"proto", "proto"}, + {"_msg", "message"}, +} + +// UTF-8 byte order mark: lets Excel detect the encoding (accents). +var utf8BOM = []byte{0xEF, 0xBB, 0xBF} + +// GET /api/export.csv?&tz=Europe/Paris&excel=1&limit=N +// Streams every stored log matching the filters (newest first, at most +// EXPORT_MAX rows) as CSV. excel=1 uses ";" as separator (French Excel), +// adds a BOM and neutralizes cells Excel would run as formulas. +func (a *API) exportCSV(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + f := FilterFromRequest(r) + filter, pipes := f.LogsQL() + if strings.TrimSpace(pipes) != "" { + writeErr(w, http.StatusBadRequest, &codedError{code: "export_pipes", msg: "export does not support LogsQL pipes (| ...): keep only the filter part"}) + return + } + limit, _ := strconv.Atoi(q.Get("limit")) + if limit <= 0 || limit > a.exportMax { + limit = a.exportMax + } + loc := time.UTC + if tz := q.Get("tz"); tz != "" { + if l, err := time.LoadLocation(tz); err == nil { + loc = l + } + } + excel := q.Get("excel") == "1" + query := fmt.Sprintf("%s | sort by (_time desc) | limit %d", filter, limit) + + // The response starts with the first row, so that a query error can + // still be returned as a proper HTTP error. + var cw *csv.Writer + start := func() { + name := "logstream-" + time.Now().In(loc).Format("20060102-150405") + ".csv" + h := w.Header() + h.Set("Content-Type", "text/csv; charset=utf-8") + h.Set("Content-Disposition", `attachment; filename="`+name+`"`) + h.Set("Cache-Control", "no-store") + w.WriteHeader(http.StatusOK) + if excel { + _, _ = w.Write(utf8BOM) + } + cw = csv.NewWriter(w) + if excel { + cw.Comma = ';' + } + zone := loc.String() + header := make([]string, len(exportColumns)) + for i, c := range exportColumns { + header[i] = c.name + if c.field == "received" || c.field == "msg_time" { + header[i] += " (" + zone + ")" + } + } + _ = cw.Write(header) + } + + rows := 0 + record := make([]string, len(exportColumns)) + err := a.store.QueryStream(r.Context(), query, func(row map[string]any) error { + if cw == nil { + start() + } + for i, c := range exportColumns { + v, _ := row[c.field].(string) + switch c.field { + case "received": + v = exportTime(v, loc) + case "msg_time": + if v == "" { // logs stored before msg_time existed + v, _ = row["_time"].(string) + } + v = exportTime(v, loc) + } + if excel { + v = excelSafe(v) + } + record[i] = v + } + if err := cw.Write(record); err != nil { + return err + } + rows++ + if rows%1000 == 0 { + cw.Flush() + } + return cw.Error() + }) + if err != nil { + if cw == nil { + writeErr(w, http.StatusBadGateway, err) + return + } + // Headers are already sent: the file is truncated, log why. + log.Printf("export interrupted after %d rows: %v", rows, err) + } + if cw == nil { + start() // no matching log: header only + } + cw.Flush() +} + +// exportTime converts an RFC 3339 timestamp to "2006-01-02 15:04:05.000" in loc, +// a format spreadsheets recognize as a date. +func exportTime(v string, loc *time.Location) string { + t, err := time.Parse(time.RFC3339Nano, v) + if err != nil { + return v + } + return t.In(loc).Format("2006-01-02 15:04:05.000") +} + +// excelSafe prevents spreadsheet formula injection: log messages come from the +// network, and a cell starting with = + - @ would be run as a formula. +func excelSafe(v string) string { + if v != "" && strings.ContainsRune("=+-@\t\r", rune(v[0])) { + return "'" + v + } + return v +} diff --git a/main.go b/main.go index 38d9db3..6cad740 100644 --- a/main.go +++ b/main.go @@ -34,6 +34,7 @@ type config struct { rdns bool dnsServer string allowPurge bool + exportMax int batchSize int queueSize int flushEvery time.Duration @@ -74,6 +75,7 @@ func main() { rdns: getenvBool("RDNS", true), dnsServer: os.Getenv("DNS_SERVER"), allowPurge: getenvBool("ALLOW_PURGE", true), + exportMax: getenvInt("EXPORT_MAX", 100000), batchSize: getenvInt("BATCH_SIZE", 1000), queueSize: getenvInt("QUEUE_SIZE", 100000), flushEvery: time.Duration(getenvInt("FLUSH_MS", 1000)) * time.Millisecond, @@ -114,7 +116,7 @@ func main() { log.Fatal(err) } mux := http.NewServeMux() - api := &API{store: store, hub: hub, tags: tags, rdns: rdns, allowPurge: cfg.allowPurge} + api := &API{store: store, hub: hub, tags: tags, rdns: rdns, allowPurge: cfg.allowPurge, exportMax: cfg.exportMax} api.Routes(mux) mux.Handle("GET /", http.FileServer(http.FS(static))) diff --git a/store.go b/store.go index 5ce27a2..ca37758 100644 --- a/store.go +++ b/store.go @@ -17,8 +17,9 @@ import ( // Store sends messages to VictoriaLogs in batches and queries it with LogsQL. type Store struct { - base string - client *http.Client + base string + client *http.Client + streamClient *http.Client in chan *Entry batchSize int flushEvery time.Duration @@ -33,6 +34,9 @@ func NewStore(base string, batchSize, queueSize int, flushEvery time.Duration) * s := &Store{ base: strings.TrimRight(base, "/"), client: &http.Client{Timeout: 60 * time.Second}, + // No global timeout: a large export can take longer than a minute. The + // request context still cancels it when the browser goes away. + streamClient: &http.Client{}, in: make(chan *Entry, queueSize), batchSize: batchSize, flushEvery: flushEvery, @@ -138,23 +142,37 @@ func (s *Store) post(u string, body []byte) error { // Query runs a LogsQL query and returns one row per result. func (s *Store) Query(ctx context.Context, query string) ([]map[string]any, error) { - form := url.Values{"query": {query}} - req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.base+"/select/logsql/query", strings.NewReader(form.Encode())) + rows := []map[string]any{} + err := s.QueryStream(ctx, query, func(row map[string]any) error { + rows = append(rows, row) + return nil + }) if err != nil { return nil, err } - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - resp, err := s.client.Do(req) + return rows, nil +} + +// QueryStream runs a LogsQL query and calls fn for each result as it arrives, +// without keeping the results in memory. fn is only called once VictoriaLogs +// has accepted the query; an error returned by fn stops the stream. +func (s *Store) QueryStream(ctx context.Context, query string, fn func(row map[string]any) error) error { + form := url.Values{"query": {query}} + req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.base+"/select/logsql/query", strings.NewReader(form.Encode())) if err != nil { - return nil, err + return err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + resp, err := s.streamClient.Do(req) + if err != nil { + return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { msg, _ := io.ReadAll(io.LimitReader(resp.Body, 2048)) - return nil, fmt.Errorf("%s", strings.TrimSpace(string(msg))) + return fmt.Errorf("%s", strings.TrimSpace(string(msg))) } - rows := []map[string]any{} sc := bufio.NewScanner(resp.Body) sc.Buffer(make([]byte, 64*1024), 16<<20) for sc.Scan() { @@ -164,11 +182,13 @@ func (s *Store) Query(ctx context.Context, query string) ([]map[string]any, erro } var row map[string]any if err := json.Unmarshal(line, &row); err != nil { - return nil, err + return err + } + if err := fn(row); err != nil { + return err } - rows = append(rows, row) } - return rows, sc.Err() + return sc.Err() } // Purge starts a VictoriaLogs task that deletes every stored log, and resets diff --git a/web/app.js b/web/app.js index 8eaeb83..a7257f8 100644 --- a/web/app.js +++ b/web/app.js @@ -66,6 +66,11 @@ const I18N = { msgTimeTitle: 'Timestamp from the message', fHostName: 'host name (DNS)', fHostIp: 'host IP', clickHost: 'Click to filter on this host', clickApp: 'Click to filter on this app', + export: 'Export', exportCsvHint: 'Comma separated, UTF-8', + exportExcel: 'CSV for Excel', exportExcelHint: 'Semicolon separated, for Excel in French', + exportNote: (n) => `Up to ${n} logs matching the current filters, newest first. Dates use the time zone chosen in Settings.`, + exporting: 'Exporting…', exportDone: (f) => `Downloaded: ${f}`, exportError: 'Export failed: ', + err_export_pipes: 'Export does not support LogsQL pipes (| …): keep only the filter part', tabLocale: 'Localization', tabFilters: 'Filters', tabInterface: 'Interface', tabData: 'Data', themeTitle: 'Theme', themeSystem: 'System', themeLight: 'Light', themeDark: 'Dark', themeHelp: 'System follows the light/dark preference of your computer or phone.', @@ -149,6 +154,11 @@ const I18N = { msgTimeTitle: 'Horodatage contenu dans le message', fHostName: 'nom d\'hôte (DNS)', fHostIp: 'IP de l\'hôte', clickHost: 'Cliquer pour filtrer sur cet hôte', clickApp: 'Cliquer pour filtrer sur cette appli', + export: 'Exporter', exportCsvHint: 'Séparateur virgule, UTF-8', + exportExcel: 'CSV pour Excel', exportExcelHint: 'Séparateur point-virgule, pour Excel en français', + exportNote: (n) => `Jusqu'à ${n} logs correspondant aux filtres, du plus récent au plus ancien. Dates dans le fuseau choisi dans Paramètres.`, + exporting: 'Export…', exportDone: (f) => `Téléchargé : ${f}`, exportError: 'Échec de l\'export : ', + err_export_pipes: 'L\'export ne gère pas les pipes LogsQL (| …) : gardez seulement la partie filtre', tabLocale: 'Localisation', tabFilters: 'Filtres', tabInterface: 'Interface', tabData: 'Données', themeTitle: 'Thème', themeSystem: 'Système', themeLight: 'Clair', themeDark: 'Sombre', themeHelp: 'Système suit la préférence clair/sombre de votre ordinateur ou de votre téléphone.', @@ -283,17 +293,19 @@ async function api(url, opts = {}) { const text = await res.text(); let data = null; try { data = text ? JSON.parse(text) : null; } catch { /* not JSON */ } - if (!res.ok) { - // Known error codes are translated; otherwise show the server message. - let msg = (data && data.error) || text || res.statusText; - if (data && data.code && I18N[lang]['err_' + data.code]) { - msg = t('err_' + data.code) + (data.detail ? (lang === 'fr' ? ' : ' : ': ') + data.detail : ''); - } - throw new Error(msg); - } + if (!res.ok) throw apiError(res, text, data); return data; } +// Known error codes are translated; otherwise the server message is shown. +function apiError(res, text, data) { + let msg = (data && data.error) || text || res.statusText; + if (data && data.code && I18N[lang]['err_' + data.code]) { + msg = t('err_' + data.code) + (data.detail ? (lang === 'fr' ? ' : ' : ': ') + data.detail : ''); + } + return new Error(msg); +} + function toast(msg) { const el = $('#toast'); // A modal dialog sits above everything: move the toast into it so it stays visible. @@ -935,6 +947,7 @@ function renderStats() { async function loadStats() { try { state.stats = await api('/api/stats'); } catch { state.stats = null; } + if (state.stats && state.stats.exportMax) exportMax = state.stats.exportMax; renderStats(); } @@ -982,6 +995,59 @@ $('#tzSelect').addEventListener('change', onTimePrefsChange); $('#fmtSelect').addEventListener('change', onTimePrefsChange); setInterval(() => { if ($('#settingsDlg').open) updateTimePreview(); }, 1000); +/* ================= CSV export ================= */ + +let exportMax = 100000; // EXPORT_MAX on the server, refreshed from /api/stats + +function setExportMenu(open) { + $('#exportMenu').hidden = !open; + $('#exportBtn').setAttribute('aria-expanded', String(open)); + if (open) $('#exportNote').textContent = t('exportNote', fmtNum(exportMax)); +} + +async function exportCSV(excel) { + const p = params(); + p.set('tz', timePrefs.tz || Intl.DateTimeFormat().resolvedOptions().timeZone || 'UTC'); + if (excel) p.set('excel', '1'); + const btn = $('#exportBtn'); + const lbl = btn.querySelector('.lbl'); + btn.disabled = true; + lbl.textContent = t('exporting'); + try { + const res = await fetch('/api/export.csv?' + p); + if (!res.ok) { + const text = await res.text(); + let data = null; + try { data = JSON.parse(text); } catch { /* not JSON */ } + throw apiError(res, text, data); + } + const blob = await res.blob(); + const name = ((res.headers.get('Content-Disposition') || '').match(/filename="([^"]+)"/) || [])[1] || 'logstream.csv'; + const url = URL.createObjectURL(blob); + const a = Object.assign(document.createElement('a'), { href: url, download: name }); + document.body.append(a); + a.click(); + a.remove(); + setTimeout(() => URL.revokeObjectURL(url), 10000); + toast(t('exportDone', name)); + } catch (e) { + toast(t('exportError') + e.message); + } finally { + btn.disabled = false; + lbl.textContent = t('export'); + } +} + +$('#exportBtn').addEventListener('click', () => setExportMenu($('#exportMenu').hidden)); +$('#exportMenu').addEventListener('click', (ev) => { + const b = ev.target.closest('[data-export]'); + if (!b) return; + setExportMenu(false); + exportCSV(b.dataset.export === 'excel'); +}); +document.addEventListener('click', (ev) => { if (!ev.target.closest('.export')) setExportMenu(false); }); +document.addEventListener('keydown', (ev) => { if (ev.key === 'Escape') setExportMenu(false); }); + /* ================= Purge ================= */ const purge = { allowed: true, running: 0, error: null, code: null, wasRunning: false, timer: null }; diff --git a/web/index.html b/web/index.html index 61fc53f..d916a75 100644 --- a/web/index.html +++ b/web/index.html @@ -68,6 +68,21 @@ +
+ + +
diff --git a/web/style.css b/web/style.css index 47416e1..ad6d94f 100644 --- a/web/style.css +++ b/web/style.css @@ -199,6 +199,25 @@ body.busy .progress::after { .spacer { flex: 1; } #count { font-size: 12.5px; font-variant-numeric: tabular-nums; } +/* Export button and menu */ +.export { position: relative; } +.btn.tool { height: 32px; padding: 0 10px; font-size: 13px; } +.btn.tool svg { width: 16px; height: 16px; } +.menu { + position: absolute; right: 0; top: calc(100% + 6px); z-index: 40; + width: 290px; padding: 6px; + background: var(--panel); border: 1px solid var(--border); border-radius: 12px; + box-shadow: var(--shadow); +} +.menu button { + display: flex; flex-direction: column; align-items: flex-start; gap: 1px; width: 100%; + padding: 8px 10px; border: 0; border-radius: 8px; background: transparent; text-align: left; +} +.menu button:hover, .menu button:focus-visible { background: var(--panel-2); outline: 0; } +.menu strong { font-size: 13px; font-weight: 600; } +.menu small { font-size: 12px; color: var(--muted); } +.menu-note { margin: 4px 0 0; padding: 8px 10px 4px; border-top: 1px solid var(--border); font-size: 11.5px; color: var(--muted); } + /* ---------- Histogram ---------- */ .histo { padding: 0 20px 6px; } .histo .bars { @@ -441,9 +460,9 @@ select.field:focus { outline: 0; border-color: var(--accent); box-shadow: 0 0 0 .search { order: 3; flex-basis: 100%; max-width: none; } .live .lbl { display: none; } .filters { padding: 10px 16px 6px; } - .filters select { flex: 1 1 calc(50% - 8px); max-width: none; } + .filters select { flex: 1 1 calc(50% - 8px); max-width: none; min-width: 0; } .spacer { display: none; } - #count { flex-basis: 100%; } + #count { flex: 1 1 50%; min-width: 0; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } .histo { padding: 0 16px 6px; } .list, .error-banner { margin-left: 16px; margin-right: 16px; } .row {