Add CSV export of the logs matching the current filters
- GET /api/export.csv streams every stored log matching the filters (newest first, up to EXPORT_MAX rows, 100000 by default) straight from VictoriaLogs, with dates in the time zone chosen in Settings. - Export button with two variants: CSV (comma, UTF-8) and CSV for Excel (semicolon + BOM, formula injection neutralized). - Store.QueryStream streams query results without buffering them. - Mobile: filter bar keeps two selects per row, count next to Export. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
84e71f741c
commit
328384032b
10 files changed
+316
-23
No files matched your search
@@ -0,0 +1,146 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Columns of the CSV export: stored field -> column name.
|
||||
var exportColumns = []struct{ field, name string }{
|
||||
{"received", "received"},
|
||||
{"msg_time", "message_time"},
|
||||
{"severity", "severity"},
|
||||
{"facility", "facility"},
|
||||
{"host", "host"},
|
||||
{"host_ip", "host_ip"},
|
||||
{"app", "app"},
|
||||
{"procid", "pid"},
|
||||
{"source", "source_ip"},
|
||||
{"proto", "proto"},
|
||||
{"_msg", "message"},
|
||||
}
|
||||
|
||||
// UTF-8 byte order mark: lets Excel detect the encoding (accents).
|
||||
var utf8BOM = []byte{0xEF, 0xBB, 0xBF}
|
||||
|
||||
// GET /api/export.csv?<filters>&tz=Europe/Paris&excel=1&limit=N
|
||||
// Streams every stored log matching the filters (newest first, at most
|
||||
// EXPORT_MAX rows) as CSV. excel=1 uses ";" as separator (French Excel),
|
||||
// adds a BOM and neutralizes cells Excel would run as formulas.
|
||||
func (a *API) exportCSV(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
f := FilterFromRequest(r)
|
||||
filter, pipes := f.LogsQL()
|
||||
if strings.TrimSpace(pipes) != "" {
|
||||
writeErr(w, http.StatusBadRequest, &codedError{code: "export_pipes", msg: "export does not support LogsQL pipes (| ...): keep only the filter part"})
|
||||
return
|
||||
}
|
||||
limit, _ := strconv.Atoi(q.Get("limit"))
|
||||
if limit <= 0 || limit > a.exportMax {
|
||||
limit = a.exportMax
|
||||
}
|
||||
loc := time.UTC
|
||||
if tz := q.Get("tz"); tz != "" {
|
||||
if l, err := time.LoadLocation(tz); err == nil {
|
||||
loc = l
|
||||
}
|
||||
}
|
||||
excel := q.Get("excel") == "1"
|
||||
query := fmt.Sprintf("%s | sort by (_time desc) | limit %d", filter, limit)
|
||||
|
||||
// The response starts with the first row, so that a query error can
|
||||
// still be returned as a proper HTTP error.
|
||||
var cw *csv.Writer
|
||||
start := func() {
|
||||
name := "logstream-" + time.Now().In(loc).Format("20060102-150405") + ".csv"
|
||||
h := w.Header()
|
||||
h.Set("Content-Type", "text/csv; charset=utf-8")
|
||||
h.Set("Content-Disposition", `attachment; filename="`+name+`"`)
|
||||
h.Set("Cache-Control", "no-store")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if excel {
|
||||
_, _ = w.Write(utf8BOM)
|
||||
}
|
||||
cw = csv.NewWriter(w)
|
||||
if excel {
|
||||
cw.Comma = ';'
|
||||
}
|
||||
zone := loc.String()
|
||||
header := make([]string, len(exportColumns))
|
||||
for i, c := range exportColumns {
|
||||
header[i] = c.name
|
||||
if c.field == "received" || c.field == "msg_time" {
|
||||
header[i] += " (" + zone + ")"
|
||||
}
|
||||
}
|
||||
_ = cw.Write(header)
|
||||
}
|
||||
|
||||
rows := 0
|
||||
record := make([]string, len(exportColumns))
|
||||
err := a.store.QueryStream(r.Context(), query, func(row map[string]any) error {
|
||||
if cw == nil {
|
||||
start()
|
||||
}
|
||||
for i, c := range exportColumns {
|
||||
v, _ := row[c.field].(string)
|
||||
switch c.field {
|
||||
case "received":
|
||||
v = exportTime(v, loc)
|
||||
case "msg_time":
|
||||
if v == "" { // logs stored before msg_time existed
|
||||
v, _ = row["_time"].(string)
|
||||
}
|
||||
v = exportTime(v, loc)
|
||||
}
|
||||
if excel {
|
||||
v = excelSafe(v)
|
||||
}
|
||||
record[i] = v
|
||||
}
|
||||
if err := cw.Write(record); err != nil {
|
||||
return err
|
||||
}
|
||||
rows++
|
||||
if rows%1000 == 0 {
|
||||
cw.Flush()
|
||||
}
|
||||
return cw.Error()
|
||||
})
|
||||
if err != nil {
|
||||
if cw == nil {
|
||||
writeErr(w, http.StatusBadGateway, err)
|
||||
return
|
||||
}
|
||||
// Headers are already sent: the file is truncated, log why.
|
||||
log.Printf("export interrupted after %d rows: %v", rows, err)
|
||||
}
|
||||
if cw == nil {
|
||||
start() // no matching log: header only
|
||||
}
|
||||
cw.Flush()
|
||||
}
|
||||
|
||||
// exportTime converts an RFC 3339 timestamp to "2006-01-02 15:04:05.000" in loc,
|
||||
// a format spreadsheets recognize as a date.
|
||||
func exportTime(v string, loc *time.Location) string {
|
||||
t, err := time.Parse(time.RFC3339Nano, v)
|
||||
if err != nil {
|
||||
return v
|
||||
}
|
||||
return t.In(loc).Format("2006-01-02 15:04:05.000")
|
||||
}
|
||||
|
||||
// excelSafe prevents spreadsheet formula injection: log messages come from the
|
||||
// network, and a cell starting with = + - @ would be run as a formula.
|
||||
func excelSafe(v string) string {
|
||||
if v != "" && strings.ContainsRune("=+-@\t\r", rune(v[0])) {
|
||||
return "'" + v
|
||||
}
|
||||
return v
|
||||
}
|
||||
Reference in new issue
Block a user