diff --git a/.env.example b/.env.example index e0c3abb..40286ca 100644 --- a/.env.example +++ b/.env.example @@ -7,3 +7,9 @@ RETENTION=30d # Web UI authentication (empty = disabled) AUTH_USER= AUTH_PASS= +# Reverse DNS: show host names instead of IP addresses (on/off) +RDNS=on +# DNS server used for reverse lookups (e.g. your router: 192.168.1.1). Empty = system resolver +DNS_SERVER= +# Allow "Delete all logs" in Settings (true/false) +ALLOW_PURGE=true diff --git a/README.md b/README.md index 7abb081..ac031bc 100644 --- a/README.md +++ b/README.md @@ -58,8 +58,25 @@ The gear icon opens the settings panel: - **Language**: English or French. The choice is remembered in the browser. - **Date & time**: time zone (browser zone, UTC, or about 80 common zones) and the display - format of the reception time, e.g. `YYYY-MM-DD - HH:MM:SS:mmm` (default), ISO 8601, - 12-hour clock or Unix epoch. The time zone applies to every date shown. Remembered in the browser. + format of the reception time: `DD/MM/YYYY HH:MM:SS` (default, the usual French display), + with milliseconds, `YYYY-MM-DD`, 12-hour clock, ISO 8601 or Unix epoch. The time zone + applies to every date shown. Remembered in the browser. +- **Danger zone**: "Delete all logs" permanently erases every stored log (you must type + `PURGE` to confirm). Tags and settings are kept. VictoriaLogs needs `-delete.enable` + (already set in `docker-compose.yml`); set `ALLOW_PURGE=false` to disable the feature. + Anyone who can open the UI can purge: set `AUTH_USER` / `AUTH_PASS` if the UI is reachable + by others. + +## Host names (reverse DNS) + +When a device sends its IP address as host name (or no host name at all), Logstream looks up +its DNS name (PTR record) and stores the name in `host` and the IP in `host_ip`. Results are +cached (1 hour, 10 minutes when there is no name). Logs stored earlier with an IP are resolved +on display, and the host filter shows `name (IP)`. + +The container uses Docker's DNS, which forwards to the host's resolvers. If your local names +are only known by your router or a local DNS (Pi-hole, AdGuard, Unbound…), set +`DNS_SERVER=192.168.1.1` (its address). Set `RDNS=off` to disable lookups. - **Color tags**: each tag has a keyword, a background color (the text automatically switches to black or white to stay readable) and options: whole word, match case, regular expression, active. Tags are stored in `/data/tags.json` (`logstream-data` volume). @@ -73,6 +90,9 @@ The gear icon opens the settings panel: | `HTTP_PORT` | `8080` | web UI port | | `RETENTION` | `30d` | how long VictoriaLogs keeps logs | | `AUTH_USER` / `AUTH_PASS` | empty | HTTP Basic authentication for the UI | +| `RDNS` | `on` | resolve IP hosts to DNS names | +| `DNS_SERVER` | empty | DNS server for reverse lookups (`ip` or `ip:port`) | +| `ALLOW_PURGE` | `true` | allow "Delete all logs" in Settings | | `TZ` | `Europe/Paris` | time zone for RFC 3164 timestamps (which carry none) | | `BATCH_SIZE`, `FLUSH_MS`, `QUEUE_SIZE` | `1000`, `1000`, `100000` | ingestion tuning | @@ -98,6 +118,7 @@ The gear icon opens the settings panel: | `store.go` | batched inserts into VictoriaLogs and LogsQL queries | | `query.go` | turns UI filters into LogsQL; live-view filter | | `hub.go` | pushes new messages to browsers (SSE) | +| `rdns.go` | cached reverse DNS lookups | | `tags.go` | color tag storage | | `api.go` | `/api/*` HTTP routes | | `web/` | UI (HTML, CSS, plain JavaScript, no build step), embedded in the binary; translations live in `web/app.js` (`I18N`) | diff --git a/api.go b/api.go index 64f33a3..3fe5dbd 100644 --- a/api.go +++ b/api.go @@ -4,6 +4,8 @@ import ( "encoding/json" "errors" "fmt" + "log" + "net" "net/http" "sort" "strconv" @@ -11,9 +13,11 @@ import ( ) type API struct { - store *Store - hub *Hub - tags *TagStore + store *Store + hub *Hub + tags *TagStore + rdns *ReverseDNS + allowPurge bool } func (a *API) Routes(mux *http.ServeMux) { @@ -28,6 +32,8 @@ func (a *API) Routes(mux *http.ServeMux) { mux.HandleFunc("POST /api/tags/reset", a.resetTags) mux.HandleFunc("PUT /api/tags/{id}", a.updateTag) mux.HandleFunc("DELETE /api/tags/{id}", a.deleteTag) + mux.HandleFunc("GET /api/purge", a.purgeStatus) + mux.HandleFunc("POST /api/purge", a.purge) } func writeJSON(w http.ResponseWriter, status int, v any) { @@ -70,6 +76,7 @@ func (a *API) logs(w http.ResponseWriter, r *http.Request) { writeErr(w, http.StatusBadGateway, err) return } + a.annotateHosts(rows) writeJSON(w, http.StatusOK, map[string]any{ "query": q, "rows": rows, @@ -122,9 +129,39 @@ func toInt(v any) int64 { return 0 } +// annotateHosts adds "host_name" to rows whose host is still an IP address +// (logs stored before DNS resolution, or resolved too slowly at reception). +func (a *API) annotateHosts(rows []map[string]any) { + seen := map[string]bool{} + var ips []string + for _, row := range rows { + if h, _ := row["host"].(string); h != "" && !seen[h] && net.ParseIP(h) != nil { + seen[h] = true + ips = append(ips, h) + } + } + if len(ips) == 0 { + return + } + if len(ips) > 100 { + ips = ips[:100] + } + names := a.rdns.LookupMany(ips, time.Second) + for _, row := range rows { + if h, _ := row["host"].(string); names[h] != "" { + row["host_name"] = names[h] + } + } +} + +type facet struct { + Value string `json:"value"` // stored value, used for filtering + Label string `json:"label"` // displayed text ("name (ip)" for resolved IPs) +} + // GET /api/facets: hosts and apps seen over 7 days, for the filter dropdowns. func (a *API) facets(w http.ResponseWriter, r *http.Request) { - res := map[string][]string{} + res := map[string][]facet{} for _, field := range []string{"host", "app"} { q := "_time:7d | stats by (" + field + ") count() hits | sort by (hits desc) | limit 300" rows, err := a.store.Query(r.Context(), q) @@ -133,13 +170,26 @@ func (a *API) facets(w http.ResponseWriter, r *http.Request) { return } vals := make([]string, 0, len(rows)) + var ips []string for _, row := range rows { if v, _ := row[field].(string); v != "" { vals = append(vals, v) + if field == "host" && net.ParseIP(v) != nil { + ips = append(ips, v) + } } } - sort.Strings(vals) - res[field] = vals + names := a.rdns.LookupMany(ips, time.Second) + list := make([]facet, 0, len(vals)) + for _, v := range vals { + label := v + if n := names[v]; n != "" { + label = n + " (" + v + ")" + } + list = append(list, facet{Value: v, Label: label}) + } + sort.Slice(list, func(i, j int) bool { return list[i].Label < list[j].Label }) + res[field] = list } writeJSON(w, http.StatusOK, res) } @@ -266,6 +316,46 @@ func (a *API) deleteTag(w http.ResponseWriter, r *http.Request) { } } +// GET /api/purge: whether purging is allowed and how many deletions are running. +func (a *API) purgeStatus(w http.ResponseWriter, r *http.Request) { + res := map[string]any{"allowed": a.allowPurge, "running": 0} + if a.allowPurge { + n, err := a.store.PurgeRunning(r.Context()) + if err != nil { + res["error"] = err.Error() + var ce *codedError + if errors.As(err, &ce) { + res["code"] = ce.code + } + } + res["running"] = n + } + writeJSON(w, http.StatusOK, res) +} + +// POST /api/purge {"confirm":"PURGE"}: deletes every stored log. +func (a *API) purge(w http.ResponseWriter, r *http.Request) { + if !a.allowPurge { + writeErr(w, http.StatusForbidden, &codedError{code: "purge_forbidden", msg: "purging is disabled (ALLOW_PURGE=false)"}) + return + } + var body struct { + Confirm string `json:"confirm"` + } + _ = json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&body) + if body.Confirm != "PURGE" { + writeErr(w, http.StatusBadRequest, &codedError{code: "purge_confirm", msg: `type "PURGE" to confirm`}) + return + } + id, err := a.store.Purge(r.Context()) + if err != nil { + writeErr(w, http.StatusBadGateway, err) + return + } + log.Printf("purge of all logs requested from %s (task %s)", r.RemoteAddr, id) + writeJSON(w, http.StatusOK, map[string]string{"task_id": id}) +} + func (a *API) resetTags(w http.ResponseWriter, r *http.Request) { tags, err := a.tags.Reset() if err != nil { diff --git a/docker-compose.yml b/docker-compose.yml index ed6f2c3..50ee92b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,9 @@ services: TZ: ${TZ:-Europe/Paris} AUTH_USER: ${AUTH_USER:-} # leave empty to disable authentication AUTH_PASS: ${AUTH_PASS:-} + RDNS: ${RDNS:-on} # replace IP hosts with their DNS name (PTR) + DNS_SERVER: ${DNS_SERVER:-} # e.g. 192.168.1.1 to query your LAN DNS; empty = system resolver + ALLOW_PURGE: ${ALLOW_PURGE:-true} volumes: - logstream-data:/data # tags.json @@ -26,6 +29,7 @@ services: - -storageDataPath=/vlogs - -retentionPeriod=${RETENTION:-30d} - -httpListenAddr=:9428 + - -delete.enable # required by "Delete all logs" in Settings volumes: - vlogs-data:/vlogs ports: diff --git a/main.go b/main.go index 070474d..38d9db3 100644 --- a/main.go +++ b/main.go @@ -15,6 +15,7 @@ import ( "os/signal" "path/filepath" "strconv" + "strings" "syscall" "time" _ "time/tzdata" // embedded time zones: TZ works without a system package @@ -30,6 +31,9 @@ type config struct { dataDir string authUser string authPass string + rdns bool + dnsServer string + allowPurge bool batchSize int queueSize int flushEvery time.Duration @@ -49,6 +53,16 @@ func getenvInt(key string, def int) int { return def } +func getenvBool(key string, def bool) bool { + switch strings.ToLower(os.Getenv(key)) { + case "1", "true", "yes", "on": + return true + case "0", "false", "no", "off": + return false + } + return def +} + func main() { cfg := config{ syslogAddr: getenv("SYSLOG_ADDR", ":5514"), @@ -57,6 +71,9 @@ func main() { dataDir: getenv("DATA_DIR", "/data"), authUser: os.Getenv("AUTH_USER"), authPass: os.Getenv("AUTH_PASS"), + rdns: getenvBool("RDNS", true), + dnsServer: os.Getenv("DNS_SERVER"), + allowPurge: getenvBool("ALLOW_PURGE", true), batchSize: getenvInt("BATCH_SIZE", 1000), queueSize: getenvInt("QUEUE_SIZE", 100000), flushEvery: time.Duration(getenvInt("FLUSH_MS", 1000)) * time.Millisecond, @@ -78,7 +95,13 @@ func main() { }() hub := NewHub() + rdns := NewReverseDNS(cfg.rdns, cfg.dnsServer) sink := func(e *Entry) { + // Host sent as an IP (or no host in the header): replace it with its DNS name. + // A new IP waits at most 300 ms; slower lookups finish in the background. + if name := rdns.Lookup(e.Host, 300*time.Millisecond); name != "" { + e.HostIP, e.Host = e.Host, name + } store.Enqueue(e) hub.Publish(e) } @@ -91,7 +114,7 @@ func main() { log.Fatal(err) } mux := http.NewServeMux() - api := &API{store: store, hub: hub, tags: tags} + api := &API{store: store, hub: hub, tags: tags, rdns: rdns, allowPurge: cfg.allowPurge} api.Routes(mux) mux.Handle("GET /", http.FileServer(http.FS(static))) diff --git a/rdns.go b/rdns.go new file mode 100644 index 0000000..4d7157c --- /dev/null +++ b/rdns.go @@ -0,0 +1,124 @@ +package main + +import ( + "context" + "net" + "strings" + "sync" + "time" +) + +// ReverseDNS resolves IP addresses to host names (PTR records), with a cache. +type ReverseDNS struct { + enabled bool + r *net.Resolver + posTTL time.Duration // cache duration of a found name + negTTL time.Duration // cache duration of "no name" + + mu sync.Mutex + cache map[string]*rdnsEntry +} + +type rdnsEntry struct { + name string + expires time.Time + done chan struct{} // closed once the lookup has finished +} + +const rdnsMaxEntries = 10000 + +// NewReverseDNS uses the system resolver, or `server` ("ip" or "ip:port") when set. +func NewReverseDNS(enabled bool, server string) *ReverseDNS { + r := net.DefaultResolver + if server != "" { + if _, _, err := net.SplitHostPort(server); err != nil { + server = net.JoinHostPort(server, "53") + } + dialer := net.Dialer{Timeout: 2 * time.Second} + r = &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, _ string) (net.Conn, error) { + return dialer.DialContext(ctx, network, server) + }, + } + } + return &ReverseDNS{ + enabled: enabled, + r: r, + posTTL: time.Hour, + negTTL: 10 * time.Minute, + cache: make(map[string]*rdnsEntry), + } +} + +func isClosed(ch chan struct{}) bool { + select { + case <-ch: + return true + default: + return false + } +} + +// Lookup returns the name of ip, or "" when ip is not an IP address, has no +// PTR record, or is not resolved within `wait`. A lookup that takes longer +// keeps running in the background and fills the cache for later calls. +func (d *ReverseDNS) Lookup(ip string, wait time.Duration) string { + if !d.enabled || net.ParseIP(ip) == nil { + return "" + } + d.mu.Lock() + e := d.cache[ip] + if e == nil || (isClosed(e.done) && time.Now().After(e.expires)) { + if len(d.cache) >= rdnsMaxEntries { + d.cache = make(map[string]*rdnsEntry) + } + e = &rdnsEntry{done: make(chan struct{})} + d.cache[ip] = e + go d.resolve(ip, e) + } + d.mu.Unlock() + + if !isClosed(e.done) { + timer := time.NewTimer(wait) + defer timer.Stop() + select { + case <-e.done: + case <-timer.C: + return "" + } + } + return e.name +} + +// LookupMany resolves several addresses in parallel; unresolved ones are absent. +func (d *ReverseDNS) LookupMany(ips []string, wait time.Duration) map[string]string { + out := make(map[string]string) + var mu sync.Mutex + var wg sync.WaitGroup + for _, ip := range ips { + wg.Add(1) + go func(ip string) { + defer wg.Done() + if name := d.Lookup(ip, wait); name != "" { + mu.Lock() + out[ip] = name + mu.Unlock() + } + }(ip) + } + wg.Wait() + return out +} + +func (d *ReverseDNS) resolve(ip string, e *rdnsEntry) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + ttl := d.negTTL + if names, err := d.r.LookupAddr(ctx, ip); err == nil && len(names) > 0 { + e.name = strings.TrimSuffix(names[0], ".") + ttl = d.posTTL + } + e.expires = time.Now().Add(ttl) + close(e.done) +} diff --git a/store.go b/store.go index 142cd26..5ce27a2 100644 --- a/store.go +++ b/store.go @@ -171,6 +171,60 @@ func (s *Store) Query(ctx context.Context, query string) ([]map[string]any, erro return rows, sc.Err() } +// Purge starts a VictoriaLogs task that deletes every stored log, and resets +// the counters. VictoriaLogs must run with -delete.enable. +func (s *Store) Purge(ctx context.Context) (string, error) { + body, err := s.deleteCall(ctx, "/delete/run_task?filter="+url.QueryEscape("*")) + if err != nil { + return "", err + } + var res struct { + TaskID string `json:"task_id"` + } + if err := json.Unmarshal(body, &res); err != nil { + return "", fmt.Errorf("unexpected VictoriaLogs answer: %s", strings.TrimSpace(string(body))) + } + s.received.Store(0) + s.ingested.Store(0) + s.dropped.Store(0) + s.lastErr.Store("") + return res.TaskID, nil +} + +// PurgeRunning returns the number of deletion tasks still running in VictoriaLogs. +func (s *Store) PurgeRunning(ctx context.Context) (int, error) { + body, err := s.deleteCall(ctx, "/delete/active_tasks") + if err != nil { + return 0, err + } + var tasks []json.RawMessage + if err := json.Unmarshal(body, &tasks); err != nil { + return 0, fmt.Errorf("unexpected VictoriaLogs answer: %s", strings.TrimSpace(string(body))) + } + return len(tasks), nil +} + +func (s *Store) deleteCall(ctx context.Context, path string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.base+path, nil) + if err != nil { + return nil, err + } + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode != http.StatusOK { + msg := strings.TrimSpace(string(body)) + if resp.StatusCode == http.StatusNotFound || strings.Contains(msg, "delete.enable") { + return nil, &codedError{code: "purge_unavailable", msg: "VictoriaLogs refuses deletions: start it with -delete.enable (see docker-compose.yml)", detail: msg} + } + return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, msg) + } + return body, nil +} + func (s *Store) Stats() map[string]any { return map[string]any{ "received": s.received.Load(), diff --git a/syslog.go b/syslog.go index ef51b7e..dbe572b 100644 --- a/syslog.go +++ b/syslog.go @@ -34,6 +34,7 @@ type Entry struct { Message string Source string // sender IP address Proto string // udp or tcp + HostIP string // original host value when it was an IP resolved through DNS } // Record returns the entry in the shape stored in VictoriaLogs and returned by the API. @@ -56,6 +57,9 @@ func (e *Entry) Record() map[string]string { if e.SD != "" { r["sd"] = e.SD } + if e.HostIP != "" { + r["host_ip"] = e.HostIP + } return r } diff --git a/web/app.js b/web/app.js index 902db24..282dcaa 100644 --- a/web/app.js +++ b/web/app.js @@ -64,6 +64,18 @@ const I18N = { fReceived: 'received', rcvTitle: 'Received by the server', msgTimeTitle: 'Timestamp from the message', + fHostName: 'host name (DNS)', fHostIp: 'host IP', + dangerZone: 'Danger zone', + purgeHelp: 'Permanently delete every stored log to start from scratch. Color tags and settings are kept.', + purgeBtn: 'Delete all logs…', + purgePrompt: 'This permanently deletes ALL stored logs.\n\nType PURGE to confirm:', + purgeStarted: 'Deletion started', + purgeRunning: 'Deletion in progress… logs disappear gradually.', + purgeDone: 'All logs have been deleted.', + purgeForbidden: 'Purging is disabled on this server (ALLOW_PURGE=false).', + err_purge_unavailable: 'VictoriaLogs refuses deletions: start it with -delete.enable (see docker-compose.yml)', + err_purge_forbidden: 'Purging is disabled on this server (ALLOW_PURGE=false)', + err_purge_confirm: 'Type PURGE to confirm', }, fr: { locale: 'fr-FR', @@ -126,6 +138,18 @@ const I18N = { fReceived: 'réception', rcvTitle: 'Reçu par le serveur', msgTimeTitle: 'Horodatage contenu dans le message', + fHostName: 'nom d\'hôte (DNS)', fHostIp: 'IP de l\'hôte', + dangerZone: 'Zone de danger', + purgeHelp: 'Supprime définitivement tous les logs stockés pour repartir de zéro. Les tags de couleur et les réglages sont conservés.', + purgeBtn: 'Supprimer tous les logs…', + purgePrompt: 'Cette action supprime définitivement TOUS les logs stockés.\n\nTapez PURGE pour confirmer :', + purgeStarted: 'Suppression lancée', + purgeRunning: 'Suppression en cours… les logs disparaissent progressivement.', + purgeDone: 'Tous les logs ont été supprimés.', + purgeForbidden: 'La purge est désactivée sur ce serveur (ALLOW_PURGE=false).', + err_purge_unavailable: 'VictoriaLogs refuse les suppressions : lancez-le avec -delete.enable (voir docker-compose.yml)', + err_purge_forbidden: 'La purge est désactivée sur ce serveur (ALLOW_PURGE=false)', + err_purge_confirm: 'Tapez PURGE pour confirmer', }, }; @@ -180,11 +204,10 @@ const TIME_ZONES = { // Each format receives the date parts in the selected time zone (and the Date itself). const TIME_FORMATS = [ - { id: 'ymd-dash', label: 'YYYY-MM-DD - HH:MM:SS:mmm', f: (p) => `${p.Y}-${p.M}-${p.D} - ${p.h}:${p.m}:${p.s}:${p.ms}` }, - { id: 'ymd-ms', label: 'YYYY-MM-DD HH:MM:SS.mmm', f: (p) => `${p.Y}-${p.M}-${p.D} ${p.h}:${p.m}:${p.s}.${p.ms}` }, - { id: 'ymd', label: 'YYYY-MM-DD HH:MM:SS', f: (p) => `${p.Y}-${p.M}-${p.D} ${p.h}:${p.m}:${p.s}` }, - { id: 'dmy-ms', label: 'DD/MM/YYYY HH:MM:SS.mmm', f: (p) => `${p.D}/${p.M}/${p.Y} ${p.h}:${p.m}:${p.s}.${p.ms}` }, { id: 'dmy', label: 'DD/MM/YYYY HH:MM:SS', f: (p) => `${p.D}/${p.M}/${p.Y} ${p.h}:${p.m}:${p.s}` }, + { id: 'dmy-ms', label: 'DD/MM/YYYY HH:MM:SS.mmm', f: (p) => `${p.D}/${p.M}/${p.Y} ${p.h}:${p.m}:${p.s}.${p.ms}` }, + { id: 'ymd', label: 'YYYY-MM-DD HH:MM:SS', f: (p) => `${p.Y}-${p.M}-${p.D} ${p.h}:${p.m}:${p.s}` }, + { id: 'ymd-ms', label: 'YYYY-MM-DD HH:MM:SS.mmm', f: (p) => `${p.Y}-${p.M}-${p.D} ${p.h}:${p.m}:${p.s}.${p.ms}` }, { id: 'mdy12', label: 'MM/DD/YYYY hh:MM:SS.mmm AM/PM', f: (p) => { const H = Number(p.h); return `${p.M}/${p.D}/${p.Y} ${pad(H % 12 || 12)}:${p.m}:${p.s}.${p.ms} ${H < 12 ? 'AM' : 'PM'}`; @@ -194,7 +217,8 @@ const TIME_FORMATS = [ { id: 'epoch', label: null, f: (p, d) => String(d.getTime()) }, ]; -const timePrefs = { tz: '', fmt: 'ymd-dash' }; // tz '' = browser time zone +const DEFAULT_TIME_FMT = 'dmy'; // usual French display: 28/09/2026 14:55:23 +const timePrefs = { tz: '', fmt: DEFAULT_TIME_FMT }; // tz '' = browser time zone const partsFormatters = new Map(); function validTimeZone(tz) { @@ -306,6 +330,7 @@ function setLang(next) { renderStats(); renderTagList(); renderTimeSettings(); + renderPurge(); } $('#langSwitch').addEventListener('click', (ev) => { @@ -434,12 +459,19 @@ function rowHTML(r, isNew) { + `` + `` + `${esc(sev || '—')}` - + `${esc(r.host)}` + + `${esc(r.host_name || r.host)}` + `${esc(r.app)}` + `