diff --git a/.env.example b/.env.example index cb8012a..715361e 100644 --- a/.env.example +++ b/.env.example @@ -5,3 +5,6 @@ GITHUB_TOKEN= CACHE_TTL=5m AUTH_USER= AUTH_PASS= +# Time zone of the scheduled searches. +TZ=Europe/Paris +HISTORY_KEEP=100 diff --git a/.gitignore b/.gitignore index c23f2ff..4316220 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,6 @@ go.work.sum # Binary /searchgit + +# Local data (scheduled searches) +/data/ diff --git a/Dockerfile b/Dockerfile index 3dee1b6..b03bdf1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,9 +11,12 @@ RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/searchgit . # ---- Final image ---- FROM alpine:3.24.2 RUN apk add --no-cache ca-certificates \ - && adduser -D -H -u 10001 searchgit + && adduser -D -H -u 10001 searchgit \ + && mkdir /data && chown searchgit:searchgit /data COPY --from=build /out/searchgit /usr/local/bin/searchgit USER searchgit +ENV DATA_DIR=/data +VOLUME /data EXPOSE 8080/tcp HEALTHCHECK --interval=30s --timeout=3s CMD wget -qO- http://127.0.0.1:8080/healthz || exit 1 ENTRYPOINT ["/usr/local/bin/searchgit"] diff --git a/README.md b/README.md index 7197839..49d0d5a 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,8 @@ Search GitHub repositories with criteria and browse the results in a clean web i ``` browser ──/api/search──▶ searchgit (Go) ──REST──▶ api.github.com/search/repositories - └── 5 min cache + ──/api/saved───▶ ├── 5 min cache + └── scheduler ──▶ /data (JSON snapshots) ``` ## Getting started @@ -18,7 +19,7 @@ docker compose up -d --build Then open . -Without Docker: `go run .` (Go 1.23 or newer), then open . +Without Docker: `go run .` (Go 1.24 or newer, data in `./data`), then open . ## Criteria @@ -42,6 +43,24 @@ shows the exact query sent to GitHub (click it to open the same search on github GitHub returns at most the first 1,000 results of a search. +## Scheduled searches (weekly review) + +**Schedule** saves the current filters as a search that the server runs on its own: +every week (day and time), every day, or every N hours, in the server time zone (`TZ`). +Each run keeps a snapshot of the first 30, 50 or 100 results. + +The **Watch** tab lists the scheduled searches with their last run, and a review page +per search shows, for any run of its history: + +- **New**: repositories never returned by a previous run of this search; +- **Rising**: repositories that gained the most stars since the previous run; +- **All**: the whole snapshot, with the star gain of each repository. + +A search runs once as soon as it is created: that first run is the baseline the next +ones are compared with. A run missed while the server was stopped is made at startup. +For a weekly "gems" review, a good start is: a topic or a language, *created < 1 month* +(or *< 1 week*), sorted by *most stars*. + ## Configuration | Variable | Default | Role | @@ -51,12 +70,20 @@ GitHub returns at most the first 1,000 results of a search. | `CACHE_TTL` | `5m` | identical searches are served from memory for this long | | `AUTH_USER` / `AUTH_PASS` | empty | HTTP basic authentication (except `/healthz`) | | `GITHUB_API` | `https://api.github.com` | API base URL (GitHub Enterprise) | +| `DATA_DIR` | `data` (`/data` in Docker) | scheduled searches and their snapshots (JSON files) | +| `TZ` | `Europe/Paris` in compose | time zone of the schedules | +| `HISTORY_KEEP` | `100` | runs kept per scheduled search (0 = all) | ## API - `GET /api/search?q=&in=&language=&stars=&maxstars=&minforks=&topic=&license=&pushed=&created=&goodfirst=1&forks=1&archived=1&sort=&order=&page=&per_page=` (`pushed` / `created`: `1w`, `1m`, `3m`, `6m`, `1y`, `2y`, `5y`) -- `GET /api/status`: token configured and last known search quota +- `GET /api/status`: token configured, last known search quota, server time zone +- `GET|POST /api/saved`, `GET|PUT|DELETE /api/saved/{id}`: scheduled searches + (`{"name", "params", "maxResults", "enabled", "schedule": {"every": "week|day|hours", "weekday", "hour", "minute", "hours"}}`, + `params` being the `/api/search` query string) +- `POST /api/saved/{id}/run`: run now +- `GET /api/saved/{id}/runs`, `GET /api/saved/{id}/runs/{run|latest}`: history and snapshots - `GET /healthz` ## Development diff --git a/api.go b/api.go index 44e0035..5b3cfad 100644 --- a/api.go +++ b/api.go @@ -4,16 +4,20 @@ import ( "encoding/json" "errors" "net/http" + "time" ) type API struct { - gh *GitHub + gh *GitHub + store *Store + sched *Scheduler } func (a *API) Routes(mux *http.ServeMux) { mux.HandleFunc("GET /healthz", a.health) mux.HandleFunc("GET /api/search", a.search) mux.HandleFunc("GET /api/status", a.status) + a.savedRoutes(mux) } // codedError carries a stable code that the UI translates. @@ -61,9 +65,19 @@ func (a *API) status(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, map[string]any{ "token": a.gh.HasToken(), "rate": a.gh.Rate(), + "tz": zoneName(), }) } +// zoneName is the server time zone used by the schedules (TZ). +func zoneName() string { + if name := time.Local.String(); name != "Local" { + return name + } + name, _ := time.Now().Zone() + return name +} + func (a *API) health(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("ok")) } diff --git a/docker-compose.yml b/docker-compose.yml index 73191fc..538c21f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,3 +10,10 @@ services: CACHE_TTL: ${CACHE_TTL:-5m} # identical searches are served from memory AUTH_USER: ${AUTH_USER:-} # leave empty to disable authentication AUTH_PASS: ${AUTH_PASS:-} + TZ: ${TZ:-Europe/Paris} # time zone of the scheduled searches + HISTORY_KEEP: ${HISTORY_KEEP:-100} # runs kept per scheduled search + volumes: + - searchgit-data:/data # scheduled searches and their history + +volumes: + searchgit-data: diff --git a/go.mod b/go.mod index 8c4224d..3338970 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module searchgit -go 1.23 +go 1.24 diff --git a/main.go b/main.go index f7d41c7..f463acc 100644 --- a/main.go +++ b/main.go @@ -12,8 +12,10 @@ import ( "net/http" "os" "os/signal" + "strconv" "syscall" "time" + _ "time/tzdata" // TZ works in the minimal Docker image ) //go:embed web @@ -33,6 +35,13 @@ func getenvDuration(key string, def time.Duration) time.Duration { return def } +func getenvInt(key string, def int) int { + if n, err := strconv.Atoi(os.Getenv(key)); err == nil && n >= 0 { + return n + } + return def +} + func main() { httpAddr := getenv("HTTP_ADDR", ":8080") token := os.Getenv("GITHUB_TOKEN") @@ -45,8 +54,15 @@ func main() { if err != nil { log.Fatal(err) } + store, err := OpenStore(getenv("DATA_DIR", "data"), getenvInt("HISTORY_KEEP", 100)) + if err != nil { + log.Fatalf("data: %v", err) + } + sched := NewScheduler(store, gh) + sched.Start(ctx) + mux := http.NewServeMux() - (&API{gh: gh}).Routes(mux) + (&API{gh: gh, store: store, sched: sched}).Routes(mux) mux.Handle("GET /", http.FileServer(http.FS(static))) srv := &http.Server{ @@ -65,7 +81,7 @@ func main() { if token == "" { log.Printf("GITHUB_TOKEN not set: GitHub allows only 10 searches per minute") } - log.Printf("web UI on %s", httpAddr) + log.Printf("web UI on %s, data in %s, time zone %s", httpAddr, getenv("DATA_DIR", "data"), time.Local) if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { log.Fatalf("http: %v", err) } diff --git a/saved_api.go b/saved_api.go new file mode 100644 index 0000000..fc3b47e --- /dev/null +++ b/saved_api.go @@ -0,0 +1,200 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "log" + "net/http" + "net/url" + "strings" + "time" +) + +// Saved search routes: +// +// GET /api/saved list +// POST /api/saved create (and run it once in the background) +// GET /api/saved/{id} +// PUT /api/saved/{id} update +// DELETE /api/saved/{id} +// POST /api/saved/{id}/run run now +// GET /api/saved/{id}/runs history (summaries) +// GET /api/saved/{id}/runs/{run} one snapshot ("latest" accepted) +func (a *API) savedRoutes(mux *http.ServeMux) { + mux.HandleFunc("GET /api/saved", a.savedList) + mux.HandleFunc("POST /api/saved", a.savedCreate) + mux.HandleFunc("GET /api/saved/{id}", a.savedGet) + mux.HandleFunc("PUT /api/saved/{id}", a.savedUpdate) + mux.HandleFunc("DELETE /api/saved/{id}", a.savedDelete) + mux.HandleFunc("POST /api/saved/{id}/run", a.savedRun) + mux.HandleFunc("GET /api/saved/{id}/runs", a.savedRuns) + mux.HandleFunc("GET /api/saved/{id}/runs/{run}", a.savedRunGet) +} + +// savedInput is what the UI sends on create and update. +type savedInput struct { + Name string `json:"name"` + Params string `json:"params"` + Schedule Schedule `json:"schedule"` + MaxResults int `json:"maxResults"` + Enabled *bool `json:"enabled"` +} + +func (in *savedInput) validate() error { + in.Name = strings.TrimSpace(in.Name) + if in.Name == "" || len(in.Name) > 120 { + return &codedError{code: "bad_param", msg: "invalid name", detail: "name"} + } + v, err := url.ParseQuery(in.Params) + if err != nil { + return &codedError{code: "bad_param", msg: "invalid parameters", detail: "params"} + } + v.Del("page") + v.Del("per_page") + in.Params = v.Encode() + if _, err := ParseCriteria(v); err != nil { + return err + } + switch in.MaxResults { + case 0: + in.MaxResults = 50 + case 30, 50, 100: + default: + return &codedError{code: "bad_param", msg: "invalid maxResults", detail: "maxResults"} + } + if !in.Schedule.Valid() { + return &codedError{code: "bad_param", msg: "invalid schedule", detail: "schedule"} + } + return nil +} + +func (a *API) savedList(w http.ResponseWriter, r *http.Request) { + writeJSON(w, http.StatusOK, a.store.List()) +} + +func (a *API) savedGet(w http.ResponseWriter, r *http.Request) { + x, err := a.store.Get(r.PathValue("id")) + if err != nil { + writeStoreErr(w, err) + return + } + writeJSON(w, http.StatusOK, x) +} + +func readInput(r *http.Request) (savedInput, error) { + var in savedInput + dec := json.NewDecoder(http.MaxBytesReader(nil, r.Body, 64<<10)) + if err := dec.Decode(&in); err != nil { + return in, &codedError{code: "bad_param", msg: "invalid JSON", detail: err.Error()} + } + return in, in.validate() +} + +func (a *API) savedCreate(w http.ResponseWriter, r *http.Request) { + in, err := readInput(r) + if err != nil { + writeErr(w, http.StatusBadRequest, err) + return + } + x := SavedSearch{Name: in.Name, Params: in.Params, Schedule: in.Schedule, MaxResults: in.MaxResults, Enabled: true} + if in.Enabled != nil { + x.Enabled = *in.Enabled + } + x.NextRunAt = x.Schedule.Next(time.Now(), time.Time{}) + if !x.Enabled { + x.NextRunAt = time.Time{} + } + x, err = a.store.Create(x) + if err != nil { + writeErr(w, http.StatusInternalServerError, err) + return + } + // First run right away: it gives a baseline to compare the next ones with. + go func(id string) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if _, err := a.sched.Run(ctx, id, "manual"); err != nil { + log.Printf("first run of saved search %s: %v", id, err) + } + }(x.ID) + writeJSON(w, http.StatusCreated, x) +} + +func (a *API) savedUpdate(w http.ResponseWriter, r *http.Request) { + in, err := readInput(r) + if err != nil { + writeErr(w, http.StatusBadRequest, err) + return + } + x, err := a.store.Update(r.PathValue("id"), func(v *SavedSearch) { + v.Name, v.Params, v.Schedule, v.MaxResults = in.Name, in.Params, in.Schedule, in.MaxResults + if in.Enabled != nil { + v.Enabled = *in.Enabled + } + v.NextRunAt = time.Time{} + if v.Enabled { + v.NextRunAt = v.Schedule.Next(time.Now(), v.LastRunAt) + } + }) + if err != nil { + writeStoreErr(w, err) + return + } + writeJSON(w, http.StatusOK, x) +} + +func (a *API) savedDelete(w http.ResponseWriter, r *http.Request) { + if err := a.store.Delete(r.PathValue("id")); err != nil { + writeStoreErr(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (a *API) savedRun(w http.ResponseWriter, r *http.Request) { + run, err := a.sched.Run(r.Context(), r.PathValue("id"), "manual") + if err != nil && run.ID == "" { + writeStoreErr(w, err) + return + } + // A failed GitHub search is still a recorded run: the error is in run.error. + writeJSON(w, http.StatusOK, run.RunInfo) +} + +func (a *API) savedRuns(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + if _, err := a.store.Get(id); err != nil { + writeStoreErr(w, err) + return + } + runs, err := a.store.Runs(id) + if err != nil { + writeStoreErr(w, err) + return + } + writeJSON(w, http.StatusOK, runs) +} + +func (a *API) savedRunGet(w http.ResponseWriter, r *http.Request) { + run, err := a.store.Run(r.PathValue("id"), r.PathValue("run")) + if err != nil { + writeStoreErr(w, err) + return + } + writeJSON(w, http.StatusOK, run) +} + +func writeStoreErr(w http.ResponseWriter, err error) { + status := http.StatusInternalServerError + var ce *codedError + if errors.As(err, &ce) { + switch ce.code { + case "not_found": + status = http.StatusNotFound + case "bad_param": + status = http.StatusBadRequest + } + } + writeErr(w, status, err) +} diff --git a/saved_api_test.go b/saved_api_test.go new file mode 100644 index 0000000..3212144 --- /dev/null +++ b/saved_api_test.go @@ -0,0 +1,124 @@ +package main + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func newTestAPI(t *testing.T) (*http.ServeMux, *API) { + t.Helper() + gh, _ := newFake(http.StatusOK, searchBody) + store, err := OpenStore(t.TempDir(), 0) + if err != nil { + t.Fatal(err) + } + a := &API{gh: gh, store: store, sched: NewScheduler(store, gh)} + mux := http.NewServeMux() + a.Routes(mux) + return mux, a +} + +func call(t *testing.T, mux *http.ServeMux, method, path, body string, out any) int { + t.Helper() + req := httptest.NewRequest(method, path, strings.NewReader(body)) + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + if out != nil && rec.Code < 300 { + if err := json.Unmarshal(rec.Body.Bytes(), out); err != nil { + t.Fatalf("%s %s: %v (%s)", method, path, err, rec.Body) + } + } + return rec.Code +} + +func TestSavedSearchAPI(t *testing.T) { + mux, a := newTestAPI(t) + + for _, bad := range []string{ + `{"name": "", "params": "q=x", "schedule": {"every": "week", "weekday": 1, "hour": 8}}`, + `{"name": "x", "params": "sort=nope", "schedule": {"every": "week", "weekday": 1, "hour": 8}}`, + `{"name": "x", "params": "q=x", "schedule": {"every": "month"}}`, + `{"name": "x", "params": "q=x", "maxResults": 7, "schedule": {"every": "day", "hour": 8}}`, + } { + if code := call(t, mux, "POST", "/api/saved", bad, nil); code != http.StatusBadRequest { + t.Errorf("%s: got %d, want 400", bad, code) + } + } + + var x SavedSearch + code := call(t, mux, "POST", "/api/saved", + `{"name": "Go syslog", "params": "q=syslog&language=Go&page=3", "schedule": {"every": "week", "weekday": 1, "hour": 8}}`, &x) + if code != http.StatusCreated || x.ID == "" || x.MaxResults != 50 || !x.Enabled || x.NextRunAt.IsZero() { + t.Fatalf("create: %d %+v", code, x) + } + if strings.Contains(x.Params, "page") { + t.Errorf("page should be dropped from params: %s", x.Params) + } + + // The first run starts in the background; run once more by hand. + var info RunInfo + if code := call(t, mux, "POST", "/api/saved/"+x.ID+"/run", "", &info); code != http.StatusOK || info.Error != "" || info.Count != 1 { + t.Fatalf("run: %d %+v", code, info) + } + deadline := time.Now().Add(2 * time.Second) + var runs []RunInfo + for time.Now().Before(deadline) { + call(t, mux, "GET", "/api/saved/"+x.ID+"/runs", "", &runs) + if len(runs) == 2 { + break + } + time.Sleep(20 * time.Millisecond) + } + if len(runs) != 2 { + t.Fatalf("want 2 runs (background + manual), got %d", len(runs)) + } + var run Run + if code := call(t, mux, "GET", "/api/saved/"+x.ID+"/runs/latest", "", &run); code != http.StatusOK || len(run.Items) != 1 || run.Items[0].FullName != "mcuadros/go-syslog" { + t.Fatalf("latest run: %d %+v", code, run) + } + if d := run.Items[0].StarsDelta; d == nil || *d != 0 || run.Items[0].New { + t.Errorf("second run: repository should be known with a 0 delta: %+v", run.Items[0]) + } + + var upd SavedSearch + code = call(t, mux, "PUT", "/api/saved/"+x.ID, + `{"name": "Renamed", "params": "q=syslog", "maxResults": 100, "enabled": false, "schedule": {"every": "hours", "hours": 6}}`, &upd) + if code != http.StatusOK || upd.Name != "Renamed" || upd.Enabled || !upd.NextRunAt.IsZero() || upd.LastRun == nil { + t.Fatalf("update: %d %+v", code, upd) + } + + if code := call(t, mux, "GET", "/api/saved/"+x.ID+"/runs/../../x", "", nil); code == http.StatusOK { + t.Error("path traversal should not succeed") + } + if code := call(t, mux, "DELETE", "/api/saved/"+x.ID, "", nil); code != http.StatusNoContent { + t.Errorf("delete: %d", code) + } + if code := call(t, mux, "GET", "/api/saved/"+x.ID, "", nil); code != http.StatusNotFound { + t.Errorf("get after delete: %d", code) + } + _ = a +} + +func TestSchedulerRunsDueSearches(t *testing.T) { + _, a := newTestAPI(t) + x, _ := a.store.Create(SavedSearch{Name: "due", Params: "q=syslog", MaxResults: 30, Enabled: true, + Schedule: Schedule{Every: "day", Hour: 8}, NextRunAt: time.Now().Add(-time.Minute)}) + a.store.Create(SavedSearch{Name: "paused", Params: "q=syslog", MaxResults: 30, + Schedule: Schedule{Every: "day", Hour: 8}, NextRunAt: time.Now().Add(-time.Minute)}) + a.sched.runDue(context.Background()) + + got, _ := a.store.Get(x.ID) + if got.LastRun == nil || !got.NextRunAt.After(time.Now()) { + t.Errorf("due search should have run and be planned again: %+v", got) + } + for _, s := range a.store.List() { + if s.Name == "paused" && s.LastRun != nil { + t.Error("a paused search must not run") + } + } +} diff --git a/scheduler.go b/scheduler.go new file mode 100644 index 0000000..5ba3f75 --- /dev/null +++ b/scheduler.go @@ -0,0 +1,163 @@ +package main + +import ( + "context" + "errors" + "log" + "net/url" + "strconv" + "sync" + "time" +) + +// Schedule says when a saved search runs, in the server time zone (TZ). +type Schedule struct { + Every string `json:"every"` // "hours", "day" or "week" + Hours int `json:"hours,omitempty"` // every: hours (1..168) + Weekday int `json:"weekday,omitempty"` // every: week, 0 = Sunday + Hour int `json:"hour"` // every: day / week + Minute int `json:"minute"` +} + +func (sc Schedule) Valid() bool { + switch sc.Every { + case "hours": + return sc.Hours >= 1 && sc.Hours <= 168 + case "day", "week": + return sc.Hour >= 0 && sc.Hour < 24 && sc.Minute >= 0 && sc.Minute < 60 && + sc.Weekday >= 0 && sc.Weekday < 7 + } + return false +} + +// Next returns the first run time strictly after "after". last is the time of +// the last run, used by the "hours" schedule. +func (sc Schedule) Next(after, last time.Time) time.Time { + switch sc.Every { + case "hours": + every := time.Duration(sc.Hours) * time.Hour + if last.IsZero() || last.Add(every).Before(after) { + return after.Add(time.Minute) + } + return last.Add(every) + case "day", "week": + t := time.Date(after.Year(), after.Month(), after.Day(), sc.Hour, sc.Minute, 0, 0, after.Location()) + for !t.After(after) || (sc.Every == "week" && int(t.Weekday()) != sc.Weekday) { + t = t.AddDate(0, 0, 1) + } + return t + } + return time.Time{} +} + +// Scheduler runs the saved searches when they are due. +type Scheduler struct { + store *Store + gh *GitHub + runMu sync.Mutex // one run at a time, to spare the GitHub quota +} + +func NewScheduler(store *Store, gh *GitHub) *Scheduler { + return &Scheduler{store: store, gh: gh} +} + +// Start checks every 30 seconds for due searches until ctx is done. +func (s *Scheduler) Start(ctx context.Context) { + now := time.Now() + for _, x := range s.store.List() { // plan searches that have no next run yet + if x.Enabled && x.NextRunAt.IsZero() { + _, _ = s.store.Update(x.ID, func(v *SavedSearch) { v.NextRunAt = v.Schedule.Next(now, v.LastRunAt) }) + } + } + go func() { + tick := time.NewTicker(30 * time.Second) + defer tick.Stop() + for { + s.runDue(ctx) + select { + case <-ctx.Done(): + return + case <-tick.C: + } + } + }() +} + +func (s *Scheduler) runDue(ctx context.Context) { + for _, x := range s.store.List() { + if ctx.Err() != nil { + return + } + if x.Enabled && !x.NextRunAt.IsZero() && !time.Now().Before(x.NextRunAt) { + if _, err := s.Run(ctx, x.ID, "schedule"); err != nil { + log.Printf("saved search %q: %v", x.Name, err) + } + } + } +} + +// Run executes a saved search now and stores the snapshot. +func (s *Scheduler) Run(ctx context.Context, id, trigger string) (Run, error) { + s.runMu.Lock() + defer s.runMu.Unlock() + x, err := s.store.Get(id) + if err != nil { + return Run{}, err + } + start := time.Now() + info := RunInfo{ID: start.UTC().Format("20060102T150405Z"), At: start, Trigger: trigger} + + var repos []Repo + c, err := criteriaFor(x) + if err == nil { + info.Query = c.Query(start) + var res SearchResult + if res, err = s.gh.Search(ctx, c); err == nil { + info.Total = res.Total + repos = res.Items + } + } + info.TookMs = time.Since(start).Milliseconds() + if err != nil { + info.Error = err.Error() + } + run, serr := s.store.AddRun(id, info, repos) + if serr != nil { + return Run{}, serr + } + + _, uerr := s.store.Update(id, func(v *SavedSearch) { + v.LastRunAt = start + v.LastError = info.Error + ri := run.RunInfo + v.LastRun = &ri + next := v.Schedule.Next(time.Now(), start) + if err != nil { + var ce *codedError + if errors.As(err, &ce) && ce.code == "rate_limited" { + // Quota exhausted: try again in a few minutes rather than next week. + next = time.Now().Add(3 * time.Minute) + } + } + v.NextRunAt = next + }) + if uerr != nil { + return run, uerr + } + if err != nil { + return run, err + } + return run, nil +} + +// criteriaFor parses the stored parameters; a saved search always reads the +// first page, sorted as saved. +func criteriaFor(x SavedSearch) (Criteria, error) { + v, err := url.ParseQuery(x.Params) + if err != nil { + return Criteria{}, &codedError{code: "bad_param", msg: "invalid parameters", detail: err.Error()} + } + v.Del("page") + v.Set("per_page", strconv.Itoa(x.MaxResults)) + return ParseCriteria(v) +} diff --git a/scheduler_test.go b/scheduler_test.go new file mode 100644 index 0000000..2c2d545 --- /dev/null +++ b/scheduler_test.go @@ -0,0 +1,100 @@ +package main + +import ( + "testing" + "time" +) + +func TestScheduleNext(t *testing.T) { + loc := time.FixedZone("CEST", 2*3600) + // Wednesday 2026-09-30 10:00 + now := time.Date(2026, 9, 30, 10, 0, 0, 0, loc) + cases := []struct { + name string + sc Schedule + last time.Time + want time.Time + }{ + {"weekly monday 8:00", Schedule{Every: "week", Weekday: 1, Hour: 8}, time.Time{}, time.Date(2026, 10, 5, 8, 0, 0, 0, loc)}, + {"weekly today later", Schedule{Every: "week", Weekday: 3, Hour: 18, Minute: 30}, time.Time{}, time.Date(2026, 9, 30, 18, 30, 0, 0, loc)}, + {"weekly today passed", Schedule{Every: "week", Weekday: 3, Hour: 9}, time.Time{}, time.Date(2026, 10, 7, 9, 0, 0, 0, loc)}, + {"daily passed", Schedule{Every: "day", Hour: 7}, time.Time{}, time.Date(2026, 10, 1, 7, 0, 0, 0, loc)}, + {"hours first", Schedule{Every: "hours", Hours: 6}, time.Time{}, now.Add(time.Minute)}, + {"hours after last", Schedule{Every: "hours", Hours: 6}, now.Add(-2 * time.Hour), now.Add(4 * time.Hour)}, + {"hours late", Schedule{Every: "hours", Hours: 6}, now.Add(-7 * time.Hour), now.Add(time.Minute)}, + } + for _, c := range cases { + if got := c.sc.Next(now, c.last); !got.Equal(c.want) { + t.Errorf("%s: got %v, want %v", c.name, got, c.want) + } + } +} + +func TestScheduleValid(t *testing.T) { + for _, sc := range []Schedule{{Every: "hours"}, {Every: "hours", Hours: 200}, {Every: "week", Weekday: 7}, {Every: "day", Hour: 24}, {Every: "month"}} { + if sc.Valid() { + t.Errorf("%+v should be invalid", sc) + } + } + if !(Schedule{Every: "week", Weekday: 1, Hour: 8}).Valid() { + t.Error("weekly schedule should be valid") + } +} + +func TestStoreRuns(t *testing.T) { + s, err := OpenStore(t.TempDir(), 2) + if err != nil { + t.Fatal(err) + } + x, err := s.Create(SavedSearch{Name: "go", Params: "language=Go", Schedule: Schedule{Every: "day"}, MaxResults: 30, Enabled: true}) + if err != nil { + t.Fatal(err) + } + t0 := time.Date(2026, 9, 1, 8, 0, 0, 0, time.UTC) + add := func(id string, at time.Time, repos ...Repo) Run { + t.Helper() + r, err := s.AddRun(x.ID, RunInfo{ID: id, At: at}, repos) + if err != nil { + t.Fatal(err) + } + return r + } + + r1 := add("r1", t0, Repo{FullName: "a/a", Stars: 10}, Repo{FullName: "b/b", Stars: 5}) + if r1.New != 0 || r1.Items[0].New || r1.Items[0].StarsDelta != nil { + t.Errorf("first run must be a baseline: %+v", r1) + } + + r2 := add("r2", t0.Add(24*time.Hour), Repo{FullName: "a/a", Stars: 25}, Repo{FullName: "c/c", Stars: 3}) + if r2.New != 1 || !r2.Items[1].New || r2.Items[0].New { + t.Errorf("c/c should be the only new repository: %+v", r2.Items) + } + if d := r2.Items[0].StarsDelta; d == nil || *d != 15 || r2.Rising != 1 { + t.Errorf("a/a should have gained 15 stars: %+v", r2.Items[0]) + } + + // A failed run is recorded but does not break the comparison. + s.AddRun(x.ID, RunInfo{ID: "r3", At: t0.Add(48 * time.Hour), Error: "boom"}, nil) + r4 := add("r4", t0.Add(72*time.Hour), Repo{FullName: "b/b", Stars: 6}, Repo{FullName: "c/c", Stars: 4}) + if r4.New != 0 { + t.Errorf("b/b was seen in the first run, not new: %+v", r4.Items) + } + + idx, _ := s.Runs(x.ID) + if len(idx) != 2 || idx[0].ID != "r4" { + t.Errorf("history should keep the 2 newest runs, got %+v", idx) + } + if _, err := s.Run(x.ID, "r1"); err == nil { + t.Error("r1 should have been pruned") + } + latest, err := s.Run(x.ID, "latest") + if err != nil || latest.ID != "r4" { + t.Errorf("latest = %v, %v", latest.ID, err) + } + if err := s.Delete(x.ID); err != nil { + t.Fatal(err) + } + if _, err := s.Get(x.ID); err == nil { + t.Error("search should be deleted") + } +} diff --git a/store.go b/store.go new file mode 100644 index 0000000..a850c65 --- /dev/null +++ b/store.go @@ -0,0 +1,301 @@ +package main + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "sync" + "time" +) + +// SavedSearch is a search that the scheduler runs on a regular basis. +type SavedSearch struct { + ID string `json:"id"` + Name string `json:"name"` + Params string `json:"params"` // same query string as /api/search (without page) + Schedule Schedule `json:"schedule"` + MaxResults int `json:"maxResults"` // 30, 50 or 100 (one GitHub page) + Enabled bool `json:"enabled"` + CreatedAt time.Time `json:"createdAt"` + LastRunAt time.Time `json:"lastRunAt,omitzero"` + NextRunAt time.Time `json:"nextRunAt,omitzero"` + LastError string `json:"lastError,omitempty"` + LastRun *RunInfo `json:"lastRun,omitempty"` +} + +// RunInfo summarizes one execution of a saved search. +type RunInfo struct { + ID string `json:"id"` + At time.Time `json:"at"` + Query string `json:"query"` + Total int `json:"total"` + Count int `json:"count"` // repositories kept in the snapshot + New int `json:"new"` // never seen in a previous run + Rising int `json:"rising"` // gained stars since the previous run + Error string `json:"error,omitempty"` + TookMs int64 `json:"tookMs"` + Trigger string `json:"trigger"` // "schedule" or "manual" +} + +// RunItem is one repository of a snapshot, compared with the previous run. +type RunItem struct { + Repo + New bool `json:"new"` + FirstSeen time.Time `json:"firstSeen"` + StarsDelta *int `json:"starsDelta,omitempty"` // nil when absent from the previous run + Rank int `json:"rank"` +} + +// Run is a full snapshot. +type Run struct { + RunInfo + Items []RunItem `json:"items"` +} + +// Store keeps saved searches and their runs as JSON files: +// +// /searches.json +// /runs//index.json run summaries, newest first +// /runs//seen.json repository -> first time seen +// /runs//.json +type Store struct { + dir string + keep int // runs kept per search (0 = all) + + mu sync.Mutex + searches []*SavedSearch +} + +var idRe = regexp.MustCompile(`^[a-zA-Z0-9-]{1,40}$`) + +var errNotFound = &codedError{code: "not_found", msg: "not found"} + +func OpenStore(dir string, keep int) (*Store, error) { + if err := os.MkdirAll(filepath.Join(dir, "runs"), 0o755); err != nil { + return nil, err + } + s := &Store{dir: dir, keep: keep} + if err := readJSON(filepath.Join(dir, "searches.json"), &s.searches); err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + return s, nil +} + +func newID() string { + b := make([]byte, 6) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} + +// List returns copies of the saved searches, sorted by name. +func (s *Store) List() []SavedSearch { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]SavedSearch, 0, len(s.searches)) + for _, x := range s.searches { + out = append(out, *x) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out +} + +func (s *Store) Get(id string) (SavedSearch, error) { + s.mu.Lock() + defer s.mu.Unlock() + if x := s.find(id); x != nil { + return *x, nil + } + return SavedSearch{}, errNotFound +} + +func (s *Store) find(id string) *SavedSearch { + for _, x := range s.searches { + if x.ID == id { + return x + } + } + return nil +} + +func (s *Store) Create(x SavedSearch) (SavedSearch, error) { + s.mu.Lock() + defer s.mu.Unlock() + x.ID = newID() + x.CreatedAt = time.Now() + s.searches = append(s.searches, &x) + return x, s.saveLocked() +} + +// Update changes what the user can edit and returns the new value. +func (s *Store) Update(id string, fn func(*SavedSearch)) (SavedSearch, error) { + s.mu.Lock() + defer s.mu.Unlock() + x := s.find(id) + if x == nil { + return SavedSearch{}, errNotFound + } + fn(x) + return *x, s.saveLocked() +} + +func (s *Store) Delete(id string) error { + s.mu.Lock() + defer s.mu.Unlock() + for i, x := range s.searches { + if x.ID == id { + s.searches = append(s.searches[:i], s.searches[i+1:]...) + if err := s.saveLocked(); err != nil { + return err + } + return os.RemoveAll(s.runDir(id)) + } + } + return errNotFound +} + +func (s *Store) saveLocked() error { + return writeFileJSON(filepath.Join(s.dir, "searches.json"), s.searches) +} + +func (s *Store) runDir(id string) string { return filepath.Join(s.dir, "runs", id) } + +// Runs returns the run summaries of a search, newest first. +func (s *Store) Runs(id string) ([]RunInfo, error) { + if !idRe.MatchString(id) { + return nil, errNotFound + } + var idx []RunInfo + err := readJSON(filepath.Join(s.runDir(id), "index.json"), &idx) + if errors.Is(err, os.ErrNotExist) { + return []RunInfo{}, nil + } + return idx, err +} + +// Run reads one snapshot ("latest" for the most recent one). +func (s *Store) Run(id, runID string) (Run, error) { + if runID == "latest" { + idx, err := s.Runs(id) + if err != nil { + return Run{}, err + } + if len(idx) == 0 { + return Run{}, errNotFound + } + runID = idx[0].ID + } + if !idRe.MatchString(id) || !idRe.MatchString(runID) { + return Run{}, errNotFound + } + var r Run + err := readJSON(filepath.Join(s.runDir(id), runID+".json"), &r) + if errors.Is(err, os.ErrNotExist) { + return Run{}, errNotFound + } + return r, err +} + +// AddRun compares the repositories with the previous run, saves the snapshot +// and updates the search. A failed run (info.Error set) is only recorded in +// the history. +func (s *Store) AddRun(id string, info RunInfo, repos []Repo) (Run, error) { + dir := s.runDir(id) + if err := os.MkdirAll(dir, 0o755); err != nil { + return Run{}, err + } + idx, err := s.Runs(id) + if err != nil { + return Run{}, err + } + run := Run{RunInfo: info, Items: []RunItem{}} + + if info.Error == "" { + seen := map[string]time.Time{} + if err := readJSON(filepath.Join(dir, "seen.json"), &seen); err != nil && !errors.Is(err, os.ErrNotExist) { + return Run{}, err + } + prevStars := map[string]int{} + for _, p := range idx { // previous successful run + if p.Error != "" { + continue + } + if prev, err := s.Run(id, p.ID); err == nil { + for _, it := range prev.Items { + prevStars[it.FullName] = it.Stars + } + } + break + } + baseline := len(seen) == 0 // the first successful run has nothing "new" + for i, r := range repos { + it := RunItem{Repo: r, Rank: i + 1} + if first, ok := seen[r.FullName]; ok { + it.FirstSeen = first + } else { + it.New = !baseline + it.FirstSeen = info.At + seen[r.FullName] = info.At + if it.New { + run.New++ + } + } + if p, ok := prevStars[r.FullName]; ok { + d := r.Stars - p + it.StarsDelta = &d + if d > 0 { + run.Rising++ + } + } + run.Items = append(run.Items, it) + } + run.Count = len(run.Items) + if err := writeFileJSON(filepath.Join(dir, "seen.json"), seen); err != nil { + return Run{}, err + } + } + + if err := writeFileJSON(filepath.Join(dir, run.ID+".json"), run); err != nil { + return Run{}, err + } + idx = append([]RunInfo{run.RunInfo}, idx...) + if s.keep > 0 && len(idx) > s.keep { + for _, old := range idx[s.keep:] { + _ = os.Remove(filepath.Join(dir, old.ID+".json")) + } + idx = idx[:s.keep] + } + if err := writeFileJSON(filepath.Join(dir, "index.json"), idx); err != nil { + return Run{}, err + } + return run, nil +} + +func readJSON(path string, v any) error { + data, err := os.ReadFile(path) + if err != nil { + return err + } + if err := json.Unmarshal(data, v); err != nil { + return fmt.Errorf("%s: %w", path, err) + } + return nil +} + +// writeFileJSON replaces the file atomically. +func writeFileJSON(path string, v any) error { + data, err := json.Marshal(v) + if err != nil { + return err + } + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return err + } + return os.Rename(tmp, path) +} diff --git a/web/app.js b/web/app.js index 7f39034..97e9c62 100644 --- a/web/app.js +++ b/web/app.js @@ -68,6 +68,41 @@ const I18N = { syntaxHelp: 'The search field accepts GitHub qualifiers, combined with the filters above the list:', synUser: 'repositories of a user', synOrg: 'repositories of an organization', synTopics: 'several topics at once', synPhrase: 'exact phrase', synNot: 'exclude a word', synSize: 'size in KB', + pushed1w: 'Pushed < 1 week', created1w: 'Created < 1 week', created3m: 'Created < 3 months', created6m: 'Created < 6 months', + viewsAria: 'Views', viewSearch: 'Search', viewSaved: 'Watch', + saveBtn: 'Schedule', saveBtnTitle: 'Run this search on a regular basis and review what changes', + savedTitle: 'Scheduled searches', savedNew: 'New from a search', + savedCount: (n) => `${n} ${n === 1 ? 'search' : 'searches'}`, + savedEmptyTitle: 'No scheduled search yet.', + savedEmptyHint: 'Set the filters in Search (for example: created < 1 week, most stars, a topic), then click Schedule.', + newTotal: (n) => `${n} new ${n === 1 ? 'repository' : 'repositories'} in the latest runs`, + schedHours: (n) => `Every ${n} h`, schedDay: (tm) => `Every day at ${tm}`, schedWeek: ({ d, tm }) => `Every ${d} at ${tm}`, + paused: 'paused', nextRun: (s) => `next ${s}`, lastRun: (s) => `last run ${s}`, neverRun: 'never run', + running: 'Running…', statNew: (n) => `${n} new`, statRising: (n) => `${n} rising`, statCount: (n) => `${n} repos`, statErr: 'error', + runNow: 'Run now', runNowTitle: 'Run the search now and keep a new snapshot', + runDone: ({ n, r }) => `Done: ${n} new, ${r} rising`, runFailed: 'Run failed: ', + edit: 'Edit', delete: 'Delete', cancel: 'Cancel', save: 'Save', back: 'Back to the list', + pause: 'Pause', resume: 'Resume', + confirmDelete: (n) => `Delete "${n}" and its history?`, deleted: 'Deleted', + openInSearch: 'Open in search', openInSearchTitle: 'Load these filters in the search view', + runAria: 'Run', revModeAria: 'Show', + modeNew: (n) => `New · ${n}`, modeRising: (n) => `Rising · ${n}`, modeAll: (n) => `All · ${n}`, + runOption: ({ d, n }) => `${d} · ${n} new`, runOptionErr: (d) => `${d} · error`, + revInfo: ({ d, total, q }) => `${d} · ${total} matching on GitHub · ${q}`, + revEmptyNew: 'Nothing new in this run.', revEmptyNewHint: 'Every repository was already in a previous run.', + revEmptyFirst: 'This is the first run.', revEmptyFirstHint: 'It is the baseline: the next runs will show what is new and what is rising. "All" shows this snapshot.', + revEmptyRising: 'No star gains since the previous run.', revEmptyRisingHint: 'Rising compares each repository with the previous snapshot.', + revEmptyAll: 'This run returned no repository.', revEmptyAllHint: '', + revNoRun: 'No run yet.', revNoRunHint: 'The first run is on its way, or click Run now.', + badgeNew: 'new', deltaTitle: 'stars gained since the previous run', rankTitle: 'position in the results', + saveNewTitle: 'Schedule this search', saveEditTitle: 'Edit the scheduled search', + svName: 'Name', svQuery: 'GitHub query', svEvery: 'Frequency', svWeekday: 'Day', svTime: 'Time', svHours: 'Hours', + everyWeek: 'Every week', everyDay: 'Every day', everyHours: 'Every N hours', hoursUnit: 'hours', + svMax: 'Results kept', svEnabled: 'Active', svEnabledHelp: 'Run on schedule', + svHintNew: (tz) => `The search runs once right away to set the baseline, then on schedule (server time zone: ${tz}).`, + svHintEdit: (tz) => `Server time zone: ${tz}. To change the filters, open the search, adjust it and schedule it again.`, + saved: 'Search scheduled', updated2: 'Saved', + err_not_found: 'Not found', }, fr: { locale: 'fr-FR', @@ -134,6 +169,41 @@ const I18N = { syntaxHelp: 'Le champ de recherche accepte les qualificatifs GitHub, combinés aux filtres au-dessus de la liste :', synUser: 'dépôts d\'un utilisateur', synOrg: 'dépôts d\'une organisation', synTopics: 'plusieurs topics à la fois', synPhrase: 'expression exacte', synNot: 'exclure un mot', synSize: 'taille en Ko', + pushed1w: 'Push < 1 semaine', created1w: 'Créé < 1 semaine', created3m: 'Créé < 3 mois', created6m: 'Créé < 6 mois', + viewsAria: 'Vues', viewSearch: 'Recherche', viewSaved: 'Veille', + saveBtn: 'Programmer', saveBtnTitle: 'Lancer cette recherche régulièrement et suivre ce qui change', + savedTitle: 'Recherches programmées', savedNew: 'Nouvelle depuis une recherche', + savedCount: (n) => `${n} recherche${n > 1 ? 's' : ''}`, + savedEmptyTitle: 'Aucune recherche programmée.', + savedEmptyHint: 'Réglez les filtres dans Recherche (par exemple : créé < 1 semaine, plus d\'étoiles, un topic), puis cliquez sur Programmer.', + newTotal: (n) => `${n} nouveau${n > 1 ? 'x' : ''} dépôt${n > 1 ? 's' : ''} dans les derniers passages`, + schedHours: (n) => `Toutes les ${n} h`, schedDay: (tm) => `Chaque jour à ${tm}`, schedWeek: ({ d, tm }) => `Chaque ${d} à ${tm}`, + paused: 'en pause', nextRun: (s) => `prochain passage ${s}`, lastRun: (s) => `dernier passage ${s}`, neverRun: 'jamais exécutée', + running: 'En cours…', statNew: (n) => `${n} nouveau${n > 1 ? 'x' : ''}`, statRising: (n) => `${n} en hausse`, statCount: (n) => `${n} dépôts`, statErr: 'erreur', + runNow: 'Exécuter', runNowTitle: 'Lancer la recherche maintenant et garder un nouvel instantané', + runDone: ({ n, r }) => `Terminé : ${n} nouveau${n > 1 ? 'x' : ''}, ${r} en hausse`, runFailed: 'Échec du passage : ', + edit: 'Modifier', delete: 'Supprimer', cancel: 'Annuler', save: 'Enregistrer', back: 'Retour à la liste', + pause: 'Mettre en pause', resume: 'Réactiver', + confirmDelete: (n) => `Supprimer « ${n} » et son historique ?`, deleted: 'Supprimée', + openInSearch: 'Ouvrir dans la recherche', openInSearchTitle: 'Charger ces filtres dans la vue Recherche', + runAria: 'Passage', revModeAria: 'Afficher', + modeNew: (n) => `Nouveautés · ${n}`, modeRising: (n) => `Progressions · ${n}`, modeAll: (n) => `Tous · ${n}`, + runOption: ({ d, n }) => `${d} · ${n} nouveau${n > 1 ? 'x' : ''}`, runOptionErr: (d) => `${d} · erreur`, + revInfo: ({ d, total, q }) => `${d} · ${total} résultats sur GitHub · ${q}`, + revEmptyNew: 'Rien de nouveau dans ce passage.', revEmptyNewHint: 'Tous les dépôts étaient déjà présents dans un passage précédent.', + revEmptyFirst: 'C\'est le premier passage.', revEmptyFirstHint: 'Il sert de référence : les suivants montreront les nouveautés et les progressions. « Tous » affiche cet instantané.', + revEmptyRising: 'Aucun gain d\'étoiles depuis le passage précédent.', revEmptyRisingHint: 'Progressions compare chaque dépôt avec l\'instantané précédent.', + revEmptyAll: 'Ce passage n\'a renvoyé aucun dépôt.', revEmptyAllHint: '', + revNoRun: 'Pas encore de passage.', revNoRunHint: 'Le premier passage est en cours, ou cliquez sur Exécuter.', + badgeNew: 'nouveau', deltaTitle: 'étoiles gagnées depuis le passage précédent', rankTitle: 'position dans les résultats', + saveNewTitle: 'Programmer cette recherche', saveEditTitle: 'Modifier la recherche programmée', + svName: 'Nom', svQuery: 'Requête GitHub', svEvery: 'Fréquence', svWeekday: 'Jour', svTime: 'Heure', svHours: 'Heures', + everyWeek: 'Chaque semaine', everyDay: 'Chaque jour', everyHours: 'Toutes les N heures', hoursUnit: 'heures', + svMax: 'Résultats gardés', svEnabled: 'Active', svEnabledHelp: 'Exécuter selon la fréquence', + svHintNew: (tz) => `La recherche est lancée une première fois tout de suite pour servir de référence, puis selon la fréquence (fuseau du serveur : ${tz}).`, + svHintEdit: (tz) => `Fuseau du serveur : ${tz}. Pour changer les filtres, ouvrez la recherche, ajustez-la et programmez-la à nouveau.`, + saved: 'Recherche programmée', updated2: 'Enregistré', + err_not_found: 'Introuvable', }, }; @@ -171,8 +241,8 @@ function fmtAgo(s) { return ''; } -async function api(url) { - const res = await fetch(url); +async function api(url, opts) { + const res = await fetch(url, opts); const text = await res.text(); let data = null; try { data = text ? JSON.parse(text) : null; } catch { /* not JSON */ } @@ -191,8 +261,8 @@ function apiError(res, text, data) { function toast(msg) { const el = $('#toast'); - const dlg = $('#settingsDlg'); - (dlg.open ? dlg : document.body).append(el); + const dlg = document.querySelector('dialog[open]'); + (dlg || document.body).append(el); el.textContent = msg; el.hidden = false; clearTimeout(toast.tm); @@ -291,7 +361,11 @@ function syncURL(f) { } function filtersFromURL() { - const src = location.search ? new URLSearchParams(location.search) : new URLSearchParams(store.get('filters', '')); + return filtersFromParams(location.search || store.get('filters', '')); +} + +function filtersFromParams(qs) { + const src = new URLSearchParams(qs); const f = {}; for (const k of FILTERS) f[k] = src.get(k) || ''; for (const k of TOGGLES) f[k] = src.get(k) === '1'; @@ -352,7 +426,9 @@ const ICON_EXT = '${esc(t('badgeNew'))}` : '') + + (r.archived ? `${esc(t('archived'))}` : '') + (r.fork ? `${esc(t('fork'))}` : ''); const topics = r.topics.length ? `
${r.topics.map((x) => ``).join('')}
` @@ -360,7 +436,10 @@ function rowHTML(r, i) { const lang = r.language ? `${esc(r.language)}` : ''; - return `
+ const delta = !review ? '' + : r.starsDelta ? `${r.starsDelta > 0 ? '+' : ''}${esc(fmtCompact(r.starsDelta))} ★` + : ''; + return `
${esc(r.owner)}/${esc(r.name)}${badges}
@@ -368,7 +447,7 @@ function rowHTML(r, i) { ${topics}
- ${lang} + ${delta}${lang} ${ICON_STAR}${fmtCompact(r.stars)} ${ICON_FORK}${fmtCompact(r.forks)} ${ICON_CLOCK}${esc(fmtAgo(r.pushedAt))} @@ -449,6 +528,7 @@ async function loadStatus() { try { const s = await api('/api/status'); state.token = s.token; + state.tz = s.tz || ''; if (s.rate && s.rate.limit) state.rate = s.rate; } catch { state.token = null; } renderStatus(); @@ -456,38 +536,49 @@ async function loadStatus() { /* ================= List interactions ================= */ -$('#list').addEventListener('click', (ev) => { - const rowEl = ev.target.closest('.row'); - if (!rowEl) return; - const r = state.items[Number(rowEl.dataset.i)]; - const topic = ev.target.closest('[data-topic]'); - if (topic) { - $('#topic').value = topic.dataset.topic; - onFilterChange(); - return; - } - if (ev.target.closest('[data-act="lang"]')) { - writeFilters({ ...readFilters(), language: r.language }); - onFilterChange(); - return; - } - const cp = ev.target.closest('[data-copy]'); - if (cp) { copy(cp.dataset.copy); return; } - if (ev.target.closest('a, .details')) return; - toggleRow(rowEl, r); -}); +// The search list and the review list share the same rows; items() returns +// the array the data-i indexes refer to. +function listEvents(listEl, items) { + listEl.addEventListener('click', (ev) => { + const rowEl = ev.target.closest('.row'); + if (!rowEl) return; + const r = items()[Number(rowEl.dataset.i)]; + const topic = ev.target.closest('[data-topic]'); + if (topic) { + filterFromList({ topic: topic.dataset.topic }); + return; + } + if (ev.target.closest('[data-act="lang"]')) { + filterFromList({ language: r.language }); + return; + } + const cp = ev.target.closest('[data-copy]'); + if (cp) { copy(cp.dataset.copy); return; } + if (ev.target.closest('a, .details')) return; + toggleRow(listEl, rowEl, r); + }); + listEl.addEventListener('keydown', (ev) => { + if (ev.key !== 'Enter' || !ev.target.classList.contains('row')) return; + toggleRow(listEl, ev.target, items()[Number(ev.target.dataset.i)]); + }); +} +listEvents($('#list'), () => state.items); -$('#list').addEventListener('keydown', (ev) => { - if (ev.key !== 'Enter' || !ev.target.classList.contains('row')) return; - toggleRow(ev.target, state.items[Number(ev.target.dataset.i)]); -}); +// A click on a topic or a language filters the search (from the review too). +function filterFromList(change) { + const base = view === 'search' ? readFilters() : review.search ? filtersFromParams(review.search.params) : {}; + writeFilters({ ...base, ...change }); + state.searchPending = false; + if (view !== 'search') location.hash = ''; + onFilterChange(); +} -function toggleRow(rowEl, r) { +function toggleRow(listEl, rowEl, r) { if (state.open.has(r.fullName)) state.open.delete(r.fullName); else state.open.add(r.fullName); const i = Number(rowEl.dataset.i); rowEl.outerHTML = rowHTML(r, i); - $(`#list .row[data-i="${i}"]`).focus({ preventScroll: true }); + listEl.querySelector(`.row[data-i="${i}"]`).focus({ preventScroll: true }); } /* ================= Filter events ================= */ @@ -516,7 +607,7 @@ $('#resetBtn').addEventListener('click', () => { $('#moreBtn').addEventListener('click', () => search(true)); document.addEventListener('keydown', (ev) => { - if (ev.key === '/' && !ev.target.closest('input, select, textarea') && !$('#settingsDlg').open) { + if (ev.key === '/' && view === 'search' && !ev.target.closest('input, select, textarea') && !document.querySelector('dialog[open]')) { ev.preventDefault(); $('#q').focus(); $('#q').select(); @@ -567,6 +658,8 @@ function setLang(next) { applyLang(); renderList(); renderStatus(); + renderSaved(); + renderReview(); } $('#langSwitch').addEventListener('click', (ev) => { @@ -663,6 +756,400 @@ $('#settingsBtn').addEventListener('click', () => { $('#closeSettings').addEventListener('click', () => $('#settingsDlg').close()); $('#settingsDlg').addEventListener('click', (ev) => { if (ev.target === ev.currentTarget) ev.currentTarget.close(); }); +/* ================= Views ================= */ + +// # search +// #saved scheduled searches +// #saved/ review of one scheduled search (#saved// for an older run) +let view = 'search'; + +function route() { + const parts = location.hash.replace(/^#\/?/, '').split('/').filter(Boolean); + if (parts[0] === 'saved' && parts[1]) { + view = 'review'; + openReview(decodeURIComponent(parts[1]), parts[2] ? decodeURIComponent(parts[2]) : 'latest'); + } else if (parts[0] === 'saved') { + view = 'saved'; + loadSaved(); + } else { + view = 'search'; + if (state.searchPending) { state.searchPending = false; search(); } + } + document.body.dataset.view = view; + $('#viewSearch').hidden = view !== 'search'; + $('#viewSaved').hidden = view !== 'saved'; + $('#viewReview').hidden = view !== 'review'; + for (const a of document.querySelectorAll('#views [data-view]')) { + const cur = a.dataset.view === (view === 'review' ? 'saved' : view); + if (cur) a.setAttribute('aria-current', 'page'); else a.removeAttribute('aria-current'); + } + window.scrollTo({ top: 0 }); +} +window.addEventListener('hashchange', route); + +/* ================= Scheduled searches ================= */ + +const ICON_PLAY = ''; +const ICON_EDIT = ''; +const ICON_PAUSE = ''; +const ICON_TRASH = ''; +const ICON_UP = ''; +const ICON_SPIN = ''; + +const saved = { list: [], loaded: false, running: new Set() }; +const pad2 = (n) => String(n).padStart(2, '0'); +const dayName = (d) => new Intl.DateTimeFormat(t('locale'), { weekday: 'long', timeZone: 'UTC' }).format(new Date(Date.UTC(2023, 0, 1 + d))); +const fmtDateTime = (s) => new Date(s).toLocaleString(t('locale'), { weekday: 'short', day: 'numeric', month: 'short', hour: '2-digit', minute: '2-digit' }); + +function schedText(sc) { + const tm = pad2(sc.hour) + ':' + pad2(sc.minute); + if (sc.every === 'hours') return t('schedHours', sc.hours); + if (sc.every === 'day') return t('schedDay', tm); + return t('schedWeek', { d: dayName(sc.weekday), tm }); +} + +// Readable GitHub query of saved parameters (computed by the server on each run). +function savedQuery(x) { + return (x.lastRun && x.lastRun.query) || new URLSearchParams(x.params).toString().replace(/&/g, ' '); +} + +async function loadSaved() { + try { + saved.list = await api('/api/saved'); + saved.loaded = true; + $('#savedError').hidden = true; + } catch (e) { + $('#savedError').textContent = e.message; + $('#savedError').hidden = false; + } + renderSaved(); +} + +function renderSaved() { + const n = saved.list.reduce((a, x) => a + (x.enabled && x.lastRun && !x.lastRun.error ? x.lastRun.new : 0), 0); + const pill = $('#newTotal'); + pill.hidden = !n; + pill.textContent = n; + pill.title = n ? t('newTotal', n) : ''; + if (!saved.loaded) return; + $('#savedCount').textContent = saved.list.length ? t('savedCount', saved.list.length) : ''; + $('#savedEmpty').hidden = saved.list.length > 0 || !$('#savedError').hidden; + $('#savedList').innerHTML = saved.list.map((x) => { + const lr = x.lastRun; + let stats = ''; + if (saved.running.has(x.id)) stats = `${ICON_SPIN}${esc(t('running'))}`; + else if (lr && lr.error) stats = `${esc(t('statErr'))}`; + else if (lr) { + stats = (lr.new ? `${esc(t('statNew', lr.new))}` : '') + + (lr.rising ? `${ICON_UP}${esc(t('statRising', lr.rising))}` : '') + + `${esc(t('statCount', lr.count))}`; + } + const when = [ + `${ICON_CLOCK}${esc(schedText(x.schedule))}`, + x.enabled ? (x.nextRunAt ? `${esc(t('nextRun', fmtAgo(x.nextRunAt)))}` : '') : `${esc(t('paused'))}`, + `${esc(x.lastRunAt ? t('lastRun', fmtAgo(x.lastRunAt)) : t('neverRun'))}`, + ].join(''); + return `
+
+
${esc(x.name)}
+
${esc(savedQuery(x))}
+
${when}
+
+
${stats}
+
+ + + + +
+
`; + }).join(''); +} + +$('#savedList').addEventListener('click', (ev) => { + const row = ev.target.closest('.srow'); + if (!row) return; + const x = saved.list.find((s) => s.id === row.dataset.id); + const act = ev.target.closest('[data-act]'); + if (!act) { location.hash = '#saved/' + encodeURIComponent(x.id); return; } + if (act.dataset.act === 'run') runSaved(x.id); + else if (act.dataset.act === 'edit') openSaveDlg(x); + else if (act.dataset.act === 'delete') deleteSaved(x); + else if (act.dataset.act === 'toggle') putSaved(x, { enabled: !x.enabled }); +}); +$('#savedList').addEventListener('keydown', (ev) => { + if (ev.key === 'Enter' && ev.target.classList.contains('srow')) location.hash = '#saved/' + encodeURIComponent(ev.target.dataset.id); +}); + +async function runSaved(id) { + saved.running.add(id); + renderSaved(); + $('#revRunBtn').disabled = true; + try { + const info = await api('/api/saved/' + encodeURIComponent(id) + '/run', { method: 'POST' }); + toast(info.error ? t('runFailed') + info.error : t('runDone', { n: info.new, r: info.rising })); + } catch (e) { + toast(t('runFailed') + e.message); + } finally { + saved.running.delete(id); + $('#revRunBtn').disabled = false; + } + await loadSaved(); + if (view === 'review' && review.id === id) openReview(id, 'latest'); +} + +async function putSaved(x, change) { + const body = { name: x.name, params: x.params, schedule: x.schedule, maxResults: x.maxResults, enabled: x.enabled, ...change }; + try { + await api('/api/saved/' + encodeURIComponent(x.id), { method: 'PUT', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + toast(t('updated2')); + } catch (e) { + toast(e.message); + } + await loadSaved(); + if (view === 'review') renderReviewHead(); +} + +async function deleteSaved(x) { + if (!confirm(t('confirmDelete', x.name))) return false; + try { + await fetch('/api/saved/' + encodeURIComponent(x.id), { method: 'DELETE' }); + toast(t('deleted')); + } catch (e) { + toast(e.message); + return false; + } + await loadSaved(); + return true; +} + +/* ---------- Review of one scheduled search ---------- */ + +const review = { id: '', search: null, runs: [], run: null, mode: '', items: [], req: 0 }; +listEvents($('#revList'), () => review.items); + +async function openReview(id, runID) { + const req = ++review.req; + if (review.id !== id) { review.mode = ''; review.run = null; review.runs = []; review.search = null; } + review.id = id; + document.body.classList.add('busy'); + try { + if (!saved.loaded) await loadSaved(); + const [x, runs] = await Promise.all([ + api('/api/saved/' + encodeURIComponent(id)), + api('/api/saved/' + encodeURIComponent(id) + '/runs'), + ]); + let run = null; + if (runs.length) { + try { run = await api('/api/saved/' + encodeURIComponent(id) + '/runs/' + encodeURIComponent(runID)); } catch { /* pruned run */ } + } + if (req !== review.req) return; + Object.assign(review, { search: x, runs, run }); + if (!review.mode || runID === 'latest') { + review.mode = run && run.new ? 'new' : run && run.rising ? 'rising' : 'all'; + } + $('#revError').hidden = true; + } catch (e) { + if (req !== review.req) return; + $('#revError').textContent = e.message; + $('#revError').hidden = false; + } finally { + if (req === review.req) document.body.classList.remove('busy'); + } + state.open.clear(); + renderReview(); +} + +function renderReviewHead() { + const x = saved.list.find((s) => s.id === review.id) || review.search; + if (!x) return; + review.search = x; + $('#revName').textContent = x.name; + $('#revSched').textContent = schedText(x.schedule) + (x.enabled + ? (x.nextRunAt ? ' · ' + t('nextRun', fmtDateTime(x.nextRunAt)) : '') + : ' · ' + t('paused')); +} + +function renderReview() { + if (view !== 'review') return; + renderReviewHead(); + const run = review.run; + const sel = $('#runSel'); + sel.innerHTML = review.runs.map((r) => ``).join(''); + sel.hidden = !review.runs.length; + if (run) sel.value = run.id; + + const items = run ? run.items : []; + const lists = { + new: items.filter((r) => r.new), + rising: items.filter((r) => r.starsDelta > 0).sort((a, b) => b.starsDelta - a.starsDelta || b.stars - a.stars), + all: items, + }; + for (const b of document.querySelectorAll('#revMode [data-mode]')) { + const m = b.dataset.mode; + b.textContent = t('mode' + m[0].toUpperCase() + m.slice(1), fmtNum(lists[m].length)); + b.setAttribute('aria-checked', String(m === review.mode)); + } + $('#revMode').hidden = !run || !!run.error; + review.items = run && !run.error ? lists[review.mode] || [] : []; + + const info = $('#revInfo'); + info.textContent = run && !run.error ? t('revInfo', { d: fmtDateTime(run.at), total: fmtNum(run.total), q: run.query }) : ''; + if (run && run.error) { + $('#revError').textContent = t('runFailed') + run.error; + $('#revError').hidden = false; + } else if (run) { + $('#revError').hidden = true; + } + + $('#revList').classList.add('review'); + $('#revList').innerHTML = review.items.map(rowHTML).join(''); + const empty = !review.items.length && !(run && run.error); + $('#revEmpty').hidden = !empty; + if (empty) { + const first = review.runs.filter((r) => !r.error).length <= 1; + const key = !run ? 'revNoRun' + : review.mode === 'new' ? (first ? 'revEmptyFirst' : 'revEmptyNew') + : review.mode === 'rising' ? (first ? 'revEmptyFirst' : 'revEmptyRising') : 'revEmptyAll'; + $('#revEmptyTitle').textContent = t(key); + $('#revEmptyHint').textContent = t(key + 'Hint'); + } +} + +$('#revMode').addEventListener('click', (ev) => { + const b = ev.target.closest('[data-mode]'); + if (!b) return; + review.mode = b.dataset.mode; + state.open.clear(); + renderReview(); +}); +$('#runSel').addEventListener('change', () => { + const latest = review.runs[0] && review.runs[0].id === $('#runSel').value; + location.hash = '#saved/' + encodeURIComponent(review.id) + (latest ? '' : '/' + encodeURIComponent($('#runSel').value)); +}); +$('#revRunBtn').addEventListener('click', () => runSaved(review.id)); +$('#revEditBtn').addEventListener('click', () => review.search && openSaveDlg(review.search)); +$('#revOpenBtn').addEventListener('click', () => { + if (!review.search) return; + writeFilters(filtersFromParams(review.search.params)); + state.searchPending = false; + location.hash = ''; + onFilterChange(); +}); + +/* ---------- Create / edit dialog ---------- */ + +const dlgState = { edit: null, params: '', max: 50 }; + +{ + const sel = $('#svWeekday'); + for (const d of [1, 2, 3, 4, 5, 6, 0]) sel.add(new Option('', String(d))); +} + +function renderSaveDlgLabels() { + for (const o of $('#svWeekday').options) o.textContent = dayName(Number(o.value)); + for (const b of document.querySelectorAll('#svMax [data-max]')) b.setAttribute('aria-checked', String(Number(b.dataset.max) === dlgState.max)); + const every = $('#svEvery').value; + $('#svWeekday').hidden = every !== 'week'; + $('#svTime').hidden = every === 'hours'; + $('#svHoursWrap').hidden = every !== 'hours'; +} + +// A readable default name: the search words, else the topic or the language. +function defaultName(f) { + const bits = [f.q, f.topic && 'topic:' + f.topic, f.language].filter(Boolean); + return bits.join(' · ') || t('savedTitle'); +} + +function openSaveDlg(x) { + dlgState.edit = x || null; + let sc = { every: 'week', weekday: 1, hour: 8, minute: 0, hours: 24 }; + if (x) { + dlgState.params = x.params; + dlgState.max = x.maxResults; + sc = { ...sc, ...x.schedule }; + $('#svName').value = x.name; + $('#svQuery').textContent = savedQuery(x); + $('#svEnabled').checked = x.enabled; + } else { + const f = readFilters(); + dlgState.params = params(f).toString(); + dlgState.max = 50; + $('#svName').value = defaultName(f); + $('#svQuery').textContent = state.query || dlgState.params; + $('#svEnabled').checked = true; + } + $('#svEvery').value = sc.every; + $('#svWeekday').value = String(sc.weekday || 0); + $('#svTime').value = pad2(sc.hour) + ':' + pad2(sc.minute); + $('#svHours').value = sc.hours || 24; + $('#saveTitle').textContent = t(x ? 'saveEditTitle' : 'saveNewTitle'); + $('#svHint').textContent = t(x ? 'svHintEdit' : 'svHintNew', state.tz || 'UTC'); + $('#svDelete').hidden = !x; + renderSaveDlgLabels(); + $('#saveDlg').showModal(); + $('#svName').select(); +} + +$('#svEvery').addEventListener('change', renderSaveDlgLabels); +$('#svMax').addEventListener('click', (ev) => { + const b = ev.target.closest('[data-max]'); + if (!b) return; + dlgState.max = Number(b.dataset.max); + renderSaveDlgLabels(); +}); +$('#closeSave').addEventListener('click', () => $('#saveDlg').close()); +$('#svCancel').addEventListener('click', () => $('#saveDlg').close()); +$('#saveDlg').addEventListener('click', (ev) => { if (ev.target === ev.currentTarget) ev.currentTarget.close(); }); +$('#svDelete').addEventListener('click', async () => { + if (dlgState.edit && await deleteSaved(dlgState.edit)) { + $('#saveDlg').close(); + if (view === 'review') location.hash = '#saved'; + } +}); + +$('#saveForm').addEventListener('submit', async (ev) => { + ev.preventDefault(); + const [h, m] = ($('#svTime').value || '08:00').split(':').map(Number); + const body = { + name: $('#svName').value.trim(), + params: dlgState.params, + maxResults: dlgState.max, + enabled: $('#svEnabled').checked, + schedule: { every: $('#svEvery').value, weekday: Number($('#svWeekday').value), hour: h || 0, minute: m || 0, hours: Number($('#svHours').value) || 24 }, + }; + const x = dlgState.edit; + $('#svSubmit').disabled = true; + try { + const res = await api(x ? '/api/saved/' + encodeURIComponent(x.id) : '/api/saved', { + method: x ? 'PUT' : 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body), + }); + $('#saveDlg').close(); + toast(t(x ? 'updated2' : 'saved')); + if (x) { + await loadSaved(); + if (view === 'review') renderReview(); + } else { + saved.running.add(res.id); // the server runs it once right away + location.hash = '#saved'; + setTimeout(async () => { saved.running.delete(res.id); await loadSaved(); }, 4000); + } + } catch (e) { + toast(e.message); + } finally { + $('#svSubmit').disabled = false; + } +}); + +$('#saveBtn').addEventListener('click', () => openSaveDlg()); + +// Keep the list fresh (next run countdown, runs made by the scheduler). +setInterval(() => { + if (document.hidden) return; + if (view === 'saved') loadSaved(); +}, 30000); +loadSaved(); + /* ================= Startup ================= */ applyDensity(store.get('density', 'comfortable')); @@ -670,5 +1157,7 @@ applyLang(); renderInterface(); writeFilters(filtersFromURL()); loadStatus(); -search(); +route(); +if (view === 'search') search(); +else state.searchPending = true; setInterval(() => { if (!document.hidden) renderStatus(); }, 5000); // reset countdown diff --git a/web/index.html b/web/index.html index 946dd2a..7c38400 100644 --- a/web/index.html +++ b/web/index.html @@ -23,7 +23,19 @@ Searchgit
-
+
+ + + + + +