From 02b51fed77173889e820ede18df2ceaded3bf53b Mon Sep 17 00:00:00 2001 From: claude Date: Thu, 1 Oct 2026 16:36:50 +0200 Subject: [PATCH] Scheduled searches with a weekly review of new and rising repositories A search can now be saved with a schedule (every N hours, daily, or weekly at a given day and time). An internal scheduler runs it and keeps a snapshot of the results on a /data volume (JSON files, no new dependency). The new Watch tab lists the scheduled searches; the review page of each one shows, for any run of its history, the repositories never seen before and the ones that gained the most stars since the previous run. Co-Authored-By: Claude Opus 5.5 (1M context) --- .env.example | 3 + .gitignore | 3 + Dockerfile | 5 +- README.md | 33 ++- api.go | 16 +- docker-compose.yml | 7 + go.mod | 2 +- main.go | 20 +- saved_api.go | 200 ++++++++++++++++ saved_api_test.go | 124 ++++++++++ scheduler.go | 163 +++++++++++++ scheduler_test.go | 100 ++++++++ store.go | 301 ++++++++++++++++++++++++ web/app.js | 561 ++++++++++++++++++++++++++++++++++++++++++--- web/index.html | 138 ++++++++++- web/style.css | 114 +++++++++ 16 files changed, 1745 insertions(+), 45 deletions(-) create mode 100644 saved_api.go create mode 100644 saved_api_test.go create mode 100644 scheduler.go create mode 100644 scheduler_test.go create mode 100644 store.go diff --git a/.env.example b/.env.example index cb8012a..715361e 100644 --- a/.env.example +++ b/.env.example @@ -5,3 +5,6 @@ GITHUB_TOKEN= CACHE_TTL=5m AUTH_USER= AUTH_PASS= +# Time zone of the scheduled searches. +TZ=Europe/Paris +HISTORY_KEEP=100 diff --git a/.gitignore b/.gitignore index c23f2ff..4316220 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,6 @@ go.work.sum # Binary /searchgit + +# Local data (scheduled searches) +/data/ diff --git a/Dockerfile b/Dockerfile index 3dee1b6..b03bdf1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,9 +11,12 @@ RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/searchgit . # ---- Final image ---- FROM alpine:3.24.2 RUN apk add --no-cache ca-certificates \ - && adduser -D -H -u 10001 searchgit + && adduser -D -H -u 10001 searchgit \ + && mkdir /data && chown searchgit:searchgit /data COPY --from=build /out/searchgit /usr/local/bin/searchgit USER searchgit +ENV DATA_DIR=/data +VOLUME /data EXPOSE 8080/tcp HEALTHCHECK --interval=30s --timeout=3s CMD wget -qO- http://127.0.0.1:8080/healthz || exit 1 ENTRYPOINT ["/usr/local/bin/searchgit"] diff --git a/README.md b/README.md index 7197839..49d0d5a 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,8 @@ Search GitHub repositories with criteria and browse the results in a clean web i ``` browser ──/api/search──▶ searchgit (Go) ──REST──▶ api.github.com/search/repositories - └── 5 min cache + ──/api/saved───▶ ├── 5 min cache + └── scheduler ──▶ /data (JSON snapshots) ``` ## Getting started @@ -18,7 +19,7 @@ docker compose up -d --build Then open . -Without Docker: `go run .` (Go 1.23 or newer), then open . +Without Docker: `go run .` (Go 1.24 or newer, data in `./data`), then open . ## Criteria @@ -42,6 +43,24 @@ shows the exact query sent to GitHub (click it to open the same search on github GitHub returns at most the first 1,000 results of a search. +## Scheduled searches (weekly review) + +**Schedule** saves the current filters as a search that the server runs on its own: +every week (day and time), every day, or every N hours, in the server time zone (`TZ`). +Each run keeps a snapshot of the first 30, 50 or 100 results. + +The **Watch** tab lists the scheduled searches with their last run, and a review page +per search shows, for any run of its history: + +- **New**: repositories never returned by a previous run of this search; +- **Rising**: repositories that gained the most stars since the previous run; +- **All**: the whole snapshot, with the star gain of each repository. + +A search runs once as soon as it is created: that first run is the baseline the next +ones are compared with. A run missed while the server was stopped is made at startup. +For a weekly "gems" review, a good start is: a topic or a language, *created < 1 month* +(or *< 1 week*), sorted by *most stars*. + ## Configuration | Variable | Default | Role | @@ -51,12 +70,20 @@ GitHub returns at most the first 1,000 results of a search. | `CACHE_TTL` | `5m` | identical searches are served from memory for this long | | `AUTH_USER` / `AUTH_PASS` | empty | HTTP basic authentication (except `/healthz`) | | `GITHUB_API` | `https://api.github.com` | API base URL (GitHub Enterprise) | +| `DATA_DIR` | `data` (`/data` in Docker) | scheduled searches and their snapshots (JSON files) | +| `TZ` | `Europe/Paris` in compose | time zone of the schedules | +| `HISTORY_KEEP` | `100` | runs kept per scheduled search (0 = all) | ## API - `GET /api/search?q=&in=&language=&stars=&maxstars=&minforks=&topic=&license=&pushed=&created=&goodfirst=1&forks=1&archived=1&sort=&order=&page=&per_page=` (`pushed` / `created`: `1w`, `1m`, `3m`, `6m`, `1y`, `2y`, `5y`) -- `GET /api/status`: token configured and last known search quota +- `GET /api/status`: token configured, last known search quota, server time zone +- `GET|POST /api/saved`, `GET|PUT|DELETE /api/saved/{id}`: scheduled searches + (`{"name", "params", "maxResults", "enabled", "schedule": {"every": "week|day|hours", "weekday", "hour", "minute", "hours"}}`, + `params` being the `/api/search` query string) +- `POST /api/saved/{id}/run`: run now +- `GET /api/saved/{id}/runs`, `GET /api/saved/{id}/runs/{run|latest}`: history and snapshots - `GET /healthz` ## Development diff --git a/api.go b/api.go index 44e0035..5b3cfad 100644 --- a/api.go +++ b/api.go @@ -4,16 +4,20 @@ import ( "encoding/json" "errors" "net/http" + "time" ) type API struct { - gh *GitHub + gh *GitHub + store *Store + sched *Scheduler } func (a *API) Routes(mux *http.ServeMux) { mux.HandleFunc("GET /healthz", a.health) mux.HandleFunc("GET /api/search", a.search) mux.HandleFunc("GET /api/status", a.status) + a.savedRoutes(mux) } // codedError carries a stable code that the UI translates. @@ -61,9 +65,19 @@ func (a *API) status(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, map[string]any{ "token": a.gh.HasToken(), "rate": a.gh.Rate(), + "tz": zoneName(), }) } +// zoneName is the server time zone used by the schedules (TZ). +func zoneName() string { + if name := time.Local.String(); name != "Local" { + return name + } + name, _ := time.Now().Zone() + return name +} + func (a *API) health(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("ok")) } diff --git a/docker-compose.yml b/docker-compose.yml index 73191fc..538c21f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,3 +10,10 @@ services: CACHE_TTL: ${CACHE_TTL:-5m} # identical searches are served from memory AUTH_USER: ${AUTH_USER:-} # leave empty to disable authentication AUTH_PASS: ${AUTH_PASS:-} + TZ: ${TZ:-Europe/Paris} # time zone of the scheduled searches + HISTORY_KEEP: ${HISTORY_KEEP:-100} # runs kept per scheduled search + volumes: + - searchgit-data:/data # scheduled searches and their history + +volumes: + searchgit-data: diff --git a/go.mod b/go.mod index 8c4224d..3338970 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module searchgit -go 1.23 +go 1.24 diff --git a/main.go b/main.go index f7d41c7..f463acc 100644 --- a/main.go +++ b/main.go @@ -12,8 +12,10 @@ import ( "net/http" "os" "os/signal" + "strconv" "syscall" "time" + _ "time/tzdata" // TZ works in the minimal Docker image ) //go:embed web @@ -33,6 +35,13 @@ func getenvDuration(key string, def time.Duration) time.Duration { return def } +func getenvInt(key string, def int) int { + if n, err := strconv.Atoi(os.Getenv(key)); err == nil && n >= 0 { + return n + } + return def +} + func main() { httpAddr := getenv("HTTP_ADDR", ":8080") token := os.Getenv("GITHUB_TOKEN") @@ -45,8 +54,15 @@ func main() { if err != nil { log.Fatal(err) } + store, err := OpenStore(getenv("DATA_DIR", "data"), getenvInt("HISTORY_KEEP", 100)) + if err != nil { + log.Fatalf("data: %v", err) + } + sched := NewScheduler(store, gh) + sched.Start(ctx) + mux := http.NewServeMux() - (&API{gh: gh}).Routes(mux) + (&API{gh: gh, store: store, sched: sched}).Routes(mux) mux.Handle("GET /", http.FileServer(http.FS(static))) srv := &http.Server{ @@ -65,7 +81,7 @@ func main() { if token == "" { log.Printf("GITHUB_TOKEN not set: GitHub allows only 10 searches per minute") } - log.Printf("web UI on %s", httpAddr) + log.Printf("web UI on %s, data in %s, time zone %s", httpAddr, getenv("DATA_DIR", "data"), time.Local) if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { log.Fatalf("http: %v", err) } diff --git a/saved_api.go b/saved_api.go new file mode 100644 index 0000000..fc3b47e --- /dev/null +++ b/saved_api.go @@ -0,0 +1,200 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "log" + "net/http" + "net/url" + "strings" + "time" +) + +// Saved search routes: +// +// GET /api/saved list +// POST /api/saved create (and run it once in the background) +// GET /api/saved/{id} +// PUT /api/saved/{id} update +// DELETE /api/saved/{id} +// POST /api/saved/{id}/run run now +// GET /api/saved/{id}/runs history (summaries) +// GET /api/saved/{id}/runs/{run} one snapshot ("latest" accepted) +func (a *API) savedRoutes(mux *http.ServeMux) { + mux.HandleFunc("GET /api/saved", a.savedList) + mux.HandleFunc("POST /api/saved", a.savedCreate) + mux.HandleFunc("GET /api/saved/{id}", a.savedGet) + mux.HandleFunc("PUT /api/saved/{id}", a.savedUpdate) + mux.HandleFunc("DELETE /api/saved/{id}", a.savedDelete) + mux.HandleFunc("POST /api/saved/{id}/run", a.savedRun) + mux.HandleFunc("GET /api/saved/{id}/runs", a.savedRuns) + mux.HandleFunc("GET /api/saved/{id}/runs/{run}", a.savedRunGet) +} + +// savedInput is what the UI sends on create and update. +type savedInput struct { + Name string `json:"name"` + Params string `json:"params"` + Schedule Schedule `json:"schedule"` + MaxResults int `json:"maxResults"` + Enabled *bool `json:"enabled"` +} + +func (in *savedInput) validate() error { + in.Name = strings.TrimSpace(in.Name) + if in.Name == "" || len(in.Name) > 120 { + return &codedError{code: "bad_param", msg: "invalid name", detail: "name"} + } + v, err := url.ParseQuery(in.Params) + if err != nil { + return &codedError{code: "bad_param", msg: "invalid parameters", detail: "params"} + } + v.Del("page") + v.Del("per_page") + in.Params = v.Encode() + if _, err := ParseCriteria(v); err != nil { + return err + } + switch in.MaxResults { + case 0: + in.MaxResults = 50 + case 30, 50, 100: + default: + return &codedError{code: "bad_param", msg: "invalid maxResults", detail: "maxResults"} + } + if !in.Schedule.Valid() { + return &codedError{code: "bad_param", msg: "invalid schedule", detail: "schedule"} + } + return nil +} + +func (a *API) savedList(w http.ResponseWriter, r *http.Request) { + writeJSON(w, http.StatusOK, a.store.List()) +} + +func (a *API) savedGet(w http.ResponseWriter, r *http.Request) { + x, err := a.store.Get(r.PathValue("id")) + if err != nil { + writeStoreErr(w, err) + return + } + writeJSON(w, http.StatusOK, x) +} + +func readInput(r *http.Request) (savedInput, error) { + var in savedInput + dec := json.NewDecoder(http.MaxBytesReader(nil, r.Body, 64<<10)) + if err := dec.Decode(&in); err != nil { + return in, &codedError{code: "bad_param", msg: "invalid JSON", detail: err.Error()} + } + return in, in.validate() +} + +func (a *API) savedCreate(w http.ResponseWriter, r *http.Request) { + in, err := readInput(r) + if err != nil { + writeErr(w, http.StatusBadRequest, err) + return + } + x := SavedSearch{Name: in.Name, Params: in.Params, Schedule: in.Schedule, MaxResults: in.MaxResults, Enabled: true} + if in.Enabled != nil { + x.Enabled = *in.Enabled + } + x.NextRunAt = x.Schedule.Next(time.Now(), time.Time{}) + if !x.Enabled { + x.NextRunAt = time.Time{} + } + x, err = a.store.Create(x) + if err != nil { + writeErr(w, http.StatusInternalServerError, err) + return + } + // First run right away: it gives a baseline to compare the next ones with. + go func(id string) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if _, err := a.sched.Run(ctx, id, "manual"); err != nil { + log.Printf("first run of saved search %s: %v", id, err) + } + }(x.ID) + writeJSON(w, http.StatusCreated, x) +} + +func (a *API) savedUpdate(w http.ResponseWriter, r *http.Request) { + in, err := readInput(r) + if err != nil { + writeErr(w, http.StatusBadRequest, err) + return + } + x, err := a.store.Update(r.PathValue("id"), func(v *SavedSearch) { + v.Name, v.Params, v.Schedule, v.MaxResults = in.Name, in.Params, in.Schedule, in.MaxResults + if in.Enabled != nil { + v.Enabled = *in.Enabled + } + v.NextRunAt = time.Time{} + if v.Enabled { + v.NextRunAt = v.Schedule.Next(time.Now(), v.LastRunAt) + } + }) + if err != nil { + writeStoreErr(w, err) + return + } + writeJSON(w, http.StatusOK, x) +} + +func (a *API) savedDelete(w http.ResponseWriter, r *http.Request) { + if err := a.store.Delete(r.PathValue("id")); err != nil { + writeStoreErr(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (a *API) savedRun(w http.ResponseWriter, r *http.Request) { + run, err := a.sched.Run(r.Context(), r.PathValue("id"), "manual") + if err != nil && run.ID == "" { + writeStoreErr(w, err) + return + } + // A failed GitHub search is still a recorded run: the error is in run.error. + writeJSON(w, http.StatusOK, run.RunInfo) +} + +func (a *API) savedRuns(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + if _, err := a.store.Get(id); err != nil { + writeStoreErr(w, err) + return + } + runs, err := a.store.Runs(id) + if err != nil { + writeStoreErr(w, err) + return + } + writeJSON(w, http.StatusOK, runs) +} + +func (a *API) savedRunGet(w http.ResponseWriter, r *http.Request) { + run, err := a.store.Run(r.PathValue("id"), r.PathValue("run")) + if err != nil { + writeStoreErr(w, err) + return + } + writeJSON(w, http.StatusOK, run) +} + +func writeStoreErr(w http.ResponseWriter, err error) { + status := http.StatusInternalServerError + var ce *codedError + if errors.As(err, &ce) { + switch ce.code { + case "not_found": + status = http.StatusNotFound + case "bad_param": + status = http.StatusBadRequest + } + } + writeErr(w, status, err) +} diff --git a/saved_api_test.go b/saved_api_test.go new file mode 100644 index 0000000..3212144 --- /dev/null +++ b/saved_api_test.go @@ -0,0 +1,124 @@ +package main + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func newTestAPI(t *testing.T) (*http.ServeMux, *API) { + t.Helper() + gh, _ := newFake(http.StatusOK, searchBody) + store, err := OpenStore(t.TempDir(), 0) + if err != nil { + t.Fatal(err) + } + a := &API{gh: gh, store: store, sched: NewScheduler(store, gh)} + mux := http.NewServeMux() + a.Routes(mux) + return mux, a +} + +func call(t *testing.T, mux *http.ServeMux, method, path, body string, out any) int { + t.Helper() + req := httptest.NewRequest(method, path, strings.NewReader(body)) + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + if out != nil && rec.Code < 300 { + if err := json.Unmarshal(rec.Body.Bytes(), out); err != nil { + t.Fatalf("%s %s: %v (%s)", method, path, err, rec.Body) + } + } + return rec.Code +} + +func TestSavedSearchAPI(t *testing.T) { + mux, a := newTestAPI(t) + + for _, bad := range []string{ + `{"name": "", "params": "q=x", "schedule": {"every": "week", "weekday": 1, "hour": 8}}`, + `{"name": "x", "params": "sort=nope", "schedule": {"every": "week", "weekday": 1, "hour": 8}}`, + `{"name": "x", "params": "q=x", "schedule": {"every": "month"}}`, + `{"name": "x", "params": "q=x", "maxResults": 7, "schedule": {"every": "day", "hour": 8}}`, + } { + if code := call(t, mux, "POST", "/api/saved", bad, nil); code != http.StatusBadRequest { + t.Errorf("%s: got %d, want 400", bad, code) + } + } + + var x SavedSearch + code := call(t, mux, "POST", "/api/saved", + `{"name": "Go syslog", "params": "q=syslog&language=Go&page=3", "schedule": {"every": "week", "weekday": 1, "hour": 8}}`, &x) + if code != http.StatusCreated || x.ID == "" || x.MaxResults != 50 || !x.Enabled || x.NextRunAt.IsZero() { + t.Fatalf("create: %d %+v", code, x) + } + if strings.Contains(x.Params, "page") { + t.Errorf("page should be dropped from params: %s", x.Params) + } + + // The first run starts in the background; run once more by hand. + var info RunInfo + if code := call(t, mux, "POST", "/api/saved/"+x.ID+"/run", "", &info); code != http.StatusOK || info.Error != "" || info.Count != 1 { + t.Fatalf("run: %d %+v", code, info) + } + deadline := time.Now().Add(2 * time.Second) + var runs []RunInfo + for time.Now().Before(deadline) { + call(t, mux, "GET", "/api/saved/"+x.ID+"/runs", "", &runs) + if len(runs) == 2 { + break + } + time.Sleep(20 * time.Millisecond) + } + if len(runs) != 2 { + t.Fatalf("want 2 runs (background + manual), got %d", len(runs)) + } + var run Run + if code := call(t, mux, "GET", "/api/saved/"+x.ID+"/runs/latest", "", &run); code != http.StatusOK || len(run.Items) != 1 || run.Items[0].FullName != "mcuadros/go-syslog" { + t.Fatalf("latest run: %d %+v", code, run) + } + if d := run.Items[0].StarsDelta; d == nil || *d != 0 || run.Items[0].New { + t.Errorf("second run: repository should be known with a 0 delta: %+v", run.Items[0]) + } + + var upd SavedSearch + code = call(t, mux, "PUT", "/api/saved/"+x.ID, + `{"name": "Renamed", "params": "q=syslog", "maxResults": 100, "enabled": false, "schedule": {"every": "hours", "hours": 6}}`, &upd) + if code != http.StatusOK || upd.Name != "Renamed" || upd.Enabled || !upd.NextRunAt.IsZero() || upd.LastRun == nil { + t.Fatalf("update: %d %+v", code, upd) + } + + if code := call(t, mux, "GET", "/api/saved/"+x.ID+"/runs/../../x", "", nil); code == http.StatusOK { + t.Error("path traversal should not succeed") + } + if code := call(t, mux, "DELETE", "/api/saved/"+x.ID, "", nil); code != http.StatusNoContent { + t.Errorf("delete: %d", code) + } + if code := call(t, mux, "GET", "/api/saved/"+x.ID, "", nil); code != http.StatusNotFound { + t.Errorf("get after delete: %d", code) + } + _ = a +} + +func TestSchedulerRunsDueSearches(t *testing.T) { + _, a := newTestAPI(t) + x, _ := a.store.Create(SavedSearch{Name: "due", Params: "q=syslog", MaxResults: 30, Enabled: true, + Schedule: Schedule{Every: "day", Hour: 8}, NextRunAt: time.Now().Add(-time.Minute)}) + a.store.Create(SavedSearch{Name: "paused", Params: "q=syslog", MaxResults: 30, + Schedule: Schedule{Every: "day", Hour: 8}, NextRunAt: time.Now().Add(-time.Minute)}) + a.sched.runDue(context.Background()) + + got, _ := a.store.Get(x.ID) + if got.LastRun == nil || !got.NextRunAt.After(time.Now()) { + t.Errorf("due search should have run and be planned again: %+v", got) + } + for _, s := range a.store.List() { + if s.Name == "paused" && s.LastRun != nil { + t.Error("a paused search must not run") + } + } +} diff --git a/scheduler.go b/scheduler.go new file mode 100644 index 0000000..5ba3f75 --- /dev/null +++ b/scheduler.go @@ -0,0 +1,163 @@ +package main + +import ( + "context" + "errors" + "log" + "net/url" + "strconv" + "sync" + "time" +) + +// Schedule says when a saved search runs, in the server time zone (TZ). +type Schedule struct { + Every string `json:"every"` // "hours", "day" or "week" + Hours int `json:"hours,omitempty"` // every: hours (1..168) + Weekday int `json:"weekday,omitempty"` // every: week, 0 = Sunday + Hour int `json:"hour"` // every: day / week + Minute int `json:"minute"` +} + +func (sc Schedule) Valid() bool { + switch sc.Every { + case "hours": + return sc.Hours >= 1 && sc.Hours <= 168 + case "day", "week": + return sc.Hour >= 0 && sc.Hour < 24 && sc.Minute >= 0 && sc.Minute < 60 && + sc.Weekday >= 0 && sc.Weekday < 7 + } + return false +} + +// Next returns the first run time strictly after "after". last is the time of +// the last run, used by the "hours" schedule. +func (sc Schedule) Next(after, last time.Time) time.Time { + switch sc.Every { + case "hours": + every := time.Duration(sc.Hours) * time.Hour + if last.IsZero() || last.Add(every).Before(after) { + return after.Add(time.Minute) + } + return last.Add(every) + case "day", "week": + t := time.Date(after.Year(), after.Month(), after.Day(), sc.Hour, sc.Minute, 0, 0, after.Location()) + for !t.After(after) || (sc.Every == "week" && int(t.Weekday()) != sc.Weekday) { + t = t.AddDate(0, 0, 1) + } + return t + } + return time.Time{} +} + +// Scheduler runs the saved searches when they are due. +type Scheduler struct { + store *Store + gh *GitHub + runMu sync.Mutex // one run at a time, to spare the GitHub quota +} + +func NewScheduler(store *Store, gh *GitHub) *Scheduler { + return &Scheduler{store: store, gh: gh} +} + +// Start checks every 30 seconds for due searches until ctx is done. +func (s *Scheduler) Start(ctx context.Context) { + now := time.Now() + for _, x := range s.store.List() { // plan searches that have no next run yet + if x.Enabled && x.NextRunAt.IsZero() { + _, _ = s.store.Update(x.ID, func(v *SavedSearch) { v.NextRunAt = v.Schedule.Next(now, v.LastRunAt) }) + } + } + go func() { + tick := time.NewTicker(30 * time.Second) + defer tick.Stop() + for { + s.runDue(ctx) + select { + case <-ctx.Done(): + return + case <-tick.C: + } + } + }() +} + +func (s *Scheduler) runDue(ctx context.Context) { + for _, x := range s.store.List() { + if ctx.Err() != nil { + return + } + if x.Enabled && !x.NextRunAt.IsZero() && !time.Now().Before(x.NextRunAt) { + if _, err := s.Run(ctx, x.ID, "schedule"); err != nil { + log.Printf("saved search %q: %v", x.Name, err) + } + } + } +} + +// Run executes a saved search now and stores the snapshot. +func (s *Scheduler) Run(ctx context.Context, id, trigger string) (Run, error) { + s.runMu.Lock() + defer s.runMu.Unlock() + x, err := s.store.Get(id) + if err != nil { + return Run{}, err + } + start := time.Now() + info := RunInfo{ID: start.UTC().Format("20060102T150405Z"), At: start, Trigger: trigger} + + var repos []Repo + c, err := criteriaFor(x) + if err == nil { + info.Query = c.Query(start) + var res SearchResult + if res, err = s.gh.Search(ctx, c); err == nil { + info.Total = res.Total + repos = res.Items + } + } + info.TookMs = time.Since(start).Milliseconds() + if err != nil { + info.Error = err.Error() + } + run, serr := s.store.AddRun(id, info, repos) + if serr != nil { + return Run{}, serr + } + + _, uerr := s.store.Update(id, func(v *SavedSearch) { + v.LastRunAt = start + v.LastError = info.Error + ri := run.RunInfo + v.LastRun = &ri + next := v.Schedule.Next(time.Now(), start) + if err != nil { + var ce *codedError + if errors.As(err, &ce) && ce.code == "rate_limited" { + // Quota exhausted: try again in a few minutes rather than next week. + next = time.Now().Add(3 * time.Minute) + } + } + v.NextRunAt = next + }) + if uerr != nil { + return run, uerr + } + if err != nil { + return run, err + } + return run, nil +} + +// criteriaFor parses the stored parameters; a saved search always reads the +// first page, sorted as saved. +func criteriaFor(x SavedSearch) (Criteria, error) { + v, err := url.ParseQuery(x.Params) + if err != nil { + return Criteria{}, &codedError{code: "bad_param", msg: "invalid parameters", detail: err.Error()} + } + v.Del("page") + v.Set("per_page", strconv.Itoa(x.MaxResults)) + return ParseCriteria(v) +} diff --git a/scheduler_test.go b/scheduler_test.go new file mode 100644 index 0000000..2c2d545 --- /dev/null +++ b/scheduler_test.go @@ -0,0 +1,100 @@ +package main + +import ( + "testing" + "time" +) + +func TestScheduleNext(t *testing.T) { + loc := time.FixedZone("CEST", 2*3600) + // Wednesday 2026-09-30 10:00 + now := time.Date(2026, 9, 30, 10, 0, 0, 0, loc) + cases := []struct { + name string + sc Schedule + last time.Time + want time.Time + }{ + {"weekly monday 8:00", Schedule{Every: "week", Weekday: 1, Hour: 8}, time.Time{}, time.Date(2026, 10, 5, 8, 0, 0, 0, loc)}, + {"weekly today later", Schedule{Every: "week", Weekday: 3, Hour: 18, Minute: 30}, time.Time{}, time.Date(2026, 9, 30, 18, 30, 0, 0, loc)}, + {"weekly today passed", Schedule{Every: "week", Weekday: 3, Hour: 9}, time.Time{}, time.Date(2026, 10, 7, 9, 0, 0, 0, loc)}, + {"daily passed", Schedule{Every: "day", Hour: 7}, time.Time{}, time.Date(2026, 10, 1, 7, 0, 0, 0, loc)}, + {"hours first", Schedule{Every: "hours", Hours: 6}, time.Time{}, now.Add(time.Minute)}, + {"hours after last", Schedule{Every: "hours", Hours: 6}, now.Add(-2 * time.Hour), now.Add(4 * time.Hour)}, + {"hours late", Schedule{Every: "hours", Hours: 6}, now.Add(-7 * time.Hour), now.Add(time.Minute)}, + } + for _, c := range cases { + if got := c.sc.Next(now, c.last); !got.Equal(c.want) { + t.Errorf("%s: got %v, want %v", c.name, got, c.want) + } + } +} + +func TestScheduleValid(t *testing.T) { + for _, sc := range []Schedule{{Every: "hours"}, {Every: "hours", Hours: 200}, {Every: "week", Weekday: 7}, {Every: "day", Hour: 24}, {Every: "month"}} { + if sc.Valid() { + t.Errorf("%+v should be invalid", sc) + } + } + if !(Schedule{Every: "week", Weekday: 1, Hour: 8}).Valid() { + t.Error("weekly schedule should be valid") + } +} + +func TestStoreRuns(t *testing.T) { + s, err := OpenStore(t.TempDir(), 2) + if err != nil { + t.Fatal(err) + } + x, err := s.Create(SavedSearch{Name: "go", Params: "language=Go", Schedule: Schedule{Every: "day"}, MaxResults: 30, Enabled: true}) + if err != nil { + t.Fatal(err) + } + t0 := time.Date(2026, 9, 1, 8, 0, 0, 0, time.UTC) + add := func(id string, at time.Time, repos ...Repo) Run { + t.Helper() + r, err := s.AddRun(x.ID, RunInfo{ID: id, At: at}, repos) + if err != nil { + t.Fatal(err) + } + return r + } + + r1 := add("r1", t0, Repo{FullName: "a/a", Stars: 10}, Repo{FullName: "b/b", Stars: 5}) + if r1.New != 0 || r1.Items[0].New || r1.Items[0].StarsDelta != nil { + t.Errorf("first run must be a baseline: %+v", r1) + } + + r2 := add("r2", t0.Add(24*time.Hour), Repo{FullName: "a/a", Stars: 25}, Repo{FullName: "c/c", Stars: 3}) + if r2.New != 1 || !r2.Items[1].New || r2.Items[0].New { + t.Errorf("c/c should be the only new repository: %+v", r2.Items) + } + if d := r2.Items[0].StarsDelta; d == nil || *d != 15 || r2.Rising != 1 { + t.Errorf("a/a should have gained 15 stars: %+v", r2.Items[0]) + } + + // A failed run is recorded but does not break the comparison. + s.AddRun(x.ID, RunInfo{ID: "r3", At: t0.Add(48 * time.Hour), Error: "boom"}, nil) + r4 := add("r4", t0.Add(72*time.Hour), Repo{FullName: "b/b", Stars: 6}, Repo{FullName: "c/c", Stars: 4}) + if r4.New != 0 { + t.Errorf("b/b was seen in the first run, not new: %+v", r4.Items) + } + + idx, _ := s.Runs(x.ID) + if len(idx) != 2 || idx[0].ID != "r4" { + t.Errorf("history should keep the 2 newest runs, got %+v", idx) + } + if _, err := s.Run(x.ID, "r1"); err == nil { + t.Error("r1 should have been pruned") + } + latest, err := s.Run(x.ID, "latest") + if err != nil || latest.ID != "r4" { + t.Errorf("latest = %v, %v", latest.ID, err) + } + if err := s.Delete(x.ID); err != nil { + t.Fatal(err) + } + if _, err := s.Get(x.ID); err == nil { + t.Error("search should be deleted") + } +} diff --git a/store.go b/store.go new file mode 100644 index 0000000..a850c65 --- /dev/null +++ b/store.go @@ -0,0 +1,301 @@ +package main + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "sync" + "time" +) + +// SavedSearch is a search that the scheduler runs on a regular basis. +type SavedSearch struct { + ID string `json:"id"` + Name string `json:"name"` + Params string `json:"params"` // same query string as /api/search (without page) + Schedule Schedule `json:"schedule"` + MaxResults int `json:"maxResults"` // 30, 50 or 100 (one GitHub page) + Enabled bool `json:"enabled"` + CreatedAt time.Time `json:"createdAt"` + LastRunAt time.Time `json:"lastRunAt,omitzero"` + NextRunAt time.Time `json:"nextRunAt,omitzero"` + LastError string `json:"lastError,omitempty"` + LastRun *RunInfo `json:"lastRun,omitempty"` +} + +// RunInfo summarizes one execution of a saved search. +type RunInfo struct { + ID string `json:"id"` + At time.Time `json:"at"` + Query string `json:"query"` + Total int `json:"total"` + Count int `json:"count"` // repositories kept in the snapshot + New int `json:"new"` // never seen in a previous run + Rising int `json:"rising"` // gained stars since the previous run + Error string `json:"error,omitempty"` + TookMs int64 `json:"tookMs"` + Trigger string `json:"trigger"` // "schedule" or "manual" +} + +// RunItem is one repository of a snapshot, compared with the previous run. +type RunItem struct { + Repo + New bool `json:"new"` + FirstSeen time.Time `json:"firstSeen"` + StarsDelta *int `json:"starsDelta,omitempty"` // nil when absent from the previous run + Rank int `json:"rank"` +} + +// Run is a full snapshot. +type Run struct { + RunInfo + Items []RunItem `json:"items"` +} + +// Store keeps saved searches and their runs as JSON files: +// +// /searches.json +// /runs//index.json run summaries, newest first +// /runs//seen.json repository -> first time seen +// /runs//.json +type Store struct { + dir string + keep int // runs kept per search (0 = all) + + mu sync.Mutex + searches []*SavedSearch +} + +var idRe = regexp.MustCompile(`^[a-zA-Z0-9-]{1,40}$`) + +var errNotFound = &codedError{code: "not_found", msg: "not found"} + +func OpenStore(dir string, keep int) (*Store, error) { + if err := os.MkdirAll(filepath.Join(dir, "runs"), 0o755); err != nil { + return nil, err + } + s := &Store{dir: dir, keep: keep} + if err := readJSON(filepath.Join(dir, "searches.json"), &s.searches); err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + return s, nil +} + +func newID() string { + b := make([]byte, 6) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} + +// List returns copies of the saved searches, sorted by name. +func (s *Store) List() []SavedSearch { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]SavedSearch, 0, len(s.searches)) + for _, x := range s.searches { + out = append(out, *x) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out +} + +func (s *Store) Get(id string) (SavedSearch, error) { + s.mu.Lock() + defer s.mu.Unlock() + if x := s.find(id); x != nil { + return *x, nil + } + return SavedSearch{}, errNotFound +} + +func (s *Store) find(id string) *SavedSearch { + for _, x := range s.searches { + if x.ID == id { + return x + } + } + return nil +} + +func (s *Store) Create(x SavedSearch) (SavedSearch, error) { + s.mu.Lock() + defer s.mu.Unlock() + x.ID = newID() + x.CreatedAt = time.Now() + s.searches = append(s.searches, &x) + return x, s.saveLocked() +} + +// Update changes what the user can edit and returns the new value. +func (s *Store) Update(id string, fn func(*SavedSearch)) (SavedSearch, error) { + s.mu.Lock() + defer s.mu.Unlock() + x := s.find(id) + if x == nil { + return SavedSearch{}, errNotFound + } + fn(x) + return *x, s.saveLocked() +} + +func (s *Store) Delete(id string) error { + s.mu.Lock() + defer s.mu.Unlock() + for i, x := range s.searches { + if x.ID == id { + s.searches = append(s.searches[:i], s.searches[i+1:]...) + if err := s.saveLocked(); err != nil { + return err + } + return os.RemoveAll(s.runDir(id)) + } + } + return errNotFound +} + +func (s *Store) saveLocked() error { + return writeFileJSON(filepath.Join(s.dir, "searches.json"), s.searches) +} + +func (s *Store) runDir(id string) string { return filepath.Join(s.dir, "runs", id) } + +// Runs returns the run summaries of a search, newest first. +func (s *Store) Runs(id string) ([]RunInfo, error) { + if !idRe.MatchString(id) { + return nil, errNotFound + } + var idx []RunInfo + err := readJSON(filepath.Join(s.runDir(id), "index.json"), &idx) + if errors.Is(err, os.ErrNotExist) { + return []RunInfo{}, nil + } + return idx, err +} + +// Run reads one snapshot ("latest" for the most recent one). +func (s *Store) Run(id, runID string) (Run, error) { + if runID == "latest" { + idx, err := s.Runs(id) + if err != nil { + return Run{}, err + } + if len(idx) == 0 { + return Run{}, errNotFound + } + runID = idx[0].ID + } + if !idRe.MatchString(id) || !idRe.MatchString(runID) { + return Run{}, errNotFound + } + var r Run + err := readJSON(filepath.Join(s.runDir(id), runID+".json"), &r) + if errors.Is(err, os.ErrNotExist) { + return Run{}, errNotFound + } + return r, err +} + +// AddRun compares the repositories with the previous run, saves the snapshot +// and updates the search. A failed run (info.Error set) is only recorded in +// the history. +func (s *Store) AddRun(id string, info RunInfo, repos []Repo) (Run, error) { + dir := s.runDir(id) + if err := os.MkdirAll(dir, 0o755); err != nil { + return Run{}, err + } + idx, err := s.Runs(id) + if err != nil { + return Run{}, err + } + run := Run{RunInfo: info, Items: []RunItem{}} + + if info.Error == "" { + seen := map[string]time.Time{} + if err := readJSON(filepath.Join(dir, "seen.json"), &seen); err != nil && !errors.Is(err, os.ErrNotExist) { + return Run{}, err + } + prevStars := map[string]int{} + for _, p := range idx { // previous successful run + if p.Error != "" { + continue + } + if prev, err := s.Run(id, p.ID); err == nil { + for _, it := range prev.Items { + prevStars[it.FullName] = it.Stars + } + } + break + } + baseline := len(seen) == 0 // the first successful run has nothing "new" + for i, r := range repos { + it := RunItem{Repo: r, Rank: i + 1} + if first, ok := seen[r.FullName]; ok { + it.FirstSeen = first + } else { + it.New = !baseline + it.FirstSeen = info.At + seen[r.FullName] = info.At + if it.New { + run.New++ + } + } + if p, ok := prevStars[r.FullName]; ok { + d := r.Stars - p + it.StarsDelta = &d + if d > 0 { + run.Rising++ + } + } + run.Items = append(run.Items, it) + } + run.Count = len(run.Items) + if err := writeFileJSON(filepath.Join(dir, "seen.json"), seen); err != nil { + return Run{}, err + } + } + + if err := writeFileJSON(filepath.Join(dir, run.ID+".json"), run); err != nil { + return Run{}, err + } + idx = append([]RunInfo{run.RunInfo}, idx...) + if s.keep > 0 && len(idx) > s.keep { + for _, old := range idx[s.keep:] { + _ = os.Remove(filepath.Join(dir, old.ID+".json")) + } + idx = idx[:s.keep] + } + if err := writeFileJSON(filepath.Join(dir, "index.json"), idx); err != nil { + return Run{}, err + } + return run, nil +} + +func readJSON(path string, v any) error { + data, err := os.ReadFile(path) + if err != nil { + return err + } + if err := json.Unmarshal(data, v); err != nil { + return fmt.Errorf("%s: %w", path, err) + } + return nil +} + +// writeFileJSON replaces the file atomically. +func writeFileJSON(path string, v any) error { + data, err := json.Marshal(v) + if err != nil { + return err + } + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return err + } + return os.Rename(tmp, path) +} diff --git a/web/app.js b/web/app.js index 7f39034..97e9c62 100644 --- a/web/app.js +++ b/web/app.js @@ -68,6 +68,41 @@ const I18N = { syntaxHelp: 'The search field accepts GitHub qualifiers, combined with the filters above the list:', synUser: 'repositories of a user', synOrg: 'repositories of an organization', synTopics: 'several topics at once', synPhrase: 'exact phrase', synNot: 'exclude a word', synSize: 'size in KB', + pushed1w: 'Pushed < 1 week', created1w: 'Created < 1 week', created3m: 'Created < 3 months', created6m: 'Created < 6 months', + viewsAria: 'Views', viewSearch: 'Search', viewSaved: 'Watch', + saveBtn: 'Schedule', saveBtnTitle: 'Run this search on a regular basis and review what changes', + savedTitle: 'Scheduled searches', savedNew: 'New from a search', + savedCount: (n) => `${n} ${n === 1 ? 'search' : 'searches'}`, + savedEmptyTitle: 'No scheduled search yet.', + savedEmptyHint: 'Set the filters in Search (for example: created < 1 week, most stars, a topic), then click Schedule.', + newTotal: (n) => `${n} new ${n === 1 ? 'repository' : 'repositories'} in the latest runs`, + schedHours: (n) => `Every ${n} h`, schedDay: (tm) => `Every day at ${tm}`, schedWeek: ({ d, tm }) => `Every ${d} at ${tm}`, + paused: 'paused', nextRun: (s) => `next ${s}`, lastRun: (s) => `last run ${s}`, neverRun: 'never run', + running: 'Running…', statNew: (n) => `${n} new`, statRising: (n) => `${n} rising`, statCount: (n) => `${n} repos`, statErr: 'error', + runNow: 'Run now', runNowTitle: 'Run the search now and keep a new snapshot', + runDone: ({ n, r }) => `Done: ${n} new, ${r} rising`, runFailed: 'Run failed: ', + edit: 'Edit', delete: 'Delete', cancel: 'Cancel', save: 'Save', back: 'Back to the list', + pause: 'Pause', resume: 'Resume', + confirmDelete: (n) => `Delete "${n}" and its history?`, deleted: 'Deleted', + openInSearch: 'Open in search', openInSearchTitle: 'Load these filters in the search view', + runAria: 'Run', revModeAria: 'Show', + modeNew: (n) => `New · ${n}`, modeRising: (n) => `Rising · ${n}`, modeAll: (n) => `All · ${n}`, + runOption: ({ d, n }) => `${d} · ${n} new`, runOptionErr: (d) => `${d} · error`, + revInfo: ({ d, total, q }) => `${d} · ${total} matching on GitHub · ${q}`, + revEmptyNew: 'Nothing new in this run.', revEmptyNewHint: 'Every repository was already in a previous run.', + revEmptyFirst: 'This is the first run.', revEmptyFirstHint: 'It is the baseline: the next runs will show what is new and what is rising. "All" shows this snapshot.', + revEmptyRising: 'No star gains since the previous run.', revEmptyRisingHint: 'Rising compares each repository with the previous snapshot.', + revEmptyAll: 'This run returned no repository.', revEmptyAllHint: '', + revNoRun: 'No run yet.', revNoRunHint: 'The first run is on its way, or click Run now.', + badgeNew: 'new', deltaTitle: 'stars gained since the previous run', rankTitle: 'position in the results', + saveNewTitle: 'Schedule this search', saveEditTitle: 'Edit the scheduled search', + svName: 'Name', svQuery: 'GitHub query', svEvery: 'Frequency', svWeekday: 'Day', svTime: 'Time', svHours: 'Hours', + everyWeek: 'Every week', everyDay: 'Every day', everyHours: 'Every N hours', hoursUnit: 'hours', + svMax: 'Results kept', svEnabled: 'Active', svEnabledHelp: 'Run on schedule', + svHintNew: (tz) => `The search runs once right away to set the baseline, then on schedule (server time zone: ${tz}).`, + svHintEdit: (tz) => `Server time zone: ${tz}. To change the filters, open the search, adjust it and schedule it again.`, + saved: 'Search scheduled', updated2: 'Saved', + err_not_found: 'Not found', }, fr: { locale: 'fr-FR', @@ -134,6 +169,41 @@ const I18N = { syntaxHelp: 'Le champ de recherche accepte les qualificatifs GitHub, combinés aux filtres au-dessus de la liste :', synUser: 'dépôts d\'un utilisateur', synOrg: 'dépôts d\'une organisation', synTopics: 'plusieurs topics à la fois', synPhrase: 'expression exacte', synNot: 'exclure un mot', synSize: 'taille en Ko', + pushed1w: 'Push < 1 semaine', created1w: 'Créé < 1 semaine', created3m: 'Créé < 3 mois', created6m: 'Créé < 6 mois', + viewsAria: 'Vues', viewSearch: 'Recherche', viewSaved: 'Veille', + saveBtn: 'Programmer', saveBtnTitle: 'Lancer cette recherche régulièrement et suivre ce qui change', + savedTitle: 'Recherches programmées', savedNew: 'Nouvelle depuis une recherche', + savedCount: (n) => `${n} recherche${n > 1 ? 's' : ''}`, + savedEmptyTitle: 'Aucune recherche programmée.', + savedEmptyHint: 'Réglez les filtres dans Recherche (par exemple : créé < 1 semaine, plus d\'étoiles, un topic), puis cliquez sur Programmer.', + newTotal: (n) => `${n} nouveau${n > 1 ? 'x' : ''} dépôt${n > 1 ? 's' : ''} dans les derniers passages`, + schedHours: (n) => `Toutes les ${n} h`, schedDay: (tm) => `Chaque jour à ${tm}`, schedWeek: ({ d, tm }) => `Chaque ${d} à ${tm}`, + paused: 'en pause', nextRun: (s) => `prochain passage ${s}`, lastRun: (s) => `dernier passage ${s}`, neverRun: 'jamais exécutée', + running: 'En cours…', statNew: (n) => `${n} nouveau${n > 1 ? 'x' : ''}`, statRising: (n) => `${n} en hausse`, statCount: (n) => `${n} dépôts`, statErr: 'erreur', + runNow: 'Exécuter', runNowTitle: 'Lancer la recherche maintenant et garder un nouvel instantané', + runDone: ({ n, r }) => `Terminé : ${n} nouveau${n > 1 ? 'x' : ''}, ${r} en hausse`, runFailed: 'Échec du passage : ', + edit: 'Modifier', delete: 'Supprimer', cancel: 'Annuler', save: 'Enregistrer', back: 'Retour à la liste', + pause: 'Mettre en pause', resume: 'Réactiver', + confirmDelete: (n) => `Supprimer « ${n} » et son historique ?`, deleted: 'Supprimée', + openInSearch: 'Ouvrir dans la recherche', openInSearchTitle: 'Charger ces filtres dans la vue Recherche', + runAria: 'Passage', revModeAria: 'Afficher', + modeNew: (n) => `Nouveautés · ${n}`, modeRising: (n) => `Progressions · ${n}`, modeAll: (n) => `Tous · ${n}`, + runOption: ({ d, n }) => `${d} · ${n} nouveau${n > 1 ? 'x' : ''}`, runOptionErr: (d) => `${d} · erreur`, + revInfo: ({ d, total, q }) => `${d} · ${total} résultats sur GitHub · ${q}`, + revEmptyNew: 'Rien de nouveau dans ce passage.', revEmptyNewHint: 'Tous les dépôts étaient déjà présents dans un passage précédent.', + revEmptyFirst: 'C\'est le premier passage.', revEmptyFirstHint: 'Il sert de référence : les suivants montreront les nouveautés et les progressions. « Tous » affiche cet instantané.', + revEmptyRising: 'Aucun gain d\'étoiles depuis le passage précédent.', revEmptyRisingHint: 'Progressions compare chaque dépôt avec l\'instantané précédent.', + revEmptyAll: 'Ce passage n\'a renvoyé aucun dépôt.', revEmptyAllHint: '', + revNoRun: 'Pas encore de passage.', revNoRunHint: 'Le premier passage est en cours, ou cliquez sur Exécuter.', + badgeNew: 'nouveau', deltaTitle: 'étoiles gagnées depuis le passage précédent', rankTitle: 'position dans les résultats', + saveNewTitle: 'Programmer cette recherche', saveEditTitle: 'Modifier la recherche programmée', + svName: 'Nom', svQuery: 'Requête GitHub', svEvery: 'Fréquence', svWeekday: 'Jour', svTime: 'Heure', svHours: 'Heures', + everyWeek: 'Chaque semaine', everyDay: 'Chaque jour', everyHours: 'Toutes les N heures', hoursUnit: 'heures', + svMax: 'Résultats gardés', svEnabled: 'Active', svEnabledHelp: 'Exécuter selon la fréquence', + svHintNew: (tz) => `La recherche est lancée une première fois tout de suite pour servir de référence, puis selon la fréquence (fuseau du serveur : ${tz}).`, + svHintEdit: (tz) => `Fuseau du serveur : ${tz}. Pour changer les filtres, ouvrez la recherche, ajustez-la et programmez-la à nouveau.`, + saved: 'Recherche programmée', updated2: 'Enregistré', + err_not_found: 'Introuvable', }, }; @@ -171,8 +241,8 @@ function fmtAgo(s) { return ''; } -async function api(url) { - const res = await fetch(url); +async function api(url, opts) { + const res = await fetch(url, opts); const text = await res.text(); let data = null; try { data = text ? JSON.parse(text) : null; } catch { /* not JSON */ } @@ -191,8 +261,8 @@ function apiError(res, text, data) { function toast(msg) { const el = $('#toast'); - const dlg = $('#settingsDlg'); - (dlg.open ? dlg : document.body).append(el); + const dlg = document.querySelector('dialog[open]'); + (dlg || document.body).append(el); el.textContent = msg; el.hidden = false; clearTimeout(toast.tm); @@ -291,7 +361,11 @@ function syncURL(f) { } function filtersFromURL() { - const src = location.search ? new URLSearchParams(location.search) : new URLSearchParams(store.get('filters', '')); + return filtersFromParams(location.search || store.get('filters', '')); +} + +function filtersFromParams(qs) { + const src = new URLSearchParams(qs); const f = {}; for (const k of FILTERS) f[k] = src.get(k) || ''; for (const k of TOGGLES) f[k] = src.get(k) === '1'; @@ -352,7 +426,9 @@ const ICON_EXT = '${esc(t('badgeNew'))}` : '') + + (r.archived ? `${esc(t('archived'))}` : '') + (r.fork ? `${esc(t('fork'))}` : ''); const topics = r.topics.length ? `
${r.topics.map((x) => ``).join('')}
` @@ -360,7 +436,10 @@ function rowHTML(r, i) { const lang = r.language ? `${esc(r.language)}` : ''; - return `
+ const delta = !review ? '' + : r.starsDelta ? `${r.starsDelta > 0 ? '+' : ''}${esc(fmtCompact(r.starsDelta))} ★` + : ''; + return `
${esc(r.owner)}/${esc(r.name)}${badges}
@@ -368,7 +447,7 @@ function rowHTML(r, i) { ${topics}
- ${lang} + ${delta}${lang} ${ICON_STAR}${fmtCompact(r.stars)} ${ICON_FORK}${fmtCompact(r.forks)} ${ICON_CLOCK}${esc(fmtAgo(r.pushedAt))} @@ -449,6 +528,7 @@ async function loadStatus() { try { const s = await api('/api/status'); state.token = s.token; + state.tz = s.tz || ''; if (s.rate && s.rate.limit) state.rate = s.rate; } catch { state.token = null; } renderStatus(); @@ -456,38 +536,49 @@ async function loadStatus() { /* ================= List interactions ================= */ -$('#list').addEventListener('click', (ev) => { - const rowEl = ev.target.closest('.row'); - if (!rowEl) return; - const r = state.items[Number(rowEl.dataset.i)]; - const topic = ev.target.closest('[data-topic]'); - if (topic) { - $('#topic').value = topic.dataset.topic; - onFilterChange(); - return; - } - if (ev.target.closest('[data-act="lang"]')) { - writeFilters({ ...readFilters(), language: r.language }); - onFilterChange(); - return; - } - const cp = ev.target.closest('[data-copy]'); - if (cp) { copy(cp.dataset.copy); return; } - if (ev.target.closest('a, .details')) return; - toggleRow(rowEl, r); -}); +// The search list and the review list share the same rows; items() returns +// the array the data-i indexes refer to. +function listEvents(listEl, items) { + listEl.addEventListener('click', (ev) => { + const rowEl = ev.target.closest('.row'); + if (!rowEl) return; + const r = items()[Number(rowEl.dataset.i)]; + const topic = ev.target.closest('[data-topic]'); + if (topic) { + filterFromList({ topic: topic.dataset.topic }); + return; + } + if (ev.target.closest('[data-act="lang"]')) { + filterFromList({ language: r.language }); + return; + } + const cp = ev.target.closest('[data-copy]'); + if (cp) { copy(cp.dataset.copy); return; } + if (ev.target.closest('a, .details')) return; + toggleRow(listEl, rowEl, r); + }); + listEl.addEventListener('keydown', (ev) => { + if (ev.key !== 'Enter' || !ev.target.classList.contains('row')) return; + toggleRow(listEl, ev.target, items()[Number(ev.target.dataset.i)]); + }); +} +listEvents($('#list'), () => state.items); -$('#list').addEventListener('keydown', (ev) => { - if (ev.key !== 'Enter' || !ev.target.classList.contains('row')) return; - toggleRow(ev.target, state.items[Number(ev.target.dataset.i)]); -}); +// A click on a topic or a language filters the search (from the review too). +function filterFromList(change) { + const base = view === 'search' ? readFilters() : review.search ? filtersFromParams(review.search.params) : {}; + writeFilters({ ...base, ...change }); + state.searchPending = false; + if (view !== 'search') location.hash = ''; + onFilterChange(); +} -function toggleRow(rowEl, r) { +function toggleRow(listEl, rowEl, r) { if (state.open.has(r.fullName)) state.open.delete(r.fullName); else state.open.add(r.fullName); const i = Number(rowEl.dataset.i); rowEl.outerHTML = rowHTML(r, i); - $(`#list .row[data-i="${i}"]`).focus({ preventScroll: true }); + listEl.querySelector(`.row[data-i="${i}"]`).focus({ preventScroll: true }); } /* ================= Filter events ================= */ @@ -516,7 +607,7 @@ $('#resetBtn').addEventListener('click', () => { $('#moreBtn').addEventListener('click', () => search(true)); document.addEventListener('keydown', (ev) => { - if (ev.key === '/' && !ev.target.closest('input, select, textarea') && !$('#settingsDlg').open) { + if (ev.key === '/' && view === 'search' && !ev.target.closest('input, select, textarea') && !document.querySelector('dialog[open]')) { ev.preventDefault(); $('#q').focus(); $('#q').select(); @@ -567,6 +658,8 @@ function setLang(next) { applyLang(); renderList(); renderStatus(); + renderSaved(); + renderReview(); } $('#langSwitch').addEventListener('click', (ev) => { @@ -663,6 +756,400 @@ $('#settingsBtn').addEventListener('click', () => { $('#closeSettings').addEventListener('click', () => $('#settingsDlg').close()); $('#settingsDlg').addEventListener('click', (ev) => { if (ev.target === ev.currentTarget) ev.currentTarget.close(); }); +/* ================= Views ================= */ + +// # search +// #saved scheduled searches +// #saved/ review of one scheduled search (#saved// for an older run) +let view = 'search'; + +function route() { + const parts = location.hash.replace(/^#\/?/, '').split('/').filter(Boolean); + if (parts[0] === 'saved' && parts[1]) { + view = 'review'; + openReview(decodeURIComponent(parts[1]), parts[2] ? decodeURIComponent(parts[2]) : 'latest'); + } else if (parts[0] === 'saved') { + view = 'saved'; + loadSaved(); + } else { + view = 'search'; + if (state.searchPending) { state.searchPending = false; search(); } + } + document.body.dataset.view = view; + $('#viewSearch').hidden = view !== 'search'; + $('#viewSaved').hidden = view !== 'saved'; + $('#viewReview').hidden = view !== 'review'; + for (const a of document.querySelectorAll('#views [data-view]')) { + const cur = a.dataset.view === (view === 'review' ? 'saved' : view); + if (cur) a.setAttribute('aria-current', 'page'); else a.removeAttribute('aria-current'); + } + window.scrollTo({ top: 0 }); +} +window.addEventListener('hashchange', route); + +/* ================= Scheduled searches ================= */ + +const ICON_PLAY = ''; +const ICON_EDIT = ''; +const ICON_PAUSE = ''; +const ICON_TRASH = ''; +const ICON_UP = ''; +const ICON_SPIN = ''; + +const saved = { list: [], loaded: false, running: new Set() }; +const pad2 = (n) => String(n).padStart(2, '0'); +const dayName = (d) => new Intl.DateTimeFormat(t('locale'), { weekday: 'long', timeZone: 'UTC' }).format(new Date(Date.UTC(2023, 0, 1 + d))); +const fmtDateTime = (s) => new Date(s).toLocaleString(t('locale'), { weekday: 'short', day: 'numeric', month: 'short', hour: '2-digit', minute: '2-digit' }); + +function schedText(sc) { + const tm = pad2(sc.hour) + ':' + pad2(sc.minute); + if (sc.every === 'hours') return t('schedHours', sc.hours); + if (sc.every === 'day') return t('schedDay', tm); + return t('schedWeek', { d: dayName(sc.weekday), tm }); +} + +// Readable GitHub query of saved parameters (computed by the server on each run). +function savedQuery(x) { + return (x.lastRun && x.lastRun.query) || new URLSearchParams(x.params).toString().replace(/&/g, ' '); +} + +async function loadSaved() { + try { + saved.list = await api('/api/saved'); + saved.loaded = true; + $('#savedError').hidden = true; + } catch (e) { + $('#savedError').textContent = e.message; + $('#savedError').hidden = false; + } + renderSaved(); +} + +function renderSaved() { + const n = saved.list.reduce((a, x) => a + (x.enabled && x.lastRun && !x.lastRun.error ? x.lastRun.new : 0), 0); + const pill = $('#newTotal'); + pill.hidden = !n; + pill.textContent = n; + pill.title = n ? t('newTotal', n) : ''; + if (!saved.loaded) return; + $('#savedCount').textContent = saved.list.length ? t('savedCount', saved.list.length) : ''; + $('#savedEmpty').hidden = saved.list.length > 0 || !$('#savedError').hidden; + $('#savedList').innerHTML = saved.list.map((x) => { + const lr = x.lastRun; + let stats = ''; + if (saved.running.has(x.id)) stats = `${ICON_SPIN}${esc(t('running'))}`; + else if (lr && lr.error) stats = `${esc(t('statErr'))}`; + else if (lr) { + stats = (lr.new ? `${esc(t('statNew', lr.new))}` : '') + + (lr.rising ? `${ICON_UP}${esc(t('statRising', lr.rising))}` : '') + + `${esc(t('statCount', lr.count))}`; + } + const when = [ + `${ICON_CLOCK}${esc(schedText(x.schedule))}`, + x.enabled ? (x.nextRunAt ? `${esc(t('nextRun', fmtAgo(x.nextRunAt)))}` : '') : `${esc(t('paused'))}`, + `${esc(x.lastRunAt ? t('lastRun', fmtAgo(x.lastRunAt)) : t('neverRun'))}`, + ].join(''); + return `
+
+
${esc(x.name)}
+
${esc(savedQuery(x))}
+
${when}
+
+
${stats}
+
+ + + + +
+
`; + }).join(''); +} + +$('#savedList').addEventListener('click', (ev) => { + const row = ev.target.closest('.srow'); + if (!row) return; + const x = saved.list.find((s) => s.id === row.dataset.id); + const act = ev.target.closest('[data-act]'); + if (!act) { location.hash = '#saved/' + encodeURIComponent(x.id); return; } + if (act.dataset.act === 'run') runSaved(x.id); + else if (act.dataset.act === 'edit') openSaveDlg(x); + else if (act.dataset.act === 'delete') deleteSaved(x); + else if (act.dataset.act === 'toggle') putSaved(x, { enabled: !x.enabled }); +}); +$('#savedList').addEventListener('keydown', (ev) => { + if (ev.key === 'Enter' && ev.target.classList.contains('srow')) location.hash = '#saved/' + encodeURIComponent(ev.target.dataset.id); +}); + +async function runSaved(id) { + saved.running.add(id); + renderSaved(); + $('#revRunBtn').disabled = true; + try { + const info = await api('/api/saved/' + encodeURIComponent(id) + '/run', { method: 'POST' }); + toast(info.error ? t('runFailed') + info.error : t('runDone', { n: info.new, r: info.rising })); + } catch (e) { + toast(t('runFailed') + e.message); + } finally { + saved.running.delete(id); + $('#revRunBtn').disabled = false; + } + await loadSaved(); + if (view === 'review' && review.id === id) openReview(id, 'latest'); +} + +async function putSaved(x, change) { + const body = { name: x.name, params: x.params, schedule: x.schedule, maxResults: x.maxResults, enabled: x.enabled, ...change }; + try { + await api('/api/saved/' + encodeURIComponent(x.id), { method: 'PUT', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + toast(t('updated2')); + } catch (e) { + toast(e.message); + } + await loadSaved(); + if (view === 'review') renderReviewHead(); +} + +async function deleteSaved(x) { + if (!confirm(t('confirmDelete', x.name))) return false; + try { + await fetch('/api/saved/' + encodeURIComponent(x.id), { method: 'DELETE' }); + toast(t('deleted')); + } catch (e) { + toast(e.message); + return false; + } + await loadSaved(); + return true; +} + +/* ---------- Review of one scheduled search ---------- */ + +const review = { id: '', search: null, runs: [], run: null, mode: '', items: [], req: 0 }; +listEvents($('#revList'), () => review.items); + +async function openReview(id, runID) { + const req = ++review.req; + if (review.id !== id) { review.mode = ''; review.run = null; review.runs = []; review.search = null; } + review.id = id; + document.body.classList.add('busy'); + try { + if (!saved.loaded) await loadSaved(); + const [x, runs] = await Promise.all([ + api('/api/saved/' + encodeURIComponent(id)), + api('/api/saved/' + encodeURIComponent(id) + '/runs'), + ]); + let run = null; + if (runs.length) { + try { run = await api('/api/saved/' + encodeURIComponent(id) + '/runs/' + encodeURIComponent(runID)); } catch { /* pruned run */ } + } + if (req !== review.req) return; + Object.assign(review, { search: x, runs, run }); + if (!review.mode || runID === 'latest') { + review.mode = run && run.new ? 'new' : run && run.rising ? 'rising' : 'all'; + } + $('#revError').hidden = true; + } catch (e) { + if (req !== review.req) return; + $('#revError').textContent = e.message; + $('#revError').hidden = false; + } finally { + if (req === review.req) document.body.classList.remove('busy'); + } + state.open.clear(); + renderReview(); +} + +function renderReviewHead() { + const x = saved.list.find((s) => s.id === review.id) || review.search; + if (!x) return; + review.search = x; + $('#revName').textContent = x.name; + $('#revSched').textContent = schedText(x.schedule) + (x.enabled + ? (x.nextRunAt ? ' · ' + t('nextRun', fmtDateTime(x.nextRunAt)) : '') + : ' · ' + t('paused')); +} + +function renderReview() { + if (view !== 'review') return; + renderReviewHead(); + const run = review.run; + const sel = $('#runSel'); + sel.innerHTML = review.runs.map((r) => ``).join(''); + sel.hidden = !review.runs.length; + if (run) sel.value = run.id; + + const items = run ? run.items : []; + const lists = { + new: items.filter((r) => r.new), + rising: items.filter((r) => r.starsDelta > 0).sort((a, b) => b.starsDelta - a.starsDelta || b.stars - a.stars), + all: items, + }; + for (const b of document.querySelectorAll('#revMode [data-mode]')) { + const m = b.dataset.mode; + b.textContent = t('mode' + m[0].toUpperCase() + m.slice(1), fmtNum(lists[m].length)); + b.setAttribute('aria-checked', String(m === review.mode)); + } + $('#revMode').hidden = !run || !!run.error; + review.items = run && !run.error ? lists[review.mode] || [] : []; + + const info = $('#revInfo'); + info.textContent = run && !run.error ? t('revInfo', { d: fmtDateTime(run.at), total: fmtNum(run.total), q: run.query }) : ''; + if (run && run.error) { + $('#revError').textContent = t('runFailed') + run.error; + $('#revError').hidden = false; + } else if (run) { + $('#revError').hidden = true; + } + + $('#revList').classList.add('review'); + $('#revList').innerHTML = review.items.map(rowHTML).join(''); + const empty = !review.items.length && !(run && run.error); + $('#revEmpty').hidden = !empty; + if (empty) { + const first = review.runs.filter((r) => !r.error).length <= 1; + const key = !run ? 'revNoRun' + : review.mode === 'new' ? (first ? 'revEmptyFirst' : 'revEmptyNew') + : review.mode === 'rising' ? (first ? 'revEmptyFirst' : 'revEmptyRising') : 'revEmptyAll'; + $('#revEmptyTitle').textContent = t(key); + $('#revEmptyHint').textContent = t(key + 'Hint'); + } +} + +$('#revMode').addEventListener('click', (ev) => { + const b = ev.target.closest('[data-mode]'); + if (!b) return; + review.mode = b.dataset.mode; + state.open.clear(); + renderReview(); +}); +$('#runSel').addEventListener('change', () => { + const latest = review.runs[0] && review.runs[0].id === $('#runSel').value; + location.hash = '#saved/' + encodeURIComponent(review.id) + (latest ? '' : '/' + encodeURIComponent($('#runSel').value)); +}); +$('#revRunBtn').addEventListener('click', () => runSaved(review.id)); +$('#revEditBtn').addEventListener('click', () => review.search && openSaveDlg(review.search)); +$('#revOpenBtn').addEventListener('click', () => { + if (!review.search) return; + writeFilters(filtersFromParams(review.search.params)); + state.searchPending = false; + location.hash = ''; + onFilterChange(); +}); + +/* ---------- Create / edit dialog ---------- */ + +const dlgState = { edit: null, params: '', max: 50 }; + +{ + const sel = $('#svWeekday'); + for (const d of [1, 2, 3, 4, 5, 6, 0]) sel.add(new Option('', String(d))); +} + +function renderSaveDlgLabels() { + for (const o of $('#svWeekday').options) o.textContent = dayName(Number(o.value)); + for (const b of document.querySelectorAll('#svMax [data-max]')) b.setAttribute('aria-checked', String(Number(b.dataset.max) === dlgState.max)); + const every = $('#svEvery').value; + $('#svWeekday').hidden = every !== 'week'; + $('#svTime').hidden = every === 'hours'; + $('#svHoursWrap').hidden = every !== 'hours'; +} + +// A readable default name: the search words, else the topic or the language. +function defaultName(f) { + const bits = [f.q, f.topic && 'topic:' + f.topic, f.language].filter(Boolean); + return bits.join(' · ') || t('savedTitle'); +} + +function openSaveDlg(x) { + dlgState.edit = x || null; + let sc = { every: 'week', weekday: 1, hour: 8, minute: 0, hours: 24 }; + if (x) { + dlgState.params = x.params; + dlgState.max = x.maxResults; + sc = { ...sc, ...x.schedule }; + $('#svName').value = x.name; + $('#svQuery').textContent = savedQuery(x); + $('#svEnabled').checked = x.enabled; + } else { + const f = readFilters(); + dlgState.params = params(f).toString(); + dlgState.max = 50; + $('#svName').value = defaultName(f); + $('#svQuery').textContent = state.query || dlgState.params; + $('#svEnabled').checked = true; + } + $('#svEvery').value = sc.every; + $('#svWeekday').value = String(sc.weekday || 0); + $('#svTime').value = pad2(sc.hour) + ':' + pad2(sc.minute); + $('#svHours').value = sc.hours || 24; + $('#saveTitle').textContent = t(x ? 'saveEditTitle' : 'saveNewTitle'); + $('#svHint').textContent = t(x ? 'svHintEdit' : 'svHintNew', state.tz || 'UTC'); + $('#svDelete').hidden = !x; + renderSaveDlgLabels(); + $('#saveDlg').showModal(); + $('#svName').select(); +} + +$('#svEvery').addEventListener('change', renderSaveDlgLabels); +$('#svMax').addEventListener('click', (ev) => { + const b = ev.target.closest('[data-max]'); + if (!b) return; + dlgState.max = Number(b.dataset.max); + renderSaveDlgLabels(); +}); +$('#closeSave').addEventListener('click', () => $('#saveDlg').close()); +$('#svCancel').addEventListener('click', () => $('#saveDlg').close()); +$('#saveDlg').addEventListener('click', (ev) => { if (ev.target === ev.currentTarget) ev.currentTarget.close(); }); +$('#svDelete').addEventListener('click', async () => { + if (dlgState.edit && await deleteSaved(dlgState.edit)) { + $('#saveDlg').close(); + if (view === 'review') location.hash = '#saved'; + } +}); + +$('#saveForm').addEventListener('submit', async (ev) => { + ev.preventDefault(); + const [h, m] = ($('#svTime').value || '08:00').split(':').map(Number); + const body = { + name: $('#svName').value.trim(), + params: dlgState.params, + maxResults: dlgState.max, + enabled: $('#svEnabled').checked, + schedule: { every: $('#svEvery').value, weekday: Number($('#svWeekday').value), hour: h || 0, minute: m || 0, hours: Number($('#svHours').value) || 24 }, + }; + const x = dlgState.edit; + $('#svSubmit').disabled = true; + try { + const res = await api(x ? '/api/saved/' + encodeURIComponent(x.id) : '/api/saved', { + method: x ? 'PUT' : 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body), + }); + $('#saveDlg').close(); + toast(t(x ? 'updated2' : 'saved')); + if (x) { + await loadSaved(); + if (view === 'review') renderReview(); + } else { + saved.running.add(res.id); // the server runs it once right away + location.hash = '#saved'; + setTimeout(async () => { saved.running.delete(res.id); await loadSaved(); }, 4000); + } + } catch (e) { + toast(e.message); + } finally { + $('#svSubmit').disabled = false; + } +}); + +$('#saveBtn').addEventListener('click', () => openSaveDlg()); + +// Keep the list fresh (next run countdown, runs made by the scheduler). +setInterval(() => { + if (document.hidden) return; + if (view === 'saved') loadSaved(); +}, 30000); +loadSaved(); + /* ================= Startup ================= */ applyDensity(store.get('density', 'comfortable')); @@ -670,5 +1157,7 @@ applyLang(); renderInterface(); writeFilters(filtersFromURL()); loadStatus(); -search(); +route(); +if (view === 'search') search(); +else state.searchPending = true; setInterval(() => { if (!document.hidden) renderStatus(); }, 5000); // reset countdown diff --git a/web/index.html b/web/index.html index 946dd2a..7c38400 100644 --- a/web/index.html +++ b/web/index.html @@ -23,7 +23,19 @@ Searchgit
-
+
+ + + + + +