Scheduled searches with a weekly review of new and rising repositories
A search can now be saved with a schedule (every N hours, daily, or weekly at a given day and time). An internal scheduler runs it and keeps a snapshot of the results on a /data volume (JSON files, no new dependency). The new Watch tab lists the scheduled searches; the review page of each one shows, for any run of its history, the repositories never seen before and the ones that gained the most stars since the previous run. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
claude BotandClaude Opus 5.5 committed
1 parent
8f4048d046
commit
02b51fed77
16 files changed
+1745
-45
No files matched your search
@@ -0,0 +1,301 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SavedSearch is a search that the scheduler runs on a regular basis.
|
||||
type SavedSearch struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Params string `json:"params"` // same query string as /api/search (without page)
|
||||
Schedule Schedule `json:"schedule"`
|
||||
MaxResults int `json:"maxResults"` // 30, 50 or 100 (one GitHub page)
|
||||
Enabled bool `json:"enabled"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
LastRunAt time.Time `json:"lastRunAt,omitzero"`
|
||||
NextRunAt time.Time `json:"nextRunAt,omitzero"`
|
||||
LastError string `json:"lastError,omitempty"`
|
||||
LastRun *RunInfo `json:"lastRun,omitempty"`
|
||||
}
|
||||
|
||||
// RunInfo summarizes one execution of a saved search.
|
||||
type RunInfo struct {
|
||||
ID string `json:"id"`
|
||||
At time.Time `json:"at"`
|
||||
Query string `json:"query"`
|
||||
Total int `json:"total"`
|
||||
Count int `json:"count"` // repositories kept in the snapshot
|
||||
New int `json:"new"` // never seen in a previous run
|
||||
Rising int `json:"rising"` // gained stars since the previous run
|
||||
Error string `json:"error,omitempty"`
|
||||
TookMs int64 `json:"tookMs"`
|
||||
Trigger string `json:"trigger"` // "schedule" or "manual"
|
||||
}
|
||||
|
||||
// RunItem is one repository of a snapshot, compared with the previous run.
|
||||
type RunItem struct {
|
||||
Repo
|
||||
New bool `json:"new"`
|
||||
FirstSeen time.Time `json:"firstSeen"`
|
||||
StarsDelta *int `json:"starsDelta,omitempty"` // nil when absent from the previous run
|
||||
Rank int `json:"rank"`
|
||||
}
|
||||
|
||||
// Run is a full snapshot.
|
||||
type Run struct {
|
||||
RunInfo
|
||||
Items []RunItem `json:"items"`
|
||||
}
|
||||
|
||||
// Store keeps saved searches and their runs as JSON files:
|
||||
//
|
||||
// <dir>/searches.json
|
||||
// <dir>/runs/<search id>/index.json run summaries, newest first
|
||||
// <dir>/runs/<search id>/seen.json repository -> first time seen
|
||||
// <dir>/runs/<search id>/<run id>.json
|
||||
type Store struct {
|
||||
dir string
|
||||
keep int // runs kept per search (0 = all)
|
||||
|
||||
mu sync.Mutex
|
||||
searches []*SavedSearch
|
||||
}
|
||||
|
||||
var idRe = regexp.MustCompile(`^[a-zA-Z0-9-]{1,40}$`)
|
||||
|
||||
var errNotFound = &codedError{code: "not_found", msg: "not found"}
|
||||
|
||||
func OpenStore(dir string, keep int) (*Store, error) {
|
||||
if err := os.MkdirAll(filepath.Join(dir, "runs"), 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s := &Store{dir: dir, keep: keep}
|
||||
if err := readJSON(filepath.Join(dir, "searches.json"), &s.searches); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return nil, err
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func newID() string {
|
||||
b := make([]byte, 6)
|
||||
_, _ = rand.Read(b)
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
|
||||
// List returns copies of the saved searches, sorted by name.
|
||||
func (s *Store) List() []SavedSearch {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
out := make([]SavedSearch, 0, len(s.searches))
|
||||
for _, x := range s.searches {
|
||||
out = append(out, *x)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Store) Get(id string) (SavedSearch, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if x := s.find(id); x != nil {
|
||||
return *x, nil
|
||||
}
|
||||
return SavedSearch{}, errNotFound
|
||||
}
|
||||
|
||||
func (s *Store) find(id string) *SavedSearch {
|
||||
for _, x := range s.searches {
|
||||
if x.ID == id {
|
||||
return x
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) Create(x SavedSearch) (SavedSearch, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
x.ID = newID()
|
||||
x.CreatedAt = time.Now()
|
||||
s.searches = append(s.searches, &x)
|
||||
return x, s.saveLocked()
|
||||
}
|
||||
|
||||
// Update changes what the user can edit and returns the new value.
|
||||
func (s *Store) Update(id string, fn func(*SavedSearch)) (SavedSearch, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
x := s.find(id)
|
||||
if x == nil {
|
||||
return SavedSearch{}, errNotFound
|
||||
}
|
||||
fn(x)
|
||||
return *x, s.saveLocked()
|
||||
}
|
||||
|
||||
func (s *Store) Delete(id string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for i, x := range s.searches {
|
||||
if x.ID == id {
|
||||
s.searches = append(s.searches[:i], s.searches[i+1:]...)
|
||||
if err := s.saveLocked(); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.RemoveAll(s.runDir(id))
|
||||
}
|
||||
}
|
||||
return errNotFound
|
||||
}
|
||||
|
||||
func (s *Store) saveLocked() error {
|
||||
return writeFileJSON(filepath.Join(s.dir, "searches.json"), s.searches)
|
||||
}
|
||||
|
||||
func (s *Store) runDir(id string) string { return filepath.Join(s.dir, "runs", id) }
|
||||
|
||||
// Runs returns the run summaries of a search, newest first.
|
||||
func (s *Store) Runs(id string) ([]RunInfo, error) {
|
||||
if !idRe.MatchString(id) {
|
||||
return nil, errNotFound
|
||||
}
|
||||
var idx []RunInfo
|
||||
err := readJSON(filepath.Join(s.runDir(id), "index.json"), &idx)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return []RunInfo{}, nil
|
||||
}
|
||||
return idx, err
|
||||
}
|
||||
|
||||
// Run reads one snapshot ("latest" for the most recent one).
|
||||
func (s *Store) Run(id, runID string) (Run, error) {
|
||||
if runID == "latest" {
|
||||
idx, err := s.Runs(id)
|
||||
if err != nil {
|
||||
return Run{}, err
|
||||
}
|
||||
if len(idx) == 0 {
|
||||
return Run{}, errNotFound
|
||||
}
|
||||
runID = idx[0].ID
|
||||
}
|
||||
if !idRe.MatchString(id) || !idRe.MatchString(runID) {
|
||||
return Run{}, errNotFound
|
||||
}
|
||||
var r Run
|
||||
err := readJSON(filepath.Join(s.runDir(id), runID+".json"), &r)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return Run{}, errNotFound
|
||||
}
|
||||
return r, err
|
||||
}
|
||||
|
||||
// AddRun compares the repositories with the previous run, saves the snapshot
|
||||
// and updates the search. A failed run (info.Error set) is only recorded in
|
||||
// the history.
|
||||
func (s *Store) AddRun(id string, info RunInfo, repos []Repo) (Run, error) {
|
||||
dir := s.runDir(id)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return Run{}, err
|
||||
}
|
||||
idx, err := s.Runs(id)
|
||||
if err != nil {
|
||||
return Run{}, err
|
||||
}
|
||||
run := Run{RunInfo: info, Items: []RunItem{}}
|
||||
|
||||
if info.Error == "" {
|
||||
seen := map[string]time.Time{}
|
||||
if err := readJSON(filepath.Join(dir, "seen.json"), &seen); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return Run{}, err
|
||||
}
|
||||
prevStars := map[string]int{}
|
||||
for _, p := range idx { // previous successful run
|
||||
if p.Error != "" {
|
||||
continue
|
||||
}
|
||||
if prev, err := s.Run(id, p.ID); err == nil {
|
||||
for _, it := range prev.Items {
|
||||
prevStars[it.FullName] = it.Stars
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
baseline := len(seen) == 0 // the first successful run has nothing "new"
|
||||
for i, r := range repos {
|
||||
it := RunItem{Repo: r, Rank: i + 1}
|
||||
if first, ok := seen[r.FullName]; ok {
|
||||
it.FirstSeen = first
|
||||
} else {
|
||||
it.New = !baseline
|
||||
it.FirstSeen = info.At
|
||||
seen[r.FullName] = info.At
|
||||
if it.New {
|
||||
run.New++
|
||||
}
|
||||
}
|
||||
if p, ok := prevStars[r.FullName]; ok {
|
||||
d := r.Stars - p
|
||||
it.StarsDelta = &d
|
||||
if d > 0 {
|
||||
run.Rising++
|
||||
}
|
||||
}
|
||||
run.Items = append(run.Items, it)
|
||||
}
|
||||
run.Count = len(run.Items)
|
||||
if err := writeFileJSON(filepath.Join(dir, "seen.json"), seen); err != nil {
|
||||
return Run{}, err
|
||||
}
|
||||
}
|
||||
|
||||
if err := writeFileJSON(filepath.Join(dir, run.ID+".json"), run); err != nil {
|
||||
return Run{}, err
|
||||
}
|
||||
idx = append([]RunInfo{run.RunInfo}, idx...)
|
||||
if s.keep > 0 && len(idx) > s.keep {
|
||||
for _, old := range idx[s.keep:] {
|
||||
_ = os.Remove(filepath.Join(dir, old.ID+".json"))
|
||||
}
|
||||
idx = idx[:s.keep]
|
||||
}
|
||||
if err := writeFileJSON(filepath.Join(dir, "index.json"), idx); err != nil {
|
||||
return Run{}, err
|
||||
}
|
||||
return run, nil
|
||||
}
|
||||
|
||||
func readJSON(path string, v any) error {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, v); err != nil {
|
||||
return fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeFileJSON replaces the file atomically.
|
||||
func writeFileJSON(path string, v any) error {
|
||||
data, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, data, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
Reference in new issue
Block a user