package main import ( "crypto/rand" "encoding/hex" "encoding/json" "errors" "fmt" "os" "path/filepath" "regexp" "sort" "sync" "time" ) // SavedSearch is a search that the scheduler runs on a regular basis. type SavedSearch struct { ID string `json:"id"` Name string `json:"name"` Params string `json:"params"` // same query string as /api/search (without page) Schedule Schedule `json:"schedule"` MaxResults int `json:"maxResults"` // 30, 50 or 100 (one GitHub page) Enabled bool `json:"enabled"` CreatedAt time.Time `json:"createdAt"` LastRunAt time.Time `json:"lastRunAt,omitzero"` NextRunAt time.Time `json:"nextRunAt,omitzero"` LastError string `json:"lastError,omitempty"` LastRun *RunInfo `json:"lastRun,omitempty"` } // RunInfo summarizes one execution of a saved search. type RunInfo struct { ID string `json:"id"` At time.Time `json:"at"` Query string `json:"query"` Total int `json:"total"` Count int `json:"count"` // repositories kept in the snapshot New int `json:"new"` // never seen in a previous run Rising int `json:"rising"` // gained stars since the previous run Error string `json:"error,omitempty"` TookMs int64 `json:"tookMs"` Trigger string `json:"trigger"` // "schedule" or "manual" } // RunItem is one repository of a snapshot, compared with the previous run. type RunItem struct { Repo New bool `json:"new"` FirstSeen time.Time `json:"firstSeen"` StarsDelta *int `json:"starsDelta,omitempty"` // nil when absent from the previous run Rank int `json:"rank"` } // Run is a full snapshot. type Run struct { RunInfo Items []RunItem `json:"items"` } // Store keeps saved searches and their runs as JSON files: // // /searches.json // /runs//index.json run summaries, newest first // /runs//seen.json repository -> first time seen // /runs//.json type Store struct { dir string keep int // runs kept per search (0 = all) mu sync.Mutex searches []*SavedSearch } var idRe = regexp.MustCompile(`^[a-zA-Z0-9-]{1,40}$`) var errNotFound = &codedError{code: "not_found", msg: "not found"} func OpenStore(dir string, keep int) (*Store, error) { if err := os.MkdirAll(filepath.Join(dir, "runs"), 0o755); err != nil { return nil, err } s := &Store{dir: dir, keep: keep} if err := readJSON(filepath.Join(dir, "searches.json"), &s.searches); err != nil && !errors.Is(err, os.ErrNotExist) { return nil, err } return s, nil } func newID() string { b := make([]byte, 6) _, _ = rand.Read(b) return hex.EncodeToString(b) } // List returns copies of the saved searches, sorted by name. func (s *Store) List() []SavedSearch { s.mu.Lock() defer s.mu.Unlock() out := make([]SavedSearch, 0, len(s.searches)) for _, x := range s.searches { out = append(out, *x) } sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) return out } func (s *Store) Get(id string) (SavedSearch, error) { s.mu.Lock() defer s.mu.Unlock() if x := s.find(id); x != nil { return *x, nil } return SavedSearch{}, errNotFound } func (s *Store) find(id string) *SavedSearch { for _, x := range s.searches { if x.ID == id { return x } } return nil } func (s *Store) Create(x SavedSearch) (SavedSearch, error) { s.mu.Lock() defer s.mu.Unlock() x.ID = newID() x.CreatedAt = time.Now() s.searches = append(s.searches, &x) return x, s.saveLocked() } // Update changes what the user can edit and returns the new value. func (s *Store) Update(id string, fn func(*SavedSearch)) (SavedSearch, error) { s.mu.Lock() defer s.mu.Unlock() x := s.find(id) if x == nil { return SavedSearch{}, errNotFound } fn(x) return *x, s.saveLocked() } func (s *Store) Delete(id string) error { s.mu.Lock() defer s.mu.Unlock() for i, x := range s.searches { if x.ID == id { s.searches = append(s.searches[:i], s.searches[i+1:]...) if err := s.saveLocked(); err != nil { return err } return os.RemoveAll(s.runDir(id)) } } return errNotFound } func (s *Store) saveLocked() error { return writeFileJSON(filepath.Join(s.dir, "searches.json"), s.searches) } func (s *Store) runDir(id string) string { return filepath.Join(s.dir, "runs", id) } // Runs returns the run summaries of a search, newest first. func (s *Store) Runs(id string) ([]RunInfo, error) { if !idRe.MatchString(id) { return nil, errNotFound } var idx []RunInfo err := readJSON(filepath.Join(s.runDir(id), "index.json"), &idx) if errors.Is(err, os.ErrNotExist) { return []RunInfo{}, nil } return idx, err } // Run reads one snapshot ("latest" for the most recent one). func (s *Store) Run(id, runID string) (Run, error) { if runID == "latest" { idx, err := s.Runs(id) if err != nil { return Run{}, err } if len(idx) == 0 { return Run{}, errNotFound } runID = idx[0].ID } if !idRe.MatchString(id) || !idRe.MatchString(runID) { return Run{}, errNotFound } var r Run err := readJSON(filepath.Join(s.runDir(id), runID+".json"), &r) if errors.Is(err, os.ErrNotExist) { return Run{}, errNotFound } return r, err } // AddRun compares the repositories with the previous run, saves the snapshot // and updates the search. A failed run (info.Error set) is only recorded in // the history. func (s *Store) AddRun(id string, info RunInfo, repos []Repo) (Run, error) { dir := s.runDir(id) if err := os.MkdirAll(dir, 0o755); err != nil { return Run{}, err } idx, err := s.Runs(id) if err != nil { return Run{}, err } run := Run{RunInfo: info, Items: []RunItem{}} if info.Error == "" { seen := map[string]time.Time{} if err := readJSON(filepath.Join(dir, "seen.json"), &seen); err != nil && !errors.Is(err, os.ErrNotExist) { return Run{}, err } prevStars := map[string]int{} for _, p := range idx { // previous successful run if p.Error != "" { continue } if prev, err := s.Run(id, p.ID); err == nil { for _, it := range prev.Items { prevStars[it.FullName] = it.Stars } } break } baseline := len(seen) == 0 // the first successful run has nothing "new" for i, r := range repos { it := RunItem{Repo: r, Rank: i + 1} if first, ok := seen[r.FullName]; ok { it.FirstSeen = first } else { it.New = !baseline it.FirstSeen = info.At seen[r.FullName] = info.At if it.New { run.New++ } } if p, ok := prevStars[r.FullName]; ok { d := r.Stars - p it.StarsDelta = &d if d > 0 { run.Rising++ } } run.Items = append(run.Items, it) } run.Count = len(run.Items) if err := writeFileJSON(filepath.Join(dir, "seen.json"), seen); err != nil { return Run{}, err } } if err := writeFileJSON(filepath.Join(dir, run.ID+".json"), run); err != nil { return Run{}, err } idx = append([]RunInfo{run.RunInfo}, idx...) if s.keep > 0 && len(idx) > s.keep { for _, old := range idx[s.keep:] { _ = os.Remove(filepath.Join(dir, old.ID+".json")) } idx = idx[:s.keep] } if err := writeFileJSON(filepath.Join(dir, "index.json"), idx); err != nil { return Run{}, err } return run, nil } func readJSON(path string, v any) error { data, err := os.ReadFile(path) if err != nil { return err } if err := json.Unmarshal(data, v); err != nil { return fmt.Errorf("%s: %w", path, err) } return nil } // writeFileJSON replaces the file atomically. func writeFileJSON(path string, v any) error { data, err := json.Marshal(v) if err != nil { return err } tmp := path + ".tmp" if err := os.WriteFile(tmp, data, 0o644); err != nil { return err } return os.Rename(tmp, path) }