package main
import (
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"sort"
"sync"
"time"
)
// SavedSearch is a search that the scheduler runs on a regular basis.
type SavedSearch struct {
ID string `json:"id"`
Name string `json:"name"`
Params string `json:"params"` // same query string as /api/search (without page)
Schedule Schedule `json:"schedule"`
MaxResults int `json:"maxResults"` // 30, 50 or 100 (one GitHub page)
Enabled bool `json:"enabled"`
CreatedAt time.Time `json:"createdAt"`
LastRunAt time.Time `json:"lastRunAt,omitzero"`
NextRunAt time.Time `json:"nextRunAt,omitzero"`
LastError string `json:"lastError,omitempty"`
LastRun *RunInfo `json:"lastRun,omitempty"`
}
// RunInfo summarizes one execution of a saved search.
type RunInfo struct {
ID string `json:"id"`
At time.Time `json:"at"`
Query string `json:"query"`
Total int `json:"total"`
Count int `json:"count"` // repositories kept in the snapshot
New int `json:"new"` // never seen in a previous run
Rising int `json:"rising"` // gained stars since the previous run
Error string `json:"error,omitempty"`
TookMs int64 `json:"tookMs"`
Trigger string `json:"trigger"` // "schedule" or "manual"
}
// RunItem is one repository of a snapshot, compared with the previous run.
type RunItem struct {
Repo
New bool `json:"new"`
FirstSeen time.Time `json:"firstSeen"`
StarsDelta *int `json:"starsDelta,omitempty"` // nil when absent from the previous run
Rank int `json:"rank"`
}
// Run is a full snapshot.
type Run struct {
RunInfo
Items []RunItem `json:"items"`
}
// Store keeps saved searches and their runs as JSON files:
//
//
/searches.json
// /runs//index.json run summaries, newest first
// /runs//seen.json repository -> first time seen
// /runs//.json
type Store struct {
dir string
keep int // runs kept per search (0 = all)
mu sync.Mutex
searches []*SavedSearch
}
var idRe = regexp.MustCompile(`^[a-zA-Z0-9-]{1,40}$`)
var errNotFound = &codedError{code: "not_found", msg: "not found"}
func OpenStore(dir string, keep int) (*Store, error) {
if err := os.MkdirAll(filepath.Join(dir, "runs"), 0o755); err != nil {
return nil, err
}
s := &Store{dir: dir, keep: keep}
if err := readJSON(filepath.Join(dir, "searches.json"), &s.searches); err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, err
}
return s, nil
}
func newID() string {
b := make([]byte, 6)
_, _ = rand.Read(b)
return hex.EncodeToString(b)
}
// List returns copies of the saved searches, sorted by name.
func (s *Store) List() []SavedSearch {
s.mu.Lock()
defer s.mu.Unlock()
out := make([]SavedSearch, 0, len(s.searches))
for _, x := range s.searches {
out = append(out, *x)
}
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
return out
}
func (s *Store) Get(id string) (SavedSearch, error) {
s.mu.Lock()
defer s.mu.Unlock()
if x := s.find(id); x != nil {
return *x, nil
}
return SavedSearch{}, errNotFound
}
func (s *Store) find(id string) *SavedSearch {
for _, x := range s.searches {
if x.ID == id {
return x
}
}
return nil
}
func (s *Store) Create(x SavedSearch) (SavedSearch, error) {
s.mu.Lock()
defer s.mu.Unlock()
x.ID = newID()
x.CreatedAt = time.Now()
s.searches = append(s.searches, &x)
return x, s.saveLocked()
}
// Update changes what the user can edit and returns the new value.
func (s *Store) Update(id string, fn func(*SavedSearch)) (SavedSearch, error) {
s.mu.Lock()
defer s.mu.Unlock()
x := s.find(id)
if x == nil {
return SavedSearch{}, errNotFound
}
fn(x)
return *x, s.saveLocked()
}
func (s *Store) Delete(id string) error {
s.mu.Lock()
defer s.mu.Unlock()
for i, x := range s.searches {
if x.ID == id {
s.searches = append(s.searches[:i], s.searches[i+1:]...)
if err := s.saveLocked(); err != nil {
return err
}
return os.RemoveAll(s.runDir(id))
}
}
return errNotFound
}
func (s *Store) saveLocked() error {
return writeFileJSON(filepath.Join(s.dir, "searches.json"), s.searches)
}
func (s *Store) runDir(id string) string { return filepath.Join(s.dir, "runs", id) }
// Runs returns the run summaries of a search, newest first.
func (s *Store) Runs(id string) ([]RunInfo, error) {
if !idRe.MatchString(id) {
return nil, errNotFound
}
var idx []RunInfo
err := readJSON(filepath.Join(s.runDir(id), "index.json"), &idx)
if errors.Is(err, os.ErrNotExist) {
return []RunInfo{}, nil
}
return idx, err
}
// Run reads one snapshot ("latest" for the most recent one).
func (s *Store) Run(id, runID string) (Run, error) {
if runID == "latest" {
idx, err := s.Runs(id)
if err != nil {
return Run{}, err
}
if len(idx) == 0 {
return Run{}, errNotFound
}
runID = idx[0].ID
}
if !idRe.MatchString(id) || !idRe.MatchString(runID) {
return Run{}, errNotFound
}
var r Run
err := readJSON(filepath.Join(s.runDir(id), runID+".json"), &r)
if errors.Is(err, os.ErrNotExist) {
return Run{}, errNotFound
}
return r, err
}
// AddRun compares the repositories with the previous run, saves the snapshot
// and updates the search. A failed run (info.Error set) is only recorded in
// the history.
func (s *Store) AddRun(id string, info RunInfo, repos []Repo) (Run, error) {
dir := s.runDir(id)
if err := os.MkdirAll(dir, 0o755); err != nil {
return Run{}, err
}
idx, err := s.Runs(id)
if err != nil {
return Run{}, err
}
run := Run{RunInfo: info, Items: []RunItem{}}
if info.Error == "" {
seen := map[string]time.Time{}
if err := readJSON(filepath.Join(dir, "seen.json"), &seen); err != nil && !errors.Is(err, os.ErrNotExist) {
return Run{}, err
}
prevStars := map[string]int{}
for _, p := range idx { // previous successful run
if p.Error != "" {
continue
}
if prev, err := s.Run(id, p.ID); err == nil {
for _, it := range prev.Items {
prevStars[it.FullName] = it.Stars
}
}
break
}
baseline := len(seen) == 0 // the first successful run has nothing "new"
for i, r := range repos {
it := RunItem{Repo: r, Rank: i + 1}
if first, ok := seen[r.FullName]; ok {
it.FirstSeen = first
} else {
it.New = !baseline
it.FirstSeen = info.At
seen[r.FullName] = info.At
if it.New {
run.New++
}
}
if p, ok := prevStars[r.FullName]; ok {
d := r.Stars - p
it.StarsDelta = &d
if d > 0 {
run.Rising++
}
}
run.Items = append(run.Items, it)
}
run.Count = len(run.Items)
if err := writeFileJSON(filepath.Join(dir, "seen.json"), seen); err != nil {
return Run{}, err
}
}
if err := writeFileJSON(filepath.Join(dir, run.ID+".json"), run); err != nil {
return Run{}, err
}
idx = append([]RunInfo{run.RunInfo}, idx...)
if s.keep > 0 && len(idx) > s.keep {
for _, old := range idx[s.keep:] {
_ = os.Remove(filepath.Join(dir, old.ID+".json"))
}
idx = idx[:s.keep]
}
if err := writeFileJSON(filepath.Join(dir, "index.json"), idx); err != nil {
return Run{}, err
}
return run, nil
}
func readJSON(path string, v any) error {
data, err := os.ReadFile(path)
if err != nil {
return err
}
if err := json.Unmarshal(data, v); err != nil {
return fmt.Errorf("%s: %w", path, err)
}
return nil
}
// writeFileJSON replaces the file atomically.
func writeFileJSON(path string, v any) error {
data, err := json.Marshal(v)
if err != nil {
return err
}
tmp := path + ".tmp"
if err := os.WriteFile(tmp, data, 0o644); err != nil {
return err
}
return os.Rename(tmp, path)
}