searchgit/store.go

package main

import (
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"sync"
	"time"
)

// SavedSearch is a search that the scheduler runs on a regular basis.
type SavedSearch struct {
	ID         string    `json:"id"`
	Name       string    `json:"name"`
	Params     string    `json:"params"` // same query string as /api/search (without page)
	Schedule   Schedule  `json:"schedule"`
	MaxResults int       `json:"maxResults"` // 30, 50 or 100 (one GitHub page)
	Enabled    bool      `json:"enabled"`
	CreatedAt  time.Time `json:"createdAt"`
	LastRunAt  time.Time `json:"lastRunAt,omitzero"`
	NextRunAt  time.Time `json:"nextRunAt,omitzero"`
	LastError  string    `json:"lastError,omitempty"`
	LastRun    *RunInfo  `json:"lastRun,omitempty"`
}

// RunInfo summarizes one execution of a saved search.
type RunInfo struct {
	ID      string    `json:"id"`
	At      time.Time `json:"at"`
	Query   string    `json:"query"`
	Total   int       `json:"total"`
	Count   int       `json:"count"`  // repositories kept in the snapshot
	New     int       `json:"new"`    // never seen in a previous run
	Rising  int       `json:"rising"` // gained stars since the previous run
	Error   string    `json:"error,omitempty"`
	TookMs  int64     `json:"tookMs"`
	Trigger string    `json:"trigger"` // "schedule" or "manual"
}

// RunItem is one repository of a snapshot, compared with the previous run.
type RunItem struct {
	Repo
	New        bool      `json:"new"`
	FirstSeen  time.Time `json:"firstSeen"`
	StarsDelta *int      `json:"starsDelta,omitempty"` // nil when absent from the previous run
	Rank       int       `json:"rank"`
}

// Run is a full snapshot.
type Run struct {
	RunInfo
	Items []RunItem `json:"items"`
}

// Store keeps saved searches and their runs as JSON files:
//
//	<dir>/searches.json
//	<dir>/runs/<search id>/index.json   run summaries, newest first
//	<dir>/runs/<search id>/seen.json    repository -> first time seen
//	<dir>/runs/<search id>/<run id>.json
type Store struct {
	dir  string
	keep int // runs kept per search (0 = all)

	mu       sync.Mutex
	searches []*SavedSearch
}

var idRe = regexp.MustCompile(`^[a-zA-Z0-9-]{1,40}$`)

var errNotFound = &codedError{code: "not_found", msg: "not found"}

func OpenStore(dir string, keep int) (*Store, error) {
	if err := os.MkdirAll(filepath.Join(dir, "runs"), 0o755); err != nil {
		return nil, err
	}
	s := &Store{dir: dir, keep: keep}
	if err := readJSON(filepath.Join(dir, "searches.json"), &s.searches); err != nil && !errors.Is(err, os.ErrNotExist) {
		return nil, err
	}
	return s, nil
}

func newID() string {
	b := make([]byte, 6)
	_, _ = rand.Read(b)
	return hex.EncodeToString(b)
}

// List returns copies of the saved searches, sorted by name.
func (s *Store) List() []SavedSearch {
	s.mu.Lock()
	defer s.mu.Unlock()
	out := make([]SavedSearch, 0, len(s.searches))
	for _, x := range s.searches {
		out = append(out, *x)
	}
	sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
	return out
}

func (s *Store) Get(id string) (SavedSearch, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	if x := s.find(id); x != nil {
		return *x, nil
	}
	return SavedSearch{}, errNotFound
}

func (s *Store) find(id string) *SavedSearch {
	for _, x := range s.searches {
		if x.ID == id {
			return x
		}
	}
	return nil
}

func (s *Store) Create(x SavedSearch) (SavedSearch, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	x.ID = newID()
	x.CreatedAt = time.Now()
	s.searches = append(s.searches, &x)
	return x, s.saveLocked()
}

// Update changes what the user can edit and returns the new value.
func (s *Store) Update(id string, fn func(*SavedSearch)) (SavedSearch, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	x := s.find(id)
	if x == nil {
		return SavedSearch{}, errNotFound
	}
	fn(x)
	return *x, s.saveLocked()
}

func (s *Store) Delete(id string) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	for i, x := range s.searches {
		if x.ID == id {
			s.searches = append(s.searches[:i], s.searches[i+1:]...)
			if err := s.saveLocked(); err != nil {
				return err
			}
			return os.RemoveAll(s.runDir(id))
		}
	}
	return errNotFound
}

func (s *Store) saveLocked() error {
	return writeFileJSON(filepath.Join(s.dir, "searches.json"), s.searches)
}

func (s *Store) runDir(id string) string { return filepath.Join(s.dir, "runs", id) }

// Runs returns the run summaries of a search, newest first.
func (s *Store) Runs(id string) ([]RunInfo, error) {
	if !idRe.MatchString(id) {
		return nil, errNotFound
	}
	var idx []RunInfo
	err := readJSON(filepath.Join(s.runDir(id), "index.json"), &idx)
	if errors.Is(err, os.ErrNotExist) {
		return []RunInfo{}, nil
	}
	return idx, err
}

// Run reads one snapshot ("latest" for the most recent one).
func (s *Store) Run(id, runID string) (Run, error) {
	if runID == "latest" {
		idx, err := s.Runs(id)
		if err != nil {
			return Run{}, err
		}
		if len(idx) == 0 {
			return Run{}, errNotFound
		}
		runID = idx[0].ID
	}
	if !idRe.MatchString(id) || !idRe.MatchString(runID) {
		return Run{}, errNotFound
	}
	var r Run
	err := readJSON(filepath.Join(s.runDir(id), runID+".json"), &r)
	if errors.Is(err, os.ErrNotExist) {
		return Run{}, errNotFound
	}
	return r, err
}

// AddRun compares the repositories with the previous run, saves the snapshot
// and updates the search. A failed run (info.Error set) is only recorded in
// the history.
func (s *Store) AddRun(id string, info RunInfo, repos []Repo) (Run, error) {
	dir := s.runDir(id)
	if err := os.MkdirAll(dir, 0o755); err != nil {
		return Run{}, err
	}
	idx, err := s.Runs(id)
	if err != nil {
		return Run{}, err
	}
	run := Run{RunInfo: info, Items: []RunItem{}}

	if info.Error == "" {
		seen := map[string]time.Time{}
		if err := readJSON(filepath.Join(dir, "seen.json"), &seen); err != nil && !errors.Is(err, os.ErrNotExist) {
			return Run{}, err
		}
		prevStars := map[string]int{}
		for _, p := range idx { // previous successful run
			if p.Error != "" {
				continue
			}
			if prev, err := s.Run(id, p.ID); err == nil {
				for _, it := range prev.Items {
					prevStars[it.FullName] = it.Stars
				}
			}
			break
		}
		baseline := len(seen) == 0 // the first successful run has nothing "new"
		for i, r := range repos {
			it := RunItem{Repo: r, Rank: i + 1}
			if first, ok := seen[r.FullName]; ok {
				it.FirstSeen = first
			} else {
				it.New = !baseline
				it.FirstSeen = info.At
				seen[r.FullName] = info.At
				if it.New {
					run.New++
				}
			}
			if p, ok := prevStars[r.FullName]; ok {
				d := r.Stars - p
				it.StarsDelta = &d
				if d > 0 {
					run.Rising++
				}
			}
			run.Items = append(run.Items, it)
		}
		run.Count = len(run.Items)
		if err := writeFileJSON(filepath.Join(dir, "seen.json"), seen); err != nil {
			return Run{}, err
		}
	}

	if err := writeFileJSON(filepath.Join(dir, run.ID+".json"), run); err != nil {
		return Run{}, err
	}
	idx = append([]RunInfo{run.RunInfo}, idx...)
	if s.keep > 0 && len(idx) > s.keep {
		for _, old := range idx[s.keep:] {
			_ = os.Remove(filepath.Join(dir, old.ID+".json"))
		}
		idx = idx[:s.keep]
	}
	if err := writeFileJSON(filepath.Join(dir, "index.json"), idx); err != nil {
		return Run{}, err
	}
	return run, nil
}

func readJSON(path string, v any) error {
	data, err := os.ReadFile(path)
	if err != nil {
		return err
	}
	if err := json.Unmarshal(data, v); err != nil {
		return fmt.Errorf("%s: %w", path, err)
	}
	return nil
}

// writeFileJSON replaces the file atomically.
func writeFileJSON(path string, v any) error {
	data, err := json.Marshal(v)
	if err != nil {
		return err
	}
	tmp := path + ".tmp"
	if err := os.WriteFile(tmp, data, 0o644); err != nil {
		return err
	}
	return os.Rename(tmp, path)
}