package main import ( "context" "encoding/json" "fmt" "io" "net/http" "net/url" "strconv" "sync" "time" ) // Repo is what the UI shows for one repository. type Repo struct { FullName string `json:"fullName"` Name string `json:"name"` Owner string `json:"owner"` Avatar string `json:"avatar"` URL string `json:"url"` Homepage string `json:"homepage,omitempty"` Description string `json:"description"` Language string `json:"language"` Topics []string `json:"topics"` License string `json:"license,omitempty"` Stars int `json:"stars"` Forks int `json:"forks"` Issues int `json:"issues"` Archived bool `json:"archived"` Fork bool `json:"fork"` CreatedAt time.Time `json:"createdAt"` PushedAt time.Time `json:"pushedAt"` } // SearchResult is the /api/search response. type SearchResult struct { Query string `json:"query"` // GitHub query actually sent Total int `json:"total"` Incomplete bool `json:"incomplete"` Page int `json:"page"` PerPage int `json:"perPage"` Items []Repo `json:"items"` Rate RateLimit `json:"rate"` Cached bool `json:"cached"` TookMs int64 `json:"tookMs"` } // RateLimit is the search quota reported by GitHub. type RateLimit struct { Limit int `json:"limit"` Remaining int `json:"remaining"` Reset time.Time `json:"reset"` } type GitHub struct { base string token string client *http.Client ttl time.Duration mu sync.Mutex cache map[string]cacheEntry rate RateLimit } type cacheEntry struct { res SearchResult expires time.Time } func NewGitHub(base, token string, ttl time.Duration) *GitHub { return &GitHub{ base: base, token: token, client: &http.Client{Timeout: 20 * time.Second}, ttl: ttl, cache: map[string]cacheEntry{}, } } func (g *GitHub) HasToken() bool { return g.token != "" } // Rate returns the last known search quota. func (g *GitHub) Rate() RateLimit { g.mu.Lock() defer g.mu.Unlock() return g.rate } // Search runs a repository search. Identical queries are served from the // cache for ttl, which saves the small search quota (10/min without token, // 30/min with one). func (g *GitHub) Search(ctx context.Context, c Criteria) (SearchResult, error) { q := c.Query(time.Now()) v := url.Values{} v.Set("q", q) if c.Sort != "" { v.Set("sort", c.Sort) v.Set("order", c.Order) } v.Set("per_page", strconv.Itoa(c.PerPage)) v.Set("page", strconv.Itoa(c.Page)) key := v.Encode() g.mu.Lock() if e, ok := g.cache[key]; ok && time.Now().Before(e.expires) { g.mu.Unlock() res := e.res res.Cached = true res.TookMs = 0 res.Rate = g.Rate() return res, nil } g.mu.Unlock() start := time.Now() var body struct { TotalCount int `json:"total_count"` IncompleteResults bool `json:"incomplete_results"` Items []struct { FullName string `json:"full_name"` Name string `json:"name"` HTMLURL string `json:"html_url"` Homepage string `json:"homepage"` Description string `json:"description"` Language string `json:"language"` Topics []string `json:"topics"` Stars int `json:"stargazers_count"` Forks int `json:"forks_count"` Issues int `json:"open_issues_count"` Archived bool `json:"archived"` Fork bool `json:"fork"` CreatedAt time.Time `json:"created_at"` PushedAt time.Time `json:"pushed_at"` Owner struct { Login string `json:"login"` AvatarURL string `json:"avatar_url"` } `json:"owner"` License *struct { SpdxID string `json:"spdx_id"` } `json:"license"` } `json:"items"` } if err := g.get(ctx, "/search/repositories?"+key, &body); err != nil { return SearchResult{}, err } res := SearchResult{ Query: q, Total: body.TotalCount, Incomplete: body.IncompleteResults, Page: c.Page, PerPage: c.PerPage, Items: make([]Repo, 0, len(body.Items)), TookMs: time.Since(start).Milliseconds(), } for _, it := range body.Items { r := Repo{ FullName: it.FullName, Name: it.Name, Owner: it.Owner.Login, Avatar: it.Owner.AvatarURL, URL: it.HTMLURL, Homepage: it.Homepage, Description: it.Description, Language: it.Language, Topics: it.Topics, Stars: it.Stars, Forks: it.Forks, Issues: it.Issues, Archived: it.Archived, Fork: it.Fork, CreatedAt: it.CreatedAt, PushedAt: it.PushedAt, } if r.Topics == nil { r.Topics = []string{} } if it.License != nil && it.License.SpdxID != "NOASSERTION" { r.License = it.License.SpdxID } res.Items = append(res.Items, r) } g.mu.Lock() now := time.Now() for k, e := range g.cache { // drop expired entries if now.After(e.expires) { delete(g.cache, k) } } g.cache[key] = cacheEntry{res: res, expires: now.Add(g.ttl)} res.Rate = g.rate g.mu.Unlock() return res, nil } // get calls the GitHub API and decodes the JSON answer into out. func (g *GitHub) get(ctx context.Context, path string, out any) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, g.base+path, nil) if err != nil { return err } req.Header.Set("Accept", "application/vnd.github+json") req.Header.Set("X-GitHub-Api-Version", "2022-11-28") req.Header.Set("User-Agent", "searchgit") if g.token != "" { req.Header.Set("Authorization", "Bearer "+g.token) } resp, err := g.client.Do(req) if err != nil { return &codedError{code: "github_unreachable", msg: "cannot reach GitHub", detail: err.Error()} } defer resp.Body.Close() g.updateRate(resp.Header) data, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) if err != nil { return err } if resp.StatusCode != http.StatusOK { var e struct { Message string `json:"message"` Errors []struct { Message string `json:"message"` } `json:"errors"` } _ = json.Unmarshal(data, &e) detail := e.Message if len(e.Errors) > 0 && e.Errors[0].Message != "" { detail = e.Errors[0].Message } switch { case resp.StatusCode == http.StatusUnauthorized: return &codedError{code: "github_token", msg: "GitHub refused the token (GITHUB_TOKEN)", detail: detail} case resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusTooManyRequests: return &codedError{code: "rate_limited", msg: "GitHub search quota exhausted, try again in a minute", detail: detail} case resp.StatusCode == http.StatusUnprocessableEntity: return &codedError{code: "bad_query", msg: "GitHub rejected the query", detail: detail} } return fmt.Errorf("GitHub returned %s: %s", resp.Status, detail) } return json.Unmarshal(data, out) } func (g *GitHub) updateRate(h http.Header) { // Only the search quota matters here (resource "search"). if res := h.Get("X-RateLimit-Resource"); res != "" && res != "search" { return } limit, err1 := strconv.Atoi(h.Get("X-RateLimit-Limit")) remaining, err2 := strconv.Atoi(h.Get("X-RateLimit-Remaining")) reset, err3 := strconv.ParseInt(h.Get("X-RateLimit-Reset"), 10, 64) if err1 != nil || err2 != nil || err3 != nil { return } g.mu.Lock() g.rate = RateLimit{Limit: limit, Remaining: remaining, Reset: time.Unix(reset, 0)} g.mu.Unlock() }