searchgit/github.go

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strconv"
	"sync"
	"time"
)

// Repo is what the UI shows for one repository.
type Repo struct {
	FullName    string    `json:"fullName"`
	Name        string    `json:"name"`
	Owner       string    `json:"owner"`
	Avatar      string    `json:"avatar"`
	URL         string    `json:"url"`
	Homepage    string    `json:"homepage,omitempty"`
	Description string    `json:"description"`
	Language    string    `json:"language"`
	Topics      []string  `json:"topics"`
	License     string    `json:"license,omitempty"`
	Stars       int       `json:"stars"`
	Forks       int       `json:"forks"`
	Issues      int       `json:"issues"`
	Archived    bool      `json:"archived"`
	Fork        bool      `json:"fork"`
	CreatedAt   time.Time `json:"createdAt"`
	PushedAt    time.Time `json:"pushedAt"`
}

// SearchResult is the /api/search response.
type SearchResult struct {
	Query      string    `json:"query"` // GitHub query actually sent
	Total      int       `json:"total"`
	Incomplete bool      `json:"incomplete"`
	Page       int       `json:"page"`
	PerPage    int       `json:"perPage"`
	Items      []Repo    `json:"items"`
	Rate       RateLimit `json:"rate"`
	Cached     bool      `json:"cached"`
	TookMs     int64     `json:"tookMs"`
}

// RateLimit is the search quota reported by GitHub.
type RateLimit struct {
	Limit     int       `json:"limit"`
	Remaining int       `json:"remaining"`
	Reset     time.Time `json:"reset"`
}

type GitHub struct {
	base   string
	token  string
	client *http.Client
	ttl    time.Duration

	mu    sync.Mutex
	cache map[string]cacheEntry
	rate  RateLimit
}

type cacheEntry struct {
	res     SearchResult
	expires time.Time
}

func NewGitHub(base, token string, ttl time.Duration) *GitHub {
	return &GitHub{
		base:   base,
		token:  token,
		client: &http.Client{Timeout: 20 * time.Second},
		ttl:    ttl,
		cache:  map[string]cacheEntry{},
	}
}

func (g *GitHub) HasToken() bool { return g.token != "" }

// Rate returns the last known search quota.
func (g *GitHub) Rate() RateLimit {
	g.mu.Lock()
	defer g.mu.Unlock()
	return g.rate
}

// Search runs a repository search. Identical queries are served from the
// cache for ttl, which saves the small search quota (10/min without token,
// 30/min with one).
func (g *GitHub) Search(ctx context.Context, c Criteria) (SearchResult, error) {
	q := c.Query(time.Now())
	v := url.Values{}
	v.Set("q", q)
	if c.Sort != "" {
		v.Set("sort", c.Sort)
		v.Set("order", c.Order)
	}
	v.Set("per_page", strconv.Itoa(c.PerPage))
	v.Set("page", strconv.Itoa(c.Page))
	key := v.Encode()

	g.mu.Lock()
	if e, ok := g.cache[key]; ok && time.Now().Before(e.expires) {
		g.mu.Unlock()
		res := e.res
		res.Cached = true
		res.TookMs = 0
		res.Rate = g.Rate()
		return res, nil
	}
	g.mu.Unlock()

	start := time.Now()
	var body struct {
		TotalCount        int  `json:"total_count"`
		IncompleteResults bool `json:"incomplete_results"`
		Items             []struct {
			FullName    string    `json:"full_name"`
			Name        string    `json:"name"`
			HTMLURL     string    `json:"html_url"`
			Homepage    string    `json:"homepage"`
			Description string    `json:"description"`
			Language    string    `json:"language"`
			Topics      []string  `json:"topics"`
			Stars       int       `json:"stargazers_count"`
			Forks       int       `json:"forks_count"`
			Issues      int       `json:"open_issues_count"`
			Archived    bool      `json:"archived"`
			Fork        bool      `json:"fork"`
			CreatedAt   time.Time `json:"created_at"`
			PushedAt    time.Time `json:"pushed_at"`
			Owner       struct {
				Login     string `json:"login"`
				AvatarURL string `json:"avatar_url"`
			} `json:"owner"`
			License *struct {
				SpdxID string `json:"spdx_id"`
			} `json:"license"`
		} `json:"items"`
	}
	if err := g.get(ctx, "/search/repositories?"+key, &body); err != nil {
		return SearchResult{}, err
	}

	res := SearchResult{
		Query:      q,
		Total:      body.TotalCount,
		Incomplete: body.IncompleteResults,
		Page:       c.Page,
		PerPage:    c.PerPage,
		Items:      make([]Repo, 0, len(body.Items)),
		TookMs:     time.Since(start).Milliseconds(),
	}
	for _, it := range body.Items {
		r := Repo{
			FullName: it.FullName, Name: it.Name, Owner: it.Owner.Login, Avatar: it.Owner.AvatarURL,
			URL: it.HTMLURL, Homepage: it.Homepage, Description: it.Description, Language: it.Language,
			Topics: it.Topics, Stars: it.Stars, Forks: it.Forks, Issues: it.Issues,
			Archived: it.Archived, Fork: it.Fork, CreatedAt: it.CreatedAt, PushedAt: it.PushedAt,
		}
		if r.Topics == nil {
			r.Topics = []string{}
		}
		if it.License != nil && it.License.SpdxID != "NOASSERTION" {
			r.License = it.License.SpdxID
		}
		res.Items = append(res.Items, r)
	}

	g.mu.Lock()
	now := time.Now()
	for k, e := range g.cache { // drop expired entries
		if now.After(e.expires) {
			delete(g.cache, k)
		}
	}
	g.cache[key] = cacheEntry{res: res, expires: now.Add(g.ttl)}
	res.Rate = g.rate
	g.mu.Unlock()
	return res, nil
}

// get calls the GitHub API and decodes the JSON answer into out.
func (g *GitHub) get(ctx context.Context, path string, out any) error {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, g.base+path, nil)
	if err != nil {
		return err
	}
	req.Header.Set("Accept", "application/vnd.github+json")
	req.Header.Set("X-GitHub-Api-Version", "2022-11-28")
	req.Header.Set("User-Agent", "searchgit")
	if g.token != "" {
		req.Header.Set("Authorization", "Bearer "+g.token)
	}
	resp, err := g.client.Do(req)
	if err != nil {
		return &codedError{code: "github_unreachable", msg: "cannot reach GitHub", detail: err.Error()}
	}
	defer resp.Body.Close()
	g.updateRate(resp.Header)

	data, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
	if err != nil {
		return err
	}
	if resp.StatusCode != http.StatusOK {
		var e struct {
			Message string `json:"message"`
			Errors  []struct {
				Message string `json:"message"`
			} `json:"errors"`
		}
		_ = json.Unmarshal(data, &e)
		detail := e.Message
		if len(e.Errors) > 0 && e.Errors[0].Message != "" {
			detail = e.Errors[0].Message
		}
		switch {
		case resp.StatusCode == http.StatusUnauthorized:
			return &codedError{code: "github_token", msg: "GitHub refused the token (GITHUB_TOKEN)", detail: detail}
		case resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusTooManyRequests:
			return &codedError{code: "rate_limited", msg: "GitHub search quota exhausted, try again in a minute", detail: detail}
		case resp.StatusCode == http.StatusUnprocessableEntity:
			return &codedError{code: "bad_query", msg: "GitHub rejected the query", detail: detail}
		}
		return fmt.Errorf("GitHub returned %s: %s", resp.Status, detail)
	}
	return json.Unmarshal(data, out)
}

func (g *GitHub) updateRate(h http.Header) {
	// Only the search quota matters here (resource "search").
	if res := h.Get("X-RateLimit-Resource"); res != "" && res != "search" {
		return
	}
	limit, err1 := strconv.Atoi(h.Get("X-RateLimit-Limit"))
	remaining, err2 := strconv.Atoi(h.Get("X-RateLimit-Remaining"))
	reset, err3 := strconv.ParseInt(h.Get("X-RateLimit-Reset"), 10, 64)
	if err1 != nil || err2 != nil || err3 != nil {
		return
	}
	g.mu.Lock()
	g.rate = RateLimit{Limit: limit, Remaining: remaining, Reset: time.Unix(reset, 0)}
	g.mu.Unlock()
}