diff --git a/README.md b/README.md index d67e1af..7197839 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,70 @@ -# searchgit +# Searchgit -Outil Go de recherche de dépôts GitHub selon des critères, avec interface web responsive (light/dark) inspirée de LogStream. \ No newline at end of file +Search GitHub repositories with criteria and browse the results in a clean web interface +(light/dark theme, responsive, English/French UI). The interface reuses the look of +[Logstream](https://gitea.vlab.bzh/cedric/logstream). + +``` +browser ──/api/search──▶ searchgit (Go) ──REST──▶ api.github.com/search/repositories + └── 5 min cache +``` + +## Getting started + +```bash +cp .env.example .env # optional: set GITHUB_TOKEN +docker compose up -d --build +``` + +Then open . + +Without Docker: `go run .` (Go 1.23 or newer), then open . + +## Criteria + +| Filter | GitHub qualifier | +|---|---| +| Search field | free text, plus any GitHub qualifier (`user:`, `org:`, `NOT`, `"exact phrase"`, `size:`…) | +| Everywhere / Name / Description / README | `in:name`, `in:description`, `in:readme` | +| Language | `language:Go` (click a language in the list to filter on it) | +| Stars | `stars:>=100` | +| Activity | `pushed:>=YYYY-MM-DD` (1 month to 2 years) | +| Creation date | `created:>=YYYY-MM-DD` | +| License | `license:mit`… | +| Topic | `topic:cli` (click a topic in the list to filter on it) | +| Good first issues | `good-first-issues:>0` | +| Forks / Archived | forks and archived repositories are hidden unless enabled | +| Sort | best match, stars, forks, recently updated, help wanted issues | + +The filters are kept in the page URL, so a search can be bookmarked or shared. The footer +shows the exact query sent to GitHub (click it to open the same search on github.com). +**CSV** downloads the loaded results (semicolon separated when the UI is in French, for Excel). + +GitHub returns at most the first 1,000 results of a search. + +## Configuration + +| Variable | Default | Role | +|---|---|---| +| `HTTP_ADDR` | `:8080` | listening address | +| `GITHUB_TOKEN` | empty | GitHub token; raises the search quota from 10 to 30 requests per minute | +| `CACHE_TTL` | `5m` | identical searches are served from memory for this long | +| `AUTH_USER` / `AUTH_PASS` | empty | HTTP basic authentication (except `/healthz`) | +| `GITHUB_API` | `https://api.github.com` | API base URL (GitHub Enterprise) | + +## API + +- `GET /api/search?q=&in=&language=&stars=&maxstars=&minforks=&topic=&license=&pushed=&created=&goodfirst=1&forks=1&archived=1&sort=&order=&page=&per_page=` + (`pushed` / `created`: `1w`, `1m`, `3m`, `6m`, `1y`, `2y`, `5y`) +- `GET /api/status`: token configured and last known search quota +- `GET /healthz` + +## Development + +```bash +go test ./... +go run . +``` + +The web interface is plain HTML, CSS and JavaScript in `web/`, embedded in the binary +(no build step). diff --git a/github.go b/github.go new file mode 100644 index 0000000..5c8fc78 --- /dev/null +++ b/github.go @@ -0,0 +1,249 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "sync" + "time" +) + +// Repo is what the UI shows for one repository. +type Repo struct { + FullName string `json:"fullName"` + Name string `json:"name"` + Owner string `json:"owner"` + Avatar string `json:"avatar"` + URL string `json:"url"` + Homepage string `json:"homepage,omitempty"` + Description string `json:"description"` + Language string `json:"language"` + Topics []string `json:"topics"` + License string `json:"license,omitempty"` + Stars int `json:"stars"` + Forks int `json:"forks"` + Issues int `json:"issues"` + Archived bool `json:"archived"` + Fork bool `json:"fork"` + CreatedAt time.Time `json:"createdAt"` + PushedAt time.Time `json:"pushedAt"` +} + +// SearchResult is the /api/search response. +type SearchResult struct { + Query string `json:"query"` // GitHub query actually sent + Total int `json:"total"` + Incomplete bool `json:"incomplete"` + Page int `json:"page"` + PerPage int `json:"perPage"` + Items []Repo `json:"items"` + Rate RateLimit `json:"rate"` + Cached bool `json:"cached"` + TookMs int64 `json:"tookMs"` +} + +// RateLimit is the search quota reported by GitHub. +type RateLimit struct { + Limit int `json:"limit"` + Remaining int `json:"remaining"` + Reset time.Time `json:"reset"` +} + +type GitHub struct { + base string + token string + client *http.Client + ttl time.Duration + + mu sync.Mutex + cache map[string]cacheEntry + rate RateLimit +} + +type cacheEntry struct { + res SearchResult + expires time.Time +} + +func NewGitHub(base, token string, ttl time.Duration) *GitHub { + return &GitHub{ + base: base, + token: token, + client: &http.Client{Timeout: 20 * time.Second}, + ttl: ttl, + cache: map[string]cacheEntry{}, + } +} + +func (g *GitHub) HasToken() bool { return g.token != "" } + +// Rate returns the last known search quota. +func (g *GitHub) Rate() RateLimit { + g.mu.Lock() + defer g.mu.Unlock() + return g.rate +} + +// Search runs a repository search. Identical queries are served from the +// cache for ttl, which saves the small search quota (10/min without token, +// 30/min with one). +func (g *GitHub) Search(ctx context.Context, c Criteria) (SearchResult, error) { + q := c.Query(time.Now()) + v := url.Values{} + v.Set("q", q) + if c.Sort != "" { + v.Set("sort", c.Sort) + v.Set("order", c.Order) + } + v.Set("per_page", strconv.Itoa(c.PerPage)) + v.Set("page", strconv.Itoa(c.Page)) + key := v.Encode() + + g.mu.Lock() + if e, ok := g.cache[key]; ok && time.Now().Before(e.expires) { + g.mu.Unlock() + res := e.res + res.Cached = true + res.TookMs = 0 + res.Rate = g.Rate() + return res, nil + } + g.mu.Unlock() + + start := time.Now() + var body struct { + TotalCount int `json:"total_count"` + IncompleteResults bool `json:"incomplete_results"` + Items []struct { + FullName string `json:"full_name"` + Name string `json:"name"` + HTMLURL string `json:"html_url"` + Homepage string `json:"homepage"` + Description string `json:"description"` + Language string `json:"language"` + Topics []string `json:"topics"` + Stars int `json:"stargazers_count"` + Forks int `json:"forks_count"` + Issues int `json:"open_issues_count"` + Archived bool `json:"archived"` + Fork bool `json:"fork"` + CreatedAt time.Time `json:"created_at"` + PushedAt time.Time `json:"pushed_at"` + Owner struct { + Login string `json:"login"` + AvatarURL string `json:"avatar_url"` + } `json:"owner"` + License *struct { + SpdxID string `json:"spdx_id"` + } `json:"license"` + } `json:"items"` + } + if err := g.get(ctx, "/search/repositories?"+key, &body); err != nil { + return SearchResult{}, err + } + + res := SearchResult{ + Query: q, + Total: body.TotalCount, + Incomplete: body.IncompleteResults, + Page: c.Page, + PerPage: c.PerPage, + Items: make([]Repo, 0, len(body.Items)), + TookMs: time.Since(start).Milliseconds(), + } + for _, it := range body.Items { + r := Repo{ + FullName: it.FullName, Name: it.Name, Owner: it.Owner.Login, Avatar: it.Owner.AvatarURL, + URL: it.HTMLURL, Homepage: it.Homepage, Description: it.Description, Language: it.Language, + Topics: it.Topics, Stars: it.Stars, Forks: it.Forks, Issues: it.Issues, + Archived: it.Archived, Fork: it.Fork, CreatedAt: it.CreatedAt, PushedAt: it.PushedAt, + } + if r.Topics == nil { + r.Topics = []string{} + } + if it.License != nil && it.License.SpdxID != "NOASSERTION" { + r.License = it.License.SpdxID + } + res.Items = append(res.Items, r) + } + + g.mu.Lock() + now := time.Now() + for k, e := range g.cache { // drop expired entries + if now.After(e.expires) { + delete(g.cache, k) + } + } + g.cache[key] = cacheEntry{res: res, expires: now.Add(g.ttl)} + res.Rate = g.rate + g.mu.Unlock() + return res, nil +} + +// get calls the GitHub API and decodes the JSON answer into out. +func (g *GitHub) get(ctx context.Context, path string, out any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, g.base+path, nil) + if err != nil { + return err + } + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("X-GitHub-Api-Version", "2022-11-28") + req.Header.Set("User-Agent", "searchgit") + if g.token != "" { + req.Header.Set("Authorization", "Bearer "+g.token) + } + resp, err := g.client.Do(req) + if err != nil { + return &codedError{code: "github_unreachable", msg: "cannot reach GitHub", detail: err.Error()} + } + defer resp.Body.Close() + g.updateRate(resp.Header) + + data, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + var e struct { + Message string `json:"message"` + Errors []struct { + Message string `json:"message"` + } `json:"errors"` + } + _ = json.Unmarshal(data, &e) + detail := e.Message + if len(e.Errors) > 0 && e.Errors[0].Message != "" { + detail = e.Errors[0].Message + } + switch { + case resp.StatusCode == http.StatusUnauthorized: + return &codedError{code: "github_token", msg: "GitHub refused the token (GITHUB_TOKEN)", detail: detail} + case resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusTooManyRequests: + return &codedError{code: "rate_limited", msg: "GitHub search quota exhausted, try again in a minute", detail: detail} + case resp.StatusCode == http.StatusUnprocessableEntity: + return &codedError{code: "bad_query", msg: "GitHub rejected the query", detail: detail} + } + return fmt.Errorf("GitHub returned %s: %s", resp.Status, detail) + } + return json.Unmarshal(data, out) +} + +func (g *GitHub) updateRate(h http.Header) { + // Only the search quota matters here (resource "search"). + if res := h.Get("X-RateLimit-Resource"); res != "" && res != "search" { + return + } + limit, err1 := strconv.Atoi(h.Get("X-RateLimit-Limit")) + remaining, err2 := strconv.Atoi(h.Get("X-RateLimit-Remaining")) + reset, err3 := strconv.ParseInt(h.Get("X-RateLimit-Reset"), 10, 64) + if err1 != nil || err2 != nil || err3 != nil { + return + } + g.mu.Lock() + g.rate = RateLimit{Limit: limit, Remaining: remaining, Reset: time.Unix(reset, 0)} + g.mu.Unlock() +} diff --git a/github_test.go b/github_test.go new file mode 100644 index 0000000..37b8a7a --- /dev/null +++ b/github_test.go @@ -0,0 +1,98 @@ +package main + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + "time" +) + +// fakeGitHub answers every request with a fixed status, headers and body. +type fakeGitHub struct { + status int + header http.Header + body string + calls int + last *http.Request +} + +func (f *fakeGitHub) RoundTrip(r *http.Request) (*http.Response, error) { + f.calls++ + f.last = r + return &http.Response{ + StatusCode: f.status, + Status: http.StatusText(f.status), + Header: f.header, + Body: io.NopCloser(strings.NewReader(f.body)), + Request: r, + }, nil +} + +func newFake(status int, body string) (*GitHub, *fakeGitHub) { + f := &fakeGitHub{status: status, body: body, header: http.Header{ + "X-Ratelimit-Limit": {"10"}, + "X-Ratelimit-Remaining": {"9"}, + "X-Ratelimit-Reset": {"1790851447"}, + "X-Ratelimit-Resource": {"search"}, + }} + gh := NewGitHub("https://api.github.test", "secret", time.Minute) + gh.client = &http.Client{Transport: f} + return gh, f +} + +const searchBody = `{"total_count": 14, "incomplete_results": false, "items": [{ + "full_name": "mcuadros/go-syslog", "name": "go-syslog", "html_url": "https://github.com/mcuadros/go-syslog", + "description": "Syslog server library for go", "language": "Go", "topics": null, + "stargazers_count": 540, "forks_count": 150, "open_issues_count": 20, "archived": false, "fork": false, + "created_at": "2014-09-01T10:00:00Z", "pushed_at": "2024-03-01T10:00:00Z", + "owner": {"login": "mcuadros", "avatar_url": "https://avatars.githubusercontent.com/u/1573114?v=4"}, + "license": {"spdx_id": "MIT"}}]}` + +func TestSearchMapsAndCaches(t *testing.T) { + gh, f := newFake(http.StatusOK, searchBody) + c := Criteria{Text: "syslog", Language: "Go", Sort: "stars", Order: "desc", Page: 1, PerPage: 30} + + res, err := gh.Search(context.Background(), c) + if err != nil { + t.Fatal(err) + } + if res.Total != 14 || len(res.Items) != 1 { + t.Fatalf("got total %d, %d items", res.Total, len(res.Items)) + } + r := res.Items[0] + if r.FullName != "mcuadros/go-syslog" || r.Owner != "mcuadros" || r.License != "MIT" || r.Stars != 540 || r.Topics == nil { + t.Errorf("bad mapping: %+v", r) + } + if res.Rate.Limit != 10 || res.Rate.Remaining != 9 { + t.Errorf("bad rate: %+v", res.Rate) + } + q := f.last.URL.Query() + if q.Get("sort") != "stars" || q.Get("q") != "syslog language:Go archived:false" { + t.Errorf("bad request: %s", f.last.URL) + } + if f.last.Header.Get("Authorization") != "Bearer secret" { + t.Errorf("token not sent") + } + + res, err = gh.Search(context.Background(), c) + if err != nil || !res.Cached || f.calls != 1 { + t.Errorf("second identical search should come from the cache (calls=%d, cached=%v, err=%v)", f.calls, res.Cached, err) + } +} + +func TestSearchErrors(t *testing.T) { + for status, code := range map[int]string{ + http.StatusForbidden: "rate_limited", + http.StatusUnauthorized: "github_token", + http.StatusUnprocessableEntity: "bad_query", + } { + gh, _ := newFake(status, `{"message": "nope", "errors": [{"message": "detail"}]}`) + _, err := gh.Search(context.Background(), Criteria{Page: 1, PerPage: 30}) + ce, ok := err.(*codedError) + if !ok || ce.code != code || ce.detail != "detail" { + t.Errorf("status %d: got %v, want code %s", status, err, code) + } + } +}