Client GitHub avec cache, quota et tests ; README
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
claude BotandClaude Opus 5.5 committed
1 parent
1885488d05
commit
9432a34e30
3 files changed
+416
-2
No files matched your search
@@ -1,3 +1,70 @@
|
||||
# searchgit
|
||||
# Searchgit
|
||||
|
||||
Outil Go de recherche de dépôts GitHub selon des critères, avec interface web responsive (light/dark) inspirée de LogStream.
|
||||
Search GitHub repositories with criteria and browse the results in a clean web interface
|
||||
(light/dark theme, responsive, English/French UI). The interface reuses the look of
|
||||
[Logstream](https://gitea.vlab.bzh/cedric/logstream).
|
||||
|
||||
```
|
||||
browser ──/api/search──▶ searchgit (Go) ──REST──▶ api.github.com/search/repositories
|
||||
└── 5 min cache
|
||||
```
|
||||
|
||||
## Getting started
|
||||
|
||||
```bash
|
||||
cp .env.example .env # optional: set GITHUB_TOKEN
|
||||
docker compose up -d --build
|
||||
```
|
||||
|
||||
Then open <http://localhost:8080>.
|
||||
|
||||
Without Docker: `go run .` (Go 1.23 or newer), then open <http://localhost:8080>.
|
||||
|
||||
## Criteria
|
||||
|
||||
| Filter | GitHub qualifier |
|
||||
|---|---|
|
||||
| Search field | free text, plus any GitHub qualifier (`user:`, `org:`, `NOT`, `"exact phrase"`, `size:`…) |
|
||||
| Everywhere / Name / Description / README | `in:name`, `in:description`, `in:readme` |
|
||||
| Language | `language:Go` (click a language in the list to filter on it) |
|
||||
| Stars | `stars:>=100` |
|
||||
| Activity | `pushed:>=YYYY-MM-DD` (1 month to 2 years) |
|
||||
| Creation date | `created:>=YYYY-MM-DD` |
|
||||
| License | `license:mit`… |
|
||||
| Topic | `topic:cli` (click a topic in the list to filter on it) |
|
||||
| Good first issues | `good-first-issues:>0` |
|
||||
| Forks / Archived | forks and archived repositories are hidden unless enabled |
|
||||
| Sort | best match, stars, forks, recently updated, help wanted issues |
|
||||
|
||||
The filters are kept in the page URL, so a search can be bookmarked or shared. The footer
|
||||
shows the exact query sent to GitHub (click it to open the same search on github.com).
|
||||
**CSV** downloads the loaded results (semicolon separated when the UI is in French, for Excel).
|
||||
|
||||
GitHub returns at most the first 1,000 results of a search.
|
||||
|
||||
## Configuration
|
||||
|
||||
| Variable | Default | Role |
|
||||
|---|---|---|
|
||||
| `HTTP_ADDR` | `:8080` | listening address |
|
||||
| `GITHUB_TOKEN` | empty | GitHub token; raises the search quota from 10 to 30 requests per minute |
|
||||
| `CACHE_TTL` | `5m` | identical searches are served from memory for this long |
|
||||
| `AUTH_USER` / `AUTH_PASS` | empty | HTTP basic authentication (except `/healthz`) |
|
||||
| `GITHUB_API` | `https://api.github.com` | API base URL (GitHub Enterprise) |
|
||||
|
||||
## API
|
||||
|
||||
- `GET /api/search?q=&in=&language=&stars=&maxstars=&minforks=&topic=&license=&pushed=&created=&goodfirst=1&forks=1&archived=1&sort=&order=&page=&per_page=`
|
||||
(`pushed` / `created`: `1w`, `1m`, `3m`, `6m`, `1y`, `2y`, `5y`)
|
||||
- `GET /api/status`: token configured and last known search quota
|
||||
- `GET /healthz`
|
||||
|
||||
## Development
|
||||
|
||||
```bash
|
||||
go test ./...
|
||||
go run .
|
||||
```
|
||||
|
||||
The web interface is plain HTML, CSS and JavaScript in `web/`, embedded in the binary
|
||||
(no build step).
|
||||
@@ -0,0 +1,249 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Repo is what the UI shows for one repository.
|
||||
type Repo struct {
|
||||
FullName string `json:"fullName"`
|
||||
Name string `json:"name"`
|
||||
Owner string `json:"owner"`
|
||||
Avatar string `json:"avatar"`
|
||||
URL string `json:"url"`
|
||||
Homepage string `json:"homepage,omitempty"`
|
||||
Description string `json:"description"`
|
||||
Language string `json:"language"`
|
||||
Topics []string `json:"topics"`
|
||||
License string `json:"license,omitempty"`
|
||||
Stars int `json:"stars"`
|
||||
Forks int `json:"forks"`
|
||||
Issues int `json:"issues"`
|
||||
Archived bool `json:"archived"`
|
||||
Fork bool `json:"fork"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
PushedAt time.Time `json:"pushedAt"`
|
||||
}
|
||||
|
||||
// SearchResult is the /api/search response.
|
||||
type SearchResult struct {
|
||||
Query string `json:"query"` // GitHub query actually sent
|
||||
Total int `json:"total"`
|
||||
Incomplete bool `json:"incomplete"`
|
||||
Page int `json:"page"`
|
||||
PerPage int `json:"perPage"`
|
||||
Items []Repo `json:"items"`
|
||||
Rate RateLimit `json:"rate"`
|
||||
Cached bool `json:"cached"`
|
||||
TookMs int64 `json:"tookMs"`
|
||||
}
|
||||
|
||||
// RateLimit is the search quota reported by GitHub.
|
||||
type RateLimit struct {
|
||||
Limit int `json:"limit"`
|
||||
Remaining int `json:"remaining"`
|
||||
Reset time.Time `json:"reset"`
|
||||
}
|
||||
|
||||
type GitHub struct {
|
||||
base string
|
||||
token string
|
||||
client *http.Client
|
||||
ttl time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
cache map[string]cacheEntry
|
||||
rate RateLimit
|
||||
}
|
||||
|
||||
type cacheEntry struct {
|
||||
res SearchResult
|
||||
expires time.Time
|
||||
}
|
||||
|
||||
func NewGitHub(base, token string, ttl time.Duration) *GitHub {
|
||||
return &GitHub{
|
||||
base: base,
|
||||
token: token,
|
||||
client: &http.Client{Timeout: 20 * time.Second},
|
||||
ttl: ttl,
|
||||
cache: map[string]cacheEntry{},
|
||||
}
|
||||
}
|
||||
|
||||
func (g *GitHub) HasToken() bool { return g.token != "" }
|
||||
|
||||
// Rate returns the last known search quota.
|
||||
func (g *GitHub) Rate() RateLimit {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
return g.rate
|
||||
}
|
||||
|
||||
// Search runs a repository search. Identical queries are served from the
|
||||
// cache for ttl, which saves the small search quota (10/min without token,
|
||||
// 30/min with one).
|
||||
func (g *GitHub) Search(ctx context.Context, c Criteria) (SearchResult, error) {
|
||||
q := c.Query(time.Now())
|
||||
v := url.Values{}
|
||||
v.Set("q", q)
|
||||
if c.Sort != "" {
|
||||
v.Set("sort", c.Sort)
|
||||
v.Set("order", c.Order)
|
||||
}
|
||||
v.Set("per_page", strconv.Itoa(c.PerPage))
|
||||
v.Set("page", strconv.Itoa(c.Page))
|
||||
key := v.Encode()
|
||||
|
||||
g.mu.Lock()
|
||||
if e, ok := g.cache[key]; ok && time.Now().Before(e.expires) {
|
||||
g.mu.Unlock()
|
||||
res := e.res
|
||||
res.Cached = true
|
||||
res.TookMs = 0
|
||||
res.Rate = g.Rate()
|
||||
return res, nil
|
||||
}
|
||||
g.mu.Unlock()
|
||||
|
||||
start := time.Now()
|
||||
var body struct {
|
||||
TotalCount int `json:"total_count"`
|
||||
IncompleteResults bool `json:"incomplete_results"`
|
||||
Items []struct {
|
||||
FullName string `json:"full_name"`
|
||||
Name string `json:"name"`
|
||||
HTMLURL string `json:"html_url"`
|
||||
Homepage string `json:"homepage"`
|
||||
Description string `json:"description"`
|
||||
Language string `json:"language"`
|
||||
Topics []string `json:"topics"`
|
||||
Stars int `json:"stargazers_count"`
|
||||
Forks int `json:"forks_count"`
|
||||
Issues int `json:"open_issues_count"`
|
||||
Archived bool `json:"archived"`
|
||||
Fork bool `json:"fork"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
PushedAt time.Time `json:"pushed_at"`
|
||||
Owner struct {
|
||||
Login string `json:"login"`
|
||||
AvatarURL string `json:"avatar_url"`
|
||||
} `json:"owner"`
|
||||
License *struct {
|
||||
SpdxID string `json:"spdx_id"`
|
||||
} `json:"license"`
|
||||
} `json:"items"`
|
||||
}
|
||||
if err := g.get(ctx, "/search/repositories?"+key, &body); err != nil {
|
||||
return SearchResult{}, err
|
||||
}
|
||||
|
||||
res := SearchResult{
|
||||
Query: q,
|
||||
Total: body.TotalCount,
|
||||
Incomplete: body.IncompleteResults,
|
||||
Page: c.Page,
|
||||
PerPage: c.PerPage,
|
||||
Items: make([]Repo, 0, len(body.Items)),
|
||||
TookMs: time.Since(start).Milliseconds(),
|
||||
}
|
||||
for _, it := range body.Items {
|
||||
r := Repo{
|
||||
FullName: it.FullName, Name: it.Name, Owner: it.Owner.Login, Avatar: it.Owner.AvatarURL,
|
||||
URL: it.HTMLURL, Homepage: it.Homepage, Description: it.Description, Language: it.Language,
|
||||
Topics: it.Topics, Stars: it.Stars, Forks: it.Forks, Issues: it.Issues,
|
||||
Archived: it.Archived, Fork: it.Fork, CreatedAt: it.CreatedAt, PushedAt: it.PushedAt,
|
||||
}
|
||||
if r.Topics == nil {
|
||||
r.Topics = []string{}
|
||||
}
|
||||
if it.License != nil && it.License.SpdxID != "NOASSERTION" {
|
||||
r.License = it.License.SpdxID
|
||||
}
|
||||
res.Items = append(res.Items, r)
|
||||
}
|
||||
|
||||
g.mu.Lock()
|
||||
now := time.Now()
|
||||
for k, e := range g.cache { // drop expired entries
|
||||
if now.After(e.expires) {
|
||||
delete(g.cache, k)
|
||||
}
|
||||
}
|
||||
g.cache[key] = cacheEntry{res: res, expires: now.Add(g.ttl)}
|
||||
res.Rate = g.rate
|
||||
g.mu.Unlock()
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// get calls the GitHub API and decodes the JSON answer into out.
|
||||
func (g *GitHub) get(ctx context.Context, path string, out any) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, g.base+path, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Accept", "application/vnd.github+json")
|
||||
req.Header.Set("X-GitHub-Api-Version", "2022-11-28")
|
||||
req.Header.Set("User-Agent", "searchgit")
|
||||
if g.token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+g.token)
|
||||
}
|
||||
resp, err := g.client.Do(req)
|
||||
if err != nil {
|
||||
return &codedError{code: "github_unreachable", msg: "cannot reach GitHub", detail: err.Error()}
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
g.updateRate(resp.Header)
|
||||
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
var e struct {
|
||||
Message string `json:"message"`
|
||||
Errors []struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"errors"`
|
||||
}
|
||||
_ = json.Unmarshal(data, &e)
|
||||
detail := e.Message
|
||||
if len(e.Errors) > 0 && e.Errors[0].Message != "" {
|
||||
detail = e.Errors[0].Message
|
||||
}
|
||||
switch {
|
||||
case resp.StatusCode == http.StatusUnauthorized:
|
||||
return &codedError{code: "github_token", msg: "GitHub refused the token (GITHUB_TOKEN)", detail: detail}
|
||||
case resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusTooManyRequests:
|
||||
return &codedError{code: "rate_limited", msg: "GitHub search quota exhausted, try again in a minute", detail: detail}
|
||||
case resp.StatusCode == http.StatusUnprocessableEntity:
|
||||
return &codedError{code: "bad_query", msg: "GitHub rejected the query", detail: detail}
|
||||
}
|
||||
return fmt.Errorf("GitHub returned %s: %s", resp.Status, detail)
|
||||
}
|
||||
return json.Unmarshal(data, out)
|
||||
}
|
||||
|
||||
func (g *GitHub) updateRate(h http.Header) {
|
||||
// Only the search quota matters here (resource "search").
|
||||
if res := h.Get("X-RateLimit-Resource"); res != "" && res != "search" {
|
||||
return
|
||||
}
|
||||
limit, err1 := strconv.Atoi(h.Get("X-RateLimit-Limit"))
|
||||
remaining, err2 := strconv.Atoi(h.Get("X-RateLimit-Remaining"))
|
||||
reset, err3 := strconv.ParseInt(h.Get("X-RateLimit-Reset"), 10, 64)
|
||||
if err1 != nil || err2 != nil || err3 != nil {
|
||||
return
|
||||
}
|
||||
g.mu.Lock()
|
||||
g.rate = RateLimit{Limit: limit, Remaining: remaining, Reset: time.Unix(reset, 0)}
|
||||
g.mu.Unlock()
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeGitHub answers every request with a fixed status, headers and body.
|
||||
type fakeGitHub struct {
|
||||
status int
|
||||
header http.Header
|
||||
body string
|
||||
calls int
|
||||
last *http.Request
|
||||
}
|
||||
|
||||
func (f *fakeGitHub) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
f.calls++
|
||||
f.last = r
|
||||
return &http.Response{
|
||||
StatusCode: f.status,
|
||||
Status: http.StatusText(f.status),
|
||||
Header: f.header,
|
||||
Body: io.NopCloser(strings.NewReader(f.body)),
|
||||
Request: r,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func newFake(status int, body string) (*GitHub, *fakeGitHub) {
|
||||
f := &fakeGitHub{status: status, body: body, header: http.Header{
|
||||
"X-Ratelimit-Limit": {"10"},
|
||||
"X-Ratelimit-Remaining": {"9"},
|
||||
"X-Ratelimit-Reset": {"1790851447"},
|
||||
"X-Ratelimit-Resource": {"search"},
|
||||
}}
|
||||
gh := NewGitHub("https://api.github.test", "secret", time.Minute)
|
||||
gh.client = &http.Client{Transport: f}
|
||||
return gh, f
|
||||
}
|
||||
|
||||
const searchBody = `{"total_count": 14, "incomplete_results": false, "items": [{
|
||||
"full_name": "mcuadros/go-syslog", "name": "go-syslog", "html_url": "https://github.com/mcuadros/go-syslog",
|
||||
"description": "Syslog server library for go", "language": "Go", "topics": null,
|
||||
"stargazers_count": 540, "forks_count": 150, "open_issues_count": 20, "archived": false, "fork": false,
|
||||
"created_at": "2014-09-01T10:00:00Z", "pushed_at": "2024-03-01T10:00:00Z",
|
||||
"owner": {"login": "mcuadros", "avatar_url": "https://avatars.githubusercontent.com/u/1573114?v=4"},
|
||||
"license": {"spdx_id": "MIT"}}]}`
|
||||
|
||||
func TestSearchMapsAndCaches(t *testing.T) {
|
||||
gh, f := newFake(http.StatusOK, searchBody)
|
||||
c := Criteria{Text: "syslog", Language: "Go", Sort: "stars", Order: "desc", Page: 1, PerPage: 30}
|
||||
|
||||
res, err := gh.Search(context.Background(), c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Total != 14 || len(res.Items) != 1 {
|
||||
t.Fatalf("got total %d, %d items", res.Total, len(res.Items))
|
||||
}
|
||||
r := res.Items[0]
|
||||
if r.FullName != "mcuadros/go-syslog" || r.Owner != "mcuadros" || r.License != "MIT" || r.Stars != 540 || r.Topics == nil {
|
||||
t.Errorf("bad mapping: %+v", r)
|
||||
}
|
||||
if res.Rate.Limit != 10 || res.Rate.Remaining != 9 {
|
||||
t.Errorf("bad rate: %+v", res.Rate)
|
||||
}
|
||||
q := f.last.URL.Query()
|
||||
if q.Get("sort") != "stars" || q.Get("q") != "syslog language:Go archived:false" {
|
||||
t.Errorf("bad request: %s", f.last.URL)
|
||||
}
|
||||
if f.last.Header.Get("Authorization") != "Bearer secret" {
|
||||
t.Errorf("token not sent")
|
||||
}
|
||||
|
||||
res, err = gh.Search(context.Background(), c)
|
||||
if err != nil || !res.Cached || f.calls != 1 {
|
||||
t.Errorf("second identical search should come from the cache (calls=%d, cached=%v, err=%v)", f.calls, res.Cached, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchErrors(t *testing.T) {
|
||||
for status, code := range map[int]string{
|
||||
http.StatusForbidden: "rate_limited",
|
||||
http.StatusUnauthorized: "github_token",
|
||||
http.StatusUnprocessableEntity: "bad_query",
|
||||
} {
|
||||
gh, _ := newFake(status, `{"message": "nope", "errors": [{"message": "detail"}]}`)
|
||||
_, err := gh.Search(context.Background(), Criteria{Page: 1, PerPage: 30})
|
||||
ce, ok := err.(*codedError)
|
||||
if !ok || ce.code != code || ce.detail != "detail" {
|
||||
t.Errorf("status %d: got %v, want code %s", status, err, code)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user