README.md

@@ -1,3 +1,70 @@
-# searchgit
+# Searchgit
 
-Outil Go de recherche de dépôts GitHub selon des critères, avec interface web responsive (light/dark) inspirée de LogStream.
+Search GitHub repositories with criteria and browse the results in a clean web interface
+(light/dark theme, responsive, English/French UI). The interface reuses the look of
+[Logstream](https://gitea.vlab.bzh/cedric/logstream).
+
+```
+browser ──/api/search──▶ searchgit (Go) ──REST──▶ api.github.com/search/repositories
+                               └── 5 min cache
+```
+
+## Getting started
+
+```bash
+cp .env.example .env      # optional: set GITHUB_TOKEN
+docker compose up -d --build
+```
+
+Then open <http://localhost:8080>.
+
+Without Docker: `go run .` (Go 1.23 or newer), then open <http://localhost:8080>.
+
+## Criteria
+
+| Filter | GitHub qualifier |
+|---|---|
+| Search field | free text, plus any GitHub qualifier (`user:`, `org:`, `NOT`, `"exact phrase"`, `size:`…) |
+| Everywhere / Name / Description / README | `in:name`, `in:description`, `in:readme` |
+| Language | `language:Go` (click a language in the list to filter on it) |
+| Stars | `stars:>=100` |
+| Activity | `pushed:>=YYYY-MM-DD` (1 month to 2 years) |
+| Creation date | `created:>=YYYY-MM-DD` |
+| License | `license:mit`… |
+| Topic | `topic:cli` (click a topic in the list to filter on it) |
+| Good first issues | `good-first-issues:>0` |
+| Forks / Archived | forks and archived repositories are hidden unless enabled |
+| Sort | best match, stars, forks, recently updated, help wanted issues |
+
+The filters are kept in the page URL, so a search can be bookmarked or shared. The footer
+shows the exact query sent to GitHub (click it to open the same search on github.com).
+**CSV** downloads the loaded results (semicolon separated when the UI is in French, for Excel).
+
+GitHub returns at most the first 1,000 results of a search.
+
+## Configuration
+
+| Variable | Default | Role |
+|---|---|---|
+| `HTTP_ADDR` | `:8080` | listening address |
+| `GITHUB_TOKEN` | empty | GitHub token; raises the search quota from 10 to 30 requests per minute |
+| `CACHE_TTL` | `5m` | identical searches are served from memory for this long |
+| `AUTH_USER` / `AUTH_PASS` | empty | HTTP basic authentication (except `/healthz`) |
+| `GITHUB_API` | `https://api.github.com` | API base URL (GitHub Enterprise) |
+
+## API
+
+- `GET /api/search?q=&in=&language=&stars=&maxstars=&minforks=&topic=&license=&pushed=&created=&goodfirst=1&forks=1&archived=1&sort=&order=&page=&per_page=`
+  (`pushed` / `created`: `1w`, `1m`, `3m`, `6m`, `1y`, `2y`, `5y`)
+- `GET /api/status`: token configured and last known search quota
+- `GET /healthz`
+
+## Development
+
+```bash
+go test ./...
+go run .
+```
+
+The web interface is plain HTML, CSS and JavaScript in `web/`, embedded in the binary
+(no build step).

github.go

@@ -0,0 +1,249 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"sync"
+	"time"
+)
+
+// Repo is what the UI shows for one repository.
+type Repo struct {
+	FullName    string    `json:"fullName"`
+	Name        string    `json:"name"`
+	Owner       string    `json:"owner"`
+	Avatar      string    `json:"avatar"`
+	URL         string    `json:"url"`
+	Homepage    string    `json:"homepage,omitempty"`
+	Description string    `json:"description"`
+	Language    string    `json:"language"`
+	Topics      []string  `json:"topics"`
+	License     string    `json:"license,omitempty"`
+	Stars       int       `json:"stars"`
+	Forks       int       `json:"forks"`
+	Issues      int       `json:"issues"`
+	Archived    bool      `json:"archived"`
+	Fork        bool      `json:"fork"`
+	CreatedAt   time.Time `json:"createdAt"`
+	PushedAt    time.Time `json:"pushedAt"`
+}
+
+// SearchResult is the /api/search response.
+type SearchResult struct {
+	Query      string    `json:"query"` // GitHub query actually sent
+	Total      int       `json:"total"`
+	Incomplete bool      `json:"incomplete"`
+	Page       int       `json:"page"`
+	PerPage    int       `json:"perPage"`
+	Items      []Repo    `json:"items"`
+	Rate       RateLimit `json:"rate"`
+	Cached     bool      `json:"cached"`
+	TookMs     int64     `json:"tookMs"`
+}
+
+// RateLimit is the search quota reported by GitHub.
+type RateLimit struct {
+	Limit     int       `json:"limit"`
+	Remaining int       `json:"remaining"`
+	Reset     time.Time `json:"reset"`
+}
+
+type GitHub struct {
+	base   string
+	token  string
+	client *http.Client
+	ttl    time.Duration
+
+	mu    sync.Mutex
+	cache map[string]cacheEntry
+	rate  RateLimit
+}
+
+type cacheEntry struct {
+	res     SearchResult
+	expires time.Time
+}
+
+func NewGitHub(base, token string, ttl time.Duration) *GitHub {
+	return &GitHub{
+		base:   base,
+		token:  token,
+		client: &http.Client{Timeout: 20 * time.Second},
+		ttl:    ttl,
+		cache:  map[string]cacheEntry{},
+	}
+}
+
+func (g *GitHub) HasToken() bool { return g.token != "" }
+
+// Rate returns the last known search quota.
+func (g *GitHub) Rate() RateLimit {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	return g.rate
+}
+
+// Search runs a repository search. Identical queries are served from the
+// cache for ttl, which saves the small search quota (10/min without token,
+// 30/min with one).
+func (g *GitHub) Search(ctx context.Context, c Criteria) (SearchResult, error) {
+	q := c.Query(time.Now())
+	v := url.Values{}
+	v.Set("q", q)
+	if c.Sort != "" {
+		v.Set("sort", c.Sort)
+		v.Set("order", c.Order)
+	}
+	v.Set("per_page", strconv.Itoa(c.PerPage))
+	v.Set("page", strconv.Itoa(c.Page))
+	key := v.Encode()
+
+	g.mu.Lock()
+	if e, ok := g.cache[key]; ok && time.Now().Before(e.expires) {
+		g.mu.Unlock()
+		res := e.res
+		res.Cached = true
+		res.TookMs = 0
+		res.Rate = g.Rate()
+		return res, nil
+	}
+	g.mu.Unlock()
+
+	start := time.Now()
+	var body struct {
+		TotalCount        int  `json:"total_count"`
+		IncompleteResults bool `json:"incomplete_results"`
+		Items             []struct {
+			FullName    string    `json:"full_name"`
+			Name        string    `json:"name"`
+			HTMLURL     string    `json:"html_url"`
+			Homepage    string    `json:"homepage"`
+			Description string    `json:"description"`
+			Language    string    `json:"language"`
+			Topics      []string  `json:"topics"`
+			Stars       int       `json:"stargazers_count"`
+			Forks       int       `json:"forks_count"`
+			Issues      int       `json:"open_issues_count"`
+			Archived    bool      `json:"archived"`
+			Fork        bool      `json:"fork"`
+			CreatedAt   time.Time `json:"created_at"`
+			PushedAt    time.Time `json:"pushed_at"`
+			Owner       struct {
+				Login     string `json:"login"`
+				AvatarURL string `json:"avatar_url"`
+			} `json:"owner"`
+			License *struct {
+				SpdxID string `json:"spdx_id"`
+			} `json:"license"`
+		} `json:"items"`
+	}
+	if err := g.get(ctx, "/search/repositories?"+key, &body); err != nil {
+		return SearchResult{}, err
+	}
+
+	res := SearchResult{
+		Query:      q,
+		Total:      body.TotalCount,
+		Incomplete: body.IncompleteResults,
+		Page:       c.Page,
+		PerPage:    c.PerPage,
+		Items:      make([]Repo, 0, len(body.Items)),
+		TookMs:     time.Since(start).Milliseconds(),
+	}
+	for _, it := range body.Items {
+		r := Repo{
+			FullName: it.FullName, Name: it.Name, Owner: it.Owner.Login, Avatar: it.Owner.AvatarURL,
+			URL: it.HTMLURL, Homepage: it.Homepage, Description: it.Description, Language: it.Language,
+			Topics: it.Topics, Stars: it.Stars, Forks: it.Forks, Issues: it.Issues,
+			Archived: it.Archived, Fork: it.Fork, CreatedAt: it.CreatedAt, PushedAt: it.PushedAt,
+		}
+		if r.Topics == nil {
+			r.Topics = []string{}
+		}
+		if it.License != nil && it.License.SpdxID != "NOASSERTION" {
+			r.License = it.License.SpdxID
+		}
+		res.Items = append(res.Items, r)
+	}
+
+	g.mu.Lock()
+	now := time.Now()
+	for k, e := range g.cache { // drop expired entries
+		if now.After(e.expires) {
+			delete(g.cache, k)
+		}
+	}
+	g.cache[key] = cacheEntry{res: res, expires: now.Add(g.ttl)}
+	res.Rate = g.rate
+	g.mu.Unlock()
+	return res, nil
+}
+
+// get calls the GitHub API and decodes the JSON answer into out.
+func (g *GitHub) get(ctx context.Context, path string, out any) error {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, g.base+path, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Accept", "application/vnd.github+json")
+	req.Header.Set("X-GitHub-Api-Version", "2022-11-28")
+	req.Header.Set("User-Agent", "searchgit")
+	if g.token != "" {
+		req.Header.Set("Authorization", "Bearer "+g.token)
+	}
+	resp, err := g.client.Do(req)
+	if err != nil {
+		return &codedError{code: "github_unreachable", msg: "cannot reach GitHub", detail: err.Error()}
+	}
+	defer resp.Body.Close()
+	g.updateRate(resp.Header)
+
+	data, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode != http.StatusOK {
+		var e struct {
+			Message string `json:"message"`
+			Errors  []struct {
+				Message string `json:"message"`
+			} `json:"errors"`
+		}
+		_ = json.Unmarshal(data, &e)
+		detail := e.Message
+		if len(e.Errors) > 0 && e.Errors[0].Message != "" {
+			detail = e.Errors[0].Message
+		}
+		switch {
+		case resp.StatusCode == http.StatusUnauthorized:
+			return &codedError{code: "github_token", msg: "GitHub refused the token (GITHUB_TOKEN)", detail: detail}
+		case resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusTooManyRequests:
+			return &codedError{code: "rate_limited", msg: "GitHub search quota exhausted, try again in a minute", detail: detail}
+		case resp.StatusCode == http.StatusUnprocessableEntity:
+			return &codedError{code: "bad_query", msg: "GitHub rejected the query", detail: detail}
+		}
+		return fmt.Errorf("GitHub returned %s: %s", resp.Status, detail)
+	}
+	return json.Unmarshal(data, out)
+}
+
+func (g *GitHub) updateRate(h http.Header) {
+	// Only the search quota matters here (resource "search").
+	if res := h.Get("X-RateLimit-Resource"); res != "" && res != "search" {
+		return
+	}
+	limit, err1 := strconv.Atoi(h.Get("X-RateLimit-Limit"))
+	remaining, err2 := strconv.Atoi(h.Get("X-RateLimit-Remaining"))
+	reset, err3 := strconv.ParseInt(h.Get("X-RateLimit-Reset"), 10, 64)
+	if err1 != nil || err2 != nil || err3 != nil {
+		return
+	}
+	g.mu.Lock()
+	g.rate = RateLimit{Limit: limit, Remaining: remaining, Reset: time.Unix(reset, 0)}
+	g.mu.Unlock()
+}

github_test.go

@@ -0,0 +1,98 @@
+package main
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+)
+
+// fakeGitHub answers every request with a fixed status, headers and body.
+type fakeGitHub struct {
+	status int
+	header http.Header
+	body   string
+	calls  int
+	last   *http.Request
+}
+
+func (f *fakeGitHub) RoundTrip(r *http.Request) (*http.Response, error) {
+	f.calls++
+	f.last = r
+	return &http.Response{
+		StatusCode: f.status,
+		Status:     http.StatusText(f.status),
+		Header:     f.header,
+		Body:       io.NopCloser(strings.NewReader(f.body)),
+		Request:    r,
+	}, nil
+}
+
+func newFake(status int, body string) (*GitHub, *fakeGitHub) {
+	f := &fakeGitHub{status: status, body: body, header: http.Header{
+		"X-Ratelimit-Limit":     {"10"},
+		"X-Ratelimit-Remaining": {"9"},
+		"X-Ratelimit-Reset":     {"1790851447"},
+		"X-Ratelimit-Resource":  {"search"},
+	}}
+	gh := NewGitHub("https://api.github.test", "secret", time.Minute)
+	gh.client = &http.Client{Transport: f}
+	return gh, f
+}
+
+const searchBody = `{"total_count": 14, "incomplete_results": false, "items": [{
+  "full_name": "mcuadros/go-syslog", "name": "go-syslog", "html_url": "https://github.com/mcuadros/go-syslog",
+  "description": "Syslog server library for go", "language": "Go", "topics": null,
+  "stargazers_count": 540, "forks_count": 150, "open_issues_count": 20, "archived": false, "fork": false,
+  "created_at": "2014-09-01T10:00:00Z", "pushed_at": "2024-03-01T10:00:00Z",
+  "owner": {"login": "mcuadros", "avatar_url": "https://avatars.githubusercontent.com/u/1573114?v=4"},
+  "license": {"spdx_id": "MIT"}}]}`
+
+func TestSearchMapsAndCaches(t *testing.T) {
+	gh, f := newFake(http.StatusOK, searchBody)
+	c := Criteria{Text: "syslog", Language: "Go", Sort: "stars", Order: "desc", Page: 1, PerPage: 30}
+
+	res, err := gh.Search(context.Background(), c)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if res.Total != 14 || len(res.Items) != 1 {
+		t.Fatalf("got total %d, %d items", res.Total, len(res.Items))
+	}
+	r := res.Items[0]
+	if r.FullName != "mcuadros/go-syslog" || r.Owner != "mcuadros" || r.License != "MIT" || r.Stars != 540 || r.Topics == nil {
+		t.Errorf("bad mapping: %+v", r)
+	}
+	if res.Rate.Limit != 10 || res.Rate.Remaining != 9 {
+		t.Errorf("bad rate: %+v", res.Rate)
+	}
+	q := f.last.URL.Query()
+	if q.Get("sort") != "stars" || q.Get("q") != "syslog language:Go archived:false" {
+		t.Errorf("bad request: %s", f.last.URL)
+	}
+	if f.last.Header.Get("Authorization") != "Bearer secret" {
+		t.Errorf("token not sent")
+	}
+
+	res, err = gh.Search(context.Background(), c)
+	if err != nil || !res.Cached || f.calls != 1 {
+		t.Errorf("second identical search should come from the cache (calls=%d, cached=%v, err=%v)", f.calls, res.Cached, err)
+	}
+}
+
+func TestSearchErrors(t *testing.T) {
+	for status, code := range map[int]string{
+		http.StatusForbidden:           "rate_limited",
+		http.StatusUnauthorized:        "github_token",
+		http.StatusUnprocessableEntity: "bad_query",
+	} {
+		gh, _ := newFake(status, `{"message": "nope", "errors": [{"message": "detail"}]}`)
+		_, err := gh.Search(context.Background(), Criteria{Page: 1, PerPage: 30})
+		ce, ok := err.(*codedError)
+		if !ok || ce.code != code || ce.detail != "detail" {
+			t.Errorf("status %d: got %v, want code %s", status, err, code)
+		}
+	}
+}