Files
upandown/internal/storage/catalog.go
T
claude BotandClaude Opus 5.5 40c0c4d758 Lot 1 (suite) : paquet catalog et liaison Store <-> PostgreSQL
Fichiers nouveaux du commit précédent : internal/catalog/catalog.go,
internal/storage/catalog.go et ses tests.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LNfwm4pfYR9C914C2EeiuB
2026-11-20 17:21:46 +01:00

337 lines
8.9 KiB
Go

package storage
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"log/slog"
"os"
"path/filepath"
"slices"
"time"
"github.com/jackc/pgx/v5"
"github.com/vblogio/upandown/internal/catalog"
)
// Catalogue PostgreSQL : quand il est configuré, les métadonnées (tags,
// icônes, auteurs) et les liens de partage d'un Store sont lus et écrits en
// base au lieu de .meta.json et .shares.json, ce qui permet à plusieurs
// instances de servir la même arborescence. Chaque modification se fait dans
// une transaction protégée par un verrou consultatif par racine : deux
// instances qui modifient la même racine passent l'une après l'autre.
// opTimeout borne une opération sur le catalogue.
const opTimeout = 30 * time.Second
type catalogRef struct {
db *catalog.DB
root string // "main", "home/<id>", "s3"
}
// UseCatalog branche le Store sur le catalogue sous la clé root. Au premier
// branchement d'une racine, ses anciens fichiers .meta.json et .shares.json
// sont repris en base puis renommés en *.imported.
func (s *Store) UseCatalog(db *catalog.DB, root string) error {
c := &catalogRef{db: db, root: root}
if err := c.importJSON(s.root); err != nil {
return fmt.Errorf("reprise des métadonnées de %s : %w", root, err)
}
s.metaMu.Lock()
s.cat, s.meta = c, nil
s.metaMu.Unlock()
return nil
}
func ctxOp() (context.Context, context.CancelFunc) {
return context.WithTimeout(context.Background(), opTimeout)
}
// --- Métadonnées -------------------------------------------------------------
func loadMetaTx(ctx context.Context, q pgx.Tx, root string) (map[string]itemMeta, error) {
rows, err := q.Query(ctx, `SELECT path, tags, favorite, icon, created, created_by, edited_by
FROM item_meta WHERE root = $1`, root)
if err != nil {
return nil, err
}
out := map[string]itemMeta{}
for rows.Next() {
var p string
var m itemMeta
var created *time.Time
if err := rows.Scan(&p, &m.Tags, &m.Favorite, &m.Icon, &created, &m.CreatedBy, &m.EditedBy); err != nil {
rows.Close()
return nil, err
}
if len(m.Tags) == 0 {
m.Tags = nil
}
if created != nil {
t := created.UTC()
m.Created = &t
}
out[p] = m
}
return out, rows.Err()
}
func (c *catalogRef) readMeta() (map[string]itemMeta, error) {
ctx, cancel := ctxOp()
defer cancel()
var out map[string]itemMeta
err := c.db.Tx(ctx, func(tx pgx.Tx) error {
var err error
out, err = loadMetaTx(ctx, tx, c.root)
return err
})
return out, err
}
func sameMeta(a, b itemMeta) bool {
return slices.Equal(a.Tags, b.Tags) && a.Favorite == b.Favorite && a.Icon == b.Icon &&
a.CreatedBy == b.CreatedBy && a.EditedBy == b.EditedBy &&
((a.Created == nil) == (b.Created == nil)) && (a.Created == nil || a.Created.Equal(*b.Created))
}
func (c *catalogRef) updateMeta(fn func(map[string]itemMeta) (bool, error)) error {
ctx, cancel := ctxOp()
defer cancel()
return c.db.Tx(ctx, func(tx pgx.Tx) error {
if err := catalog.Lock(ctx, tx, "meta:"+c.root); err != nil {
return err
}
cur, err := loadMetaTx(ctx, tx, c.root)
if err != nil {
return err
}
next := make(map[string]itemMeta, len(cur))
for k, v := range cur {
next[k] = v
}
changed, err := fn(next)
if err != nil || !changed {
return err
}
return writeMetaDiff(ctx, tx, c.root, cur, next)
})
}
func writeMetaDiff(ctx context.Context, tx pgx.Tx, root string, cur, next map[string]itemMeta) error {
b := &pgx.Batch{}
for k := range cur {
if m, ok := next[k]; !ok || m.empty() {
b.Queue("DELETE FROM item_meta WHERE root = $1 AND path = $2", root, k)
}
}
for k, m := range next {
if m.empty() {
continue
}
if old, ok := cur[k]; ok && sameMeta(old, m) {
continue
}
tags := m.Tags
if tags == nil {
tags = []string{}
}
b.Queue(`INSERT INTO item_meta (root, path, tags, favorite, icon, created, created_by, edited_by)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (root, path) DO UPDATE SET tags = EXCLUDED.tags, favorite = EXCLUDED.favorite,
icon = EXCLUDED.icon, created = EXCLUDED.created, created_by = EXCLUDED.created_by,
edited_by = EXCLUDED.edited_by`,
root, k, tags, m.Favorite, m.Icon, m.Created, m.CreatedBy, m.EditedBy)
}
if b.Len() == 0 {
return nil
}
return tx.SendBatch(ctx, b).Close()
}
// --- Liens de partage --------------------------------------------------------
func loadSharesTx(ctx context.Context, tx pgx.Tx, root string) (map[string]shareDisk, error) {
rows, err := tx.Query(ctx, "SELECT id, data FROM shares WHERE root = $1", root)
if err != nil {
return nil, err
}
out := map[string]shareDisk{}
for rows.Next() {
var id string
var d shareDisk
if err := rows.Scan(&id, &d); err != nil {
rows.Close()
return nil, err
}
out[id] = d
}
return out, rows.Err()
}
func (c *catalogRef) readShares() (map[string]shareDisk, error) {
ctx, cancel := ctxOp()
defer cancel()
var out map[string]shareDisk
err := c.db.Tx(ctx, func(tx pgx.Tx) error {
var err error
out, err = loadSharesTx(ctx, tx, c.root)
return err
})
return out, err
}
func (c *catalogRef) editShares(edit func(map[string]shareDisk) (bool, error)) error {
ctx, cancel := ctxOp()
defer cancel()
return c.db.Tx(ctx, func(tx pgx.Tx) error {
if err := catalog.Lock(ctx, tx, "shares:"+c.root); err != nil {
return err
}
cur, err := loadSharesTx(ctx, tx, c.root)
if err != nil {
return err
}
next := make(map[string]shareDisk, len(cur))
for k, v := range cur {
next[k] = v
}
changed, err := edit(next)
if err != nil || !changed {
return err
}
return writeSharesDiff(ctx, tx, c.root, cur, next)
})
}
func writeSharesDiff(ctx context.Context, tx pgx.Tx, root string, cur, next map[string]shareDisk) error {
b := &pgx.Batch{}
for id := range cur {
if _, ok := next[id]; !ok {
b.Queue("DELETE FROM shares WHERE id = $1", id)
}
}
for id, d := range next {
if old, ok := cur[id]; ok && sameShare(old, d) {
continue
}
b.Queue(`INSERT INTO shares (id, root, data) VALUES ($1, $2, $3)
ON CONFLICT (id) DO UPDATE SET root = EXCLUDED.root, data = EXCLUDED.data`, id, root, d)
}
if b.Len() == 0 {
return nil
}
return tx.SendBatch(ctx, b).Close()
}
func sameShare(a, b shareDisk) bool {
ja, _ := json.Marshal(a)
jb, _ := json.Marshal(b)
return string(ja) == string(jb)
}
// --- Reprise des anciens fichiers JSON --------------------------------------
// importJSON reprend .meta.json et .shares.json d'un dossier racine, une seule
// fois par racine (table imports), sous verrou : deux instances qui démarrent
// ensemble ne reprennent pas deux fois.
func (c *catalogRef) importJSON(dir string) error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
var imported []string
err := c.db.Tx(ctx, func(tx pgx.Tx) error {
imported = imported[:0]
if err := catalog.Lock(ctx, tx, "import:"+c.root); err != nil {
return err
}
for _, kind := range []string{"meta", "shares"} {
var done bool
if err := tx.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM imports WHERE root = $1 AND kind = $2)",
c.root, kind).Scan(&done); err != nil {
return err
}
if done {
continue
}
file := metaFile
if kind == "shares" {
file = sharesFile
}
b, err := os.ReadFile(filepath.Join(dir, file))
switch {
case errors.Is(err, fs.ErrNotExist):
b = nil
case err != nil:
return err
}
if b != nil {
if err := importFile(ctx, tx, c.root, kind, file, b); err != nil {
return err
}
imported = append(imported, file)
}
if _, err := tx.Exec(ctx, "INSERT INTO imports (root, kind) VALUES ($1, $2)", c.root, kind); err != nil {
return err
}
}
return nil
})
if err != nil {
return err
}
for _, f := range imported {
p := filepath.Join(dir, f)
if err := os.Rename(p, p+".imported"); err != nil && !errors.Is(err, fs.ErrNotExist) {
slog.Warn("catalogue : fichier repris non renommé", "fichier", p, "err", err)
}
slog.Info("catalogue : fichier repris en base", "racine", c.root, "fichier", f)
}
return nil
}
func importFile(ctx context.Context, tx pgx.Tx, root, kind, file string, b []byte) error {
if kind == "meta" {
var d struct {
Items map[string]itemMeta `json:"items"`
}
if err := json.Unmarshal(b, &d); err != nil {
return fmt.Errorf("%s : %w", file, err)
}
cur, err := loadMetaTx(ctx, tx, root)
if err != nil {
return err
}
next := make(map[string]itemMeta, len(cur)+len(d.Items))
for k, v := range cur {
next[k] = v
}
for k, v := range d.Items {
if _, ok := next[k]; !ok {
next[k] = v
}
}
return writeMetaDiff(ctx, tx, root, cur, next)
}
var d struct {
Links map[string]shareDisk `json:"links"`
}
if err := json.Unmarshal(b, &d); err != nil {
return fmt.Errorf("%s : %w", file, err)
}
cur, err := loadSharesTx(ctx, tx, root)
if err != nil {
return err
}
next := make(map[string]shareDisk, len(cur)+len(d.Links))
for k, v := range cur {
next[k] = v
}
for k, v := range d.Links {
if _, ok := next[k]; !ok {
next[k] = v
}
}
return writeSharesDiff(ctx, tx, root, cur, next)
}