Lot 1 (suite) : paquet catalog et liaison Store <-> PostgreSQL
Fichiers nouveaux du commit précédent : internal/catalog/catalog.go, internal/storage/catalog.go et ses tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LNfwm4pfYR9C914C2EeiuB
This commit is contained in:
1 parent
f8801f00f9
commit
40c0c4d758
3 files changed
+694
No files matched your search
@@ -0,0 +1,336 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/vblogio/upandown/internal/catalog"
|
||||
)
|
||||
|
||||
// Catalogue PostgreSQL : quand il est configuré, les métadonnées (tags,
|
||||
// icônes, auteurs) et les liens de partage d'un Store sont lus et écrits en
|
||||
// base au lieu de .meta.json et .shares.json, ce qui permet à plusieurs
|
||||
// instances de servir la même arborescence. Chaque modification se fait dans
|
||||
// une transaction protégée par un verrou consultatif par racine : deux
|
||||
// instances qui modifient la même racine passent l'une après l'autre.
|
||||
|
||||
// opTimeout borne une opération sur le catalogue.
|
||||
const opTimeout = 30 * time.Second
|
||||
|
||||
type catalogRef struct {
|
||||
db *catalog.DB
|
||||
root string // "main", "home/<id>", "s3"
|
||||
}
|
||||
|
||||
// UseCatalog branche le Store sur le catalogue sous la clé root. Au premier
|
||||
// branchement d'une racine, ses anciens fichiers .meta.json et .shares.json
|
||||
// sont repris en base puis renommés en *.imported.
|
||||
func (s *Store) UseCatalog(db *catalog.DB, root string) error {
|
||||
c := &catalogRef{db: db, root: root}
|
||||
if err := c.importJSON(s.root); err != nil {
|
||||
return fmt.Errorf("reprise des métadonnées de %s : %w", root, err)
|
||||
}
|
||||
s.metaMu.Lock()
|
||||
s.cat, s.meta = c, nil
|
||||
s.metaMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func ctxOp() (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(context.Background(), opTimeout)
|
||||
}
|
||||
|
||||
// --- Métadonnées -------------------------------------------------------------
|
||||
|
||||
func loadMetaTx(ctx context.Context, q pgx.Tx, root string) (map[string]itemMeta, error) {
|
||||
rows, err := q.Query(ctx, `SELECT path, tags, favorite, icon, created, created_by, edited_by
|
||||
FROM item_meta WHERE root = $1`, root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]itemMeta{}
|
||||
for rows.Next() {
|
||||
var p string
|
||||
var m itemMeta
|
||||
var created *time.Time
|
||||
if err := rows.Scan(&p, &m.Tags, &m.Favorite, &m.Icon, &created, &m.CreatedBy, &m.EditedBy); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
if len(m.Tags) == 0 {
|
||||
m.Tags = nil
|
||||
}
|
||||
if created != nil {
|
||||
t := created.UTC()
|
||||
m.Created = &t
|
||||
}
|
||||
out[p] = m
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (c *catalogRef) readMeta() (map[string]itemMeta, error) {
|
||||
ctx, cancel := ctxOp()
|
||||
defer cancel()
|
||||
var out map[string]itemMeta
|
||||
err := c.db.Tx(ctx, func(tx pgx.Tx) error {
|
||||
var err error
|
||||
out, err = loadMetaTx(ctx, tx, c.root)
|
||||
return err
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
func sameMeta(a, b itemMeta) bool {
|
||||
return slices.Equal(a.Tags, b.Tags) && a.Favorite == b.Favorite && a.Icon == b.Icon &&
|
||||
a.CreatedBy == b.CreatedBy && a.EditedBy == b.EditedBy &&
|
||||
((a.Created == nil) == (b.Created == nil)) && (a.Created == nil || a.Created.Equal(*b.Created))
|
||||
}
|
||||
|
||||
func (c *catalogRef) updateMeta(fn func(map[string]itemMeta) (bool, error)) error {
|
||||
ctx, cancel := ctxOp()
|
||||
defer cancel()
|
||||
return c.db.Tx(ctx, func(tx pgx.Tx) error {
|
||||
if err := catalog.Lock(ctx, tx, "meta:"+c.root); err != nil {
|
||||
return err
|
||||
}
|
||||
cur, err := loadMetaTx(ctx, tx, c.root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next := make(map[string]itemMeta, len(cur))
|
||||
for k, v := range cur {
|
||||
next[k] = v
|
||||
}
|
||||
changed, err := fn(next)
|
||||
if err != nil || !changed {
|
||||
return err
|
||||
}
|
||||
return writeMetaDiff(ctx, tx, c.root, cur, next)
|
||||
})
|
||||
}
|
||||
|
||||
func writeMetaDiff(ctx context.Context, tx pgx.Tx, root string, cur, next map[string]itemMeta) error {
|
||||
b := &pgx.Batch{}
|
||||
for k := range cur {
|
||||
if m, ok := next[k]; !ok || m.empty() {
|
||||
b.Queue("DELETE FROM item_meta WHERE root = $1 AND path = $2", root, k)
|
||||
}
|
||||
}
|
||||
for k, m := range next {
|
||||
if m.empty() {
|
||||
continue
|
||||
}
|
||||
if old, ok := cur[k]; ok && sameMeta(old, m) {
|
||||
continue
|
||||
}
|
||||
tags := m.Tags
|
||||
if tags == nil {
|
||||
tags = []string{}
|
||||
}
|
||||
b.Queue(`INSERT INTO item_meta (root, path, tags, favorite, icon, created, created_by, edited_by)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
ON CONFLICT (root, path) DO UPDATE SET tags = EXCLUDED.tags, favorite = EXCLUDED.favorite,
|
||||
icon = EXCLUDED.icon, created = EXCLUDED.created, created_by = EXCLUDED.created_by,
|
||||
edited_by = EXCLUDED.edited_by`,
|
||||
root, k, tags, m.Favorite, m.Icon, m.Created, m.CreatedBy, m.EditedBy)
|
||||
}
|
||||
if b.Len() == 0 {
|
||||
return nil
|
||||
}
|
||||
return tx.SendBatch(ctx, b).Close()
|
||||
}
|
||||
|
||||
// --- Liens de partage --------------------------------------------------------
|
||||
|
||||
func loadSharesTx(ctx context.Context, tx pgx.Tx, root string) (map[string]shareDisk, error) {
|
||||
rows, err := tx.Query(ctx, "SELECT id, data FROM shares WHERE root = $1", root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]shareDisk{}
|
||||
for rows.Next() {
|
||||
var id string
|
||||
var d shareDisk
|
||||
if err := rows.Scan(&id, &d); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
out[id] = d
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (c *catalogRef) readShares() (map[string]shareDisk, error) {
|
||||
ctx, cancel := ctxOp()
|
||||
defer cancel()
|
||||
var out map[string]shareDisk
|
||||
err := c.db.Tx(ctx, func(tx pgx.Tx) error {
|
||||
var err error
|
||||
out, err = loadSharesTx(ctx, tx, c.root)
|
||||
return err
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (c *catalogRef) editShares(edit func(map[string]shareDisk) (bool, error)) error {
|
||||
ctx, cancel := ctxOp()
|
||||
defer cancel()
|
||||
return c.db.Tx(ctx, func(tx pgx.Tx) error {
|
||||
if err := catalog.Lock(ctx, tx, "shares:"+c.root); err != nil {
|
||||
return err
|
||||
}
|
||||
cur, err := loadSharesTx(ctx, tx, c.root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next := make(map[string]shareDisk, len(cur))
|
||||
for k, v := range cur {
|
||||
next[k] = v
|
||||
}
|
||||
changed, err := edit(next)
|
||||
if err != nil || !changed {
|
||||
return err
|
||||
}
|
||||
return writeSharesDiff(ctx, tx, c.root, cur, next)
|
||||
})
|
||||
}
|
||||
|
||||
func writeSharesDiff(ctx context.Context, tx pgx.Tx, root string, cur, next map[string]shareDisk) error {
|
||||
b := &pgx.Batch{}
|
||||
for id := range cur {
|
||||
if _, ok := next[id]; !ok {
|
||||
b.Queue("DELETE FROM shares WHERE id = $1", id)
|
||||
}
|
||||
}
|
||||
for id, d := range next {
|
||||
if old, ok := cur[id]; ok && sameShare(old, d) {
|
||||
continue
|
||||
}
|
||||
b.Queue(`INSERT INTO shares (id, root, data) VALUES ($1, $2, $3)
|
||||
ON CONFLICT (id) DO UPDATE SET root = EXCLUDED.root, data = EXCLUDED.data`, id, root, d)
|
||||
}
|
||||
if b.Len() == 0 {
|
||||
return nil
|
||||
}
|
||||
return tx.SendBatch(ctx, b).Close()
|
||||
}
|
||||
|
||||
func sameShare(a, b shareDisk) bool {
|
||||
ja, _ := json.Marshal(a)
|
||||
jb, _ := json.Marshal(b)
|
||||
return string(ja) == string(jb)
|
||||
}
|
||||
|
||||
// --- Reprise des anciens fichiers JSON --------------------------------------
|
||||
|
||||
// importJSON reprend .meta.json et .shares.json d'un dossier racine, une seule
|
||||
// fois par racine (table imports), sous verrou : deux instances qui démarrent
|
||||
// ensemble ne reprennent pas deux fois.
|
||||
func (c *catalogRef) importJSON(dir string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
var imported []string
|
||||
err := c.db.Tx(ctx, func(tx pgx.Tx) error {
|
||||
imported = imported[:0]
|
||||
if err := catalog.Lock(ctx, tx, "import:"+c.root); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, kind := range []string{"meta", "shares"} {
|
||||
var done bool
|
||||
if err := tx.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM imports WHERE root = $1 AND kind = $2)",
|
||||
c.root, kind).Scan(&done); err != nil {
|
||||
return err
|
||||
}
|
||||
if done {
|
||||
continue
|
||||
}
|
||||
file := metaFile
|
||||
if kind == "shares" {
|
||||
file = sharesFile
|
||||
}
|
||||
b, err := os.ReadFile(filepath.Join(dir, file))
|
||||
switch {
|
||||
case errors.Is(err, fs.ErrNotExist):
|
||||
b = nil
|
||||
case err != nil:
|
||||
return err
|
||||
}
|
||||
if b != nil {
|
||||
if err := importFile(ctx, tx, c.root, kind, file, b); err != nil {
|
||||
return err
|
||||
}
|
||||
imported = append(imported, file)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, "INSERT INTO imports (root, kind) VALUES ($1, $2)", c.root, kind); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, f := range imported {
|
||||
p := filepath.Join(dir, f)
|
||||
if err := os.Rename(p, p+".imported"); err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
slog.Warn("catalogue : fichier repris non renommé", "fichier", p, "err", err)
|
||||
}
|
||||
slog.Info("catalogue : fichier repris en base", "racine", c.root, "fichier", f)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func importFile(ctx context.Context, tx pgx.Tx, root, kind, file string, b []byte) error {
|
||||
if kind == "meta" {
|
||||
var d struct {
|
||||
Items map[string]itemMeta `json:"items"`
|
||||
}
|
||||
if err := json.Unmarshal(b, &d); err != nil {
|
||||
return fmt.Errorf("%s : %w", file, err)
|
||||
}
|
||||
cur, err := loadMetaTx(ctx, tx, root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next := make(map[string]itemMeta, len(cur)+len(d.Items))
|
||||
for k, v := range cur {
|
||||
next[k] = v
|
||||
}
|
||||
for k, v := range d.Items {
|
||||
if _, ok := next[k]; !ok {
|
||||
next[k] = v
|
||||
}
|
||||
}
|
||||
return writeMetaDiff(ctx, tx, root, cur, next)
|
||||
}
|
||||
var d struct {
|
||||
Links map[string]shareDisk `json:"links"`
|
||||
}
|
||||
if err := json.Unmarshal(b, &d); err != nil {
|
||||
return fmt.Errorf("%s : %w", file, err)
|
||||
}
|
||||
cur, err := loadSharesTx(ctx, tx, root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next := make(map[string]shareDisk, len(cur)+len(d.Links))
|
||||
for k, v := range cur {
|
||||
next[k] = v
|
||||
}
|
||||
for k, v := range d.Links {
|
||||
if _, ok := next[k]; !ok {
|
||||
next[k] = v
|
||||
}
|
||||
}
|
||||
return writeSharesDiff(ctx, tx, root, cur, next)
|
||||
}
|
||||
Reference in new issue
Block a user