Complète le commit précédent (s3.go manquant). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LNfwm4pfYR9C914C2EeiuB
968 lines
25 KiB
Go
968 lines
25 KiB
Go
package storage
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"log/slog"
|
|
"net/url"
|
|
"os"
|
|
"path"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/vblogio/upandown/internal/catalog"
|
|
)
|
|
|
|
// Racine S3 : l'arborescence (noms, dossiers, tailles, dates) vit dans le
|
|
// catalogue PostgreSQL (table s3_nodes), les contenus dans le bucket sous une
|
|
// clé fixée au dépôt (« o/<id> »). Renommer ou déplacer ne touche donc que le
|
|
// catalogue ; le chemin de dépôt reste dans les métadonnées x-amz-meta de
|
|
// l'objet (reconstruction de secours). Corbeille : les nœuds passent sous
|
|
// « .trash/<id>/ » dans le catalogue, les objets ne sont supprimés qu'à la
|
|
// purge. Pas d'historique de versions ni de recherche plein texte.
|
|
|
|
// ObjectStore est le minimum attendu du stockage objet (minio-go pour Garage,
|
|
// mémoire pour les tests).
|
|
type ObjectStore interface {
|
|
// Put envoie r (taille inconnue : -1) sous key et renvoie la taille écrite.
|
|
Put(ctx context.Context, key string, r io.Reader, meta map[string]string) (int64, error)
|
|
// Get ouvre key en lecture par plages.
|
|
Get(ctx context.Context, key string) (File, error)
|
|
// Copy duplique src vers dst côté serveur.
|
|
Copy(ctx context.Context, src, dst string, meta map[string]string) error
|
|
Remove(ctx context.Context, key string) error
|
|
// Empty indique si le bucket ne contient aucun objet.
|
|
Empty(ctx context.Context) (bool, error)
|
|
}
|
|
|
|
const (
|
|
s3TrashPrefix = ".trash"
|
|
s3ObjPrefix = "o/"
|
|
s3LockName = "s3tree"
|
|
)
|
|
|
|
// s3Root porte les opérations propres à la racine S3 d'un Store.
|
|
type s3Root struct {
|
|
db *catalog.DB
|
|
objs ObjectStore
|
|
}
|
|
|
|
type s3Node struct {
|
|
Path, Dir, Name string
|
|
IsDir bool
|
|
Size int64
|
|
ModTime time.Time
|
|
Key string
|
|
}
|
|
|
|
// NewS3 crée le Store de la racine S3 : cacheDir (sur DATA_DIR) ne reçoit que
|
|
// les miniatures ; métadonnées et partages vont dans le catalogue sous la
|
|
// racine "s3".
|
|
func NewS3(cacheDir string, db *catalog.DB, objs ObjectStore) (*Store, error) {
|
|
s, err := New(cacheDir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
s.History = HistoryPolicy{}
|
|
s.s3 = &s3Root{db: db, objs: objs}
|
|
if err := s.UseCatalog(db, "s3"); err != nil {
|
|
return nil, err
|
|
}
|
|
// Décision : la racine S3 démarre toujours sur un bucket vide. Un catalogue
|
|
// vide face à un bucket qui contient déjà des objets (autre usage, ou
|
|
// catalogue perdu) est refusé plutôt que de mélanger les deux.
|
|
empty, err := s.s3Empty()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if empty {
|
|
ctx, cancel := ctxOp()
|
|
defer cancel()
|
|
bucketEmpty, err := objs.Empty(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !bucketEmpty {
|
|
return nil, errors.New("racine S3 refusée : le bucket contient déjà des objets alors que le catalogue est vide (bucket vide attendu)")
|
|
}
|
|
}
|
|
return s, nil
|
|
}
|
|
|
|
// IsS3 indique si le Store est la racine S3.
|
|
func (s *Store) IsS3() bool { return s.s3 != nil }
|
|
|
|
func newObjectKey() string {
|
|
b := make([]byte, 16)
|
|
_, _ = rand.Read(b)
|
|
return s3ObjPrefix + hex.EncodeToString(b)
|
|
}
|
|
|
|
func s3Now() time.Time { return time.Now().UTC().Truncate(time.Microsecond) }
|
|
|
|
func parentOf(c string) string {
|
|
if d := path.Dir(c); d != "." {
|
|
return d
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// --- Lecture du catalogue ----------------------------------------------------------
|
|
|
|
const s3Cols = "path, dir, name, is_dir, size, mtime, object_key"
|
|
|
|
func scanNode(row pgx.Row) (s3Node, error) {
|
|
var n s3Node
|
|
var key *string
|
|
err := row.Scan(&n.Path, &n.Dir, &n.Name, &n.IsDir, &n.Size, &n.ModTime, &key)
|
|
if key != nil {
|
|
n.Key = *key
|
|
}
|
|
n.ModTime = n.ModTime.UTC()
|
|
return n, err
|
|
}
|
|
|
|
func getNode(ctx context.Context, q pgx.Tx, c string) (s3Node, bool, error) {
|
|
n, err := scanNode(q.QueryRow(ctx, "SELECT "+s3Cols+" FROM s3_nodes WHERE path = $1", c))
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return s3Node{}, false, nil
|
|
}
|
|
return n, err == nil, err
|
|
}
|
|
|
|
func queryNodes(ctx context.Context, q pgx.Tx, where string, args ...any) ([]s3Node, error) {
|
|
rows, err := q.Query(ctx, "SELECT "+s3Cols+" FROM s3_nodes WHERE "+where, args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var out []s3Node
|
|
for rows.Next() {
|
|
n, err := scanNode(rows)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, n)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
// subtree : le nœud c et tout ce qu'il contient.
|
|
const subtreeWhere = "(path = $1 OR starts_with(path, $1 || '/'))"
|
|
|
|
func (r *s3Root) read(fn func(ctx context.Context, tx pgx.Tx) error) error {
|
|
ctx, cancel := ctxOp()
|
|
defer cancel()
|
|
return r.db.Tx(ctx, func(tx pgx.Tx) error { return fn(ctx, tx) })
|
|
}
|
|
|
|
// write exécute fn sous le verrou de l'arborescence S3 (toutes instances).
|
|
func (r *s3Root) write(fn func(ctx context.Context, tx pgx.Tx) error) error {
|
|
ctx, cancel := ctxOp()
|
|
defer cancel()
|
|
return r.db.Tx(ctx, func(tx pgx.Tx) error {
|
|
if err := catalog.Lock(ctx, tx, s3LockName); err != nil {
|
|
return err
|
|
}
|
|
return fn(ctx, tx)
|
|
})
|
|
}
|
|
|
|
func (n s3Node) entry() Entry {
|
|
e := Entry{Name: n.Name, Path: n.Path, IsDir: n.IsDir, ModTime: n.ModTime}
|
|
if !n.IsDir {
|
|
e.Size = n.Size
|
|
}
|
|
return e
|
|
}
|
|
|
|
// s3Info : fs.FileInfo d'un objet (ETag dérivé de la taille et de la date).
|
|
type s3Info struct{ n s3Node }
|
|
|
|
func (i s3Info) Name() string { return i.n.Name }
|
|
func (i s3Info) Size() int64 { return i.n.Size }
|
|
func (i s3Info) Mode() fs.FileMode { return 0o644 }
|
|
func (i s3Info) ModTime() time.Time { return i.n.ModTime }
|
|
func (i s3Info) IsDir() bool { return i.n.IsDir }
|
|
func (i s3Info) Sys() any { return nil }
|
|
|
|
// ensureDirs crée les dossiers manquants de dir (et refuse un fichier en chemin).
|
|
func ensureDirs(ctx context.Context, tx pgx.Tx, dir string, now time.Time) error {
|
|
if dir == "" {
|
|
return nil
|
|
}
|
|
n, ok, err := getNode(ctx, tx, dir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if ok {
|
|
if !n.IsDir {
|
|
return fmt.Errorf("%w : un fichier porte le nom d'un dossier du chemin", ErrExists)
|
|
}
|
|
return nil
|
|
}
|
|
if err := ensureDirs(ctx, tx, parentOf(dir), now); err != nil {
|
|
return err
|
|
}
|
|
_, err = tx.Exec(ctx, "INSERT INTO s3_nodes (path, dir, name, is_dir, size, mtime) VALUES ($1, $2, $3, true, 0, $4)",
|
|
dir, parentOf(dir), path.Base(dir), now)
|
|
return err
|
|
}
|
|
|
|
// --- Opérations de base (Root) ---------------------------------------------------
|
|
|
|
func (s *Store) s3List(c string) ([]Entry, error) {
|
|
var nodes []s3Node
|
|
err := s.s3.read(func(ctx context.Context, tx pgx.Tx) error {
|
|
if c != "" {
|
|
n, ok, err := getNode(ctx, tx, c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !ok || !n.IsDir {
|
|
return ErrNotFound
|
|
}
|
|
}
|
|
var err error
|
|
nodes, err = queryNodes(ctx, tx, "dir = $1", c)
|
|
return err
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
meta := s.metaSnapshot()
|
|
out := make([]Entry, 0, len(nodes))
|
|
for _, n := range nodes {
|
|
if !visible(n.Name) {
|
|
continue
|
|
}
|
|
e := n.entry()
|
|
decorate(&e, meta)
|
|
out = append(out, e)
|
|
}
|
|
sortEntries(out)
|
|
return out, nil
|
|
}
|
|
|
|
func (s *Store) s3Node(c string) (s3Node, error) {
|
|
var n s3Node
|
|
err := s.s3.read(func(ctx context.Context, tx pgx.Tx) error {
|
|
var ok bool
|
|
var err error
|
|
n, ok, err = getNode(ctx, tx, c)
|
|
if err == nil && !ok {
|
|
err = ErrNotFound
|
|
}
|
|
return err
|
|
})
|
|
return n, err
|
|
}
|
|
|
|
func (s *Store) s3Stat(c string) (Entry, error) {
|
|
if c == "" {
|
|
return Entry{Path: "", IsDir: true}, nil
|
|
}
|
|
n, err := s.s3Node(c)
|
|
if err != nil {
|
|
return Entry{}, err
|
|
}
|
|
e := n.entry()
|
|
decorate(&e, s.metaSnapshot())
|
|
return e, nil
|
|
}
|
|
|
|
func (s *Store) s3Open(c string) (File, fs.FileInfo, error) {
|
|
n, err := s.s3Node(c)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
if n.IsDir || n.Key == "" {
|
|
return nil, nil, ErrNotFound
|
|
}
|
|
ctx, cancel := ctxOp()
|
|
defer cancel()
|
|
f, err := s.s3.objs.Get(ctx, n.Key)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
return f, s3Info{n}, nil
|
|
}
|
|
|
|
// countingReader compte les octets lus et s'arrête au-delà de max (0 : illimité).
|
|
type countingReader struct {
|
|
r io.Reader
|
|
n int64
|
|
max int64
|
|
}
|
|
|
|
var errTooMuch = errors.New("limite dépassée")
|
|
|
|
func (c *countingReader) Read(p []byte) (int, error) {
|
|
n, err := c.r.Read(p)
|
|
c.n += int64(n)
|
|
if c.max > 0 && c.n > c.max {
|
|
return n, errTooMuch
|
|
}
|
|
return n, err
|
|
}
|
|
|
|
// freeNameTx : comme Store.freeName, dans le catalogue.
|
|
func freeNameTx(ctx context.Context, tx pgx.Tx, c string, isDir bool, format string) (string, error) {
|
|
_, ok, err := getNode(ctx, tx, c)
|
|
if err != nil || !ok {
|
|
return c, err
|
|
}
|
|
dir, base := parentOf(c), path.Base(c)
|
|
ext := ""
|
|
if !isDir {
|
|
ext = path.Ext(base)
|
|
if ext == base {
|
|
ext = ""
|
|
}
|
|
}
|
|
stem := strings.TrimSuffix(base, ext)
|
|
for i := 2; ; i++ {
|
|
cand := path.Join(dir, stem+fmt.Sprintf(format, i)+ext)
|
|
_, ok, err := getNode(ctx, tx, cand)
|
|
if err != nil || !ok {
|
|
return cand, err
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *Store) s3Put(c string, r io.Reader, o PutOptions) (Entry, error) {
|
|
if c == "" {
|
|
return Entry{}, ErrInvalidPath
|
|
}
|
|
// Contrôles préalables (refaits sous verrou après l'envoi).
|
|
target := c
|
|
if err := s.s3.read(func(ctx context.Context, tx pgx.Tx) error {
|
|
n, ok, err := getNode(ctx, tx, c)
|
|
if err != nil || !ok {
|
|
if err == nil && o.IfMatch != "" && o.IfMatch != "*" {
|
|
err = ErrConflict
|
|
}
|
|
return err
|
|
}
|
|
if n.IsDir {
|
|
return fmt.Errorf("%w : un dossier porte déjà ce nom", ErrExists)
|
|
}
|
|
switch o.Conflict {
|
|
case "overwrite":
|
|
if o.IfMatch != "" && o.IfMatch != "*" && o.IfMatch != ETag(s3Info{n}) {
|
|
return ErrConflict
|
|
}
|
|
case "rename":
|
|
target, err = freeNameTx(ctx, tx, c, false, " (%d)")
|
|
return err
|
|
default:
|
|
return ErrExists
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
return Entry{}, err
|
|
}
|
|
|
|
key := newObjectKey()
|
|
cr := &countingReader{r: r}
|
|
if o.Limit > 0 {
|
|
cr.max = o.Limit
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
meta := map[string]string{"path": url.PathEscape(target), "by": url.PathEscape(o.By)}
|
|
size, err := s.s3.objs.Put(ctx, key, cr, meta)
|
|
if err != nil || (o.Limit > 0 && cr.n > o.Limit) {
|
|
s.s3RemoveObjects([]string{key})
|
|
if errors.Is(err, errTooMuch) || (o.Limit > 0 && cr.n > o.Limit) {
|
|
if o.LimitErr != nil {
|
|
return Entry{}, o.LimitErr
|
|
}
|
|
return Entry{}, ErrTooLarge
|
|
}
|
|
return Entry{}, err
|
|
}
|
|
mtime := s3Now()
|
|
if m := o.ModTime; !m.IsZero() && m.After(time.Date(1980, 1, 1, 0, 0, 0, 0, time.UTC)) && m.Before(time.Now().Add(24*time.Hour)) {
|
|
mtime = m.UTC().Truncate(time.Microsecond)
|
|
}
|
|
|
|
var old string
|
|
created := false
|
|
err = s.s3.write(func(ctx context.Context, tx pgx.Tx) error {
|
|
old, created = "", false
|
|
n, ok, err := getNode(ctx, tx, target)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if ok {
|
|
if n.IsDir {
|
|
return fmt.Errorf("%w : un dossier porte déjà ce nom", ErrExists)
|
|
}
|
|
switch o.Conflict {
|
|
case "overwrite":
|
|
if o.IfMatch != "" && o.IfMatch != "*" && o.IfMatch != ETag(s3Info{n}) {
|
|
return ErrConflict
|
|
}
|
|
case "rename": // pris entre-temps
|
|
if target, err = freeNameTx(ctx, tx, target, false, " (%d)"); err != nil {
|
|
return err
|
|
}
|
|
ok = false
|
|
default:
|
|
return ErrExists
|
|
}
|
|
} else if o.IfMatch != "" && o.IfMatch != "*" && o.Conflict == "overwrite" && target == c {
|
|
return ErrConflict
|
|
}
|
|
if err := ensureDirs(ctx, tx, parentOf(target), mtime); err != nil {
|
|
return err
|
|
}
|
|
if ok {
|
|
old = n.Key
|
|
_, err = tx.Exec(ctx, "UPDATE s3_nodes SET size = $2, mtime = $3, object_key = $4 WHERE path = $1",
|
|
target, size, mtime, key)
|
|
return err
|
|
}
|
|
created = true
|
|
_, err = tx.Exec(ctx, `INSERT INTO s3_nodes (path, dir, name, is_dir, size, mtime, object_key, created_by)
|
|
VALUES ($1, $2, $3, false, $4, $5, $6, $7)`, target, parentOf(target), path.Base(target), size, mtime, key, o.By)
|
|
return err
|
|
})
|
|
if err != nil {
|
|
s.s3RemoveObjects([]string{key})
|
|
return Entry{}, err
|
|
}
|
|
if old != "" {
|
|
s.s3RemoveObjects([]string{old})
|
|
}
|
|
s.removeThumb(target)
|
|
s.noteEdit(target, o.By, created)
|
|
e := s3Node{Path: target, Name: path.Base(target), Size: size, ModTime: mtime}.entry()
|
|
e.EditedBy = o.By
|
|
return e, nil
|
|
}
|
|
|
|
func (s *Store) s3Mkdir(c string) (Entry, error) {
|
|
if c == "" {
|
|
return Entry{}, ErrInvalidPath
|
|
}
|
|
now := s3Now()
|
|
err := s.s3.write(func(ctx context.Context, tx pgx.Tx) error {
|
|
if _, ok, err := getNode(ctx, tx, c); err != nil || ok {
|
|
if err == nil {
|
|
err = ErrExists
|
|
}
|
|
return err
|
|
}
|
|
return ensureDirs(ctx, tx, c, now)
|
|
})
|
|
if err != nil {
|
|
return Entry{}, err
|
|
}
|
|
return Entry{Name: path.Base(c), Path: c, IsDir: true, ModTime: now}, nil
|
|
}
|
|
|
|
// moveTx renomme le sous-arbre from en to dans le catalogue.
|
|
func moveTx(ctx context.Context, tx pgx.Tx, from, to string) error {
|
|
if _, err := tx.Exec(ctx, `UPDATE s3_nodes SET path = $2 || substr(path, length($1) + 1),
|
|
dir = CASE WHEN path = $1 THEN $3 ELSE $2 || substr(dir, length($1) + 1) END,
|
|
name = CASE WHEN path = $1 THEN $4 ELSE name END
|
|
WHERE `+subtreeWhere, from, to, parentOf(to), path.Base(to)); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Store) s3Move(from, to string) (Entry, error) {
|
|
if from == "" || to == "" {
|
|
return Entry{}, ErrInvalidPath
|
|
}
|
|
var n s3Node
|
|
err := s.s3.write(func(ctx context.Context, tx pgx.Tx) error {
|
|
var ok bool
|
|
var err error
|
|
if n, ok, err = getNode(ctx, tx, from); err != nil || !ok {
|
|
if err == nil {
|
|
err = ErrNotFound
|
|
}
|
|
return err
|
|
}
|
|
if n.IsDir && (to == from || strings.HasPrefix(to, from+"/")) {
|
|
return fmt.Errorf("%w : déplacement d'un dossier dans lui-même", ErrInvalidPath)
|
|
}
|
|
if _, ok, err := getNode(ctx, tx, to); err != nil || ok {
|
|
if err == nil {
|
|
err = ErrExists
|
|
}
|
|
return err
|
|
}
|
|
if err := ensureDirs(ctx, tx, parentOf(to), s3Now()); err != nil {
|
|
return err
|
|
}
|
|
return moveTx(ctx, tx, from, to)
|
|
})
|
|
if err != nil {
|
|
return Entry{}, err
|
|
}
|
|
s.moveThumb(from, to, n.IsDir)
|
|
s.moveMeta(from, to)
|
|
s.moveShares(from, to)
|
|
n.Path, n.Dir, n.Name = to, parentOf(to), path.Base(to)
|
|
e := n.entry()
|
|
decorate(&e, s.metaSnapshot())
|
|
return e, nil
|
|
}
|
|
|
|
func (s *Store) s3Copy(from, to string) (Entry, error) {
|
|
if from == "" || to == "" {
|
|
return Entry{}, ErrInvalidPath
|
|
}
|
|
// Liste des nœuds à copier, puis copie des objets hors transaction.
|
|
var nodes []s3Node
|
|
err := s.s3.read(func(ctx context.Context, tx pgx.Tx) error {
|
|
n, ok, err := getNode(ctx, tx, from)
|
|
if err != nil || !ok {
|
|
if err == nil {
|
|
err = ErrNotFound
|
|
}
|
|
return err
|
|
}
|
|
if n.IsDir && (to == from || strings.HasPrefix(to, from+"/")) {
|
|
return fmt.Errorf("%w : copie d'un dossier dans lui-même", ErrInvalidPath)
|
|
}
|
|
if _, ok, err := getNode(ctx, tx, to); err != nil || ok {
|
|
if err == nil {
|
|
err = ErrExists
|
|
}
|
|
return err
|
|
}
|
|
nodes, err = queryNodes(ctx, tx, subtreeWhere+" ORDER BY path", from)
|
|
return err
|
|
})
|
|
if err != nil {
|
|
return Entry{}, err
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
newKeys := map[string]string{}
|
|
var created []string
|
|
for _, n := range nodes {
|
|
if n.IsDir || n.Key == "" {
|
|
continue
|
|
}
|
|
k := newObjectKey()
|
|
dst := to + strings.TrimPrefix(n.Path, from)
|
|
if err := s.s3.objs.Copy(ctx, n.Key, k, map[string]string{"path": url.PathEscape(dst)}); err != nil {
|
|
s.s3RemoveObjects(created)
|
|
return Entry{}, err
|
|
}
|
|
newKeys[n.Path] = k
|
|
created = append(created, k)
|
|
}
|
|
now := s3Now()
|
|
err = s.s3.write(func(ctx context.Context, tx pgx.Tx) error {
|
|
if _, ok, err := getNode(ctx, tx, to); err != nil || ok {
|
|
if err == nil {
|
|
err = ErrExists
|
|
}
|
|
return err
|
|
}
|
|
if err := ensureDirs(ctx, tx, parentOf(to), now); err != nil {
|
|
return err
|
|
}
|
|
for _, n := range nodes {
|
|
p := to + strings.TrimPrefix(n.Path, from)
|
|
var key *string
|
|
if k, ok := newKeys[n.Path]; ok {
|
|
key = &k
|
|
}
|
|
if _, err := tx.Exec(ctx, `INSERT INTO s3_nodes (path, dir, name, is_dir, size, mtime, object_key)
|
|
VALUES ($1, $2, $3, $4, $5, $6, $7)`, p, parentOf(p), path.Base(p), n.IsDir, n.Size, n.ModTime, key); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
s.s3RemoveObjects(created)
|
|
return Entry{}, err
|
|
}
|
|
s.copyMeta(from, to)
|
|
root := nodes[0]
|
|
root.Path, root.Dir, root.Name = to, parentOf(to), path.Base(to)
|
|
return root.entry(), nil
|
|
}
|
|
|
|
// s3RemoveObjects supprime des objets ; un échec laisse un orphelin, signalé
|
|
// dans le journal (la réconciliation le retrouvera).
|
|
func (s *Store) s3RemoveObjects(keys []string) {
|
|
if len(keys) == 0 {
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
|
defer cancel()
|
|
for _, k := range keys {
|
|
if err := s.s3.objs.Remove(ctx, k); err != nil {
|
|
slog.Warn("s3 : objet non supprimé", "key", k, "err", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// deleteTx retire le sous-arbre c du catalogue et renvoie les clés à supprimer.
|
|
func deleteTx(ctx context.Context, tx pgx.Tx, c string) ([]string, error) {
|
|
rows, err := tx.Query(ctx, "DELETE FROM s3_nodes WHERE "+subtreeWhere+" RETURNING object_key", c)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var keys []string
|
|
for rows.Next() {
|
|
var k *string
|
|
if err := rows.Scan(&k); err != nil {
|
|
return nil, err
|
|
}
|
|
if k != nil {
|
|
keys = append(keys, *k)
|
|
}
|
|
}
|
|
return keys, rows.Err()
|
|
}
|
|
|
|
func (s *Store) s3Remove(c string) error {
|
|
if c == "" {
|
|
return ErrInvalidPath
|
|
}
|
|
var keys []string
|
|
err := s.s3.write(func(ctx context.Context, tx pgx.Tx) error {
|
|
if _, ok, err := getNode(ctx, tx, c); err != nil || !ok {
|
|
if err == nil {
|
|
err = ErrNotFound
|
|
}
|
|
return err
|
|
}
|
|
var err error
|
|
keys, err = deleteTx(ctx, tx, c)
|
|
return err
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
s.s3RemoveObjects(keys)
|
|
s.removeThumb(c)
|
|
s.removeMeta(c)
|
|
s.removeShares(c)
|
|
return nil
|
|
}
|
|
|
|
func (s *Store) s3Walk(c string, fn WalkFunc) error {
|
|
var nodes []s3Node
|
|
err := s.s3.read(func(ctx context.Context, tx pgx.Tx) error {
|
|
var err error
|
|
if c == "" {
|
|
nodes, err = queryNodes(ctx, tx, "NOT starts_with(path, '.') ORDER BY path")
|
|
return err
|
|
}
|
|
if _, ok, err := getNode(ctx, tx, c); err != nil || !ok {
|
|
if err == nil {
|
|
err = ErrNotFound
|
|
}
|
|
return err
|
|
}
|
|
nodes, err = queryNodes(ctx, tx, subtreeWhere+" ORDER BY path", c)
|
|
return err
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if c == "" {
|
|
if err := fn("", Entry{IsDir: true}); err != nil {
|
|
if errors.Is(err, fs.SkipDir) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
}
|
|
skip := ""
|
|
for _, n := range nodes {
|
|
if skip != "" && strings.HasPrefix(n.Path, skip) {
|
|
continue
|
|
}
|
|
if err := fn(n.Path, n.entry()); err != nil {
|
|
if errors.Is(err, fs.SkipDir) {
|
|
if n.IsDir {
|
|
skip = n.Path + "/"
|
|
continue
|
|
}
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Store) s3Usage() int64 {
|
|
var total int64
|
|
_ = s.s3.read(func(ctx context.Context, tx pgx.Tx) error {
|
|
return tx.QueryRow(ctx, "SELECT coalesce(sum(size), 0) FROM s3_nodes").Scan(&total)
|
|
})
|
|
return total
|
|
}
|
|
|
|
// s3Empty indique si le catalogue S3 ne contient aucun élément.
|
|
func (s *Store) s3Empty() (bool, error) {
|
|
var n int64
|
|
err := s.s3.read(func(ctx context.Context, tx pgx.Tx) error {
|
|
return tx.QueryRow(ctx, "SELECT count(*) FROM s3_nodes").Scan(&n)
|
|
})
|
|
return n == 0, err
|
|
}
|
|
|
|
// --- Corbeille -------------------------------------------------------------------
|
|
|
|
func (s *Store) s3Delete(c string, recursive bool, by string) error {
|
|
if c == "" {
|
|
return ErrInvalidPath
|
|
}
|
|
now := time.Now().UTC()
|
|
id := now.Format(trashLayout)
|
|
var n s3Node
|
|
meta := s.takeMeta(c)
|
|
err := s.s3.write(func(ctx context.Context, tx pgx.Tx) error {
|
|
var ok bool
|
|
var err error
|
|
if n, ok, err = getNode(ctx, tx, c); err != nil || !ok {
|
|
if err == nil {
|
|
err = ErrNotFound
|
|
}
|
|
return err
|
|
}
|
|
if n.IsDir && !recursive {
|
|
var child bool
|
|
if err := tx.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM s3_nodes WHERE dir = $1)", c).Scan(&child); err != nil {
|
|
return err
|
|
}
|
|
if child {
|
|
return ErrNotEmpty
|
|
}
|
|
}
|
|
for i := 1; ; i++ { // deux suppressions dans la même nanoseconde
|
|
var taken bool
|
|
if err := tx.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM s3_trash WHERE id = $1)", id).Scan(&taken); err != nil {
|
|
return err
|
|
}
|
|
if !taken {
|
|
break
|
|
}
|
|
id = now.Add(time.Duration(i)).Format(trashLayout)
|
|
}
|
|
var size int64
|
|
if err := tx.QueryRow(ctx, "SELECT coalesce(sum(size), 0) FROM s3_nodes WHERE "+subtreeWhere, c).Scan(&size); err != nil {
|
|
return err
|
|
}
|
|
if err := moveTx(ctx, tx, c, s3TrashPrefix+"/"+id+"/"+n.Name); err != nil {
|
|
return err
|
|
}
|
|
_, err = tx.Exec(ctx, `INSERT INTO s3_trash (id, origin, name, is_dir, size, deleted, deleted_by, meta)
|
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, id, c, n.Name, n.IsDir, size, now, by, meta)
|
|
return err
|
|
})
|
|
if err != nil {
|
|
s.putMeta(c, meta)
|
|
return err
|
|
}
|
|
s.removeThumb(c)
|
|
s.removeShares(c)
|
|
return nil
|
|
}
|
|
|
|
func (s *Store) s3TrashItems() ([]TrashItem, error) {
|
|
out := []TrashItem{}
|
|
err := s.s3.read(func(ctx context.Context, tx pgx.Tx) error {
|
|
rows, err := tx.Query(ctx, "SELECT id, origin, name, is_dir, size, deleted, deleted_by FROM s3_trash ORDER BY id DESC")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer rows.Close()
|
|
out = out[:0]
|
|
for rows.Next() {
|
|
var it TrashItem
|
|
if err := rows.Scan(&it.ID, &it.Path, &it.Name, &it.IsDir, &it.Size, &it.Deleted, &it.By); err != nil {
|
|
return err
|
|
}
|
|
it.Deleted = it.Deleted.UTC()
|
|
if it.IsDir {
|
|
it.Size = 0
|
|
}
|
|
out = append(out, it)
|
|
}
|
|
return rows.Err()
|
|
})
|
|
return out, err
|
|
}
|
|
|
|
func (s *Store) s3TrashCount() int {
|
|
var n int
|
|
_ = s.s3.read(func(ctx context.Context, tx pgx.Tx) error {
|
|
return tx.QueryRow(ctx, "SELECT count(*) FROM s3_trash").Scan(&n)
|
|
})
|
|
return n
|
|
}
|
|
|
|
func (s *Store) s3RestoreTrash(id string) (Entry, error) {
|
|
if !trashID.MatchString(id) {
|
|
return Entry{}, fmt.Errorf("%w : élément de corbeille %q", ErrInvalidPath, id)
|
|
}
|
|
var c string
|
|
var meta map[string]itemMeta
|
|
var n s3Node
|
|
err := s.s3.write(func(ctx context.Context, tx pgx.Tx) error {
|
|
var origin, name string
|
|
var isDir bool
|
|
meta = nil
|
|
err := tx.QueryRow(ctx, "SELECT origin, name, is_dir, meta FROM s3_trash WHERE id = $1", id).Scan(&origin, &name, &isDir, &meta)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return ErrNotFound
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if cc, err := Clean(origin); err != nil || cc == "" {
|
|
origin = name
|
|
}
|
|
if c, err = freeNameTx(ctx, tx, origin, isDir, " (restauré %d)"); err != nil {
|
|
return err
|
|
}
|
|
if err := ensureDirs(ctx, tx, parentOf(c), s3Now()); err != nil {
|
|
return err
|
|
}
|
|
src := s3TrashPrefix + "/" + id + "/" + name
|
|
var ok bool
|
|
if n, ok, err = getNode(ctx, tx, src); err != nil || !ok {
|
|
if err == nil {
|
|
err = ErrNotFound
|
|
}
|
|
return err
|
|
}
|
|
if err := moveTx(ctx, tx, src, c); err != nil {
|
|
return err
|
|
}
|
|
_, err = tx.Exec(ctx, "DELETE FROM s3_trash WHERE id = $1", id)
|
|
return err
|
|
})
|
|
if err != nil {
|
|
return Entry{}, err
|
|
}
|
|
s.putMeta(c, meta)
|
|
n.Path, n.Dir, n.Name = c, parentOf(c), path.Base(c)
|
|
e := n.entry()
|
|
decorate(&e, s.metaSnapshot())
|
|
return e, nil
|
|
}
|
|
|
|
// s3DropTrash supprime définitivement des entrées de corbeille (ids nil : toutes
|
|
// celles que drop retient).
|
|
func (s *Store) s3DropTrash(drop func(id string) bool) error {
|
|
var keys []string
|
|
err := s.s3.write(func(ctx context.Context, tx pgx.Tx) error {
|
|
keys = keys[:0]
|
|
rows, err := tx.Query(ctx, "SELECT id FROM s3_trash")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var ids []string
|
|
for rows.Next() {
|
|
var id string
|
|
if err := rows.Scan(&id); err != nil {
|
|
rows.Close()
|
|
return err
|
|
}
|
|
if drop(id) {
|
|
ids = append(ids, id)
|
|
}
|
|
}
|
|
rows.Close()
|
|
if err := rows.Err(); err != nil {
|
|
return err
|
|
}
|
|
for _, id := range ids {
|
|
k, err := deleteTx(ctx, tx, s3TrashPrefix+"/"+id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
keys = append(keys, k...)
|
|
if _, err := tx.Exec(ctx, "DELETE FROM s3_trash WHERE id = $1", id); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
s.s3RemoveObjects(keys)
|
|
return nil
|
|
}
|
|
|
|
// lstat renvoie le chemin nettoyé et la nature d'un élément, sur disque comme
|
|
// sur S3 (liens symboliques refusés sur disque).
|
|
func (s *Store) lstat(rel string) (string, fs.FileInfo, error) {
|
|
if s.s3 != nil {
|
|
c, err := Clean(rel)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
if c == "" {
|
|
return "", s3Info{s3Node{IsDir: true}}, nil
|
|
}
|
|
n, err := s.s3Node(c)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
return c, s3Info{n}, nil
|
|
}
|
|
c, abs, err := s.resolve(rel)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
fi, err := os.Lstat(abs)
|
|
if err != nil {
|
|
return "", nil, mapErr(err)
|
|
}
|
|
if fi.Mode()&fs.ModeSymlink != 0 {
|
|
return "", nil, ErrNotFound
|
|
}
|
|
return c, fi, nil
|
|
}
|
|
|
|
// s3DirDetails : comme dirDetails, depuis le catalogue.
|
|
func (s *Store) s3DirDetails(c string) (*Details, error) {
|
|
es, err := s.s3List(c)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
d := &Details{}
|
|
for _, e := range es {
|
|
if e.IsDir {
|
|
d.Dirs++
|
|
} else {
|
|
d.Files++
|
|
}
|
|
}
|
|
err = s.s3.read(func(ctx context.Context, tx pgx.Tx) error {
|
|
var n int64
|
|
err := tx.QueryRow(ctx, "SELECT coalesce(sum(size), 0), count(*) FROM s3_nodes WHERE NOT is_dir AND starts_with(path, $1) AND NOT starts_with(path, '.trash/')",
|
|
strings.TrimPrefix(c+"/", "/")).Scan(&d.Total, &n)
|
|
d.Count = int(n)
|
|
return err
|
|
})
|
|
return d, err
|
|
}
|