From ec3b7c9a011a2a8de5eace1cb0e58d2389d9a42a Mon Sep 17 00:00:00 2001 From: claude Date: Fri, 23 Oct 2026 17:23:14 +0200 Subject: [PATCH] =?UTF-8?q?Full=20web=20:=20clusters=20g=C3=A9r=C3=A9s=20d?= =?UTF-8?q?ans=20l'UI,=20primaire=20et=20cibles,=20comptes=20pve?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/pve-confsync/main.go | 500 +++++++++++------------------------- internal/adapter/access.go | 345 +++++++++++++++++++++++++ internal/adapter/adapter.go | 56 +++- internal/config/config.go | 141 +++++----- internal/model/model.go | 28 +- 5 files changed, 617 insertions(+), 453 deletions(-) create mode 100644 internal/adapter/access.go diff --git a/cmd/pve-confsync/main.go b/cmd/pve-confsync/main.go index 14eb719..3d33ef8 100644 --- a/cmd/pve-confsync/main.go +++ b/cmd/pve-confsync/main.go @@ -1,88 +1,39 @@ -// pve-confsync synchronizes part of the configuration of a reference Proxmox VE -// cluster (datacenter options, SDN VLAN/VXLAN zones, vnets, subnets) to client -// clusters through the PVE API: command line (check, export, diff, apply) and -// web UI (serve). +// pve-confsync replicates part of the configuration of a primary Proxmox VE 9.x +// cluster (datacenter options, accounts of the pve realm with their passwords, +// groups, roles, SDN VLAN/VXLAN zones, vnets, subnets) to target clusters. +// Everything is done from the web UI: this program only starts the server. +// Settings come from the environment, like Logstream (see README). package main import ( - "bufio" "context" - "encoding/json" - "errors" "flag" "fmt" - "io" + "io/fs" + "log" + "net/http" "os" "os/signal" - "sort" + "path/filepath" + "strconv" "strings" "syscall" + "time" - "gopkg.in/yaml.v3" - - "pve-confsync/internal/adapter" - "pve-confsync/internal/config" - "pve-confsync/internal/engine" - "pve-confsync/internal/model" - "pve-confsync/internal/pve" - "pve-confsync/internal/service" + "pve-confsync/internal/demo" + "pve-confsync/internal/server" + "pve-confsync/internal/store" + "pve-confsync/web" ) var version = "dev" -const usage = `pve-confsync %s - synchronisation de configuration Proxmox VE 9.x (référence -> clients) - -Usage : pve-confsync [-config fichier] [options] - -Commandes : - check vérifie l'accès, la version PVE (9.x) et les droits des tokens - export exporte l'état normalisé d'un cluster en YAML (défaut : la référence) - diff affiche les écarts référence -> clients (code retour 2 s'il y en a) - apply dry-run par défaut ; applique avec -confirm - serve interface web (clusters, diff, dry-run, synchronisation, journal) - version affiche la version - -Lancer "pve-confsync -h" pour les options d'une commande. -` - func main() { - global := flag.NewFlagSet("pve-confsync", flag.ExitOnError) - cfgFile := global.String("config", envOr("CONFSYNC_CONFIG", "confsync.yaml"), "fichier de configuration") - global.Usage = func() { fmt.Fprintf(os.Stderr, usage, version); global.PrintDefaults() } - _ = global.Parse(os.Args[1:]) - args := global.Args() - if len(args) == 0 { - global.Usage() - os.Exit(1) - } ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() - - var err error - code := 0 - switch args[0] { - case "version": - fmt.Println(version) - return - case "check": - err = cmdCheck(ctx, *cfgFile, args[1:]) - case "export": - err = cmdExport(ctx, *cfgFile, args[1:]) - case "diff": - code, err = cmdDiff(ctx, *cfgFile, args[1:]) - case "apply": - err = cmdApply(ctx, *cfgFile, args[1:]) - case "serve": - err = cmdServe(ctx, args[1:]) - default: - global.Usage() - os.Exit(1) + if err := serve(ctx, os.Args[1:]); err != nil { + log.Fatal(err) } - if err != nil { - fmt.Fprintln(os.Stderr, "erreur :", err) - os.Exit(1) - } - os.Exit(code) } func envOr(k, def string) string { @@ -92,318 +43,153 @@ func envOr(k, def string) string { return def } -// ---- check ---- +// serve starts the web UI. +func serve(ctx context.Context, args []string) error { + fl := flag.NewFlagSet("pve-confsync", flag.ExitOnError) + listen := fl.String("listen", envOr("LISTEN", ":8080"), "adresse d'écoute HTTP") + dataDir := fl.String("data", envOr("DATA_DIR", "/data"), "répertoire des données (base SQLite, clés)") + demoMode := fl.Bool("demo", envBool("CONFSYNC_DEMO", false), "mode démo : faux clusters PVE en mémoire, base temporaire") + webDir := fl.String("web", os.Getenv("WEB_DIR"), "servir les pages depuis ce répertoire au lieu de celles embarquées (développement)") + _ = fl.Parse(args) -func cmdCheck(ctx context.Context, cfgFile string, args []string) error { - fs := flag.NewFlagSet("check", flag.ExitOnError) - _ = fs.Parse(args) - cfg, err := config.Load(cfgFile) - if err != nil { - return err - } - var errs []error - check := func(cl config.Cluster, role string, wantWrite bool) { - c, err := cl.PVE() - if err != nil { - errs = append(errs, err) - return - } - vs, err := c.CheckVersion(ctx) - for _, v := range vs { - fmt.Printf("%-10s %-16s nœud %-12s PVE %s\n", role, cl.Name, v.Node, v.Version) - } - if err != nil { - errs = append(errs, err) - return - } - perms, err := c.Permissions(ctx) - if err != nil { - fmt.Printf(" avertissement : droits du token non lisibles (%v)\n", err) - return - } - for _, w := range service.PrivilegeWarnings(perms, wantWrite, cfg.HasKind(model.KindDatacenter)) { - fmt.Println(" avertissement :", w) - } - } - check(cfg.Reference, "référence", false) - for _, cl := range cfg.Clients { - check(cl.Cluster, "client", true) - } - return errors.Join(errs...) -} - -// ---- export ---- - -func cmdExport(ctx context.Context, cfgFile string, args []string) error { - fs := flag.NewFlagSet("export", flag.ExitOnError) - name := fs.String("cluster", "", "cluster à exporter (défaut : la référence)") - out := fs.String("o", "-", "fichier de sortie YAML") - _ = fs.Parse(args) - cfg, err := config.Load(cfgFile) - if err != nil { - return err - } - cl, mode := cfg.Reference, adapter.ReadRunning - if *name != "" && *name != cfg.Reference.Name { - c, ok := cfg.Client(*name) - if !ok { - return fmt.Errorf("cluster %q inconnu", *name) - } - cl, mode = c.Cluster, adapter.ReadCurrent - } - c, err := cl.PVE() - if err != nil { - return err - } - snap, err := engine.Read(ctx, c, cfg.Kinds, mode) - if err != nil { - return err - } - warn(snap.Warnings) - return writeYAML(*out, snap) -} - -func writeYAML(file string, v any) error { - var w io.Writer = os.Stdout - if file != "-" { - f, err := os.Create(file) + if *demoMode && os.Getenv("DATA_DIR") == "" && !flagSet(fl, "data") { + dir, err := os.MkdirTemp("", "confsync-demo-") if err != nil { return err } - defer f.Close() - w = f + defer os.RemoveAll(dir) + *dataDir = dir } - enc := yaml.NewEncoder(w) - enc.SetIndent(2) - if err := enc.Encode(v); err != nil { + if err := os.MkdirAll(*dataDir, 0o700); err != nil { return err } - return enc.Close() -} - -func loadSnapshot(file string) (model.Snapshot, error) { - var s model.Snapshot - raw, err := os.ReadFile(file) + key, keySrc, err := store.LoadKey(*dataDir) if err != nil { - return s, err - } - if err := yaml.Unmarshal(raw, &s); err != nil { - return s, fmt.Errorf("%s: %w", file, err) - } - for i := range s.Objects { - s.Objects[i].Fields = model.Normalize(s.Objects[i].Fields) - if a, err := adapter.ByKind(s.Objects[i].Kind); err == nil { - a.Canonicalize(s.Objects[i].Fields) - } else { - return s, fmt.Errorf("%s: %w", file, err) - } - } - return s, nil -} - -// ---- diff / apply ---- - -type target struct { - cl *config.Client - client *pve.Client - plan engine.Plan -} - -// prepare reads the source (live reference or YAML snapshot) and each selected -// client, and computes the plans. -func prepare(ctx context.Context, cfg *config.Config, source, only string) ([]target, error) { - var ref model.Snapshot - var err error - if source != "" { - ref, err = loadSnapshot(source) - } else { - var c *pve.Client - if c, err = cfg.Reference.PVE(); err == nil { - ref, err = engine.Read(ctx, c, cfg.Kinds, adapter.ReadRunning) - } + return err } + st, err := store.Open(filepath.Join(*dataDir, "confsync.db"), key) if err != nil { - return nil, err + return err } - warn(ref.Warnings) - var ts []target - for i := range cfg.Clients { - cl := &cfg.Clients[i] - if only != "" && cl.Name != only { - continue - } - c, err := cl.PVE() + defer st.Close() + log.Printf("database %s, encryption key from %s", filepath.Join(*dataDir, "confsync.db"), keySrc) + + if *demoMode { + clusters, err := demo.Start(ctx, st) if err != nil { - return nil, err + return fmt.Errorf("demo: %w", err) } - cur, err := engine.Read(ctx, c, cfg.Kinds, adapter.ReadCurrent) + defer clusters.Close() + log.Printf("DEMO mode: fake PVE clusters started; accounts admin, operateur, lecteur, password %q", demo.Password) + } else if err := bootstrap(ctx, st); err != nil { + return err + } else if n, _ := st.CountUsers(ctx); n == 0 { + log.Printf("empty database: open the web UI to create the first administrator") + } + + ttl, err := time.ParseDuration(envOr("SESSION_TTL", "12h")) + if err != nil { + return fmt.Errorf("SESSION_TTL: %w", err) + } + var static fs.FS = web.FS + if *webDir != "" { + static = os.DirFS(*webDir) + } + srv, err := server.New(server.Options{ + Store: st, + Static: static, + Version: version, + Demo: *demoMode, + Auth: server.AuthConfig{ + Local: envBool("AUTH_LOCAL", true), + Issuer: os.Getenv("OIDC_ISSUER"), + ClientID: os.Getenv("OIDC_CLIENT_ID"), + ClientSecret: secretEnv("OIDC_CLIENT_SECRET"), + RedirectURL: os.Getenv("OIDC_REDIRECT_URL"), + Scopes: os.Getenv("OIDC_SCOPES"), + GroupsClaim: os.Getenv("OIDC_GROUPS_CLAIM"), + AdminGroup: os.Getenv("OIDC_ADMIN_GROUP"), + OperatorGroup: os.Getenv("OIDC_OPERATOR_GROUP"), + DefaultRole: os.Getenv("OIDC_DEFAULT_ROLE"), + SessionTTL: ttl, + DataDir: *dataDir, + Demo: *demoMode, + }, + }) + if err != nil { + return err + } + hs := &http.Server{Addr: *listen, Handler: srv.Handler(), ReadHeaderTimeout: 10 * time.Second} + errc := make(chan error, 1) + go func() { errc <- hs.ListenAndServe() }() + log.Printf("pve-confsync %s: web UI on %s", version, *listen) + select { + case err := <-errc: + return err + case <-ctx.Done(): + } + log.Printf("stopping: waiting for the synchronizations in progress") + sctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + _ = hs.Shutdown(sctx) + srv.Wait() + return nil +} + +// bootstrap: the database starts empty, the first administrator is created +// from the login page. CONFSYNC_ADMIN_USER / CONFSYNC_ADMIN_PASSWORD (optional) +// create it at startup instead, for an unattended deployment. +func bootstrap(ctx context.Context, st *store.Store) error { + pass := secretEnv("CONFSYNC_ADMIN_PASSWORD") + if pass == "" { + return nil + } + user := envOr("CONFSYNC_ADMIN_USER", "admin") + err := st.CreateFirstAdmin(ctx, user, pass) + switch { + case err == nil: + log.Printf("first administrator %q created from the environment", user) + case err != store.ErrSetupDone: + return fmt.Errorf("first administrator: %w", err) + } + return nil +} + +// secretEnv reads NAME, or the file named by NAME_FILE (docker secret). +func secretEnv(name string) string { + if v := os.Getenv(name); v != "" { + return v + } + if f := os.Getenv(name + "_FILE"); f != "" { + b, err := os.ReadFile(f) if err != nil { - return nil, err + log.Printf("%s_FILE: %v", name, err) + return "" } - p := engine.MakePlan(cfg, cl, ref.Objects, cur.Objects) - p.Warnings = append(p.Warnings, cur.Warnings...) - ts = append(ts, target{cl: cl, client: c, plan: p}) + return strings.TrimSpace(string(b)) } - if only != "" && len(ts) == 0 { - return nil, fmt.Errorf("client %q inconnu", only) - } - return ts, nil + return "" } -func cmdDiff(ctx context.Context, cfgFile string, args []string) (int, error) { - fs := flag.NewFlagSet("diff", flag.ExitOnError) - only := fs.String("client", "", "limiter à un client") - source := fs.String("source", "", "lire la référence depuis un export YAML au lieu de l'API") - format := fs.String("format", "text", "text ou json") - _ = fs.Parse(args) - cfg, err := config.Load(cfgFile) +func envBool(name string, def bool) bool { + v := os.Getenv(name) + if v == "" { + return def + } + b, err := strconv.ParseBool(v) if err != nil { - return 1, err + log.Printf("%s=%q: true or false expected, using %t", name, v, def) + return def } - ts, err := prepare(ctx, cfg, *source, *only) - if err != nil { - return 1, err - } - code := 0 - var plans []engine.Plan - for _, t := range ts { - plans = append(plans, t.plan) - if !t.plan.Empty() || len(t.plan.Errors) > 0 { - code = 2 - } - } - if *format == "json" { - enc := json.NewEncoder(os.Stdout) - enc.SetIndent("", " ") - return code, enc.Encode(plans) - } - for _, p := range plans { - printPlan(os.Stdout, p) - } - return code, nil + return b } -func cmdApply(ctx context.Context, cfgFile string, args []string) error { - fs := flag.NewFlagSet("apply", flag.ExitOnError) - only := fs.String("client", "", "limiter à un client") - source := fs.String("source", "", "lire la référence depuis un export YAML au lieu de l'API") - confirm := fs.Bool("confirm", false, "appliquer réellement (sans cette option : dry-run)") - yes := fs.Bool("yes", false, "ne pas demander la confirmation interactive des changements SDN") - prune := fs.Bool("prune", false, "supprimer chez les clients les objets absents de la référence") - _ = fs.Parse(args) - cfg, err := config.Load(cfgFile) - if err != nil { - return err - } - cfg.Prune = cfg.Prune || *prune - ts, err := prepare(ctx, cfg, *source, *only) - if err != nil { - return err - } - var src model.Snapshot - if *source != "" { - src, _ = loadSnapshot(*source) - } - var errs []error - for _, t := range ts { - printPlan(os.Stdout, t.plan) - if t.plan.Empty() || len(t.plan.Errors) > 0 { - if len(t.plan.Errors) > 0 { - errs = append(errs, fmt.Errorf("client %s : plan en erreur, rien appliqué", t.cl.Name)) - } - continue +func flagSet(fs *flag.FlagSet, name string) bool { + found := false + fs.Visit(func(f *flag.Flag) { + if f.Name == name { + found = true } - if !*confirm { - fmt.Printf("[%s] dry-run : rien n'a été appliqué (ajouter -confirm)\n\n", t.cl.Name) - continue - } - if t.plan.HasSDN() && !*yes && !askSDN(t.cl.Name) { - fmt.Printf("[%s] changements SDN non confirmés, client ignoré\n\n", t.cl.Name) - continue - } - res := engine.Apply(ctx, t.client, t.plan) - for _, s := range res.Steps { - status := "ok" - if s.Error != "" { - status = "ERREUR " + s.Error - } - fmt.Printf("[%s] %s %s : %s\n", t.cl.Name, s.At.Format("15:04:05"), s.Action, status) - } - if res.Err != nil { - errs = append(errs, fmt.Errorf("client %s : %w", t.cl.Name, res.Err)) - continue - } - replan := func(actual []model.Object) engine.Plan { - ref := src.Objects - if *source == "" { - // the reference is re-read so that verification uses the same input - c, err := cfg.Reference.PVE() - var r model.Snapshot - if err == nil { - r, err = engine.Read(ctx, c, cfg.Kinds, adapter.ReadRunning) - } - if err != nil { - return engine.Plan{Errors: []string{err.Error()}} - } - ref = r.Objects - } - return engine.MakePlan(cfg, t.cl, ref, actual) - } - if err := engine.Verify(ctx, t.client, replan, cfg.Kinds); err != nil { - errs = append(errs, fmt.Errorf("client %s : vérification : %w", t.cl.Name, err)) - continue - } - fmt.Printf("[%s] appliqué et vérifié\n\n", t.cl.Name) - } - return errors.Join(errs...) -} - -// askSDN requires typing the client name: an SDN apply reloads the network of every node. -func askSDN(client string) bool { - fi, err := os.Stdin.Stat() - if err != nil || fi.Mode()&os.ModeCharDevice == 0 { - fmt.Fprintln(os.Stderr, "entrée non interactive : utiliser -yes pour confirmer les changements SDN") - return false - } - fmt.Printf("Appliquer les changements SDN sur %q (rechargement réseau de tous les nœuds) ?\nTaper le nom du client pour confirmer : ", client) - line, _ := bufio.NewReader(os.Stdin).ReadString('\n') - return strings.TrimSpace(line) == client -} - -func printPlan(w io.Writer, p engine.Plan) { - for _, m := range p.Warnings { - fmt.Fprintf(w, "[%s] avertissement : %s\n", p.Client, m) - } - for _, e := range p.Errors { - fmt.Fprintf(w, "[%s] ERREUR : %s\n", p.Client, e) - } - if p.Empty() { - fmt.Fprintf(w, "[%s] aucun écart\n\n", p.Client) - return - } - for _, ch := range p.Changes { - fmt.Fprintf(w, "[%s] %-6s %s/%s\n", p.Client, ch.Op, ch.Kind, ch.ID) - keys := make([]string, 0, len(ch.Set)) - for k := range ch.Set { - keys = append(keys, k) - } - sort.Strings(keys) - for _, k := range keys { - if old, ok := ch.Old[k]; ok { - fmt.Fprintf(w, " ~ %s : %s -> %s\n", k, model.Format(old), model.Format(ch.Set[k])) - } else { - fmt.Fprintf(w, " + %s = %s\n", k, model.Format(ch.Set[k])) - } - } - for _, k := range ch.Unset { - fmt.Fprintf(w, " - %s (était %s)\n", k, model.Format(ch.Old[k])) - } - } - fmt.Fprintln(w) -} - -func warn(ws []string) { - for _, w := range ws { - fmt.Fprintln(os.Stderr, "avertissement :", w) - } + }) + return found } diff --git a/internal/adapter/access.go b/internal/adapter/access.go new file mode 100644 index 0000000..7a86ab9 --- /dev/null +++ b/internal/adapter/access.go @@ -0,0 +1,345 @@ +package adapter + +import ( + "context" + "errors" + "fmt" + "net/url" + "sort" + "strings" + + "pve-confsync/internal/model" + "pve-confsync/internal/pve" +) + +// Access control objects of user.cfg, through /access (pve-access-control, +// src/PVE/API2/Group.pm, Role.pm, User.pm). Only the accounts of the pve realm +// (Proxmox VE authentication server) are replicated: pam accounts are Linux +// accounts of each node, the other realms (LDAP, AD, OpenID) are synchronized +// from their directory. + +// ---- groups: /access/groups ---- + +type group struct{} + +func (group) Kind() model.Kind { return model.KindGroup } +func (group) SDN() bool { return false } +func (group) Singleton() bool { return false } + +func (group) List(ctx context.Context, c *pve.Client, _ ReadMode, _ []model.Object) ([]model.Object, []string, error) { + var raw []map[string]any + if err := c.Get(ctx, "access/groups", nil, &raw); err != nil { + return nil, nil, err + } + var objs []model.Object + for _, r := range raw { + // members come from the "groups" field of the accounts + f := dropEmpty(normalizeAPI(r, "groupid", "users")) + objs = append(objs, model.Object{Kind: model.KindGroup, ID: pve.Str(r["groupid"]), Fields: f}) + } + return objs, nil, nil +} + +func (group) DefaultPolicy(string) Policy { return Sync } +func (group) Canonicalize(map[string]any) {} +func (group) Forbidden(f string) bool { return f == "users" } +func (group) Immutable(string) bool { return false } +func (group) Required(model.Object) []string { return nil } + +func (group) Create(ctx context.Context, c *pve.Client, o model.Object, _ string) error { + f := form(o.Fields, nil, "") + f.Set("groupid", o.ID) + return c.Post(ctx, "access/groups", f, nil) +} + +func (group) Update(ctx context.Context, c *pve.Client, t model.Object, set map[string]any, unset []string, _ string) error { + return c.Put(ctx, "access/groups/"+url.PathEscape(t.ID), form(blank(set, unset), nil, ""), nil) +} + +func (group) Delete(ctx context.Context, c *pve.Client, o model.Object, _ string) error { + return c.Delete(ctx, "access/groups/"+url.PathEscape(o.ID), nil, nil) +} + +// ---- roles: /access/roles (custom roles only) ---- + +type role struct{} + +func (role) Kind() model.Kind { return model.KindRole } +func (role) SDN() bool { return false } +func (role) Singleton() bool { return false } + +func (role) List(ctx context.Context, c *pve.Client, _ ReadMode, _ []model.Object) ([]model.Object, []string, error) { + var raw []map[string]any + if err := c.Get(ctx, "access/roles", nil, &raw); err != nil { + return nil, nil, err + } + var objs []model.Object + for _, r := range raw { + if pve.Str(r["special"]) == "1" { // built-in roles (Administrator, PVEAuditor…) + continue + } + f := normalizeAPI(r, "roleid", "special") + role{}.Canonicalize(f) + objs = append(objs, model.Object{Kind: model.KindRole, ID: pve.Str(r["roleid"]), Fields: f}) + } + return objs, nil, nil +} + +func (role) DefaultPolicy(string) Policy { return Sync } +func (role) Canonicalize(f map[string]any) { sortList(f, "privs") } +func (role) Forbidden(string) bool { return false } +func (role) Immutable(string) bool { return false } +func (role) Required(model.Object) []string { return nil } + +func (role) Create(ctx context.Context, c *pve.Client, o model.Object, _ string) error { + f := form(o.Fields, nil, "") + f.Set("roleid", o.ID) + return c.Post(ctx, "access/roles", f, nil) +} + +// Update replaces the privileges (no "append"). +func (role) Update(ctx context.Context, c *pve.Client, t model.Object, set map[string]any, unset []string, _ string) error { + return c.Put(ctx, "access/roles/"+url.PathEscape(t.ID), form(blank(set, unset), nil, ""), nil) +} + +func (role) Delete(ctx context.Context, c *pve.Client, o model.Object, _ string) error { + return c.Delete(ctx, "access/roles/"+url.PathEscape(o.ID), nil, nil) +} + +// ---- accounts of the pve realm: /access/users ---- + +// PVERealm is the realm whose accounts are replicated (type "pve"). +const PVERealm = "pve" + +type user struct{} + +func (user) Kind() model.Kind { return model.KindUser } +func (user) SDN() bool { return false } +func (user) Singleton() bool { return false } + +func (user) List(ctx context.Context, c *pve.Client, _ ReadMode, _ []model.Object) ([]model.Object, []string, error) { + var raw []map[string]any + if err := c.Get(ctx, "access/users", url.Values{"full": {"1"}}, &raw); err != nil { + return nil, nil, err + } + var objs []model.Object + for _, r := range raw { + id := pve.Str(r["userid"]) + if !strings.HasSuffix(id, "@"+PVERealm) { + continue + } + // tokens hold no secret here, but are specific to each cluster (their secret is) + f := dropEmpty(normalizeAPI(r, "userid", "tokens", "realm-type", "totp-locked", "tfa-locked-until")) + user{}.Canonicalize(f) + objs = append(objs, model.Object{Kind: model.KindUser, ID: id, Fields: f}) + } + return objs, nil, nil +} + +func (user) DefaultPolicy(string) Policy { return Sync } + +// Canonicalize: "groups" is a comma separated list; expire 0 means no expiration (absent). +func (user) Canonicalize(f map[string]any) { + if l, ok := f["groups"].([]string); ok { + f["groups"] = strings.Join(l, ",") + } + sortList(f, "groups") + if f["expire"] == "0" { + delete(f, "expire") + } +} + +// Forbidden: the password is never set through the API (only its hash is copied, see password). +func (user) Forbidden(f string) bool { return f == "password" || f == "tokens" } +func (user) Immutable(string) bool { return false } +func (user) Required(model.Object) []string { return nil } + +func (user) Parents(o model.Object) []string { + g, _ := o.Fields["groups"].(string) + var out []string + for _, x := range strings.Split(g, ",") { + if x = strings.TrimSpace(x); x != "" { + out = append(out, string(model.KindGroup)+"/"+x) + } + } + return out +} + +func (user) Create(ctx context.Context, c *pve.Client, o model.Object, _ string) error { + f := form(o.Fields, nil, "") + f.Set("userid", o.ID) + return c.Post(ctx, "access/users", f, nil) +} + +// Update: PUT /access/users/{userid} has no "delete" parameter; a removed field +// is written empty (expire 0, groups "" removes every membership). +func (user) Update(ctx context.Context, c *pve.Client, t model.Object, set map[string]any, unset []string, _ string) error { + f := blank(set, unset) + if _, ok := f["expire"]; ok && f["expire"] == "" { + f["expire"] = "0" + } + if _, ok := f["enable"]; ok && f["enable"] == "" { + f["enable"] = "1" + } + return c.Put(ctx, "access/users/"+url.PathEscape(t.ID), form(f, nil, ""), nil) +} + +func (user) Delete(ctx context.Context, c *pve.Client, o model.Object, _ string) error { + if o.ID == "root@pam" { // never, whatever the selection + return errors.New("root@pam cannot be deleted") + } + return c.Delete(ctx, "access/users/"+url.PathEscape(o.ID), nil, nil) +} + +// ---- password hashes: /etc/pve/priv/shadow.cfg over SSH ---- +// The API only takes clear passwords (PUT /access/password); the hashes are +// copied as they are, so the accounts keep their password on every cluster. +// Format (src/PVE/Auth/PVE.pm): one "user:hash:" line per account, the user +// without its realm. Writes go through the pmxcfs lock of the file, like PVE +// (PVE::Auth::PVE::lock_shadow_config, cfs_read_file / cfs_write_file). + +type password struct{} + +const shadowRead = "cat /etc/pve/priv/shadow.cfg 2>/dev/null || true" + +// shadowWrite reads "set\tuser\thash" / "del\tuser" lines on stdin: the hashes +// never appear on a command line. +const shadowWrite = `perl -e ' +use strict; use warnings; +use PVE::Cluster; +use PVE::Auth::PVE; +my @ops; +while (my $l = ) { chomp $l; push @ops, [split(/\t/, $l, 3)]; } +PVE::Auth::PVE::lock_shadow_config(sub { + my $cfg = PVE::Cluster::cfs_read_file("priv/shadow.cfg"); + for my $o (@ops) { + my ($op, $u, $h) = @$o; + die "invalid user\n" if !defined($u) || $u !~ m/^[^\s:]+$/; + if ($op eq "set") { + die "invalid hash\n" if !defined($h) || $h !~ m/^[^\s:]+$/; + $cfg->{users}->{$u}->{shadow} = $h; + } elsif ($op eq "del") { + delete $cfg->{users}->{$u}; + } else { + die "invalid operation\n"; + } + } + PVE::Cluster::cfs_write_file("priv/shadow.cfg", $cfg); +}, "confsync"); +print "ok\n"; +' # confsync-shadow-write` + +func (password) Kind() model.Kind { return model.KindPassword } +func (password) SDN() bool { return false } +func (password) Singleton() bool { return false } + +// List reads the hashes of the known accounts of the pve realm (the accounts +// are read before, see KindDeps): orphan lines are left aside. +func (password) List(ctx context.Context, c *pve.Client, _ ReadMode, known []model.Object) ([]model.Object, []string, error) { + sh, err := c.Shell() + if err != nil { + return nil, nil, err + } + out, err := sh.Run(ctx, shadowRead, nil) + if err != nil { + return nil, nil, err + } + users := ids(known, model.KindUser) + var objs []model.Object + for _, line := range strings.Split(string(out), "\n") { + line = strings.TrimSpace(line) + p := strings.Split(line, ":") + if len(p) != 3 || p[0] == "" || p[1] == "" { + continue + } + id := p[0] + "@" + PVERealm + if !users[id] { + continue + } + objs = append(objs, model.Object{Kind: model.KindPassword, ID: id, Fields: map[string]any{"hash": p[1]}}) + } + sort.Slice(objs, func(i, j int) bool { return objs[i].ID < objs[j].ID }) + return objs, nil, nil +} + +func (password) DefaultPolicy(string) Policy { return Sync } +func (password) Canonicalize(map[string]any) {} +func (password) Forbidden(f string) bool { return f != "hash" } +func (password) Immutable(string) bool { return false } +func (password) Required(model.Object) []string { return []string{"hash"} } +func (password) Sensitive(f string) bool { return f == "hash" } + +func (password) Parents(o model.Object) []string { + return []string{string(model.KindUser) + "/" + o.ID} +} + +func shadowUser(id string) (string, error) { + u, ok := strings.CutSuffix(id, "@"+PVERealm) + if !ok || u == "" { + return "", fmt.Errorf("%s: not an account of the %s realm", id, PVERealm) + } + return u, nil +} + +func (password) write(ctx context.Context, c *pve.Client, line string) error { + sh, err := c.Shell() + if err != nil { + return err + } + out, err := sh.Run(ctx, shadowWrite, []byte(line+"\n")) + if err != nil { + return err + } + if strings.TrimSpace(string(out)) != "ok" { + return errors.New("shadow.cfg: unexpected answer") + } + return nil +} + +func (p password) Create(ctx context.Context, c *pve.Client, o model.Object, _ string) error { + return p.Update(ctx, c, o, o.Fields, nil, "") +} + +func (p password) Update(ctx context.Context, c *pve.Client, t model.Object, set map[string]any, _ []string, _ string) error { + u, err := shadowUser(t.ID) + if err != nil { + return err + } + h, _ := set["hash"].(string) + if h == "" { + return errors.New("empty password hash") + } + return p.write(ctx, c, "set\t"+u+"\t"+h) +} + +func (p password) Delete(ctx context.Context, c *pve.Client, o model.Object, _ string) error { + u, err := shadowUser(o.ID) + if err != nil { + return err + } + return p.write(ctx, c, "del\t"+u) +} + +// ---- helpers ---- + +// dropEmpty removes empty values: PVE returns some optional fields empty. +func dropEmpty(f map[string]any) map[string]any { + for k, v := range f { + if v == "" { + delete(f, k) + } + } + return f +} + +// blank merges the fields to remove as empty values (APIs without "delete"). +func blank(set map[string]any, unset []string) map[string]any { + out := make(map[string]any, len(set)+len(unset)) + for k, v := range set { + out[k] = v + } + for _, k := range unset { + out[k] = "" + } + return out +} diff --git a/internal/adapter/adapter.go b/internal/adapter/adapter.go index b418641..60ec4ff 100644 --- a/internal/adapter/adapter.go +++ b/internal/adapter/adapter.go @@ -5,6 +5,8 @@ package adapter import ( "context" + "crypto/sha256" + "encoding/hex" "fmt" "net/url" "sort" @@ -60,7 +62,59 @@ type Adapter interface { // All returns the adapters in dependency order. func All() []Adapter { - return []Adapter{datacenter{}, zone{}, vnet{}, subnet{}} + return []Adapter{datacenter{}, group{}, role{}, user{}, password{}, zone{}, vnet{}, subnet{}} +} + +// KindDeps: replicating a kind needs these kinds too (a password needs its +// account on the target, an account its groups). +var KindDeps = map[model.Kind][]model.Kind{ + model.KindPassword: {model.KindUser}, + model.KindUser: {model.KindGroup}, +} + +// Sensitive is implemented by adapters whose fields must never be shown nor +// journaled (password hashes): they are replaced by Redact. +type Sensitive interface { + Sensitive(field string) bool +} + +// Parented is implemented by adapters whose objects need other objects on the +// target ("kind/id" keys): an account needs its groups, a password its account. +type Parented interface { + Parents(o model.Object) []string +} + +// IsSensitive tells whether a field of a kind is secret. +func IsSensitive(k model.Kind, field string) bool { + a, err := ByKind(k) + if err != nil { + return false + } + s, ok := a.(Sensitive) + return ok && s.Sensitive(field) +} + +// Redact replaces a secret value by a short digest: two different hashes stay +// distinguishable in a diff, the value itself is never shown. +func Redact(v any) any { + s, ok := v.(string) + if !ok || s == "" { + return v + } + sum := sha256.Sum256([]byte(s)) + return "" +} + +// Parents returns the keys of the objects o needs on the target. +func Parents(o model.Object) []string { + a, err := ByKind(o.Kind) + if err != nil { + return nil + } + if p, ok := a.(Parented); ok { + return p.Parents(o) + } + return nil } // ByKind returns the adapter of a kind. diff --git a/internal/config/config.go b/internal/config/config.go index a92b3c2..76c5267 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,81 +1,60 @@ -// Package config loads the YAML configuration of the CLI (phase 0): clusters, -// field policies, per client mappings and object selection. -// Token secrets are never written in this file: they are read from an -// environment variable or a file (docker secret), see README. +// Package config is the synchronization configuration given to the engine: +// the primary cluster, the targets with their mappings and policies, the +// replicated kinds and the object selection. It is built from the database of +// the web UI (internal/store); credentials are decrypted only in memory. package config import ( "errors" "fmt" - "os" + "net/url" "path" - "strings" "time" - "gopkg.in/yaml.v3" - "pve-confsync/internal/adapter" "pve-confsync/internal/model" "pve-confsync/internal/pve" + "pve-confsync/internal/sshx" ) -// Cluster is the connection to one PVE cluster. +// Cluster is the connection to one PVE cluster: the API with the root@pam +// credentials and SSH as root on one node for what the API does not cover. type Cluster struct { - Name string `yaml:"name"` - URL string `yaml:"url"` - TokenID string `yaml:"token_id"` - TokenSecretEnv string `yaml:"token_secret_env,omitempty"` - TokenSecretFile string `yaml:"token_secret_file,omitempty"` - CAFile string `yaml:"ca_file,omitempty"` - Fingerprint string `yaml:"fingerprint,omitempty"` - Insecure bool `yaml:"insecure,omitempty"` - Timeout string `yaml:"timeout,omitempty"` - - // Set by the web UI from its database, never read from nor written to YAML. - Secret string `yaml:"-"` // token secret, decrypted - CAPEM string `yaml:"-"` // CA bundle, PEM + Name string + URL string + User string // API user, root@pam + Password string // decrypted, never logged + Fingerprint string // pinned TLS certificate (SHA-256) + CAPEM string // CA bundle, PEM + Insecure bool + Timeout string + SSHHost string // empty: the host of URL + SSHPort int + SSHHostKey string // pinned host key (authorized_keys format) } // Client is a target cluster with its mapping (kind -> id -> field -> value) // and its own policy overrides. type Client struct { - Cluster `yaml:",inline"` - Mapping map[model.Kind]map[string]map[string]any `yaml:"mapping,omitempty"` - Policies map[model.Kind]map[string]adapter.Policy `yaml:"policies,omitempty"` + Cluster + Mapping map[model.Kind]map[string]map[string]any + Policies map[model.Kind]map[string]adapter.Policy } // Selection filters the objects of a kind with glob patterns on their id. type Selection struct { - Include []string `yaml:"include,omitempty"` - Exclude []string `yaml:"exclude,omitempty"` + Include []string `json:"include,omitempty"` + Exclude []string `json:"exclude,omitempty"` } -// Config is the whole file. +// Config is the whole synchronization: Reference is the primary cluster. type Config struct { - Reference Cluster `yaml:"reference"` - Clients []Client `yaml:"clients"` - Kinds []model.Kind `yaml:"kinds,omitempty"` - Objects map[model.Kind]Selection `yaml:"objects,omitempty"` - Policies map[model.Kind]map[string]adapter.Policy `yaml:"policies,omitempty"` - Prune bool `yaml:"prune,omitempty"` -} - -// Load reads and validates a configuration file. -func Load(file string) (*Config, error) { - raw, err := os.ReadFile(file) - if err != nil { - return nil, err - } - var c Config - dec := yaml.NewDecoder(strings.NewReader(string(raw))) - dec.KnownFields(true) // typos must fail, not be silently ignored - if err := dec.Decode(&c); err != nil { - return nil, fmt.Errorf("%s: %w", file, err) - } - if len(c.Kinds) == 0 { - c.Kinds = append(c.Kinds, model.Kinds...) - } - return &c, c.validate() + Reference Cluster + Clients []Client + Kinds []model.Kind + Objects map[model.Kind]Selection + Policies map[model.Kind]map[string]adapter.Policy + Prune bool } // Validate checks a configuration built elsewhere (web UI database, YAML import). @@ -93,14 +72,16 @@ func (c *Config) validate() error { errs = append(errs, fmt.Errorf("cluster %s: duplicate name", cl.Name)) } seen[cl.Name] = true - if cl.Secret == "" && cl.TokenSecretEnv == "" && cl.TokenSecretFile == "" { - errs = append(errs, fmt.Errorf("cluster %s: token_secret_env or token_secret_file is required", cl.Name)) - } } for _, k := range c.Kinds { if _, err := adapter.ByKind(k); err != nil { errs = append(errs, err) } + for _, d := range adapter.KindDeps[k] { + if !c.HasKind(d) { + errs = append(errs, fmt.Errorf("%s needs %s", k, d)) + } + } } check := func(where string, p map[model.Kind]map[string]adapter.Policy) { for k, fields := range p { @@ -151,45 +132,39 @@ func (c *Config) Client(name string) (*Client, bool) { return nil, false } -// PVE builds the API client of a cluster, resolving its token secret. +// PVE builds the API client of a cluster, with its SSH access when the host +// key is recorded. func (cl Cluster) PVE() (*pve.Client, error) { - secret, err := cl.secret() - if err != nil { - return nil, err - } var timeout time.Duration + var err error if cl.Timeout != "" { if timeout, err = time.ParseDuration(cl.Timeout); err != nil { return nil, fmt.Errorf("cluster %s: timeout: %w", cl.Name, err) } } - return pve.New(pve.Options{ - Name: cl.Name, BaseURL: cl.URL, TokenID: cl.TokenID, TokenSecret: secret, - CAFile: cl.CAFile, CAPEM: []byte(cl.CAPEM), Fingerprint: cl.Fingerprint, Insecure: cl.Insecure, Timeout: timeout, - }) + o := pve.Options{ + Name: cl.Name, BaseURL: cl.URL, User: cl.User, Password: cl.Password, + CAPEM: []byte(cl.CAPEM), Fingerprint: cl.Fingerprint, Insecure: cl.Insecure, Timeout: timeout, + } + if cl.SSHHostKey != "" { + sh, err := sshx.New(cl.SSHOptions()) + if err != nil { + return nil, fmt.Errorf("cluster %s: %w", cl.Name, err) + } + o.Shell = sh + } + return pve.New(o) } -// ResolveSecret reads the token secret from the environment variable or file of -// the YAML configuration (used when a YAML file is imported in the web UI). -func (cl Cluster) ResolveSecret() (string, error) { return cl.secret() } - -func (cl Cluster) secret() (string, error) { - if cl.Secret != "" { - return cl.Secret, nil - } - if cl.TokenSecretEnv != "" { - if v := os.Getenv(cl.TokenSecretEnv); v != "" { - return v, nil - } - if cl.TokenSecretFile == "" { - return "", fmt.Errorf("cluster %s: environment variable %s is empty", cl.Name, cl.TokenSecretEnv) +// SSHOptions is the SSH access of the cluster: root on SSHHost, or on the host of the API URL. +func (cl Cluster) SSHOptions() sshx.Options { + host := cl.SSHHost + if host == "" { + if u, err := url.Parse(cl.URL); err == nil { + host = u.Hostname() } } - b, err := os.ReadFile(cl.TokenSecretFile) - if err != nil { - return "", fmt.Errorf("cluster %s: token secret: %w", cl.Name, err) - } - return strings.TrimSpace(string(b)), nil + return sshx.Options{Host: host, Port: cl.SSHPort, User: "root", Password: cl.Password, HostKey: cl.SSHHostKey} } // Policy resolves the policy of a field for a client: client policy, then diff --git a/internal/model/model.go b/internal/model/model.go index 5df00d4..ee876c3 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -1,6 +1,6 @@ // Package model holds the generic, normalized representation of synchronized objects. // Field values are canonical strings or sorted string lists, so that two clusters -// can be compared field by field and the result serialized to YAML without loss. +// can be compared field by field. package model import ( @@ -15,32 +15,36 @@ type Kind string const ( KindDatacenter Kind = "datacenter-options" + KindGroup Kind = "pve-group" // groups of user.cfg + KindRole Kind = "pve-role" // custom roles of user.cfg + KindUser Kind = "pve-user" // accounts of the pve realm + KindPassword Kind = "pve-password" // their password hashes (SSH, priv/shadow.cfg) KindZone Kind = "sdn-zone" KindVNet Kind = "sdn-vnet" KindSubnet Kind = "sdn-subnet" ) // Kinds lists the kinds in dependency order (creation order). -var Kinds = []Kind{KindDatacenter, KindZone, KindVNet, KindSubnet} +var Kinds = []Kind{KindDatacenter, KindGroup, KindRole, KindUser, KindPassword, KindZone, KindVNet, KindSubnet} // Object is one configuration object of a cluster. // Field values are either string or []string (see Normalize). type Object struct { - Kind Kind `yaml:"kind" json:"kind"` - ID string `yaml:"id" json:"id"` - Fields map[string]any `yaml:"fields" json:"fields"` + Kind Kind `json:"kind"` + ID string `json:"id"` + Fields map[string]any `json:"fields"` } // Key is the unique identity of an object inside a cluster. func (o Object) Key() string { return string(o.Kind) + "/" + o.ID } -// Snapshot is the exported state of one cluster (YAML, versionable in Git). +// Snapshot is the state read from one cluster. type Snapshot struct { - Cluster string `yaml:"cluster"` - PVEVersion string `yaml:"pve_version,omitempty"` - ExportedAt time.Time `yaml:"exported_at"` - Objects []Object `yaml:"objects"` - Warnings []string `yaml:"warnings,omitempty"` + Cluster string `json:"cluster"` + PVEVersion string `json:"pveVersion,omitempty"` + ExportedAt time.Time `json:"readAt"` + Objects []Object `json:"objects"` + Warnings []string `json:"warnings,omitempty"` } // Sort orders objects by kind (dependency order) then by id. @@ -57,7 +61,7 @@ func Sort(objs []Object) { }) } -// Normalize turns a YAML/JSON decoded field map into canonical values: +// Normalize turns a JSON decoded field map into canonical values: // scalars become strings, lists become sorted []string. func Normalize(fields map[string]any) map[string]any { out := make(map[string]any, len(fields))