Réplication de nœuds : réseau et DNS par l'API, multipath, chrony, sysctl par SSH

Deuxième mode de réplication, séparé de la réplication cluster vers cluster
(sélecteur « Clusters (API) | Nœuds (fichiers) » en haut de page) : un profil
réplique la configuration système d'un nœud maître vers des nœuds cibles.

- réseau par /nodes/{node}/network (interfaces.new puis rechargement), adresses
  et passerelles propres à chaque nœud, mapping des noms d'interfaces par cible ;
  DNS par /nodes/{node}/dns ; multipath.conf, wwids (ajout seul), chrony,
  sysctl.d, interfaces.d (hors sdn) par SSH
- découverte des nœuds (/cluster/status), accès SSH par nœud (mot de passe du
  cluster ou propre), clé d'hôte épinglée, contrôle du nom du nœud
- dry-run avec diff des fichiers ; apply (admin) avec contrôle SHA-256,
  sauvegarde et restore.sh sur le nœud, écriture atomique, rechargements,
  restauration si un rechargement échoue, timer systemd de restauration si le
  nœud est injoignable après le rechargement du réseau, confirmation tapée
- vue Nœuds et profils, journal séparé par mode, démo avec nœuds simulés

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

internal/demo/demo.go

@@ -8,9 +8,11 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
+	"strings"
 
 	"pve-confsync/internal/adapter"
 	"pve-confsync/internal/model"
+	"pve-confsync/internal/nodesync"
 	"pve-confsync/internal/pve"
 	"pve-confsync/internal/pve/pvetest"
 	"pve-confsync/internal/store"
@@ -104,6 +106,45 @@ func Start(ctx context.Context, st *store.Store) (*Clusters, error) {
 	b.Shadow["alice"] = ref.Shadow["alice"]
 	b.Shadow["bob"] = ref.Shadow["bob"]
 
+	// nodes of the node replication: pve1 (master) and pve2 in pve-rennes,
+	// pve-a1 in site-a with other NIC names
+	n1, n2, a1 := ref.AddNode("pve1"), ref.AddNode("pve2"), a.AddNode("pve-a1")
+	n1.Net = map[string]map[string]any{
+		"eno1":     {"type": "eth", "mtu": "9000"},
+		"eno2":     {"type": "eth", "mtu": "9000"},
+		"bond0":    {"type": "bond", "slaves": "eno1 eno2", "bond_mode": "802.3ad", "bond_xmit_hash_policy": "layer3+4", "mtu": "9000"},
+		"vmbr0":    {"type": "bridge", "bridge_ports": "bond0", "bridge_vlan_aware": "1", "cidr": "10.0.0.11/24", "gateway": "10.0.0.1", "autostart": "1"},
+		"bond0.50": {"type": "vlan", "mtu": "9000"},
+		"vmbr1":    {"type": "bridge", "bridge_ports": "bond0.50", "cidr": "10.5.0.11/24", "comments": "stockage iSCSI", "autostart": "1"},
+	}
+	n2.Net = map[string]map[string]any{
+		"eno1":  {"type": "eth", "mtu": "9000"},
+		"eno2":  {"type": "eth", "mtu": "9000"},
+		"bond0": {"type": "bond", "slaves": "eno1 eno2", "bond_mode": "active-backup", "mtu": "9000"},
+		"vmbr0": {"type": "bridge", "bridge_ports": "bond0", "cidr": "10.0.0.12/24", "gateway": "10.0.0.1", "autostart": "1"},
+	}
+	a1.Net = map[string]map[string]any{
+		"enp65s0f0": {"type": "eth"},
+		"enp65s0f1": {"type": "eth"},
+		"bond0":     {"type": "bond", "slaves": "enp65s0f0 enp65s0f1", "bond_mode": "802.3ad", "bond_xmit_hash_policy": "layer3+4"},
+		"vmbr0":     {"type": "bridge", "bridge_ports": "bond0", "bridge_vlan_aware": "1", "cidr": "192.168.10.21/24", "gateway": "192.168.10.1", "autostart": "1"},
+	}
+	n1.DNS = map[string]string{"search": "rennes.lab", "dns1": "10.0.0.53", "dns2": "10.0.0.54"}
+	n2.DNS = map[string]string{"search": "rennes.lab", "dns1": "10.0.0.53"}
+	a1.DNS = map[string]string{"search": "rennes.lab", "dns1": "10.0.0.53", "dns2": "10.0.0.54"}
+	mp := "defaults {\n    user_friendly_names yes\n    find_multipaths no\n}\nblacklist {\n    devnode \"^sda\"\n}\ndevices {\n    device {\n        vendor \"NETAPP\"\n        product \"LUN.*\"\n        path_grouping_policy group_by_prio\n        no_path_retry queue\n    }\n}\n"
+	n1.Files["/etc/multipath.conf"] = pvetest.FakeFile{Content: mp}
+	n2.Files["/etc/multipath.conf"] = pvetest.FakeFile{Content: strings.Replace(mp, "no_path_retry queue", "no_path_retry 12", 1)}
+	n1.Files["/etc/multipath/wwids"] = pvetest.FakeFile{Content: "# Multipath wwids, Version : 1.0\n/3600a098038303053453f463045747a41/\n/3600a098038303053453f463045747a42/\n"}
+	n2.Files["/etc/multipath/wwids"] = pvetest.FakeFile{Content: "# Multipath wwids, Version : 1.0\n/3600a098038303053453f463045747a41/\n"}
+	n1.Files["/etc/chrony/chrony.conf"] = pvetest.FakeFile{Content: "server ntp1.rennes.lab iburst\nserver ntp2.rennes.lab iburst\ndriftfile /var/lib/chrony/chrony.drift\nmakestep 1.0 3\n"}
+	n2.Files["/etc/chrony/chrony.conf"] = pvetest.FakeFile{Content: "pool 2.debian.pool.ntp.org iburst\ndriftfile /var/lib/chrony/chrony.drift\nmakestep 1.0 3\n"}
+	a1.Files["/etc/chrony/chrony.conf"] = n1.Files["/etc/chrony/chrony.conf"]
+	n1.Files["/etc/sysctl.d/90-network.conf"] = pvetest.FakeFile{Content: "net.core.rmem_max = 16777216\nnet.core.wmem_max = 16777216\n"}
+	a1.Files["/etc/sysctl.d/90-network.conf"] = n1.Files["/etc/sysctl.d/90-network.conf"]
+	a1.Files["/etc/sysctl.d/99-local.conf"] = pvetest.FakeFile{Content: "vm.swappiness = 10\n"}
+	a1.Files["/etc/multipath.conf"] = n1.Files["/etc/multipath.conf"]
+
 	// legacy: PVE 8, refused by the version check
 	old := pvetest.New("8.4.1")
 
@@ -140,6 +181,24 @@ func Start(ctx context.Context, st *store.Store) (*Clusters, error) {
 		st.SaveSettings(ctx, store.Settings{Kinds: model.Kinds,
 			Policies: map[model.Kind]map[string]adapter.Policy{model.KindDatacenter: {"keyboard": adapter.Sync}}}),
 	}
+	node := func(cluster string, n *pvetest.Node) error {
+		host, port := n.SSHAddr()
+		if err := st.SaveNode(ctx, cluster, n.Name, host, port, "", false); err != nil {
+			return err
+		}
+		return st.PinNode(ctx, cluster, n.Name, n.SSHHostKey(), nil)
+	}
+	steps = append(steps,
+		st.SetNodes(ctx, "pve-rennes", []store.Discovered{{Name: "pve1", IP: "127.0.0.1"}, {Name: "pve2", IP: "127.0.0.1"}}),
+		st.SetNodes(ctx, "site-a", []store.Discovered{{Name: "pve-a1", IP: "127.0.0.1"}}),
+		node("pve-rennes", n1), node("pve-rennes", n2), node("site-a", a1),
+		st.SaveProfile(ctx, "", nodesync.Profile{Name: "réseau et stockage", Master: "pve-rennes/pve1",
+			Targets: []string{"pve-rennes/pve2", "site-a/pve-a1"},
+			Items:   []string{"network", "dns", "multipath", "wwids", "chrony", "sysctl"},
+			Mapping: map[string]nodesync.TargetMap{
+				"pve-rennes/pve2": {Values: map[string]map[string]string{"network/vmbr1": {"cidr": "10.5.0.12/24"}}},
+				"site-a/pve-a1":   {NICs: map[string]string{"eno1": "enp65s0f0", "eno2": "enp65s0f1"}},
+			}}))
 	for _, u := range []struct{ name, role string }{{"admin", store.RoleAdmin}, {"operateur", store.RoleOperator}, {"lecteur", store.RoleViewer}} {
 		steps = append(steps, st.SaveUser(ctx, u.name, u.role, Password))
 	}

internal/nodesync/analyze.go

@@ -0,0 +1,341 @@
+package nodesync
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sort"
+	"sync"
+	"time"
+
+	"pve-confsync/internal/model"
+)
+
+// NodeInfo is what was read from one node.
+type NodeInfo struct {
+	Key      string    `json:"name"` // "cluster/node", "name" like the cluster cards
+	Cluster  string    `json:"cluster"`
+	Node     string    `json:"node"`
+	Version  string    `json:"version,omitempty"`
+	Error    string    `json:"error,omitempty"`
+	Warnings []string  `json:"warnings,omitempty"`
+	ReadAt   time.Time `json:"readAt"`
+	Millis   int64     `json:"ms"`
+}
+
+// Field compares one field of an interface or of the DNS servers.
+type Field struct {
+	Name    string `json:"name"`
+	Policy  Policy `json:"policy"`
+	Ref     any    `json:"ref,omitempty"`     // master
+	Mapping any    `json:"mapping,omitempty"` // value of the target mapping
+	Current any    `json:"current,omitempty"` // target
+	Target  any    `json:"target,omitempty"`  // after the replication
+	Change  string `json:"change,omitempty"`  // add, set, unset or empty
+}
+
+// Row statuses (same as the cluster rows).
+const (
+	StatusSame   = "same"
+	StatusCreate = "create"
+	StatusUpdate = "update"
+	StatusDelete = "delete"
+	StatusExtra  = "extra" // only on the target, kept
+	StatusError  = "error"
+)
+
+// Row is one object of a target: an interface, the DNS servers or a file.
+type Row struct {
+	Kind   string   `json:"kind"` // item id
+	ID     string   `json:"id"`
+	Key    string   `json:"key"`
+	Status string   `json:"status"`
+	Fields []Field  `json:"fields,omitempty"`
+	Diff   []Line   `json:"diff,omitempty"`
+	Errors []string `json:"errors,omitempty"`
+}
+
+// Target is the state of one target node.
+type Target struct {
+	NodeInfo
+	Plan   Plan           `json:"plan"`
+	Digest string         `json:"digest"`
+	Rows   []Row          `json:"rows"`
+	Counts map[string]int `json:"counts"`
+}
+
+// State is a full comparison of the master with its targets.
+type State struct {
+	At      time.Time `json:"at"`
+	Profile string    `json:"profile"`
+	Items   []string  `json:"items"`
+	Prune   bool      `json:"prune"`
+	Master  NodeInfo  `json:"reference"` // "reference" like the cluster state
+	Targets []Target  `json:"clients"`
+
+	prof    *Profile
+	master  Snapshot
+	targets map[string]Snapshot
+	eps     map[string]Endpoint
+}
+
+// Analyze reads the master and the targets (in parallel) and compares them.
+// only limits the targets (empty: all). A node that cannot be read is
+// reported in its Error field, it does not stop the others.
+func Analyze(ctx context.Context, prof *Profile, eps map[string]Endpoint, only []string) *State {
+	st := &State{At: time.Now().UTC(), Profile: prof.Name, Items: prof.Items, Prune: prof.Prune, Targets: []Target{},
+		prof: prof, targets: map[string]Snapshot{}, eps: eps}
+	st.Master, st.master = read(ctx, eps, prof.Master, prof.Items)
+	want := map[string]bool{}
+	for _, n := range only {
+		want[n] = true
+	}
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	for _, key := range prof.Targets {
+		if len(only) > 0 && !want[key] {
+			continue
+		}
+		st.Targets = append(st.Targets, Target{NodeInfo: info(key)})
+		idx := len(st.Targets) - 1
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			ni, snap := read(ctx, eps, key, prof.Items)
+			mu.Lock()
+			defer mu.Unlock()
+			st.Targets[idx].NodeInfo = ni
+			if ni.Error == "" {
+				st.targets[key] = snap
+			}
+		}()
+	}
+	wg.Wait()
+	for i := range st.Targets {
+		t := &st.Targets[i]
+		t.Counts = map[string]int{}
+		if t.Error != "" || st.Master.Error != "" {
+			continue
+		}
+		t.Plan = MakePlan(prof, st.master, st.targets[t.Key])
+		t.Digest = t.Plan.Digest()
+		t.Rows = rows(prof, st.master, st.targets[t.Key], t.Plan)
+		for _, r := range t.Rows {
+			t.Counts[r.Status]++
+		}
+	}
+	return st
+}
+
+func info(key string) NodeInfo {
+	ni := NodeInfo{Key: key}
+	for i := len(key) - 1; i >= 0; i-- {
+		if key[i] == '/' {
+			ni.Cluster, ni.Node = key[:i], key[i+1:]
+			break
+		}
+	}
+	return ni
+}
+
+func read(ctx context.Context, eps map[string]Endpoint, key string, items []string) (NodeInfo, Snapshot) {
+	ni := info(key)
+	ni.ReadAt = time.Now().UTC()
+	start := time.Now()
+	ep, ok := eps[key]
+	if !ok {
+		ni.Error = fmt.Sprintf("unknown node %s (discover the nodes of its cluster)", key)
+		return ni, Snapshot{}
+	}
+	snap, err := Read(ctx, ep, items)
+	if err != nil {
+		ni.Error = err.Error()
+	}
+	ni.Version = snap.Version
+	ni.Warnings = snap.Warnings
+	ni.Millis = time.Since(start).Milliseconds()
+	return ni, snap
+}
+
+// rows describes every object of the master and of the target.
+func rows(prof *Profile, master, target Snapshot, plan Plan) []Row {
+	changes := map[string]Change{}
+	for _, c := range plan.Changes {
+		changes[c.Key()] = c
+	}
+	errs := map[string][]string{}
+	for _, e := range plan.Errors {
+		if e.Key != "" {
+			errs[e.Key] = append(errs[e.Key], e.Message)
+		}
+	}
+	tm := prof.Mapping[target.Key]
+	var out []Row
+	seen := map[string]bool{}
+	status := func(key string, inMaster bool) string {
+		ch, has := changes[key]
+		switch {
+		case len(errs[key]) > 0:
+			return StatusError
+		case has:
+			return string(ch.Op)
+		case !inMaster:
+			return StatusExtra
+		}
+		return StatusSame
+	}
+	addAPI := func(item, id string, ref, cur map[string]any) {
+		key := item + "/" + id
+		seen[key] = true
+		ch := changes[key]
+		r := Row{Kind: item, ID: id, Key: key, Status: status(key, ref != nil), Errors: errs[key]}
+		names := map[string]bool{}
+		for _, m := range []map[string]any{ref, cur, ch.Set} {
+			for f := range m {
+				names[f] = true
+			}
+		}
+		for f := range tm.Values[key] {
+			names[f] = true
+		}
+		list := make([]string, 0, len(names))
+		for f := range names {
+			list = append(list, f)
+		}
+		sort.Strings(list)
+		unset := map[string]bool{}
+		for _, f := range ch.Unset {
+			unset[f] = true
+		}
+		for _, f := range list {
+			fd := Field{Name: f, Policy: prof.Policy(item, f), Ref: ref[f], Current: cur[f]}
+			if v, ok := tm.Values[key][f]; ok {
+				fd.Mapping = v
+			}
+			switch {
+			case ch.Op == OpDelete || unset[f]:
+				fd.Change = "unset"
+			case ch.Set[f] != nil:
+				fd.Target = ch.Set[f]
+				fd.Change = "set"
+				if fd.Current == nil {
+					fd.Change = "add"
+				}
+			case fd.Current != nil:
+				fd.Target = fd.Current
+			}
+			r.Fields = append(r.Fields, fd)
+		}
+		out = append(out, r)
+	}
+	for _, id := range prof.Items {
+		it, _ := ItemByID(id)
+		switch it.Kind {
+		case KindNetwork:
+			cur := map[string]model.Object{}
+			for _, o := range target.Network {
+				cur[o.ID] = o
+			}
+			for _, m := range master.Network {
+				tid := rename(m.ID, tm.NICs)
+				addAPI("network", tid, m.Fields, cur[tid].Fields)
+			}
+			for _, o := range target.Network {
+				if !seen["network/"+o.ID] {
+					addAPI("network", o.ID, nil, o.Fields)
+				}
+			}
+		case KindDNS:
+			addAPI("dns", "dns", master.DNS, target.DNS)
+		case KindFile:
+			paths := map[string]bool{}
+			for p := range master.Files[id] {
+				paths[p] = true
+			}
+			for p := range target.Files[id] {
+				paths[p] = true
+			}
+			if len(master.Files[id]) == 0 && !it.Glob {
+				continue
+			}
+			for p := range paths {
+				key := id + "/" + p
+				_, inMaster := master.Files[id][p]
+				r := Row{Kind: id, ID: p, Key: key, Status: status(key, inMaster), Errors: errs[key], Diff: changes[key].Diff}
+				out = append(out, r)
+			}
+		}
+	}
+	sort.SliceStable(out, func(i, j int) bool {
+		if out[i].Kind != out[j].Kind {
+			return itemRank(out[i].Kind) < itemRank(out[j].Kind)
+		}
+		return out[i].ID < out[j].ID
+	})
+	return out
+}
+
+// TargetPlan is the plan of one target restricted to a selection of objects.
+type TargetPlan struct {
+	Client  string   `json:"client"` // "client" like the cluster plans
+	Plan    Plan     `json:"plan"`
+	Digest  string   `json:"digest"`
+	Network bool     `json:"network"`
+	Reloads []string `json:"reloads,omitempty"`
+	Error   string   `json:"error,omitempty"`
+}
+
+// Plans returns the plans of the targets restricted to the given objects
+// (empty: everything), as a dry-run shows them and an apply checks them.
+func (st *State) Plans(keys []string) []TargetPlan {
+	var out []TargetPlan
+	for _, t := range st.Targets {
+		tp := TargetPlan{Client: t.Key}
+		switch {
+		case st.Master.Error != "":
+			tp.Error = "master: " + st.Master.Error
+		case t.Error != "":
+			tp.Error = t.Error
+		default:
+			tp.Plan = Filter(t.Plan, keys)
+			tp.Digest = tp.Plan.Digest()
+			tp.Network = tp.Plan.Network()
+			tp.Reloads = tp.Plan.Reloads()
+		}
+		out = append(out, tp)
+	}
+	return out
+}
+
+// Apply writes one plan on its target, then reads the target again and checks
+// that the selected objects are in line with the master.
+func (st *State) Apply(ctx context.Context, tp TargetPlan, keys []string, log func(Step)) error {
+	ep, ok := st.eps[tp.Client]
+	if !ok {
+		return fmt.Errorf("unknown node %q", tp.Client)
+	}
+	if err := Apply(ctx, ep, tp.Plan, log); err != nil {
+		return err
+	}
+	start := time.Now()
+	err := st.verify(ctx, ep, keys)
+	s := Step{At: start.UTC(), Action: "verify"}
+	if err != nil {
+		s.Error = err.Error()
+	}
+	log(s)
+	return err
+}
+
+func (st *State) verify(ctx context.Context, ep Endpoint, keys []string) error {
+	snap, err := Read(ctx, ep, st.prof.Items)
+	if err != nil {
+		return err
+	}
+	p := Filter(MakePlan(st.prof, st.master, snap), keys)
+	var errs []error
+	for _, c := range p.Changes {
+		errs = append(errs, fmt.Errorf("still different after apply: %s", c.Key()))
+	}
+	return errors.Join(errs...)
+}

internal/nodesync/apply.go

@@ -0,0 +1,333 @@
+package nodesync
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"net/url"
+	"sort"
+	"strings"
+	"time"
+
+	"pve-confsync/internal/pve"
+)
+
+// Step is the outcome of one action, for the execution journal.
+type Step struct {
+	At     time.Time `json:"at"`
+	Action string    `json:"action"`
+	Error  string    `json:"error,omitempty"`
+}
+
+// Timing of an apply; variables for the tests.
+var (
+	// RestoreDelay: when the network was reloaded, the node restores its
+	// previous configuration by itself after this delay unless pve-confsync
+	// reaches it again over SSH and cancels the restore.
+	RestoreDelay = 120 * time.Second
+	// ProbeTries and ProbeWait: attempts to reach the node after the reload.
+	ProbeTries = 6
+	ProbeWait  = 5 * time.Second
+	TaskPoll   = 2 * time.Second
+)
+
+// BackupRoot is where the previous files are kept on the node, one folder
+// per apply, with the restore.sh script that puts them back.
+const BackupRoot = "/var/lib/pve-confsync/backup"
+
+// Apply writes a plan on a target node:
+//  1. one SSH script checks that every file is still the one of the dry-run,
+//     backs up the files (and /etc/network/interfaces, /etc/resolv.conf when
+//     the API writes them), writes restore.sh, then writes the new files
+//     atomically (temporary file then mv, mode and owner kept);
+//  2. the API writes the interfaces (in interfaces.new) and the DNS servers;
+//  3. when the network is reloaded, a systemd timer is armed on the node: it
+//     runs restore.sh and ifreload -a after RestoreDelay;
+//  4. the reload commands run (sysctl, chrony, multipath, network last);
+//  5. the node must answer again over SSH and the API: the timer is stopped.
+//
+// A failure before the network reload restores the backups (and drops
+// interfaces.new); a reload failure restores them and runs the reloads again.
+func Apply(ctx context.Context, ep Endpoint, plan Plan, notify func(Step)) error {
+	log := func(action string, err error) error {
+		s := Step{At: time.Now().UTC(), Action: action}
+		if err != nil {
+			s.Error = err.Error()
+		}
+		if notify != nil {
+			notify(s)
+		}
+		return err
+	}
+	if len(plan.Errors) > 0 {
+		return log("plan check", fmt.Errorf("plan has %d error(s), nothing applied", len(plan.Errors)))
+	}
+	if plan.Empty() {
+		return nil
+	}
+	if ep.Shell == nil {
+		return log("ssh", fmt.Errorf("%s: SSH host key not recorded yet, test the node first", ep.Key))
+	}
+	if _, err := NodeVersion(ctx, ep); err != nil {
+		return log("version check", err)
+	}
+	stamp := time.Now().UTC().Format("20060102-150405")
+	dir := BackupRoot + "/" + stamp
+	unit := "pve-confsync-restore-" + stamp
+
+	var netAPI, dns bool
+	for _, c := range plan.Changes {
+		netAPI = netAPI || c.Item == "network"
+		dns = dns || c.Item == "dns"
+	}
+	if _, err := ep.Shell.Run(ctx, prepareScript(dir), prepareInput(plan, netAPI, dns)); err != nil {
+		return log("check, back up and write the files ("+dir+")", err)
+	}
+	log("check, back up and write the files ("+dir+")", nil)
+	restore := func(cause error) error {
+		_, rerr := ep.Shell.Run(ctx, restoreScript(dir), nil)
+		log("restore the previous files", rerr)
+		if rerr != nil {
+			return errors.Join(cause, fmt.Errorf("restore failed, run sh %s/restore.sh on the node: %w", dir, rerr))
+		}
+		return cause
+	}
+
+	for _, c := range plan.Changes {
+		if c.Item != "network" && c.Item != "dns" {
+			continue
+		}
+		if err := log(describe(ep, c), writeAPI(ctx, ep, c)); err != nil {
+			return restore(err)
+		}
+	}
+
+	network := plan.Network()
+	if network {
+		_, err := ep.Shell.Run(ctx, armScript(dir, unit), nil)
+		if err := log(fmt.Sprintf("arm the automatic restore (%s, %s)", unit, RestoreDelay), err); err != nil {
+			return restore(err)
+		}
+	}
+	var done []string // reload commands already run
+	for _, cmd := range plan.Reloads() {
+		var err error
+		action := cmd
+		if cmd == "ifreload -a" && netAPI {
+			action = "PUT /nodes/" + ep.Node + "/network (ifreload -a)"
+			err = reloadNetworkAPI(ctx, ep)
+		} else {
+			_, err = ep.Shell.Run(ctx, reloadScript(cmd), nil)
+		}
+		done = append(done, cmd)
+		if log("reload: "+action, err) == nil {
+			continue
+		}
+		cause := fmt.Errorf("reload %q: %w", cmd, err)
+		if network && !reachable(ctx, ep) {
+			return log("node unreachable", fmt.Errorf("%w; the node is unreachable: it restores its previous configuration by itself within %s", cause, RestoreDelay))
+		}
+		// back to the previous files, which the reloads put in effect again
+		err = restore(cause)
+		for _, c := range done {
+			_, rerr := ep.Shell.Run(ctx, reloadScript(c), nil)
+			log("reload after restore: "+c, rerr)
+		}
+		if network {
+			_, derr := ep.Shell.Run(ctx, disarmScript(unit), nil)
+			log("stop the automatic restore", derr)
+		}
+		return err
+	}
+	if network {
+		if !reachable(ctx, ep) {
+			return log("reach the node again", fmt.Errorf("%s unreachable after the network reload: it restores its previous configuration by itself within %s", ep.Key, RestoreDelay))
+		}
+		log("reach the node again (SSH and API)", nil)
+		_, err := ep.Shell.Run(ctx, disarmScript(unit), nil)
+		if err := log("stop the automatic restore", err); err != nil {
+			return fmt.Errorf("stop the automatic restore (systemctl stop %s.timer on the node): %w", unit, err)
+		}
+	}
+	return nil
+}
+
+func describe(ep Endpoint, c Change) string {
+	if c.Item == "dns" {
+		return "PUT /nodes/" + ep.Node + "/dns"
+	}
+	return fmt.Sprintf("%s %s/%s", c.Op, c.Item, c.ID)
+}
+
+// reachable opens a new SSH connection and calls the API, a few times.
+func reachable(ctx context.Context, ep Endpoint) bool {
+	for i := 0; i < ProbeTries; i++ {
+		if i > 0 {
+			select {
+			case <-ctx.Done():
+				return false
+			case <-time.After(ProbeWait):
+			}
+		}
+		sh := ep.Shell
+		if ep.Dial != nil {
+			var err error
+			if sh, err = ep.Dial(); err != nil {
+				continue
+			}
+		}
+		if _, err := sh.Run(ctx, "# confsync:probe\ntrue", nil); err != nil {
+			continue
+		}
+		if _, err := NodeVersion(ctx, ep); err == nil {
+			return true
+		}
+	}
+	return false
+}
+
+func nodePath(ep Endpoint, rest string) string {
+	return "nodes/" + url.PathEscape(ep.Node) + "/" + rest
+}
+
+// writeAPI writes one interface (POST, PUT with every field, DELETE) or the
+// DNS servers (PUT with every server).
+func writeAPI(ctx context.Context, ep Endpoint, c Change) error {
+	if c.Item == "dns" {
+		f := url.Values{}
+		for k, v := range c.Full {
+			f.Set(k, pve.Str(v))
+		}
+		return ep.API.Put(ctx, nodePath(ep, "dns"), f, nil)
+	}
+	switch c.Op {
+	case OpCreate:
+		f := form(c.Full)
+		f.Set("iface", c.ID)
+		return ep.API.Post(ctx, nodePath(ep, "network"), f, nil)
+	case OpUpdate:
+		f := form(c.Full)
+		if len(c.Unset) > 0 {
+			f.Set("delete", strings.Join(c.Unset, ","))
+		}
+		return ep.API.Put(ctx, nodePath(ep, "network/"+url.PathEscape(c.ID)), f, nil)
+	case OpDelete:
+		return ep.API.Delete(ctx, nodePath(ep, "network/"+url.PathEscape(c.ID)), nil, nil)
+	}
+	return fmt.Errorf("unknown op %q", c.Op)
+}
+
+func form(fields map[string]any) url.Values {
+	f := url.Values{}
+	keys := make([]string, 0, len(fields))
+	for k := range fields {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		f.Set(k, pve.Str(fields[k]))
+	}
+	return f
+}
+
+// reloadNetworkAPI applies interfaces.new (PUT /nodes/{node}/network: rename
+// then ifreload -a in a task) and waits for the task.
+func reloadNetworkAPI(ctx context.Context, ep Endpoint) error {
+	var upid string
+	if err := ep.API.Put(ctx, nodePath(ep, "network"), url.Values{}, &upid); err != nil {
+		return err
+	}
+	return ep.API.WaitTask(ctx, upid, TaskPoll)
+}
+
+// ---- scripts run on the node (the first line is a marker for the fake node
+// of the tests) ----
+
+// prepareInput is the plan given to prepareScript on its standard input:
+// "W <mode> <owner> <hash> <path>" then the content in base64 on one line,
+// "D - - <hash> <path>" (delete) or "B - - - <path>" (backup only).
+func prepareInput(plan Plan, netAPI, dns bool) []byte {
+	var b bytes.Buffer
+	if netAPI {
+		b.WriteString("B - - - /etc/network/interfaces\nB - - - /etc/network/interfaces.new\n")
+	}
+	if dns {
+		b.WriteString("B - - - /etc/resolv.conf\n")
+	}
+	for _, c := range plan.Changes {
+		switch {
+		case c.Item == "network" || c.Item == "dns":
+		case c.Op == OpDelete:
+			fmt.Fprintf(&b, "D - - %s %s\n", c.Hash, c.ID)
+		default:
+			fmt.Fprintf(&b, "W %s %s %s %s\n%s\n", c.Mode, c.Owner, c.Hash, c.ID, base64.StdEncoding.EncodeToString(c.Content))
+		}
+	}
+	return b.Bytes()
+}
+
+func prepareScript(dir string) string { return prepareScriptIn(dir, "") }
+
+// prepareScriptIn prefixes every path with root (empty on a node): the tests
+// run the script for real in a temporary folder.
+func prepareScriptIn(dir, root string) string {
+	return `# confsync:prepare ` + dir + `
+set -u
+R='` + root + `'
+B="$R` + dir + `"
+umask 077
+mkdir -p "$B/files" || exit 1
+cat > "$B/plan" || exit 1
+sum() { if [ -e "$1" ]; then sha256sum < "$1" | cut -d' ' -f1; else echo absent; fi; }
+# 1. every file must be the one read by the dry-run
+while read -r op mode owner h p; do
+  case "$op" in W|D) ;; *) continue ;; esac
+  [ "$(sum "$R$p")" = "$h" ] || { echo "changed since the dry-run: $p" >&2; exit 3; }
+done < "$B/plan"
+# 2. backups and restore script
+: > "$B/restore.sh"
+while read -r op mode owner h p; do
+  case "$op" in W|D|B) ;; *) continue ;; esac
+  if [ -e "$R$p" ]; then
+    mkdir -p "$B/files$(dirname "$p")" && cp -p "$R$p" "$B/files$p" || exit 1
+    printf "cp -p '%s' '%s'\n" "$B/files$p" "$R$p" >> "$B/restore.sh"
+  else
+    printf "rm -f '%s'\n" "$R$p" >> "$B/restore.sh"
+  fi
+done < "$B/plan"
+# 3. writes: temporary file in the same folder, then mv
+while read -r op mode owner h p; do
+  case "$op" in
+  W)
+    read -r data
+    mkdir -p "$(dirname "$R$p")" || exit 4
+    t=$(mktemp "$R$p.confsync.XXXXXX") || exit 4
+    if printf '%s' "$data" | base64 -d > "$t" && chmod "$mode" "$t" && chown "$owner" "$t" && mv -f "$t" "$R$p"; then :; else
+      rm -f "$t"; sh "$B/restore.sh"; echo "write failed: $p" >&2; exit 4
+    fi ;;
+  D)
+    rm -f "$R$p" || { sh "$B/restore.sh"; echo "delete failed: $p" >&2; exit 4; } ;;
+  esac
+done < "$B/plan"
+echo ok
+`
+}
+
+func restoreScript(dir string) string {
+	return "# confsync:restore " + dir + "\nsh '" + dir + "/restore.sh'\n"
+}
+
+func armScript(dir, unit string) string {
+	return fmt.Sprintf("# confsync:arm %s\nsystemd-run --quiet --unit=%s --on-active=%ds --timer-property=AccuracySec=1s /bin/sh -c 'sh %s/restore.sh; ifreload -a'\n",
+		unit, unit, int(RestoreDelay.Seconds()), dir)
+}
+
+func disarmScript(unit string) string {
+	return "# confsync:disarm " + unit + "\nsystemctl stop " + unit + ".timer\n"
+}
+
+func reloadScript(cmd string) string {
+	return "# confsync:reload " + cmd + "\n" + cmd + "\n"
+}

internal/nodesync/diff.go

@@ -0,0 +1,148 @@
+package nodesync
+
+import (
+	"strconv"
+	"strings"
+)
+
+// Line is one line of a unified diff: Op is ' ' (context), '-' (target only)
+// or '+' (written by the replication). A hunk starts with Op '@' and Text
+// "@@ -a,b +c,d @@".
+type Line struct {
+	Op   string `json:"op"`
+	Text string `json:"text"`
+}
+
+const diffContext = 3
+
+// maxCells bounds the LCS table; beyond, the diff shows every line replaced.
+const maxCells = 4_000_000
+
+// Diff compares the current content of a file with the content to write.
+func Diff(oldText, newText []byte) []Line {
+	a, b := lines(oldText), lines(newText)
+	ops := editScript(a, b)
+	// group the changes into hunks with their context
+	var out []Line
+	n := len(ops)
+	i := 0
+	for i < n {
+		if ops[i].Op == " " {
+			i++
+			continue
+		}
+		start := i - diffContext
+		if start < 0 {
+			start = 0
+		}
+		end := i
+		for end < n {
+			if ops[end].Op != " " {
+				end++
+				continue
+			}
+			// a run of context: the hunk ends when it is longer than 2*context
+			j := end
+			for j < n && ops[j].Op == " " {
+				j++
+			}
+			if j == n || j-end > 2*diffContext {
+				end += diffContext
+				if end > n {
+					end = n
+				}
+				break
+			}
+			end = j
+		}
+		oa, ob := ops[start].A, ops[start].B
+		ca, cb := 0, 0
+		for _, o := range ops[start:end] {
+			if o.Op != "+" {
+				ca++
+			}
+			if o.Op != "-" {
+				cb++
+			}
+		}
+		out = append(out, Line{Op: "@", Text: hunkHeader(oa, ca, ob, cb)})
+		for _, o := range ops[start:end] {
+			out = append(out, Line{Op: o.Op, Text: o.Text})
+		}
+		i = end
+	}
+	return out
+}
+
+type editOp struct {
+	Op   string
+	Text string
+	A, B int // line numbers (1-based) where this op starts in a and b
+}
+
+func editScript(a, b []string) []editOp {
+	var ops []editOp
+	if len(a)*len(b) > maxCells {
+		for i, l := range a {
+			ops = append(ops, editOp{"-", l, i + 1, 1})
+		}
+		for i, l := range b {
+			ops = append(ops, editOp{"+", l, len(a) + 1, i + 1})
+		}
+		return ops
+	}
+	// lcs[i][j]: length of the LCS of a[i:] and b[j:]
+	lcs := make([][]int, len(a)+1)
+	for i := range lcs {
+		lcs[i] = make([]int, len(b)+1)
+	}
+	for i := len(a) - 1; i >= 0; i-- {
+		for j := len(b) - 1; j >= 0; j-- {
+			if a[i] == b[j] {
+				lcs[i][j] = lcs[i+1][j+1] + 1
+			} else if lcs[i+1][j] >= lcs[i][j+1] {
+				lcs[i][j] = lcs[i+1][j]
+			} else {
+				lcs[i][j] = lcs[i][j+1]
+			}
+		}
+	}
+	i, j := 0, 0
+	for i < len(a) || j < len(b) {
+		switch {
+		case i < len(a) && j < len(b) && a[i] == b[j]:
+			ops = append(ops, editOp{" ", a[i], i + 1, j + 1})
+			i++
+			j++
+		case j < len(b) && (i == len(a) || lcs[i][j+1] > lcs[i+1][j]):
+			ops = append(ops, editOp{"+", b[j], i + 1, j + 1})
+			j++
+		default:
+			ops = append(ops, editOp{"-", a[i], i + 1, j + 1})
+			i++
+		}
+	}
+	return ops
+}
+
+func lines(b []byte) []string {
+	s := string(b)
+	if s == "" {
+		return nil
+	}
+	s = strings.TrimSuffix(s, "\n")
+	return strings.Split(s, "\n")
+}
+
+func hunkHeader(a, ca, b, cb int) string {
+	r := func(start, n int) string {
+		if n == 0 {
+			start--
+		}
+		if n == 1 {
+			return strconv.Itoa(start)
+		}
+		return strconv.Itoa(start) + "," + strconv.Itoa(n)
+	}
+	return "@@ -" + r(a, ca) + " +" + r(b, cb) + " @@"
+}

internal/nodesync/items.go

@@ -0,0 +1,100 @@
+// Package nodesync replicates the system configuration of one PVE node (the
+// master) to other nodes (the targets), of the same cluster or not. It is the
+// second replication mode of pve-confsync, next to the cluster to cluster one
+// (package engine). What the PVE API covers goes through the API: the network
+// interfaces (/nodes/{node}/network, written in interfaces.new then reloaded)
+// and the DNS servers (/nodes/{node}/dns). The other files go through SSH as
+// root on the node: multipath, chrony, sysctl, interfaces.d. Same pipeline as
+// the engine: read, normalize, diff, plan, apply with backup, reload, verify.
+package nodesync
+
+import "fmt"
+
+// ItemKind tells how an item is read and written.
+type ItemKind string
+
+const (
+	KindNetwork ItemKind = "network" // API /nodes/{node}/network
+	KindDNS     ItemKind = "dns"     // API /nodes/{node}/dns
+	KindFile    ItemKind = "file"    // files over SSH
+)
+
+// Item is one replicable element.
+type Item struct {
+	ID   string   `json:"id"`
+	Kind ItemKind `json:"kind"`
+	// Files: paths or glob patterns read on the nodes, and paths never touched.
+	Paths   []string `json:"paths,omitempty"`
+	Exclude []string `json:"exclude,omitempty"`
+	// Glob items hold several files: a file only on the target is kept, or
+	// deleted with the prune option. Single file items are never deleted.
+	Glob bool `json:"glob,omitempty"`
+	// Additive files (multipath wwids) only get the lines of the master they
+	// miss: nothing is removed from the target.
+	Additive bool `json:"additive,omitempty"`
+	// Reload is the command run on the node after a write (empty: none).
+	Reload string `json:"reload,omitempty"`
+	// Network items reload the network of the node: typed confirmation,
+	// automatic restore when the node is unreachable afterwards.
+	Network bool `json:"network,omitempty"`
+}
+
+// Items is the catalog, in apply order (reloads follow this order, the
+// network last).
+var Items = []Item{
+	{ID: "sysctl", Kind: KindFile, Paths: []string{"/etc/sysctl.d/*.conf"}, Glob: true, Reload: "sysctl --system"},
+	{ID: "chrony", Kind: KindFile, Paths: []string{"/etc/chrony/chrony.conf", "/etc/chrony/sources.d/*.sources"}, Glob: true,
+		Reload: "systemctl restart chrony"},
+	{ID: "multipath", Kind: KindFile, Paths: []string{"/etc/multipath.conf"}, Reload: "multipathd reconfigure"},
+	{ID: "wwids", Kind: KindFile, Paths: []string{"/etc/multipath/wwids"}, Additive: true, Reload: "multipathd reconfigure"},
+	{ID: "dns", Kind: KindDNS},
+	// interfaces.d/sdn is generated by the SDN of PVE: never replicated
+	{ID: "interfaces-d", Kind: KindFile, Paths: []string{"/etc/network/interfaces.d/*"}, Exclude: []string{"/etc/network/interfaces.d/sdn"},
+		Glob: true, Reload: "ifreload -a", Network: true},
+	{ID: "network", Kind: KindNetwork, Network: true},
+}
+
+// ItemByID returns an item of the catalog.
+func ItemByID(id string) (Item, error) {
+	for _, it := range Items {
+		if it.ID == id {
+			return it, nil
+		}
+	}
+	return Item{}, fmt.Errorf("unknown item %q", id)
+}
+
+func itemRank(id string) int {
+	for i, it := range Items {
+		if it.ID == id {
+			return i
+		}
+	}
+	return len(Items)
+}
+
+// Policy of a field of an API item (network, dns).
+type Policy string
+
+const (
+	Sync     Policy = "sync"     // copied from the master
+	Ignore   Policy = "ignore"   // never written
+	Override Policy = "override" // only written from the target mapping
+)
+
+// defaultPolicy: the addresses and gateways are those of each node; they are
+// only written from the mapping of a target.
+func defaultPolicy(item, field string) Policy {
+	if item == "network" {
+		switch field {
+		case "cidr", "cidr6", "gateway", "gateway6":
+			return Override
+		}
+	}
+	return Sync
+}
+
+// DefaultPolicies lists the fields whose default is not "sync", for the UI.
+var DefaultPolicies = map[string]map[string]Policy{
+	"network": {"cidr": Override, "cidr6": Override, "gateway": Override, "gateway6": Override},
+}

internal/nodesync/nodesync_test.go

@@ -0,0 +1,304 @@
+package nodesync
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"strings"
+	"testing"
+	"time"
+
+	"pve-confsync/internal/config"
+	"pve-confsync/internal/pve/pvetest"
+	"pve-confsync/internal/sshx"
+)
+
+func init() {
+	ProbeWait, TaskPoll = time.Millisecond, time.Millisecond
+}
+
+// endpoint connects to a node of a fake cluster like the web UI does.
+func endpoint(t *testing.T, s *pvetest.Server, cluster string, n *pvetest.Node) Endpoint {
+	t.Helper()
+	sum := sha256.Sum256(s.Certificate().Raw)
+	c, err := config.Cluster{Name: cluster, URL: s.URL, User: "root@pam", Password: pvetest.Password,
+		Fingerprint: hex.EncodeToString(sum[:])}.PVE()
+	if err != nil {
+		t.Fatal(err)
+	}
+	host, port := n.SSHAddr()
+	o := sshx.Options{Host: host, Port: port, Password: pvetest.Password, HostKey: n.SSHHostKey()}
+	sh, err := sshx.New(o)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return Endpoint{Key: cluster + "/" + n.Name, Cluster: cluster, Node: n.Name, API: c, Shell: sh}
+}
+
+const multipath = "defaults {\n    user_friendly_names yes\n    find_multipaths no\n}\nblacklist {\n    devnode \"^sda\"\n}\n"
+
+// lab: one cluster, a master pve1 and a target pve2 with differences.
+func lab(t *testing.T) (*pvetest.Server, *pvetest.Node, *pvetest.Node, map[string]Endpoint) {
+	s := pvetest.New("9.1.4")
+	t.Cleanup(s.Close)
+	m, c := s.AddNode("pve1"), s.AddNode("pve2")
+	m.Net = map[string]map[string]any{
+		"eno1":     {"type": "eth", "mtu": "9000"},
+		"eno2":     {"type": "eth", "mtu": "9000"},
+		"bond0":    {"type": "bond", "slaves": "eno1 eno2", "bond_mode": "802.3ad", "bond_xmit_hash_policy": "layer3+4", "mtu": "9000"},
+		"vmbr0":    {"type": "bridge", "bridge_ports": "bond0", "bridge_vlan_aware": "1", "cidr": "10.0.0.11/24", "gateway": "10.0.0.1", "autostart": "1"},
+		"vmbr1":    {"type": "bridge", "bridge_ports": "bond0.50", "cidr": "10.5.0.11/24", "comments": "stockage"},
+		"bond0.50": {"type": "vlan", "mtu": "9000"},
+	}
+	// the target has other NIC names and an active-backup bond
+	c.Net = map[string]map[string]any{
+		"enp1s0f0": {"type": "eth"},
+		"enp1s0f1": {"type": "eth"},
+		"bond0":    {"type": "bond", "slaves": "enp1s0f0 enp1s0f1", "bond_mode": "active-backup"},
+		"vmbr0":    {"type": "bridge", "bridge_ports": "bond0", "cidr": "10.0.0.12/24", "gateway": "10.0.0.1", "autostart": "1"},
+	}
+	m.DNS = map[string]string{"search": "lab.local", "dns1": "10.0.0.53", "dns2": "10.0.0.54"}
+	c.DNS = map[string]string{"search": "lab.local", "dns1": "8.8.8.8"}
+	m.Files["/etc/multipath.conf"] = pvetest.FakeFile{Content: multipath}
+	m.Files["/etc/multipath/wwids"] = pvetest.FakeFile{Content: "# wwids\n/3600a0980383030/\n/3600a0980383031/\n"}
+	c.Files["/etc/multipath/wwids"] = pvetest.FakeFile{Content: "# wwids\n/3600a0980383031/\n/36000local/\n"}
+	m.Files["/etc/sysctl.d/90-net.conf"] = pvetest.FakeFile{Content: "net.core.rmem_max = 16777216\n"}
+	c.Files["/etc/sysctl.d/90-net.conf"] = pvetest.FakeFile{Content: "net.core.rmem_max = 8388608\n", Mode: "600"}
+	c.Files["/etc/sysctl.d/99-local.conf"] = pvetest.FakeFile{Content: "vm.swappiness = 10\n"}
+	m.Files["/etc/network/interfaces.d/sdn"] = pvetest.FakeFile{Content: "auto vnet1\n"}
+	eps := map[string]Endpoint{"lab/pve1": endpoint(t, s, "lab", m), "lab/pve2": endpoint(t, s, "lab", c)}
+	return s, m, c, eps
+}
+
+func profile() *Profile {
+	return &Profile{Name: "stockage", Master: "lab/pve1", Targets: []string{"lab/pve2"},
+		Items: []string{"network", "dns", "multipath", "wwids", "sysctl", "interfaces-d"},
+		Mapping: map[string]TargetMap{"lab/pve2": {
+			NICs:   map[string]string{"eno1": "enp1s0f0", "eno2": "enp1s0f1"},
+			Values: map[string]map[string]string{"network/vmbr1": {"cidr": "10.5.0.12/24"}},
+		}}}
+}
+
+func keys(p Plan) []string {
+	var out []string
+	for _, c := range p.Changes {
+		out = append(out, string(c.Op)+" "+c.Key())
+	}
+	return out
+}
+
+func TestPlan(t *testing.T) {
+	_, _, _, eps := lab(t)
+	prof := profile()
+	st := Analyze(context.Background(), prof, eps, nil)
+	if st.Master.Error != "" || st.Targets[0].Error != "" {
+		t.Fatal(st.Master.Error, st.Targets[0].Error)
+	}
+	p := st.Targets[0].Plan
+	if len(p.Errors) > 0 {
+		t.Fatalf("errors: %+v", p.Errors)
+	}
+	got := strings.Join(keys(p), "\n")
+	want := strings.Join([]string{
+		"update sysctl//etc/sysctl.d/90-net.conf",
+		"create multipath//etc/multipath.conf",
+		"update wwids//etc/multipath/wwids",
+		"update dns/dns",
+		"update network/enp1s0f0",
+		"update network/enp1s0f1",
+		"update network/bond0",
+		"create network/bond0.50",
+		"update network/vmbr0",
+		"create network/vmbr1",
+	}, "\n")
+	if got != want {
+		t.Fatalf("plan:\n%s\nwant:\n%s", got, want)
+	}
+	for _, c := range p.Changes {
+		switch c.Key() {
+		case "network/bond0":
+			if c.Set["slaves"] != "enp1s0f0 enp1s0f1" && c.Set["slaves"] != nil {
+				t.Errorf("slaves not renamed: %v", c.Set["slaves"])
+			}
+			if c.Set["bond_mode"] != "802.3ad" {
+				t.Errorf("bond0: %+v", c.Set)
+			}
+		case "network/vmbr0":
+			// the address of the target is kept (override without mapping)
+			if _, ok := c.Set["cidr"]; ok || c.Full["cidr"] != "10.0.0.12/24" {
+				t.Errorf("vmbr0: set %+v full %+v", c.Set, c.Full)
+			}
+		case "network/vmbr1":
+			if c.Full["cidr"] != "10.5.0.12/24" || c.Full["bridge_ports"] != "bond0.50" {
+				t.Errorf("vmbr1: %+v", c.Full)
+			}
+		case "wwids//etc/multipath/wwids":
+			// additive: the local WWID of the target stays
+			if string(c.Content) != "# wwids\n/3600a0980383031/\n/36000local/\n/3600a0980383030/\n" {
+				t.Errorf("wwids: %q", c.Content)
+			}
+		case "sysctl//etc/sysctl.d/90-net.conf":
+			if c.Mode != "600" || len(c.Diff) == 0 {
+				t.Errorf("sysctl: mode %s diff %+v", c.Mode, c.Diff)
+			}
+		}
+	}
+	// the file only on the target is kept without prune; interfaces.d/sdn is never read
+	for _, r := range st.Targets[0].Rows {
+		if r.ID == "/etc/sysctl.d/99-local.conf" && r.Status != StatusExtra {
+			t.Errorf("99-local.conf: %s", r.Status)
+		}
+		if strings.Contains(r.ID, "interfaces.d/sdn") {
+			t.Errorf("sdn file listed")
+		}
+	}
+	if !p.Network() {
+		t.Error("network plan not flagged")
+	}
+	if strings.Join(p.Reloads(), ",") != "sysctl --system,multipathd reconfigure,ifreload -a" {
+		t.Errorf("reloads: %v", p.Reloads())
+	}
+}
+
+func TestApplyAndVerify(t *testing.T) {
+	_, _, c, eps := lab(t)
+	prof := profile()
+	st := Analyze(context.Background(), prof, eps, nil)
+	tp := st.Plans(nil)[0]
+	var steps []string
+	if err := st.Apply(context.Background(), tp, nil, func(s Step) {
+		steps = append(steps, s.Action+" "+s.Error)
+	}); err != nil {
+		t.Fatalf("apply: %v\n%s", err, strings.Join(steps, "\n"))
+	}
+	if c.Net["vmbr0"]["cidr"] != "10.0.0.12/24" || c.Net["vmbr1"]["cidr"] != "10.5.0.12/24" {
+		t.Errorf("addresses: vmbr0 %v vmbr1 %v", c.Net["vmbr0"], c.Net["vmbr1"])
+	}
+	if c.Net["vmbr0"]["bridge_vlan_aware"] != "1" || c.Pending != nil {
+		t.Errorf("vmbr0 %v pending %v", c.Net["vmbr0"], c.Pending)
+	}
+	if c.DNS["dns2"] != "10.0.0.54" || c.Files["/etc/multipath.conf"].Content != multipath {
+		t.Errorf("dns %v multipath %q", c.DNS, c.Files["/etc/multipath.conf"].Content)
+	}
+	if len(c.Armed) != 0 {
+		t.Errorf("restore timer still armed: %v", c.Armed)
+	}
+	if got := strings.Join(c.Reloads, ","); got != "sysctl --system,multipathd reconfigure,network" {
+		t.Errorf("reloads: %s", got)
+	}
+	if len(c.Backups()) != 1 {
+		t.Errorf("backups: %v", c.Backups())
+	}
+	st = Analyze(context.Background(), prof, eps, nil)
+	if p := st.Targets[0].Plan; !p.Empty() {
+		t.Errorf("still to do: %v", keys(p))
+	}
+}
+
+func TestReloadFailureRestores(t *testing.T) {
+	_, _, c, eps := lab(t)
+	prof := profile()
+	prof.Items = []string{"multipath", "sysctl"}
+	c.FailOn = "reload multipathd reconfigure"
+	st := Analyze(context.Background(), prof, eps, nil)
+	err := st.Apply(context.Background(), st.Plans(nil)[0], nil, func(Step) {})
+	if err == nil || !strings.Contains(err.Error(), "multipathd reconfigure") {
+		t.Fatalf("err = %v", err)
+	}
+	if _, ok := c.Files["/etc/multipath.conf"]; ok {
+		t.Error("multipath.conf not removed by the restore")
+	}
+	if c.Files["/etc/sysctl.d/90-net.conf"].Content != "net.core.rmem_max = 8388608\n" {
+		t.Errorf("sysctl not restored: %q", c.Files["/etc/sysctl.d/90-net.conf"].Content)
+	}
+}
+
+func TestUnreachableAfterReloadKeepsTimer(t *testing.T) {
+	_, _, c, eps := lab(t)
+	prof := profile()
+	prof.Items = []string{"network"}
+	st := Analyze(context.Background(), prof, eps, nil)
+	c.FailOn = "probe"
+	err := st.Apply(context.Background(), st.Plans(nil)[0], nil, func(Step) {})
+	if err == nil || !strings.Contains(err.Error(), "unreachable") {
+		t.Fatalf("err = %v", err)
+	}
+	if len(c.Armed) != 1 {
+		t.Fatalf("timer: %v", c.Armed)
+	}
+	// systemd runs the restore: the previous interfaces come back
+	c.Fire()
+	if c.Net["bond0"]["bond_mode"] != "active-backup" || c.Net["vmbr1"] != nil {
+		t.Errorf("not restored: %v", c.Net)
+	}
+}
+
+func TestStaleFileRefused(t *testing.T) {
+	_, _, c, eps := lab(t)
+	prof := profile()
+	prof.Items = []string{"sysctl"}
+	st := Analyze(context.Background(), prof, eps, nil)
+	c.Files["/etc/sysctl.d/90-net.conf"] = pvetest.FakeFile{Content: "changed by hand\n"}
+	err := st.Apply(context.Background(), st.Plans(nil)[0], nil, func(Step) {})
+	if err == nil || !strings.Contains(err.Error(), "changed since the dry-run") {
+		t.Fatalf("err = %v", err)
+	}
+	if c.Files["/etc/sysctl.d/90-net.conf"].Content != "changed by hand\n" {
+		t.Error("file overwritten")
+	}
+}
+
+func TestPendingNetworkBlocks(t *testing.T) {
+	_, _, c, eps := lab(t)
+	c.Pending = map[string]map[string]any{"vmbr0": {"type": "bridge"}}
+	st := Analyze(context.Background(), profile(), eps, nil)
+	if len(st.Targets[0].Plan.Errors) == 0 {
+		t.Fatal("pending changes not blocking")
+	}
+}
+
+func TestMissingNICIsAnError(t *testing.T) {
+	_, _, _, eps := lab(t)
+	prof := profile()
+	prof.Items = []string{"network"}
+	prof.Mapping = nil
+	st := Analyze(context.Background(), prof, eps, nil)
+	p := st.Targets[0].Plan
+	if len(p.Errors) != 2 || p.Errors[0].Key != "network/eno1" {
+		t.Fatalf("errors: %+v", p.Errors)
+	}
+	// the selection of another object is not blocked
+	if f := Filter(p, []string{"network/vmbr0"}); len(f.Errors) != 0 || len(f.Changes) != 1 {
+		t.Errorf("filter: %+v", f)
+	}
+}
+
+func TestPruneAndProtect(t *testing.T) {
+	_, m, _, eps := lab(t)
+	m.Files["/etc/sysctl.d/90-net.conf"] = pvetest.FakeFile{Content: "# host pve1\nnet.core.rmem_max = 8388608\n"}
+	prof := profile()
+	prof.Items = []string{"sysctl"}
+	prof.Prune = true
+	prof.Protect = map[string][]string{"sysctl": {"^# host "}}
+	st := Analyze(context.Background(), prof, eps, nil)
+	got := strings.Join(keys(st.Targets[0].Plan), ",")
+	// the protected line is absent on the target: dropped, the rest is equal
+	if got != "delete sysctl//etc/sysctl.d/99-local.conf" {
+		t.Fatalf("plan: %s", got)
+	}
+}
+
+func TestDiff(t *testing.T) {
+	d := Diff([]byte("a\nb\nc\nd\ne\nf\ng\nh\ni\n"), []byte("a\nb\nc\nd\nE\nf\ng\nh\ni\nj\n"))
+	var b strings.Builder
+	for _, l := range d {
+		b.WriteString(l.Op + l.Text + "\n")
+	}
+	want := "@@@ -2,8 +2,9 @@\n b\n c\n d\n-e\n+E\n f\n g\n h\n i\n+j\n"
+	if b.String() != want {
+		t.Fatalf("diff:\n%s\nwant:\n%s", b.String(), want)
+	}
+	if d := Diff(nil, []byte("x\n")); len(d) != 2 || d[0].Text != "@@ -0,0 +1 @@" {
+		t.Errorf("creation diff: %+v", d)
+	}
+}

internal/nodesync/plan.go

@@ -0,0 +1,661 @@
+package nodesync
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"regexp"
+	"sort"
+	"strings"
+
+	"pve-confsync/internal/model"
+)
+
+// Profile is one node replication: a master node, target nodes, the
+// replicated items and their rules.
+type Profile struct {
+	Name    string   `json:"name"`
+	Master  string   `json:"master"`  // "cluster/node"
+	Targets []string `json:"targets"` // "cluster/node"
+	Items   []string `json:"items"`
+	// Prune deletes on the targets what the master does not have: network
+	// interfaces (never the physical ones) and the files of the glob items.
+	Prune bool `json:"prune"`
+	// Policies of the fields of the API items (item -> field -> policy),
+	// over the defaults (addresses and gateways: override).
+	Policies map[string]map[string]Policy `json:"policies"`
+	// Protect: per file item, regular expressions of the lines kept from the
+	// target (hostname, local address…).
+	Protect map[string][]string `json:"protect"`
+	// Mapping: per target, its interface names and its own values.
+	Mapping  map[string]TargetMap `json:"mapping"`
+	Position int                  `json:"position"`
+}
+
+// TargetMap holds what is specific to one target.
+type TargetMap struct {
+	// NICs renames the interfaces of the master (eno1 -> enp1s0f0), in the
+	// interface names and in the fields that list interfaces.
+	NICs map[string]string `json:"nics,omitempty"`
+	// Values: "item/id" -> field -> value written on this target (cidr of
+	// vmbr0…), whatever the master has.
+	Values map[string]map[string]string `json:"values,omitempty"`
+}
+
+// Validate checks a profile.
+func (p *Profile) Validate() error {
+	var errs []error
+	p.Name = strings.TrimSpace(p.Name)
+	if p.Name == "" {
+		errs = append(errs, errors.New("name is required"))
+	}
+	if p.Master == "" {
+		errs = append(errs, errors.New("master node is required"))
+	}
+	seen := map[string]bool{}
+	for _, t := range p.Targets {
+		if t == p.Master {
+			errs = append(errs, fmt.Errorf("target %s is the master", t))
+		}
+		if seen[t] {
+			errs = append(errs, fmt.Errorf("target %s listed twice", t))
+		}
+		seen[t] = true
+	}
+	if len(p.Items) == 0 {
+		errs = append(errs, errors.New("select at least one item"))
+	}
+	for _, id := range p.Items {
+		if _, err := ItemByID(id); err != nil {
+			errs = append(errs, err)
+		}
+	}
+	for item, fields := range p.Policies {
+		for f, pol := range fields {
+			switch pol {
+			case Sync, Ignore, Override:
+			default:
+				errs = append(errs, fmt.Errorf("%s.%s: unknown policy %q", item, f, pol))
+			}
+			if item == "network" && f == "type" && pol != Sync {
+				errs = append(errs, errors.New("network.type: always synchronized"))
+			}
+		}
+	}
+	for item, res := range p.Protect {
+		for _, re := range res {
+			if _, err := regexp.Compile(re); err != nil {
+				errs = append(errs, fmt.Errorf("%s: protected lines: %w", item, err))
+			}
+		}
+	}
+	return errors.Join(errs...)
+}
+
+// Policy resolves the policy of a field.
+func (p *Profile) Policy(item, field string) Policy {
+	if item == "network" && field == "type" {
+		return Sync
+	}
+	if pol, ok := p.Policies[item][field]; ok {
+		return pol
+	}
+	return defaultPolicy(item, field)
+}
+
+// Has tells whether an item is replicated.
+func (p *Profile) Has(item string) bool {
+	for _, x := range p.Items {
+		if x == item {
+			return true
+		}
+	}
+	return false
+}
+
+// Op is the action planned on one object.
+type Op string
+
+const (
+	OpCreate Op = "create"
+	OpUpdate Op = "update"
+	OpDelete Op = "delete"
+)
+
+// Change is one planned action: an interface or the DNS servers (Set, Unset,
+// Old), or a file (Diff, Hash of the target file, content to write).
+type Change struct {
+	Item  string         `json:"kind"` // item id ("kind" like the cluster plans, for the UI)
+	ID    string         `json:"id"`   // interface, "dns" or file path
+	Op    Op             `json:"op"`
+	Set   map[string]any `json:"set,omitempty"`
+	Unset []string       `json:"unset,omitempty"`
+	Old   map[string]any `json:"old,omitempty"`
+	Diff  []Line         `json:"diff,omitempty"`
+	Hash  string         `json:"hash,omitempty"` // current file on the target ("absent" when created)
+	Mode  string         `json:"mode,omitempty"`
+	Owner string         `json:"owner,omitempty"`
+	// Full is the whole writable configuration of an interface after the
+	// change: PUT /nodes/{node}/network/{iface} recomputes the method and the
+	// families from the parameters, so every field is sent again.
+	Full    map[string]any `json:"-"`
+	Content []byte         `json:"-"`
+}
+
+// Key identifies the object of a change ("item/id").
+func (c Change) Key() string { return c.Item + "/" + c.ID }
+
+// Issue is a blocking error or a warning, about one object (Key) or the
+// whole plan.
+type Issue struct {
+	Key     string `json:"key,omitempty"`
+	Message string `json:"message"`
+}
+
+// Plan is the list of changes for one target, in execution order.
+type Plan struct {
+	Target   string   `json:"client"`
+	Changes  []Change `json:"changes"`
+	Errors   []Issue  `json:"errors,omitempty"`
+	Warnings []Issue  `json:"warnings,omitempty"`
+}
+
+// Empty tells whether there is nothing to do.
+func (p Plan) Empty() bool { return len(p.Changes) == 0 }
+
+// Network tells whether the plan reloads the network of the node.
+func (p Plan) Network() bool {
+	for _, c := range p.Changes {
+		if it, _ := ItemByID(c.Item); it.Network {
+			return true
+		}
+	}
+	return false
+}
+
+// Reloads lists the commands run after the writes, in order.
+func (p Plan) Reloads() []string {
+	seen := map[string]bool{}
+	var out []string
+	for _, it := range Items {
+		for _, c := range p.Changes {
+			if c.Item != it.ID {
+				continue
+			}
+			cmd := it.Reload
+			if it.Kind == KindNetwork {
+				cmd = "ifreload -a"
+			}
+			if cmd != "" && !seen[cmd] {
+				seen[cmd] = true
+				out = append(out, cmd)
+			}
+			break
+		}
+	}
+	return out
+}
+
+func (p *Plan) errorf(key, format string, a ...any) {
+	p.Errors = append(p.Errors, Issue{key, fmt.Sprintf(format, a...)})
+}
+
+func (p *Plan) warnf(key, format string, a ...any) {
+	p.Warnings = append(p.Warnings, Issue{key, fmt.Sprintf(format, a...)})
+}
+
+// MakePlan computes the changes that bring the target in line with the master.
+func MakePlan(prof *Profile, master, target Snapshot) Plan {
+	plan := Plan{Target: target.Key}
+	tm := prof.Mapping[target.Key]
+	if target.Pending && prof.Has("network") {
+		plan.errorf("", "pending network changes on %s (interfaces.new): apply or revert them in PVE first", target.Key)
+	}
+	if master.Pending && prof.Has("network") {
+		plan.warnf("", "the master has pending network changes: its pending configuration is replicated")
+	}
+	for _, id := range prof.Items {
+		it, _ := ItemByID(id)
+		switch it.Kind {
+		case KindNetwork:
+			planNetwork(&plan, prof, tm, master, target)
+		case KindDNS:
+			planDNS(&plan, prof, tm, master, target)
+		case KindFile:
+			planFiles(&plan, prof, it, master.Files[it.ID], target.Files[it.ID])
+		}
+	}
+	sortChanges(plan.Changes)
+	return plan
+}
+
+// rank orders the interfaces: what is referenced first (ports, bonds, VLAN
+// devices), bridges last; deletions run in the reverse order.
+func netRank(typ string) int {
+	switch typ {
+	case "eth", "alias":
+		return 0
+	case "bond", "OVSBond":
+		return 1
+	case "vlan":
+		return 2
+	case "bridge", "OVSBridge":
+		return 3
+	}
+	return 4
+}
+
+func sortChanges(chs []Change) {
+	typ := func(c Change) string {
+		if t, ok := c.Full["type"].(string); ok {
+			return t
+		}
+		t, _ := c.Old["type"].(string)
+		return t
+	}
+	sort.SliceStable(chs, func(i, j int) bool {
+		a, b := chs[i], chs[j]
+		if (a.Op == OpDelete) != (b.Op == OpDelete) {
+			return b.Op == OpDelete
+		}
+		if a.Item != b.Item {
+			return itemRank(a.Item) < itemRank(b.Item)
+		}
+		if a.Item == "network" {
+			ra, rb := netRank(typ(a)), netRank(typ(b))
+			if a.Op == OpDelete {
+				ra, rb = rb, ra
+			}
+			if ra != rb {
+				return ra < rb
+			}
+		}
+		return a.ID < b.ID
+	})
+}
+
+// netLists are the fields naming other interfaces (space separated lists or
+// one name): renamed with the NIC mapping of the target.
+var netLists = map[string]bool{
+	"bridge_ports": true, "slaves": true, "ovs_ports": true, "ovs_bonds": true,
+	"bond-primary": true, "vlan-raw-device": true, "ovs_bridge": true,
+}
+
+// rename maps an interface name of the master to the target: eno1, and the
+// VLAN interfaces built on it (eno1.100).
+func rename(name string, nics map[string]string) string {
+	if to, ok := nics[name]; ok {
+		return to
+	}
+	if base, vlan, ok := strings.Cut(name, "."); ok {
+		if to, ok := nics[base]; ok {
+			return to + "." + vlan
+		}
+	}
+	return name
+}
+
+func renameList(v string, nics map[string]string) string {
+	parts := strings.Fields(v)
+	for i, p := range parts {
+		parts[i] = rename(p, nics)
+	}
+	return strings.Join(parts, " ")
+}
+
+func planNetwork(plan *Plan, prof *Profile, tm TargetMap, master, target Snapshot) {
+	cur := map[string]model.Object{}
+	for _, o := range target.Network {
+		cur[o.ID] = o
+	}
+	inMaster := map[string]bool{}
+	for _, m := range master.Network {
+		id := rename(m.ID, tm.NICs)
+		key := "network/" + id
+		inMaster[id] = true
+		desired := map[string]any{}
+		managed := map[string]bool{}
+		for f, v := range m.Fields {
+			if prof.Policy("network", f) != Sync {
+				continue
+			}
+			if netLists[f] {
+				v = renameList(v.(string), tm.NICs)
+			}
+			desired[f] = v
+		}
+		for f := range netWritable {
+			if prof.Policy("network", f) == Sync {
+				managed[f] = true
+			}
+		}
+		for f, v := range tm.Values[key] {
+			if prof.Policy("network", f) == Ignore || !netWritable[f] || f == "type" {
+				continue
+			}
+			desired[f] = v
+			managed[f] = true
+		}
+		typ, _ := desired["type"].(string)
+		o, exists := cur[id]
+		if !exists {
+			if physical(typ) {
+				plan.errorf(key, "physical interface %s absent on %s: add it to the NIC mapping of the target", id, target.Key)
+				continue
+			}
+			plan.Changes = append(plan.Changes, Change{Item: "network", ID: id, Op: OpCreate, Set: desired, Full: desired})
+			continue
+		}
+		if t, _ := o.Fields["type"].(string); t != typ {
+			plan.errorf(key, "type differs (%s on the target, %s on the master): re-create the interface by hand", t, typ)
+			continue
+		}
+		if mo, to := strings.Join(master.Options[m.ID], "\n"), strings.Join(target.Options[id], "\n"); mo != to {
+			plan.warnf(key, "options the API does not write differ (%s): not replicated", optionNames(master.Options[m.ID], target.Options[id]))
+		}
+		ch := Change{Item: "network", ID: id, Op: OpUpdate, Set: map[string]any{}, Old: map[string]any{}}
+		full := map[string]any{}
+		for f, v := range o.Fields {
+			full[f] = v
+		}
+		for f, v := range desired {
+			if !model.Equal(v, o.Fields[f]) {
+				ch.Set[f] = v
+				if old, ok := o.Fields[f]; ok {
+					ch.Old[f] = old
+				}
+			}
+			full[f] = v
+		}
+		for f := range managed {
+			if _, want := desired[f]; want {
+				continue
+			}
+			if old, has := o.Fields[f]; has {
+				ch.Unset = append(ch.Unset, f)
+				ch.Old[f] = old
+				delete(full, f)
+			}
+		}
+		sort.Strings(ch.Unset)
+		if len(ch.Set) > 0 || len(ch.Unset) > 0 {
+			ch.Full = full
+			plan.Changes = append(plan.Changes, ch)
+		}
+	}
+	if !prof.Prune {
+		return
+	}
+	for id, o := range cur {
+		if inMaster[id] {
+			continue
+		}
+		if t, _ := o.Fields["type"].(string); physical(t) {
+			continue
+		}
+		plan.Changes = append(plan.Changes, Change{Item: "network", ID: id, Op: OpDelete, Old: o.Fields})
+	}
+}
+
+// optionNames gives the option keywords that differ (bond-miimon, post-up…).
+func optionNames(a, b []string) string {
+	set := map[string]bool{}
+	count := map[string]int{}
+	for _, l := range a {
+		count[l]++
+	}
+	for _, l := range b {
+		count[l]--
+	}
+	for l, n := range count {
+		if n != 0 {
+			set[strings.Fields(l + " ")[0]] = true
+		}
+	}
+	var out []string
+	for k := range set {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return strings.Join(out, ", ")
+}
+
+func planDNS(plan *Plan, prof *Profile, tm TargetMap, master, target Snapshot) {
+	const key = "dns/dns"
+	desired := map[string]any{}
+	for _, f := range dnsFields {
+		pol := prof.Policy("dns", f)
+		if v, ok := master.DNS[f]; ok && pol == Sync {
+			desired[f] = v
+		}
+		if pol != Sync {
+			if v, ok := target.DNS[f]; ok {
+				desired[f] = v // PUT /nodes/{node}/dns rewrites every server
+			}
+		}
+		if v, ok := tm.Values[key][f]; ok && pol != Ignore {
+			desired[f] = v
+		}
+	}
+	ch := Change{Item: "dns", ID: "dns", Op: OpUpdate, Set: map[string]any{}, Old: map[string]any{}, Full: desired}
+	for _, f := range dnsFields {
+		v, want := desired[f]
+		old, has := target.DNS[f]
+		switch {
+		case want && !model.Equal(v, old):
+			ch.Set[f] = v
+			if has {
+				ch.Old[f] = old
+			}
+		case !want && has:
+			ch.Unset = append(ch.Unset, f)
+			ch.Old[f] = old
+		}
+	}
+	if len(ch.Set) == 0 && len(ch.Unset) == 0 {
+		return
+	}
+	if desired["search"] == nil {
+		plan.errorf(key, "a search domain is required by the API (PUT /nodes/{node}/dns)")
+		return
+	}
+	plan.Changes = append(plan.Changes, ch)
+}
+
+func planFiles(plan *Plan, prof *Profile, it Item, master, target map[string]File) {
+	var protect []*regexp.Regexp
+	for _, s := range prof.Protect[it.ID] {
+		if re, err := regexp.Compile(s); err == nil {
+			protect = append(protect, re)
+		}
+	}
+	if len(master) == 0 && !it.Glob {
+		plan.warnf(it.ID+"/"+it.Paths[0], "%s absent on the master: not replicated", it.Paths[0])
+		return
+	}
+	paths := make([]string, 0, len(master))
+	for p := range master {
+		paths = append(paths, p)
+	}
+	sort.Strings(paths)
+	for _, p := range paths {
+		m := master[p]
+		t, exists := target[p]
+		var content []byte
+		switch {
+		case it.Additive && exists:
+			content = additive(t.Content, m.Content)
+		case exists:
+			content = protectLines(m.Content, t.Content, protect)
+		default:
+			content = protectLines(m.Content, nil, protect)
+		}
+		if exists && hash(content) == t.Hash {
+			continue
+		}
+		ch := Change{Item: it.ID, ID: p, Content: content, Mode: m.Mode, Owner: m.Owner, Hash: "absent", Op: OpCreate}
+		var old []byte
+		if exists {
+			// an update keeps the mode and owner of the target file
+			ch.Op, ch.Hash, ch.Mode, ch.Owner, old = OpUpdate, t.Hash, t.Mode, t.Owner, t.Content
+		}
+		ch.Diff = Diff(old, content)
+		plan.Changes = append(plan.Changes, ch)
+	}
+	if !prof.Prune || !it.Glob {
+		return
+	}
+	for p, t := range target {
+		if _, ok := master[p]; ok {
+			continue
+		}
+		plan.Changes = append(plan.Changes, Change{Item: it.ID, ID: p, Op: OpDelete, Hash: t.Hash, Diff: Diff(t.Content, nil)})
+	}
+}
+
+// protectLines: each line of the master matching a protected expression takes
+// the value of the first line of the target matching the same expression
+// (dropped when the target has none).
+func protectLines(master, target []byte, protect []*regexp.Regexp) []byte {
+	if len(protect) == 0 {
+		return master
+	}
+	tl := lines(target)
+	used := map[int]bool{}
+	var out []string
+	for _, l := range lines(master) {
+		var re *regexp.Regexp
+		for _, r := range protect {
+			if r.MatchString(l) {
+				re = r
+				break
+			}
+		}
+		if re == nil {
+			out = append(out, l)
+			continue
+		}
+		for i, x := range tl {
+			if !used[i] && re.MatchString(x) {
+				used[i] = true
+				out = append(out, x)
+				break
+			}
+		}
+	}
+	return joinLines(out)
+}
+
+// additive appends to the target the lines of the master it does not have.
+func additive(target, master []byte) []byte {
+	tl := lines(target)
+	have := map[string]bool{}
+	for _, l := range tl {
+		have[l] = true
+	}
+	out := tl
+	for _, l := range lines(master) {
+		if !have[l] {
+			have[l] = true
+			out = append(out, l)
+		}
+	}
+	return joinLines(out)
+}
+
+func joinLines(l []string) []byte {
+	if len(l) == 0 {
+		return nil
+	}
+	return []byte(strings.Join(l, "\n") + "\n")
+}
+
+// Filter keeps the changes of the given objects ("item/id"), plus the
+// interfaces they need that the plan creates (ports of a bridge, raw device
+// of a VLAN). Issues about other objects do not block the selection.
+func Filter(p Plan, keys []string) Plan {
+	if len(keys) == 0 {
+		return p
+	}
+	byKey := map[string]Change{}
+	for _, c := range p.Changes {
+		byKey[c.Key()] = c
+	}
+	want := map[string]bool{}
+	for _, k := range keys {
+		want[k] = true
+	}
+	for changed := true; changed; {
+		changed = false
+		for k := range want {
+			c, ok := byKey[k]
+			if !ok || c.Item != "network" || c.Op == OpDelete {
+				continue
+			}
+			for f, v := range c.Full {
+				if !netLists[f] {
+					continue
+				}
+				for _, n := range strings.Fields(v.(string)) {
+					dep := "network/" + n
+					if d, ok := byKey[dep]; ok && d.Op == OpCreate && !want[dep] {
+						want[dep] = true
+						changed = true
+					}
+				}
+			}
+		}
+	}
+	out := Plan{Target: p.Target}
+	for _, e := range p.Errors {
+		if e.Key == "" || want[e.Key] {
+			out.Errors = append(out.Errors, e)
+		}
+	}
+	for _, w := range p.Warnings {
+		if w.Key == "" || want[w.Key] {
+			out.Warnings = append(out.Warnings, w)
+		}
+	}
+	for _, c := range p.Changes {
+		if want[c.Key()] {
+			out.Changes = append(out.Changes, c)
+		}
+	}
+	return out
+}
+
+// Digest identifies the content of a plan: the apply only runs when the plan
+// computed again is the one reviewed in the dry-run.
+func (p Plan) Digest() string {
+	type ch struct {
+		Op    Op             `json:"op"`
+		Key   string         `json:"key"`
+		Set   map[string]any `json:"set,omitempty"`
+		Unset []string       `json:"unset,omitempty"`
+		Hash  string         `json:"hash,omitempty"`
+		New   string         `json:"new,omitempty"`
+	}
+	l := make([]ch, 0, len(p.Changes))
+	for _, c := range p.Changes {
+		x := ch{Op: c.Op, Key: c.Key(), Set: c.Set, Unset: c.Unset, Hash: c.Hash}
+		if c.Content != nil || c.Op != OpDelete && c.Hash != "" {
+			x.New = hash(c.Content)
+		}
+		l = append(l, x)
+	}
+	errs := make([]string, 0, len(p.Errors))
+	for _, e := range p.Errors {
+		errs = append(errs, e.Key+" "+e.Message)
+	}
+	sort.Strings(errs)
+	b, _ := json.Marshal(struct {
+		Target  string   `json:"target"`
+		Changes []ch     `json:"changes"`
+		Errors  []string `json:"errors"`
+	}{p.Target, l, errs})
+	sum := sha256.Sum256(b)
+	return hex.EncodeToString(sum[:12])
+}

internal/nodesync/read.go

@@ -0,0 +1,337 @@
+package nodesync
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+	"math/bits"
+	"net/url"
+	"path"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"pve-confsync/internal/model"
+	"pve-confsync/internal/pve"
+)
+
+// Shell runs a command as root on the node, stdin fed with in (sshx.Client).
+type Shell interface {
+	Run(ctx context.Context, cmd string, in []byte) ([]byte, error)
+}
+
+// Endpoint is one node: the API of its cluster (calls proxied to the node)
+// and SSH as root on the node itself.
+type Endpoint struct {
+	Key     string // "cluster/node"
+	Cluster string
+	Node    string
+	API     *pve.Client
+	Shell   Shell // nil: SSH host key not recorded yet
+	// Dial opens a new SSH access, to check that the node still answers after
+	// a network reload (nil: Shell is used).
+	Dial func() (Shell, error)
+}
+
+// File is a file read on a node.
+type File struct {
+	Path    string `json:"path"`
+	Mode    string `json:"mode"`  // octal, e.g. 644
+	Owner   string `json:"owner"` // user:group
+	Hash    string `json:"hash"`  // SHA-256 of the content
+	Content []byte `json:"-"`
+}
+
+// Snapshot is what was read on one node.
+type Snapshot struct {
+	Key     string                     `json:"key"`
+	Version string                     `json:"version,omitempty"`
+	Network []model.Object             `json:"-"` // interfaces (id = iface), canonical fields
+	Options map[string][]string        `json:"-"` // iface -> lines the API does not write (bond-miimon…)
+	DNS     map[string]any             `json:"-"`
+	Files   map[string]map[string]File `json:"-"` // item -> path -> file
+	// Pending: /etc/network/interfaces.new exists (network changes not applied yet).
+	Pending  bool      `json:"pending,omitempty"`
+	Warnings []string  `json:"warnings,omitempty"`
+	ReadAt   time.Time `json:"readAt"`
+}
+
+// maxFile: larger files are not replicated (configuration files are small).
+const maxFile = 1 << 20
+
+// Read loads the given items from a node.
+func Read(ctx context.Context, ep Endpoint, items []string) (Snapshot, error) {
+	snap := Snapshot{Key: ep.Key, Files: map[string]map[string]File{}, ReadAt: time.Now().UTC()}
+	v, err := NodeVersion(ctx, ep)
+	if err != nil {
+		return snap, err
+	}
+	snap.Version = v
+	var files []Item
+	for _, id := range items {
+		it, err := ItemByID(id)
+		if err != nil {
+			return snap, err
+		}
+		switch it.Kind {
+		case KindNetwork:
+			if snap.Network, snap.Options, err = readNetwork(ctx, ep); err != nil {
+				return snap, fmt.Errorf("%s: read network: %w", ep.Key, err)
+			}
+		case KindDNS:
+			if snap.DNS, err = readDNS(ctx, ep); err != nil {
+				return snap, fmt.Errorf("%s: read dns: %w", ep.Key, err)
+			}
+		case KindFile:
+			files = append(files, it)
+		}
+	}
+	// SSH is needed for the files, and for the pending network changes
+	if ep.Shell == nil {
+		return snap, fmt.Errorf("%s: SSH host key not recorded yet, test the node first", ep.Key)
+	}
+	out, err := ep.Shell.Run(ctx, readScript(files), nil)
+	if err != nil {
+		return snap, fmt.Errorf("%s: read files: %w", ep.Key, err)
+	}
+	got, pending, warns, err := parseRead(out)
+	if err != nil {
+		return snap, fmt.Errorf("%s: read files: %w", ep.Key, err)
+	}
+	snap.Pending = pending
+	snap.Warnings = append(snap.Warnings, warns...)
+	for _, it := range files {
+		m := map[string]File{}
+		for p, f := range got {
+			if matchItem(it, p) {
+				m[p] = f
+			}
+		}
+		snap.Files[it.ID] = m
+	}
+	return snap, nil
+}
+
+// NodeVersion reads the pve-manager version of the node and refuses anything
+// but PVE 9.x.
+func NodeVersion(ctx context.Context, ep Endpoint) (string, error) {
+	var v struct {
+		Version string `json:"version"`
+		Release string `json:"release"`
+	}
+	if err := ep.API.Get(ctx, "nodes/"+url.PathEscape(ep.Node)+"/version", nil, &v); err != nil {
+		return "", fmt.Errorf("%s: version check: %w", ep.Key, err)
+	}
+	ref := v.Release
+	if ref == "" {
+		ref = v.Version
+	}
+	if major, err := strconv.Atoi(strings.SplitN(ref, ".", 2)[0]); err != nil || major != pve.SupportedMajor {
+		return v.Version, fmt.Errorf("%s: node runs PVE %q, only %d.x is supported", ep.Key, ref, pve.SupportedMajor)
+	}
+	return v.Version, nil
+}
+
+func matchItem(it Item, p string) bool {
+	for _, x := range it.Exclude {
+		if x == p {
+			return false
+		}
+	}
+	for _, pat := range it.Paths {
+		if ok, _ := path.Match(pat, p); ok {
+			return true
+		}
+	}
+	return false
+}
+
+// readScript lists the files of the items: one header line per file
+// ("F <mode> <user:group> <path>", "L <size> <path>" when too large) then
+// its content in base64 on one line; "PENDING" when interfaces.new exists.
+// The first line is a marker the fake node of the tests understands.
+func readScript(files []Item) string {
+	var pats []string
+	for _, it := range files {
+		pats = append(pats, it.Paths...)
+	}
+	var b strings.Builder
+	b.WriteString("# confsync:read\n")
+	if len(pats) > 0 {
+		fmt.Fprintf(&b, "for f in %s; do\n", strings.Join(pats, " "))
+		b.WriteString(`  [ -f "$f" ] && [ ! -L "$f" ] || continue
+  s=$(stat -c %s "$f")
+  if [ "$s" -gt ` + strconv.Itoa(maxFile) + ` ]; then printf 'L %s %s\n' "$s" "$f"; continue; fi
+  printf 'F %s %s\n' "$(stat -c '%a %U:%G' "$f")" "$f"
+  base64 -w0 < "$f"; echo
+done
+`)
+	}
+	b.WriteString("[ -e /etc/network/interfaces.new ] && echo PENDING\ntrue\n")
+	return b.String()
+}
+
+func parseRead(out []byte) (map[string]File, bool, []string, error) {
+	files := map[string]File{}
+	pending := false
+	var warns []string
+	sc := bufio.NewScanner(bytes.NewReader(out))
+	sc.Buffer(make([]byte, 64<<10), 4*maxFile)
+	for sc.Scan() {
+		line := sc.Text()
+		switch {
+		case line == "PENDING":
+			pending = true
+		case strings.HasPrefix(line, "L "):
+			p := strings.SplitN(line, " ", 3)
+			if len(p) == 3 {
+				warns = append(warns, fmt.Sprintf("%s: %s bytes, too large, not replicated", p[2], p[1]))
+			}
+		case strings.HasPrefix(line, "F "):
+			p := strings.SplitN(line, " ", 4)
+			if len(p) != 4 || !sc.Scan() {
+				return nil, false, nil, fmt.Errorf("unexpected output %q", line)
+			}
+			data, err := base64.StdEncoding.DecodeString(strings.TrimSpace(sc.Text()))
+			if err != nil {
+				return nil, false, nil, fmt.Errorf("%s: %w", p[3], err)
+			}
+			if !safePath(p[3]) {
+				warns = append(warns, fmt.Sprintf("%s: unexpected characters in the path, not replicated", p[3]))
+				continue
+			}
+			files[p[3]] = File{Path: p[3], Mode: p[1], Owner: p[2], Hash: hash(data), Content: data}
+		}
+	}
+	return files, pending, warns, sc.Err()
+}
+
+// safePath: the paths go into shell scripts between single quotes.
+func safePath(p string) bool {
+	if !strings.HasPrefix(p, "/") || strings.Contains(p, "..") {
+		return false
+	}
+	for _, r := range p {
+		if !(r == '/' || r == '.' || r == '-' || r == '_' || r == '+' || r == '@' ||
+			(r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9')) {
+			return false
+		}
+	}
+	return true
+}
+
+func hash(b []byte) string {
+	s := sha256.Sum256(b)
+	return hex.EncodeToString(s[:])
+}
+
+// ---- network (GET /nodes/{node}/network) ----
+
+// netWritable are the parameters of POST/PUT /nodes/{node}/network[/{iface}]
+// (pve-manager PVE/API2/Network.pm, $confdesc), in their cidr form: the GET
+// returns address/netmask and cidr, the canonical form keeps cidr only.
+var netWritable = map[string]bool{
+	"type": true, "comments": true, "comments6": true, "autostart": true,
+	"bridge_vlan_aware": true, "bridge_vids": true, "bridge_ports": true,
+	"ovs_ports": true, "ovs_tag": true, "ovs_options": true, "ovs_bridge": true, "ovs_bonds": true,
+	"slaves": true, "bond_mode": true, "bond-primary": true, "bond_xmit_hash_policy": true,
+	"vlan-raw-device": true, "vlan-id": true,
+	"gateway": true, "cidr": true, "gateway6": true, "cidr6": true, "mtu": true,
+}
+
+// netTypes are the interface types handled (fabric, vnet and unknown are not).
+var netTypes = map[string]bool{
+	"eth": true, "bond": true, "bridge": true, "vlan": true, "alias": true,
+	"OVSBridge": true, "OVSBond": true, "OVSPort": true, "OVSIntPort": true,
+}
+
+// physical interfaces cannot be created: they must exist on the target.
+func physical(typ string) bool { return typ == "eth" }
+
+func readNetwork(ctx context.Context, ep Endpoint) ([]model.Object, map[string][]string, error) {
+	var raw []map[string]any
+	if err := ep.API.Get(ctx, "nodes/"+url.PathEscape(ep.Node)+"/network", nil, &raw); err != nil {
+		return nil, nil, err
+	}
+	var objs []model.Object
+	opts := map[string][]string{}
+	for _, r := range raw {
+		id := pve.Str(r["iface"])
+		typ := pve.Str(r["type"])
+		if id == "" || id == "lo" || !netTypes[typ] {
+			continue
+		}
+		f := map[string]any{}
+		for k, v := range r {
+			if !netWritable[k] || v == nil {
+				continue
+			}
+			s := strings.TrimRight(pve.Str(v), "\n")
+			if s == "" {
+				continue
+			}
+			f[k] = s
+		}
+		// a static address without cidr (older parsers): rebuild it
+		if _, ok := f["cidr"]; !ok {
+			if a, m := pve.Str(r["address"]), pve.Str(r["netmask"]); a != "" && m != "" {
+				f["cidr"] = a + "/" + maskBits(m)
+			}
+		}
+		if l := listOf(r["options"]); len(l) > 0 {
+			opts[id] = l
+		}
+		if l := listOf(r["options6"]); len(l) > 0 {
+			opts[id] = append(opts[id], l...)
+		}
+		objs = append(objs, model.Object{Kind: "network", ID: id, Fields: f})
+	}
+	sort.Slice(objs, func(i, j int) bool { return objs[i].ID < objs[j].ID })
+	return objs, opts, nil
+}
+
+func listOf(v any) []string {
+	l, _ := v.([]any)
+	out := make([]string, 0, len(l))
+	for _, e := range l {
+		out = append(out, pve.Str(e))
+	}
+	sort.Strings(out)
+	return out
+}
+
+// maskBits converts 255.255.255.0 to 24 (a prefix length is kept).
+func maskBits(m string) string {
+	if !strings.Contains(m, ".") {
+		return m
+	}
+	n := 0
+	for _, p := range strings.Split(m, ".") {
+		b, _ := strconv.Atoi(p)
+		n += bits.OnesCount8(uint8(b))
+	}
+	return strconv.Itoa(n)
+}
+
+// ---- dns (GET /nodes/{node}/dns) ----
+
+var dnsFields = []string{"search", "dns1", "dns2", "dns3"}
+
+func readDNS(ctx context.Context, ep Endpoint) (map[string]any, error) {
+	var raw map[string]any
+	if err := ep.API.Get(ctx, "nodes/"+url.PathEscape(ep.Node)+"/dns", nil, &raw); err != nil {
+		return nil, err
+	}
+	out := map[string]any{}
+	for _, k := range dnsFields {
+		if s := pve.Str(raw[k]); s != "" {
+			out[k] = s
+		}
+	}
+	return out, nil
+}

internal/nodesync/script_test.go

@@ -0,0 +1,77 @@
+package nodesync
+
+import (
+	"os"
+	"os/exec"
+	"os/user"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// TestPrepareScriptForReal runs the prepare script and restore.sh with the
+// local shell in a temporary folder (every path prefixed with it).
+func TestPrepareScriptForReal(t *testing.T) {
+	if _, err := exec.LookPath("sha256sum"); err != nil {
+		t.Skip("sha256sum not installed")
+	}
+	root := t.TempDir()
+	u, _ := user.Current()
+	g, _ := user.LookupGroupId(u.Gid)
+	owner := u.Username + ":" + g.Name
+	write := func(p, s string) {
+		if err := os.MkdirAll(filepath.Dir(root+p), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(root+p, []byte(s), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	read := func(p string) string {
+		b, err := os.ReadFile(root + p)
+		if err != nil {
+			return "<absent>"
+		}
+		return string(b)
+	}
+	write("/etc/sysctl.d/90-net.conf", "old\n")
+	write("/etc/sysctl.d/99-local.conf", "local\n")
+	write("/etc/network/interfaces", "auto lo\n")
+	plan := Plan{Changes: []Change{
+		{Item: "sysctl", ID: "/etc/sysctl.d/90-net.conf", Op: OpUpdate, Hash: hash([]byte("old\n")), Mode: "640", Owner: owner, Content: []byte("new\n")},
+		{Item: "multipath", ID: "/etc/multipath.conf", Op: OpCreate, Hash: "absent", Mode: "644", Owner: owner, Content: []byte("defaults {}\n")},
+		{Item: "sysctl", ID: "/etc/sysctl.d/99-local.conf", Op: OpDelete, Hash: hash([]byte("local\n"))},
+		{Item: "network", ID: "vmbr0", Op: OpUpdate},
+	}}
+	run := func(script string, in []byte) (string, error) {
+		cmd := exec.Command("sh", "-c", script)
+		cmd.Stdin = strings.NewReader(string(in))
+		out, err := cmd.CombinedOutput()
+		return string(out), err
+	}
+	const dir = "/var/lib/pve-confsync/backup/test"
+	if out, err := run(prepareScriptIn(dir, root), prepareInput(plan, true, false)); err != nil || strings.TrimSpace(out) != "ok" {
+		t.Fatalf("prepare: %v %s", err, out)
+	}
+	if read("/etc/sysctl.d/90-net.conf") != "new\n" || read("/etc/multipath.conf") != "defaults {}\n" || read("/etc/sysctl.d/99-local.conf") != "<absent>" {
+		t.Fatalf("writes: %q %q %q", read("/etc/sysctl.d/90-net.conf"), read("/etc/multipath.conf"), read("/etc/sysctl.d/99-local.conf"))
+	}
+	if fi, _ := os.Stat(root + "/etc/sysctl.d/90-net.conf"); fi.Mode().Perm() != 0o640 {
+		t.Errorf("mode %v", fi.Mode())
+	}
+	if m, _ := filepath.Glob(root + "/etc/sysctl.d/*.confsync.*"); len(m) > 0 {
+		t.Errorf("temporary files left: %v", m)
+	}
+	// a second run with the same plan: the files changed, nothing is written
+	if out, err := run(prepareScriptIn(dir+"2", root), prepareInput(plan, false, false)); err == nil || !strings.Contains(out, "changed since the dry-run") {
+		t.Errorf("stale check: %v %s", err, out)
+	}
+	write("/etc/network/interfaces.new", "pending\n")
+	if out, err := run("sh '"+root+dir+"/restore.sh'", nil); err != nil {
+		t.Fatalf("restore: %v %s", err, out)
+	}
+	if read("/etc/sysctl.d/90-net.conf") != "old\n" || read("/etc/multipath.conf") != "<absent>" || read("/etc/sysctl.d/99-local.conf") != "local\n" ||
+		read("/etc/network/interfaces") != "auto lo\n" || read("/etc/network/interfaces.new") != "<absent>" {
+		t.Errorf("restore: %q %q %q %q", read("/etc/sysctl.d/90-net.conf"), read("/etc/multipath.conf"), read("/etc/sysctl.d/99-local.conf"), read("/etc/network/interfaces.new"))
+	}
+}

internal/pve/pvetest/fake.go

@@ -64,6 +64,7 @@ type Server struct {
 	Shadow   map[string]string         // user without realm -> hash (priv/shadow.cfg)
 	FailOn   string                    // "METHOD path" answering 500, to test rollbacks
 	Calls    []string                  // "METHOD path" of every write, "SSH write" for shadow.cfg
+	Nodes    []*Node                   // nodes of the node replication (AddNode); none: one node "n1"
 	locks    int
 	ssh      *sshServer
 }
@@ -83,6 +84,9 @@ func New(version string) *Server {
 // Close stops the API and the SSH server.
 func (s *Server) Close() {
 	s.ssh.close()
+	for _, n := range s.Nodes {
+		n.ssh.close()
+	}
 	s.Server.Close()
 }
 
@@ -146,14 +150,26 @@ func (s *Server) handle(w http.ResponseWriter, r *http.Request) {
 	if running {
 		cfg = &s.Running
 	}
+	var nd *Node
+	if len(seg) >= 3 && seg[0] == "nodes" {
+		nd = s.node(seg[1])
+	}
 	switch {
+	case nd != nil && r.Method == "GET" && len(seg) == 3 && seg[2] == "version":
+		reply(w, map[string]any{"version": nd.version(), "release": release(nd.version()), "repoid": "abcdef"})
+	case nd != nil && (seg[2] == "network" || seg[2] == "dns"):
+		s.nodeAPI(w, r, nd, seg)
 	case key == "GET version" || (r.Method == "GET" && len(seg) == 3 && seg[0] == "nodes" && seg[2] == "version"):
 		reply(w, map[string]any{"version": s.Version, "release": release(s.Version), "repoid": "abcdef"})
 	case key == "GET cluster/status":
-		reply(w, []any{
-			map[string]any{"type": "cluster", "name": "fake", "quorate": 1},
-			map[string]any{"type": "node", "name": "n1", "online": 1},
-		})
+		st := []any{map[string]any{"type": "cluster", "name": "fake", "quorate": 1}}
+		if len(s.Nodes) == 0 {
+			st = append(st, map[string]any{"type": "node", "name": "n1", "online": 1})
+		}
+		for i, n := range s.Nodes {
+			st = append(st, map[string]any{"type": "node", "name": n.Name, "online": 1, "nodeid": i + 1, "ip": "127.0.0.1"})
+		}
+		reply(w, st)
 	case r.Method == "GET" && len(seg) == 4 && seg[0] == "nodes" && seg[2] == "sdn" && seg[3] == "zones":
 		var out []any
 		for id := range s.Running.Zones {

internal/pve/pvetest/nodes.go

@@ -0,0 +1,419 @@
+package pvetest
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"path"
+	"sort"
+	"strings"
+
+	"golang.org/x/crypto/ssh"
+)
+
+// FakeFile is a file of a fake node.
+type FakeFile struct {
+	Content string
+	Mode    string // 644 by default
+	Owner   string // root:root by default
+}
+
+// Node is a node of a fake cluster for the node replication: its own SSH
+// server (files, reloads, restore timer), and its network interfaces and DNS
+// servers served by the API of the cluster (/nodes/{node}/network, /dns).
+// Like PVE, the interface writes go to a pending configuration
+// (interfaces.new) applied by PUT /nodes/{node}/network.
+type Node struct {
+	Name     string
+	Hostname string // answered by hostname (default: Name)
+	Version  string // pve-manager version (default: the cluster's)
+	Files    map[string]FakeFile
+	Net      map[string]map[string]any // iface -> fields as written (cidr, type…)
+	Pending  map[string]map[string]any // interfaces.new; nil: no pending change
+	DNS      map[string]string
+	Reloads  []string        // reload commands run, "network" for PUT /network
+	Armed    map[string]bool // restore timers armed (systemd-run units)
+	FailOn   string          // "reload <cmd>", "prepare" or "probe": that step fails
+	Calls    []string        // SSH steps run
+	backups  map[string]backup
+	srv      *Server
+	ssh      *sshServer
+}
+
+type backup struct {
+	files   map[string]*FakeFile // nil: absent before
+	net     map[string]map[string]any
+	pending map[string]map[string]any
+	dns     map[string]string
+	hasNet  bool
+	hasDNS  bool
+}
+
+// AddNode adds a node to the fake cluster (listed by GET /cluster/status).
+func (s *Server) AddNode(name string) *Node {
+	n := &Node{Name: name, Hostname: name, Files: map[string]FakeFile{}, Net: map[string]map[string]any{},
+		DNS: map[string]string{}, Armed: map[string]bool{}, backups: map[string]backup{}, srv: s}
+	n.ssh = newSSH(func(user, pass string) bool {
+		s.mu.Lock()
+		defer s.mu.Unlock()
+		return user == "root" && pass == s.Password
+	}, n.exec)
+	s.mu.Lock()
+	s.Nodes = append(s.Nodes, n)
+	s.mu.Unlock()
+	return n
+}
+
+// SSHAddr returns the host and port of the SSH server of the node.
+func (n *Node) SSHAddr() (string, int) {
+	a := n.ssh.ln.Addr().(*net.TCPAddr)
+	return a.IP.String(), a.Port
+}
+
+// SSHHostKey returns the host key of the node (authorized_keys format).
+func (n *Node) SSHHostKey() string {
+	return strings.TrimSpace(string(ssh.MarshalAuthorizedKey(n.ssh.key)))
+}
+
+func (s *Server) node(name string) *Node {
+	for _, n := range s.Nodes {
+		if n.Name == name {
+			return n
+		}
+	}
+	return nil
+}
+
+func (n *Node) version() string {
+	if n.Version != "" {
+		return n.Version
+	}
+	return n.srv.Version
+}
+
+func cloneNet(m map[string]map[string]any) map[string]map[string]any {
+	if m == nil {
+		return nil
+	}
+	out := map[string]map[string]any{}
+	for k, v := range m {
+		f := map[string]any{}
+		for x, y := range v {
+			f[x] = y
+		}
+		out[k] = f
+	}
+	return out
+}
+
+// ---- API (called with s.mu held) ----
+
+func (s *Server) nodeAPI(w http.ResponseWriter, r *http.Request, n *Node, seg []string) {
+	switch {
+	case len(seg) == 3 && seg[2] == "dns" && r.Method == "GET":
+		out := map[string]any{}
+		for k, v := range n.DNS {
+			out[k] = v
+		}
+		reply(w, out)
+	case len(seg) == 3 && seg[2] == "dns" && r.Method == "PUT":
+		if r.PostForm.Get("search") == "" {
+			fail(w, 400, "search: property is missing and it is not optional")
+			return
+		}
+		n.DNS = map[string]string{}
+		for _, k := range []string{"search", "dns1", "dns2", "dns3"} {
+			if v := r.PostForm.Get(k); v != "" {
+				n.DNS[k] = v
+			}
+		}
+		reply(w, nil)
+	case len(seg) == 3 && seg[2] == "network" && r.Method == "GET":
+		cfg := n.Net
+		if n.Pending != nil {
+			cfg = n.Pending
+		}
+		var out []any
+		prio := 0
+		for _, id := range sortedKeys(cfg) {
+			prio++
+			f := map[string]any{"iface": id, "priority": prio, "families": []any{"inet"}, "method": "manual", "active": 1}
+			for k, v := range cfg[id] {
+				f[k] = v
+			}
+			if c, ok := f["cidr"].(string); ok {
+				a, m, _ := strings.Cut(c, "/")
+				f["address"], f["netmask"], f["method"] = a, m, "static"
+			}
+			if f["type"] == "eth" {
+				f["exists"] = 1
+			}
+			out = append(out, f)
+		}
+		reply(w, out)
+	case len(seg) == 3 && seg[2] == "network" && r.Method == "DELETE":
+		n.Pending = nil
+		reply(w, nil)
+	case len(seg) == 3 && seg[2] == "network" && r.Method == "PUT":
+		if n.Pending != nil {
+			n.Net, n.Pending = n.Pending, nil
+		}
+		n.Reloads = append(n.Reloads, "network")
+		reply(w, "UPID:"+n.Name+":00001234:00005678:65000000:srvreload:networking:root@pam:")
+	case len(seg) == 3 && seg[2] == "network" && r.Method == "POST":
+		id := r.PostForm.Get("iface")
+		cfg := n.pending()
+		if cfg[id] != nil {
+			fail(w, 400, "iface: interface already exists")
+			return
+		}
+		cfg[id] = fields(r.PostForm, "iface")
+		reply(w, nil)
+	case len(seg) == 4 && seg[2] == "network":
+		cfg := n.pending()
+		cur := cfg[seg[3]]
+		if cur == nil {
+			fail(w, 400, "iface: interface does not exist")
+			return
+		}
+		switch r.Method {
+		case "DELETE":
+			delete(cfg, seg[3])
+		case "PUT":
+			if r.PostForm.Get("type") == "" {
+				fail(w, 400, "type: property is missing and it is not optional")
+				return
+			}
+			for _, d := range strings.Split(r.PostForm.Get("delete"), ",") {
+				delete(cur, d)
+			}
+			// like PVE: without an address in the parameters, the interface
+			// becomes "manual" and loses its address
+			if r.PostForm.Get("cidr") == "" {
+				delete(cur, "cidr")
+			}
+			for k, v := range fields(r.PostForm, "delete") {
+				cur[k] = v
+			}
+		default:
+			fail(w, 501, "not implemented in fake")
+			return
+		}
+		reply(w, nil)
+	default:
+		fail(w, 501, "not implemented in fake: "+r.Method+" "+strings.Join(seg, "/"))
+	}
+}
+
+// pending returns interfaces.new, created from the current configuration.
+func (n *Node) pending() map[string]map[string]any {
+	if n.Pending == nil {
+		n.Pending = cloneNet(n.Net)
+	}
+	return n.Pending
+}
+
+// ---- SSH ----
+
+func (n *Node) exec(cmd string, in io.Reader, out, errw io.Writer) int {
+	s := n.srv
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	first, _, _ := strings.Cut(cmd, "\n")
+	verb, arg, _ := strings.Cut(strings.TrimPrefix(first, "# confsync:"), " ")
+	if strings.HasPrefix(first, "# confsync:") {
+		n.Calls = append(n.Calls, verb)
+		if n.FailOn == verb || n.FailOn == verb+" "+arg {
+			fmt.Fprintln(errw, "injected failure")
+			return 1
+		}
+	}
+	switch {
+	case strings.HasPrefix(cmd, "pveversion"):
+		fmt.Fprintf(out, "pve-manager/%s/abcdef (running kernel: 6.14.8-2-pve)\n", n.version())
+		if strings.Contains(cmd, "hostname") {
+			fmt.Fprintln(out, n.Hostname)
+		}
+		return 0
+	case verb == "read":
+		return n.read(cmd, out)
+	case verb == "prepare":
+		data, _ := io.ReadAll(in)
+		return n.prepare(arg, data, errw)
+	case verb == "restore":
+		b, ok := n.backups[arg]
+		if !ok {
+			fmt.Fprintln(errw, "no backup "+arg)
+			return 1
+		}
+		n.restore(b)
+		return 0
+	case verb == "arm":
+		n.Armed[arg] = true
+		return 0
+	case verb == "disarm":
+		delete(n.Armed, arg)
+		return 0
+	case verb == "reload", verb == "probe":
+		if verb == "reload" {
+			n.Reloads = append(n.Reloads, arg)
+		}
+		return 0
+	}
+	fmt.Fprintf(errw, "fake node: unknown command %q\n", first)
+	return 127
+}
+
+// Fire runs the armed restore timers (as systemd would after the delay).
+func (n *Node) Fire() {
+	n.srv.mu.Lock()
+	defer n.srv.mu.Unlock()
+	for unit := range n.Armed {
+		dir := strings.Replace(unit, "pve-confsync-restore-", "/var/lib/pve-confsync/backup/", 1)
+		if b, ok := n.backups[dir]; ok {
+			n.restore(b)
+		}
+		n.Reloads = append(n.Reloads, "ifreload -a")
+		delete(n.Armed, unit)
+	}
+}
+
+func (n *Node) read(cmd string, out io.Writer) int {
+	var pats []string
+	for _, l := range strings.Split(cmd, "\n") {
+		if strings.HasPrefix(l, "for f in ") {
+			pats = strings.Fields(strings.TrimSuffix(strings.TrimPrefix(l, "for f in "), "; do"))
+		}
+	}
+	for _, p := range sortedKeys(n.Files) {
+		for _, pat := range pats {
+			if ok, _ := path.Match(pat, p); ok {
+				f := n.Files[p]
+				fmt.Fprintf(out, "F %s %s %s\n%s\n", def(f.Mode, "644"), def(f.Owner, "root:root"), p,
+					base64.StdEncoding.EncodeToString([]byte(f.Content)))
+				break
+			}
+		}
+	}
+	if n.Pending != nil {
+		fmt.Fprintln(out, "PENDING")
+	}
+	return 0
+}
+
+func def(v, d string) string {
+	if v == "" {
+		return d
+	}
+	return v
+}
+
+// prepare emulates the prepare script: checks, backups, writes.
+func (n *Node) prepare(dir string, plan []byte, errw io.Writer) int {
+	type op struct{ op, mode, owner, hash, path, data string }
+	var ops []op
+	sc := bufio.NewScanner(bytes.NewReader(plan))
+	sc.Buffer(make([]byte, 64<<10), 8<<20)
+	for sc.Scan() {
+		p := strings.SplitN(sc.Text(), " ", 5)
+		if len(p) != 5 {
+			continue
+		}
+		o := op{p[0], p[1], p[2], p[3], p[4], ""}
+		if o.op == "W" && sc.Scan() {
+			o.data = sc.Text()
+		}
+		ops = append(ops, o)
+	}
+	for _, o := range ops {
+		if o.op != "W" && o.op != "D" {
+			continue
+		}
+		h := "absent"
+		if f, ok := n.Files[o.path]; ok {
+			h = sha([]byte(f.Content))
+		}
+		if h != o.hash {
+			fmt.Fprintf(errw, "changed since the dry-run: %s\n", o.path)
+			return 3
+		}
+	}
+	b := backup{files: map[string]*FakeFile{}}
+	for _, o := range ops {
+		switch o.path {
+		case "/etc/network/interfaces", "/etc/network/interfaces.new":
+			b.hasNet, b.net, b.pending = true, cloneNet(n.Net), cloneNet(n.Pending)
+			continue
+		case "/etc/resolv.conf":
+			b.hasDNS, b.dns = true, map[string]string{}
+			for k, v := range n.DNS {
+				b.dns[k] = v
+			}
+			continue
+		}
+		if f, ok := n.Files[o.path]; ok {
+			c := f
+			b.files[o.path] = &c
+		} else {
+			b.files[o.path] = nil
+		}
+	}
+	n.backups[dir] = b
+	for _, o := range ops {
+		switch o.op {
+		case "W":
+			data, err := base64.StdEncoding.DecodeString(o.data)
+			if err != nil {
+				n.restore(b)
+				fmt.Fprintln(errw, "write failed: "+o.path)
+				return 4
+			}
+			n.Files[o.path] = FakeFile{Content: string(data), Mode: o.mode, Owner: o.owner}
+		case "D":
+			delete(n.Files, o.path)
+		}
+	}
+	return 0
+}
+
+func (n *Node) restore(b backup) {
+	for p, f := range b.files {
+		if f == nil {
+			delete(n.Files, p)
+		} else {
+			n.Files[p] = *f
+		}
+	}
+	if b.hasNet {
+		n.Net, n.Pending = cloneNet(b.net), cloneNet(b.pending)
+	}
+	if b.hasDNS {
+		n.DNS = map[string]string{}
+		for k, v := range b.dns {
+			n.DNS[k] = v
+		}
+	}
+}
+
+// Backups lists the backup folders written on the node.
+func (n *Node) Backups() []string {
+	n.srv.mu.Lock()
+	defer n.srv.mu.Unlock()
+	var out []string
+	for d := range n.backups {
+		out = append(out, d)
+	}
+	sort.Strings(out)
+	return out
+}
+
+func sha(b []byte) string {
+	h := sha256.Sum256(b)
+	return hex.EncodeToString(h[:])
+}

internal/pve/pvetest/ssh.go

@@ -17,18 +17,27 @@ import (
 // sshServer answers the commands pve-confsync runs on a node: pveversion, the
 // read of priv/shadow.cfg and its locked write (adapter password).
 type sshServer struct {
-	ln  net.Listener
-	key ssh.PublicKey
+	ln   net.Listener
+	key  ssh.PublicKey
+	exec func(cmd string, in io.Reader, out, errw io.Writer) int
 }
 
 func startSSH(s *Server) *sshServer {
+	return newSSH(func(user, pass string) bool {
+		s.mu.Lock()
+		defer s.mu.Unlock()
+		return user == "root" && pass == s.Password
+	}, s.exec)
+}
+
+// newSSH starts an SSH server accepting the password check ok and running
+// the commands with exec.
+func newSSH(ok func(user, pass string) bool, exec func(cmd string, in io.Reader, out, errw io.Writer) int) *sshServer {
 	_, priv, _ := ed25519.GenerateKey(rand.Reader)
 	signer, _ := ssh.NewSignerFromKey(priv)
 	cfg := &ssh.ServerConfig{
 		PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
-			s.mu.Lock()
-			defer s.mu.Unlock()
-			if c.User() == "root" && string(pass) == s.Password {
+			if ok(c.User(), string(pass)) {
 				return nil, nil
 			}
 			return nil, fmt.Errorf("denied")
@@ -39,14 +48,14 @@ func startSSH(s *Server) *sshServer {
 	if err != nil {
 		panic(err)
 	}
-	srv := &sshServer{ln: ln, key: signer.PublicKey()}
+	srv := &sshServer{ln: ln, key: signer.PublicKey(), exec: exec}
 	go func() {
 		for {
 			nc, err := ln.Accept()
 			if err != nil {
 				return
 			}
-			go s.serveSSH(nc, cfg)
+			go srv.serve(nc, cfg)
 		}
 	}()
 	return srv
@@ -65,7 +74,7 @@ func (s *Server) SSHHostKey() string {
 	return strings.TrimSpace(string(ssh.MarshalAuthorizedKey(s.ssh.key)))
 }
 
-func (s *Server) serveSSH(nc net.Conn, cfg *ssh.ServerConfig) {
+func (x *sshServer) serve(nc net.Conn, cfg *ssh.ServerConfig) {
 	_, chans, reqs, err := ssh.NewServerConn(nc, cfg)
 	if err != nil {
 		nc.Close()
@@ -91,7 +100,7 @@ func (s *Server) serveSSH(nc net.Conn, cfg *ssh.ServerConfig) {
 				var p struct{ Cmd string }
 				_ = ssh.Unmarshal(req.Payload, &p)
 				_ = req.Reply(true, nil)
-				status := s.exec(p.Cmd, ch, ch, ch.Stderr())
+				status := x.exec(p.Cmd, ch, ch, ch.Stderr())
 				b := make([]byte, 4)
 				binary.BigEndian.PutUint32(b, uint32(status))
 				_, _ = ch.SendRequest("exit-status", false, b)

internal/server/nodes.go

@@ -0,0 +1,580 @@
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"pve-confsync/internal/nodesync"
+	"pve-confsync/internal/pve"
+	"pve-confsync/internal/service"
+	"pve-confsync/internal/sshx"
+	"pve-confsync/internal/store"
+)
+
+// ---- node replication: nodes (SSH access), profiles, state, dry-run, apply ----
+
+// handleNodes lists the nodes, the profiles and the catalog of items.
+func (s *Server) handleNodes(w http.ResponseWriter, r *http.Request) {
+	nodes, err := s.store.Nodes(r.Context())
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error(), "store")
+		return
+	}
+	profiles, err := s.store.Profiles(r.Context())
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error(), "store")
+		return
+	}
+	if identity(r).Role != store.RoleAdmin {
+		for i, n := range nodes {
+			nodes[i] = store.Node{Key: n.Key, Cluster: n.Cluster, Node: n.Node, ClusterRole: n.ClusterRole, LastCheck: n.LastCheck}
+		}
+	}
+	writeJSON(w, http.StatusOK, map[string]any{"nodes": nodes, "profiles": profiles, "items": nodesync.Items,
+		"defaults": nodesync.DefaultPolicies})
+}
+
+// handleDiscover reads the nodes of every cluster (GET /cluster/status) and
+// records them with their address in the cluster.
+func (s *Server) handleDiscover(w http.ResponseWriter, r *http.Request) {
+	cls, err := s.store.Clusters(r.Context())
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error(), "store")
+		return
+	}
+	ctx, cancel := context.WithTimeout(r.Context(), readTimeout)
+	defer cancel()
+	errs := map[string]string{}
+	found := 0
+	for _, c := range cls {
+		if c.Role == store.ClusterDisabled {
+			continue
+		}
+		_, cc, err := s.store.Cluster(ctx, c.Name)
+		var nodes []service.DiscoveredNode
+		if err == nil {
+			nodes, err = service.DiscoverNodes(ctx, cc)
+		}
+		if err == nil {
+			var d []store.Discovered
+			for _, n := range nodes {
+				d = append(d, store.Discovered{Name: n.Name, IP: n.IP})
+			}
+			err = s.store.SetNodes(ctx, c.Name, d)
+			found += len(d)
+		}
+		if err != nil {
+			errs[c.Name] = err.Error()
+		}
+	}
+	s.audit(r, Identity{}, "nodes-discover", "all clusters", len(errs) == 0, fmt.Sprintf("%d node(s)", found))
+	s.invalidateNodes()
+	writeJSON(w, http.StatusOK, map[string]any{"found": found, "errors": errs})
+}
+
+func (s *Server) handleSaveNode(w http.ResponseWriter, r *http.Request) {
+	var req struct {
+		SSHHost       string `json:"sshHost"`
+		SSHPort       int    `json:"sshPort"`
+		Password      string `json:"password"`
+		ClearPassword bool   `json:"clearPassword"`
+	}
+	if err := readJSON(r, &req); err != nil {
+		writeError(w, http.StatusBadRequest, err.Error(), "request")
+		return
+	}
+	cl, node := r.PathValue("cluster"), r.PathValue("node")
+	err := s.store.SaveNode(r.Context(), cl, node, req.SSHHost, req.SSHPort, req.Password, req.ClearPassword)
+	detail := fmt.Sprintf("ssh=%s:%d", req.SSHHost, req.SSHPort)
+	if req.Password != "" {
+		detail += " password=changed"
+	}
+	if req.ClearPassword {
+		detail += " password=cluster"
+	}
+	s.audit(r, Identity{}, "node-save", store.NodeKey(cl, node), err == nil, errText(err, detail))
+	switch {
+	case errors.Is(err, store.ErrNotFound):
+		writeError(w, http.StatusNotFound, "unknown node", "notfound")
+	case err != nil:
+		writeError(w, http.StatusBadRequest, err.Error(), "invalid")
+	default:
+		s.invalidateNodes()
+		writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
+	}
+}
+
+// handleCheckNode tests SSH on a node; the first test records its host key.
+func (s *Server) handleCheckNode(w http.ResponseWriter, r *http.Request) {
+	var req struct {
+		Trust bool `json:"trust"`
+	}
+	if r.ContentLength > 0 {
+		if err := readJSON(r, &req); err != nil {
+			writeError(w, http.StatusBadRequest, err.Error(), "request")
+			return
+		}
+	}
+	cl, node := r.PathValue("cluster"), r.PathValue("node")
+	a, err := s.store.NodeAccessOf(r.Context(), cl, node)
+	if errors.Is(err, store.ErrNotFound) {
+		writeError(w, http.StatusNotFound, "unknown node", "notfound")
+		return
+	}
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error(), "store")
+		return
+	}
+	ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
+	defer cancel()
+	res := service.CheckNode(ctx, node, a.SSH, req.Trust)
+	chk := &store.Check{At: time.Now().UTC(), OK: res.OK, Error: res.Error, Version: res.Version, SSH: res.OK}
+	if err := s.store.PinNode(r.Context(), cl, node, res.SSHHostKey, chk); err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error(), "store")
+		return
+	}
+	detail := res.Error
+	if res.SSHHostKey != "" {
+		detail += " SSH key recorded " + res.SSHFingerprint
+	}
+	s.audit(r, Identity{}, "node-check", store.NodeKey(cl, node), res.OK, strings.TrimSpace(detail))
+	s.invalidateNodes()
+	writeJSON(w, http.StatusOK, res)
+}
+
+func (s *Server) handleForgetNode(w http.ResponseWriter, r *http.Request) {
+	cl, node := r.PathValue("cluster"), r.PathValue("node")
+	err := s.store.ForgetNode(r.Context(), cl, node)
+	s.audit(r, Identity{}, "node-forget", store.NodeKey(cl, node), err == nil, errText(err, "SSH key forgotten"))
+	if err != nil {
+		writeError(w, http.StatusNotFound, err.Error(), "notfound")
+		return
+	}
+	s.invalidateNodes()
+	writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
+}
+
+func (s *Server) handleSaveProfile(w http.ResponseWriter, r *http.Request) {
+	var p nodesync.Profile
+	if err := readJSON(r, &p); err != nil {
+		writeError(w, http.StatusBadRequest, err.Error(), "request")
+		return
+	}
+	old := r.PathValue("name")
+	if old == "-" {
+		old = ""
+	}
+	err := s.store.SaveProfile(r.Context(), old, p)
+	b, _ := json.Marshal(p)
+	s.audit(r, Identity{}, "profile-save", p.Name, err == nil, errText(err, string(b)))
+	if err != nil {
+		writeError(w, http.StatusBadRequest, err.Error(), "invalid")
+		return
+	}
+	s.invalidateNodes()
+	writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
+}
+
+func (s *Server) handleDeleteProfile(w http.ResponseWriter, r *http.Request) {
+	name := r.PathValue("name")
+	err := s.store.DeleteProfile(r.Context(), name)
+	s.audit(r, Identity{}, "profile-delete", name, err == nil, errText(err, ""))
+	if err != nil {
+		writeError(w, http.StatusNotFound, err.Error(), "notfound")
+		return
+	}
+	s.invalidateNodes()
+	writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
+}
+
+// endpoints builds the access to every node: one API client per cluster,
+// SSH as root on each node whose host key is recorded.
+func (s *Server) endpoints(ctx context.Context) (map[string]nodesync.Endpoint, error) {
+	acc, err := s.store.NodeAccesses(ctx)
+	if err != nil {
+		return nil, err
+	}
+	apis := map[string]*pve.Client{}
+	out := map[string]nodesync.Endpoint{}
+	for key, a := range acc {
+		c, ok := apis[a.Cluster.Name]
+		if !ok {
+			if c, err = a.Cluster.PVE(); err != nil {
+				return nil, err
+			}
+			apis[a.Cluster.Name] = c
+		}
+		ep := nodesync.Endpoint{Key: key, Cluster: a.Node.Cluster, Node: a.Node.Node, API: c}
+		if a.SSH.HostKey != "" {
+			if sh, err := sshx.New(a.SSH); err == nil {
+				ep.Shell = sh
+			}
+		}
+		out[key] = ep
+	}
+	return out, nil
+}
+
+func (s *Server) invalidateNodes() {
+	s.nodeMu.Lock()
+	s.nodeStates = nil
+	s.nodeMu.Unlock()
+}
+
+// handleNodeState returns the last comparison of a profile; refresh=1 reads
+// the nodes again.
+func (s *Server) handleNodeState(w http.ResponseWriter, r *http.Request) {
+	name := r.URL.Query().Get("profile")
+	s.nodeMu.Lock()
+	st := s.nodeStates[name]
+	s.nodeMu.Unlock()
+	if st == nil || r.URL.Query().Get("refresh") == "1" {
+		prof, err := s.store.Profile(r.Context(), name)
+		if errors.Is(err, store.ErrNotFound) {
+			writeError(w, http.StatusNotFound, "unknown profile", "notfound")
+			return
+		}
+		if err != nil {
+			writeError(w, http.StatusInternalServerError, err.Error(), "store")
+			return
+		}
+		eps, err := s.endpoints(r.Context())
+		if err != nil {
+			writeError(w, http.StatusInternalServerError, err.Error(), "config")
+			return
+		}
+		ctx, cancel := context.WithTimeout(r.Context(), readTimeout)
+		defer cancel()
+		st = nodesync.Analyze(ctx, &prof, eps, nil)
+		s.rememberNodes(st, true)
+	}
+	writeJSON(w, http.StatusOK, map[string]any{"state": st})
+}
+
+// rememberNodes keeps a comparison; a partial one (some targets) only
+// replaces the targets it read.
+func (s *Server) rememberNodes(st *nodesync.State, all bool) {
+	s.nodeMu.Lock()
+	defer s.nodeMu.Unlock()
+	if s.nodeStates == nil {
+		s.nodeStates = map[string]*nodesync.State{}
+	}
+	old := s.nodeStates[st.Profile]
+	if all || old == nil {
+		if all {
+			s.nodeStates[st.Profile] = st
+		}
+		return
+	}
+	cp := *old
+	cp.Master = st.Master
+	cp.Targets = append([]nodesync.Target(nil), old.Targets...)
+	for _, t := range st.Targets {
+		for i := range cp.Targets {
+			if cp.Targets[i].Key == t.Key {
+				cp.Targets[i] = t
+			}
+		}
+	}
+	s.nodeStates[st.Profile] = &cp
+}
+
+type nodeSelection struct {
+	Profile string            `json:"profile"`
+	Scope   string            `json:"scope"`   // global (several targets), client (one target) or object
+	Targets []string          `json:"targets"` // global: the targets (empty: all of them)
+	Client  string            `json:"client"`  // client and object scopes
+	Objects []string          `json:"objects"` // object scope: "item/id"
+	Digests map[string]string `json:"digests"`
+	Confirm string            `json:"confirm"` // network changes: target (profile name for several)
+}
+
+func (sel *nodeSelection) check() error {
+	if sel.Profile == "" {
+		return errors.New("profile required")
+	}
+	s := selection{Scope: sel.Scope, Targets: sel.Targets, Client: sel.Client, Objects: sel.Objects}
+	if err := s.check(); err != nil {
+		return err
+	}
+	sel.Targets, sel.Client, sel.Objects = s.Targets, s.Client, s.Objects
+	return nil
+}
+
+func (sel nodeSelection) clients() []string {
+	if sel.Client == "" {
+		return sel.Targets
+	}
+	return []string{sel.Client}
+}
+
+func (sel nodeSelection) target() string {
+	t := sel.Profile + ": "
+	switch {
+	case sel.Scope == "client":
+		t += sel.Client
+	case sel.Scope == "object":
+		t += sel.Client + ": " + strings.Join(sel.Objects, ", ")
+	case len(sel.Targets) > 0:
+		t += "targets " + strings.Join(sel.Targets, ", ")
+	default:
+		t += "all targets"
+	}
+	return t
+}
+
+func (s *Server) nodePlan(ctx context.Context, sel nodeSelection) (*nodesync.State, []nodesync.TargetPlan, error) {
+	prof, err := s.store.Profile(ctx, sel.Profile)
+	if err != nil {
+		if errors.Is(err, store.ErrNotFound) {
+			err = fmt.Errorf("unknown profile %q", sel.Profile)
+		}
+		return nil, nil, err
+	}
+	for _, c := range sel.clients() {
+		found := false
+		for _, t := range prof.Targets {
+			found = found || t == c
+		}
+		if !found {
+			return nil, nil, fmt.Errorf("unknown target %q", c)
+		}
+	}
+	eps, err := s.endpoints(ctx)
+	if err != nil {
+		return nil, nil, err
+	}
+	rctx, cancel := context.WithTimeout(ctx, readTimeout)
+	defer cancel()
+	st := nodesync.Analyze(rctx, &prof, eps, sel.clients())
+	s.rememberNodes(st, sel.Scope == "global" && len(sel.Targets) == 0)
+	return st, st.Plans(sel.Objects), nil
+}
+
+// node runs are journaled with the scopes "nodes", "node" and "node-object".
+func nodeScope(scope string) string {
+	switch scope {
+	case "global":
+		return "nodes"
+	case "client":
+		return "node"
+	}
+	return "node-object"
+}
+
+type nodeRunDetail struct {
+	Profile string                 `json:"profile"`
+	Plans   []nodesync.TargetPlan  `json:"plans"`
+	Results map[string]*nodeResult `json:"results,omitempty"`
+}
+
+type nodeResult struct {
+	Status string          `json:"status"`
+	Error  string          `json:"error,omitempty"`
+	Steps  []nodesync.Step `json:"steps"`
+}
+
+func nodeSummary(plans []nodesync.TargetPlan) (string, string) {
+	changes, targets, blocked := 0, 0, 0
+	for _, p := range plans {
+		if p.Error != "" || len(p.Plan.Errors) > 0 {
+			blocked++
+			continue
+		}
+		if n := len(p.Plan.Changes); n > 0 {
+			changes += n
+			targets++
+		}
+	}
+	switch {
+	case blocked > 0:
+		return store.RunBlocked, fmt.Sprintf("%d change(s) on %d target(s), %d target(s) in error", changes, targets, blocked)
+	case changes == 0:
+		return store.RunNoop, "no difference"
+	}
+	return store.RunOK, fmt.Sprintf("%d change(s) on %d target(s)", changes, targets)
+}
+
+func nodePlanTargets(plans []nodesync.TargetPlan) []string {
+	out := make([]string, 0, len(plans))
+	for _, p := range plans {
+		out = append(out, p.Client)
+	}
+	return out
+}
+
+func (s *Server) handleNodeDryRun(w http.ResponseWriter, r *http.Request) {
+	var sel nodeSelection
+	if err := readJSON(r, &sel); err != nil {
+		writeError(w, http.StatusBadRequest, err.Error(), "request")
+		return
+	}
+	if err := sel.check(); err != nil {
+		writeError(w, http.StatusBadRequest, err.Error(), "request")
+		return
+	}
+	id := identity(r)
+	_, plans, err := s.nodePlan(r.Context(), sel)
+	if err != nil {
+		writeError(w, http.StatusBadRequest, err.Error(), "config")
+		return
+	}
+	status, summary := nodeSummary(plans)
+	d, _ := json.Marshal(nodeRunDetail{Profile: sel.Profile, Plans: plans})
+	runID, err := s.store.StartRun(r.Context(), store.Run{User: id.User, Mode: "dry-run", Scope: nodeScope(sel.Scope),
+		Clients: nodePlanTargets(plans), Objects: sel.Objects, Status: status, Summary: summary, Detail: d})
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error(), "store")
+		return
+	}
+	_ = s.store.UpdateRun(r.Context(), runID, status, summary, d)
+	s.audit(r, id, "node-dry-run", sel.target(), true, summary)
+	writeJSON(w, http.StatusOK, map[string]any{"run": runID, "plans": plans, "summary": summary})
+}
+
+// handleNodeApply recomputes the plans, checks the digests of the dry-run and
+// the typed confirmation of the network changes, then applies them in the
+// background, one target after the other.
+func (s *Server) handleNodeApply(w http.ResponseWriter, r *http.Request) {
+	var sel nodeSelection
+	if err := readJSON(r, &sel); err != nil {
+		writeError(w, http.StatusBadRequest, err.Error(), "request")
+		return
+	}
+	if err := sel.check(); err != nil {
+		writeError(w, http.StatusBadRequest, err.Error(), "request")
+		return
+	}
+	id := identity(r)
+	if !s.applyMu.TryLock() {
+		writeError(w, http.StatusConflict, "a replication is already running", "busy")
+		return
+	}
+	started := false
+	defer func() {
+		if !started {
+			s.applyMu.Unlock()
+		}
+	}()
+	st, plans, err := s.nodePlan(r.Context(), sel)
+	if err != nil {
+		writeError(w, http.StatusBadRequest, err.Error(), "config")
+		return
+	}
+	var todo []nodesync.TargetPlan
+	network := false
+	for _, p := range plans {
+		switch {
+		case p.Error != "":
+			s.audit(r, id, "node-apply", sel.target(), false, p.Client+": "+p.Error)
+			writeError(w, http.StatusConflict, p.Client+": "+p.Error, "read")
+			return
+		case len(p.Plan.Errors) > 0:
+			s.audit(r, id, "node-apply", sel.target(), false, p.Client+": plan en erreur")
+			writeError(w, http.StatusConflict, p.Client+": "+p.Plan.Errors[0].Message, "plan")
+			return
+		case sel.Digests[p.Client] != p.Digest:
+			s.audit(r, id, "node-apply", sel.target(), false, p.Client+": the state changed since the dry-run")
+			writeJSON(w, http.StatusConflict, map[string]any{"error": p.Client + ": the state changed since the dry-run, run it again",
+				"code": "stale", "plans": plans})
+			return
+		case p.Plan.Empty():
+			continue
+		}
+		network = network || p.Network
+		todo = append(todo, p)
+	}
+	if len(todo) == 0 {
+		writeJSON(w, http.StatusOK, map[string]any{"run": 0, "summary": "no difference"})
+		return
+	}
+	if network {
+		want := sel.Client
+		if sel.Scope == "global" {
+			want = sel.Profile
+		}
+		if sel.Confirm != want {
+			writeError(w, http.StatusBadRequest, "network changes: confirmation required ("+want+")", "confirm")
+			return
+		}
+	}
+	det := &nodeRunDetail{Profile: sel.Profile, Plans: todo, Results: map[string]*nodeResult{}}
+	for _, p := range todo {
+		det.Results[p.Client] = &nodeResult{Status: "pending", Steps: []nodesync.Step{}}
+	}
+	d, _ := json.Marshal(det)
+	runID, err := s.store.StartRun(r.Context(), store.Run{User: id.User, Mode: "apply", Scope: nodeScope(sel.Scope),
+		Clients: nodePlanTargets(todo), Objects: sel.Objects, Status: store.RunRunning, Summary: "en cours", Detail: d})
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error(), "store")
+		return
+	}
+	s.audit(r, id, "node-apply-start", sel.target(), true, fmt.Sprintf("journal #%d", runID))
+	started = true
+	s.jobs.Add(1)
+	rr := r.Clone(context.WithoutCancel(r.Context()))
+	go func() {
+		defer s.jobs.Done()
+		defer s.applyMu.Unlock()
+		s.runNodeApply(rr, id, runID, sel, st, det)
+	}()
+	writeJSON(w, http.StatusAccepted, map[string]any{"run": runID})
+}
+
+// runNodeApply applies the plans one target after the other and stops at the
+// first failure: the nodes are often of the same cluster, a configuration
+// that breaks one would break the next.
+func (s *Server) runNodeApply(r *http.Request, id Identity, runID int64, sel nodeSelection, st *nodesync.State, det *nodeRunDetail) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+	var mu sync.Mutex
+	save := func(status, summary string) {
+		mu.Lock()
+		d, _ := json.Marshal(det)
+		mu.Unlock()
+		if err := s.store.UpdateRun(ctx, runID, status, summary, d); err != nil {
+			log.Printf("journal #%d: %v", runID, err)
+		}
+	}
+	failed, done := 0, 0
+	for _, p := range det.Plans {
+		res := det.Results[p.Client]
+		if failed > 0 {
+			res.Status = "skipped"
+			continue
+		}
+		res.Status = store.RunRunning
+		save(store.RunRunning, p.Client+" en cours")
+		err := st.Apply(ctx, p, sel.Objects, func(step nodesync.Step) {
+			mu.Lock()
+			res.Steps = append(res.Steps, step)
+			mu.Unlock()
+			save(store.RunRunning, p.Client+" : "+step.Action)
+		})
+		if err != nil {
+			failed++
+			res.Status, res.Error = store.RunError, err.Error()
+			log.Printf("node apply %s (journal #%d): %v", p.Client, runID, err)
+			s.audit(r, id, "node-apply", p.Client, false, err.Error())
+			continue
+		}
+		done++
+		res.Status = store.RunOK
+		s.audit(r, id, "node-apply", p.Client, true, fmt.Sprintf("%d change(s), verified", len(p.Plan.Changes)))
+	}
+	status := store.RunOK
+	summary := fmt.Sprintf("%d target(s) replicated and verified", done)
+	if failed > 0 {
+		status = store.RunError
+		summary = fmt.Sprintf("%d target(s) failed, %d replicated", failed, done)
+	}
+	save(status, summary)
+	s.invalidateNodes()
+}

internal/server/server.go

@@ -16,6 +16,7 @@ import (
 	"sync"
 	"time"
 
+	"pve-confsync/internal/nodesync"
 	"pve-confsync/internal/service"
 	"pve-confsync/internal/store"
 )
@@ -40,6 +41,10 @@ type Server struct {
 	state   *service.State
 	stateAt time.Time
 
+	// last comparison of each node replication profile
+	nodeMu     sync.Mutex
+	nodeStates map[string]*nodesync.State
+
 	applyMu sync.Mutex // one synchronization at a time
 	jobs    sync.WaitGroup
 }
@@ -83,6 +88,18 @@ func (s *Server) Handler() http.Handler {
 	mux.Handle("GET /api/settings", s.role(viewer, s.handleSettings))
 	mux.Handle("PUT /api/settings", s.role(admin, s.handleSaveSettings))
 
+	mux.Handle("GET /api/nodes", s.role(viewer, s.handleNodes))
+	mux.Handle("POST /api/nodes/discover", s.role(admin, s.handleDiscover))
+	mux.Handle("PUT /api/nodes/{cluster}/{node}", s.role(admin, s.handleSaveNode))
+	mux.Handle("POST /api/nodes/{cluster}/{node}/check", s.role(admin, s.handleCheckNode))
+	mux.Handle("POST /api/nodes/{cluster}/{node}/forget", s.role(admin, s.handleForgetNode))
+	mux.Handle("PUT /api/profiles/{name}", s.role(admin, s.handleSaveProfile))
+	mux.Handle("DELETE /api/profiles/{name}", s.role(admin, s.handleDeleteProfile))
+	mux.Handle("GET /api/nodestate", s.role(viewer, s.handleNodeState))
+	mux.Handle("POST /api/nodedryrun", s.role(operator, s.handleNodeDryRun))
+	// writing system files as root on the nodes: administrators only
+	mux.Handle("POST /api/nodeapply", s.role(admin, s.handleNodeApply))
+
 	mux.Handle("GET /api/users", s.role(admin, s.handleUsers))
 	mux.Handle("PUT /api/users/{name}", s.role(admin, s.handleSaveUser))
 	mux.Handle("DELETE /api/users/{name}", s.role(admin, s.handleDeleteUser))

internal/service/nodes.go

@@ -0,0 +1,103 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"strings"
+
+	"pve-confsync/internal/config"
+	"pve-confsync/internal/pve"
+	"pve-confsync/internal/sshx"
+)
+
+// DiscoveredNode is a node listed by GET /cluster/status.
+type DiscoveredNode struct {
+	Name   string `json:"name"`
+	IP     string `json:"ip"`
+	Online bool   `json:"online"`
+}
+
+// DiscoverNodes lists the nodes of a cluster with their address in the
+// cluster. A standalone node without address gets the host of the API URL.
+func DiscoverNodes(ctx context.Context, cl config.Cluster) ([]DiscoveredNode, error) {
+	cl.SSHHostKey = ""
+	c, err := cl.PVE()
+	if err != nil {
+		return nil, err
+	}
+	var status []map[string]any
+	if err := c.Get(ctx, "cluster/status", nil, &status); err != nil {
+		return nil, err
+	}
+	var out []DiscoveredNode
+	for _, s := range status {
+		if pve.Str(s["type"]) != "node" {
+			continue
+		}
+		n := DiscoveredNode{Name: pve.Str(s["name"]), IP: pve.Str(s["ip"]), Online: pve.Str(s["online"]) == "1"}
+		if n.IP == "" {
+			if u, err := url.Parse(cl.URL); err == nil {
+				n.IP = u.Hostname()
+			}
+		}
+		out = append(out, n)
+	}
+	return out, nil
+}
+
+// NodeCheck is the result of the SSH test of a node.
+type NodeCheck struct {
+	OK             bool   `json:"ok"`
+	Error          string `json:"error,omitempty"`
+	Version        string `json:"version,omitempty"`
+	SSHHostKey     string `json:"-"`
+	SSHFingerprint string `json:"sshFingerprint,omitempty"`
+	SSHMismatch    bool   `json:"sshMismatch,omitempty"`
+}
+
+// CheckNode tests SSH as root on a node: the host key (recorded on the first
+// connection, trust accepts a new one), pveversion, and the host name, which
+// must be the name of the node in its cluster (a wrong address is refused).
+func CheckNode(ctx context.Context, node string, o sshx.Options, trust bool) NodeCheck {
+	var res NodeCheck
+	fail := func(err error) NodeCheck {
+		res.Error = err.Error()
+		return res
+	}
+	if o.HostKey == "" || trust {
+		seen, fp, err := sshx.Scan(ctx, o)
+		if err != nil {
+			return fail(fmt.Errorf("SSH: %w", err))
+		}
+		o.HostKey, res.SSHHostKey, res.SSHFingerprint = seen, seen, fp
+	}
+	sh, err := sshx.New(o)
+	if err != nil {
+		return fail(fmt.Errorf("SSH: %w", err))
+	}
+	out, err := sh.Run(ctx, "pveversion; hostname", nil)
+	if err != nil {
+		if strings.Contains(err.Error(), "host key mismatch") {
+			_, fp, _ := sshx.Scan(ctx, o)
+			res.SSHMismatch, res.SSHFingerprint = true, fp
+			return fail(fmt.Errorf("SSH: node %s presents host key %s instead of the recorded %s", o.Host, fp, sshx.Fingerprint(o.HostKey)))
+		}
+		return fail(fmt.Errorf("SSH: %w", err))
+	}
+	lines := strings.Split(strings.TrimSpace(string(out)), "\n")
+	if !strings.HasPrefix(lines[0], "pve-manager/") {
+		return fail(fmt.Errorf("SSH: %s is not a PVE node (pveversion: %q)", o.Host, lines[0]))
+	}
+	res.Version = strings.SplitN(strings.TrimPrefix(lines[0], "pve-manager/"), "/", 2)[0]
+	host := ""
+	if len(lines) > 1 {
+		host = strings.TrimSpace(lines[len(lines)-1])
+	}
+	if short, _, _ := strings.Cut(host, "."); !strings.EqualFold(short, node) {
+		res.SSHHostKey = "" // not the expected node: nothing is recorded
+		return fail(fmt.Errorf("SSH: %s answers as %q, not as node %s", o.Host, host, node))
+	}
+	res.OK = true
+	return res
+}

internal/store/nodes.go

@@ -0,0 +1,433 @@
+package store
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"pve-confsync/internal/config"
+	"pve-confsync/internal/nodesync"
+	"pve-confsync/internal/sshx"
+)
+
+// nodeSchema: the nodes of the clusters (node replication) and the profiles.
+// A node is reached over SSH as root at its address in the cluster (GET
+// /cluster/status) or at an address given by the administrator, with the
+// root password of its cluster or its own.
+const nodeSchema = `
+CREATE TABLE IF NOT EXISTS nodes (
+  cluster     TEXT NOT NULL,
+  node        TEXT NOT NULL,
+  ip          TEXT NOT NULL DEFAULT '',
+  ssh_host    TEXT NOT NULL DEFAULT '',
+  ssh_port    INTEGER NOT NULL DEFAULT 22,
+  password    BLOB,
+  ssh_hostkey TEXT NOT NULL DEFAULT '',
+  last_check  TEXT NOT NULL DEFAULT '',
+  PRIMARY KEY (cluster, node)
+);
+CREATE TABLE IF NOT EXISTS node_profiles (
+  name       TEXT PRIMARY KEY,
+  position   INTEGER NOT NULL DEFAULT 0,
+  body       TEXT NOT NULL,
+  updated_at INTEGER NOT NULL
+);
+`
+
+// Node is a node as stored. The password is never returned to the UI.
+type Node struct {
+	Key            string `json:"key"` // "cluster/node"
+	Cluster        string `json:"cluster"`
+	Node           string `json:"node"`
+	IP             string `json:"ip"`      // address in the cluster
+	SSHHost        string `json:"sshHost"` // override (empty: IP)
+	SSHPort        int    `json:"sshPort"`
+	Password       string `json:"password,omitempty"` // write only (empty: the cluster's)
+	HasPassword    bool   `json:"hasPassword"`
+	SSHFingerprint string `json:"sshFingerprint"`
+	ClusterRole    string `json:"clusterRole"`
+	LastCheck      *Check `json:"lastCheck,omitempty"`
+}
+
+// NodeKey is the identity of a node in the profiles.
+func NodeKey(cluster, node string) string { return cluster + "/" + node }
+
+const nodeCols = `n.cluster, n.node, n.ip, n.ssh_host, n.ssh_port, n.password, n.ssh_hostkey, n.last_check, c.role`
+
+func scanNode(scan func(...any) error) (Node, []byte, string, error) {
+	var n Node
+	var blob []byte
+	var key, check string
+	err := scan(&n.Cluster, &n.Node, &n.IP, &n.SSHHost, &n.SSHPort, &blob, &key, &check, &n.ClusterRole)
+	if err != nil {
+		return n, nil, "", err
+	}
+	n.Key = NodeKey(n.Cluster, n.Node)
+	n.HasPassword = len(blob) > 0
+	n.SSHFingerprint = sshx.Fingerprint(key)
+	if check != "" {
+		n.LastCheck = &Check{}
+		_ = json.Unmarshal([]byte(check), n.LastCheck)
+	}
+	return n, blob, key, nil
+}
+
+// Nodes lists the known nodes, by cluster then name.
+func (s *Store) Nodes(ctx context.Context) ([]Node, error) {
+	rows, err := s.db.QueryContext(ctx, `SELECT `+nodeCols+` FROM nodes n JOIN clusters c ON c.name = n.cluster
+		ORDER BY c.role = 'primary' DESC, c.position, n.cluster, n.node`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	out := []Node{}
+	for rows.Next() {
+		n, _, _, err := scanNode(rows.Scan)
+		if err != nil {
+			return nil, err
+		}
+		out = append(out, n)
+	}
+	return out, rows.Err()
+}
+
+// Discovered is a node read in GET /cluster/status.
+type Discovered struct {
+	Name string
+	IP   string
+}
+
+// SetNodes records the nodes of a cluster: new ones are added, the address of
+// the known ones is updated (their pinned host key is forgotten when the SSH
+// address changes), the nodes gone from the cluster are removed.
+func (s *Store) SetNodes(ctx context.Context, cluster string, nodes []Discovered) error {
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+	keep := map[string]bool{}
+	for _, d := range nodes {
+		keep[d.Name] = true
+		var ip, host string
+		err := tx.QueryRowContext(ctx, `SELECT ip, ssh_host FROM nodes WHERE cluster = ? AND node = ?`, cluster, d.Name).Scan(&ip, &host)
+		switch {
+		case errors.Is(err, sql.ErrNoRows):
+			_, err = tx.ExecContext(ctx, `INSERT INTO nodes (cluster, node, ip) VALUES (?,?,?)`, cluster, d.Name, d.IP)
+		case err == nil && ip != d.IP:
+			q := `UPDATE nodes SET ip = ? WHERE cluster = ? AND node = ?`
+			if host == "" {
+				q = `UPDATE nodes SET ip = ?, ssh_hostkey = '' WHERE cluster = ? AND node = ?`
+			}
+			_, err = tx.ExecContext(ctx, q, d.IP, cluster, d.Name)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	rows, err := tx.QueryContext(ctx, `SELECT node FROM nodes WHERE cluster = ?`, cluster)
+	if err != nil {
+		return err
+	}
+	var gone []string
+	for rows.Next() {
+		var n string
+		if err := rows.Scan(&n); err != nil {
+			rows.Close()
+			return err
+		}
+		if !keep[n] {
+			gone = append(gone, n)
+		}
+	}
+	rows.Close()
+	for _, n := range gone {
+		if _, err := tx.ExecContext(ctx, `DELETE FROM nodes WHERE cluster = ? AND node = ?`, cluster, n); err != nil {
+			return err
+		}
+	}
+	return tx.Commit()
+}
+
+// SaveNode sets the SSH access of a node: address (empty: its address in the
+// cluster), port, password (empty: unchanged; clear: back to the password
+// of the cluster). A new address forgets the pinned host key.
+func (s *Store) SaveNode(ctx context.Context, cluster, node, host string, port int, password string, clear bool) error {
+	if port == 0 {
+		port = 22
+	}
+	if port < 1 || port > 65535 {
+		return errors.New("ssh port: 1-65535 expected")
+	}
+	var oldHost string
+	var oldPort int
+	err := s.db.QueryRowContext(ctx, `SELECT ssh_host, ssh_port FROM nodes WHERE cluster = ? AND node = ?`, cluster, node).Scan(&oldHost, &oldPort)
+	if errors.Is(err, sql.ErrNoRows) {
+		return ErrNotFound
+	}
+	if err != nil {
+		return err
+	}
+	q := `UPDATE nodes SET ssh_host = ?, ssh_port = ?`
+	args := []any{strings.TrimSpace(host), port}
+	if oldHost != strings.TrimSpace(host) || oldPort != port {
+		q += `, ssh_hostkey = ''`
+	}
+	switch {
+	case clear:
+		q += `, password = NULL`
+	case password != "":
+		blob, err := s.box.Seal([]byte(password))
+		if err != nil {
+			return err
+		}
+		q += `, password = ?`
+		args = append(args, blob)
+	}
+	_, err = s.db.ExecContext(ctx, q+` WHERE cluster = ? AND node = ?`, append(args, cluster, node)...)
+	return err
+}
+
+// PinNode records the SSH host key seen on the first connection (empty: left
+// unchanged) and the result of the test.
+func (s *Store) PinNode(ctx context.Context, cluster, node, hostKey string, check *Check) error {
+	var chk []byte
+	if check != nil {
+		chk, _ = json.Marshal(check)
+	}
+	_, err := s.db.ExecContext(ctx, `UPDATE nodes SET
+		ssh_hostkey = CASE WHEN ? <> '' THEN ? ELSE ssh_hostkey END,
+		last_check = CASE WHEN ? <> '' THEN ? ELSE last_check END
+		WHERE cluster = ? AND node = ?`, hostKey, hostKey, string(chk), string(chk), cluster, node)
+	return err
+}
+
+// ForgetNode clears the pinned SSH host key of a node.
+func (s *Store) ForgetNode(ctx context.Context, cluster, node string) error {
+	res, err := s.db.ExecContext(ctx, `UPDATE nodes SET ssh_hostkey = '' WHERE cluster = ? AND node = ?`, cluster, node)
+	if err != nil {
+		return err
+	}
+	if n, _ := res.RowsAffected(); n == 0 {
+		return ErrNotFound
+	}
+	return nil
+}
+
+// NodeAccess is the decrypted access to one node: the connection of its
+// cluster (API) and SSH as root on the node.
+type NodeAccess struct {
+	Node
+	Cluster config.Cluster
+	SSH     sshx.Options
+}
+
+// NodeAccesses returns the access to every node of the clusters not disabled.
+func (s *Store) NodeAccesses(ctx context.Context) (map[string]NodeAccess, error) {
+	rows, err := s.db.QueryContext(ctx, `SELECT `+nodeCols+` FROM nodes n JOIN clusters c ON c.name = n.cluster WHERE c.role <> 'disabled'`)
+	if err != nil {
+		return nil, err
+	}
+	type row struct {
+		n    Node
+		blob []byte
+		key  string
+	}
+	var list []row
+	for rows.Next() {
+		n, blob, key, err := scanNode(rows.Scan)
+		if err != nil {
+			rows.Close()
+			return nil, err
+		}
+		list = append(list, row{n, blob, key})
+	}
+	rows.Close()
+	out := map[string]NodeAccess{}
+	clusters := map[string]config.Cluster{}
+	for _, r := range list {
+		cc, ok := clusters[r.n.Cluster]
+		if !ok {
+			var err error
+			if _, cc, err = s.Cluster(ctx, r.n.Cluster); err != nil {
+				return nil, err
+			}
+			clusters[r.n.Cluster] = cc
+		}
+		a, err := s.access(r.n, r.blob, r.key, cc)
+		if err != nil {
+			return nil, err
+		}
+		out[r.n.Key] = a
+	}
+	return out, nil
+}
+
+// NodeAccessOf returns the access to one node.
+func (s *Store) NodeAccessOf(ctx context.Context, cluster, node string) (NodeAccess, error) {
+	n, blob, key, err := scanNode(s.db.QueryRowContext(ctx, `SELECT `+nodeCols+` FROM nodes n JOIN clusters c ON c.name = n.cluster
+		WHERE n.cluster = ? AND n.node = ?`, cluster, node).Scan)
+	if errors.Is(err, sql.ErrNoRows) {
+		return NodeAccess{}, ErrNotFound
+	}
+	if err != nil {
+		return NodeAccess{}, err
+	}
+	_, cc, err := s.Cluster(ctx, cluster)
+	if err != nil {
+		return NodeAccess{}, err
+	}
+	return s.access(n, blob, key, cc)
+}
+
+func (s *Store) access(n Node, blob []byte, key string, cc config.Cluster) (NodeAccess, error) {
+	pass := cc.Password
+	if len(blob) > 0 {
+		p, err := s.box.Open(blob)
+		if err != nil {
+			return NodeAccess{}, fmt.Errorf("node %s: password: %w", n.Key, err)
+		}
+		pass = string(p)
+	}
+	host := n.SSHHost
+	if host == "" {
+		host = n.IP
+	}
+	cc.SSHHostKey = "" // the cluster SSH access is not used here
+	return NodeAccess{Node: n, Cluster: cc,
+		SSH: sshx.Options{Host: host, Port: n.SSHPort, User: "root", Password: pass, HostKey: key}}, nil
+}
+
+// ---- profiles ----
+
+// Profiles returns the node replication profiles, in order.
+func (s *Store) Profiles(ctx context.Context) ([]nodesync.Profile, error) {
+	rows, err := s.db.QueryContext(ctx, `SELECT name, position, body FROM node_profiles ORDER BY position, name`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	out := []nodesync.Profile{}
+	for rows.Next() {
+		var name, body string
+		var pos int
+		if err := rows.Scan(&name, &pos, &body); err != nil {
+			return nil, err
+		}
+		var p nodesync.Profile
+		if err := json.Unmarshal([]byte(body), &p); err != nil {
+			return nil, fmt.Errorf("profile %s: %w", name, err)
+		}
+		p.Name, p.Position = name, pos
+		out = append(out, p)
+	}
+	return out, rows.Err()
+}
+
+// Profile returns one profile.
+func (s *Store) Profile(ctx context.Context, name string) (nodesync.Profile, error) {
+	ps, err := s.Profiles(ctx)
+	if err != nil {
+		return nodesync.Profile{}, err
+	}
+	for _, p := range ps {
+		if p.Name == name {
+			return p, nil
+		}
+	}
+	return nodesync.Profile{}, ErrNotFound
+}
+
+// SaveProfile creates or updates a profile; oldName renames it.
+func (s *Store) SaveProfile(ctx context.Context, oldName string, p nodesync.Profile) error {
+	if err := p.Validate(); err != nil {
+		return err
+	}
+	if oldName == "" {
+		oldName = p.Name
+	}
+	body, _ := json.Marshal(p)
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+	res, err := tx.ExecContext(ctx, `UPDATE node_profiles SET name = ?, position = ?, body = ?, updated_at = ? WHERE name = ?`,
+		p.Name, p.Position, string(body), time.Now().Unix(), oldName)
+	if err == nil {
+		if n, _ := res.RowsAffected(); n == 0 {
+			_, err = tx.ExecContext(ctx, `INSERT INTO node_profiles (name, position, body, updated_at) VALUES (?,?,?,?)`,
+				p.Name, p.Position, string(body), time.Now().Unix())
+		}
+	}
+	if err != nil {
+		if strings.Contains(err.Error(), "UNIQUE") {
+			return fmt.Errorf("profile %q already exists", p.Name)
+		}
+		return err
+	}
+	return tx.Commit()
+}
+
+// DeleteProfile removes a profile.
+func (s *Store) DeleteProfile(ctx context.Context, name string) error {
+	res, err := s.db.ExecContext(ctx, `DELETE FROM node_profiles WHERE name = ?`, name)
+	if err != nil {
+		return err
+	}
+	if n, _ := res.RowsAffected(); n == 0 {
+		return ErrNotFound
+	}
+	return nil
+}
+
+// renameClusterNodes follows a renamed cluster in the nodes and the profiles.
+func renameClusterNodes(ctx context.Context, tx *sql.Tx, old, name string) error {
+	if _, err := tx.ExecContext(ctx, `UPDATE nodes SET cluster = ? WHERE cluster = ?`, name, old); err != nil {
+		return err
+	}
+	rows, err := tx.QueryContext(ctx, `SELECT name, body FROM node_profiles`)
+	if err != nil {
+		return err
+	}
+	bodies := map[string]string{}
+	for rows.Next() {
+		var n, b string
+		if err := rows.Scan(&n, &b); err != nil {
+			rows.Close()
+			return err
+		}
+		bodies[n] = b
+	}
+	rows.Close()
+	ren := func(k string) string {
+		if strings.HasPrefix(k, old+"/") {
+			return name + k[len(old):]
+		}
+		return k
+	}
+	for n, b := range bodies {
+		var p nodesync.Profile
+		if json.Unmarshal([]byte(b), &p) != nil {
+			continue
+		}
+		p.Master = ren(p.Master)
+		for i, t := range p.Targets {
+			p.Targets[i] = ren(t)
+		}
+		m := map[string]nodesync.TargetMap{}
+		for k, v := range p.Mapping {
+			m[ren(k)] = v
+		}
+		p.Mapping = m
+		nb, _ := json.Marshal(p)
+		if _, err := tx.ExecContext(ctx, `UPDATE node_profiles SET body = ? WHERE name = ?`, string(nb), n); err != nil {
+			return err
+		}
+	}
+	return nil
+}

internal/store/nodes_test.go

@@ -0,0 +1,66 @@
+package store
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+
+	"pve-confsync/internal/nodesync"
+)
+
+func TestNodesAndProfiles(t *testing.T) {
+	ctx := context.Background()
+	st, err := Open(filepath.Join(t.TempDir(), "t.db"), testKey(1))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := st.SaveCluster(ctx, "", Cluster{Name: "lab", Role: ClusterPrimary, URL: "https://lab:8006", Password: "cluster-pass"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := st.SetNodes(ctx, "lab", []Discovered{{"pve1", "10.0.0.11"}, {"pve2", "10.0.0.12"}}); err != nil {
+		t.Fatal(err)
+	}
+	if err := st.PinNode(ctx, "lab", "pve2", "ssh-ed25519 AAAA", nil); err != nil {
+		t.Fatal(err)
+	}
+	// the cluster password by default, the node's own one once set
+	a, err := st.NodeAccessOf(ctx, "lab", "pve2")
+	if err != nil || a.SSH.Password != "cluster-pass" || a.SSH.Host != "10.0.0.12" || a.SSH.HostKey == "" {
+		t.Fatalf("%+v %v", a.SSH, err)
+	}
+	if err := st.SaveNode(ctx, "lab", "pve2", "192.168.0.12", 22, "own-pass", false); err != nil {
+		t.Fatal(err)
+	}
+	a, _ = st.NodeAccessOf(ctx, "lab", "pve2")
+	if a.SSH.Password != "own-pass" || a.SSH.Host != "192.168.0.12" || a.SSH.HostKey != "" {
+		t.Fatalf("new address must forget the host key: %+v", a.SSH)
+	}
+	// a node gone from the cluster is removed
+	if err := st.SetNodes(ctx, "lab", []Discovered{{"pve1", "10.0.0.11"}}); err != nil {
+		t.Fatal(err)
+	}
+	if ns, _ := st.Nodes(ctx); len(ns) != 1 || ns[0].Key != "lab/pve1" {
+		t.Fatalf("nodes: %+v", ns)
+	}
+	p := nodesync.Profile{Name: "net", Master: "lab/pve1", Targets: []string{"lab/pve2"}, Items: []string{"network"},
+		Mapping: map[string]nodesync.TargetMap{"lab/pve2": {NICs: map[string]string{"eno1": "enp1s0"}}}}
+	if err := st.SaveProfile(ctx, "", p); err != nil {
+		t.Fatal(err)
+	}
+	if err := st.SaveProfile(ctx, "", nodesync.Profile{Name: "bad", Master: "lab/pve1", Targets: []string{"lab/pve1"}, Items: []string{"network"}}); err == nil {
+		t.Error("master as target accepted")
+	}
+	// renaming the cluster follows in the nodes and the profiles
+	c, _, _ := st.Cluster(ctx, "lab")
+	c.Name = "rennes"
+	if err := st.SaveCluster(ctx, "lab", c); err != nil {
+		t.Fatal(err)
+	}
+	got, err := st.Profile(ctx, "net")
+	if err != nil || got.Master != "rennes/pve1" || got.Targets[0] != "rennes/pve2" || got.Mapping["rennes/pve2"].NICs["eno1"] != "enp1s0" {
+		t.Fatalf("profile after rename: %+v %v", got, err)
+	}
+	if ns, _ := st.Nodes(ctx); len(ns) != 1 || ns[0].Key != "rennes/pve1" {
+		t.Fatalf("nodes after rename: %+v", ns)
+	}
+}

internal/store/store.go

@@ -112,7 +112,7 @@ func Open(path string, key []byte) (*Store, error) {
 		db.Close()
 		return nil, fmt.Errorf("%s: %w", path, err)
 	}
-	if _, err := db.Exec(schema); err != nil {
+	if _, err := db.Exec(schema + nodeSchema); err != nil {
 		db.Close()
 		return nil, fmt.Errorf("%s: %w", path, err)
 	}
@@ -391,6 +391,11 @@ func (s *Store) SaveCluster(ctx context.Context, oldName string, c Cluster) erro
 		}
 		return err
 	}
+	if exists && oldName != c.Name {
+		if err := renameClusterNodes(ctx, tx, oldName, c.Name); err != nil {
+			return err
+		}
+	}
 	return tx.Commit()
 }
 
@@ -465,7 +470,8 @@ func (s *Store) DeleteCluster(ctx context.Context, name string) error {
 	if n, _ := res.RowsAffected(); n == 0 {
 		return ErrNotFound
 	}
-	return nil
+	_, err = s.db.ExecContext(ctx, `DELETE FROM nodes WHERE cluster = ?`, name)
+	return err
 }
 
 func nonNil(m map[model.Kind]map[string]map[string]any) any {