Réplication de nœuds : réseau et DNS par l'API, multipath, chrony, sysctl par SSH

Deuxième mode de réplication, séparé de la réplication cluster vers cluster
(sélecteur « Clusters (API) | Nœuds (fichiers) » en haut de page) : un profil
réplique la configuration système d'un nœud maître vers des nœuds cibles.

- réseau par /nodes/{node}/network (interfaces.new puis rechargement), adresses
  et passerelles propres à chaque nœud, mapping des noms d'interfaces par cible ;
  DNS par /nodes/{node}/dns ; multipath.conf, wwids (ajout seul), chrony,
  sysctl.d, interfaces.d (hors sdn) par SSH
- découverte des nœuds (/cluster/status), accès SSH par nœud (mot de passe du
  cluster ou propre), clé d'hôte épinglée, contrôle du nom du nœud
- dry-run avec diff des fichiers ; apply (admin) avec contrôle SHA-256,
  sauvegarde et restore.sh sur le nœud, écriture atomique, rechargements,
  restauration si un rechargement échoue, timer systemd de restauration si le
  nœud est injoignable après le rechargement du réseau, confirmation tapée
- vue Nœuds et profils, journal séparé par mode, démo avec nœuds simulés

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

internal/nodesync/analyze.go

@@ -0,0 +1,341 @@
+package nodesync
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sort"
+	"sync"
+	"time"
+
+	"pve-confsync/internal/model"
+)
+
+// NodeInfo is what was read from one node.
+type NodeInfo struct {
+	Key      string    `json:"name"` // "cluster/node", "name" like the cluster cards
+	Cluster  string    `json:"cluster"`
+	Node     string    `json:"node"`
+	Version  string    `json:"version,omitempty"`
+	Error    string    `json:"error,omitempty"`
+	Warnings []string  `json:"warnings,omitempty"`
+	ReadAt   time.Time `json:"readAt"`
+	Millis   int64     `json:"ms"`
+}
+
+// Field compares one field of an interface or of the DNS servers.
+type Field struct {
+	Name    string `json:"name"`
+	Policy  Policy `json:"policy"`
+	Ref     any    `json:"ref,omitempty"`     // master
+	Mapping any    `json:"mapping,omitempty"` // value of the target mapping
+	Current any    `json:"current,omitempty"` // target
+	Target  any    `json:"target,omitempty"`  // after the replication
+	Change  string `json:"change,omitempty"`  // add, set, unset or empty
+}
+
+// Row statuses (same as the cluster rows).
+const (
+	StatusSame   = "same"
+	StatusCreate = "create"
+	StatusUpdate = "update"
+	StatusDelete = "delete"
+	StatusExtra  = "extra" // only on the target, kept
+	StatusError  = "error"
+)
+
+// Row is one object of a target: an interface, the DNS servers or a file.
+type Row struct {
+	Kind   string   `json:"kind"` // item id
+	ID     string   `json:"id"`
+	Key    string   `json:"key"`
+	Status string   `json:"status"`
+	Fields []Field  `json:"fields,omitempty"`
+	Diff   []Line   `json:"diff,omitempty"`
+	Errors []string `json:"errors,omitempty"`
+}
+
+// Target is the state of one target node.
+type Target struct {
+	NodeInfo
+	Plan   Plan           `json:"plan"`
+	Digest string         `json:"digest"`
+	Rows   []Row          `json:"rows"`
+	Counts map[string]int `json:"counts"`
+}
+
+// State is a full comparison of the master with its targets.
+type State struct {
+	At      time.Time `json:"at"`
+	Profile string    `json:"profile"`
+	Items   []string  `json:"items"`
+	Prune   bool      `json:"prune"`
+	Master  NodeInfo  `json:"reference"` // "reference" like the cluster state
+	Targets []Target  `json:"clients"`
+
+	prof    *Profile
+	master  Snapshot
+	targets map[string]Snapshot
+	eps     map[string]Endpoint
+}
+
+// Analyze reads the master and the targets (in parallel) and compares them.
+// only limits the targets (empty: all). A node that cannot be read is
+// reported in its Error field, it does not stop the others.
+func Analyze(ctx context.Context, prof *Profile, eps map[string]Endpoint, only []string) *State {
+	st := &State{At: time.Now().UTC(), Profile: prof.Name, Items: prof.Items, Prune: prof.Prune, Targets: []Target{},
+		prof: prof, targets: map[string]Snapshot{}, eps: eps}
+	st.Master, st.master = read(ctx, eps, prof.Master, prof.Items)
+	want := map[string]bool{}
+	for _, n := range only {
+		want[n] = true
+	}
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	for _, key := range prof.Targets {
+		if len(only) > 0 && !want[key] {
+			continue
+		}
+		st.Targets = append(st.Targets, Target{NodeInfo: info(key)})
+		idx := len(st.Targets) - 1
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			ni, snap := read(ctx, eps, key, prof.Items)
+			mu.Lock()
+			defer mu.Unlock()
+			st.Targets[idx].NodeInfo = ni
+			if ni.Error == "" {
+				st.targets[key] = snap
+			}
+		}()
+	}
+	wg.Wait()
+	for i := range st.Targets {
+		t := &st.Targets[i]
+		t.Counts = map[string]int{}
+		if t.Error != "" || st.Master.Error != "" {
+			continue
+		}
+		t.Plan = MakePlan(prof, st.master, st.targets[t.Key])
+		t.Digest = t.Plan.Digest()
+		t.Rows = rows(prof, st.master, st.targets[t.Key], t.Plan)
+		for _, r := range t.Rows {
+			t.Counts[r.Status]++
+		}
+	}
+	return st
+}
+
+func info(key string) NodeInfo {
+	ni := NodeInfo{Key: key}
+	for i := len(key) - 1; i >= 0; i-- {
+		if key[i] == '/' {
+			ni.Cluster, ni.Node = key[:i], key[i+1:]
+			break
+		}
+	}
+	return ni
+}
+
+func read(ctx context.Context, eps map[string]Endpoint, key string, items []string) (NodeInfo, Snapshot) {
+	ni := info(key)
+	ni.ReadAt = time.Now().UTC()
+	start := time.Now()
+	ep, ok := eps[key]
+	if !ok {
+		ni.Error = fmt.Sprintf("unknown node %s (discover the nodes of its cluster)", key)
+		return ni, Snapshot{}
+	}
+	snap, err := Read(ctx, ep, items)
+	if err != nil {
+		ni.Error = err.Error()
+	}
+	ni.Version = snap.Version
+	ni.Warnings = snap.Warnings
+	ni.Millis = time.Since(start).Milliseconds()
+	return ni, snap
+}
+
+// rows describes every object of the master and of the target.
+func rows(prof *Profile, master, target Snapshot, plan Plan) []Row {
+	changes := map[string]Change{}
+	for _, c := range plan.Changes {
+		changes[c.Key()] = c
+	}
+	errs := map[string][]string{}
+	for _, e := range plan.Errors {
+		if e.Key != "" {
+			errs[e.Key] = append(errs[e.Key], e.Message)
+		}
+	}
+	tm := prof.Mapping[target.Key]
+	var out []Row
+	seen := map[string]bool{}
+	status := func(key string, inMaster bool) string {
+		ch, has := changes[key]
+		switch {
+		case len(errs[key]) > 0:
+			return StatusError
+		case has:
+			return string(ch.Op)
+		case !inMaster:
+			return StatusExtra
+		}
+		return StatusSame
+	}
+	addAPI := func(item, id string, ref, cur map[string]any) {
+		key := item + "/" + id
+		seen[key] = true
+		ch := changes[key]
+		r := Row{Kind: item, ID: id, Key: key, Status: status(key, ref != nil), Errors: errs[key]}
+		names := map[string]bool{}
+		for _, m := range []map[string]any{ref, cur, ch.Set} {
+			for f := range m {
+				names[f] = true
+			}
+		}
+		for f := range tm.Values[key] {
+			names[f] = true
+		}
+		list := make([]string, 0, len(names))
+		for f := range names {
+			list = append(list, f)
+		}
+		sort.Strings(list)
+		unset := map[string]bool{}
+		for _, f := range ch.Unset {
+			unset[f] = true
+		}
+		for _, f := range list {
+			fd := Field{Name: f, Policy: prof.Policy(item, f), Ref: ref[f], Current: cur[f]}
+			if v, ok := tm.Values[key][f]; ok {
+				fd.Mapping = v
+			}
+			switch {
+			case ch.Op == OpDelete || unset[f]:
+				fd.Change = "unset"
+			case ch.Set[f] != nil:
+				fd.Target = ch.Set[f]
+				fd.Change = "set"
+				if fd.Current == nil {
+					fd.Change = "add"
+				}
+			case fd.Current != nil:
+				fd.Target = fd.Current
+			}
+			r.Fields = append(r.Fields, fd)
+		}
+		out = append(out, r)
+	}
+	for _, id := range prof.Items {
+		it, _ := ItemByID(id)
+		switch it.Kind {
+		case KindNetwork:
+			cur := map[string]model.Object{}
+			for _, o := range target.Network {
+				cur[o.ID] = o
+			}
+			for _, m := range master.Network {
+				tid := rename(m.ID, tm.NICs)
+				addAPI("network", tid, m.Fields, cur[tid].Fields)
+			}
+			for _, o := range target.Network {
+				if !seen["network/"+o.ID] {
+					addAPI("network", o.ID, nil, o.Fields)
+				}
+			}
+		case KindDNS:
+			addAPI("dns", "dns", master.DNS, target.DNS)
+		case KindFile:
+			paths := map[string]bool{}
+			for p := range master.Files[id] {
+				paths[p] = true
+			}
+			for p := range target.Files[id] {
+				paths[p] = true
+			}
+			if len(master.Files[id]) == 0 && !it.Glob {
+				continue
+			}
+			for p := range paths {
+				key := id + "/" + p
+				_, inMaster := master.Files[id][p]
+				r := Row{Kind: id, ID: p, Key: key, Status: status(key, inMaster), Errors: errs[key], Diff: changes[key].Diff}
+				out = append(out, r)
+			}
+		}
+	}
+	sort.SliceStable(out, func(i, j int) bool {
+		if out[i].Kind != out[j].Kind {
+			return itemRank(out[i].Kind) < itemRank(out[j].Kind)
+		}
+		return out[i].ID < out[j].ID
+	})
+	return out
+}
+
+// TargetPlan is the plan of one target restricted to a selection of objects.
+type TargetPlan struct {
+	Client  string   `json:"client"` // "client" like the cluster plans
+	Plan    Plan     `json:"plan"`
+	Digest  string   `json:"digest"`
+	Network bool     `json:"network"`
+	Reloads []string `json:"reloads,omitempty"`
+	Error   string   `json:"error,omitempty"`
+}
+
+// Plans returns the plans of the targets restricted to the given objects
+// (empty: everything), as a dry-run shows them and an apply checks them.
+func (st *State) Plans(keys []string) []TargetPlan {
+	var out []TargetPlan
+	for _, t := range st.Targets {
+		tp := TargetPlan{Client: t.Key}
+		switch {
+		case st.Master.Error != "":
+			tp.Error = "master: " + st.Master.Error
+		case t.Error != "":
+			tp.Error = t.Error
+		default:
+			tp.Plan = Filter(t.Plan, keys)
+			tp.Digest = tp.Plan.Digest()
+			tp.Network = tp.Plan.Network()
+			tp.Reloads = tp.Plan.Reloads()
+		}
+		out = append(out, tp)
+	}
+	return out
+}
+
+// Apply writes one plan on its target, then reads the target again and checks
+// that the selected objects are in line with the master.
+func (st *State) Apply(ctx context.Context, tp TargetPlan, keys []string, log func(Step)) error {
+	ep, ok := st.eps[tp.Client]
+	if !ok {
+		return fmt.Errorf("unknown node %q", tp.Client)
+	}
+	if err := Apply(ctx, ep, tp.Plan, log); err != nil {
+		return err
+	}
+	start := time.Now()
+	err := st.verify(ctx, ep, keys)
+	s := Step{At: start.UTC(), Action: "verify"}
+	if err != nil {
+		s.Error = err.Error()
+	}
+	log(s)
+	return err
+}
+
+func (st *State) verify(ctx context.Context, ep Endpoint, keys []string) error {
+	snap, err := Read(ctx, ep, st.prof.Items)
+	if err != nil {
+		return err
+	}
+	p := Filter(MakePlan(st.prof, st.master, snap), keys)
+	var errs []error
+	for _, c := range p.Changes {
+		errs = append(errs, fmt.Errorf("still different after apply: %s", c.Key()))
+	}
+	return errors.Join(errs...)
+}

internal/nodesync/apply.go

@@ -0,0 +1,333 @@
+package nodesync
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"net/url"
+	"sort"
+	"strings"
+	"time"
+
+	"pve-confsync/internal/pve"
+)
+
+// Step is the outcome of one action, for the execution journal.
+type Step struct {
+	At     time.Time `json:"at"`
+	Action string    `json:"action"`
+	Error  string    `json:"error,omitempty"`
+}
+
+// Timing of an apply; variables for the tests.
+var (
+	// RestoreDelay: when the network was reloaded, the node restores its
+	// previous configuration by itself after this delay unless pve-confsync
+	// reaches it again over SSH and cancels the restore.
+	RestoreDelay = 120 * time.Second
+	// ProbeTries and ProbeWait: attempts to reach the node after the reload.
+	ProbeTries = 6
+	ProbeWait  = 5 * time.Second
+	TaskPoll   = 2 * time.Second
+)
+
+// BackupRoot is where the previous files are kept on the node, one folder
+// per apply, with the restore.sh script that puts them back.
+const BackupRoot = "/var/lib/pve-confsync/backup"
+
+// Apply writes a plan on a target node:
+//  1. one SSH script checks that every file is still the one of the dry-run,
+//     backs up the files (and /etc/network/interfaces, /etc/resolv.conf when
+//     the API writes them), writes restore.sh, then writes the new files
+//     atomically (temporary file then mv, mode and owner kept);
+//  2. the API writes the interfaces (in interfaces.new) and the DNS servers;
+//  3. when the network is reloaded, a systemd timer is armed on the node: it
+//     runs restore.sh and ifreload -a after RestoreDelay;
+//  4. the reload commands run (sysctl, chrony, multipath, network last);
+//  5. the node must answer again over SSH and the API: the timer is stopped.
+//
+// A failure before the network reload restores the backups (and drops
+// interfaces.new); a reload failure restores them and runs the reloads again.
+func Apply(ctx context.Context, ep Endpoint, plan Plan, notify func(Step)) error {
+	log := func(action string, err error) error {
+		s := Step{At: time.Now().UTC(), Action: action}
+		if err != nil {
+			s.Error = err.Error()
+		}
+		if notify != nil {
+			notify(s)
+		}
+		return err
+	}
+	if len(plan.Errors) > 0 {
+		return log("plan check", fmt.Errorf("plan has %d error(s), nothing applied", len(plan.Errors)))
+	}
+	if plan.Empty() {
+		return nil
+	}
+	if ep.Shell == nil {
+		return log("ssh", fmt.Errorf("%s: SSH host key not recorded yet, test the node first", ep.Key))
+	}
+	if _, err := NodeVersion(ctx, ep); err != nil {
+		return log("version check", err)
+	}
+	stamp := time.Now().UTC().Format("20060102-150405")
+	dir := BackupRoot + "/" + stamp
+	unit := "pve-confsync-restore-" + stamp
+
+	var netAPI, dns bool
+	for _, c := range plan.Changes {
+		netAPI = netAPI || c.Item == "network"
+		dns = dns || c.Item == "dns"
+	}
+	if _, err := ep.Shell.Run(ctx, prepareScript(dir), prepareInput(plan, netAPI, dns)); err != nil {
+		return log("check, back up and write the files ("+dir+")", err)
+	}
+	log("check, back up and write the files ("+dir+")", nil)
+	restore := func(cause error) error {
+		_, rerr := ep.Shell.Run(ctx, restoreScript(dir), nil)
+		log("restore the previous files", rerr)
+		if rerr != nil {
+			return errors.Join(cause, fmt.Errorf("restore failed, run sh %s/restore.sh on the node: %w", dir, rerr))
+		}
+		return cause
+	}
+
+	for _, c := range plan.Changes {
+		if c.Item != "network" && c.Item != "dns" {
+			continue
+		}
+		if err := log(describe(ep, c), writeAPI(ctx, ep, c)); err != nil {
+			return restore(err)
+		}
+	}
+
+	network := plan.Network()
+	if network {
+		_, err := ep.Shell.Run(ctx, armScript(dir, unit), nil)
+		if err := log(fmt.Sprintf("arm the automatic restore (%s, %s)", unit, RestoreDelay), err); err != nil {
+			return restore(err)
+		}
+	}
+	var done []string // reload commands already run
+	for _, cmd := range plan.Reloads() {
+		var err error
+		action := cmd
+		if cmd == "ifreload -a" && netAPI {
+			action = "PUT /nodes/" + ep.Node + "/network (ifreload -a)"
+			err = reloadNetworkAPI(ctx, ep)
+		} else {
+			_, err = ep.Shell.Run(ctx, reloadScript(cmd), nil)
+		}
+		done = append(done, cmd)
+		if log("reload: "+action, err) == nil {
+			continue
+		}
+		cause := fmt.Errorf("reload %q: %w", cmd, err)
+		if network && !reachable(ctx, ep) {
+			return log("node unreachable", fmt.Errorf("%w; the node is unreachable: it restores its previous configuration by itself within %s", cause, RestoreDelay))
+		}
+		// back to the previous files, which the reloads put in effect again
+		err = restore(cause)
+		for _, c := range done {
+			_, rerr := ep.Shell.Run(ctx, reloadScript(c), nil)
+			log("reload after restore: "+c, rerr)
+		}
+		if network {
+			_, derr := ep.Shell.Run(ctx, disarmScript(unit), nil)
+			log("stop the automatic restore", derr)
+		}
+		return err
+	}
+	if network {
+		if !reachable(ctx, ep) {
+			return log("reach the node again", fmt.Errorf("%s unreachable after the network reload: it restores its previous configuration by itself within %s", ep.Key, RestoreDelay))
+		}
+		log("reach the node again (SSH and API)", nil)
+		_, err := ep.Shell.Run(ctx, disarmScript(unit), nil)
+		if err := log("stop the automatic restore", err); err != nil {
+			return fmt.Errorf("stop the automatic restore (systemctl stop %s.timer on the node): %w", unit, err)
+		}
+	}
+	return nil
+}
+
+func describe(ep Endpoint, c Change) string {
+	if c.Item == "dns" {
+		return "PUT /nodes/" + ep.Node + "/dns"
+	}
+	return fmt.Sprintf("%s %s/%s", c.Op, c.Item, c.ID)
+}
+
+// reachable opens a new SSH connection and calls the API, a few times.
+func reachable(ctx context.Context, ep Endpoint) bool {
+	for i := 0; i < ProbeTries; i++ {
+		if i > 0 {
+			select {
+			case <-ctx.Done():
+				return false
+			case <-time.After(ProbeWait):
+			}
+		}
+		sh := ep.Shell
+		if ep.Dial != nil {
+			var err error
+			if sh, err = ep.Dial(); err != nil {
+				continue
+			}
+		}
+		if _, err := sh.Run(ctx, "# confsync:probe\ntrue", nil); err != nil {
+			continue
+		}
+		if _, err := NodeVersion(ctx, ep); err == nil {
+			return true
+		}
+	}
+	return false
+}
+
+func nodePath(ep Endpoint, rest string) string {
+	return "nodes/" + url.PathEscape(ep.Node) + "/" + rest
+}
+
+// writeAPI writes one interface (POST, PUT with every field, DELETE) or the
+// DNS servers (PUT with every server).
+func writeAPI(ctx context.Context, ep Endpoint, c Change) error {
+	if c.Item == "dns" {
+		f := url.Values{}
+		for k, v := range c.Full {
+			f.Set(k, pve.Str(v))
+		}
+		return ep.API.Put(ctx, nodePath(ep, "dns"), f, nil)
+	}
+	switch c.Op {
+	case OpCreate:
+		f := form(c.Full)
+		f.Set("iface", c.ID)
+		return ep.API.Post(ctx, nodePath(ep, "network"), f, nil)
+	case OpUpdate:
+		f := form(c.Full)
+		if len(c.Unset) > 0 {
+			f.Set("delete", strings.Join(c.Unset, ","))
+		}
+		return ep.API.Put(ctx, nodePath(ep, "network/"+url.PathEscape(c.ID)), f, nil)
+	case OpDelete:
+		return ep.API.Delete(ctx, nodePath(ep, "network/"+url.PathEscape(c.ID)), nil, nil)
+	}
+	return fmt.Errorf("unknown op %q", c.Op)
+}
+
+func form(fields map[string]any) url.Values {
+	f := url.Values{}
+	keys := make([]string, 0, len(fields))
+	for k := range fields {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		f.Set(k, pve.Str(fields[k]))
+	}
+	return f
+}
+
+// reloadNetworkAPI applies interfaces.new (PUT /nodes/{node}/network: rename
+// then ifreload -a in a task) and waits for the task.
+func reloadNetworkAPI(ctx context.Context, ep Endpoint) error {
+	var upid string
+	if err := ep.API.Put(ctx, nodePath(ep, "network"), url.Values{}, &upid); err != nil {
+		return err
+	}
+	return ep.API.WaitTask(ctx, upid, TaskPoll)
+}
+
+// ---- scripts run on the node (the first line is a marker for the fake node
+// of the tests) ----
+
+// prepareInput is the plan given to prepareScript on its standard input:
+// "W <mode> <owner> <hash> <path>" then the content in base64 on one line,
+// "D - - <hash> <path>" (delete) or "B - - - <path>" (backup only).
+func prepareInput(plan Plan, netAPI, dns bool) []byte {
+	var b bytes.Buffer
+	if netAPI {
+		b.WriteString("B - - - /etc/network/interfaces\nB - - - /etc/network/interfaces.new\n")
+	}
+	if dns {
+		b.WriteString("B - - - /etc/resolv.conf\n")
+	}
+	for _, c := range plan.Changes {
+		switch {
+		case c.Item == "network" || c.Item == "dns":
+		case c.Op == OpDelete:
+			fmt.Fprintf(&b, "D - - %s %s\n", c.Hash, c.ID)
+		default:
+			fmt.Fprintf(&b, "W %s %s %s %s\n%s\n", c.Mode, c.Owner, c.Hash, c.ID, base64.StdEncoding.EncodeToString(c.Content))
+		}
+	}
+	return b.Bytes()
+}
+
+func prepareScript(dir string) string { return prepareScriptIn(dir, "") }
+
+// prepareScriptIn prefixes every path with root (empty on a node): the tests
+// run the script for real in a temporary folder.
+func prepareScriptIn(dir, root string) string {
+	return `# confsync:prepare ` + dir + `
+set -u
+R='` + root + `'
+B="$R` + dir + `"
+umask 077
+mkdir -p "$B/files" || exit 1
+cat > "$B/plan" || exit 1
+sum() { if [ -e "$1" ]; then sha256sum < "$1" | cut -d' ' -f1; else echo absent; fi; }
+# 1. every file must be the one read by the dry-run
+while read -r op mode owner h p; do
+  case "$op" in W|D) ;; *) continue ;; esac
+  [ "$(sum "$R$p")" = "$h" ] || { echo "changed since the dry-run: $p" >&2; exit 3; }
+done < "$B/plan"
+# 2. backups and restore script
+: > "$B/restore.sh"
+while read -r op mode owner h p; do
+  case "$op" in W|D|B) ;; *) continue ;; esac
+  if [ -e "$R$p" ]; then
+    mkdir -p "$B/files$(dirname "$p")" && cp -p "$R$p" "$B/files$p" || exit 1
+    printf "cp -p '%s' '%s'\n" "$B/files$p" "$R$p" >> "$B/restore.sh"
+  else
+    printf "rm -f '%s'\n" "$R$p" >> "$B/restore.sh"
+  fi
+done < "$B/plan"
+# 3. writes: temporary file in the same folder, then mv
+while read -r op mode owner h p; do
+  case "$op" in
+  W)
+    read -r data
+    mkdir -p "$(dirname "$R$p")" || exit 4
+    t=$(mktemp "$R$p.confsync.XXXXXX") || exit 4
+    if printf '%s' "$data" | base64 -d > "$t" && chmod "$mode" "$t" && chown "$owner" "$t" && mv -f "$t" "$R$p"; then :; else
+      rm -f "$t"; sh "$B/restore.sh"; echo "write failed: $p" >&2; exit 4
+    fi ;;
+  D)
+    rm -f "$R$p" || { sh "$B/restore.sh"; echo "delete failed: $p" >&2; exit 4; } ;;
+  esac
+done < "$B/plan"
+echo ok
+`
+}
+
+func restoreScript(dir string) string {
+	return "# confsync:restore " + dir + "\nsh '" + dir + "/restore.sh'\n"
+}
+
+func armScript(dir, unit string) string {
+	return fmt.Sprintf("# confsync:arm %s\nsystemd-run --quiet --unit=%s --on-active=%ds --timer-property=AccuracySec=1s /bin/sh -c 'sh %s/restore.sh; ifreload -a'\n",
+		unit, unit, int(RestoreDelay.Seconds()), dir)
+}
+
+func disarmScript(unit string) string {
+	return "# confsync:disarm " + unit + "\nsystemctl stop " + unit + ".timer\n"
+}
+
+func reloadScript(cmd string) string {
+	return "# confsync:reload " + cmd + "\n" + cmd + "\n"
+}

internal/nodesync/diff.go

@@ -0,0 +1,148 @@
+package nodesync
+
+import (
+	"strconv"
+	"strings"
+)
+
+// Line is one line of a unified diff: Op is ' ' (context), '-' (target only)
+// or '+' (written by the replication). A hunk starts with Op '@' and Text
+// "@@ -a,b +c,d @@".
+type Line struct {
+	Op   string `json:"op"`
+	Text string `json:"text"`
+}
+
+const diffContext = 3
+
+// maxCells bounds the LCS table; beyond, the diff shows every line replaced.
+const maxCells = 4_000_000
+
+// Diff compares the current content of a file with the content to write.
+func Diff(oldText, newText []byte) []Line {
+	a, b := lines(oldText), lines(newText)
+	ops := editScript(a, b)
+	// group the changes into hunks with their context
+	var out []Line
+	n := len(ops)
+	i := 0
+	for i < n {
+		if ops[i].Op == " " {
+			i++
+			continue
+		}
+		start := i - diffContext
+		if start < 0 {
+			start = 0
+		}
+		end := i
+		for end < n {
+			if ops[end].Op != " " {
+				end++
+				continue
+			}
+			// a run of context: the hunk ends when it is longer than 2*context
+			j := end
+			for j < n && ops[j].Op == " " {
+				j++
+			}
+			if j == n || j-end > 2*diffContext {
+				end += diffContext
+				if end > n {
+					end = n
+				}
+				break
+			}
+			end = j
+		}
+		oa, ob := ops[start].A, ops[start].B
+		ca, cb := 0, 0
+		for _, o := range ops[start:end] {
+			if o.Op != "+" {
+				ca++
+			}
+			if o.Op != "-" {
+				cb++
+			}
+		}
+		out = append(out, Line{Op: "@", Text: hunkHeader(oa, ca, ob, cb)})
+		for _, o := range ops[start:end] {
+			out = append(out, Line{Op: o.Op, Text: o.Text})
+		}
+		i = end
+	}
+	return out
+}
+
+type editOp struct {
+	Op   string
+	Text string
+	A, B int // line numbers (1-based) where this op starts in a and b
+}
+
+func editScript(a, b []string) []editOp {
+	var ops []editOp
+	if len(a)*len(b) > maxCells {
+		for i, l := range a {
+			ops = append(ops, editOp{"-", l, i + 1, 1})
+		}
+		for i, l := range b {
+			ops = append(ops, editOp{"+", l, len(a) + 1, i + 1})
+		}
+		return ops
+	}
+	// lcs[i][j]: length of the LCS of a[i:] and b[j:]
+	lcs := make([][]int, len(a)+1)
+	for i := range lcs {
+		lcs[i] = make([]int, len(b)+1)
+	}
+	for i := len(a) - 1; i >= 0; i-- {
+		for j := len(b) - 1; j >= 0; j-- {
+			if a[i] == b[j] {
+				lcs[i][j] = lcs[i+1][j+1] + 1
+			} else if lcs[i+1][j] >= lcs[i][j+1] {
+				lcs[i][j] = lcs[i+1][j]
+			} else {
+				lcs[i][j] = lcs[i][j+1]
+			}
+		}
+	}
+	i, j := 0, 0
+	for i < len(a) || j < len(b) {
+		switch {
+		case i < len(a) && j < len(b) && a[i] == b[j]:
+			ops = append(ops, editOp{" ", a[i], i + 1, j + 1})
+			i++
+			j++
+		case j < len(b) && (i == len(a) || lcs[i][j+1] > lcs[i+1][j]):
+			ops = append(ops, editOp{"+", b[j], i + 1, j + 1})
+			j++
+		default:
+			ops = append(ops, editOp{"-", a[i], i + 1, j + 1})
+			i++
+		}
+	}
+	return ops
+}
+
+func lines(b []byte) []string {
+	s := string(b)
+	if s == "" {
+		return nil
+	}
+	s = strings.TrimSuffix(s, "\n")
+	return strings.Split(s, "\n")
+}
+
+func hunkHeader(a, ca, b, cb int) string {
+	r := func(start, n int) string {
+		if n == 0 {
+			start--
+		}
+		if n == 1 {
+			return strconv.Itoa(start)
+		}
+		return strconv.Itoa(start) + "," + strconv.Itoa(n)
+	}
+	return "@@ -" + r(a, ca) + " +" + r(b, cb) + " @@"
+}

internal/nodesync/items.go

@@ -0,0 +1,100 @@
+// Package nodesync replicates the system configuration of one PVE node (the
+// master) to other nodes (the targets), of the same cluster or not. It is the
+// second replication mode of pve-confsync, next to the cluster to cluster one
+// (package engine). What the PVE API covers goes through the API: the network
+// interfaces (/nodes/{node}/network, written in interfaces.new then reloaded)
+// and the DNS servers (/nodes/{node}/dns). The other files go through SSH as
+// root on the node: multipath, chrony, sysctl, interfaces.d. Same pipeline as
+// the engine: read, normalize, diff, plan, apply with backup, reload, verify.
+package nodesync
+
+import "fmt"
+
+// ItemKind tells how an item is read and written.
+type ItemKind string
+
+const (
+	KindNetwork ItemKind = "network" // API /nodes/{node}/network
+	KindDNS     ItemKind = "dns"     // API /nodes/{node}/dns
+	KindFile    ItemKind = "file"    // files over SSH
+)
+
+// Item is one replicable element.
+type Item struct {
+	ID   string   `json:"id"`
+	Kind ItemKind `json:"kind"`
+	// Files: paths or glob patterns read on the nodes, and paths never touched.
+	Paths   []string `json:"paths,omitempty"`
+	Exclude []string `json:"exclude,omitempty"`
+	// Glob items hold several files: a file only on the target is kept, or
+	// deleted with the prune option. Single file items are never deleted.
+	Glob bool `json:"glob,omitempty"`
+	// Additive files (multipath wwids) only get the lines of the master they
+	// miss: nothing is removed from the target.
+	Additive bool `json:"additive,omitempty"`
+	// Reload is the command run on the node after a write (empty: none).
+	Reload string `json:"reload,omitempty"`
+	// Network items reload the network of the node: typed confirmation,
+	// automatic restore when the node is unreachable afterwards.
+	Network bool `json:"network,omitempty"`
+}
+
+// Items is the catalog, in apply order (reloads follow this order, the
+// network last).
+var Items = []Item{
+	{ID: "sysctl", Kind: KindFile, Paths: []string{"/etc/sysctl.d/*.conf"}, Glob: true, Reload: "sysctl --system"},
+	{ID: "chrony", Kind: KindFile, Paths: []string{"/etc/chrony/chrony.conf", "/etc/chrony/sources.d/*.sources"}, Glob: true,
+		Reload: "systemctl restart chrony"},
+	{ID: "multipath", Kind: KindFile, Paths: []string{"/etc/multipath.conf"}, Reload: "multipathd reconfigure"},
+	{ID: "wwids", Kind: KindFile, Paths: []string{"/etc/multipath/wwids"}, Additive: true, Reload: "multipathd reconfigure"},
+	{ID: "dns", Kind: KindDNS},
+	// interfaces.d/sdn is generated by the SDN of PVE: never replicated
+	{ID: "interfaces-d", Kind: KindFile, Paths: []string{"/etc/network/interfaces.d/*"}, Exclude: []string{"/etc/network/interfaces.d/sdn"},
+		Glob: true, Reload: "ifreload -a", Network: true},
+	{ID: "network", Kind: KindNetwork, Network: true},
+}
+
+// ItemByID returns an item of the catalog.
+func ItemByID(id string) (Item, error) {
+	for _, it := range Items {
+		if it.ID == id {
+			return it, nil
+		}
+	}
+	return Item{}, fmt.Errorf("unknown item %q", id)
+}
+
+func itemRank(id string) int {
+	for i, it := range Items {
+		if it.ID == id {
+			return i
+		}
+	}
+	return len(Items)
+}
+
+// Policy of a field of an API item (network, dns).
+type Policy string
+
+const (
+	Sync     Policy = "sync"     // copied from the master
+	Ignore   Policy = "ignore"   // never written
+	Override Policy = "override" // only written from the target mapping
+)
+
+// defaultPolicy: the addresses and gateways are those of each node; they are
+// only written from the mapping of a target.
+func defaultPolicy(item, field string) Policy {
+	if item == "network" {
+		switch field {
+		case "cidr", "cidr6", "gateway", "gateway6":
+			return Override
+		}
+	}
+	return Sync
+}
+
+// DefaultPolicies lists the fields whose default is not "sync", for the UI.
+var DefaultPolicies = map[string]map[string]Policy{
+	"network": {"cidr": Override, "cidr6": Override, "gateway": Override, "gateway6": Override},
+}

internal/nodesync/nodesync_test.go

@@ -0,0 +1,304 @@
+package nodesync
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"strings"
+	"testing"
+	"time"
+
+	"pve-confsync/internal/config"
+	"pve-confsync/internal/pve/pvetest"
+	"pve-confsync/internal/sshx"
+)
+
+func init() {
+	ProbeWait, TaskPoll = time.Millisecond, time.Millisecond
+}
+
+// endpoint connects to a node of a fake cluster like the web UI does.
+func endpoint(t *testing.T, s *pvetest.Server, cluster string, n *pvetest.Node) Endpoint {
+	t.Helper()
+	sum := sha256.Sum256(s.Certificate().Raw)
+	c, err := config.Cluster{Name: cluster, URL: s.URL, User: "root@pam", Password: pvetest.Password,
+		Fingerprint: hex.EncodeToString(sum[:])}.PVE()
+	if err != nil {
+		t.Fatal(err)
+	}
+	host, port := n.SSHAddr()
+	o := sshx.Options{Host: host, Port: port, Password: pvetest.Password, HostKey: n.SSHHostKey()}
+	sh, err := sshx.New(o)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return Endpoint{Key: cluster + "/" + n.Name, Cluster: cluster, Node: n.Name, API: c, Shell: sh}
+}
+
+const multipath = "defaults {\n    user_friendly_names yes\n    find_multipaths no\n}\nblacklist {\n    devnode \"^sda\"\n}\n"
+
+// lab: one cluster, a master pve1 and a target pve2 with differences.
+func lab(t *testing.T) (*pvetest.Server, *pvetest.Node, *pvetest.Node, map[string]Endpoint) {
+	s := pvetest.New("9.1.4")
+	t.Cleanup(s.Close)
+	m, c := s.AddNode("pve1"), s.AddNode("pve2")
+	m.Net = map[string]map[string]any{
+		"eno1":     {"type": "eth", "mtu": "9000"},
+		"eno2":     {"type": "eth", "mtu": "9000"},
+		"bond0":    {"type": "bond", "slaves": "eno1 eno2", "bond_mode": "802.3ad", "bond_xmit_hash_policy": "layer3+4", "mtu": "9000"},
+		"vmbr0":    {"type": "bridge", "bridge_ports": "bond0", "bridge_vlan_aware": "1", "cidr": "10.0.0.11/24", "gateway": "10.0.0.1", "autostart": "1"},
+		"vmbr1":    {"type": "bridge", "bridge_ports": "bond0.50", "cidr": "10.5.0.11/24", "comments": "stockage"},
+		"bond0.50": {"type": "vlan", "mtu": "9000"},
+	}
+	// the target has other NIC names and an active-backup bond
+	c.Net = map[string]map[string]any{
+		"enp1s0f0": {"type": "eth"},
+		"enp1s0f1": {"type": "eth"},
+		"bond0":    {"type": "bond", "slaves": "enp1s0f0 enp1s0f1", "bond_mode": "active-backup"},
+		"vmbr0":    {"type": "bridge", "bridge_ports": "bond0", "cidr": "10.0.0.12/24", "gateway": "10.0.0.1", "autostart": "1"},
+	}
+	m.DNS = map[string]string{"search": "lab.local", "dns1": "10.0.0.53", "dns2": "10.0.0.54"}
+	c.DNS = map[string]string{"search": "lab.local", "dns1": "8.8.8.8"}
+	m.Files["/etc/multipath.conf"] = pvetest.FakeFile{Content: multipath}
+	m.Files["/etc/multipath/wwids"] = pvetest.FakeFile{Content: "# wwids\n/3600a0980383030/\n/3600a0980383031/\n"}
+	c.Files["/etc/multipath/wwids"] = pvetest.FakeFile{Content: "# wwids\n/3600a0980383031/\n/36000local/\n"}
+	m.Files["/etc/sysctl.d/90-net.conf"] = pvetest.FakeFile{Content: "net.core.rmem_max = 16777216\n"}
+	c.Files["/etc/sysctl.d/90-net.conf"] = pvetest.FakeFile{Content: "net.core.rmem_max = 8388608\n", Mode: "600"}
+	c.Files["/etc/sysctl.d/99-local.conf"] = pvetest.FakeFile{Content: "vm.swappiness = 10\n"}
+	m.Files["/etc/network/interfaces.d/sdn"] = pvetest.FakeFile{Content: "auto vnet1\n"}
+	eps := map[string]Endpoint{"lab/pve1": endpoint(t, s, "lab", m), "lab/pve2": endpoint(t, s, "lab", c)}
+	return s, m, c, eps
+}
+
+func profile() *Profile {
+	return &Profile{Name: "stockage", Master: "lab/pve1", Targets: []string{"lab/pve2"},
+		Items: []string{"network", "dns", "multipath", "wwids", "sysctl", "interfaces-d"},
+		Mapping: map[string]TargetMap{"lab/pve2": {
+			NICs:   map[string]string{"eno1": "enp1s0f0", "eno2": "enp1s0f1"},
+			Values: map[string]map[string]string{"network/vmbr1": {"cidr": "10.5.0.12/24"}},
+		}}}
+}
+
+func keys(p Plan) []string {
+	var out []string
+	for _, c := range p.Changes {
+		out = append(out, string(c.Op)+" "+c.Key())
+	}
+	return out
+}
+
+func TestPlan(t *testing.T) {
+	_, _, _, eps := lab(t)
+	prof := profile()
+	st := Analyze(context.Background(), prof, eps, nil)
+	if st.Master.Error != "" || st.Targets[0].Error != "" {
+		t.Fatal(st.Master.Error, st.Targets[0].Error)
+	}
+	p := st.Targets[0].Plan
+	if len(p.Errors) > 0 {
+		t.Fatalf("errors: %+v", p.Errors)
+	}
+	got := strings.Join(keys(p), "\n")
+	want := strings.Join([]string{
+		"update sysctl//etc/sysctl.d/90-net.conf",
+		"create multipath//etc/multipath.conf",
+		"update wwids//etc/multipath/wwids",
+		"update dns/dns",
+		"update network/enp1s0f0",
+		"update network/enp1s0f1",
+		"update network/bond0",
+		"create network/bond0.50",
+		"update network/vmbr0",
+		"create network/vmbr1",
+	}, "\n")
+	if got != want {
+		t.Fatalf("plan:\n%s\nwant:\n%s", got, want)
+	}
+	for _, c := range p.Changes {
+		switch c.Key() {
+		case "network/bond0":
+			if c.Set["slaves"] != "enp1s0f0 enp1s0f1" && c.Set["slaves"] != nil {
+				t.Errorf("slaves not renamed: %v", c.Set["slaves"])
+			}
+			if c.Set["bond_mode"] != "802.3ad" {
+				t.Errorf("bond0: %+v", c.Set)
+			}
+		case "network/vmbr0":
+			// the address of the target is kept (override without mapping)
+			if _, ok := c.Set["cidr"]; ok || c.Full["cidr"] != "10.0.0.12/24" {
+				t.Errorf("vmbr0: set %+v full %+v", c.Set, c.Full)
+			}
+		case "network/vmbr1":
+			if c.Full["cidr"] != "10.5.0.12/24" || c.Full["bridge_ports"] != "bond0.50" {
+				t.Errorf("vmbr1: %+v", c.Full)
+			}
+		case "wwids//etc/multipath/wwids":
+			// additive: the local WWID of the target stays
+			if string(c.Content) != "# wwids\n/3600a0980383031/\n/36000local/\n/3600a0980383030/\n" {
+				t.Errorf("wwids: %q", c.Content)
+			}
+		case "sysctl//etc/sysctl.d/90-net.conf":
+			if c.Mode != "600" || len(c.Diff) == 0 {
+				t.Errorf("sysctl: mode %s diff %+v", c.Mode, c.Diff)
+			}
+		}
+	}
+	// the file only on the target is kept without prune; interfaces.d/sdn is never read
+	for _, r := range st.Targets[0].Rows {
+		if r.ID == "/etc/sysctl.d/99-local.conf" && r.Status != StatusExtra {
+			t.Errorf("99-local.conf: %s", r.Status)
+		}
+		if strings.Contains(r.ID, "interfaces.d/sdn") {
+			t.Errorf("sdn file listed")
+		}
+	}
+	if !p.Network() {
+		t.Error("network plan not flagged")
+	}
+	if strings.Join(p.Reloads(), ",") != "sysctl --system,multipathd reconfigure,ifreload -a" {
+		t.Errorf("reloads: %v", p.Reloads())
+	}
+}
+
+func TestApplyAndVerify(t *testing.T) {
+	_, _, c, eps := lab(t)
+	prof := profile()
+	st := Analyze(context.Background(), prof, eps, nil)
+	tp := st.Plans(nil)[0]
+	var steps []string
+	if err := st.Apply(context.Background(), tp, nil, func(s Step) {
+		steps = append(steps, s.Action+" "+s.Error)
+	}); err != nil {
+		t.Fatalf("apply: %v\n%s", err, strings.Join(steps, "\n"))
+	}
+	if c.Net["vmbr0"]["cidr"] != "10.0.0.12/24" || c.Net["vmbr1"]["cidr"] != "10.5.0.12/24" {
+		t.Errorf("addresses: vmbr0 %v vmbr1 %v", c.Net["vmbr0"], c.Net["vmbr1"])
+	}
+	if c.Net["vmbr0"]["bridge_vlan_aware"] != "1" || c.Pending != nil {
+		t.Errorf("vmbr0 %v pending %v", c.Net["vmbr0"], c.Pending)
+	}
+	if c.DNS["dns2"] != "10.0.0.54" || c.Files["/etc/multipath.conf"].Content != multipath {
+		t.Errorf("dns %v multipath %q", c.DNS, c.Files["/etc/multipath.conf"].Content)
+	}
+	if len(c.Armed) != 0 {
+		t.Errorf("restore timer still armed: %v", c.Armed)
+	}
+	if got := strings.Join(c.Reloads, ","); got != "sysctl --system,multipathd reconfigure,network" {
+		t.Errorf("reloads: %s", got)
+	}
+	if len(c.Backups()) != 1 {
+		t.Errorf("backups: %v", c.Backups())
+	}
+	st = Analyze(context.Background(), prof, eps, nil)
+	if p := st.Targets[0].Plan; !p.Empty() {
+		t.Errorf("still to do: %v", keys(p))
+	}
+}
+
+func TestReloadFailureRestores(t *testing.T) {
+	_, _, c, eps := lab(t)
+	prof := profile()
+	prof.Items = []string{"multipath", "sysctl"}
+	c.FailOn = "reload multipathd reconfigure"
+	st := Analyze(context.Background(), prof, eps, nil)
+	err := st.Apply(context.Background(), st.Plans(nil)[0], nil, func(Step) {})
+	if err == nil || !strings.Contains(err.Error(), "multipathd reconfigure") {
+		t.Fatalf("err = %v", err)
+	}
+	if _, ok := c.Files["/etc/multipath.conf"]; ok {
+		t.Error("multipath.conf not removed by the restore")
+	}
+	if c.Files["/etc/sysctl.d/90-net.conf"].Content != "net.core.rmem_max = 8388608\n" {
+		t.Errorf("sysctl not restored: %q", c.Files["/etc/sysctl.d/90-net.conf"].Content)
+	}
+}
+
+func TestUnreachableAfterReloadKeepsTimer(t *testing.T) {
+	_, _, c, eps := lab(t)
+	prof := profile()
+	prof.Items = []string{"network"}
+	st := Analyze(context.Background(), prof, eps, nil)
+	c.FailOn = "probe"
+	err := st.Apply(context.Background(), st.Plans(nil)[0], nil, func(Step) {})
+	if err == nil || !strings.Contains(err.Error(), "unreachable") {
+		t.Fatalf("err = %v", err)
+	}
+	if len(c.Armed) != 1 {
+		t.Fatalf("timer: %v", c.Armed)
+	}
+	// systemd runs the restore: the previous interfaces come back
+	c.Fire()
+	if c.Net["bond0"]["bond_mode"] != "active-backup" || c.Net["vmbr1"] != nil {
+		t.Errorf("not restored: %v", c.Net)
+	}
+}
+
+func TestStaleFileRefused(t *testing.T) {
+	_, _, c, eps := lab(t)
+	prof := profile()
+	prof.Items = []string{"sysctl"}
+	st := Analyze(context.Background(), prof, eps, nil)
+	c.Files["/etc/sysctl.d/90-net.conf"] = pvetest.FakeFile{Content: "changed by hand\n"}
+	err := st.Apply(context.Background(), st.Plans(nil)[0], nil, func(Step) {})
+	if err == nil || !strings.Contains(err.Error(), "changed since the dry-run") {
+		t.Fatalf("err = %v", err)
+	}
+	if c.Files["/etc/sysctl.d/90-net.conf"].Content != "changed by hand\n" {
+		t.Error("file overwritten")
+	}
+}
+
+func TestPendingNetworkBlocks(t *testing.T) {
+	_, _, c, eps := lab(t)
+	c.Pending = map[string]map[string]any{"vmbr0": {"type": "bridge"}}
+	st := Analyze(context.Background(), profile(), eps, nil)
+	if len(st.Targets[0].Plan.Errors) == 0 {
+		t.Fatal("pending changes not blocking")
+	}
+}
+
+func TestMissingNICIsAnError(t *testing.T) {
+	_, _, _, eps := lab(t)
+	prof := profile()
+	prof.Items = []string{"network"}
+	prof.Mapping = nil
+	st := Analyze(context.Background(), prof, eps, nil)
+	p := st.Targets[0].Plan
+	if len(p.Errors) != 2 || p.Errors[0].Key != "network/eno1" {
+		t.Fatalf("errors: %+v", p.Errors)
+	}
+	// the selection of another object is not blocked
+	if f := Filter(p, []string{"network/vmbr0"}); len(f.Errors) != 0 || len(f.Changes) != 1 {
+		t.Errorf("filter: %+v", f)
+	}
+}
+
+func TestPruneAndProtect(t *testing.T) {
+	_, m, _, eps := lab(t)
+	m.Files["/etc/sysctl.d/90-net.conf"] = pvetest.FakeFile{Content: "# host pve1\nnet.core.rmem_max = 8388608\n"}
+	prof := profile()
+	prof.Items = []string{"sysctl"}
+	prof.Prune = true
+	prof.Protect = map[string][]string{"sysctl": {"^# host "}}
+	st := Analyze(context.Background(), prof, eps, nil)
+	got := strings.Join(keys(st.Targets[0].Plan), ",")
+	// the protected line is absent on the target: dropped, the rest is equal
+	if got != "delete sysctl//etc/sysctl.d/99-local.conf" {
+		t.Fatalf("plan: %s", got)
+	}
+}
+
+func TestDiff(t *testing.T) {
+	d := Diff([]byte("a\nb\nc\nd\ne\nf\ng\nh\ni\n"), []byte("a\nb\nc\nd\nE\nf\ng\nh\ni\nj\n"))
+	var b strings.Builder
+	for _, l := range d {
+		b.WriteString(l.Op + l.Text + "\n")
+	}
+	want := "@@@ -2,8 +2,9 @@\n b\n c\n d\n-e\n+E\n f\n g\n h\n i\n+j\n"
+	if b.String() != want {
+		t.Fatalf("diff:\n%s\nwant:\n%s", b.String(), want)
+	}
+	if d := Diff(nil, []byte("x\n")); len(d) != 2 || d[0].Text != "@@ -0,0 +1 @@" {
+		t.Errorf("creation diff: %+v", d)
+	}
+}

internal/nodesync/plan.go

@@ -0,0 +1,661 @@
+package nodesync
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"regexp"
+	"sort"
+	"strings"
+
+	"pve-confsync/internal/model"
+)
+
+// Profile is one node replication: a master node, target nodes, the
+// replicated items and their rules.
+type Profile struct {
+	Name    string   `json:"name"`
+	Master  string   `json:"master"`  // "cluster/node"
+	Targets []string `json:"targets"` // "cluster/node"
+	Items   []string `json:"items"`
+	// Prune deletes on the targets what the master does not have: network
+	// interfaces (never the physical ones) and the files of the glob items.
+	Prune bool `json:"prune"`
+	// Policies of the fields of the API items (item -> field -> policy),
+	// over the defaults (addresses and gateways: override).
+	Policies map[string]map[string]Policy `json:"policies"`
+	// Protect: per file item, regular expressions of the lines kept from the
+	// target (hostname, local address…).
+	Protect map[string][]string `json:"protect"`
+	// Mapping: per target, its interface names and its own values.
+	Mapping  map[string]TargetMap `json:"mapping"`
+	Position int                  `json:"position"`
+}
+
+// TargetMap holds what is specific to one target.
+type TargetMap struct {
+	// NICs renames the interfaces of the master (eno1 -> enp1s0f0), in the
+	// interface names and in the fields that list interfaces.
+	NICs map[string]string `json:"nics,omitempty"`
+	// Values: "item/id" -> field -> value written on this target (cidr of
+	// vmbr0…), whatever the master has.
+	Values map[string]map[string]string `json:"values,omitempty"`
+}
+
+// Validate checks a profile.
+func (p *Profile) Validate() error {
+	var errs []error
+	p.Name = strings.TrimSpace(p.Name)
+	if p.Name == "" {
+		errs = append(errs, errors.New("name is required"))
+	}
+	if p.Master == "" {
+		errs = append(errs, errors.New("master node is required"))
+	}
+	seen := map[string]bool{}
+	for _, t := range p.Targets {
+		if t == p.Master {
+			errs = append(errs, fmt.Errorf("target %s is the master", t))
+		}
+		if seen[t] {
+			errs = append(errs, fmt.Errorf("target %s listed twice", t))
+		}
+		seen[t] = true
+	}
+	if len(p.Items) == 0 {
+		errs = append(errs, errors.New("select at least one item"))
+	}
+	for _, id := range p.Items {
+		if _, err := ItemByID(id); err != nil {
+			errs = append(errs, err)
+		}
+	}
+	for item, fields := range p.Policies {
+		for f, pol := range fields {
+			switch pol {
+			case Sync, Ignore, Override:
+			default:
+				errs = append(errs, fmt.Errorf("%s.%s: unknown policy %q", item, f, pol))
+			}
+			if item == "network" && f == "type" && pol != Sync {
+				errs = append(errs, errors.New("network.type: always synchronized"))
+			}
+		}
+	}
+	for item, res := range p.Protect {
+		for _, re := range res {
+			if _, err := regexp.Compile(re); err != nil {
+				errs = append(errs, fmt.Errorf("%s: protected lines: %w", item, err))
+			}
+		}
+	}
+	return errors.Join(errs...)
+}
+
+// Policy resolves the policy of a field.
+func (p *Profile) Policy(item, field string) Policy {
+	if item == "network" && field == "type" {
+		return Sync
+	}
+	if pol, ok := p.Policies[item][field]; ok {
+		return pol
+	}
+	return defaultPolicy(item, field)
+}
+
+// Has tells whether an item is replicated.
+func (p *Profile) Has(item string) bool {
+	for _, x := range p.Items {
+		if x == item {
+			return true
+		}
+	}
+	return false
+}
+
+// Op is the action planned on one object.
+type Op string
+
+const (
+	OpCreate Op = "create"
+	OpUpdate Op = "update"
+	OpDelete Op = "delete"
+)
+
+// Change is one planned action: an interface or the DNS servers (Set, Unset,
+// Old), or a file (Diff, Hash of the target file, content to write).
+type Change struct {
+	Item  string         `json:"kind"` // item id ("kind" like the cluster plans, for the UI)
+	ID    string         `json:"id"`   // interface, "dns" or file path
+	Op    Op             `json:"op"`
+	Set   map[string]any `json:"set,omitempty"`
+	Unset []string       `json:"unset,omitempty"`
+	Old   map[string]any `json:"old,omitempty"`
+	Diff  []Line         `json:"diff,omitempty"`
+	Hash  string         `json:"hash,omitempty"` // current file on the target ("absent" when created)
+	Mode  string         `json:"mode,omitempty"`
+	Owner string         `json:"owner,omitempty"`
+	// Full is the whole writable configuration of an interface after the
+	// change: PUT /nodes/{node}/network/{iface} recomputes the method and the
+	// families from the parameters, so every field is sent again.
+	Full    map[string]any `json:"-"`
+	Content []byte         `json:"-"`
+}
+
+// Key identifies the object of a change ("item/id").
+func (c Change) Key() string { return c.Item + "/" + c.ID }
+
+// Issue is a blocking error or a warning, about one object (Key) or the
+// whole plan.
+type Issue struct {
+	Key     string `json:"key,omitempty"`
+	Message string `json:"message"`
+}
+
+// Plan is the list of changes for one target, in execution order.
+type Plan struct {
+	Target   string   `json:"client"`
+	Changes  []Change `json:"changes"`
+	Errors   []Issue  `json:"errors,omitempty"`
+	Warnings []Issue  `json:"warnings,omitempty"`
+}
+
+// Empty tells whether there is nothing to do.
+func (p Plan) Empty() bool { return len(p.Changes) == 0 }
+
+// Network tells whether the plan reloads the network of the node.
+func (p Plan) Network() bool {
+	for _, c := range p.Changes {
+		if it, _ := ItemByID(c.Item); it.Network {
+			return true
+		}
+	}
+	return false
+}
+
+// Reloads lists the commands run after the writes, in order.
+func (p Plan) Reloads() []string {
+	seen := map[string]bool{}
+	var out []string
+	for _, it := range Items {
+		for _, c := range p.Changes {
+			if c.Item != it.ID {
+				continue
+			}
+			cmd := it.Reload
+			if it.Kind == KindNetwork {
+				cmd = "ifreload -a"
+			}
+			if cmd != "" && !seen[cmd] {
+				seen[cmd] = true
+				out = append(out, cmd)
+			}
+			break
+		}
+	}
+	return out
+}
+
+func (p *Plan) errorf(key, format string, a ...any) {
+	p.Errors = append(p.Errors, Issue{key, fmt.Sprintf(format, a...)})
+}
+
+func (p *Plan) warnf(key, format string, a ...any) {
+	p.Warnings = append(p.Warnings, Issue{key, fmt.Sprintf(format, a...)})
+}
+
+// MakePlan computes the changes that bring the target in line with the master.
+func MakePlan(prof *Profile, master, target Snapshot) Plan {
+	plan := Plan{Target: target.Key}
+	tm := prof.Mapping[target.Key]
+	if target.Pending && prof.Has("network") {
+		plan.errorf("", "pending network changes on %s (interfaces.new): apply or revert them in PVE first", target.Key)
+	}
+	if master.Pending && prof.Has("network") {
+		plan.warnf("", "the master has pending network changes: its pending configuration is replicated")
+	}
+	for _, id := range prof.Items {
+		it, _ := ItemByID(id)
+		switch it.Kind {
+		case KindNetwork:
+			planNetwork(&plan, prof, tm, master, target)
+		case KindDNS:
+			planDNS(&plan, prof, tm, master, target)
+		case KindFile:
+			planFiles(&plan, prof, it, master.Files[it.ID], target.Files[it.ID])
+		}
+	}
+	sortChanges(plan.Changes)
+	return plan
+}
+
+// rank orders the interfaces: what is referenced first (ports, bonds, VLAN
+// devices), bridges last; deletions run in the reverse order.
+func netRank(typ string) int {
+	switch typ {
+	case "eth", "alias":
+		return 0
+	case "bond", "OVSBond":
+		return 1
+	case "vlan":
+		return 2
+	case "bridge", "OVSBridge":
+		return 3
+	}
+	return 4
+}
+
+func sortChanges(chs []Change) {
+	typ := func(c Change) string {
+		if t, ok := c.Full["type"].(string); ok {
+			return t
+		}
+		t, _ := c.Old["type"].(string)
+		return t
+	}
+	sort.SliceStable(chs, func(i, j int) bool {
+		a, b := chs[i], chs[j]
+		if (a.Op == OpDelete) != (b.Op == OpDelete) {
+			return b.Op == OpDelete
+		}
+		if a.Item != b.Item {
+			return itemRank(a.Item) < itemRank(b.Item)
+		}
+		if a.Item == "network" {
+			ra, rb := netRank(typ(a)), netRank(typ(b))
+			if a.Op == OpDelete {
+				ra, rb = rb, ra
+			}
+			if ra != rb {
+				return ra < rb
+			}
+		}
+		return a.ID < b.ID
+	})
+}
+
+// netLists are the fields naming other interfaces (space separated lists or
+// one name): renamed with the NIC mapping of the target.
+var netLists = map[string]bool{
+	"bridge_ports": true, "slaves": true, "ovs_ports": true, "ovs_bonds": true,
+	"bond-primary": true, "vlan-raw-device": true, "ovs_bridge": true,
+}
+
+// rename maps an interface name of the master to the target: eno1, and the
+// VLAN interfaces built on it (eno1.100).
+func rename(name string, nics map[string]string) string {
+	if to, ok := nics[name]; ok {
+		return to
+	}
+	if base, vlan, ok := strings.Cut(name, "."); ok {
+		if to, ok := nics[base]; ok {
+			return to + "." + vlan
+		}
+	}
+	return name
+}
+
+func renameList(v string, nics map[string]string) string {
+	parts := strings.Fields(v)
+	for i, p := range parts {
+		parts[i] = rename(p, nics)
+	}
+	return strings.Join(parts, " ")
+}
+
+func planNetwork(plan *Plan, prof *Profile, tm TargetMap, master, target Snapshot) {
+	cur := map[string]model.Object{}
+	for _, o := range target.Network {
+		cur[o.ID] = o
+	}
+	inMaster := map[string]bool{}
+	for _, m := range master.Network {
+		id := rename(m.ID, tm.NICs)
+		key := "network/" + id
+		inMaster[id] = true
+		desired := map[string]any{}
+		managed := map[string]bool{}
+		for f, v := range m.Fields {
+			if prof.Policy("network", f) != Sync {
+				continue
+			}
+			if netLists[f] {
+				v = renameList(v.(string), tm.NICs)
+			}
+			desired[f] = v
+		}
+		for f := range netWritable {
+			if prof.Policy("network", f) == Sync {
+				managed[f] = true
+			}
+		}
+		for f, v := range tm.Values[key] {
+			if prof.Policy("network", f) == Ignore || !netWritable[f] || f == "type" {
+				continue
+			}
+			desired[f] = v
+			managed[f] = true
+		}
+		typ, _ := desired["type"].(string)
+		o, exists := cur[id]
+		if !exists {
+			if physical(typ) {
+				plan.errorf(key, "physical interface %s absent on %s: add it to the NIC mapping of the target", id, target.Key)
+				continue
+			}
+			plan.Changes = append(plan.Changes, Change{Item: "network", ID: id, Op: OpCreate, Set: desired, Full: desired})
+			continue
+		}
+		if t, _ := o.Fields["type"].(string); t != typ {
+			plan.errorf(key, "type differs (%s on the target, %s on the master): re-create the interface by hand", t, typ)
+			continue
+		}
+		if mo, to := strings.Join(master.Options[m.ID], "\n"), strings.Join(target.Options[id], "\n"); mo != to {
+			plan.warnf(key, "options the API does not write differ (%s): not replicated", optionNames(master.Options[m.ID], target.Options[id]))
+		}
+		ch := Change{Item: "network", ID: id, Op: OpUpdate, Set: map[string]any{}, Old: map[string]any{}}
+		full := map[string]any{}
+		for f, v := range o.Fields {
+			full[f] = v
+		}
+		for f, v := range desired {
+			if !model.Equal(v, o.Fields[f]) {
+				ch.Set[f] = v
+				if old, ok := o.Fields[f]; ok {
+					ch.Old[f] = old
+				}
+			}
+			full[f] = v
+		}
+		for f := range managed {
+			if _, want := desired[f]; want {
+				continue
+			}
+			if old, has := o.Fields[f]; has {
+				ch.Unset = append(ch.Unset, f)
+				ch.Old[f] = old
+				delete(full, f)
+			}
+		}
+		sort.Strings(ch.Unset)
+		if len(ch.Set) > 0 || len(ch.Unset) > 0 {
+			ch.Full = full
+			plan.Changes = append(plan.Changes, ch)
+		}
+	}
+	if !prof.Prune {
+		return
+	}
+	for id, o := range cur {
+		if inMaster[id] {
+			continue
+		}
+		if t, _ := o.Fields["type"].(string); physical(t) {
+			continue
+		}
+		plan.Changes = append(plan.Changes, Change{Item: "network", ID: id, Op: OpDelete, Old: o.Fields})
+	}
+}
+
+// optionNames gives the option keywords that differ (bond-miimon, post-up…).
+func optionNames(a, b []string) string {
+	set := map[string]bool{}
+	count := map[string]int{}
+	for _, l := range a {
+		count[l]++
+	}
+	for _, l := range b {
+		count[l]--
+	}
+	for l, n := range count {
+		if n != 0 {
+			set[strings.Fields(l + " ")[0]] = true
+		}
+	}
+	var out []string
+	for k := range set {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return strings.Join(out, ", ")
+}
+
+func planDNS(plan *Plan, prof *Profile, tm TargetMap, master, target Snapshot) {
+	const key = "dns/dns"
+	desired := map[string]any{}
+	for _, f := range dnsFields {
+		pol := prof.Policy("dns", f)
+		if v, ok := master.DNS[f]; ok && pol == Sync {
+			desired[f] = v
+		}
+		if pol != Sync {
+			if v, ok := target.DNS[f]; ok {
+				desired[f] = v // PUT /nodes/{node}/dns rewrites every server
+			}
+		}
+		if v, ok := tm.Values[key][f]; ok && pol != Ignore {
+			desired[f] = v
+		}
+	}
+	ch := Change{Item: "dns", ID: "dns", Op: OpUpdate, Set: map[string]any{}, Old: map[string]any{}, Full: desired}
+	for _, f := range dnsFields {
+		v, want := desired[f]
+		old, has := target.DNS[f]
+		switch {
+		case want && !model.Equal(v, old):
+			ch.Set[f] = v
+			if has {
+				ch.Old[f] = old
+			}
+		case !want && has:
+			ch.Unset = append(ch.Unset, f)
+			ch.Old[f] = old
+		}
+	}
+	if len(ch.Set) == 0 && len(ch.Unset) == 0 {
+		return
+	}
+	if desired["search"] == nil {
+		plan.errorf(key, "a search domain is required by the API (PUT /nodes/{node}/dns)")
+		return
+	}
+	plan.Changes = append(plan.Changes, ch)
+}
+
+func planFiles(plan *Plan, prof *Profile, it Item, master, target map[string]File) {
+	var protect []*regexp.Regexp
+	for _, s := range prof.Protect[it.ID] {
+		if re, err := regexp.Compile(s); err == nil {
+			protect = append(protect, re)
+		}
+	}
+	if len(master) == 0 && !it.Glob {
+		plan.warnf(it.ID+"/"+it.Paths[0], "%s absent on the master: not replicated", it.Paths[0])
+		return
+	}
+	paths := make([]string, 0, len(master))
+	for p := range master {
+		paths = append(paths, p)
+	}
+	sort.Strings(paths)
+	for _, p := range paths {
+		m := master[p]
+		t, exists := target[p]
+		var content []byte
+		switch {
+		case it.Additive && exists:
+			content = additive(t.Content, m.Content)
+		case exists:
+			content = protectLines(m.Content, t.Content, protect)
+		default:
+			content = protectLines(m.Content, nil, protect)
+		}
+		if exists && hash(content) == t.Hash {
+			continue
+		}
+		ch := Change{Item: it.ID, ID: p, Content: content, Mode: m.Mode, Owner: m.Owner, Hash: "absent", Op: OpCreate}
+		var old []byte
+		if exists {
+			// an update keeps the mode and owner of the target file
+			ch.Op, ch.Hash, ch.Mode, ch.Owner, old = OpUpdate, t.Hash, t.Mode, t.Owner, t.Content
+		}
+		ch.Diff = Diff(old, content)
+		plan.Changes = append(plan.Changes, ch)
+	}
+	if !prof.Prune || !it.Glob {
+		return
+	}
+	for p, t := range target {
+		if _, ok := master[p]; ok {
+			continue
+		}
+		plan.Changes = append(plan.Changes, Change{Item: it.ID, ID: p, Op: OpDelete, Hash: t.Hash, Diff: Diff(t.Content, nil)})
+	}
+}
+
+// protectLines: each line of the master matching a protected expression takes
+// the value of the first line of the target matching the same expression
+// (dropped when the target has none).
+func protectLines(master, target []byte, protect []*regexp.Regexp) []byte {
+	if len(protect) == 0 {
+		return master
+	}
+	tl := lines(target)
+	used := map[int]bool{}
+	var out []string
+	for _, l := range lines(master) {
+		var re *regexp.Regexp
+		for _, r := range protect {
+			if r.MatchString(l) {
+				re = r
+				break
+			}
+		}
+		if re == nil {
+			out = append(out, l)
+			continue
+		}
+		for i, x := range tl {
+			if !used[i] && re.MatchString(x) {
+				used[i] = true
+				out = append(out, x)
+				break
+			}
+		}
+	}
+	return joinLines(out)
+}
+
+// additive appends to the target the lines of the master it does not have.
+func additive(target, master []byte) []byte {
+	tl := lines(target)
+	have := map[string]bool{}
+	for _, l := range tl {
+		have[l] = true
+	}
+	out := tl
+	for _, l := range lines(master) {
+		if !have[l] {
+			have[l] = true
+			out = append(out, l)
+		}
+	}
+	return joinLines(out)
+}
+
+func joinLines(l []string) []byte {
+	if len(l) == 0 {
+		return nil
+	}
+	return []byte(strings.Join(l, "\n") + "\n")
+}
+
+// Filter keeps the changes of the given objects ("item/id"), plus the
+// interfaces they need that the plan creates (ports of a bridge, raw device
+// of a VLAN). Issues about other objects do not block the selection.
+func Filter(p Plan, keys []string) Plan {
+	if len(keys) == 0 {
+		return p
+	}
+	byKey := map[string]Change{}
+	for _, c := range p.Changes {
+		byKey[c.Key()] = c
+	}
+	want := map[string]bool{}
+	for _, k := range keys {
+		want[k] = true
+	}
+	for changed := true; changed; {
+		changed = false
+		for k := range want {
+			c, ok := byKey[k]
+			if !ok || c.Item != "network" || c.Op == OpDelete {
+				continue
+			}
+			for f, v := range c.Full {
+				if !netLists[f] {
+					continue
+				}
+				for _, n := range strings.Fields(v.(string)) {
+					dep := "network/" + n
+					if d, ok := byKey[dep]; ok && d.Op == OpCreate && !want[dep] {
+						want[dep] = true
+						changed = true
+					}
+				}
+			}
+		}
+	}
+	out := Plan{Target: p.Target}
+	for _, e := range p.Errors {
+		if e.Key == "" || want[e.Key] {
+			out.Errors = append(out.Errors, e)
+		}
+	}
+	for _, w := range p.Warnings {
+		if w.Key == "" || want[w.Key] {
+			out.Warnings = append(out.Warnings, w)
+		}
+	}
+	for _, c := range p.Changes {
+		if want[c.Key()] {
+			out.Changes = append(out.Changes, c)
+		}
+	}
+	return out
+}
+
+// Digest identifies the content of a plan: the apply only runs when the plan
+// computed again is the one reviewed in the dry-run.
+func (p Plan) Digest() string {
+	type ch struct {
+		Op    Op             `json:"op"`
+		Key   string         `json:"key"`
+		Set   map[string]any `json:"set,omitempty"`
+		Unset []string       `json:"unset,omitempty"`
+		Hash  string         `json:"hash,omitempty"`
+		New   string         `json:"new,omitempty"`
+	}
+	l := make([]ch, 0, len(p.Changes))
+	for _, c := range p.Changes {
+		x := ch{Op: c.Op, Key: c.Key(), Set: c.Set, Unset: c.Unset, Hash: c.Hash}
+		if c.Content != nil || c.Op != OpDelete && c.Hash != "" {
+			x.New = hash(c.Content)
+		}
+		l = append(l, x)
+	}
+	errs := make([]string, 0, len(p.Errors))
+	for _, e := range p.Errors {
+		errs = append(errs, e.Key+" "+e.Message)
+	}
+	sort.Strings(errs)
+	b, _ := json.Marshal(struct {
+		Target  string   `json:"target"`
+		Changes []ch     `json:"changes"`
+		Errors  []string `json:"errors"`
+	}{p.Target, l, errs})
+	sum := sha256.Sum256(b)
+	return hex.EncodeToString(sum[:12])
+}

internal/nodesync/read.go

@@ -0,0 +1,337 @@
+package nodesync
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+	"math/bits"
+	"net/url"
+	"path"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"pve-confsync/internal/model"
+	"pve-confsync/internal/pve"
+)
+
+// Shell runs a command as root on the node, stdin fed with in (sshx.Client).
+type Shell interface {
+	Run(ctx context.Context, cmd string, in []byte) ([]byte, error)
+}
+
+// Endpoint is one node: the API of its cluster (calls proxied to the node)
+// and SSH as root on the node itself.
+type Endpoint struct {
+	Key     string // "cluster/node"
+	Cluster string
+	Node    string
+	API     *pve.Client
+	Shell   Shell // nil: SSH host key not recorded yet
+	// Dial opens a new SSH access, to check that the node still answers after
+	// a network reload (nil: Shell is used).
+	Dial func() (Shell, error)
+}
+
+// File is a file read on a node.
+type File struct {
+	Path    string `json:"path"`
+	Mode    string `json:"mode"`  // octal, e.g. 644
+	Owner   string `json:"owner"` // user:group
+	Hash    string `json:"hash"`  // SHA-256 of the content
+	Content []byte `json:"-"`
+}
+
+// Snapshot is what was read on one node.
+type Snapshot struct {
+	Key     string                     `json:"key"`
+	Version string                     `json:"version,omitempty"`
+	Network []model.Object             `json:"-"` // interfaces (id = iface), canonical fields
+	Options map[string][]string        `json:"-"` // iface -> lines the API does not write (bond-miimon…)
+	DNS     map[string]any             `json:"-"`
+	Files   map[string]map[string]File `json:"-"` // item -> path -> file
+	// Pending: /etc/network/interfaces.new exists (network changes not applied yet).
+	Pending  bool      `json:"pending,omitempty"`
+	Warnings []string  `json:"warnings,omitempty"`
+	ReadAt   time.Time `json:"readAt"`
+}
+
+// maxFile: larger files are not replicated (configuration files are small).
+const maxFile = 1 << 20
+
+// Read loads the given items from a node.
+func Read(ctx context.Context, ep Endpoint, items []string) (Snapshot, error) {
+	snap := Snapshot{Key: ep.Key, Files: map[string]map[string]File{}, ReadAt: time.Now().UTC()}
+	v, err := NodeVersion(ctx, ep)
+	if err != nil {
+		return snap, err
+	}
+	snap.Version = v
+	var files []Item
+	for _, id := range items {
+		it, err := ItemByID(id)
+		if err != nil {
+			return snap, err
+		}
+		switch it.Kind {
+		case KindNetwork:
+			if snap.Network, snap.Options, err = readNetwork(ctx, ep); err != nil {
+				return snap, fmt.Errorf("%s: read network: %w", ep.Key, err)
+			}
+		case KindDNS:
+			if snap.DNS, err = readDNS(ctx, ep); err != nil {
+				return snap, fmt.Errorf("%s: read dns: %w", ep.Key, err)
+			}
+		case KindFile:
+			files = append(files, it)
+		}
+	}
+	// SSH is needed for the files, and for the pending network changes
+	if ep.Shell == nil {
+		return snap, fmt.Errorf("%s: SSH host key not recorded yet, test the node first", ep.Key)
+	}
+	out, err := ep.Shell.Run(ctx, readScript(files), nil)
+	if err != nil {
+		return snap, fmt.Errorf("%s: read files: %w", ep.Key, err)
+	}
+	got, pending, warns, err := parseRead(out)
+	if err != nil {
+		return snap, fmt.Errorf("%s: read files: %w", ep.Key, err)
+	}
+	snap.Pending = pending
+	snap.Warnings = append(snap.Warnings, warns...)
+	for _, it := range files {
+		m := map[string]File{}
+		for p, f := range got {
+			if matchItem(it, p) {
+				m[p] = f
+			}
+		}
+		snap.Files[it.ID] = m
+	}
+	return snap, nil
+}
+
+// NodeVersion reads the pve-manager version of the node and refuses anything
+// but PVE 9.x.
+func NodeVersion(ctx context.Context, ep Endpoint) (string, error) {
+	var v struct {
+		Version string `json:"version"`
+		Release string `json:"release"`
+	}
+	if err := ep.API.Get(ctx, "nodes/"+url.PathEscape(ep.Node)+"/version", nil, &v); err != nil {
+		return "", fmt.Errorf("%s: version check: %w", ep.Key, err)
+	}
+	ref := v.Release
+	if ref == "" {
+		ref = v.Version
+	}
+	if major, err := strconv.Atoi(strings.SplitN(ref, ".", 2)[0]); err != nil || major != pve.SupportedMajor {
+		return v.Version, fmt.Errorf("%s: node runs PVE %q, only %d.x is supported", ep.Key, ref, pve.SupportedMajor)
+	}
+	return v.Version, nil
+}
+
+func matchItem(it Item, p string) bool {
+	for _, x := range it.Exclude {
+		if x == p {
+			return false
+		}
+	}
+	for _, pat := range it.Paths {
+		if ok, _ := path.Match(pat, p); ok {
+			return true
+		}
+	}
+	return false
+}
+
+// readScript lists the files of the items: one header line per file
+// ("F <mode> <user:group> <path>", "L <size> <path>" when too large) then
+// its content in base64 on one line; "PENDING" when interfaces.new exists.
+// The first line is a marker the fake node of the tests understands.
+func readScript(files []Item) string {
+	var pats []string
+	for _, it := range files {
+		pats = append(pats, it.Paths...)
+	}
+	var b strings.Builder
+	b.WriteString("# confsync:read\n")
+	if len(pats) > 0 {
+		fmt.Fprintf(&b, "for f in %s; do\n", strings.Join(pats, " "))
+		b.WriteString(`  [ -f "$f" ] && [ ! -L "$f" ] || continue
+  s=$(stat -c %s "$f")
+  if [ "$s" -gt ` + strconv.Itoa(maxFile) + ` ]; then printf 'L %s %s\n' "$s" "$f"; continue; fi
+  printf 'F %s %s\n' "$(stat -c '%a %U:%G' "$f")" "$f"
+  base64 -w0 < "$f"; echo
+done
+`)
+	}
+	b.WriteString("[ -e /etc/network/interfaces.new ] && echo PENDING\ntrue\n")
+	return b.String()
+}
+
+func parseRead(out []byte) (map[string]File, bool, []string, error) {
+	files := map[string]File{}
+	pending := false
+	var warns []string
+	sc := bufio.NewScanner(bytes.NewReader(out))
+	sc.Buffer(make([]byte, 64<<10), 4*maxFile)
+	for sc.Scan() {
+		line := sc.Text()
+		switch {
+		case line == "PENDING":
+			pending = true
+		case strings.HasPrefix(line, "L "):
+			p := strings.SplitN(line, " ", 3)
+			if len(p) == 3 {
+				warns = append(warns, fmt.Sprintf("%s: %s bytes, too large, not replicated", p[2], p[1]))
+			}
+		case strings.HasPrefix(line, "F "):
+			p := strings.SplitN(line, " ", 4)
+			if len(p) != 4 || !sc.Scan() {
+				return nil, false, nil, fmt.Errorf("unexpected output %q", line)
+			}
+			data, err := base64.StdEncoding.DecodeString(strings.TrimSpace(sc.Text()))
+			if err != nil {
+				return nil, false, nil, fmt.Errorf("%s: %w", p[3], err)
+			}
+			if !safePath(p[3]) {
+				warns = append(warns, fmt.Sprintf("%s: unexpected characters in the path, not replicated", p[3]))
+				continue
+			}
+			files[p[3]] = File{Path: p[3], Mode: p[1], Owner: p[2], Hash: hash(data), Content: data}
+		}
+	}
+	return files, pending, warns, sc.Err()
+}
+
+// safePath: the paths go into shell scripts between single quotes.
+func safePath(p string) bool {
+	if !strings.HasPrefix(p, "/") || strings.Contains(p, "..") {
+		return false
+	}
+	for _, r := range p {
+		if !(r == '/' || r == '.' || r == '-' || r == '_' || r == '+' || r == '@' ||
+			(r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9')) {
+			return false
+		}
+	}
+	return true
+}
+
+func hash(b []byte) string {
+	s := sha256.Sum256(b)
+	return hex.EncodeToString(s[:])
+}
+
+// ---- network (GET /nodes/{node}/network) ----
+
+// netWritable are the parameters of POST/PUT /nodes/{node}/network[/{iface}]
+// (pve-manager PVE/API2/Network.pm, $confdesc), in their cidr form: the GET
+// returns address/netmask and cidr, the canonical form keeps cidr only.
+var netWritable = map[string]bool{
+	"type": true, "comments": true, "comments6": true, "autostart": true,
+	"bridge_vlan_aware": true, "bridge_vids": true, "bridge_ports": true,
+	"ovs_ports": true, "ovs_tag": true, "ovs_options": true, "ovs_bridge": true, "ovs_bonds": true,
+	"slaves": true, "bond_mode": true, "bond-primary": true, "bond_xmit_hash_policy": true,
+	"vlan-raw-device": true, "vlan-id": true,
+	"gateway": true, "cidr": true, "gateway6": true, "cidr6": true, "mtu": true,
+}
+
+// netTypes are the interface types handled (fabric, vnet and unknown are not).
+var netTypes = map[string]bool{
+	"eth": true, "bond": true, "bridge": true, "vlan": true, "alias": true,
+	"OVSBridge": true, "OVSBond": true, "OVSPort": true, "OVSIntPort": true,
+}
+
+// physical interfaces cannot be created: they must exist on the target.
+func physical(typ string) bool { return typ == "eth" }
+
+func readNetwork(ctx context.Context, ep Endpoint) ([]model.Object, map[string][]string, error) {
+	var raw []map[string]any
+	if err := ep.API.Get(ctx, "nodes/"+url.PathEscape(ep.Node)+"/network", nil, &raw); err != nil {
+		return nil, nil, err
+	}
+	var objs []model.Object
+	opts := map[string][]string{}
+	for _, r := range raw {
+		id := pve.Str(r["iface"])
+		typ := pve.Str(r["type"])
+		if id == "" || id == "lo" || !netTypes[typ] {
+			continue
+		}
+		f := map[string]any{}
+		for k, v := range r {
+			if !netWritable[k] || v == nil {
+				continue
+			}
+			s := strings.TrimRight(pve.Str(v), "\n")
+			if s == "" {
+				continue
+			}
+			f[k] = s
+		}
+		// a static address without cidr (older parsers): rebuild it
+		if _, ok := f["cidr"]; !ok {
+			if a, m := pve.Str(r["address"]), pve.Str(r["netmask"]); a != "" && m != "" {
+				f["cidr"] = a + "/" + maskBits(m)
+			}
+		}
+		if l := listOf(r["options"]); len(l) > 0 {
+			opts[id] = l
+		}
+		if l := listOf(r["options6"]); len(l) > 0 {
+			opts[id] = append(opts[id], l...)
+		}
+		objs = append(objs, model.Object{Kind: "network", ID: id, Fields: f})
+	}
+	sort.Slice(objs, func(i, j int) bool { return objs[i].ID < objs[j].ID })
+	return objs, opts, nil
+}
+
+func listOf(v any) []string {
+	l, _ := v.([]any)
+	out := make([]string, 0, len(l))
+	for _, e := range l {
+		out = append(out, pve.Str(e))
+	}
+	sort.Strings(out)
+	return out
+}
+
+// maskBits converts 255.255.255.0 to 24 (a prefix length is kept).
+func maskBits(m string) string {
+	if !strings.Contains(m, ".") {
+		return m
+	}
+	n := 0
+	for _, p := range strings.Split(m, ".") {
+		b, _ := strconv.Atoi(p)
+		n += bits.OnesCount8(uint8(b))
+	}
+	return strconv.Itoa(n)
+}
+
+// ---- dns (GET /nodes/{node}/dns) ----
+
+var dnsFields = []string{"search", "dns1", "dns2", "dns3"}
+
+func readDNS(ctx context.Context, ep Endpoint) (map[string]any, error) {
+	var raw map[string]any
+	if err := ep.API.Get(ctx, "nodes/"+url.PathEscape(ep.Node)+"/dns", nil, &raw); err != nil {
+		return nil, err
+	}
+	out := map[string]any{}
+	for _, k := range dnsFields {
+		if s := pve.Str(raw[k]); s != "" {
+			out[k] = s
+		}
+	}
+	return out, nil
+}

internal/nodesync/script_test.go

@@ -0,0 +1,77 @@
+package nodesync
+
+import (
+	"os"
+	"os/exec"
+	"os/user"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// TestPrepareScriptForReal runs the prepare script and restore.sh with the
+// local shell in a temporary folder (every path prefixed with it).
+func TestPrepareScriptForReal(t *testing.T) {
+	if _, err := exec.LookPath("sha256sum"); err != nil {
+		t.Skip("sha256sum not installed")
+	}
+	root := t.TempDir()
+	u, _ := user.Current()
+	g, _ := user.LookupGroupId(u.Gid)
+	owner := u.Username + ":" + g.Name
+	write := func(p, s string) {
+		if err := os.MkdirAll(filepath.Dir(root+p), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(root+p, []byte(s), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	read := func(p string) string {
+		b, err := os.ReadFile(root + p)
+		if err != nil {
+			return "<absent>"
+		}
+		return string(b)
+	}
+	write("/etc/sysctl.d/90-net.conf", "old\n")
+	write("/etc/sysctl.d/99-local.conf", "local\n")
+	write("/etc/network/interfaces", "auto lo\n")
+	plan := Plan{Changes: []Change{
+		{Item: "sysctl", ID: "/etc/sysctl.d/90-net.conf", Op: OpUpdate, Hash: hash([]byte("old\n")), Mode: "640", Owner: owner, Content: []byte("new\n")},
+		{Item: "multipath", ID: "/etc/multipath.conf", Op: OpCreate, Hash: "absent", Mode: "644", Owner: owner, Content: []byte("defaults {}\n")},
+		{Item: "sysctl", ID: "/etc/sysctl.d/99-local.conf", Op: OpDelete, Hash: hash([]byte("local\n"))},
+		{Item: "network", ID: "vmbr0", Op: OpUpdate},
+	}}
+	run := func(script string, in []byte) (string, error) {
+		cmd := exec.Command("sh", "-c", script)
+		cmd.Stdin = strings.NewReader(string(in))
+		out, err := cmd.CombinedOutput()
+		return string(out), err
+	}
+	const dir = "/var/lib/pve-confsync/backup/test"
+	if out, err := run(prepareScriptIn(dir, root), prepareInput(plan, true, false)); err != nil || strings.TrimSpace(out) != "ok" {
+		t.Fatalf("prepare: %v %s", err, out)
+	}
+	if read("/etc/sysctl.d/90-net.conf") != "new\n" || read("/etc/multipath.conf") != "defaults {}\n" || read("/etc/sysctl.d/99-local.conf") != "<absent>" {
+		t.Fatalf("writes: %q %q %q", read("/etc/sysctl.d/90-net.conf"), read("/etc/multipath.conf"), read("/etc/sysctl.d/99-local.conf"))
+	}
+	if fi, _ := os.Stat(root + "/etc/sysctl.d/90-net.conf"); fi.Mode().Perm() != 0o640 {
+		t.Errorf("mode %v", fi.Mode())
+	}
+	if m, _ := filepath.Glob(root + "/etc/sysctl.d/*.confsync.*"); len(m) > 0 {
+		t.Errorf("temporary files left: %v", m)
+	}
+	// a second run with the same plan: the files changed, nothing is written
+	if out, err := run(prepareScriptIn(dir+"2", root), prepareInput(plan, false, false)); err == nil || !strings.Contains(out, "changed since the dry-run") {
+		t.Errorf("stale check: %v %s", err, out)
+	}
+	write("/etc/network/interfaces.new", "pending\n")
+	if out, err := run("sh '"+root+dir+"/restore.sh'", nil); err != nil {
+		t.Fatalf("restore: %v %s", err, out)
+	}
+	if read("/etc/sysctl.d/90-net.conf") != "old\n" || read("/etc/multipath.conf") != "<absent>" || read("/etc/sysctl.d/99-local.conf") != "local\n" ||
+		read("/etc/network/interfaces") != "auto lo\n" || read("/etc/network/interfaces.new") != "<absent>" {
+		t.Errorf("restore: %q %q %q %q", read("/etc/sysctl.d/90-net.conf"), read("/etc/multipath.conf"), read("/etc/sysctl.d/99-local.conf"), read("/etc/network/interfaces.new"))
+	}
+}