Réplication de nœuds : réseau et DNS par l'API, multipath, chrony, sysctl par SSH

Deuxième mode de réplication, séparé de la réplication cluster vers cluster
(sélecteur « Clusters (API) | Nœuds (fichiers) » en haut de page) : un profil
réplique la configuration système d'un nœud maître vers des nœuds cibles.

- réseau par /nodes/{node}/network (interfaces.new puis rechargement), adresses
  et passerelles propres à chaque nœud, mapping des noms d'interfaces par cible ;
  DNS par /nodes/{node}/dns ; multipath.conf, wwids (ajout seul), chrony,
  sysctl.d, interfaces.d (hors sdn) par SSH
- découverte des nœuds (/cluster/status), accès SSH par nœud (mot de passe du
  cluster ou propre), clé d'hôte épinglée, contrôle du nom du nœud
- dry-run avec diff des fichiers ; apply (admin) avec contrôle SHA-256,
  sauvegarde et restore.sh sur le nœud, écriture atomique, rechargements,
  restauration si un rechargement échoue, timer systemd de restauration si le
  nœud est injoignable après le rechargement du réseau, confirmation tapée
- vue Nœuds et profils, journal séparé par mode, démo avec nœuds simulés

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

internal/pve/pvetest/fake.go

@@ -64,6 +64,7 @@ type Server struct {
 	Shadow   map[string]string         // user without realm -> hash (priv/shadow.cfg)
 	FailOn   string                    // "METHOD path" answering 500, to test rollbacks
 	Calls    []string                  // "METHOD path" of every write, "SSH write" for shadow.cfg
+	Nodes    []*Node                   // nodes of the node replication (AddNode); none: one node "n1"
 	locks    int
 	ssh      *sshServer
 }
@@ -83,6 +84,9 @@ func New(version string) *Server {
 // Close stops the API and the SSH server.
 func (s *Server) Close() {
 	s.ssh.close()
+	for _, n := range s.Nodes {
+		n.ssh.close()
+	}
 	s.Server.Close()
 }
 
@@ -146,14 +150,26 @@ func (s *Server) handle(w http.ResponseWriter, r *http.Request) {
 	if running {
 		cfg = &s.Running
 	}
+	var nd *Node
+	if len(seg) >= 3 && seg[0] == "nodes" {
+		nd = s.node(seg[1])
+	}
 	switch {
+	case nd != nil && r.Method == "GET" && len(seg) == 3 && seg[2] == "version":
+		reply(w, map[string]any{"version": nd.version(), "release": release(nd.version()), "repoid": "abcdef"})
+	case nd != nil && (seg[2] == "network" || seg[2] == "dns"):
+		s.nodeAPI(w, r, nd, seg)
 	case key == "GET version" || (r.Method == "GET" && len(seg) == 3 && seg[0] == "nodes" && seg[2] == "version"):
 		reply(w, map[string]any{"version": s.Version, "release": release(s.Version), "repoid": "abcdef"})
 	case key == "GET cluster/status":
-		reply(w, []any{
-			map[string]any{"type": "cluster", "name": "fake", "quorate": 1},
-			map[string]any{"type": "node", "name": "n1", "online": 1},
-		})
+		st := []any{map[string]any{"type": "cluster", "name": "fake", "quorate": 1}}
+		if len(s.Nodes) == 0 {
+			st = append(st, map[string]any{"type": "node", "name": "n1", "online": 1})
+		}
+		for i, n := range s.Nodes {
+			st = append(st, map[string]any{"type": "node", "name": n.Name, "online": 1, "nodeid": i + 1, "ip": "127.0.0.1"})
+		}
+		reply(w, st)
 	case r.Method == "GET" && len(seg) == 4 && seg[0] == "nodes" && seg[2] == "sdn" && seg[3] == "zones":
 		var out []any
 		for id := range s.Running.Zones {

internal/pve/pvetest/nodes.go

@@ -0,0 +1,419 @@
+package pvetest
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"path"
+	"sort"
+	"strings"
+
+	"golang.org/x/crypto/ssh"
+)
+
+// FakeFile is a file of a fake node.
+type FakeFile struct {
+	Content string
+	Mode    string // 644 by default
+	Owner   string // root:root by default
+}
+
+// Node is a node of a fake cluster for the node replication: its own SSH
+// server (files, reloads, restore timer), and its network interfaces and DNS
+// servers served by the API of the cluster (/nodes/{node}/network, /dns).
+// Like PVE, the interface writes go to a pending configuration
+// (interfaces.new) applied by PUT /nodes/{node}/network.
+type Node struct {
+	Name     string
+	Hostname string // answered by hostname (default: Name)
+	Version  string // pve-manager version (default: the cluster's)
+	Files    map[string]FakeFile
+	Net      map[string]map[string]any // iface -> fields as written (cidr, type…)
+	Pending  map[string]map[string]any // interfaces.new; nil: no pending change
+	DNS      map[string]string
+	Reloads  []string        // reload commands run, "network" for PUT /network
+	Armed    map[string]bool // restore timers armed (systemd-run units)
+	FailOn   string          // "reload <cmd>", "prepare" or "probe": that step fails
+	Calls    []string        // SSH steps run
+	backups  map[string]backup
+	srv      *Server
+	ssh      *sshServer
+}
+
+type backup struct {
+	files   map[string]*FakeFile // nil: absent before
+	net     map[string]map[string]any
+	pending map[string]map[string]any
+	dns     map[string]string
+	hasNet  bool
+	hasDNS  bool
+}
+
+// AddNode adds a node to the fake cluster (listed by GET /cluster/status).
+func (s *Server) AddNode(name string) *Node {
+	n := &Node{Name: name, Hostname: name, Files: map[string]FakeFile{}, Net: map[string]map[string]any{},
+		DNS: map[string]string{}, Armed: map[string]bool{}, backups: map[string]backup{}, srv: s}
+	n.ssh = newSSH(func(user, pass string) bool {
+		s.mu.Lock()
+		defer s.mu.Unlock()
+		return user == "root" && pass == s.Password
+	}, n.exec)
+	s.mu.Lock()
+	s.Nodes = append(s.Nodes, n)
+	s.mu.Unlock()
+	return n
+}
+
+// SSHAddr returns the host and port of the SSH server of the node.
+func (n *Node) SSHAddr() (string, int) {
+	a := n.ssh.ln.Addr().(*net.TCPAddr)
+	return a.IP.String(), a.Port
+}
+
+// SSHHostKey returns the host key of the node (authorized_keys format).
+func (n *Node) SSHHostKey() string {
+	return strings.TrimSpace(string(ssh.MarshalAuthorizedKey(n.ssh.key)))
+}
+
+func (s *Server) node(name string) *Node {
+	for _, n := range s.Nodes {
+		if n.Name == name {
+			return n
+		}
+	}
+	return nil
+}
+
+func (n *Node) version() string {
+	if n.Version != "" {
+		return n.Version
+	}
+	return n.srv.Version
+}
+
+func cloneNet(m map[string]map[string]any) map[string]map[string]any {
+	if m == nil {
+		return nil
+	}
+	out := map[string]map[string]any{}
+	for k, v := range m {
+		f := map[string]any{}
+		for x, y := range v {
+			f[x] = y
+		}
+		out[k] = f
+	}
+	return out
+}
+
+// ---- API (called with s.mu held) ----
+
+func (s *Server) nodeAPI(w http.ResponseWriter, r *http.Request, n *Node, seg []string) {
+	switch {
+	case len(seg) == 3 && seg[2] == "dns" && r.Method == "GET":
+		out := map[string]any{}
+		for k, v := range n.DNS {
+			out[k] = v
+		}
+		reply(w, out)
+	case len(seg) == 3 && seg[2] == "dns" && r.Method == "PUT":
+		if r.PostForm.Get("search") == "" {
+			fail(w, 400, "search: property is missing and it is not optional")
+			return
+		}
+		n.DNS = map[string]string{}
+		for _, k := range []string{"search", "dns1", "dns2", "dns3"} {
+			if v := r.PostForm.Get(k); v != "" {
+				n.DNS[k] = v
+			}
+		}
+		reply(w, nil)
+	case len(seg) == 3 && seg[2] == "network" && r.Method == "GET":
+		cfg := n.Net
+		if n.Pending != nil {
+			cfg = n.Pending
+		}
+		var out []any
+		prio := 0
+		for _, id := range sortedKeys(cfg) {
+			prio++
+			f := map[string]any{"iface": id, "priority": prio, "families": []any{"inet"}, "method": "manual", "active": 1}
+			for k, v := range cfg[id] {
+				f[k] = v
+			}
+			if c, ok := f["cidr"].(string); ok {
+				a, m, _ := strings.Cut(c, "/")
+				f["address"], f["netmask"], f["method"] = a, m, "static"
+			}
+			if f["type"] == "eth" {
+				f["exists"] = 1
+			}
+			out = append(out, f)
+		}
+		reply(w, out)
+	case len(seg) == 3 && seg[2] == "network" && r.Method == "DELETE":
+		n.Pending = nil
+		reply(w, nil)
+	case len(seg) == 3 && seg[2] == "network" && r.Method == "PUT":
+		if n.Pending != nil {
+			n.Net, n.Pending = n.Pending, nil
+		}
+		n.Reloads = append(n.Reloads, "network")
+		reply(w, "UPID:"+n.Name+":00001234:00005678:65000000:srvreload:networking:root@pam:")
+	case len(seg) == 3 && seg[2] == "network" && r.Method == "POST":
+		id := r.PostForm.Get("iface")
+		cfg := n.pending()
+		if cfg[id] != nil {
+			fail(w, 400, "iface: interface already exists")
+			return
+		}
+		cfg[id] = fields(r.PostForm, "iface")
+		reply(w, nil)
+	case len(seg) == 4 && seg[2] == "network":
+		cfg := n.pending()
+		cur := cfg[seg[3]]
+		if cur == nil {
+			fail(w, 400, "iface: interface does not exist")
+			return
+		}
+		switch r.Method {
+		case "DELETE":
+			delete(cfg, seg[3])
+		case "PUT":
+			if r.PostForm.Get("type") == "" {
+				fail(w, 400, "type: property is missing and it is not optional")
+				return
+			}
+			for _, d := range strings.Split(r.PostForm.Get("delete"), ",") {
+				delete(cur, d)
+			}
+			// like PVE: without an address in the parameters, the interface
+			// becomes "manual" and loses its address
+			if r.PostForm.Get("cidr") == "" {
+				delete(cur, "cidr")
+			}
+			for k, v := range fields(r.PostForm, "delete") {
+				cur[k] = v
+			}
+		default:
+			fail(w, 501, "not implemented in fake")
+			return
+		}
+		reply(w, nil)
+	default:
+		fail(w, 501, "not implemented in fake: "+r.Method+" "+strings.Join(seg, "/"))
+	}
+}
+
+// pending returns interfaces.new, created from the current configuration.
+func (n *Node) pending() map[string]map[string]any {
+	if n.Pending == nil {
+		n.Pending = cloneNet(n.Net)
+	}
+	return n.Pending
+}
+
+// ---- SSH ----
+
+func (n *Node) exec(cmd string, in io.Reader, out, errw io.Writer) int {
+	s := n.srv
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	first, _, _ := strings.Cut(cmd, "\n")
+	verb, arg, _ := strings.Cut(strings.TrimPrefix(first, "# confsync:"), " ")
+	if strings.HasPrefix(first, "# confsync:") {
+		n.Calls = append(n.Calls, verb)
+		if n.FailOn == verb || n.FailOn == verb+" "+arg {
+			fmt.Fprintln(errw, "injected failure")
+			return 1
+		}
+	}
+	switch {
+	case strings.HasPrefix(cmd, "pveversion"):
+		fmt.Fprintf(out, "pve-manager/%s/abcdef (running kernel: 6.14.8-2-pve)\n", n.version())
+		if strings.Contains(cmd, "hostname") {
+			fmt.Fprintln(out, n.Hostname)
+		}
+		return 0
+	case verb == "read":
+		return n.read(cmd, out)
+	case verb == "prepare":
+		data, _ := io.ReadAll(in)
+		return n.prepare(arg, data, errw)
+	case verb == "restore":
+		b, ok := n.backups[arg]
+		if !ok {
+			fmt.Fprintln(errw, "no backup "+arg)
+			return 1
+		}
+		n.restore(b)
+		return 0
+	case verb == "arm":
+		n.Armed[arg] = true
+		return 0
+	case verb == "disarm":
+		delete(n.Armed, arg)
+		return 0
+	case verb == "reload", verb == "probe":
+		if verb == "reload" {
+			n.Reloads = append(n.Reloads, arg)
+		}
+		return 0
+	}
+	fmt.Fprintf(errw, "fake node: unknown command %q\n", first)
+	return 127
+}
+
+// Fire runs the armed restore timers (as systemd would after the delay).
+func (n *Node) Fire() {
+	n.srv.mu.Lock()
+	defer n.srv.mu.Unlock()
+	for unit := range n.Armed {
+		dir := strings.Replace(unit, "pve-confsync-restore-", "/var/lib/pve-confsync/backup/", 1)
+		if b, ok := n.backups[dir]; ok {
+			n.restore(b)
+		}
+		n.Reloads = append(n.Reloads, "ifreload -a")
+		delete(n.Armed, unit)
+	}
+}
+
+func (n *Node) read(cmd string, out io.Writer) int {
+	var pats []string
+	for _, l := range strings.Split(cmd, "\n") {
+		if strings.HasPrefix(l, "for f in ") {
+			pats = strings.Fields(strings.TrimSuffix(strings.TrimPrefix(l, "for f in "), "; do"))
+		}
+	}
+	for _, p := range sortedKeys(n.Files) {
+		for _, pat := range pats {
+			if ok, _ := path.Match(pat, p); ok {
+				f := n.Files[p]
+				fmt.Fprintf(out, "F %s %s %s\n%s\n", def(f.Mode, "644"), def(f.Owner, "root:root"), p,
+					base64.StdEncoding.EncodeToString([]byte(f.Content)))
+				break
+			}
+		}
+	}
+	if n.Pending != nil {
+		fmt.Fprintln(out, "PENDING")
+	}
+	return 0
+}
+
+func def(v, d string) string {
+	if v == "" {
+		return d
+	}
+	return v
+}
+
+// prepare emulates the prepare script: checks, backups, writes.
+func (n *Node) prepare(dir string, plan []byte, errw io.Writer) int {
+	type op struct{ op, mode, owner, hash, path, data string }
+	var ops []op
+	sc := bufio.NewScanner(bytes.NewReader(plan))
+	sc.Buffer(make([]byte, 64<<10), 8<<20)
+	for sc.Scan() {
+		p := strings.SplitN(sc.Text(), " ", 5)
+		if len(p) != 5 {
+			continue
+		}
+		o := op{p[0], p[1], p[2], p[3], p[4], ""}
+		if o.op == "W" && sc.Scan() {
+			o.data = sc.Text()
+		}
+		ops = append(ops, o)
+	}
+	for _, o := range ops {
+		if o.op != "W" && o.op != "D" {
+			continue
+		}
+		h := "absent"
+		if f, ok := n.Files[o.path]; ok {
+			h = sha([]byte(f.Content))
+		}
+		if h != o.hash {
+			fmt.Fprintf(errw, "changed since the dry-run: %s\n", o.path)
+			return 3
+		}
+	}
+	b := backup{files: map[string]*FakeFile{}}
+	for _, o := range ops {
+		switch o.path {
+		case "/etc/network/interfaces", "/etc/network/interfaces.new":
+			b.hasNet, b.net, b.pending = true, cloneNet(n.Net), cloneNet(n.Pending)
+			continue
+		case "/etc/resolv.conf":
+			b.hasDNS, b.dns = true, map[string]string{}
+			for k, v := range n.DNS {
+				b.dns[k] = v
+			}
+			continue
+		}
+		if f, ok := n.Files[o.path]; ok {
+			c := f
+			b.files[o.path] = &c
+		} else {
+			b.files[o.path] = nil
+		}
+	}
+	n.backups[dir] = b
+	for _, o := range ops {
+		switch o.op {
+		case "W":
+			data, err := base64.StdEncoding.DecodeString(o.data)
+			if err != nil {
+				n.restore(b)
+				fmt.Fprintln(errw, "write failed: "+o.path)
+				return 4
+			}
+			n.Files[o.path] = FakeFile{Content: string(data), Mode: o.mode, Owner: o.owner}
+		case "D":
+			delete(n.Files, o.path)
+		}
+	}
+	return 0
+}
+
+func (n *Node) restore(b backup) {
+	for p, f := range b.files {
+		if f == nil {
+			delete(n.Files, p)
+		} else {
+			n.Files[p] = *f
+		}
+	}
+	if b.hasNet {
+		n.Net, n.Pending = cloneNet(b.net), cloneNet(b.pending)
+	}
+	if b.hasDNS {
+		n.DNS = map[string]string{}
+		for k, v := range b.dns {
+			n.DNS[k] = v
+		}
+	}
+}
+
+// Backups lists the backup folders written on the node.
+func (n *Node) Backups() []string {
+	n.srv.mu.Lock()
+	defer n.srv.mu.Unlock()
+	var out []string
+	for d := range n.backups {
+		out = append(out, d)
+	}
+	sort.Strings(out)
+	return out
+}
+
+func sha(b []byte) string {
+	h := sha256.Sum256(b)
+	return hex.EncodeToString(h[:])
+}

internal/pve/pvetest/ssh.go

@@ -17,18 +17,27 @@ import (
 // sshServer answers the commands pve-confsync runs on a node: pveversion, the
 // read of priv/shadow.cfg and its locked write (adapter password).
 type sshServer struct {
-	ln  net.Listener
-	key ssh.PublicKey
+	ln   net.Listener
+	key  ssh.PublicKey
+	exec func(cmd string, in io.Reader, out, errw io.Writer) int
 }
 
 func startSSH(s *Server) *sshServer {
+	return newSSH(func(user, pass string) bool {
+		s.mu.Lock()
+		defer s.mu.Unlock()
+		return user == "root" && pass == s.Password
+	}, s.exec)
+}
+
+// newSSH starts an SSH server accepting the password check ok and running
+// the commands with exec.
+func newSSH(ok func(user, pass string) bool, exec func(cmd string, in io.Reader, out, errw io.Writer) int) *sshServer {
 	_, priv, _ := ed25519.GenerateKey(rand.Reader)
 	signer, _ := ssh.NewSignerFromKey(priv)
 	cfg := &ssh.ServerConfig{
 		PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
-			s.mu.Lock()
-			defer s.mu.Unlock()
-			if c.User() == "root" && string(pass) == s.Password {
+			if ok(c.User(), string(pass)) {
 				return nil, nil
 			}
 			return nil, fmt.Errorf("denied")
@@ -39,14 +48,14 @@ func startSSH(s *Server) *sshServer {
 	if err != nil {
 		panic(err)
 	}
-	srv := &sshServer{ln: ln, key: signer.PublicKey()}
+	srv := &sshServer{ln: ln, key: signer.PublicKey(), exec: exec}
 	go func() {
 		for {
 			nc, err := ln.Accept()
 			if err != nil {
 				return
 			}
-			go s.serveSSH(nc, cfg)
+			go srv.serve(nc, cfg)
 		}
 	}()
 	return srv
@@ -65,7 +74,7 @@ func (s *Server) SSHHostKey() string {
 	return strings.TrimSpace(string(ssh.MarshalAuthorizedKey(s.ssh.key)))
 }
 
-func (s *Server) serveSSH(nc net.Conn, cfg *ssh.ServerConfig) {
+func (x *sshServer) serve(nc net.Conn, cfg *ssh.ServerConfig) {
 	_, chans, reqs, err := ssh.NewServerConn(nc, cfg)
 	if err != nil {
 		nc.Close()
@@ -91,7 +100,7 @@ func (s *Server) serveSSH(nc net.Conn, cfg *ssh.ServerConfig) {
 				var p struct{ Cmd string }
 				_ = ssh.Unmarshal(req.Payload, &p)
 				_ = req.Reply(true, nil)
-				status := s.exec(p.Cmd, ch, ch, ch.Stderr())
+				status := x.exec(p.Cmd, ch, ch, ch.Stderr())
 				b := make([]byte, 4)
 				binary.BigEndian.PutUint32(b, uint32(status))
 				_, _ = ch.SendRequest("exit-status", false, b)