pve-confsync/internal/nodesync/nodesync_test.go

package nodesync

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"strings"
	"testing"
	"time"

	"pve-confsync/internal/config"
	"pve-confsync/internal/pve/pvetest"
	"pve-confsync/internal/sshx"
)

func init() {
	ProbeWait, TaskPoll = time.Millisecond, time.Millisecond
}

// endpoint connects to a node of a fake cluster like the web UI does.
func endpoint(t *testing.T, s *pvetest.Server, cluster string, n *pvetest.Node) Endpoint {
	t.Helper()
	sum := sha256.Sum256(s.Certificate().Raw)
	c, err := config.Cluster{Name: cluster, URL: s.URL, User: "root@pam", Password: pvetest.Password,
		Fingerprint: hex.EncodeToString(sum[:])}.PVE()
	if err != nil {
		t.Fatal(err)
	}
	host, port := n.SSHAddr()
	o := sshx.Options{Host: host, Port: port, Password: pvetest.Password, HostKey: n.SSHHostKey()}
	sh, err := sshx.New(o)
	if err != nil {
		t.Fatal(err)
	}
	return Endpoint{Key: cluster + "/" + n.Name, Cluster: cluster, Node: n.Name, API: c, Shell: sh}
}

const multipath = "defaults {\n    user_friendly_names yes\n    find_multipaths no\n}\nblacklist {\n    devnode \"^sda\"\n}\n"

// lab: one cluster, a master pve1 and a target pve2 with differences.
func lab(t *testing.T) (*pvetest.Server, *pvetest.Node, *pvetest.Node, map[string]Endpoint) {
	s := pvetest.New("9.1.4")
	t.Cleanup(s.Close)
	m, c := s.AddNode("pve1"), s.AddNode("pve2")
	m.Net = map[string]map[string]any{
		"eno1":     {"type": "eth", "mtu": "9000"},
		"eno2":     {"type": "eth", "mtu": "9000"},
		"bond0":    {"type": "bond", "slaves": "eno1 eno2", "bond_mode": "802.3ad", "bond_xmit_hash_policy": "layer3+4", "mtu": "9000"},
		"vmbr0":    {"type": "bridge", "bridge_ports": "bond0", "bridge_vlan_aware": "1", "cidr": "10.0.0.11/24", "gateway": "10.0.0.1", "autostart": "1"},
		"vmbr1":    {"type": "bridge", "bridge_ports": "bond0.50", "cidr": "10.5.0.11/24", "comments": "stockage"},
		"bond0.50": {"type": "vlan", "mtu": "9000"},
	}
	// the target has other NIC names and an active-backup bond
	c.Net = map[string]map[string]any{
		"enp1s0f0": {"type": "eth"},
		"enp1s0f1": {"type": "eth"},
		"bond0":    {"type": "bond", "slaves": "enp1s0f0 enp1s0f1", "bond_mode": "active-backup"},
		"vmbr0":    {"type": "bridge", "bridge_ports": "bond0", "cidr": "10.0.0.12/24", "gateway": "10.0.0.1", "autostart": "1"},
	}
	m.DNS = map[string]string{"search": "lab.local", "dns1": "10.0.0.53", "dns2": "10.0.0.54"}
	c.DNS = map[string]string{"search": "lab.local", "dns1": "8.8.8.8"}
	m.Files["/etc/multipath.conf"] = pvetest.FakeFile{Content: multipath}
	m.Files["/etc/multipath/wwids"] = pvetest.FakeFile{Content: "# wwids\n/3600a0980383030/\n/3600a0980383031/\n"}
	c.Files["/etc/multipath/wwids"] = pvetest.FakeFile{Content: "# wwids\n/3600a0980383031/\n/36000local/\n"}
	m.Files["/etc/sysctl.d/90-net.conf"] = pvetest.FakeFile{Content: "net.core.rmem_max = 16777216\n"}
	c.Files["/etc/sysctl.d/90-net.conf"] = pvetest.FakeFile{Content: "net.core.rmem_max = 8388608\n", Mode: "600"}
	c.Files["/etc/sysctl.d/99-local.conf"] = pvetest.FakeFile{Content: "vm.swappiness = 10\n"}
	m.Files["/etc/network/interfaces.d/sdn"] = pvetest.FakeFile{Content: "auto vnet1\n"}
	eps := map[string]Endpoint{"lab/pve1": endpoint(t, s, "lab", m), "lab/pve2": endpoint(t, s, "lab", c)}
	return s, m, c, eps
}

func profile() *Profile {
	return &Profile{Name: "stockage", Master: "lab/pve1", Targets: []string{"lab/pve2"},
		Items: []string{"network", "dns", "multipath", "wwids", "sysctl", "interfaces-d"},
		Mapping: map[string]TargetMap{"lab/pve2": {
			NICs:   map[string]string{"eno1": "enp1s0f0", "eno2": "enp1s0f1"},
			Values: map[string]map[string]string{"network/vmbr1": {"cidr": "10.5.0.12/24"}},
		}}}
}

func keys(p Plan) []string {
	var out []string
	for _, c := range p.Changes {
		out = append(out, string(c.Op)+" "+c.Key())
	}
	return out
}

func TestPlan(t *testing.T) {
	_, _, _, eps := lab(t)
	prof := profile()
	st := Analyze(context.Background(), prof, eps, nil)
	if st.Master.Error != "" || st.Targets[0].Error != "" {
		t.Fatal(st.Master.Error, st.Targets[0].Error)
	}
	p := st.Targets[0].Plan
	if len(p.Errors) > 0 {
		t.Fatalf("errors: %+v", p.Errors)
	}
	got := strings.Join(keys(p), "\n")
	want := strings.Join([]string{
		"update sysctl//etc/sysctl.d/90-net.conf",
		"create multipath//etc/multipath.conf",
		"update wwids//etc/multipath/wwids",
		"update dns/dns",
		"update network/enp1s0f0",
		"update network/enp1s0f1",
		"update network/bond0",
		"create network/bond0.50",
		"update network/vmbr0",
		"create network/vmbr1",
	}, "\n")
	if got != want {
		t.Fatalf("plan:\n%s\nwant:\n%s", got, want)
	}
	for _, c := range p.Changes {
		switch c.Key() {
		case "network/bond0":
			if c.Set["slaves"] != "enp1s0f0 enp1s0f1" && c.Set["slaves"] != nil {
				t.Errorf("slaves not renamed: %v", c.Set["slaves"])
			}
			if c.Set["bond_mode"] != "802.3ad" {
				t.Errorf("bond0: %+v", c.Set)
			}
		case "network/vmbr0":
			// the address of the target is kept (override without mapping)
			if _, ok := c.Set["cidr"]; ok || c.Full["cidr"] != "10.0.0.12/24" {
				t.Errorf("vmbr0: set %+v full %+v", c.Set, c.Full)
			}
		case "network/vmbr1":
			if c.Full["cidr"] != "10.5.0.12/24" || c.Full["bridge_ports"] != "bond0.50" {
				t.Errorf("vmbr1: %+v", c.Full)
			}
		case "wwids//etc/multipath/wwids":
			// additive: the local WWID of the target stays
			if string(c.Content) != "# wwids\n/3600a0980383031/\n/36000local/\n/3600a0980383030/\n" {
				t.Errorf("wwids: %q", c.Content)
			}
		case "sysctl//etc/sysctl.d/90-net.conf":
			if c.Mode != "600" || len(c.Diff) == 0 {
				t.Errorf("sysctl: mode %s diff %+v", c.Mode, c.Diff)
			}
		}
	}
	// the file only on the target is kept without prune; interfaces.d/sdn is never read
	for _, r := range st.Targets[0].Rows {
		if r.ID == "/etc/sysctl.d/99-local.conf" && r.Status != StatusExtra {
			t.Errorf("99-local.conf: %s", r.Status)
		}
		if strings.Contains(r.ID, "interfaces.d/sdn") {
			t.Errorf("sdn file listed")
		}
	}
	if !p.Network() {
		t.Error("network plan not flagged")
	}
	if strings.Join(p.Reloads(), ",") != "sysctl --system,multipathd reconfigure,ifreload -a" {
		t.Errorf("reloads: %v", p.Reloads())
	}
}

func TestApplyAndVerify(t *testing.T) {
	_, _, c, eps := lab(t)
	prof := profile()
	st := Analyze(context.Background(), prof, eps, nil)
	tp := st.Plans(nil)[0]
	var steps []string
	if err := st.Apply(context.Background(), tp, nil, func(s Step) {
		steps = append(steps, s.Action+" "+s.Error)
	}); err != nil {
		t.Fatalf("apply: %v\n%s", err, strings.Join(steps, "\n"))
	}
	if c.Net["vmbr0"]["cidr"] != "10.0.0.12/24" || c.Net["vmbr1"]["cidr"] != "10.5.0.12/24" {
		t.Errorf("addresses: vmbr0 %v vmbr1 %v", c.Net["vmbr0"], c.Net["vmbr1"])
	}
	if c.Net["vmbr0"]["bridge_vlan_aware"] != "1" || c.Pending != nil {
		t.Errorf("vmbr0 %v pending %v", c.Net["vmbr0"], c.Pending)
	}
	if c.DNS["dns2"] != "10.0.0.54" || c.Files["/etc/multipath.conf"].Content != multipath {
		t.Errorf("dns %v multipath %q", c.DNS, c.Files["/etc/multipath.conf"].Content)
	}
	if len(c.Armed) != 0 {
		t.Errorf("restore timer still armed: %v", c.Armed)
	}
	if got := strings.Join(c.Reloads, ","); got != "sysctl --system,multipathd reconfigure,network" {
		t.Errorf("reloads: %s", got)
	}
	if len(c.Backups()) != 1 {
		t.Errorf("backups: %v", c.Backups())
	}
	st = Analyze(context.Background(), prof, eps, nil)
	if p := st.Targets[0].Plan; !p.Empty() {
		t.Errorf("still to do: %v", keys(p))
	}
}

func TestReloadFailureRestores(t *testing.T) {
	_, _, c, eps := lab(t)
	prof := profile()
	prof.Items = []string{"multipath", "sysctl"}
	c.FailOn = "reload multipathd reconfigure"
	st := Analyze(context.Background(), prof, eps, nil)
	err := st.Apply(context.Background(), st.Plans(nil)[0], nil, func(Step) {})
	if err == nil || !strings.Contains(err.Error(), "multipathd reconfigure") {
		t.Fatalf("err = %v", err)
	}
	if _, ok := c.Files["/etc/multipath.conf"]; ok {
		t.Error("multipath.conf not removed by the restore")
	}
	if c.Files["/etc/sysctl.d/90-net.conf"].Content != "net.core.rmem_max = 8388608\n" {
		t.Errorf("sysctl not restored: %q", c.Files["/etc/sysctl.d/90-net.conf"].Content)
	}
}

func TestUnreachableAfterReloadKeepsTimer(t *testing.T) {
	_, _, c, eps := lab(t)
	prof := profile()
	prof.Items = []string{"network"}
	st := Analyze(context.Background(), prof, eps, nil)
	c.FailOn = "probe"
	err := st.Apply(context.Background(), st.Plans(nil)[0], nil, func(Step) {})
	if err == nil || !strings.Contains(err.Error(), "unreachable") {
		t.Fatalf("err = %v", err)
	}
	if len(c.Armed) != 1 {
		t.Fatalf("timer: %v", c.Armed)
	}
	// systemd runs the restore: the previous interfaces come back
	c.Fire()
	if c.Net["bond0"]["bond_mode"] != "active-backup" || c.Net["vmbr1"] != nil {
		t.Errorf("not restored: %v", c.Net)
	}
}

func TestStaleFileRefused(t *testing.T) {
	_, _, c, eps := lab(t)
	prof := profile()
	prof.Items = []string{"sysctl"}
	st := Analyze(context.Background(), prof, eps, nil)
	c.Files["/etc/sysctl.d/90-net.conf"] = pvetest.FakeFile{Content: "changed by hand\n"}
	err := st.Apply(context.Background(), st.Plans(nil)[0], nil, func(Step) {})
	if err == nil || !strings.Contains(err.Error(), "changed since the dry-run") {
		t.Fatalf("err = %v", err)
	}
	if c.Files["/etc/sysctl.d/90-net.conf"].Content != "changed by hand\n" {
		t.Error("file overwritten")
	}
}

func TestPendingNetworkBlocks(t *testing.T) {
	_, _, c, eps := lab(t)
	c.Pending = map[string]map[string]any{"vmbr0": {"type": "bridge"}}
	st := Analyze(context.Background(), profile(), eps, nil)
	if len(st.Targets[0].Plan.Errors) == 0 {
		t.Fatal("pending changes not blocking")
	}
}

func TestMissingNICIsAnError(t *testing.T) {
	_, _, _, eps := lab(t)
	prof := profile()
	prof.Items = []string{"network"}
	prof.Mapping = nil
	st := Analyze(context.Background(), prof, eps, nil)
	p := st.Targets[0].Plan
	if len(p.Errors) != 2 || p.Errors[0].Key != "network/eno1" {
		t.Fatalf("errors: %+v", p.Errors)
	}
	// the selection of another object is not blocked
	if f := Filter(p, []string{"network/vmbr0"}); len(f.Errors) != 0 || len(f.Changes) != 1 {
		t.Errorf("filter: %+v", f)
	}
}

func TestPruneAndProtect(t *testing.T) {
	_, m, _, eps := lab(t)
	m.Files["/etc/sysctl.d/90-net.conf"] = pvetest.FakeFile{Content: "# host pve1\nnet.core.rmem_max = 8388608\n"}
	prof := profile()
	prof.Items = []string{"sysctl"}
	prof.Prune = true
	prof.Protect = map[string][]string{"sysctl": {"^# host "}}
	st := Analyze(context.Background(), prof, eps, nil)
	got := strings.Join(keys(st.Targets[0].Plan), ",")
	// the protected line is absent on the target: dropped, the rest is equal
	if got != "delete sysctl//etc/sysctl.d/99-local.conf" {
		t.Fatalf("plan: %s", got)
	}
}

func TestDiff(t *testing.T) {
	d := Diff([]byte("a\nb\nc\nd\ne\nf\ng\nh\ni\n"), []byte("a\nb\nc\nd\nE\nf\ng\nh\ni\nj\n"))
	var b strings.Builder
	for _, l := range d {
		b.WriteString(l.Op + l.Text + "\n")
	}
	want := "@@@ -2,8 +2,9 @@\n b\n c\n d\n-e\n+E\n f\n g\n h\n i\n+j\n"
	if b.String() != want {
		t.Fatalf("diff:\n%s\nwant:\n%s", b.String(), want)
	}
	if d := Diff(nil, []byte("x\n")); len(d) != 2 || d[0].Text != "@@ -0,0 +1 @@" {
		t.Errorf("creation diff: %+v", d)
	}
}