package nodesync import ( "context" "crypto/sha256" "encoding/hex" "strings" "testing" "time" "pve-confsync/internal/config" "pve-confsync/internal/pve/pvetest" "pve-confsync/internal/sshx" ) func init() { ProbeWait, TaskPoll = time.Millisecond, time.Millisecond } // endpoint connects to a node of a fake cluster like the web UI does. func endpoint(t *testing.T, s *pvetest.Server, cluster string, n *pvetest.Node) Endpoint { t.Helper() sum := sha256.Sum256(s.Certificate().Raw) c, err := config.Cluster{Name: cluster, URL: s.URL, User: "root@pam", Password: pvetest.Password, Fingerprint: hex.EncodeToString(sum[:])}.PVE() if err != nil { t.Fatal(err) } host, port := n.SSHAddr() o := sshx.Options{Host: host, Port: port, Password: pvetest.Password, HostKey: n.SSHHostKey()} sh, err := sshx.New(o) if err != nil { t.Fatal(err) } return Endpoint{Key: cluster + "/" + n.Name, Cluster: cluster, Node: n.Name, API: c, Shell: sh} } const multipath = "defaults {\n user_friendly_names yes\n find_multipaths no\n}\nblacklist {\n devnode \"^sda\"\n}\n" // lab: one cluster, a master pve1 and a target pve2 with differences. func lab(t *testing.T) (*pvetest.Server, *pvetest.Node, *pvetest.Node, map[string]Endpoint) { s := pvetest.New("9.1.4") t.Cleanup(s.Close) m, c := s.AddNode("pve1"), s.AddNode("pve2") m.Net = map[string]map[string]any{ "eno1": {"type": "eth", "mtu": "9000"}, "eno2": {"type": "eth", "mtu": "9000"}, "bond0": {"type": "bond", "slaves": "eno1 eno2", "bond_mode": "802.3ad", "bond_xmit_hash_policy": "layer3+4", "mtu": "9000"}, "vmbr0": {"type": "bridge", "bridge_ports": "bond0", "bridge_vlan_aware": "1", "cidr": "10.0.0.11/24", "gateway": "10.0.0.1", "autostart": "1"}, "vmbr1": {"type": "bridge", "bridge_ports": "bond0.50", "cidr": "10.5.0.11/24", "comments": "stockage"}, "bond0.50": {"type": "vlan", "mtu": "9000"}, } // the target has other NIC names and an active-backup bond c.Net = map[string]map[string]any{ "enp1s0f0": {"type": "eth"}, "enp1s0f1": {"type": "eth"}, "bond0": {"type": "bond", "slaves": "enp1s0f0 enp1s0f1", "bond_mode": "active-backup"}, "vmbr0": {"type": "bridge", "bridge_ports": "bond0", "cidr": "10.0.0.12/24", "gateway": "10.0.0.1", "autostart": "1"}, } m.DNS = map[string]string{"search": "lab.local", "dns1": "10.0.0.53", "dns2": "10.0.0.54"} c.DNS = map[string]string{"search": "lab.local", "dns1": "8.8.8.8"} m.Files["/etc/multipath.conf"] = pvetest.FakeFile{Content: multipath} m.Files["/etc/multipath/wwids"] = pvetest.FakeFile{Content: "# wwids\n/3600a0980383030/\n/3600a0980383031/\n"} c.Files["/etc/multipath/wwids"] = pvetest.FakeFile{Content: "# wwids\n/3600a0980383031/\n/36000local/\n"} m.Files["/etc/sysctl.d/90-net.conf"] = pvetest.FakeFile{Content: "net.core.rmem_max = 16777216\n"} c.Files["/etc/sysctl.d/90-net.conf"] = pvetest.FakeFile{Content: "net.core.rmem_max = 8388608\n", Mode: "600"} c.Files["/etc/sysctl.d/99-local.conf"] = pvetest.FakeFile{Content: "vm.swappiness = 10\n"} m.Files["/etc/network/interfaces.d/sdn"] = pvetest.FakeFile{Content: "auto vnet1\n"} eps := map[string]Endpoint{"lab/pve1": endpoint(t, s, "lab", m), "lab/pve2": endpoint(t, s, "lab", c)} return s, m, c, eps } func profile() *Profile { return &Profile{Name: "stockage", Master: "lab/pve1", Targets: []string{"lab/pve2"}, Items: []string{"network", "dns", "multipath", "wwids", "sysctl", "interfaces-d"}, Mapping: map[string]TargetMap{"lab/pve2": { NICs: map[string]string{"eno1": "enp1s0f0", "eno2": "enp1s0f1"}, Values: map[string]map[string]string{"network/vmbr1": {"cidr": "10.5.0.12/24"}}, }}} } func keys(p Plan) []string { var out []string for _, c := range p.Changes { out = append(out, string(c.Op)+" "+c.Key()) } return out } func TestPlan(t *testing.T) { _, _, _, eps := lab(t) prof := profile() st := Analyze(context.Background(), prof, eps, nil) if st.Master.Error != "" || st.Targets[0].Error != "" { t.Fatal(st.Master.Error, st.Targets[0].Error) } p := st.Targets[0].Plan if len(p.Errors) > 0 { t.Fatalf("errors: %+v", p.Errors) } got := strings.Join(keys(p), "\n") want := strings.Join([]string{ "update sysctl//etc/sysctl.d/90-net.conf", "create multipath//etc/multipath.conf", "update wwids//etc/multipath/wwids", "update dns/dns", "update network/enp1s0f0", "update network/enp1s0f1", "update network/bond0", "create network/bond0.50", "update network/vmbr0", "create network/vmbr1", }, "\n") if got != want { t.Fatalf("plan:\n%s\nwant:\n%s", got, want) } for _, c := range p.Changes { switch c.Key() { case "network/bond0": if c.Set["slaves"] != "enp1s0f0 enp1s0f1" && c.Set["slaves"] != nil { t.Errorf("slaves not renamed: %v", c.Set["slaves"]) } if c.Set["bond_mode"] != "802.3ad" { t.Errorf("bond0: %+v", c.Set) } case "network/vmbr0": // the address of the target is kept (override without mapping) if _, ok := c.Set["cidr"]; ok || c.Full["cidr"] != "10.0.0.12/24" { t.Errorf("vmbr0: set %+v full %+v", c.Set, c.Full) } case "network/vmbr1": if c.Full["cidr"] != "10.5.0.12/24" || c.Full["bridge_ports"] != "bond0.50" { t.Errorf("vmbr1: %+v", c.Full) } case "wwids//etc/multipath/wwids": // additive: the local WWID of the target stays if string(c.Content) != "# wwids\n/3600a0980383031/\n/36000local/\n/3600a0980383030/\n" { t.Errorf("wwids: %q", c.Content) } case "sysctl//etc/sysctl.d/90-net.conf": if c.Mode != "600" || len(c.Diff) == 0 { t.Errorf("sysctl: mode %s diff %+v", c.Mode, c.Diff) } } } // the file only on the target is kept without prune; interfaces.d/sdn is never read for _, r := range st.Targets[0].Rows { if r.ID == "/etc/sysctl.d/99-local.conf" && r.Status != StatusExtra { t.Errorf("99-local.conf: %s", r.Status) } if strings.Contains(r.ID, "interfaces.d/sdn") { t.Errorf("sdn file listed") } } if !p.Network() { t.Error("network plan not flagged") } if strings.Join(p.Reloads(), ",") != "sysctl --system,multipathd reconfigure,ifreload -a" { t.Errorf("reloads: %v", p.Reloads()) } } func TestApplyAndVerify(t *testing.T) { _, _, c, eps := lab(t) prof := profile() st := Analyze(context.Background(), prof, eps, nil) tp := st.Plans(nil)[0] var steps []string if err := st.Apply(context.Background(), tp, nil, func(s Step) { steps = append(steps, s.Action+" "+s.Error) }); err != nil { t.Fatalf("apply: %v\n%s", err, strings.Join(steps, "\n")) } if c.Net["vmbr0"]["cidr"] != "10.0.0.12/24" || c.Net["vmbr1"]["cidr"] != "10.5.0.12/24" { t.Errorf("addresses: vmbr0 %v vmbr1 %v", c.Net["vmbr0"], c.Net["vmbr1"]) } if c.Net["vmbr0"]["bridge_vlan_aware"] != "1" || c.Pending != nil { t.Errorf("vmbr0 %v pending %v", c.Net["vmbr0"], c.Pending) } if c.DNS["dns2"] != "10.0.0.54" || c.Files["/etc/multipath.conf"].Content != multipath { t.Errorf("dns %v multipath %q", c.DNS, c.Files["/etc/multipath.conf"].Content) } if len(c.Armed) != 0 { t.Errorf("restore timer still armed: %v", c.Armed) } if got := strings.Join(c.Reloads, ","); got != "sysctl --system,multipathd reconfigure,network" { t.Errorf("reloads: %s", got) } if len(c.Backups()) != 1 { t.Errorf("backups: %v", c.Backups()) } st = Analyze(context.Background(), prof, eps, nil) if p := st.Targets[0].Plan; !p.Empty() { t.Errorf("still to do: %v", keys(p)) } } func TestReloadFailureRestores(t *testing.T) { _, _, c, eps := lab(t) prof := profile() prof.Items = []string{"multipath", "sysctl"} c.FailOn = "reload multipathd reconfigure" st := Analyze(context.Background(), prof, eps, nil) err := st.Apply(context.Background(), st.Plans(nil)[0], nil, func(Step) {}) if err == nil || !strings.Contains(err.Error(), "multipathd reconfigure") { t.Fatalf("err = %v", err) } if _, ok := c.Files["/etc/multipath.conf"]; ok { t.Error("multipath.conf not removed by the restore") } if c.Files["/etc/sysctl.d/90-net.conf"].Content != "net.core.rmem_max = 8388608\n" { t.Errorf("sysctl not restored: %q", c.Files["/etc/sysctl.d/90-net.conf"].Content) } } func TestUnreachableAfterReloadKeepsTimer(t *testing.T) { _, _, c, eps := lab(t) prof := profile() prof.Items = []string{"network"} st := Analyze(context.Background(), prof, eps, nil) c.FailOn = "probe" err := st.Apply(context.Background(), st.Plans(nil)[0], nil, func(Step) {}) if err == nil || !strings.Contains(err.Error(), "unreachable") { t.Fatalf("err = %v", err) } if len(c.Armed) != 1 { t.Fatalf("timer: %v", c.Armed) } // systemd runs the restore: the previous interfaces come back c.Fire() if c.Net["bond0"]["bond_mode"] != "active-backup" || c.Net["vmbr1"] != nil { t.Errorf("not restored: %v", c.Net) } } func TestStaleFileRefused(t *testing.T) { _, _, c, eps := lab(t) prof := profile() prof.Items = []string{"sysctl"} st := Analyze(context.Background(), prof, eps, nil) c.Files["/etc/sysctl.d/90-net.conf"] = pvetest.FakeFile{Content: "changed by hand\n"} err := st.Apply(context.Background(), st.Plans(nil)[0], nil, func(Step) {}) if err == nil || !strings.Contains(err.Error(), "changed since the dry-run") { t.Fatalf("err = %v", err) } if c.Files["/etc/sysctl.d/90-net.conf"].Content != "changed by hand\n" { t.Error("file overwritten") } } func TestPendingNetworkBlocks(t *testing.T) { _, _, c, eps := lab(t) c.Pending = map[string]map[string]any{"vmbr0": {"type": "bridge"}} st := Analyze(context.Background(), profile(), eps, nil) if len(st.Targets[0].Plan.Errors) == 0 { t.Fatal("pending changes not blocking") } } func TestMissingNICIsAnError(t *testing.T) { _, _, _, eps := lab(t) prof := profile() prof.Items = []string{"network"} prof.Mapping = nil st := Analyze(context.Background(), prof, eps, nil) p := st.Targets[0].Plan if len(p.Errors) != 2 || p.Errors[0].Key != "network/eno1" { t.Fatalf("errors: %+v", p.Errors) } // the selection of another object is not blocked if f := Filter(p, []string{"network/vmbr0"}); len(f.Errors) != 0 || len(f.Changes) != 1 { t.Errorf("filter: %+v", f) } } func TestPruneAndProtect(t *testing.T) { _, m, _, eps := lab(t) m.Files["/etc/sysctl.d/90-net.conf"] = pvetest.FakeFile{Content: "# host pve1\nnet.core.rmem_max = 8388608\n"} prof := profile() prof.Items = []string{"sysctl"} prof.Prune = true prof.Protect = map[string][]string{"sysctl": {"^# host "}} st := Analyze(context.Background(), prof, eps, nil) got := strings.Join(keys(st.Targets[0].Plan), ",") // the protected line is absent on the target: dropped, the rest is equal if got != "delete sysctl//etc/sysctl.d/99-local.conf" { t.Fatalf("plan: %s", got) } } func TestDiff(t *testing.T) { d := Diff([]byte("a\nb\nc\nd\ne\nf\ng\nh\ni\n"), []byte("a\nb\nc\nd\nE\nf\ng\nh\ni\nj\n")) var b strings.Builder for _, l := range d { b.WriteString(l.Op + l.Text + "\n") } want := "@@@ -2,8 +2,9 @@\n b\n c\n d\n-e\n+E\n f\n g\n h\n i\n+j\n" if b.String() != want { t.Fatalf("diff:\n%s\nwant:\n%s", b.String(), want) } if d := Diff(nil, []byte("x\n")); len(d) != 2 || d[0].Text != "@@ -0,0 +1 @@" { t.Errorf("creation diff: %+v", d) } }