Réplication de nœuds : réseau et DNS par l'API, multipath, chrony, sysctl par SSH
Deuxième mode de réplication, séparé de la réplication cluster vers cluster
(sélecteur « Clusters (API) | Nœuds (fichiers) » en haut de page) : un profil
réplique la configuration système d'un nœud maître vers des nœuds cibles.
- réseau par /nodes/{node}/network (interfaces.new puis rechargement), adresses
et passerelles propres à chaque nœud, mapping des noms d'interfaces par cible ;
DNS par /nodes/{node}/dns ; multipath.conf, wwids (ajout seul), chrony,
sysctl.d, interfaces.d (hors sdn) par SSH
- découverte des nœuds (/cluster/status), accès SSH par nœud (mot de passe du
cluster ou propre), clé d'hôte épinglée, contrôle du nom du nœud
- dry-run avec diff des fichiers ; apply (admin) avec contrôle SHA-256,
sauvegarde et restore.sh sur le nœud, écriture atomique, rechargements,
restauration si un rechargement échoue, timer systemd de restauration si le
nœud est injoignable après le rechargement du réseau, confirmation tapée
- vue Nœuds et profils, journal séparé par mode, démo avec nœuds simulés
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
cedricandClaude Opus 5.5 committed
1 parent
dc1a7ff40e
commit
1e667ce8b9
24 files changed
+4973
-65
No files matched your search
@@ -0,0 +1,433 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pve-confsync/internal/config"
|
||||
"pve-confsync/internal/nodesync"
|
||||
"pve-confsync/internal/sshx"
|
||||
)
|
||||
|
||||
// nodeSchema: the nodes of the clusters (node replication) and the profiles.
|
||||
// A node is reached over SSH as root at its address in the cluster (GET
|
||||
// /cluster/status) or at an address given by the administrator, with the
|
||||
// root password of its cluster or its own.
|
||||
const nodeSchema = `
|
||||
CREATE TABLE IF NOT EXISTS nodes (
|
||||
cluster TEXT NOT NULL,
|
||||
node TEXT NOT NULL,
|
||||
ip TEXT NOT NULL DEFAULT '',
|
||||
ssh_host TEXT NOT NULL DEFAULT '',
|
||||
ssh_port INTEGER NOT NULL DEFAULT 22,
|
||||
password BLOB,
|
||||
ssh_hostkey TEXT NOT NULL DEFAULT '',
|
||||
last_check TEXT NOT NULL DEFAULT '',
|
||||
PRIMARY KEY (cluster, node)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS node_profiles (
|
||||
name TEXT PRIMARY KEY,
|
||||
position INTEGER NOT NULL DEFAULT 0,
|
||||
body TEXT NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
`
|
||||
|
||||
// Node is a node as stored. The password is never returned to the UI.
|
||||
type Node struct {
|
||||
Key string `json:"key"` // "cluster/node"
|
||||
Cluster string `json:"cluster"`
|
||||
Node string `json:"node"`
|
||||
IP string `json:"ip"` // address in the cluster
|
||||
SSHHost string `json:"sshHost"` // override (empty: IP)
|
||||
SSHPort int `json:"sshPort"`
|
||||
Password string `json:"password,omitempty"` // write only (empty: the cluster's)
|
||||
HasPassword bool `json:"hasPassword"`
|
||||
SSHFingerprint string `json:"sshFingerprint"`
|
||||
ClusterRole string `json:"clusterRole"`
|
||||
LastCheck *Check `json:"lastCheck,omitempty"`
|
||||
}
|
||||
|
||||
// NodeKey is the identity of a node in the profiles.
|
||||
func NodeKey(cluster, node string) string { return cluster + "/" + node }
|
||||
|
||||
const nodeCols = `n.cluster, n.node, n.ip, n.ssh_host, n.ssh_port, n.password, n.ssh_hostkey, n.last_check, c.role`
|
||||
|
||||
func scanNode(scan func(...any) error) (Node, []byte, string, error) {
|
||||
var n Node
|
||||
var blob []byte
|
||||
var key, check string
|
||||
err := scan(&n.Cluster, &n.Node, &n.IP, &n.SSHHost, &n.SSHPort, &blob, &key, &check, &n.ClusterRole)
|
||||
if err != nil {
|
||||
return n, nil, "", err
|
||||
}
|
||||
n.Key = NodeKey(n.Cluster, n.Node)
|
||||
n.HasPassword = len(blob) > 0
|
||||
n.SSHFingerprint = sshx.Fingerprint(key)
|
||||
if check != "" {
|
||||
n.LastCheck = &Check{}
|
||||
_ = json.Unmarshal([]byte(check), n.LastCheck)
|
||||
}
|
||||
return n, blob, key, nil
|
||||
}
|
||||
|
||||
// Nodes lists the known nodes, by cluster then name.
|
||||
func (s *Store) Nodes(ctx context.Context) ([]Node, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT `+nodeCols+` FROM nodes n JOIN clusters c ON c.name = n.cluster
|
||||
ORDER BY c.role = 'primary' DESC, c.position, n.cluster, n.node`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []Node{}
|
||||
for rows.Next() {
|
||||
n, _, _, err := scanNode(rows.Scan)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, n)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// Discovered is a node read in GET /cluster/status.
|
||||
type Discovered struct {
|
||||
Name string
|
||||
IP string
|
||||
}
|
||||
|
||||
// SetNodes records the nodes of a cluster: new ones are added, the address of
|
||||
// the known ones is updated (their pinned host key is forgotten when the SSH
|
||||
// address changes), the nodes gone from the cluster are removed.
|
||||
func (s *Store) SetNodes(ctx context.Context, cluster string, nodes []Discovered) error {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
keep := map[string]bool{}
|
||||
for _, d := range nodes {
|
||||
keep[d.Name] = true
|
||||
var ip, host string
|
||||
err := tx.QueryRowContext(ctx, `SELECT ip, ssh_host FROM nodes WHERE cluster = ? AND node = ?`, cluster, d.Name).Scan(&ip, &host)
|
||||
switch {
|
||||
case errors.Is(err, sql.ErrNoRows):
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO nodes (cluster, node, ip) VALUES (?,?,?)`, cluster, d.Name, d.IP)
|
||||
case err == nil && ip != d.IP:
|
||||
q := `UPDATE nodes SET ip = ? WHERE cluster = ? AND node = ?`
|
||||
if host == "" {
|
||||
q = `UPDATE nodes SET ip = ?, ssh_hostkey = '' WHERE cluster = ? AND node = ?`
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, q, d.IP, cluster, d.Name)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
rows, err := tx.QueryContext(ctx, `SELECT node FROM nodes WHERE cluster = ?`, cluster)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var gone []string
|
||||
for rows.Next() {
|
||||
var n string
|
||||
if err := rows.Scan(&n); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
if !keep[n] {
|
||||
gone = append(gone, n)
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
for _, n := range gone {
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM nodes WHERE cluster = ? AND node = ?`, cluster, n); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// SaveNode sets the SSH access of a node: address (empty: its address in the
|
||||
// cluster), port, password (empty: unchanged; clear: back to the password
|
||||
// of the cluster). A new address forgets the pinned host key.
|
||||
func (s *Store) SaveNode(ctx context.Context, cluster, node, host string, port int, password string, clear bool) error {
|
||||
if port == 0 {
|
||||
port = 22
|
||||
}
|
||||
if port < 1 || port > 65535 {
|
||||
return errors.New("ssh port: 1-65535 expected")
|
||||
}
|
||||
var oldHost string
|
||||
var oldPort int
|
||||
err := s.db.QueryRowContext(ctx, `SELECT ssh_host, ssh_port FROM nodes WHERE cluster = ? AND node = ?`, cluster, node).Scan(&oldHost, &oldPort)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
q := `UPDATE nodes SET ssh_host = ?, ssh_port = ?`
|
||||
args := []any{strings.TrimSpace(host), port}
|
||||
if oldHost != strings.TrimSpace(host) || oldPort != port {
|
||||
q += `, ssh_hostkey = ''`
|
||||
}
|
||||
switch {
|
||||
case clear:
|
||||
q += `, password = NULL`
|
||||
case password != "":
|
||||
blob, err := s.box.Seal([]byte(password))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
q += `, password = ?`
|
||||
args = append(args, blob)
|
||||
}
|
||||
_, err = s.db.ExecContext(ctx, q+` WHERE cluster = ? AND node = ?`, append(args, cluster, node)...)
|
||||
return err
|
||||
}
|
||||
|
||||
// PinNode records the SSH host key seen on the first connection (empty: left
|
||||
// unchanged) and the result of the test.
|
||||
func (s *Store) PinNode(ctx context.Context, cluster, node, hostKey string, check *Check) error {
|
||||
var chk []byte
|
||||
if check != nil {
|
||||
chk, _ = json.Marshal(check)
|
||||
}
|
||||
_, err := s.db.ExecContext(ctx, `UPDATE nodes SET
|
||||
ssh_hostkey = CASE WHEN ? <> '' THEN ? ELSE ssh_hostkey END,
|
||||
last_check = CASE WHEN ? <> '' THEN ? ELSE last_check END
|
||||
WHERE cluster = ? AND node = ?`, hostKey, hostKey, string(chk), string(chk), cluster, node)
|
||||
return err
|
||||
}
|
||||
|
||||
// ForgetNode clears the pinned SSH host key of a node.
|
||||
func (s *Store) ForgetNode(ctx context.Context, cluster, node string) error {
|
||||
res, err := s.db.ExecContext(ctx, `UPDATE nodes SET ssh_hostkey = '' WHERE cluster = ? AND node = ?`, cluster, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n, _ := res.RowsAffected(); n == 0 {
|
||||
return ErrNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NodeAccess is the decrypted access to one node: the connection of its
|
||||
// cluster (API) and SSH as root on the node.
|
||||
type NodeAccess struct {
|
||||
Node
|
||||
Cluster config.Cluster
|
||||
SSH sshx.Options
|
||||
}
|
||||
|
||||
// NodeAccesses returns the access to every node of the clusters not disabled.
|
||||
func (s *Store) NodeAccesses(ctx context.Context) (map[string]NodeAccess, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT `+nodeCols+` FROM nodes n JOIN clusters c ON c.name = n.cluster WHERE c.role <> 'disabled'`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
type row struct {
|
||||
n Node
|
||||
blob []byte
|
||||
key string
|
||||
}
|
||||
var list []row
|
||||
for rows.Next() {
|
||||
n, blob, key, err := scanNode(rows.Scan)
|
||||
if err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
list = append(list, row{n, blob, key})
|
||||
}
|
||||
rows.Close()
|
||||
out := map[string]NodeAccess{}
|
||||
clusters := map[string]config.Cluster{}
|
||||
for _, r := range list {
|
||||
cc, ok := clusters[r.n.Cluster]
|
||||
if !ok {
|
||||
var err error
|
||||
if _, cc, err = s.Cluster(ctx, r.n.Cluster); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clusters[r.n.Cluster] = cc
|
||||
}
|
||||
a, err := s.access(r.n, r.blob, r.key, cc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[r.n.Key] = a
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// NodeAccessOf returns the access to one node.
|
||||
func (s *Store) NodeAccessOf(ctx context.Context, cluster, node string) (NodeAccess, error) {
|
||||
n, blob, key, err := scanNode(s.db.QueryRowContext(ctx, `SELECT `+nodeCols+` FROM nodes n JOIN clusters c ON c.name = n.cluster
|
||||
WHERE n.cluster = ? AND n.node = ?`, cluster, node).Scan)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return NodeAccess{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return NodeAccess{}, err
|
||||
}
|
||||
_, cc, err := s.Cluster(ctx, cluster)
|
||||
if err != nil {
|
||||
return NodeAccess{}, err
|
||||
}
|
||||
return s.access(n, blob, key, cc)
|
||||
}
|
||||
|
||||
func (s *Store) access(n Node, blob []byte, key string, cc config.Cluster) (NodeAccess, error) {
|
||||
pass := cc.Password
|
||||
if len(blob) > 0 {
|
||||
p, err := s.box.Open(blob)
|
||||
if err != nil {
|
||||
return NodeAccess{}, fmt.Errorf("node %s: password: %w", n.Key, err)
|
||||
}
|
||||
pass = string(p)
|
||||
}
|
||||
host := n.SSHHost
|
||||
if host == "" {
|
||||
host = n.IP
|
||||
}
|
||||
cc.SSHHostKey = "" // the cluster SSH access is not used here
|
||||
return NodeAccess{Node: n, Cluster: cc,
|
||||
SSH: sshx.Options{Host: host, Port: n.SSHPort, User: "root", Password: pass, HostKey: key}}, nil
|
||||
}
|
||||
|
||||
// ---- profiles ----
|
||||
|
||||
// Profiles returns the node replication profiles, in order.
|
||||
func (s *Store) Profiles(ctx context.Context) ([]nodesync.Profile, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT name, position, body FROM node_profiles ORDER BY position, name`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []nodesync.Profile{}
|
||||
for rows.Next() {
|
||||
var name, body string
|
||||
var pos int
|
||||
if err := rows.Scan(&name, &pos, &body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var p nodesync.Profile
|
||||
if err := json.Unmarshal([]byte(body), &p); err != nil {
|
||||
return nil, fmt.Errorf("profile %s: %w", name, err)
|
||||
}
|
||||
p.Name, p.Position = name, pos
|
||||
out = append(out, p)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// Profile returns one profile.
|
||||
func (s *Store) Profile(ctx context.Context, name string) (nodesync.Profile, error) {
|
||||
ps, err := s.Profiles(ctx)
|
||||
if err != nil {
|
||||
return nodesync.Profile{}, err
|
||||
}
|
||||
for _, p := range ps {
|
||||
if p.Name == name {
|
||||
return p, nil
|
||||
}
|
||||
}
|
||||
return nodesync.Profile{}, ErrNotFound
|
||||
}
|
||||
|
||||
// SaveProfile creates or updates a profile; oldName renames it.
|
||||
func (s *Store) SaveProfile(ctx context.Context, oldName string, p nodesync.Profile) error {
|
||||
if err := p.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if oldName == "" {
|
||||
oldName = p.Name
|
||||
}
|
||||
body, _ := json.Marshal(p)
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
res, err := tx.ExecContext(ctx, `UPDATE node_profiles SET name = ?, position = ?, body = ?, updated_at = ? WHERE name = ?`,
|
||||
p.Name, p.Position, string(body), time.Now().Unix(), oldName)
|
||||
if err == nil {
|
||||
if n, _ := res.RowsAffected(); n == 0 {
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO node_profiles (name, position, body, updated_at) VALUES (?,?,?,?)`,
|
||||
p.Name, p.Position, string(body), time.Now().Unix())
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "UNIQUE") {
|
||||
return fmt.Errorf("profile %q already exists", p.Name)
|
||||
}
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// DeleteProfile removes a profile.
|
||||
func (s *Store) DeleteProfile(ctx context.Context, name string) error {
|
||||
res, err := s.db.ExecContext(ctx, `DELETE FROM node_profiles WHERE name = ?`, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n, _ := res.RowsAffected(); n == 0 {
|
||||
return ErrNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// renameClusterNodes follows a renamed cluster in the nodes and the profiles.
|
||||
func renameClusterNodes(ctx context.Context, tx *sql.Tx, old, name string) error {
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE nodes SET cluster = ? WHERE cluster = ?`, name, old); err != nil {
|
||||
return err
|
||||
}
|
||||
rows, err := tx.QueryContext(ctx, `SELECT name, body FROM node_profiles`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bodies := map[string]string{}
|
||||
for rows.Next() {
|
||||
var n, b string
|
||||
if err := rows.Scan(&n, &b); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
bodies[n] = b
|
||||
}
|
||||
rows.Close()
|
||||
ren := func(k string) string {
|
||||
if strings.HasPrefix(k, old+"/") {
|
||||
return name + k[len(old):]
|
||||
}
|
||||
return k
|
||||
}
|
||||
for n, b := range bodies {
|
||||
var p nodesync.Profile
|
||||
if json.Unmarshal([]byte(b), &p) != nil {
|
||||
continue
|
||||
}
|
||||
p.Master = ren(p.Master)
|
||||
for i, t := range p.Targets {
|
||||
p.Targets[i] = ren(t)
|
||||
}
|
||||
m := map[string]nodesync.TargetMap{}
|
||||
for k, v := range p.Mapping {
|
||||
m[ren(k)] = v
|
||||
}
|
||||
p.Mapping = m
|
||||
nb, _ := json.Marshal(p)
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE node_profiles SET body = ? WHERE name = ?`, string(nb), n); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user