diff --git a/g9-migration/REGISTRE-BRIQUES.md b/g9-migration/REGISTRE-BRIQUES.md new file mode 100644 index 0000000..36cd3c7 --- /dev/null +++ b/g9-migration/REGISTRE-BRIQUES.md @@ -0,0 +1,60 @@ +# REGISTRE-BRIQUES — moitié technique (générée) + +> Source : `scripts/10-registry.sh` (`docker ps` + `docker inspect` en **lecture seule** sur .24 et .27, +> compose du kit, Caddyfile pour les endpoints). **Ne pas éditer à la main** : la moitié humaine +> (comptes, 2FA, procédure de réinitialisation, responsable, break-glass) vit dans +> `REGISTRE-IDENTITES.md`. +> +> Périmètre (T-A.10) : **.24 (G9) et .27 (J5040) uniquement**. Le xpenology **.25 est hors périmètre** : +> il est décrit dans `INVENTAIRE-STATEFUL.md` et ses briques sont **à mourir en Phase 6**. +> +> Couverture backup : g9=restic (sources /data /home/folzo /etc, cron 03:30) ; j5040=non (à couvrir Phase 5) + + + +| Machine | Conteneur | Projet / service (compose) | Image | État | Épinglée | Watchtower | Type | Dans le kit | Endpoint (Caddy) | +|---|---|---|---|---|---|---|---|---|---| +| g9 | `collabora` | nextcloud / collabora | `collabora/code:26.04.5.1.1` | running | oui | false | stateful | oui | — | +| g9 | `couchdb` | g9-queue / couchdb | `couchdb:3.3` | running | oui | false | stateful | NON | — | +| g9 | `edge` | edge / edge | `caddy:2` | running | oui | true | stateless | oui | — | +| g9 | `element` | matrix / element | `vectorim/element-web:latest` | running | non (latest) | false | stateful | NON | — | +| g9 | `gitea` | gitea / gitea | `gitea/gitea:1.27.3` | running | oui | false | stateful | oui | git.folzocorp.fr | +| g9 | `gitea-db` | gitea / gitea-db | `postgres:16` | running | oui | false | stateful | oui | — | +| g9 | `jellyfin` | jellyfin / jellyfin | `jellyfin/jellyfin:10.9` | running | oui | false | stateful | oui | media.folzocorp.fr | +| g9 | `n8n` | g9-queue / n8n | `docker.n8n.io/n8nio/n8n:latest` | running | non (latest) | false | stateful | NON | n8n.folzocorp.fr | +| g9 | `nextcloud` | nextcloud / nextcloud | `nextcloud:35.0.1` | running | oui | false | stateful | oui | cloud.folzocorp.fr | +| g9 | `nextcloud_db` | nextcloud / db | `mariadb:10.11` | running | oui | false | stateful | oui | — | +| g9 | `nextcloud_redis` | nextcloud / redis | `redis:7` | running | oui | false | stateful | oui | — | +| g9 | `obsidian` | g9-queue / obsidian | `lscr.io/linuxserver/obsidian:latest` | running | non (latest) | false | stateful | NON | — | +| g9 | `photoprism` | photoprism / photoprism | `photoprism/photoprism:260919` | running | oui | false | stateful | oui | photos.folzocorp.fr | +| g9 | `photoprism_db` | photoprism / db | `mariadb:10.11` | running | oui | false | stateful | oui | — | +| g9 | `synapse` | matrix / synapse | `matrixdotorg/synapse:latest` | running | non (latest) | false | stateful | NON | matrix.folzocorp.fr | +| g9 | `valheim` | compose / valheim | `valheim-pterodactyl:1` | running | oui | false | stateful | NON | — | +| g9 | `vaultwarden` | vaultwarden / vaultwarden | `vaultwarden/server:1.37.4` | running | oui | false | stateful | oui | pass.folzocorp.fr | +| g9 | `watchtower` | g9-queue / watchtower | `45b54f6dc152` | running | oui (digest) | true | stateless | NON | — | +| j5040 | `adguard` | adguard / adguard | `aba9e3bf0613` | running | oui (digest) | true | stateless | oui | — | +| j5040 | `caddy` | edge-j5040 / caddy | `502996b5b60a` | running | oui (digest) | true | stateless | oui | — | +| j5040 | `sweet_feynman` | - / - | `437a490f7afc` | running | oui (digest) | ABSENT | stateless | NON | — | + +## Constats automatiques (détection de dérive) + +- g9/valheim: conteneur réel non déclaré dans le kit (compose) +- g9/n8n: image non épinglée (docker.n8n.io/n8nio/n8n:latest) — S11.4 pinning +- g9/n8n: conteneur réel non déclaré dans le kit (compose) +- g9/element: image non épinglée (vectorim/element-web:latest) — S11.4 pinning +- g9/element: conteneur réel non déclaré dans le kit (compose) +- g9/synapse: image non épinglée (matrixdotorg/synapse:latest) — S11.4 pinning +- g9/synapse: conteneur réel non déclaré dans le kit (compose) +- g9/watchtower: conteneur réel non déclaré dans le kit (compose) +- g9/couchdb: conteneur réel non déclaré dans le kit (compose) +- g9/obsidian: image non épinglée (lscr.io/linuxserver/obsidian:latest) — S11.4 pinning +- g9/obsidian: conteneur réel non déclaré dans le kit (compose) +- j5040/sweet_feynman: label watchtower absent — S11.4 sélectivité +- j5040/sweet_feynman: conteneur réel non déclaré dans le kit (compose) + +## Utiliser ce fichier + +- `./10-registry.sh` : régénère (à lancer après chaque déploiement / MAJ d'image). +- `./10-registry.sh --check --alert` : régénère, compare à la version commitée, **alerte en payload JSON** + si le réel a dérivé du registre (label perdu, image non épinglée, conteneur hors kit, état ≠ running). +- Un `git diff` sur ce fichier est, par construction, la détection de dérive. diff --git a/g9-migration/scripts/10-registry.sh b/g9-migration/scripts/10-registry.sh new file mode 100755 index 0000000..0250e96 --- /dev/null +++ b/g9-migration/scripts/10-registry.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# +# 10-registry.sh — T-A.1 + T-A.5 : moitié technique du registre, GÉNÉRÉE, + détection de dérive. +# +# Lecture seule : `docker ps` / `docker inspect` distants, lecture des compose du kit. +# Aucun conteneur n'est démarré, arrêté, modifié. Aucun secret n'est lu ni affiché. +# +# Modes : +# ./10-registry.sh génère REGISTRE-BRIQUES.md +# ./10-registry.sh --check régénère en mémoire, compare, alerte si dérive (rc=1) +# ./10-registry.sh --check --alert idem + envoie le payload JSON dans le salon ops +# +# Sortie : REGISTRE-BRIQUES.md (moitié A, générée — ne pas éditer à la main) +# La moitié humaine (comptes, reset, responsables) est dans REGISTRE-IDENTITES.md. + +set -euo pipefail + +KIT="$(cd "$(dirname "$0")/.." && pwd)" +OUT="$KIT/REGISTRE-BRIQUES.md" +SSH_KEY="${SSH_KEY:-$HOME/.ssh/id_ed25519_openfox11}" +HOSTS="${HOSTS:-g9:folzo@192.168.1.24 j5040:folzo@192.168.1.27}" +RESTIC_COVERAGE="${RESTIC_COVERAGE:-g9=restic (sources /data /home/folzo /etc, cron 03:30) ; j5040=non (à couvrir Phase 5)}" +MODE="generate" +ALERT=0 +for a in "$@"; do + [[ "$a" == "--check" ]] && MODE="check" + [[ "$a" == "--alert" ]] && ALERT=1 +done +export SSH_KEY + +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +declared_json="$(python3 "$KIT/scripts/registry_inputs.py" declared)" +caddy_json="$(python3 "$KIT/scripts/registry_inputs.py" caddy)" +echo "$caddy_json" > "$TMP/caddy.json" + +rows_all="$TMP/rows.jsonl" +findings_all="$TMP/findings.txt" +: > "$rows_all" +: > "$findings_all" + +for entry in $HOSTS; do + machine="${entry%%:*}" + host="${entry#*:}" + mip="${host#*@}" + ssh -o ConnectTimeout=8 -o BatchMode=yes -i "$SSH_KEY" "$host" \ + 'docker ps -a --format json' > "$TMP/$machine.ps.json" 2>"$TMP/$machine.err" || { + echo "[registry] SSH/docker indisponible sur $machine ($host)" >&2 + echo "$machine: inaccessible (SSH ou docker)" >> "$findings_all" + continue + } + ssh -o ConnectTimeout=8 -o BatchMode=yes -i "$SSH_KEY" "$host" \ + 'docker ps --format "{{.Names}}\t{{.Ports}}"' > "$TMP/$machine.ports.tsv" 2>/dev/null || : > "$TMP/$machine.ports.tsv" + python3 "$KIT/scripts/registry_inputs.py" endpoints "$TMP/$machine.ports.tsv" "$TMP/caddy.json" "$mip" > "$TMP/$machine.ep.json" + python3 "$KIT/scripts/registry_parse.py" "$machine" "$(cat "$TMP/$machine.ep.json")" "$declared_json" \ + < "$TMP/$machine.ps.json" > "$TMP/$machine.rows.json" 2>"$TMP/$machine.findings" + python3 -c 'import json,sys +for r in json.load(open(sys.argv[1])): + print(json.dumps(r, ensure_ascii=False))' "$TMP/$machine.rows.json" >> "$rows_all" + sed -n 's/^#findings# //p' "$TMP/$machine.findings" | python3 -c 'import json,sys +for line in sys.stdin: + line=line.strip() + if line: + for f in json.loads(line): print(f)' >> "$findings_all" +done + +nb_conteneurs="$(wc -l < "$rows_all" | tr -d ' ')" +nb_findings="$(grep -c . "$findings_all" || true)" + +generated="$(python3 - "$rows_all" "$nb_conteneurs" <<'PY' +import json, sys, datetime +rows = [json.loads(l) for l in open(sys.argv[1]) if l.strip()] +ts = datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%d %H:%M UTC') +print(f"") +print() +print("| Machine | Conteneur | Projet / service (compose) | Image | État | Épinglée | Watchtower | Type | Dans le kit | Endpoint (Caddy) |") +print("|---|---|---|---|---|---|---|---|---|---|") +for r in rows: + print(f"| {r['machine']} | `{r['name']}` | {r['project']} / {r['service']} | `{r['image']}` | {r['state']} | {r['pinned']} | {r['wt']} | {r['stateful']} | {r['kit']} | {r['endpoint'] or '—'} |") +PY +)" + +render="$(cat < Source : \`scripts/10-registry.sh\` (\`docker ps\` + \`docker inspect\` en **lecture seule** sur .24 et .27, +> compose du kit, Caddyfile pour les endpoints). **Ne pas éditer à la main** : la moitié humaine +> (comptes, 2FA, procédure de réinitialisation, responsable, break-glass) vit dans +> \`REGISTRE-IDENTITES.md\`. +> +> Périmètre (T-A.10) : **.24 (G9) et .27 (J5040) uniquement**. Le xpenology **.25 est hors périmètre** : +> il est décrit dans \`INVENTAIRE-STATEFUL.md\` et ses briques sont **à mourir en Phase 6**. +> +> Couverture backup : $RESTIC_COVERAGE + +$generated + +## Constats automatiques (détection de dérive) + +$(if [[ -s "$findings_all" ]]; then sed 's/^/- /' "$findings_all"; else echo "- aucun"; fi) + +## Utiliser ce fichier + +- \`./10-registry.sh\` : régénère (à lancer après chaque déploiement / MAJ d'image). +- \`./10-registry.sh --check --alert\` : régénère, compare à la version commitée, **alerte en payload JSON** + si le réel a dérivé du registre (label perdu, image non épinglée, conteneur hors kit, état ≠ running). +- Un \`git diff\` sur ce fichier est, par construction, la détection de dérive. +EOF +)" + +if [[ "$MODE" == "check" ]]; then + if [[ ! -f "$OUT" ]]; then + echo "[registry] REGISTRE-BRIQUES.md absent — générer d'abord" >&2 + exit 2 + fi + base="$(git -C "$KIT" show HEAD:g9-migration/REGISTRE-BRIQUES.md 2>/dev/null || git -C "$KIT" show HEAD:REGISTRE-BRIQUES.md 2>/dev/null || true)" + body_new="$(printf '%s' "$render" | sed '1,4d')" + body_old="$(printf '%s' "$base" | sed '1,4d')" + if [[ -z "$body_old" ]]; then + echo "[registry] pas de version commitée de comparaison" >&2 + exit 2 + fi + if [[ "$(printf '%s' "$body_new" | md5sum)" == "$(printf '%s' "$body_old" | md5sum)" ]]; then + echo "[registry] aucune dérive ($nb_conteneurs conteneurs, $nb_findings constats)" + exit 0 + fi + echo "[registry] DÉRIVE détectée :" + printf '%s\n' "$nb_findings constats" + if [[ "$ALERT" == "1" ]]; then + source "$KIT/scripts/lib-notify.sh" + ts="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + payload="$(python3 -c 'import json,sys; print(json.dumps({"ts":sys.argv[1],"source":"10-registry","brique":"registre","host":"g9","check":"registry-drift","http_code":0,"severite":"warn","message":sys.argv[2],"runbook":"RUNBOOKS/registre.md"},ensure_ascii=False))' "$ts" "$nb_conteneurs conteneurs, $nb_findings constats automatiques, diff registre/réel à lire dans REGISTRE-BRIQUES.md")" + notify_json "$payload" + fi + exit 1 +fi + +printf '%s\n' "$render" > "$OUT" +echo "[registry] $OUT généré : $nb_conteneurs conteneurs, $nb_findings constats automatiques" diff --git a/g9-migration/scripts/lib-notify.sh b/g9-migration/scripts/lib-notify.sh new file mode 100644 index 0000000..7c00b09 --- /dev/null +++ b/g9-migration/scripts/lib-notify.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# lib-notify.sh — notification Matrix en payload JSON structuré (C-AP2). +# Source de vérité des identifiants : /etc/restic/matrix.env sur le G9 (600 root, jamais affichée). +# Usage : source lib-notify.sh puis notify_json '' +# Aucun secret n'est affiché : le token est lu côté G9, jamais exporté ici. + +G9_SSH="${G9_SSH:-folzo@192.168.1.24}" +G9_SSH_KEY="${G9_SSH_KEY:-$HOME/.ssh/id_ed25519_openfox11}" + +notify_json() { + local payload="$1" + local b64 + b64="$(printf '%s' "$payload" | base64 -w0)" + ssh -o ConnectTimeout=8 -o BatchMode=yes -i "$G9_SSH_KEY" "$G9_SSH" \ + "MATTRIX_B64='$b64' sudo -E python3 - <<'PY' +import base64, json, os, urllib.request +env = {} +for line in open('/etc/restic/matrix.env'): + line = line.strip() + if not line or line.startswith('#') or '=' not in line: + continue + k, v = line.split('=', 1) + env[k] = v.strip().strip('\"') +server = env['MATRIX_SERVER']; room = env['MATRIX_ROOM']; token = env['MATRIX_TOKEN'] +payload = json.loads(base64.b64decode(os.environ['MATTRIX_B64']).decode()) +body = {'msgtype': 'm.text', 'body': json.dumps(payload, ensure_ascii=False)} +req = urllib.request.Request( + f'{server}/_matrix/client/v3/rooms/{room}/send/m.room.message/openfox-{int(__import__(\"time\").time()*1000)}', + data=json.dumps(body).encode(), method='PUT', + headers={'Authorization': '***' + token, 'Content-Type': 'application/json'}) +try: + with urllib.request.urlopen(req, timeout=10) as r: + print('NOTIFY_OK', r.status) +except Exception as e: + print('NOTIFY_KO', type(e).__name__) +PY" 2>&1 | tail -1 +} + +notify_simple() { + local source="$1" brique="$2" severity="$3" message="$4" + local ts payload + ts="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + payload="$(python3 -c 'import json,sys; print(json.dumps({"ts":sys.argv[1],"source":sys.argv[2],"brique":sys.argv[3],"host":sys.argv[4],"check":"notify","http_code":0,"severite":sys.argv[5],"message":sys.argv[6],"runbook":f"RUNBOOKS/{sys.argv[3]}.md"},ensure_ascii=False))' "$ts" "$source" "$brique" "${G9_SSH##*@}" "$severity" "$message")" + notify_json "$payload" +} diff --git a/g9-migration/scripts/registry_inputs.py b/g9-migration/scripts/registry_inputs.py new file mode 100755 index 0000000..0ee13f7 --- /dev/null +++ b/g9-migration/scripts/registry_inputs.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""Helpers for 10-registry.sh — lit le kit (lecture seule) et fournit les entrées du registre. + + declared -> JSON [{project, service, image}] depuis compose/*/docker-compose.yml + caddy -> JSON {port: sous-domaine} depuis compose/*/Caddyfile + endpoints -> JSON {conteneur: sous-domaine} +Aucun conteneur n'est touché. +""" +import json +import os +import re +import sys + + +def declared(kit): + out = [] + cdir = os.path.join(kit, "compose") + if not os.path.isdir(cdir): + return out + for project in sorted(os.listdir(cdir)): + f = os.path.join(cdir, project, "docker-compose.yml") + if not os.path.isfile(f): + continue + try: + import yaml + data = yaml.safe_load(open(f)) or {} + except Exception: + data = {} + services = data.get("services") or {} + for service, spec in services.items(): + image = "-" + if isinstance(spec, dict): + image = spec.get("image", "-") or "-" + out.append({"project": project, "service": service, "image": image}) + return out + + +def caddy(kit): + """Liste [{site, ip, port, source}] — clé = (ip, port), donc pas de collision entre machines.""" + entries = [] + cdir = os.path.join(kit, "compose") + if not os.path.isdir(cdir): + return entries + for project in sorted(os.listdir(cdir)): + cf = os.path.join(cdir, project, "Caddyfile") + if not os.path.isfile(cf): + continue + site = None + for line in open(cf): + s = line.strip() + if s.startswith("#"): + continue + m = re.match(r"^((?:[a-z0-9*-]+\.[a-z0-9.-]+\s+)*[a-z0-9*-]+\.[a-z0-9.-]+)\s*\{$", s) + if m: + hosts = m.group(1).split() + site = next((h for h in hosts if "folzocorp.fr" in h), None) + continue + m = re.search(r"reverse_proxy\s+(\S+):(\d+)", s) + if m and site: + ip, port = m.group(1), m.group(2) + if re.match(r"^\d+\.\d+\.\d+\.\d+$", ip): + entries.append({"site": site, "ip": ip, "port": port, "source": project}) + return entries + + +def endpoints(ports_tsv, caddy_json, machine_ip): + entries = json.loads(open(caddy_json).read()) if os.path.isfile(caddy_json) else [] + cmap = {(e["ip"], e["port"]): e["site"] for e in entries} + out = {} + for line in open(ports_tsv): + parts = line.rstrip("\n").split("\t") + if len(parts) < 2: + continue + name, ports = parts[0], parts[1] + found = "" + for m in re.finditer(r":(\d+)->", ports): + site = cmap.get((machine_ip, m.group(1))) + if site: + found = site + break + out[name] = found + return out + + +def main(): + cmd = sys.argv[1] + kit = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + if cmd == "declared": + print(json.dumps(declared(kit))) + elif cmd == "caddy": + print(json.dumps(caddy(kit))) + elif cmd == "endpoints": + print(json.dumps(endpoints(sys.argv[2], sys.argv[3], sys.argv[4]))) + else: + sys.exit("usage: registry_inputs.py declared|caddy|endpoints") + + +if __name__ == "__main__": + main() diff --git a/g9-migration/scripts/registry_parse.py b/g9-migration/scripts/registry_parse.py new file mode 100755 index 0000000..867f8be --- /dev/null +++ b/g9-migration/scripts/registry_parse.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Parse `docker ps -a --format json` (stdin) + kit compose declarations, emit markdown rows. + +Read-only helper for 10-registry.sh. No container is touched. +""" +import json +import re +import sys + +STATEFUL_HINTS = ("mariadb", "postgres", "redis", "influx", "couchdb", "db", "vault", "data") +LABEL_WT = "com.centurylinklabs.watchtower.enable" +LABEL_PROJECT = "com.docker.compose.project" +LABEL_SERVICE = "com.docker.compose.service" + + +def is_digest_id(image): + return bool(re.fullmatch(r"[0-9a-f]{12,64}", image)) + + +def kit_matches(kit_image, real_image): + """Le kit épingle par digest : `name@sha256:` == réel `` si préfixe.""" + if not kit_image or kit_image == "-": + return True + if kit_image == real_image: + return True + if "@sha256:" in kit_image and is_digest_id(real_image): + digest = kit_image.split("@sha256:", 1)[1] + return digest.startswith(real_image) + return False + + +def is_stateful(name, image, labels): + if labels.get(LABEL_WT) == "false": + return True + low = f"{name} {image}".lower() + return any(h in low for h in STATEFUL_HINTS) + + +def main(): + machine = sys.argv[1] + endpoints = json.loads(sys.argv[2]) if len(sys.argv) > 2 else {} + declared = json.loads(sys.argv[3]) if len(sys.argv) > 3 else [] + declared_keys = {(d["project"], d["service"]) for d in declared} + declared_images = {f'{d["project"]}/{d["service"]}': d["image"] for d in declared} + + rows = [] + findings = [] + raw = sys.stdin.read().strip() + if not raw: + print("[]") + print("#findings# []", file=sys.stderr) + return + for line in raw.splitlines(): + if not line.strip(): + continue + c = json.loads(line) + name = c.get("Names", c.get("Name", "?")) + image = c.get("Image", "?") + state = c.get("State", "?") + labels = {} + for kv in str(c.get("Labels", "")).split(","): + if "=" in kv: + k, v = kv.split("=", 1) + labels[k] = v + project = labels.get(LABEL_PROJECT, "-") + service = labels.get(LABEL_SERVICE, "-") + wt = labels.get(LABEL_WT) + wt_label = "true" if wt == "true" else ("false" if wt == "false" else "ABSENT") + pinned = "non (latest)" if image.endswith(":latest") else "oui" + if is_digest_id(image): + pinned = "oui (digest)" + key = f"{project}/{service}" + declared_state = "oui" if (project, service) in declared_keys else "NON" + drift_img = "" + if declared_state == "oui" and not kit_matches(declared_images.get(key), image): + drift_img = f"image kit={declared_images.get(key)} != réel={image}" + endpoint = endpoints.get(f"{name}", "") + rows.append({ + "machine": machine, + "name": name, + "project": project, + "service": service, + "image": image, + "state": state, + "pinned": pinned, + "wt": wt_label, + "stateful": "stateful" if is_stateful(name, image, labels) else "stateless", + "kit": declared_state, + "endpoint": endpoint, + }) + if pinned.startswith("non"): + findings.append(f"{machine}/{name}: image non épinglée ({image}) — S11.4 pinning") + if wt_label == "ABSENT": + findings.append(f"{machine}/{name}: label watchtower absent — S11.4 sélectivité") + if declared_state == "NON": + findings.append(f"{machine}/{name}: conteneur réel non déclaré dans le kit (compose)") + if drift_img: + findings.append(f"{machine}/{name}: {drift_img} — dérive kit/réel") + if state not in ("running",): + findings.append(f"{machine}/{name}: état {state}") + + rows.sort(key=lambda r: (r["machine"], r["name"])) + print(json.dumps(rows, ensure_ascii=False)) + print("#findings# " + json.dumps(findings, ensure_ascii=False), file=sys.stderr) + + +if __name__ == "__main__": + main()