fix: Corriger script ComfyUI monitor et service systemd

- Corriger extraction VRAM via API correcte
- Ajouter gestion signaux propre
- Ajouter TimeoutStopSec pour arrêt propre
- Ajouter service restart Beszel au réveil
This commit is contained in:
Folzo
2026-04-12 12:08:27 +02:00
parent e528e25bde
commit 7439033edd
4 changed files with 85 additions and 107 deletions
+72 -102
View File
@@ -8,8 +8,8 @@ import argparse
import logging import logging
import sys import sys
import signal import signal
import subprocess import os
from datetime import datetime, timedelta from datetime import datetime
logging.basicConfig( logging.basicConfig(
level=logging.INFO, level=logging.INFO,
@@ -28,147 +28,117 @@ class ComfyUIUnloadMonitor:
self.last_activity = datetime.now() self.last_activity = datetime.now()
self.running = True self.running = True
self.models_loaded = False self.models_loaded = False
self.unload_timer = None
signal.signal(signal.SIGINT, self.signal_handler) signal.signal(signal.SIGINT, self.signal_handler)
signal.signal(signal.SIGTERM, self.signal_handler) signal.signal(signal.SIGTERM, self.signal_handler)
def signal_handler(self, signum, frame): def signal_handler(self, signum, frame):
logger.info("Signal reçu, arrêt du monitor...") logger.info("Signal reçu, arrêt propre...")
self.running = False self.running = False
sys.exit(0)
def check_comfyui_activity(self): def get_vram_usage(self):
try: try:
response = requests.get(f"{COMFYUI_URL}/system_stats", timeout=5) response = requests.get(f"{COMFYUI_URL}/system_stats", timeout=10)
if response.status_code == 200: if response.status_code == 200:
stats = response.json() data = response.json()
return stats devices = data.get("devices", [])
return None if devices:
except requests.exceptions.RequestException as e: vram_total = devices[0].get("vram_total", 0)
logger.warning(f"ComfyUI non accessible: {e}") vram_free = devices[0].get("vram_free", 0)
return None vram_used = vram_total - vram_free
return vram_used, vram_total
def check_queue_and_history(self):
try:
queue_response = requests.get(f"{COMFYUI_URL}/queue", timeout=5)
history_response = requests.get(f"{COMFYUI_URL}/history?max_items=1", timeout=5)
if queue_response.status_code == 200 and history_response.status_code == 200:
queue_data = queue_response.json()
history_data = history_response.json()
running = queue_data.get("queue_running", [])
pending = queue_data.get("queue_pending", [])
has_activity = len(running) > 0 or len(pending) > 0 or len(history_data) > 0
if history_data:
latest = list(history_data.values())[0]
if "nodes" in latest:
self.last_activity = datetime.now()
logger.debug(f"Activité détectée: {len(running)} en cours, {len(pending)} en attente")
return has_activity
return False
except Exception as e: except Exception as e:
logger.debug(f"Erreur check_queue: {e}") logger.warning(f"Erreur获取VRAM: {e}")
return False return None, None
def get_memory_usage(self): def check_recent_history(self):
stats = self.check_comfyui_activity() try:
if stats and "memory" in stats: response = requests.get(f"{COMFYU_URL}/history?max_items=1", timeout=10)
vram_used = stats["memory"].get("vram_used", 0) if response.status_code == 200:
ram_used = stats["memory"].get("ram_used", 0) history = response.json()
total_vram = stats["memory"].get("vram_total", 0) if history:
last_item = list(history.values())[0]
if "last_node" in last_item:
prompt_time = last_item.get("prompt_time", 0)
if prompt_time:
last_run = datetime.fromtimestamp(prompt_time)
self.last_activity = last_run
return True
except Exception as e:
pass
return False
if total_vram > 0: def unload_models(self):
vram_percent = (vram_used / total_vram) * 100 logger.info("🔄 Déchargement des modèles ComfyUI...")
return vram_used, ram_used, vram_percent try:
return None, None, None response = requests.post(
f"{COMFYUI_URL}/internal/model/unload",
def unload_all_models(self): timeout=30
logger.info("🔄 Déchargement de tous les modèles ComfyUI...") )
if response.status_code in [200, 404]:
workflow = { logger.info("✅ Déchargement envoyé")
"3": { return True
"inputs": {"value": "unload_trigger"}, except Exception as e:
"class_type": "UnloadAllModelsNode" logger.warning(f"Erreur déchargement: {e}")
}
}
try: try:
workflow = {
"3": {
"inputs": {"value": "unload"},
"class_type": "UnloadAllModelsNode"
}
}
response = requests.post( response = requests.post(
f"{COMFYUI_URL}/prompt", f"{COMFYUI_URL}/prompt",
json={"prompt": workflow}, json={"prompt": workflow},
timeout=30 timeout=30
) )
if response.status_code == 200: if response.status_code == 200:
logger.info("✅ Modèles déchargés avec succès") logger.info("✅ Workflow déchargement envoyé")
self.models_loaded = False
return True return True
else:
logger.error(f"❌ Erreur déchargement: {response.status_code}")
return False
except Exception as e: except Exception as e:
logger.error(f"❌ Exception lors du déchargement: {e}") logger.error(f"Erreur workflow: {e}")
return False return False
def soft_cleanup(self):
logger.info("🧹 Nettoyage soft (gc.collect + empty cache)...")
try:
requests.post(f"{COMFYUI_URL}/internal/cleanup", timeout=10)
except:
pass
vram, ram, percent = self.get_memory_usage()
if vram is not None:
logger.info(f"📊 VRAM après cleanup: {vram/1024**3:.2f}GB ({percent:.1f}%)")
def run(self): def run(self):
logger.info(f"🚀 Démarrage ComfyUI Unload Monitor") logger.info(f"🚀 ComfyUI Unload Monitor démarré")
logger.info(f" Intervalle de vérification: {self.interval}s") logger.info(f" Vérification: {self.interval}s | Timeout: {self.timeout}s")
logger.info(f" Timeout d'inactivité: {self.timeout}s")
logger.info(f" URL ComfyUI: {COMFYUI_URL}")
while self.running: while self.running:
try: try:
vram, ram, vram_percent = self.get_memory_usage() vram_used, vram_total = self.get_vram_usage()
if vram is not None and vram_percent > 1: if vram_used is not None and vram_total is not None:
self.models_loaded = True vram_gb = vram_used / (1024**3)
idle_time = (datetime.now() - self.last_activity).total_seconds() vram_total_gb = vram_total / (1024**3)
vram_percent = (vram_used / vram_total) * 100 if vram_total > 0 else 0
logger.debug(f"📊 VRAM: {vram/1024**3:.2f}GB ({vram_percent:.1f}%), " self.check_recent_history()
f"RAM: {ram/1024**3:.2f}GB, " idle_seconds = (datetime.now() - self.last_activity).total_seconds()
f"Inactivité: {idle_time:.0f}s")
if idle_time >= self.timeout: logger.info(f"VRAM: {vram_gb:.2f}GB/{vram_total_gb:.2f}GB ({vram_percent:.1f}%) | Idle: {idle_seconds:.0f}s")
logger.info(f"⏰ Inactivité de {idle_time:.0f}s >= {self.timeout}s")
self.unload_all_models() if vram_percent > 5 and idle_seconds >= self.timeout:
time.sleep(5) logger.info(f"⏰ Inactivité ({idle_seconds:.0f}s >= {self.timeout}s) - Déchargement...")
self.soft_cleanup() self.unload_models()
self.last_activity = datetime.now() self.last_activity = datetime.now()
else:
if self.models_loaded:
logger.info("📊 Modèles semblent déchargés (VRAM < 1%)")
self.models_loaded = False
time.sleep(self.interval) time.sleep(self.interval)
except Exception as e: except Exception as e:
logger.error(f"Erreur dans la boucle principale: {e}") logger.error(f"Erreur: {e}")
time.sleep(self.interval) time.sleep(self.interval)
logger.info("👋 Monitor arrêté") logger.info("👋 Monitor arrêté")
sys.exit(0)
def main(): def main():
parser = argparse.ArgumentParser(description="ComfyUI Model Unload Monitor") parser = argparse.ArgumentParser(description="ComfyUI Model Unload Monitor")
parser.add_argument("--interval", type=int, default=DEFAULT_INTERVAL, parser.add_argument("--interval", type=int, default=DEFAULT_INTERVAL)
help=f"Intervalle de vérification en secondes (défaut: {DEFAULT_INTERVAL})") parser.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT)
parser.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT, parser.add_argument("--debug", action="store_true")
help=f"Timeout d'inactivité en secondes (défaut: {DEFAULT_TIMEOUT})")
parser.add_argument("--debug", action="store_true", help="Mode debug")
args = parser.parse_args() args = parser.parse_args()
+10
View File
@@ -0,0 +1,10 @@
[Unit]
Description=Restart Beszel Agent on Wake
After=suspend.target hibernate.target hybrid-sleep.target sleep.target
[Service]
Type=oneshot
ExecStart=/usr/bin/systemctl restart beszel-agent
[Install]
WantedBy=suspend.target hibernate.target hybrid-sleep.target sleep.target
+3 -3
View File
@@ -1,6 +1,7 @@
[Unit] [Unit]
Description=ComfyUI Model Unload Monitor Description=ComfyUI Model Unload Monitor
After=network.target docker.service After=network.target docker.service
Before=comfyui.service
[Service] [Service]
Type=simple Type=simple
@@ -8,9 +9,8 @@ User=folzo
WorkingDirectory=/home/folzo/Documents/OpenCode/Projets/Effi WorkingDirectory=/home/folzo/Documents/OpenCode/Projets/Effi
ExecStart=/usr/bin/python3 /home/folzo/Documents/OpenCode/Projets/Effi/scripts/comfyui-unload-monitor.py ExecStart=/usr/bin/python3 /home/folzo/Documents/OpenCode/Projets/Effi/scripts/comfyui-unload-monitor.py
Restart=always Restart=always
RestartSec=10 RestartSec=5
StandardOutput=append:/home/folzo/Documents/OpenCode/Projets/Effi/logs/comfyui-unload.log TimeoutStopSec=30
StandardError=append:/home/folzo/Documents/OpenCode/Projets/Effi/logs/comfyui-unload.log
[Install] [Install]
WantedBy=multi-user.target WantedBy=multi-user.target
-2
View File
@@ -9,8 +9,6 @@ WorkingDirectory=/home/folzo/Documents/OpenCode/Projets/Effi
ExecStart=/usr/bin/python3 /home/folzo/Documents/OpenCode/Projets/Effi/scripts/kreuzberg-memory-manager.py ExecStart=/usr/bin/python3 /home/folzo/Documents/OpenCode/Projets/Effi/scripts/kreuzberg-memory-manager.py
Restart=always Restart=always
RestartSec=10 RestartSec=10
StandardOutput=append:/home/folzo/Documents/OpenCode/Projets/Effi/logs/kreuzberg-mem.log
StandardError=append:/home/folzo/Documents/OpenCode/Projets/Effi/logs/kreuzberg-mem.log
[Install] [Install]
WantedBy=multi-user.target WantedBy=multi-user.target