From fcad41521e78d4e33e78f0a7d6fafb6ae447739b Mon Sep 17 00:00:00 2001 From: Folzo Date: Mon, 20 Jul 2026 10:10:18 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20jean-router=20=E2=80=94=20routing=20dyn?= =?UTF-8?q?amique=20Jean=20+=20Ollama?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - jean-router interroge /api/presets pour lister dynamiquement les presets Jean - Support Ollama via /api/tags pour les modèles vision - Switch automatique de preset quand nécessaire - Expose /v1/models avec tous les modèles disponibles - Route les requêtes vers Jean ou Ollama selon le modèle demandé - Nouveaux presets Jean automatiquement visibles dans OpenWebUI Architecture: open-webui → LiteLLM:4000 → jean-router:3014 ├→ jean-proxy:3012 → Jean (.11:8080) └→ wol-proxy:3011 → Ollama (.11:11434) --- docker-compose.litellm.yml | 23 +++ jean-proxy/Dockerfile | 4 +- jean-router/Dockerfile | 12 ++ jean-router/app.py | 320 +++++++++++++++++++++++++++++++++++++ litellm/config.yaml | 112 ++----------- 5 files changed, 370 insertions(+), 101 deletions(-) create mode 100644 jean-router/Dockerfile create mode 100644 jean-router/app.py diff --git a/docker-compose.litellm.yml b/docker-compose.litellm.yml index 57bb203..233f54f 100644 --- a/docker-compose.litellm.yml +++ b/docker-compose.litellm.yml @@ -48,6 +48,29 @@ services: - BROADCAST_IP=${BROADCAST_IP} - HEALTH_PATH=/health + # ───────────────────────────────────────────────────────── + # jean-router — Proxy dynamique qui interroge Jean pour + # lister les presets et route les requêtes vers Jean + # ou Ollama selon le modèle demandé. + # Ecoute :3014 sur le host (network_mode: host) + # ───────────────────────────────────────────────────────── + jean-router: + build: ./jean-router + container_name: jean-router + restart: unless-stopped + network_mode: host + environment: + - JEAN_WEB_HOST=${OLLAMA_HOST} + - JEAN_WEB_PORT=8090 + - JEAN_PROXY_HOST=localhost + - JEAN_PROXY_PORT=3012 + - OLLAMA_PROXY_HOST=localhost + - OLLAMA_PROXY_PORT=3011 + - ENABLE_OLLAMA=true + - LISTEN_PORT=3014 + - SWITCH_TIMEOUT=90 + - CHECK_INTERVAL=2 + # ───────────────────────────────────────────────────────── # 2. LiteLLM proxy — Endpoint OpenAI unique pour open-webui # Ecoute :4000 sur le host (network_mode: host pour joindre diff --git a/jean-proxy/Dockerfile b/jean-proxy/Dockerfile index efc5cee..8177697 100644 --- a/jean-proxy/Dockerfile +++ b/jean-proxy/Dockerfile @@ -7,8 +7,8 @@ RUN pip install --no-cache-dir flask requests COPY app.py . -ENV LISTEN_PORT=3010 -EXPOSE 3010 +ENV LISTEN_PORT=3012 +EXPOSE 3012 CMD ["python", "app.py"] diff --git a/jean-router/Dockerfile b/jean-router/Dockerfile new file mode 100644 index 0000000..f151b84 --- /dev/null +++ b/jean-router/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.12-slim + +WORKDIR /app + +RUN pip install --no-cache-dir flask requests + +COPY app.py . + +ENV LISTEN_PORT=3014 +EXPOSE 3014 + +CMD ["python", "app.py"] diff --git a/jean-router/app.py b/jean-router/app.py new file mode 100644 index 0000000..8c5c201 --- /dev/null +++ b/jean-router/app.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 +""" +Jean Router — Proxy dynamique pour Jean presets +Interroge Jean pour lister les presets disponibles et route les requêtes +vers le bon preset en faisant le switch automatique si nécessaire. +""" + +import os +import time +import logging +import threading +from typing import Optional + +import requests +from flask import Flask, request, Response, jsonify + +JEAN_WEB_HOST = os.getenv("JEAN_WEB_HOST", "192.168.1.11") +JEAN_WEB_PORT = int(os.getenv("JEAN_WEB_PORT", "8090")) +JEAN_PROXY_HOST = os.getenv("JEAN_PROXY_HOST", "localhost") +JEAN_PROXY_PORT = int(os.getenv("JEAN_PROXY_PORT", "3012")) +OLLAMA_PROXY_HOST = os.getenv("OLLAMA_PROXY_HOST", "localhost") +OLLAMA_PROXY_PORT = int(os.getenv("OLLAMA_PROXY_PORT", "3011")) +ENABLE_OLLAMA = os.getenv("ENABLE_OLLAMA", "true").lower() == "true" +LISTEN_PORT = int(os.getenv("LISTEN_PORT", "3014")) +SWITCH_TIMEOUT = int(os.getenv("SWITCH_TIMEOUT", "90")) +CHECK_INTERVAL = int(os.getenv("CHECK_INTERVAL", "2")) + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") +log = logging.getLogger(__name__) + +app = Flask(__name__) + +presets_cache = {"data": [], "timestamp": 0} +ollama_cache = {"data": [], "timestamp": 0} +CACHE_TTL = 30 +switch_lock = threading.Lock() + + +def get_presets(force_refresh=False): + """Récupère la liste des presets depuis Jean""" + now = time.time() + if not force_refresh and presets_cache["data"] and (now - presets_cache["timestamp"]) < CACHE_TTL: + return presets_cache["data"] + + try: + url = f"http://{JEAN_WEB_HOST}:{JEAN_WEB_PORT}/api/presets" + r = requests.get(url, timeout=3) + if r.ok: + presets_cache["data"] = r.json() + presets_cache["timestamp"] = now + return presets_cache["data"] + + +def get_ollama_models(force_refresh=False): + """Récupère la liste des modèles depuis Ollama""" + if not ENABLE_OLLAMA: + return [] + + now = time.time() + if not force_refresh and ollama_cache["data"] and (now - ollama_cache["timestamp"]) < CACHE_TTL: + return ollama_cache["data"] + + try: + url = f"http://{OLLAMA_PROXY_HOST}:{OLLAMA_PROXY_PORT}/api/tags" + r = requests.get(url, timeout=3) + if r.ok: + data = r.json().get("models", []) + ollama_cache["data"] = data + ollama_cache["timestamp"] = now + return data + except Exception as e: + log.warning(f"[Ollama] Erreur récupération: {e}") + + return ollama_cache["data"] + except Exception as e: + log.warning(f"[Presets] Erreur récupération: {e}") + + return presets_cache["data"] + + +def get_active_preset(): + """Retourne le preset actuellement actif""" + presets = get_presets() + for p in presets: + if p.get("active"): + return p + return None + + +def switch_preset(preset_id: str) -> bool: + """Switch vers un preset spécifique par son ID""" + presets = get_presets(force_refresh=True) + + for i, p in enumerate(presets): + if p["id"] == preset_id: + if p.get("active"): + log.info(f"[Switch] Preset {preset_id} déjà actif") + return True + + log.info(f"[Switch] Activation preset {preset_id} (index {i+1})") + try: + url = f"http://{JEAN_WEB_HOST}:{JEAN_WEB_PORT}/api/switch" + r = requests.post(url, json={"n": i + 1}, timeout=5) + if not r.ok: + log.error(f"[Switch] Erreur API: {r.text}") + return False + except Exception as e: + log.error(f"[Switch] Erreur requête: {e}") + return False + + log.info(f"[Switch] Attente chargement modèle...") + deadline = time.time() + SWITCH_TIMEOUT + while time.time() < deadline: + try: + health_url = f"http://{JEAN_PROXY_HOST}:{JEAN_PROXY_PORT}/health" + hr = requests.get(health_url, timeout=2) + if hr.ok: + models_url = f"http://{JEAN_PROXY_HOST}:{JEAN_PROXY_PORT}/v1/models" + mr = requests.get(models_url, timeout=2) + if mr.ok: + models = mr.json().get("data", []) + if models: + log.info(f"[Switch] Modèle chargé après {int(time.time() - deadline + SWITCH_TIMEOUT)}s") + presets_cache["timestamp"] = 0 + return True + except Exception: + pass + time.sleep(CHECK_INTERVAL) + + log.error(f"[Switch] Timeout après {SWITCH_TIMEOUT}s") + return False + + log.error(f"[Switch] Preset {preset_id} non trouvé") + return False + + +@app.route("/v1/models", methods=["GET"]) +def list_models(): + """Expose tous les presets Jean + modèles Ollama comme modèles OpenAI""" + presets = get_presets() + models = [] + + for p in presets: + model_id = p["id"] + model_name = p.get("name", model_id) + + model_info = { + "id": model_id, + "object": "model", + "created": int(time.time()), + "owned_by": "jean", + "permission": [], + "root": model_id, + "parent": None, + } + + meta = {} + if p.get("quant"): + meta["quantization"] = p["quant"] + if p.get("reasoning"): + meta["reasoning"] = p["reasoning"] + if p.get("active"): + meta["active"] = True + if p.get("bench"): + meta["bench"] = p["bench"] + + if meta: + model_info["meta"] = meta + + models.append(model_info) + + if ENABLE_OLLAMA: + ollama_models = get_ollama_models() + for m in ollama_models: + model_id = m.get("name", "").replace(":latest", "") + if not model_id: + continue + + model_info = { + "id": model_id, + "object": "model", + "created": int(time.time()), + "owned_by": "ollama", + "permission": [], + "root": model_id, + "parent": None, + } + + details = m.get("details", {}) + meta = {} + if details.get("parameter_size"): + meta["parameter_size"] = details["parameter_size"] + if details.get("quantization_level"): + meta["quantization"] = details["quantization_level"] + if details.get("family"): + meta["family"] = details["family"] + if m.get("capabilities"): + meta["capabilities"] = m["capabilities"] + + if meta: + model_info["meta"] = meta + + models.append(model_info) + + return jsonify({"object": "list", "data": models}) + + +@app.route("/v1/chat/completions", methods=["POST"]) +def chat_completions(): + """Route les requêtes chat vers Jean ou Ollama selon le modèle""" + data = request.get_json() + model_id = data.get("model") + + if not model_id: + return jsonify({"error": "model parameter required"}), 400 + + presets = get_presets() + jean_preset_ids = [p["id"] for p in presets] + + if model_id in jean_preset_ids: + with switch_lock: + active = get_active_preset() + if not active or active["id"] != model_id: + log.info(f"[Router] Switch Jean requis: {active['id'] if active else 'none'} → {model_id}") + if not switch_preset(model_id): + return jsonify({"error": f"Failed to switch to preset {model_id}"}), 503 + + upstream_url = f"http://{JEAN_PROXY_HOST}:{JEAN_PROXY_PORT}/v1/chat/completions" + log.info(f"[Router] → Jean ({model_id})") + elif ENABLE_OLLAMA: + ollama_models = get_ollama_models() + ollama_ids = [m.get("name", "").replace(":latest", "") for m in ollama_models] + + if model_id in ollama_ids: + upstream_url = f"http://{OLLAMA_PROXY_HOST}:{OLLAMA_PROXY_PORT}/v1/chat/completions" + log.info(f"[Router] → Ollama ({model_id})") + else: + return jsonify({"error": f"Unknown model: {model_id}"}), 404 + else: + return jsonify({"error": f"Unknown model: {model_id}"}), 404 + + headers = dict(request.headers) + headers.pop("Host", None) + + try: + resp = requests.request( + method=request.method, + url=upstream_url, + headers=headers, + json=data, + stream=True, + timeout=None, + ) + except Exception as e: + log.error(f"[Router] Erreur upstream: {e}") + return jsonify({"error": str(e)}), 502 + + excluded = {"content-encoding", "content-length", "transfer-encoding", "connection"} + response_headers = [ + (name, value) for name, value in resp.headers.items() + if name.lower() not in excluded + ] + + return Response( + resp.raw, + status=resp.status_code, + headers=response_headers, + direct_passthrough=True, + ) + + +@app.route("/", defaults={"path": ""}, methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS"]) +@app.route("/", methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS"]) +def proxy(path): + """Proxy générique pour les autres endpoints""" + if path.startswith("v1/chat/completions"): + return chat_completions() + + upstream_url = f"http://{JEAN_PROXY_HOST}:{JEAN_PROXY_PORT}/{path}" + headers = dict(request.headers) + headers.pop("Host", None) + + try: + resp = requests.request( + method=request.method, + url=upstream_url, + headers=headers, + data=request.get_data(), + cookies=request.cookies, + params=request.args, + allow_redirects=False, + stream=True, + timeout=None, + ) + except Exception as e: + return Response(str(e), status=502) + + excluded = {"content-encoding", "content-length", "transfer-encoding", "connection"} + response_headers = [ + (name, value) for name, value in resp.headers.items() + if name.lower() not in excluded + ] + + return Response( + resp.raw, + status=resp.status_code, + headers=response_headers, + direct_passthrough=True, + ) + + +if __name__ == "__main__": + log.info(f"[Router] Démarrage sur port {LISTEN_PORT}") + log.info(f"[Router] Jean Web: {JEAN_WEB_HOST}:{JEAN_WEB_PORT}") + log.info(f"[Router] Jean Proxy: {JEAN_PROXY_HOST}:{JEAN_PROXY_PORT}") + if ENABLE_OLLAMA: + log.info(f"[Router] Ollama Proxy: {OLLAMA_PROXY_HOST}:{OLLAMA_PROXY_PORT}") + else: + log.info(f"[Router] Ollama: désactivé") + app.run(host="0.0.0.0", port=LISTEN_PORT, threaded=True) diff --git a/litellm/config.yaml b/litellm/config.yaml index 10826c5..bf25ec7 100644 --- a/litellm/config.yaml +++ b/litellm/config.yaml @@ -1,112 +1,26 @@ # ═══════════════════════════════════════════════════════════ -# LiteLLM Proxy — Routing pour Effi +# LiteLLM Proxy — Routing pour Effi via jean-router # Endpoint OpenAI-compatible sur :4000 (host network) -# ‎ -# Backend unique : JeanOrchestrator (port 8080 sur PC gaming .11) -# via jean-proxy :3012 + WOL auto si PC eteint. -# ‎ -# L'orchestrateur gere automatiquement le switch entre : -# - Qwen3.6-35B MoE I-Nano (defaut, 11.7 Go, 100k ctx, ~70 tok/s) -# - Qwen3.6-27B IQ4_XS (code/analyse, 14.6 Go, 55k ctx, ~60 tok/s) -# ‎ -# Tous les modeles passent par le meme proxy :3012. -# L'orchestrateur detecte le modele demande et charge le bon LLM. -# Retour automatique au MoE apres 15 min sans 27B. +# +# jean-router (:3014) gère dynamiquement : +# - Liste des presets Jean (avec switch automatique) +# - Liste des modèles Ollama +# - Routing vers le bon backend +# +# LiteLLM sert uniquement de proxy OpenAI unifié pour OpenWebUI. # ═══════════════════════════════════════════════════════════ model_list: - # ═════════════════════════════════════════════════════════ - # Profil 1 — Raisonnement ON (défaut, pour tâches complexes) - # Le modele reflechit (jusqu'a 2048 tokens de reasoning_content) - # avant de produire sa reponse. Bon pour code, analyse, RAG, rapports. - # ═════════════════════════════════════════════════════════ - - model_name: qwen3.6-reason + - model_name: jean litellm_params: - model: openai/qwen3.6-reason - api_base: http://localhost:3012/v1 + model: openai/* + api_base: http://localhost:3014/v1 api_key: "null" timeout: 600 - extra_body: - chat_template_kwargs: - enable_thinking: true - - - model_name: qwen3.6-27b - litellm_params: - model: openai/qwen3.6-27b - api_base: http://localhost:3012/v1 - api_key: "null" - timeout: 600 - extra_body: - chat_template_kwargs: - enable_thinking: true - - # ═════════════════════════════════════════════════════════ - # Profil 2 — Raisonnement OFF (chat rapide, ~5x moins de latence) - # Reponse immediate (TTFT <1s) sans phase de reflexion. - # Ideal pour : chat interactif, redaction de mail, extraction - # structuree simple, Q/R directes. - # ═════════════════════════════════════════════════════════ - - model_name: qwen3.6-fast - litellm_params: - model: openai/qwen3.6-fast - api_base: http://localhost:3012/v1 - api_key: "null" - timeout: 600 - extra_body: - chat_template_kwargs: - enable_thinking: false - - # Alias court "qwen3.6" = fast par defaut (usage chat le plus courant) - - model_name: qwen3.6 - litellm_params: - model: openai/qwen3.6 - api_base: http://localhost:3012/v1 - api_key: "null" - timeout: 600 - extra_body: - chat_template_kwargs: - enable_thinking: false - - # ═════════════════════════════════════════════════════════ - # Profil 3 — Qwen3.6-35B MoE APEX I-Nano (chat Hermes, usage quotidien) - # Modele MoE : 35B total / 3B actifs, 11.7 Go, cache KV 5x plus petit - # que le 27B -> contexte 100k possible. Cohabite avec ComfyUI. - # Route via le meme proxy :3012 que le 27B (orchestrateur fait le switch). - # ═════════════════════════════════════════════════════════ - - model_name: qwen3.6-moe - litellm_params: - model: openai/qwen3.6-moe - api_base: http://localhost:3012/v1 - api_key: "null" - timeout: 600 - extra_body: - chat_template_kwargs: - enable_thinking: false - - - model_name: qwen3.6-moe-reason - litellm_params: - model: openai/qwen3.6-moe-reason - api_base: http://localhost:3012/v1 - api_key: "null" - timeout: 600 - extra_body: - chat_template_kwargs: - enable_thinking: true - - # ═════════════════════════════════════════════════════════ - # Backend Ollama — COMMENTE (contention VRAM, voir entete) - # Pour reactiver si dispo (autre machine / GPU dedie), decommenter - # et s'assurer que le GPU peut charger les deux modeles. - # ═════════════════════════════════════════════════════════ - # - model_name: llama3.1-8b - # litellm_params: - # model: ollama/llama3.1:8b - # api_base: http://localhost:3011 - # timeout: 600 litellm_settings: - drop_params: false # ne pas dropper: on veut passer chat_template_kwargs a Jean + drop_params: false telemetry: false general_settings: - master_key: sk-effi-litellm-master \ No newline at end of file + master_key: sk-effi-litellm-master