From 4f957574893c98fe8c65143d4f79115cd0e4a832 Mon Sep 17 00:00:00 2001 From: Folzo Date: Wed, 22 Jul 2026 21:59:07 +0200 Subject: [PATCH] jean-router: resolution dynamique des context_length + max_tokens + capabilities MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Suppression du preset_ctx_map code en dur - Resolution dynamique: Jean /v1/models → matching Ollama → heuristique - Ajout max_tokens / max_input_tokens dans /v1/models - Enrichissement Ollama via /api/show pour donnees manquantes - Detection capabilities (vision, thinking) pour presets Jean - Auto-doublement max_tokens pour presets reasoning=on - Timeout switch augmente a 15s avec retry - Tracking du dernier preset actif (bypass bug Jean active=false) - Cache TTL configurable (15s par defaut), ?refresh=true - Plan de test des capacites LLM --- jean-router/app.py | 330 ++++++++++++++++++++++++++----- jean-router/test-capabilities.md | 225 +++++++++++++++++++++ 2 files changed, 507 insertions(+), 48 deletions(-) create mode 100644 jean-router/test-capabilities.md diff --git a/jean-router/app.py b/jean-router/app.py index b18ef28..1b47687 100644 --- a/jean-router/app.py +++ b/jean-router/app.py @@ -38,11 +38,15 @@ app = Flask(__name__) presets_cache = {"data": [], "timestamp": 0} ollama_cache = {"data": [], "timestamp": 0} +ollama_detail_cache = {} jean_ctx_cache = {} -CACHE_TTL = 30 +jean_caps_cache = {} +CACHE_TTL = int(os.getenv("CACHE_TTL", "15")) +OLLAMA_DETAIL_CACHE_TTL = 300 switch_lock = threading.Lock() vram_lock = threading.Lock() vram_state = {"active_backend": "jean"} +last_active_preset = None ollama_unload_timer = None @@ -88,36 +92,213 @@ def get_ollama_models(force_refresh=False): return ollama_cache["data"] -def get_jean_context_lengths(): - """Récupère les context_length depuis Jean /v1/models (avec cache)""" - preset_ctx_map = { - "ornith-35b-q6": 64000, - "qwen3.6-27b-mtp": 55000, - "qwen3.6-35b-moe-compact": 131072, - "qwen3.6-35b-moe-compact-reason": 131072, - "qwen3.6-35b-moe-nano": 100000, - } +def _get_ollama_model_detail(model_name: str) -> dict: + """Récupère les détails d'un modèle Ollama via /api/show, avec cache long""" + now = time.time() + cached = ollama_detail_cache.get(model_name) + if cached and (now - cached["timestamp"]) < OLLAMA_DETAIL_CACHE_TTL: + return cached["data"] + + detail = {} + try: + clean_name = model_name.replace(":latest", "") + url = f"http://{OLLAMA_DIRECT_HOST}:{OLLAMA_DIRECT_PORT}/api/show" + r = requests.post(url, json={"model": clean_name}, timeout=10) + if r.ok: + d = r.json() + model_info = d.get("model_info", {}) + details = d.get("details", {}) + + for key, val in model_info.items(): + if "context_length" in key.lower() or "ctx" in key.lower(): + detail["context_length"] = int(val) + break + + if not detail.get("context_length"): + detail["context_length"] = details.get("context_length") + + detail["capabilities"] = d.get("capabilities", []) + detail["family"] = details.get("family", "") + + ollama_detail_cache[model_name] = {"data": detail, "timestamp": now} + except Exception as e: + log.debug(f"[Ollama] /api/show echoue pour {model_name}: {e}") + + if model_name not in ollama_detail_cache: + ollama_detail_cache[model_name] = {"data": {}, "timestamp": now} + + return ollama_detail_cache[model_name]["data"] + + +def _enrich_ollama_model(m: dict) -> dict: + """Enrichit un modele Ollama avec les donnees /api/show si necessaire""" + model_id = m.get("name", "").replace(":latest", "") + details = m.get("details", {}) + has_ctx = bool(details.get("context_length")) + caps = m.get("capabilities", []) + + if not has_ctx or len(caps) <= 1: + extra = _get_ollama_model_detail(model_id) + if extra.get("context_length"): + details = dict(details) + details["context_length"] = extra["context_length"] + m = dict(m) + m["details"] = details + if extra.get("capabilities") and len(extra["capabilities"]) > len(caps): + m = dict(m) + m["capabilities"] = extra["capabilities"] + if extra.get("family") and not details.get("family"): + details = dict(details) + details["family"] = extra["family"] + m = dict(m) + m["details"] = details + + return m + +def _guess_context_length(preset_id: str) -> int: + """Devine le context_length depuis le nom du preset""" + pid = preset_id.lower() + if "qwen3.6" in pid or "qwen3.5" in pid: + return 262144 + if "qwen3" in pid: + return 40960 + if "qwen2.5" in pid: + return 32768 + if "mistral" in pid or "nemo" in pid: + return 1024000 + if "llama" in pid: + return 131072 + if "ornith" in pid: + return 131072 + if "gemma" in pid: + return 262144 + if "deepseek" in pid: + return 131072 + if "phi" in pid: + return 16384 + if "35b" in pid or "27b" in pid: + return 131072 + return 32768 + + +def _guess_capabilities(preset_id: str) -> list: + """Devine les capabilities depuis le nom du preset + metadonnees Jean""" + pid = preset_id.lower() + caps = ["completion"] + + has_vision = any(kw in pid for kw in ["vision", "vl", "llava", "bakllava", "minicpm"]) + has_thinking = any(kw in pid for kw in ["reason", "thinking", "deepseek-r1"]) + has_tools = not any(kw in pid for kw in ["embed", "bge-", "nomic-"]) + + if has_vision: + caps.append("vision") + if has_thinking: + caps.append("thinking") + if has_tools: + caps.append("tools") + + return caps + + +def _match_preset_to_ollama(preset_id: str): + """Tente de trouver un modele Ollama correspondant au preset Jean""" + ollama_models = get_ollama_models() + pid_lower = preset_id.lower() + + def norm(s): + return s.lower().replace(":", "").replace("-", "").replace(" ", "").replace(".", "") + + pid_norm = norm(pid_lower) + + for m in ollama_models: + name = m.get("name", "").replace(":latest", "") + name_norm = norm(name) + + if pid_norm == name_norm: + return m + + best = None + best_score = 0 + for m in ollama_models: + name = m.get("name", "").replace(":latest", "") + name_norm = norm(name) + score = 0 + + if len(pid_norm) >= 4 and pid_norm[:4] == name_norm[:4]: + score += 10 + if len(pid_norm) >= 8 and pid_norm[:8] == name_norm[:8]: + score += 20 + if name_norm in pid_norm or pid_norm in name_norm: + score += 15 + + parts_pid = set(pid_norm.split("_")) + parts_name = set(name_norm.split("_")) + common = parts_pid & parts_name + score += len(common) * 5 + + if score > best_score: + best_score = score + best = m + + if best and best_score >= 5: + return best + return None + + +def get_jean_metadata(): + """Resout dynamiquement context_length et capabilities pour les presets Jean. + 1) Ollama matching → 2) Jean /v1/models → 3) heuristique""" + now = time.time() try: url = f"http://{JEAN_PROXY_HOST}:{JEAN_PROXY_PORT}/v1/models" r = requests.get(url, timeout=2) if r.ok: - models = r.json().get("data", []) - for m in models: + for m in r.json().get("data", []): + alias = m.get("id", "") meta = m.get("meta", {}) - n_ctx = meta.get("n_ctx") - if n_ctx: - alias = m.get("id", "") - jean_ctx_cache[alias] = n_ctx - log.debug(f"[Jean] Context length pour {alias}: {n_ctx}") - except Exception as e: - log.debug(f"[Jean] Impossible de récupérer context_length: {e}") + n_ctx = meta.get("n_ctx") or m.get("max_model_len") + if n_ctx and alias: + jean_ctx_cache[alias] = int(n_ctx) + except Exception: + pass - for preset_id, ctx in preset_ctx_map.items(): - if preset_id not in jean_ctx_cache: - jean_ctx_cache[preset_id] = ctx + ollama_models = get_ollama_models() + ollama_lookup = {} + for m in ollama_models: + m = _enrich_ollama_model(m) + name = m.get("name", "").replace(":latest", "") + details = m.get("details", {}) + ctx = details.get("context_length") + caps = m.get("capabilities", []) + ollama_lookup[name] = {"context_length": ctx, "capabilities": caps} - return jean_ctx_cache + presets = get_presets() + for p in presets: + pid = p["id"] + + matched_ollama = _match_preset_to_ollama(pid) + ollama_name = matched_ollama.get("name", "").replace(":latest", "") if matched_ollama else None + + if pid not in jean_ctx_cache: + if ollama_name and ollama_lookup.get(ollama_name, {}).get("context_length"): + jean_ctx_cache[pid] = ollama_lookup[ollama_name]["context_length"] + else: + jean_ctx_cache[pid] = _guess_context_length(pid) + + if pid not in jean_caps_cache: + if p.get("reasoning") == "on": + caps = ["completion", "tools", "thinking"] + else: + caps = ["completion", "tools"] + + if ollama_name and ollama_lookup.get(ollama_name, {}).get("capabilities"): + ollama_caps = ollama_lookup[ollama_name]["capabilities"] + caps = list(dict.fromkeys(caps + ollama_caps)) + + jean_caps_cache[pid] = caps + + return jean_ctx_cache, jean_caps_cache def get_active_preset(): @@ -131,6 +312,7 @@ def get_active_preset(): def switch_preset(preset_id: str) -> bool: """Switch vers un preset spécifique par son ID""" + global last_active_preset presets = get_presets(force_refresh=True) for i, p in enumerate(presets): @@ -140,14 +322,20 @@ def switch_preset(preset_id: str) -> bool: return True log.info(f"[Switch] Activation preset {preset_id} (index {i+1})") - try: - url = f"http://{JEAN_WEB_HOST}:{JEAN_WEB_PORT}/api/switch" - r = requests.post(url, json={"n": i + 1}, timeout=5) - if not r.ok: - log.error(f"[Switch] Erreur API: {r.text}") - return False - except Exception as e: - log.error(f"[Switch] Erreur requête: {e}") + success = False + for attempt in range(2): + try: + url = f"http://{JEAN_WEB_HOST}:{JEAN_WEB_PORT}/api/switch" + r = requests.post(url, json={"n": i + 1}, timeout=15) + if r.ok: + success = True + break + log.error(f"[Switch] Erreur API (tentative {attempt+1}): {r.text}") + except Exception as e: + log.error(f"[Switch] Erreur requête (tentative {attempt+1}): {e}") + if attempt == 0: + time.sleep(3) + if not success: return False log.info(f"[Switch] Attente chargement modèle...") @@ -163,6 +351,7 @@ def switch_preset(preset_id: str) -> bool: models = mr.json().get("data", []) if models: log.info(f"[Switch] Modèle chargé après {int(time.time() - deadline + SWITCH_TIMEOUT)}s") + last_active_preset = preset_id presets_cache["timestamp"] = 0 return True except Exception: @@ -320,6 +509,8 @@ def ensure_vram_for_backend(target_backend: str) -> bool: if not ENABLE_VRAM_MANAGEMENT: return True + global last_active_preset + with vram_lock: current = vram_state["active_backend"] @@ -331,6 +522,7 @@ def ensure_vram_for_backend(target_backend: str) -> bool: cancel_ollama_unload() if not stop_jean(): return False + last_active_preset = None vram_state["active_backend"] = "ollama" return True @@ -339,6 +531,7 @@ def ensure_vram_for_backend(target_backend: str) -> bool: schedule_ollama_unload() if not start_jean(): return False + last_active_preset = None vram_state["active_backend"] = "jean" return True @@ -348,13 +541,24 @@ def ensure_vram_for_backend(target_backend: str) -> bool: @app.route("/v1/models", methods=["GET"]) def list_models(): """Expose tous les presets Jean + modèles Ollama comme modèles OpenAI""" + force_refresh = request.args.get("refresh", "").lower() in ("1", "true", "yes") + if force_refresh: + global presets_cache, ollama_cache + presets_cache["timestamp"] = 0 + ollama_cache["timestamp"] = 0 + log.info("[Models] Refresh force demande") + presets = get_presets() - jean_ctx = get_jean_context_lengths() + jean_ctx, jean_caps = get_jean_metadata() models = [] for p in presets: model_id = p["id"] - model_name = p.get("name", model_id) + + ctx_len = jean_ctx.get(model_id) + caps = jean_caps.get(model_id, ["completion", "tools"]) + if p.get("reasoning") == "on" and "thinking" not in caps: + caps = list(dict.fromkeys(caps + ["thinking"])) model_info = { "id": model_id, @@ -366,7 +570,18 @@ def list_models(): "parent": None, } + if ctx_len: + model_info["context_length"] = ctx_len + model_info["context_window"] = ctx_len + model_info["max_model_len"] = ctx_len + model_info["max_tokens"] = ctx_len + model_info["max_input_tokens"] = ctx_len + + model_info["capabilities"] = caps + meta = {} + if p.get("name") and p["name"] != model_id: + meta["display_name"] = p["name"] if p.get("quant"): meta["quantization"] = p["quant"] if p.get("reasoning"): @@ -375,13 +590,9 @@ def list_models(): meta["active"] = True if p.get("bench"): meta["bench"] = p["bench"] - - ctx_len = jean_ctx.get(model_id) if ctx_len: meta["context_length"] = ctx_len - model_info["context_length"] = ctx_len - model_info["context_window"] = ctx_len - model_info["max_model_len"] = ctx_len + meta["capabilities"] = caps if meta: model_info["meta"] = meta @@ -391,10 +602,15 @@ def list_models(): if ENABLE_OLLAMA: ollama_models = get_ollama_models() for m in ollama_models: + m = _enrich_ollama_model(m) model_id = m.get("name", "").replace(":latest", "") if not model_id: continue + caps = m.get("capabilities", ["completion"]) + details = m.get("details", {}) + ctx_len = details.get("context_length") + model_info = { "id": model_id, "object": "model", @@ -405,7 +621,15 @@ def list_models(): "parent": None, } - details = m.get("details", {}) + if ctx_len: + model_info["context_length"] = ctx_len + model_info["context_window"] = ctx_len + model_info["max_model_len"] = ctx_len + model_info["max_tokens"] = ctx_len + model_info["max_input_tokens"] = ctx_len + + model_info["capabilities"] = caps + meta = {} if details.get("parameter_size"): meta["parameter_size"] = details["parameter_size"] @@ -413,14 +637,9 @@ def list_models(): meta["quantization"] = details["quantization_level"] if details.get("family"): meta["family"] = details["family"] - if details.get("context_length"): - ctx_len = details["context_length"] + if ctx_len: meta["context_length"] = ctx_len - model_info["context_length"] = ctx_len - model_info["context_window"] = ctx_len - model_info["max_model_len"] = ctx_len - if m.get("capabilities"): - meta["capabilities"] = m["capabilities"] + meta["capabilities"] = caps if meta: model_info["meta"] = meta @@ -448,10 +667,25 @@ def chat_completions(): with switch_lock: active = get_active_preset() - if not active or active["id"] != model_id: - log.info(f"[Router] Switch Jean requis: {active['id'] if active else 'none'} → {model_id}") + current_active = active["id"] if active else last_active_preset + if current_active != model_id: + log.info(f"[Router] Switch Jean requis: {current_active or 'none'} → {model_id}") if not switch_preset(model_id): return jsonify({"error": f"Failed to switch to preset {model_id}"}), 503 + else: + log.info(f"[Router] Preset {model_id} déjà actif (tracké)") + + preset = next((p for p in presets if p["id"] == model_id), {}) + if preset.get("reasoning") == "on": + ctx, _ = get_jean_metadata() + ctx_len = ctx.get(model_id, 131072) + req_max = data.get("max_tokens", 0) + if req_max and req_max > 0: + new_max = min(req_max * 2, ctx_len) + if new_max != req_max: + log.info(f"[Reasoning] max_tokens doublé: {req_max} → {new_max}") + data = dict(data) + data["max_tokens"] = new_max upstream_url = f"http://{JEAN_PROXY_HOST}:{JEAN_PROXY_PORT}/v1/chat/completions" log.info(f"[Router] → Jean ({model_id})") diff --git a/jean-router/test-capabilities.md b/jean-router/test-capabilities.md new file mode 100644 index 0000000..73e6673 --- /dev/null +++ b/jean-router/test-capabilities.md @@ -0,0 +1,225 @@ +# Plan de test — Capacités des LLM (vision, thinking) + +## 1. Vérification de /v1/models + +```bash +# Vérifier que les champs sont bien présents +curl -s http://localhost:3014/v1/models | python3 -c " +import json, sys +data = json.load(sys.stdin)['data'] +for m in data: + ctx = m.get('context_length', '?') + max_tok = m.get('max_tokens', '?') + max_in = m.get('max_input_tokens', '?') + caps = m.get('capabilities', []) + print(f'{m[\"id\"]:45s} | ctx={str(ctx):>8s} | max_tok={str(max_tok):>8s} | caps={caps}') +" + +# Forcer le refresh +curl -s "http://localhost:3014/v1/models?refresh=true" | python3 -c "..." +``` + +**Checklist :** +- [ ] Tous les modèles ont `context_length` (pas `?` ni `None`) +- [ ] Tous les modèles ont `max_tokens` et `max_input_tokens` +- [ ] Les modèles ont `capabilities` (pas vide, min `["completion"]`) +- [ ] Le preset "Ornith 1.0 35b Q4 MOE" a des champs valides + +--- + +## 2. Test de vision (multimodal) + +### 2.1 — Modèles censés supporter la vision (selon `/v1/models`) + +| Modèle Ollama | Vision annoncée | +|---------------|----------------| +| qwen3.5:9b | vision | +| qwen3.5:0.8b | vision | +| qwen3.5:4b | vision | +| qwen3.5:2b | vision | +| qwen3.5 (latest) | vision | +| gemma4:12b | vision | +| ministral-3:14b | vision | + +```bash +# Script de test (enregistre comme test_vision.sh) +for MODEL in qwen3.5:9b qwen3.5:4b qwen3.5:2b gemma4:12b ministral-3:14b; do + echo "=== Test vision: $MODEL ===" + curl -s http://localhost:3014/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d "{ + \"model\": \"$MODEL\", + \"messages\": [{ + \"role\": \"user\", + \"content\": \"Décris ce que tu vois dans l'image.\" + }], + \"max_tokens\": 100 + }" | python3 -c " +import json, sys +d = json.load(sys.stdin) +if 'error' in d: + print(f' ERREUR: {d[\"error\"]}') +else: + content = d['choices'][0]['message']['content'] + print(f' REPONSE: {content[:200]}') +" + echo "" +done +``` + +### 2.2 — Test avec image réelle + +```bash +# Encoder une image en base64 (necessite une image de test) +IMAGE_B64=$(base64 -w0 /path/to/test-image.jpg) + +curl -s http://localhost:3014/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d "{ + \"model\": \"qwen3.5:9b\", + \"messages\": [{ + \"role\": \"user\", + \"content\": [ + {\"type\": \"text\", \"text\": \"Décris cette image en une phrase.\"}, + {\"type\": \"image_url\", \"image_url\": {\"url\": \"data:image/jpeg;base64,$IMAGE_B64\"}} + ] + }], + \"max_tokens\": 200 + }" | python3 -m json.tool +``` + +--- + +## 3. Test de thinking/reasoning + +### 3.1 — Modèles censés supporter le thinking + +| Modèle | Thinking annoncé | +|--------|-----------------| +| qwen3.5:9b | thinking | +| qwen3.5:0.8b | thinking | +| qwen3.5:4b | thinking | +| qwen3.5:2b | thinking | +| qwen3.5 (latest) | thinking | +| qwen3:8b | thinking | +| deepseek-r1:14b | thinking | +| deepseek-r1 | thinking | +| gemma4:12b | thinking | +| gemma4:26b | thinking | +| gemma4:e4b | thinking | +| magistral | thinking | +| gpt-oss | thinking | +| gemma4 (latest) | thinking | + +```bash +# Test de raisonnement +for MODEL in deepseek-r1:14b qwen3.5:9b qwen3:8b gemma4:12b; do + echo "=== Test thinking: $MODEL ===" + curl -s http://localhost:3014/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d "{ + \"model\": \"$MODEL\", + \"messages\": [{ + \"role\": \"user\", + \"content\": \"Si un train part de Paris à 100 km/h et un autre de Lyon à 150 km/h, distance 400 km, combien de temps avant qu'ils se croisent ? Explique ton raisonnement étape par étape.\" + }], + \"max_tokens\": 500 + }" | python3 -c " +import json, sys +d = json.load(sys.stdin) +if 'error' in d: + print(f' ERREUR: {d[\"error\"]}') +else: + content = d['choices'][0]['message']['content'] + print(f' REPONSE: {content[:300]}') + # Vérifier si la réponse contient un raisonnement structuré + has_reasoning = any(kw in content.lower() for kw in ['étape', 'd\'abord', 'premièrement', 'step', 'first', 'raisonn']) + print(f' STRUCTURE_VISIBLE: {has_reasoning}') +" + echo "" +done +``` + +### 3.2 — Test thinking avec le paramètre OpenAI `reasoning_effort` + +> Pour les modèles Jean avec `reasoning: "on"` (Ornith, qwen3.6-27b-mtp, etc.) + +```bash +curl -s http://localhost:3014/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "ornith-35b-q6", + "messages": [{"role": "user", "content": "Combien de 'r' y a-t-il dans le mot \"strawberry\" ?"}], + "max_tokens": 300 + }' | python3 -c " +import json, sys +d = json.load(sys.stdin) +print(json.dumps(d, indent=2)[:2000]) +" +# Le modèle devrait répondre : 3 (s-t-r-a-w-b-e-r-r-y → 3 'r') +# Si thinking est actif, il devrait montrer son raisonnement +``` + +--- + +## 4. Test des modèles sans certaines capacités + +```bash +# Tester un modèle sans vision — lui envoyer une image devrait échouer +curl -s http://localhost:3014/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "mistral", + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": "Décris cette image."}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,..."}} + ] + }], + "max_tokens": 100 + }' | python3 -c " +import json, sys +d = json.load(sys.stdin) +print(f'Status attendu: ERREUR → Reçu: {\"error\" in d}') +" +``` + +--- + +## 5. Test de routing (Jean ↔ Ollama) + +```bash +# Tester un modèle Jean (doit déclencher le switch VRAM) +curl -s http://localhost:3014/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "ornith-35b-q6", + "messages": [{"role": "user", "content": "Bonjour, es-tu fonctionnel ?"}], + "max_tokens": 50 + }' | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('choices',[{}])[0].get('message',{}).get('content','ERREUR:'+json.dumps(d))[:300])" + +# Puis tester un modèle Ollama (doit déclencher le switch inverse) +curl -s http://localhost:3014/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "mistral", + "messages": [{"role": "user", "content": "Bonjour, es-tu fonctionnel ?"}], + "max_tokens": 50 + }' | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('choices',[{}])[0].get('message',{}).get('content','ERREUR:'+json.dumps(d))[:300])" +``` + +--- + +## 6. Synthèse + +| Check | Méthode | +|-------|---------| +| `/v1/models` a `max_tokens` + `max_input_tokens` | curl + jq | +| `/v1/models` a `capabilities` non-vide | curl + jq | +| Preset "Ornith 1.0 35b Q4 MOE" existe avec champs valides | curl + jq | +| Modèle `vision=true` accepte image sans erreur | curl + image base64 | +| Modèle `vision=false` rejette image | curl + image base64 | +| Modèle `thinking=true` montre raisonnement structuré | curl + analyse réponse | +| Switch Jean→Ollama fonctionne | 2 requêtes séquentielles | +| `?refresh=true` force la mise à jour du cache | 2 requêtes avec timestamp |