Folzo/Effi
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

jean-router: resolution dynamique des context_length + max_tokens + capabilities

Browse Source 
- Suppression du preset_ctx_map code en dur
- Resolution dynamique: Jean /v1/models → matching Ollama → heuristique
- Ajout max_tokens / max_input_tokens dans /v1/models
- Enrichissement Ollama via /api/show pour donnees manquantes
- Detection capabilities (vision, thinking) pour presets Jean
- Auto-doublement max_tokens pour presets reasoning=on
- Timeout switch augmente a 15s avec retry
- Tracking du dernier preset actif (bypass bug Jean active=false)
- Cache TTL configurable (15s par defaut), ?refresh=true
- Plan de test des capacites LLM
This commit is contained in:
Folzo
2026-07-22 21:59:07 +02:00
parent 2e83369483
commit 4f95757489
2 changed files with 507 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jean-router/app.py
+282 -48
View File
@@ -38,11 +38,15 @@ app = Flask(__name__)
presets_cache = {"data": [], "timestamp": 0}
ollama_cache = {"data": [], "timestamp": 0}
ollama_detail_cache = {}
jean_ctx_cache = {}
CACHE_TTL = 30
jean_caps_cache = {}
CACHE_TTL = int(os.getenv("CACHE_TTL", "15"))
OLLAMA_DETAIL_CACHE_TTL = 300
switch_lock = threading.Lock()
vram_lock = threading.Lock()
vram_state = {"active_backend": "jean"}
last_active_preset = None
ollama_unload_timer = None
@@ -88,36 +92,213 @@ def get_ollama_models(force_refresh=False):
return ollama_cache["data"]
def get_jean_context_lengths():
"""Récupère les context_length depuis Jean /v1/models (avec cache)"""
preset_ctx_map = {
"ornith-35b-q6": 64000,
"qwen3.6-27b-mtp": 55000,
"qwen3.6-35b-moe-compact": 131072,
"qwen3.6-35b-moe-compact-reason": 131072,
"qwen3.6-35b-moe-nano": 100000,
}
def _get_ollama_model_detail(model_name: str) -> dict:
"""Récupère les détails d'un modèle Ollama via /api/show, avec cache long"""
now = time.time()
cached = ollama_detail_cache.get(model_name)
if cached and (now - cached["timestamp"]) < OLLAMA_DETAIL_CACHE_TTL:
return cached["data"]
detail = {}
try:
clean_name = model_name.replace(":latest", "")
url = f"http://{OLLAMA_DIRECT_HOST}:{OLLAMA_DIRECT_PORT}/api/show"
r = requests.post(url, json={"model": clean_name}, timeout=10)
if r.ok:
d = r.json()
model_info = d.get("model_info", {})
details = d.get("details", {})
for key, val in model_info.items():
if "context_length" in key.lower() or "ctx" in key.lower():
detail["context_length"] = int(val)
break
if not detail.get("context_length"):
detail["context_length"] = details.get("context_length")
detail["capabilities"] = d.get("capabilities", [])
detail["family"] = details.get("family", "")
ollama_detail_cache[model_name] = {"data": detail, "timestamp": now}
except Exception as e:
log.debug(f"[Ollama] /api/show echoue pour {model_name}: {e}")
if model_name not in ollama_detail_cache:
ollama_detail_cache[model_name] = {"data": {}, "timestamp": now}
return ollama_detail_cache[model_name]["data"]
def _enrich_ollama_model(m: dict) -> dict:
"""Enrichit un modele Ollama avec les donnees /api/show si necessaire"""
model_id = m.get("name", "").replace(":latest", "")
details = m.get("details", {})
has_ctx = bool(details.get("context_length"))
caps = m.get("capabilities", [])
if not has_ctx or len(caps) <= 1:
extra = _get_ollama_model_detail(model_id)
if extra.get("context_length"):
details = dict(details)
details["context_length"] = extra["context_length"]
m = dict(m)
m["details"] = details
if extra.get("capabilities") and len(extra["capabilities"]) > len(caps):
m = dict(m)
m["capabilities"] = extra["capabilities"]
if extra.get("family") and not details.get("family"):
details = dict(details)
details["family"] = extra["family"]
m = dict(m)
m["details"] = details
return m
def _guess_context_length(preset_id: str) -> int:
"""Devine le context_length depuis le nom du preset"""
pid = preset_id.lower()
if "qwen3.6" in pid or "qwen3.5" in pid:
return 262144
if "qwen3" in pid:
return 40960
if "qwen2.5" in pid:
return 32768
if "mistral" in pid or "nemo" in pid:
return 1024000
if "llama" in pid:
return 131072
if "ornith" in pid:
return 131072
if "gemma" in pid:
return 262144
if "deepseek" in pid:
return 131072
if "phi" in pid:
return 16384
if "35b" in pid or "27b" in pid:
return 131072
return 32768
def _guess_capabilities(preset_id: str) -> list:
"""Devine les capabilities depuis le nom du preset + metadonnees Jean"""
pid = preset_id.lower()
caps = ["completion"]
has_vision = any(kw in pid for kw in ["vision", "vl", "llava", "bakllava", "minicpm"])
has_thinking = any(kw in pid for kw in ["reason", "thinking", "deepseek-r1"])
has_tools = not any(kw in pid for kw in ["embed", "bge-", "nomic-"])
if has_vision:
caps.append("vision")
if has_thinking:
caps.append("thinking")
if has_tools:
caps.append("tools")
return caps
def _match_preset_to_ollama(preset_id: str):
"""Tente de trouver un modele Ollama correspondant au preset Jean"""
ollama_models = get_ollama_models()
pid_lower = preset_id.lower()
def norm(s):
return s.lower().replace(":", "").replace("-", "").replace(" ", "").replace(".", "")
pid_norm = norm(pid_lower)
for m in ollama_models:
name = m.get("name", "").replace(":latest", "")
name_norm = norm(name)
if pid_norm == name_norm:
return m
best = None
best_score = 0
for m in ollama_models:
name = m.get("name", "").replace(":latest", "")
name_norm = norm(name)
score = 0
if len(pid_norm) >= 4 and pid_norm[:4] == name_norm[:4]:
score += 10
if len(pid_norm) >= 8 and pid_norm[:8] == name_norm[:8]:
score += 20
if name_norm in pid_norm or pid_norm in name_norm:
score += 15
parts_pid = set(pid_norm.split("_"))
parts_name = set(name_norm.split("_"))
common = parts_pid & parts_name
score += len(common) * 5
if score > best_score:
best_score = score
best = m
if best and best_score >= 5:
return best
return None
def get_jean_metadata():
"""Resout dynamiquement context_length et capabilities pour les presets Jean.
1) Ollama matching → 2) Jean /v1/models → 3) heuristique"""
now = time.time()
try:
url = f"http://{JEAN_PROXY_HOST}:{JEAN_PROXY_PORT}/v1/models"
r = requests.get(url, timeout=2)
if r.ok:
models = r.json().get("data", [])
for m in models:
for m in r.json().get("data", []):
alias = m.get("id", "")
meta = m.get("meta", {})
n_ctx = meta.get("n_ctx")
if n_ctx:
alias = m.get("id", "")
jean_ctx_cache[alias] = n_ctx
log.debug(f"[Jean] Context length pour {alias}: {n_ctx}")
except Exception as e:
log.debug(f"[Jean] Impossible de récupérer context_length: {e}")
n_ctx = meta.get("n_ctx") or m.get("max_model_len")
if n_ctx and alias:
jean_ctx_cache[alias] = int(n_ctx)
except Exception:
pass
for preset_id, ctx in preset_ctx_map.items():
if preset_id not in jean_ctx_cache:
jean_ctx_cache[preset_id] = ctx
ollama_models = get_ollama_models()
ollama_lookup = {}
for m in ollama_models:
m = _enrich_ollama_model(m)
name = m.get("name", "").replace(":latest", "")
details = m.get("details", {})
ctx = details.get("context_length")
caps = m.get("capabilities", [])
ollama_lookup[name] = {"context_length": ctx, "capabilities": caps}
return jean_ctx_cache
presets = get_presets()
for p in presets:
pid = p["id"]
matched_ollama = _match_preset_to_ollama(pid)
ollama_name = matched_ollama.get("name", "").replace(":latest", "") if matched_ollama else None
if pid not in jean_ctx_cache:
if ollama_name and ollama_lookup.get(ollama_name, {}).get("context_length"):
jean_ctx_cache[pid] = ollama_lookup[ollama_name]["context_length"]
else:
jean_ctx_cache[pid] = _guess_context_length(pid)
if pid not in jean_caps_cache:
if p.get("reasoning") == "on":
caps = ["completion", "tools", "thinking"]
else:
caps = ["completion", "tools"]
if ollama_name and ollama_lookup.get(ollama_name, {}).get("capabilities"):
ollama_caps = ollama_lookup[ollama_name]["capabilities"]
caps = list(dict.fromkeys(caps + ollama_caps))
jean_caps_cache[pid] = caps
return jean_ctx_cache, jean_caps_cache
def get_active_preset():
@@ -131,6 +312,7 @@ def get_active_preset():
def switch_preset(preset_id: str) -> bool:
"""Switch vers un preset spécifique par son ID"""
global last_active_preset
presets = get_presets(force_refresh=True)
for i, p in enumerate(presets):
@@ -140,14 +322,20 @@ def switch_preset(preset_id: str) -> bool:
return True
log.info(f"[Switch] Activation preset {preset_id} (index {i+1})")
try:
url = f"http://{JEAN_WEB_HOST}:{JEAN_WEB_PORT}/api/switch"
r = requests.post(url, json={"n": i + 1}, timeout=5)
if not r.ok:
log.error(f"[Switch] Erreur API: {r.text}")
return False
except Exception as e:
log.error(f"[Switch] Erreur requête: {e}")
success = False
for attempt in range(2):
try:
url = f"http://{JEAN_WEB_HOST}:{JEAN_WEB_PORT}/api/switch"
r = requests.post(url, json={"n": i + 1}, timeout=15)
if r.ok:
success = True
break
log.error(f"[Switch] Erreur API (tentative {attempt+1}): {r.text}")
except Exception as e:
log.error(f"[Switch] Erreur requête (tentative {attempt+1}): {e}")
if attempt == 0:
time.sleep(3)
if not success:
return False
log.info(f"[Switch] Attente chargement modèle...")
@@ -163,6 +351,7 @@ def switch_preset(preset_id: str) -> bool:
models = mr.json().get("data", [])
if models:
log.info(f"[Switch] Modèle chargé après {int(time.time() - deadline + SWITCH_TIMEOUT)}s")
last_active_preset = preset_id
presets_cache["timestamp"] = 0
return True
except Exception:
@@ -320,6 +509,8 @@ def ensure_vram_for_backend(target_backend: str) -> bool:
if not ENABLE_VRAM_MANAGEMENT:
return True
global last_active_preset
with vram_lock:
current = vram_state["active_backend"]
@@ -331,6 +522,7 @@ def ensure_vram_for_backend(target_backend: str) -> bool:
cancel_ollama_unload()
if not stop_jean():
return False
last_active_preset = None
vram_state["active_backend"] = "ollama"
return True
@@ -339,6 +531,7 @@ def ensure_vram_for_backend(target_backend: str) -> bool:
schedule_ollama_unload()
if not start_jean():
return False
last_active_preset = None
vram_state["active_backend"] = "jean"
return True
@@ -348,13 +541,24 @@ def ensure_vram_for_backend(target_backend: str) -> bool:
@app.route("/v1/models", methods=["GET"])
def list_models():
"""Expose tous les presets Jean + modèles Ollama comme modèles OpenAI"""
force_refresh = request.args.get("refresh", "").lower() in ("1", "true", "yes")
if force_refresh:
global presets_cache, ollama_cache
presets_cache["timestamp"] = 0
ollama_cache["timestamp"] = 0
log.info("[Models] Refresh force demande")
presets = get_presets()
jean_ctx = get_jean_context_lengths()
jean_ctx, jean_caps = get_jean_metadata()
models = []
for p in presets:
model_id = p["id"]
model_name = p.get("name", model_id)
ctx_len = jean_ctx.get(model_id)
caps = jean_caps.get(model_id, ["completion", "tools"])
if p.get("reasoning") == "on" and "thinking" not in caps:
caps = list(dict.fromkeys(caps + ["thinking"]))
model_info = {
"id": model_id,
@@ -366,7 +570,18 @@ def list_models():
"parent": None,
}
if ctx_len:
model_info["context_length"] = ctx_len
model_info["context_window"] = ctx_len
model_info["max_model_len"] = ctx_len
model_info["max_tokens"] = ctx_len
model_info["max_input_tokens"] = ctx_len
model_info["capabilities"] = caps
meta = {}
if p.get("name") and p["name"] != model_id:
meta["display_name"] = p["name"]
if p.get("quant"):
meta["quantization"] = p["quant"]
if p.get("reasoning"):
@@ -375,13 +590,9 @@ def list_models():
meta["active"] = True
if p.get("bench"):
meta["bench"] = p["bench"]
ctx_len = jean_ctx.get(model_id)
if ctx_len:
meta["context_length"] = ctx_len
model_info["context_length"] = ctx_len
model_info["context_window"] = ctx_len
model_info["max_model_len"] = ctx_len
meta["capabilities"] = caps
if meta:
model_info["meta"] = meta
@@ -391,10 +602,15 @@ def list_models():
if ENABLE_OLLAMA:
ollama_models = get_ollama_models()
for m in ollama_models:
m = _enrich_ollama_model(m)
model_id = m.get("name", "").replace(":latest", "")
if not model_id:
continue
caps = m.get("capabilities", ["completion"])
details = m.get("details", {})
ctx_len = details.get("context_length")
model_info = {
"id": model_id,
"object": "model",
@@ -405,7 +621,15 @@ def list_models():
"parent": None,
}
details = m.get("details", {})
if ctx_len:
model_info["context_length"] = ctx_len
model_info["context_window"] = ctx_len
model_info["max_model_len"] = ctx_len
model_info["max_tokens"] = ctx_len
model_info["max_input_tokens"] = ctx_len
model_info["capabilities"] = caps
meta = {}
if details.get("parameter_size"):
meta["parameter_size"] = details["parameter_size"]
@@ -413,14 +637,9 @@ def list_models():
meta["quantization"] = details["quantization_level"]
if details.get("family"):
meta["family"] = details["family"]
if details.get("context_length"):
ctx_len = details["context_length"]
if ctx_len:
meta["context_length"] = ctx_len
model_info["context_length"] = ctx_len
model_info["context_window"] = ctx_len
model_info["max_model_len"] = ctx_len
if m.get("capabilities"):
meta["capabilities"] = m["capabilities"]
meta["capabilities"] = caps
if meta:
model_info["meta"] = meta
@@ -448,10 +667,25 @@ def chat_completions():
with switch_lock:
active = get_active_preset()
if not active or active["id"] != model_id:
log.info(f"[Router] Switch Jean requis: {active['id'] if active else 'none'} → {model_id}")
current_active = active["id"] if active else last_active_preset
if current_active != model_id:
log.info(f"[Router] Switch Jean requis: {current_active or 'none'} → {model_id}")
if not switch_preset(model_id):
return jsonify({"error": f"Failed to switch to preset {model_id}"}), 503
else:
log.info(f"[Router] Preset {model_id} déjà actif (tracké)")
preset = next((p for p in presets if p["id"] == model_id), {})
if preset.get("reasoning") == "on":
ctx, _ = get_jean_metadata()
ctx_len = ctx.get(model_id, 131072)
req_max = data.get("max_tokens", 0)
if req_max and req_max > 0:
new_max = min(req_max * 2, ctx_len)
if new_max != req_max:
log.info(f"[Reasoning] max_tokens doublé: {req_max} → {new_max}")
data = dict(data)
data["max_tokens"] = new_max
upstream_url = f"http://{JEAN_PROXY_HOST}:{JEAN_PROXY_PORT}/v1/chat/completions"
log.info(f"[Router] → Jean ({model_id})")
jean-router/test-capabilities.md
+225
View File
@@ -0,0 +1,225 @@
# Plan de test — Capacités des LLM (vision, thinking)
## 1. Vérification de /v1/models
```bash
# Vérifier que les champs sont bien présents
curl -s http://localhost:3014/v1/models | python3 -c "
import json, sys
data = json.load(sys.stdin)['data']
for m in data:
ctx = m.get('context_length', '?')
max_tok = m.get('max_tokens', '?')
max_in = m.get('max_input_tokens', '?')
caps = m.get('capabilities', [])
print(f'{m[\"id\"]:45s} | ctx={str(ctx):>8s} | max_tok={str(max_tok):>8s} | caps={caps}')
"
# Forcer le refresh
curl -s "http://localhost:3014/v1/models?refresh=true" | python3 -c "..."
```
**Checklist :**
- [ ] Tous les modèles ont `context_length` (pas `?` ni `None`)
- [ ] Tous les modèles ont `max_tokens` et `max_input_tokens`
- [ ] Les modèles ont `capabilities` (pas vide, min `["completion"]`)
- [ ] Le preset "Ornith 1.0 35b Q4 MOE" a des champs valides
---
## 2. Test de vision (multimodal)
### 2.1 — Modèles censés supporter la vision (selon `/v1/models`)
| Modèle Ollama | Vision annoncée |
|---------------|----------------|
| qwen3.5:9b | vision |
| qwen3.5:0.8b | vision |
| qwen3.5:4b | vision |
| qwen3.5:2b | vision |
| qwen3.5 (latest) | vision |
| gemma4:12b | vision |
| ministral-3:14b | vision |
```bash
# Script de test (enregistre comme test_vision.sh)
for MODEL in qwen3.5:9b qwen3.5:4b qwen3.5:2b gemma4:12b ministral-3:14b; do
echo "=== Test vision: $MODEL ==="
curl -s http://localhost:3014/v1/chat/completions \
-H "Content-Type: application/json" \
-d "{
\"model\": \"$MODEL\",
\"messages\": [{
\"role\": \"user\",
\"content\": \"Décris ce que tu vois dans l'image.\"
}],
\"max_tokens\": 100
}" | python3 -c "
import json, sys
d = json.load(sys.stdin)
if 'error' in d:
print(f' ERREUR: {d[\"error\"]}')
else:
content = d['choices'][0]['message']['content']
print(f' REPONSE: {content[:200]}')
"
echo ""
done
```
### 2.2 — Test avec image réelle
```bash
# Encoder une image en base64 (necessite une image de test)
IMAGE_B64=$(base64 -w0 /path/to/test-image.jpg)
curl -s http://localhost:3014/v1/chat/completions \
-H "Content-Type: application/json" \
-d "{
\"model\": \"qwen3.5:9b\",
\"messages\": [{
\"role\": \"user\",
\"content\": [
{\"type\": \"text\", \"text\": \"Décris cette image en une phrase.\"},
{\"type\": \"image_url\", \"image_url\": {\"url\": \"data:image/jpeg;base64,$IMAGE_B64\"}}
]
}],
\"max_tokens\": 200
}" | python3 -m json.tool
```
---
## 3. Test de thinking/reasoning
### 3.1 — Modèles censés supporter le thinking
| Modèle | Thinking annoncé |
|--------|-----------------|
| qwen3.5:9b | thinking |
| qwen3.5:0.8b | thinking |
| qwen3.5:4b | thinking |
| qwen3.5:2b | thinking |
| qwen3.5 (latest) | thinking |
| qwen3:8b | thinking |
| deepseek-r1:14b | thinking |
| deepseek-r1 | thinking |
| gemma4:12b | thinking |
| gemma4:26b | thinking |
| gemma4:e4b | thinking |
| magistral | thinking |
| gpt-oss | thinking |
| gemma4 (latest) | thinking |
```bash
# Test de raisonnement
for MODEL in deepseek-r1:14b qwen3.5:9b qwen3:8b gemma4:12b; do
echo "=== Test thinking: $MODEL ==="
curl -s http://localhost:3014/v1/chat/completions \
-H "Content-Type: application/json" \
-d "{
\"model\": \"$MODEL\",
\"messages\": [{
\"role\": \"user\",
\"content\": \"Si un train part de Paris à 100 km/h et un autre de Lyon à 150 km/h, distance 400 km, combien de temps avant qu'ils se croisent ? Explique ton raisonnement étape par étape.\"
}],
\"max_tokens\": 500
}" | python3 -c "
import json, sys
d = json.load(sys.stdin)
if 'error' in d:
print(f' ERREUR: {d[\"error\"]}')
else:
content = d['choices'][0]['message']['content']
print(f' REPONSE: {content[:300]}')
# Vérifier si la réponse contient un raisonnement structuré
has_reasoning = any(kw in content.lower() for kw in ['étape', 'd\'abord', 'premièrement', 'step', 'first', 'raisonn'])
print(f' STRUCTURE_VISIBLE: {has_reasoning}')
"
echo ""
done
```
### 3.2 — Test thinking avec le paramètre OpenAI `reasoning_effort`
> Pour les modèles Jean avec `reasoning: "on"` (Ornith, qwen3.6-27b-mtp, etc.)
```bash
curl -s http://localhost:3014/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "ornith-35b-q6",
"messages": [{"role": "user", "content": "Combien de 'r' y a-t-il dans le mot \"strawberry\" ?"}],
"max_tokens": 300
}' | python3 -c "
import json, sys
d = json.load(sys.stdin)
print(json.dumps(d, indent=2)[:2000])
"
# Le modèle devrait répondre : 3 (s-t-r-a-w-b-e-r-r-y → 3 'r')
# Si thinking est actif, il devrait montrer son raisonnement
```
---
## 4. Test des modèles sans certaines capacités
```bash
# Tester un modèle sans vision — lui envoyer une image devrait échouer
curl -s http://localhost:3014/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "mistral",
"messages": [{
"role": "user",
"content": [
{"type": "text", "text": "Décris cette image."},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,..."}}
]
}],
"max_tokens": 100
}' | python3 -c "
import json, sys
d = json.load(sys.stdin)
print(f'Status attendu: ERREUR → Reçu: {\"error\" in d}')
"
```
---
## 5. Test de routing (Jean ↔ Ollama)
```bash
# Tester un modèle Jean (doit déclencher le switch VRAM)
curl -s http://localhost:3014/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "ornith-35b-q6",
"messages": [{"role": "user", "content": "Bonjour, es-tu fonctionnel ?"}],
"max_tokens": 50
}' | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('choices',[{}])[0].get('message',{}).get('content','ERREUR:'+json.dumps(d))[:300])"
# Puis tester un modèle Ollama (doit déclencher le switch inverse)
curl -s http://localhost:3014/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "mistral",
"messages": [{"role": "user", "content": "Bonjour, es-tu fonctionnel ?"}],
"max_tokens": 50
}' | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('choices',[{}])[0].get('message',{}).get('content','ERREUR:'+json.dumps(d))[:300])"
```
---
## 6. Synthèse
| Check | Méthode |
|-------|---------|
| `/v1/models` a `max_tokens` + `max_input_tokens` | curl + jq |
| `/v1/models` a `capabilities` non-vide | curl + jq |
| Preset "Ornith 1.0 35b Q4 MOE" existe avec champs valides | curl + jq |
| Modèle `vision=true` accepte image sans erreur | curl + image base64 |
| Modèle `vision=false` rejette image | curl + image base64 |
| Modèle `thinking=true` montre raisonnement structuré | curl + analyse réponse |
| Switch Jean→Ollama fonctionne | 2 requêtes séquentielles |
| `?refresh=true` force la mise à jour du cache | 2 requêtes avec timestamp |