jean-router: resolution dynamique des context_length + max_tokens + capabilities
- Suppression du preset_ctx_map code en dur - Resolution dynamique: Jean /v1/models → matching Ollama → heuristique - Ajout max_tokens / max_input_tokens dans /v1/models - Enrichissement Ollama via /api/show pour donnees manquantes - Detection capabilities (vision, thinking) pour presets Jean - Auto-doublement max_tokens pour presets reasoning=on - Timeout switch augmente a 15s avec retry - Tracking du dernier preset actif (bypass bug Jean active=false) - Cache TTL configurable (15s par defaut), ?refresh=true - Plan de test des capacites LLM
This commit is contained in:
Folzo
+282
-48
@@ -38,11 +38,15 @@ app = Flask(__name__)
|
||||
|
||||
presets_cache = {"data": [], "timestamp": 0}
|
||||
ollama_cache = {"data": [], "timestamp": 0}
|
||||
ollama_detail_cache = {}
|
||||
jean_ctx_cache = {}
|
||||
CACHE_TTL = 30
|
||||
jean_caps_cache = {}
|
||||
CACHE_TTL = int(os.getenv("CACHE_TTL", "15"))
|
||||
OLLAMA_DETAIL_CACHE_TTL = 300
|
||||
switch_lock = threading.Lock()
|
||||
vram_lock = threading.Lock()
|
||||
vram_state = {"active_backend": "jean"}
|
||||
last_active_preset = None
|
||||
ollama_unload_timer = None
|
||||
|
||||
|
||||
@@ -88,36 +92,213 @@ def get_ollama_models(force_refresh=False):
|
||||
return ollama_cache["data"]
|
||||
|
||||
|
||||
def get_jean_context_lengths():
|
||||
"""Récupère les context_length depuis Jean /v1/models (avec cache)"""
|
||||
preset_ctx_map = {
|
||||
"ornith-35b-q6": 64000,
|
||||
"qwen3.6-27b-mtp": 55000,
|
||||
"qwen3.6-35b-moe-compact": 131072,
|
||||
"qwen3.6-35b-moe-compact-reason": 131072,
|
||||
"qwen3.6-35b-moe-nano": 100000,
|
||||
}
|
||||
def _get_ollama_model_detail(model_name: str) -> dict:
|
||||
"""Récupère les détails d'un modèle Ollama via /api/show, avec cache long"""
|
||||
now = time.time()
|
||||
cached = ollama_detail_cache.get(model_name)
|
||||
if cached and (now - cached["timestamp"]) < OLLAMA_DETAIL_CACHE_TTL:
|
||||
return cached["data"]
|
||||
|
||||
detail = {}
|
||||
try:
|
||||
clean_name = model_name.replace(":latest", "")
|
||||
url = f"http://{OLLAMA_DIRECT_HOST}:{OLLAMA_DIRECT_PORT}/api/show"
|
||||
r = requests.post(url, json={"model": clean_name}, timeout=10)
|
||||
if r.ok:
|
||||
d = r.json()
|
||||
model_info = d.get("model_info", {})
|
||||
details = d.get("details", {})
|
||||
|
||||
for key, val in model_info.items():
|
||||
if "context_length" in key.lower() or "ctx" in key.lower():
|
||||
detail["context_length"] = int(val)
|
||||
break
|
||||
|
||||
if not detail.get("context_length"):
|
||||
detail["context_length"] = details.get("context_length")
|
||||
|
||||
detail["capabilities"] = d.get("capabilities", [])
|
||||
detail["family"] = details.get("family", "")
|
||||
|
||||
ollama_detail_cache[model_name] = {"data": detail, "timestamp": now}
|
||||
except Exception as e:
|
||||
log.debug(f"[Ollama] /api/show echoue pour {model_name}: {e}")
|
||||
|
||||
if model_name not in ollama_detail_cache:
|
||||
ollama_detail_cache[model_name] = {"data": {}, "timestamp": now}
|
||||
|
||||
return ollama_detail_cache[model_name]["data"]
|
||||
|
||||
|
||||
def _enrich_ollama_model(m: dict) -> dict:
|
||||
"""Enrichit un modele Ollama avec les donnees /api/show si necessaire"""
|
||||
model_id = m.get("name", "").replace(":latest", "")
|
||||
details = m.get("details", {})
|
||||
has_ctx = bool(details.get("context_length"))
|
||||
caps = m.get("capabilities", [])
|
||||
|
||||
if not has_ctx or len(caps) <= 1:
|
||||
extra = _get_ollama_model_detail(model_id)
|
||||
if extra.get("context_length"):
|
||||
details = dict(details)
|
||||
details["context_length"] = extra["context_length"]
|
||||
m = dict(m)
|
||||
m["details"] = details
|
||||
if extra.get("capabilities") and len(extra["capabilities"]) > len(caps):
|
||||
m = dict(m)
|
||||
m["capabilities"] = extra["capabilities"]
|
||||
if extra.get("family") and not details.get("family"):
|
||||
details = dict(details)
|
||||
details["family"] = extra["family"]
|
||||
m = dict(m)
|
||||
m["details"] = details
|
||||
|
||||
return m
|
||||
|
||||
def _guess_context_length(preset_id: str) -> int:
|
||||
"""Devine le context_length depuis le nom du preset"""
|
||||
pid = preset_id.lower()
|
||||
if "qwen3.6" in pid or "qwen3.5" in pid:
|
||||
return 262144
|
||||
if "qwen3" in pid:
|
||||
return 40960
|
||||
if "qwen2.5" in pid:
|
||||
return 32768
|
||||
if "mistral" in pid or "nemo" in pid:
|
||||
return 1024000
|
||||
if "llama" in pid:
|
||||
return 131072
|
||||
if "ornith" in pid:
|
||||
return 131072
|
||||
if "gemma" in pid:
|
||||
return 262144
|
||||
if "deepseek" in pid:
|
||||
return 131072
|
||||
if "phi" in pid:
|
||||
return 16384
|
||||
if "35b" in pid or "27b" in pid:
|
||||
return 131072
|
||||
return 32768
|
||||
|
||||
|
||||
def _guess_capabilities(preset_id: str) -> list:
|
||||
"""Devine les capabilities depuis le nom du preset + metadonnees Jean"""
|
||||
pid = preset_id.lower()
|
||||
caps = ["completion"]
|
||||
|
||||
has_vision = any(kw in pid for kw in ["vision", "vl", "llava", "bakllava", "minicpm"])
|
||||
has_thinking = any(kw in pid for kw in ["reason", "thinking", "deepseek-r1"])
|
||||
has_tools = not any(kw in pid for kw in ["embed", "bge-", "nomic-"])
|
||||
|
||||
if has_vision:
|
||||
caps.append("vision")
|
||||
if has_thinking:
|
||||
caps.append("thinking")
|
||||
if has_tools:
|
||||
caps.append("tools")
|
||||
|
||||
return caps
|
||||
|
||||
|
||||
def _match_preset_to_ollama(preset_id: str):
|
||||
"""Tente de trouver un modele Ollama correspondant au preset Jean"""
|
||||
ollama_models = get_ollama_models()
|
||||
pid_lower = preset_id.lower()
|
||||
|
||||
def norm(s):
|
||||
return s.lower().replace(":", "").replace("-", "").replace(" ", "").replace(".", "")
|
||||
|
||||
pid_norm = norm(pid_lower)
|
||||
|
||||
for m in ollama_models:
|
||||
name = m.get("name", "").replace(":latest", "")
|
||||
name_norm = norm(name)
|
||||
|
||||
if pid_norm == name_norm:
|
||||
return m
|
||||
|
||||
best = None
|
||||
best_score = 0
|
||||
for m in ollama_models:
|
||||
name = m.get("name", "").replace(":latest", "")
|
||||
name_norm = norm(name)
|
||||
score = 0
|
||||
|
||||
if len(pid_norm) >= 4 and pid_norm[:4] == name_norm[:4]:
|
||||
score += 10
|
||||
if len(pid_norm) >= 8 and pid_norm[:8] == name_norm[:8]:
|
||||
score += 20
|
||||
if name_norm in pid_norm or pid_norm in name_norm:
|
||||
score += 15
|
||||
|
||||
parts_pid = set(pid_norm.split("_"))
|
||||
parts_name = set(name_norm.split("_"))
|
||||
common = parts_pid & parts_name
|
||||
score += len(common) * 5
|
||||
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best = m
|
||||
|
||||
if best and best_score >= 5:
|
||||
return best
|
||||
return None
|
||||
|
||||
|
||||
def get_jean_metadata():
|
||||
"""Resout dynamiquement context_length et capabilities pour les presets Jean.
|
||||
1) Ollama matching → 2) Jean /v1/models → 3) heuristique"""
|
||||
now = time.time()
|
||||
|
||||
try:
|
||||
url = f"http://{JEAN_PROXY_HOST}:{JEAN_PROXY_PORT}/v1/models"
|
||||
r = requests.get(url, timeout=2)
|
||||
if r.ok:
|
||||
models = r.json().get("data", [])
|
||||
for m in models:
|
||||
for m in r.json().get("data", []):
|
||||
alias = m.get("id", "")
|
||||
meta = m.get("meta", {})
|
||||
n_ctx = meta.get("n_ctx")
|
||||
if n_ctx:
|
||||
alias = m.get("id", "")
|
||||
jean_ctx_cache[alias] = n_ctx
|
||||
log.debug(f"[Jean] Context length pour {alias}: {n_ctx}")
|
||||
except Exception as e:
|
||||
log.debug(f"[Jean] Impossible de récupérer context_length: {e}")
|
||||
n_ctx = meta.get("n_ctx") or m.get("max_model_len")
|
||||
if n_ctx and alias:
|
||||
jean_ctx_cache[alias] = int(n_ctx)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for preset_id, ctx in preset_ctx_map.items():
|
||||
if preset_id not in jean_ctx_cache:
|
||||
jean_ctx_cache[preset_id] = ctx
|
||||
ollama_models = get_ollama_models()
|
||||
ollama_lookup = {}
|
||||
for m in ollama_models:
|
||||
m = _enrich_ollama_model(m)
|
||||
name = m.get("name", "").replace(":latest", "")
|
||||
details = m.get("details", {})
|
||||
ctx = details.get("context_length")
|
||||
caps = m.get("capabilities", [])
|
||||
ollama_lookup[name] = {"context_length": ctx, "capabilities": caps}
|
||||
|
||||
return jean_ctx_cache
|
||||
presets = get_presets()
|
||||
for p in presets:
|
||||
pid = p["id"]
|
||||
|
||||
matched_ollama = _match_preset_to_ollama(pid)
|
||||
ollama_name = matched_ollama.get("name", "").replace(":latest", "") if matched_ollama else None
|
||||
|
||||
if pid not in jean_ctx_cache:
|
||||
if ollama_name and ollama_lookup.get(ollama_name, {}).get("context_length"):
|
||||
jean_ctx_cache[pid] = ollama_lookup[ollama_name]["context_length"]
|
||||
else:
|
||||
jean_ctx_cache[pid] = _guess_context_length(pid)
|
||||
|
||||
if pid not in jean_caps_cache:
|
||||
if p.get("reasoning") == "on":
|
||||
caps = ["completion", "tools", "thinking"]
|
||||
else:
|
||||
caps = ["completion", "tools"]
|
||||
|
||||
if ollama_name and ollama_lookup.get(ollama_name, {}).get("capabilities"):
|
||||
ollama_caps = ollama_lookup[ollama_name]["capabilities"]
|
||||
caps = list(dict.fromkeys(caps + ollama_caps))
|
||||
|
||||
jean_caps_cache[pid] = caps
|
||||
|
||||
return jean_ctx_cache, jean_caps_cache
|
||||
|
||||
|
||||
def get_active_preset():
|
||||
@@ -131,6 +312,7 @@ def get_active_preset():
|
||||
|
||||
def switch_preset(preset_id: str) -> bool:
|
||||
"""Switch vers un preset spécifique par son ID"""
|
||||
global last_active_preset
|
||||
presets = get_presets(force_refresh=True)
|
||||
|
||||
for i, p in enumerate(presets):
|
||||
@@ -140,14 +322,20 @@ def switch_preset(preset_id: str) -> bool:
|
||||
return True
|
||||
|
||||
log.info(f"[Switch] Activation preset {preset_id} (index {i+1})")
|
||||
try:
|
||||
url = f"http://{JEAN_WEB_HOST}:{JEAN_WEB_PORT}/api/switch"
|
||||
r = requests.post(url, json={"n": i + 1}, timeout=5)
|
||||
if not r.ok:
|
||||
log.error(f"[Switch] Erreur API: {r.text}")
|
||||
return False
|
||||
except Exception as e:
|
||||
log.error(f"[Switch] Erreur requête: {e}")
|
||||
success = False
|
||||
for attempt in range(2):
|
||||
try:
|
||||
url = f"http://{JEAN_WEB_HOST}:{JEAN_WEB_PORT}/api/switch"
|
||||
r = requests.post(url, json={"n": i + 1}, timeout=15)
|
||||
if r.ok:
|
||||
success = True
|
||||
break
|
||||
log.error(f"[Switch] Erreur API (tentative {attempt+1}): {r.text}")
|
||||
except Exception as e:
|
||||
log.error(f"[Switch] Erreur requête (tentative {attempt+1}): {e}")
|
||||
if attempt == 0:
|
||||
time.sleep(3)
|
||||
if not success:
|
||||
return False
|
||||
|
||||
log.info(f"[Switch] Attente chargement modèle...")
|
||||
@@ -163,6 +351,7 @@ def switch_preset(preset_id: str) -> bool:
|
||||
models = mr.json().get("data", [])
|
||||
if models:
|
||||
log.info(f"[Switch] Modèle chargé après {int(time.time() - deadline + SWITCH_TIMEOUT)}s")
|
||||
last_active_preset = preset_id
|
||||
presets_cache["timestamp"] = 0
|
||||
return True
|
||||
except Exception:
|
||||
@@ -320,6 +509,8 @@ def ensure_vram_for_backend(target_backend: str) -> bool:
|
||||
if not ENABLE_VRAM_MANAGEMENT:
|
||||
return True
|
||||
|
||||
global last_active_preset
|
||||
|
||||
with vram_lock:
|
||||
current = vram_state["active_backend"]
|
||||
|
||||
@@ -331,6 +522,7 @@ def ensure_vram_for_backend(target_backend: str) -> bool:
|
||||
cancel_ollama_unload()
|
||||
if not stop_jean():
|
||||
return False
|
||||
last_active_preset = None
|
||||
vram_state["active_backend"] = "ollama"
|
||||
return True
|
||||
|
||||
@@ -339,6 +531,7 @@ def ensure_vram_for_backend(target_backend: str) -> bool:
|
||||
schedule_ollama_unload()
|
||||
if not start_jean():
|
||||
return False
|
||||
last_active_preset = None
|
||||
vram_state["active_backend"] = "jean"
|
||||
return True
|
||||
|
||||
@@ -348,13 +541,24 @@ def ensure_vram_for_backend(target_backend: str) -> bool:
|
||||
@app.route("/v1/models", methods=["GET"])
|
||||
def list_models():
|
||||
"""Expose tous les presets Jean + modèles Ollama comme modèles OpenAI"""
|
||||
force_refresh = request.args.get("refresh", "").lower() in ("1", "true", "yes")
|
||||
if force_refresh:
|
||||
global presets_cache, ollama_cache
|
||||
presets_cache["timestamp"] = 0
|
||||
ollama_cache["timestamp"] = 0
|
||||
log.info("[Models] Refresh force demande")
|
||||
|
||||
presets = get_presets()
|
||||
jean_ctx = get_jean_context_lengths()
|
||||
jean_ctx, jean_caps = get_jean_metadata()
|
||||
models = []
|
||||
|
||||
for p in presets:
|
||||
model_id = p["id"]
|
||||
model_name = p.get("name", model_id)
|
||||
|
||||
ctx_len = jean_ctx.get(model_id)
|
||||
caps = jean_caps.get(model_id, ["completion", "tools"])
|
||||
if p.get("reasoning") == "on" and "thinking" not in caps:
|
||||
caps = list(dict.fromkeys(caps + ["thinking"]))
|
||||
|
||||
model_info = {
|
||||
"id": model_id,
|
||||
@@ -366,7 +570,18 @@ def list_models():
|
||||
"parent": None,
|
||||
}
|
||||
|
||||
if ctx_len:
|
||||
model_info["context_length"] = ctx_len
|
||||
model_info["context_window"] = ctx_len
|
||||
model_info["max_model_len"] = ctx_len
|
||||
model_info["max_tokens"] = ctx_len
|
||||
model_info["max_input_tokens"] = ctx_len
|
||||
|
||||
model_info["capabilities"] = caps
|
||||
|
||||
meta = {}
|
||||
if p.get("name") and p["name"] != model_id:
|
||||
meta["display_name"] = p["name"]
|
||||
if p.get("quant"):
|
||||
meta["quantization"] = p["quant"]
|
||||
if p.get("reasoning"):
|
||||
@@ -375,13 +590,9 @@ def list_models():
|
||||
meta["active"] = True
|
||||
if p.get("bench"):
|
||||
meta["bench"] = p["bench"]
|
||||
|
||||
ctx_len = jean_ctx.get(model_id)
|
||||
if ctx_len:
|
||||
meta["context_length"] = ctx_len
|
||||
model_info["context_length"] = ctx_len
|
||||
model_info["context_window"] = ctx_len
|
||||
model_info["max_model_len"] = ctx_len
|
||||
meta["capabilities"] = caps
|
||||
|
||||
if meta:
|
||||
model_info["meta"] = meta
|
||||
@@ -391,10 +602,15 @@ def list_models():
|
||||
if ENABLE_OLLAMA:
|
||||
ollama_models = get_ollama_models()
|
||||
for m in ollama_models:
|
||||
m = _enrich_ollama_model(m)
|
||||
model_id = m.get("name", "").replace(":latest", "")
|
||||
if not model_id:
|
||||
continue
|
||||
|
||||
caps = m.get("capabilities", ["completion"])
|
||||
details = m.get("details", {})
|
||||
ctx_len = details.get("context_length")
|
||||
|
||||
model_info = {
|
||||
"id": model_id,
|
||||
"object": "model",
|
||||
@@ -405,7 +621,15 @@ def list_models():
|
||||
"parent": None,
|
||||
}
|
||||
|
||||
details = m.get("details", {})
|
||||
if ctx_len:
|
||||
model_info["context_length"] = ctx_len
|
||||
model_info["context_window"] = ctx_len
|
||||
model_info["max_model_len"] = ctx_len
|
||||
model_info["max_tokens"] = ctx_len
|
||||
model_info["max_input_tokens"] = ctx_len
|
||||
|
||||
model_info["capabilities"] = caps
|
||||
|
||||
meta = {}
|
||||
if details.get("parameter_size"):
|
||||
meta["parameter_size"] = details["parameter_size"]
|
||||
@@ -413,14 +637,9 @@ def list_models():
|
||||
meta["quantization"] = details["quantization_level"]
|
||||
if details.get("family"):
|
||||
meta["family"] = details["family"]
|
||||
if details.get("context_length"):
|
||||
ctx_len = details["context_length"]
|
||||
if ctx_len:
|
||||
meta["context_length"] = ctx_len
|
||||
model_info["context_length"] = ctx_len
|
||||
model_info["context_window"] = ctx_len
|
||||
model_info["max_model_len"] = ctx_len
|
||||
if m.get("capabilities"):
|
||||
meta["capabilities"] = m["capabilities"]
|
||||
meta["capabilities"] = caps
|
||||
|
||||
if meta:
|
||||
model_info["meta"] = meta
|
||||
@@ -448,10 +667,25 @@ def chat_completions():
|
||||
|
||||
with switch_lock:
|
||||
active = get_active_preset()
|
||||
if not active or active["id"] != model_id:
|
||||
log.info(f"[Router] Switch Jean requis: {active['id'] if active else 'none'} → {model_id}")
|
||||
current_active = active["id"] if active else last_active_preset
|
||||
if current_active != model_id:
|
||||
log.info(f"[Router] Switch Jean requis: {current_active or 'none'} → {model_id}")
|
||||
if not switch_preset(model_id):
|
||||
return jsonify({"error": f"Failed to switch to preset {model_id}"}), 503
|
||||
else:
|
||||
log.info(f"[Router] Preset {model_id} déjà actif (tracké)")
|
||||
|
||||
preset = next((p for p in presets if p["id"] == model_id), {})
|
||||
if preset.get("reasoning") == "on":
|
||||
ctx, _ = get_jean_metadata()
|
||||
ctx_len = ctx.get(model_id, 131072)
|
||||
req_max = data.get("max_tokens", 0)
|
||||
if req_max and req_max > 0:
|
||||
new_max = min(req_max * 2, ctx_len)
|
||||
if new_max != req_max:
|
||||
log.info(f"[Reasoning] max_tokens doublé: {req_max} → {new_max}")
|
||||
data = dict(data)
|
||||
data["max_tokens"] = new_max
|
||||
|
||||
upstream_url = f"http://{JEAN_PROXY_HOST}:{JEAN_PROXY_PORT}/v1/chat/completions"
|
||||
log.info(f"[Router] → Jean ({model_id})")
|
||||
|
||||
Reference in New Issue
Block a user