refactor: LiteLLM unified routing via JeanOrchestrator
- All models now route through single jean-proxy:3012 → orchestrator:8080 - Removed jean-proxy-moe (no longer needed with auto-switch orchestrator) - Added qwen3.6-moe and qwen3.6-moe-reason model routes - Updated architecture comments
This commit is contained in:
@@ -29,8 +29,8 @@ services:
|
|||||||
- ENABLE_OPENAI_API=True
|
- ENABLE_OPENAI_API=True
|
||||||
|
|
||||||
# ─────────────────────────────────────────────────────────
|
# ─────────────────────────────────────────────────────────
|
||||||
# 1. jean-proxy — Reverse-proxy vers Jean (.11:8080) + WOL
|
# jean-proxy — Reverse-proxy vers JeanOrchestrator (.11:8080) + WOL
|
||||||
# Clone du wol-proxy, meme MAC cible, port cible = 8080
|
# L'orchestrateur gere le switch auto MoE/27B.
|
||||||
# Ecoute :3012 sur le host (network_mode: host)
|
# Ecoute :3012 sur le host (network_mode: host)
|
||||||
# ─────────────────────────────────────────────────────────
|
# ─────────────────────────────────────────────────────────
|
||||||
jean-proxy:
|
jean-proxy:
|
||||||
@@ -46,6 +46,7 @@ services:
|
|||||||
- WAKE_TIMEOUT=60
|
- WAKE_TIMEOUT=60
|
||||||
- CHECK_INTERVAL=5
|
- CHECK_INTERVAL=5
|
||||||
- BROADCAST_IP=${BROADCAST_IP}
|
- BROADCAST_IP=${BROADCAST_IP}
|
||||||
|
- HEALTH_PATH=/health
|
||||||
|
|
||||||
# ─────────────────────────────────────────────────────────
|
# ─────────────────────────────────────────────────────────
|
||||||
# 2. LiteLLM proxy — Endpoint OpenAI unique pour open-webui
|
# 2. LiteLLM proxy — Endpoint OpenAI unique pour open-webui
|
||||||
|
|||||||
+33
-12
@@ -2,21 +2,16 @@
|
|||||||
# LiteLLM Proxy — Routing pour Effi
|
# LiteLLM Proxy — Routing pour Effi
|
||||||
# Endpoint OpenAI-compatible sur :4000 (host network)
|
# Endpoint OpenAI-compatible sur :4000 (host network)
|
||||||
#
|
#
|
||||||
# Backend unique : Jean (llama.cpp/ROCm sur PC gaming .11:8080)
|
# Backend unique : JeanOrchestrator (port 8080 sur PC gaming .11)
|
||||||
# via jean-proxy :3012 + WOL auto si PC eteint.
|
# via jean-proxy :3012 + WOL auto si PC eteint.
|
||||||
#
|
#
|
||||||
# CONTRAINTE VRAM : la RX 7800 XT a 16 Go. Le modele Qwen3.6-27B IQ4_XS
|
# L'orchestrateur gere automatiquement le switch entre :
|
||||||
# occupe ~15.3 Go. Ollama ne peut PAS cohabiter (llama3.1-8b = ~5 Go
|
# - Qwen3.6-35B MoE I-Nano (defaut, 11.7 Go, 100k ctx, ~70 tok/s)
|
||||||
# supplémentaires -> OOM). On abandonne Ollama et on expose 3 profils
|
# - Qwen3.6-27B IQ4_XS (code/analyse, 14.6 Go, 55k ctx, ~60 tok/s)
|
||||||
# du meme modele Qwen3.6, avec ou sans raisonnement (toggle natif
|
|
||||||
# chat_template_kwargs.enable_thinking):
|
|
||||||
# - qwen3.6-reason : raisonnement ON (tâches complexes, code, analyse)
|
|
||||||
# - qwen3.6-fast : raisonnement OFF (chat rapide, mail, extraction simple)
|
|
||||||
# - qwen3.6-27b : alias (= reason, défaut)
|
|
||||||
#
|
#
|
||||||
# Latence observee (campagne de tests 2026-06-27) :
|
# Tous les modeles passent par le meme proxy :3012.
|
||||||
# reason : TTFT 11-38s, 60-75 tok/s (le modele reflechit avant de repondre)
|
# L'orchestrateur detecte le modele demande et charge le bon LLM.
|
||||||
# fast : TTFT 0.4-1s, 60-75 tok/s (reponse quasi immediate)
|
# Retour automatique au MoE apres 15 min sans 27B.
|
||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
|
||||||
model_list:
|
model_list:
|
||||||
@@ -72,6 +67,32 @@ model_list:
|
|||||||
chat_template_kwargs:
|
chat_template_kwargs:
|
||||||
enable_thinking: false
|
enable_thinking: false
|
||||||
|
|
||||||
|
# ═════════════════════════════════════════════════════════
|
||||||
|
# Profil 3 — Qwen3.6-35B MoE APEX I-Nano (chat Hermes, usage quotidien)
|
||||||
|
# Modele MoE : 35B total / 3B actifs, 11.7 Go, cache KV 5x plus petit
|
||||||
|
# que le 27B -> contexte 100k possible. Cohabite avec ComfyUI.
|
||||||
|
# Route via le meme proxy :3012 que le 27B (orchestrateur fait le switch).
|
||||||
|
# ═════════════════════════════════════════════════════════
|
||||||
|
- model_name: qwen3.6-moe
|
||||||
|
litellm_params:
|
||||||
|
model: openai/qwen3.6-moe
|
||||||
|
api_base: http://localhost:3012/v1
|
||||||
|
api_key: "null"
|
||||||
|
timeout: 600
|
||||||
|
extra_body:
|
||||||
|
chat_template_kwargs:
|
||||||
|
enable_thinking: false
|
||||||
|
|
||||||
|
- model_name: qwen3.6-moe-reason
|
||||||
|
litellm_params:
|
||||||
|
model: openai/qwen3.6-moe-reason
|
||||||
|
api_base: http://localhost:3012/v1
|
||||||
|
api_key: "null"
|
||||||
|
timeout: 600
|
||||||
|
extra_body:
|
||||||
|
chat_template_kwargs:
|
||||||
|
enable_thinking: true
|
||||||
|
|
||||||
# ═════════════════════════════════════════════════════════
|
# ═════════════════════════════════════════════════════════
|
||||||
# Backend Ollama — COMMENTE (contention VRAM, voir entete)
|
# Backend Ollama — COMMENTE (contention VRAM, voir entete)
|
||||||
# Pour reactiver si dispo (autre machine / GPU dedie), decommenter
|
# Pour reactiver si dispo (autre machine / GPU dedie), decommenter
|
||||||
|
|||||||
Reference in New Issue
Block a user