From 7a8dc342125efdee847e7e9dde5126d5322d71f0 Mon Sep 17 00:00:00 2001 From: Folzo Date: Sat, 27 Jun 2026 16:40:24 +0200 Subject: [PATCH] feat(litellm): 3 profils Qwen3.6 (reason/fast/default) via toggle enable_thinking Contrainte VRAM: RX 7800 XT 16Go ne peut pas co-heberger Qwen3.6-27B (14.6 Go) + llama3.1-8b (5 Go). Ollama commente, on expose 3 profils du meme modele Qwen3.6 via le toggle chat_template_kwargs.enable_thinking: - qwen3.6-reason / qwen3.6-27b : raisonnement ON (taches complexes) - qwen3.6-fast / qwen3.6 : raisonnement OFF (chat rapide ~400ms TTFT) drop_params=false pour passer chat_template_kwargs a Jean. Teste en prod: qwen3.6-fast repond BONJOUR en 400ms, MTP 3/3 acceptes. --- litellm/config.yaml | 96 ++++++++++++++++++++++++++++++--------------- 1 file changed, 65 insertions(+), 31 deletions(-) diff --git a/litellm/config.yaml b/litellm/config.yaml index 53e1bb2..034ad82 100644 --- a/litellm/config.yaml +++ b/litellm/config.yaml @@ -1,57 +1,91 @@ # ═══════════════════════════════════════════════════════════ -# LiteLLM Proxy — Routing multi-backends pour Effi -# Expose un endpoint OpenAI-compatible sur :4000 +# LiteLLM Proxy — Routing pour Effi +# Endpoint OpenAI-compatible sur :4000 (host network) # ‎ -# Backends : -# 1. Jean (llama.cpp/ROCm sur PC gaming .11:8080) → modele "qwen3.6-27b" -# 2. Ollama (PC gaming .11:11434 via wol-proxy :3011) → modele "llama3.1-8b" (+ autres) +# Backend unique : Jean (llama.cpp/ROCm sur PC gaming .11:8080) +# via jean-proxy :3012 + WOL auto si PC eteint. # ‎ -# Les deux backends sont sur le meme PC gaming (192.168.1.11) -# qui s eteint la nuit. Les proxies (wol-proxy + jean-proxy) declenchent -# le WOL si besoin avec la meme adresse MAC cible. +# CONTRAINTE VRAM : la RX 7800 XT a 16 Go. Le modele Qwen3.6-27B IQ4_XS +# occupe ~15.3 Go. Ollama ne peut PAS cohabiter (llama3.1-8b = ~5 Go +# supplémentaires -> OOM). On abandonne Ollama et on expose 3 profils +# du meme modele Qwen3.6, avec ou sans raisonnement (toggle natif +# chat_template_kwargs.enable_thinking): +# - qwen3.6-reason : raisonnement ON (tâches complexes, code, analyse) +# - qwen3.6-fast : raisonnement OFF (chat rapide, mail, extraction simple) +# - qwen3.6-27b : alias (= reason, défaut) +# ‎ +# Latence observee (campagne de tests 2026-06-27) : +# reason : TTFT 11-38s, 60-75 tok/s (le modele reflechit avant de repondre) +# fast : TTFT 0.4-1s, 60-75 tok/s (reponse quasi immediate) # ═══════════════════════════════════════════════════════════ model_list: # ═════════════════════════════════════════════════════════ - # Backend 1 — Jean (llama.cpp/v1 OpenAI-compatible) + # Profil 1 — Raisonnement ON (défaut, pour tâches complexes) + # Le modele reflechit (jusqu'a 2048 tokens de reasoning_content) + # avant de produire sa reponse. Bon pour code, analyse, RAG, rapports. # ═════════════════════════════════════════════════════════ + - model_name: qwen3.6-reason + litellm_params: + model: openai/qwen3.6-reason + api_base: http://localhost:3012/v1 + api_key: "null" + timeout: 600 + extra_body: + chat_template_kwargs: + enable_thinking: true + - model_name: qwen3.6-27b litellm_params: model: openai/qwen3.6-27b api_base: http://localhost:3012/v1 api_key: "null" timeout: 600 - # Alias court pour open-webui (sans version) - - model_name: qwen3.6 + extra_body: + chat_template_kwargs: + enable_thinking: true + + # ═════════════════════════════════════════════════════════ + # Profil 2 — Raisonnement OFF (chat rapide, ~5x moins de latence) + # Reponse immediate (TTFT <1s) sans phase de reflexion. + # Ideal pour : chat interactif, redaction de mail, extraction + # structuree simple, Q/R directes. + # ═════════════════════════════════════════════════════════ + - model_name: qwen3.6-fast litellm_params: - model: openai/qwen3.6-27b + model: openai/qwen3.6-fast api_base: http://localhost:3012/v1 api_key: "null" timeout: 600 + extra_body: + chat_template_kwargs: + enable_thinking: false + + # Alias court "qwen3.6" = fast par defaut (usage chat le plus courant) + - model_name: qwen3.6 + litellm_params: + model: openai/qwen3.6 + api_base: http://localhost:3012/v1 + api_key: "null" + timeout: 600 + extra_body: + chat_template_kwargs: + enable_thinking: false # ═════════════════════════════════════════════════════════ - # Backend 2 — Ollama (via wol-proxy :3011 + WOL auto) + # Backend Ollama — COMMENTE (contention VRAM, voir entete) + # Pour reactiver si dispo (autre machine / GPU dedie), decommenter + # et s'assurer que le GPU peut charger les deux modeles. # ═════════════════════════════════════════════════════════ - - model_name: llama3.1-8b - litellm_params: - model: ollama/llama3.1:8b - api_base: http://localhost:3011 - timeout: 600 - - - model_name: llama3.1 - litellm_params: - model: ollama/llama3.1:8b - api_base: http://localhost:3011 - timeout: 600 - -# ═════════════════════════════════════════════════════════ -# Ollama "fallback" auto : tout modele ollama/* est route vers wol-proxy -# (laisse open-webui lister les autres modeles ollama si besoin) -# ═════════════════════════════════════════════════════════ + # - model_name: llama3.1-8b + # litellm_params: + # model: ollama/llama3.1:8b + # api_base: http://localhost:3011 + # timeout: 600 litellm_settings: - drop_params: true + drop_params: false # ne pas dropper: on veut passer chat_template_kwargs a Jean telemetry: false general_settings: - master_key: sk-effi-litellm-master + master_key: sk-effi-litellm-master \ No newline at end of file