Effi/effi-crawl4ai/update-effiserver.sh

#!/bin/bash
# Script de mise à jour du Web Scraper sur effi-server
# À exécuter dans /root/effi/

set -e

echo "================================"
echo "Mise à jour Web Scraper"
echo "================================"
echo ""

# 1. Nettoyer l'ancien conteneur standalone
echo "🧹 Nettoyage de l'ancien conteneur..."
docker stop effi-web-scraper 2>/dev/null || true
docker rm effi-web-scraper 2>/dev/null || true

# 2. Nettoyer le réseau si créé
echo "🗑️  Suppression du réseau effi-net temporaire..."
docker network rm effi-net 2>/dev/null || true

# 3. Supprimer l'ancienne section crawl4ai du docker-compose
echo "📝 Mise à jour du docker-compose.yml..."

# Créer le service web-scraper à insérer
cat > /tmp/web-scraper-service.txt << 'EOF'
  # ─────────────────────────────────────────────────────────
  # 8. WEB SCRAPER - Scraping pour OpenWebUI
  #    → http://localhost:11235
  # ─────────────────────────────────────────────────────────
  web-scraper:
    build: ./web-scraper
    container_name: effi-web-scraper
    restart: unless-stopped
    ports:
      - "11235:11235"
    networks:
      - effi-net

EOF

# Backup du docker-compose
cp docker-compose.yml docker-compose.yml.backup.$(date +%Y%m%d_%H%M%S)

# Utiliser awk pour remplacer la section 8 (crawl4ai) par web-scraper
awk '
/^networks:/ {
    # On est à la fin du fichier services, inserer le nouveau service avant
    system("cat /tmp/web-scraper-service.txt")
}
{ print }
' docker-compose.yml > /tmp/docker-compose-new.yml

mv /tmp/docker-compose-new.yml docker-compose.yml

echo "✅ Section #8 mise à jour (crawl4ai → web-scraper)"

# 4. Créer le répertoire et les fichiers du web-scraper
echo ""
echo "📁 Création du répertoire web-scraper..."
mkdir -p web-scraper

cat > web-scraper/Dockerfile << 'EOF'
FROM python:3.11-slim

WORKDIR /app

RUN apt-get update && apt-get install -y --no-install-recommends \
    curl \
    && rm -rf /var/lib/apt/lists/*

RUN pip install --no-cache-dir fastapi uvicorn requests beautifulsoup4

COPY scraper.py .

EXPOSE 11235

HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
    CMD curl -f http://localhost:11235/health || exit 1

CMD ["uvicorn", "scraper:app", "--host", "0.0.0.0", "--port", "11235"]
EOF

cat > web-scraper/scraper.py << 'EOF'
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from typing import Optional, List
import requests
from bs4 import BeautifulSoup

app = FastAPI(title="Effi Web Scraper", version="1.0.0")

class ScrapeRequest(BaseModel):
    url: str
    max_length: Optional[int] = 10000

class ScrapeResponse(BaseModel):
    url: str
    title: str
    content: str
    links: List[str]
    success: bool
    error: Optional[str] = None

@app.get("/health")
def health():
    return {"status": "healthy"}

@app.post("/scrape", response_model=ScrapeResponse)
def scrape(request: ScrapeRequest):
    try:
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        }
        
        response = requests.get(request.url, headers=headers, timeout=30)
        response.raise_for_status()
        
        soup = BeautifulSoup(response.content, 'html.parser')
        
        title = soup.title.string if soup.title else "No title"
        
        for script in soup(["script", "style"]):
            script.decompose()
        
        main_content = None
        for selector in ['main', 'article', '[role="main"]', '.content', '#content', '.post-content']:
            main_content = soup.select_one(selector)
            if main_content:
                break
        
        if not main_content:
            main_content = soup.body if soup.body else soup
        
        text = main_content.get_text(separator='\n', strip=True)
        lines = [line.strip() for line in text.split('\n') if line.strip()]
        text = '\n'.join(lines)
        
        if len(text) > request.max_length:
            text = text[:request.max_length] + f"\n\n[...] {len(text) - request.max_length} caractères supplémentaires"
        
        links = []
        for a in soup.find_all('a', href=True):
            href = a['href']
            if href.startswith('http'):
                links.append(href)
        links = links[:20]
        
        return ScrapeResponse(
            url=request.url,
            title=title,
            content=text,
            links=links,
            success=True
        )
        
    except Exception as e:
        return ScrapeResponse(
            url=request.url,
            title="Error",
            content="",
            links=[],
            success=False,
            error=str(e)
        )

@app.post("/scrape/sync")
def scrape_sync(url: str, max_length: int = 10000):
    request = ScrapeRequest(url=url, max_length=max_length)
    result = scrape(request)
    if not result.success:
        raise HTTPException(status_code=400, detail=result.error)
    return {
        "url": result.url,
        "html": result.content,
        "markdown": result.content,
        "success": True
    }

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(app, host="0.0.0.0", port=11235)
EOF

echo "✅ Fichiers créés dans web-scraper/"

# 5. Relancer docker-compose
echo ""
echo "🚀 Démarrage du service web-scraper..."
docker-compose up -d web-scraper

echo ""
echo "⏳ Attente du démarrage (30s)..."
sleep 30

# 6. Vérification
echo ""
echo "🔍 Vérification..."
if curl -s http://localhost:11235/health > /dev/null 2>&1; then
    echo "✅ Web Scraper est fonctionnel !"
    curl http://localhost:11235/health
else
    echo "⚠️  Le service ne répond pas encore"
    docker logs effi-web-scraper
fi

echo ""
echo "================================"
echo "Installation terminée !"
echo "================================"
echo ""
echo "📋 Configuration OpenWebUI:"
echo "   Admin → Tools → Web Search and Crawl → ⚙️"
echo "   Crawl4AI Base URL: http://localhost:11235"
echo "   Timeout: 30000"
echo "   ✅ Activer le Tool"