Effi/effi-crawl4ai/install-on-effiserver-v2.sh

#!/bin/bash
# Installation du Web Scraper sur effi-server
# À exécuter dans /root/effi/

set -e

echo "================================"
echo "Installation Web Scraper (Effi)"
echo "================================"
echo ""

# Vérifier qu'on est dans le bon répertoire
if [ ! -f "docker-compose.yml" ]; then
    echo "❌ Erreur: docker-compose.yml non trouvé"
    exit 1
fi

echo "📁 Répertoire: $(pwd)"

# Créer le répertoire pour le web scraper
mkdir -p web-scraper
cd web-scraper

echo "📝 Création des fichiers..."

# Dockerfile
cat > Dockerfile << 'EOF'
FROM python:3.11-slim

WORKDIR /app

RUN apt-get update && apt-get install -y --no-install-recommends \
    curl \
    && rm -rf /var/lib/apt/lists/*

RUN pip install --no-cache-dir fastapi uvicorn requests beautifulsoup4

COPY scraper.py .

EXPOSE 11235

HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
    CMD curl -f http://localhost:11235/health || exit 1

CMD ["uvicorn", "scraper:app", "--host", "0.0.0.0", "--port", "11235"]
EOF

# Scraper Python
cat > scraper.py << 'EOF'
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from typing import Optional, List
import requests
from bs4 import BeautifulSoup

app = FastAPI(title="Effi Web Scraper", version="1.0.0")

class ScrapeRequest(BaseModel):
    url: str
    max_length: Optional[int] = 10000

class ScrapeResponse(BaseModel):
    url: str
    title: str
    content: str
    links: List[str]
    success: bool
    error: Optional[str] = None

@app.get("/health")
def health():
    return {"status": "healthy"}

@app.post("/scrape", response_model=ScrapeResponse)
def scrape(request: ScrapeRequest):
    try:
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        }
        
        response = requests.get(request.url, headers=headers, timeout=30)
        response.raise_for_status()
        
        soup = BeautifulSoup(response.content, 'html.parser')
        
        title = soup.title.string if soup.title else "No title"
        
        for script in soup(["script", "style"]):
            script.decompose()
        
        main_content = None
        for selector in ['main', 'article', '[role="main"]', '.content', '#content', '.post-content']:
            main_content = soup.select_one(selector)
            if main_content:
                break
        
        if not main_content:
            main_content = soup.body if soup.body else soup
        
        text = main_content.get_text(separator='\n', strip=True)
        lines = [line.strip() for line in text.split('\n') if line.strip()]
        text = '\n'.join(lines)
        
        if len(text) > request.max_length:
            text = text[:request.max_length] + f"\n\n[...] {len(text) - request.max_length} caractères supplémentaires"
        
        links = []
        for a in soup.find_all('a', href=True):
            href = a['href']
            if href.startswith('http'):
                links.append(href)
        links = links[:20]
        
        return ScrapeResponse(
            url=request.url,
            title=title,
            content=text,
            links=links,
            success=True
        )
        
    except Exception as e:
        return ScrapeResponse(
            url=request.url,
            title="Error",
            content="",
            links=[],
            success=False,
            error=str(e)
        )

@app.post("/scrape/sync")
def scrape_sync(url: str, max_length: int = 10000):
    request = ScrapeRequest(url=url, max_length=max_length)
    result = scrape(request)
    if not result.success:
        raise HTTPException(status_code=400, detail=result.error)
    return {
        "url": result.url,
        "html": result.content,
        "markdown": result.content,
        "success": True
    }

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(app, host="0.0.0.0", port=11235)
EOF

echo "✅ Fichiers créés"
echo ""

# Construire et démarrer
echo "🔨 Construction de l'image..."
docker build -t effi-web-scraper .

echo ""
echo "🚀 Démarrage du service..."
docker run -d \
    --name effi-web-scraper \
    --restart unless-stopped \
    -p 11235:11235 \
    --network effi-net \
    effi-web-scraper

echo ""
echo "⏳ Attente du démarrage (15s)..."
sleep 15

# Vérification
echo ""
echo "🔍 Vérification..."
if curl -s http://localhost:11235/health > /dev/null 2>&1; then
    echo "✅ Web Scraper est démarré !"
    curl http://localhost:11235/health
else
    echo "⚠️  Le service ne répond pas encore"
    docker logs effi-web-scraper
fi

echo ""
echo "================================"
echo "Installation terminée !"
echo "================================"
echo ""
echo "📋 Configuration OpenWebUI:"
echo "   Admin → Tools → Web Search and Crawl → ⚙️"
echo "   Crawl4AI Base URL: http://localhost:11235"
echo "   Timeout: 30000"
echo ""