Ajout Effi-OCR (Tesseract) + Web Scraper pour OpenWebUI
- Effi-OCR: Service OCR avec Tesseract (multilingue FR/EN) * API FastAPI sur port 8300 * Prétraitement images (contraste, niveaux de gris) * Tool OpenWebUI intégré - Web Scraper: Service Python (BeautifulSoup) sur port 11235 * Alternative à Crawl4AI (incompatible ARM64/J5040) * Scraping HTML basique mais fonctionnel * Intégré au docker-compose d'effi-server - Documentation: * ROADMAP-FONCTIONNALITES.md: Plan des 9 fonctionnalités * WEB-SCRAPER-CONFIG.md: Configuration OpenWebUI * WEB-SCRAPER-TOOL.md: Code du Tool - Script start-all-services.sh mis à jour (6 services)
This commit is contained in:
Folzo
@@ -0,0 +1,215 @@
|
||||
#!/bin/bash
|
||||
# Script de mise à jour du Web Scraper sur effi-server
|
||||
# À exécuter dans /root/effi/
|
||||
|
||||
set -e
|
||||
|
||||
echo "================================"
|
||||
echo "Mise à jour Web Scraper"
|
||||
echo "================================"
|
||||
echo ""
|
||||
|
||||
# 1. Nettoyer l'ancien conteneur standalone
|
||||
echo "🧹 Nettoyage de l'ancien conteneur..."
|
||||
docker stop effi-web-scraper 2>/dev/null || true
|
||||
docker rm effi-web-scraper 2>/dev/null || true
|
||||
|
||||
# 2. Nettoyer le réseau si créé
|
||||
echo "🗑️ Suppression du réseau effi-net temporaire..."
|
||||
docker network rm effi-net 2>/dev/null || true
|
||||
|
||||
# 3. Supprimer l'ancienne section crawl4ai du docker-compose
|
||||
echo "📝 Mise à jour du docker-compose.yml..."
|
||||
|
||||
# Créer le service web-scraper à insérer
|
||||
cat > /tmp/web-scraper-service.txt << 'EOF'
|
||||
# ─────────────────────────────────────────────────────────
|
||||
# 8. WEB SCRAPER - Scraping pour OpenWebUI
|
||||
# → http://localhost:11235
|
||||
# ─────────────────────────────────────────────────────────
|
||||
web-scraper:
|
||||
build: ./web-scraper
|
||||
container_name: effi-web-scraper
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "11235:11235"
|
||||
networks:
|
||||
- effi-net
|
||||
|
||||
EOF
|
||||
|
||||
# Backup du docker-compose
|
||||
cp docker-compose.yml docker-compose.yml.backup.$(date +%Y%m%d_%H%M%S)
|
||||
|
||||
# Utiliser awk pour remplacer la section 8 (crawl4ai) par web-scraper
|
||||
awk '
|
||||
/^networks:/ {
|
||||
# On est à la fin du fichier services, inserer le nouveau service avant
|
||||
system("cat /tmp/web-scraper-service.txt")
|
||||
}
|
||||
{ print }
|
||||
' docker-compose.yml > /tmp/docker-compose-new.yml
|
||||
|
||||
mv /tmp/docker-compose-new.yml docker-compose.yml
|
||||
|
||||
echo "✅ Section #8 mise à jour (crawl4ai → web-scraper)"
|
||||
|
||||
# 4. Créer le répertoire et les fichiers du web-scraper
|
||||
echo ""
|
||||
echo "📁 Création du répertoire web-scraper..."
|
||||
mkdir -p web-scraper
|
||||
|
||||
cat > web-scraper/Dockerfile << 'EOF'
|
||||
FROM python:3.11-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN pip install --no-cache-dir fastapi uvicorn requests beautifulsoup4
|
||||
|
||||
COPY scraper.py .
|
||||
|
||||
EXPOSE 11235
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
|
||||
CMD curl -f http://localhost:11235/health || exit 1
|
||||
|
||||
CMD ["uvicorn", "scraper:app", "--host", "0.0.0.0", "--port", "11235"]
|
||||
EOF
|
||||
|
||||
cat > web-scraper/scraper.py << 'EOF'
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional, List
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
app = FastAPI(title="Effi Web Scraper", version="1.0.0")
|
||||
|
||||
class ScrapeRequest(BaseModel):
|
||||
url: str
|
||||
max_length: Optional[int] = 10000
|
||||
|
||||
class ScrapeResponse(BaseModel):
|
||||
url: str
|
||||
title: str
|
||||
content: str
|
||||
links: List[str]
|
||||
success: bool
|
||||
error: Optional[str] = None
|
||||
|
||||
@app.get("/health")
|
||||
def health():
|
||||
return {"status": "healthy"}
|
||||
|
||||
@app.post("/scrape", response_model=ScrapeResponse)
|
||||
def scrape(request: ScrapeRequest):
|
||||
try:
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
|
||||
}
|
||||
|
||||
response = requests.get(request.url, headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
|
||||
soup = BeautifulSoup(response.content, 'html.parser')
|
||||
|
||||
title = soup.title.string if soup.title else "No title"
|
||||
|
||||
for script in soup(["script", "style"]):
|
||||
script.decompose()
|
||||
|
||||
main_content = None
|
||||
for selector in ['main', 'article', '[role="main"]', '.content', '#content', '.post-content']:
|
||||
main_content = soup.select_one(selector)
|
||||
if main_content:
|
||||
break
|
||||
|
||||
if not main_content:
|
||||
main_content = soup.body if soup.body else soup
|
||||
|
||||
text = main_content.get_text(separator='\n', strip=True)
|
||||
lines = [line.strip() for line in text.split('\n') if line.strip()]
|
||||
text = '\n'.join(lines)
|
||||
|
||||
if len(text) > request.max_length:
|
||||
text = text[:request.max_length] + f"\n\n[...] {len(text) - request.max_length} caractères supplémentaires"
|
||||
|
||||
links = []
|
||||
for a in soup.find_all('a', href=True):
|
||||
href = a['href']
|
||||
if href.startswith('http'):
|
||||
links.append(href)
|
||||
links = links[:20]
|
||||
|
||||
return ScrapeResponse(
|
||||
url=request.url,
|
||||
title=title,
|
||||
content=text,
|
||||
links=links,
|
||||
success=True
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
return ScrapeResponse(
|
||||
url=request.url,
|
||||
title="Error",
|
||||
content="",
|
||||
links=[],
|
||||
success=False,
|
||||
error=str(e)
|
||||
)
|
||||
|
||||
@app.post("/scrape/sync")
|
||||
def scrape_sync(url: str, max_length: int = 10000):
|
||||
request = ScrapeRequest(url=url, max_length=max_length)
|
||||
result = scrape(request)
|
||||
if not result.success:
|
||||
raise HTTPException(status_code=400, detail=result.error)
|
||||
return {
|
||||
"url": result.url,
|
||||
"html": result.content,
|
||||
"markdown": result.content,
|
||||
"success": True
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run(app, host="0.0.0.0", port=11235)
|
||||
EOF
|
||||
|
||||
echo "✅ Fichiers créés dans web-scraper/"
|
||||
|
||||
# 5. Relancer docker-compose
|
||||
echo ""
|
||||
echo "🚀 Démarrage du service web-scraper..."
|
||||
docker-compose up -d web-scraper
|
||||
|
||||
echo ""
|
||||
echo "⏳ Attente du démarrage (30s)..."
|
||||
sleep 30
|
||||
|
||||
# 6. Vérification
|
||||
echo ""
|
||||
echo "🔍 Vérification..."
|
||||
if curl -s http://localhost:11235/health > /dev/null 2>&1; then
|
||||
echo "✅ Web Scraper est fonctionnel !"
|
||||
curl http://localhost:11235/health
|
||||
else
|
||||
echo "⚠️ Le service ne répond pas encore"
|
||||
docker logs effi-web-scraper
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "================================"
|
||||
echo "Installation terminée !"
|
||||
echo "================================"
|
||||
echo ""
|
||||
echo "📋 Configuration OpenWebUI:"
|
||||
echo " Admin → Tools → Web Search and Crawl → ⚙️"
|
||||
echo " Crawl4AI Base URL: http://localhost:11235"
|
||||
echo " Timeout: 30000"
|
||||
echo " ✅ Activer le Tool"
|
||||
Reference in New Issue
Block a user