Folzo
93f31cdaeb
- Effi-OCR: Service OCR avec Tesseract (multilingue FR/EN) * API FastAPI sur port 8300 * Prétraitement images (contraste, niveaux de gris) * Tool OpenWebUI intégré - Web Scraper: Service Python (BeautifulSoup) sur port 11235 * Alternative à Crawl4AI (incompatible ARM64/J5040) * Scraping HTML basique mais fonctionnel * Intégré au docker-compose d'effi-server - Documentation: * ROADMAP-FONCTIONNALITES.md: Plan des 9 fonctionnalités * WEB-SCRAPER-CONFIG.md: Configuration OpenWebUI * WEB-SCRAPER-TOOL.md: Code du Tool - Script start-all-services.sh mis à jour (6 services)
116 lines
3.3 KiB
Python
116 lines
3.3 KiB
Python
"""
|
|
Service de scraping web simple pour OpenWebUI
|
|
Alternative à Crawl4AI compatible x86_64
|
|
"""
|
|
|
|
from fastapi import FastAPI, HTTPException
|
|
from pydantic import BaseModel
|
|
from typing import Optional, List
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
import re
|
|
|
|
app = FastAPI(title="Effi Web Scraper", version="1.0.0")
|
|
|
|
class ScrapeRequest(BaseModel):
|
|
url: str
|
|
max_length: Optional[int] = 10000
|
|
extract_markdown: Optional[bool] = True
|
|
|
|
class ScrapeResponse(BaseModel):
|
|
url: str
|
|
title: str
|
|
content: str
|
|
links: List[str]
|
|
success: bool
|
|
error: Optional[str] = None
|
|
|
|
@app.get("/health")
|
|
def health():
|
|
return {"status": "healthy"}
|
|
|
|
@app.post("/scrape", response_model=ScrapeResponse)
|
|
def scrape(request: ScrapeRequest):
|
|
try:
|
|
headers = {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
|
}
|
|
|
|
response = requests.get(request.url, headers=headers, timeout=30)
|
|
response.raise_for_status()
|
|
|
|
soup = BeautifulSoup(response.content, 'html.parser')
|
|
|
|
# Titre
|
|
title = soup.title.string if soup.title else "No title"
|
|
|
|
# Supprimer scripts et styles
|
|
for script in soup(["script", "style"]):
|
|
script.decompose()
|
|
|
|
# Extraction du texte principal
|
|
# Essayer de trouver le contenu principal
|
|
main_content = None
|
|
for selector in ['main', 'article', '[role="main"]', '.content', '#content', '.post-content']:
|
|
main_content = soup.select_one(selector)
|
|
if main_content:
|
|
break
|
|
|
|
if not main_content:
|
|
main_content = soup.body if soup.body else soup
|
|
|
|
# Extraire le texte
|
|
text = main_content.get_text(separator='\n', strip=True)
|
|
|
|
# Nettoyer
|
|
lines = [line.strip() for line in text.split('\n') if line.strip()]
|
|
text = '\n'.join(lines)
|
|
|
|
# Limiter la taille
|
|
if len(text) > request.max_length:
|
|
text = text[:request.max_length] + f"\n\n[...] {len(text) - request.max_length} caractères supplémentaires"
|
|
|
|
# Extraire les liens
|
|
links = []
|
|
for a in soup.find_all('a', href=True):
|
|
href = a['href']
|
|
if href.startswith('http'):
|
|
links.append(href)
|
|
links = links[:20] # Limiter à 20 liens
|
|
|
|
return ScrapeResponse(
|
|
url=request.url,
|
|
title=title,
|
|
content=text,
|
|
links=links,
|
|
success=True
|
|
)
|
|
|
|
except Exception as e:
|
|
return ScrapeResponse(
|
|
url=request.url,
|
|
title="Error",
|
|
content="",
|
|
links=[],
|
|
success=False,
|
|
error=str(e)
|
|
)
|
|
|
|
@app.post("/scrape/sync")
|
|
def scrape_sync(url: str, max_length: int = 10000):
|
|
"""Endpoint compatible avec l'API Crawl4AI"""
|
|
request = ScrapeRequest(url=url, max_length=max_length)
|
|
result = scrape(request)
|
|
if not result.success:
|
|
raise HTTPException(status_code=400, detail=result.error)
|
|
return {
|
|
"url": result.url,
|
|
"html": result.content,
|
|
"markdown": result.content,
|
|
"success": True
|
|
}
|
|
|
|
if __name__ == "__main__":
|
|
import uvicorn
|
|
uvicorn.run(app, host="0.0.0.0", port=11235)
|