"""
Service de scraping web simple pour OpenWebUI
Alternative à Crawl4AI compatible x86_64
"""

from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from typing import Optional, List
import requests
from bs4 import BeautifulSoup
import re

app = FastAPI(title="Effi Web Scraper", version="1.0.0")

class ScrapeRequest(BaseModel):
    url: str
    max_length: Optional[int] = 10000
    extract_markdown: Optional[bool] = True

class ScrapeResponse(BaseModel):
    url: str
    title: str
    content: str
    links: List[str]
    success: bool
    error: Optional[str] = None

@app.get("/health")
def health():
    return {"status": "healthy"}

@app.post("/scrape", response_model=ScrapeResponse)
def scrape(request: ScrapeRequest):
    try:
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
        }
        
        response = requests.get(request.url, headers=headers, timeout=30)
        response.raise_for_status()
        
        soup = BeautifulSoup(response.content, 'html.parser')
        
        # Titre
        title = soup.title.string if soup.title else "No title"
        
        # Supprimer scripts et styles
        for script in soup(["script", "style"]):
            script.decompose()
        
        # Extraction du texte principal
        # Essayer de trouver le contenu principal
        main_content = None
        for selector in ['main', 'article', '[role="main"]', '.content', '#content', '.post-content']:
            main_content = soup.select_one(selector)
            if main_content:
                break
        
        if not main_content:
            main_content = soup.body if soup.body else soup
        
        # Extraire le texte
        text = main_content.get_text(separator='\n', strip=True)
        
        # Nettoyer
        lines = [line.strip() for line in text.split('\n') if line.strip()]
        text = '\n'.join(lines)
        
        # Limiter la taille
        if len(text) > request.max_length:
            text = text[:request.max_length] + f"\n\n[...] {len(text) - request.max_length} caractères supplémentaires"
        
        # Extraire les liens
        links = []
        for a in soup.find_all('a', href=True):
            href = a['href']
            if href.startswith('http'):
                links.append(href)
        links = links[:20]  # Limiter à 20 liens
        
        return ScrapeResponse(
            url=request.url,
            title=title,
            content=text,
            links=links,
            success=True
        )
        
    except Exception as e:
        return ScrapeResponse(
            url=request.url,
            title="Error",
            content="",
            links=[],
            success=False,
            error=str(e)
        )

@app.post("/scrape/sync")
def scrape_sync(url: str, max_length: int = 10000):
    """Endpoint compatible avec l'API Crawl4AI"""
    request = ScrapeRequest(url=url, max_length=max_length)
    result = scrape(request)
    if not result.success:
        raise HTTPException(status_code=400, detail=result.error)
    return {
        "url": result.url,
        "html": result.content,
        "markdown": result.content,
        "success": True
    }

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(app, host="0.0.0.0", port=11235)