Folzo/Effi
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(kreuzberg): rewrite tool code to fix parsing errors

Browse Source 
- Remove complex docstrings that caused parsing issues
- Use simple string concatenation instead of f-strings with special chars
- Remove duplicate code
- Keep functionality intact: single file, multi-file, list, search
This commit is contained in:
Folzo
2026-04-11 00:47:21 +02:00
parent 98b695bc31
commit e9d6de3513
1 changed files with 48 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
effi-kreuzberg/TOOL-OPENWEBUI.md
+48 -122
View File
@@ -11,17 +11,11 @@
## Tool Code ## Tool Code
```python ```python
"""
Kreuzberg OCR Tool for OpenWebUI
Extracts text from scanned documents, images, and PDFs using Kreuzberg OCR.
Supports: PDF, JPEG, PNG, TIFF, DOCX, PPTX, XLSX, EPUB, etc.
"""
import os import os
import glob import glob
import requests import requests
import hashlib
import io import io
import json
from PIL import Image, ImageEnhance from PIL import Image, ImageEnhance
@@ -31,8 +25,7 @@ class Tools:
self.uploads_dir = "/app/backend/data/uploads" self.uploads_dir = "/app/backend/data/uploads"
self.host_uploads_dir = "/home/effi/effi/data/uploads" self.host_uploads_dir = "/home/effi/effi/data/uploads"
def _preprocess_image(self, image: Image.Image) -> Image.Image: def _preprocess_image(self, image):
"""Preprocess image for better OCR quality (grayscale + contrast)"""
if image.mode in ('RGBA', 'LA', 'P'): if image.mode in ('RGBA', 'LA', 'P'):
image = image.convert('RGBA').convert('RGB') image = image.convert('RGBA').convert('RGB')
elif image.mode != 'RGB': elif image.mode != 'RGB':
@@ -42,47 +35,24 @@ class Tools:
image = enhancer.enhance(2.0) image = enhancer.enhance(2.0)
return image return image
def extract_file(self, filename: str) -> str: def extract_file(self, filename):
"""
Extract text from a single file using Kreuzberg OCR.
Args:
filename: The filename shown in the chat (e.g., "document.pdf", "image.jpg")
or the full name like "UUID_name.pdf"
Returns:
Extracted text content with markdown formatting
"""
return self._extract_single_file(filename) return self._extract_single_file(filename)
def extract_files(self, filenames: str) -> str: def extract_files(self, filenames):
"""
Extract text from multiple files, one at a time.
Args:
filenames: Comma-separated list of filenames (e.g., "doc1.pdf,doc2.jpg")
or JSON array string (e.g., '["doc1.pdf","doc2.jpg"]')
Returns:
Combined extracted text from all files, processed sequentially
"""
import json
filenames = filenames.strip() filenames = filenames.strip()
if not filenames: if not filenames:
return "❌ No filenames provided. Usage: extract_files(\"file1.pdf,file2.jpg\")" return "[ERROR] No filenames provided. Usage: extract_files(\"file1.pdf,file2.jpg\")"
if filenames.startswith('['): if filenames.startswith('['):
try: try:
file_list = json.loads(filenames) file_list = json.loads(filenames)
except json.JSONDecodeError: except json.JSONDecodeError:
return f"❌ Invalid JSON array: {filenames}" return "[ERROR] Invalid JSON array: " + filenames
else: else:
file_list = [f.strip() for f in filenames.split(',')] file_list = [f.strip() for f in filenames.split(',')]
if not file_list: if not file_list:
return "❌ No files to process" return "[ERROR] No files to process"
if len(file_list) == 1: if len(file_list) == 1:
return self._extract_single_file(file_list[0]) return self._extract_single_file(file_list[0])
@@ -94,22 +64,19 @@ class Tools:
filename = filename.strip() filename = filename.strip()
if not filename: if not filename:
continue continue
if total > 1: if total > 1:
results.append(f"--- Document {i}/{total}: `{filename}` ---") results.append("--- Document " + str(i) + "/" + str(total) + ": " + filename + " ---")
result = self._extract_single_file(filename) result = self._extract_single_file(filename)
results.append(result) results.append(result)
return "\n\n".join(results) return "\n\n".join(results)
def _extract_single_file(self, filename: str) -> str: def _extract_single_file(self, filename):
"""Internal method to extract text from a single file"""
try: try:
file_path = self._find_file(filename) file_path = self._find_file(filename)
if not file_path: if not file_path:
return f"❌ File not found: {filename}\n\nTip: Use `list_recent_files()` to see available files." return "[ERROR] File not found: " + filename + ". Use list_recent_files() to see available files."
mime_type = self._get_mime_type(file_path) mime_type = self._get_mime_type(file_path)
ext = os.path.splitext(file_path)[1].lower() ext = os.path.splitext(file_path)[1].lower()
@@ -137,7 +104,7 @@ class Tools:
} }
response = requests.post( response = requests.post(
f"{self.kreuzberg_url}/v1/convert/file", self.kreuzberg_url + "/v1/convert/file",
files=files, files=files,
data=data, data=data,
timeout=300 timeout=300
@@ -152,31 +119,30 @@ class Tools:
content = self._clean_content(content) content = self._clean_content(content)
if len(content) > 15000: if len(content) > 15000:
content = content[:15000] + "\n\n... (document truncated - too long for context)" content = content[:15000] + "\n\n... (document truncated)"
return f"📄 **{os.path.basename(file_path)}**\n\n{content}" return "FILE: " + os.path.basename(file_path) + "\n\n" + content
else: else:
return f"⚠️ No text extracted from {os.path.basename(file_path)}\n\nThe file may be empty, protected, or contain only images without selectable text." return "[WARNING] No text extracted from " + os.path.basename(file_path) + ". File may be empty or contain only images."
return f"⚠️ Unexpected response format from Kreuzberg: {str(result)[:500]}" return "[WARNING] Unexpected response from Kreuzberg"
elif response.status_code == 404: elif response.status_code == 404:
return f"❌ Kreuzberg endpoint not found. Is the server running at {self.kreuzberg_url}?" return "[ERROR] Kreuzberg endpoint not found at " + self.kreuzberg_url
else: else:
return f"❌ Kreuzberg error ({response.status_code}): {response.text[:500]}" return "[ERROR] Kreuzberg error (" + str(response.status_code) + ")"
except requests.Timeout: except requests.Timeout:
return f"⏱️ Timeout - File is too large or Kreuzberg is overloaded. Try a smaller file." return "[ERROR] Timeout - file is too large or Kreuzberg is overloaded"
except requests.ConnectionError: except requests.ConnectionError:
return f"❌ Cannot connect to Kreuzberg at {self.kreuzberg_url}. Is the server running?" return "[ERROR] Cannot connect to Kreuzberg at " + self.kreuzberg_url
except Exception as e: except Exception as e:
return f"❌ Error: {str(e)}" return "[ERROR] " + str(e)
def list_recent_files(self, count: int = 10) -> str: def list_recent_files(self, count=10):
"""List recent uploaded files"""
try: try:
files = glob.glob(os.path.join(self.uploads_dir, "*")) files = glob.glob(os.path.join(self.uploads_dir, "*"))
@@ -187,81 +153,70 @@ class Tools:
files = sorted(files, key=os.path.getmtime, reverse=True)[:count] files = sorted(files, key=os.path.getmtime, reverse=True)[:count]
if not files: if not files:
return "📂 No files found in uploads directory." return "[INFO] No files found in uploads directory."
result = f"📂 **{len(files)} recent files:**\n\n" result = "[INFO] " + str(len(files)) + " recent files:\n\n"
for f in files: for f in files:
name = os.path.basename(f) name = os.path.basename(f)
size = os.path.getsize(f) size = os.path.getsize(f)
size_str = self._format_size(size) size_str = self._format_size(size)
result += f"- `{name}` ({size_str})\n" result += "- " + name + " (" + size_str + ")\n"
result += "\n💡 Use `extract_file(\"filename\")` to OCR a file." result += "\nUse extract_file(\"filename\") to OCR a file."
return result return result
except Exception as e: except Exception as e:
return f"❌ Error listing files: {str(e)}" return "[ERROR] " + str(e)
def search_files(self, pattern: str) -> str: def search_files(self, pattern):
"""Search for files matching a pattern"""
try: try:
# Search in uploads files = glob.glob(os.path.join(self.uploads_dir, "*" + pattern + "*"))
files = glob.glob(os.path.join(self.uploads_dir, f"*{pattern}*"))
if not files and os.path.exists(self.host_uploads_dir): if not files and os.path.exists(self.host_uploads_dir):
files = glob.glob(os.path.join(self.host_uploads_dir, f"*{pattern}*")) files = glob.glob(os.path.join(self.host_uploads_dir, "*" + pattern + "*"))
files = [f for f in files if os.path.isfile(f)] files = [f for f in files if os.path.isfile(f)]
if not files: if not files:
return f"🔍 No files found matching: {pattern}" return "[INFO] No files found matching: " + pattern
result = f"🔍 **{len(files)} files found:**\n\n" result = "[INFO] " + str(len(files)) + " files found:\n\n"
for f in files[:20]: # Limit to 20 for f in files[:20]:
name = os.path.basename(f) name = os.path.basename(f)
size = self._format_size(os.path.getsize(f)) size = self._format_size(os.path.getsize(f))
result += f"- `{name}` ({size})\n" result += "- " + name + " (" + size + ")\n"
return result return result
except Exception as e: except Exception as e:
return f"❌ Error searching files: {str(e)}" return "[ERROR] " + str(e)
def _find_file(self, filename: str) -> str: def _find_file(self, filename):
"""Find file in uploads directory"""
# Clean up filename
filename = filename.strip().strip('"').strip("'") filename = filename.strip().strip('"').strip("'")
# Try exact path
if os.path.exists(filename): if os.path.exists(filename):
return filename return filename
# Try in uploads directory with exact name
path = os.path.join(self.uploads_dir, filename) path = os.path.join(self.uploads_dir, filename)
if os.path.exists(path): if os.path.exists(path):
return path return path
# Try host mount path
path = os.path.join(self.host_uploads_dir, filename) path = os.path.join(self.host_uploads_dir, filename)
if os.path.exists(path): if os.path.exists(path):
return path return path
# Try glob patterns
for directory in [self.uploads_dir, self.host_uploads_dir]: for directory in [self.uploads_dir, self.host_uploads_dir]:
if not os.path.exists(directory): if not os.path.exists(directory):
continue continue
# UUID prefix match pattern = os.path.join(directory, filename + "*")
pattern = os.path.join(directory, f"{filename}*")
matches = glob.glob(pattern) matches = glob.glob(pattern)
if matches: if matches:
return matches[0] return matches[0]
# Filename contains match pattern = os.path.join(directory, "*" + filename + "*")
pattern = os.path.join(directory, f"*{filename}*")
matches = glob.glob(pattern) matches = glob.glob(pattern)
if matches: if matches:
# Prefer exact matches
for m in matches: for m in matches:
if filename in os.path.basename(m): if filename in os.path.basename(m):
return m return m
@@ -269,8 +224,7 @@ class Tools:
return None return None
def _get_mime_type(self, file_path: str) -> str: def _get_mime_type(self, file_path):
"""Determine MIME type from file extension"""
ext = os.path.splitext(file_path)[1].lower() ext = os.path.splitext(file_path)[1].lower()
mime_types = { mime_types = {
'.pdf': 'application/pdf', '.pdf': 'application/pdf',
@@ -295,12 +249,10 @@ class Tools:
} }
return mime_types.get(ext, 'application/octet-stream') return mime_types.get(ext, 'application/octet-stream')
def _clean_content(self, content: str) -> str: def _clean_content(self, content):
"""Clean up extracted content"""
if not content: if not content:
return "" return ""
# Remove excessive whitespace
lines = content.split('\n') lines = content.split('\n')
cleaned_lines = [] cleaned_lines = []
prev_empty = False prev_empty = False
@@ -315,64 +267,38 @@ class Tools:
cleaned_lines.append(stripped) cleaned_lines.append(stripped)
prev_empty = False prev_empty = False
# Join and limit consecutive empty lines
result = '\n'.join(cleaned_lines) result = '\n'.join(cleaned_lines)
# Remove very long lines (>200 chars wrap)
if len(result) > 500: if len(result) > 500:
# Check if content looks garbled (lots of short fragments)
short_words = sum(1 for w in result.split() if len(w) <= 3) short_words = sum(1 for w in result.split() if len(w) <= 3)
total_words = len(result.split()) total_words = len(result.split())
if total_words > 50 and short_words / total_words > 0.3: if total_words > 50 and short_words / total_words > 0.3:
result += "\n\n⚠️ Warning: This text may contain OCR errors due to complex formatting or handwriting." result += "\n\n[WARNING] This text may contain OCR errors."
return result return result
def _format_size(self, size: int) -> str: def _format_size(self, size):
"""Format file size"""
if size < 1024: if size < 1024:
return f"{size} B" return str(size) + " B"
elif size < 1024 * 1024: elif size < 1024 * 1024:
return f"{size / 1024:.1f} KB" return str(size / 1024) + " KB"
else: else:
return f"{size / (1024 * 1024):.1f} MB" return str(size / (1024 * 1024)) + " MB"
# OpenWebUI Tool Functions def extract_file(filename):
def extract_file(filename: str) -> str:
"""
Extract text from a single file using Kreuzberg OCR.
Args:
filename: The filename or UUID from the chat (e.g., "document.pdf", "abc123_document.pdf")
Returns:
Extracted text with markdown formatting
"""
tools = Tools() tools = Tools()
return tools.extract_file(filename) return tools.extract_file(filename)
def extract_files(filenames: str) -> str: def extract_files(filenames):
"""
Extract text from multiple files, one at a time.
Args:
filenames: Comma-separated list (e.g., "doc1.pdf,doc2.jpg")
or JSON array (e.g., '["doc1.pdf","doc2.jpg"]')
Returns:
Combined extracted text from all files
"""
tools = Tools() tools = Tools()
return tools.extract_files(filenames) return tools.extract_files(filenames)
def list_recent_files(count: int = 10) -> str: def list_recent_files(count=10):
"""List recent uploaded files"""
tools = Tools() tools = Tools()
return tools.list_recent_files(count) return tools.list_recent_files(count)
def search_files(pattern: str) -> str: def search_files(pattern):
"""Search for files by name pattern"""
tools = Tools() tools = Tools()
return tools.search_files(pattern) return tools.search_files(pattern)
``` ```