import base64 import io import re from typing import Optional import requests # type: ignore[import-untyped] from PIL import Image SMART_PUNCT_TRANSLATION = str.maketrans( { "\u2018": "'", "\u2019": "'", "\u201c": '"', "\u201d": '"', } ) SPEECHABLE_PATTERN = re.compile( r"[^\w\s.,!?;:'\"\-()\/\\@#%&*+=$€£¥₹₽¢\[\]{}<>~`^|…—–\n\r\t]", flags=re.UNICODE, ) def remove_unspeechable(text: str) -> str: """Keep only speechable characters: letters, digits, punctuation, whitespace. support unicode characters (english, arabic, chinese, japanese, korean, etc.) """ text = text.translate(SMART_PUNCT_TRANSLATION) return SPEECHABLE_PATTERN.sub("", text) WHISPER_LANGUAGE_TO_LLM_LANGUAGE = { "en": "english", "fr": "french", "es": "spanish", "zh": "chinese", "ja": "japanese", "ko": "korean", "hi": "hindi", "de": "german", "pt": "portuguese", "pl": "polish", "it": "italian", "nl": "dutch", } def resolve_auto_language(language_code: Optional[str]) -> tuple[Optional[str], Optional[str]]: """Strip the ``-auto`` suffix and resolve the human-readable language name. Returns ``(clean_code, language_name)``. ``language_name`` is non-None when the code (with or without ``-auto``) maps to a known language. """ if not language_code: return language_code, None if language_code.endswith("-auto"): language_code = language_code[:-5] if language_code not in WHISPER_LANGUAGE_TO_LLM_LANGUAGE: return language_code, None return language_code, WHISPER_LANGUAGE_TO_LLM_LANGUAGE.get(language_code) def image_url_to_pil(image_url: str) -> Image.Image: """Convert an image URL or base64 data URI to a PIL Image. Accepts: - 'data:image/...;base64,' data URIs - 'https://...`` or ``http://...' URLs (fetched with a 10s timeout) """ if image_url.startswith("data:"): _, b64_data = image_url.split(",", 1) return Image.open(io.BytesIO(base64.b64decode(b64_data))) resp = requests.get(image_url, timeout=10) resp.raise_for_status() return Image.open(io.BytesIO(resp.content))