HTML to Text: Saubere Textextraktion aus jeder Webseite

📰 Warum HTML-zu-Text-Konvertierung?

Webseiten bestehen zu 60–80% aus Boilerplate: Navigation, Footer, Werbung, Sidebar-Widgets. Für KI-Training, Content-Analyse oder Archivierung brauchen Sie nur den reinen Inhalt. Der HTML-to-Text-Konverter filtert alles Störende zuverlässig heraus.

💻 Der Extraktor-Kern

HTMLToText Klasse mit Readability-Heuristiken

from bs4 import BeautifulSoup
from markdownify import markdownify as md
import re, requests

class HTMLToText:
    REMOVE_TAGS = [
        'script', 'style', 'nav', 'footer', 'header',
        'noscript', 'iframe', 'svg', 'form',
        '.sidebar', '.widget', '.advertisement',
        '.cookie-banner', '.social-share', '.comments',
        '[role="navigation"]', '[role="banner"]'
    ]

    def __init__(self, output_format='text'):
        self.output_format = output_format

    def extract(self, html, url=None):
        soup = BeautifulSoup(html, 'lxml')
        self._remove_boilerplate(soup)
        main_content = self._extract_main_content(soup)

        if self.output_format == 'markdown':
            text = md(str(main_content), heading_style='ATX')
        elif self.output_format == 'text':
            text = main_content.get_text(
                separator='\n', strip=True)

        text = self._clean_text(text)
        return {
            'url': url,
            'title': self._extract_title(soup),
            'text': text,
            'char_count': len(text),
            'word_count': len(text.split())}

Boilerplate-Entfernung & Content-Erkennung

    def _remove_boilerplate(self, soup):
        for selector in self.REMOVE_TAGS:
            for element in soup.select(selector):
                element.decompose()
        for el in soup.find_all(['div','section','aside']):
            text = el.get_text(strip=True)
            links = el.find_all('a')
            if len(text) < 200 and len(links) > 3:
                el.decompose()

    def _extract_main_content(self, soup):
        content_selectors = [
            'article', 'main', '[role="main"]',
            '.post-content', '.article-content',
            '.entry-content', '#content', '.post']
        candidates = []
        for sel in content_selectors:
            for el in soup.select(sel):
                tlen = len(el.get_text(strip=True))
                if tlen > 200:
                    candidates.append((tlen, el))
        if candidates:
            return max(candidates, key=lambda x: x[0])[1]
        return soup.find('body') or soup

Text-Bereinigung & URL-Verarbeitung

    @staticmethod
    def _clean_text(text):
        # Mehrfache Leerzeilen entfernen
        text = re.sub(r'\n{3,}', '\n\n', text)
        # Leerraum am Anfang/Ende trimmen
        text = text.strip()
        return text

    @staticmethod
    def from_url(url, **kwargs):
        headers = {'User-Agent': 'Mozilla/5.0'}
        resp = requests.get(url, headers=headers, timeout=30)
        extractor = HTMLToText(**kwargs)
        return extractor.extract(resp.text, url=url)