🕷️ Website Crawler — Komplette Websites automatisch spiegeln & offline verfügbar machen

📊 Business Value

Ob Website-Migration, Offline-Archivierung oder Wettbewerbsanalyse — dieser Crawler spiegelt komplette Websites in eine lokale Ordnerstruktur. Alle Links werden korrekt relativiert.

⚙️ Crawler-Architektur

💻 Code — Core Crawler mit BeautifulSoup

def save_page(url, base_url, output_dir):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
                      'AppleWebKit/537.36 Chrome/58.0.3029.110 Safari/537.3'
    }
    response = requests.get(url, headers=headers)
    response.raise_for_status()
    
    soup = BeautifulSoup(response.text, 'html.parser')
    
    # Alle Ressourcen-Links in relative Pfade umwandeln
    for tag in soup.find_all(['a', 'img', 'script', 'link']):
        attr = 'href' if tag.name == 'a' else 'src'
        if tag.has_attr(attr):
            tag[attr] = urljoin(base_url, tag[attr])
            if urlparse(tag[attr]).netloc == urlparse(base_url).netloc:
                tag[attr] = os.path.relpath(tag[attr], base_url)
    
    # Dateipfad aus URL-Struktur ableiten
    parsed_url = urlparse(url)
    file_path = os.path.join(output_dir, parsed_url.netloc, 
                             parsed_url.path.lstrip('/'))
    if file_path.endswith('/') or file_path == '':
        file_path = os.path.join(file_path, 'index.html')
    
    os.makedirs(os.path.dirname(file_path), exist_ok=True)
    with open(file_path, 'w', encoding='utf-8') as file:
        file.write(soup.prettify())
    
    return soup

💻 Code — BFS-Crawling mit tqdm-Progress

def crawl_site(start_url, output_dir):
    global visited
    visited = set()
    to_visit = [start_url]
    base_url = '{uri.scheme}://{uri.netloc}'.format(
        uri=urlparse(start_url))

    with tqdm(total=0, unit='pages') as pbar:
        while to_visit:
            url = to_visit.pop(0)  # BFS: FIFO
            if url not in visited:
                visited.add(url)
                pbar.set_description(f"Crawling {url}")
                pbar.update(1)
                
                soup = save_page(url, base_url, output_dir)
                if soup:
                    for link in soup.find_all('a', href=True):
                        abs_url = urljoin(base_url, link['href'])
                        if (abs_url.startswith(base_url) 
                            and abs_url not in visited):
                            to_visit.append(abs_url)
                
                # Fortschrittsbalken aktualisieren
                pbar.total = len(visited) + len(to_visit)

if __name__ == '__main__':
    start_url = 'https://www.computech.gmbh/'
    output_dir = 'downloaded_site'
    os.makedirs(output_dir, exist_ok=True)
    crawl_site(start_url, output_dir)
    print("Website successfully downloaded.")

🎯 Strategische Erkenntnisse