Grok vs Perplexity for Coding

#!/usr/bin/env python3
"""
Broken Link Checker Script
==========================

This script:
1. Discovers sitemaps from robots.txt or common locations.
2. Parses sitemap(s) recursively to extract page URLs (limited to same domain).
3. For each page (up to --max-pages), fetches the page and extracts all <a href> links.
4. Checks each link (HEAD request, internal only by default) for broken status (>=400 or error).
5. Outputs ONLY broken links (including broken pages themselves) to a CSV report.

Usage:
    python broken_link_checker.py https://example.com --max-pages 50 --output report.csv

Dependencies:
    pip install requests beautifulsoup4 lxml

Notes:
- Sequential checking to respect servers (use --delay for slower sites).
- Filters out mailto:, tel:, javascript:, #, non-http(s) links.
- HEAD requests for speed; falls back to reporting errors.
- Limited to --max-pages to avoid overload.
- Add --check-external to include external links (slower).
"""

import argparse
import csv
import time
import xml.etree.ElementTree as ET
from datetime import datetime
from urllib.parse import urljoin, urlparse
import requests
from bs4 import BeautifulSoup


def get_sitemaps(base_url):
    """Fetch sitemaps from robots.txt or fallback locations."""
    sitemaps = []
    try:
        robots_url = urljoin(base_url, '/robots.txt')
        r = requests.get(robots_url, timeout=10)
        r.raise_for_status()
        for line in r.text.splitlines():
            if line.lower().startswith('sitemap:'):
                sitemap_url = line.split(':', 1)[1].strip()
                sitemaps.append(sitemap_url)
    except Exception:
        pass

    if not sitemaps:
        # Common fallback locations
        sitemaps = [
            urljoin(base_url, '/sitemap.xml'),
            urljoin(base_url, '/sitemap_index.xml'),
            urljoin(base_url, '/sitemap-index.xml')
        ]
    return sitemaps


def parse_sitemap(url):
    """Recursively parse sitemap XML and return list of page URLs."""
    try:
        r = requests.get(url, timeout=10)
        r.raise_for_status()
        root = ET.fromstring(r.content)
        ns = {'s': 'http://www.sitemaps.org/schemas/sitemap/0.9'}
        urls = []

        # Find all <loc> elements (works for both urlset and sitemapindex)
        for loc_elem in root.findall('.//s:loc', ns):
            loc_url = loc_elem.text.strip()
            if loc_url:
                # Recurse if it's a sitemapindex entry (contains <sitemap>)
                sub_root = None
                try:
                    sub_r = requests.get(loc_url, timeout=10)
                    sub_root = ET.fromstring(sub_r.content)
                except Exception:
                    pass
                if sub_root and sub_root.find('.//s:sitemap', ns) is not None:
                    urls.extend(parse_sitemap(loc_url))
                else:
                    urls.append(loc_url)

        return urls
    except Exception as e:
        print(f"Error parsing sitemap {url}: {e}")
        return []


def check_link(link):
    """Check a single link with HEAD request."""
    try:
        start_time = time.time()
        resp = requests.head(link, timeout=10, allow_redirects=True,
                             headers={'User-Agent': 'Mozilla/5.0 (compatible; LinkChecker/1.0)'})
        elapsed_ms = (time.time() - start_time) * 1000
        return resp.status_code, elapsed_ms, None
    except Exception as e:
        return 'ERROR', 0, str(e)


def check_page(page_url, base_domain, check_external=False):
    """Check a single page for broken links. Returns list of broken [source, link, status, time, timestamp, error]."""
    broken = []
    headers = {'User-Agent': 'Mozilla/5.0 (compatible; LinkChecker/1.0)'}
    try:
        page_resp = requests.get(page_url, timeout=10, headers=headers)
        timestamp = datetime.now().isoformat()
        if page_resp.status_code >= 400:
            broken.append([page_url, page_url, page_resp.status_code,
                           page_resp.elapsed.total_seconds() * 1000, timestamp, None])
            return broken

        soup = BeautifulSoup(page_resp.content, 'html.parser')
        links = set()
        for a in soup.find_all('a', href=True):
            href = a['href'].strip()
            if not href or href.startswith(('#', 'mailto:', 'tel:', 'javascript:')):
                continue
            full_url = urljoin(page_url, href)
            parsed = urlparse(full_url)
            if parsed.scheme not in ('http', 'https'):
                continue
            if not check_external and parsed.netloc != base_domain:
                continue
            links.add(full_url)

        for link in links:
            status, elapsed_ms, error = check_link(link)
            row_timestamp = datetime.now().isoformat()
            if status == 'ERROR' or (isinstance(status, int) and status >= 400):
                broken.append([page_url, link, status, elapsed_ms, row_timestamp, error])

        return broken
    except Exception as e:
        timestamp = datetime.now().isoformat()
        broken.append([page_url, page_url, 'ERROR_PAGE', 0, timestamp, str(e)])
        return broken


def main():
    parser = argparse.ArgumentParser(description='Check website sitemap for broken links.')
    parser.add_argument('url', help='Base website URL (e.g., https://example.com)')
    parser.add_argument('--output', '-o', default='broken_links_report.csv',
                        help='Output CSV file (default: broken_links_report.csv)')
    parser.add_argument('--max-pages', '-m', type=int, default=100,
                        help='Max pages to check (default: 100)')
    parser.add_argument('--delay', type=float, default=0.5,
                        help='Delay between requests in seconds (default: 0.5)')
    parser.add_argument('--check-external', '-e', action='store_true',
                        help='Check external links too (default: internal only)')
    args = parser.parse_args()

    base_url = args.url.rstrip('/')
    base_parsed = urlparse(base_url)
    base_domain = base_parsed.netloc

    print(f"Discovering sitemaps for {base_url}...")
    sitemaps = get_sitemaps(base_url)

    all_page_urls = []
    for sitemap in sitemaps:
        print(f"Parsing sitemap: {sitemap}")
        all_page_urls.extend(parse_sitemap(sitemap))

    # Filter to same domain and unique
    page_urls = list(set(
        u for u in all_page_urls
        if u and urlparse(u).netloc == base_domain
    ))[:args.max_pages]

    print(f"Found {len(page_urls)} unique internal pages to check (limited to {args.max_pages}).")

    results = []
    total_pages = len(page_urls)
    for i, page_url in enumerate(page_urls, 1):
        print(f"[{i}/{total_pages}] Checking: {page_url}")
        broken = check_page(page_url, base_domain, args.check_external)
        results.extend(broken)
        time.sleep(args.delay)

    # Write CSV report
    with open(args.output, 'w', newline='', encoding='utf-8') as f:
        if results:
            writer = csv.writer(f)
            writer.writerow(['Source_Page', 'Link_URL', 'Status_Code', 'Response_Time_ms', 'Timestamp', 'Error'])
            writer.writerows(results)
            print(f"\nReport saved: {args.output} ({len(results)} broken links found)")
        else:
            print(f"\nNo broken links found. Empty report: {args.output}")

    print("Done!")


if __name__ == '__main__':
    main()

Features & Usage Notes

• Robust sitemap discovery: Checks robots.txt first, falls back to /sitemap.xml etc.
• Recursive sitemap parsing: Handles sitemap indexes automatically.
• Efficient link checking: Unique links per page, HEAD requests (fast), filters junk protocols.
• Internal by default: --check-external to include outbound links.
• Rate limiting: --delay prevents bans.
• Limits: --max-pages avoids overload (e.g., huge sites).
• CSV Output: Only broken items (pages/links with 4xx/5xx/ERROR), with timings/timestamps.
• Error resilient: Continues on failures, logs errors.
• Example:

  python broken_link_checker.py https://example.com --max-pages 20 --delay 1.0 --output my_report.csv
  

• Customization: Edit check_link() for GET fallback if HEAD fails often.
• Performance: Sequential for politeness; for speed, replace loop with concurrent.futures.ThreadPoolExecutor(max_workers=5) (but monitor server response).

import httpx
import csv
import time
import urllib.robotparser
from urllib.parse import urljoin, urlparse
from parsel import Selector
from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import List, Dict, Optional
import sys

def find_sitemaps(base_url: str) -> List[str]:
    """Find sitemap URLs from robots.txt."""
    try:
        rp = urllib.robotparser.RobotFileParser()
        rp.set_url(urljoin(base_url, '/robots.txt'))
        rp.read()
        sitemaps = rp.sitemaps()
        if not sitemaps:
            # Fallback: check common sitemap locations
            common_sitemaps = [                urljoin(base_url, '/sitemap.xml'),
                urljoin(base_url, '/sitemap_index.xml'),
                urljoin(base_url, '/sitemap.xml.gz')
            ]
            return common_sitemaps
        return sitemaps
    except:
        # Fallback to common locations if robots.txt fails
        return [            urljoin(base_url, '/sitemap.xml'),
            urljoin(base_url, '/sitemap_index.xml')
        ][1]

def extract_urls_from_sitemap(sitemap_url: str, client: httpx.Client) -> List[str]:
    """Recursively extract all URLs from sitemap(s)."""
    urls = set()
    
    try:
        response = client.get(sitemap_url, timeout=30.0)
        response.raise_for_status()
        selector = Selector(response.text)
        
        # Check if this is a sitemap index
        sitemap_locs = selector.xpath('//sitemap/loc/text()').getall()
        if sitemap_locs:
            for loc in sitemap_locs:
                urls.update(extract_urls_from_sitemap(loc, client))
            return list(urls)
        
        # Extract URLs from regular sitemap
        url_locs = selector.xpath('//url/loc/text()').getall()
        urls.update(url_locs)
        
    except Exception as e:
        print(f"Error parsing sitemap {sitemap_url}: {e}", file=sys.stderr)
    
    return list(urls)

def check_link_status(url: str, client: httpx.Client, timeout: float = 10.0) -> Dict[str, any]:
    """Check if a URL returns a successful HTTP status."""
    try:
        response = client.get(url, timeout=timeout, follow_redirects=True)
        return {
            'url': url,
            'status_code': response.status_code,
            'ok': response.status_code < 400,
            'response_time': response.elapsed.total_seconds()
        }
    except httpx.RequestError as e:
        return {
            'url': url,
            'status_code': None,
            'ok': False,
            'error': str(e),
            'response_time': None
        }

def main(base_url: str, max_workers: int = 20, output_file: str = "broken_links_report.csv"):
    """Main function to scrape sitemaps and check for broken links."""
    print(f"🔍 Discovering sitemaps for {base_url}...")
    
    with httpx.Client(
        timeout=httpx.Timeout(30.0),
        limits=httpx.Limits(max_keepalive_connections=50, max_connections=100)
    ) as client:
        
        # Step 1: Find sitemaps
        sitemaps = find_sitemaps(base_url)[1]
        print(f"📋 Found sitemaps: {sitemaps}")
        
        # Step 2: Extract all URLs
        all_urls = set()
        print("📄 Extracting URLs from sitemaps...")
        for sitemap_url in sitemaps:
            print(f"  Parsing {sitemap_url}...")
            urls = extract_urls_from_sitemap(sitemap_url, client)
            all_urls.update(urls)
        
        unique_urls = list(all_urls)
        print(f"✅ Extracted {len(unique_urls)} unique URLs")
        
        # Step 3: Check each URL for broken links
        print("🔍 Checking links for broken status...")
        results = []
        
        with ThreadPoolExecutor(max_workers=max_workers) as executor:
            # Submit all link checks
            future_to_url = {
                executor.submit(check_link_status, url, client): url 
                for url in unique_urls
            }
            
            # Process results as they complete
            for i, future in enumerate(as_completed(future_to_url), 1):
                result = future.result()
                results.append(result)
                
                if not result['ok']:
                    status = result.get('status_code', 'ERROR')
                    print(f"  ❌ {status} {result['url'][:80]}...")
                else:
                    if i % 50 == 0 or i == len(unique_urls):
                        print(f"  ✅ Progress: {i}/{len(unique_urls)}")
                
                # Be respectful to servers
                time.sleep(0.1)
        
        # Step 4: Generate CSV report
        print("📊 Generating CSV report...")
        fieldnames = ['url', 'status_code', 'ok', 'response_time', 'error']
        
        with open(output_file, 'w', newline='', encoding='utf-8') as csvfile:
            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
            writer.writeheader()
            writer.writerows(results)
        
        # Summary statistics
        total = len(results)
        broken = len([r for r in results if not r['ok']])
        ok = total - broken
        
        print(f"\n📈 **Report Summary**")
        print(f"   Total URLs: {total}")
        print(f"   ✅ Working: {ok}")
        print(f"   ❌ Broken: {broken}")
        print(f"   📄 Report saved: {output_file}")
        
        if broken > 0:
            print(f"\n🔗 First 5 broken links:")
            for result in sorted([r for r in results if not r['ok']], 
                               key=lambda x: x.get('status_code', 999))[:5]:
                status = result.get('status_code', 'ERROR')
                print(f"   {status}: {result['url'][:100]}")

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Usage: python sitemap_checker.py <base_url> [output.csv]")
        print("Example: python sitemap_checker.py https://example.com")
        sys.exit(1)
    
    base_url = sys.argv[1]
    output_file = sys.argv[2] if len(sys.argv) > 2 else "broken_links_report.csv"
    
    main(base_url, output_file=output_file)

Installation

pip install httpx parsel

Usage

# Check example.com and save results
python sitemap_checker.py https://example.com

# Custom output file
python sitemap_checker.py https://example.com my_site_report.csv

Features

• 📋 Auto-discovers sitemaps from robots.txt + common locations[1]
• 🔗 Handles nested sitemap indexes recursively[1][2]
• ⚡ Concurrent link checking with thread pool (configurable workers)
• 📊 CSV output with status codes, response times, and errors
• ⏱️ Rate limiting and timeouts to be server-friendly
• 📈 Progress tracking and summary statistics[1][2]

Sample CSV Output

url,status_code,ok,response_time,error
https://example.com/,200,true,0.234,
https://example.com/missing,404,false,0.156,
https://example.com/timeout,-,false,,"Request timed out"

The script uses httpx for robust HTTP requests and parsel for XML parsing (recommended in search results[1][4]). It handles large sites efficiently with concurrency while respecting servers through rate limiting.[1][2]