radionica3d/scripts/seo_audit.py

import sys
import hashlib
import requests
import xml.etree.ElementTree as ET
from bs4 import BeautifulSoup
from collections import defaultdict
from urllib.parse import urlparse

# Force UTF-8 for console output if possible
if sys.stdout.encoding != 'utf-8':
    try:
        import codecs
        sys.stdout = codecs.getwriter('utf-8')(sys.stdout.buffer, 'strict')
        sys.stderr = codecs.getwriter('utf-8')(sys.stderr.buffer, 'strict')
    except:
        pass

# Production Settings
PRODUCTION_DOMAIN = "https://radionica3d.me"
PRODUCTION_SITEMAP = f"{PRODUCTION_DOMAIN}/sitemap.xml"

def get_content_hash(soup):
    # Remove elements that change frequently or are boilerplate
    clean_soup = BeautifulSoup(str(soup), "lxml")
    for tag in clean_soup(["script", "style", "header", "footer", "nav", "aside", "noscript"]):
        tag.decompose()
    
    text = clean_soup.get_text(separator=' ', strip=True)
    return hashlib.md5(text.encode('utf-8')).hexdigest()

def load_sitemap(url):
    print(f"Loading production sitemap from {url}...")
    urls = set()
    try:
        response = requests.get(url, timeout=15)
        response.raise_for_status()
        
        root = ET.fromstring(response.content)
        ns = {'ns': 'http://www.sitemaps.org/schemas/sitemap/0.9'}
        for loc in root.findall('.//ns:loc', ns):
            urls.add(loc.text.strip())
    except Exception as e:
        print(f"Error loading sitemap: {e}")
    return urls

def audit_production():
    sitemap_urls = load_sitemap(PRODUCTION_SITEMAP)
    if not sitemap_urls:
        print("No URLs found. Is the site online?")
        return

    print(f"\n--- Starting Production SEO Audit ({len(sitemap_urls)} URLs) ---")
    print(f"Target: {PRODUCTION_DOMAIN}\n")

    results = []
    content_map = defaultdict(list)
    errors = []
    warnings = []
    info = []

    session = requests.Session()
    # Mask as a real browser to see what Google/Users see
    session.headers.update({
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
    })

    for i, url in enumerate(sorted(sitemap_urls), 1):
        print(f"[{i}/{len(sitemap_urls)}] Checking: {url} ...", end="\r")
        try:
            response = session.get(url, timeout=15, allow_redirects=True)
            
            if response.history:
                # Page with redirect
                info.append(f"Page with redirect: {url} (Google sees redirect to {response.url})")
            
            if response.status_code != 200:
                errors.append(f"HTTP {response.status_code} for {url}")
                continue

            soup = BeautifulSoup(response.text, "lxml")
            
            title = soup.title.string if soup.title else None
            description = soup.find("meta", attrs={"name": "description"})
            description = description["content"] if description else None
            canonical = soup.find("link", rel="canonical")
            canonical = canonical["href"] if canonical else None
            h1 = [h.get_text(strip=True) for h in soup.find_all("h1")]
            
            content_hash = get_content_hash(soup)
            
            page_data = {
                "url": url,
                "title": title,
                "description": description,
                "canonical": canonical,
                "h1": h1,
                "hash": content_hash,
                "status": response.status_code
            }
            
            results.append(page_data)
            content_map[content_hash].append(page_data)

        except Exception as e:
            errors.append(f"Failed to fetch {url}: {e}")

    print("\n\nAnalyzing results...")

    # 1. Duplicates Analysis (GSC Style)
    for content_hash, pages in content_map.items():
        if len(pages) > 1:
            canonicals = set(p['canonical'] for p in pages if p['canonical'])
            urls_str = "\n    ".join(p['url'] for p in pages)
            
            if not canonicals:
                errors.append(f"Page is a duplicate. Canonical version not selected by user:\n    {urls_str}")
            elif len(canonicals) > 1:
                warnings.append(f"Duplicate, submitted URL not selected as canonical (Multiple canonicals found):\n    {urls_str}")
            else:
                info.append(f"Duplicate content correctly handled by canonical ({list(canonicals)[0]})")

    # 2. Canonical & SEO Tag Analysis
    for p in results:
        # Canonical Tag
        if not p['canonical']:
            warnings.append(f"Indexing issue: Missing canonical tag (Google will choose one): {p['url']}")
        elif p['canonical'] != p['url']:
            info.append(f"Variant of page with canonical tag: {p['url']} -> {p['canonical']}")
            if not p['canonical'].startswith(PRODUCTION_DOMAIN):
                warnings.append(f"Canonical points to EXTERNAL domain: {p['url']} -> {p['canonical']}")

        # Meta Tags
        if not p['title']: errors.append(f"SEO Error: Missing <title> tag: {p['url']}")
        if not p['description']: warnings.append(f"SEO Warning: Missing <meta description>: {p['url']}")
        if not p['h1']: warnings.append(f"SEO Warning: Missing <h1> tag: {p['url']}")
        elif len(p['h1']) > 1: warnings.append(f"SEO Warning: Multiple <h1> tags: {p['url']} ({len(p['h1'])})")

    # Report
    print(f"\nAudit Summary for {PRODUCTION_DOMAIN}:")
    print(f"- Total pages checked: {len(results)}")
    print(f"- Errors: {len(errors)}")
    print(f"- Warnings: {len(warnings)}")

    print("\n--- ERRORS (Fix immediately) ---")
    if not errors: print("None")
    for e in errors:
        try: print(f"❌ {e}")
        except: print(f"ERROR: {e.encode('ascii', 'ignore').decode()}")

    print("\n--- WARNINGS (Recommended fixes) ---")
    if not warnings: print("None")
    for w in warnings:
        try: print(f"⚠️ {w}")
        except: print(f"WARNING: {w.encode('ascii', 'ignore').decode()}")

    print("\n--- INFO (Status OK / Handled) ---")
    if not info: print("None")
    for i in info:
        try: print(f"✅ {i}")
        except: print(f"INFO: {i.encode('ascii', 'ignore').decode()}")

    print("\n--- Audit Complete ---")

if __name__ == "__main__":
    audit_production()