|
|
@@ -0,0 +1,165 @@
|
|
|
+import sys
|
|
|
+import hashlib
|
|
|
+import requests
|
|
|
+import xml.etree.ElementTree as ET
|
|
|
+from bs4 import BeautifulSoup
|
|
|
+from collections import defaultdict
|
|
|
+from urllib.parse import urlparse
|
|
|
+
|
|
|
+# Force UTF-8 for console output if possible
|
|
|
+if sys.stdout.encoding != 'utf-8':
|
|
|
+ try:
|
|
|
+ import codecs
|
|
|
+ sys.stdout = codecs.getwriter('utf-8')(sys.stdout.buffer, 'strict')
|
|
|
+ sys.stderr = codecs.getwriter('utf-8')(sys.stderr.buffer, 'strict')
|
|
|
+ except:
|
|
|
+ pass
|
|
|
+
|
|
|
+# Production Settings
|
|
|
+PRODUCTION_DOMAIN = "https://radionica3d.me"
|
|
|
+PRODUCTION_SITEMAP = f"{PRODUCTION_DOMAIN}/sitemap.xml"
|
|
|
+
|
|
|
+def get_content_hash(soup):
|
|
|
+ # Remove elements that change frequently or are boilerplate
|
|
|
+ clean_soup = BeautifulSoup(str(soup), "lxml")
|
|
|
+ for tag in clean_soup(["script", "style", "header", "footer", "nav", "aside", "noscript"]):
|
|
|
+ tag.decompose()
|
|
|
+
|
|
|
+ text = clean_soup.get_text(separator=' ', strip=True)
|
|
|
+ return hashlib.md5(text.encode('utf-8')).hexdigest()
|
|
|
+
|
|
|
+def load_sitemap(url):
|
|
|
+ print(f"Loading production sitemap from {url}...")
|
|
|
+ urls = set()
|
|
|
+ try:
|
|
|
+ response = requests.get(url, timeout=15)
|
|
|
+ response.raise_for_status()
|
|
|
+
|
|
|
+ root = ET.fromstring(response.content)
|
|
|
+ ns = {'ns': 'http://www.sitemaps.org/schemas/sitemap/0.9'}
|
|
|
+ for loc in root.findall('.//ns:loc', ns):
|
|
|
+ urls.add(loc.text.strip())
|
|
|
+ except Exception as e:
|
|
|
+ print(f"Error loading sitemap: {e}")
|
|
|
+ return urls
|
|
|
+
|
|
|
+def audit_production():
|
|
|
+ sitemap_urls = load_sitemap(PRODUCTION_SITEMAP)
|
|
|
+ if not sitemap_urls:
|
|
|
+ print("No URLs found. Is the site online?")
|
|
|
+ return
|
|
|
+
|
|
|
+ print(f"\n--- Starting Production SEO Audit ({len(sitemap_urls)} URLs) ---")
|
|
|
+ print(f"Target: {PRODUCTION_DOMAIN}\n")
|
|
|
+
|
|
|
+ results = []
|
|
|
+ content_map = defaultdict(list)
|
|
|
+ errors = []
|
|
|
+ warnings = []
|
|
|
+ info = []
|
|
|
+
|
|
|
+ session = requests.Session()
|
|
|
+ # Mask as a real browser to see what Google/Users see
|
|
|
+ session.headers.update({
|
|
|
+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
|
+ })
|
|
|
+
|
|
|
+ for i, url in enumerate(sorted(sitemap_urls), 1):
|
|
|
+ print(f"[{i}/{len(sitemap_urls)}] Checking: {url} ...", end="\r")
|
|
|
+ try:
|
|
|
+ response = session.get(url, timeout=15, allow_redirects=True)
|
|
|
+
|
|
|
+ if response.history:
|
|
|
+ # Page with redirect
|
|
|
+ info.append(f"Page with redirect: {url} (Google sees redirect to {response.url})")
|
|
|
+
|
|
|
+ if response.status_code != 200:
|
|
|
+ errors.append(f"HTTP {response.status_code} for {url}")
|
|
|
+ continue
|
|
|
+
|
|
|
+ soup = BeautifulSoup(response.text, "lxml")
|
|
|
+
|
|
|
+ title = soup.title.string if soup.title else None
|
|
|
+ description = soup.find("meta", attrs={"name": "description"})
|
|
|
+ description = description["content"] if description else None
|
|
|
+ canonical = soup.find("link", rel="canonical")
|
|
|
+ canonical = canonical["href"] if canonical else None
|
|
|
+ h1 = [h.get_text(strip=True) for h in soup.find_all("h1")]
|
|
|
+
|
|
|
+ content_hash = get_content_hash(soup)
|
|
|
+
|
|
|
+ page_data = {
|
|
|
+ "url": url,
|
|
|
+ "title": title,
|
|
|
+ "description": description,
|
|
|
+ "canonical": canonical,
|
|
|
+ "h1": h1,
|
|
|
+ "hash": content_hash,
|
|
|
+ "status": response.status_code
|
|
|
+ }
|
|
|
+
|
|
|
+ results.append(page_data)
|
|
|
+ content_map[content_hash].append(page_data)
|
|
|
+
|
|
|
+ except Exception as e:
|
|
|
+ errors.append(f"Failed to fetch {url}: {e}")
|
|
|
+
|
|
|
+ print("\n\nAnalyzing results...")
|
|
|
+
|
|
|
+ # 1. Duplicates Analysis (GSC Style)
|
|
|
+ for content_hash, pages in content_map.items():
|
|
|
+ if len(pages) > 1:
|
|
|
+ canonicals = set(p['canonical'] for p in pages if p['canonical'])
|
|
|
+ urls_str = "\n ".join(p['url'] for p in pages)
|
|
|
+
|
|
|
+ if not canonicals:
|
|
|
+ errors.append(f"Page is a duplicate. Canonical version not selected by user:\n {urls_str}")
|
|
|
+ elif len(canonicals) > 1:
|
|
|
+ warnings.append(f"Duplicate, submitted URL not selected as canonical (Multiple canonicals found):\n {urls_str}")
|
|
|
+ else:
|
|
|
+ info.append(f"Duplicate content correctly handled by canonical ({list(canonicals)[0]})")
|
|
|
+
|
|
|
+ # 2. Canonical & SEO Tag Analysis
|
|
|
+ for p in results:
|
|
|
+ # Canonical Tag
|
|
|
+ if not p['canonical']:
|
|
|
+ warnings.append(f"Indexing issue: Missing canonical tag (Google will choose one): {p['url']}")
|
|
|
+ elif p['canonical'] != p['url']:
|
|
|
+ info.append(f"Variant of page with canonical tag: {p['url']} -> {p['canonical']}")
|
|
|
+ if not p['canonical'].startswith(PRODUCTION_DOMAIN):
|
|
|
+ warnings.append(f"Canonical points to EXTERNAL domain: {p['url']} -> {p['canonical']}")
|
|
|
+
|
|
|
+ # Meta Tags
|
|
|
+ if not p['title']: errors.append(f"SEO Error: Missing <title> tag: {p['url']}")
|
|
|
+ if not p['description']: warnings.append(f"SEO Warning: Missing <meta description>: {p['url']}")
|
|
|
+ if not p['h1']: warnings.append(f"SEO Warning: Missing <h1> tag: {p['url']}")
|
|
|
+ elif len(p['h1']) > 1: warnings.append(f"SEO Warning: Multiple <h1> tags: {p['url']} ({len(p['h1'])})")
|
|
|
+
|
|
|
+ # Report
|
|
|
+ print(f"\nAudit Summary for {PRODUCTION_DOMAIN}:")
|
|
|
+ print(f"- Total pages checked: {len(results)}")
|
|
|
+ print(f"- Errors: {len(errors)}")
|
|
|
+ print(f"- Warnings: {len(warnings)}")
|
|
|
+
|
|
|
+ print("\n--- ERRORS (Fix immediately) ---")
|
|
|
+ if not errors: print("None")
|
|
|
+ for e in errors:
|
|
|
+ try: print(f"❌ {e}")
|
|
|
+ except: print(f"ERROR: {e.encode('ascii', 'ignore').decode()}")
|
|
|
+
|
|
|
+ print("\n--- WARNINGS (Recommended fixes) ---")
|
|
|
+ if not warnings: print("None")
|
|
|
+ for w in warnings:
|
|
|
+ try: print(f"⚠️ {w}")
|
|
|
+ except: print(f"WARNING: {w.encode('ascii', 'ignore').decode()}")
|
|
|
+
|
|
|
+ print("\n--- INFO (Status OK / Handled) ---")
|
|
|
+ if not info: print("None")
|
|
|
+ for i in info:
|
|
|
+ try: print(f"✅ {i}")
|
|
|
+ except: print(f"INFO: {i.encode('ascii', 'ignore').decode()}")
|
|
|
+
|
|
|
+ print("\n--- Audit Complete ---")
|
|
|
+
|
|
|
+if __name__ == "__main__":
|
|
|
+ audit_production()
|
unknown