ELO scraper: discover the embed link and retry transient failures

Read the current OneDrive link from the stat-check.com/elo iframe on
each run, falling back to EMBED_URL if the page can't be read or its
link 404s. Retry 429/5xx and network errors up to three times with
backoff, after a one-off 503 from the Badger token endpoint failed a run.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

sites/40k-rankings/statcheck-elo-scraper/pure_python_elo_scrape.py

@@ -1,14 +1,22 @@
 #!/usr/bin/env python3
 import base64
+import html
 import json
+import re
 import sys
 from pathlib import Path
+from urllib.error import HTTPError
 from urllib.parse import quote
-from urllib.request import Request, urlopen
+from urllib.request import Request
 
-from download_and_parse import download, xlsx_to_csv
+from download_and_parse import download, fetch, xlsx_to_csv
 
 
+STATCHECK_PAGE_URL = "https://www.stat-check.com/elo"
+IFRAME_SRC_RE = re.compile(r'<iframe[^>]+src="(https://1drv\.ms/x/[^"]+)"', re.I)
+
+# Fallback for when the link can't be read from the Stat Check page. Stat Check
+# replaces the share link from time to time, so this can go stale.
 EMBED_URL = (
     "https://1drv.ms/x/c/9f33f29504402fa1/"
     "IQSM-tN0M6OcTaaLY3OtMA2vAYx131Ze6y5N0aHYpoCCJ1g"
@@ -21,8 +29,16 @@ BADGER_APP_ID = "00000000-0000-0000-0000-0000481710a4"
 
 def request_json(url: str, *, method: str = "GET", data: bytes | None = None, headers: dict[str, str] | None = None) -> dict:
     req = Request(url, data=data, method=method, headers=headers or {})
-    with urlopen(req, timeout=60) as resp:
-        return json.loads(resp.read().decode("utf-8"))
+    return json.loads(fetch(req).decode("utf-8"))
+
+
+def discover_embed_url(page_url: str = STATCHECK_PAGE_URL) -> str:
+    req = Request(page_url, headers={"User-Agent": "Mozilla/5.0"})
+    page = fetch(req).decode("utf-8", errors="replace")
+    match = IFRAME_SRC_RE.search(page)
+    if not match:
+        raise RuntimeError(f"No OneDrive workbook iframe found on {page_url}")
+    return html.unescape(match.group(1))
 
 
 def badger_token() -> str:
@@ -73,10 +89,31 @@ def download_url_for_embed(embed_url: str) -> str:
     return data["@content.downloadUrl"]
 
 
+def resolve_download_url() -> str:
+    """Resolve the workbook download URL, preferring the link currently on the Stat Check page."""
+    try:
+        embed_url = discover_embed_url()
+    except Exception as e:
+        print(f"  Could not read link from {STATCHECK_PAGE_URL} ({e}); using fallback EMBED_URL", file=sys.stderr)
+        return download_url_for_embed(EMBED_URL)
+
+    print(f"  Using link from {STATCHECK_PAGE_URL}: {embed_url}")
+    if embed_url == EMBED_URL:
+        return download_url_for_embed(embed_url)
+    print("  (differs from EMBED_URL; consider updating the fallback)", file=sys.stderr)
+    try:
+        return download_url_for_embed(embed_url)
+    except HTTPError as e:
+        if e.code != 404:
+            raise
+        print("  Page link returned 404; trying fallback EMBED_URL", file=sys.stderr)
+        return download_url_for_embed(EMBED_URL)
+
+
 def main() -> None:
     xlsx_path = Path(sys.argv[1] if len(sys.argv) > 1 else "elo-pure.xlsx")
     csv_path = Path(sys.argv[2] if len(sys.argv) > 2 else "elo-pure.csv")
-    url = download_url_for_embed(EMBED_URL)
+    url = resolve_download_url()
     download(url, xlsx_path)
     xlsx_to_csv(xlsx_path, csv_path)
     print(f"downloaded={xlsx_path.stat().st_size} bytes csv={csv_path.stat().st_size} bytes")