gateway-gamers.net/sites/40k-rankings/statcheck-elo-scraper/pure_python_elo_scrape.py

#!/usr/bin/env python3
import base64
import html
import json
import re
import sys
from pathlib import Path
from urllib.error import HTTPError
from urllib.parse import quote
from urllib.request import Request

from download_and_parse import download, fetch, xlsx_to_csv


STATCHECK_PAGE_URL = "https://www.stat-check.com/elo"
IFRAME_SRC_RE = re.compile(r'<iframe[^>]+src="(https://1drv\.ms/x/[^"]+)"', re.I)

# Fallback for when the link can't be read from the Stat Check page. Stat Check
# replaces the share link from time to time, so this can go stale.
EMBED_URL = (
    "https://1drv.ms/x/c/9f33f29504402fa1/"
    "IQSM-tN0M6OcTaaLY3OtMA2vAYx131Ze6y5N0aHYpoCCJ1g"
    "?Item='Sheet1'!A1%3AL44394"
    "&wdHideGridlines=True&wdInConfigurator=True&wdInConfigurator=True"
)

BADGER_APP_ID = "00000000-0000-0000-0000-0000481710a4"


def request_json(url: str, *, method: str = "GET", data: bytes | None = None, headers: dict[str, str] | None = None) -> dict:
    req = Request(url, data=data, method=method, headers=headers or {})
    return json.loads(fetch(req).decode("utf-8"))


def discover_embed_url(page_url: str = STATCHECK_PAGE_URL) -> str:
    req = Request(page_url, headers={"User-Agent": "Mozilla/5.0"})
    page = fetch(req).decode("utf-8", errors="replace")
    match = IFRAME_SRC_RE.search(page)
    if not match:
        raise RuntimeError(f"No OneDrive workbook iframe found on {page_url}")
    return html.unescape(match.group(1))


def badger_token() -> str:
    payload = json.dumps({"appid": BADGER_APP_ID}, separators=(",", ":")).encode()
    data = request_json(
        "https://api-badgerp.svc.ms/v1.0/token",
        method="POST",
        data=payload,
        headers={
            "Content-Type": "application/json",
            "Origin": "https://onedrive.live.com",
            "Referer": "https://onedrive.live.com/",
            "User-Agent": "Mozilla/5.0",
        },
    )
    if data.get("authScheme") != "badger" or not data.get("token"):
        raise RuntimeError(f"Unexpected Badger token response: {data!r}")
    return data["token"]


def share_id(url: str) -> str:
    encoded = base64.urlsafe_b64encode(url.encode()).decode().rstrip("=")
    return "u!" + encoded


def download_url_for_embed(embed_url: str) -> str:
    token = badger_token()
    select = "id,openWith,officebundle,currentUserRole,eTag,name,size,content.downloadUrl,file,sharepointIds,sensitivityLabel,webUrl,webDavUrl,parentReference,vault"
    url = (
        "https://my.microsoftpersonalcontent.com/_api/v2.0/shares/"
        f"{share_id(embed_url)}/driveItem?action=EmbedView&$select={quote(select, safe=',')}"
    )
    data = request_json(
        url,
        method="POST",
        data=b"",
        headers={
            "Accept": "application/json",
            "Authorization": f"Badger {token}",
            "Origin": "https://onedrive.live.com",
            "Prefer": "autoredeem",
            "Referer": "https://onedrive.live.com/",
            "User-Agent": "Mozilla/5.0",
        },
    )
    if "@content.downloadUrl" not in data:
        raise RuntimeError(f"No download URL in metadata response: {data!r}")
    return data["@content.downloadUrl"]


def resolve_download_url() -> str:
    """Resolve the workbook download URL, preferring the link currently on the Stat Check page."""
    try:
        embed_url = discover_embed_url()
    except Exception as e:
        print(f"  Could not read link from {STATCHECK_PAGE_URL} ({e}); using fallback EMBED_URL", file=sys.stderr)
        return download_url_for_embed(EMBED_URL)

    print(f"  Using link from {STATCHECK_PAGE_URL}: {embed_url}")
    if embed_url == EMBED_URL:
        return download_url_for_embed(embed_url)
    print("  (differs from EMBED_URL; consider updating the fallback)", file=sys.stderr)
    try:
        return download_url_for_embed(embed_url)
    except HTTPError as e:
        if e.code != 404:
            raise
        print("  Page link returned 404; trying fallback EMBED_URL", file=sys.stderr)
        return download_url_for_embed(EMBED_URL)


def main() -> None:
    xlsx_path = Path(sys.argv[1] if len(sys.argv) > 1 else "elo-pure.xlsx")
    csv_path = Path(sys.argv[2] if len(sys.argv) > 2 else "elo-pure.csv")
    url = resolve_download_url()
    download(url, xlsx_path)
    xlsx_to_csv(xlsx_path, csv_path)
    print(f"downloaded={xlsx_path.stat().st_size} bytes csv={csv_path.stat().st_size} bytes")


if __name__ == "__main__":
    main()