#!/usr/bin/env python3
"""Fetch fresh App Store reviews and ADD them to your dataset.

Free. No API key, no account. Apple publishes a public RSS feed of recent
reviews for every app, 10 pages deep, ~50 reviews per page.

Usage:
    python3 fetch.py --app spotify

Run it whenever you want. It only appends reviews you don't already have, so
your dataset grows over time instead of being replaced. That growing history is
what lets you eventually answer "is this complaint getting worse or better?",
which a single snapshot can never tell you.
"""

import argparse
import csv
import json
import ssl
import time
import urllib.request
from datetime import date
from pathlib import Path

# Add any app here. The number is its App Store ID, visible in the app's URL.
APPS = {
    "spotify": 324684580,
    "applemusic": 1108187390,
    "youtubemusic": 1017492454,
    "sonos": 1488977981,
}

FEED = (
    "https://itunes.apple.com/rss/customerreviews/page={page}/id={app_id}"
    "/sortby=mostrecent/json?l=en&cc=us"
)

FIELDS = ["review_id", "date", "rating", "author", "title", "review", "fetched_on"]


def _ssl_context():
    """Build an SSL context that works on stock macOS Python.

    Python installed from python.org does not wire itself into Apple's
    certificate store, so HTTPS fails with CERTIFICATE_VERIFY_FAILED until you
    run its "Install Certificates.command". Using the `certifi` bundle directly
    sidesteps that entirely.
    """
    try:
        import certifi

        return ssl.create_default_context(cafile=certifi.where())
    except ImportError:
        return ssl.create_default_context()


def fetch_page(app_id, page):
    url = FEED.format(page=page, app_id=app_id)
    request = urllib.request.Request(
        url, headers={"User-Agent": "opportunity-agent/1.0 (personal PM research)"}
    )
    with urllib.request.urlopen(request, timeout=30, context=_ssl_context()) as response:
        data = json.loads(response.read().decode("utf-8"))

    entries = data.get("feed", {}).get("entry", [])
    # Page 1 sometimes leads with an app-info entry that has no rating.
    return [e for e in entries if "im:rating" in e]


def to_record(entry):
    return {
        "review_id": entry["id"]["label"],
        "date": entry["updated"]["label"][:10],
        "rating": entry["im:rating"]["label"],
        "author": entry["author"]["name"]["label"],
        "title": entry["title"]["label"],
        # Newlines inside a CSV cell make the file painful to read by hand.
        "review": entry["content"]["label"].replace("\n", " ").strip(),
        "fetched_on": date.today().isoformat(),
    }


def load_existing(path):
    """Return existing rows and the set of review IDs already saved."""
    if not path.exists():
        return [], set()
    with open(path, newline="", encoding="utf-8") as f:
        rows = list(csv.DictReader(f))
    return rows, {r.get("review_id") for r in rows}


def main():
    parser = argparse.ArgumentParser(description="Fetch App Store reviews (free).")
    parser.add_argument("--app", default="spotify", choices=sorted(APPS))
    parser.add_argument("--pages", type=int, default=10, help="1-10, ~50 reviews each")
    parser.add_argument("--out", default=None, help="CSV to append to")
    args = parser.parse_args()

    app_id = APPS[args.app]
    out_path = Path(args.out or f"data/{args.app}-appstore-reviews.csv")
    out_path.parent.mkdir(parents=True, exist_ok=True)

    existing_rows, seen_ids = load_existing(out_path)
    print(f"\nDataset: {out_path}")
    print(f"  already have {len(existing_rows)} reviews")

    new_rows = []
    for page in range(1, min(args.pages, 10) + 1):
        try:
            entries = fetch_page(app_id, page)
        except Exception as error:
            print(f"  page {page}: failed ({error}), skipping")
            continue

        fresh = 0
        for entry in entries:
            record = to_record(entry)
            if record["review_id"] not in seen_ids:
                seen_ids.add(record["review_id"])
                new_rows.append(record)
                fresh += 1
        print(f"  page {page}: {len(entries)} fetched, {fresh} new")
        time.sleep(1)  # be polite to Apple's servers

    if not new_rows:
        print("\nNo new reviews since last fetch. Dataset unchanged.\n")
        return

    all_rows = existing_rows + new_rows
    with open(out_path, "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=FIELDS, extrasaction="ignore")
        writer.writeheader()
        writer.writerows(all_rows)

    dates = sorted(r["date"] for r in all_rows if r.get("date"))
    print(f"\nAdded {len(new_rows)} new reviews. Total now {len(all_rows)}.")
    if dates:
        print(f"Coverage: {dates[0]} to {dates[-1]}")
    print(f"Saved to {out_path}\n")


if __name__ == "__main__":
    main()