"""Turn the cached AniList pages into clean tables (data/clean/anime.csv, novels.csv).

Definitions (they are said in the video, so keep them simple):
- isekai: AniList tag "Isekai" with a rank of at least ISEKAI_RANK (rank = how strongly users voted the tag applies)
- first season: no PREQUEL relation to another anime, and no sequel marker in the title ("Season 2", "2nd Season", "Part 2", "II")
- main dataset: TV and ONA, not adult, Japanese, first seasons
"""
import json
import re
from pathlib import Path

import pandas as pd

ROOT = Path(__file__).resolve().parent.parent
RAW = ROOT / "data" / "raw"
CLEAN = ROOT / "data" / "clean"
ISEKAI_RANK = 60

SEQUEL_RE = re.compile(
    r"(\bseason\s*\d|\b\d+(st|nd|rd|th)\s+season|\bpart\s*(\d|ii)|\bcour\s*\d|\b(ii|iii|iv|v|vi)\s*$|"
    r"第.{1,3}期|第.{1,3}部|\bfinal season|2nd|3rd|\bs\d\b)",
    re.I,
)
WORD_RE = re.compile(r"[A-Za-z0-9À-ɏ]+(?:['’][A-Za-z]+)*")
SENTENCE_WORDS = {"i", "i'm", "i’m", "my", "me", "so", "but", "because", "when", "if", "and", "was", "is", "got", "became"}


def words(title):
    return WORD_RE.findall(title or "")


def load(kind):
    rows = {}
    for path in sorted((RAW / kind).glob("*.json")):
        for m in json.loads(path.read_text(encoding="utf-8"))["data"]["Page"]["media"]:
            rows[m["id"]] = m
    return list(rows.values())


def row(m):
    t = m["title"]
    tags = {x["name"]: x["rank"] for x in m["tags"] or []}
    rel = [(e["relationType"], e["node"]["type"]) for e in m["relations"]["edges"]]
    english = t["english"]
    shown = english or t["romaji"] or ""
    w = words(shown)
    lw = [x.lower() for x in w]
    return {
        "id": m["id"],
        "year": (m["startDate"] or {}).get("year"),
        "month": (m["startDate"] or {}).get("month"),
        "format": m["format"],
        "source": m["source"],
        "adult": m["isAdult"],
        "title_english": english,
        "title_romaji": t["romaji"],
        "title_native": t["native"],
        "title_used": shown,
        "has_english": bool(english),
        "words_en": len(w),
        "chars_en": len(shown),
        "words_romaji": len(words(t["romaji"])),
        "chars_native": len(re.sub(r"\s", "", t["native"] or "")),
        "has_comma": "," in shown,
        "has_colon": ":" in shown,
        "sentence_title": "," in shown or any(x in SENTENCE_WORDS for x in lw),
        "first_person": any(x in {"i", "i'm", "i’m", "my", "me"} for x in lw),
        "isekai_rank": tags.get("Isekai", 0),
        "isekai": tags.get("Isekai", 0) >= ISEKAI_RANK,
        "has_prequel": any(r == "PREQUEL" and ty == "ANIME" for r, ty in rel),
        "has_parent": any(r == "PARENT" for r, _ in rel),
        "source_novel_ids": "|".join(str(e["node"]["id"]) for e in m["relations"]["edges"]
                                     # from the anime's side, its source book is an ADAPTATION edge (relation API version 1)
                                     if e["relationType"] == "ADAPTATION" and e["node"]["format"] == "NOVEL"),
        "sequel_marker": bool(SEQUEL_RE.search(shown) or SEQUEL_RE.search(t["romaji"] or "") or SEQUEL_RE.search(t["native"] or "")),
        "score": m["averageScore"],
        "popularity": m["popularity"],
        "favourites": m["favourites"],
        "episodes": m["episodes"],
        "studio": ", ".join(n["name"] for n in m["studios"]["nodes"]),
        "genres": "|".join(m["genres"] or []),
        "tags": "|".join(f"{k}:{v}" for k, v in sorted(tags.items(), key=lambda kv: -kv[1])),
    }


def main():
    CLEAN.mkdir(parents=True, exist_ok=True)
    anime = pd.DataFrame([row(m) for m in load("anime")])
    anime["first_season"] = ~anime.has_prequel & ~anime.sequel_marker
    anime["main"] = anime.format.isin(["TV", "ONA"]) & ~anime.adult & anime.first_season & anime.year.notna()
    anime.sort_values(["year", "month", "id"]).to_csv(CLEAN / "anime.csv", index=False, encoding="utf-8")
    print("anime", len(anime), "main", int(anime.main.sum()), "isekai main", int((anime.main & anime.isekai).sum()))

    if (RAW / "novel").exists():
        nov = pd.DataFrame([row(m) for m in load("novel")])
        nov["first_season"] = ~nov.has_prequel
        nov.sort_values(["year", "id"]).to_csv(CLEAN / "novels.csv", index=False, encoding="utf-8")
        print("novels", len(nov), "isekai", int(nov.isekai.sum()))


if __name__ == "__main__":
    main()