"""Assemble web/index.html from web/template.html + data.

  python3 build.py            -> web/index.html
"""
import glob, json, math, os, re
from collections import defaultdict
from measure import TARGETS

HERE = os.path.dirname(os.path.abspath(__file__))
P = lambda *a: os.path.join(HERE, *a)

PROBE_TO_SITE = {"Sao Paulo": "São Paulo"}


def land():
    """TopoJSON land-110m -> list of polygons (rings of [lon, lat]), 2 dp."""
    t = json.load(open(P("raw/geo/land-110m.json")))
    sx, sy = t["transform"]["scale"]
    tx, ty = t["transform"]["translate"]
    arcs = []
    for arc in t["arcs"]:
        x = y = 0
        pts = []
        for dx, dy in arc:
            x += dx
            y += dy
            pts.append([round(x * sx + tx, 2), round(y * sy + ty, 2)])
        arcs.append(pts)

    def ring(idx):
        out = []
        for i in idx:
            a = arcs[i] if i >= 0 else arcs[~i][::-1]
            out.extend(a if not out else a[1:])
        return out

    polys = []
    for g in t["objects"]["land"]["geometries"]:
        ps = g["arcs"] if g["type"] == "MultiPolygon" else [g["arcs"]]
        for p in ps:
            polys.append([ring(r) for r in p])
    return polys


def wonder():
    d = json.load(open(P("data/wn_clean.json")))
    ids = sorted(d["servers"], key=int)
    index = {s: i for i, s in enumerate(ids)}
    sites = [[d["servers"][s]["name"], d["servers"][s]["country"],
              round(d["servers"][s]["lat"], 4), round(d["servers"][s]["lon"], 4)] for s in ids]
    flat = []
    for r in d["pairs"]:
        flat += [index[r["a"]], index[r["b"]], round(r["rtt"], 2)]
    return dict(sites=sites, pairs=flat, dropped=[f"{m['name']} ({m['country']})" for m in d["mislocated"]])


def live():
    from measure import EXTRA_SITES
    sites = [[t[0], t[2], t[3], t[4], t[1]] for t in TARGETS]   # name, anchor, lat, lon, ip
    sites += [[n, "vultr-" + n.lower(), la, lo, ""] for n, la, lo in EXTRA_SITES]
    sidx = {s[0]: i for i, s in enumerate(sites)}
    runs, cells = [], []
    for f in sorted(glob.glob(P("data/live_*.json"))):
        run = json.load(open(f))
        rn = len(runs)
        runs.append(dict(started=run["started"], packets=run["packets"], file="data/" + os.path.basename(f)))
        probe_key = lambda c: (c["src_city"], c["src_network"], c["src_asn"])
        local = {}
        for c in run["cells"]:
            site = PROBE_TO_SITE.get(c["src_city"], c["src_city"])
            if c["dst_city"] == site and c["min"] is not None:
                local[probe_key(c)] = c["min"]
        for c in run["cells"]:
            src = PROBE_TO_SITE.get(c["src_city"], c["src_city"])
            loc = local.get(probe_key(c))
            if c["min"] is None or loc is None or src == c["dst_city"]:
                continue
            cells.append([sidx[src], sidx[c["dst_city"]], round(c["min"], 1), round(loc, 1), rn, c["src_network"], c["measurement"]])
    from measure import ROUTE_ONLY_SITES, HOME_NEAR
    hf = P("data/home/ping.log")                                # the home connection in Michigan, pinned to its nearest anchor
    vstate = json.load(open(P("data/vps/state.json"))) if os.path.exists(P("data/vps/state.json")) else {"nodes": []}
    vlocal = json.load(open(P("data/vps_cells.json")))["local"] if os.path.exists(P("data/vps_cells.json")) else {}
    if os.path.exists(hf):
        rx = re.compile(r"^(\S+) (\S+) (\S+) .*?= ([\d.]+)/")
        best = {}
        for line in open(hf):
            m = rx.match(line)
            if m:
                best[m.group(2)] = min(best.get(m.group(2), 1e9), float(m.group(4)))
        near = {n: best[n] for n, _ in HOME_NEAR if n in best}
        ref = min(near, key=near.get)
        rname, rla, rlo = ROUTE_ONLY_SITES[0]
        sites.append([rname.replace("(home reference)", "(residential reference)"), ref, rla, rlo, ""])   # display name; data files keep the original
        sidx[rname] = len(sites) - 1
        rn = len(runs)
        runs.append(dict(started=open(hf).readline().split()[0], packets=20, source="home", file="data/home/ping.log"))
        fq_city = {t[2]: t[0] for t in TARGETS}
        vcity = {"syd": "Sydney", "mel": "Melbourne", "jnb": "Johannesburg", "scl": "Santiago", "sao": "São Paulo",
                 "lax": "Los Angeles", "icn": "Seoul", "hnl": "Honolulu"}
        for tgt, rtt in best.items():
            if tgt in dict(HOME_NEAR):
                continue
            city = fq_city.get(tgt) or vcity.get(tgt)
            if city in sidx:
                cells.append([sidx[rname], sidx[city], round(rtt, 1), round(near[ref] + vlocal.get(tgt, 0.0), 1), rn, "home connection, Michigan", "home"])
    vf = P("data/vps_cells.json")                               # our own Vultr servers, as one more run
    if os.path.exists(vf):
        v = json.load(open(vf))
        rn = len(runs)
        st = min(n.get("started", "") for n in v["nodes"])
        ends = [ln.split()[0] for f in glob.glob(P("data/vps/*/ping.log")) for ln in open(f) if ln.strip()]
        runs.append(dict(started=st, ended=max(ends), rounds=max(c["runs"] for c in v["cells"]), packets=20, source="vultr", file="data/vps/"))
        for c in v["cells"]:
            if c["a"] in sidx and c["b"] in sidx:
                cells.append([sidx[c["a"]], sidx[c["b"]], round(c["rtt"], 1), round(c["allow"], 1), rn, f"our Vultr server gl-{c['src']}", c["src"]])
    return dict(sites=sites, runs=runs, cells=cells)


def sweep(wn):
    """Impossible-pair counts across each flat model's free parameter (WN set)."""
    from models import dualpole_xy, mercator_xy, floor_ms, C_FIBER, C_VACUUM
    S, flat = wn["sites"], wn["pairs"]
    out = {}
    for name, f in (("dualpole", dualpole_xy), ("mercator", mercator_xy)):
        rows = []
        for p in range(-180, 180, 5):
            xy = [f(s[2], s[3], p) for s in S]
            fib = vac = 0
            for k in range(0, len(flat), 3):
                a, b, rtt = flat[k], flat[k + 1], flat[k + 2]
                km = max(0.0, math.dist(xy[a], xy[b]) - 200)
                fib += rtt + 1 < floor_ms(km, C_FIBER)
                vac += rtt + 1 < floor_ms(km, C_VACUUM)
            rows.append([p, fib, vac])
        out[name] = rows
    return out


BYO_CITIES = ["Sydney", "Melbourne", "Brisbane", "Perth", "Adelaide", "Hobart", "Auckland", "Wellington", "Christchurch",
              "Nouméa", "Suva", "Jakarta", "Singapore", "Tokyo", "Seoul", "Hong Kong", "Mumbai", "Dubai", "Johannesburg",
              "Cape Town", "Durban", "Luanda", "Nairobi", "Lagos", "São Paulo", "Fortaleza", "Buenos Aires", "Santiago",
              "Montevideo", "Lima", "Bogotá", "Mexico City", "Los Angeles", "Seattle", "Chicago", "New York", "Miami",
              "London", "Frankfurt", "Stockholm", "Reykjavik", "Honolulu"]


def byo():
    """Cities for the 'your location' box (GeoNames) and one RIPE Atlas anchor per target city."""
    import csv
    oce = {"AU", "NZ", "NC", "FJ", "PG", "PF", "WS", "TO", "VU"}
    cities = []
    for r in csv.reader(open(P("raw/geonames/cities15000.txt"), encoding="utf-8"), delimiter="\t", quoting=csv.QUOTE_NONE):
        pop = int(r[14] or 0)
        if pop >= 150000 or (r[8] in oce and pop >= 30000):
            cities.append([r[1], r[8], round(float(r[4]), 3), round(float(r[5]), 3), pop])
    cities.sort(key=lambda c: -c[4])
    anchors = json.load(open(P("raw/ripe_anchors_all.json")))
    live_ip = {t[0]: t[1] for t in TARGETS}
    norm = lambda s: s.lower().replace("ã", "a").replace("é", "e").replace("á", "a").replace("ó", "o").replace("í", "i")
    picked = []
    for city in BYO_CITIES:
        cand = [a for a in anchors if not a.get("is_disabled") and not a.get("date_decommissioned") and a.get("ip_v4")
                and norm(a["city"]).startswith(norm(city))]
        if not cand:
            continue
        pref = [a for a in cand if a["ip_v4"] == live_ip.get(city)]
        a = (pref or sorted(cand, key=lambda a: a["id"]))[0]
        lon, lat = a["geometry"]["coordinates"]
        picked.append([city, a["country"], round(lat, 3), round(lon, 3), a["ip_v4"], a["fqdn"].split(".")[0]])
    return dict(cities=[c[:4] for c in cities], anchors=picked)


def traces():
    t = json.load(open(P("data/trace.json")))
    A = {x["fqdn"].split(".")[0]: x for x in json.load(open(P("raw/ripe_anchors_all.json")))}
    for r in t:
        for k in ("src", "dst"):
            lon, lat = A[r[k]]["geometry"]["coordinates"]
            r[k + "_ll"] = [round(lat, 4), round(lon, 4)]
        if r["cable"] == "tasman-global-access-tga-cable":
            r["assumed"] = "the cable is assumed from its length; both routers at the jump belong to Vocus, which is not a listed owner of TGA."
    return t


def nets():
    """Network attribution from asmap.py, compacted for the page."""
    f = P("data/asmap.json")
    if not os.path.exists(f):
        return None
    a = json.load(open(f))
    anc = {x["fqdn"].split(".")[0]: x for x in json.load(open(P("raw/ripe_anchors_all.json")))}
    vst = {n["code"]: n["city"] for n in json.load(open(P("data/vps/state.json")))["nodes"]}
    city = lambda fq: (anc.get(fq) or {}).get("city", fq)
    ends, gp = [], {}
    for e in a["endpoints"]:
        sid = e["site"].split(":", 1)[1]
        if e["kind"] == "RIPE anchor":
            ends.append(dict(kind="anchor", id=sid, city=city(sid), company=(anc.get(sid) or {}).get("company", ""), claimed_asn=e["claimed_asn"],
                             bgp_origin=e["bgp_origin"], name=e["name"], prefix=e["prefix"], match=e["match"], rpki=e.get("rpki", [])))
        elif e["kind"] == "our Vultr server":
            ends.append(dict(kind="vultr", id=sid, city=vst.get(sid, sid), bgp_origin=e["bgp_origin"], name=e["name"], prefix=e["prefix"], match=e["match"], rpki=e.get("rpki", [])))
        else:
            k = e["claimed_asn"]
            gp.setdefault(k, dict(asn=k, network=e["network"], probes=0))["probes"] += 1
    ends.sort(key=lambda e: (e["kind"], e["city"]))
    cname = {c["id"]: c["name"].replace(" Cable", "") for c in json.load(open(P("data/cables.json")))["cables"]}
    xs = [dict(src_city=city(x["src"].split(":")[-1]), dst_city=city(x["dst"].split(":")[-1]), cable_name=cname.get(x["cable"]) if x["cable"] else None,
               owners=x["owners"], path=x["path"], reading=x["reading"], assumed=x["cable"] == "tasman-global-access-tga-cable") for x in a["crossings"]]
    return dict(summary=a["summary"], traces=a["traces"], endpoints=ends, gp=sorted(gp.values(), key=lambda x: -x["probes"]),
                transit=a["transit"], ixps=a["ixps"], crossings=xs)


def cables():
    d = json.load(open(P("data/cables.json")))
    keep = ("id", "name", "length", "rfs", "south", "landings", "span", "route", "span_ends")
    return dict(cables=[{k: c[k] for k in keep} for c in d["cables"]], dropped=d["dropped"])


def netlines():
    """Drawn route of every cable in service, for the background network."""
    meta = {}
    for f in glob.glob(P("raw/tg/cables/*.json")):
        c = json.load(open(f))
        meta[c["id"]] = c
    out = {}
    for f in json.load(open(P("raw/tg/cable-geo.json")))["features"]:
        c = meta.get(f["properties"]["id"])
        if not c or c.get("is_planned") or (c.get("rfs_year") or 0) > 2026:
            continue
        out.setdefault(c["id"], []).extend([[round(x, 2), round(y, 2)] for x, y in line] for line in f["geometry"]["coordinates"])
    return out


if __name__ == "__main__":
    wn = wonder()
    data = dict(land=land(), wn=wn, live=live(), sweep=sweep(wn),
                cables=cables(), trace=traces(), byo=byo(),
                network=json.load(open(P("data/network.json"))), netlines=netlines(),
                flat=json.load(open(P("data/flatfit.json"))) if os.path.exists(P("data/flatfit.json")) else None,
                pingmap=json.load(open(P("data/pingmap.json")))["stress"], net=nets())
    blob = json.dumps(data, separators=(",", ":"), ensure_ascii=False)
    html = open(P("web/template.html")).read().replace("/*__DATA__*/null", blob)
    open(P("web/index.html"), "w").write(html)
    # standalone copy for hand-off: full document, d3 embedded so it opens without a network connection
    d3 = open(P("raw/lib/d3.min.js")).read()
    tag = '<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/7.9.0/d3.min.js"></script>'
    assert html.count(tag) == 1 and "</script>" not in d3.replace("<\\/script>", "")
    head, body = html.split("</style>", 1)
    doc = ('<!doctype html>\n<html lang="en">\n<head>\n<meta charset="utf-8">\n'
           '<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover">\n'
           + head + "</style>\n</head>\n<body>\n" + body.replace(tag, "<script>" + d3 + "</script>") + "\n</body>\n</html>\n")
    os.makedirs(P("handoff"), exist_ok=True)
    open(P("handoff/cable_latency_survey_report.html"), "w").write(doc)
    os.makedirs(P("site"), exist_ok=True)                      # deployed as https://pingstats.aethercosmology.com/
    open(P("site/index.html"), "w").write(doc)
    print(f"web/index.html {len(html)/1e6:.2f} MB; live cells {len(data['live']['cells'])}, "
          f"runs {len(data['live']['runs'])}, wn pairs {len(wn['pairs'])//3}, cables {len(data['cables']['cables'])}, "
          f"cities {len(data['byo']['cities'])}, anchors {[a[0] for a in data['byo']['anchors']]}")