Toby
258 lines
13 KiB
Python
258 lines
13 KiB
Python
"""Assemble web/index.html from web/template.html + data.
|
|
|
|
python3 build.py -> web/index.html
|
|
"""
|
|
import glob, json, math, os, re
|
|
from collections import defaultdict
|
|
from measure import TARGETS
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
P = lambda *a: os.path.join(HERE, *a)
|
|
|
|
PROBE_TO_SITE = {"Sao Paulo": "São Paulo"}
|
|
|
|
|
|
def land():
|
|
"""TopoJSON land-110m -> list of polygons (rings of [lon, lat]), 2 dp."""
|
|
t = json.load(open(P("raw/geo/land-110m.json")))
|
|
sx, sy = t["transform"]["scale"]
|
|
tx, ty = t["transform"]["translate"]
|
|
arcs = []
|
|
for arc in t["arcs"]:
|
|
x = y = 0
|
|
pts = []
|
|
for dx, dy in arc:
|
|
x += dx
|
|
y += dy
|
|
pts.append([round(x * sx + tx, 2), round(y * sy + ty, 2)])
|
|
arcs.append(pts)
|
|
|
|
def ring(idx):
|
|
out = []
|
|
for i in idx:
|
|
a = arcs[i] if i >= 0 else arcs[~i][::-1]
|
|
out.extend(a if not out else a[1:])
|
|
return out
|
|
|
|
polys = []
|
|
for g in t["objects"]["land"]["geometries"]:
|
|
ps = g["arcs"] if g["type"] == "MultiPolygon" else [g["arcs"]]
|
|
for p in ps:
|
|
polys.append([ring(r) for r in p])
|
|
return polys
|
|
|
|
|
|
def wonder():
|
|
d = json.load(open(P("data/wn_clean.json")))
|
|
ids = sorted(d["servers"], key=int)
|
|
index = {s: i for i, s in enumerate(ids)}
|
|
sites = [[d["servers"][s]["name"], d["servers"][s]["country"],
|
|
round(d["servers"][s]["lat"], 4), round(d["servers"][s]["lon"], 4)] for s in ids]
|
|
flat = []
|
|
for r in d["pairs"]:
|
|
flat += [index[r["a"]], index[r["b"]], round(r["rtt"], 2)]
|
|
return dict(sites=sites, pairs=flat, dropped=[f"{m['name']} ({m['country']})" for m in d["mislocated"]])
|
|
|
|
|
|
def live():
|
|
from measure import EXTRA_SITES
|
|
sites = [[t[0], t[2], t[3], t[4], t[1]] for t in TARGETS] # name, anchor, lat, lon, ip
|
|
sites += [[n, "vultr-" + n.lower(), la, lo, ""] for n, la, lo in EXTRA_SITES]
|
|
sidx = {s[0]: i for i, s in enumerate(sites)}
|
|
runs, cells = [], []
|
|
for f in sorted(glob.glob(P("data/live_*.json"))):
|
|
run = json.load(open(f))
|
|
rn = len(runs)
|
|
runs.append(dict(started=run["started"], packets=run["packets"], file="data/" + os.path.basename(f)))
|
|
probe_key = lambda c: (c["src_city"], c["src_network"], c["src_asn"])
|
|
local = {}
|
|
for c in run["cells"]:
|
|
site = PROBE_TO_SITE.get(c["src_city"], c["src_city"])
|
|
if c["dst_city"] == site and c["min"] is not None:
|
|
local[probe_key(c)] = c["min"]
|
|
for c in run["cells"]:
|
|
src = PROBE_TO_SITE.get(c["src_city"], c["src_city"])
|
|
loc = local.get(probe_key(c))
|
|
if c["min"] is None or loc is None or src == c["dst_city"]:
|
|
continue
|
|
cells.append([sidx[src], sidx[c["dst_city"]], round(c["min"], 1), round(loc, 1), rn, c["src_network"], c["measurement"]])
|
|
from measure import ROUTE_ONLY_SITES, HOME_NEAR
|
|
hf = P("data/home/ping.log") # the home connection in Michigan, pinned to its nearest anchor
|
|
vstate = json.load(open(P("data/vps/state.json"))) if os.path.exists(P("data/vps/state.json")) else {"nodes": []}
|
|
vlocal = json.load(open(P("data/vps_cells.json")))["local"] if os.path.exists(P("data/vps_cells.json")) else {}
|
|
if os.path.exists(hf):
|
|
rx = re.compile(r"^(\S+) (\S+) (\S+) .*?= ([\d.]+)/")
|
|
best = {}
|
|
for line in open(hf):
|
|
m = rx.match(line)
|
|
if m:
|
|
best[m.group(2)] = min(best.get(m.group(2), 1e9), float(m.group(4)))
|
|
near = {n: best[n] for n, _ in HOME_NEAR if n in best}
|
|
ref = min(near, key=near.get)
|
|
rname, rla, rlo = ROUTE_ONLY_SITES[0]
|
|
sites.append([rname.replace("(home reference)", "(residential reference)"), ref, rla, rlo, ""]) # display name; data files keep the original
|
|
sidx[rname] = len(sites) - 1
|
|
rn = len(runs)
|
|
runs.append(dict(started=open(hf).readline().split()[0], packets=20, source="home", file="data/home/ping.log"))
|
|
fq_city = {t[2]: t[0] for t in TARGETS}
|
|
vcity = {"syd": "Sydney", "mel": "Melbourne", "jnb": "Johannesburg", "scl": "Santiago", "sao": "São Paulo",
|
|
"lax": "Los Angeles", "icn": "Seoul", "hnl": "Honolulu"}
|
|
for tgt, rtt in best.items():
|
|
if tgt in dict(HOME_NEAR):
|
|
continue
|
|
city = fq_city.get(tgt) or vcity.get(tgt)
|
|
if city in sidx:
|
|
cells.append([sidx[rname], sidx[city], round(rtt, 1), round(near[ref] + vlocal.get(tgt, 0.0), 1), rn, "home connection, Michigan", "home"])
|
|
vf = P("data/vps_cells.json") # our own Vultr servers, as one more run
|
|
if os.path.exists(vf):
|
|
v = json.load(open(vf))
|
|
rn = len(runs)
|
|
st = min(n.get("started", "") for n in v["nodes"])
|
|
ends = [ln.split()[0] for f in glob.glob(P("data/vps/*/ping.log")) for ln in open(f) if ln.strip()]
|
|
runs.append(dict(started=st, ended=max(ends), rounds=max(c["runs"] for c in v["cells"]), packets=20, source="vultr", file="data/vps/"))
|
|
for c in v["cells"]:
|
|
if c["a"] in sidx and c["b"] in sidx:
|
|
cells.append([sidx[c["a"]], sidx[c["b"]], round(c["rtt"], 1), round(c["allow"], 1), rn, f"our Vultr server gl-{c['src']}", c["src"]])
|
|
return dict(sites=sites, runs=runs, cells=cells)
|
|
|
|
|
|
def sweep(wn):
|
|
"""Impossible-pair counts across each flat model's free parameter (WN set)."""
|
|
from models import dualpole_xy, mercator_xy, floor_ms, C_FIBER, C_VACUUM
|
|
S, flat = wn["sites"], wn["pairs"]
|
|
out = {}
|
|
for name, f in (("dualpole", dualpole_xy), ("mercator", mercator_xy)):
|
|
rows = []
|
|
for p in range(-180, 180, 5):
|
|
xy = [f(s[2], s[3], p) for s in S]
|
|
fib = vac = 0
|
|
for k in range(0, len(flat), 3):
|
|
a, b, rtt = flat[k], flat[k + 1], flat[k + 2]
|
|
km = max(0.0, math.dist(xy[a], xy[b]) - 200)
|
|
fib += rtt + 1 < floor_ms(km, C_FIBER)
|
|
vac += rtt + 1 < floor_ms(km, C_VACUUM)
|
|
rows.append([p, fib, vac])
|
|
out[name] = rows
|
|
return out
|
|
|
|
|
|
BYO_CITIES = ["Sydney", "Melbourne", "Brisbane", "Perth", "Adelaide", "Hobart", "Auckland", "Wellington", "Christchurch",
|
|
"Nouméa", "Suva", "Jakarta", "Singapore", "Tokyo", "Seoul", "Hong Kong", "Mumbai", "Dubai", "Johannesburg",
|
|
"Cape Town", "Durban", "Luanda", "Nairobi", "Lagos", "São Paulo", "Fortaleza", "Buenos Aires", "Santiago",
|
|
"Montevideo", "Lima", "Bogotá", "Mexico City", "Los Angeles", "Seattle", "Chicago", "New York", "Miami",
|
|
"London", "Frankfurt", "Stockholm", "Reykjavik", "Honolulu"]
|
|
|
|
|
|
def byo():
|
|
"""Cities for the 'your location' box (GeoNames) and one RIPE Atlas anchor per target city."""
|
|
import csv
|
|
oce = {"AU", "NZ", "NC", "FJ", "PG", "PF", "WS", "TO", "VU"}
|
|
cities = []
|
|
for r in csv.reader(open(P("raw/geonames/cities15000.txt"), encoding="utf-8"), delimiter="\t", quoting=csv.QUOTE_NONE):
|
|
pop = int(r[14] or 0)
|
|
if pop >= 150000 or (r[8] in oce and pop >= 30000):
|
|
cities.append([r[1], r[8], round(float(r[4]), 3), round(float(r[5]), 3), pop])
|
|
cities.sort(key=lambda c: -c[4])
|
|
anchors = json.load(open(P("raw/ripe_anchors_all.json")))
|
|
live_ip = {t[0]: t[1] for t in TARGETS}
|
|
norm = lambda s: s.lower().replace("ã", "a").replace("é", "e").replace("á", "a").replace("ó", "o").replace("í", "i")
|
|
picked = []
|
|
for city in BYO_CITIES:
|
|
cand = [a for a in anchors if not a.get("is_disabled") and not a.get("date_decommissioned") and a.get("ip_v4")
|
|
and norm(a["city"]).startswith(norm(city))]
|
|
if not cand:
|
|
continue
|
|
pref = [a for a in cand if a["ip_v4"] == live_ip.get(city)]
|
|
a = (pref or sorted(cand, key=lambda a: a["id"]))[0]
|
|
lon, lat = a["geometry"]["coordinates"]
|
|
picked.append([city, a["country"], round(lat, 3), round(lon, 3), a["ip_v4"], a["fqdn"].split(".")[0]])
|
|
return dict(cities=[c[:4] for c in cities], anchors=picked)
|
|
|
|
|
|
def traces():
|
|
t = json.load(open(P("data/trace.json")))
|
|
A = {x["fqdn"].split(".")[0]: x for x in json.load(open(P("raw/ripe_anchors_all.json")))}
|
|
for r in t:
|
|
for k in ("src", "dst"):
|
|
lon, lat = A[r[k]]["geometry"]["coordinates"]
|
|
r[k + "_ll"] = [round(lat, 4), round(lon, 4)]
|
|
if r["cable"] == "tasman-global-access-tga-cable":
|
|
r["assumed"] = "the cable is assumed from its length; both routers at the jump belong to Vocus, which is not a listed owner of TGA."
|
|
return t
|
|
|
|
|
|
def nets():
|
|
"""Network attribution from asmap.py, compacted for the page."""
|
|
f = P("data/asmap.json")
|
|
if not os.path.exists(f):
|
|
return None
|
|
a = json.load(open(f))
|
|
anc = {x["fqdn"].split(".")[0]: x for x in json.load(open(P("raw/ripe_anchors_all.json")))}
|
|
vst = {n["code"]: n["city"] for n in json.load(open(P("data/vps/state.json")))["nodes"]}
|
|
city = lambda fq: (anc.get(fq) or {}).get("city", fq)
|
|
ends, gp = [], {}
|
|
for e in a["endpoints"]:
|
|
sid = e["site"].split(":", 1)[1]
|
|
if e["kind"] == "RIPE anchor":
|
|
ends.append(dict(kind="anchor", id=sid, city=city(sid), company=(anc.get(sid) or {}).get("company", ""), claimed_asn=e["claimed_asn"],
|
|
bgp_origin=e["bgp_origin"], name=e["name"], prefix=e["prefix"], match=e["match"], rpki=e.get("rpki", [])))
|
|
elif e["kind"] == "our Vultr server":
|
|
ends.append(dict(kind="vultr", id=sid, city=vst.get(sid, sid), bgp_origin=e["bgp_origin"], name=e["name"], prefix=e["prefix"], match=e["match"], rpki=e.get("rpki", [])))
|
|
else:
|
|
k = e["claimed_asn"]
|
|
gp.setdefault(k, dict(asn=k, network=e["network"], probes=0))["probes"] += 1
|
|
ends.sort(key=lambda e: (e["kind"], e["city"]))
|
|
cname = {c["id"]: c["name"].replace(" Cable", "") for c in json.load(open(P("data/cables.json")))["cables"]}
|
|
xs = [dict(src_city=city(x["src"].split(":")[-1]), dst_city=city(x["dst"].split(":")[-1]), cable_name=cname.get(x["cable"]) if x["cable"] else None,
|
|
owners=x["owners"], path=x["path"], reading=x["reading"], assumed=x["cable"] == "tasman-global-access-tga-cable") for x in a["crossings"]]
|
|
return dict(summary=a["summary"], traces=a["traces"], endpoints=ends, gp=sorted(gp.values(), key=lambda x: -x["probes"]),
|
|
transit=a["transit"], ixps=a["ixps"], crossings=xs)
|
|
|
|
|
|
def cables():
|
|
d = json.load(open(P("data/cables.json")))
|
|
keep = ("id", "name", "length", "rfs", "south", "landings", "span", "route", "span_ends")
|
|
return dict(cables=[{k: c[k] for k in keep} for c in d["cables"]], dropped=d["dropped"])
|
|
|
|
|
|
def netlines():
|
|
"""Drawn route of every cable in service, for the background network."""
|
|
meta = {}
|
|
for f in glob.glob(P("raw/tg/cables/*.json")):
|
|
c = json.load(open(f))
|
|
meta[c["id"]] = c
|
|
out = {}
|
|
for f in json.load(open(P("raw/tg/cable-geo.json")))["features"]:
|
|
c = meta.get(f["properties"]["id"])
|
|
if not c or c.get("is_planned") or (c.get("rfs_year") or 0) > 2026:
|
|
continue
|
|
out.setdefault(c["id"], []).extend([[round(x, 2), round(y, 2)] for x, y in line] for line in f["geometry"]["coordinates"])
|
|
return out
|
|
|
|
|
|
if __name__ == "__main__":
|
|
wn = wonder()
|
|
data = dict(land=land(), wn=wn, live=live(), sweep=sweep(wn),
|
|
cables=cables(), trace=traces(), byo=byo(),
|
|
network=json.load(open(P("data/network.json"))), netlines=netlines(),
|
|
flat=json.load(open(P("data/flatfit.json"))) if os.path.exists(P("data/flatfit.json")) else None,
|
|
pingmap=json.load(open(P("data/pingmap.json")))["stress"], net=nets())
|
|
blob = json.dumps(data, separators=(",", ":"), ensure_ascii=False)
|
|
html = open(P("web/template.html")).read().replace("/*__DATA__*/null", blob)
|
|
open(P("web/index.html"), "w").write(html)
|
|
# standalone copy for hand-off: full document, d3 embedded so it opens without a network connection
|
|
d3 = open(P("raw/lib/d3.min.js")).read()
|
|
tag = '<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/7.9.0/d3.min.js"></script>'
|
|
assert html.count(tag) == 1 and "</script>" not in d3.replace("<\\/script>", "")
|
|
head, body = html.split("</style>", 1)
|
|
doc = ('<!doctype html>\n<html lang="en">\n<head>\n<meta charset="utf-8">\n'
|
|
'<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover">\n'
|
|
+ head + "</style>\n</head>\n<body>\n" + body.replace(tag, "<script>" + d3 + "</script>") + "\n</body>\n</html>\n")
|
|
os.makedirs(P("handoff"), exist_ok=True)
|
|
open(P("handoff/cable_latency_survey_report.html"), "w").write(doc)
|
|
os.makedirs(P("site"), exist_ok=True) # deployed as https://pingstats.aethercosmology.com/
|
|
open(P("site/index.html"), "w").write(doc)
|
|
print(f"web/index.html {len(html)/1e6:.2f} MB; live cells {len(data['live']['cells'])}, "
|
|
f"runs {len(data['live']['runs'])}, wn pairs {len(wn['pairs'])//3}, cables {len(data['cables']['cables'])}, "
|
|
f"cities {len(data['byo']['cities'])}, anchors {[a[0] for a in data['byo']['anchors']]}")
|