Open-source release: pipeline code, raw logs, derived data, licences, tests
Adds src/ (scripts, raw measurement logs, derived data, report source), tests/, LICENSE (MIT, code) and LICENSE-DATA.md (CC BY 4.0 for the survey's data; third-party terms). The report is updated to the current site; the v1 and v2 data hand-off files are unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Toby
+257
@@ -0,0 +1,257 @@
|
||||
"""Assemble web/index.html from web/template.html + data.
|
||||
|
||||
python3 build.py -> web/index.html
|
||||
"""
|
||||
import glob, json, math, os, re
|
||||
from collections import defaultdict
|
||||
from measure import TARGETS
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
P = lambda *a: os.path.join(HERE, *a)
|
||||
|
||||
PROBE_TO_SITE = {"Sao Paulo": "São Paulo"}
|
||||
|
||||
|
||||
def land():
|
||||
"""TopoJSON land-110m -> list of polygons (rings of [lon, lat]), 2 dp."""
|
||||
t = json.load(open(P("raw/geo/land-110m.json")))
|
||||
sx, sy = t["transform"]["scale"]
|
||||
tx, ty = t["transform"]["translate"]
|
||||
arcs = []
|
||||
for arc in t["arcs"]:
|
||||
x = y = 0
|
||||
pts = []
|
||||
for dx, dy in arc:
|
||||
x += dx
|
||||
y += dy
|
||||
pts.append([round(x * sx + tx, 2), round(y * sy + ty, 2)])
|
||||
arcs.append(pts)
|
||||
|
||||
def ring(idx):
|
||||
out = []
|
||||
for i in idx:
|
||||
a = arcs[i] if i >= 0 else arcs[~i][::-1]
|
||||
out.extend(a if not out else a[1:])
|
||||
return out
|
||||
|
||||
polys = []
|
||||
for g in t["objects"]["land"]["geometries"]:
|
||||
ps = g["arcs"] if g["type"] == "MultiPolygon" else [g["arcs"]]
|
||||
for p in ps:
|
||||
polys.append([ring(r) for r in p])
|
||||
return polys
|
||||
|
||||
|
||||
def wonder():
|
||||
d = json.load(open(P("data/wn_clean.json")))
|
||||
ids = sorted(d["servers"], key=int)
|
||||
index = {s: i for i, s in enumerate(ids)}
|
||||
sites = [[d["servers"][s]["name"], d["servers"][s]["country"],
|
||||
round(d["servers"][s]["lat"], 4), round(d["servers"][s]["lon"], 4)] for s in ids]
|
||||
flat = []
|
||||
for r in d["pairs"]:
|
||||
flat += [index[r["a"]], index[r["b"]], round(r["rtt"], 2)]
|
||||
return dict(sites=sites, pairs=flat, dropped=[f"{m['name']} ({m['country']})" for m in d["mislocated"]])
|
||||
|
||||
|
||||
def live():
|
||||
from measure import EXTRA_SITES
|
||||
sites = [[t[0], t[2], t[3], t[4], t[1]] for t in TARGETS] # name, anchor, lat, lon, ip
|
||||
sites += [[n, "vultr-" + n.lower(), la, lo, ""] for n, la, lo in EXTRA_SITES]
|
||||
sidx = {s[0]: i for i, s in enumerate(sites)}
|
||||
runs, cells = [], []
|
||||
for f in sorted(glob.glob(P("data/live_*.json"))):
|
||||
run = json.load(open(f))
|
||||
rn = len(runs)
|
||||
runs.append(dict(started=run["started"], packets=run["packets"], file="data/" + os.path.basename(f)))
|
||||
probe_key = lambda c: (c["src_city"], c["src_network"], c["src_asn"])
|
||||
local = {}
|
||||
for c in run["cells"]:
|
||||
site = PROBE_TO_SITE.get(c["src_city"], c["src_city"])
|
||||
if c["dst_city"] == site and c["min"] is not None:
|
||||
local[probe_key(c)] = c["min"]
|
||||
for c in run["cells"]:
|
||||
src = PROBE_TO_SITE.get(c["src_city"], c["src_city"])
|
||||
loc = local.get(probe_key(c))
|
||||
if c["min"] is None or loc is None or src == c["dst_city"]:
|
||||
continue
|
||||
cells.append([sidx[src], sidx[c["dst_city"]], round(c["min"], 1), round(loc, 1), rn, c["src_network"], c["measurement"]])
|
||||
from measure import ROUTE_ONLY_SITES, HOME_NEAR
|
||||
hf = P("data/home/ping.log") # the home connection in Michigan, pinned to its nearest anchor
|
||||
vstate = json.load(open(P("data/vps/state.json"))) if os.path.exists(P("data/vps/state.json")) else {"nodes": []}
|
||||
vlocal = json.load(open(P("data/vps_cells.json")))["local"] if os.path.exists(P("data/vps_cells.json")) else {}
|
||||
if os.path.exists(hf):
|
||||
rx = re.compile(r"^(\S+) (\S+) (\S+) .*?= ([\d.]+)/")
|
||||
best = {}
|
||||
for line in open(hf):
|
||||
m = rx.match(line)
|
||||
if m:
|
||||
best[m.group(2)] = min(best.get(m.group(2), 1e9), float(m.group(4)))
|
||||
near = {n: best[n] for n, _ in HOME_NEAR if n in best}
|
||||
ref = min(near, key=near.get)
|
||||
rname, rla, rlo = ROUTE_ONLY_SITES[0]
|
||||
sites.append([rname.replace("(home reference)", "(residential reference)"), ref, rla, rlo, ""]) # display name; data files keep the original
|
||||
sidx[rname] = len(sites) - 1
|
||||
rn = len(runs)
|
||||
runs.append(dict(started=open(hf).readline().split()[0], packets=20, source="home", file="data/home/ping.log"))
|
||||
fq_city = {t[2]: t[0] for t in TARGETS}
|
||||
vcity = {"syd": "Sydney", "mel": "Melbourne", "jnb": "Johannesburg", "scl": "Santiago", "sao": "São Paulo",
|
||||
"lax": "Los Angeles", "icn": "Seoul", "hnl": "Honolulu"}
|
||||
for tgt, rtt in best.items():
|
||||
if tgt in dict(HOME_NEAR):
|
||||
continue
|
||||
city = fq_city.get(tgt) or vcity.get(tgt)
|
||||
if city in sidx:
|
||||
cells.append([sidx[rname], sidx[city], round(rtt, 1), round(near[ref] + vlocal.get(tgt, 0.0), 1), rn, "home connection, Michigan", "home"])
|
||||
vf = P("data/vps_cells.json") # our own Vultr servers, as one more run
|
||||
if os.path.exists(vf):
|
||||
v = json.load(open(vf))
|
||||
rn = len(runs)
|
||||
st = min(n.get("started", "") for n in v["nodes"])
|
||||
ends = [ln.split()[0] for f in glob.glob(P("data/vps/*/ping.log")) for ln in open(f) if ln.strip()]
|
||||
runs.append(dict(started=st, ended=max(ends), rounds=max(c["runs"] for c in v["cells"]), packets=20, source="vultr", file="data/vps/"))
|
||||
for c in v["cells"]:
|
||||
if c["a"] in sidx and c["b"] in sidx:
|
||||
cells.append([sidx[c["a"]], sidx[c["b"]], round(c["rtt"], 1), round(c["allow"], 1), rn, f"our Vultr server gl-{c['src']}", c["src"]])
|
||||
return dict(sites=sites, runs=runs, cells=cells)
|
||||
|
||||
|
||||
def sweep(wn):
|
||||
"""Impossible-pair counts across each flat model's free parameter (WN set)."""
|
||||
from models import dualpole_xy, mercator_xy, floor_ms, C_FIBER, C_VACUUM
|
||||
S, flat = wn["sites"], wn["pairs"]
|
||||
out = {}
|
||||
for name, f in (("dualpole", dualpole_xy), ("mercator", mercator_xy)):
|
||||
rows = []
|
||||
for p in range(-180, 180, 5):
|
||||
xy = [f(s[2], s[3], p) for s in S]
|
||||
fib = vac = 0
|
||||
for k in range(0, len(flat), 3):
|
||||
a, b, rtt = flat[k], flat[k + 1], flat[k + 2]
|
||||
km = max(0.0, math.dist(xy[a], xy[b]) - 200)
|
||||
fib += rtt + 1 < floor_ms(km, C_FIBER)
|
||||
vac += rtt + 1 < floor_ms(km, C_VACUUM)
|
||||
rows.append([p, fib, vac])
|
||||
out[name] = rows
|
||||
return out
|
||||
|
||||
|
||||
BYO_CITIES = ["Sydney", "Melbourne", "Brisbane", "Perth", "Adelaide", "Hobart", "Auckland", "Wellington", "Christchurch",
|
||||
"Nouméa", "Suva", "Jakarta", "Singapore", "Tokyo", "Seoul", "Hong Kong", "Mumbai", "Dubai", "Johannesburg",
|
||||
"Cape Town", "Durban", "Luanda", "Nairobi", "Lagos", "São Paulo", "Fortaleza", "Buenos Aires", "Santiago",
|
||||
"Montevideo", "Lima", "Bogotá", "Mexico City", "Los Angeles", "Seattle", "Chicago", "New York", "Miami",
|
||||
"London", "Frankfurt", "Stockholm", "Reykjavik", "Honolulu"]
|
||||
|
||||
|
||||
def byo():
|
||||
"""Cities for the 'your location' box (GeoNames) and one RIPE Atlas anchor per target city."""
|
||||
import csv
|
||||
oce = {"AU", "NZ", "NC", "FJ", "PG", "PF", "WS", "TO", "VU"}
|
||||
cities = []
|
||||
for r in csv.reader(open(P("raw/geonames/cities15000.txt"), encoding="utf-8"), delimiter="\t", quoting=csv.QUOTE_NONE):
|
||||
pop = int(r[14] or 0)
|
||||
if pop >= 150000 or (r[8] in oce and pop >= 30000):
|
||||
cities.append([r[1], r[8], round(float(r[4]), 3), round(float(r[5]), 3), pop])
|
||||
cities.sort(key=lambda c: -c[4])
|
||||
anchors = json.load(open(P("raw/ripe_anchors_all.json")))
|
||||
live_ip = {t[0]: t[1] for t in TARGETS}
|
||||
norm = lambda s: s.lower().replace("ã", "a").replace("é", "e").replace("á", "a").replace("ó", "o").replace("í", "i")
|
||||
picked = []
|
||||
for city in BYO_CITIES:
|
||||
cand = [a for a in anchors if not a.get("is_disabled") and not a.get("date_decommissioned") and a.get("ip_v4")
|
||||
and norm(a["city"]).startswith(norm(city))]
|
||||
if not cand:
|
||||
continue
|
||||
pref = [a for a in cand if a["ip_v4"] == live_ip.get(city)]
|
||||
a = (pref or sorted(cand, key=lambda a: a["id"]))[0]
|
||||
lon, lat = a["geometry"]["coordinates"]
|
||||
picked.append([city, a["country"], round(lat, 3), round(lon, 3), a["ip_v4"], a["fqdn"].split(".")[0]])
|
||||
return dict(cities=[c[:4] for c in cities], anchors=picked)
|
||||
|
||||
|
||||
def traces():
|
||||
t = json.load(open(P("data/trace.json")))
|
||||
A = {x["fqdn"].split(".")[0]: x for x in json.load(open(P("raw/ripe_anchors_all.json")))}
|
||||
for r in t:
|
||||
for k in ("src", "dst"):
|
||||
lon, lat = A[r[k]]["geometry"]["coordinates"]
|
||||
r[k + "_ll"] = [round(lat, 4), round(lon, 4)]
|
||||
if r["cable"] == "tasman-global-access-tga-cable":
|
||||
r["assumed"] = "the cable is assumed from its length; both routers at the jump belong to Vocus, which is not a listed owner of TGA."
|
||||
return t
|
||||
|
||||
|
||||
def nets():
|
||||
"""Network attribution from asmap.py, compacted for the page."""
|
||||
f = P("data/asmap.json")
|
||||
if not os.path.exists(f):
|
||||
return None
|
||||
a = json.load(open(f))
|
||||
anc = {x["fqdn"].split(".")[0]: x for x in json.load(open(P("raw/ripe_anchors_all.json")))}
|
||||
vst = {n["code"]: n["city"] for n in json.load(open(P("data/vps/state.json")))["nodes"]}
|
||||
city = lambda fq: (anc.get(fq) or {}).get("city", fq)
|
||||
ends, gp = [], {}
|
||||
for e in a["endpoints"]:
|
||||
sid = e["site"].split(":", 1)[1]
|
||||
if e["kind"] == "RIPE anchor":
|
||||
ends.append(dict(kind="anchor", id=sid, city=city(sid), company=(anc.get(sid) or {}).get("company", ""), claimed_asn=e["claimed_asn"],
|
||||
bgp_origin=e["bgp_origin"], name=e["name"], prefix=e["prefix"], match=e["match"], rpki=e.get("rpki", [])))
|
||||
elif e["kind"] == "our Vultr server":
|
||||
ends.append(dict(kind="vultr", id=sid, city=vst.get(sid, sid), bgp_origin=e["bgp_origin"], name=e["name"], prefix=e["prefix"], match=e["match"], rpki=e.get("rpki", [])))
|
||||
else:
|
||||
k = e["claimed_asn"]
|
||||
gp.setdefault(k, dict(asn=k, network=e["network"], probes=0))["probes"] += 1
|
||||
ends.sort(key=lambda e: (e["kind"], e["city"]))
|
||||
cname = {c["id"]: c["name"].replace(" Cable", "") for c in json.load(open(P("data/cables.json")))["cables"]}
|
||||
xs = [dict(src_city=city(x["src"].split(":")[-1]), dst_city=city(x["dst"].split(":")[-1]), cable_name=cname.get(x["cable"]) if x["cable"] else None,
|
||||
owners=x["owners"], before=x["before"], after=x["after"]) for x in a["crossings"]]
|
||||
return dict(summary=a["summary"], traces=a["traces"], endpoints=ends, gp=sorted(gp.values(), key=lambda x: -x["probes"]),
|
||||
transit=a["transit"], ixps=a["ixps"], crossings=xs)
|
||||
|
||||
|
||||
def cables():
|
||||
d = json.load(open(P("data/cables.json")))
|
||||
keep = ("id", "name", "length", "rfs", "south", "landings", "span", "route", "span_ends")
|
||||
return dict(cables=[{k: c[k] for k in keep} for c in d["cables"]], dropped=d["dropped"])
|
||||
|
||||
|
||||
def netlines():
|
||||
"""Drawn route of every cable in service, for the background network."""
|
||||
meta = {}
|
||||
for f in glob.glob(P("raw/tg/cables/*.json")):
|
||||
c = json.load(open(f))
|
||||
meta[c["id"]] = c
|
||||
out = {}
|
||||
for f in json.load(open(P("raw/tg/cable-geo.json")))["features"]:
|
||||
c = meta.get(f["properties"]["id"])
|
||||
if not c or c.get("is_planned") or (c.get("rfs_year") or 0) > 2026:
|
||||
continue
|
||||
out.setdefault(c["id"], []).extend([[round(x, 2), round(y, 2)] for x, y in line] for line in f["geometry"]["coordinates"])
|
||||
return out
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
wn = wonder()
|
||||
data = dict(land=land(), wn=wn, live=live(), sweep=sweep(wn),
|
||||
cables=cables(), trace=traces(), byo=byo(),
|
||||
network=json.load(open(P("data/network.json"))), netlines=netlines(),
|
||||
flat=json.load(open(P("data/flatfit.json"))) if os.path.exists(P("data/flatfit.json")) else None,
|
||||
pingmap=json.load(open(P("data/pingmap.json")))["stress"], net=nets())
|
||||
blob = json.dumps(data, separators=(",", ":"), ensure_ascii=False)
|
||||
html = open(P("web/template.html")).read().replace("/*__DATA__*/null", blob)
|
||||
open(P("web/index.html"), "w").write(html)
|
||||
# standalone copy for hand-off: full document, d3 embedded so it opens without a network connection
|
||||
d3 = open(P("raw/lib/d3.min.js")).read()
|
||||
tag = '<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/7.9.0/d3.min.js"></script>'
|
||||
assert html.count(tag) == 1 and "</script>" not in d3.replace("<\\/script>", "")
|
||||
head, body = html.split("</style>", 1)
|
||||
doc = ('<!doctype html>\n<html lang="en">\n<head>\n<meta charset="utf-8">\n'
|
||||
'<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover">\n'
|
||||
+ head + "</style>\n</head>\n<body>\n" + body.replace(tag, "<script>" + d3 + "</script>") + "\n</body>\n</html>\n")
|
||||
os.makedirs(P("handoff"), exist_ok=True)
|
||||
open(P("handoff/cable_latency_survey_report.html"), "w").write(doc)
|
||||
os.makedirs(P("site"), exist_ok=True) # deployed as https://pingstats.aethercosmology.com/
|
||||
open(P("site/index.html"), "w").write(doc)
|
||||
print(f"web/index.html {len(html)/1e6:.2f} MB; live cells {len(data['live']['cells'])}, "
|
||||
f"runs {len(data['live']['runs'])}, wn pairs {len(wn['pairs'])//3}, cables {len(data['cables']['cables'])}, "
|
||||
f"cities {len(data['byo']['cities'])}, anchors {[a[0] for a in data['byo']['anchors']]}")
|
||||
Reference in New Issue
Block a user