Open-source release: pipeline code, raw logs, derived data, licences, tests

Adds src/ (scripts, raw measurement logs, derived data, report source),
tests/, LICENSE (MIT, code) and LICENSE-DATA.md (CC BY 4.0 for the survey's
data; third-party terms). The report is updated to the current site; the
v1 and v2 data hand-off files are unchanged.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Toby
2026-10-27 20:09:54 -04:00
parent 75da69fc31
commit c094eb88fa
81 changed files with 249210 additions and 127 deletions
+45
View File
@@ -0,0 +1,45 @@
"""Turn our own servers' ping logs into live-grid cells.
Each server is pinned to the RIPE anchor in its own city, with its ping to that
anchor as the allowance (the same rule as the Globalping probes). Server-to-
server and server-to-anchor measurements then become cells between anchor
cities, so they share the cable routes already computed for those cities.
Honolulu has no anchor; it is its own site at the data-centre city (within
about 1 km of the coordinates used). Every pair gets the page's 25 km slack.
python3 vps_analyze.py -> data/vps_cells.json
"""
import glob, json, os, re, statistics
from collections import defaultdict
from measure import TARGETS
HERE = os.path.dirname(os.path.abspath(__file__))
STATE = json.load(open(os.path.join(HERE, "data/vps/state.json")))
ANCHOR = {t[2]: t[0] for t in TARGETS} # anchor fqdn -> city
LOCAL = {"syd": "Sydney", "mel": "Melbourne", "jnb": "Johannesburg", "scl": "Santiago", "sao": "São Paulo",
"lax": "Los Angeles", "icn": "Seoul"} # server -> city of the anchor it is pinned to
RX = re.compile(r"^(\S+) (\S+) (\S+) .*?(\d+) packets transmitted, (\d+) received.*?= ([\d.]+)/([\d.]+)/([\d.]+)/([\d.]+) ms")
runs = defaultdict(list) # (src code, target name) -> per-run minimum RTTs
for n in STATE["nodes"]:
f = os.path.join(HERE, "data/vps", n["code"], "ping.log")
if not os.path.exists(f):
continue
for line in open(f):
m = RX.match(line)
if m:
runs[(n["code"], m.group(2))].append(float(m.group(6)))
site = lambda code_or_target: LOCAL.get(code_or_target) or ANCHOR.get(code_or_target) or ("Honolulu" if code_or_target == "hnl" else None)
local = {c: min(runs[(c, a)]) for c, city in LOCAL.items() for a, ac in ANCHOR.items() if ac == city and runs.get((c, a))}
cells = []
for (src, tgt), mins in runs.items():
a, b = site(src), site(tgt)
if not a or not b or a == b:
continue
allow = local.get(src, 0.0) + (local.get(tgt, 0.0) if tgt in LOCAL else 0.0)
cells.append(dict(src=src, tgt=tgt, a=a, b=b, rtt=round(min(mins), 2), median=round(statistics.median(mins), 2),
runs=len(mins), allow=round(allow, 2), slack_km=25))
json.dump(dict(cells=cells, local=local, nodes=STATE["nodes"]), open(os.path.join(HERE, "data/vps_cells.json"), "w"), indent=1)
print(f"{len(cells)} cells from {len(runs)} server-target series; local allowances: { {k: round(v, 2) for k, v in local.items()} }")