The Penguin Test: SCAR RAATD penguin tracks on the four maps, 3-day stretches against a penguin's top sustained speed (penguins.aethercosmology.com)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Toby
2026-11-03 18:14:10 -05:00
parent 4268d22dac
commit ad1b2e3a4d
13 changed files with 1365 additions and 2 deletions
+295
View File
@@ -0,0 +1,295 @@
"""The Penguin Test: tracked penguins on the globe and three flat maps.
./fetch.sh -> raw/raatd/ (SCAR archive, checked against raw/raatd.sha256)
../.venv/bin/python build.py -> data/penguins.json, site/index.html
Data: The Retrospective Analysis of Antarctic Tracking (Standardised) Data from the
Scientific Committee on Antarctic Research, version 1.5 (Ropert-Coudert et al. 2020,
Scientific Data 7:94; GBIF DOI 10.15468/uzmdpm; CC BY 4.0). Penguin species only.
For every tracked penguin:
daily one position per UTC day: median time, median latitude, median longitude
(longitudes unwrapped around the day's first fix), to damp Argos error
lengths the track's length on each map, summing straight steps between daily
positions (models.py: globe, North-pole AE, dual-pole AE on 0N 0E; Mercator
with its east and west edges joined so that a track can cross 180 deg)
stretches every pair of daily positions three days apart; the speed each needs on each
map is (map distance - position allowance) / elapsed time
The limit: no penguin can cover more than (swim + current) x time + allowance:
swim 2.3 m/s, the fastest mean swimming speed measured in any penguin species
(gentoo and chinstrap; Sato et al. 2007, Proc. R. Soc. B 274:471-477,
table 1: emperor 1.7, Adelie 2.0, king 2.1 m/s)
current 1.0 m/s, about twice the strongest jets of the Antarctic Circumpolar Current
(typically 0.1 m/s, up to 0.5-0.6 m/s at the fronts)
allowance 50 km per stretch for position error at its two ends
and the limit assumes the bird swims flat out for the whole stretch, never diving,
resting or walking.
"""
import collections, datetime as dt, json, math, os, re, statistics, sys
HERE = os.path.dirname(os.path.abspath(__file__))
SRC = os.path.join(HERE, "..", "src")
sys.path.insert(0, SRC)
from models import R, ae_xy, dualpole_xy, globe # 6371.0 km sphere shared with the cable survey
RAW = os.path.join(HERE, "raw", "raatd")
GENERA = ("Aptenodytes", "Pygoscelis", "Eudyptes", "Spheniscus", "Eudyptula", "Megadyptes")
V_SWIM = 2.3 # m/s
V_CURRENT = 1.0 # m/s
ERR_KM = 50.0 # km per stretch
WINDOW = 3 # days
LIMIT_KMH = (V_SWIM + V_CURRENT) * 3.6
MAPS = ["globe", "ae", "dualpole", "mercator"]
DEFAULT = ("Aptenodytes forsteri", "Pointe Geologie, Terre Adelie", "2005")
SOUTH_DETAIL = -35.0 # land polygons reaching south of this use 1:50m, the rest 1:110m
def merc_dist(a, b):
"""Straight line on a Mercator map, true on the equator, east and west edges joined."""
y = lambda lat: R * math.log(math.tan(math.pi / 4 + math.radians(max(-85.0, min(85.0, lat))) / 2))
dl = (b[1] - a[1] + 540.0) % 360.0 - 180.0
return math.hypot(R * math.radians(dl), y(b[0]) - y(a[0]))
def dist(m, a, b):
if m == "globe":
return globe(*a, *b)
if m == "ae":
(x1, y1), (x2, y2) = ae_xy(*a), ae_xy(*b)
elif m == "dualpole":
(x1, y1), (x2, y2) = dualpole_xy(*a), dualpole_xy(*b)
else:
return merc_dist(a, b)
return math.hypot(x2 - x1, y2 - y1)
def stamp(date, time):
"""'2005-11-01T00:46:02' or '2005-11-1' + '03:45:00Z' -> POSIX seconds (UTC)."""
s = date if "T" in date else date + "T" + (time.rstrip("Z") or "00:00:00")
m = re.match(r"(\d{4})-(\d{1,2})-(\d{1,2})T(\d{1,2}):(\d{2})(?::(\d{2}(?:\.\d+)?))?", s)
if not m:
return None
y, mo, d, h, mi, sec = m.groups()
return dt.datetime(int(y), int(mo), int(d), int(h), int(mi), int(float(sec or 0)), tzinfo=dt.timezone.utc).timestamp()
def read_rows(name):
with open(os.path.join(RAW, name), encoding="utf-8") as f:
head = f.readline().rstrip("\n").split("\t")
for line in f:
r = line.rstrip("\n").split("\t")
if len(r) >= len(head):
yield dict(zip(head, r))
def load():
meta = {}
for r in read_rows("event.txt"):
if not r["eventID"].startswith("capture:"):
continue
sp = r["samplingProtocol"]
dev = sp.split("device type=")[1].split(":")[0].strip() if "device type=" in sp else ""
meta[r["eventID"]] = dict(dev=dev, colony=r["locality"].strip(), year=r["year"], dataset=r["datasetID"],
clat=float(r["decimalLatitude"] or "nan"), clon=float(r["decimalLongitude"] or "nan"))
fixes = collections.defaultdict(list)
who = {}
for r in read_rows("occurrence.txt"):
if not r["scientificName"].startswith(GENERA):
continue
if "::capture" in r["occurrenceID"]:
who.setdefault(r["id"], {}).update(sex=r["sex"], stage=r["lifeStage"])
continue
t = stamp(r["eventDate"], r["eventTime"])
try:
lat, lon = float(r["decimalLatitude"]), float(r["decimalLongitude"])
except ValueError:
continue
if t is None or not (-90 <= lat <= 90 and -180 <= lon <= 180):
continue
fixes[r["id"]].append((t, lat, lon))
who.setdefault(r["id"], {}).update(sp=r["scientificName"], name=r["vernacularName"])
return meta, fixes, who
def daily(fx):
"""One position per UTC day: median time, latitude and (unwrapped) longitude."""
by = collections.defaultdict(list)
for t, la, lo in sorted(fx):
by[int(t // 86400)].append((t, la, lo))
out = []
for d in sorted(by):
v = by[d]
l0 = v[0][2]
lons = [l0 + ((lo - l0 + 540.0) % 360.0 - 180.0) for _, _, lo in v]
lon = (statistics.median(lons) + 540.0) % 360.0 - 180.0
out.append((d, statistics.median(t for t, _, _ in v), statistics.median(la for _, la, _ in v), lon, len(v)))
return out
def track_stats(D):
"""Lengths on each map and the 3-day stretch that needs the highest speed on each map."""
P = [(x[2], x[3]) for x in D]
st = {}
for m in MAPS:
length = sum(dist(m, P[i - 1], P[i]) for i in range(1, len(P)))
best = None
for i in range(len(D)):
for j in range(i + 1, len(D)):
if D[j][0] - D[i][0] > WINDOW:
break
if D[j][0] - D[i][0] < WINDOW:
continue
hours = (D[j][1] - D[i][1]) / 3600
if hours < 24 * (WINDOW - 1):
continue
km = dist(m, P[i], P[j])
need = max(0.0, km - ERR_KM) / hours
if best is None or need > best[0]:
best = (need, km, hours, i, j)
st[m] = dict(len=length, need=best[0] if best else None, km=best[1] if best else None,
h=best[2] if best else None, i=best[3] if best else None, j=best[4] if best else None)
return st
def land(path, nd=2):
"""TopoJSON land -> list of polygons (rings of [lon, lat])."""
t = json.load(open(path))
sx, sy = t["transform"]["scale"]
tx, ty = t["transform"]["translate"]
arcs = []
for arc in t["arcs"]:
x = y = 0
pts = []
for dx, dy in arc:
x += dx; y += dy
pts.append([round(x * sx + tx, nd), round(y * sy + ty, nd)])
arcs.append(pts)
def ring(idx):
out = []
for i in idx:
a = arcs[i] if i >= 0 else arcs[~i][::-1]
out.extend(a if not out else a[1:])
return out
polys = []
for g in t["objects"]["land"]["geometries"]:
if g["type"] == "Polygon":
polys.append([ring(r) for r in g["arcs"]])
elif g["type"] == "MultiPolygon":
polys.extend([[ring(r) for r in p] for p in g["arcs"]])
return polys
def coast():
"""1:50m for land reaching south of SOUTH_DETAIL (Antarctica, the sub-Antarctic islands, the
southern continents' tips), 1:110m elsewhere."""
south = lambda p: min(y for _, y in p[0]) < SOUTH_DETAIL
fine = [p for p in land(os.path.join(SRC, "raw", "geo", "land-50m.json")) if south(p)]
coarse = [p for p in land(os.path.join(SRC, "raw", "geo", "land-110m.json")) if not south(p)]
return coarse + fine
SPECIES = {"Aptenodytes forsteri": "Emperor penguin", "Aptenodytes patagonicus": "King penguin",
"Pygoscelis adeliae": "Adélie penguin", "Eudyptes chrysolophus": "Macaroni penguin",
"Eudyptes schlegeli": "Royal penguin"}
COLONY_FIX = {"Pointe Geologie, Terre Adelie": "Pointe Géologie, Terre Adélie"}
def main():
meta, fixes, who = load()
tracks = []
for iid, fx in fixes.items():
D = daily(fx)
if len(D) < 2:
continue
m = meta.get(iid, {})
st = track_stats(D)
d0 = D[0][0]
tracks.append(dict(
id=iid.replace("capture:", ""), sp=who[iid]["sp"], colony=m.get("colony", ""), year=m.get("year", ""),
dev=m.get("dev", ""), sex=who[iid].get("sex", ""), stage=who[iid].get("stage", ""), dataset=m.get("dataset", ""),
start=dt.datetime.fromtimestamp(D[0][1], dt.timezone.utc).strftime("%Y-%m-%d"),
end=dt.datetime.fromtimestamp(D[-1][1], dt.timezone.utc).strftime("%Y-%m-%d"),
fixes=len(fx), d=[x[0] - d0 for x in D], lat=[round(x[2], 3) for x in D], lon=[round(x[3], 3) for x in D],
st={k: {kk: (round(vv, 3) if isinstance(vv, float) else vv) for kk, vv in v.items()} for k, v in st.items()}))
tracks.sort(key=lambda t: (t["sp"], t["colony"], t["year"], t["start"], t["id"]))
tested = [t for t in tracks if t["st"]["globe"]["need"] is not None]
# summaries at the default limit
def beyond(t, m, lim=LIMIT_KMH):
return t["st"][m]["need"] is not None and t["st"][m]["need"] > lim
by_sp = collections.OrderedDict()
for sp in sorted({t["sp"] for t in tracks}, key=lambda s: -sum(t["sp"] == s for t in tracks)):
v = [t for t in tested if t["sp"] == sp]
by_sp[sp] = dict(name=SPECIES.get(sp, sp), tracks=sum(t["sp"] == sp for t in tracks), tested=len(v),
beyond={m: sum(beyond(t, m) for t in v) for m in MAPS},
med_need={m: round(statistics.median(t["st"][m]["need"] for t in v), 2) if v else None for m in MAPS},
med_len={m: round(statistics.median(t["st"][m]["len"] for t in v)) if v else None for m in MAPS})
tot = dict(tracks=len(tracks), tested=len(tested), beyond={m: sum(beyond(t, m) for t in tested) for m in MAPS},
days=sum(len(t["d"]) for t in tracks), fixes=sum(t["fixes"] for t in tracks),
colonies=len({t["colony"] for t in tracks}), years=[min(t["year"] for t in tracks), max(t["year"] for t in tracks)],
devices=dict(collections.Counter(t["dev"] or "unrecorded" for t in tracks)),
ratio_ae=round(statistics.median(t["st"]["ae"]["len"] / t["st"]["globe"]["len"] for t in tracks if t["st"]["globe"]["len"] > 50), 2))
groups = collections.OrderedDict()
for t in tracks:
k = f'{t["sp"]}|{t["colony"]}|{t["year"]}'
groups.setdefault(k, dict(sp=t["sp"], colony=COLONY_FIX.get(t["colony"], t["colony"]), year=t["year"], n=0))["n"] += 1
film = [t for t in tested if (t["sp"], t["colony"], t["year"]) == DEFAULT]
out = dict(tracks=tracks, groups=groups, species=by_sp, total=tot, film=dict(n=len(film), beyond={m: sum(beyond(t, m) for t in film) for m in MAPS}),
default="|".join(DEFAULT), limit=dict(v_swim=V_SWIM, v_current=V_CURRENT, err_km=ERR_KM, window=WINDOW, kmh=round(LIMIT_KMH, 2)),
names=SPECIES, colony_fix=COLONY_FIX, land=coast(),
built=dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%d %H:%M UTC"))
os.makedirs(os.path.join(HERE, "data"), exist_ok=True)
with open(os.path.join(HERE, "data", "penguins.json"), "w") as fh:
json.dump(out, fh, separators=(",", ":"))
print(f"{tot['tracks']} tracks ({tot['tested']} with a {WINDOW}-day stretch), {tot['days']} daily positions from {tot['fixes']} fixes, "
f"{len(groups)} colony-seasons; limit {LIMIT_KMH:.2f} km/h + {ERR_KM:.0f} km")
for sp, s in by_sp.items():
print(f" {s['name']:16s} {s['tested']:4d} tested | beyond the limit: " + " ".join(f"{m} {s['beyond'][m]}" for m in MAPS))
print(" all", tot["beyond"], "| film colony 2005:", out["film"])
render(out)
def fmt(n):
return f"{n:,}"
def copy_values(o):
t, sp, f, L = o["total"], o["species"], o["film"], o["limit"]
pct = lambda k: f"{100 * k / t['tested']:.0f}%"
v = dict(
N_TRACKS=fmt(t["tracks"]), N_TESTED=fmt(t["tested"]), N_DAYS=fmt(t["days"]), N_FIXES=fmt(t["fixes"]),
N_SPECIES=str(len(sp)), N_COLONIES=fmt(t["colonies"]), Y0=t["years"][0], Y1=t["years"][1],
LIMIT=f"{L['kmh']:.1f}", V_SWIM=f"{L['v_swim']:.1f}", V_CURRENT=f"{L['v_current']:.1f}", ERR=f"{L['err_km']:.0f}", WINDOW=str(L["window"]),
MIN_H=str(24 * (L["window"] - 1)),
RATIO_AE=f"{t['ratio_ae']:.1f}",
G_BEYOND=fmt(t["beyond"]["globe"]), AE_BEYOND=fmt(t["beyond"]["ae"]), DP_BEYOND=fmt(t["beyond"]["dualpole"]), ME_BEYOND=fmt(t["beyond"]["mercator"]),
AE_PCT=pct(t["beyond"]["ae"]), DP_PCT=pct(t["beyond"]["dualpole"]), ME_PCT=pct(t["beyond"]["mercator"]), G_PCT=pct(t["beyond"]["globe"]),
FILM_N=str(f["n"]), FILM_AE=str(f["beyond"]["ae"]), FILM_G=str(f["beyond"]["globe"]), FILM_DP=str(f["beyond"]["dualpole"]), FILM_ME=str(f["beyond"]["mercator"]),
DEVICES=", ".join(f"{fmt(n)} {k}" for k, n in sorted(t["devices"].items(), key=lambda x: -x[1])),
BUILT=o["built"],
)
return v
def render(out):
tpl = os.path.join(HERE, "template.html")
if not os.path.exists(tpl):
return
html = open(tpl, encoding="utf-8").read()
for k, v in copy_values(out).items():
html = html.replace("{{" + k + "}}", v)
left = sorted(set(re.findall(r"\{\{(\w+)\}\}", html)))
assert not left, left
d3 = open(os.path.join(SRC, "raw", "lib", "d3.min.js"), encoding="utf-8").read()
html = html.replace("/*D3*/", d3).replace("/*DATA*/null", json.dumps(out, separators=(",", ":"), ensure_ascii=False))
os.makedirs(os.path.join(HERE, "site"), exist_ok=True)
open(os.path.join(HERE, "site", "index.html"), "w", encoding="utf-8").write(html)
print("site/index.html", round(len(html) / 1e6, 2), "MB")
if __name__ == "__main__":
if len(sys.argv) > 1 and sys.argv[1] == "render":
render(json.load(open(os.path.join(HERE, "data", "penguins.json"))))
else:
main()