The Penguin Test: SCAR RAATD penguin tracks on the four maps, 3-day stretches against a penguin's top sustained speed (penguins.aethercosmology.com)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,295 @@
|
||||
"""The Penguin Test: tracked penguins on the globe and three flat maps.
|
||||
|
||||
./fetch.sh -> raw/raatd/ (SCAR archive, checked against raw/raatd.sha256)
|
||||
../.venv/bin/python build.py -> data/penguins.json, site/index.html
|
||||
|
||||
Data: The Retrospective Analysis of Antarctic Tracking (Standardised) Data from the
|
||||
Scientific Committee on Antarctic Research, version 1.5 (Ropert-Coudert et al. 2020,
|
||||
Scientific Data 7:94; GBIF DOI 10.15468/uzmdpm; CC BY 4.0). Penguin species only.
|
||||
|
||||
For every tracked penguin:
|
||||
daily one position per UTC day: median time, median latitude, median longitude
|
||||
(longitudes unwrapped around the day's first fix), to damp Argos error
|
||||
lengths the track's length on each map, summing straight steps between daily
|
||||
positions (models.py: globe, North-pole AE, dual-pole AE on 0N 0E; Mercator
|
||||
with its east and west edges joined so that a track can cross 180 deg)
|
||||
stretches every pair of daily positions three days apart; the speed each needs on each
|
||||
map is (map distance - position allowance) / elapsed time
|
||||
The limit: no penguin can cover more than (swim + current) x time + allowance:
|
||||
swim 2.3 m/s, the fastest mean swimming speed measured in any penguin species
|
||||
(gentoo and chinstrap; Sato et al. 2007, Proc. R. Soc. B 274:471-477,
|
||||
table 1: emperor 1.7, Adelie 2.0, king 2.1 m/s)
|
||||
current 1.0 m/s, about twice the strongest jets of the Antarctic Circumpolar Current
|
||||
(typically 0.1 m/s, up to 0.5-0.6 m/s at the fronts)
|
||||
allowance 50 km per stretch for position error at its two ends
|
||||
and the limit assumes the bird swims flat out for the whole stretch, never diving,
|
||||
resting or walking.
|
||||
"""
|
||||
import collections, datetime as dt, json, math, os, re, statistics, sys
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
SRC = os.path.join(HERE, "..", "src")
|
||||
sys.path.insert(0, SRC)
|
||||
from models import R, ae_xy, dualpole_xy, globe # 6371.0 km sphere shared with the cable survey
|
||||
|
||||
RAW = os.path.join(HERE, "raw", "raatd")
|
||||
GENERA = ("Aptenodytes", "Pygoscelis", "Eudyptes", "Spheniscus", "Eudyptula", "Megadyptes")
|
||||
V_SWIM = 2.3 # m/s
|
||||
V_CURRENT = 1.0 # m/s
|
||||
ERR_KM = 50.0 # km per stretch
|
||||
WINDOW = 3 # days
|
||||
LIMIT_KMH = (V_SWIM + V_CURRENT) * 3.6
|
||||
MAPS = ["globe", "ae", "dualpole", "mercator"]
|
||||
DEFAULT = ("Aptenodytes forsteri", "Pointe Geologie, Terre Adelie", "2005")
|
||||
SOUTH_DETAIL = -35.0 # land polygons reaching south of this use 1:50m, the rest 1:110m
|
||||
|
||||
|
||||
def merc_dist(a, b):
|
||||
"""Straight line on a Mercator map, true on the equator, east and west edges joined."""
|
||||
y = lambda lat: R * math.log(math.tan(math.pi / 4 + math.radians(max(-85.0, min(85.0, lat))) / 2))
|
||||
dl = (b[1] - a[1] + 540.0) % 360.0 - 180.0
|
||||
return math.hypot(R * math.radians(dl), y(b[0]) - y(a[0]))
|
||||
|
||||
|
||||
def dist(m, a, b):
|
||||
if m == "globe":
|
||||
return globe(*a, *b)
|
||||
if m == "ae":
|
||||
(x1, y1), (x2, y2) = ae_xy(*a), ae_xy(*b)
|
||||
elif m == "dualpole":
|
||||
(x1, y1), (x2, y2) = dualpole_xy(*a), dualpole_xy(*b)
|
||||
else:
|
||||
return merc_dist(a, b)
|
||||
return math.hypot(x2 - x1, y2 - y1)
|
||||
|
||||
|
||||
def stamp(date, time):
|
||||
"""'2005-11-01T00:46:02' or '2005-11-1' + '03:45:00Z' -> POSIX seconds (UTC)."""
|
||||
s = date if "T" in date else date + "T" + (time.rstrip("Z") or "00:00:00")
|
||||
m = re.match(r"(\d{4})-(\d{1,2})-(\d{1,2})T(\d{1,2}):(\d{2})(?::(\d{2}(?:\.\d+)?))?", s)
|
||||
if not m:
|
||||
return None
|
||||
y, mo, d, h, mi, sec = m.groups()
|
||||
return dt.datetime(int(y), int(mo), int(d), int(h), int(mi), int(float(sec or 0)), tzinfo=dt.timezone.utc).timestamp()
|
||||
|
||||
|
||||
def read_rows(name):
|
||||
with open(os.path.join(RAW, name), encoding="utf-8") as f:
|
||||
head = f.readline().rstrip("\n").split("\t")
|
||||
for line in f:
|
||||
r = line.rstrip("\n").split("\t")
|
||||
if len(r) >= len(head):
|
||||
yield dict(zip(head, r))
|
||||
|
||||
|
||||
def load():
|
||||
meta = {}
|
||||
for r in read_rows("event.txt"):
|
||||
if not r["eventID"].startswith("capture:"):
|
||||
continue
|
||||
sp = r["samplingProtocol"]
|
||||
dev = sp.split("device type=")[1].split(":")[0].strip() if "device type=" in sp else ""
|
||||
meta[r["eventID"]] = dict(dev=dev, colony=r["locality"].strip(), year=r["year"], dataset=r["datasetID"],
|
||||
clat=float(r["decimalLatitude"] or "nan"), clon=float(r["decimalLongitude"] or "nan"))
|
||||
fixes = collections.defaultdict(list)
|
||||
who = {}
|
||||
for r in read_rows("occurrence.txt"):
|
||||
if not r["scientificName"].startswith(GENERA):
|
||||
continue
|
||||
if "::capture" in r["occurrenceID"]:
|
||||
who.setdefault(r["id"], {}).update(sex=r["sex"], stage=r["lifeStage"])
|
||||
continue
|
||||
t = stamp(r["eventDate"], r["eventTime"])
|
||||
try:
|
||||
lat, lon = float(r["decimalLatitude"]), float(r["decimalLongitude"])
|
||||
except ValueError:
|
||||
continue
|
||||
if t is None or not (-90 <= lat <= 90 and -180 <= lon <= 180):
|
||||
continue
|
||||
fixes[r["id"]].append((t, lat, lon))
|
||||
who.setdefault(r["id"], {}).update(sp=r["scientificName"], name=r["vernacularName"])
|
||||
return meta, fixes, who
|
||||
|
||||
|
||||
def daily(fx):
|
||||
"""One position per UTC day: median time, latitude and (unwrapped) longitude."""
|
||||
by = collections.defaultdict(list)
|
||||
for t, la, lo in sorted(fx):
|
||||
by[int(t // 86400)].append((t, la, lo))
|
||||
out = []
|
||||
for d in sorted(by):
|
||||
v = by[d]
|
||||
l0 = v[0][2]
|
||||
lons = [l0 + ((lo - l0 + 540.0) % 360.0 - 180.0) for _, _, lo in v]
|
||||
lon = (statistics.median(lons) + 540.0) % 360.0 - 180.0
|
||||
out.append((d, statistics.median(t for t, _, _ in v), statistics.median(la for _, la, _ in v), lon, len(v)))
|
||||
return out
|
||||
|
||||
|
||||
def track_stats(D):
|
||||
"""Lengths on each map and the 3-day stretch that needs the highest speed on each map."""
|
||||
P = [(x[2], x[3]) for x in D]
|
||||
st = {}
|
||||
for m in MAPS:
|
||||
length = sum(dist(m, P[i - 1], P[i]) for i in range(1, len(P)))
|
||||
best = None
|
||||
for i in range(len(D)):
|
||||
for j in range(i + 1, len(D)):
|
||||
if D[j][0] - D[i][0] > WINDOW:
|
||||
break
|
||||
if D[j][0] - D[i][0] < WINDOW:
|
||||
continue
|
||||
hours = (D[j][1] - D[i][1]) / 3600
|
||||
if hours < 24 * (WINDOW - 1):
|
||||
continue
|
||||
km = dist(m, P[i], P[j])
|
||||
need = max(0.0, km - ERR_KM) / hours
|
||||
if best is None or need > best[0]:
|
||||
best = (need, km, hours, i, j)
|
||||
st[m] = dict(len=length, need=best[0] if best else None, km=best[1] if best else None,
|
||||
h=best[2] if best else None, i=best[3] if best else None, j=best[4] if best else None)
|
||||
return st
|
||||
|
||||
|
||||
def land(path, nd=2):
|
||||
"""TopoJSON land -> list of polygons (rings of [lon, lat])."""
|
||||
t = json.load(open(path))
|
||||
sx, sy = t["transform"]["scale"]
|
||||
tx, ty = t["transform"]["translate"]
|
||||
arcs = []
|
||||
for arc in t["arcs"]:
|
||||
x = y = 0
|
||||
pts = []
|
||||
for dx, dy in arc:
|
||||
x += dx; y += dy
|
||||
pts.append([round(x * sx + tx, nd), round(y * sy + ty, nd)])
|
||||
arcs.append(pts)
|
||||
|
||||
def ring(idx):
|
||||
out = []
|
||||
for i in idx:
|
||||
a = arcs[i] if i >= 0 else arcs[~i][::-1]
|
||||
out.extend(a if not out else a[1:])
|
||||
return out
|
||||
polys = []
|
||||
for g in t["objects"]["land"]["geometries"]:
|
||||
if g["type"] == "Polygon":
|
||||
polys.append([ring(r) for r in g["arcs"]])
|
||||
elif g["type"] == "MultiPolygon":
|
||||
polys.extend([[ring(r) for r in p] for p in g["arcs"]])
|
||||
return polys
|
||||
|
||||
|
||||
def coast():
|
||||
"""1:50m for land reaching south of SOUTH_DETAIL (Antarctica, the sub-Antarctic islands, the
|
||||
southern continents' tips), 1:110m elsewhere."""
|
||||
south = lambda p: min(y for _, y in p[0]) < SOUTH_DETAIL
|
||||
fine = [p for p in land(os.path.join(SRC, "raw", "geo", "land-50m.json")) if south(p)]
|
||||
coarse = [p for p in land(os.path.join(SRC, "raw", "geo", "land-110m.json")) if not south(p)]
|
||||
return coarse + fine
|
||||
|
||||
|
||||
SPECIES = {"Aptenodytes forsteri": "Emperor penguin", "Aptenodytes patagonicus": "King penguin",
|
||||
"Pygoscelis adeliae": "Adélie penguin", "Eudyptes chrysolophus": "Macaroni penguin",
|
||||
"Eudyptes schlegeli": "Royal penguin"}
|
||||
COLONY_FIX = {"Pointe Geologie, Terre Adelie": "Pointe Géologie, Terre Adélie"}
|
||||
|
||||
|
||||
def main():
|
||||
meta, fixes, who = load()
|
||||
tracks = []
|
||||
for iid, fx in fixes.items():
|
||||
D = daily(fx)
|
||||
if len(D) < 2:
|
||||
continue
|
||||
m = meta.get(iid, {})
|
||||
st = track_stats(D)
|
||||
d0 = D[0][0]
|
||||
tracks.append(dict(
|
||||
id=iid.replace("capture:", ""), sp=who[iid]["sp"], colony=m.get("colony", ""), year=m.get("year", ""),
|
||||
dev=m.get("dev", ""), sex=who[iid].get("sex", ""), stage=who[iid].get("stage", ""), dataset=m.get("dataset", ""),
|
||||
start=dt.datetime.fromtimestamp(D[0][1], dt.timezone.utc).strftime("%Y-%m-%d"),
|
||||
end=dt.datetime.fromtimestamp(D[-1][1], dt.timezone.utc).strftime("%Y-%m-%d"),
|
||||
fixes=len(fx), d=[x[0] - d0 for x in D], lat=[round(x[2], 3) for x in D], lon=[round(x[3], 3) for x in D],
|
||||
st={k: {kk: (round(vv, 3) if isinstance(vv, float) else vv) for kk, vv in v.items()} for k, v in st.items()}))
|
||||
tracks.sort(key=lambda t: (t["sp"], t["colony"], t["year"], t["start"], t["id"]))
|
||||
tested = [t for t in tracks if t["st"]["globe"]["need"] is not None]
|
||||
|
||||
# summaries at the default limit
|
||||
def beyond(t, m, lim=LIMIT_KMH):
|
||||
return t["st"][m]["need"] is not None and t["st"][m]["need"] > lim
|
||||
by_sp = collections.OrderedDict()
|
||||
for sp in sorted({t["sp"] for t in tracks}, key=lambda s: -sum(t["sp"] == s for t in tracks)):
|
||||
v = [t for t in tested if t["sp"] == sp]
|
||||
by_sp[sp] = dict(name=SPECIES.get(sp, sp), tracks=sum(t["sp"] == sp for t in tracks), tested=len(v),
|
||||
beyond={m: sum(beyond(t, m) for t in v) for m in MAPS},
|
||||
med_need={m: round(statistics.median(t["st"][m]["need"] for t in v), 2) if v else None for m in MAPS},
|
||||
med_len={m: round(statistics.median(t["st"][m]["len"] for t in v)) if v else None for m in MAPS})
|
||||
tot = dict(tracks=len(tracks), tested=len(tested), beyond={m: sum(beyond(t, m) for t in tested) for m in MAPS},
|
||||
days=sum(len(t["d"]) for t in tracks), fixes=sum(t["fixes"] for t in tracks),
|
||||
colonies=len({t["colony"] for t in tracks}), years=[min(t["year"] for t in tracks), max(t["year"] for t in tracks)],
|
||||
devices=dict(collections.Counter(t["dev"] or "unrecorded" for t in tracks)),
|
||||
ratio_ae=round(statistics.median(t["st"]["ae"]["len"] / t["st"]["globe"]["len"] for t in tracks if t["st"]["globe"]["len"] > 50), 2))
|
||||
groups = collections.OrderedDict()
|
||||
for t in tracks:
|
||||
k = f'{t["sp"]}|{t["colony"]}|{t["year"]}'
|
||||
groups.setdefault(k, dict(sp=t["sp"], colony=COLONY_FIX.get(t["colony"], t["colony"]), year=t["year"], n=0))["n"] += 1
|
||||
film = [t for t in tested if (t["sp"], t["colony"], t["year"]) == DEFAULT]
|
||||
out = dict(tracks=tracks, groups=groups, species=by_sp, total=tot, film=dict(n=len(film), beyond={m: sum(beyond(t, m) for t in film) for m in MAPS}),
|
||||
default="|".join(DEFAULT), limit=dict(v_swim=V_SWIM, v_current=V_CURRENT, err_km=ERR_KM, window=WINDOW, kmh=round(LIMIT_KMH, 2)),
|
||||
names=SPECIES, colony_fix=COLONY_FIX, land=coast(),
|
||||
built=dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%d %H:%M UTC"))
|
||||
os.makedirs(os.path.join(HERE, "data"), exist_ok=True)
|
||||
with open(os.path.join(HERE, "data", "penguins.json"), "w") as fh:
|
||||
json.dump(out, fh, separators=(",", ":"))
|
||||
print(f"{tot['tracks']} tracks ({tot['tested']} with a {WINDOW}-day stretch), {tot['days']} daily positions from {tot['fixes']} fixes, "
|
||||
f"{len(groups)} colony-seasons; limit {LIMIT_KMH:.2f} km/h + {ERR_KM:.0f} km")
|
||||
for sp, s in by_sp.items():
|
||||
print(f" {s['name']:16s} {s['tested']:4d} tested | beyond the limit: " + " ".join(f"{m} {s['beyond'][m]}" for m in MAPS))
|
||||
print(" all", tot["beyond"], "| film colony 2005:", out["film"])
|
||||
render(out)
|
||||
|
||||
|
||||
def fmt(n):
|
||||
return f"{n:,}"
|
||||
|
||||
|
||||
def copy_values(o):
|
||||
t, sp, f, L = o["total"], o["species"], o["film"], o["limit"]
|
||||
pct = lambda k: f"{100 * k / t['tested']:.0f}%"
|
||||
v = dict(
|
||||
N_TRACKS=fmt(t["tracks"]), N_TESTED=fmt(t["tested"]), N_DAYS=fmt(t["days"]), N_FIXES=fmt(t["fixes"]),
|
||||
N_SPECIES=str(len(sp)), N_COLONIES=fmt(t["colonies"]), Y0=t["years"][0], Y1=t["years"][1],
|
||||
LIMIT=f"{L['kmh']:.1f}", V_SWIM=f"{L['v_swim']:.1f}", V_CURRENT=f"{L['v_current']:.1f}", ERR=f"{L['err_km']:.0f}", WINDOW=str(L["window"]),
|
||||
MIN_H=str(24 * (L["window"] - 1)),
|
||||
RATIO_AE=f"{t['ratio_ae']:.1f}",
|
||||
G_BEYOND=fmt(t["beyond"]["globe"]), AE_BEYOND=fmt(t["beyond"]["ae"]), DP_BEYOND=fmt(t["beyond"]["dualpole"]), ME_BEYOND=fmt(t["beyond"]["mercator"]),
|
||||
AE_PCT=pct(t["beyond"]["ae"]), DP_PCT=pct(t["beyond"]["dualpole"]), ME_PCT=pct(t["beyond"]["mercator"]), G_PCT=pct(t["beyond"]["globe"]),
|
||||
FILM_N=str(f["n"]), FILM_AE=str(f["beyond"]["ae"]), FILM_G=str(f["beyond"]["globe"]), FILM_DP=str(f["beyond"]["dualpole"]), FILM_ME=str(f["beyond"]["mercator"]),
|
||||
DEVICES=", ".join(f"{fmt(n)} {k}" for k, n in sorted(t["devices"].items(), key=lambda x: -x[1])),
|
||||
BUILT=o["built"],
|
||||
)
|
||||
return v
|
||||
|
||||
|
||||
def render(out):
|
||||
tpl = os.path.join(HERE, "template.html")
|
||||
if not os.path.exists(tpl):
|
||||
return
|
||||
html = open(tpl, encoding="utf-8").read()
|
||||
for k, v in copy_values(out).items():
|
||||
html = html.replace("{{" + k + "}}", v)
|
||||
left = sorted(set(re.findall(r"\{\{(\w+)\}\}", html)))
|
||||
assert not left, left
|
||||
d3 = open(os.path.join(SRC, "raw", "lib", "d3.min.js"), encoding="utf-8").read()
|
||||
html = html.replace("/*D3*/", d3).replace("/*DATA*/null", json.dumps(out, separators=(",", ":"), ensure_ascii=False))
|
||||
os.makedirs(os.path.join(HERE, "site"), exist_ok=True)
|
||||
open(os.path.join(HERE, "site", "index.html"), "w", encoding="utf-8").write(html)
|
||||
print("site/index.html", round(len(html) / 1e6, 2), "MB")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) > 1 and sys.argv[1] == "render":
|
||||
render(json.load(open(os.path.join(HERE, "data", "penguins.json"))))
|
||||
else:
|
||||
main()
|
||||
Reference in New Issue
Block a user