"""The Penguin Test: tracked penguins on the globe and three flat maps. ./fetch.sh -> raw/raatd/ (SCAR archive, checked against raw/raatd.sha256) ../.venv/bin/python build.py -> data/penguins.json, site/index.html Data: The Retrospective Analysis of Antarctic Tracking (Standardised) Data from the Scientific Committee on Antarctic Research, version 1.5 (Ropert-Coudert et al. 2020, Scientific Data 7:94; GBIF DOI 10.15468/uzmdpm; CC BY 4.0). Penguin species only. For every tracked penguin: daily one position per UTC day: median time, median latitude, median longitude (longitudes unwrapped around the day's first fix), to damp Argos error lengths the track's length on each map, summing straight steps between daily positions (models.py: globe, North-pole AE, dual-pole AE on 0N 0E; Mercator with its east and west edges joined so that a track can cross 180 deg) stretches every pair of daily positions three days apart; the speed each needs on each map is (map distance - position allowance) / elapsed time The limit: no penguin can cover more than (swim + current) x time + allowance: swim 2.3 m/s, the fastest mean swimming speed measured in any penguin species (gentoo and chinstrap; Sato et al. 2007, Proc. R. Soc. B 274:471-477, table 1: emperor 1.7, Adelie 2.0, king 2.1 m/s) current 1.0 m/s, about twice the strongest jets of the Antarctic Circumpolar Current (typically 0.1 m/s, up to 0.5-0.6 m/s at the fronts) allowance 50 km per stretch for position error at its two ends and the limit assumes the bird swims flat out for the whole stretch, never diving, resting or walking. """ import collections, datetime as dt, json, math, os, re, statistics, sys HERE = os.path.dirname(os.path.abspath(__file__)) SRC = os.path.join(HERE, "..", "src") sys.path.insert(0, SRC) from models import R, ae_xy, dualpole_xy, globe # 6371.0 km sphere shared with the cable survey RAW = os.path.join(HERE, "raw", "raatd") GENERA = ("Aptenodytes", "Pygoscelis", "Eudyptes", "Spheniscus", "Eudyptula", "Megadyptes") V_SWIM = 2.3 # m/s V_CURRENT = 1.0 # m/s ERR_KM = 50.0 # km per stretch WINDOW = 3 # days LIMIT_KMH = (V_SWIM + V_CURRENT) * 3.6 MAPS = ["globe", "ae", "dualpole", "mercator"] DEFAULT = ("Aptenodytes forsteri", "Pointe Geologie, Terre Adelie", "2005") SOUTH_DETAIL = -35.0 # land polygons reaching south of this use 1:50m, the rest 1:110m def merc_dist(a, b): """Straight line on a Mercator map, true on the equator, east and west edges joined.""" y = lambda lat: R * math.log(math.tan(math.pi / 4 + math.radians(max(-85.0, min(85.0, lat))) / 2)) dl = (b[1] - a[1] + 540.0) % 360.0 - 180.0 return math.hypot(R * math.radians(dl), y(b[0]) - y(a[0])) def dist(m, a, b): if m == "globe": return globe(*a, *b) if m == "ae": (x1, y1), (x2, y2) = ae_xy(*a), ae_xy(*b) elif m == "dualpole": (x1, y1), (x2, y2) = dualpole_xy(*a), dualpole_xy(*b) else: return merc_dist(a, b) return math.hypot(x2 - x1, y2 - y1) def stamp(date, time): """'2005-11-01T00:46:02' or '2005-11-1' + '03:45:00Z' -> POSIX seconds (UTC).""" s = date if "T" in date else date + "T" + (time.rstrip("Z") or "00:00:00") m = re.match(r"(\d{4})-(\d{1,2})-(\d{1,2})T(\d{1,2}):(\d{2})(?::(\d{2}(?:\.\d+)?))?", s) if not m: return None y, mo, d, h, mi, sec = m.groups() return dt.datetime(int(y), int(mo), int(d), int(h), int(mi), int(float(sec or 0)), tzinfo=dt.timezone.utc).timestamp() def read_rows(name): with open(os.path.join(RAW, name), encoding="utf-8") as f: head = f.readline().rstrip("\n").split("\t") for line in f: r = line.rstrip("\n").split("\t") if len(r) >= len(head): yield dict(zip(head, r)) def load(): meta = {} for r in read_rows("event.txt"): if not r["eventID"].startswith("capture:"): continue sp = r["samplingProtocol"] dev = sp.split("device type=")[1].split(":")[0].strip() if "device type=" in sp else "" meta[r["eventID"]] = dict(dev=dev, colony=r["locality"].strip(), year=r["year"], dataset=r["datasetID"], clat=float(r["decimalLatitude"] or "nan"), clon=float(r["decimalLongitude"] or "nan")) fixes = collections.defaultdict(list) who = {} for r in read_rows("occurrence.txt"): if not r["scientificName"].startswith(GENERA): continue if "::capture" in r["occurrenceID"]: who.setdefault(r["id"], {}).update(sex=r["sex"], stage=r["lifeStage"]) continue t = stamp(r["eventDate"], r["eventTime"]) try: lat, lon = float(r["decimalLatitude"]), float(r["decimalLongitude"]) except ValueError: continue if t is None or not (-90 <= lat <= 90 and -180 <= lon <= 180): continue fixes[r["id"]].append((t, lat, lon)) who.setdefault(r["id"], {}).update(sp=r["scientificName"], name=r["vernacularName"]) return meta, fixes, who def daily(fx): """One position per UTC day: median time, latitude and (unwrapped) longitude.""" by = collections.defaultdict(list) for t, la, lo in sorted(fx): by[int(t // 86400)].append((t, la, lo)) out = [] for d in sorted(by): v = by[d] l0 = v[0][2] lons = [l0 + ((lo - l0 + 540.0) % 360.0 - 180.0) for _, _, lo in v] lon = (statistics.median(lons) + 540.0) % 360.0 - 180.0 out.append((d, statistics.median(t for t, _, _ in v), statistics.median(la for _, la, _ in v), lon, len(v))) return out def track_stats(D): """Lengths on each map and the 3-day stretch that needs the highest speed on each map.""" P = [(x[2], x[3]) for x in D] st = {} for m in MAPS: length = sum(dist(m, P[i - 1], P[i]) for i in range(1, len(P))) best = None for i in range(len(D)): for j in range(i + 1, len(D)): if D[j][0] - D[i][0] > WINDOW: break if D[j][0] - D[i][0] < WINDOW: continue hours = (D[j][1] - D[i][1]) / 3600 if hours < 24 * (WINDOW - 1): continue km = dist(m, P[i], P[j]) need = max(0.0, km - ERR_KM) / hours if best is None or need > best[0]: best = (need, km, hours, i, j) st[m] = dict(len=length, need=best[0] if best else None, km=best[1] if best else None, h=best[2] if best else None, i=best[3] if best else None, j=best[4] if best else None) return st def land(path, nd=2): """TopoJSON land -> list of polygons (rings of [lon, lat]).""" t = json.load(open(path)) sx, sy = t["transform"]["scale"] tx, ty = t["transform"]["translate"] arcs = [] for arc in t["arcs"]: x = y = 0 pts = [] for dx, dy in arc: x += dx; y += dy pts.append([round(x * sx + tx, nd), round(y * sy + ty, nd)]) arcs.append(pts) def ring(idx): out = [] for i in idx: a = arcs[i] if i >= 0 else arcs[~i][::-1] out.extend(a if not out else a[1:]) return out polys = [] for g in t["objects"]["land"]["geometries"]: if g["type"] == "Polygon": polys.append([ring(r) for r in g["arcs"]]) elif g["type"] == "MultiPolygon": polys.extend([[ring(r) for r in p] for p in g["arcs"]]) return polys def coast(): """1:50m for land reaching south of SOUTH_DETAIL (Antarctica, the sub-Antarctic islands, the southern continents' tips), 1:110m elsewhere.""" south = lambda p: min(y for _, y in p[0]) < SOUTH_DETAIL fine = [p for p in land(os.path.join(SRC, "raw", "geo", "land-50m.json")) if south(p)] coarse = [p for p in land(os.path.join(SRC, "raw", "geo", "land-110m.json")) if not south(p)] return coarse + fine SPECIES = {"Aptenodytes forsteri": "Emperor penguin", "Aptenodytes patagonicus": "King penguin", "Pygoscelis adeliae": "Adélie penguin", "Eudyptes chrysolophus": "Macaroni penguin", "Eudyptes schlegeli": "Royal penguin"} COLONY_FIX = {"Pointe Geologie, Terre Adelie": "Pointe Géologie, Terre Adélie"} def main(): meta, fixes, who = load() tracks = [] for iid, fx in fixes.items(): D = daily(fx) if len(D) < 2: continue m = meta.get(iid, {}) st = track_stats(D) d0 = D[0][0] tracks.append(dict( id=iid.replace("capture:", ""), sp=who[iid]["sp"], colony=m.get("colony", ""), year=m.get("year", ""), dev=m.get("dev", ""), sex=who[iid].get("sex", ""), stage=who[iid].get("stage", ""), dataset=m.get("dataset", ""), start=dt.datetime.fromtimestamp(D[0][1], dt.timezone.utc).strftime("%Y-%m-%d"), end=dt.datetime.fromtimestamp(D[-1][1], dt.timezone.utc).strftime("%Y-%m-%d"), fixes=len(fx), d=[x[0] - d0 for x in D], lat=[round(x[2], 3) for x in D], lon=[round(x[3], 3) for x in D], st={k: {kk: (round(vv, 3) if isinstance(vv, float) else vv) for kk, vv in v.items()} for k, v in st.items()})) tracks.sort(key=lambda t: (t["sp"], t["colony"], t["year"], t["start"], t["id"])) tested = [t for t in tracks if t["st"]["globe"]["need"] is not None] # summaries at the default limit def beyond(t, m, lim=LIMIT_KMH): return t["st"][m]["need"] is not None and t["st"][m]["need"] > lim by_sp = collections.OrderedDict() for sp in sorted({t["sp"] for t in tracks}, key=lambda s: -sum(t["sp"] == s for t in tracks)): v = [t for t in tested if t["sp"] == sp] by_sp[sp] = dict(name=SPECIES.get(sp, sp), tracks=sum(t["sp"] == sp for t in tracks), tested=len(v), beyond={m: sum(beyond(t, m) for t in v) for m in MAPS}, med_need={m: round(statistics.median(t["st"][m]["need"] for t in v), 2) if v else None for m in MAPS}, med_len={m: round(statistics.median(t["st"][m]["len"] for t in v)) if v else None for m in MAPS}) tot = dict(tracks=len(tracks), tested=len(tested), beyond={m: sum(beyond(t, m) for t in tested) for m in MAPS}, days=sum(len(t["d"]) for t in tracks), fixes=sum(t["fixes"] for t in tracks), colonies=len({t["colony"] for t in tracks}), years=[min(t["year"] for t in tracks), max(t["year"] for t in tracks)], devices=dict(collections.Counter(t["dev"] or "unrecorded" for t in tracks)), ratio_ae=round(statistics.median(t["st"]["ae"]["len"] / t["st"]["globe"]["len"] for t in tracks if t["st"]["globe"]["len"] > 50), 2)) groups = collections.OrderedDict() for t in tracks: k = f'{t["sp"]}|{t["colony"]}|{t["year"]}' groups.setdefault(k, dict(sp=t["sp"], colony=COLONY_FIX.get(t["colony"], t["colony"]), year=t["year"], n=0))["n"] += 1 film = [t for t in tested if (t["sp"], t["colony"], t["year"]) == DEFAULT] out = dict(tracks=tracks, groups=groups, species=by_sp, total=tot, film=dict(n=len(film), beyond={m: sum(beyond(t, m) for t in film) for m in MAPS}), default="|".join(DEFAULT), limit=dict(v_swim=V_SWIM, v_current=V_CURRENT, err_km=ERR_KM, window=WINDOW, kmh=round(LIMIT_KMH, 2)), names=SPECIES, colony_fix=COLONY_FIX, land=coast(), built=dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%d %H:%M UTC")) os.makedirs(os.path.join(HERE, "data"), exist_ok=True) with open(os.path.join(HERE, "data", "penguins.json"), "w") as fh: json.dump(out, fh, separators=(",", ":")) print(f"{tot['tracks']} tracks ({tot['tested']} with a {WINDOW}-day stretch), {tot['days']} daily positions from {tot['fixes']} fixes, " f"{len(groups)} colony-seasons; limit {LIMIT_KMH:.2f} km/h + {ERR_KM:.0f} km") for sp, s in by_sp.items(): print(f" {s['name']:16s} {s['tested']:4d} tested | beyond the limit: " + " ".join(f"{m} {s['beyond'][m]}" for m in MAPS)) print(" all", tot["beyond"], "| film colony 2005:", out["film"]) render(out) def fmt(n): return f"{n:,}" def copy_values(o): t, sp, f, L = o["total"], o["species"], o["film"], o["limit"] pct = lambda k: f"{100 * k / t['tested']:.0f}%" v = dict( N_TRACKS=fmt(t["tracks"]), N_TESTED=fmt(t["tested"]), N_DAYS=fmt(t["days"]), N_FIXES=fmt(t["fixes"]), N_SPECIES=str(len(sp)), N_COLONIES=fmt(t["colonies"]), Y0=t["years"][0], Y1=t["years"][1], LIMIT=f"{L['kmh']:.1f}", V_SWIM=f"{L['v_swim']:.1f}", V_CURRENT=f"{L['v_current']:.1f}", ERR=f"{L['err_km']:.0f}", WINDOW=str(L["window"]), MIN_H=str(24 * (L["window"] - 1)), RATIO_AE=f"{t['ratio_ae']:.1f}", G_BEYOND=fmt(t["beyond"]["globe"]), AE_BEYOND=fmt(t["beyond"]["ae"]), DP_BEYOND=fmt(t["beyond"]["dualpole"]), ME_BEYOND=fmt(t["beyond"]["mercator"]), AE_PCT=pct(t["beyond"]["ae"]), DP_PCT=pct(t["beyond"]["dualpole"]), ME_PCT=pct(t["beyond"]["mercator"]), G_PCT=pct(t["beyond"]["globe"]), FILM_N=str(f["n"]), FILM_AE=str(f["beyond"]["ae"]), FILM_G=str(f["beyond"]["globe"]), FILM_DP=str(f["beyond"]["dualpole"]), FILM_ME=str(f["beyond"]["mercator"]), DEVICES=", ".join(f"{fmt(n)} {k}" for k, n in sorted(t["devices"].items(), key=lambda x: -x[1])), BUILT=o["built"], ) return v def render(out): tpl = os.path.join(HERE, "template.html") if not os.path.exists(tpl): return html = open(tpl, encoding="utf-8").read() for k, v in copy_values(out).items(): html = html.replace("{{" + k + "}}", v) left = sorted(set(re.findall(r"\{\{(\w+)\}\}", html))) assert not left, left d3 = open(os.path.join(SRC, "raw", "lib", "d3.min.js"), encoding="utf-8").read() html = html.replace("/*D3*/", d3).replace("/*DATA*/null", json.dumps(out, separators=(",", ":"), ensure_ascii=False)) os.makedirs(os.path.join(HERE, "site"), exist_ok=True) open(os.path.join(HERE, "site", "index.html"), "w", encoding="utf-8").write(html) print("site/index.html", round(len(html) / 1e6, 2), "MB") if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == "render": render(json.load(open(os.path.join(HERE, "data", "penguins.json")))) else: main()