#!/usr/bin/env python3 """Pull scene photographs for the AfriX post corpus from Unsplash. Sibling of fetch_portraits.py, and it borrows that file's rules exactly: * The access key is read from ./.unsplash-key (or $UNSPLASH_ACCESS_KEY) and is NEVER printed, logged, or written into any output file. * Only the official API is used. If the API says no, this stops and says so — nothing here scrapes the site or works around a bot check. * Every photograph is recorded in credits.json with its photographer and its page, because the licence asks for attribution and because a corpus whose provenance is unknown is not one you can ship. Each query is written to match the post it illustrates. A photograph that does not show what its post describes is set dressing, which is the same defect as a trend that leads nowhere or a reply count with no thread behind it. python3 fetch_scenes.py # writes img/scenes/ and credits-scenes.json python3 fetch_scenes.py --check # says what it would fetch, touches nothing Then `python3 build.py` (canvas) and `npm run media` (app) pick them up. """ import json, os, sys, urllib.parse, urllib.request from fetch_scenes_commons import thumb, FULL # same two-rung output as the Commons path HERE = os.path.dirname(os.path.abspath(__file__)) OUT = os.path.join(HERE, "img", "scenes") API = "https://api.unsplash.com/search/photos" # post id -> (query, what the photograph has to show, alt text for the shipped app) SCENES = { "p_88240912": ("container port terminal cranes africa|container port terminal cranes", "A working container terminal. Cranes and stacked boxes, not a cruise dock.", "Container cranes and stacked freight at a working port, seen from above the city."), "p_ad33380e": ("old customs house building facade|old colonial building door weathered|abandoned official building entrance", "An empty official building, weathered. The post is about disuse.", "A freshly painted blue door in the weathered wall of an old building."), "p_05d5c579": ("african market tomatoes stall|outdoor market vegetable stall morning africa|outdoor market vegetable stall morning", "An open-air produce market at trading hours.", "Baskets of ripe tomatoes on the ground at an open-air market."), # The second batch: every other feed post whose text is a scene. Data and # process posts (a gazette, an inflation table, a portal outage) stay text. "p_a2e89d24": ("freight train kenya|cargo train containers africa|freight train containers railway", "A freight train on a rail corridor, containers or wagons visible.", "Freight wagons lined up in a railway yard, the city skyline behind."), "p_97df02f4": ("dar es salaam bus rapid transit|bus lane dar es salaam|bus rapid transit africa|city bus lane", "A dedicated bus lane or BRT bus in a city.", "Bus rapid transit buses in their own lane, Dar es Salaam's towers behind."), "p_8e581cb5": ("morocco highway interchange|casablanca highway|highway interchange aerial africa|road interchange overpass", "A road interchange, signage or overpasses visible.", "A road junction seen from above, its lanes and turning bays marked out."), "p_ab22b450": ("south africa power lines pylons|electricity pylons sunset south africa|candle power cut|power lines dusk", "Electricity pylons or lines; the post is about outages.", "Power lines crossing an empty road at dusk."), "p_a8bd9b7e": ("harmattan haze|harmattan dust west africa|hazy sky sahel|dusty haze city africa", "Harmattan haze over a city or road.", "A dusty street under the flat light of a Harmattan haze."), "p_48c1b4d1": ("borehole namibia|water pump namibia|windmill water pump africa|hand pump borehole africa", "A borehole or water pump in dry country.", "A windmill water pump on a dry plain, mountains behind."), "p_f0a2e173": ("lake volta ghana|volta river ghana|akosombo dam|rain ghana village", "The Volta: the lake, the river or the dam.", "Fishing boats drawn up on a palm-lined shore of the Volta."), "p_6005a5c3": ("queue government office africa|people waiting in line africa|gaborone|waiting room queue", "People waiting in a queue at an office.", "Gaborone's business district from above, its office blocks among the trees."), "p_36928e7e": ("cape town city hall|city hall south africa|council chamber public hearing|town hall meeting", "A city hall or council chamber.", "Civic buildings on Church Square, Pretoria, at dusk."), "p_75b2a5ea": ("port of abidjan|abidjan port containers|abidjan skyline lagoon|container port west africa", "Abidjan, ideally its port.", "Abidjan's waterfront with the port beyond."), "p_25e6452c": ("cairo metro|cairo metro station|metro train egypt|cairo metro line 3", "The Cairo metro, a train or a station.", "Trains on the tracks in Cairo, the city's towers in the haze."), "p_fe834e19": ("book fair outdoor|second hand books stall street|books market outdoor table|outdoor bookshop stall", "Books outdoors. The post moved the fair into gardens.", "Second-hand books laid out on a table under trees."), } def key() -> str: k = os.environ.get("UNSPLASH_ACCESS_KEY", "").strip() if not k: p = os.path.join(HERE, ".unsplash-key") if os.path.exists(p): with open(p) as f: k = f.read().strip() if not k: sys.exit("No Unsplash access key. Put one in concept-mobile/.unsplash-key " "or set UNSPLASH_ACCESS_KEY, then run this again. " "The key is never printed and is gitignored.") return k CACHE = os.path.join(HERE, "img", ".unsplash-cache.json") # gitignored: raw API results, no key def search(query: str, k: str) -> dict: cache = json.load(open(CACHE)) if os.path.exists(CACHE) else {} if query in cache: return cache[query] url = API + "?" + urllib.parse.urlencode( {"query": query, "per_page": 8, "orientation": "landscape", "content_filter": "high"}) req = urllib.request.Request(url, headers={ "Authorization": "Client-ID " + k, "Accept-Version": "v1", }) try: with urllib.request.urlopen(req, timeout=30) as r: data = json.load(r) except urllib.error.HTTPError as e: if e.code == 403: sys.exit("Unsplash refused (403): the hourly request limit is spent. Results so far are " "cached; run again in an hour, or pick from what is already cached.") raise cache[query] = data json.dump(cache, open(CACHE, "w")) return data def main() -> None: check = "--check" in sys.argv if check: print("Would fetch, one landscape photograph each:") for pid, (q, need, _alt) in SCENES.items(): print(f" {pid} {q.replace('|', ' | ')!r}\n needs: {need}") print("\nNothing was fetched and no key was read.") return k = key() os.makedirs(OUT, exist_ok=True) credits = [] only = [a.split("=", 1)[1] for a in sys.argv if a.startswith("--only=")] cand = [a.split("=", 1)[1] for a in sys.argv if a.startswith("--candidates=")] picks = dict(a.split("=", 1)[1].split(":", 1) for a in sys.argv if a.startswith("--pick=")) if cand: # Previews for a person to look at. Nothing here touches the corpus. os.makedirs(cand[0], exist_ok=True) manifest = {} for pid, (queries, need, _alt) in SCENES.items(): if only and pid not in only: continue manifest[pid] = [] for qi, q in enumerate(queries.split("|")): if len(manifest[pid]) >= 12: break for ri, photo in enumerate(search(q, k).get("results") or []): name = f"{pid}-{qi}-{ri}.jpg" with urllib.request.urlopen(photo["urls"]["small"], timeout=60) as r, \ open(os.path.join(cand[0], name), "wb") as f: f.write(r.read()) manifest[pid].append({"file": name, "q": q, "by": photo["user"]["name"], "desc": (photo.get("description") or photo.get("alt_description") or "")[:90]}) print(f"{pid}: {len(manifest[pid])} candidates ({need})") json.dump(manifest, open(os.path.join(cand[0], "manifest.json"), "w"), indent=1) return for pid, (queries, _need, alt) in SCENES.items(): if only and pid not in only: continue photo, q = None, None if pid in picks: qi, ri = [int(x) for x in picks[pid].split(":")] q = queries.split("|")[qi] hits = search(q, k).get("results") or [] photo = hits[ri] if ri < len(hits) else None else: for q in queries.split("|"): hits = (search(q, k).get("results") or []) if hits: photo = hits[0] break if not photo: print(f" {pid}: nothing for any of {queries!r} — left as is") continue # `regular` is ~1080px on the long edge, which is above the 1600 the # media ladder caps at only in theory; NET-1 never serves the full file # to a feed, so this is the right rung to store. # `regular` (~1080 px) is the full rung, stored but never served to a # feed; the served thumb is cut from it under the 25 KB cap (NET-1). src = photo["urls"]["regular"] name = pid + ".jpg" os.makedirs(FULL, exist_ok=True) full = os.path.join(FULL, name) with urllib.request.urlopen(src, timeout=60) as r, open(full, "wb") as f: f.write(r.read()) size = thumb(full, os.path.join(OUT, name), None) # Unsplash asks for a download event per use; it is a notification, not a fetch. try: urllib.request.urlopen(urllib.request.Request( photo["links"]["download_location"], headers={"Authorization": "Client-ID " + k}), timeout=30).read() except Exception: pass credits.append({ "post": pid, "file": "scenes/" + name, "full": "scenes/full/" + name, "alt": alt, "source": "Unsplash", "title": photo.get("description") or photo.get("alt_description") or "", "author": photo["user"]["name"], "author_url": photo["user"]["links"]["html"], "licence": "Unsplash License", "licence_url": "https://unsplash.com/license", "page": photo["links"]["html"], "query": q, }) print(f"{name} <- {q!r} by {photo['user']['name']} thumb {size} B") path = os.path.join(HERE, "img", "credits-scenes.json") fetched = {c["post"] for c in credits} if os.path.exists(path): # keep credits for posts this run did not touch credits += [c for c in json.load(open(path)) if c["post"] not in fetched] credits.sort(key=lambda c: c["post"]) with open(path, "w") as f: json.dump(credits, f, indent=1, ensure_ascii=False) f.write("\n") print(f"\n{len(credits)} photographs written to img/scenes/, credits recorded.") print("Next: python3 build.py && python3 reseed.py, and `npm run media` in afrix-web.") if __name__ == "__main__": main()