← A tiny storyteller: how a computer learns to write, one letter at a time

pictures.py

Python · 459 lines · Download raw ·

1"""
2Pictures and animations for "A tiny storyteller", made from what experiment.py
3saved: results.csv, samples.json and probes.json (and the book, grimm.txt).
4
5Every number and letter in these pictures comes from those files: the model's
6real odds, its real guesses and its real stories. Nothing is made up or tidied.
7
8Outputs:
9 book.png the book as one long line of letters, split into practice and hidden parts
10 alphabet.png every letter and symbol the model can choose from, shaded by how common it is
11 practice.png where the first practice step read from, and how one stretch becomes many guessing questions
12 model.png what happens inside the model when it makes one guess
13 torn.png "torn between": a coin, a dice, and our model before and after practice
14 odds.gif its odds for the letter after "Once upon a ti", at each checkpoint (animation)
15 odds.png the same odds, before and after practice, for two beginnings
16 read-along.gif the trained model guessing a hidden sentence, letter by letter (animation)
17 read-along.png the same sentence at every checkpoint: green = first guess right
18 first-guess.png out of 100 guesses, how many first guesses were right, at each checkpoint
19 typewriter.gif the trained model writing a story, one letter at a time, with its odds (animation)
20 growing.gif the story it writes at each checkpoint (animation)
21 attention.png which earlier letters it looks back at most, for one guess
22 dial.png the same model writing with the "adventure dial" set to careful, normal and wild
23
24Run (after experiment.py):
25 python3 pictures.py
26"""
27
28import csv
29import io
30import json
31import math
32import textwrap
33from collections import Counter
34
35import matplotlib
36from PIL import Image
37
38matplotlib.use("Agg")
39import matplotlib.pyplot as plt
40from matplotlib.patches import Circle, FancyBboxPatch, Rectangle, Wedge
41
42BLUE, ORANGE, GREEN, RED, GREY, INK = "#2557a7", "#c2410c", "#15803d", "#b91c1c", "#9ca3af", "#1f2937"
43PALE_BLUE, PALE_ORANGE, PALE_GREEN, PALE_RED = "#dbe6f7", "#fbe3d6", "#dcfce7", "#fee2e2"
44MONO = "DejaVu Sans Mono"
45plt.rcParams.update({"font.size": 11, "axes.spines.top": False, "axes.spines.right": False})
46
47
48def show(ch):
49 """How to draw one letter on a tile: spaces and line breaks get a visible sign."""
50 return {" ": "␣", "\n": "↵", None: ""}.get(ch, ch)
51
52
53def flat(text):
54 return text.replace("\n", " ")
55
56
57def save(fig, path):
58 fig.savefig(path, facecolor="white", dpi=150, bbox_inches="tight", pad_inches=0.15)
59 plt.close(fig)
60
61
62def frame(fig):
63 """Render a figure to a picture, for an animation."""
64 buf = io.BytesIO()
65 fig.savefig(buf, format="png", facecolor="white", dpi=100)
66 plt.close(fig)
67 buf.seek(0)
68 return Image.open(buf).convert("RGB")
69
70
71def save_gif(frames, durations, path):
72 """frames: list of pictures; durations in milliseconds (one per frame). Loops forever."""
73 pal = [f.quantize(colors=64, method=Image.Quantize.MEDIANCUT) for f in frames]
74 pal[0].save(path, save_all=True, append_images=pal[1:], duration=durations, loop=0, optimize=True)
75
76
77def torn_between(odds):
78 """How many choices a set of odds is 'torn between' (e to the power of its entropy)."""
79 return math.exp(-sum(p * math.log(p) for p in odds if p > 0))
80
81
82def tile(ax, x, y, ch, face="white", edge=GREY, color=INK, w=0.9, h=0.9, size=12, weight="normal"):
83 ax.add_patch(FancyBboxPatch((x, y), w, h, boxstyle="round,pad=0,rounding_size=0.12", facecolor=face, edgecolor=edge, lw=1))
84 ax.text(x + w / 2, y + h / 2, show(ch), ha="center", va="center", family=MONO, fontsize=size, color=color, weight=weight)
85
86
87# ---------------------------------------------------------------------------
88# The setting: the book, the letters, the practice
89# ---------------------------------------------------------------------------
90
91def book_picture(P, book):
92 total, train, test = P["book_letters"], P["training_letters"], P["test_letters"]
93 fig, ax = plt.subplots(figsize=(8, 3.4))
94 ax.set_xlim(0, 100), ax.set_ylim(0, 34), ax.axis("off")
95 split = 100 * train / total
96 ax.add_patch(Rectangle((0, 14), split, 6, color=BLUE))
97 ax.add_patch(Rectangle((split, 14), 100 - split, 6, color=ORANGE))
98 ax.text(split / 2, 17, f"practice part: {train:,} letters (90%)", ha="center", va="center", color="white", weight="bold")
99 ax.text(100, 11.5, f"hidden part: {test:,} letters (10%)", ha="right", va="top", color=ORANGE, fontsize=9.5, weight="bold")
100 ax.text(0, 31, f"Grimms' Fairy Tales, written out as one long line of {total:,} letters", fontsize=12, weight="bold", color=INK)
101 ax.text(0, 27, "The model practises on the blue part. We keep the orange part hidden, to test it later.", fontsize=10, color=INK)
102 # zoom in on the very start of the book
103 ax.plot([0, 0], [14, 8.2], color=GREY, lw=1), ax.plot([0.3, 70], [14, 8.2], color=GREY, lw=1)
104 piece = book[:44]
105 for i, ch in enumerate(piece):
106 tile(ax, i * 1.6, 3.2, ch, w=1.45, h=5, size=9)
107 ax.text(0, 0.2, "Zoomed in: every letter, space (␣), comma and line break (↵) counts as a 'letter'.", fontsize=9.5, color=INK)
108 save(fig, "book.png")
109
110
111def alphabet_picture(P, book):
112 alphabet, counts = P["alphabet"], Counter(book)
113 order = sorted(alphabet, key=lambda c: -counts[c])
114 cols = 14
115 rows = math.ceil(len(order) / cols)
116 fig, ax = plt.subplots(figsize=(8, 0.75 * rows + 1.3))
117 ax.set_xlim(0, cols), ax.set_ylim(-0.2, rows + 1.1), ax.axis("off"), ax.set_aspect("equal")
118 top = counts[order[0]]
119 cmap = plt.get_cmap("Blues")
120 for k, ch in enumerate(order):
121 r, c = divmod(k, cols)
122 share = math.log(counts[ch]) / math.log(top)
123 face = cmap(0.08 + 0.85 * share)
124 tile(ax, c, rows - 1 - r, ch, face=face, color="white" if share > 0.6 else INK, size=11)
125 ax.text(0, rows + 0.4, f"The {len(alphabet)} 'letters' the model can choose from, most common first",
126 fontsize=12, weight="bold", color=INK)
127 common = ", ".join(f"'{show(c)}' {100 * counts[c] / len(book):.0f}%" for c in order[:3])
128 ax.text(0, -0.15, f"Darker = more common in the book. The most common: {common}.", fontsize=9.5, color=INK, va="top")
129 save(fig, "alphabet.png")
130
131
132def practice_picture(P, book):
133 starts, ctx, train = P["first_batch_starts"], P["context"], P["training_letters"]
134 fig = plt.figure(figsize=(8, 5.4))
135 ax = fig.add_axes([0.03, 0.62, 0.94, 0.36])
136 ax.set_xlim(0, 100), ax.set_ylim(0, 12), ax.axis("off")
137 ax.add_patch(Rectangle((0, 2), 100, 3, color=PALE_BLUE))
138 for s in starts:
139 x = 100 * s / train
140 ax.plot([x, x], [5, 8.2], color=BLUE, lw=1)
141 ax.add_patch(Circle((x, 8.6), 0.55, color=BLUE))
142 ax.text(0, 10.6, f"The first practice step: {len(starts)} stretches of {ctx} letters, from random places",
143 fontsize=11.5, weight="bold", color=INK)
144 ax.text(0, 0, "start of the practice part", fontsize=8.5, color=GREY), ax.text(100, 0, "end", fontsize=8.5, color=GREY, ha="right")
145
146 bx = fig.add_axes([0.03, 0.02, 0.94, 0.56])
147 n = 16
148 s = starts[0]
149 piece = book[s:s + n + 1]
150 bx.set_xlim(-3.2, n + 0.5), bx.set_ylim(-0.6, 6.4), bx.axis("off")
151 bx.text(-3.2, 5.9, f"Zoom in on the first stretch (its first {n} letters). Every letter is a guessing question:",
152 fontsize=10.5, color=INK)
153 bx.text(-3.2, 3.95, "the model\nreads", fontsize=9, color=BLUE, va="center")
154 bx.text(-3.2, 1.45, "and must\nguess", fontsize=9, color=ORANGE, va="center")
155 for i in range(n):
156 tile(bx, i, 3.5, piece[i], face=PALE_BLUE, edge=BLUE, size=11)
157 tile(bx, i, 1.0, piece[i + 1], face=PALE_ORANGE, edge=ORANGE, size=11)
158 bx.annotate("", xy=(i + 0.45, 1.95), xytext=(i + 0.45, 3.45), arrowprops=dict(arrowstyle="->", color=GREY, lw=0.8))
159 bx.text(-3.2, 0.1, f"Reading up to letter 1 it guesses letter 2, reading up to letter 2 it guesses letter 3, and so on: "
160 f"{ctx} questions per stretch,\n{len(starts) * ctx:,} questions per practice step. After each step it nudges its "
161 f"{P['parameters']:,} numbers a little, towards the right answers.", fontsize=8.8, color=INK, va="top")
162 save(fig, "practice.png")
163
164
165def model_picture(P):
166 final = P["odds"][-1][P["guess_prompts"][0]]
167 best = sorted(range(len(final)), key=lambda i: -final[i])[:2]
168 example = "\n".join(f"{show(P['alphabet'][i])} {100 * final[i]:.0f}%" for i in best)
169 fig, ax = plt.subplots(figsize=(8, 3.6))
170 ax.set_xlim(0, 100), ax.set_ylim(0, 40), ax.axis("off")
171
172 def box(x, w, title, body, face, edge):
173 ax.add_patch(FancyBboxPatch((x, 7), w, 24, boxstyle="round,pad=0.4,rounding_size=1.5", facecolor=face, edgecolor=edge, lw=1.5))
174 ax.text(x + w / 2, 27.5, title, ha="center", va="center", weight="bold", fontsize=10.5, color=INK)
175 ax.text(x + w / 2, 17, body, ha="center", va="center", fontsize=8.6, color=INK, linespacing=1.4)
176
177 box(0, 21, "1. Read", f"the last {P['context']}\nletters, e.g.\n“…upon a ti”", PALE_BLUE, BLUE)
178 box(26, 21, "2. Turn into\nnumbers", f"each letter becomes\na list of {P['width']}\nnumbers", "#f3f4f6", GREY)
179 box(52, 21, f"3. {P['layers']} layers of\nlook back + think", "look back at earlier\nletters (attention),\nthen mix it all up", "#f3f4f6", GREY)
180 box(78, 21, "4. Odds for the\nnext letter", f"{example}\n… for all {len(P['alphabet'])} letters", PALE_ORANGE, ORANGE)
181 for x in (21.5, 47.5, 73.5):
182 ax.annotate("", xy=(x + 4, 19), xytext=(x, 19), arrowprops=dict(arrowstyle="-|>", color=INK, lw=1.5))
183 ax.text(50, 36.5, "What happens inside the model when it makes one guess", ha="center", fontsize=12, weight="bold", color=INK)
184 ax.text(50, 0, f"All of this is done with {P['parameters']:,} adjustable numbers. Practice nudges those numbers; nothing else changes.\n"
185 f"Box 4 shows the trained model's real top 2 odds after “{P['guess_prompts'][0]}”.", ha="center", fontsize=8.6, color=INK)
186 save(fig, "model.png")
187
188
189def torn_picture(P, rows):
190 start, end = math.exp(rows[0]["test_loss"]), math.exp(rows[-1]["test_loss"])
191 fig, axes = plt.subplots(1, 4, figsize=(8.4, 3.6), gridspec_kw={"width_ratios": [1, 1, 1.5, 1]})
192 items = [("a coin", 2, "heads or tails:\ntorn between 2"),
193 ("a dice", 6, "1 to 6:\ntorn between 6"),
194 ("our model,\nbefore practice", start, f"torn between\nabout {start:.0f} letters"),
195 ("our model,\nafter practice", end, f"torn between\nabout {end:.1f} letters")]
196 for ax, (name, n, label) in zip(axes, items):
197 ax.axis("off"), ax.set_aspect("equal")
198 k = math.ceil(n)
199 cols = 2 if k <= 2 else 3 if k <= 6 else 12 if k > 40 else 4
200 r = math.ceil(k / cols)
201 ax.set_xlim(-0.2, max(cols, 3) + 0.2), ax.set_ylim(-3.2, 10.6)
202 for i in range(k):
203 rr, cc = divmod(i, cols)
204 part = min(1.0, n - i) # the last square is only partly filled for 3.8 and 93.2
205 ax.add_patch(Rectangle((cc + 0.06, r - 1 - rr + 0.06), 0.88, 0.88, facecolor="white", edgecolor=ORANGE if "model" in name else BLUE, lw=1))
206 ax.add_patch(Rectangle((cc + 0.06, r - 1 - rr + 0.06), 0.88 * part, 0.88, color=ORANGE if "model" in name else BLUE, alpha=0.75))
207 ax.text(max(cols, 3) / 2, 9.4, name, ha="center", va="center", fontsize=10, weight="bold", color=INK)
208 ax.text(max(cols, 3) / 2, -0.4, label, ha="center", va="top", fontsize=9.5, color=INK)
209 fig.suptitle("“Torn between”: how many choices it is really stuck between (smaller is better)",
210 fontsize=11, weight="bold", color=INK, y=0.99)
211 save(fig, "torn.png")
212
213
214# ---------------------------------------------------------------------------
215# Its guesses
216# ---------------------------------------------------------------------------
217
218def odds_axes(ax, odds, alphabet, real, k=10, ymax=1.0):
219 best = sorted(range(len(odds)), key=lambda i: -odds[i])[:k]
220 if real in alphabet and alphabet.index(real) not in best: # always show the real letter, even when it is far down
221 best[-1] = alphabet.index(real)
222 xs = range(len(best))
223 ax.bar(xs, [100 * odds[i] for i in best], color=[ORANGE if alphabet[i] == real else BLUE for i in best], width=0.75)
224 ax.set_xticks(list(xs), [show(alphabet[i]) for i in best], family=MONO, fontsize=13)
225 ax.set_ylim(0, 100 * ymax), ax.set_ylabel("odds (%)")
226 for x, i in zip(xs, best):
227 if odds[i] >= 0.01:
228 ax.text(x, 100 * odds[i] + 1.5, f"{100 * odds[i]:.0f}%", ha="center", fontsize=8.5, color=INK)
229
230
231def odds_animation(P):
232 prompt, real = P["guess_prompts"][0], "m"
233 frames, durations = [], []
234 for snap in P["odds"]:
235 odds = snap[prompt]
236 fig, ax = plt.subplots(figsize=(6.4, 3.8))
237 odds_axes(ax, odds, P["alphabet"], real)
238 ax.set_title(f"“{prompt}…” what comes next?\nAfter {snap['step']:,} practice steps: torn between about {torn_between(odds):.1f}",
239 fontsize=11, color=INK)
240 ax.text(0.99, 0.95, "orange = the real next letter", transform=ax.transAxes, ha="right", fontsize=9, color=ORANGE)
241 fig.tight_layout()
242 frames.append(frame(fig)), durations.append(1800)
243 durations[-1] = 3500
244 save_gif(frames, durations, "odds.gif")
245
246
247def odds_picture(P):
248 reals = {P["guess_prompts"][0]: "m", P["guess_prompts"][1]: "e"}
249 first, last = P["odds"][0], P["odds"][-1]
250 fig, axes = plt.subplots(2, 2, figsize=(8, 5.6), sharey=True)
251 for r, prompt in enumerate(P["guess_prompts"]):
252 for c, snap in enumerate((first, last)):
253 ax = axes[r][c]
254 odds_axes(ax, snap[prompt], P["alphabet"], reals[prompt], k=8)
255 ax.set_title(f"“{prompt}…”, after {snap['step']:,} steps\ntorn between about {torn_between(snap[prompt]):.1f}", fontsize=9.5, color=INK)
256 if c:
257 ax.set_ylabel("")
258 fig.suptitle("Its odds for the next letter, before and after practice (orange = the real next letter)", fontsize=11, weight="bold", color=INK)
259 fig.tight_layout()
260 save(fig, "odds.png")
261
262
263def sentence_row(ax, y, sentence, guesses, upto=None, size=10.5):
264 """Draw sentence as tiles: green if the first guess was right, red if not, plain if not reached yet."""
265 for i, ch in enumerate(sentence):
266 if upto is not None and i > upto:
267 tile(ax, i, y, None, face="white", edge="#e5e7eb", size=size)
268 continue
269 if upto is not None and i == upto:
270 tile(ax, i, y, "?", face="#fef9c3", edge="#ca8a04", size=size, weight="bold")
271 continue
272 right = guesses[i]["top"][0][0] == ch
273 tile(ax, i, y, ch, face=PALE_GREEN if right else PALE_RED, edge=GREEN if right else RED, size=size)
274
275
276def read_along_animation(P):
277 sentence, last = P["read_along_sentence"], P["read_along"][-1]
278 guesses = last["guesses"]
279 n = len(sentence)
280 frames, durations = [], []
281 for i in range(n + 1):
282 fig = plt.figure(figsize=(8.6, 4.2))
283 ax = fig.add_axes([0.02, 0.62, 0.96, 0.33])
284 ax.set_xlim(-0.2, n + 0.2), ax.set_ylim(-0.6, 2.2), ax.axis("off")
285 sentence_row(ax, 0, sentence, guesses, upto=i if i < n else None, size=9)
286 done = guesses[:i]
287 right = sum(g["top"][0][0] == g["real"] for g in done)
288 ax.text(0, 1.6, f"A sentence from the hidden part of the book. First guess right so far: {right} of {len(done)}",
289 fontsize=11, weight="bold", color=INK)
290 bx = fig.add_axes([0.1, 0.14, 0.55, 0.38])
291 if i < n:
292 g = guesses[i]
293 labels = [show(c) for c, _ in g["top"]]
294 bx.barh(range(3)[::-1], [100 * p for _, p in g["top"]], color=[GREEN if c == g["real"] else BLUE for c, _ in g["top"]])
295 bx.set_yticks(range(3)[::-1], labels, family=MONO, fontsize=14)
296 bx.set_xlim(0, 100), bx.set_xlabel("odds (%)")
297 for k, (_, p) in enumerate(g["top"]):
298 bx.text(100 * p + 1, 2 - k, f"{100 * p:.0f}%", va="center", fontsize=9)
299 bx.set_title("its top 3 guesses for the ? letter", fontsize=10, color=INK)
300 verdict = "first guess right!" if g["top"][0][0] == g["real"] else "first guess wrong"
301 fig.text(0.72, 0.32, f"real letter: '{show(g['real'])}'\n{verdict}\n(it gave the real letter\n{100 * g['real_odds']:.0f}% odds)",
302 fontsize=11, color=GREEN if g["top"][0][0] == g["real"] else RED, va="center")
303 else:
304 bx.axis("off")
305 fig.text(0.1, 0.3, f"Done: its first guess was right for {right} of {n} letters.\n"
306 "Green = first guess right. Red = first guess wrong.", fontsize=12, color=INK)
307 frames.append(frame(fig)), durations.append(900 if i < n else 4000)
308 save_gif(frames, durations, "read-along.gif")
309
310
311def read_along_picture(P):
312 sentence = P["read_along_sentence"]
313 n, snaps = len(sentence), P["read_along"]
314 fig, ax = plt.subplots(figsize=(8.8, 0.62 * len(snaps) + 1.0))
315 ax.set_xlim(-6.5, n + 0.2), ax.set_ylim(-0.4, 1.25 * len(snaps) + 0.5), ax.axis("off")
316 for r, snap in enumerate(snaps):
317 y = 1.25 * (len(snaps) - 1 - r)
318 sentence_row(ax, y, sentence, snap["guesses"], size=8.5)
319 right = sum(g["top"][0][0] == g["real"] for g in snap["guesses"])
320 ax.text(-6.4, y + 0.45, f"{snap['step']:>5,} steps\n{right:>3} of {n}", va="center", fontsize=8.5, family=MONO, color=INK)
321 ax.text(-6.4, 1.25 * len(snaps) + 0.2, "The same hidden sentence at each checkpoint. Green = first guess right, red = wrong.",
322 fontsize=10.5, weight="bold", color=INK)
323 save(fig, "read-along.png")
324
325
326def first_guess_picture(rows):
327 fig, axes = plt.subplots(1, len(rows), figsize=(8.4, 2.9))
328 for ax, row in zip(axes, rows):
329 right = round(100 * row["test_first_guess_right"])
330 ax.set_xlim(0, 10), ax.set_ylim(-3.2, 11.6), ax.axis("off"), ax.set_aspect("equal")
331 for k in range(100):
332 r, c = divmod(k, 10)
333 ax.add_patch(Rectangle((c + 0.08, 9 - r + 0.08), 0.84, 0.84, color=GREEN if k < right else "#e5e7eb"))
334 ax.text(5, 10.8, f"after {row['step']:,} steps", ha="center", fontsize=9.5, weight="bold", color=INK)
335 ax.text(5, -0.6, f"{right} of 100\nright", ha="center", va="top", fontsize=10, color=GREEN)
336 fig.suptitle("Out of 100 guesses on the hidden part of the book, how many first guesses were right?",
337 fontsize=11, weight="bold", color=INK)
338 save(fig, "first-guess.png")
339
340
341# ---------------------------------------------------------------------------
342# Its writing
343# ---------------------------------------------------------------------------
344
345def typewriter_animation(P):
346 prompt, steps = P["prompt"], P["typewriter"]
347 frames, durations = [], []
348 written = ""
349 for i, st in enumerate(steps + [None]):
350 fig = plt.figure(figsize=(8, 4.4))
351 tx = fig.add_axes([0.03, 0.55, 0.94, 0.42])
352 tx.axis("off")
353 tx.text(0, 1, "The trained model writes a story, one letter at a time", fontsize=12, weight="bold", color=INK, va="top")
354 body = " " * len(prompt) + flat(written) + ("▌" if st else "")
355 lines = "\n".join(body[k:k + 56] for k in range(0, len(body), 56))
356 tx.text(0, 0.72, lines, fontsize=13, family=MONO, color=INK, va="top")
357 tx.text(0, 0.72, prompt, fontsize=13, family=MONO, color=GREY, va="top")
358 bx = fig.add_axes([0.12, 0.12, 0.5, 0.34])
359 if st:
360 letters = [show(c) for c, _ in st["top"]]
361 bx.barh(range(5)[::-1], [100 * p for _, p in st["top"]], color=[ORANGE if c == st["picked"] else BLUE for c, _ in st["top"]])
362 bx.set_yticks(range(5)[::-1], letters, family=MONO, fontsize=13)
363 bx.set_xlim(0, 100), bx.set_xlabel("odds (%)")
364 bx.set_title("its top 5 choices for the next letter", fontsize=10, color=INK)
365 picked_rank = [c for c, _ in st["top"]].index(st["picked"]) + 1 if st["picked"] in [c for c, _ in st["top"]] else None
366 note = "the most likely letter" if picked_rank == 1 else "not the most likely letter:\nlike rolling a weighted dice,\nless likely letters sometimes win" if picked_rank else "a rare letter outside the top 5"
367 fig.text(0.67, 0.27, f"picked: '{show(st['picked'])}'\n{note}", fontsize=10.5, color=ORANGE, va="center")
368 written += st["picked"]
369 else:
370 bx.axis("off")
371 frames.append(frame(fig)), durations.append(450 if st else 4000)
372 save_gif(frames, durations, "typewriter.gif")
373
374
375def growing_animation(samples, rows):
376 frames, durations = [], []
377 for s, row in zip(samples, rows):
378 fig = plt.figure(figsize=(8, 2.9))
379 ax = fig.add_axes([0.03, 0.03, 0.94, 0.92])
380 ax.axis("off")
381 ax.text(0, 1, f"After {s['step']:,} practice steps", fontsize=15, weight="bold", color=BLUE, va="top")
382 ax.text(0, 0.84, f"torn between about {math.exp(row['test_loss']):.1f} letters · first guess right "
383 f"{100 * row['test_first_guess_right']:.0f} times in 100", fontsize=10, color=INK, va="top")
384 ax.text(0, 0.66, "\n".join(textwrap.wrap(flat(s["story"])[:230], 62)), fontsize=11.5, family=MONO, color=INK, va="top")
385 frames.append(frame(fig)), durations.append(3200)
386 durations[-1] = 5000
387 save_gif(frames, durations, "growing.gif")
388
389
390def attention_picture(P):
391 A = P["attention"]
392 text, layers = A["text"], A["layers"]
393 n = len(text)
394 fig, ax = plt.subplots(figsize=(9, 0.75 * len(layers) + 2.1))
395 ax.set_xlim(-4.2, n + 1.5), ax.set_ylim(-1.9, 1.15 * len(layers) + 1.4), ax.axis("off")
396 cmap = plt.get_cmap("Oranges")
397 for r, w in enumerate(layers):
398 y = 1.15 * (len(layers) - 1 - r)
399 biggest = max(w)
400 for i, ch in enumerate(text):
401 share = w[i] / biggest
402 tile(ax, i, y, ch, face=cmap(0.05 + 0.85 * share), edge="#e5e7eb", size=9, color="white" if share > 0.65 else INK)
403 tile(ax, n + 0.3, y, "?", face="#fef9c3", edge="#ca8a04", size=9, weight="bold")
404 ax.text(-4.1, y + 0.45, f"layer {r + 1}", va="center", fontsize=9, color=INK)
405 guess = ", ".join(f"'{show(c)}' {100 * p:.0f}%" for c, p in A["top"][:3])
406 ax.text(-4.1, 1.15 * len(layers) + 0.6, "Guessing the ? letter: which earlier letters does each layer look back at most?",
407 fontsize=11, weight="bold", color=INK)
408 ax.text(-4.1, -0.5, f"Darker = looked at more (each row is scaled to its own biggest value). Its top guesses for ?: {guess}.",
409 fontsize=9, color=INK, va="top")
410 save(fig, "attention.png")
411
412
413def dial_picture(P):
414 dial = P["dial"]
415 fig = plt.figure(figsize=(8.6, 5.2))
416 fig.text(0.01, 0.99, "The same trained model, with the “adventure dial” turned to three settings", fontsize=12, weight="bold", color=INK, va="top")
417 names = {0: "careful", 1: "normal", 2: "wild"}
418 for k, d in enumerate(dial):
419 y0 = 0.92 - (k + 1) * 0.3
420 gx = fig.add_axes([0.01, y0 + 0.02, 0.17, 0.26])
421 gx.set_xlim(-1.2, 1.2), gx.set_ylim(-0.5, 1.2), gx.axis("off"), gx.set_aspect("equal")
422 for a0, col in ((120, GREEN), (60, "#ca8a04"), (0, RED)):
423 gx.add_patch(Wedge((0, 0), 1, a0, a0 + 60, width=0.3, color=col, alpha=0.7))
424 # needle: temperature 0.3 -> left, 1.5 -> right
425 angle = math.radians(180 - 180 * min(1, d["temperature"] / 1.8))
426 gx.annotate("", xy=(0.85 * math.cos(angle), 0.85 * math.sin(angle)), xytext=(0, 0), arrowprops=dict(arrowstyle="-|>", color=INK, lw=2))
427 gx.text(0, -0.4, f"{names[k]}\n(temperature {d['temperature']})", ha="center", va="top", fontsize=9, color=INK)
428 tx = fig.add_axes([0.21, y0 + 0.02, 0.77, 0.26])
429 tx.axis("off")
430 tx.text(0, 0.95, "\n".join(textwrap.wrap(flat(d["story"])[:190], 64)), fontsize=10, family=MONO, color=INK, va="top")
431 save(fig, "dial.png")
432
433
434def main():
435 P = json.load(open("probes.json", encoding="utf-8"))
436 samples = json.load(open("samples.json", encoding="utf-8"))
437 with open("results.csv") as f:
438 rows = [{k: (int(v) if k == "step" else float(v)) for k, v in r.items()} for r in csv.DictReader(f)]
439 book = open("grimm.txt", encoding="utf-8").read()
440
441 book_picture(P, book)
442 alphabet_picture(P, book)
443 practice_picture(P, book)
444 model_picture(P)
445 torn_picture(P, rows)
446 odds_animation(P)
447 odds_picture(P)
448 read_along_animation(P)
449 read_along_picture(P)
450 first_guess_picture(rows)
451 typewriter_animation(P)
452 growing_animation(samples, rows)
453 attention_picture(P)
454 dial_picture(P)
455 print("pictures done")
456
457
458if __name__ == "__main__":
459 main()