← A tiny storyteller: how a computer learns to write, one letter at a time
pictures.py
| 1 | """ |
| 2 | Pictures and animations for "A tiny storyteller", made from what experiment.py |
| 3 | saved: results.csv, samples.json and probes.json (and the book, grimm.txt). |
| 4 | |
| 5 | Every number and letter in these pictures comes from those files: the model's |
| 6 | real odds, its real guesses and its real stories. Nothing is made up or tidied. |
| 7 | |
| 8 | Outputs: |
| 9 | book.png the book as one long line of letters, split into practice and hidden parts |
| 10 | alphabet.png every letter and symbol the model can choose from, shaded by how common it is |
| 11 | practice.png where the first practice step read from, and how one stretch becomes many guessing questions |
| 12 | model.png what happens inside the model when it makes one guess |
| 13 | torn.png "torn between": a coin, a dice, and our model before and after practice |
| 14 | odds.gif its odds for the letter after "Once upon a ti", at each checkpoint (animation) |
| 15 | odds.png the same odds, before and after practice, for two beginnings |
| 16 | read-along.gif the trained model guessing a hidden sentence, letter by letter (animation) |
| 17 | read-along.png the same sentence at every checkpoint: green = first guess right |
| 18 | first-guess.png out of 100 guesses, how many first guesses were right, at each checkpoint |
| 19 | typewriter.gif the trained model writing a story, one letter at a time, with its odds (animation) |
| 20 | growing.gif the story it writes at each checkpoint (animation) |
| 21 | attention.png which earlier letters it looks back at most, for one guess |
| 22 | dial.png the same model writing with the "adventure dial" set to careful, normal and wild |
| 23 | |
| 24 | Run (after experiment.py): |
| 25 | python3 pictures.py |
| 26 | """ |
| 27 | |
| 28 | import csv |
| 29 | import io |
| 30 | import json |
| 31 | import math |
| 32 | import textwrap |
| 33 | from collections import Counter |
| 34 | |
| 35 | import matplotlib |
| 36 | from PIL import Image |
| 37 | |
| 38 | matplotlib.use("Agg") |
| 39 | import matplotlib.pyplot as plt |
| 40 | from matplotlib.patches import Circle, FancyBboxPatch, Rectangle, Wedge |
| 41 | |
| 42 | BLUE, ORANGE, GREEN, RED, GREY, INK = "#2557a7", "#c2410c", "#15803d", "#b91c1c", "#9ca3af", "#1f2937" |
| 43 | PALE_BLUE, PALE_ORANGE, PALE_GREEN, PALE_RED = "#dbe6f7", "#fbe3d6", "#dcfce7", "#fee2e2" |
| 44 | MONO = "DejaVu Sans Mono" |
| 45 | plt.rcParams.update({"font.size": 11, "axes.spines.top": False, "axes.spines.right": False}) |
| 46 | |
| 47 | |
| 48 | def show(ch): |
| 49 | """How to draw one letter on a tile: spaces and line breaks get a visible sign.""" |
| 50 | return {" ": "␣", "\n": "↵", None: ""}.get(ch, ch) |
| 51 | |
| 52 | |
| 53 | def flat(text): |
| 54 | return text.replace("\n", " ") |
| 55 | |
| 56 | |
| 57 | def save(fig, path): |
| 58 | fig.savefig(path, facecolor="white", dpi=150, bbox_inches="tight", pad_inches=0.15) |
| 59 | plt.close(fig) |
| 60 | |
| 61 | |
| 62 | def frame(fig): |
| 63 | """Render a figure to a picture, for an animation.""" |
| 64 | buf = io.BytesIO() |
| 65 | fig.savefig(buf, format="png", facecolor="white", dpi=100) |
| 66 | plt.close(fig) |
| 67 | buf.seek(0) |
| 68 | return Image.open(buf).convert("RGB") |
| 69 | |
| 70 | |
| 71 | def save_gif(frames, durations, path): |
| 72 | """frames: list of pictures; durations in milliseconds (one per frame). Loops forever.""" |
| 73 | pal = [f.quantize(colors=64, method=Image.Quantize.MEDIANCUT) for f in frames] |
| 74 | pal[0].save(path, save_all=True, append_images=pal[1:], duration=durations, loop=0, optimize=True) |
| 75 | |
| 76 | |
| 77 | def torn_between(odds): |
| 78 | """How many choices a set of odds is 'torn between' (e to the power of its entropy).""" |
| 79 | return math.exp(-sum(p * math.log(p) for p in odds if p > 0)) |
| 80 | |
| 81 | |
| 82 | def tile(ax, x, y, ch, face="white", edge=GREY, color=INK, w=0.9, h=0.9, size=12, weight="normal"): |
| 83 | ax.add_patch(FancyBboxPatch((x, y), w, h, boxstyle="round,pad=0,rounding_size=0.12", facecolor=face, edgecolor=edge, lw=1)) |
| 84 | ax.text(x + w / 2, y + h / 2, show(ch), ha="center", va="center", family=MONO, fontsize=size, color=color, weight=weight) |
| 85 | |
| 86 | |
| 87 | # --------------------------------------------------------------------------- |
| 88 | # The setting: the book, the letters, the practice |
| 89 | # --------------------------------------------------------------------------- |
| 90 | |
| 91 | def book_picture(P, book): |
| 92 | total, train, test = P["book_letters"], P["training_letters"], P["test_letters"] |
| 93 | fig, ax = plt.subplots(figsize=(8, 3.4)) |
| 94 | ax.set_xlim(0, 100), ax.set_ylim(0, 34), ax.axis("off") |
| 95 | split = 100 * train / total |
| 96 | ax.add_patch(Rectangle((0, 14), split, 6, color=BLUE)) |
| 97 | ax.add_patch(Rectangle((split, 14), 100 - split, 6, color=ORANGE)) |
| 98 | ax.text(split / 2, 17, f"practice part: {train:,} letters (90%)", ha="center", va="center", color="white", weight="bold") |
| 99 | ax.text(100, 11.5, f"hidden part: {test:,} letters (10%)", ha="right", va="top", color=ORANGE, fontsize=9.5, weight="bold") |
| 100 | ax.text(0, 31, f"Grimms' Fairy Tales, written out as one long line of {total:,} letters", fontsize=12, weight="bold", color=INK) |
| 101 | ax.text(0, 27, "The model practises on the blue part. We keep the orange part hidden, to test it later.", fontsize=10, color=INK) |
| 102 | # zoom in on the very start of the book |
| 103 | ax.plot([0, 0], [14, 8.2], color=GREY, lw=1), ax.plot([0.3, 70], [14, 8.2], color=GREY, lw=1) |
| 104 | piece = book[:44] |
| 105 | for i, ch in enumerate(piece): |
| 106 | tile(ax, i * 1.6, 3.2, ch, w=1.45, h=5, size=9) |
| 107 | ax.text(0, 0.2, "Zoomed in: every letter, space (␣), comma and line break (↵) counts as a 'letter'.", fontsize=9.5, color=INK) |
| 108 | save(fig, "book.png") |
| 109 | |
| 110 | |
| 111 | def alphabet_picture(P, book): |
| 112 | alphabet, counts = P["alphabet"], Counter(book) |
| 113 | order = sorted(alphabet, key=lambda c: -counts[c]) |
| 114 | cols = 14 |
| 115 | rows = math.ceil(len(order) / cols) |
| 116 | fig, ax = plt.subplots(figsize=(8, 0.75 * rows + 1.3)) |
| 117 | ax.set_xlim(0, cols), ax.set_ylim(-0.2, rows + 1.1), ax.axis("off"), ax.set_aspect("equal") |
| 118 | top = counts[order[0]] |
| 119 | cmap = plt.get_cmap("Blues") |
| 120 | for k, ch in enumerate(order): |
| 121 | r, c = divmod(k, cols) |
| 122 | share = math.log(counts[ch]) / math.log(top) |
| 123 | face = cmap(0.08 + 0.85 * share) |
| 124 | tile(ax, c, rows - 1 - r, ch, face=face, color="white" if share > 0.6 else INK, size=11) |
| 125 | ax.text(0, rows + 0.4, f"The {len(alphabet)} 'letters' the model can choose from, most common first", |
| 126 | fontsize=12, weight="bold", color=INK) |
| 127 | common = ", ".join(f"'{show(c)}' {100 * counts[c] / len(book):.0f}%" for c in order[:3]) |
| 128 | ax.text(0, -0.15, f"Darker = more common in the book. The most common: {common}.", fontsize=9.5, color=INK, va="top") |
| 129 | save(fig, "alphabet.png") |
| 130 | |
| 131 | |
| 132 | def practice_picture(P, book): |
| 133 | starts, ctx, train = P["first_batch_starts"], P["context"], P["training_letters"] |
| 134 | fig = plt.figure(figsize=(8, 5.4)) |
| 135 | ax = fig.add_axes([0.03, 0.62, 0.94, 0.36]) |
| 136 | ax.set_xlim(0, 100), ax.set_ylim(0, 12), ax.axis("off") |
| 137 | ax.add_patch(Rectangle((0, 2), 100, 3, color=PALE_BLUE)) |
| 138 | for s in starts: |
| 139 | x = 100 * s / train |
| 140 | ax.plot([x, x], [5, 8.2], color=BLUE, lw=1) |
| 141 | ax.add_patch(Circle((x, 8.6), 0.55, color=BLUE)) |
| 142 | ax.text(0, 10.6, f"The first practice step: {len(starts)} stretches of {ctx} letters, from random places", |
| 143 | fontsize=11.5, weight="bold", color=INK) |
| 144 | ax.text(0, 0, "start of the practice part", fontsize=8.5, color=GREY), ax.text(100, 0, "end", fontsize=8.5, color=GREY, ha="right") |
| 145 | |
| 146 | bx = fig.add_axes([0.03, 0.02, 0.94, 0.56]) |
| 147 | n = 16 |
| 148 | s = starts[0] |
| 149 | piece = book[s:s + n + 1] |
| 150 | bx.set_xlim(-3.2, n + 0.5), bx.set_ylim(-0.6, 6.4), bx.axis("off") |
| 151 | bx.text(-3.2, 5.9, f"Zoom in on the first stretch (its first {n} letters). Every letter is a guessing question:", |
| 152 | fontsize=10.5, color=INK) |
| 153 | bx.text(-3.2, 3.95, "the model\nreads", fontsize=9, color=BLUE, va="center") |
| 154 | bx.text(-3.2, 1.45, "and must\nguess", fontsize=9, color=ORANGE, va="center") |
| 155 | for i in range(n): |
| 156 | tile(bx, i, 3.5, piece[i], face=PALE_BLUE, edge=BLUE, size=11) |
| 157 | tile(bx, i, 1.0, piece[i + 1], face=PALE_ORANGE, edge=ORANGE, size=11) |
| 158 | bx.annotate("", xy=(i + 0.45, 1.95), xytext=(i + 0.45, 3.45), arrowprops=dict(arrowstyle="->", color=GREY, lw=0.8)) |
| 159 | bx.text(-3.2, 0.1, f"Reading up to letter 1 it guesses letter 2, reading up to letter 2 it guesses letter 3, and so on: " |
| 160 | f"{ctx} questions per stretch,\n{len(starts) * ctx:,} questions per practice step. After each step it nudges its " |
| 161 | f"{P['parameters']:,} numbers a little, towards the right answers.", fontsize=8.8, color=INK, va="top") |
| 162 | save(fig, "practice.png") |
| 163 | |
| 164 | |
| 165 | def model_picture(P): |
| 166 | final = P["odds"][-1][P["guess_prompts"][0]] |
| 167 | best = sorted(range(len(final)), key=lambda i: -final[i])[:2] |
| 168 | example = "\n".join(f"{show(P['alphabet'][i])} {100 * final[i]:.0f}%" for i in best) |
| 169 | fig, ax = plt.subplots(figsize=(8, 3.6)) |
| 170 | ax.set_xlim(0, 100), ax.set_ylim(0, 40), ax.axis("off") |
| 171 | |
| 172 | def box(x, w, title, body, face, edge): |
| 173 | ax.add_patch(FancyBboxPatch((x, 7), w, 24, boxstyle="round,pad=0.4,rounding_size=1.5", facecolor=face, edgecolor=edge, lw=1.5)) |
| 174 | ax.text(x + w / 2, 27.5, title, ha="center", va="center", weight="bold", fontsize=10.5, color=INK) |
| 175 | ax.text(x + w / 2, 17, body, ha="center", va="center", fontsize=8.6, color=INK, linespacing=1.4) |
| 176 | |
| 177 | box(0, 21, "1. Read", f"the last {P['context']}\nletters, e.g.\n“…upon a ti”", PALE_BLUE, BLUE) |
| 178 | box(26, 21, "2. Turn into\nnumbers", f"each letter becomes\na list of {P['width']}\nnumbers", "#f3f4f6", GREY) |
| 179 | box(52, 21, f"3. {P['layers']} layers of\nlook back + think", "look back at earlier\nletters (attention),\nthen mix it all up", "#f3f4f6", GREY) |
| 180 | box(78, 21, "4. Odds for the\nnext letter", f"{example}\n… for all {len(P['alphabet'])} letters", PALE_ORANGE, ORANGE) |
| 181 | for x in (21.5, 47.5, 73.5): |
| 182 | ax.annotate("", xy=(x + 4, 19), xytext=(x, 19), arrowprops=dict(arrowstyle="-|>", color=INK, lw=1.5)) |
| 183 | ax.text(50, 36.5, "What happens inside the model when it makes one guess", ha="center", fontsize=12, weight="bold", color=INK) |
| 184 | ax.text(50, 0, f"All of this is done with {P['parameters']:,} adjustable numbers. Practice nudges those numbers; nothing else changes.\n" |
| 185 | f"Box 4 shows the trained model's real top 2 odds after “{P['guess_prompts'][0]}”.", ha="center", fontsize=8.6, color=INK) |
| 186 | save(fig, "model.png") |
| 187 | |
| 188 | |
| 189 | def torn_picture(P, rows): |
| 190 | start, end = math.exp(rows[0]["test_loss"]), math.exp(rows[-1]["test_loss"]) |
| 191 | fig, axes = plt.subplots(1, 4, figsize=(8.4, 3.6), gridspec_kw={"width_ratios": [1, 1, 1.5, 1]}) |
| 192 | items = [("a coin", 2, "heads or tails:\ntorn between 2"), |
| 193 | ("a dice", 6, "1 to 6:\ntorn between 6"), |
| 194 | ("our model,\nbefore practice", start, f"torn between\nabout {start:.0f} letters"), |
| 195 | ("our model,\nafter practice", end, f"torn between\nabout {end:.1f} letters")] |
| 196 | for ax, (name, n, label) in zip(axes, items): |
| 197 | ax.axis("off"), ax.set_aspect("equal") |
| 198 | k = math.ceil(n) |
| 199 | cols = 2 if k <= 2 else 3 if k <= 6 else 12 if k > 40 else 4 |
| 200 | r = math.ceil(k / cols) |
| 201 | ax.set_xlim(-0.2, max(cols, 3) + 0.2), ax.set_ylim(-3.2, 10.6) |
| 202 | for i in range(k): |
| 203 | rr, cc = divmod(i, cols) |
| 204 | part = min(1.0, n - i) # the last square is only partly filled for 3.8 and 93.2 |
| 205 | ax.add_patch(Rectangle((cc + 0.06, r - 1 - rr + 0.06), 0.88, 0.88, facecolor="white", edgecolor=ORANGE if "model" in name else BLUE, lw=1)) |
| 206 | ax.add_patch(Rectangle((cc + 0.06, r - 1 - rr + 0.06), 0.88 * part, 0.88, color=ORANGE if "model" in name else BLUE, alpha=0.75)) |
| 207 | ax.text(max(cols, 3) / 2, 9.4, name, ha="center", va="center", fontsize=10, weight="bold", color=INK) |
| 208 | ax.text(max(cols, 3) / 2, -0.4, label, ha="center", va="top", fontsize=9.5, color=INK) |
| 209 | fig.suptitle("“Torn between”: how many choices it is really stuck between (smaller is better)", |
| 210 | fontsize=11, weight="bold", color=INK, y=0.99) |
| 211 | save(fig, "torn.png") |
| 212 | |
| 213 | |
| 214 | # --------------------------------------------------------------------------- |
| 215 | # Its guesses |
| 216 | # --------------------------------------------------------------------------- |
| 217 | |
| 218 | def odds_axes(ax, odds, alphabet, real, k=10, ymax=1.0): |
| 219 | best = sorted(range(len(odds)), key=lambda i: -odds[i])[:k] |
| 220 | if real in alphabet and alphabet.index(real) not in best: # always show the real letter, even when it is far down |
| 221 | best[-1] = alphabet.index(real) |
| 222 | xs = range(len(best)) |
| 223 | ax.bar(xs, [100 * odds[i] for i in best], color=[ORANGE if alphabet[i] == real else BLUE for i in best], width=0.75) |
| 224 | ax.set_xticks(list(xs), [show(alphabet[i]) for i in best], family=MONO, fontsize=13) |
| 225 | ax.set_ylim(0, 100 * ymax), ax.set_ylabel("odds (%)") |
| 226 | for x, i in zip(xs, best): |
| 227 | if odds[i] >= 0.01: |
| 228 | ax.text(x, 100 * odds[i] + 1.5, f"{100 * odds[i]:.0f}%", ha="center", fontsize=8.5, color=INK) |
| 229 | |
| 230 | |
| 231 | def odds_animation(P): |
| 232 | prompt, real = P["guess_prompts"][0], "m" |
| 233 | frames, durations = [], [] |
| 234 | for snap in P["odds"]: |
| 235 | odds = snap[prompt] |
| 236 | fig, ax = plt.subplots(figsize=(6.4, 3.8)) |
| 237 | odds_axes(ax, odds, P["alphabet"], real) |
| 238 | ax.set_title(f"“{prompt}…” what comes next?\nAfter {snap['step']:,} practice steps: torn between about {torn_between(odds):.1f}", |
| 239 | fontsize=11, color=INK) |
| 240 | ax.text(0.99, 0.95, "orange = the real next letter", transform=ax.transAxes, ha="right", fontsize=9, color=ORANGE) |
| 241 | fig.tight_layout() |
| 242 | frames.append(frame(fig)), durations.append(1800) |
| 243 | durations[-1] = 3500 |
| 244 | save_gif(frames, durations, "odds.gif") |
| 245 | |
| 246 | |
| 247 | def odds_picture(P): |
| 248 | reals = {P["guess_prompts"][0]: "m", P["guess_prompts"][1]: "e"} |
| 249 | first, last = P["odds"][0], P["odds"][-1] |
| 250 | fig, axes = plt.subplots(2, 2, figsize=(8, 5.6), sharey=True) |
| 251 | for r, prompt in enumerate(P["guess_prompts"]): |
| 252 | for c, snap in enumerate((first, last)): |
| 253 | ax = axes[r][c] |
| 254 | odds_axes(ax, snap[prompt], P["alphabet"], reals[prompt], k=8) |
| 255 | ax.set_title(f"“{prompt}…”, after {snap['step']:,} steps\ntorn between about {torn_between(snap[prompt]):.1f}", fontsize=9.5, color=INK) |
| 256 | if c: |
| 257 | ax.set_ylabel("") |
| 258 | fig.suptitle("Its odds for the next letter, before and after practice (orange = the real next letter)", fontsize=11, weight="bold", color=INK) |
| 259 | fig.tight_layout() |
| 260 | save(fig, "odds.png") |
| 261 | |
| 262 | |
| 263 | def sentence_row(ax, y, sentence, guesses, upto=None, size=10.5): |
| 264 | """Draw sentence as tiles: green if the first guess was right, red if not, plain if not reached yet.""" |
| 265 | for i, ch in enumerate(sentence): |
| 266 | if upto is not None and i > upto: |
| 267 | tile(ax, i, y, None, face="white", edge="#e5e7eb", size=size) |
| 268 | continue |
| 269 | if upto is not None and i == upto: |
| 270 | tile(ax, i, y, "?", face="#fef9c3", edge="#ca8a04", size=size, weight="bold") |
| 271 | continue |
| 272 | right = guesses[i]["top"][0][0] == ch |
| 273 | tile(ax, i, y, ch, face=PALE_GREEN if right else PALE_RED, edge=GREEN if right else RED, size=size) |
| 274 | |
| 275 | |
| 276 | def read_along_animation(P): |
| 277 | sentence, last = P["read_along_sentence"], P["read_along"][-1] |
| 278 | guesses = last["guesses"] |
| 279 | n = len(sentence) |
| 280 | frames, durations = [], [] |
| 281 | for i in range(n + 1): |
| 282 | fig = plt.figure(figsize=(8.6, 4.2)) |
| 283 | ax = fig.add_axes([0.02, 0.62, 0.96, 0.33]) |
| 284 | ax.set_xlim(-0.2, n + 0.2), ax.set_ylim(-0.6, 2.2), ax.axis("off") |
| 285 | sentence_row(ax, 0, sentence, guesses, upto=i if i < n else None, size=9) |
| 286 | done = guesses[:i] |
| 287 | right = sum(g["top"][0][0] == g["real"] for g in done) |
| 288 | ax.text(0, 1.6, f"A sentence from the hidden part of the book. First guess right so far: {right} of {len(done)}", |
| 289 | fontsize=11, weight="bold", color=INK) |
| 290 | bx = fig.add_axes([0.1, 0.14, 0.55, 0.38]) |
| 291 | if i < n: |
| 292 | g = guesses[i] |
| 293 | labels = [show(c) for c, _ in g["top"]] |
| 294 | bx.barh(range(3)[::-1], [100 * p for _, p in g["top"]], color=[GREEN if c == g["real"] else BLUE for c, _ in g["top"]]) |
| 295 | bx.set_yticks(range(3)[::-1], labels, family=MONO, fontsize=14) |
| 296 | bx.set_xlim(0, 100), bx.set_xlabel("odds (%)") |
| 297 | for k, (_, p) in enumerate(g["top"]): |
| 298 | bx.text(100 * p + 1, 2 - k, f"{100 * p:.0f}%", va="center", fontsize=9) |
| 299 | bx.set_title("its top 3 guesses for the ? letter", fontsize=10, color=INK) |
| 300 | verdict = "first guess right!" if g["top"][0][0] == g["real"] else "first guess wrong" |
| 301 | fig.text(0.72, 0.32, f"real letter: '{show(g['real'])}'\n{verdict}\n(it gave the real letter\n{100 * g['real_odds']:.0f}% odds)", |
| 302 | fontsize=11, color=GREEN if g["top"][0][0] == g["real"] else RED, va="center") |
| 303 | else: |
| 304 | bx.axis("off") |
| 305 | fig.text(0.1, 0.3, f"Done: its first guess was right for {right} of {n} letters.\n" |
| 306 | "Green = first guess right. Red = first guess wrong.", fontsize=12, color=INK) |
| 307 | frames.append(frame(fig)), durations.append(900 if i < n else 4000) |
| 308 | save_gif(frames, durations, "read-along.gif") |
| 309 | |
| 310 | |
| 311 | def read_along_picture(P): |
| 312 | sentence = P["read_along_sentence"] |
| 313 | n, snaps = len(sentence), P["read_along"] |
| 314 | fig, ax = plt.subplots(figsize=(8.8, 0.62 * len(snaps) + 1.0)) |
| 315 | ax.set_xlim(-6.5, n + 0.2), ax.set_ylim(-0.4, 1.25 * len(snaps) + 0.5), ax.axis("off") |
| 316 | for r, snap in enumerate(snaps): |
| 317 | y = 1.25 * (len(snaps) - 1 - r) |
| 318 | sentence_row(ax, y, sentence, snap["guesses"], size=8.5) |
| 319 | right = sum(g["top"][0][0] == g["real"] for g in snap["guesses"]) |
| 320 | ax.text(-6.4, y + 0.45, f"{snap['step']:>5,} steps\n{right:>3} of {n}", va="center", fontsize=8.5, family=MONO, color=INK) |
| 321 | ax.text(-6.4, 1.25 * len(snaps) + 0.2, "The same hidden sentence at each checkpoint. Green = first guess right, red = wrong.", |
| 322 | fontsize=10.5, weight="bold", color=INK) |
| 323 | save(fig, "read-along.png") |
| 324 | |
| 325 | |
| 326 | def first_guess_picture(rows): |
| 327 | fig, axes = plt.subplots(1, len(rows), figsize=(8.4, 2.9)) |
| 328 | for ax, row in zip(axes, rows): |
| 329 | right = round(100 * row["test_first_guess_right"]) |
| 330 | ax.set_xlim(0, 10), ax.set_ylim(-3.2, 11.6), ax.axis("off"), ax.set_aspect("equal") |
| 331 | for k in range(100): |
| 332 | r, c = divmod(k, 10) |
| 333 | ax.add_patch(Rectangle((c + 0.08, 9 - r + 0.08), 0.84, 0.84, color=GREEN if k < right else "#e5e7eb")) |
| 334 | ax.text(5, 10.8, f"after {row['step']:,} steps", ha="center", fontsize=9.5, weight="bold", color=INK) |
| 335 | ax.text(5, -0.6, f"{right} of 100\nright", ha="center", va="top", fontsize=10, color=GREEN) |
| 336 | fig.suptitle("Out of 100 guesses on the hidden part of the book, how many first guesses were right?", |
| 337 | fontsize=11, weight="bold", color=INK) |
| 338 | save(fig, "first-guess.png") |
| 339 | |
| 340 | |
| 341 | # --------------------------------------------------------------------------- |
| 342 | # Its writing |
| 343 | # --------------------------------------------------------------------------- |
| 344 | |
| 345 | def typewriter_animation(P): |
| 346 | prompt, steps = P["prompt"], P["typewriter"] |
| 347 | frames, durations = [], [] |
| 348 | written = "" |
| 349 | for i, st in enumerate(steps + [None]): |
| 350 | fig = plt.figure(figsize=(8, 4.4)) |
| 351 | tx = fig.add_axes([0.03, 0.55, 0.94, 0.42]) |
| 352 | tx.axis("off") |
| 353 | tx.text(0, 1, "The trained model writes a story, one letter at a time", fontsize=12, weight="bold", color=INK, va="top") |
| 354 | body = " " * len(prompt) + flat(written) + ("▌" if st else "") |
| 355 | lines = "\n".join(body[k:k + 56] for k in range(0, len(body), 56)) |
| 356 | tx.text(0, 0.72, lines, fontsize=13, family=MONO, color=INK, va="top") |
| 357 | tx.text(0, 0.72, prompt, fontsize=13, family=MONO, color=GREY, va="top") |
| 358 | bx = fig.add_axes([0.12, 0.12, 0.5, 0.34]) |
| 359 | if st: |
| 360 | letters = [show(c) for c, _ in st["top"]] |
| 361 | bx.barh(range(5)[::-1], [100 * p for _, p in st["top"]], color=[ORANGE if c == st["picked"] else BLUE for c, _ in st["top"]]) |
| 362 | bx.set_yticks(range(5)[::-1], letters, family=MONO, fontsize=13) |
| 363 | bx.set_xlim(0, 100), bx.set_xlabel("odds (%)") |
| 364 | bx.set_title("its top 5 choices for the next letter", fontsize=10, color=INK) |
| 365 | picked_rank = [c for c, _ in st["top"]].index(st["picked"]) + 1 if st["picked"] in [c for c, _ in st["top"]] else None |
| 366 | note = "the most likely letter" if picked_rank == 1 else "not the most likely letter:\nlike rolling a weighted dice,\nless likely letters sometimes win" if picked_rank else "a rare letter outside the top 5" |
| 367 | fig.text(0.67, 0.27, f"picked: '{show(st['picked'])}'\n{note}", fontsize=10.5, color=ORANGE, va="center") |
| 368 | written += st["picked"] |
| 369 | else: |
| 370 | bx.axis("off") |
| 371 | frames.append(frame(fig)), durations.append(450 if st else 4000) |
| 372 | save_gif(frames, durations, "typewriter.gif") |
| 373 | |
| 374 | |
| 375 | def growing_animation(samples, rows): |
| 376 | frames, durations = [], [] |
| 377 | for s, row in zip(samples, rows): |
| 378 | fig = plt.figure(figsize=(8, 2.9)) |
| 379 | ax = fig.add_axes([0.03, 0.03, 0.94, 0.92]) |
| 380 | ax.axis("off") |
| 381 | ax.text(0, 1, f"After {s['step']:,} practice steps", fontsize=15, weight="bold", color=BLUE, va="top") |
| 382 | ax.text(0, 0.84, f"torn between about {math.exp(row['test_loss']):.1f} letters · first guess right " |
| 383 | f"{100 * row['test_first_guess_right']:.0f} times in 100", fontsize=10, color=INK, va="top") |
| 384 | ax.text(0, 0.66, "\n".join(textwrap.wrap(flat(s["story"])[:230], 62)), fontsize=11.5, family=MONO, color=INK, va="top") |
| 385 | frames.append(frame(fig)), durations.append(3200) |
| 386 | durations[-1] = 5000 |
| 387 | save_gif(frames, durations, "growing.gif") |
| 388 | |
| 389 | |
| 390 | def attention_picture(P): |
| 391 | A = P["attention"] |
| 392 | text, layers = A["text"], A["layers"] |
| 393 | n = len(text) |
| 394 | fig, ax = plt.subplots(figsize=(9, 0.75 * len(layers) + 2.1)) |
| 395 | ax.set_xlim(-4.2, n + 1.5), ax.set_ylim(-1.9, 1.15 * len(layers) + 1.4), ax.axis("off") |
| 396 | cmap = plt.get_cmap("Oranges") |
| 397 | for r, w in enumerate(layers): |
| 398 | y = 1.15 * (len(layers) - 1 - r) |
| 399 | biggest = max(w) |
| 400 | for i, ch in enumerate(text): |
| 401 | share = w[i] / biggest |
| 402 | tile(ax, i, y, ch, face=cmap(0.05 + 0.85 * share), edge="#e5e7eb", size=9, color="white" if share > 0.65 else INK) |
| 403 | tile(ax, n + 0.3, y, "?", face="#fef9c3", edge="#ca8a04", size=9, weight="bold") |
| 404 | ax.text(-4.1, y + 0.45, f"layer {r + 1}", va="center", fontsize=9, color=INK) |
| 405 | guess = ", ".join(f"'{show(c)}' {100 * p:.0f}%" for c, p in A["top"][:3]) |
| 406 | ax.text(-4.1, 1.15 * len(layers) + 0.6, "Guessing the ? letter: which earlier letters does each layer look back at most?", |
| 407 | fontsize=11, weight="bold", color=INK) |
| 408 | ax.text(-4.1, -0.5, f"Darker = looked at more (each row is scaled to its own biggest value). Its top guesses for ?: {guess}.", |
| 409 | fontsize=9, color=INK, va="top") |
| 410 | save(fig, "attention.png") |
| 411 | |
| 412 | |
| 413 | def dial_picture(P): |
| 414 | dial = P["dial"] |
| 415 | fig = plt.figure(figsize=(8.6, 5.2)) |
| 416 | fig.text(0.01, 0.99, "The same trained model, with the “adventure dial” turned to three settings", fontsize=12, weight="bold", color=INK, va="top") |
| 417 | names = {0: "careful", 1: "normal", 2: "wild"} |
| 418 | for k, d in enumerate(dial): |
| 419 | y0 = 0.92 - (k + 1) * 0.3 |
| 420 | gx = fig.add_axes([0.01, y0 + 0.02, 0.17, 0.26]) |
| 421 | gx.set_xlim(-1.2, 1.2), gx.set_ylim(-0.5, 1.2), gx.axis("off"), gx.set_aspect("equal") |
| 422 | for a0, col in ((120, GREEN), (60, "#ca8a04"), (0, RED)): |
| 423 | gx.add_patch(Wedge((0, 0), 1, a0, a0 + 60, width=0.3, color=col, alpha=0.7)) |
| 424 | # needle: temperature 0.3 -> left, 1.5 -> right |
| 425 | angle = math.radians(180 - 180 * min(1, d["temperature"] / 1.8)) |
| 426 | gx.annotate("", xy=(0.85 * math.cos(angle), 0.85 * math.sin(angle)), xytext=(0, 0), arrowprops=dict(arrowstyle="-|>", color=INK, lw=2)) |
| 427 | gx.text(0, -0.4, f"{names[k]}\n(temperature {d['temperature']})", ha="center", va="top", fontsize=9, color=INK) |
| 428 | tx = fig.add_axes([0.21, y0 + 0.02, 0.77, 0.26]) |
| 429 | tx.axis("off") |
| 430 | tx.text(0, 0.95, "\n".join(textwrap.wrap(flat(d["story"])[:190], 64)), fontsize=10, family=MONO, color=INK, va="top") |
| 431 | save(fig, "dial.png") |
| 432 | |
| 433 | |
| 434 | def main(): |
| 435 | P = json.load(open("probes.json", encoding="utf-8")) |
| 436 | samples = json.load(open("samples.json", encoding="utf-8")) |
| 437 | with open("results.csv") as f: |
| 438 | rows = [{k: (int(v) if k == "step" else float(v)) for k, v in r.items()} for r in csv.DictReader(f)] |
| 439 | book = open("grimm.txt", encoding="utf-8").read() |
| 440 | |
| 441 | book_picture(P, book) |
| 442 | alphabet_picture(P, book) |
| 443 | practice_picture(P, book) |
| 444 | model_picture(P) |
| 445 | torn_picture(P, rows) |
| 446 | odds_animation(P) |
| 447 | odds_picture(P) |
| 448 | read_along_animation(P) |
| 449 | read_along_picture(P) |
| 450 | first_guess_picture(rows) |
| 451 | typewriter_animation(P) |
| 452 | growing_animation(samples, rows) |
| 453 | attention_picture(P) |
| 454 | dial_picture(P) |
| 455 | print("pictures done") |
| 456 | |
| 457 | |
| 458 | if __name__ == "__main__": |
| 459 | main() |