← A tiny storyteller: how a computer learns to write, one letter at a time
experiment.py
| 1 | """ |
| 2 | Train a tiny storyteller: a small language model that learns to write like |
| 3 | Grimms' Fairy Tales, one letter at a time. |
| 4 | |
| 5 | Data: grimm.txt, the text of Grimms' Fairy Tales (public domain, from |
| 6 | Project Gutenberg, with the Gutenberg header and licence removed). |
| 7 | |
| 8 | Model: a very small "transformer", the same kind of model that powers AI |
| 9 | chatbots, but about a hundred thousand times smaller. It reads the last |
| 10 | CONTEXT letters and guesses the next one. We train it on 90% of the text and |
| 11 | keep 10% aside to test it on writing it has never seen. |
| 12 | |
| 13 | While it learns, we stop at a few checkpoints and ask it to continue the |
| 14 | words "Once upon a time", so we can watch its writing improve. |
| 15 | |
| 16 | Outputs: |
| 17 | results.csv training and test "surprise" (loss), and how often the |
| 18 | first guess is right, at every checkpoint |
| 19 | samples.json the story the model writes at each checkpoint |
| 20 | probes.json a close look inside the model, for the pictures: its odds |
| 21 | for the next letter, which letters it looks back at, the |
| 22 | same story written with the "adventure dial" turned to |
| 23 | three settings, and more |
| 24 | learning.png how surprised the model is by new text, as it learns |
| 25 | samples.png the stories from a few checkpoints, side by side |
| 26 | |
| 27 | pictures.py turns results.csv, samples.json and probes.json into the rest of |
| 28 | the pictures and animations in the post. |
| 29 | |
| 30 | The last line printed is a JSON summary of the headline numbers. |
| 31 | |
| 32 | Run: |
| 33 | python3 experiment.py && python3 pictures.py |
| 34 | """ |
| 35 | |
| 36 | import csv |
| 37 | import json |
| 38 | import math |
| 39 | import os |
| 40 | import textwrap |
| 41 | import time |
| 42 | |
| 43 | import matplotlib |
| 44 | import torch |
| 45 | import torch.nn as nn |
| 46 | import torch.nn.functional as F |
| 47 | |
| 48 | matplotlib.use("Agg") # draw to a file, no window needed |
| 49 | import matplotlib.pyplot as plt |
| 50 | |
| 51 | # --------------------------------------------------------------------------- |
| 52 | # Settings |
| 53 | # --------------------------------------------------------------------------- |
| 54 | |
| 55 | SEED = 0 |
| 56 | STEPS = int(os.environ.get("STEPS", 2000)) # training steps (more makes it memorise the book) |
| 57 | CHECKPOINTS = sorted({0, 100, 300, 1000, STEPS}) |
| 58 | CONTEXT = 128 # letters the model can look back at |
| 59 | BATCH_SIZE = 64 |
| 60 | LAYERS, HEADS, WIDTH = 4, 4, 192 |
| 61 | LEARNING_RATE = 1e-3 |
| 62 | PROMPT = "Once upon a time" |
| 63 | SAMPLE_LENGTH = 300 # letters written at each checkpoint |
| 64 | TEMPERATURE = 0.8 # below 1 = a little less random when picking letters |
| 65 | DEVICE = "cuda" if torch.cuda.is_available() else "cpu" |
| 66 | |
| 67 | # Probes: small, fixed questions we ask the model at every checkpoint |
| 68 | GUESS_PROMPTS = ["Once upon a ti", "The king and the qu"] |
| 69 | READ_ALONG = "Once upon a time the king held a great feast," # a sentence from the hidden 10% of the book |
| 70 | ATTENTION_TEXT = "Once upon a time there was a king who had a daugh" |
| 71 | DIAL_SETTINGS = [0.3, 0.8, 1.5] # temperature: low = careful, high = adventurous |
| 72 | TYPEWRITER_LETTERS = 60 |
| 73 | |
| 74 | |
| 75 | # --------------------------------------------------------------------------- |
| 76 | # Data |
| 77 | # --------------------------------------------------------------------------- |
| 78 | |
| 79 | def load_text(path="grimm.txt"): |
| 80 | """Read the book and turn every letter into a number (its position in the alphabet we found).""" |
| 81 | text = open(path, encoding="utf-8").read() |
| 82 | alphabet = sorted(set(text)) |
| 83 | to_id = {ch: i for i, ch in enumerate(alphabet)} |
| 84 | ids = torch.tensor([to_id[ch] for ch in text], dtype=torch.long) |
| 85 | split = int(0.9 * len(ids)) |
| 86 | return ids[:split], ids[split:], alphabet |
| 87 | |
| 88 | |
| 89 | def random_batch(ids, generator, keep=None): |
| 90 | """BATCH_SIZE random stretches of text, and the same stretches shifted by one letter (the answers). |
| 91 | If keep is a list, the start positions are added to it (for the pictures).""" |
| 92 | starts = torch.randint(0, len(ids) - CONTEXT - 1, (BATCH_SIZE,), generator=generator) |
| 93 | if keep is not None: |
| 94 | keep.extend(starts.tolist()) |
| 95 | x = torch.stack([ids[s:s + CONTEXT] for s in starts]) |
| 96 | y = torch.stack([ids[s + 1:s + CONTEXT + 1] for s in starts]) |
| 97 | return x.to(DEVICE), y.to(DEVICE) |
| 98 | |
| 99 | |
| 100 | # --------------------------------------------------------------------------- |
| 101 | # Model |
| 102 | # --------------------------------------------------------------------------- |
| 103 | |
| 104 | class Block(nn.Module): |
| 105 | """One transformer layer: look back at earlier letters (attention), then think (MLP).""" |
| 106 | |
| 107 | def __init__(self): |
| 108 | super().__init__() |
| 109 | self.norm1 = nn.LayerNorm(WIDTH) |
| 110 | self.attention = nn.MultiheadAttention(WIDTH, HEADS, batch_first=True) |
| 111 | self.norm2 = nn.LayerNorm(WIDTH) |
| 112 | self.mlp = nn.Sequential(nn.Linear(WIDTH, 4 * WIDTH), nn.GELU(), nn.Linear(4 * WIDTH, WIDTH)) |
| 113 | |
| 114 | def forward(self, x, mask): |
| 115 | h = self.norm1(x) |
| 116 | x = x + self.attention(h, h, h, attn_mask=mask, need_weights=False)[0] |
| 117 | return x + self.mlp(self.norm2(x)) |
| 118 | |
| 119 | |
| 120 | class TinyStoryteller(nn.Module): |
| 121 | """Reads up to CONTEXT letters and gives a score for every possible next letter.""" |
| 122 | |
| 123 | def __init__(self, alphabet_size): |
| 124 | super().__init__() |
| 125 | self.letter = nn.Embedding(alphabet_size, WIDTH) |
| 126 | self.position = nn.Embedding(CONTEXT, WIDTH) |
| 127 | self.blocks = nn.ModuleList(Block() for _ in range(LAYERS)) |
| 128 | self.norm = nn.LayerNorm(WIDTH) |
| 129 | self.out = nn.Linear(WIDTH, alphabet_size) |
| 130 | |
| 131 | def forward(self, ids): |
| 132 | n = ids.shape[1] |
| 133 | x = self.letter(ids) + self.position(torch.arange(n, device=ids.device)) |
| 134 | # each letter may only look at letters before it, never ahead |
| 135 | mask = torch.triu(torch.ones(n, n, dtype=torch.bool, device=ids.device), diagonal=1) |
| 136 | for block in self.blocks: |
| 137 | x = block(x, mask) |
| 138 | return self.out(self.norm(x)) |
| 139 | |
| 140 | |
| 141 | |
| 142 | def write_story(model, alphabet, generator): |
| 143 | """Continue PROMPT one letter at a time, picking each letter by the model's odds.""" |
| 144 | model.eval() |
| 145 | to_id = {ch: i for i, ch in enumerate(alphabet)} |
| 146 | ids = torch.tensor([[to_id[ch] for ch in PROMPT]], device=DEVICE) |
| 147 | for _ in range(SAMPLE_LENGTH): |
| 148 | scores = model(ids[:, -CONTEXT:])[0, -1] / TEMPERATURE |
| 149 | odds = F.softmax(scores, dim=-1).cpu() |
| 150 | next_id = torch.multinomial(odds, 1, generator=generator).to(DEVICE) |
| 151 | ids = torch.cat([ids, next_id.view(1, 1)], dim=1) |
| 152 | model.train() |
| 153 | return "".join(alphabet[i] for i in ids[0].tolist()) |
| 154 | |
| 155 | |
| 156 | |
| 157 | def average_loss(model, ids, generator, batches=20): |
| 158 | """Average 'surprise' (cross-entropy, in nats per letter) on random stretches of ids, |
| 159 | and the share of letters where the model's first choice was the right one.""" |
| 160 | model.eval() |
| 161 | total, right = 0.0, 0.0 |
| 162 | for _ in range(batches): |
| 163 | x, y = random_batch(ids, generator) |
| 164 | scores = model(x) |
| 165 | total += F.cross_entropy(scores.flatten(0, 1), y.flatten()).item() |
| 166 | right += (scores.argmax(-1) == y).float().mean().item() |
| 167 | model.train() |
| 168 | return total / batches, right / batches |
| 169 | |
| 170 | |
| 171 | # --------------------------------------------------------------------------- |
| 172 | # Probes: looking inside the model, for the pictures |
| 173 | # --------------------------------------------------------------------------- |
| 174 | |
| 175 | def encode(text, alphabet): |
| 176 | to_id = {ch: i for i, ch in enumerate(alphabet)} |
| 177 | return torch.tensor([[to_id[ch] for ch in text]], device=DEVICE) |
| 178 | |
| 179 | |
| 180 | |
| 181 | def next_letter_odds(model, alphabet, text): |
| 182 | """The model's odds (0 to 1) for every possible next letter after text.""" |
| 183 | model.eval() |
| 184 | odds = F.softmax(model(encode(text, alphabet)[:, -CONTEXT:])[0, -1], dim=-1).cpu() |
| 185 | model.train() |
| 186 | return [round(p, 5) for p in odds.tolist()] |
| 187 | |
| 188 | |
| 189 | def top(odds, alphabet, k): |
| 190 | best = sorted(range(len(odds)), key=lambda i: -odds[i])[:k] |
| 191 | return [[alphabet[i], odds[i]] for i in best] |
| 192 | |
| 193 | |
| 194 | |
| 195 | def read_along(model, alphabet, before, sentence): |
| 196 | """Guess each letter of sentence from the letters before it; report the top 3 guesses and the real letter.""" |
| 197 | out = [] |
| 198 | for i, real in enumerate(sentence): |
| 199 | odds = next_letter_odds(model, alphabet, before + sentence[:i]) |
| 200 | out.append({"real": real, "real_odds": odds[alphabet.index(real)], "top": top(odds, alphabet, 3)}) |
| 201 | return out |
| 202 | |
| 203 | |
| 204 | |
| 205 | def attention_weights(model, alphabet, text): |
| 206 | """For the last letter of text: how much each layer looks back at each earlier letter (heads averaged).""" |
| 207 | model.eval() |
| 208 | ids = encode(text, alphabet) |
| 209 | n = ids.shape[1] |
| 210 | x = model.letter(ids) + model.position(torch.arange(n, device=DEVICE)) |
| 211 | mask = torch.triu(torch.ones(n, n, dtype=torch.bool, device=DEVICE), diagonal=1) |
| 212 | layers = [] |
| 213 | for block in model.blocks: |
| 214 | h = block.norm1(x) |
| 215 | a, w = block.attention(h, h, h, attn_mask=mask, need_weights=True, average_attn_weights=True) |
| 216 | layers.append([round(v, 4) for v in w[0, -1].tolist()]) |
| 217 | x = x + a |
| 218 | x = x + block.mlp(block.norm2(x)) |
| 219 | odds = F.softmax(model.out(model.norm(x))[0, -1], dim=-1).tolist() |
| 220 | model.train() |
| 221 | return {"text": text, "layers": layers, "top": top(odds, alphabet, 5)} |
| 222 | |
| 223 | |
| 224 | |
| 225 | def typewriter(model, alphabet, generator, letters): |
| 226 | """Write after PROMPT, keeping the top 5 odds at each letter and the letter that was picked.""" |
| 227 | model.eval() |
| 228 | ids = encode(PROMPT, alphabet) |
| 229 | frames = [] |
| 230 | for _ in range(letters): |
| 231 | odds = F.softmax(model(ids[:, -CONTEXT:])[0, -1] / TEMPERATURE, dim=-1).cpu() |
| 232 | next_id = torch.multinomial(odds, 1, generator=generator).item() |
| 233 | frames.append({"top": top([round(p, 4) for p in odds.tolist()], alphabet, 5), "picked": alphabet[next_id]}) |
| 234 | ids = torch.cat([ids, torch.tensor([[next_id]], device=DEVICE)], dim=1) |
| 235 | model.train() |
| 236 | return frames |
| 237 | |
| 238 | |
| 239 | |
| 240 | def write_with_dial(model, alphabet, temperature, seed, letters=200): |
| 241 | """Like write_story, with the 'adventure dial' (temperature) set by hand.""" |
| 242 | model.eval() |
| 243 | generator = torch.Generator().manual_seed(seed) |
| 244 | ids = encode(PROMPT, alphabet) |
| 245 | for _ in range(letters): |
| 246 | odds = F.softmax(model(ids[:, -CONTEXT:])[0, -1] / temperature, dim=-1).cpu() |
| 247 | ids = torch.cat([ids, torch.multinomial(odds, 1, generator=generator).view(1, 1).to(DEVICE)], dim=1) |
| 248 | model.train() |
| 249 | return "".join(alphabet[i] for i in ids[0].tolist()) |
| 250 | |
| 251 | |
| 252 | # --------------------------------------------------------------------------- |
| 253 | # Pictures |
| 254 | # --------------------------------------------------------------------------- |
| 255 | |
| 256 | def save_learning(rows, path="learning.png"): |
| 257 | fig, ax = plt.subplots(figsize=(6.4, 3.8), dpi=160) |
| 258 | steps = [r["step"] for r in rows] |
| 259 | # perplexity = e^loss: "how many letters it is torn between", easier to read than loss |
| 260 | ax.plot(steps, [math.exp(r["train_loss"]) for r in rows], "o-", color="#2557a7", label="text it learned from") |
| 261 | ax.plot(steps, [math.exp(r["test_loss"]) for r in rows], "s-", color="#c2410c", label="text it never saw") |
| 262 | ax.set_xscale("symlog", linthresh=100) |
| 263 | ax.set_xlabel("training steps") |
| 264 | ax.set_ylabel("letters it is torn between") |
| 265 | ax.grid(True, alpha=0.3) |
| 266 | ax.legend(frameon=False) |
| 267 | ax.spines[["top", "right"]].set_visible(False) |
| 268 | fig.tight_layout() |
| 269 | fig.savefig(path, facecolor="white") |
| 270 | |
| 271 | |
| 272 | def save_samples(samples, path="samples.png"): |
| 273 | shown = [s for s in samples if s["step"] in (0, 300, 1000, STEPS)] |
| 274 | fig, axes = plt.subplots(len(shown), 1, figsize=(7.2, 1.9 * len(shown)), dpi=160) |
| 275 | for ax, s in zip(axes, shown): |
| 276 | story = s["story"][:200].replace("\n", " ") |
| 277 | ax.text(0, 1, f"After {s['step']:,} steps:", fontsize=10, weight="bold", va="top", transform=ax.transAxes) |
| 278 | ax.text(0, 0.78, "\n".join(textwrap.wrap(story, 78)), fontsize=9, family="monospace", va="top", |
| 279 | transform=ax.transAxes) |
| 280 | ax.axis("off") |
| 281 | fig.tight_layout() |
| 282 | fig.savefig(path, facecolor="white") |
| 283 | |
| 284 | |
| 285 | # --------------------------------------------------------------------------- |
| 286 | # Main |
| 287 | # --------------------------------------------------------------------------- |
| 288 | |
| 289 | def main(): |
| 290 | began = time.time() |
| 291 | torch.manual_seed(SEED) |
| 292 | batches = torch.Generator().manual_seed(SEED) |
| 293 | sampler = torch.Generator().manual_seed(SEED + 1) |
| 294 | train_ids, test_ids, alphabet = load_text() |
| 295 | model = TinyStoryteller(len(alphabet)).to(DEVICE) |
| 296 | optimizer = torch.optim.AdamW(model.parameters(), lr=LEARNING_RATE) |
| 297 | parameters = sum(p.numel() for p in model.parameters()) |
| 298 | print(f"{parameters:,} parameters, {len(alphabet)} different letters, {len(train_ids):,} training letters") |
| 299 | |
| 300 | # the hidden sentence and the 128 letters before it, straight from the test part of the book |
| 301 | test_text = "".join(alphabet[i] for i in test_ids.tolist()) |
| 302 | at = test_text.index(READ_ALONG) |
| 303 | read_before = test_text[max(0, at - CONTEXT):at] |
| 304 | probes = {"alphabet": alphabet, "prompt": PROMPT, "context": CONTEXT, "batch_size": BATCH_SIZE, |
| 305 | "temperature": TEMPERATURE, "book_letters": len(train_ids) + len(test_ids), |
| 306 | "training_letters": len(train_ids), "test_letters": len(test_ids), |
| 307 | "first_batch_starts": [], "guess_prompts": GUESS_PROMPTS, |
| 308 | "read_along_sentence": READ_ALONG, "odds": [], "read_along": [], |
| 309 | "parameters": parameters, "layers": LAYERS, "heads": HEADS, "width": WIDTH} |
| 310 | |
| 311 | rows, samples = [], [] |
| 312 | for step in range(STEPS + 1): |
| 313 | if step in CHECKPOINTS: |
| 314 | train_loss, train_right = average_loss(model, train_ids, batches) |
| 315 | test_loss, test_right = average_loss(model, test_ids, batches) |
| 316 | row = {"step": step, "train_loss": train_loss, "test_loss": test_loss, |
| 317 | "train_first_guess_right": train_right, "test_first_guess_right": test_right} |
| 318 | rows.append(row) |
| 319 | samples.append({"step": step, "story": write_story(model, alphabet, sampler)}) |
| 320 | probes["odds"].append({"step": step, **{t: next_letter_odds(model, alphabet, t) for t in GUESS_PROMPTS}}) |
| 321 | probes["read_along"].append({"step": step, "guesses": read_along(model, alphabet, read_before, READ_ALONG)}) |
| 322 | print(f"step {step:>5} train {row['train_loss']:.3f} test {row['test_loss']:.3f} " |
| 323 | f"first guess right on new text {100 * test_right:.1f}%", flush=True) |
| 324 | if step == STEPS: |
| 325 | break |
| 326 | # learning rate fades to zero by the end (cosine schedule) |
| 327 | for group in optimizer.param_groups: |
| 328 | group["lr"] = LEARNING_RATE * 0.5 * (1 + math.cos(math.pi * step / STEPS)) |
| 329 | # remember where the very first practice step reads from, for the pictures |
| 330 | x, y = random_batch(train_ids, batches, keep=probes["first_batch_starts"] if step == 0 else None) |
| 331 | loss = F.cross_entropy(model(x).flatten(0, 1), y.flatten()) |
| 332 | optimizer.zero_grad() |
| 333 | loss.backward() |
| 334 | optimizer.step() |
| 335 | if step % 50 == 0: |
| 336 | print(f"step {step:>5} practice loss {loss.item():.3f}", flush=True) # also keeps the log stream alive |
| 337 | probes["attention"] = attention_weights(model, alphabet, ATTENTION_TEXT) |
| 338 | probes["typewriter"] = typewriter(model, alphabet, torch.Generator().manual_seed(SEED + 2), TYPEWRITER_LETTERS) |
| 339 | probes["dial"] = [{"temperature": t, "story": write_with_dial(model, alphabet, t, SEED + 3)} for t in DIAL_SETTINGS] |
| 340 | seconds = time.time() - began |
| 341 | |
| 342 | with open("results.csv", "w", newline="") as f: |
| 343 | writer = csv.DictWriter(f, fieldnames=["step", "train_loss", "test_loss", "train_first_guess_right", "test_first_guess_right"]) |
| 344 | writer.writeheader() |
| 345 | for r in rows: |
| 346 | writer.writerow({k: (f"{v:.4f}" if isinstance(v, float) else v) for k, v in r.items()}) |
| 347 | with open("samples.json", "w", encoding="utf-8") as f: |
| 348 | json.dump(samples, f, indent=2, ensure_ascii=False) |
| 349 | with open("probes.json", "w", encoding="utf-8") as f: |
| 350 | json.dump(probes, f, ensure_ascii=False) |
| 351 | save_learning(rows) |
| 352 | save_samples(samples) |
| 353 | |
| 354 | print(f"\n--- story after {STEPS} steps ---\n{samples[-1]['story']}\n") |
| 355 | first, last = rows[0], rows[-1] |
| 356 | print(json.dumps({ |
| 357 | "device": torch.cuda.get_device_name(0) if DEVICE == "cuda" else "cpu", |
| 358 | "seconds": round(seconds, 1), |
| 359 | "parameters": parameters, |
| 360 | "alphabet_size": len(alphabet), |
| 361 | "training_letters": len(train_ids), |
| 362 | "steps": STEPS, |
| 363 | "torn_between_start": round(math.exp(first["test_loss"]), 1), |
| 364 | "torn_between_end": round(math.exp(last["test_loss"]), 2), |
| 365 | "test_loss_end": round(last["test_loss"], 3), |
| 366 | "train_loss_end": round(last["train_loss"], 3), |
| 367 | "first_guess_right_start_pct": round(100 * first["test_first_guess_right"], 1), |
| 368 | "first_guess_right_end_pct": round(100 * last["test_first_guess_right"], 1), |
| 369 | "torn_between_train_end": round(math.exp(last["train_loss"]), 2), |
| 370 | })) |
| 371 | |
| 372 | |
| 373 | if __name__ == "__main__": |
| 374 | main() |