← A tiny storyteller: how a computer learns to write, one letter at a time

experiment.py

Python · 374 lines · Download raw ·

1"""
2Train a tiny storyteller: a small language model that learns to write like
3Grimms' Fairy Tales, one letter at a time.
4
5Data: grimm.txt, the text of Grimms' Fairy Tales (public domain, from
6Project Gutenberg, with the Gutenberg header and licence removed).
7
8Model: a very small "transformer", the same kind of model that powers AI
9chatbots, but about a hundred thousand times smaller. It reads the last
10CONTEXT letters and guesses the next one. We train it on 90% of the text and
11keep 10% aside to test it on writing it has never seen.
12
13While it learns, we stop at a few checkpoints and ask it to continue the
14words "Once upon a time", so we can watch its writing improve.
15
16Outputs:
17 results.csv training and test "surprise" (loss), and how often the
18 first guess is right, at every checkpoint
19 samples.json the story the model writes at each checkpoint
20 probes.json a close look inside the model, for the pictures: its odds
21 for the next letter, which letters it looks back at, the
22 same story written with the "adventure dial" turned to
23 three settings, and more
24 learning.png how surprised the model is by new text, as it learns
25 samples.png the stories from a few checkpoints, side by side
26
27pictures.py turns results.csv, samples.json and probes.json into the rest of
28the pictures and animations in the post.
29
30The last line printed is a JSON summary of the headline numbers.
31
32Run:
33 python3 experiment.py && python3 pictures.py
34"""
35
36import csv
37import json
38import math
39import os
40import textwrap
41import time
42
43import matplotlib
44import torch
45import torch.nn as nn
46import torch.nn.functional as F
47
48matplotlib.use("Agg") # draw to a file, no window needed
49import matplotlib.pyplot as plt
50
51# ---------------------------------------------------------------------------
52# Settings
53# ---------------------------------------------------------------------------
54
55SEED = 0
56STEPS = int(os.environ.get("STEPS", 2000)) # training steps (more makes it memorise the book)
57CHECKPOINTS = sorted({0, 100, 300, 1000, STEPS})
58CONTEXT = 128 # letters the model can look back at
59BATCH_SIZE = 64
60LAYERS, HEADS, WIDTH = 4, 4, 192
61LEARNING_RATE = 1e-3
62PROMPT = "Once upon a time"
63SAMPLE_LENGTH = 300 # letters written at each checkpoint
64TEMPERATURE = 0.8 # below 1 = a little less random when picking letters
65DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
66
67# Probes: small, fixed questions we ask the model at every checkpoint
68GUESS_PROMPTS = ["Once upon a ti", "The king and the qu"]
69READ_ALONG = "Once upon a time the king held a great feast," # a sentence from the hidden 10% of the book
70ATTENTION_TEXT = "Once upon a time there was a king who had a daugh"
71DIAL_SETTINGS = [0.3, 0.8, 1.5] # temperature: low = careful, high = adventurous
72TYPEWRITER_LETTERS = 60
73
74
75# ---------------------------------------------------------------------------
76# Data
77# ---------------------------------------------------------------------------
78
79def load_text(path="grimm.txt"):
80 """Read the book and turn every letter into a number (its position in the alphabet we found)."""
81 text = open(path, encoding="utf-8").read()
82 alphabet = sorted(set(text))
83 to_id = {ch: i for i, ch in enumerate(alphabet)}
84 ids = torch.tensor([to_id[ch] for ch in text], dtype=torch.long)
85 split = int(0.9 * len(ids))
86 return ids[:split], ids[split:], alphabet
87
88
89def random_batch(ids, generator, keep=None):
90 """BATCH_SIZE random stretches of text, and the same stretches shifted by one letter (the answers).
91 If keep is a list, the start positions are added to it (for the pictures)."""
92 starts = torch.randint(0, len(ids) - CONTEXT - 1, (BATCH_SIZE,), generator=generator)
93 if keep is not None:
94 keep.extend(starts.tolist())
95 x = torch.stack([ids[s:s + CONTEXT] for s in starts])
96 y = torch.stack([ids[s + 1:s + CONTEXT + 1] for s in starts])
97 return x.to(DEVICE), y.to(DEVICE)
98
99
100# ---------------------------------------------------------------------------
101# Model
102# ---------------------------------------------------------------------------
103
104class Block(nn.Module):
105 """One transformer layer: look back at earlier letters (attention), then think (MLP)."""
106
107 def __init__(self):
108 super().__init__()
109 self.norm1 = nn.LayerNorm(WIDTH)
110 self.attention = nn.MultiheadAttention(WIDTH, HEADS, batch_first=True)
111 self.norm2 = nn.LayerNorm(WIDTH)
112 self.mlp = nn.Sequential(nn.Linear(WIDTH, 4 * WIDTH), nn.GELU(), nn.Linear(4 * WIDTH, WIDTH))
113
114 def forward(self, x, mask):
115 h = self.norm1(x)
116 x = x + self.attention(h, h, h, attn_mask=mask, need_weights=False)[0]
117 return x + self.mlp(self.norm2(x))
118
119
120class TinyStoryteller(nn.Module):
121 """Reads up to CONTEXT letters and gives a score for every possible next letter."""
122
123 def __init__(self, alphabet_size):
124 super().__init__()
125 self.letter = nn.Embedding(alphabet_size, WIDTH)
126 self.position = nn.Embedding(CONTEXT, WIDTH)
127 self.blocks = nn.ModuleList(Block() for _ in range(LAYERS))
128 self.norm = nn.LayerNorm(WIDTH)
129 self.out = nn.Linear(WIDTH, alphabet_size)
130
131 def forward(self, ids):
132 n = ids.shape[1]
133 x = self.letter(ids) + self.position(torch.arange(n, device=ids.device))
134 # each letter may only look at letters before it, never ahead
135 mask = torch.triu(torch.ones(n, n, dtype=torch.bool, device=ids.device), diagonal=1)
136 for block in self.blocks:
137 x = block(x, mask)
138 return self.out(self.norm(x))
139
140
141@torch.no_grad()
142def write_story(model, alphabet, generator):
143 """Continue PROMPT one letter at a time, picking each letter by the model's odds."""
144 model.eval()
145 to_id = {ch: i for i, ch in enumerate(alphabet)}
146 ids = torch.tensor([[to_id[ch] for ch in PROMPT]], device=DEVICE)
147 for _ in range(SAMPLE_LENGTH):
148 scores = model(ids[:, -CONTEXT:])[0, -1] / TEMPERATURE
149 odds = F.softmax(scores, dim=-1).cpu()
150 next_id = torch.multinomial(odds, 1, generator=generator).to(DEVICE)
151 ids = torch.cat([ids, next_id.view(1, 1)], dim=1)
152 model.train()
153 return "".join(alphabet[i] for i in ids[0].tolist())
154
155
156@torch.no_grad()
157def average_loss(model, ids, generator, batches=20):
158 """Average 'surprise' (cross-entropy, in nats per letter) on random stretches of ids,
159 and the share of letters where the model's first choice was the right one."""
160 model.eval()
161 total, right = 0.0, 0.0
162 for _ in range(batches):
163 x, y = random_batch(ids, generator)
164 scores = model(x)
165 total += F.cross_entropy(scores.flatten(0, 1), y.flatten()).item()
166 right += (scores.argmax(-1) == y).float().mean().item()
167 model.train()
168 return total / batches, right / batches
169
170
171# ---------------------------------------------------------------------------
172# Probes: looking inside the model, for the pictures
173# ---------------------------------------------------------------------------
174
175def encode(text, alphabet):
176 to_id = {ch: i for i, ch in enumerate(alphabet)}
177 return torch.tensor([[to_id[ch] for ch in text]], device=DEVICE)
178
179
180@torch.no_grad()
181def next_letter_odds(model, alphabet, text):
182 """The model's odds (0 to 1) for every possible next letter after text."""
183 model.eval()
184 odds = F.softmax(model(encode(text, alphabet)[:, -CONTEXT:])[0, -1], dim=-1).cpu()
185 model.train()
186 return [round(p, 5) for p in odds.tolist()]
187
188
189def top(odds, alphabet, k):
190 best = sorted(range(len(odds)), key=lambda i: -odds[i])[:k]
191 return [[alphabet[i], odds[i]] for i in best]
192
193
194@torch.no_grad()
195def read_along(model, alphabet, before, sentence):
196 """Guess each letter of sentence from the letters before it; report the top 3 guesses and the real letter."""
197 out = []
198 for i, real in enumerate(sentence):
199 odds = next_letter_odds(model, alphabet, before + sentence[:i])
200 out.append({"real": real, "real_odds": odds[alphabet.index(real)], "top": top(odds, alphabet, 3)})
201 return out
202
203
204@torch.no_grad()
205def attention_weights(model, alphabet, text):
206 """For the last letter of text: how much each layer looks back at each earlier letter (heads averaged)."""
207 model.eval()
208 ids = encode(text, alphabet)
209 n = ids.shape[1]
210 x = model.letter(ids) + model.position(torch.arange(n, device=DEVICE))
211 mask = torch.triu(torch.ones(n, n, dtype=torch.bool, device=DEVICE), diagonal=1)
212 layers = []
213 for block in model.blocks:
214 h = block.norm1(x)
215 a, w = block.attention(h, h, h, attn_mask=mask, need_weights=True, average_attn_weights=True)
216 layers.append([round(v, 4) for v in w[0, -1].tolist()])
217 x = x + a
218 x = x + block.mlp(block.norm2(x))
219 odds = F.softmax(model.out(model.norm(x))[0, -1], dim=-1).tolist()
220 model.train()
221 return {"text": text, "layers": layers, "top": top(odds, alphabet, 5)}
222
223
224@torch.no_grad()
225def typewriter(model, alphabet, generator, letters):
226 """Write after PROMPT, keeping the top 5 odds at each letter and the letter that was picked."""
227 model.eval()
228 ids = encode(PROMPT, alphabet)
229 frames = []
230 for _ in range(letters):
231 odds = F.softmax(model(ids[:, -CONTEXT:])[0, -1] / TEMPERATURE, dim=-1).cpu()
232 next_id = torch.multinomial(odds, 1, generator=generator).item()
233 frames.append({"top": top([round(p, 4) for p in odds.tolist()], alphabet, 5), "picked": alphabet[next_id]})
234 ids = torch.cat([ids, torch.tensor([[next_id]], device=DEVICE)], dim=1)
235 model.train()
236 return frames
237
238
239@torch.no_grad()
240def write_with_dial(model, alphabet, temperature, seed, letters=200):
241 """Like write_story, with the 'adventure dial' (temperature) set by hand."""
242 model.eval()
243 generator = torch.Generator().manual_seed(seed)
244 ids = encode(PROMPT, alphabet)
245 for _ in range(letters):
246 odds = F.softmax(model(ids[:, -CONTEXT:])[0, -1] / temperature, dim=-1).cpu()
247 ids = torch.cat([ids, torch.multinomial(odds, 1, generator=generator).view(1, 1).to(DEVICE)], dim=1)
248 model.train()
249 return "".join(alphabet[i] for i in ids[0].tolist())
250
251
252# ---------------------------------------------------------------------------
253# Pictures
254# ---------------------------------------------------------------------------
255
256def save_learning(rows, path="learning.png"):
257 fig, ax = plt.subplots(figsize=(6.4, 3.8), dpi=160)
258 steps = [r["step"] for r in rows]
259 # perplexity = e^loss: "how many letters it is torn between", easier to read than loss
260 ax.plot(steps, [math.exp(r["train_loss"]) for r in rows], "o-", color="#2557a7", label="text it learned from")
261 ax.plot(steps, [math.exp(r["test_loss"]) for r in rows], "s-", color="#c2410c", label="text it never saw")
262 ax.set_xscale("symlog", linthresh=100)
263 ax.set_xlabel("training steps")
264 ax.set_ylabel("letters it is torn between")
265 ax.grid(True, alpha=0.3)
266 ax.legend(frameon=False)
267 ax.spines[["top", "right"]].set_visible(False)
268 fig.tight_layout()
269 fig.savefig(path, facecolor="white")
270
271
272def save_samples(samples, path="samples.png"):
273 shown = [s for s in samples if s["step"] in (0, 300, 1000, STEPS)]
274 fig, axes = plt.subplots(len(shown), 1, figsize=(7.2, 1.9 * len(shown)), dpi=160)
275 for ax, s in zip(axes, shown):
276 story = s["story"][:200].replace("\n", " ")
277 ax.text(0, 1, f"After {s['step']:,} steps:", fontsize=10, weight="bold", va="top", transform=ax.transAxes)
278 ax.text(0, 0.78, "\n".join(textwrap.wrap(story, 78)), fontsize=9, family="monospace", va="top",
279 transform=ax.transAxes)
280 ax.axis("off")
281 fig.tight_layout()
282 fig.savefig(path, facecolor="white")
283
284
285# ---------------------------------------------------------------------------
286# Main
287# ---------------------------------------------------------------------------
288
289def main():
290 began = time.time()
291 torch.manual_seed(SEED)
292 batches = torch.Generator().manual_seed(SEED)
293 sampler = torch.Generator().manual_seed(SEED + 1)
294 train_ids, test_ids, alphabet = load_text()
295 model = TinyStoryteller(len(alphabet)).to(DEVICE)
296 optimizer = torch.optim.AdamW(model.parameters(), lr=LEARNING_RATE)
297 parameters = sum(p.numel() for p in model.parameters())
298 print(f"{parameters:,} parameters, {len(alphabet)} different letters, {len(train_ids):,} training letters")
299
300 # the hidden sentence and the 128 letters before it, straight from the test part of the book
301 test_text = "".join(alphabet[i] for i in test_ids.tolist())
302 at = test_text.index(READ_ALONG)
303 read_before = test_text[max(0, at - CONTEXT):at]
304 probes = {"alphabet": alphabet, "prompt": PROMPT, "context": CONTEXT, "batch_size": BATCH_SIZE,
305 "temperature": TEMPERATURE, "book_letters": len(train_ids) + len(test_ids),
306 "training_letters": len(train_ids), "test_letters": len(test_ids),
307 "first_batch_starts": [], "guess_prompts": GUESS_PROMPTS,
308 "read_along_sentence": READ_ALONG, "odds": [], "read_along": [],
309 "parameters": parameters, "layers": LAYERS, "heads": HEADS, "width": WIDTH}
310
311 rows, samples = [], []
312 for step in range(STEPS + 1):
313 if step in CHECKPOINTS:
314 train_loss, train_right = average_loss(model, train_ids, batches)
315 test_loss, test_right = average_loss(model, test_ids, batches)
316 row = {"step": step, "train_loss": train_loss, "test_loss": test_loss,
317 "train_first_guess_right": train_right, "test_first_guess_right": test_right}
318 rows.append(row)
319 samples.append({"step": step, "story": write_story(model, alphabet, sampler)})
320 probes["odds"].append({"step": step, **{t: next_letter_odds(model, alphabet, t) for t in GUESS_PROMPTS}})
321 probes["read_along"].append({"step": step, "guesses": read_along(model, alphabet, read_before, READ_ALONG)})
322 print(f"step {step:>5} train {row['train_loss']:.3f} test {row['test_loss']:.3f} "
323 f"first guess right on new text {100 * test_right:.1f}%", flush=True)
324 if step == STEPS:
325 break
326 # learning rate fades to zero by the end (cosine schedule)
327 for group in optimizer.param_groups:
328 group["lr"] = LEARNING_RATE * 0.5 * (1 + math.cos(math.pi * step / STEPS))
329 # remember where the very first practice step reads from, for the pictures
330 x, y = random_batch(train_ids, batches, keep=probes["first_batch_starts"] if step == 0 else None)
331 loss = F.cross_entropy(model(x).flatten(0, 1), y.flatten())
332 optimizer.zero_grad()
333 loss.backward()
334 optimizer.step()
335 if step % 50 == 0:
336 print(f"step {step:>5} practice loss {loss.item():.3f}", flush=True) # also keeps the log stream alive
337 probes["attention"] = attention_weights(model, alphabet, ATTENTION_TEXT)
338 probes["typewriter"] = typewriter(model, alphabet, torch.Generator().manual_seed(SEED + 2), TYPEWRITER_LETTERS)
339 probes["dial"] = [{"temperature": t, "story": write_with_dial(model, alphabet, t, SEED + 3)} for t in DIAL_SETTINGS]
340 seconds = time.time() - began
341
342 with open("results.csv", "w", newline="") as f:
343 writer = csv.DictWriter(f, fieldnames=["step", "train_loss", "test_loss", "train_first_guess_right", "test_first_guess_right"])
344 writer.writeheader()
345 for r in rows:
346 writer.writerow({k: (f"{v:.4f}" if isinstance(v, float) else v) for k, v in r.items()})
347 with open("samples.json", "w", encoding="utf-8") as f:
348 json.dump(samples, f, indent=2, ensure_ascii=False)
349 with open("probes.json", "w", encoding="utf-8") as f:
350 json.dump(probes, f, ensure_ascii=False)
351 save_learning(rows)
352 save_samples(samples)
353
354 print(f"\n--- story after {STEPS} steps ---\n{samples[-1]['story']}\n")
355 first, last = rows[0], rows[-1]
356 print(json.dumps({
357 "device": torch.cuda.get_device_name(0) if DEVICE == "cuda" else "cpu",
358 "seconds": round(seconds, 1),
359 "parameters": parameters,
360 "alphabet_size": len(alphabet),
361 "training_letters": len(train_ids),
362 "steps": STEPS,
363 "torn_between_start": round(math.exp(first["test_loss"]), 1),
364 "torn_between_end": round(math.exp(last["test_loss"]), 2),
365 "test_loss_end": round(last["test_loss"], 3),
366 "train_loss_end": round(last["train_loss"], 3),
367 "first_guess_right_start_pct": round(100 * first["test_first_guess_right"], 1),
368 "first_guess_right_end_pct": round(100 * last["test_first_guess_right"], 1),
369 "torn_between_train_end": round(math.exp(last["train_loss"]), 2),
370 }))
371
372
373if __name__ == "__main__":
374 main()