← Can a computer learn to read your numbers?

experiment.py

Python · 256 lines · Download raw ·

1"""
2Teach a small neural network to read handwritten digits (0-9).
3
4Data: MNIST, 60,000 training pictures and 10,000 test pictures of
5handwritten digits, each 28 x 28 pixels. It is downloaded from the Hugging
6Face dataset "ylecun/mnist".
7
8Model: a small convolutional network (two "pattern finder" layers, then two
9ordinary layers). We train it 3 times with different random seeds, for
10EPOCHS passes over the training pictures each, and test it on the 10,000
11pictures it never saw while learning.
12
13Outputs:
14 results.csv test accuracy after every epoch, for every seed
15 examples.png a few training pictures with their labels
16 learning.png test accuracy after each epoch
17 mistakes.png test pictures the network got wrong (from the first seed)
18 mixups.png which digits get mixed up with which (all seeds)
19
20The last line printed is a JSON summary of the headline numbers.
21
22Run:
23 python3 experiment.py
24"""
25
26import csv
27import io
28import json
29import os
30import time
31
32import matplotlib
33import numpy as np
34import pyarrow.parquet as pq
35import torch
36import torch.nn as nn
37import torch.nn.functional as F
38from huggingface_hub import hf_hub_download
39from PIL import Image
40
41matplotlib.use("Agg") # draw to a file, no window needed
42import matplotlib.pyplot as plt
43
44# ---------------------------------------------------------------------------
45# Settings
46# ---------------------------------------------------------------------------
47
48SEEDS = [0, 1, 2]
49EPOCHS = int(os.environ.get("EPOCHS", 5))
50BATCH_SIZE = 128
51LEARNING_RATE = 1e-3
52TRAIN_LIMIT = int(os.environ.get("TRAIN_LIMIT", 60_000)) # set lower for a quick local test
53DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
54
55
56# ---------------------------------------------------------------------------
57# Data
58# ---------------------------------------------------------------------------
59
60def load_split(split):
61 """Download one MNIST split and return (images as floats 0-1, labels)."""
62 path = hf_hub_download("ylecun/mnist", f"mnist/{split}-00000-of-00001.parquet", repo_type="dataset")
63 table = pq.read_table(path).to_pydict()
64 images = np.stack([np.array(Image.open(io.BytesIO(img["bytes"]))) for img in table["image"]])
65 return images.astype(np.float32) / 255.0, np.array(table["label"], dtype=np.int64)
66
67
68# ---------------------------------------------------------------------------
69# Model
70# ---------------------------------------------------------------------------
71
72class SmallConvNet(nn.Module):
73 """Two pattern-finder (convolution) layers, then two ordinary layers.
74
75 The pattern finders slide small 3x3 windows over the picture and learn to
76 spot strokes, curves and corners. The last layers combine those into a
77 score for each digit 0-9; the highest score is the network's answer.
78 """
79
80 def __init__(self):
81 super().__init__()
82 self.conv1 = nn.Conv2d(1, 16, 3, padding=1) # 28x28 -> 16 maps of 28x28
83 self.conv2 = nn.Conv2d(16, 32, 3, padding=1) # after pooling 14x14 -> 32 maps
84 self.fc1 = nn.Linear(32 * 7 * 7, 128)
85 self.fc2 = nn.Linear(128, 10)
86
87 def forward(self, x):
88 x = F.max_pool2d(F.relu(self.conv1(x)), 2) # -> 16 x 14 x 14
89 x = F.max_pool2d(F.relu(self.conv2(x)), 2) # -> 32 x 7 x 7
90 x = F.relu(self.fc1(x.flatten(1)))
91 return self.fc2(x)
92
93
94def count_parameters(model):
95 return sum(p.numel() for p in model.parameters())
96
97
98# ---------------------------------------------------------------------------
99# Training and testing
100# ---------------------------------------------------------------------------
101
102def predict(model, images):
103 """The network's answer (0-9) for every picture."""
104 model.eval()
105 answers = []
106 with torch.no_grad():
107 for start in range(0, len(images), 1000):
108 answers.append(model(images[start:start + 1000]).argmax(1))
109 return torch.cat(answers)
110
111
112def train_one_seed(seed, train_x, train_y, test_x, test_y):
113 """Train a fresh network; return accuracy after each epoch and final test answers."""
114 torch.manual_seed(seed)
115 model = SmallConvNet().to(DEVICE)
116 optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)
117 accuracies = []
118 for epoch in range(EPOCHS):
119 model.train()
120 order = torch.randperm(len(train_x), device=DEVICE)
121 for start in range(0, len(order), BATCH_SIZE):
122 batch = order[start:start + BATCH_SIZE]
123 loss = F.cross_entropy(model(train_x[batch]), train_y[batch])
124 optimizer.zero_grad()
125 loss.backward()
126 optimizer.step()
127 answers = predict(model, test_x)
128 accuracies.append((answers == test_y).float().mean().item())
129 print(f"seed {seed} epoch {epoch + 1} test accuracy {accuracies[-1]:.4f}", flush=True)
130 return model, accuracies, answers
131
132
133# ---------------------------------------------------------------------------
134# Pictures
135# ---------------------------------------------------------------------------
136
137def save_examples(images, labels, path="examples.png"):
138 fig, axes = plt.subplots(2, 8, figsize=(7.2, 2.4), dpi=160)
139 for i, ax in enumerate(axes.flat):
140 ax.imshow(images[i], cmap="gray_r")
141 ax.set_title(str(labels[i]), fontsize=11)
142 ax.axis("off")
143 fig.tight_layout()
144 fig.savefig(path, facecolor="white")
145
146
147def save_learning(all_acc, path="learning.png"):
148 fig, ax = plt.subplots(figsize=(6.4, 3.8), dpi=160)
149 epochs = np.arange(1, EPOCHS + 1)
150 for seed, acc in zip(SEEDS, all_acc):
151 ax.plot(epochs, 100 * np.array(acc), "o-", ms=4, label=f"try {seed + 1}")
152 ax.set_xlabel("passes through the 60,000 training pictures")
153 ax.set_ylabel("test pictures read correctly (%)")
154 ax.set_xticks(epochs)
155 ax.grid(True, alpha=0.3)
156 ax.legend(frameon=False)
157 ax.spines[["top", "right"]].set_visible(False)
158 fig.tight_layout()
159 fig.savefig(path, facecolor="white")
160
161
162def save_mistakes(images, labels, answers, path="mistakes.png", count=16):
163 wrong = np.flatnonzero(answers != labels)[:count]
164 fig, axes = plt.subplots(2, 8, figsize=(7.2, 2.8), dpi=160)
165 for ax, i in zip(axes.flat, wrong):
166 ax.imshow(images[i], cmap="gray_r")
167 ax.set_title(f"is {labels[i]}\nsaid {answers[i]}", fontsize=9)
168 ax.axis("off")
169 for ax in list(axes.flat)[len(wrong):]:
170 ax.axis("off")
171 fig.tight_layout()
172 fig.savefig(path, facecolor="white")
173
174
175def save_mixups(confusion, path="mixups.png"):
176 """Grid of how often each true digit (row) was read as each other digit (column)."""
177 off = confusion.copy()
178 np.fill_diagonal(off, 0)
179 fig, ax = plt.subplots(figsize=(5.2, 4.6), dpi=160)
180 im = ax.imshow(off, cmap="Oranges")
181 for r in range(10):
182 for c in range(10):
183 if off[r, c]:
184 ax.text(c, r, str(off[r, c]), ha="center", va="center", fontsize=8,
185 color="white" if off[r, c] > off.max() / 2 else "black")
186 ax.set_xticks(range(10))
187 ax.set_yticks(range(10))
188 ax.set_xlabel("what the network said")
189 ax.set_ylabel("what the digit really was")
190 ax.set_title(f"Mistakes, added up over {len(SEEDS)} tries", fontsize=10)
191 fig.colorbar(im, ax=ax, shrink=0.8)
192 fig.tight_layout()
193 fig.savefig(path, facecolor="white")
194
195
196# ---------------------------------------------------------------------------
197# Main
198# ---------------------------------------------------------------------------
199
200def save_csv(all_acc, path="results.csv"):
201 with open(path, "w", newline="") as f:
202 writer = csv.writer(f)
203 writer.writerow(["seed", "epoch", "test_accuracy"])
204 for seed, acc in zip(SEEDS, all_acc):
205 for epoch, a in enumerate(acc, 1):
206 writer.writerow([seed, epoch, f"{a:.4f}"])
207
208
209def main():
210 began = time.time()
211 train_images, train_labels = load_split("train")
212 test_images, test_labels = load_split("test")
213 train_images, train_labels = train_images[:TRAIN_LIMIT], train_labels[:TRAIN_LIMIT]
214 to_tensor = lambda a: torch.tensor(a, device=DEVICE).unsqueeze(1)
215 train_x, test_x = to_tensor(train_images), to_tensor(test_images)
216 train_y = torch.tensor(train_labels, device=DEVICE)
217 test_y = torch.tensor(test_labels, device=DEVICE)
218
219 all_acc, confusion, first_answers = [], np.zeros((10, 10), dtype=np.int64), None
220 for seed in SEEDS:
221 model, acc, answers = train_one_seed(seed, train_x, train_y, test_x, test_y)
222 all_acc.append(acc)
223 answers = answers.cpu().numpy()
224 np.add.at(confusion, (test_labels, answers), 1)
225 if first_answers is None:
226 first_answers = answers
227 seconds = time.time() - began
228
229 save_csv(all_acc)
230 save_examples(train_images, train_labels)
231 save_learning(all_acc)
232 save_mistakes(test_images, test_labels, first_answers)
233 save_mixups(confusion)
234
235 final = np.array([acc[-1] for acc in all_acc])
236 off = confusion.copy()
237 np.fill_diagonal(off, 0)
238 r, c = np.unravel_index(off.argmax(), off.shape)
239 print(json.dumps({
240 "device": torch.cuda.get_device_name(0) if DEVICE == "cuda" else "cpu",
241 "seconds": round(seconds, 1),
242 "parameters": count_parameters(SmallConvNet()),
243 "train_pictures": len(train_images),
244 "test_pictures": len(test_images),
245 "epochs": EPOCHS,
246 "accuracy_after_1_epoch_pct": round(100 * float(np.mean([a[0] for a in all_acc])), 2),
247 "final_accuracy_pct": round(100 * float(final.mean()), 2),
248 "final_accuracy_spread_pct": round(100 * float(final.std()), 2),
249 "mistakes_per_10000": round(float((1 - final.mean()) * 10_000)),
250 "most_common_mixup": f"{r} read as {c}",
251 "most_common_mixup_count": int(off[r, c]),
252 }))
253
254
255if __name__ == "__main__":
256 main()