Merge pull request karpathy#71 from cchan/patch-1

Zero-grad more aggressively to save memory

Author: karpathy

train.py

@@ -259,7 +259,6 @@ def get_lr(iter):
         break
 
     # forward backward update, with optional gradient accumulation to simulate larger batch size
-    optimizer.zero_grad(set_to_none=True)
     for micro_step in range(gradient_accumulation_steps):
         X, Y = get_batch('train')
         if ddp:
@@ -272,6 +271,7 @@ def get_lr(iter):
             logits, loss = model(X, Y)
         loss.backward()
     optimizer.step()
+    optimizer.zero_grad(set_to_none=True)
 
     # timing and logging
     t1 = time.time()