Compute/memory requirements scripts

Estimations for memory and compute requirements for training large language models.

scripts/memory_compute_estimations/memory_req.py

@@ -13,15 +13,15 @@ setups = {
     "1T": { "L": 128, "H": 160, "D": 25600, }
 }
 
-CURRENT = setups["284M"]
+CURRENT = setups["65B"]
 
 L = CURRENT["L"] # number of layers
 H = CURRENT["H"] # number of heads
 D = CURRENT["D"] # embedding dimension
 
-BS = 32 # batch size
-SEQ = 512 # sequence length
-TOKS = 16_000 # number of tokens in the vocab
+BS = 1 # batch size
+SEQ = 4096 # sequence length
+TOKS = 32_000 # number of tokens in the vocab
 
 # -- END OF GLOBALS --