Compute/memory requirements scripts

Estimations for memory and compute requirements for training large language models.

scripts/memory_compute_estimations/memory_req.py

@@ -81,10 +81,15 @@ gradients = 1 * model_params
 moms = 2
 optimizer = moms * model_params
 
-gigabytes_used = (
+# 4 bytes (fp32) used; for 2 bytes activations (fp16), adjust the percent value;
+# ideally, it should be 0.5, however that value can realistically not be
+# reached; real values could be between 0.2 and 0.3
+activations_saved_percent = 0.25
+gigabytes_used = 4 * (
     model_params +
     optimizer +
-    max(activations, gradients)
+    (1 - activations_saved_percent) * activations +
+    gradients
 ) / bytes_to_gigs
 
 print(f"Model params: {model_params:,} - {model_params * 4 / bytes_to_gigs:,.2f}GB")