cuda: new download

labs/cuda/Makefile

@@ -17,19 +17,27 @@ DOWNLOAD_PARTITION ?= nehalem
 
 # Times
 RUN_TIME      ?= 00:05:00
-BUILD_TIME    ?= 00:02:00
+BUILD_TIME    ?= 00:05:00
 DOWNLOAD_TIME ?= 00:20:00
 
 # Commands
 BUILD_CMD    ?= make LOCAL=y
 RUN_CMD      ?= ./$(EXEC)
-DOWNLOAD_CMD ?= apptainer pull $(IMG_PATH) $(IMG_URL)
 LOAD_PREFIX  ?= apptainer exec --env='TMPDIR=$(HOME)' --nv $(IMG_PATH)
 
+# Download Config
+GIT_CLONE_PATH    := $(shell git rev-parse --show-toplevel)
+DOWNLOAD_TOOL     := $(GIT_CLONE_PATH)/labs/utils/multithreaded_downloader.sh
+DOWNLOAD_NUM_CPUS ?= 8
+DOWNLOAD_CMD      ?= $(DOWNLOAD_TOOL) $(SIF_IMG_URL) $(IMG_PATH) $(DOWNLOAD_NUM_CPUS)
+
 # Image config
 IMG_TAG  ?= 12.3.2
-IMG_URL  ?= docker://gitlab.cs.pub.ro:5050/asc/asc-public/cuda-labs:$(IMG_TAG)
 IMG_PATH ?= ~/DO_NOT_DELETE_IMGS/cuda-labs_$(IMG_TAG).sif
+
+# URLs
+SIF_IMG_URL    ?= https://repository.grid.pub.ro/cs/asc/cuda-labs_$(IMG_TAG).sif
+DOCKER_IMG_URL ?= docker://gitlab.cs.pub.ro:5050/asc/asc-public/cuda-labs:$(IMG_TAG)
 # Older images: https://gitlab.cs.pub.ro/asc/asc-public/container_registry/10
 
 compile: $(IMG_PATH)
@@ -47,13 +55,14 @@ run: $(EXEC) $(IMG_PATH)
 
 $(IMG_PATH):
 	@mkdir -p $(dir $(IMG_PATH))
-	@sbatch                                   \
-		--cpus-per-task=8                 \
-		--time $(DOWNLOAD_TIME)           \
-		--partition $(DOWNLOAD_PARTITION) \
+	@sbatch                                      \
+		--cpus-per-task=$(DOWNLOAD_NUM_CPUS) \
+		--time $(DOWNLOAD_TIME)              \
+		--partition $(DOWNLOAD_PARTITION)    \
 		--wrap="$(DOWNLOAD_CMD)"
 	$(info Please wait! It will take a while.)
 	@while [ ! -f $(IMG_PATH) ]; do sleep 10; done
+	@while [ -f $(IMG_PATH).part$$(( $(DOWNLOAD_NUM_CPUS) - 1 )) ]; do sleep 10; done
 
 # To run on your local machine define LOCAL=y or LOCAL=whatever
 else
@@ -69,3 +78,4 @@ endif
 
 clean:
 	rm -f $(EXEC) slurm-*.out slurm-*.err profile.ncu-rep
+	rm -f $(IMG_PATH).part*