minor fix: Save model at epoch

Author: Phuc Le

scripts/evaluate.sh

@@ -1,9 +1,9 @@
-python evaluate.py --model ORIG_STG2 --experiment adam_trueWD_cyclical \
-	--loadPath ORIG_STG2_adam_trueWD_cyclical \
-	--chunkSize 32 --batchSize 32 \
-	--gpu 0
-
-# python evaluate.py --model ORIG_STG2 --experiment sgd_trueWD_restart \
+# python evaluate.py --model ORIG_STG2 --experiment adam_trueWD_cyclical \
 # 	--loadPath ORIG_STG2_adam_trueWD_cyclical \
 # 	--chunkSize 32 --batchSize 32 \
 # 	--gpu 0
+
+python evaluate.py --model ORIG_STG2 --experiment sgd_trueWD_restart \
+	--loadPath ORIG_STG2_sgd_trueWD_restart \
+	--chunkSize 32 --batchSize 32 \
+	--gpu 0

scripts/evaluate_dist.sh

@@ -1,8 +1,13 @@
-python evaluate_dist.py --model ORIG_STG2 --experiment adam_trueWD_cyclical \
-	--loadPath ORIG_STG2_adam_trueWD_cyclical \
-	--chunkSize 32 --batchSize 32 
-
-# python evaluate.py --model ORIG_STG2 --experiment sgd_trueWD_restart \
+# python evaluate_dist.py --model ORIG_STG2 --experiment adam_trueWD_cyclical \
 # 	--loadPath ORIG_STG2_adam_trueWD_cyclical \
+# 	--chunkSize 32 --batchSize 32 
+
+# python evaluate_dist.py --model ORIG_STG2 --experiment sgd_trueWD_restart \
+# 	--loadPath ORIG_STG2_sgd_trueWD_restart \
 # 	--chunkSize 32 --batchSize 32 \
 # 	--gpu 0
+
+python evaluate_dist.py --model ORIG_STG2 --experiment orig_tf \
+	--loadPath "~/3D-point-cloud-generation/results_0/orig-ft_it100000" \
+	--chunkSize 32 --batchSize 32 \
+	--gpu 1

scripts/train_stg2.sh

@@ -27,9 +27,17 @@
 # 	--lrSched restart --T_0 5 --T_mult 2 --lrBase 5e-2 \
 # 	--gpu 0
 
-python train_stg2.py --model ORIG_STG2 --experiment sgd_trueWD_restart \
-	--loadPath ORIG_STG1_sgd_trueWD_restart \
-	--chunkSize 32 --batchSize 32 \
+# python train_stg2.py --model ORIG_STG2 --experiment sgd_trueWD_restart \
+# 	--loadPath ORIG_STG1_sgd_trueWD_restart \
+# 	--chunkSize 32 --batchSize 32 \
+# 	--optim sgd --trueWD 1e-4 --lr 1e-1 \
+# 	--lrSched restart --T_0 5 --T_mult 2 --lrBase 5e-3 \
+# 	--gpu 1
+
+# Continue training
+python train_stg2.py --model ORIG_STG2 --experiment sgd_trueWD_restart_cont \
+	--loadPath ORIG_STG2_sgd_trueWD_restart_cont \
+	--chunkSize 32 --batchSize 32 --saveEpoch 10 \
 	--optim sgd --trueWD 1e-4 --lr 1e-1 \
 	--lrSched restart --T_0 5 --T_mult 2 --lrBase 5e-3 \
 	--gpu 1

utils.py

@@ -165,7 +165,7 @@ def save_best_model(model_path, model, df_hist):
         torch.save(model.state_dict(), f"{model_path}/best.pth")
 
 def checkpoint_model(model_path, model, epoch, saveEpoch):
-    if (saveEpoch is not None) and ((epoch+1) % saveEpoch == 0):
+    if (saveEpoch is not None) and (epoch % saveEpoch == 0):
         torch.save(model.state_dict(), f"{model_path}/{epoch}.pth")