Update run_checkpoints.sh

run_checkpoints.sh

@@ -18,15 +18,17 @@ for model_name in "${model_names[@]}"; do
     model_path="${model_dir}/${checkpoint}"
     checkpoint_num=$(echo $checkpoint | cut -d'-' -f2)
     echo "Running model ${model_name} checkpoint ${checkpoint_num}"
-    python -W ignore main.py \
-      -db postgres \
-      -q data/instruct_basic_postgres.csv data/instruct_advanced_postgres.csv data/questions_gen_postgres.csv \
-      -o "results/${model_name}/c${checkpoint_num}_basic_api_a100.csv" "results/${model_name}/c${checkpoint_num}_advanced_api_a100.csv" "results/${model_name}/c${checkpoint_num}_v1_api_a100.csv" \
-      -g vllm \
-      -m "$model_path" \
-      -c 0 \
-      -b 1 \
-      -bs 10 \
-      -f prompts/prompt.md
+    # first, get the API up
+    python3 utils/api_server.py --model "$model_path" --tensor-parallel-size 1 --dtype float16 --max-model-len 8192 --gpu-memory-utilization 0.90 --block-size 16 --disable-log-requests --port 8080 && python main.py \  
+  -db postgres \
+  -q "data/questions_gen_postgres.csv" "data/instruct_basic_postgres.csv" "data/instruct_advanced_postgres.csv" \
+  -o "results/${model_name}/c${checkpoint_num}_v1_api.csv" "results/${model_name}/c${checkpoint_num}_v1_basic.csv" "results/${model_name}/c${checkpoint_num}_v1_advanced.csv"
+  -g api \
+  -b 1 \
+  -f prompts/prompt.md \
+  --api_url "http://localhost:8080/generate" \
+  --api_type "vllm" \
+  -p 10 \
+  -c 0
   done
 done