Add magicoder

analysis/all-metrics.sh

@@ -8,7 +8,8 @@
 #python metrics.py ../results/a8724ee8/phind-v2_prompted_temp0.2/results.csv --model-name Phind-V2 --output ../results/a8724ee8/phind-v2_prompted_temp0.2/metrics.csv
 #python metrics.py ../results/a8724ee8/gpt-3.5_temp0.2/results.csv --model-name GPT-3.5 --output ../results/a8724ee8/gpt-3.5_temp0.2/metrics.csv
 #python metrics.py ../results/a8724ee8/gpt-4_temp0.2/results.csv --model-name GPT-4 --output ../results/a8724ee8/gpt-4_temp0.2/metrics.csv
-python metrics.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro --output ../results/015cff6f/gemini-pro_temp0.2/metrics.csv
+#python metrics.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro --output ../results/015cff6f/gemini-pro_temp0.2/metrics.csv
+python metrics.py ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/results.csv --model-name Magicoder-S-DS --output ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/metrics.csv
 #(cd ../results/a8724ee8 && head -n 1 codellama-7b-hf_prompted_temp0.2/metrics.csv > all-metrics.csv && tail -n+2 -q */metrics.csv >> all-metrics.csv)
 
 # mpi scaling metrics
@@ -19,7 +20,8 @@ python metrics.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-nam
 #python metrics-scaling.py ../results/a8724ee8/phind-v2_prompted_temp0.2/results.csv --model-name Phind-V2 -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/a8724ee8/phind-v2_prompted_temp0.2/metrics-scaling-mpi.csv
 #python metrics-scaling.py ../results/a8724ee8/gpt-3.5_temp0.2/results.csv --model-name GPT-3.5 -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/a8724ee8/gpt-3.5_temp0.2/metrics-scaling-mpi.csv
 #python metrics-scaling.py ../results/a8724ee8/gpt-4_temp0.2/results.csv --model-name GPT-4 -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/a8724ee8/gpt-4_temp0.2/metrics-scaling-mpi.csv
-python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/015cff6f/gemini-pro_temp0.2/metrics-scaling-mpi.csv
+#python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/015cff6f/gemini-pro_temp0.2/metrics-scaling-mpi.csv
+python metrics-scaling.py ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/results.csv --model-name Magicoder-S-DS -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/metrics-scaling-mpi.csv
 #(cd ../results/a8724ee8 && head -n 1 codellama-7b-hf_prompted_temp0.2/metrics-scaling-mpi.csv > all-metrics-scaling-mpi.csv && tail -n+2 -q */metrics-scaling-mpi.csv >> all-metrics-scaling-mpi.csv)
 
 # omp scaling metrics
@@ -30,7 +32,8 @@ python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --m
 #python metrics-scaling.py ../results/a8724ee8/phind-v2_prompted_temp0.2/results.csv --model-name Phind-V2 -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/a8724ee8/phind-v2_prompted_temp0.2/metrics-scaling-omp.csv
 #python metrics-scaling.py ../results/a8724ee8/gpt-3.5_temp0.2/results.csv --model-name GPT-3.5 -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/a8724ee8/gpt-3.5_temp0.2/metrics-scaling-omp.csv
 #python metrics-scaling.py ../results/a8724ee8/gpt-4_temp0.2/results.csv --model-name GPT-4 -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/a8724ee8/gpt-4_temp0.2/metrics-scaling-omp.csv
-python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/015cff6f/gemini-pro_temp0.2/metrics-scaling-omp.csv
+#python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/015cff6f/gemini-pro_temp0.2/metrics-scaling-omp.csv
+python metrics-scaling.py ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/results.csv --model-name Magicoder-S-DS -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/metrics-scaling-omp.csv
 #(cd ../results/a8724ee8 && head -n 1 codellama-7b-hf_prompted_temp0.2/metrics-scaling-omp.csv > all-metrics-scaling-omp.csv && tail -n+2 -q */metrics-scaling-omp.csv >> all-metrics-scaling-omp.csv)
 
 # kokkos scaling metrics
@@ -41,5 +44,6 @@ python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --m
 #python metrics-scaling.py ../results/a8724ee8/phind-v2_prompted_temp0.2/results.csv --model-name Phind-V2 -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/a8724ee8/phind-v2_prompted_temp0.2/metrics-scaling-kokkos.csv
 #python metrics-scaling.py ../results/a8724ee8/gpt-3.5_temp0.2/results.csv --model-name GPT-3.5 -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/a8724ee8/gpt-3.5_temp0.2/metrics-scaling-kokkos.csv
 #python metrics-scaling.py ../results/a8724ee8/gpt-4_temp0.2/results.csv --model-name GPT-4 -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/a8724ee8/gpt-4_temp0.2/metrics-scaling-kokkos.csv
-python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/015cff6f/gemini-pro_temp0.2/metrics-scaling-kokkos.csv
+#python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/015cff6f/gemini-pro_temp0.2/metrics-scaling-kokkos.csv
+python metrics-scaling.py ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/results.csv --model-name Magicoder-S-DS -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/metrics-scaling-kokkos.csv
 #(cd ../results/a8724ee8 && head -n 1 codellama-7b-hf_prompted_temp0.2/metrics-scaling-kokkos.csv > all-metrics-scaling-kokkos.csv && tail -n+2 -q */metrics-scaling-kokkos.csv >> all-metrics-scaling-kokkos.csv)

drivers/cpp/cpp_driver_wrapper.py

@@ -106,6 +106,9 @@ def run(self, executable: PathLike, **run_config) -> RunOutput:
             run_process = run_command(launch_cmd, timeout=self.run_timeout, dry=self.dry)
         except subprocess.TimeoutExpired as e:
             return RunOutput(-1, str(e.stdout), f"[Timeout] {str(e.stderr)}", config=run_config)
+        except UnicodeDecodeError as e:
+            logging.warning(f"UnicodeDecodeError: {str(e)}\nRunnning command: {launch_cmd}")
+            return RunOutput(-1, "", f"UnicodeDecodeError: {str(e)}", config=run_config)
         return RunOutput(run_process.returncode, run_process.stdout, run_process.stderr, config=run_config)
 
     def test_single_output(self, prompt: str, output: str, test_driver_file: PathLike, problem_size: str) -> GeneratedTextResult:

drivers/driver_wrapper.py

@@ -46,7 +46,7 @@ def __init__(self, exit_code: int, stdout: str, stderr: str, config: dict = {}):
             logging.warning(f"Runtime is 0 for run with config {self.config}. Try increasing the problem size.")
         if self.is_valid and self.best_sequential_runtime == 0:
             logging.warning(f"The best sequential runtime is 0 for run with config {self.config}. Try increasing the problem size.")
-        if self.is_valid and self.best_sequential_runtime < 0.001:
+        if self.is_valid and self.best_sequential_runtime and self.best_sequential_runtime < 0.001:
             logging.warning(f"The best sequential runtime is very small ({self.best_sequential_runtime}) for run with config {self.config}. Try increasing the problem size.")
 
     def __repr__(self) -> str:

drivers/run-cuda.sbatch

@@ -4,12 +4,12 @@
 #SBATCH --gpus=a100:1
 #SBATCH -t 05:00:00
 #SBATCH -A bhatele-lab-cmsc
-#SBATCH -J cuda-gemini-pro
-#SBATCH -o run-outputs/gemini-pro-cuda-%A.out
+#SBATCH -J cuda-magicoder-s-ds-6.7b
+#SBATCH -o run-outputs/magicoder-s-ds-6.7b-cuda-%A.out
 
 # settings
 HASH="015cff6f"
-MODEL="gemini-pro"
+MODEL="magicoder-s-ds-6.7b"
 TEMP="0.2"
 
 GENERATED_PROMPTS="../results/${HASH}/${MODEL}_temp${TEMP}/all.json"

drivers/run-hip.sbatch

@@ -1,12 +1,12 @@
 #!/bin/bash
 #SBATCH -N 1
 #SBATCH -t 08:00:00
-#SBATCH -J hip-gemini-pro
-#SBATCH -o run-outputs/gemini-pro-hip-{{id}}.out
+#SBATCH -J hip-magicoder-s-ds-6.7b
+#SBATCH -o run-outputs/magicoder-s-ds-6.7b-hip-{{id}}.out
 
 # settings
 HASH="015cff6f"
-MODEL="gemini-pro"
+MODEL="magicoder-s-ds-6.7b"
 TEMP="0.2"
 
 GENERATED_PROMPTS="../results/${HASH}/${MODEL}_temp${TEMP}/all.json"

drivers/run-kokkos.sbatch

@@ -4,12 +4,12 @@
 #SBATCH -p serial
 #SBATCH -t 05:00:00
 #SBATCH -A bhatele-lab-cmsc
-#SBATCH -J kokkos-gemini-pro
-#SBATCH -o run-outputs/gemini-pro-kokkos-%A.out
+#SBATCH -J kokkos-magicoder-s-ds-6.7b
+#SBATCH -o run-outputs/magicoder-s-ds-6.7b-kokkos-%A.out
 
 # settings
 HASH="015cff6f"
-MODEL="gemini-pro"
+MODEL="magicoder-s-ds-6.7b"
 TEMP="0.2"
 
 GENERATED_PROMPTS="../results/${HASH}/${MODEL}_temp${TEMP}/all.json"

drivers/run-mpi+omp.sbatch

@@ -2,13 +2,14 @@
 #SBATCH -N 4
 #SBATCH --exclusive
 #SBATCH -t 05:00:00
+#SBATCH -t 05:00:00
 #SBATCH -A bhatele-lab-cmsc
-#SBATCH -J mpi+omp-gemini-pro
-#SBATCH -o run-outputs/gemini-pro-mpi+omp-%A.out
+#SBATCH -J mpi+omp-magicoder-s-ds-6.7b
+#SBATCH -o run-outputs/magicoder-s-ds-6.7b-mpi+omp-%A.out
 
 # settings
 HASH="015cff6f"
-MODEL="gemini-pro"
+MODEL="magicoder-s-ds-6.7b"
 TEMP="0.2"
 
 GENERATED_PROMPTS="../results/${HASH}/${MODEL}_temp${TEMP}/all.json"

drivers/run-mpi.sbatch

@@ -2,12 +2,12 @@
 #SBATCH -n 512
 #SBATCH -t 04:00:00
 #SBATCH -A bhatele-lab-cmsc
-#SBATCH -J mpi-gemini-pro
-#SBATCH -o run-outputs/gemini-pro-mpi-%A.out
+#SBATCH -J mpi-magicoder-s-ds-6.7b
+#SBATCH -o run-outputs/magicoder-s-ds-6.7b-mpi-%A.out
 
 # settings
 HASH="015cff6f"
-MODEL="gemini-pro"
+MODEL="magicoder-s-ds-6.7b"
 TEMP="0.2"
 
 GENERATED_PROMPTS="../results/${HASH}/${MODEL}_temp${TEMP}/all.json"
@@ -19,7 +19,7 @@ ml python gcc openmpi
 export OMPI_MCA_opal_warn_on_missing_libcuda=0
 
 python run-all.py \
-    $OUTPUT \
+    $GENERATED_PROMPTS \
     --output $OUTPUT \
     --scratch-dir $SCRATCH_DIR \
     --launch-configs launch-configs.json \

drivers/run-omp.sbatch

@@ -3,13 +3,14 @@
 #SBATCH --exclusive
 #SBATCH -p serial
 #SBATCH -t 05:00:00
+#SBATCH -t 05:00:00
 #SBATCH -A bhatele-lab-cmsc
-#SBATCH -J omp-gemini-pro
-#SBATCH -o run-outputs/gemini-pro-omp-%A.out
+#SBATCH -J omp-magicoder-s-ds-6.7b
+#SBATCH -o run-outputs/magicoder-s-ds-6.7b-omp-%A.out
 
 # settings
 HASH="015cff6f"
-MODEL="gemini-pro"
+MODEL="magicoder-s-ds-6.7b"
 TEMP="0.2"
 
 GENERATED_PROMPTS="../results/${HASH}/${MODEL}_temp${TEMP}/all.json"

drivers/run-serial.sbatch

@@ -3,13 +3,14 @@
 #SBATCH --exclusive
 #SBATCH -p serial
 #SBATCH -t 04:00:00
+#SBATCH -t 04:00:00
 #SBATCH -A bhatele-lab-cmsc
-#SBATCH -J serial-gemini-pro
-#SBATCH -o run-outputs/gemini-pro-serial-%A.out
+#SBATCH -J serial-magicoder-s-ds-6.7b
+#SBATCH -o run-outputs/magicoder-s-ds-6.7b-serial-%A.out
 
 # settings
 HASH="015cff6f"
-MODEL="gemini-pro"
+MODEL="magicoder-s-ds-6.7b"
 TEMP="0.2"
 
 GENERATED_PROMPTS="../results/${HASH}/${MODEL}_temp${TEMP}/all.json"

generate/generate-magicoder-s-ds-prompted.sbatch

@@ -0,0 +1,45 @@
+#!/bin/bash
+#SBATCH -n 1
+#SBATCH -c 4
+#SBATCH --ntasks-per-node=1
+#SBATCH --gpus-per-task=1
+#SBATCH --mem=128000
+#SBATCH -t 05:00:00
+#SBATCH -A m2404
+#SBATCH -C gpu&hbm80g
+#SBATCH -q regular
+#SBATCH -J generate-magicoder-s-ds-prompted
+#SBATCH -o generation-job-logs/generate-magicoder-s-ds-prompted-%A.out
+
+# settings
+MODEL="ise-uiuc/Magicoder-S-DS-6.7B"
+TEMP=0.2
+TOPP=0.95
+MAX_NEW_TKNS=2048
+SAMPLES_PER_PROMPT=20
+BATCH_SIZE=16
+hash=$(md5sum ../prompts/generation-prompts.json | cut -d' ' -f1)
+OUTPUT="../outputs/output_${hash:0:8}_${MODEL//\//--}_prompted_temp${TEMP}.json"
+CACHE="../outputs/cache/cache_${hash:0:8}_${MODEL//\//--}_prompted_temp${TEMP}.jsonl"
+echo "Writing to $OUTPUT"
+echo "model=$MODEL   MAX_NEW_TKNS=$MAX_NEW_TKNS   SAMPLES_PER_PROMPT=$SAMPLES_PER_PROMPT   BATCH_SIZE=$BATCH_SIZE"
+
+# setup
+#ml cuda/11.8.0
+source .env/bin/activate
+export HF_HOME=/pscratch/sd/d/dnicho/.cache/huggingface
+export OMP_NUM_THREADS=4
+
+# generate
+srun python generate.py \
+    --model $MODEL \
+    --prompts ../prompts/generation-prompts.json \
+    --output $OUTPUT \
+    --cache $CACHE \
+    --temperature $TEMP \
+    --top_p $TOPP \
+    --do_sample \
+    --max_new_tokens $MAX_NEW_TKNS \
+    --num_samples_per_prompt $SAMPLES_PER_PROMPT \
+    --batch_size $BATCH_SIZE \
+    --prompted

generate/generate.py

@@ -114,7 +114,7 @@
 )
 
 """ Iterate over prompts and generate code """
-if not args.restart and args.cache is not None:
+if not args.restart and args.cache is not None and os.path.exists(args.cache):
     with open(args.cache, 'r') as jsonl_file:
         responses = [json.loads(line) for line in jsonl_file]
         responses = [r for r in responses if r["temperature"] == args.temperature and r["prompted"] == args.prompted
@@ -133,7 +133,7 @@
         prompt_str = cur_prompt["prompt"]
 
     total_tokens += len(generator.tokenizer.encode(output[0]["generated_text"]))
-    cleaned_output = clean_output(output[0]["generated_text"], prompt_str)
+    cleaned_output = inference_config.clean_output(output[0]["generated_text"], prompt_str)
     cur_prompt["outputs"].append(cleaned_output)
 
     if idx % args.num_samples_per_prompt == args.num_samples_per_prompt - 1: