Skip to content

Commit 962462d

Browse files
Dando18Dando18
authored
Add magicoder (#20)
add magicoder outputs
1 parent 244bb43 commit 962462d

31 files changed

+595697
-60
lines changed

‎analysis/all-metrics.sh

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@
88
#python metrics.py ../results/a8724ee8/phind-v2_prompted_temp0.2/results.csv --model-name Phind-V2 --output ../results/a8724ee8/phind-v2_prompted_temp0.2/metrics.csv
99
#python metrics.py ../results/a8724ee8/gpt-3.5_temp0.2/results.csv --model-name GPT-3.5 --output ../results/a8724ee8/gpt-3.5_temp0.2/metrics.csv
1010
#python metrics.py ../results/a8724ee8/gpt-4_temp0.2/results.csv --model-name GPT-4 --output ../results/a8724ee8/gpt-4_temp0.2/metrics.csv
11-
python metrics.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro --output ../results/015cff6f/gemini-pro_temp0.2/metrics.csv
11+
#python metrics.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro --output ../results/015cff6f/gemini-pro_temp0.2/metrics.csv
12+
python metrics.py ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/results.csv --model-name Magicoder-S-DS --output ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/metrics.csv
1213
#(cd ../results/a8724ee8 && head -n 1 codellama-7b-hf_prompted_temp0.2/metrics.csv > all-metrics.csv && tail -n+2 -q */metrics.csv >> all-metrics.csv)
1314

1415
# mpi scaling metrics
@@ -19,7 +20,8 @@ python metrics.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-nam
1920
#python metrics-scaling.py ../results/a8724ee8/phind-v2_prompted_temp0.2/results.csv --model-name Phind-V2 -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/a8724ee8/phind-v2_prompted_temp0.2/metrics-scaling-mpi.csv
2021
#python metrics-scaling.py ../results/a8724ee8/gpt-3.5_temp0.2/results.csv --model-name GPT-3.5 -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/a8724ee8/gpt-3.5_temp0.2/metrics-scaling-mpi.csv
2122
#python metrics-scaling.py ../results/a8724ee8/gpt-4_temp0.2/results.csv --model-name GPT-4 -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/a8724ee8/gpt-4_temp0.2/metrics-scaling-mpi.csv
22-
python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/015cff6f/gemini-pro_temp0.2/metrics-scaling-mpi.csv
23+
#python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/015cff6f/gemini-pro_temp0.2/metrics-scaling-mpi.csv
24+
python metrics-scaling.py ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/results.csv --model-name Magicoder-S-DS -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/metrics-scaling-mpi.csv
2325
#(cd ../results/a8724ee8 && head -n 1 codellama-7b-hf_prompted_temp0.2/metrics-scaling-mpi.csv > all-metrics-scaling-mpi.csv && tail -n+2 -q */metrics-scaling-mpi.csv >> all-metrics-scaling-mpi.csv)
2426

2527
# omp scaling metrics
@@ -30,7 +32,8 @@ python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --m
3032
#python metrics-scaling.py ../results/a8724ee8/phind-v2_prompted_temp0.2/results.csv --model-name Phind-V2 -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/a8724ee8/phind-v2_prompted_temp0.2/metrics-scaling-omp.csv
3133
#python metrics-scaling.py ../results/a8724ee8/gpt-3.5_temp0.2/results.csv --model-name GPT-3.5 -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/a8724ee8/gpt-3.5_temp0.2/metrics-scaling-omp.csv
3234
#python metrics-scaling.py ../results/a8724ee8/gpt-4_temp0.2/results.csv --model-name GPT-4 -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/a8724ee8/gpt-4_temp0.2/metrics-scaling-omp.csv
33-
python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/015cff6f/gemini-pro_temp0.2/metrics-scaling-omp.csv
35+
#python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/015cff6f/gemini-pro_temp0.2/metrics-scaling-omp.csv
36+
python metrics-scaling.py ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/results.csv --model-name Magicoder-S-DS -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/metrics-scaling-omp.csv
3437
#(cd ../results/a8724ee8 && head -n 1 codellama-7b-hf_prompted_temp0.2/metrics-scaling-omp.csv > all-metrics-scaling-omp.csv && tail -n+2 -q */metrics-scaling-omp.csv >> all-metrics-scaling-omp.csv)
3538

3639
# kokkos scaling metrics
@@ -41,5 +44,6 @@ python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --m
4144
#python metrics-scaling.py ../results/a8724ee8/phind-v2_prompted_temp0.2/results.csv --model-name Phind-V2 -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/a8724ee8/phind-v2_prompted_temp0.2/metrics-scaling-kokkos.csv
4245
#python metrics-scaling.py ../results/a8724ee8/gpt-3.5_temp0.2/results.csv --model-name GPT-3.5 -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/a8724ee8/gpt-3.5_temp0.2/metrics-scaling-kokkos.csv
4346
#python metrics-scaling.py ../results/a8724ee8/gpt-4_temp0.2/results.csv --model-name GPT-4 -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/a8724ee8/gpt-4_temp0.2/metrics-scaling-kokkos.csv
44-
python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/015cff6f/gemini-pro_temp0.2/metrics-scaling-kokkos.csv
47+
#python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/015cff6f/gemini-pro_temp0.2/metrics-scaling-kokkos.csv
48+
python metrics-scaling.py ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/results.csv --model-name Magicoder-S-DS -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/metrics-scaling-kokkos.csv
4549
#(cd ../results/a8724ee8 && head -n 1 codellama-7b-hf_prompted_temp0.2/metrics-scaling-kokkos.csv > all-metrics-scaling-kokkos.csv && tail -n+2 -q */metrics-scaling-kokkos.csv >> all-metrics-scaling-kokkos.csv)

‎drivers/cpp/cpp_driver_wrapper.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,9 @@ def run(self, executable: PathLike, **run_config) -> RunOutput:
106106
run_process = run_command(launch_cmd, timeout=self.run_timeout, dry=self.dry)
107107
except subprocess.TimeoutExpired as e:
108108
return RunOutput(-1, str(e.stdout), f"[Timeout] {str(e.stderr)}", config=run_config)
109+
except UnicodeDecodeError as e:
110+
logging.warning(f"UnicodeDecodeError: {str(e)}\nRunnning command: {launch_cmd}")
111+
return RunOutput(-1, "", f"UnicodeDecodeError: {str(e)}", config=run_config)
109112
return RunOutput(run_process.returncode, run_process.stdout, run_process.stderr, config=run_config)
110113

111114
def test_single_output(self, prompt: str, output: str, test_driver_file: PathLike, problem_size: str) -> GeneratedTextResult:

‎drivers/driver_wrapper.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ def __init__(self, exit_code: int, stdout: str, stderr: str, config: dict = {}):
4646
logging.warning(f"Runtime is 0 for run with config {self.config}. Try increasing the problem size.")
4747
if self.is_valid and self.best_sequential_runtime == 0:
4848
logging.warning(f"The best sequential runtime is 0 for run with config {self.config}. Try increasing the problem size.")
49-
if self.is_valid and self.best_sequential_runtime < 0.001:
49+
if self.is_valid and self.best_sequential_runtime and self.best_sequential_runtime < 0.001:
5050
logging.warning(f"The best sequential runtime is very small ({self.best_sequential_runtime}) for run with config {self.config}. Try increasing the problem size.")
5151

5252
def __repr__(self) -> str:

‎drivers/run-cuda.sbatch

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,12 @@
44
#SBATCH --gpus=a100:1
55
#SBATCH -t 05:00:00
66
#SBATCH -A bhatele-lab-cmsc
7-
#SBATCH -J cuda-gemini-pro
8-
#SBATCH -o run-outputs/gemini-pro-cuda-%A.out
7+
#SBATCH -J cuda-magicoder-s-ds-6.7b
8+
#SBATCH -o run-outputs/magicoder-s-ds-6.7b-cuda-%A.out
99

1010
# settings
1111
HASH="015cff6f"
12-
MODEL="gemini-pro"
12+
MODEL="magicoder-s-ds-6.7b"
1313
TEMP="0.2"
1414

1515
GENERATED_PROMPTS="../results/${HASH}/${MODEL}_temp${TEMP}/all.json"

‎drivers/run-hip.sbatch

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
#!/bin/bash
22
#SBATCH -N 1
33
#SBATCH -t 08:00:00
4-
#SBATCH -J hip-gemini-pro
5-
#SBATCH -o run-outputs/gemini-pro-hip-{{id}}.out
4+
#SBATCH -J hip-magicoder-s-ds-6.7b
5+
#SBATCH -o run-outputs/magicoder-s-ds-6.7b-hip-{{id}}.out
66

77
# settings
88
HASH="015cff6f"
9-
MODEL="gemini-pro"
9+
MODEL="magicoder-s-ds-6.7b"
1010
TEMP="0.2"
1111

1212
GENERATED_PROMPTS="../results/${HASH}/${MODEL}_temp${TEMP}/all.json"

‎drivers/run-kokkos.sbatch

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,12 @@
44
#SBATCH -p serial
55
#SBATCH -t 05:00:00
66
#SBATCH -A bhatele-lab-cmsc
7-
#SBATCH -J kokkos-gemini-pro
8-
#SBATCH -o run-outputs/gemini-pro-kokkos-%A.out
7+
#SBATCH -J kokkos-magicoder-s-ds-6.7b
8+
#SBATCH -o run-outputs/magicoder-s-ds-6.7b-kokkos-%A.out
99

1010
# settings
1111
HASH="015cff6f"
12-
MODEL="gemini-pro"
12+
MODEL="magicoder-s-ds-6.7b"
1313
TEMP="0.2"
1414

1515
GENERATED_PROMPTS="../results/${HASH}/${MODEL}_temp${TEMP}/all.json"

‎drivers/run-mpi+omp.sbatch

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,14 @@
22
#SBATCH -N 4
33
#SBATCH --exclusive
44
#SBATCH -t 05:00:00
5+
#SBATCH -t 05:00:00
56
#SBATCH -A bhatele-lab-cmsc
6-
#SBATCH -J mpi+omp-gemini-pro
7-
#SBATCH -o run-outputs/gemini-pro-mpi+omp-%A.out
7+
#SBATCH -J mpi+omp-magicoder-s-ds-6.7b
8+
#SBATCH -o run-outputs/magicoder-s-ds-6.7b-mpi+omp-%A.out
89

910
# settings
1011
HASH="015cff6f"
11-
MODEL="gemini-pro"
12+
MODEL="magicoder-s-ds-6.7b"
1213
TEMP="0.2"
1314

1415
GENERATED_PROMPTS="../results/${HASH}/${MODEL}_temp${TEMP}/all.json"

‎drivers/run-mpi.sbatch

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,12 @@
22
#SBATCH -n 512
33
#SBATCH -t 04:00:00
44
#SBATCH -A bhatele-lab-cmsc
5-
#SBATCH -J mpi-gemini-pro
6-
#SBATCH -o run-outputs/gemini-pro-mpi-%A.out
5+
#SBATCH -J mpi-magicoder-s-ds-6.7b
6+
#SBATCH -o run-outputs/magicoder-s-ds-6.7b-mpi-%A.out
77

88
# settings
99
HASH="015cff6f"
10-
MODEL="gemini-pro"
10+
MODEL="magicoder-s-ds-6.7b"
1111
TEMP="0.2"
1212

1313
GENERATED_PROMPTS="../results/${HASH}/${MODEL}_temp${TEMP}/all.json"
@@ -19,7 +19,7 @@ ml python gcc openmpi
1919
export OMPI_MCA_opal_warn_on_missing_libcuda=0
2020

2121
python run-all.py \
22-
$OUTPUT \
22+
$GENERATED_PROMPTS \
2323
--output $OUTPUT \
2424
--scratch-dir $SCRATCH_DIR \
2525
--launch-configs launch-configs.json \

‎drivers/run-omp.sbatch

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,14 @@
33
#SBATCH --exclusive
44
#SBATCH -p serial
55
#SBATCH -t 05:00:00
6+
#SBATCH -t 05:00:00
67
#SBATCH -A bhatele-lab-cmsc
7-
#SBATCH -J omp-gemini-pro
8-
#SBATCH -o run-outputs/gemini-pro-omp-%A.out
8+
#SBATCH -J omp-magicoder-s-ds-6.7b
9+
#SBATCH -o run-outputs/magicoder-s-ds-6.7b-omp-%A.out
910

1011
# settings
1112
HASH="015cff6f"
12-
MODEL="gemini-pro"
13+
MODEL="magicoder-s-ds-6.7b"
1314
TEMP="0.2"
1415

1516
GENERATED_PROMPTS="../results/${HASH}/${MODEL}_temp${TEMP}/all.json"

‎drivers/run-serial.sbatch

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,14 @@
33
#SBATCH --exclusive
44
#SBATCH -p serial
55
#SBATCH -t 04:00:00
6+
#SBATCH -t 04:00:00
67
#SBATCH -A bhatele-lab-cmsc
7-
#SBATCH -J serial-gemini-pro
8-
#SBATCH -o run-outputs/gemini-pro-serial-%A.out
8+
#SBATCH -J serial-magicoder-s-ds-6.7b
9+
#SBATCH -o run-outputs/magicoder-s-ds-6.7b-serial-%A.out
910

1011
# settings
1112
HASH="015cff6f"
12-
MODEL="gemini-pro"
13+
MODEL="magicoder-s-ds-6.7b"
1314
TEMP="0.2"
1415

1516
GENERATED_PROMPTS="../results/${HASH}/${MODEL}_temp${TEMP}/all.json"

‎generate/generate-magicoder-s-ds-prompted.sbatch

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#!/bin/bash
2+
#SBATCH -n 1
3+
#SBATCH -c 4
4+
#SBATCH --ntasks-per-node=1
5+
#SBATCH --gpus-per-task=1
6+
#SBATCH --mem=128000
7+
#SBATCH -t 05:00:00
8+
#SBATCH -A m2404
9+
#SBATCH -C gpu&hbm80g
10+
#SBATCH -q regular
11+
#SBATCH -J generate-magicoder-s-ds-prompted
12+
#SBATCH -o generation-job-logs/generate-magicoder-s-ds-prompted-%A.out
13+
14+
# settings
15+
MODEL="ise-uiuc/Magicoder-S-DS-6.7B"
16+
TEMP=0.2
17+
TOPP=0.95
18+
MAX_NEW_TKNS=2048
19+
SAMPLES_PER_PROMPT=20
20+
BATCH_SIZE=16
21+
hash=$(md5sum ../prompts/generation-prompts.json | cut -d' ' -f1)
22+
OUTPUT="../outputs/output_${hash:0:8}_${MODEL//\//--}_prompted_temp${TEMP}.json"
23+
CACHE="../outputs/cache/cache_${hash:0:8}_${MODEL//\//--}_prompted_temp${TEMP}.jsonl"
24+
echo "Writing to $OUTPUT"
25+
echo "model=$MODEL MAX_NEW_TKNS=$MAX_NEW_TKNS SAMPLES_PER_PROMPT=$SAMPLES_PER_PROMPT BATCH_SIZE=$BATCH_SIZE"
26+
27+
# setup
28+
#ml cuda/11.8.0
29+
source .env/bin/activate
30+
export HF_HOME=/pscratch/sd/d/dnicho/.cache/huggingface
31+
export OMP_NUM_THREADS=4
32+
33+
# generate
34+
srun python generate.py \
35+
--model $MODEL \
36+
--prompts ../prompts/generation-prompts.json \
37+
--output $OUTPUT \
38+
--cache $CACHE \
39+
--temperature $TEMP \
40+
--top_p $TOPP \
41+
--do_sample \
42+
--max_new_tokens $MAX_NEW_TKNS \
43+
--num_samples_per_prompt $SAMPLES_PER_PROMPT \
44+
--batch_size $BATCH_SIZE \
45+
--prompted

‎generate/generate.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@
114114
)
115115

116116
""" Iterate over prompts and generate code """
117-
if not args.restart and args.cache is not None:
117+
if not args.restart and args.cache is not None and os.path.exists(args.cache):
118118
with open(args.cache, 'r') as jsonl_file:
119119
responses = [json.loads(line) for line in jsonl_file]
120120
responses = [r for r in responses if r["temperature"] == args.temperature and r["prompted"] == args.prompted
@@ -133,7 +133,7 @@
133133
prompt_str = cur_prompt["prompt"]
134134

135135
total_tokens += len(generator.tokenizer.encode(output[0]["generated_text"]))
136-
cleaned_output = clean_output(output[0]["generated_text"], prompt_str)
136+
cleaned_output = inference_config.clean_output(output[0]["generated_text"], prompt_str)
137137
cur_prompt["outputs"].append(cleaned_output)
138138

139139
if idx % args.num_samples_per_prompt == args.num_samples_per_prompt - 1:

0 commit comments

Comments
 (0)