magicoder metrics

Author: Dando18

analysis/all-metrics.sh

@@ -8,7 +8,8 @@
 #python metrics.py ../results/a8724ee8/phind-v2_prompted_temp0.2/results.csv --model-name Phind-V2 --output ../results/a8724ee8/phind-v2_prompted_temp0.2/metrics.csv
 #python metrics.py ../results/a8724ee8/gpt-3.5_temp0.2/results.csv --model-name GPT-3.5 --output ../results/a8724ee8/gpt-3.5_temp0.2/metrics.csv
 #python metrics.py ../results/a8724ee8/gpt-4_temp0.2/results.csv --model-name GPT-4 --output ../results/a8724ee8/gpt-4_temp0.2/metrics.csv
-python metrics.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro --output ../results/015cff6f/gemini-pro_temp0.2/metrics.csv
+#python metrics.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro --output ../results/015cff6f/gemini-pro_temp0.2/metrics.csv
+python metrics.py ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/results.csv --model-name Magicoder-S-DS --output ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/metrics.csv
 #(cd ../results/a8724ee8 && head -n 1 codellama-7b-hf_prompted_temp0.2/metrics.csv > all-metrics.csv && tail -n+2 -q */metrics.csv >> all-metrics.csv)
 
 # mpi scaling metrics
@@ -19,7 +20,8 @@ python metrics.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-nam
 #python metrics-scaling.py ../results/a8724ee8/phind-v2_prompted_temp0.2/results.csv --model-name Phind-V2 -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/a8724ee8/phind-v2_prompted_temp0.2/metrics-scaling-mpi.csv
 #python metrics-scaling.py ../results/a8724ee8/gpt-3.5_temp0.2/results.csv --model-name GPT-3.5 -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/a8724ee8/gpt-3.5_temp0.2/metrics-scaling-mpi.csv
 #python metrics-scaling.py ../results/a8724ee8/gpt-4_temp0.2/results.csv --model-name GPT-4 -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/a8724ee8/gpt-4_temp0.2/metrics-scaling-mpi.csv
-python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/015cff6f/gemini-pro_temp0.2/metrics-scaling-mpi.csv
+#python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/015cff6f/gemini-pro_temp0.2/metrics-scaling-mpi.csv
+python metrics-scaling.py ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/results.csv --model-name Magicoder-S-DS -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/metrics-scaling-mpi.csv
 #(cd ../results/a8724ee8 && head -n 1 codellama-7b-hf_prompted_temp0.2/metrics-scaling-mpi.csv > all-metrics-scaling-mpi.csv && tail -n+2 -q */metrics-scaling-mpi.csv >> all-metrics-scaling-mpi.csv)
 
 # omp scaling metrics
@@ -30,7 +32,8 @@ python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --m
 #python metrics-scaling.py ../results/a8724ee8/phind-v2_prompted_temp0.2/results.csv --model-name Phind-V2 -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/a8724ee8/phind-v2_prompted_temp0.2/metrics-scaling-omp.csv
 #python metrics-scaling.py ../results/a8724ee8/gpt-3.5_temp0.2/results.csv --model-name GPT-3.5 -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/a8724ee8/gpt-3.5_temp0.2/metrics-scaling-omp.csv
 #python metrics-scaling.py ../results/a8724ee8/gpt-4_temp0.2/results.csv --model-name GPT-4 -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/a8724ee8/gpt-4_temp0.2/metrics-scaling-omp.csv
-python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/015cff6f/gemini-pro_temp0.2/metrics-scaling-omp.csv
+#python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/015cff6f/gemini-pro_temp0.2/metrics-scaling-omp.csv
+python metrics-scaling.py ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/results.csv --model-name Magicoder-S-DS -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/metrics-scaling-omp.csv
 #(cd ../results/a8724ee8 && head -n 1 codellama-7b-hf_prompted_temp0.2/metrics-scaling-omp.csv > all-metrics-scaling-omp.csv && tail -n+2 -q */metrics-scaling-omp.csv >> all-metrics-scaling-omp.csv)
 
 # kokkos scaling metrics
@@ -41,5 +44,6 @@ python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --m
 #python metrics-scaling.py ../results/a8724ee8/phind-v2_prompted_temp0.2/results.csv --model-name Phind-V2 -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/a8724ee8/phind-v2_prompted_temp0.2/metrics-scaling-kokkos.csv
 #python metrics-scaling.py ../results/a8724ee8/gpt-3.5_temp0.2/results.csv --model-name GPT-3.5 -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/a8724ee8/gpt-3.5_temp0.2/metrics-scaling-kokkos.csv
 #python metrics-scaling.py ../results/a8724ee8/gpt-4_temp0.2/results.csv --model-name GPT-4 -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/a8724ee8/gpt-4_temp0.2/metrics-scaling-kokkos.csv
-python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/015cff6f/gemini-pro_temp0.2/metrics-scaling-kokkos.csv
+#python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/015cff6f/gemini-pro_temp0.2/metrics-scaling-kokkos.csv
+python metrics-scaling.py ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/results.csv --model-name Magicoder-S-DS -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/015cff6f/magicoder-s-ds-6.7b_temp0.2/metrics-scaling-kokkos.csv
 #(cd ../results/a8724ee8 && head -n 1 codellama-7b-hf_prompted_temp0.2/metrics-scaling-kokkos.csv > all-metrics-scaling-kokkos.csv && tail -n+2 -q */metrics-scaling-kokkos.csv >> all-metrics-scaling-kokkos.csv)

results/015cff6f/magicoder-s-ds-6.7b_temp0.2/metrics-scaling-kokkos.csv

@@ -0,0 +1,11 @@
+model,execution model,problem type,speedup_1@1,efficiency_1@1,speedup_2@1,efficiency_2@1,speedup_4@1,efficiency_4@1,speedup_8@1,efficiency_8@1,speedup_16@1,efficiency_16@1,speedup_32@1,efficiency_32@1
+Magicoder-S-DS,kokkos,dense_la,0.39106183585574383,0.39106183585574383,0.7899047737221508,0.3949523868610754,1.4938234435657416,0.3734558608914354,2.9651372147706776,0.3706421518463347,5.757547193868735,0.35984669961679594,10.68512060857563,0.33391001901798845
+Magicoder-S-DS,kokkos,geometry,0.2201953172941331,0.2201953172941331,0.2834083272103796,0.1417041636051898,0.46834308444558803,0.11708577111139701,0.842179955885303,0.10527249448566288,1.6516759672037034,0.10322974795023146,3.262063855463885,0.1019394954832464
+Magicoder-S-DS,kokkos,graph,50.36095628020942,50.36095628020942,60.02545891045853,30.012729455229266,59.13201356311438,14.783003390778594,61.058644118902954,7.632330514862869,58.52499703283761,3.6578123145523507,59.95297112491764,1.8735303476536762
+Magicoder-S-DS,kokkos,histogram,0.6294846188404293,0.6294846188404293,1.1346875795289515,0.5673437897644757,2.1539229426845035,0.5384807356711259,3.7668351154407813,0.47085438943009766,7.204953981570652,0.45030962384816575,9.46797648530848,0.29587426516589
+Magicoder-S-DS,kokkos,reduce,1.6580677763641496,1.6580677763641496,3.287842828135793,1.6439214140678966,6.50209251694157,1.6255231292353924,12.708704471595444,1.5885880589494306,24.42294944200172,1.5264343401251075,44.033568229119595,1.3760490071599873
+Magicoder-S-DS,kokkos,scan,0.923662632561,0.923662632561,1.8199674036240787,0.9099837018120394,3.4853761163317643,0.8713440290829411,5.958938551492954,0.7448673189366193,10.610163331796041,0.6631352082372526,15.670620270939763,0.4897068834668676
+Magicoder-S-DS,kokkos,search,0.2803052618291233,0.2803052618291233,0.559614668746061,0.2798073343730305,1.0869637040505062,0.27174092601262656,2.021152639624306,0.2526440799530382,3.8176944461004627,0.23860590288127892,5.613757780342226,0.17542993063569456
+Magicoder-S-DS,kokkos,sort,3.786284714326595,3.786284714326595,4.183879574023857,2.0919397870119285,4.444089129408004,1.111022282352001,4.570403182254889,0.5713003977818611,4.631718511686261,0.2894824069803913,4.580961189333846,0.14315503716668268
+Magicoder-S-DS,kokkos,stencil,0.42321453821324073,0.42321453821324073,0.8408494166293078,0.4204247083146539,1.6714336676258674,0.41785841690646686,3.3252138766364743,0.4156517345795593,6.6565207652228295,0.41603254782642685,13.354612445063513,0.41733163890823477
+Magicoder-S-DS,kokkos,transform,1.2523314446657041,1.2523314446657041,2.4848645538418044,1.2424322769209022,4.920660773191196,1.230165193297799,9.802673914685776,1.225334239335722,19.413856125946047,1.213366007871628,38.87091465972441,1.214716083116388

results/015cff6f/magicoder-s-ds-6.7b_temp0.2/metrics-scaling-mpi.csv

@@ -0,0 +1,13 @@
+model,execution model,problem type,speedup_1@1,efficiency_1@1,speedup_2@1,efficiency_2@1,speedup_4@1,efficiency_4@1,speedup_8@1,efficiency_8@1,speedup_16@1,efficiency_16@1,speedup_32@1,efficiency_32@1,speedup_64@1,efficiency_64@1,speedup_128@1,efficiency_128@1,speedup_256@1,efficiency_256@1,speedup_512@1,efficiency_512@1
+Magicoder-S-DS,mpi,dense_la,0.5058218682993209,0.5058218682993209,0.6423424356920785,0.32117121784603925,1.011039125427997,0.2527597813569992,2.726965656936606,0.34087070711707573,5.369141017133533,0.3355713135708458,10.402126001738486,0.3250664375543277,14.53478753237885,0.22710605519341953,14.44152078219993,0.11282438111093696,25.402843586598284,0.09922985776014955,40.16537980505637,0.07844800743175072
+Magicoder-S-DS,mpi,fft,0.922008550228447,0.922008550228447,0.8467626624307939,0.42338133121539695,0.8980000867267577,0.22450002168168942,0.9216802504799207,0.1152100313099901,0.8911802395942714,0.05569876497464196,0.8025972239250879,0.025081163247658998,0.829701659807939,0.012964088434499046,0.7561414806327932,0.005907355317443697,0.7194583644105186,0.002810384235978588,0.6173862735560207,0.001205832565539103
+Magicoder-S-DS,mpi,geometry,13.69937957700998,13.69937957700998,4.494174309855799,2.2470871549278995,7.119760998285893,1.7799402495714733,15.8699874831777,1.9837484353972126,20.70860545193749,1.2942878407460932,32.4572221276596,1.0142881914893624,56.360264751866566,0.8806291367479151,100.83503036204984,0.7877736747035143,179.69024785874205,0.7019150306982112,265.5441115350919,0.5186408428419763
+Magicoder-S-DS,mpi,graph,0.6024788215205545,0.6024788215205545,0.7999449465315039,0.39997247326575197,1.459625765584768,0.364906441396192,2.6290701561947416,0.3286337695243427,4.802700808639124,0.30016880053994527,8.624119618851855,0.2695037380891205,10.35825366867548,0.16184771357305439,31.87762406586661,0.24904393801458288,53.636082597761224,0.20951594764750478,96.99097552592332,0.18943549907406898
+Magicoder-S-DS,mpi,histogram,0.9674802433333337,0.9674802433333337,1.5507796600687263,0.7753898300343631,2.763606408555868,0.690901602138967,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+Magicoder-S-DS,mpi,reduce,0.7833078341893968,0.7833078341893968,0.9174562983189004,0.4587281491594502,1.5601322345309967,0.3900330586327492,2.502848484363848,0.312856060545481,3.8289717366064897,0.2393107335379056,4.548710455044885,0.14214720172015266,7.1013609288486395,0.11095876451325999,9.363786095769637,0.07315457887320029,12.582801710327546,0.04915156918096698,20.50351119067844,0.04004592029429383
+Magicoder-S-DS,mpi,scan,0.30625791108853684,0.30625791108853684,0.11831831640471721,0.059159158202358605,0.12230630930364884,0.03057657732591221,0.09059757501454016,0.01132469687681752,0.09528242107398258,0.005955151317123911,0.07695879899464572,0.002404962468582679,0.07574874099265005,0.001183574078010157,0.05582761600043836,0.0004361532500034247,0.04716947454708777,0.0001842557599495616,0.026215048169701976,5.120126595644917e-05
+Magicoder-S-DS,mpi,search,0.5206824830187461,0.5206824830187461,0.4061031573596422,0.2030515786798211,0.6747767482203461,0.16869418705508651,0.944377143666739,0.11804714295834237,1.332673431965616,0.083292089497851,2.018442609780322,0.06307633155563507,2.1589103704643215,0.033732974538505024,3.3130696066817693,0.025883356302201323,3.698819284717165,0.014448512830926425,3.95156382252175,0.007717898090862793
+Magicoder-S-DS,mpi,sort,0.6197972497288179,0.6197972497288179,0.642781686518811,0.3213908432594055,1.4924246382137352,0.3731061595534338,2.8644497670151714,0.3580562208768964,4.409353739867294,0.27558460874170587,9.673795522597308,0.30230611008116587,14.188579510639201,0.22169655485373752,15.275016521382431,0.11933606657330025,4.327278643239143,0.0169034322001529,8.255580818921418,0.016124181286955894
+Magicoder-S-DS,mpi,sparse_la,1.8287222564135144,1.8287222564135144,0.15343260088788016,0.07671630044394008,0.12599062518935794,0.031497656297339485,0.18258161854087573,0.022822702317609467,0.1764997647528723,0.011031235297054518,0.1652083574969639,0.0051627611717801215,0.11777022904566697,0.0018401598288385465,0.09431016778650894,0.0007367981858321011,0.07080482137659413,0.00027658133350232084,0.04499606313691153,8.788293581428033e-05
+Magicoder-S-DS,mpi,stencil,0.6389154314382036,0.6389154314382036,0.19717903189449726,0.09858951594724863,0.3219174030892935,0.08047935077232338,0.20870680189453494,0.026088350236816868,0.24359963264388404,0.015224977040242752,0.20274628457787663,0.006335821393058645,0.1535627221975792,0.002399417534337175,0.20935726361948143,0.0016356036220271986,0.1691174197972692,0.0006606149210830828,0.11012761145009235,0.00021509299111346163
+Magicoder-S-DS,mpi,transform,0.3202465234754115,0.3202465234754115,0.3516902846376103,0.17584514231880516,0.41471255269385143,0.10367813817346286,0.39324901492933295,0.04915612686616662,0.585156067331126,0.036572254208195376,0.41096259320013373,0.012842581037504179,0.46718039594560323,0.0072996936866500505,1.1218171490289939,0.008764196476789015,1.2511395360394237,0.004887263812653999,0.7799240296661313,0.0015232891204416627

results/015cff6f/magicoder-s-ds-6.7b_temp0.2/metrics-scaling-omp.csv

@@ -0,0 +1,13 @@
+model,execution model,problem type,speedup_1@1,efficiency_1@1,speedup_2@1,efficiency_2@1,speedup_4@1,efficiency_4@1,speedup_8@1,efficiency_8@1,speedup_16@1,efficiency_16@1,speedup_32@1,efficiency_32@1
+Magicoder-S-DS,omp,dense_la,0.751024872851404,0.751024872851404,1.2293477762356035,0.6146738881178018,1.855595151114692,0.463898787778673,2.1496528262435373,0.26870660328044216,3.002367127891538,0.1876479454932211,3.6773904294480624,0.11491845092025195
+Magicoder-S-DS,omp,fft,0.7405474361365446,0.7405474361365446,1.0587016691765558,0.5293508345882779,1.7521080690275725,0.43802701725689314,3.0973830081811116,0.38717287602263895,5.92863965369192,0.370539978355745,11.504591833541683,0.3595184947981776
+Magicoder-S-DS,omp,geometry,0.8869219150374367,0.8869219150374367,1.1214214490845087,0.5607107245422543,1.8510505471882652,0.4627626367970663,3.3714006092952045,0.42142507616190056,6.48634033329022,0.4053962708306387,12.447845346068931,0.3889951670646541
+Magicoder-S-DS,omp,graph,0.7530671023810317,0.7530671023810317,0.9734075628599994,0.4867037814299997,1.1644351065030634,0.29110877662576584,1.4181012344654684,0.17726265430818355,2.0910190754939832,0.13068869221837395,2.3147143432800426,0.07233482322750133
+Magicoder-S-DS,omp,histogram,0.3608897274635466,0.3608897274635466,0.14044317602521814,0.07022158801260907,0.17869640748297497,0.04467410187074374,0.09088586703383955,0.011360733379229944,0.07796127819989067,0.004872579887493167,0.06439016735082191,0.0020121927297131847
+Magicoder-S-DS,omp,reduce,1.6861173712574602,1.6861173712574602,3.105187250770329,1.5525936253851644,4.881385485750482,1.2203463714376206,6.56203780546908,0.820254725683635,10.074564178087861,0.6296602611304913,14.793351793005229,0.4622922435314134
+Magicoder-S-DS,omp,scan,0.6989489600918425,0.6989489600918425,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+Magicoder-S-DS,omp,search,0.40835836984781315,0.40835836984781315,0.25876240073038564,0.12938120036519282,0.3726375788965017,0.09315939472412542,0.36993383308376415,0.04624172913547052,0.271399920315098,0.016962495019693626,0.2228717980281848,0.006964743688380775
+Magicoder-S-DS,omp,sort,0.686431387650754,0.686431387650754,0.889318272639466,0.444659136319733,0.9344587933486199,0.23361469833715498,0.9725648415994244,0.12157060519992804,0.999642743772619,0.062477671485788686,1.0117046178095856,0.03161576930654955
+Magicoder-S-DS,omp,sparse_la,68.93834948915425,68.93834948915425,78.68821247091749,39.344106235458746,116.31442126586782,29.078605316466955,145.84193711360484,18.230242139200605,187.58251520188688,11.72390720011793,186.74327422027767,5.835727319383677
+Magicoder-S-DS,omp,stencil,0.7495508770866339,0.7495508770866339,1.43730229096625,0.718651145483125,2.4957853390644305,0.6239463347661076,4.014605140028361,0.5018256425035451,7.106545859479557,0.44415911621747234,12.160170622601827,0.3800053319563071
+Magicoder-S-DS,omp,transform,0.6665533575570962,0.6665533575570962,1.3386033418091574,0.6693016709045787,2.282756627326909,0.5706891568317273,2.7788661799845307,0.34735827249806633,2.9368952856405754,0.18355595535253597,1.4259193818414053,0.044559980682543916