From 64c5062c738e6e3a7f040611631a87b75f133dde Mon Sep 17 00:00:00 2001 From: Kai Fricke Date: Tue, 14 Jul 2020 21:56:18 +0200 Subject: [PATCH 1/3] Added Ray Tune Hyperparameter Tuning Tutorial --- _static/img/ray-tune.png | Bin 0 -> 59762 bytes .../hyperparameter_tuning_tutorial.py | 454 ++++++++++++++++++ index.rst | 8 + requirements.txt | 1 + 4 files changed, 463 insertions(+) create mode 100644 _static/img/ray-tune.png create mode 100644 beginner_source/hyperparameter_tuning_tutorial.py diff --git a/_static/img/ray-tune.png b/_static/img/ray-tune.png new file mode 100644 index 0000000000000000000000000000000000000000..febd6de282e1d62a08b5ebbdf9806d6ce6fe566e GIT binary patch literal 59762 zcmdqIgNc@A^1tIHBA-V912YR$B`102OeZTOR ze|+t2!ttT(Is9L=!T)wI^zt}4?aVk3C5ubuKUT_kqOJ(}F$?%}x3}|>d%x#HRiuZBeOY8`f?f;#!FF0X z%34fspyHa8+I+`SCR3V5Xl|REp-53gNQmwt#^ZK}y3((+;B~{%m$&ke-;rzh9~>Fm zD|0k`gR=_*UpBd7Iob(d+`3^%>`zA|b+ zPe>{>yil1tYyN+IGr5Be%=zTY1@Q zI}?{s-~}93=0^Z|v2Y&Mr$y-VtrLeSgR`wuK^C#Wz_H3l)wF+;1@AI9>7X@&ew}p* z`gMgNmP3b0IKMb7ZV2-bilP@vU=0{`prtP31SXV1%PGGkxrPFK;B6&2+=$`pPfr#z zpwL|#ywt56UXuFw#FwS~1EASoZkY9v-^1odAs4RY_!JM60ykdX!~hxfllQo91MLfxtM4$Xwlq zc?W^6tGM5umGyW_);MLO2aqMjvu82$91FS9mW6nyEmA_M7K7e~+6eCB7m9-o|8CM? z%nZ@BsH|iQB?oS-NIc$Kh6>mMKU^#{#N0L8EX4Gs1#FH+1a-9N4zy@urmnQ!x>~}> z=xnH}i@M zVwCGfg`oFk-2}d9`RT$4!wE(lZ&qu(Gb+>Cdi~lp@VR$}H1J0Ub=qXDPhwh6BhA*v zuGGhyMy>x07K7F|&yUz}5o?xq%|?WPuiLyIfEF)?wyHoxR>0q9`)P+{if8fSu_KRu zFQ6W%s1yNTHqcu#U-_X-==1RSeAuBcg^>pi^5W!^eCT8qFMCO8F3-_YwzepDa@9u^U zy9up!=jCYG0M&S3q-N8~Ik5C_I>ds3L0$GZgD`N??n=A+x9gOxOIKi#vPND-34A(m`oIWFRhZb1I$u+lpwD@lb?@CI7{1R|AfD!9;yO+sf%Bk63 zz?%v5AA>V3+&;q?6=j+7(|sT>f1nkj(t+taY|!UgpFj}VpbFw+kkkr=y4ZU30Nwbn zM5`0PL%Z%_=?@&VuQQDfyvk$C5jw^S%gbXqdG%HB>cYBNVY@MHX}!Ys_^+~di$83k zgW+Ynb{(wm%>jX|+gj6ptXlKNq+YUk-LLSWSh315WyF{NAPe;QfQ6gqGBxdLs@pi5 z3gt1yHo-l(-lp1!Up7Y%2!jVI()^`uMSB5i{E^wP`iwdlhyI#6qgmM)Jl^ zrBQlhL6L4R^(u|Vf2L)|ilZ~X9BtqV^f4e>GOwp1jLo_ht-NfZkzVBy$rH*VbS8Ar zGsSz*8*rSqG#TMLv=mw*!Y?fa;ALl|j$E8azNK~%$z>aJXM3~=@Y%M z_FX{T2~2A+w-pwRxAdRdaw_xLTi%|4sQ$>sk$@-Li;XeiaHLJDCI@m9&K4S$inco# z%{n0_Ct#n+H-pv+XXWI(YKy{{_X==fr)P{-G-4<3zq+MHy^!m+SezUOumqe~YtAEq zvK4O3s1qp0Q*i#jf=zHh22nG`mHpNG`rT-o=$IHFkRe#**!nTn&ZT zts8RKd+0b|O8X4SF<2Us7%F$N6QI$XJkGt_=TUdUbK7*q z-Ea=(sWHHiNlhM}^1SF@#PDNRnlUaXYxIu=(tf6D;PR_eALYZ$tOtm0;w`y3mr^=r zI5;7DBj18E(fN#=IOD9CV#u>ei&ICS@%?i#eSLT9juLUBatY+{&e7! z46C8?y|F^hY^M`rAA%>KJhMN!0^!!az!GA6S89Vg5jhqGwPHws7Pf@TNN}8h#FCKG2rCg-ZMS__~h-L_%Oy zq?#qbxCCz~qhGBW*9rsfcEemMZ1qQmRbMav|2)+m9c4M$qQ~0)Tmx)sBHR{iin%d^ z?idS*+snxXAcdlq7t;#rev8#PqdQvjd9ucvMxFkmd20s1A#+1p%o*F?vH(LXfTvWM zc z3#5O0gu}rJdzo=fanG9~>a>ZV1M(CB=(Rz8k9S{+{XIfcCok6hRe}w_*O&pM?cL+( z+DZfV6mD!`X>nz?F|wJOch+?xwBW8AzrrDR-T=#ZCj?YQ+8GU7G7G3TqqgGHJ``Cd zi#GzL+=?j<6?i`|vHlMFlxI~auxXADXmU~Bw#$*#7fu=RdZq!JFp39 z_btT7J`*(^pC(8>fobQ+7y>?#*X$|1cqCI*6o(sCwLe=y>8zdl&SvSJp z13%@plV^bHQ>CQOm;&zL7wF<#7bf(9GROEhl*g&(AcLw@0L7|457e;VZC)6BsQ~Y~ zN`~hO1@sd~FY3+*&Wh~_^AcBH@koB{`)&q^N=95f5-9exRv@7p$yIpFEL>qg;Jt*9 z;(|MiAqjk2qA9-dHEPhI=DGafc+7(kS}{yKeI+I&Zs^qyh3vn?SAMTA;k%3RKt2KnrX&fVh3AHZA&P+dSJ#Ff!cu?@0D@`;h}x_b`Q(x= z88W_BvamPY_g>i1T*3IrdLv+cIAEf_8S|UlTl%%qKu|SyJMrMQ7M}qMFgMB1D27%m zX9cSPH60%0@tg$)f|NhomgZVR+t}Q81|l-Ql>0cvRe)WJ{J(dp*;D2^E2AJ>SzzT# z11+^A2exT2Ysn0e+tf$@4}y&zzu<8X3|$<#SXK0}Q1hV5oqUV`Qwj^s_8+m1sCdgy zJG4@vE1(#CF#3n+jZtkBw8=h)I@V25s|CYK+Qwz-C0EI7>{&x3?l7R1ntSH?m_ zdjwx*@RX%Gd7!Avq4i>^&247f5G0B)hc(HuI)g3II>*BHHE?YAO{6IRU8|?vtuW># zJl(Yhg}QG10&DA3>~Yz^{b0E94;(e7?YYR9U$VaJG82imvEIN}q(|Tx zzOu!?)<3pRD=Zj6h826VGhwsu8KUJ`9*>AOY5wo-Cxrb5^_J@Chqc< zodIRhgiSW4C?3R98~#XKAAkxgi>9Wa+{UpYX6az~^|2rJU}rJ8K#bB!GXGDb*Qp580rqR|IQjcdUG9k zb7U<@kB68U2qV$Wb`m}ZU+NWPdhlIKvCZ4bLqJ@ghLEqH3Ox&43nc<@Jk^Ryg7RMR zKYJWwsoL_@W0PKqWA$i06M&M-;Wse4c<1 z`aDg@PC2?)o`7>VuvnR0x;n^2L%@hJs?_xF&t6xtnnbO_(; zVc$Pm@}O2T-qlkWWTX7!txJNfTR{O@X2l+an!jHwZIvw1TC2$;;j;0}P@*C{)5hQk z^*uDNczOy!Wd@U{uA{fa&I_Kcas=7HF||LP zydIXI-2|(2=7#A0@597$!1I8R8uotFtWwcrM(%J*g}&#AL+v)K>gddwB2u%fz`6uI zpee*;A!H?7z#B~@E*J(xoFrf4CMW}3C%3~nGv_aQK-8K=e%i=I-Tiy~?qc+BKn%Rs z@=>^ZWpRf~ttt~yQ(AesUFuK4bKY~_RTS7x%PzaE+BVdviG5U@Z3Lzf4WdhihkFVT zUf%NvsP9tc_+XgWs}XXg#0v)cQUv?DTL~jjB>)?H6PvA5KvHyfmmM7x@Az5FGP#8(~xSi$8#vK6aFER29Vm9AH#*uj&( zAcnU4$ENJ&fBGJClce%6@nIY&ZqYYyw$_l*QU>T?AVx9LYb94Ps_*+1H$Oj(hj^1x z^R&^{61`MmXcWjQ<%jnd%M0UlGxzdQkrD9i{Op8=)$H%H$B#a9lsJV{UkiPvx+tKL zE;o@{i9J!y5dQ(K8u30OPVJDk!ZQR)r>qGjpWPQfH-YlRJby9^3I`Ix3m*F5yQT~ zLD2AhM};XjXj2fO2yNLm%?aD5-!NaIA%eT(21|0BeH({|^*P8q*6uG>I#ev? zF}9RS$>Ygfp`6?cuzBIto;f&TT zY-%mS=aRLKW8thF&p#bD%rBBc^z4d{N(OTL<nM9C94miZkM<5t>uXY!iowh8>tIZWjFRKko5Z495>()6^Xpy zR?)m*&;(jzUfc+PB+9mrgjm!ng)`7Ij!Q@H^R9|im|h5!W+R-~T^nuAB`qm`wFEfW z@5V7-Pb_PvhvT(et;!;iC}+MP3CCL*b4R@nZ(|1N)kCH^F%@3hmDT~!s}Bh5kBdxp z`om>wAbcix>_Z?KxFqIrE49^T?iiq_Pa9#grXwYmD^8EGfv1G+o^yH~=KJBlx~H`N z7g^ctY@ZQ{6niPIB3kijq6>6ye(Xa^n~e{^Qfn<+)^Glaz_ejV6p8XbSsycdx1S?uI(F3^QX-ZV( z=6l%W!ETB={cCRA5RmpRZg-$frnXoocg(m$u|UAgeB{$KcP);R&{ym8YqFs3+YWEV zegYi}`T%a6z`?O`Lk?xKteoczI7a>C*-N@s$ zQ>&;XByWU2PDQH|+2tY06Ccmn{(ad(0F#t|r1SM(*Kx3(o7q!Ej0>O{ex_vmkG5qO zKFF!Je{o`kUcVllAesQ)4eG0Y$eCZP&w>bGmP`J5$~@Wb7ps~^MmSU)E@h5@0v04iFdFiK*pBW!yvQZp~t>=mRR@Am<1@CNS}c_$0yJgY}M=s98UXV*$cPW74v z2P(O82L~<{aeF@CLcL)~x~TF)E>z>%>|)!I<11$ON`ut;i#k)!{Z-gRRg>(Df)NuW zueB=eg?Q-o8XG$Fu+VAV_lH7Eej$KItn6EWXJ`ibmQYL~E#@UB8P@BFfCJsO2T;2{ zw_7X9qj~n){PndL$47r>!JsXwUOjEZ?ZL%vhN1rTglc9P>I*{D*?Ksr4|PP(v%r%8 z$GZdWOz_xu0FAX9#nAUYfnGD3CLn}V3pt`)c+^%-*dKo=zVGh?3WCUki`PKmPzG@` ze$B|J2`Gp^PznC!EXc|`x}RK=YMxL}Eu)*8KLVM?il31oovikET)snF`KORO1P@+mCT+v7g=+i(dl-i4JNRsFY_3!VqFATF^T z6p#)Rp*v3AyHOx4U92qhbOtnb<+40smhR;t+kKh`ec3~yDiU_QjLmq7?Ar}}tMKjj zm7+0`L75ZdGhD6E0Y9ZD-?H=9Ym-e+ykj%0cMW)l#cSs6wkGIZ{RY@mwT1JGUrVHO z+rQn4DF{7WW6ef*jP#g zt3+OI{9gO**wg6hFUy+HX>CC1M$3YwOVgi>S2o}P>y`B@FYtgts&ZZ|qU~OdRm;0()VS8vgOw{Y)#7?yp_x6F`1f?nOzuj+@w9 zkT#q~gI3Po3{agsm0sdU0e&1ePbHv?6h4a`Es~Ft5n0Qlcx(vI?_i9Iv6zG%RC56_ zI@4)Y!`DAj7#0I4)Kx6IyeQtD!s*SC(%O|BQmGoyu#~f$5U+W<86D`bH2@0I#z`lM zgu}NqkYBMtqU5Py7S`WGEs=vEiXg|9o z87IUg+Y|_Q8Q)k3b*+ZDr{Yz-)+YzOaEjFHzj{PfE3pPr2mMa z!-R(#27Qy=p^?j#IE@4H5ugsLh_;hexS#^+_vOJ*KwJLmG24^me;GEO6s%Bnmj!Y` zSCY`iBqxraJ@lX(^Lb=90<3XHpm`6EAe{jEGP#fTV=B6EsC7&w4*W5wPZQA7(SsfF z*RbwY;0Le#g3O;iiPMWQq&Rewa*(NQN)&F32t9t_o~oW5UfFtFvK-(fT)2$(TS{Wf za!lfnq#PK!P0-m&i=}$< zgg|9=#M-Hz4txq6Wf4alqkjg-u1Jlv2xkkuP!f@br-xiuGBi1&5^N?$h)Box+HF1e zcBBo(i{d>SYT9l$^t-`uI|t8LSgD0wo-%mZhSz_JU>pXyoZV;iTuMtp%P@O@TLV>ms~s4h|Z8DXg1Fk=rjh@8Lz1jW%{BQ?#> zKe_lKL)SJR;yluUWZ$#qI2`pV=Q`YkT{irC7hqiLqv+UAY@2O%b`LU z(g@HKEoTNxd1uHDHzq%sMLKOqbbcv=)j8Aafu`l3AI}f4JlM~>OEGeZhExF=9e@b; z>A5Fx9xE=~oX&QOdmlHU3FeD%kXZ`yh%i#fC5hcZFqhjLd+pvUR6ap7gI5UbViI0z+!F!wm>EFyfM_4S&Iy zniWp>{o6c;E@G9t2}X_1@gfNmT_2f`lCYLG2x;ag>2ifSqh}x;9l(48K(no3i>ZUUpo3TQ7npRLRe?n9$UfDJ5kk)!H5 z&{}k#%4^Wmk?hM#p$-1hqzKL+alyzigwU$Hm&X2Gn(hzK zWDwzbINSo$JgBb*q%QZqDDJlVDynj&;ujD!g0o#vNhKK5rIB$R2@%gQKj2RI;0I~7 z1h%E?cXs5ETzA}(_x8Xs6PZUJ>9p3py=u^_l6;xD@epSlk21f5b9=^@9XD8CYu@qUhj3^ObBQ!;42R%;{YxvfYi4xa&DA3ZLA84V`qFa z5N;r`*2>HY$L1<3YmY)yzMS7a0s_WT&XIE>ZbXl1Kaj~sEpnv(>&l=@PPoy2atyxO zou%VHvbt%g7m-U&vyC>o7m@8brGGTWHt#8e()bqD1rRs#chdqF&s8RC%sXz9{-(dp z-Gs)EkSR_G1aU$GUSlff0s3qIA37`i36(O#2#5_eLT8`*XUm3=6e)b%7=gJhA~K7+F?3@zyk z0j}61^O{mM6fKQd5_J@Zm-)0OfO`@}*k%|8{TVzYT@md^e=pcPKYBK&fFIhEzLW^c zHFAJ2RO9y!;I9&T6#t zn+D8U!00W$hN^M=Q4Jj8rmYLp%M%8+b-h?^zdZKRhNp=COl)1KP_p0RO``@ zk@j3gL(GV*e8G8=aKeCEay}1MSSG)`h37^m7Q(u}eh*j#3gTIDgm+=xT*-ngymHIm z$8|c9h`-I2r1>fuI2(*Tw|~&nwL)amwVriMHWH^^@hsWFXXQNj?^wm&n`Ww5PogBz z(n*KJ`%%r=?B${$zF%Lhs7#_VJplAqd+s`Fn*%wXMsM&>s}6hc*F-vN(s|hXBAMMY z3guJSHl#(aRWQ}uB`yM;I#4D-?&w+vvr*frC<@KIfjeHs227x}Gf9KfQ!La4MBWT~ z5VD1(usVo!yq!Ttk9#sJXCs)ru-wcrgCrS}^+Okcnp8;;Kkc)sgPS%pTj-({6Q@-e z3}Bk084}%YQiHV5qq2B++k~bch*!Cg;Bp(mFx}^o#7!s9vm^ybd=E5!moguh%yCb!^UIl^97p=}6b5`HXuoT6Se%Hjn8W1=)J}*`R4iruAG8D; zbeH3ByH$Cz5;dl%9M|_7r~sc^lm?Indm$VJh%7@RY<3z;g3FslvidB{ebWbR@YVdI zKGOFrXM3lY@YeF2(4m&6Y%ORo;V+5QT16i0w?RDBm{7t8#$C$po4F)!LVMLJ!>di#^RTMgM09h9(VgfXTf?{hBn&8(EZ@2*+UMuA4Q zWe-mhiO^6Oe9(te+JW~x=>tf0+9Y$@1PI-)V0P&7|DTiw23^}^MxJ8O7&7NW`A{F- zIHV7L{2JP8t$ZotIin6s(hXcW(wvU1?Lycw-w!N>xqu9EW7agop1l|f&k{3iwUO!H z{-r|O#47H@0On~L95wbB+iXa>;g>KjqBz%C6c|tR_072FgWXj*W^VMF1c^-|hUwP3 z!@B61%2LBV<%kuu%d*hR)-G{xJ`YHkoDP>y50?j%N6w}NM~j4toUrChL*Jw}7}$Ma zrlFHFQIy1=-%V?V|JkHKx$Z>;fBP$EEt?3CxzQB56g`M3M8=awAjE=DC~lL)!tclp zd^jlJ3fC7^-(aLd!HodS!H|6)b_QX!+l`w2x-Cd^qXYs+5117#WvQv813{$Ql6(Fa z!%QZ#)MhD{amEvc!f2>%eXuQ2dKc&+(6^+>&xg~Ra6>U!0S&r9$I{%^oT*tFOm2?4 zH^V!iU>bsAyZk2+CDZ=;oXE1$hBPghBHMu*BHI$4yNiM}U635?rZrx4yrq(KWx<+UmEHr#6942U^$f{1p%1AsLm86-GqMK$4E+<5#g@9rr z3|xtD+kf`Fr5HpP0G;*HS_ioEW|n9F$)D2fus`1L0=;}>o#wo!=E6G^By+BfizP?KtHCPqmfo^~z*GlCPp3fX0enhjk}D+iMdrE+*vQ z>Ai=b9SZJPjPqbTxPhasZE(>lVuA_giIzv`<1GMH0NPR@Eiim@mnbd?U9^cSe7wG= zXPzpfmi=V)gmz(v2h*x=G05gMk69TC0WjyEUo%YI36UHNAOoh6FrVRarAM0zrm3)` zeOlcmg3BRZX4gcbqhg!^Ou38)XU3LfAyke6v#awWlH4_egRzsUosdN$n z?IwgQwTC(rxOX`zaX8cP{ktQ+a(}fflu1BcNdOuNcz?YO7JNc2=kW{~kEA+N^)1pN zg45QWqRWyQG`+=ls-r#xb_?)f>%9*0!-QSkte)4xnHsLj69J2i(+ya7mlN76H#Bmk;bw#TwDpm(S2 z%M8Zd5CzvvMF@!X$-0-v6D=h)cRrF#g?NDX8srIiu{5Qma>j)BN$B`BSQlOlXeh;^ zIZ%!?>}B9Ea5=mmD9piV?CS*&A}Qz!Rpr)33AmS$sG7)J*=;d5jkeMb@rdCW=N^--Ai)sL-pWU8nIsF42Ly2h}*S z*lLFD(S$hKQYqWU40qz$B4@X4BhRrG5<;m-a1~G3xE!JcRb`M|^^zkrWHv=mjg~oK z4w0b`5Ij|r9^ew#IHoKm4-`>Qj^7!Z$}8`uO8?_*k`XC!IN1k{+Ptd5nBJipBW#eN|xOe9mJq6O1QIy#Tn~KQ8dJDw}8%HTGl0vc8ivSO5VPoN9fJHdvS0LnFi zuwldBbYUiG`qS$9z$nw%>Lgd*wCV0W3!q}}1dM~5A$h#8=pJQTbF)IYhJXhsn!!b9 zs9u=a3}|W*6cF#0bstO!fp)zMD#XpyY1X)ohFP-EYE9u*RU76GGS*C*P4g?O>Q+HL z0`1?;P~RiunO(;Qow{MAl3g>PE7(=2{4t?lu|l%}+^guAyTmV|rjR*#02#R(1}&*3 z2gW+#OU(Q0cJW_DLKQtZ!2~M((%pTkyVULi9x~Bg4(?Jnc)+#ie3-^pPzIaiM^<`T z++|fkLY@;4(pAXRJ$89-fv3u|>VXYg8&Ivn^ zTLBR#fqSYw5?=Q}g0Eb4msy>)nwweoV}+FTZ%Y_%5Bm(NE!*+7;+xZ#N>;m_YNg*o zJW_Q48(lHIacTc1%z76%bVsjxm(`gyL zho!&SN|s=B6OfTyUOv3u2za%~d1q$xJy0Bq-+S!G{}d|#;|TQCp>4|xShO&Ane+E@ zkDLmqSXx9b1v>xN*|~P{c~Fl%2oRZP)N{@-@@5U(9^@^8 zEFpi2C8NTSPL?p;vU0>Ul6i=DLSIBHB||OK)Ga;Ats=B?_dP5L+#~wJhX2{20GbPz zo=k&4nCP;_?8Mop^<-=p8kODwICNmCrJ3A;M2A)!DRdflXEGNYx13tS*MGB`IJ1n= z%UYUPy1l&{Q>ekwP+g)4UEKkLU7(`-u#1O>^Fy1F`kNk;+ndeT%S5K683<0FJ%5$t zn{0z-4rfECbz`C+KDN-=X)1O<+sVlZRlcG9?X-+B)5p^?-(*9soew5>7fU@fP0QXB zxLaf6{fSHM(W2gZc}w-5jh5Vy zd!6DT_2}ql9MkqDdyu;09D;43vU&8A^;Pdml9w{>t_hz4NV7~vMQ((Rxk?k-?{#cu zzwk(t8k9;yp0?IFW9Vt}cKg?jNCXXPq=TGR{7;a@H$$T<$3G=J*S3CV=3X3LJ0mT0 zc*x54hUPN5Fmf|wPWMA#(yOV%mB69-&+N+yhXYc|@k31K%QCup$ewWO%%g37yahFW<8KkZ`<$UeCnr>jQ)MN73*WY>M1oU_8HVIz3=o(|tA$72muTHYz zE_#@#Ra=a}1k@7N*o|TKd)l!>b9jsh`m~oIXYsMNQ^+zr^t4-UzOV zd6l$Ccy3;N5WGC)B~rT>byc`ENyB07&0oXIq{B(=-jlpbx9~&mQk&k5w)gvr@9aM1 zfUhd5F66!RnVM&L&BRB;>J$ZnX3KNX(dts-oA5%?rPyr#%HAKmBQfiO6mR~JuezvidP>9?H3u=#Ll&-_RnyT*>dQ=S8HP-F(IX&uFD&q)en;v zNNVh-dq@6Xmn3k%{fH&qfDq+>#h(;rc=WU*;@{ss{;@-7%rKx^UR1>>l8L^5O~TDU z=ymS(rFUm?I2UGwcm;HRI&vONu3FSB5c9OWT(OK}n8(x%MrXbYL!^r$4Qlw6`s3aW z2{6}6ujyYq@f#e1L0q^Bt4TBMqd@DdTi=t#FmnIx4@wiVlM-z{dG}3l++*aIFn6=i zvd?0=i>$XM>*)HJ6(=XU1S433^K@nPyv!#QTjO|rFW>}P3y+h121}WIpI%Q#J}i!E zzDs%I+UA210({0@30K$4Q^G30AK@Uw^{;GAD7 zDH%Us1j6>~uh~+SYpe!nEQ6x9;lhgzwLnX*B#oT3HR0ed|FE>J3T-;zs{mQy*k!ap zDdCxb-CLF9iYj`g20UsJa%s=UdRfR5g#nll`I7?ZKex}9B0FKqUo^??c_p{J<#4R} zl0C1=-Pc?6@Gm~MpNd?UwGA3c3O}!T;Rob(+ho7iU#~U<&g}dq6>j}0c|{RHI(3}fu4HPVdCMBsYGbuGyUz~^$R%38wH{0T$c18 zxm5h!W(&lqaj!=O=5cIJ@!({5B8PsQ6|dOiq+s;BZ*Ne_xW2#Kw{&DIZ~G#4XW&oi z*++f)ejP*}DzD%qzG)uK=X(n4r#RWo{Ng&KoRldDd;-%|@~^kOohoUY`Ke#hJv`p0 z#Mx--_RZTQhDpw?IPS#S^K8rE@GFI1#E1i(8CC>}z0Zs2t!k_=oM03%$;Y36 zN=SkCVfiIzYkb=igK-y9Qn{KLdJ1#6e4bU0K*|!(aAnL-`!pX8?2wCCLzX`eOc!1WnQq*drmg<`>88Gt7*_WFYYu=jhICF9LTgzjt{>CV%ww=^($B$X|J*y%Ol-P2Ty*hM0s(>{ZY$i{sI%yFt6E#F zJy-g=ch@@U0{114G0JWs6js>C3~ao~^hyukh1YM?9#?dpoMDAD2u$%B@8V0%jj9q+ zzLim7*6Um1r?J;QXKfVHa9c8W$@_S3FYM{9T#vdw@n>w+v$Aq#xHiN|6DUgzce83 z@J?Gf-6}5p#w13+44VW3vMhh(6`S&NLCHx9-KG|p|L7a}zG{1}*Swm|KF4^MjZ_=v zaIW|MkF=2d59Z=eDt*aN<&6k3ANc8%M4N?&Sur=qBmQBjjYLP=wqxd}LCNI3S8j~s zqt-rQhz!omB0dtzdFKZA8@|R*pMNtrKG>uTD3pM`g$omh zQE*(hkv|s`i^xSfM?I@HcN|L9C3{yUu*n>7RVe*bzMgt>@nh`P3VN@4De9Vlq4LY0tXKaPG$;;&Ci9W)^nHcwXV`qErMn!SX z`rkSRZRpErU@?O7ayR5N$GpBV;$A5>I*lMCS*}f(e#;L>MW^O#edsd98x$U4UWPg=c zk!{gukDFJ_#)_X5anH^{As|pVw02MfFgwj2={4f{(Sig4DO?*rQBe()fB>dQ7&9MCH$50Lr9LpXqhqZEe+3DIaG#OxtM;KG)IBWS! z?(PDeH%X!AAo8*&3w|rpwHVVo>5R;Ld0-c3*Jf_bM5ISlM&KJVL zf8&ndQ2H|DudO@t!Vx2PndMV}kZ|j55^_z&vxx zVWO^z>a4n0ikQ1`fg-yZMQb5IDs!f1(UxBR#OT)W-Ln2Ewb*jiWOKT;#Yln;m7A@-TN7ISDY;> zkf#gae4uVA_e6!iO&0Pn1GukujbOidf|H^H@ zStr}O2(#&ol6A@Y_@4X}B-vxQozUU@?!=E|(SYyWNytZq3Z}lg3e|{VgsyE$8pE_* z7c%x9WLZ6C@9!iQ9jAy`4dC++*BcDEXTbWi4v0{xZG0+Z%HZ zQ{xh)RC4Df1-j+^&maYAW!>%TC0W)FQ)}=&d#>`YweNtnWufQ4u?x0%SB7>;c}MxR zKX~ES?r#(#0rK3P>(+Mhzc52)0q0^0FD2-T&G&;dfV8NM{p=pl%bGg6HYR$$um5Hz zqDO!{JaWm_JN}eN3%R4nbM)D65~zq_?1(8e8k0VwDgSeskH7Qsx!1WGUV!00kiLoD zWw~EQE>PJ!`ZG!iO3^kob@obx?A%3WPbp{U|JHbD!BRHoNaI%qavWC_$-eRWyq9I@ zebe$xSHU>4ARGU+B#$OF{OASZXmbdaUCGlGA))-s*qAo}H3O?O5;%-?QKzvp&jzU{`&;?R?CV|ARF z1s#9TG3rL`p16`ZE~t^9pfv|j`GO8qgBr^m$3CrH`{exTUTfgIsthvGqogIXc9}E``Z+1 z+~Dx{f58Dy0XgIf`6z=}L8hxvrb|7$^Vy5;)!Z4~4;Bxkdih@(ya(y+d6mhRQ8mt9 zhrQ2kj6car4^`3ke1ROo7ZTQ~WZEy<1%6DpIjzav2cu^_361SBU}l9erNuB!mgE@8 zn=b1tM*nJuFU4LN-PK*#-xVd4dMqi%y!_KEK(`0YXYz_pU-9K@Ht!^_UJCTbmAt+2 zzR^ZeAZ}ji>azaoj4x2 z;IkmAKcys@?}Oj;=|2p;Z)=&+{q@tvPQajKk??kl{O|5V@E4&S z_7)R1-ewLve{E<(_WDiCwZFZPBnV;=_Xm9^rKXz@ z=5rM!y8-9k=Z4TSS)6%qlQH|?IM&=E0mE;GGF@_yL9_?>DF#&2y&650(`aGv zw*HJAMCT_gdhrrm{N+nLak3lD+}7drWe0zwEi9o8L(bFa=yK z728z0`rPq0kae;5@*5iRpZrZi?W^Db_CwD85haV8yMKdNhFtdcGRv3c%i3=Rn{K!U zQ@m(YJp4=uPYs|IS2bSB;`d1mt}JKUI;jU;VLX6UHe-!bULoCY=u+HT z5hx8l8!N9nH@pS{tmqOb*@1NJF>U>LHHdy5fm`JFF!R$SBdP>#y1b&1u@aUu*C_`0 z-+epV_%Hs46afC+VW&YwB&aOe*|f}&Lhx&UnAB&+0%@k_id-&3YDLwNZLBUY@=vHO zovU0>{y8ciV+E4>z%66Sj>Vs!QyyUR|N2_C2h>BGGFKlA%3IOBzMiKuclpc(m#|>m zCCw`&lI|P>ALn#k45{}Phc0^V?&^1Eoa%DBf0>C%>_v{=HRkWOTeR?!YNRLn$KEsI z9zMCNFX#Z^y2DOd)TJMRt5rU%Ef;mWJ~w*H)HKN-$`rTAb(5g%yDm{*x9!M{+hs?{4%XpCh#3la44Jz*j-NiX!4@9X-vtS?wL=-G1Q7gy5J z58Wb6eE-}%z&>Coj!^$6X%O`E$-!t)#i=9T{CwO_9wbRy)9>4wA*1CA`hyXg1M_`4 zEOJ-D`sIkU$rFU~x5|bxwbd|eO68m?G@CW}P2L@?;g6QTwfGS) zO9JU1ffA+?<>>FVw|MNTp%@A>FJ*4;7sywtZ8&fUP}%qoyzWXotVCu+S&uG zt(CS5zF!x#BKkKfLLs6#;3Hx1&AAAAMcRIf2d7<6r<{4~#{fHL&K+lXqhU~?y(&); z!i8mdAWn*m&8kwW=*K<2t&YJr}V?6Ig+x+hf8kINz}en zB&qomCUeXXMO+}Fy_hdBi3_%-c9F_SyQ9V~&x_YfoOKeuCXd_kD@=DG zPL4EAx`oV%_Qky9h?$dpR^|O;acx+LBfBEJDgUTkwdbsru@m*oN$}bAl-RoT<8!eL zi)Rg+aX$5nMYT9wMMdpEDlE##+L&6Ey)XFw<=WZpKdvg_xA6p2J7f1)nl)up$&+#JYu;?iAiHF>+!c1R zj5A?{ffnu=+Ze1o`4B7n@S6w%w0BjrgJ)nFRPyE8L?@1JZih> z512mVZ0!iTgs;bnkKQx3Vf@k;@Xl@gDa_j?W6{_+?N?fUOu4IFT*p*KC-tpgua!zn z&sa$PQ_ikX4|g9ysi%&K`BP5tSO7XA2H!o>W%D9Zqy#st&qzHDl?T1j+?Oc`^i9pRMK=2 z8DFAKy`{~XUr0(gtej+k0R-geDo=sqo-JYb=RLVl-pbx@eF1JM zIOn;G0Hq~P6%9z7{sbuXwQ+0y-`?;l9y1a738WU6rkufn-DkSMw6IaLdKMC~lu@TTYz|37TK1yq$=7dA?Rq$r(|N~5GwBBHcP_Xg?QbaxBV-GX#? zvniDh3F$8B?)ulp^ZobUG43Ac49DTfUNz@4pZU!BuD8oI?tdk@6&Pm}z2q@xmwWu% zAvLd6>h`6h1rAYVPnsD@--FVQuJ~5Uv6#~1te>mA9uGei23{YIG29+Bs_=W5rW&$j z5<_3anSeUqA%MffGstYew~&G6c$U1w28)18`+woS`AVkX0mS(UHrGyTXwt@_MH}Wr zBU`raKZ_PWZ@_UULhDWQ*t=Vi zb*I(8oBf?RvtNvB^~Qw$8$c)ne7#ksGTABnHCE9rcBc9}h&XQywQv623mI4 z`(FQSvp*4!{5Q-DB~XgcJ2{6X=XG$BjA#ms>}Vd4$hlZ)SYr0gG@_4uIiBgA`oCzh zn@FqI@iG8i$b9-C`c?>oMB+QXZ1Owq5INQ#&LQ{|j(k zlt4De3)9Wff_=hDT7RLv_0VS){UWfH-8Vl>ZsQ>SOnKi-4N%cCpvyhQ!-N0f9V{J1 z52`1+3CE*9phzgX?LF#&wQ|t?qj^^A>ru24y?+|0$HU0M{Qo(mWjGSVx%F$+qc({5 zAMqf8NzHeUa?O~w;0)+Ra*0dKlgv5`Gt2@#PTHilPvM1C?^9t+V}U-=V7LK-7Z;QEyZHrE_6<8ConLv#R1*yyjC`5be)M!`^DQfDHNU+>G=Sm(f%QUA-nP+pd7@3i zm~{Cc5Q|(p3rI%2#q$;XD2en5hgaLb{ti3T_H|5uwJHP2HNjU;WX!A99y3pIOVMWh zA>()FqCW)I3*7c{ZJUla9|4sDaPTFnCOuNGeye{>Y&zB{z0Vn-BxHzH*R&C!`O8uUgZEx!QrLObRIg;Z)`^oIpIe-?vCA zkhs+Z()$b`vEDr-N(YtY3bKs}ch2LNBzLb?3|pCvJ;irhmu$#IfwkqZ zJ04V6g*(Z+%cm1wYt9&E`Za%O9*k74IOepKZkY6_tXR(0u|`WxX3IM~$MFSn3!Hj8 zwSNDDVZ!|;Gmq~nJiY@(raYSb`Y@JZK)XPf1Z~VkqL6ZtpRvZ^+4VoW+V~8rd^-$9 zZ+0va?$Gm^6Bn5v$W4+YSD4uDL`~M5T05m(T5DJM2xBf?pvYyoo(v+#K)9!Ms=|7~ z=q94j%&8Y5kPNlB#D#B@x_n#@6XRiIsobYY|h`=r|(jiWs#haljHG^uvf^$;#M@15Yaz>D{Blv z^dtkMqr(uq;U3Zsv30!IPvIrE*a zg|ERw#C+A)V8i7fXhn$wHz#kfcA7@2GNJ)3#*_~xva5Q>Hx+=J-_(_>F~%8uzNd7c z^svyCf#@(uz(izz{RHNmpjIjer**IJ<4btYS6QT+9Nc0-D+7!g>c8}{PXqJ6ewd2$ zdR1V-0(`j85P_jYe;vx8%KA8dlsEf~Pxx>QMFB3(JiV{Cuh?#k?kxis?u!&X-P_R9 zAL@YXLF&l(XI8(VBn+%j*_+$T69bTkCsMF1ykZPBlxU;`IfcuceSjSJEM&v)5**7pKg=Z-R6sjIM-7SH`x>n34D&;|fb<#2^KypW5Evb5Wc}m97Qr@nyXKuCjITR?Q zPyX(R%{>wNTdZm0v>oXIyBHTZAFccj+BB;6`t(12p#KLb77}3dAi6l}D#NXgI}Aaa zlUgA{5pow{9;i0aI)qg%w9{{!@gH`Y%ijEP-kBgPrP5m58A;Ymg@Uq^U!-Mhm<1fcqr_z8#V7j#1fpYBAINv+t$c#8T+G*U7-5 zQxNSzkmTA+y)5QIl1rOhiTz#(gy&yiB^6~enIpePOaJp1&~L+}B>ay?nAZ*h#$%v? z#hY=<%j!nX3-PT>{OzBx5Z2xW$%%%|5vV2wJ)SI?+QAT!Fw+Oh-C;5Hz@cw?zxAS2 zs=5%Ms&+@$3q66LJXBJ6N%n6uu4KVxKtGZD7^q%B%FcJpKU^eH#ZU)+xkFfe+PXUu zQ!0VR%3h0@_Xk;9sG%J+t{kj&_Suw;`QMvcD}hMDEwvP|a;J5GFX@*lF!7=aDA3PIkF*Ewh`JJy2hj6_`MvEi-f z^KT9i_z5|WSq5pyhl>ie{y zjg9NxQjeI-)AwykJ%65DIB!p7@n7ug zngxA16*Q)zM}n+?I<}zb@-&(~7kWnNQx1K-N@{i4T2|djOL_(Z#cDOLh_i|J-R%-Z zE{CGpJT``pszM7NyDz5aKPcKIMTJG-NMhV#7TvuPYASXX?ofa+I_a5&?{&pLsYNaV zeRWx&U}%TogCY{d?_AS!cU3HXy%a01Jnr5uQ9C=|NXz^mR@Z}GEGY*+d*d`;e`wof z>5~EjM>a#;o)4|(T3u&!L@L8Qf$Bml@ulnsgB+Tx=lS-Vp&#w>?kJb*6Z)>u8SJJQaaoxnt@}G)^-umE#`+tC)wz`7+*a|-`!%~2Y_-3;=(^j zyk^1(Mb)AtTI(zjV*|<>`L=h@-h7 zt0(X*WB;G{ywU-P?_?B6MEt7L+m;+h_T!%waIR540ycX};LuCH@i>xGBuX~Y=f;&tz{S?hplZa=tpEe~{F$e>3&bVCOenw^PeXgJMx$|dkz zpsO7{NrNrd3(WI(8x){8S0xY$z6ha{7iexr*@H6s)GfCUqxI^O?_~d59lY@(O6cCZ zJW1oUCGc#deTrZxZ5x5!A6h5IGs@8K54V$zz4pNOxR!YI>?)%qi$AP;!YYiz&BdqX zs)wEN-8}g%k>zdCCHlR~Zd?vZz?G9Il+1Tz$Lzv*TP;3im};S~v5BSb9bwf3l)FDZ zp`m3uxtV)4{UR@hGR?`(tL1zYrO_xDDCUV|gIv7VmJf^m+rNfLBuIA-zn7H|j%(>1 zpRtx7@P=N_)I-Wk!LK(0R-JA6#^K^CXL);PH6%D%mQ=HF6#uogIv=P7dW z`;#Ff7FZ4m`9S{BKTD{%Nd!G!`s$-TYtE|IyhV4V0ME#HR~$R`iHF5znykyDr}XeI zh6rxweq|Gv)x7eg7MU1Kzw71~$i)H~2np$NLYfob^$yDxAIH66&?BeYf~n;&(44cB zUDC2PBNR)M?jEA{3C7^I{f*DKQ?o*|_eu_$i39M(VkUfd=v(M^must;W-+w;_V3!A zaJ(+E=D5B-XD4iy;dGbzXggXAR0@ao{9Cd!LrutBTJOivpN9JZv3ZsPOnP2A*R8`e zfLPB*07)#}|96Ye47MujPI=Hw8w@{(%KAy4jAAd`)5S#<4xqZa$nb+jo*K1Zj=(d`tk4s6_vg@zvI#T<`Cb# z1J!e+D}W;%$1od8^BqfX?fH;ViSTw(D_ki6_H-C|rYGD&N#Ggl!${XXJpUGv>q&c} zI1Nzh+CTGUpUn{+;36gMDF>x2kf_6x!9+SSnXu-opIYTQCYS%ra2F7OVWB9_ox@^E z->o|w(@q*oLnl#2eqYXWNs=40XWJu)mb5nWvlyQ4B5Qp=$vB$$J4+!DKx@UD@tF~B z^;B$-LAo4KOG35pbgd)ec{9nB=5CV!lP5d>59%WuCA~}S;`35Z&)DxC#%X&E*uM~3 zHLLsh3S$}K+rtHS7b=qZaMgM2;dI|1#{s2_xex2-CuY5x8piTaKahbJITG%9N~e)P==1-iagC&c&91^2$z9ig96g}avq>1DcadFdts zAO0j&{N&jKPl$(mBB*151m@unOiz)&I3yFYa3|vIB(B=dOq`%k%Y?aETlQH zJClm_-#Jtdk^^Jf`XP+6^-%~jWIo_h66BB`u~Q4OZC%80uQC_d!|_A}WedT(Wi^k( zVfzwIGx=h4FvRw1TGg#CVD`kqLT&S)KjQKCJ*rDy-lz_H83Z5-a^naY-{*UWP!SD~ zsFp<2o3u_hU`x*jR;T;6ApMvOMJS5GwR#*WqpIm~hQ;gO;`r+YP7gyevg_sfsO@)( ziIL6;iLir63eb{zDRtWKxz%vlk9R+FNf0b{WBeW#iSAecs8b9Fw0PgA2G@%G9OdgR zA!w|jDa+Vk$Df4V+Mi!dl`Fw{arPuyydgPjX^lxW9cnA}8oSQxfy#tL*8TF_0C0(w zSGjKZmvX_!-<^L2%%`HT6I#gL-TT2YCorbwOrrCUvO@*c3;Zg>?o&idT!PbV1sJj? z(ryrZ5q)`o=8D}_HBD@fcX+hjZDT=OoMw~y`lURm3=5shLS=6Gd%3%u8NYYDbdV*U zv{s3arp|VHQ^cZ@jbVZ-~dQ8%kt$(k?1lut1upUslHoJd@=G&nt!<` zx<9f`Ji%qK#M&r1;(>CnTrs}YVIN~Dh1TkiM4SQ5JZp;#$!7dV0pN&6lV7&l!#fEC zaxu44*lnyJ{_ANJOGCGqt4oc9uiXvC5bv$b?#2iOEOG2Jbvw?48x9c+2WalQ`d50* zTnNaXm~kZo3fx`raQ;~jl$+S9}H~r_DS<)iJaBCpu0S&t>BHjSO)?X!-I&|&%Ee>`J z8G&v9mQr`3$Ud`Lx~bIT%Wd@M;Lw^J%wypSc&wT10Hmb6Vy^AXet$a9fNbMyIyl-0f1`aNGp3c zx0x*3%K0R{5)`La^giGmtt=vfI>G-)L`vhup2fYhS>3O5`a}qZ1la)fhfm04xw?yx zhGhi4j({0Ck_%ewTT&-IkY0_`4xcQ6G=5}m-t>REDpCA2XkS~6FJ-m9e8ZD^t90qQ zRZHAzEfavQ1@EA+{ea8G8wJu=JvVL~Z~KQSaBAAc>+zwqRmJ5g&EFd283Zo$?Dx9c zEFYc8ilRU#^+vJum1{dhBnZ?jX0bgv`gxrWRIr+yKc|8wD-v+wl|{1}3XoV8M@9e5 zJ<$md{%t+3cvc!0)EkJFk_kIJiKu^FDc;?oU8bQm;>pNre35`2-ZLld7%Q-hfnyu$|F;#IN?|kc?A-Ujcyj$FMGW|=v z7Svr%>}#g4*9-)wKC#j)KmHe)8x?^>E3<5PM+flck(f#4`P40N~;;&;bv6 zW6R2Br0~2#xDnmZ9hq2rzsWnm<6gRpWla7-`xO<63iA}E*L}WZd3I9^Y)sNRA^;Z3 zRd!4{;e^_5dMY9}ee>}GOy0bP3!!`S7YVDJYWi?tcEVf()4YxAWb1E1C_Y;v)S1(GN5 zC{UaI1kJlU3!9HX3ey7}fAH477c0-XJ}m$eA!w^Fq1AftMzx2x>cDEM@*bp$t6se< zy2emfd7-`YdTM%0HV3^9#~p3vQyC^)4%!|1mE(D|u1*J<-kIS-#b`aV$~u{5qFDuO zBlo7orEYg3a0jX6xc)eH?xt~T)8i?8gl3TSAJ@?Pl;QSQmRQc%R?4TeQPd`J0Yemt{;6 zv=v(-5GvuQONWk5LV0TGfb`)*lPgJzNv4So+!r>Q&-y{YU;jyo!I$BI8Rv;IHcm6~ zWZI}Q*~AmNYC>|06a$(?5l6;6LV{+HVbrtN^`OzOC}*TqG{I`^O)T}bd74UCNmgzwl@p}61Nbve&A_y@{t#iVcQhRdO1$e4(V}U1B_&`R@WV9n1W}lBu&zgn3 zCH3?dh?tL*yTH9wQ{T~vhx-Gb1YC!Qxk3R*9sI`=V(S@gv1Ex3m?{62pwbV&1P1y2>K#$UCf zz+U_BNu6dY^3(J;J;|Pr*?w7kO5?^yGS}5g5k4 zS=?;<%I34A(ZH&`3vKSCo#g5Z!q&pu!)M(UZ=@LLrmQvMsoZs#19-SlEH&y88W^cDiiOiqT8%e6~@hB^6Zq_N0U3+H1o@V_;0QW={JwNeXpx@}L zZ!%nyN)l8V+IEw?!hMdho7AGIe2EC%3^8-d5)(^))La5jnTG4qNeMk$v3!GO^KDf`T)aEfC9{ROZ)ljvsz)dBVa5VMX}bvx3cc%e_|kDd z&EU7%Tef|BYdx;N$NF{&5O z;E~(Q0h^GFFZpgeM=}Gy8>#!Y{8e)7_Wx$C5yzpoUBEDWN`K;3^+_AOr!DILo_e|M zj@;n-zVy=NGud{ynf|kk@{+FZwFsA5*gB|S<)SFbWhwZb;dkk} zOhzj6H%-9xBvH$~!P!>+cbCnAfZ&8GzKkpUcB#=*wg+zB()(H?gM6NEl5Xy@{Q0sE zt2T}r=kVByar2Le^7mALd&(kE$R*U^A$Z}eU-un0s`k_rhCj7&x$CGprAirJsrMO> zBewIX$&>jLxdkpz?Nl*WQrC4VTFf2WeQi=KUs0etm!+B!?J#_hTxqtVvT zZ}H}8N4fWMtQ#{jpVo~+j*(%X1P)#hwgC_b#kJ1^OP9T|wrH%D#+#cirL46tWU0PZ zPuu?bY^>f*_))nrKCrHbqweBJ5S{GzFoVaJhNn_o98!sR^sb^&jRVvP4F1vbHjO4b zLsx1{!pv&VUBJ`ProrMWKZg&^v~5KeY)4SxQ*gx_Fmr&hPM>0Um7aw)XMODQtN$I8 zk1_Y+PyVdHRn_vT<;lqp?g{Ggu84e$T`Bfj>r7g3?bValSNd7AU_a*4>Y~;H@T~^?q}X&EhiZ z!q`3`4&1q-$#{ua>Whr1RY&V|Wudhx^-i`oXb~4|pRtp|XXnzkD0xY=X7EgSQTL4? z5@Z3h&N*LD&*fqOM!y;+q5YgB&|^_MSlN7IXJ-}l5kEg$8}|8_eF7I;HBS@hM#{5oiU z>gj^Ru{#T+nyoGT0H(7kC(iRKW@y1{<1q!mgX!U?;HVZZpX{TSoEwkv7E@y8W?w)g zHiYCizOKX~?e{&5%tVe&gME&*@Q42_islVYRAN# zN)GrS{dO)>-??;8lPxECfE>Oz)}h#my!cx7 z9ZMus;9{R4!@J;CVpUYjIg}Rv$HRSCh6x(Za#7@7Mqi}8drgyJ((^c?b?6w#1Dok zxfm1E(CY_lPLFMuWSU{}=l4QFD6*JLVe3?zTTM-4;jIcvkd8k&;v3xZJWUzj_>%JC zrG~+D^zu3O7Y+2H2!aqbvcI8bcKjKNMv6Dh&;P&BH6y@3N&2av(wb3}_17(UodO0f z0IuqD*a~6xu$sw1{P4Az{BF<+tJEQq&D+?BSC+A8E%YZsy3C%%e);F92s)`rkZ0XB z@s3}T1AnBZGnu-S-qvNTn>ofmxUo|yrDsZZjOpdLqvMOVw_90>g5yZ1oB_9x z%ILAVm9f(@PwJ`nnJkeP^P1JYV^XzRY&j)?d7#mDr&1Psz_PxLVchhdy6iYt}w{;N51 zQ<7)n)0AB4&|tIiQ>vZ0$y_$Q)L`-WUN6^%?`#=Xj>LOWz>f=?(%&p#X^P4H@@ool z%}%72Ohjfx!Ik+-&NtTV{vko(b0xt>WD$voD?)3)s&h`Wn9}5Fj5aGrI*02kcgEZ^ zysVSOMO3(3>E;(zu7@OPJj>TST`3Qw`s&M89nh|oAM`HpmEGO!&6K-JX+Bly2nD7_ zT#VEsOt1mH6X-MmP@e;)W0dvNF{}PA5|BwtId9SxqF||mPx*V+`FBhs^W(u%k?bC) z2nbb+&ajI5Uvsxc18Y27kl~g3rnnbjh9eA=f(?0icpSzqHF<7rv0J~N@<*WSn_~N4 z{r`IMNIn6$IhA;}1rF!W=ZlYX+Lv-3=1G_^K7Ajv4aLD&_eNn7zbg zs?ZP?oZD_GHkV0GL$}x9gvjXlLWD3w@jbI z^ajtSP&oOm%-`fD5Y^Q0qgt}*{KF#}M1(prfaIS|oYOPRT>*OWYOZfzWD4`A{Fy^G zNlkukL2J0)n7VAl_J{rU$TzFq z*hp@W4#r7LgB_osf;Zl(_e|V_yuCxix5{-{LMy<`{&s&o^^E!RLs5G&g|732l&TgV z56QrAJKznCS{&fmn;T)K3J}fifgqPgy(vJF;AnrQ<<7>&AfFl;_bnj~+)Z7SG428g z$E!p0?%IB!h9w(DXzZ$HH;a9|CFqc+XRx2sEpT(Cih9JCXh9QV3IVJmdW{!g=)Ed@*^c7M+1Y;dKR;L9AtC6DK{^5BC2#GP&=ysn;^>Fb z8=RsSHv7KG@r0VIZo&q!?W8TEGm~YqIo!eRZ*_};gV20TVO#cjE8%!Y`|Z)FQ!1IQ zm8WVTh;nnO3^ZRTm@> zj$89Iyu6GXNWxrb73_E``*-?Chv9gZM*J$v>Zyn)0>U5f&%c!>wW(etcTXn0f(+Gf zWE!qXGZScm@!+wWlD{JF__JDe18&d?DEpmUKsd`W!)Lw#5)6UwjdJ$-0&UJ~1K&Nb^4(We~Or z2~xzqNreRdAmZvP-m{hO><6#Tn<7T=dd)z{4YVGpwG%l-Y#IT6lM z5MF@K4u=@@XG{02YQ1F7;8eEY81?J;B>g)m+loI+fRTPAmjS6g6_F0y{W7ai(x3;|1<)u2~aVVa2eq**b zheW_}3DTsKZ`&p*ZhWmNPqoG5=|3}(Jy7!bEd9kV z8^8wTm$FCJ(jdB7&ZUeBIi7B=^-E?)g=<)6twfghLAVugej z(I*UB(0ns#OmaYshP3drCl%v8eZ}Nn9FIod4U>6JWvF-9#ADx(% zjXk_YZC|tLx45LMC9gB`S^w4f$c${}DZXIo(;Xy506eukiq$)77A4y` z(`$#tKUMmh*BO{*k6k_sq!GoDK|W}XkAKdYu@NDPH)k`jSUn5_jwwGmCjaX14gGKf8^H@2 z``((IE&q;Up+xt2X#SxlQ?F@dj%=lp+dOn~tpTQ4dqCQEq)}U6!ZB_(`=`&^%~=*- z$(E;3zd)0GFIn*!uVp3WhwO@6^8rDX>9Aaz$><%Y*d8@-o7b*c{x(x`trDvX4o$)G z<)NVGOTm-)jl!~dX_MH_L9juT3`Hu2cYvwDwdcuf**Q#{o50ZiYrG@fna9JuBFVrXABa?4X&5& zf{`Qb@@oE>g>G%&0G0E%6I;DkQ4xxiM_(_HJAu=2S%~e=7i2TQlf;5bB>F}PDdeBx zB$W~DMm*AxO%4R&ZdkJHg4xfF9dIw+pFK;h0nphWechHwW1Bo09u_vy4Uf6$@@w@rJ9v^f z#P^AGvqoXlU32sHE5QzUNBtOxSUSiimQ0^&gxFYlsuQ~!UkPXp^=V2TVHvDqh7l3T zN`LtqcoL5J=a77EN8yp_DD(8yriuTfi#IVEnmRO+ZyGMmd# zW5HouEGLt1ii2S<2Bv)o-He{-Zl(0jj@gSTC@@~E&jFvpRF;MkzP*H$PHu!l`p z34duN7@x&d(wIwmTM+2 zDtbt4NJ1PjIfeYIhY`JEBr7_ag__7?|^{V_tS7> zJV#K>L_wNw1&Y|88?I_SAIQDBdhvaD$yNUJ8|mM*HNrkEjb4-%kad{Vg0w6DH1s&} z7KUvPQb~*yoBHPj>~T#2A627loZr{G>Fq6n3WUSRNzS^|<==@+Tq>>6V@*jgqWq)+ z``$B%mA*hu&xF1E%aR9Sh3^L`-Xg1xKS%P0E`V4lx;SY#KGK~;W0{DTo$U^1{e$R@ z{bF@uQs%KPA}l9DGz^w6I%bOa+u$cVrLM}#w<~w$A^kC>H(B{{=rn~va!I^4E1=j6 z?sjgH>xRKSYuoBhDZLzlooVa6*0O^umvq*+J&2kgP+)Dqf>Aau0@HMOTs!#P_DfRfLLKgQU+A zGwmprmk>+ZAe-1CwujL!O*)<%Wbi=S276tz4AMSDW>*^u2+YknZRM($y3p7N@f(#l zbRR6)+wIgrwr(pn8;Ey{I9#K}`(IXw2>uR@2TOy5;>aO?-6Q`>m`PY}ANY=!ecO=? zs#WvnZZpb=cqsU{`z0F$%C2R~4;Ao`?$UhL05l8}-R?hdifI9xzkZ{(ZWJuVk<{ig zwz&e%yvE%bW@0Pag{=uWd2F`XIqnBfIxa{sC6%9L8FOc;j>_{>$a80bBq4OPSG3-6W>BWV=TP2ArXOd(4HEIJgA3 z%+Jh8Wt@yuDS=RRyfrC_*_42XVdkv;dN47X$1&`l?EW1zKootG@sWWOC!Y@g6Eo>2 z?nhAu{si~rCN?OLFhqTgpmKDYQURniLR!V*?{At9e$h=><*m9LBxJu8oGD|&$19}! zUB{+=krq1rCb*NVbR6*O_QecY3Ow}Nv*cFd2@F@Ov4z?v2Dnm3EkJhWv!HSlXl>-x zeZ2k7MaDI7bk?v=W8Ff&uJVwT#Ns+0{#tdfYTIWcj9*U$yR9{lH|38$wqzE{*B`j} zXXYzfHWg1F3LX0YaTH&}3SF#OFJvG%<0vbxC!4l*i?yg-wXT;PKXMq}*OQQ#0F8y5gcd!rN=wteq!sFxAv=`U*GXt$K@xZ3I(314?z#AB!DJ@=*2Q z&M6D5+L@Az`&B9oO~rogsbF{a)GO!o?5f)ZQ6cn4zqH&tAVY=hB`M=wL>p@8ObhyD zeSC|VoAo1M)HyJwNkM8(m)V03uuN8eR$C$XaRPh$3N}RIS}Jc#KYN%IAX5HuKULgz zy;O4G6A=bQW{>a5GO zR1UJ*nYy(xoRZNcD`fJ>H)0=YWUcTjTueJ_WN-2*IR|9T0#E@TRS102J!cFp=9gDf zw(_?zF+#?2{v+h{=QxQDxQlX3ywB`+BN}#j5jl6L{u}Qt7|#x-SK!`UVRBSfr=sqn z(vu$1GscUzi>4Y>rvEehJYv*38HkW5EtGtTL=VCLp&{G7AJMBiq?OofaWW@!oJBfY za%4^Sz&+o}W*N0*5f{`pK$&6oOggnzB2t(sMbq-^Nr(d=P~fA}4+MdsJmGV;lsKX} zpq0d6rb}6BlhYl?%6ub_%xsqLWRoBCl^?fcs_8~)StL+u`G5UhnhLK{a~d0=NaV0< zmVa&5(hoE&4;FWiVlXD7{mzRvDERHF!Q(reT$yypmwosvJP;6OfWnHvKENU^r))DU zzgt&zoECHOky%z!`}Z-z9A`V@P^g_WrABs%tL!;BbzJ+bEF1OPb9m?l!B7CabbxsM zDvDWbcBW{BRWiFWr*m&uTj6F)pME~J>QkB=vr*mSOH9MK;>|nxoK3zN zHOI&8ApHOzs~rGbs)?riTds`f_D4}dEC#rV1@klb-D!MnWlf>S-0BZJ_2kVzZZ^pR zvJo@GF7Cq8#u58(>G!n)tBY%?J8gKVk-Ugz-0!F474qQI-)CV9Z26?Hkz6OaS^q(5 zR$=2z`hOE=0~1#ns8ljL(;1ZX{{Zmv@%37mj4Zw{}*b%9=`7P&vzcDO)eIY2!%upls+q)rn zK0;so;s#_-;A4S*UgBf-if!DUhm-T$$7er$5Ki3QAF_-!$tm|xMiI?NR=m#|&H^W< zZH-*(nrD<~s?7rSRVEwSseJTY*i<+?RAMo#!RX-GeY8SA$fMcULLC&jYl0o6?mZ(4 zF>!O#)YH==qXu=huAq1C-<##`?phLIktZkfQ8!mSaOI+wO7xXqRLoxD)5ynmP-8hF z&?3Di;vsB}v{cw3r;d!8&MI*QiZMaK)*1OOcZT(jw4d6$`|0yOOnOt2X;WVmE^@0% z?Rjn8;V^$ji0L0A!)WHcKQWO2A1`Ac3>?)>9}=X4<(L$nVVrH>#x~qKk~0W3Oy4-s zIDc!iCd;8l$bodWja0@J`Z0b|OCV})wV}|`U!9$1V{K#MbaZrNR8-1>R7TJ7v6U6& zt8u&;fPX#ZGjq9Shdo{WAd$^9a+vXFYg}(Sw&gdH-CRr%2#%IJF8-1V$}}#C&jey-o4}I=g*Vyqkj4)!z%N@ZBR`9 z>?TTx!vIez5jkhG)=kcaDDT>$m&-lM{r#a-RHZSqw^_5-`>Y>CkI&y-WlfajV}{`- zm~Jt%{2{^MdIEp73P9mNKP$?_w6s6R$5p3$^Yb=p{1hchs4COqqqSdkzES+~H|<5A z=NT)wN$1NoUELeLlD}2VK8)W#YiC9*+mf&xC>kH0>oJIJv#OA(Bj4b4_j;@$Lqf5^ ztJLN?k&|=9LY=P>~-jb zUt2jM6R+Zf0EO5;_#LsY>pJ{e7jcg&vG@YpU&Qk~c?N&a7lIDY!*fe3?3j)Q+w%Qd zeg;B0>Fi4R#jJ}aLC{>i)`QY2Z00n%iT-nv@nH>M*T2_{VphYeO&5(<-DHC8665fH z!+N~x{T>8=N6t~_Pf^7UF2+@moEZ|%>{0Ko)JT@K&^~;la#{VRcv>v-bUI0F>tj*B zSa6NU;97*Be)Ywr#V)$&OZZgGjZt8#k+HZj)6K56xBGZ^5vUy}pQvvGXypr3q;_Ib ziWHNIP+%iDhqdSV5ByS5xhDD?#m^h3u2;ihDh~p&dWomGR1T_DSX!}_c8!efci-o+ zuavYD5e-}xoXFyeZ@*)@-yGY2ZfR91Gt6MP9P)l2NAiJv~Q0_X{E99R-C+U$e5h96+1SVNaBh$^*Bj*~2gM_sd^G2PG{thDYKy=d1f6 ztEJ13=|;jJNcIqFjZzJt->m7Bwr8`ILY~Ueq2X_w93u2ptJBT)>W|5Ej|I0FkU zlX(fXTtZpn%de+5e=NpIr>`f9axBL7#>^ugL zk&SBQT#olA-nLkZ+Psjzt&6#M&72ao?QBaqpI?ZCR#tw_(qi7>w_uTO&QWpmCc|mA zlcfL~FFB&=4;q7dedriLutqQ2$2XA;TR)izSIc>YVAH5Q%2)d{!Bv)K<2m{aV@nxj z%A?dXrYZ-sC8k2d4WYLSU}L;N8l$s};oqkL+rkU$NR2}4cjkloEM(j%y-bg_d7qWt zEWh|TUy!a|I_G_vO4cJ11XJY%hsVJX{VS=0pB%rboBnJGG&D75s5cE$H>vYjh$~^` zm1e2CWp^-7D|z#wkB)T?H}SG9o=R!t6_u$sSvbj6^8=DLkKu-Bu9*=*h9)N7ppJ*+ zO8q=nonzrazMXQ#h`e)+jmNMey6c!nt6wXEmtlOQ19U>ity(=#vp~J+1;$7)#~xNy zF;OYrU@?|rhzhzDBl}0T5?-Y-i>ZR5I(5NeSgPdoDNB3o7^*8 zkZOZPBM?8@)x4c4P(*%OlW=s>u^?q6dlh&nqb0Jy(9My|tn)?(=ptT3%q*4drUwBJ z&*jIK78^FVKVA_rk1ZxCR}%(j$0~)qkJHE-MYGh_-HM7cwJvlwo_*nWWQt~4Qf71> ztX^#FKekl$5fMR?5(IbE-{neK)-A&Nl7E5-sGqMlSG*ikME{W*8*D`H+Ex{m7tp?{ zHX<@nD+|?irhN$PHr^qYO_AMRjniR<-Fk9WF+{qTO>wiZvXF58_bRAbc?3o?l9b}^ zVEZTgSew=N%mgQHsh+SYPGR?(3tYdLfe}0P)|~MARcfKWv`|7=052awY9*w6|CY>A zHQ6!39TI#j1u%TD!!S9y6#GlBgypA0Dfx>gQUaCpABqpgOQN~r#{+)iqXC**ggv&T zQXcIZxifBsVI+;}TxXAAR9)&=Tkq(XxhBe2y|$z=*yC9ugQ*;RjNJ+wCJ{Fwx9=s5{RiR z>yTS}Uc18BDbe2sn$YYWm7>*N^{LX)YQ+aXnfZ!qAd-ojR%0g|;0cp_v^nLsm_N6e z2Mc#pmwLs7$aib9YhwRc1nSr155vIh*pT_gTtqfmIs`1&KJYI9ztHL5zVhoE_>ryM zM>kvS8mO)COcTd(%Nh0YqekbW`aSl*_H5gns0kJoOO1#Wne1Xey@GJ}Mw@z;5ySRE zb10e2dSh(3_6QYg+Y7K`W#tzt4nhSz$#_&MPLrqwz0}U}ke^P6N5mq-gF;<_Fk9p0MC_)9+Je=hiA_kS(ucS~Ns@oe~?%W|!8W&_9 zvq|T%M;^;JCpS5rulL~`rN}d0jeUToH2~y%2;}0^-|F-XrptWXOL!40_KFu=Zc%*> z9T4r4E!gHe%xrZO5>-*1CyG0P1PVb%>J7dExTr4FIL2c@-VB>J$ls4{3b7uZC z$0J5=6$aL`9($!#53L~W@vtn9vJ42d(^c_VuEB?R1kfRe@OmHk1VR}ZJAY`sTCylU zrcT%Hm1%L;X>vi>7{P*aZM{X!q$!T|&Y!)l>il=m{1<1k7_(N5yl3CFoV1IFEH9{b zv-gtslrAla<5`-_lon2Q6qwl*O=Fj0qvQDqS5AtR3J@Fix=~h#$CSLKcf_oGtiiZC z&7fcjIt4fg!B= zz>9*}Tj^bey%H)zPF{Yy26#%d))k^17Spoy8~s;&mYE`K3T4dF^fGCerO3gtd)UHVzvDer(&v75_GME-L(*W_cTJ*JfZ=x$<{mSbpPZTuP~ zQ(`VT)EafQJ#A)3lt4gZ#%re6WLR#*Hk=}>covJB@MDjrzDpUeTl2IFRX>cZG7eXC zV$H+~&YW$U>B&oDacee79uzRhe}CAmqv@gfiye&h;?n0^OH8L;SaLU=5n}g#y5Ikm zc*S<-#=ZcnoDQQk0q>lCG3V(E@<5+t3Z|1RNG~hh+%MF@RI(&V-%_04wl6ye?k>~f zY;&(9$;l&Szx3GXlJe7e0g=&_-4c^r*~TIH8C-77>fKryuLpElCu5y(2qw%32ykax zM-22;KhEjbLspk(R(tYJw^ofXht);!iDl~F8M3GzZ#nx_d`-b;p~T6!s$s+WW@n!A zz9!Z#3Hc=*y6uTIT1$K~znr#OGPUWvd(+ZxmM(gVB|mj+%JJReSzLQV<&;Pe_L-uk znM)3tM_+As zQ+f2%#eP=noR{zZOcVY_^Ow0S1Gxj!(PyeBvw0twF4%QN=`ct4tMEI8Di0R5PI*n% zxfMkQ)IY0U{YlgIN(OOA@+XI@)~UP7(j#M4nv2{t#}OZjQHC4w_r zbmKkxyuliO-w-rDkZzut%c1+^p;@y;G<6XC6B0~4KJs7MkpKJT|J{fGpBTaVa`Fq| z@u$t~$~TdO58|=!#bT;O@i+E;Qw{gBC7+*3R(gC_xMJDt^2ed|)Zp)aO-M~H;BQi0 zYI&FYpx?$71`7PHpVmfbM1d7aF%mPL`Skg7+L0}y)k&w=%?79u9-m-x+oknRLW1dz z_@@8$BP%bT^7%8AG&N#3-ihFY@$Zdysrn-OuKRs46}gF_pV7f|%61rvMainDWRsPC_|Po&CcU*^{9P7zvsZUf zmR9n|^nmtht^AUb5_1Rr%=H;2QN|hXrl+S@hQI!^i6%#k?30{a^44QD4u=k@-?(vO zdaPbgZu91QmQt0pnVorEZH1mYyocJBWN!t*CTa1^VW26KkYPRVWAl7|AB~8J07y2y zu$Y|=Yb9T=iC0W{_f9i@yXV7Gcfh`2anaGy+x|0eFfB^1cf{W{il=@i<)<-JfM3dE zw>y4Z=i9e$AewpxUsKE_BqjU){Q9=TXY>K#q;3f7t?0KFZVc2P`chij{UkV;ru}67 z{-U{wR_=Ex(7%-BNPuj4C>d&rCKw zI~bj8;Z|(in(x-U#=GsF;!we^{0`p?-!o?p$|Np2a>1MQsKaE4U*9UdzQ5&&-{Eyh zf07}`#Ue#F{$rY9in)~b#uY6k{?DJeeaUKdeV%Dzps{InPQd&3@B7Mv>2{nRNhHg? zzcQ%V*JT?=tWokcGBh-l_-}{41lRb>rR#o#E>})-XgjK+WD<@0?urp{Wzg=3h$(7@V0e`CX%!cV^PH z-4nl|d}ipm&(NE59HXM*1y28n(j4~dLj~X1{1;}v924Aj^$UI%c3oe63OrU$Svl+R zI!Q9N`0NI5T-!FXZlVyHN!@|l+ z)|)5LHPPy3s=$BLE@iTxsvE^_xAYATHvai@T5@I9^3e0=&--L-q}W=CtX#Ik?@!LeU~3A_w!iabDB_XC ze+;rpN|~nCK6|oXj9%I(dRwRe?wO&w;>nJTGsDHE-W@)|GxB9}GKeTP^P@4o-y-D4 zI?fEcqzXnC_%=o>FO1tPY|`ZA>-kW+@wLI0JZ;1*iIwiR{k}BAaL2F}jvwwSaqqlK$$Hh$a5Laf z#k&6UDzob`4x*Vakqx0NSRxb;O=lAUc-@3?(?HP@2nc6>hxzdKbYQh7S{)L=`pgKSlj zZM#{0@bHGno#lFr!J~yA{B;B8ngiFQR-RO%{i1#G^&!^5_33KvS(V z&uXruuL%oN33=GsGUXEHLBmZqG+jDgOob?YlCNs5Q~FSUZ`b*;>4EHl@n)OSu59N| zJXpclpKp-^b$izlBXe8`VwJw}?nk4kO`~+cFXm!+LT>fTJ0BYxlgW`M%(V5A*Z+=- zkc*t$M`t!!)%`?y{?B3m*S?pN-FC+>M^kR4S&v?*3kB zvq3AD{CdkZmj2#n?GFx2JdV-Y1Oq(N^7L4wb>N&ABE>J}-~lHmCoST2cLV2rHmzV% z8mryq`}1+-*ht0lt-BfdoK)sxN@tU;1E%bYCt94tUYI@Ia{Ds%zq6y&T0%Fb8*S!{ z{eOkqG#pd@ecP*8C9D&Idv>!%f$-oH6nFcFDJgRB1vtku%cj|(OTT{oI_GWn==D9} zDL?Bo6F=)RvP;tCzxWxxK6Gu}mZq>nk5iki&+N}Gp18Bov9~7Px99e;u?9muM_b#V zCF-60J5TGY%=PKvm%B?l+t~##d0BM5y4%S=5FR=H-_{R5`ce{_T=LuBe6@WKPs!=p ztj|V~`f|3z%Aj%fL+-S<8A5?ov-0VsZ2Yz1(j}=0^J6Jh-6v*8a(et>&BSBoC)^fH zmaEL>kaJf0^@{U_IMYDOt5;zG)Ab4c$uZU+U%jGd4pCJIoPSZWFgH9f^JSrg1VL&4 zK5gIrv=nq7-hS;ow?3H?q@8L~kY}i(Kja-a={;UXw>-y|k{G_Q_5Blxh|8CAbHh`g zUH&mJ@VtLm1;HT3Cg7mT{I5u99xXcOgCZ-h56}3{4Cap`Xo;_l9Qd7pa|)PtR2nIx zV_9Ba)TwZVoTSpi?C+M+Kov|WqaTX{cLF!O{}%+HiASSFV6o~SuiBq)OUmJZSIz-*wQIJ|H)3#?Q?kXDf zw;3*7D4CyJnEwL*Sk6qFd+q(v@z8a%stYiNhaUpxfBQ~+KIZj7X);Hq;OHVow2hCA zjjdOitx)-qFyEIj2Bb5PA25)g>pydfi=p}Q@XYk*4*?GYr(y!@E^-Q+vD=;vlqU(n z+NXO}yH9WYbiZk7y0l*x^VrnubdX4xZBP($y!4Ba5bM)jO9nfNElTHFOEZPsrc zcs{^P$*pwmS;+`d z8@LcK_@&gw;@R=Kb+a$m{YXo>n{W+xddRyp1D^SWr%Y`7sh_vz$MhDACfmIe zkgT@1`%kt@EX(4}#=QpIK(yuLVgC@oIY(fH#gTmFyOr1 z>(nVR-Uu4`##)@M;4dZTnx@(K!4r>1-saVGGvv#+l? zK<((xNIchN6i*KPA*?cW@vi^GLH|K#8^dGAgu_EtElzJ+;_01P07T5Y6D3ARfuNWLGs@h&sN~$R&tgx`KrLwSadz98D zu^QK_01SJFU;Msu*Kbr)X|m0;sPAE{8RSX#c1=&@m%g{C%6{6>VcqUgxu)fbefTri z_Vi!BJdlkJ0v}zf{AN0yx8d4_9XobN&76AXGVt^BQ&KM6-g-=S5!J@;^f|C~>(<%Y zEV@dMF=F*93sWD)Gu`~Pmn;7f2oH&xYkx`)D^PWRb!c-$WTf-|Fm}_(NDknCLwkGX z-MeeOyuEF_f^y7J1vsgdrz8ZljMP1I2}o>yNlkFS-tbMgG?zTg4j9kCK1ftC7$H4J zF8qm{o(x>@ZjrYxBm8`Q&09GI2vW!Vt9IemBF}X>qE4u~!j%~?v<_>=n z`~si7UpNh5qF&@huVv#dSdMlHWdZvgocLl^7A)v>g^PKH^!js z*>hQWc4SvE5HN}COhMVf>*jv!r6Ry%ffL6_s5{7R$xfxQax}b7o^(J-GQH6He6m1f zV0P|Xvkd=kw#~GH0(Ut~cXn=OF{nN%ng8tLqn!^v_q)zNX*w=xVbMHhaj?+dJ}bHD zq07=dm;9e)EiXUHw(mT=R(~`7?%fwc85w`iR9sWrwPbbY-P_H-e|6pqN%m~duQ-{f#wa_6W{mkk5e!R2nY!KT0Z*wcbh}VGUyq52saEMpO%>U*XOH8j9J}5o)hp9DnfewdUaHSMl%zGjagQTdCtwcU0q$eDoc9y?T_7NobTG+ zkZBVd|rnC!QRNVygN_lFtG3lgunjUbnx`))2W6jx@0|XBKT6u%hfY&O06|DHK*ri zh6`86(fR#)%}pF~ZNMKp)-^kBs=BX5Jwc%)a^$+Snke~>>7yoT%yTw9d-g1cmgLm* zw3xE8H4u_7DyOL1x6@epwuy*}YCbucl~~k2qCwFe0#dO?xA=TlakVHFY+>sk+Kr`(64s-bb-cY zo_ib{G(?n?tZe*#oxOhj`pJ<_weqyk2M;#7d?~TgQyRZ{=6VU7^(G8d zELSlyFyQF24E|ObIe>#`MnXJLn6f2K+r%V#Ei^HMpP!zW$hA95(2)xjZksAR^q-S7AN7e1zXFN%xhPLJ5KuH9)yfbr6Geu+Tlmd9%11mnyVCVSct z>(cEi&T(AeyH5oR{Ufw_dI`!}Iyy$`>a;Niyq2n}L2pp8D-ORfT<1S_ts`K@WsOQe zoUKzIBD!98as^z1{*e36<}=+%NlCIt|KpjPZZ7R%y~1#E=%^b>{iLuW1;!KeOnSpz z_V)I76uq-Z4Jm5)Fef`Zc8g!vfW$)Tej(0dH#pZw*TDJQlgZ2-Y7?sldGBm<(Gt;p z)7)&jVegd&x4;<-|M{t2^T&tIB1$tpT`92e;kV8$}_Ws5U?qHq<=j`H#@*Sr` z1)S0#7~|v(2UR$)y0~ar>_$N~f6G_(usyl^@?${o%uh%UwJSxo%&SB@aH?J5S1TS~|lT1v2VOjg__eY6k$hWzNHf7sncUQ-F)+h4fnC_?qmQ+<& zd-cY9uehKlVe#zplYFj_&`>$Aej`?1*$Xl(5<)^6@RK+|F2(l=DmJTEuMRsvNBF0) zzWVXm-yiC23OOdICM8w-+}n*9WdD~^m6>ft95DD-uXb+|5^}14xSxE~m97tFLokKX#bs8QGB}JTb#3vEiL1srL59{V9sZ5a<)Yp zr;-LZJS`jfE;WKG%03tS0Nv5dtHeTUzJ>?}aN@ng+5VOTbw<-?RaK2LBmOiLOsVcK6`hm$k%Idg{d zt^JG9D)E~otUsTAP_8dO%gpq3iv(9VmxNg&U!DZu`4L(?6fV$>v&ZHcG_9KxwJ}r3{J`s7M?_mF{t$iX5yWq5x<0< zp42TLZ;Bdz+B&-8kqMrw6jS=+)kV%Q13gkHB|mt;{%X&y_ebB~Rk8rb8=aCO99+Pp zr>8eJKUvyrsHgP0X-&-LV>SGUXdGe8hrEa>Q1XxnWt|#CBrOZBx~NGB3JdRjdi4EA zAY;agmxQ(?lmNX4hqAJ=?@(c{?2)(Z4c;kCMBxJ+P+28E>-re1Y@<)7Tsz`vE z-6;j?9v&X%xegpLmu$+LGA%xfe|*WdX2*kL3I5RnQtJf-REd>OPYi1FxMq0|clb_q z$M_r`T0yAXW}Cox0ZW#%(qwuKYQSK54t48YdrmQtpp*Y4K%1RS-_r6CrkMWVL6}TB z2S5KK1Uh4@=chKY?7I%DB>#<)?~(e=U+nQsBg9>BB`Jd``g?(TF{^7y_wO@(Ti^5F z?fYY6Cqq*2;|%*px+>@mB5)e8@qL4>`NAx5DHzc*!eh0b^3(&}yE`@6r9!A_kisvp zapP`M#*$QjpLfZUC4>iHN@5!w9nJNdIuVk3E;2F_CY1tASFW0eEKOnzf^eja)adLe zd>SX5NDViXrA#a=4WNPd_MPNVnH>q;cKnknyQ#y}Xt$Vz#KsHEw{(Qw-`;v&-opJL zRGziy9&~r9tucv*af@%*5Gr6vIR*_Bg`#lv^`W}iso$$suH4PjXoh9NX_%6tw`|>- zccNP>B=rqoKoTNZh-$>E%1Xk)ZMxtTx7|~tI$ClABe&{eoyx|wYE?4M$)uyBBg`W> zbm{W`8+yuu7tD#rS5;Lpex*J9=5ifkf-ZY%l!a^I9-s{23r>#?KGhHv6{YX`PIV4S z;@XpaANRnP)?aXX0lW$d_)_A}l*CH{8olt| z#hHwM2Y1*!95AUEQlNV3)H8xCLQ(~*Vr)jo#~c598p5PsF_)tc<7H)?D3 z_a`XF3#31Wt*N^o`fD(eJC&5i0e|0CR-Tth7ZDTt2uPyJ9`g%~B}e%Z;NZkqx02fe zo$R3utRIo9n7-1Iq=MwkRIa&*+ffBam=iaBKwLYx+&v4*^Hls29k|lk5 zfzzC7W%nNH#;1b?ydxkrf*isba)avqlpqFoR7m2=%6A3Q_i1a_qb6Yby8a6P_9W2z zoV!u#UZ`gV!)`|rqJWKZ9> ze}DITVc}O#HxY}M&AGN=ub7C)1%cxwf4(wD#l&Dw$m!F^{`ld5guQ~EcQ?94vhJO# zA*mtqiE+ypql&CdU|?Xx>aFJlj-T-KjJk72i+!6(hVi?1@96C+B3f!|T}$Uqsiz)7 zAr;nniuQR%!YU(s@gs=Ny$c- z8$9IS7LAUL9l2h26lAw+0!{MdK{Yisr^fVEYKe6)m2mdspIqPg{Cn;F5hZbHX+1|r zN7wk5$k5w4Hlmz;cVr6(J&T9bSBWgtn184Kw&>KL7UQ-dOOfwAJ%lfeqyM^}-Cc^8 zW9E_K^1rh^Qaat3{k^YGll{c7Beb`gSqOgk8A62#g&kuo1~MwV>dZKpf?- z0d29dvGMv*8!Yg>!-ChnFxM)aOOlc1TwRI(8TG42^2|x|FT|a8aUQl5mzG>X`uO-0 zlA@v>)s;2##m7($iEP{$CSc3crdlhtY13Kue`E7dI3te1mYo;o=Uk^gwU2w)q}o?U zx8<;&Qxx`Sg2^+!WLn&eAe!BK_Hcx}N4RMp+RV+xC5AFR^h4>;=ih8Zm-zjOwzjtC^UAopcWZ)s zztFOXA=#nZO)n^lLKO@Jy`B5Nb%9oBL@3h;G?(z$7CIC;IT=ijbF!5jsds|4&O{{VQh;4Thy{Lb9?q z*=_fsB-Tl2<==kd)!@WI?`?lq%#Lr97$7)#XC!$fkC}-{ke=r{8aF^&i-rwQ%~)Rc zZx+ZOqA!lOrH$;@)9Vdtb)TEL(2yqB?L^rbCFv8 z5aY733DNU#ozY2H;Aj1|7+WcH_ePX5a&sG8T#xkV&aCpBueiOr%Q@!uZB0gozuAuA z=uUtT?%axyi08)(zdc%9 zWy0&TasRXcWvaJE@;uMKYmSTKXUk=csSfpQfamh^{WS(gL_uU>tNG3I07U!_d=G`~ zPQoOTlamq11sFsB&sy`UtE)YyD))6cSI!s96a@uQP(y?X94}CRS@3szRbdVY;25oF zSDKZfsg>6%!Ju9TVf8L&%fAOE0Lumcwt+ETa+kQ6nC0~5EnUvs?Ch$HGA|Mn1+Z7= zUA*H~{hc1mncH_!97QE041bKgR^BWnBXgDA_Own?D|Y(dkTRBAj`#MTO;tQ@VPL59 zr+hW62YLr=@ZYSGeN%_ZhiXrzv>s_{NJw5dW zIABvzo|XT0;cWIA&XS?v>hCWL`{P9XBK~G*mUA#1+_Kc@s@}f5x1wo7uehkFAU)mR zGYh>p`^2-G$;LF`c-(JyV{2>cF#RQ*C2w{wrX7Y|Q}*7Y-!)_WkK@3R&js<>C{NT= zL&}?P{hM6Dkxy9#1)uDtS4&Ar^-x;>087xug#dC08oi8W``=&{VZ&1h7nY+Qkj0z} zb*_lzcjr3#^x)dH4UWAqIL>3_J}iZYrMr2`GH;~&fP>TRo<#}g9b*Zf;6 zXGyPTqo&!V6*WEz!c?dEKg)eymhizi+yLKw*z(dd7a&JY=x?}cVV0XXzjL|8jsgTQ z1W5Yiw*IJ)e-Dl@0I`BaLn-!Yq1woFO!SBMuF?4A@?ylAqPM zhZU2MUzZMy11EX-tK?xagv{4)IX%4zja)nJrOo+h1a$ZIa?;1CgV+=NG~!>C==|@q z>~JWU3f zFPBgh@P?orT|;%JbR009^Cf2A9=b%qDxDoE6sMwd0<;vbu5aIVF>XuIOAsQANy9B- z5)=ym6?}TQ(b9XQm{N6-*-5kbN&N_NSE&1-&v*Ck-7s);3fKWJga%VfOC18TY$aJK z4n~+&e^__crLmppZxB_-_f#W%e0(@)DP>)YZ@-GPx@y%b4l+;b>ydZK zEGPYt@vv+>!Q&$(NTY)77nvqIIUY*tFXj`jEheHeGWuwxEmx!H3Q0*_VZZ!m?kppv z;kB`;)gDG;8RE71K3vdT#Db@@DEQR8=Y7+*u*-s9U?T^?v*?3NcKT4ThIk zL)f90=cq%=ky@6~*q!bO#hrzmo(7UZG|~B|iVFFCAe(tbjl4%)U7ay_j~IU7bWzFg z19?p+4>{S(&zW0^=Mb*L=~Zj1#r*s{Fn;5SzduLSdNjNH^ zWrBYG{`j?WYV6l9g@+e*k==8;OigK&RHRS}TYb8*xO&cZpV5tQc1ohgVwC*%L$pJ& z?Qh=-P`91YwCXc_PqiKYxbwJfvx|w2)*zmqdhO~}cJ!83gcZ%+RZEQy#|sL&cnfn(6?PrhRli1N(Q}wwKs5(pE0qr0voJTyFlHby!xa)%<8qD z&)BWx=H{fQG|-UOe0}3Gan?ywlSz#_VK_6-T}n_90#v6y&aJPnFAP5j;=j4y@Ilw+qPvf*GAj516z@f}IY2x1}n8w3RfL4j;yB#X5o zizPZPoZ+H#N1*sszk2e|h@CnmXa{V$Q{TYA01YR4LDy~gfS{JXw!*D~pPK4`y69;r zrdUl-1+}Dm^*+NSRxBl#QoP`nWnH&)6vROeqAicNVERBBil;{e1Sw^! zkpDZ~t%^~(I)|2(%{eCD0ITmqX@xWDH0&8-M>A18Zi5dbWSe`Q1|2Ut( zk}x8#BlO9CZoIkb!v|qvx0WU-D_0^0vR@}XWI|d6S!_oG{pf$U?tM_6(LpJrs_#q! z-F{Tz-GhVs*{^G9Yv-LBG!Mc1%j^*~Tth5h6UvRo)gGMUG&nW6GZV*2cbYU+S2ipo ztNb=Nn2B1NGt8xKe(3pdIzkm5qDyfM(!Dutsk~^}F z*N?uxllyS|Z_|#JF!1o$tcIC%UN5hxxILPAsNx)J%QF`~ zwX$SF)TX7SanJcWI;IiU{rON)_?*87bqHfy+eE@Ivv2$S`SWyV=yFD4ZZV|02x6x$ zX-^m>X8iN91jp{GD7gTuck?9$D`7YiMtq;3Oim67p`~u%CRQF9)uT^u3mhj(N%#l{ zy=4tPYYcjbz`Xh4PhYsXcflw-HvMH!j^m)l-|)FntxoFQp~vodZA-W$&G8I+cTr+4 zmsf8+iS>)m2j-MYO92}vc7QKSLpzn;;5E9I_typd2^PRy-8FZeV-XA!BXl3NkJq^Z zr7uylrQ%P`g7n1X9%Jkl?<-dMHdRLMTZ-ON7I>&Pwra(S6%7wGIYUwpHKrR+<2)H@ zbtUEHP0_zH1d+OHq4ei4GYnRhPRa=Ux;DEpzv3w!Xs&3fTb+a!W_qhzX}n`G^Um=G zLny~Ls+EbNZ{UdL5X8Rj#dE8F4i9u7Hpwyq!nX%5%+Y%U>#58cp)0;*aS+m|DJui< zwjN;v#wwv!*2ll&w_Wgk|Ayd%uT03T7V%4@x1p6zaX(2BXQspP9YsI zjE~TiUO?BFiB|V81Or58;sxa#!<^^mFU@l>JO_QWJcPoSzxVbIbe&&kqN~ehhanQp z^V?6wl6d-+7H!KXO<7jj>@qJPDsVuH3~uI@TS_-GQ$)BRRGf5sM*xiP6y$E3lB%*dDu6*4DqRTJBrMFyz-x(S8^~~i2Uee8KHm@YLFju7 zO8lq}vXE#ui5NnIJGbOIk`@Rap-nLw{q?I3J*?!QAWC_vhPCxwBESZ%c2TBQ0LPGf zcX-_?%?Im^m6|SLU^u~Ug;r9Kh8!AK5j>ki1f;&6Ul&k|Hrh$lqGS(L zli$AGH`gs4D0m^9=&Fgp1dJ#LLx1DK{GX^r%lWcp_z%EE$YUN`%rd}CM~|< zSyUPbla{k9r)PUWx_-4kAQq8DpH6B+VVLKcYeW=HqzEB99-C)EOT!HJ>ZFBJD_eQK zm!tP!B91W*+2(@b(W63ki8afXEz3Rq>kw9}jt8JTHdHmfKCOnUG~dgLNR@Z)`mJ00`YhYGpn&P~3#b5g znHRXJ7k%2m|kPGbm^l;Xla`)(4Pm!Xepf! zNw={|JX>VTmg_Q3wor@^!D*gTqZnzBT;OZPl^KE`@Oq;6GDJSs-YEr<15R6P^X6-z zKQ(v~=xoiHn3$-_b6h7#0~}D5Iy^T$PA*KjJ|Emhw7bieFBe`ZuMb5Cl*86UY*RAT z!-p`&4uf%(0sv&@mTG{fq5jhlh%rEq0~<=@Ne!Ulp_UOKegDgR3b>erz?>oH+~fex zTWa#pUk9kNmPdXptIaB&FsF*3A^youa zDJ*7sAMWi411`JY1$skwTc2pW4j39TJz!v4yE9cd!S4XNjZHW-S?Ehh%j$ACVsahi z6Jzg$6C8juq!d4|L2udTcaJDiL+H>sBr6_kKff2z5(ksW5s*xEm6x~>09NkN_q+rx zpr|l`V2tt2JK$$%GWJiyw+i1OqZK?tn~@kTRpO}Fpbj|}Y28a&6&Y>NxLk`CbR+oN z5RXH!D3$p?_mvlBJz?*n($XK2*STS}vVrq{&rW$9hE>F=lg>E89-6~@zxe%T4Q?oh z#4XpgeaHBZx?Ske8epbi$6qR3L`8GCWwaD{+&^*RMB7)7%Ga-{!Y+M%cHv+m#Ojot zotvZHJs@fcE@^8WfxKL=0TYNKjqB>_cok1Qf=XtGkMF2U*#)xu;{(kD02sz6PHYKA zF|lKF?E5N?g&>gK zWs=dDxuu}%9G|Rbj}AI3?&<~Gt^4TQ#e{BXqI@&0ys`$yCBcm0xi z6GfnPv9Cdv+Nt89AcP!bdr5>TJ1dfkkVx4Qic(l^r1sxigy^l*S{j{UeF znU0j`3t1b)O*$f`KhR>H#&wi&$Xbvv(vXFtLRUh}v|eE8qx*&j4xGgM*C@EF!el*L zMLYW2PPMw+*1@THu6)k-4p0*;ieFJZeH9sLk;4+|pr2Z7R%ZKDFa z(L>MD1m?)}V+$ygdr-5X+qjPy1f;4ruU%6^-BRZ6TGS^*`mupCtzQZ@KcO?rG$Ha# z(wird)+Ezhbn|8d1hm6fziz?OVQdd?-du@3p97Q#ngQhW^KuYwMw@-^-KA zg*}YHg~q)^#x`6$NxcCe73qET#fzOpE=b%Q6HmQC zq)uD9_yGt2^|Yl}WfSB?d2rCG52S3u7fzp_B*QQ6Vd6-62at0i1(E&lOEH#+43D-_ zEi|M^u8*oBQK*7>&;c&^c?^$$XuFhiU;0}iZb3VVPInNM(q^)^5x4( zhP)$V7lO#hL@!6Uu1zS$QF-Vf98mRgKp7)aC+MImIRJ?)bmZbH$MVeWx7=RI&YlRu z3cie}r33J2h)qSf&MN8`Ll0~>IxC|S6A*Z%Ky?Z`+5~p?V5SKqa7ffT_y!ck^~mO= z&aC=iI_G0&cMoOhhnYW+2a7XzBqgn9nKKPf#=o|LsqG(i$A$i=Ya_9PXVKrTw~wicaY{_9<8YQey- zRPu@jT@Zz*pC6Ii!LW=bQXCV(o{|Zlfvam48h=dX`-zAKIJ|bheEatU7Cra6PNSpY zUHimdo9N)k9j@ugVLbtY^I6Gji5wcTTcUI(=wq3_9uA0z6N^z(@@_lMV=tY>15eLi zv1Uys+_De*V~9(jOoT;B4F=mm{WuIkZrT{k(z1Prl9F6$%`nYw0-nl1g`%z&PsmVh zaAI{tW=PO0Wb*X17^6XT-@XfLeSZj2KM+#Uol2iYR}>t8z>efelWglu%^O0i`!$-~ zU3YG!b9eTi9y5UCdmOS}D&WCQ!CfL4sLY;}PN{A}|A)$U#POX9CAgG_dEEuge0N zp++_alLq6ntUS0iJR+j;?X?YZ(Bw>%YJ`HG3<{Sp8wQ+bML^?|cWpqzA4lTnTpSE3 zyx4dAK(=);B4ctOu0v1b=0j9$thR!?I2wM$G6--g^gKqZM!Kaz+9M+)`s(T-2%5x> zU|W4;L8LJcpQjXr0T7Z>{o9)e>SYJ0u`I}?0G@(bpM}^jvl^vygKn#Md9?%_R+7c; zLYY~I7o8!T84|P6fdkhGUPYxqe(xbgs=TstFSweztAe|bF*9*z3o|F zdJUl;*eIjYfH~NywulUAqomTI6-uBgxlI6vBgL3M?)HIKi$%|k=B4|K7cXvJy_%>N z%Fss?%y3@m=`-lZdwqSyWPjv{QFK|aRu_P-n6z}HDjrI|im(;9mUX}K+6`|!jXOhg zx0=zJp~Af+;o(YW8m>pU-eK7le(_>mgnUOT1{Q_-Mb&m6Hn>xOhK7b*7xF^!h+YXl z!O0ngg##U`2X_&AT?y(fa(3s~)}%jqvW477BsPldaPmj3Agy}w?_a-&dp4E^29hm> zlAUc~D`Ltj#E&08e%Sa8LaA6)Q=^>CqHM~TlAJ86qLL7bXp+8)lk@(qTRiOu9$=!H z!7ApGi$nnGpe*Ube0x#XhK11+%mcC9{^Cc;!PBIdKK>u{Ti_u~amz~1XY@#NN{TbV z=Q(zv0e`xd61=#AKM8zyQO_Och7`SZUJzm;yhP-ejf@$=&B3HLLc?a`SDcvG$!+hh zp>YmZ6K>+R4Yq>~u?06n5*b)|A7Hz|c8P7;l>B729ruNBWntGo_X`lUjt{n)0zDbx zPSJO;&LZySILO*El%6D15Ko3C3m(u!KrFe&)>dc;M+^(&eQYLiDKgR&>7pUSR63<) zmxO>o;tAhgbwG_Bps>16aiB#4J5Oy>EXc{x2gR+eh+=xH(2caw{v_P%&cJs8BE-qEsq?2Mf$zFN@vo+BldR`KPLcai4GoL^P2IA^w`lb#Bh88kd#7{dzmPABdqb(8#3~6cmJK$2A(0#4lSC>&1qibe+k!qHH-V6x2$?^9GJE~z%@dJ4h=kEhO~ehTKjGOp zVx;(tmcCBtmr3(i08yw4Q3zcLV?rh&yJ>y|jDGO_`w@(+4PNSimew-0LK>v>Teogi zT^xbsGoGe(}o@!)BF1gc@gF>fEM&UuduZ%`S>gj zj^K`Qb#rr!t8DQ&phHBW6g;te_Xv)Q&#M%Q2K_)>NFXeIKf=+?jd0w4 zRPN-iIqE^P!iPQk?Rs%V%Ne3NHI$4_UrLI}1r`*{A;A{NUKC+Y9-dU(DM`fvMOppXCoWOnU3dh5a0~~>HGLVKAAwQ7`q6juZ$HJ`Al_b5)K24 z-oAbCAZ>zp2rl1#7OPT4StAzS?%C}M!vY~|SWY8l`(*Uz5{eWYfZ#kWJ-ucu20L#u zbr{4Aq4Lt!BK(rN33sNlBu=&FI^G99;6Q&mh*Lq{$ip=})w6XEd78rvLj!3Mlf|Ki zr7jM$1!5(d2{wH?5(UXfNFQ_ITVMsdz+UwBaLHMQY3$xDvUBH)(Mr&^;t~=%P?#PI zM_8k9o3#7W)%b(Czsx6Xb99YA7+-N&StGn4d)5meLw(mSNJAwcbq%5I4}vF*FvA5Q z;H*cEzX=i&9bh1|0o-7Q;}wu<3PeGS__^ZZ;#Fwm5Pxv$T5POE;Tz@rho>gQhtb3a!uanfDL%xi+dNh7il4v}i3z2_upx+n z7=OVbBJ={_1tP^|y<{-humoIpOdb8*8z3Pff==Z%Bt({MIsESN4otT(V z9{z0{Xw^QA{~ch9&LCRqVq|jhjGwm^6``*;;5gjRVg%f~6GXBur?e4{aPdz8 z`_ is an industry standard tool for +distributed hyperparameter tuning. Ray Tune includes the latest hyperparameter search +algorithms, integrates with TensorBoard and other analysis libraries, and natively +supports distributed training through `Ray's distributed machine learning engine +`_. + +In this tutorial, we will show you how to integrate Tune into your PyTorch +training workflow. We will extend `this tutorial from the PyTorch documentation +`_ for training +a CIFAR10 image classifier. + +As you will see, we only need to add some slight modifications. In particular, we +need to + +1. wrap data loading and training in functions, +2. make some network parameters configurable, +3. add checkpointing (optional), +4. and define the search space for the model tuning + +| + +To run this tutorial, please make sure the following packages are +installed: + +- ``ray[tune]``: Distributed hyperparameter tuning library +- ``torchvision``: For the data transformers + +Setup / Imports +--------------- +Let's start with the imports: +""" +from functools import partial +import numpy as np +import os +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torch.utils.data import random_split +import torchvision +import torchvision.transforms as transforms +from ray import tune +from ray.tune import CLIReporter +from ray.tune.schedulers import ASHAScheduler + +###################################################################### +# Most of the imports are needed for building the PyTorch model. Only the last three +# imports are for Ray Tune. +# +# Data loaders +# ------------ +# We wrap the data loaders in their own function and pass a global data directory. +# This way we can share a data directory between different trials. + +def load_data(data_dir="./data"): + transform = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) + ]) + + trainset = torchvision.datasets.CIFAR10( + root=data_dir, train=True, download=True, transform=transform) + + testset = torchvision.datasets.CIFAR10( + root=data_dir, train=False, download=True, transform=transform) + + return trainset, testset + +###################################################################### +# Configurable neural network +# --------------------------- +# We can only tune those parameters that are configurable. In this example, we can specify +# the layer sizes of the fully connected layers: + +class Net(nn.Module): + def __init__(self, l1=120, l2=84): + super(Net, self).__init__() + self.conv1 = nn.Conv2d(3, 6, 5) + self.pool = nn.MaxPool2d(2, 2) + self.conv2 = nn.Conv2d(6, 16, 5) + self.fc1 = nn.Linear(16 * 5 * 5, l1) + self.fc2 = nn.Linear(l1, l2) + self.fc3 = nn.Linear(l2, 10) + + def forward(self, x): + x = self.pool(F.relu(self.conv1(x))) + x = self.pool(F.relu(self.conv2(x))) + x = x.view(-1, 16 * 5 * 5) + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = self.fc3(x) + return x + +###################################################################### +# The train function +# ------------------ +# Now it gets interesting, because we introduce some changes to the example `from the PyTorch +# documentation `_. +# +# We wrap the training script in a function ``train_cifar(config, checkpoint=None, data_dir=None)``. +# As you can guess, the ``config`` parameter will receive the hyperparameters we would like to +# train with. The ``checkpoint`` parameter is used to restore checkpoints. The ``data_dir`` specifies +# the directory where we load and store the data, so multiple runs can share the same data source. +# +# .. code-block:: python +# +# net = Net(config["l1"], config["l2"]) +# +# if checkpoint: +# net.load_state_dict(torch.load(checkpoint)) +# +# The learning rate of the optimizer is made configurable, too: +# +# .. code-block:: python +# +# optimizer = optim.SGD(net.parameters(), lr=config["lr"], momentum=0.9) +# +# We also split the training data into a training and validation subset. We thus train on +# 80% of the data and calculate the validation loss on the remaining 20%. The batch sizes +# with which we iterate through the training and test sets are configurable as well. +# +# Adding (multi) GPU support with DataParallel +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Image classification benefits largely from GPUs. Luckily, we can continue to use +# PyTorch's abstractions in Ray Tune. Thus, we can wrap our model in ``nn.DataParallel`` +# to support data parallel training on multiple GPUs: +# +# .. code-block:: python +# +# device = "cpu" +# if torch.cuda.is_available(): +# device = "cuda:0" +# if torch.cuda.device_count() > 1: +# net = nn.DataParallel(net) +# net.to(device) +# +# By using a ``device`` variable we make sure that training also works when we have +# no GPUs available. PyTorch requires us to send our data to the GPU memory explicitly, +# like this: +# +# .. code-block:: python +# +# for i, data in enumerate(trainloader, 0): +# inputs, labels = data +# inputs, labels = inputs.to(device), labels.to(device) +# +# The code now supports training on CPUs, on a single GPU, and on multiple GPUs. Notably, Ray +# also supports `fractional GPUs `_ +# so we can share GPUs among trials, as long as the model still fits on the GPU memory. We'll come back +# to that later. +# +# Communicating with Ray Tune +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# The most interesting part is the communication with Tune: +# +# .. code-block:: python +# +# checkpoint_dir = tune.make_checkpoint_dir(epoch) +# path = os.path.join(checkpoint_dir, "checkpoint") +# torch.save((net.state_dict(), optimizer.state_dict()), path) +# tune.save_checkpoint(path) +# +# tune.report(loss=(val_loss / val_steps), accuracy=correct / total) +# +# Here we first save a checkpoint and then report some metrics back to Tune. Specifically, +# we send the validation loss and accuracy back to Tune. Tune can then use these metrics +# to decide which hyperparameter configuration lead to the best results. These metrics +# can also be used to stop bad performing trials early in order to avoid wasting +# resources on those trials. +# +# The checkpoint saving is optional, however, it is necessary if we wanted to use advanced +# schedulers like `Population Based Training `_. +# Also, by saving the checkpoint we can later load the trained models and validate them +# on a test set. +# +# Full training function +# ~~~~~~~~~~~~~~~~~~~~~~ +# +# The full code example looks like this: + +def train_cifar(config, checkpoint=None, data_dir=None): + net = Net(config["l1"], config["l2"]) + + device = "cpu" + if torch.cuda.is_available(): + device = "cuda:0" + if torch.cuda.device_count() > 1: + net = nn.DataParallel(net) + net.to(device) + + criterion = nn.CrossEntropyLoss() + optimizer = optim.SGD(net.parameters(), lr=config["lr"], momentum=0.9) + + if checkpoint: + print("loading checkpoint {}".format(checkpoint)) + model_state, optimizer_state = torch.load(checkpoint) + net.load_state_dict(model_state) + optimizer.load_state_dict(optimizer_state) + + trainset, testset = load_data(data_dir) + + test_abs = int(len(trainset) * 0.8) + train_subset, val_subset = random_split( + trainset, [test_abs, len(trainset) - test_abs]) + + trainloader = torch.utils.data.DataLoader( + train_subset, + batch_size=int(config["batch_size"]), + shuffle=True, + num_workers=8) + valloader = torch.utils.data.DataLoader( + val_subset, + batch_size=int(config["batch_size"]), + shuffle=True, + num_workers=8) + + for epoch in range(10): # loop over the dataset multiple times + running_loss = 0.0 + epoch_steps = 0 + for i, data in enumerate(trainloader, 0): + # get the inputs; data is a list of [inputs, labels] + inputs, labels = data + inputs, labels = inputs.to(device), labels.to(device) + + # zero the parameter gradients + optimizer.zero_grad() + + # forward + backward + optimize + outputs = net(inputs) + loss = criterion(outputs, labels) + loss.backward() + optimizer.step() + + # print statistics + running_loss += loss.item() + epoch_steps += 1 + if i % 2000 == 1999: # print every 2000 mini-batches + print("[%d, %5d] loss: %.3f" % (epoch + 1, i + 1, + running_loss / epoch_steps)) + running_loss = 0.0 + + # Validation loss + val_loss = 0.0 + val_steps = 0 + total = 0 + correct = 0 + for i, data in enumerate(valloader, 0): + with torch.no_grad(): + inputs, labels = data + inputs, labels = inputs.to(device), labels.to(device) + + outputs = net(inputs) + _, predicted = torch.max(outputs.data, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + + loss = criterion(outputs, labels) + val_loss += loss.cpu().numpy() + val_steps += 1 + + checkpoint_dir = tune.make_checkpoint_dir(epoch) + path = os.path.join(checkpoint_dir, "checkpoint") + torch.save((net.state_dict(), optimizer.state_dict()), path) + tune.save_checkpoint(path) + + tune.report(loss=(val_loss / val_steps), accuracy=correct / total) + print("Finished Training") + +###################################################################### +# As you can see, most of the code is adapted directly from the original example. +# +# Test set accuracy +# ----------------- +# Commonly the performance of a machine learning model is tested on a hold-out test +# set with data that has not been used for training the model. We also wrap this in a +# function: + +def test_accuracy(net, device="cpu"): + trainset, testset = load_data() + + testloader = torch.utils.data.DataLoader( + testset, batch_size=4, shuffle=False, num_workers=2) + + correct = 0 + total = 0 + with torch.no_grad(): + for data in testloader: + images, labels = data + images, labels = images.to(device), labels.to(device) + outputs = net(images) + _, predicted = torch.max(outputs.data, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + + return correct / total + +###################################################################### +# The function also expects a ``device`` parameter, so we can do the +# test set validation on a GPU. +# +# Configuring the search space +# ---------------------------- +# Lastly, we need to define Tune's search space. Here is an example: +# +# .. code-block:: python +# +# config = { +# "l1": tune.sample_from(lambda _: 2**np.random.randint(2, 9)), +# "l2": tune.sample_from(lambda _: 2**np.random.randint(2, 9)), +# "lr": tune.loguniform(1e-4, 1e-1), +# "batch_size": tune.choice([2, 4, 8, 16]) +# } +# +# The ``tune.sample_from()`` function makes it possible to define your own sample +# methods to obtain hyperparameters. In this example, the ``l1`` and ``l2`` parameters +# should be powers of 2 between 4 and 256, so either 4, 8, 16, 32, 64, 128, or 256. +# The ``lr`` (learning rate) should be uniformly sampled between 0.0001 and 0.1. Lastly, +# the batch size is a choice between 2, 4, 8, and 16. +# +# At each trial, Tune will now randomly sample a combination of parameters from these +# search spaces. It will then train a number of models in parallel and find the best +# performing one among these. We also use the ``ASHAScheduler`` which will terminate bad +# performing trials early. +# +# We wrap the ``train_cifar`` function with ``functools.partial`` to set the constant +# ``data_dir`` parameter. We can also tell Ray Tune what resources should be +# available for each trial: +# +# .. code-block:: python +# +# gpus_per_trial = 2 +# # ... +# result = tune.run( +# partial(train_cifar, data_dir=data_dir), +# resources_per_trial={"cpu": 8, "gpu": gpus_per_trial}, +# config=config, +# num_samples=num_samples, +# scheduler=scheduler, +# progress_reporter=reporter, +# checkpoint_at_end=True) +# +# You can specify the number of CPUs, which are then available e.g. +# to increase the ``num_workers`` of the PyTorch ``DataLoader`` instances. The selected +# number of GPUs are made visible to PyTorch in each trial. Trials do not have access to +# GPUs that haven't been requested for them - so you don't have to care about two trials +# using the same set of resources. +# +# Here we can also specify fractional GPUs, so something like ``gpus_per_trial=0.5`` is +# completely valid. The trials will then share GPUs among each other. +# You just have to make sure that the models still fit in the GPU memory. +# +# After training the models, we will find the best performing one and load the trained +# network from the checkpoint file. We then obtain the test set accuracy and report +# everything by printing. +# +# The full main function looks like this: + +def main(num_samples=10, max_num_epochs=10, gpus_per_trial=2): + data_dir = os.path.abspath("./data") + load_data(data_dir) + config = { + "l1": tune.sample_from(lambda _: 2 ** np.random.randint(2, 9)), + "l2": tune.sample_from(lambda _: 2 ** np.random.randint(2, 9)), + "lr": tune.loguniform(1e-4, 1e-1), + "batch_size": tune.choice([2, 4, 8, 16]) + } + scheduler = ASHAScheduler( + metric="loss", + mode="min", + max_t=max_num_epochs, + grace_period=1, + reduction_factor=2) + reporter = CLIReporter( + # parameter_columns=["l1", "l2", "lr", "batch_size"], + metric_columns=["loss", "accuracy", "training_iteration"]) + result = tune.run( + partial(train_cifar, data_dir=data_dir), + resources_per_trial={"cpu": 2, "gpu": gpus_per_trial}, + config=config, + num_samples=num_samples, + scheduler=scheduler, + progress_reporter=reporter, + checkpoint_at_end=True) + + best_trial = result.get_best_trial("loss", "min", "last") + print("Best trial config: {}".format(best_trial.config)) + print("Best trial final validation loss: {}".format( + best_trial.last_result["loss"])) + print("Best trial final validation accuracy: {}".format( + best_trial.last_result["accuracy"])) + + best_trained_model = Net(best_trial.config["l1"], best_trial.config["l2"]) + device = "cpu" + if torch.cuda.is_available(): + device = "cuda:0" + if gpus_per_trial > 1: + best_trained_model = nn.DataParallel(best_trained_model) + best_trained_model.to(device) + + model_state, optimizer_state = torch.load(best_trial.checkpoint.value) + best_trained_model.load_state_dict(model_state) + + test_acc = test_accuracy(best_trained_model, device) + print("Best trial test set accuracy: {}".format(test_acc)) + + +if __name__ == "__main__": + # You can change the number of GPUs per trial here: + main(num_samples=10, max_num_epochs=10, gpus_per_trial=0) + + +###################################################################### +# If you run the code, an example output could look like this: +# +# .. code-block:: +# +# Number of trials: 10 (10 TERMINATED) +# +-----+------+------+-------------+--------------+---------+------------+--------------------+ +# | ... | l1 | l2 | lr | batch_size | loss | accuracy | training_iteration | +# |-----+------+------+-------------+--------------+---------+------------+--------------------| +# | ... | 64 | 4 | 0.00011629 | 2 | 1.87273 | 0.244 | 2 | +# | ... | 32 | 64 | 0.000339763 | 8 | 1.23603 | 0.567 | 8 | +# | ... | 8 | 16 | 0.00276249 | 16 | 1.1815 | 0.5836 | 10 | +# | ... | 4 | 64 | 0.000648721 | 4 | 1.31131 | 0.5224 | 8 | +# | ... | 32 | 16 | 0.000340753 | 8 | 1.26454 | 0.5444 | 8 | +# | ... | 8 | 4 | 0.000699775 | 8 | 1.99594 | 0.1983 | 2 | +# | ... | 256 | 8 | 0.0839654 | 16 | 2.3119 | 0.0993 | 1 | +# | ... | 16 | 128 | 0.0758154 | 16 | 2.33575 | 0.1327 | 1 | +# | ... | 16 | 8 | 0.0763312 | 16 | 2.31129 | 0.1042 | 4 | +# | ... | 128 | 16 | 0.000124903 | 4 | 2.26917 | 0.1945 | 1 | +# +-----+------+------+-------------+--------------+---------+------------+--------------------+ +# +# +# Best trial config: {'l1': 8, 'l2': 16, 'lr': 0.00276249, 'batch_size': 16, 'data_dir': '...'} +# Best trial final validation loss: 1.181501 +# Best trial final validation accuracy: 0.5836 +# Best trial test set accuracy: 0.5806 +# +# Most trials have been stopped early in order to avoid wasting resources. +# The best performing trial achieved a validation accuracy of about 58%, which could +# be confirmed on the test set. +# +# So that's it! You can now tune the parameters of your PyTorch models. diff --git a/index.rst b/index.rst index 9a8762e8026..8274715b367 100644 --- a/index.rst +++ b/index.rst @@ -253,6 +253,13 @@ Welcome to PyTorch Tutorials .. Model Optimization +.. customcarditem:: + :header: Hyperparameter Tuning Tutorial + :card_description: Learn how to use Ray Tune to find the best performing set of hyperparameters for your model. + :image: _static/img/ray-tune.png + :link: beginner/hyperparameter_tuning_tutorial.html + :tags: Model-Optimization,Best-Practice + .. customcarditem:: :header: Pruning Tutorial :card_description: Learn how to use torch.nn.utils.prune to sparsify your neural networks, and how to extend it to implement your own custom pruning technique. @@ -479,6 +486,7 @@ Additional Resources :hidden: :caption: Model Optimization + beginner/hyperparameter_tuning_tutorial intermediate/pruning_tutorial advanced/dynamic_quantization_tutorial intermediate/dynamic_quantization_bert_tutorial diff --git a/requirements.txt b/requirements.txt index af5eec5392b..87f0fcfed1b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,7 @@ bs4 awscli==1.16.35 flask spacy +ray[tune] # PyTorch Theme -e git+git://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme From f075938e0a09e28a10f0a19b757a21c589695a30 Mon Sep 17 00:00:00 2001 From: Kai Fricke Date: Wed, 15 Jul 2020 12:46:16 +0200 Subject: [PATCH 2/3] Use nightly ray release --- requirements.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 87f0fcfed1b..a211f877eac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,12 @@ bs4 awscli==1.16.35 flask spacy -ray[tune] + +# Replace this block with ray[tune] after next PyPI release +https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.9.0.dev0-cp36-cp36m-manylinux1_x86_64.whl +tabulate +tensorboardX +pandas # PyTorch Theme -e git+git://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme From faccec1cb9cc593deec7c3378fd89e0760064948 Mon Sep 17 00:00:00 2001 From: Kai Fricke Date: Wed, 19 Aug 2020 10:58:20 +0200 Subject: [PATCH 3/3] Fix checkpoint API --- .../hyperparameter_tuning_tutorial.py | 60 +++++++++++-------- requirements.txt | 7 +-- 2 files changed, 35 insertions(+), 32 deletions(-) diff --git a/beginner_source/hyperparameter_tuning_tutorial.py b/beginner_source/hyperparameter_tuning_tutorial.py index 7f1dda1287b..11524618cba 100644 --- a/beginner_source/hyperparameter_tuning_tutorial.py +++ b/beginner_source/hyperparameter_tuning_tutorial.py @@ -14,7 +14,7 @@ supports distributed training through `Ray's distributed machine learning engine `_. -In this tutorial, we will show you how to integrate Tune into your PyTorch +In this tutorial, we will show you how to integrate Ray Tune into your PyTorch training workflow. We will extend `this tutorial from the PyTorch documentation `_ for training a CIFAR10 image classifier. @@ -62,6 +62,7 @@ # We wrap the data loaders in their own function and pass a global data directory. # This way we can share a data directory between different trials. + def load_data(data_dir="./data"): transform = transforms.Compose([ transforms.ToTensor(), @@ -82,6 +83,7 @@ def load_data(data_dir="./data"): # We can only tune those parameters that are configurable. In this example, we can specify # the layer sizes of the fully connected layers: + class Net(nn.Module): def __init__(self, l1=120, l2=84): super(Net, self).__init__() @@ -107,17 +109,20 @@ def forward(self, x): # Now it gets interesting, because we introduce some changes to the example `from the PyTorch # documentation `_. # -# We wrap the training script in a function ``train_cifar(config, checkpoint=None, data_dir=None)``. +# We wrap the training script in a function ``train_cifar(config, checkpoint_dir=None, data_dir=None)``. # As you can guess, the ``config`` parameter will receive the hyperparameters we would like to -# train with. The ``checkpoint`` parameter is used to restore checkpoints. The ``data_dir`` specifies +# train with. The ``checkpoint_dir`` parameter is used to restore checkpoints. The ``data_dir`` specifies # the directory where we load and store the data, so multiple runs can share the same data source. # # .. code-block:: python # # net = Net(config["l1"], config["l2"]) # -# if checkpoint: -# net.load_state_dict(torch.load(checkpoint)) +# if checkpoint_dir: +# model_state, optimizer_state = torch.load( +# os.path.join(checkpoint_dir, "checkpoint")) +# net.load_state_dict(model_state) +# optimizer.load_state_dict(optimizer_state) # # The learning rate of the optimizer is made configurable, too: # @@ -162,25 +167,25 @@ def forward(self, x): # Communicating with Ray Tune # ~~~~~~~~~~~~~~~~~~~~~~~~~~~ # -# The most interesting part is the communication with Tune: +# The most interesting part is the communication with Ray Tune: # # .. code-block:: python # -# checkpoint_dir = tune.make_checkpoint_dir(epoch) -# path = os.path.join(checkpoint_dir, "checkpoint") -# torch.save((net.state_dict(), optimizer.state_dict()), path) -# tune.save_checkpoint(path) +# with tune.checkpoint_dir(epoch) as checkpoint_dir: +# path = os.path.join(checkpoint_dir, "checkpoint") +# torch.save((net.state_dict(), optimizer.state_dict()), path) # # tune.report(loss=(val_loss / val_steps), accuracy=correct / total) # -# Here we first save a checkpoint and then report some metrics back to Tune. Specifically, -# we send the validation loss and accuracy back to Tune. Tune can then use these metrics +# Here we first save a checkpoint and then report some metrics back to Ray Tune. Specifically, +# we send the validation loss and accuracy back to Ray Tune. Ray Tune can then use these metrics # to decide which hyperparameter configuration lead to the best results. These metrics # can also be used to stop bad performing trials early in order to avoid wasting # resources on those trials. # # The checkpoint saving is optional, however, it is necessary if we wanted to use advanced -# schedulers like `Population Based Training `_. +# schedulers like +# `Population Based Training `_. # Also, by saving the checkpoint we can later load the trained models and validate them # on a test set. # @@ -189,7 +194,8 @@ def forward(self, x): # # The full code example looks like this: -def train_cifar(config, checkpoint=None, data_dir=None): + +def train_cifar(config, checkpoint_dir=None, data_dir=None): net = Net(config["l1"], config["l2"]) device = "cpu" @@ -202,9 +208,9 @@ def train_cifar(config, checkpoint=None, data_dir=None): criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(net.parameters(), lr=config["lr"], momentum=0.9) - if checkpoint: - print("loading checkpoint {}".format(checkpoint)) - model_state, optimizer_state = torch.load(checkpoint) + if checkpoint_dir: + model_state, optimizer_state = torch.load( + os.path.join(checkpoint_dir, "checkpoint")) net.load_state_dict(model_state) optimizer.load_state_dict(optimizer_state) @@ -269,10 +275,9 @@ def train_cifar(config, checkpoint=None, data_dir=None): val_loss += loss.cpu().numpy() val_steps += 1 - checkpoint_dir = tune.make_checkpoint_dir(epoch) - path = os.path.join(checkpoint_dir, "checkpoint") - torch.save((net.state_dict(), optimizer.state_dict()), path) - tune.save_checkpoint(path) + with tune.checkpoint_dir(epoch) as checkpoint_dir: + path = os.path.join(checkpoint_dir, "checkpoint") + torch.save((net.state_dict(), optimizer.state_dict()), path) tune.report(loss=(val_loss / val_steps), accuracy=correct / total) print("Finished Training") @@ -286,6 +291,7 @@ def train_cifar(config, checkpoint=None, data_dir=None): # set with data that has not been used for training the model. We also wrap this in a # function: + def test_accuracy(net, device="cpu"): trainset, testset = load_data() @@ -311,7 +317,7 @@ def test_accuracy(net, device="cpu"): # # Configuring the search space # ---------------------------- -# Lastly, we need to define Tune's search space. Here is an example: +# Lastly, we need to define Ray Tune's search space. Here is an example: # # .. code-block:: python # @@ -328,7 +334,7 @@ def test_accuracy(net, device="cpu"): # The ``lr`` (learning rate) should be uniformly sampled between 0.0001 and 0.1. Lastly, # the batch size is a choice between 2, 4, 8, and 16. # -# At each trial, Tune will now randomly sample a combination of parameters from these +# At each trial, Ray Tune will now randomly sample a combination of parameters from these # search spaces. It will then train a number of models in parallel and find the best # performing one among these. We also use the ``ASHAScheduler`` which will terminate bad # performing trials early. @@ -366,6 +372,7 @@ def test_accuracy(net, device="cpu"): # # The full main function looks like this: + def main(num_samples=10, max_num_epochs=10, gpus_per_trial=2): data_dir = os.path.abspath("./data") load_data(data_dir) @@ -390,8 +397,7 @@ def main(num_samples=10, max_num_epochs=10, gpus_per_trial=2): config=config, num_samples=num_samples, scheduler=scheduler, - progress_reporter=reporter, - checkpoint_at_end=True) + progress_reporter=reporter) best_trial = result.get_best_trial("loss", "min", "last") print("Best trial config: {}".format(best_trial.config)) @@ -408,7 +414,9 @@ def main(num_samples=10, max_num_epochs=10, gpus_per_trial=2): best_trained_model = nn.DataParallel(best_trained_model) best_trained_model.to(device) - model_state, optimizer_state = torch.load(best_trial.checkpoint.value) + best_checkpoint_dir = best_trial.checkpoint.value + model_state, optimizer_state = torch.load(os.path.join( + best_checkpoint_dir, "checkpoint")) best_trained_model.load_state_dict(model_state) test_acc = test_accuracy(best_trained_model, device) diff --git a/requirements.txt b/requirements.txt index a211f877eac..87f0fcfed1b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,12 +14,7 @@ bs4 awscli==1.16.35 flask spacy - -# Replace this block with ray[tune] after next PyPI release -https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.9.0.dev0-cp36-cp36m-manylinux1_x86_64.whl -tabulate -tensorboardX -pandas +ray[tune] # PyTorch Theme -e git+git://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme