From b474806c9a3d5fd5af671bca370b60bf26fdb139 Mon Sep 17 00:00:00 2001 From: Quarto GHA Workflow Runner Date: Wed, 25 Oct 2023 20:19:24 +0000 Subject: [PATCH] Built site for gh-pages --- .nojekyll | 2 +- labs/06/scf.pdf | Bin 264001 -> 264000 bytes labs/lab0-setup.pdf | Bin 24956 -> 24957 bytes labs/lab1-submission.pdf | Bin 34616 -> 34619 bytes labs/lab2-testing.pdf | Bin 22443 -> 22453 bytes labs/lab3-debugging.pdf | Bin 32912 -> 32915 bytes labs/lab5-codereview.pdf | Bin 17094 -> 17090 bytes labs/py_vs_R.pdf | Bin 44895 -> 44897 bytes ps/ps1.pdf | Bin 41411 -> 41416 bytes ps/ps2.pdf | Bin 31654 -> 31655 bytes ps/ps3.pdf | Bin 37769 -> 37772 bytes ps/ps4.pdf | Bin 35989 -> 35991 bytes ps/ps5.pdf | Bin 42206 -> 42203 bytes ps/ps6.pdf | Bin 35789 -> 35785 bytes ps/regex.pdf | Bin 16934 -> 16936 bytes search.json | 2474 ++++++++--------- sitemap.xml | 152 +- units/unit1-intro.pdf | Bin 48643 -> 48639 bytes units/unit10-linalg.pdf | Bin 184452 -> 184450 bytes units/unit2-dataTech.pdf | Bin 175082 -> 175080 bytes units/unit3-bash.pdf | Bin 66836 -> 66843 bytes units/unit4-goodPractices.pdf | Bin 105815 -> 105823 bytes units/unit5-programming.pdf | Bin 312990 -> 312992 bytes units/unit6-parallel.pdf | Bin 129505 -> 129501 bytes units/unit7-bigData.pdf | Bin 139133 -> 139132 bytes units/unit8-numbers.pdf | Bin 113267 -> 113264 bytes units/unit9-sim.html | 14 +- units/unit9-sim.pdf | Bin 146780 -> 146705 bytes .../figure-html/unnamed-chunk-11-1.png | Bin 0 -> 30386 bytes .../figure-html/unnamed-chunk-2-1.png | Bin 0 -> 32569 bytes .../figure-html/unnamed-chunk-4-3.png | Bin 0 -> 239812 bytes 31 files changed, 1321 insertions(+), 1321 deletions(-) create mode 100644 units/unit9-sim_files/figure-html/unnamed-chunk-11-1.png create mode 100644 units/unit9-sim_files/figure-html/unnamed-chunk-2-1.png create mode 100644 units/unit9-sim_files/figure-html/unnamed-chunk-4-3.png diff --git a/.nojekyll b/.nojekyll index a06bb69..22f10c6 100644 --- a/.nojekyll +++ b/.nojekyll @@ -1 +1 @@ -88d310e3 \ No newline at end of file +3a54877f \ No newline at end of file diff --git a/labs/06/scf.pdf b/labs/06/scf.pdf index 1d9d9ca5027bc4e8a12d8fa7e8060f6b65d23e0c..369013cae92f9c8476baaae2b05787d080c32e56 100644 GIT binary patch delta 9396 zcmai4Q*b5FvW;!qw(Xhd#C9?IA1j^6U z9J;F>=<98bSr%_y5t>!;tFBl9TK#~xKbgAJS@(@6}79E9=$Y6lSTs8v>c;_S(!hHVLCi%>_LvhzO+jF7 z?XAgaFET8pnY3nFqZ8+J@dL7VJSBX0q2p$#r*J7BQYHtRy8GU7zYszq>4}Ob*~_AL z`j3qkbKuh2trc%LtavwlHBW+CX1=PvoXSmmD*v$D5za#xgNmG4Xr;1G8P#`URT2f$Bz6#H=St_vnglj-RL z65AoeAPwCB5-Y7*4Uz`P$1fs+;Nt3RZfu9(xnYnvZxbwrCb|0&80D93)#}q>1NyGj^|R=c*6e&SI_qR8sGq0Z>yE%J)glzD`uk|UVB2dkwu{Rz~UfH=HvdvLYx{R;<8itH(yr_Alf5^UeD07`f{8&>qkqPhw zJ0LTQ0!t=1fVTp|rl2L0V`4o=NRq{+Q^RABZHfm~PfRHPDKJ84WFRG*!nYS=9AeY4 z;E1!1O69`ZdWr)p%PpI?Oy$1q#&302Hyw*U`L+G#qJSRZFONkd%)*+sVDWG4Ln}b7d8xm95T3buibPuVDmmg!aZa-G(uu zNBg#8gu6bll8n#^=1lGm#GLsH6#L}fCnW@-VFje77NRvv#K61lV>C-yKusjTBlr!O zCQVi&>gY$bN z11D6Y zv282)kBJdaW2*B~g7mF6o5>weQ(GgDRS(1evuBey`gFC;Z#9($$#7-c;p49xlLWPm zz~P^}our8LwwHZ>;_~fG7a(n_oDv*RRH_%{k;S5_Q*eIq{$_B+jL-$WsqS*`lgQrh zOO8219Lg}W(~;b{sx!0JPbYzSI<(uIO{78_=P=3J>>TJ$zwa)fC0ddb7ryQ(1gD;^ zh&7xK*X&GXI!C}m^E}L`44&nR6@DuxhHfj5zyrNd&)yIOW9R6M0yKe+Mrfi)dM2mhLnCdo-15&5QGPA9H4g6Kq>p=>neU(RX)tzJCOK7Afhp@S;Ckj>p$ zSdg3V9=o}W$QvQ#M6oEN7=6E1bZ&9Qw`RY&H6g#ajpxl|w)>Ej=JN%1`*BM5AF>2= zAZdX-|K)B8{WgPaNdM%7KrARFo9%uELq1oE47Fj$#eA8$0l>AgJ!}u}{lK8VXA4Ho zD>fSYJuG^8-i%4M;w8t&iL{DexkNjJe}0$O~(Vy_@Rma zdZq4du-~fbi6YZsN@02ySSCT0U%$VxI7l&snYMX8?{f**#*P#_H5GGIpO~j8gab>9AJthz6}Abp?e=8#}PGf?-fohgUrB$RQKGZw7Ajz{*Oq zUXt*;g>X_*Bw1R8f}=3Vs)aHNH#<6mM;@^dT8rl9`y?~wMPfu3cfEZ)W&?|C#6i$m zu!|hcb)$FDd>Q_zQp}EMO`69k45O8}+223XRd*fVeqNRrzQIO0#b*BpzcSr_`2F{G zNLGp75JL;t{lI?nEFsFYb=MMyVpT!&wwv#2tz*}*+mn}wLH&HAc|#^qSAKTq6x>xD z$tdlz)za$8{JhLO*z<9#JZ}W8o~&RS5`T15(yCNXRQ?GaaBv#m9Ov?z-n@Akt~$%x z%`$pht=lzxng&wW6|IAI@*GILp58xOW2+fHBFlEKYqT0}Sp>K0dVXJ?s}-eAMyf87 zS9d;c8wl4-p8WY8^Ha2Q!9&AOQ|ac6yqd_~?rop7_j=j3Q8)CE;y$YMewBySH*M8Y zRA(k^7Qoy#Xvux&_OjcXh3PJY4FuWy3VGv`@%Lq~D zc}k4OT;fx$Aj(vS)+?3|IiA(ihP3zv{yVD%wP0f?jnn2zED5ZOEo~q;!Nx}S3#?0M z>ycM;10T4wQNaW2-6d4L7>wJ$G2z$8joa7rIMb{sMtBo=kDHu?uyNie4o|_$;wp1S zXPm_9OmW?8N8vv;rQ`2(l^|xL3Q4D)x|>W7!51!->SdVkJ8vApRg$Sm^YG2UwkmipYLzu^h;u!CZnc+*%@^O+!cUokMh?82wZd61vki3~ceamqt1)*=Mk4~5tDaU}I)>?`HM52HdZbmxJVv4# zgHPvp)9)8~%Qsi&lWk?dvfc~O<%yP%{!ZoH$?*&OQ)#3Rj_7aHd8~sgizk19(Zklw zCyR!3e9pYqmNL+@y+}>wC%1pE$QwJ4lWkY<%YEZc32oOTTlcW(mn8{KO| zuRLUINQHqnu2J?%N_6+JT~S&3Q0|f zOL&ETf7*uC#pwO$ZoPUseEVd(m!G%NQM#mfX#@&ua*;y_Z25LulQno4@U%Gv^=fX5 zH8PP=39WUFj&F?}=pfO0rk{$$f>ADKFl`k6dlhJ0Al~QGU_g6a*A}G{lS0L0IjZgtarYoS#S z+E6NJEr&R!OfJWGIA|^?M`&>eP?kYa7N=(*ABAL;_%lly-($JR`h`F>HBsN~ir`w> zpZAo#+r%(8Ty+23(YLJF`*isuK3ld0f(^0HbNN4H+c=CNO{4!?v)lUX&VVsbct5MV6b2<_p7t~8PNIlSefzV?xTG2%ZWN3 z!d^Gi)l4!55fU2`6WU|Sbi|0%h;~lOp?7?L2H;ib)I42PTH!7aqW(B8;;KSvSj8&? zC2)0dREwfDL%Bw9b%UD+=dl(-;TreIB?}D5i@*s{ExKV$p$&^|9Tx+dgO#NMCF4UI zLZi?o^P|Cp5j82=2NE^vL#Yul5eUU}8Q_o>Sg=qEC9RH;WJa|2QVYUZaHL=jJi!;@ zSA&~z3vQM9J@vB&5NHumaR7U83zw)smFiGczEIs#vJ0>(s0geYy8OZ-a%8@5w1bpZ z(&@^BH1ppuA}GZdbk?*G^!O~TC1LlGrXYT{m+v9M2P2MRj!`oU9D>(bNwg3BTnPN= zLH8sIds^w=uzH+XL~ghxp-HJPK}ADDq4&KPf|o=KG}M}-qaqcjjvylku-#JtneeNN zL*`ZnW>gN~lB9(Wsp95#@BuTJcrt}*w-gX~d<2Q3l(~&?LYTKo-+zhO(kY{2;YNj_ z(euW_#(<0Pfz!)!O0y+{DPbi7f~v^m$fyt+Lt=}l`iXc9Y(4~3skfHCcOKSV{O2UCBe=2D4q?Lp#r1NvF;>a^_e!-r@M~u)}nchY@ zGD{IF=37%`LUFfrGg=wg&_h+Tj_b~`FQ~v0&k>q}*B3d*$rEqy$qbhPHB}=W3x?Q& z8xw0XtwQ<|XP~^4Z`CITcNUts3391hWcWM85-Mr+74p@)E#ZfX(P5;-eLM+L_=fpL z0#0khc%Ui|(k1%Hg!1dD@t|U*y));ED>FqBhMH2vDv;nMJre~eg#wgiJcD{ga&CASTLz2$kON6Mj}Bs+TS}Y|IiJQ2{(b$f_RrAHh{j$Jg2t z*&%lLC9d?^yHdf1ul@K{!YK*SfM4oGVD`{;7x@76!C;q_w`}H-I+H$J>cRHM!tW{% zd)ZI08~~v2cN9Fr7~~^3YSqVJ(8HT=20m3%Gk86FNne1bt}x$eJuIv2VyR8V5EO(9 zaj?9(NN@ukAo?q-+UO}aC#7fjwy~Soz};Kvt?w60AXCG~LFSx7(mrV{w#uw4AAu36Nz(;GsL07K z;{J$o68jb8rv-l*T*{rA19a&w(Ex6h7$U4ciBs&URnwlk2w3ZGc2W*?9K71Hr$>Wj zpMdUcr1CV&X75knu+k20+kQQ)4WY!u7j(tM2{crB88^LixvNd62KmiC1`761C0r!q z2^Wk6DoK;%@tW(p6SPg^wx1{NY}E*mk!)O+i`3VmN#!?-+!pUIP@Mg~17|uS(M(f| z_|Z?DI;D+-W${PLzC6Ba%x^tfN_h|d)j$yzA69!9t>q+N&ummyk_{7JSJy`fxblMY zCeLOQiUkX`WmkGvivE`q`y3#-eAnmhX#B9hkf1=dLNdi)Bo!@+yfZQ*^#yT{_4*2Wye7nxT5hVbXD+hLt9 zUgOJj8{N)2+Pj`EySAz);(`cdXxX=c0@QRXhDFiVv?lB3+E2tb5NHbjXltne@__MbZRSKdob2skP!Y7tr4H7d`ZM~VQN{?Wx zoh+msa(RbRG?)YDoM{A=!YCFbM~fmxCV;o8f4GS;U)RTwbLz>Cy0L!YoIW~8s8Cld z#7Q5+)X??OgP_|35#Y@Y%;@RTqYCVj$ygkwuH1?Z3iwNX2%GM| z?4a>%M|s_ry8ULXZYJ|VDRjs|SI41`Pil6aF z$29XYW*q8R98R@v=Dg}d<`#Dr&p$l1dp@u4N(X@`u#H%Jq$mjRFfW>Z-)!xZS#}w= zU%dRE-|nV8JO1Pe=OA}EwENHV`!rPB1C~4fe3+;0KR^!p1?Kyra^rg1$&%7Hty z1w2Cj1_uvL6MKea1HRmR(LbY%6+<8wmw6d) zWz@ai%5QhmW~kv5*)3llf$tRLw6ho7cebp1cg#<&A69#0wcKTef7n;ymXfa-HYGHC zS7^0~>uPplD{aw{?A5W-Lgd8Ww8y+sPHy9oTe>A2f9$r#2JULrWh`=x>$H5-)z{8Ods|$`5 zv!pn{O5B(5gg5u1I$@erpv2XDQ=0U_6rH>L5_b{W*!sR-VBwLuqYrUki7ktMxX~(7 z&E_5AK-!@TOe-J`^Tl;F(?^t#F%_JNvH70;(Ez*RT!M4(v1rqvU+mXyEs%`^J~Ao$ zFm3MUMIX|i=OMwDK3MfCw6gm7zP8h$MTb!~*<6rB@eZwa(Y$z1Mq1S$4a`U7fhdw} zC-;m;o#|kAq8{ud#uR1!J71w&Yb;Xw$;j{af%K*UeCB*`E4ME9oqZc=5f*I~caUz= z0-gJ9C~YdMld_jrWbk4a?pe9?;&fNuXgz$SfZx9~5ZnFimq&b~$%0|brFB1heP6Ta zu{4zrSX4B&dHcwBx-)Z!Ju&D&|Hi9uplep?Cn?(Dp%ZrmN}0W92R<{mxGj>Fj&HSr z&6))Rjz>gAPJRN%(zs>3a;|?7AcTh9>7_*LO}vzfGz_QL!rYpl zJk?cb?PRwVX0c22?mquJLVbe&k|W zxkqA{fv8SZR@G5iw65_`a|WQOj%-P*ooFZvtRR$Ii(fqc&{a3A zW<1BP=)TlEN@=BFt;K#|&@iN7bo~J8pQLHkMs*`4oUg1_EvHk|i5+Wc)&1K@FuYaD zyaor}^Rrc2xGx_o6n;j@cJs4{ijs!e>*RT9de`D2b*o%syqx#vXOeas3XQiN5=s{= z>m*qZP5y=s^tOCt+E=T7{XuK!O!kO4M}QlmQ;)MupM_~S`#B9-#_Y1BP(5H>*@H_E zv$G8|hkmqBCjwALf0vVh1%mpXkyL{z{C$I_lJ4WV3|uMvnJ#4p~~h1T~Z4S0#mM4cv6eq%cB*;q?83 zw`aK)`#0i+^D-aSj@8?{OH-it@9CQ$Km99xAV(d{*fLFD}a8;+P zWot(C38;wEu!vVuFi%Nm@sgHfQGioQpk2?M{=O26R=~pX}qM-zjNsel5bVtTNcNAB?o*3v^6c2zQ*i$Rq6c4 z>{@OR?i`+fCmG)k+Hw0OH~#Fw7*Zm`STUTD(zq#;d>Gtz=E#Ncw(b!1vzYX++RWmT z_pK$*A$v2cCz7|&qDKc-^t$X1MD?Oy4`lXYo)1V-m3!{%8|gUhYlvt=k+|*q?E<{e z&CW68JELl6Z<3uRQ14lb>$^wTOHZOj?^C4txS$#K5$zH>rb6faimpN%tOmjMR+x5T z<43V=8>}MP;e*SvPt;_NSjJO&<(6c55X|4iJE;h*lw0XASSNttg2}96Qvt!j+7t=$ zCI_upy}fZ2ZOMrP2Uf3!oz9v{=ZuD7XGj+79J)rWs~qbWBv#%GorgeRO+|B#ehz{J zcoLRzT3^tZ968$e(N+_<1Nsor-sSUvmYu$n{E0W@5X|YtsTQJf@c0)x{ ze~a%WiJVo@X58N?e}7;PLKsC2T{sO>C&w}CH=LX5uXH4a_a2;ne0LYdsw$HS$rR|l ztbbXc>RrOp2)LthTugegkvk(vsuq_+h*w$Hf=ygLvhPuv_LJ(0b}e?DuN*#tGe ztH3AW1hZCOH!YWtrZ~cDzJti^a7!{Uy`IHEgiK3Zm-f_JwgwRK2=Hj_3WCz`vGXnu zSdo2Gxv0S9W`kF9c(fzlVDLjh<&bFYC)GfbTUR>$RDqG|kti^0nkjKf3?0NAFN;fh z|K+;pDjWvh+inqo4j!;du5#a&WdtY5#sO4{4}>|UkcXpM-iwVWZ{JgI@&JRJ-1h{od|} zlL1=R+`pN~g`;&Isuy`AJ@1dxn_thL#~16;Cin z%Cot91nh6*lIco@!RM5OR!Lrjj3_ch+XK)r9{~@QRH2SWgWP+TCYW(*SnLTCFyf%h zD{mQnPT24|8&&f)^K)C`0Go{XCe&)B+So+OFt~e`ef#*Fjs{=n!8_YT9g`sP}^>bL^|Pm5NvJHcZ+! zIFvB3jP>{1B9ObpODHR*7e@V(Mg2WZ4BAbeP?IP~5t6Z7IWQ&NpD0XJU?~XuIJvnH zn{6NN2TE3bBu-mwsw0Fh3Wg{)gV)6okLHo(kK;7aua{BFpE^TYgMh96sAQuMY-`aIu-6RYLo z`cqmn1QY;xeAOoU+kYMGdX-V9csEq;y)dza8(9Z4%W>ZRr8IzztLc? zQVn5?=e2A=l6dUfiEI1&*;P@VQFZ;B#(beWqQ2X2h}E|_I2Hi01masW3~uCJ(B0(= zqPs-r;Ey}$z6BDkjq26Ds0|i}W%^?~#FwN3jHd#iiBkB)tmxPADXr31@aZoR_2_j4 z^StS*$~Oxhd|<3?@rpT%@GH*;-hR(Ow8Ko^viE0oYw7o1_P{tpUNlo?QVnm`r5 zp9VgBC>xmw>&mBO#9M)qskK}sb13*iYZ@Wwc`3g(%PE&R^J}zTU4nBS}c5_Y< zq5rQUtq2Or8mO0Qo1&abmqL*uY=*32RtLA9W=l=~mb=}~ZEw)dUbLx?$7`WK_V#&t zcEbDRwF8=Y%gx{%#WU5_6&iucVCW8#cI4aX-}J=2xC7rA9DPQMUCWalnG2e|h4;Rt z7LkrNv<>!O!}(|+?j4b`{YA_M2FVT=+(79*FtY~T1&m_ve+}QR$CCcx;T!HS^vr!R zL1OYAoxPnAU{otkEpS7hD@wjW?uqoRAN_J#R6QQ^v9X?b{ zM=bk6SKyQ+>C7-+c5IOolKntUJNl>zPi>zDABxr?fqwL%6YBe?U52y9IcxfPj~WeL%NxjwU(3BTt@YY(y^FiebMBrVAWf&LwVw;UVY^eht>+@0M5+X^^0)w6zN;G}BAC&~TWbD< zw=JIxqtzz!o^%#fC`^^w)-;|$*}vHL!L-VadjtbI|tHmZ@dM6nuwa~s1Xvef& z@0B73T8SM5?OhoIn_q)#H3C^f-%Ssw+^XFt>o2J(_}gn!o3$@qp)^@C1JhYFp)k`c zRiVCvc$ftR5nbI}%#H04y;k*;bZtX~(89Lfv0gJ&g3~!VXrM`F;c4V`T?(fy2&>UF zS27~rUQhBUQL!+)b2Hgo*RSXGo98Ec-yG-5$-h?FxLF_l*@z|cC~TiK0Ms`qhmgOj z+t2z2v@^X-7f~MMbFSr2zMc+$%ru3e6h8-nPF|asZ*OT(Vd%<59*$?fMH5^$?Co(X zb|YtY5Bfaq-{!dG@m-xi%&O>M*_&I&GH*WRwpS=VZoBkfQdEC zApPiICA9F%cdWw(bA+C08u-F0lID2I=q}D&9nyH0hUky?6YmH{4JEIsiOyTzIEQv^ z2e(Cp$2E3wiQHC=cJ@1~yfPB!LfukAAeya z@<9&RHt`bJsYg zVZ7;Sax&op?B9MU4aB8}7lyEMpg|Ny8pu$>nG&WmlPKU|6)ud|LPkV0O^TETC5XgC zz0#oBD$t8pvBHRxctsV^;E-5J&{zcn<}n%@Fnve#Lbi2SjT!xaP~NG^IboYxV9@?R zpk=Z6zN5qP+cFRZW6ETr3dE8<3Z0M0rn(NHs!|n2L*o%na}T58YNHv{^IV77Pus)Q z=7l$3o3`QE(bylLp0(K{w$>;LpW5~LWrT4m;C~=d16?$vys%6)DnZ-oQror20#SS$ z@j65Z&23QMA1iOeiT=SbLE+6NUAE%U#Np0Vw$;oKw-8;ySz7A*S_cz_q;IZ`!1k`SBad8o}hoqFLaJsuYlM0ix@s-}~gPGB2 zq}=G}B+1UE^F{)BVbm`pEAA%Je6Ey+u9eaTxbhL9Hte)sR+8Y-#0Zg6yHk+QU~qna z5~s0(%67wqxr!DhwUa~Sgqf>jafbb>v-cMnG=M8K=ZRS(l;eo9>nHb9N(7uz$08i9 z6Kqj{JkaC{t1%S1x{ucw9E(svZWdpt>L4^zo>9QoH#w|S=8skpI|~;^orR!mawkms z!{P_}x5siN<(L{|oLvRh!tTg-CBp$+r9_Egc1uaQ$>$mJ$%!ygq$}|R!?wmG!-t1c-V9Kv^}EUl^)Xkr**isp{O7@(v7sCNI-RE zNYGjK>bIW(nAPqmTqN^1>ay2N0W(j zhlWwA>yv+65Elx>R{SSPWjat)AnyO7{}kouU1qevfAsIYzNclE{aqPNSQn}9e53wC zaNX3tSW|0F<=*)7z3CYiM?vW+N#S7JEyufg`qAE1^>#~ClkiPGH>;WGt*HYwRZ|k= zu&SN&v%TCwWT&M)<=rl?E9HIqW_R&Tv763g?Gk8U&;gQN8JfMV1a1MxcAnp>BdaUk zqrh8&zO@)T{sB*&n{Vb@dAv+>b0*Fx**lVbtC(A55} zOW@mJ3m4YppHnBj7=y*dkJa>4;<1M_%Sx-as>KQb@HEr$^z&gOxNfifMHgj$@7lwU z|HqscA&45utdPQ(DTv)y1pk43fJP%y^$vCnDmFUEW1v(?n4rY;0a?t0PRye#5ZQ9K zoWU8}WdMO8GFde(;cqe@x6}m*MP8C3e<3@WFG|!s28JzZl`1WZ0q+@tv3(2L9~yO; z`eu<`n&$!4NF6X=q>e7>MS(Mm78oY939C;>6{Hw)4?u&rrJ_g$V}$cjBte{pp26CZ zTbi7Q{sb4M(_1LOvxzckPvj)HL2&%au-?oVl)K%m&Ts}sXYO?Aw)Z0Jlv6_G>FE8e zDfFdZbWiwZd&5&dG#J&Cu#lX^Qw~P^6OPDRm5U}|e1tPh`0p52$3gp&Lt_8Sn_4U$ zN&Ckp5kvF&=g~VZVwyOdgs=u7=btiqR@5OX)DiTJ$WW5s$Ju?#GM@pNi~fKi2y9qe zK8^ZM}lv37CE5u~W-zjU~A@4BA9ReAk# zzt8em7R86%dZgbU$%B{QJ;a!mfzeoJpW`|bCLDI5YWc|%EIYv*U{DH zqi;W-eC`d{+#~g%N}fy`chghwaNio(Q(APcYWL4!XgL1G9zA^gD`&9g`SLy1d~L1f zLpL^k|M|U50iEJ93tY7CjS0J7oi+XjWp;|xP%_{Fj%X^}7b>^Whh z0=t0RU2iuA85`!SFAdpS)zrW=ZTlM}1u(Hf z7KR}p?L_C9hV5Cn^lju<$`%6!5Iukri~jnS>`kU!1+;r8{*=1VJ}@2qmntdWB{3hq z2csAR$kaDYtN4caa&eVS^u2F7`s$b4;LhRCf_|pR1)$Br6ju>p%Wc6(j-zX=FF}1 z!_N<^s;Bq3W4qZC=asr9dYYs2MSYJV#%u20;j#fLrT?L+3d4Ui#qwX3HD_oW!i@Ii z$`d#!;1N?)U+qECSCP!~$9~}fs^t1C6i-x+_~qQvrSN}r{>|kSUw;v!?Fv3T=#CnX zedfMlkvo=)0d30b*u2-o%GbDaT^Cso#<>3QBpMZdNAeeta5EqH7~#B~BtE`+Wy_QX z2AK|GSk)I#4tIz|cAYk~DZ)nRc#xZEe?cQU=%6aTtn;!TW)oa#bKEGp*oVOe7df*{ z%9qojP=6H@&g(b@m{Sd5Qw_q{FfswmX8B!4D!l(fj6H1e${sJbee2O%JFm)UkyP$; zsq~W6Nu-@ywZ=;3r08jO*LiKiQ4v{9RA?><{!5-xl5}^_Xk7B6(L;tD97L>zOQhFY)Uh8tm>JDit??*Kgr0y7SHc ze_~r$jiF4V|GQ^5^sHU+hnwE>3|Y2_u2lec-6eWe;K3iMQfaac$A z5}iM+%x z=7{3C#HZLD{6IM5y^wRE6?~%0yjL$^JLFqJ61yw3D8iK(X*oqJVgs~D5sBjcQBkO5?GiBW?*mKZNCR;+{VTcR%cyd)KLP8-aIWb8Ath}3rZ&q=6KEcj+ z3&}pvCs7B0B2}+?<K4x`zk24!rh zl2S4?JZ~!kB6vmPPFnMf@br;3WIIR(%4D(p$z(*c{qVB7+OZ@B0Dp$};z%j@LIC=) zY|Ig~(J+g24>c`d#llGSWj2J<9|yn`1;YejKPvs}*&a@+H^YIAQv3@Nmu8j7Lp^xb zu&h;dZH9W(;Ruw-j7D%@CQb$eIXbdkvW&Ec9ZD|BJA`E|Cbr22*j(?@y*U~5da=E{wiB#4 z@DHN@&gipspZAz|XWs1{MM^N}E9z|9$ZJ`I0E9dehvth~cYINhjUcb$fC8+<4T)hJB!Tqf&C84&mjzsHI4wE?OKal}cLo;>ZX!N+wECjN|#Cpa(;JR>Q za}5CbFi1-q_M9loe%Z9rNE4YJ=3d~v(iFEZq!Rz}oZcN5O;RnvTnxZ6mM+zXI;r); ze+sfpoLUooMxsvmgs~Xplv=t@C>xnv$55QY2y?n-H6sh;Hf;j4x+otf+Fi8hlbmekw8pEE%LwR}kggeh5iKf#OFXOhf8}2nxl<0-U-7@|oB|%#t|2 zaF$^LW@IYi4hbH%DXmx47*aEYjA~31$|o$DvBj;gLCN>0o4VtIt&;caX?4+*yF!lIc%AK;Rkl@ z1Mij?_x|leF~EGI?-`2R0^^p6Vcxu^*9KhUY94!-NpH;gU$@l811=-L(>_2A5OI3f zAy-J{iL@a9RKy{weLZM>BcSgKwRM|)_PI73TYAzj{fP~kn$K2%6{XC=r2y#W@3oO~ ztWvp3el#4`&C7-p4{;3G$p|W98a)17)sf?l<8)Bs%Y(bMsZqRHqC;;nFRmDB*!{XQ z%4GWRV$MwX4$0w&y?=jW(NI1FI$?&6B?HfCq=w((s#%j&i9EFsr-S^UZq@W?i`G_y z>aNf5AoNIFJB+Me-VZe|KOnTP-frs^gZC2LxMPvHOcpPoQX%L$>L>NA z)Dfp%wRX)8AwyK{Nuy4$HjNvW&jXEZwdiO)~%J=o1T@GewAuc#&b=abT z(XxU@@IU)_ks%0)b@IkR*Cj89UaYuoj)!a#@$4E)bAV%+8j|V~(ucN^_tcVdQP!CS zYTPdn=5Dou9!$!(ZmCM`EjgO%m$0*)T)3U08JlvX$OHSl*;qwlSo(nN2?zzK ztqmUvDpl`FU)n*O8Y=v_`ENcJa(2jytNfJ{;azs@c0}LLzR*D1M}={f+pp=vO^4%j zx;0Cq7mmM0&HIciFEo?Dt2dQm2h&&wTROcqp~B;XU0Sf&e@11|&2`LTKTfp|f!tdy zlOPTm&Tk3BjUuTAC382_7HMG|Qr9ek3DtsZxmyPMn^U!$g||ish2Spm;_G#t&-?Ij zHp(X(!HCMXA2&4x;IHoe*ZMZ)cE3uu!^7ttoW1O8msy}MFTTf?(|eKYgP-p} zYe4$WJrp5?ZTjau6g$LlI{O2ZEJSTf-~&_-1VnVY&=V9h=;L$?u^WoDTnOs;c*D%) zx+7rOo>L*EtSQwI&Uo>UMYQ)*c=M|DL8TzSD%Yj-LCT<&E(f1?pr;j;gB*32gMG7s z!=1|m_vwa({o8?Y&)P|tHGK1uof*Aya&^`oqdh^4wY^57BlhZIpWI;cniwDEe)S{< z$D2fhfTPYJ=rOg@qRY17XzXZJ-l7e)^xPOJW%Q2*)=WG0+L`k~CFvFl`Kb8tzd(5M z+^eZwb?lJa* zd3jroYi=?s3-eEw55b!L*5T$KVE1eD7t8lvYcO+@KZ}(ff96Jzx2D6ujC^t{cBYNt ztm1`!8u_!|KAZmpae%D-#$ALp>mx%i+X@NDtR1NjI&$VqT6L^j+jV^W^D6O;`BxPK z@N03nNTKnY#{O=aOLw`V9ZOjygkR?KyX8qL*Wo#}hA#twa8wmsd{r2GQ^Wm;0=$hF zoxn2Z7YDHIqFd`YN!{+-@}E(x6VY$?^-(^H!F|{ZYf!KcixSNYmh6y2&7%Kqd#Dc@ zwaa3CEt_bkOe%++>$Gx~T10bcy!K&Jt5vE;9IjML!X#Ljt~_f^{gw1hy?+Hk{ZR6e zzozkOQg2V!CH9$2cskx8XdRs*d!FlC$kI73vpBzQB>`L7ywG~o9FOQ?nT-*(!cN6G{ox}{%hZA8U0OX7Z7bd-lbSw?s1F3CKkuhj zk}-*vwBO&`aFmVEDLpSPqJ~u3gk-D#%w}}%IyzjejD8ESP3&eGYd$)J0h2LJoh3|< zGdz;yFWKV#&Bpb)zboQA|E~R{R|s}bw~L>A(FWGGU~&?mRXv<`8wsDmW4FK{xs-aw z-FCQ53LINNa~)MAV@=JE_Em$H=3jVzWAij&4yO+Y8$Xo`4=kXrtpm^Bf;ApB zz)a1MeY?#Zfl;5izyv$dTz-vTyXyMwY-xXZd*15RkgzF5o zTz#C-S~$oY=%*(yV}Is*l`Q+(i?~8`f@U8uv&5fx$4_FDDtNwWO=_G%T5xcdOaXd|Cx2pvF=DnWBhR9R^wHUWQeMJB zR6NA!JqJ?CE~(#Z8D%HASkmd|)Hw)|hcilnOGgdGsXo6U1tuWp#x)BLxmB?^!t6K| z9=knl@#3Xy4dY%2-;uswSB@+|TR#_Ld}n|))Y`DDlm*i6U{pOFimo!suu46ijA)Eu_G{oP#bzo-@RqH-dRN0^on;2^4y zM{P&$d<}DV%kyG}w^#jI}cO0Da0jJHD>o#Xl%^^Ul@$g*~~F z5(1H=v`-td^lsTGk$$i(@cmx?O?QBo4#isoxCeZ_nM|?Oi#sI!$_)lgtB#es$_rIg zxW;H$OYvE%Byb{M3?LD$`o;G9d!J5*Ddqw9Iq=2e4KGXn^t``;mlQ!C@6qYCmRup_ zcNCj2>M!?y%9e+9d*37{i z0d{rT+rfL_sx8;XJa8j*rDN;9I~T>0{NmPbeH<45E6D)8Jc;Y```#J&7|=x*aOx!& z({J^Becj?n{AtY)`{HAz>>aWJo4x@7EdXj^`gnV9sl}V)&y$xhgrfDpq5Y682<9#O zN{1p04#HCV*()OlMCm6cwE6wRMMh^9F~bR(IYNF}EP72smK%ZrE-D`}wbR`NpYF!3 zs`KNtrPe@DXPm4XPh%HFqvVaON^1TGR&|yZ!hp>4`8|FJ+6Gf)zi`rjK|;%bPUju-{a;%l)@wBzqA;Ua?v?BP5)ZPtbIh!5pnp&^R%c(}s25<_KY<-jxb8j=CfH z$f>O$R4iX!wY;8}kK^K{p2O#2)I}2y)fLOzij(e4`5{Ac{Brk9+4ydGpoCZL)o%8! zWm6uovYIB*QLdnP>Fa>j_%Odv7_Q&dMaof18)(+jy=bVmrL)AH7t&lN!nwdB1?Op$SV|{P(VaiE zV7Xt+smfVb)N-olFO$0_^6_^+4Qg<_BJ>4F73M*gzJ2=dtAivoG_HuLg$cVE2P-oV z7Z(o;y9GBZGdn8>7mtMrmoY1csRg$n|Npfky$BlG8suw6_oG3q9JBn4v_mYhi_FZh z0ZV*k>1?Rl`Lr(fsTH}aav#lsRZpba+ls1xk<5o#5&&4N5=u=}i8 zuIOtD^MH>3qXLaTA1}{&7EeWX7B~Olne0 z=O&2Bw98zN5X&1sz@8K4%#E?^o)K~C45={Sw;m(QjCi;QDc2|6iiTs%sxXA97gG+1Rb_@^bEZ)kG|vbd zcg8!|mXRAcZiV66Cddh50YtA|qh0joYQ=QaeL<>+_uM1m5A<~=PTLOPk4&#eM%`29 z2LJFU3$VP3SCDIW3*}2t>G>xcaYZ4j!o6 zF7drC~bi5?Fz4D%O|y~Oap%!16p*>sA?bR zz_(4d0|87+0jF-s4-ECdZYvPl8s@)eY+busM@vrw$4-_{2ns)ch;$q{Xi|7?L`q6= IMG3_J2SHOVT>t<8 diff --git a/labs/lab0-setup.pdf b/labs/lab0-setup.pdf index d365ecf12e510c762ba301db84d8decb649be2fb..0e4de3ddaaf1084ac31b9b4350588cdee55eb5d4 100644 GIT binary patch delta 4514 zcmai&S3DaGpvASHJwC*Wpf<5rC~6aX&!YCG6h&jEh+Q*5&7xL|qH6D1dym#EQ5v&G z5c7VI_deaX^KgFW;s0{Z@7(r1y6byXr$qsWC@7G=4)Awz@Fs;V+oT(LMT=5>-?*c_ z{2`W@u&k|&aKun8dYiV(SBzjDOOhC|ULU$;%*GJ_uGSuou6BjL?5WmlowJPte>D6l9%#a!GhDd=WsSX+XuQhzpIIU*IY9+g~Y-EAJll(s}$8P>*loa5w z)?Kd49R##hpb?~b<@C&`SIM)iET)U82G2i69rIS4BNOO)3ZL<(K@3LbiL5^+QB3Jx z*}J+9mJSqJLJ1j6zkBMFoqbkVuBMW_ceaM-Ad%g^w@x{o)pn$rMSwsc^{~;MaO9wR zkOj;ybi+K#tc^WM#|T&83E3eO`V*<9TzG>9H}T4nbZ$NtT%y3Ygw*6a-+26f3~ zAX)y>c!gdZ!31S!7^%G05WGN%m&zM&@s(G;U>B>NKy<;9Avb9m!!#@GHTC9fn}>Xh z<}lZHouy2rK$1+cpo@zXS{yrao|2h>(%m-;iNsQxz?)FNx?A$1h!dntHJ1U%i9|?{%lnepYH5HYxd(PGZPcHh{?)V(&j%L>W5!Ejy59q~#Ku!Hfum5Pf+@t|zremc!x zDbeOF4cd87pw+TqEgLwP^y?@&Dsr71s^IQKW(5K~0gKc2=nN=V_Q->0s5P>PS^Qws zSudISeWmkAW$L_8S&M(BhI#|t1$r8lj{D5N!>KFTe(AEg{mD`OnJ~_o$8cEhwxGA~ z@z<4ULVIHF6}LRSIo2~p{Ft>lz1;%&iCPP;=YZg0{dHCO8@>ex{Gume50I>UN1gP39rUhRb4d9KcV52fRq$$`_T zcW>};I-V9+zsmkeP813n^kE>bw;a75JMh_JKOXvst8cmsD)8%+Pl`(|xe&qz=5~$S z9|-objok)NA2d3-37gD;mVPvl1Z|$WI9dqV(F0e)_bvEFQz|YR{PzF4cau~0Lq_H3 z3TeS9(+p97^T8tKa1!DMFu+yIA8v^8@Hjt@YNItR{ugCv5aoEDUQ{R-&9#G=QY+Sv zgzRZfRiXv0sjw}C6k8z80K!fP!j`0DN+@1I>u<>g8%(SIg4UKcAtawBr%Z&+xm*db z=RpwF#-S{)3;^(H9lY0-_W(%rCvt@i`3h|cmOn90tS3xBanb#z23gH9CgiYlCy#R( z&-&JA$?0fX?8*gRA}cIv7a}^-<(S%T6^EMD7G#~6)ve){!2(NO?OjhVcgs6vn_omc zfN^dv;g3SRj*b^0`d*nt4|o5NQ?T12u{7pzu^Dp zAWo>8(;fM@gl7{qRyBrBh&_4LxM159RvcR|Dm68A1KzuGenF*h!GDU2_r3gR%B5w+ zq($^`KG{QG*|FvOi{JJPL)?r*q{b?qniLo|2TNZ~vQKRCr7mydG8vbZI8HuraJ?nK zR|(NP7f>2BR`JmMVWh7{dM3$0-G(4E7vvUv=kcy#5lWSEjW_0`1FGxLV7i(0rT*&x zSWG9?q-d&Q>_A7-*u$&w^?|(+bZ#8tKZI{fzW2M8qVAztO+>z-F7TObk02J=Wh}2^I4WLMLpPt;rDj9WtdFW z(Q({wVAJVUL`UBMqY`_y_SkbV80l=3`sk16WZ3f}9;gI(-&MNc2vH2Hros!OsaKgt zE!IojV;pYZz{T>PE;OeRvkFWn3&d7FdjF2XVhULLhH}Ej>^ zU+|5dL={{+!$@)xZ2;{A>hHUX34ex}uJH;$j6Xe>g+;M*X`0O@0C6gQtkygqK;yep zAJ8Fw5<*93JP)&SQCk#j)oF&6%*Kln%B>^2@ZWQ{@M61EAE)U{A3qufvsuAhPh0Ye zJA$q+_Y-x(>R1rRr7%5V&dRUQlJf+hwg9+caPoG89nedRUIP_Vkq=8zMg1|qa#PVf z;nEOdA6y8_z^`b$XKKWJvhDvzFnPWM!CMy3h+g?S`PoCVJC{O3MqiuWhKIrIXVF6x z`aQ8XeEZ@8iWVdm$@&V1ouQ6TV!1aas%FgAOGnzzlD0tz9m30gt%$|MIdk%RpM^4v zD#avEa|3RhFgi{TIW@K|_h9<&bS%4K?d{TF_{= zE=!w`{19(lRoJdk&A>R*A2;kdJu+Epn|A=eF0-{Zlt%%ovW~RcrwTDF6LnCwA5Cr{ zv$aA$3LT3VlFh>6I*VxJnLp{N%Gr)zQ!Y5JPmLK2cJLpceZ@%deoNbLY?I+XcoPnE zB4}KT7HiwbS?`a&EV^k0E@>vts|GTXa9h>;YlE^3B5v+dhcQ5cSaI1V9)5;7k0!Wk zmEQQOtq?#oWgy&Y#Ml35!Jnc3IP_cPos=NirKWe9&z z|HH|$zpnLW1)I>6x-$#&T5g>;aEgK_pTycWqGO!Mb;L>dMDyT^Dxk@nL}Hkg2wPm)yR zv}bm3@-hAb*_xGmrtkBP{R!ysi%=D%Xch;E$J6MtFr-_X%9=i~7=< zE%Y+HsmP}NC@<^~^$AmM&AV1OcaD&-_)suff!}^6Qpx z0%2k&;}LEl07AbrAg9rx3++HG6-!?v6C$3PNdeQcNt6QjfhSEpF_Xy-=SZcM-Dvhn zu^}hHH#M|1zb6@Uec(HSNb$jN*(+$CZo>SQlX<{Wv#=a}jHhAmXS6ci1+U30D6Arb zagQ`LP_}X;b9vfEp=5(x6w>gQc&)2GW*Xtuf0#>fmO`D9$zsJ1^erqLZ>`Hed;%Qp zP*)S{OA0DIUhR9nPk#{YTALO~<~O-A@ekQhe<9T$$fFk(?(lFZjUP9W_BZPXcl8e6 zvC8Ic$ZTBY$1WsuEl(gZ2^K(tpfGA~OKyAh@kq5r4ADnX!27vM$(}$s2B@MV1{seH z^e}E3bu%^pGJ4d|E7Pden=$85Ky*0Pn@7o7`*3$X;SzJ2vvTjEV5uAT8We#w{5SwY zHJc6BB1?RB;FKD?=slie?pL28G+JLnfSh^s6RMN~6Rvaldj;{EVeB{35REg(*a1z& z?&-_Wd+MJkZdS}v9#*SOt^LV|{CUo=+waRiKJB<{T^gYi4e!y4<`NCfWI(nDjgM4} ze0N-3WlxcMFIy(JrV)~KeclhuK3`#QL~;MamFgH^4A(CY9>_?-_!?^v2C=63+p(>n z%nG{g#}p@;?o-+ejE0tvVK*JI4$yCkIU%EoSLGagVaft=(c z>UR5IPN=B(W&gCPp0$P8(FbUxe!HemTvNm0mK61CBWxn6r!;6eS*f9LUZZgyBU269 zkInIpaH)8X=J+U6ypmu4@?+eTQVdQ>6230|x5bUD)x?OXTs~k`6SlfI5*6~ntajaP z3O99+cE07Ne2C<}pw_uft;25(VAUfxo*>lo#XZ)O+q2VY4s4C%>6<>@4Q<`(>djpH zca<%1g)7nJzpx67s%76#3y4_U-1hn2C9p2H|9!6v-X?@TgUTm7mnb$N4pK z%8pk>QcBGAsf&ZSl#44wN=6joA}R@SkdTyia*~pI`kz*i`~MtBX?ykm`(1EPX&Lt! zQoY^u=>hYT%nT%qP!THAZs9$4v;SnB+9NO65xr~QFOQO3>bbOd)pfm5QgxFs9Pxq1 zu5CJ!R^Ne`@-IFoEXEBMEfya!4+Y0CZx`bc{pw|BR;Jz&H6wX>0kpEZt=}bi(a)H# zO2QGp`TTKM3qqh$1y8ail61b9oE^dzY3;6RqLXm*uG=3auh9~uR>*Q*dD6U4V18dM yUZYF)J%Vd-GZv*hRGo-ISyf-ZwjRbD#tTmCk-cBunNRda&E9*ZYHn?tDFu`DVD+^jk$H8@IB>2rFAbroK7b zvE$>O`wCxpK8B}-g&~08F$0Dd_nbN{QfUB=n$py~vzi|kX9g2t!Tp|XGVMUe%TxdR zg!5_eV5Vs7;7N5bCGTnDYcPWrG0hmvmuZ)X(W=1feBEA0bX@Dl1^LnyYfLY20b-4?QFzHq`G>iupJy5M47h*TkM$!E+xRJERE zYkBRvgFW^@=s?x|bi__G9--KcSA|Q;thuEtN1{7}Qb1Q}h@bTHJXK%KxK?(BXUu|} zlE0x?QA}i+(r-==6B#bJkr-;WyZ4d@-j^beshfe2# zd0{caz1-R6HCPbY<8?MN%<8-V8uOh_9>!1Pin%xj6p27I&c+eHGCfm#uft7K*u=AI z+|f%u82E3_q4ys`e>VNb9o{smyz%&QZQHuLH+>j3cadXiq7f2u+DnKIHdBmCpR=B3 zPjUd_q|fAG9NKutB*?VQmWT4y%!4iM#b2vAjO(bNV|=cqG4D|hpY&E(5Yk3V zl-#_F+3U}5i#8i;;j?4VbvzN$tX&UTMym(dxXnv3#q8rn{eMd*s_=tpC=IPO`rF>M zv9|}f^UF-tL35I>uYNy?qjb{!lYYF-#tx6Z(g?1_i}mDry1U6FJ@-;@mROr^)MguL zZC1;Pul90A+^QG+s4P{OCyW9) zQkQsH=JYgb7`%PhN&Xi`sA*m`-dlJ_vc5ebu8=i#9X)rt{bVg9l#!bAYOZ-|*7Xt; z^|d}eQt=GaS3CPIx77gRX6-O0@Jkz=mE_SF7HsQ&_Nr^h=#63{z|u^W8i+m+8d_bz zEFlT`r7wAm5<7`a@+I%uF>3XQxdH?tO5!P>KSQ%3M5ZE)xAlx7mFcE1f@!@u2iYQ1 z!fD#EUj@-wc}QT2|==#Q&hx~%rl?6lX#%1emwPmnWKjp zbk&tARE=Knm6htM^YACIBeMk|_D<&M(PPBb{5kwn8v0~2TBRbnorq@pj`}l^nuZBkQ*R{C$h1vq53dnxv=~F2rN@Z2e}T;l;*# z^gbv5ZDpmTs@Dj9@ngSh)xE>6uC@{GXyh@y88G2eAsRnEM?M+A+PL@`gZ&%*G1`Um zaz6`;qNWfE8?>R@r$mQXp*{IlvY;Ii=?%FIBqRZHY(MW-=Tf>DXbueM-He2mMAa%v z;k!e)R?ar;ZtDTK3Y+N|U&=ab0R7R?mqKCMar@1!f{0$^j>i#`_NLo!5yZxS5}Mcp z6qDE0oZJc|4p!HdQ|@2}MYD6!{bQiODzEVqb&w3{o&bq%+8Th1K_>AS8l2$nH5s__dY8_bmD&e}(mOVMTamhr!y<3F-A za1)(kE!XsbD%9|6nZ6WWhXpUHlUY-uVT=Qq%vYnsq3~e%G z+VtR=qF%p*!YuAU?Y&DjxUP&=Sn%db>Ata?&X6W+sEyr6Ric!ry&u{d*4kpLPgV(g zG<(kD=XIqY_&jv&qjfiHCd_MdiPw<)42(oApRf(*)2!K?{ z^alz{$Qmi88u1Dp2%A(WJ5;mDKmtHtp%zI_S$xHxHPLGN6<;Ykza!fQ3v3w(7g#)xrG^6=TIM?jn>xFtbtdHy26IzvyX25m9H z8MtxY);5bTXEG6uUuZXHa`cSaPnZ<67plr1W>olZqD#!1+$0%83O_i{5(?oGb9~8wSfxnw7J7qI03g`;OJ;G_XR# zVBgLANn7@)_AJvv%}$uCUdRPp`llqINCk9_l z%H0L!^=bbu5vNf|$jE7griZLoYoS{ru@SG5t zwESY<{9uOhM6TqkL%ep9_sL;-b(s;>vhz<5Ev3;{3s^ZhN-(#BwoELKtYZJ#;6L+z zVhzqM^DD!)PfgE2a8Opi6=t~jEek#1PC0vA&Me91V2;q!jqst%6^eYZZ$u89e?uG= zst|#{9KYP#Z1`Xbet^R!%HgN4^d1iw875=9q5jXwdO$t0g$2^ooI;Bxk$+NLTEssR zt5?{pKj@@4k@R;fFzvDPiJrzn%jITG(tF zLm%v({SJRK(&lw!H=!cY73+r%|2LDQWtfM9=NO2yPu*Er&5wDlUTVX7Dcgaf@R;d{ zld6!#9(qSvQdpG~>yw6d1uy^C%uMRaI_xdP>hy~&ah5kiA)F0K+o!ZEznDkwUHtvC zbI|#L%4@Gv8(<@itzYsz)hPuqf9d(QPj;^J&nQgj0>$osvTl=bxb5vSTtghWDnBMq zc2lXw6pltBmkW&zmR?+3ba1RGL&17ko-rWziFxCCGBi&Zw~-*+WDys?DP&#VForS& zZ)6*P85hHdnGQR?FP-~1{3ZWLmUPr#q%rasVs&5}@hBRSP9G^={0M-tt~_@rbJB)y zS~&5YtQJxp5z1&f1#HWk(F$)pmf`#H%a-L4AE!aR;~Z(OFq0iYN&iN4Xh;codSFD< zTa5{+FX5Wb<-c8wuDpN5POVWZ-Ee1Q0vTNC{{88;af2GIwoi%i(?CzkqCDb{%AAxw z@1!{#THR1CV$GxD{xX1bc82gCrdx8(Q6I&~Szp-R8L;wI$Rvu~jGel?RwWxT^G6|b z*yu%2B}NTWJr%omNQ-w0FUXG3**r}Vj~1|I^_&&^5SZ*OwmWXz9e#A|I2wa4I1$MHP zPmy&n<)S)Yi#K^WCuI=w_iSesiY|Zx0Q@^V>RRsjCI(gZoBuYfbL0bP|ES>*$uM_D zetLwRg+iz6fw-t5yWreFgkSkGbdN3PemW%SvLGt61mk0PV%Y25nFeS37-b=?Wsk~x zwRyD**OY|~>Rg^U1t28GtNR6Yro+V^muzj`-?!Nd@V#rm={?CsX7CEvx7oz(=dW^U zPPt>%J3;b%AfvQh*9)5UmT3*~_DVgu4of*$#KZ;xwAU2v2e(>O%6W*|sdZZ=(>w%V znO(L?DkHKtRP(ve{<&Qx79Y!GKjv(KxO9ahmfoehEm-_pDcQvsbbAdVi+-VvHrBDK z#%H#cnv|d2f4AVGI!r})^Bsrli4wzn!r|4xv%HsY%_C{YbQpL!X|-;-pu)lTlj8_xJig+2I9 z2g!O18+Bd;|0(U$>#HID1n~w{SAfI)y{#;Dk^K< zL@U8W*cA=z(-z#K=eHb4`U->H{!zL-n))6v`gMe(N6Tf~&jnQ4R)jeB>YxmWP4m9bxhgcJfkojxc4#|Bpb*+iC+c|C@5+V}quI zYBb?1BC7X{Vm)GQXw(=pVQG+*h$3r9w_*@oRIoQUbq6cS>Fp5u>_*unFe^T1O!{9j zwbJHyuz@YH%!eMr&vy5p(kwP<3dJz>auYoCCgb4azUy^Rh-mx`@&})I42YJg6C|cZ z+))>bpnd@>QzfeNCIXjf(EFyvF*d9)?yVTgcr+*9;(i9dyY8k9IxYAb8(I-D161;O yR93~#J+>HWY_AI$cX4i<+??Hp%H3XdZm(mu^_Elj3aA6OI0x`dAf&0C(EkC$@~R^M diff --git a/labs/lab1-submission.pdf b/labs/lab1-submission.pdf index 101226aaf6598a2e85ef89cb317899f9174fa480..9f8c2727791a4e61af05df7fbfea314be955f9f1 100644 GIT binary patch delta 5153 zcmai!Rag@MqlOt$k^&>7yHiq{!6*d;8Q};S(nvWvCaoYjTDpbNohso70cntK7~N9i z{O4Sqb9L^&o9}(z=WU6@ZI8pPVWLAwDJcva3QHvP!1cTgmmk`le($&xe{+6NiAH#|{j7ryBQm6SC37bfMN2XgS@Qyc zze2MiD>g!)1B1_ymfohaUPFQ328>^X-(ZN1Lj3Ea`7j^5r5-7m0pD=vK=$*Gv&5&t z0OGwQFx2Q%_kB{owEb?VBeHdQ`{Go4uXXN{a@JXr*^OIa`7#gc)k$5M|CSZ|K(J&P&G_ySwExvsXt!L7m&`Gn(n zCxf3V7-MacPL@g>&xjHTz*fBpmOk(RM(aMIC3KDJctmv=uY8NSx;S$mXEsR|*8lTG z#x*VNEUrWy6pJ}q#Icr0o5w6s{7EUF<{Coag1QH$FT*;KeHs|Sj+RbQ!Jkd#>@hD4 zuCtvaw+NWaLey1rZrAymcpJi(+ySW-1ffEi#(H&&4W~HCf#nl;?`PiJdF*YR+azfk zfIesnT_!&(Vm4gkFDWx;7e@_;clpLj-X|9`H{a``Uf!EMhF8f1JW6 z?tOU~2fSwAc3fx?Db;)lYw-4$WG#vB=`^ocNo*eb)p~bD0h>xijyVI7*H8HnV*kfQ zq^@(A%9F?=Y_wuY@(LQ08A|k7<4I;iFP^`YTnWhBRx0Z57Sr{DoCcVa9q(i3-GEdW zwv`vZgoz@w;FkLi8vfo#>dDiT2ydp=}aoRm>_<1XIAz1W+ zOmt?TM>heTh^{ONlbTG1mYm#2;-l_4w4)vcB5As#1^uw!O2ZjwZN)h!=#_~-bfn8j zw>(9rsf{Z_iJzs%y)aNng zfR^_y>e~6BDP21f(5fsBgll=E8_sJid^izofWJhSNMb) z_=-qO6%g?!VF1@Fga!AdkL#NNhEQA?kDYi%{BycB7ErH;rd3z#rY%|OX3&A_#VaRE zp3YUEziwKr6L)8DXg+Hy-#r!}wlmcKADgL>Xhsmi`YCMQ+Xt6DVTULrd(k$lX1{Pu zPb0ky{#y+j+<3%I!lJTT46I8;@O^^p5qJ!EQP8#vLwrC%#ml znC9fan@)53L^}tSx+@dW6T;q!P|V-WUo~H04ZWTP`}u6on`~NH|(&Y9i?L=S!5^wy*!{FUu}_+n>Pu2i=Q1$a>P+TU>!2 z4RhgW?VYKh^(r-pyp@+dUFG_2J>?$k#)V@>>S3{&)*Ai#-VC|SBnB`+K4CThs3Kv^ zNy-R&ckZvHLKzQR)Y`3hPhZh$MGo3pt5np$T3&3~O1KT6lTdC;`htR}@fy4n`_qC=m+%KyVlgGC0h=o%_UPq6Z#svt@vX4T|Rcmm%i>x_90e}JhH)92)p zl@K50oq1m|Dy^K$1JT@US~K>)iF3;3+uX4zM{uRX4r?@Z`*RxTzW?>@sv??hz zn?>*%;?=u<4sQu$T2hWL?Wzdh6Y{z5w6q*)Wvt6&P#@LT{w*O=cluoK%Op*~t{D^b z%Ur4BmbH<7)9mzp%F*xd*AkYR_yEaU5khAv)nz6l0BbtIRX`HefkYcz#QmcUZaq zKgU;D`H&^_6o^HqaIq&5M?ul%uKY2dy;ATg-`M_ax*T03{PV6M2RJtF~z!iMEFj>((Z3#PeKJ5Z*bT(lTj6{|AV4Gr}>l*07VujP?mtm7{0oLF|4` z#Nmo{`)va*%IMLYMf9&l$ry1Hdez}@0>8r?n_1==T%HNfno0(?{+%|@r0nSw!o1$^ zvhr`=%$zMxyLuXvIxVKhU{Urb2)SS9W&$?fS~cdQCV47l9TXQjj+j8kB1{ zx0*^0cr6q>Y?)eN*2N&+!XiD=H8fIh!pS1R>)@TyDS{BHpC}Z_od&o5dm%u8M$h4U zavr@dDBqHnIwtPk0~$>vl^XAzoSfw5)It5Kp(U?>^UI265T^r|*uC@@EB+qQusmzr z0JUnb+JEdW(Nk<~LikUJ_9x2ek5Jn6?V}s?g#VH8e!kLQtZrSd4lxs#YL)nw@6|sr zIrI?I{noB~K2F&@YvPDcw|@TgwT1EAJGtSCa?^!CQ(&%DijNPnkF@Ue+_ee6PmI_) z1TeK?qk1u^^Cl+HDI?JTl?*tjs|+&`7)%t2_w-BAW0YZpAA5xgLB+yE&%gJIW>cGL zT0h8b2JCB-vsnABI|F~I`acvS+kF^_ZRa3eH&VIVR*}eMA8~*ilG2yv(akwOlHZpb zu!_|8rNu*YJWL`TeRXhi_mGXg>~y(4wbnyO!-QqnYnF;kUlErH&(u=Z803L33*x)A zAn24Q7e%`frqm(-;PVg-0aZ9K6|;blK&94)I)1MAEQ3cVyX z@uF5Ge-}OT)lYeIBf?=?-s*Rg;A&Ppb%Uu=F(r({PJiCXEVlfR7U@PrK1A`p2*J&R zouJp3`rAaepsd@FEr zz+RP!zbmHEJJ3+Po$h_UtCd(icJnZn@Iw8N=T~+ES=ua-zl$7~H~QD$c6$BsY(3L@ z+ixFU)DPdt(Hc_?zCmlD;?d`z#|ZahW&iSNW?TClrOo{t0U-ZLlw8Fe z1Q-BVk!t}mfF>kEpbq|gTRN5j=@=M5MxY=9vRcH)e{s)wdx1O&WXH=zT<0QbgVq6( z$eSQ@5^OMt6{lyD;2ByTX&o$%xT?6bX&JADsMin@=u2tXL&LlQ6SuGQh%^2zMUUi3 zd1(U-f8lmyF)72_mT=>0-h5dT(*P5=k|nyuGP~EP70Cwgga=h0ISBm>xbbd(1l*UH zzoa2>`8!D&z?A^yz}r6+H|;rys)s?QgxmIqi~goR{(2?VKp&m+^XN%4f^z&gpR!vH z#D&iBy*%Z=7=_^K${lhhRq|YuCC;*W@*)?w*<`;7=hO9-e`F=Z&B+DA8qYXh)#}sK zmVbhPQ35c?Ar9_v7gfk9B7o@4%=7WJ1oMj&&&CC$vmEL@)a0P-*G@yNgOQ*H>H7YA zB|#q1q0v=mVD?Y5zHw2DdR=JSvXTe zR`0g5H|0+z^Oaz0klqRxSh?g zMs!B)W&ZE<?`-aKtb-VrEPlL7Gg_4sI&ZI}bR5?wN z>WuklVi0{H3m;*hWSR|`!j67>VqrzOMqKO<&Nd^WFi8u9ZKzt3O$;VSnw7`yqW^`a zlbW#u!Bk?eCe4z7w#oa&*?LL~?J(}-{IA)32JZl>WC`VoUL2FI_z7M_oZmp#zjWZj zX|4%xWKtc%V_T-}Q1rwWAZ3!#itfcV9ycQtt_K`jutO#K3;1tEW~YQ~faF;=IrxpS2+s`Cs$W)pbLInIE63T^IV1NISvQ?4$u_DZu zU)=vJa!ZhEhO2e~oh8Gbp990#MCZU5oa1Fkh_i`Y;>0bpa}9HdO)otSH}lVLku#!* zYb>JLZOJm(2Z`U@6LV*$n2Rd&?-+9LY7yxP^?7Y{ff*VbKy?ff&QZWw424|dM(p9| zmWZc$rJJ5P6Mli+e0L#%`${Dhu+QB5=BM0N-Z#|D9rnr*0X^*`E3Ps7MVU!LXvqUu z_gO+fwgv7Kf)f>@X&#(LC(~RzGJyV?i-7tSqfTgM9Un(qSw`Q+wN!ZCqA9 zidVRVH|DEfPR+iHQtLOWN3B*h3!NX2ERrGE(8dSJ`D>qr9SxR_%X(ea7C_KUPTw2KL5}&BbpCK|Dl@+#5l`{t^?42u&6`X^$xfqZ^zbskD?ACw}Yw-0*T=M7Fa5Ko`iNzyLhx&4zNS@SzfPVSVoO?D~4H-G^CHbm)5jX&IFd9&wGCpM#y6J)fPUvjuzpSHtmzYo=)*MS9>ZDWeOK4GaeJZ?8~if4i?s?sg0S~T5@ z^|6J!9EW4zI`qWmRNo%OgYRO2glIJ2AiB-c=r1HYB=eGEW}Gn4JfQS9}-g?u)b= zqGj)89|d{IQHty(;TJKDFK?RN#4tp5UaD5j5;Dac6>H~Sw5iX}($Y35y;L}N%_3^bk-n7eX{aY{% z@UGhKY^uTxF6y6bh(NFisJ{;(F=l%jw}PX$N-wfPTD3}*++vknfT!?-y5k39;19aH zAFQ}+M7ZqqAU0q(8xsc`Re^d020T_nz|#`ODAvQdphjxShM-+zvkV;e*THpBDNql@ zG;=wGduoNa?u1xsMJVk=Nz#8*r4NPof3&6#d0F;RoIXk@{iAk&B!7COPCr1$b}|+7g;%KE%r$krI6i*>}>@G3hmpi<0SX!Z>b>8`)H)>`*Uix@$0-X z{(d7b#&^4cYueeu(%EH$*@PP6t{<)TWAWt{;}IW!E@#PVCDW$m{HYIfZ53!~fqah_ zX2Y*d3u)zIZ1pjkyzvOjB`arf*e6=&wS5xXlboZa4u~l7IW`u}2^vcrV*NR!AF2C2-h9GL&{QGg`;rzLI;f!|%-(D8wgmW@O zPB2+dSh&urHMc@b4#|(vp6lW+4xK?A$vC6ACV+m9*%zi5i9Giq6fKGf7_2_a`?VEy2@)4)p=(+tf3#8vy-@l4_kn`UxRRa&9Vo zje-pRiAoN#NChG-KahhCGhrW0Pf_$Qk8kcnGoZzUPd7r1|CEra*y81y-ducqCAI;d67c9v%YC9qv7yvtv#GyoL9;2XL0o# zfbEND8cHup@;MVnbMC!GV#t+ndQIl!Bqh{iHqN6P z2kqyik_-w_jCo8L5Oy_5onU4YFp`wbAe`u3Gpo=$?OU+lFnNAJgSn>^|3~cMB|Aj$ zf4m5P;}kAO^!emDM!Np%YVUqV7PU1Nc27L+Ovi2jy)a5kLJ!sih%R32IGiEBOm0>08Ic*q%{_2lxV$C2u0}iv zwG7*tBiN?;@q)zJB4jv^JKYOln|-ugFdqrZI{lD04Y_41$>$0Nn;Ke!dCq}Wu{KX5 z!y(YjZ~F~zYns1l9K)Cw{n_HfrADc{1RZ`BIVYmaNAz0gP{NpyxZWrxx)1C->ZKB97=14bz+7+l6JRnLfi}6G&b=!l z_gxMQWaG#ovIqm<<^tgu`v7V5wByOzYhiQ@*Y!~bQQTd=E@^1g2COyMYv6)^uJ2l%z*YViA*M|_-{&jNj^|f`kGjS( z7kHDR?dJnoB3ABo@yD2g%0FjMbw`8Do}Vv#Z=W}=TH4-b#$~T8-CY~q?Zhhrzt}J0 z6cOIEM?-&&yc>A%@lFl#+wTsf!>LN^m_OJ(S$9~t1mC!{UJr;4H!K-Rh1l34b(cS{ zO_-X9#)!>3cWkg+?zei(2y%#;FfervOk58Nke)mgOH2pe3P&j9C$2>Th;tI`i26FtQXbk4V%`g)1oIu$>atO zhMzB%9VKsO>;L7o_CSa`2vYx#7l>XIV*Jl7rNUedf`E7>mCp=a&!@bZP?qvP_A zz6x&?YAK&?CVu5ol9FX@4vmdy#+I!v^&Psg?k6cv`~K1qj%r3ko9@ng{A1pUsUL~Y z(;=)plG1O_Yxbk(PTXYY#Dkv1XhG&4Ge*Z<x9wQZ7L>zP8SLonA}0L`3e5`-R46+iS0pq?bm0oaKQ14z z5FF*F{|&y%zyT5apFK|0`S+*vt~`+i98z_685J-p*l}We(bOCTWqj8xR*kwJb(4kT zg%~o1njb?Nn;$n2GSiM}s*f&p_Lgi*Ir$N)dY7+f_)>uO{63m62<=gJF80t>P{ zV=iIPDWRCJj7N#O`OGBONm#7cN!C|VH_VZ9i>Jf2MRrzIzE27#Wum=L8#${Etz$j5 zcTZRhHff#zlB_WXrt9hn{}0O{V<)c#v6QFQwy>4yd7ean*+SiAwRMl)1=WUL&8C_k zV^~IcE(Qa6@AJ=R|IrPQukESjeq+B7GHz6 zsk~gIq(9{1xJnM4;Vdp>)heIf*w}ys^}BAhxTb3g6s`WUd1ayNA2})Q^V6~IoqykC z65bYDL^dk!H;JpwdC2~oHb(HilsGtzyoO;yqHb06JIk`HPuup#Hp~y*lbgf&pQnNB zdVGJ$BiaX{4RjEs*pC&Rj1LW4J56QPf4no^=~p4|Dmhwz@tz#B#lE!J+FNqQi?Jl@ z0H7h-d&Blil4~jMCI#;*di13k{=Ci;o+(AyG4LaQddS}n z1L&rUj1f;CR>TQk8Nq&ouY=@k*QlgPXX2a;qiYm#K13l+a30zxgh_V!$Vh3cpL0bY1(&bEa0nCpNyGOsci1lc6+)Z z;&w|he4&yT%X$SVt+CiCj;>iU`Njqgb*)WYMAM-^+WeTBK9tNX7Oc^Pb)GBII6$JK z|6Za$py9zf_FZ3uXbBoJI&ejC{2q&BnbkNyiNDk%m-@?Fu+W0{$QGiL8@gTF*3qgK z42dfJATJR|>>=o~CtgSwBB8tKRQn7O-XuwEj`m=8>0}iNqyI(_y;N0oF22nzbFMK_u>Xl5wd~G z0>2M2!TTtaOZaKc6D@s%9d7wi0CJ)K%xSVW`-*86Exd^OM@_kW^QXNK4)zF*U^kH6 zXhu^$3a2%ynI?=u!qg2FY{;YRLG^JTDLl1%2EKrwL*1My1#!on9RXU`32^IG` z-Q@unO~hqCN}8mPhZx{{M^(0V-RV$;*rzbP_SJP6gbqDWzkdN5S@nxrfM7e^j$&KV z0h7YMQq8bh3K8BZc@D$j`CdOgEJ=6o0 zy|U6LE{!ovmi#1FnL*&RFs(0%5Rh=;rk;u+?i343 z9f|o%q$x3pK z)Pwex?9YRMs^9#P`1;6B@YVc`F<8!I)}3Q?<0?OEaDav>&z-s8$PTIC-St!<%gQ%ys{#0KTvA9)ATm~{_PP7@@O$K_B774)tlWsTuxEJoFRU9Ab7FGr~+uw>EaSTT&Q z*c!*zJ?J6%Qb{~BA4jg3tG;Wu7NCzbUflc=qt6H%HYYP(?@j|SJYp`@0t4g=Cvvg- zBVBiCTJzO=AX3;0m6}uG3V&ivji@{KBvp7JD?QYsu~J;7z7QAVK|ab<$H2jIpfAQB}Yf zDHZUS7Itpyt~h2SS^g_+kPvZ>$Xft}yf5^Mb2Ob9;-u_t;F3=mOd@is>czy}(iNQR z$p6hUrI^NZ6$V?BCtv?E6)Mz6tjg{3%bui!-;|7s7CY2x7?c1_mdwZJXKE;7$pydQ zV^wxr9t{SqXRxF^q+PJFOzLTbXpHG;SRb8zkk2Iz9TMm{s3GmwLme=CgOK|A?C){6 zLMi(Q*;(+T+*-0}KRRVlCdOvmXq;in9#54N+RREOX8EfOD0&ZG4E_yTf%T}09{vSBysl& zA6a!=Bm`xe0;c$|omPoc-OksyMWzGp_O@prRt78ML*tD5f(~>+JBD(ZGnO5BP zdTniVkLY`N z>u`4u*{M$o#I<#``7djNt9jwUNh4k=@be4US_xQNTMJkU2#J5Nw&E8Q6SA}x5)>4% zvJw*$kdgZTb--OC3?Q#{!*$5y*8HkgU4ckaDWsyi-^^-4PP^$1DwhT2;is&ytgz1j z%x9sWciV$p$2`mWYEMrA#TIoKyoM1rVqs`v5!T30LdehR+Up;6Vn|L=v`ztTr>La2 zVGOs5jOcPi=!)1-1uCc_x>5}8SZtmJUEUgjDE+S5KY;FOfVPFV+p9XASq;|DYV3V1;i@T#+XXL zd##P%;<7V)suTJ0#;kKnR>%)pUkK$@CT&P^ir;q7A>!7!@B=O9p5maDdD=~CZs^hBImK26Ce z^C`3CBX7lzDhBFxsN%1p8uYrR{li{ePrG7eTRqt{WbT}W4t5_f+&8Rzhe>8s15~xN zKu-e$P)<)k!SnX79lW9;tbHq&tRah>aF&`~otPORs$J65bMI8ofM+lh*i?MP-(0)x zp6j5_#i!{no_3ecTjO~U|2Q>OIEk3x%3^iRtgbcdEL29y38_P6eqNOl#Hi3WlzrPM znIj?&B8KX4vpp7Pb`~GhVCweGfPI0Ps7XQ$#WXITted^|_vsVPlDd0u5Lx6gF%305 z1}}seY%66Cw|+0z++3*4(!iqU52i3s>n+`65fE`v!0?@Bi70IxO}$+cqw9EEKzio! z&<%52vpL&2;@ri{^L<^t3W#vgduGCDQgrrK3{{C8|XK~=p(OmIG)^j9?KeLVTS z4!eLtj&=OZF=4F(uH>)+JlQx_|s^i}2 zThB+@^oEl(-zLvz7N;-5=8v%w9-iCVo-f~GcI}UO4Ys_NxUE>3T#kTocYcF2*9K)& z?IH7K2lo2xHcx&*6@`0Vd{fYtZmLwm5c8xz-h`VxnvEGdru-_gH#ME@h4v6t;~$Q* zrqvH#w}BX%uOS=RV8qDR;y~Ry2-}J4O+PBU5qBh%JmEtYbF2$#;#v306#K;XjSebA z+XYwN5N|e>+5`+Wm)bxAhTabw@g%;!xY5*4fpp-uVWK}gxcJ$lx?-+@(!h^RoA#`v zIwsS#9{wSh7|d(I#IvXYivQKX43pG+2rK;T0wjyW8;~XKC$I``xCp)737kiizqoFi zs6eLwMd+&I$FUk(NOYp>ck7wG%jit&x7tcd#lLeOHhy4YPHR+wEXIE z2$Qc+89FwX>3TG0MVe=Odw|~Ddjr?dYPRtn0H=oCy;Sz${hER!nvI)HVy0CzQyN4V z8UcFO5(Si0tbz;7?By^hPmkwSo|WFzl(7ekM;MpK*I}tN(=mC*9V@D5mp4`WrD8aw z<&R3=mIwQ%ZCQw-kD_*wQ-rh%8Wf|ZD|_h~tt=NGD)=4DxL;qljq0`WZ`#*rw$O6Q zDY}t8_U>G_RN4_cS}wIx?dZ{a_#aa6F5pKWf-L~LS0l;+jwy6emJZF+_*dceM3b~O zlmM0y;ZYgY!cLwJ*((yIrc^zOiV~)dZeekUrQd8<$8v=RiLA!_q!E+{4=w<$1>U$h zPwa3HQ+k6OU->rsKW#UMY*$hZ&*!7eJ%)3CYqKSB`-+V0Sej~5DL@AW&=zx zdCa_PjOXLC9i&Zu!b=zfI=|CvA{~2#w=;WrGEJt9oCN>KB|h_TVoc}3PCIkLQs1PB zAYRhADHFBxE5t13EH4q@m@P|SicmEBbV0H*J@_hlJpkthv=*id`|-k#PfM64lJoH| zhS_lj^dKVkntl~>ct*qVY~+3_AS)9DpjQgU4zFMucHo+}o{f%-ES+o$ugJgczR{wimBH5yrhX-fs(kq{GlD))OOJxZs z%@N_I!mR7aU@;MQOGu^0Br+ibF`DuEyW_l#nCPSg#aCLb?Hnklf;@k;la-@U?lir7 zZd05DFoI=4N^FJ<9cBlX4dZShvbaY@Sa>JMS!_JYq1>xEO2`K;#mUsT5_;Zco!8MJ zGxA$Z5dAKl4HbC^f|J6N9^4qkmWHs3wD#K~)bo1nvqtmLYk>wc;_ZZUHvQm^8T6YE z$`!%0j^{F~wB*tpH-~ zANv%rfj#v+Bc!64%&lI z=Rc&-q-}9H>!&36So``PK6Xsmz;UIC1rZq>&6USy`=4AQmv)4OvGAFfWX=6lA(7q z?Mz~QsnnRT%-!jI(O+(!)yN2Gut_%E3|!MpD+ck9cK2^^zI0-Y@x~5J3N}822Ky_u zprs$flRh@shb6GjhUKP4rUv_pQySK7B3Dx-v(8!UYg@vesRo8tu9sts6;vbA57@MU z0$@SOrtZSu7^|Om!}J$0@5@8jW|YVc9IomqTTIDL$=d+C6yz)>_pJShAhu)8e0&b( z1Sr#u2Mc}xC!zME_R(J7|)})Y`-~UhpNUp?UXv+be~-b;+h&IW;rQI0X(n_ZiEg>73Rt z;xXSEl|%TYb;({?dGJk7Tx;4Iu(3e@Y`)(}<4|nA7BgHh*jC@1%{5N`QNiGmAbe4H zoPU5f;lo5)^cQv@;rW)nLB(PYQkP4^@FRN*9HGeT(tbxB5ej2}B7Xy!p)4p%BE3!BLcySjQ(d z8xKD0(BkIHmMgUXRnvHjQI7jM@&!e?jGO|P;#GOIzTkv@CtbFMypa1BN{qUiW6r~V zduA1@`^jp@e-HryN~MZS!E#c0jFp|dovok#5J%4hWOf!xnQH;2@}LH1-MwrE6CR&Y ze7>61oaY?Sh>nk5M_ekNq(2Q^$e2F);T@G{^MOIrBX87wHA_;)F#p#17GfQ)Y8X;x z$hA~4?%pU~7*?*lRBlmIy>tj3_?9CQ;#1Mk7vfMR)*NF}G^Xq)O*nw5Ud|lx0!0xr z;Cd&aheD2@1KgCzf;1vXlwV&F^+A zN;twy_D7ye@8L=9l3jd<%i-S=vtJSyvR6;HaZOMnfec;+)`4nMv9{8&w%<%tOpSB~ zK1P=ElaX7q>U8z+bL2#>k?s`lB`vd?p9mL!*Nl8`{HH_bXf)FKH|5aj;N0oPMi#Uu zROy^r+~3SRoy3(vRro6^0bV&5MQy|92)oiy{S$AvADPf-$=Shjo5WV##`{^i_@5`m zoE?wQg#wiNy;7OG_P_xLj2rBWBb2{^~ z3|X?+;17?)!+69$yE{;$uL$HjjE@MiYrXAJm_V@U^FALMvENL{3x3F)V3;p$h6Yen zUHy;1lHbrRjDVFCOx{%v<)rGWtf+)?hAKf-+?-KvE>Kq#)LB(oP8I5+rSbnIFv(VM zKpFDC7pC;6Dt(kAW8CVwxt%$gZf|&(v|xW!x1I;gQzKo8{R8{*=l~|qV zQ+W*2>(ct(%(R=m!lH5_;c|k)a^Sn~MW~s6Ixx+|Ro^O1xuu0|O<}KO%%4io!?TK22^$vNrav?lt7^KUyXA x7<(nJU&Znh^Hl@q(HGZxxtZ_%pOx_MPjdM$`FvKDBR2d^dGNwQ0D*u delta 3846 zcmai$Ra6v=x5Y&SWRPZP=^+J&oB^b}MarQW8Wb2pN*Ees2w`aW=#r3fq#5auMnpP? zZbnh5``xwf!~g$uANE@N;k@kMTIZaNsUxVaBd9l^L=qKg=}6KAE!LC(H`;1~hOl+x zPo#a7*0YoSW3mxV8!sz8;KvnA4y3g1GA1mHvGba?W-hY0-vrgN(y$)vVf34&LmoT~ z|7B^vLI&wPaZ*uN#4wWFY%@6yvHP4Ucqv_7RJ6PQF`Os^Wgp1Sw@Wtr=!q8Vz!IpI zh0n(snX4jax$cSy$zs;6F#p`@05Tte9kAdFdAFrq|HCC05i7hTk6GbghG4xOX|Dh*| zpQ8X&&BU$t{Zg6Eh|N{qI2_A)q>0{9rGBA!$cadfYuhemGMamv{HbI-LwUDJwKFP> z!D37d4@oo@lvda4=(XZyGmR{=z9x_D9sOd=Srqe6JgqjZ{=tL?RBvf7=;N$D< zX!jZruwa#9=pG5Ci(b6}1|Bl1(bgU)M(GQnYZohze&z9+F;t4GhU{VshSTMr6rSSp z(M!3s^d>M@$<))Yzlvj0B(!zNR1{zYTlZ_H2^>`gMMbvmiZ^*5q6i`E9fUNutk~6& zxxvfchLe`^S0Pbv+PvG5@&&&nKs3z81s=o?Em{N*7QxD9d*ysEdMcW6DanLi>ZY>u zHf!BSA?Bo}dl@z3YDfP2!&ySUg&H(Ofyc=uKk|1$k7CH`Et{zvraHV+k9N{Y3UOE& zy~#JOeNUV9^=+I2=|B?xeQ*N-l?854K0JPeoThg{c^B2JiK0{zc=xP<#ubIRPwHvV z@VQ&u{t&dx*{@ld(9l%jTslPAs=GPhC%tS>gT`Ex`~ios9^=wtf>+8R<2bB>)l5br znuJCMDfM3zr~j8C)x9-;q#7GUed=ZIXAWg|Gb&Rvx3Bp#TZd6OQ7-I%Pb&PUV~LEr zEWPs?dOLmq%lBXbcvF+frZ&+96=!y9g7%(TQG(=b4`T(=ld5!)hZm8)Wv{}c9Rr^r zrPys^(XY$a&tAEo1ig_yLAKg5yO{@t%GjQ5!Oi5_*%o@fWt6#oc^SYg=HH2IY)<@a z7QnvK)k(aX#S+}U1spe~+&P|%*n`nIFqLl_=Qc9S@Bsr{J*Y~jz+xH4u_;8hc$lj0mvwJo>6>KY%Pvr6)GD>v&}msWZiMw%M+*Dv!E7J8jUask(kQ&we>x zZ5+_PvRW63eU)oeO^?$2FyB(sMR8*#_oVvQE>id87N34{uwBDOQ5_P{g!Nml!2Wi6 zj!4@nvW)0#ti8qvX|FPj&1;|n-g;wxBp9&$DgP_w!TL3+;@oq}U7MUIJVPqVn%&JG)TSa=Mw58yY<=@+&-ZEF>3iQj}4eP^`4z{PvS}i?NBy`qOA1 zDkEp3U#VCl{mB5?M{&afm12=JCH#9>+=_UIgKye;KR}p?mgtO2Rb-w_Rm5~Rn(a9R z+QjnEIY&+m>Lco~8+ZFi-snR_6CF_`l7Y`;sK35B!!nO5C~5iAuIu@~`qe!GQ992E z@lxg~Dg*4B(`ok9BQtndxBAEcNj#7C_47!zXz<5gPP@w~QM9Q~qek<1Jnv54G-f+h zAjBv&G{;ALv&t>j4rus2LloGCy-cEd<^TEGc4pJ8HTPzw>hy{PN&4tJz|jN!1qef$ zK9sP4neQeg)jGrsNG16UVQpT;vhmihr{J|W5=Yvq%Ic}C65zt-yU|<3e3C3Nlj|Hj zS})cj<`b_c)T^|NrHql&)SQ1mjAsU41bHxnOlVH9h%EMRhn{mr-Vc&OT)IS`*880gplGp=?yKlEcda--p**Pxe(-Ns9K(@Z@&tkP4`ts zYTiJ<)=26iklT5-+M*LiG1yXWM|mQx#%}?<*=z4bvL=>yezNOfCVZ_6CHGr9 zC5~quZOT_l4&ww63l3(7YxRINeV|3anlRnQ`yCM~3bDAC?iGxOUZ;kd<tt9CZQ6Hxfj#g>Y9 z4y$vq1?$ESzjK`sBfj2F-h_q5kD<&%E8R-o(Iy$?k>AkN9=6oJcB34amqMam16w2{ zcM)NmLFN;6&i(uxLU~poKEGM`o$8m17GCGgzAv57cNG0FZ4q1I^vsJO-^UA2T3PdX zAnOf!Nc|I(7W6o#!~=SRG|w#d2SRg~#_FV ziGR{A%F7n@c=Le=e-JnKOqJ|_Q&^XM6PNwiKl~ELmT|(?MLG`dAWM-XGvN$9-Zs21 za;~Omq+z;~p11#!6#X{)0CxmBcCxF{mCx8L|4MvNoX@voXGyQ|)4pSjUahChJCJj` z$mQS<3d*wFPmkdmSezH>QboCU4QYJQ=$-&A znvM_nUnU<;$PJfm>7ObDcZz&gT&Z{IT{*>a2;hkbvwJG#ckTu}>vVc6V3icyx(p0^ zrLQiseq02xrcUVn#!u>h1q3D$MVr$!2>UYx92eF{={^`lOwrE-DKln>GzlPc^0j;% z#kf^Jnv_dpr1fn+S~cf7(L<=ymP(l#EOM!r-;F|Z+mpsnnin!3U*rmuTzs?F^y&6< z68;HGY(*s~|IiIa?@ZCqg}Fq3Hxwp+jtZ>-Vl?wD)Y?yp03iLLrR-6%+~90#D=!`U zE#QkjIhrHHXo`yvxpdsC6c=QyzC+YV;xJuWH+MAfStEs9X1PdynAw1!w}yrOu}f`< zECo(mh9JJrVs*~^ZhF?$nJBNaDMsZY`f2;;sXzJ8PTTW(!(!$F-dZJ$Gio% zmJr5dt1W@T?P0T(;&u5vgB2>~wBdXjYP4KGx07>Gi<@_O|Jcl4s(~!;1a;ZU`}2O( z734@Q6`1<_)ik0TU3ZWM!Lgfvb#_4oyRVH*(Va7*Zf^7PKMP%@a$|z4T3BC4Y?bgO zZsQ)|QrH}QOJbT`44<`@L6+MLg=?)Y_+rQ6cmh2toB9K7N_ksj^svRAYv$wfQQqg6 zP3dc?R^h=Tv%WbsLk;8FD}HFr(=!TS{EA_meSY4COlC>1n;4QjfsbznJ`E;oHQ?{K zYPmC6jtc3z7QLRI=&WI>!bl}CvWuAd+`m315wdHYSNT9sLfo(Yx#V6-A^WXl!g$lCVel?m{x`Jz zK?HQ}`2_tQ_mFCGgn*CyVG*%%B4C348?@p)b0>M>uCro;>^;X^yjA#y4MG1QMH&kA z6}8w)ZlAQj_nV}!uTxR$um5wdfHrbo`Z6a%UKhVG(wmE|U3NAM5%09N-EY&H-~RRI zZ|*_DHXz~f-xP|0M);5X)Ym}W!(*_tg2L6?KL|&_9TM@gFK$Oubl-CpU8^W6v-}{w zcp!2xeX{3f^QVJ5{ySLooJki?0iNd#`L`>ZX(HOWahL1!IM1E>?T(N8>;Srp`~1i4 zVH6XO<2rBOPgwR=6xys<~R(Hpu8R1M3FrC}PaEL}3sEFZbCsg`UOey!Q-6QbjK+Z+iu@o!J-=PU} qoh|lS#=++A#C#e;N(hKbX5o^HdLjgiVPb|XGdHi4pW-ROW^o+IizJaN8lE%J z46Nr}kH#J-Aa`Z(*Ex)>WPzfAC$wL`!q*ejnO5s{#LW1W*rFo;z7}%{uNQT{|0y1E zj`Z*Tvb%EOCCsqEZJgV{aJE`gn~t>t{Sj5kOA=4YYjk?7C}pO1I3WE62V)CSdew!se4T())%5RxB~=hqwLB%~>Sbx0*m`}HfG@X#EzyYLScz490${_yLG z14_eIWu=C!cXfQXI%A~RZuBy9JA#A#CC?Js=e&7G4Q18ObVVdZh}nSY_1O1ETrt-r zpYvM9JDHhZ@~quXr$66U9Q!RbPE+g&qe91%cMmr)zT3H?scG;`H-)@?NQ#4XF^fS& zBqa$vy*#YVoeBJwjZ>x_!ucqp&Vu8Nd=iRc=UbR$9aEktO1C=+<>pL@Gcz)-3c%3SLLIS!gnt8P>a~h%?!J9JCeCE; zkLIp|-!B|mh}FI_G%a9YT&;11ETopo%Z48v&tjSb^ak&iNTzf;`B_IXW&4N!h1{?M z?>NKyUXLrV@6}rBTuu3dx`e@fC^1sHh-vb{opOajLDsbcW6&EQL#@UI)CbkP#o&3I zvxl|@vV?SG?;`nzi#ZBWs@&IUZ%W8tT~L0|cKP&do?6zl8*C7yq%8d_mLa9}BiE<% zYf4ADw*mtuGZOC;B1WEx-!C$QLx!xg{8vF`da4#)E&+m!rOB2dps!n>+k||4P-i3* zGGDAO%2n(xW|^;{wxdq5cLOnpC6AW#eTU7%_J7}FizKA}((`k1*<$@6)qH2Qr{ zIuHMi?PJSCujx&+<-(U4lD+KT5ca+O4||^M)4ltA0-M9qe8B5=;MI;)eVqN91@*f7R)*NB~sjOB?zkH{DlBm(Y z?G!{FU2;o9qPyr*QMK`2WAwqx=(qo>20IRbBXnM26sK8c%prP_e1tR{Lj8nNmh1;} zwgA*goh#egx*6GUNz@3c(bXhyF3i=634R+cTZ{pcBW4ENB)rrfZQzv;rr!taivBZq z*i^>|9?fDLK)=cL3yVlw6tCZM1Ke!#08NRXm=Jza4|&Kle1QX44PTFdbS;?pUZ(kfE} zXkO>U)#(LRuB`_Wlun;pS!9uDP=BgH_vq1@M_Zykv|n@vG1Mw*aSCX7YS2;w3!VwW z$FU(1qT>zoXFhgq~ewqe`rXn z!e{Ua0HlGLp^JbI1hT;uxIIu4>2hIf?}^xNB`TzMMKT0O_QcRm8)yJ@nF^tXz5T5K z7l)TWh|@1t|A`5q?bdS{8M!sHR@x(iSp-B=M0KcQ$}w?*j@?UgD|<>s6o{lOt!WjThlhg_l373(@o>5~ zwNvf)H*>so6GZU`Qlue22c4^V)_R$ZR}!_*HvnuLz}x?({@=m%j|g00bU;o_Q9hU= z@FfJ;{%5kBCpc;NFyf!YA_=+Al)tN&jxhB|H&U14X_Rfox|K3$B*!Z^>=U~7KoA!8 zmr}iW8rJ6Y(WZU=qcEtO-fi1Tz_+<*W0m<>p``SBkgm5}!T z1na6lgc>tDJas>La=VZE6*5IpE!kduJ`AbO6;xb{!_{4AZsyu8M=*#}Vqu%4F7Cus zB`PQN6cGwh5e2yOC1Y;~y@-v5KCWzZR#)2eYG>P}PE5^vDp!vJ#TQO3#;z+inoS#pRo5Y!gQYd$yJGxK(%)aaea|;aThcY}r<#G@H{X^AKR>jbN&Q`F3x5au`7&zMP0zcf z7iqmKx0Dz3Mq|*na4-8ppY>Z!Z?_0&ka+OO^m*y&lEZ3Vsj#7Hnj?fG`o+sk-~pL| zoW==)R0~ONph!!@;=HIYKLfI2LL#xWr8(^6D0A!wL4Yk)70QckrfC+N6!ORnFm;7r z#E*E=qc8LXfRwlICEr{JDNIld&{L;s}Ng@k2RU-fyet_SmJ<4>_wxwEdz=?%#G|)MpXR=qrjT z-v=*FG#ZOw4GsK1MzC-o5Ef+OV{&cE;XHf?5&GXE=+QUyTvnnE$v2&Lb@TxsZbo%S zT72k%d5ay3!+y4c0ZdF3a~ASpCu-=2kTN#uXKtCXO_|I*e>(EfcV3KOj)6=*RVBhN z%Db88s?P^Yte0uou>@rJ*)n%h?e`+hHvbq1SZB87*-t>Rlm(cv6*M3tqO)O7Bel#; zD!!+|?f5y_1*h$`pP3sOCQw^`$CqTmVbMrPAuQ_mJD_sYABEck*KtZ|RqJtQ|vExkQZbTh#hN`{s-#?9F7WM1jVnKUg z@=KGUFdkFhS@N`Rc9xmpG_Ar_{8$<+UQ72VYG$-4HAaAT1$i zN2<9p94a(fk}grQ=C*JZKz1_fVKh-&`XG~J+`nY)VW7@AmJe+R8H6H8a2@Fx>1#Oi z%=Y2c^hUcnSwRp{kecM~M&+-QR;dxwdAPeoj!>&faFvOv&I4yzjZ&5m?FQfH99*bs zP*F{FNbSSd*hj_sNi!$0&ddn~)9X-KxpG?0=n(+U1oT8CP+Ly_Xg~J2T z-R(S;{sC_9i964IbiIkAQ0ZOM^dzSFC?Tus{2d>7ggV7q03+Xr8(|) zc>a_U#>qJT>9e`7m!9A@s``GT=x(_u-xqI}?~GoRjMnFL1`R9GjEivh60c}PP~$`sA95<@H=~aui>HNpLm{T>lhMh!hKzSjLGP!7Lv#G1Wu<*1 zYra%}Cz;Q2&QtDksJ47T{Ntj|zgV~j#W#9(h}(3jdNw3eyv9=vFM_qCk9G{m5oPhW zj1|AgHo&`3O5SIWE1z7r3I&Q(POD_L>mS0l&KoGa3`=Ud^QoKsTVfu%rA&V$y@BBI zgCX~t@*$mN1I)jd@iRt)OxDM7Q8_L(tdt~=B;{{Zno6~zMnq(^$h^tQdGS3vsa{Ly zzk*%ck63lRY5M`E`1zpg&X&(!E3XnXEa6ICmQAi##vSnEXl?Y!IE#%2Re!*kzOkJ< zwB3IQHCnZ?@$cod1t5qVA~J1$s)fh3OF?AS;?KHUOufR%z1x#E71Eta%R<@bTtEFu zlknCHHU_af#4K-N9n4-8;GoI{*KwkJ!-IUzo&8Kzqea>_7W45^i)GxPk4uLLcdRa; zC<}=?`e`b;5$$l~ZsRCw;~Y|^K7DU535t_;JzBQcvUY#^+AWSfe;;+<*4|#df zU8EklNZmBbAQRvEakV{x_g)8Ll!T9(DO-+E&7_o&Efr!K6mb49Xl&ybH^C;r-1w2s zUcF>sX{>{TsXbA?PcA5#>Md9+iMC~;xhndTK@Kmi9+7tZ7-tp^X`zy=tj$D;$>P{( z+JP3&9|t5iIA^q|NkRO*^Vi|#2nfFMTrqt8{f5PmJ&U%DYNgr2Eh13tFXa+zr%(m1 z0hOPb81ZZJx0qPjcno^ve;YA?Q=-tAVng_=Trmeq8FKo#>E;xs&KU&D4ST4forgmT zURGpa8o+Yc99WS87KkiO44IgzaIp-T%1U`&%yeXlkrwoeWW4%{-J!8j7ZTT4I#uIuyb0d$7=Fw#{eRg z4LG8kJ*z3FriLt-ImROZkP*y3$LtPOjK_L3&k0>872^0+o`289trE^~avpGw6K~=N zo4kJUCyQpC_m%mqQh6f&x70F(lvO;P%LTU;4~v(8u$UDh#d`EkEpe>TKG@$!_Dp_r z*d3Cmc=Yr5YO>DcVdw95WA~{>#IeadGp)CBX?G7IjsTJZC!HSNci0=D%8QKb`@N;$aqE5IZdJbYHki%~Zccpu|#U54GwEt?J(weE)pnS~3)^)DQ$IG3tHz zvxf2>&Cb`>>=#q*zAc2lL{319tmuGw1n&wU!P^Z5<+i*WD-`Xrar@3CejRx4TPxRT z*gYuNw|R{;CBOBcZ(r0JrNJ(3=P4dHUmoZCq<}?QG2_z2Jq#t7^X9={+T5Yn->M?4 zks85vG$%e8?TRDHi@_30DP7bG^R*2j`Hm;VyMe{>t_4%qb_|HDhGpVdDD?=H%75E(& zEF;q(gN=>T5PUuPnxFMVR?Q8NCF%jl%4mPG&>zzv?zSh5H_?DL0-B;>+V6F2grBi; zTRPSgP(6!eScFO}M#?QlQ;1^D!aqM}QyOZNglR@n=ExAHyVYvqKSIe7h1Da@w}vf@d4nbfrP>>NKI!c*zgLM7q7SN4WVsr_}=n|yU(b6?y zbT>Rc2k&(~|D*TpKDduQ-?;AUMg|bZ1rU~nu@Oii!`aw@DmH^^ZG*aTaj;QF2k0ns z&X0%JcS4b{`C^Vc5okqEC+Ob;`Axxer4>prm<|=wW*;;%h^f8_Q%{F-D6x~7dy=~SpAYp?fNGOFP z%vK)yC?ny7CLF@AEv9M|E@xrwr6QX}-SYkBq56 zigMB*``CE_>F3WWKDjuXnb=WyOd5UBwh0oU4Vk;5^W1y@r777^2p-UJEH-*;aQuzu zEknMjvhP}RdUyPD@r+$u8ge?7h8|ZnV$HUIbwdXRjHs#K@s-jiSf=YAlkQl zgU0cYMafB#&JvOXv0?qIgB#=5BbA1)t7H|$W^0T9)Z`L{gYHDZ{f72M-tr0t z_&QEFJ%}3izx-hI|L`reZ}}RK>#c_hJ%Qrf3$aNLF&U)P#Gc!+Srch%T!*1|$_%AI zrdW*g*R9-m(hS_|?>v_EV~eo{ym_oh*+}Y?XNdAV6#G^jwWcT(M-gFS$8*?jll?bL z5v80qL2vcBl91d%v%IQR?Bgc)3|sq){OIzEYWM6eDzN5KhbvePL7i1OoOLGR%e;+4 zg{scxiVfhJ0p0Y3@P8y_fT#bVI5ZU#C;}px_fA76+DnE8=}ZU5@pXlo&IGLyv9TCZ z^J}!+ne)p8mfF95*caWxEJdf%!`Y|4)XvqFsx!fd)iSNtcItTCGHnD|Xc_wE_D+J-!3CCEj^?@gV|Szr@FyWdDfM z7+jn%>63#chiS`CAOz3vH1^FkVc2GR1P#6o5Bc!#sLkYaf}ak)q(%;O&ftyzWE7^j zaF;H(%^q?I4dn4Uy@&+%J_Ok%=3bW^HcV^g;nF&P?GT1DocV9u@;%ul$aiVdsa99!zJHBpWh$<}Ac<`S5eiMWf=nG9ZIL7S-`dB5zePV*pJ0(@p?R84*qM!@Om;1R2|4YL$%=eFn*+GWfYH^c|oW$&_EXIP` zi<0csUxmi~^?etlRg3jj6lV6O|7%_(p6e+f{vQoc8c|aspwOLbIvIO@^-FFmBKA0q zMe4OGvkLIx+Ju_*#Dt0y<=IK)(Ov$LxL=$$E4~(?d$x^Jo0H~R74Mshj_tMba!VTP z-K5@wU0qglKY!I7)sP?yxxwokV{NbWj^%rzURO9YQ*GGo9bHU$1Md7&Tz-D;t5X*1 z=~S#xgzgq>`nu)>meX3_f9=OIZ8n}JQ~6F=t-qseMdq~LOMi3*>zh2_cRqQRD_4~F z+lv);T-|iz!Z1!L5R=dw_oT&mbaZ<%G5)~J+YQU{C_P7?cBvxT(BS68_+e{I9$s@j zZ=-v|LoVjPl+ls}=u)sE$@vqR%BcM9KF1RS#Z-kQlZ?ksU|kTY8TpPlY9ICjNi3X< z`yQ~e_LAPooZX3s3ojg(ujCosN4s?Vg@d`0!E9{3FIS(cJ+`%#sS&bYJ zLT1P^)3obVW;)^yLzu*kb$qNFveEtwf^bt6z6T?ZDd`i0bg(Md^KW!^F6qm zyPY<@qgz%e#)@$s?I2Ci#xCC=k0t&zC<`cMG=XHJ+mP-n&jp0mw7TXi}{wi=(eGy+fYb#iB)li9ite<*Pn_EBdSCx{`;RgEB ztYZ3-v`sRYU;$~)Cj*H7HxO9sB@qzxU$uK}{zOeAa94r|t%ejP`t9M>tn(9UV}Jey z;?e$z&Bk5dcS?k+2`jV7Sda3*4kt#>3fm3VHilyHBW@bI7VOgfF&7if~W0twmXENVg}Dy1UR8T;xm)^J)>n9)PN1A}8t(nhYd*Etwfd>w z?#Q@}Z`vErEcDH*Z9w;MdEQvSJiFUd0a=7u$0fE08f`@p6aF$k$XDdcBIh< z;+5ZT&Trqn5=oD22K-L0iT;xTQwM}Kg`_x}2;0A*B28db4G{akCTN6DIZvrj`wqVC zn~JQ5BDNDz>Xmg#KM}opT+h_#TZl9u;@12SC`A8a=j8Afmtur&L^(6>;jQ75G%>m1 zGOcJalccx@LdYJmh2*C(FMBZ(uSm=T5o%96xrh7iwEWtrH1B!pAUFJtnDnbTge*D@ zKsn_S(NJWyHs}QKhhY)AuPo8S`68_~RSzC__}}+Yp-@SCz{#i_viC=DK11u3dbBBv z^BAM~e9h7W(Vc7riI<$`z513VmTNUMrwn-(MNmkI|A#$+`vofrEi?rSX^LGW*hTp$ zEp}#*6m`J6yPP+*?tO}}j%na9j}I-sf%kW!9{Ip5L#Lf$2)3zH72W2UoXz6k8=!3J zyGo9`!M{~6akmlv9Qy>gw-8zPy{w?tyDR=YVSQVB_zG;}sa4$zxi*X+G?y4%3SZ`NnLcqpl}XavN4T zMuY6moRGn0%$ut}g_IOyvPk3OIJe(MgB#E?Rb2Krqgl#Oup7Ar!^`GxdPzG$<$j%k zBUB_d%&g2M0%;#see@`W4+Jy@fY^)f&P$v4Z?kth&a~y2$9`LV+&CJ?FaKoH1=~S~ zFr%lOuV2c8Oc!bF%#J>832R)kv^4pbPwgFT*52n zbCRVZeYLqWey|gnicZP%?!=Ydajzwk*C4rfNTtr0-au7P0FtbW7MG;h zIze-~AnTV!;pw(BFwv6zy}NKm*G_jRXWC%hjbNeh%!Q}>yO{lf*~gIIKa3`U1XQF* zi5Zl%-usN_nSkvs_mHkd<)^kDgwik~2kNf%d%dK+bd;pAq;7aNUVY|qo#Y0S3HHa} zuR@!@9tP1X1(C8Yf$fUpK+yZL9El{qnUinT+R)6a2i7pDu$oUFMTHArUgKq&n>sur#YUMf9UC3R>sANNtj4A4nSy@nEzDbe7mrJcyT&CHJ2o4@5FpE#lw8{v{ zz`bj~*~pG$Fuv}UT`wim&B%^_UsJ`uql%7G=$e3X{^>&Q)c7}w& zy{hf=WtU7l%Jd8{hUG;`v0lRQ{FgZK*oj zAZcRp0>z->)y;x>d~c7aw$ezK?9CG?r#vEV{Y)9>ibO@`9Oe z^PZ-Ehp|YNc}I&m@0?o1q2`Aa^0exD&*XQ3JQy}?%Et>7FgsG|r)}m$KHwP1lfH5m zR&TBIs+}ASrntH-c8p;s|Hz%@OK9LtiQH{EY7R5WV>QW3z}9z&$BmmQlz2zR6pxmOpYqOcs#)Hxrn3?YEIWr7_P;| z&Ms!5BW$~3rgoU0;3eUez%gKkg=eu8R1KNxLxO=Hc&v#-frIEW=DvU!&zmwEAG{ok zU!*PdOo}d135Bc(3n>Hnqkw?^pcp}NKe#`QC9`51wqyYXW08ntBd2V)oO}T?vciAy zruoGw4zBDX>Ti|n2`OJLLymNzlOe+@@)qm5)uQs2{aw-8pd-;+YR~t>%*Q-wB;asC zC;->4S>Qi(XtVV-bqhB-m&tjaODlU1SnC}-hV|%VdYpL>12Qj2#bdO+zOj?`CtWSkjyuqy!ZgA^m5mWd(8amND z<$MN#wvbk?$0(&qupWh_rD-_`W!_3`1_YeD(o~Md9u-C14Xw7|Y){I$I}Zzb6Psjf z*C9vmzJjIoto0bNG4^QA4(?Qe*eUPE%jR6O8%#S!S+LtMW~P>7*|TDV48|O+O%pUW zu#r3Gv-GTwv|3Sd;goAZXgG3C`d?{JFpE|tm$~cGwoJ32_ z=Z+~=aH|bSpNwsWt+v^9vOHC7-;G~GTU%VsnH3>R89Y}t0rxE0AFIjxHe^J)_=NZ+ z^1eO1XX?rg_o!dgq+C%NTCUCXHuTw!{P0!%^zh+;1*&pN%2V`O=YV-Swlq@vC}eUi zA0G!pKk;S4t2wM^-Ty-&Zr;rg^q7dsCdKfAmtF!-9}Fgk%sNQx8fW^!dOfcrg|8np z9a}U7}tI_M-{o*D=HIKd`r+ zpj^tvPg0jZo>}Q@sdhwr_eAw8)XD(BZf~i&#WL0HNbg&y-B%eCl3FfANLODvK$sj= z^4Er-y*gm=lv3euuJ7B7Q&ok%dLrEB=JMkBQi50*&h&7f&k4c84fCZ^(|TXs(uN+v zd{`7}34YG;?3^#V>X-76>YFAy!E5qjTi84WAM6JKbmtc8bD=vlVm|f*wOd$sH*0!~ z@k!#3UDmgn-#quH0>FP zKVc^Zm30lhsYWJ9$>(PTyC!+;84*L*N{FG|k}z31l)}w<+ebxhVPh>xUdIsH!KoK} z&ZOD4x{ZRbmOmziH{U|`R$R9j)Hv(GdRgI`mqAg*z$}!0W`crGr$+;P6zrkT0H5t3 zpS<`%oq&};$}}oVHCasK<288(sapOK@}<(Icn-%@4?P>;!^Ud;RKVX5ymRr~M(!EW zAjkX|A!Z-N#h;3rAWXy&A4N<=#ih+npMFG0OGtlc5UBY{N^?bS4> zIDdEZ(*q_zz`{4?7wz(g`yFglpR|IVlER3SLWTMi2+U0gc}&O!v;xielxPYIXM&qY zITN*j;(F>Jh1 q=X)YEZZvA6bn<>-KTorQz<`uB;Qt(`NH>2vGARmfZm6aT#s2`|H^wkfAp17f%Gmza%$Tu`y&+>F`@Zi>X0mH0Arc z*>gHSRgmTal_n`iQ=xDoQAa+#N9mxu)9K9plL!zb&0X~RyMkF6(bP_m7Nnjoyl2S> zmeQyCgK*UNXA(8$V{wgih$!{e_2O@o!A(aFGg90!`)QnSMC@BK_!_;2m8%#Nq6>}j z6aRM6;DrVfe^o-#2{*k4n`55)ne^^Gq4T(rzQ#&>&Z&B4skuodQcxEu2^g7v`r=Tl zmhsWUuzE8ZCoied)RH_Z$8c-awV*sVf^w+TWNuz509kenEM}lHOqIV85O(x8ZW1c3KX{C%W1Y08G;e zYsU?mA7eUevn69zh6CPRs+*9OFEMuTHI8~2afm%qT*QjO_irqO8O>2IZAJYZfs5AR z17CrFD|HswIMp!i)gh-BoU4GyX|RN3!H5_9v#@Z1#&DCWXV?9r*`S#Pu_e# zTVO6!jr9EBw*{68Er#}to65Cm1~rFSV^8Up1BkzguuGOWBQZ^5k{uGt>E6Ouwxt%3 z^mbtFXJTHWpZL+1FZv}1Yvl=by=yedsO7a4Jmc4VfJL8Shl>g8i+A>PWl1;d=z%;Jdn}X~D zZtWeq=|;OlCN>k-ICO_`6mY+1ur#(keBNIr^vP9YOhr@@-a?oRurB4Uvwmc15y)$> z)s+dY`}1uUK<01RHI;p32y&qe5sYNUl2*=Vaurzna%I2wE`}-m2EhX`{hc}5no=P% z6Sf=2Kk~nAH`x9PdP^E8q)v+C*mHS^EiCFdHTD0&jyM|7r_5<|a0XCtH2rOmL)KN6 zG-r$7)BA|M?}@TBP!R67U)u@XH1Dju5`WGA`y$D275!#zE`qVVywS+ldMO450G-JpzOG3EEmZIcgm$4`UXai8Jc9QT4huHk+cPR#Cn(Z*t} zs`wQO?;L$o%N)v$%{I?mC6Gki+Waj0w9=ilVGPZQ#((LYg{ z8_mD6Ol~nbFphxy9&6*IJ`D5u@%Xv0S*>`9A$%klA3TVo7y9Wx#j}67M*qS7Gl8x_ zpj=@eCS9FCy*Q!voL5>vB4VISFOaj#&nl})%vW#g4}anp2trS(PIu~~*%YxYFeyKm zf&TA!HsXUo^7Cc+1QesM5as=|m4Zs=5**Nljfc#BR8b-wmH@=XaNnD%VV(uX0Pam~ z;NA6YCD#)LhGW`5Ng)AWGwFmPn6&#Qmiu)Vo;!TQ*aV)}wxb^IIG)#*Wj`6pyvtRj z)g63(cuGEF55|Oth?OXRafh9Xgy)=<<3xUQOWa>_C2G{^aB2p5|p| zN;)~#PR)mh*FmClV1lalAjR2?tdV-xjfB<9I(2}(8JHIwy$0mp09 z%%XhW`dsxkcO$x)e{}gS0_@ki;fc#lZf+iP1gHJs6&AV4Vp~sso*~bOe}FYtmUUQ` zS;kzDO;*b`%Lp4$O$y(XzwOuMp3602D)nqi+5UpaD@dYt7I3UgHxW3zQWWn>&+l;9 zuW$|Xczvt~-pD(G906xtiZb2S=)ruxDyiy~U@M=*qAaRQ$>$6`CU0wIXRAH<(+wtSGn+aBRB%$u z?jpq74W7%!KMngx38kaq+PKck!8uMvMIh)3Ccj{bk8o!iGU$w1q&O&q*i!nLF9`$- z&sd{@I@wKkXw)`=HkqA~)?wocS`d1HpAw;^5!xpjH$p5Xbk@I-EVVDZFPLCEpCLl< z+Z3n_L(IsIhX%V@(po}7W2II8DnQO~iR)vk}F zEwP-v+6V1eCq25_7X@u`6lG|iiB)IkX&B`=z?-O-+ZZ;;yuf=;v}~+m`$Xr3TfOsIdBnHc7eV2{G9R zw+k=VYG^_dY-Fn|a6WhHkXz6_;AjQ~G!_1WvZ|9WtJP*UiHe)sNmRh|v5e+{b~dtE!CxL74d^-Cigfj^y0WhTM#F5VA?4zh9B5rgOIoBzgZv6x_!K5 zO>ifsMh#JG(P~y3;x8KJ=7XBUPqxXR(B3ZjG|FT{vxAk-(-9=nDY^9ZY(8RtF<-}0 z=FE9d#?=1}|6m+^Rb~3e9r$1Kh=^F;Usfbq%a4P-7{7@j9Zg|pa%7gQ*Xp~|LqExQ z%aQd$&Ul>Rt&bIT%#Yzhl>HMd|ICZYUDV$J*w}+-r|OwbrrQ_KF zui4XT+IH{wLW6z?M&hOkl`g?H@ zS(U8y>s|peym}+)V=o7V8N|Q4=lv!vBZ`XfZvMxzF-O~7Zh%J+jO4KON zgz7=ucv2mOLv6a)xT>|(2krmzz46IINDwuV(_iY&--hF;bYX@SST7_py!)dXx3g1! z@aXd4ojt^AjT4hrYbUqv3Y&r$3kvoYNrI1!d+|_*tJjPm)$0g@M%CG*K$o7xyxPn2 z3k*Orc zq{|z}jgvjP)MraRkQ{N_i%7}b=I&4N<@COE+HUyUmCH!0Sf<;#3wye~kft(7L!X`6 zd#nk_lg=N44pYQRQ;n+TN?3Gi^?2fZgm1Ve2i%vMHg!LguswmcQSV+5PxbgGW=u7O zD!GTTzj|m3&X}wio+lK$7B~eVSI(WaRrXo0mmlJ+QB0{GcROzK8VPWSA@o@f7AV92 zwDDWr_si5tkTTeq^ueFtWnV$Rg-Ug}uC6!g@+R8_qo$}nYb7!0fq^H5dyP=`W1 zu7j0TJ)ux{9WBrQ;~>`P8UODMfhJ72-{0ajUWqk8TqGHa-!trtw|XWh_gB3zMp(H) zl(nAz+9>A^IX^CZdu>MPYVwQ&0i7JD zE+76@f=1%9{>5NFeOA+34=?XMXEoTUj)kzAh9*QRsvRnIdY8N(oUwb+`8wO!)sg4% rlU6k#JO~Lf3EqOV&*k~=gHb4f40u0*8%8YB7iUp{06-uklri96X|qbP delta 3289 zcmai$XEYm*0>z6evDH?@EUH>bi4{>1sl8e>wOV^rjM%EtT2(buRB2SGy)|n4Q%X>y z`P-{Du~%*0JMa8Iy!YvSzaQ@T-FwcxZIM(zBB?6EXt5wU7>ohq>*M&$gCTI-I%&o| zT#l`C=3HiX6!aEZiWY`f(xcZPZVkW$S09z^n<5m=%hUAC{p?p)VRc(mi@OK0ZWbi8 z_(i@+V^os-C(div6!=&cm^>ua_^%J{c#8z=;$8di{?Tu#7J74ea(b|LzI#-oUVzv6 zSjIYFh~;q_&bQa$5dExiuv7FZg3+SkMCILpMU{zo4-|f;TzUhEkti-Tb`Gov03yRy z9rVoe3I2))xd*htiDH08XuzTrEJ21U+)025CmV>8E6;RTZ1iExt&Jt~lKYr1?SFK; zf_lP*v+KUd!5NZj;^*iADTCaF2*-6Ph9n3=+o%BXHf{8t>01TH2RGhUUI)0ND?(le zN&6eo!D3RdBN zIhy>#kuzwWSBpb+|oZqdRRPXH}JetyO z0~TvBi@B;u$T9_2QFk$q8`8dKAB*u8X^F*W2=^vTK5C@9ZmvaCH!HX8tlq36JP(cX zug5erV{KUEMXvKdvK6{UZ&`br(kTbGEkfetndt$`RcPLg@RC2KGrv1Ng<1k?9?GJu z_r|mS#(@G>G~&b=s(Ni7RH|i5Fhwy|SP|KB2L~A+H%%xdM<^!vt zZz1xSb~_`0ri!o!w1YFnQ=19HDBE3{Z?F&p{U(SgbG5&Yj3q&TxnOw&Egd4O|czn{_hBtZi;N# zM6L)wOZOf;d0{=`V}#3pSbbq-EI8X;ZtRJ4Y$`NUgdcejUA!V{9{!s9>}f8Xx$~5md}-d_a#a`tC-=o{tLp_H)H9yN9Oikt^-^w4b}K?V#D_qSp*C zK;*jnkXzT@6=5t?KPh@}`dLl#$-BGg%^uyapX&cwe*}4%SI39x5E`O4G3JB`~Y<}jmR#WXWE~LdLlQ~B#eX~@D(Ns&fksBl} zA*;jZx|Y%yWB_b2Pln`{T6Al3N$)&C+Mo~wEAYuovX3lD%1O7*Ve4z^Q8JKNH)q|0 z7vxZM&zIQ`J^1N;I4b?3X4+-m@OYW^K4aPr?khkUs|fy|*5Nc_C*;^7w=TF;(9*H8 zJ;@L@CLICIr{vV1clIfxxW6k>7>>Kbr5BT%GXz5E}!s*CXdFMB87i?xPer6^3 z^7+|bXNYXv%F1iLXjU+r*?=d#xTodY1zoS3XHf}VNv^8Z0 z4P7_%Ui8h%Gve?1DXP~E&cjuV#0yU4~jgry325a`==xnkt&?f>YLpL0<)N`{XKd0MoJmXM31M?n6lQKBT+`^6(=eJX zQ$X7R-aJB7>GPMFghy2iz!WkUOX!?aF$Dj(3%Qc6q86piMHhyQc7w zpL@w~d2%FORWqB{xeY9$OxD-!A4oEnD@^|t{|v6z`Q9!nXRz1pdxZ0_`%TlOs}R7F z{Dxy+wn;ArD<2i4t=H42HUGh=vbFNP7p)#MKgsm7)XMTz9Dz2T8b+7s+iB#I-*JXv_mZ1}QzoHhn&8r+dV?|`?5lje zRPm*R07xj}F*F*dBb;;z^#zV-C+qgW>^i}^g}HK!LAorpMeNq|=$p zp2qawwQl7%r>iK>h}~Hzzo!{ORzqm&w=AW)YOSsOPq^%oStw@HM!qVwl=XI|Ctt87 z_z$g35aP^$`|Yf*)}|=U@(?F8a?!)!*KTrOEsi9M6H8Do<5Asi&=`5l!7vW&t=#&%#yfuom(s=Z!j3?kQYYg}xS&#JTH})OUnkjJtH5ipMhONlGn&yxo=ihs* zzN1cQJMb4QautX7k5yOwv|qTprltMTqirJE-^^SaQDWn3fE)F%an?^O_K33oV;EcW zcu3DKWEi-Fhkh zBZt4Dvb{7;QN||fjTt_3z9ePOm1sr#16OTU`TPUzrdZL4j6h!zx=p6}{NCB`t=hwP zEJt4Z+H)mVw9uM{>-T4wWQ73zN=yjy$~@Q#1pc1t;)_muYGnQf6|I0B?HCHa;ROUEjn9eKKW;P7uV z^4@gvI=1(5BKth|v|e^_4rT0MPimgJFgT(e{yFbBgodbwl7iARC5W=T{XHj;gFOVS zXa~8cY!6Ygd#0rP%t09jwf{c~T)qYp3;OS&k?2$9ZZ+ma&SK!&5$$5rJMk7Du5gv= zy2FAw26*EDysx{(=QjN^JU7Uh2eI0C)$6*{nk(Tnj2~JVOh?=J|0KRz83bvIw-p-G zvA&SHPfJ|@XLne`Inr|aeQbiNEdQ#9LNpqn4eY;V_! zrbIxC(buO260Teyo98$|U#oT?dF~P7FKKd|%#&;mVz|SLz56r*j8q-j+{(t?G_%$Z z=s`^+@QT8fqxd{Ne+U>g<6xNTz{`VW6(p~gRd}GpKb4@5!ql6+=NLVuokamv3H4uh#->)M}bd^TqJLT9Zr$Nr;K@aMiBw9Rt+1s6#?nSqlBAxVEMav!@BmNDpIX% z*yfTHdZn0$;YJw@02jm5y43Qw;1MNW1ntY=f&k$tto-|(t4t7XEK6coH zOrFtD3qNSuxu6V$-F=*a!!z_i&_bMu!_zk!JVR8?QQ3}b3QZoWUM8)fve2B*(HCxR zcD?$wXZ~77%t|c}s2fYS#`=n_q`%%9`eDt|-!CDQvKW)CB1wPUii0o^h3|ceT<)?I zf9g*QBD7ke&~BI7K7dWeu25r$_M&gUniG1l0hKQ^bDYDBo9~OgY)9t)=$=bvfUxrfbedpvy|j_1Csr5`M|{X;CSm(q`! zpa!Xzmm(He-WPw>YZ$59%cf^(>M-V>C*@((FZQt!{f$27yOS2ATX|<6*Ss<})l;bl zbtFgTIay}peDub9L4&F}=~~{&NxkW0EG?&#IV8ouc#xp%*6`n2&*^?XrF&lNXEOzX zp)dd~5wo;GYzf*TW~l^)hL%u?BTxd0LAEXd*}9dX(8p#U)*!25X$8tbR;2(Hpq-%z zDsc&dhsXnkh7wC~6v2ChULv@SJ!D!!FA3bn=p};NsX-I7gkBPOPpk^OPai{TGa5UC zc}VkWy*GmoJb=fodZ`l-GFwJ&F_gOSFtp3@3<=N}UZr?VARq}1wUaPm`VWYID z=nU!M(UAz5w|}eraqFqs+H))B^FWB{esI3^?ja_Nsy)%m`0vYAyZw;(ko* zIr9{==Q{|VdI_3=u(OywS0MIeLM}0qh;bqfN)%`VLca(JCP*ofpcr{FpPs1)y8g@Zu2DuodWZ)9{eJVc_p;VC{D4v{g zw8jhZyc`eZ-ML{v{jKq%KTc_F>3Olbwi{RZ_YY^A+*gx6COUsOK0XR_baG{3Z3=kW zU6k8u<1i40@BI|I>{3d6lBR)RNbDK{n>{4$130#l8XDUu&PCpSZT&x4s3C#RI?Sih zcozA#*mmux7Jpn{AMB`3ZPz$^?QgEO>S)hx7eujY+Il4U|N2r5!6M&P!>{UMr`5at z-_7><=WgF$sxE&DU-tHV`>Nc_$9owW)z}kezuyO|V)C{)+^?g(Y`ashR;xg%#k8!w zj_$6W)=hu3KjwG8U1MF_oz>Gve(-M&w_*6Q7u$_046@8`7eTM?8+#q9+PbPc+h7$= zPo`Go^kjkUns4L6Wye$fS-E+7w3()dVH#zga!-a-dXiQ*PjQTyF(|=I(i&q42zwe5 z_Dt5G!qfE9(;L>MAjC7EB*K^mA-W+Mj2Vz%jP#0loVf&XPkIW7o+P@wg6INDLs%Ce znk_*zBOxy-PCFgUkC8OSHj~XKBmowa@+WFc%}Rb$v&^pLX8x{OVN(2`gE#qH|MC2R zP89iXI1VF!)~Ga~rEdrP?&roZceF=Ct8dOt?Zvk}-?-);({}dJ8HawDSNY#RU*t5| zlW{B)le;J<0XCE4C?bDBShrEz!*)xPXD8GcVtVMmpXP-g7Et)|US7gW!m_Cv)msvN znNhT0N!KfBU>a{m06D{@R{@QQ?1lAme=%?!gyo*=4?X}1fA&@VT|d9?597cUjn<9j z*Iuko$C@%Cp~K|kT?2`|o2tD}7KTQ3BgylCknk|qo~*f()r)@|&+xR&eHsa-R~NE6 zm93j;Z(3hr0A)!O6oo5C@pNL8FljEpaCLzokLFR3yq-r9fRg{>bc@T*o%rQuNvGwU z9z~26i{cF`M!ljosLClWQ8mUAr%jAXQ9ibHN_=dG@--_I)?{x^Y2=jDD2g$!#3_rM z7-nX8PKKE&o|7=)ewo(}h390rUzWP&_%1AMTCVqTyW(cd+-&y-+%+y8?-e|ri5~T` z%6)$f+@U0saV!&)M=CadU2EGg7=`csD|Fd0ifze~S_p>JNruPLjiqJRj;&;dMmCCb zk$*p1=g^B85_qhS_tBB1w5m6??Y=Q zX~rb2_RaX(T%1V%Z0px=-@pFY?*~8R+2e0_zI`;|@%?R=8_~Ic6FR@$M<*n?-W=}N z$z5uH8pLXq2$9ai`s)<#;$htlNB3oU_cL@Z^!_X!-m61=bGVJ;hr2jGi7YA0I9(3A zcIe!7Y+4r@e|E_#o1apwcJosv&UgQ{8;N1ZQ~S3GOZWU`?w(~uUPh`S8Ia7;mVp%7 zBuGcH73e9FegQoqgEpWwW>Fy2GbGegYtSZ=&E3TJqTwQur-qA0n?Yul1Tzg!12aqF-eML5 z7lqw4{06aq2ANqBGEu4OtWmNqA0|2%d|Fdx{Y;yrIlir^qe~1!y<0fmT z&8DoTqe!%)Sy|LdbejD8Jr|%P>Ot2^oK92_z{S1iUL1fIpt(v|!d=1z3ZDqY594ZkqDVs{De)Mc8Tx_XG|qhC!C`qNrCkVs*1Tpp->oZ60R{;quD5z7zsY9 z09|+`6Z{k2szf0Wi~)y32|L=NgfWf4Fpm=bT)=-S(P*jkEU}Ome+ZX_yznk@2)u$l zn39Aa0hN~G1(zIfTMY;t{V>fD94Zhl2Lu8#u%Lh!lLutL?0{4W!~g|QP#j^*BW{fI z1okvXFgZ;S4daLuqK6ik&=&vWTp<>gS^%EG4ktDU2wDVJ$^?^QEJFpc#B9Jd=1k&g ziV;M3rWr8=9>~BLf4GeJkqZm}xrD30!@(pc5is;Q!%>1|R?!Fq#S5ZO;yKYqCtkuH zNC!t8=20cyLboJ`NF^j;N74+CYaZbs9~Ci{D9#MgmM}ONc=jxN`(;*SC(~(JW!Lu~ zDyqF2Pyd^pmh(|D&l#2n>t`>si|j1tv2uc8*V&8mZJC`Pe=Uk(H7=)rPeteWIJ-hE zcRFRjq`6I{Bd*w#QP?F<4fFlu;~$>Vu3XvmKK0-RVxB5+iff)aT9n+U+N-Cwt5$Zw ziqwH&%F9%PA>7E+YXV|ZO#D+D*4qwan@Tm&M~T#eRLbkrxF>hm>^f|GYKYLDr_$1X zDN`$+++n+Ge^?{3wB2!)dMsdNd1|fv3d1Z(HU2eCUuC_cG@E_-15|Q z%X;Igzr}DRJ^ZBwtW!@LzLu#|yoT}obM@l78dOC>@0hFXtelkd>)Bvf;5|VNy>ePX zuE()|ggW+v_&~c+dKRp7@n|V3Be=e##dvo5)-WH#$?BZ^4Tb#w> zG!_?6G$VI|@wEHD`(i=!vRKWER3>G#3p$l@l!{Y0B=$s?3ZV`S$J)`g^I)}P+p$^2 z0gswCwXX=t<0olpp#|XA&dKcdW;Vb50O3JZ74vC!UVIu4i>nu>Yk_|b2RvnX9u5)? z1F@-Ye_-o|)WoLL93RKQuf})dYLOqEm%}dHoL8Nri>kQ8X5G(z9^4fhU0^gEhyuK$ zSfj=|iZ`n06^*LsV7Av)*->g;MQ4a)e-lyYlbxf%Xf!4|4JHj{t-#e%$cR?@R7bgW zO?OlSwdv>vYTMBb(5|B!n0-e_$b0Q;wFmLAf4rLw=HuyYxB6IgKNKGapT_09Ti$eM zU#gGgv^$uNy7SJ_46_T&D<2*?P6tjMji-xhem|^nM8CJzosRxb%kJW1`6ucz0>Lit z7`Cp7j^XQ?>=?1fueIXlmb-$#?ly5&S3utn)L6(-ClCD*lB9vZU$y#Fns{Tkb}ql6@x-cKg|A9CTgx{(s6 zNZneyxlC~;XzSM4`$6=!Zl&}?%7qbFM(y2#V@F+fOS5BJHyqpTmO5P_lRLl!-r;S% zn=#q$)rO_WRogTRn`+nAD;1KL2jayUe_pYz-4fBZw!K@*hXO*Afpg9-GDW>ss?_bB zi#^><-G9*?Lxxnrw09$KO^t2smKJX7_HOv4_ybX~RJ2Wq0ms@`haw8fcHqd!+ykZ! zrKEK?*6%s?lm#7t!y7Z?CgW;SJi94wZg8+J7&9mZLt25OrE#%tE!B@7O32>~f9A!s zq9f&g%^sSHsMRX`U;>9UX6m}}(NdH-e(1>28_yRNongt-SA+HQS$PixUB1E+WA!r< z0Xov|$2EdYUfVCC=bG2p^cwHu@R?>L{Mx$XE!N4hs(*ghQb0ERR#bKu)thp#5YkmTje^%be+Aih^Bn%ND?kwla*qW`XW~+{MCE{~vqcLXf z1C1w{px4yVB^|z@=Vr}^FbhSTr64PgA3BQH{Im^z`u+avwa@h+(->qLgE+lbc}N53 z(kv7~DP5X{g2zI{BCPMQ;IE2Bc|RW(3%Wuhh2bpn25HF>h7+}+X@%+{e;2`UKb}_> zbmbgWbM>tF&Ym}%g^D= zMMr4sMXl#-UJR;oz9il-PP|r;NZQpzTM~)a^(}$~;$VSnkhs=mYT`)@l7qh{9@(M= zD2fL0?vc7KLFz$0pk}=`fAy*rHK3)kdJjsZj8{H3!`O)Xmr!fxmcb}8rj|3BfS=~f1NlruV=;dB&@Oe?MZh6Va_khQIY+!C|=)JlkpT32+1?ue{_=AvbA1nktItu z1>T$us==hZ4fJ-o(!MWst~u{3l?%=KMuwW@xwfWnZKydOB%RESaDC^3q}Mq@*7q)# z=tB^K3}+g3)#qH~%4w7?l=QB)IdT@d9PKNaE2;ZRqY;$EIZG?m*DSPZo0E+8p&&qX z?<-8-01cCUt8>RAe>MyvY@P+J-sIBpfXEpjiF!EmL6CD!_ED%5iyLyWCUOXmxM~GRDx5zGb;Vw1PjVsT^oT;*oEA ziV+0KEAWke)|t>mZkceW9bChSNm&(KBWkcPAW%gi+`&BzBKJ&~7)7{)o1reEyl^U% zHI;Axr)gQce{C6Pgh#k-1>MCf`pOV~)~MJ8OZ0t#W=`cs(`An#HQ{IJ69-fp5w+u> zjQl=+qp@_0uY$OxXRHH+1kDr@Pxt7MtVi`7%Y#s@mrMp_jy~Y{uz)bo*HZL3!3qkL zmGn(TL}8j}EUKjv0v1RC*GSO{bZs+`1FEp4z+WNIe_%pG5D>(3M$C!>@sa~zwCNB% z?mc0NA%-E%qHiJ8Ab_*W-EF6sxxoDdNJFww`Y-FgaKRQs$x1u?QV~#g_fszW|I1zb!!w*lAix z5efFGe=|WM?2mhTt!*}kmC*tV9H7F>R*5kAtLI&;Av|GM;A=Uq8_&^1SU!T5%`$hO zo5t0>?AgUR)v1GJr-MZidVhL<^6Kr?kFPGT%Dcgo+wN6ydp{Y>N&NcDqAKn#rZ;8C zWp2ldYX0TuWK@1AI@xQw;2lqIk1j^YT*lRxe-13oW|QKM@*28AB>gy*4d>$-vJd*Q zMK7oGcgS2=_KrTAtrO!V+O>RkvKWR8iD^39h378!I=cp@Uy&=~4L5sPjBh_inUhbq z@5ZC*BYh(Ys!k~x>f#-a#1y*~m-I;$>!6=q;LSK5o=k5iMZ&V)WN^Dk4D^0MUh@pE zf6{5WD4wz9f0}LlcU*f$ujBUUkwkAiDP-c~d!52rRM<}UIK7&pN6t-m8S?JS?D=$9 zjxY^bqb-JOGysH>lFFDah$1(Lf{h78VIT^9K;?LAe?B2mTBFw>irpZJKY%E0+7Bg)-yw=QoG7nO zo}XU6Zh$#0C!?ol34QzCpb33;{|265mu(}8!DiDS3eK+}ich$vPr_Wb5JeclHEU&y zdw?zbL%602600UxIFkpEWv}u>nZo%FRqWwZdGp)l>+7=>6!zVyqPD4GzKJSaf3;FY ziY8SYQN`0=TZBKcKqB_kRM{WGCROw%RqO#&*{l3us>mIxxUZnf<<-AWFB(w3+v|t^ zL}rsMoPE;^sC8e_$~Va(DO14-mn&|c`~}n+za~}u230segeooi4`$15vf>YC%SG>( zUwfB7((}om#iG0`bZ=f1-Rp8PfBqgD3}5B8n1eu>M<%znncRw1aw|5H7Td^fy_F5l zXidW`PO6^%z{?!Mn0QAgTN(3k3>&oJa+5ZPq___nIg~p>Z(p81f-1jVp7-AO8pMqe z{2nAR+xyBlCAQkM?<*VmigxwBBIy{ka{}sel`?6#mrmD&vFxk%n6^+hX}MxIxpGLB z+oJwZmdKsd#k0eAn0L=Fetz*2z+9Ho^1Hr{#CD!59+&4@>(;apLhO4Tkxb*@%r
Vma%Ev{3V7Nr&__-}Q51mTd$56`SW&Sc zhz~^U*ua7XJ1X{uI5Q4hfC&kS0~0o2j1yMi1{}E-H-hiaVe;j_H|Hid#~4$oIE=@v z!740)8=7Drb|C`?Z~{kg23ast{xpWVgmVyZKn>JF9n`}OTnTTf21tSv?tkGH@{ki( zSm#`@EZ%oLHbOtNLl`133Q_2QX6S@2@Ip8AKri@U0$QLCVlW2d5P}h~zyp&o0E6I% zAqYSahM^VOUr^s1txd^0wJs$8thFVDSFLL){As17=+(ND;*-|4 cl-ybmQmWqVNI7AD^83-4e<)2X`;#}XC{Mz51poj5 delta 6530 zcmV-|8GYvA-U8p=0HGC`zNmponr$!8%@1a_w`)aH zBA~_S>r(>>SFVrEbDUtPWvfV@dxZE)T0c(aNj3+e?(kyoK8*m=%Z_YrW#ev|S?dQ3 zASV)dMd8X(d>&VS0!Ga^7$!UL@?cp7$*W}*9!T*|rwOLrTmHk04X5Q29t0^ZEQJwL zh=f{na1>Rj?Ht9KqqsymqW0z}p-2@H)=UDVouhD##F`2ATiRO6BBU6*XGnY0Vn`_dpGhrn;qV|_GcQOjak0XS?kQP|JY0Tb6+cMv)uj#1vHGAc9RGpNq}<#t`+P|9+Yi`mlh8FXx_{o8;O_p5~&PnEkDW zs-THAov0O!!>lS|S86+9xt!HqH5tlVB8xG5v3295?(At-4_A6#*S=IusuL}H zIwyr*DP}2e)RUN)W5uyf=c2r7sA-gCMXD4nS&_4hm_^w#Vv9umPv<^t*QWX{)f(=; zFX4{kyRoK3Qvl-70uPjG;t6OEpAX6q3Gz5-3wqP!8O`JjLB{v0$zOM{LkjY+!wzJ8 zMnf&ZUeneEWgzVC;S2$Ch8zSf#EAmr^o)j_A*v2g**4G=8Xu}&Cas~e(45cF7jABL zz52Ch9xYSNN-Ynl8%wvw`ih;PNAJl&t}gxk0ZFB@DU+@uNq<{wgFq04-~AQ)*iwR* zcu7EnL@NkQQ(^*nHM?Uf#9i5#hy45Mo+S@mQ2BOdj&qoC^Qx$1H!}A}D_+yc+D-~h zclFSm*oCg0bo_vcj*phioA=G&xcRObzMGzydp=kD-Ra|NcTjyJ{a~?GH?cS!OFw3U zTBIJI1F^tzRe#ja`$&ByTgB4UVaz>D%J)$}+1o~_3%$>Gdo4(J^2**$d2Md$$1vQe zCpj|D$ucA7qc^Su-G_#gZse7m)SFJm(sDYPLsI;U2MNk9E&pxwobKmSy644yHd7E7 z3Ios*F-sf7mY^+SmP$})XbF`#0wtgrWa|=;ty>uieP;Y&4YDehR-hbYRSHl6`ZN?l zB`!hm5P6``P+|#=B6yF`O9Z#EhfGW8C4t)*y+m+3HE3d%&`SdEiB*C3>0@YZMq_6% z4{2Vl*JkmD*3+oX{+T73=iNzqt>~bJxyn3$ZLYY+lUpQ9e}ga-#ozrDdF&X)#6L*~ zMQH5`g>Eag{Qy>PJ7Gpi?L)u)H0QF12?{^&$+^klCd_x6ooxmY9$Z%yG>B_sYe)C) z;VKdrx-~XXk*J$$VDjIpE&CwMKg#~IY)ORq`Ec4FU-uu!t}U&S^L6)A&(Ei=(xRp- zq^D<3B4XZce~RaQpmt}komegdA;RP6d>`CX%vN=Gp_l33m#fJ&_9kZMyzqaDN8kTY zOLh=4$TDxIu(ztK>E4$WxzgS!SjzFq#Bw`6SwOb_S5G9CU9Q!4=_a{ak8&v!m3hiN zX{N0w^X@5%po~B-7zeIumT*@*N~^iJ`M6T>;k1>7i4CTi$O{TF0tRI@-q=h75jnW$(crL zx)6W&9HDwRHx8)3HGcG`DQztMU97(Ar&a#_1z_adRg*s^I)6DnJ_>Vma%Ev{3V7OG zl-r8iFc60C{S>;~QkM86%?5%YvDc8c+m=oC0UTRN4eJAnbCI{Nt^X&58WQ-d!+aWz zXO-`Zecw)M_1g{g!A|Pj_KmaG;pS?qPWICFK@_W|ttXQIue)jtR{2{s{;WEyR^_+y zxcmO~$J_hRReyaHKEK<`?Xz;vAICB>s`E_WgNKn@yn9YF^e} zCwEs*+h#b~FUz}MuCcD|FY4(dKlnF?+c<0XLK5C?bEEuy#Y+OK)j>c7Ym0w1@utX7;56jKkdsAQ!kw<cgcup~f`z2006rPjeeo1Vbg<4)JFCOmw+SD6u=e>fC}e*$em8_c3WsAov1r#7H%B*{4|^vokEx@eF@8wqCW5tM^aT^WosAd7J%SS!u~ zYsGr>v>@h!B-9J$WIcL<@)T=8^_&#2}xKa zVYy>zJcK-L%W+(;k?&iiy@oI!>Uh`0>tZ?Cy?5W&(Kw9DsQCK_W^)-93T19&b98cL zvv@3Ia{(}u-Jc?Vs2ev)2+L_3<5MXf}q@xR}50ZO7CbgjhcLRup|r=!G;ovg+3u!LNCgAzt1^X(?WC$pN?3fWXlY(;UH}0^xE%ARq$^3V1Pj zKnBbXNR>bgPyhwR5ym{?#yC%4Pjdv5(*)5ljz}STXn_fB@juQLVqvKT;2G?2VuOI7 zMR27|Fe%0|R1izd23%v#B%Y=iL4;?T5kugC42*$)%ZMMjzyOd-xC%TROmY$dL!UDo zC0J$^jX+SmAo?Vp6K!cRP5f1WE5o3ws%n)q} zgM)z=FS6^0X^|aGCS{df-G8X4_Ifn=Z+2YHhQ%yrSRSmGz0A(DlbpxO5r$o5KbF^J zc6u;>F9y}9ocuEtox{WI61CjvlmV0GHkFRJVpB$8mpnDh_YM!gdrrG@Y1jMIgByr> zs=z6(dFp6Ua-V9ip4+Zk+66082ZkvxQw@f2BU7&lh)pr^&uv(5I*e^9)kGg9QVUWk zuT$fm-C?usu<@xOLVKP{OZ%lvt$22a?Y3cmjmXk=$5raFfR*K`wekxLvn18{*ED^Z zogJjv^x@A>^FJ@s>F}nLo#re%8&$mxhAZjeFD+o5dfM={Or7F2j2}K%KVDV+sz~UJ zxy(+=aXGu1_6G&d32HbiCl%z*IQEZF$9@nWXg7*aVN(cCo_2xo#A*MhN5Hw5m4mB) zqRO+2)3fZl_*`Y@cm3PqBo@c9IDe)Yx$BQ6-T&PebDEd=a#o}=DWhG`sg$EsoWdco zC%RM!wQo39j;@>st0miv%`y&n)U>I6K}eoHNec@t0KarjrnfiK+3g1i_p7RyO|sMC z(`Zm!{&>6+`1hdCQ-SBJj&7i~9o+!!I=X?`cXWij)4o$m zn%s7)k45)G@v;ADRL;8PO?UcGeJm&4{$$vlbq=POU0`1N@WgRCaOz+*nOC#>L5(B& zy|M0e^nX%z=O4?zP>&G^c6rCJRZVmZU)5yCh#h{d6gRirCHz&liL<&Cf-Sw?0()k? z1vc$^i|QD+-l99kueX?vW$P_}wqtn%`>WA#o~vMs(rYSAm`_TsYq2~uWWWCQdqVp) zwrNKRNiw+~k9*(c!fkXTB~FpLwRUrv;!e=kt+Drm=xyCf>Bp1{Be0Cxy9LLNy6Bc> z%eHPfw%sjtx5d^os$kl?k+-JCwsuPkw{?3r{8Idps8}l6 zCd7ba?W;o(1!X&MWMu9U(}q&gx*O~F6nn~?F2Lc$47tgu8W%5aikll;tP92rO2Lp; z;A&}HtXoULD_Rn=_Ov93gX4sA5XtbCyHBop+SI=ZC8H}u@B_z-5Hh_e)A#o=RD@tU8u!B4;2 zf4%a#9%LGWOk)tIW0i+AfL@w~A}FPoW})D*5U~jBJ1qFiVqV_Q2F09Sp^?IH7r8)M zvVh@4ZD>-VI?qLaFx-pxl{vlii70d7cW9ORX&HP;~m^^eQX;m$=@XzN9-=VVs&t8%s=-XKoAR*^{B)kIqmiP!ZFf&}7V zfvl0Z)@5qqNeq(xzb78qq6H|52J!BZx-LNKem$UOy*BlKsueY$g|d1NQ4OHBULn>t z#x7RAt-&4GtDPaVt9nckck>vGa2Xrk4~p5r=i#RjF7tPvlRpjw6M|XJ{^)dK zys>OfaS1#){TI3h*QR-c~qP9V(rn{rrW@8-qZ`)WLz zfC3?TrZ=5`B(`X+ms(`Wf=z)pC;h5FE^h<9U9Pn6Nu6uXdrIX(v!0QmW_hly=~)|U zjt5C6b0b{OxghCvj*#`d%O&~{gdoG2MqTwd7rAm8r3)pU)iy`YLN7;qO6E%Hp3-Op zC2`NvO7%1gt=i@!qkSj{5WV*mre}bL$)44@;}IKw1`#&Tf>v*G>3Bfo43I=UocSQg zIVbxlRc@I@iFWULTm%gPp@XFDNtJ^Oi4j03V2fU`0kh4i1t2w-JqZ;_=B;3ZGO>;f*3IcTIFazfaih7CbEF7IiVd&(NE4$eTv z7#h;EEH{W&@CP-O1C2;L@@-Esf*^SXzR}M*6Pm~^6W(bD*KlJ}Rt49H8Y~P5R8a_b za1VpXJrgEI5$@n-sEa5soC;-4C0xL1T2^j;b-X+2UHpnwd10U{62o8vGf*S1#wHySO*9Rnkgin-lIdZ9@TR!4??wG zG8vRP`herZ0>VIFOVQ^9D=1J_(l-?mg=wO(s1`~HSRe&lBSlNlwaq{dsKS;4e}zDQ zg9!~mKoHLvF)I$lAqT){(;<4?d%_Y!3`3el-$JNC0B0A=16;y}K}4lBOa+*QNm6mA zpe1aUy9iN;xI+szECOvr2o8O-p?yUKF|1n!uh5w&R&OCv#DyzuJ>Objam2Q-T<<)wGl%66{icXM#l7A9wUx+iVUiqXiZ?K!umB5@GU}&)Zl- zc*3s0S8`l8o}-Ddd;~3e?$3+Rd-m$&-Rt+?zkYLB-t{Nk zb}x(D`*D9p;#Uvzs<=C!+>{}gxgE``*~7unu>4STvbXetcQm;@I3FT&8C4H|9ax%9 z$Hg7xHS`LR^y5}Gn2n~$KIqF99ZsjeAah~aFZ9`Ll^8G3uH>_$`560qfK`%SU$v7Gu zO>W0U!m_h*|8|}j=zTz5^8!bI>7<+&FWBNg&DQ=qtUaUSxIKC#aW)zkGV$?Tr*IY( zw$mL>FQ@2+;=X1nB}d=*Lj(Cd#kx zkIvq&z=&^16th7T_e7$YCQ(#P6r12V!nVVzn;H+^R5BrxEv3kpC$C|dyOXa+5Ibcf?c+aCHhV!nzhT(wevMT#a>98ty7Ut5Gfu|OjB+*H{e!X{PpI#ui*RN1L~Z>q>G zs<{2Ba`w~N_3^6)ly7$Y(4WYxvxT#-I)GaD6|H=eERr%6+;F+#_Sp}h*7!B4;@7Cc z`98Tyi~haYvYo8>{n>JU{GX#Y@4u(#qrZxIc~|JOSy6Pa%JJxbTWm0VncHFx0%e|< z+}dVxE0)QvSW8-LBfIrhHaMd-4YRnZdinz|vkzn9J386Qn8#z-pbeMnwAm-cebmUl z+!1>7<>?csa{co6%b$O55I08fTad(T?kiuF*lN?hudL-O+SU7tq-)UD4XDdi%B105 zI$aaSvM<_W+CtfAo_8r#>JVf(S|2o3C>ys3<+n;-eMnNif=!STB6rridau^#Xc!+i~7Bp zvX!~`#gX`bs052GlTNKILS{5GWnnltH)S|AIXE~tWivH4Ha226Gi75rIb$?qFg`qH zG&5ykI5;vlvG-EK6y{$)oHaFSIDr>%<6%4q&hKOM<)6t+ zGI?W+DU~XW%dEl*%tIqIz#i;C3icrlhj0qVV5InA40Qo#AW#97PzBXc1J`gV+@)$^ z5$fO$ZXgSP_u>faIR`9>`%Rbi&;zXyh6oHn6xzTE?a%>k@IWVcp$kT#3A!N$!!QCt z7=#eGU<`Vp4}8!Meh9z-G(!uF!z|3eBuv0GOhFu$VGb6=_i0D$_gVr!mf*7`l(s~^ zEzzV-V^?CYS_z3SX&p%XU29De0~-7~M*YX3{vCNi!HB2uLH{NP~2jG$NsdG`{cQ zJAB{iyFG{h>ACy{1~5hjFzP~hV2n~KACS#IahU7a$%PiZL_i)L63yu}v}@v{MczHj z+%j6W1FwE&Idbt;Hq5%Q{V5kNj!kRQ(0mb-1jP3|@0OE)-Ju!eaa z@y&ZH?&8t+%#ufkw4#`Gt7U<%R3-oq`)q5I znX0J$9`O=RcZ)ycNL^hMU&&9d=^+;EYI*X%UX^0}q^KZhvQv{{o3Ew@EUr(>a}hQT zQW?xoazzvfHWN*5wqJNY1kr5`gE$Tkrs6(D*WcrD65e!~+jnv2%Yp9`J|jNzs>8Gs zq6A1{&!6G+9lT)sr{zdoYfpvcGgVLi5KY7k;XHLQ!~OY^LuniiATa#~sqVBxWo*tR z0rgw)M})<(|+|bfpxC%3+9e#Px*%K_5C~PYoMWs zRUr!LHSvZ25K}w;J725#Qo!(N3Y`hstS2j`x|o8oruP2 ztZ9-}43)Yry(W~@W5w%~4%p_jNx8wmnG~lkv-sU$%OEjIbU7lUxAQZ<%%$V?V+svV z`5-TGWb{F1|E#_l(V!HQo#i^BBN;xCJOkv$+`L(L9>)$5j?{eUY68rGpb5=4ii>3OVA~8OZ$o+e}w&N ztH&o{7;XhC2nIe$n2@Y2zOSE;qn#&y;Hq`{hF6pTLCkp&{u^j6H!i479k(LJM9K2T za13+$o1Yr{2aK+xwUlq4sS=b+`DK%dp^u%s8Y@XZV042#^hKwo0^*NGGudFg#gq6!ADA;Pw&RMfY3ul4- z8C6a+SGd&-iF@UA#E47U1|Y2SG=(QCS)?wKK@ni2=~-#JYuF&eK5j&o28-w`<7-#(ld9-iJX z62o*)>s``Ut_{VlHsR0r`#}hC!x)*W!mSQlgFn!&5IN+)yR{Qb;X>WS9HO=3xi*-X z>3C%r;nq<7a5sL~2K~gsxEqx!G^-}$O?zh9DoW2Y}urdxKoKOU-x{MNnm*H z{0;e|8=wHSw!%S4_V&TmW34Y;YR0IeK#R)Z?VsR4i?biUuUNXIX#BDrJDU(Rr1Vai zSqPRLby^vTqc5gn*$x%6OcS4J z6SSem+c3gJgqV7+goJnVJft+RZ!^N6L4Teuy4M0cUHfgDkNJY3(jJOjA`6@{>mdNr zOQ`jFE#_BC!vGkIwhZ&*)8|m&iM2GyKcM)~^=!_qAxR>k+}@@&XJ_$-%59W@<#)rx zXv}XD|J<4|^EA+8PNxX9D#z@kxKK*sbBka#d;VTb^vVyKh2bCb9W)sOY-(-&V*e1qj`0h%yC^1lI%2eep)7nAxJ z+HL6D&FW}yAXgdIO;i@vAy{0dFNjpc@qO4)PPA?epSlsp(YVWIwP0V4VitK7N)um} z>t@KXI0B089fU%;Qik*Q{-Q&`uQKLk?GMYh& zvMAsqUWwAd(4%v4WL%5{e^;1JN3|d1YGo&cV-(3@ju}kTV#&ndIUS8~pa^^;8VPLJ z@*RJ(v~lQ?Af^NqK0Iqh=H5D?oCSjh$oABZxF#m*7(#V0@q%uNak_YFA~z{`dJ_h} z5Oz9zhOE3K=M9Ny-`||6-al1nq1;%dz_l>~WPWU6eP^?Dm;HonIM{h=y4vQA-4*c; zKRh~bC3+HYj(MO`3pkxe?1@UE3m_O@Gr;U|iO^?Cx!ygZH#9-9XE4oWZ_nLFIpX-f z*qR6roN$QB7=UEh8CQ}+uX^-0HP=A`cl0TCXPlyut%83dTsS7xs%O>L zvZsJwtR(yUtjzTZ^IdmWVFL!%U!FTz%jnR7tL2FjRj|J|iWg7b$FfuP+-VsY{S)?^ zeWFGOg$RCcZ7KF{R%=mKujp@TdL_569Ja45yD?3%46?+cjlY-m`;XK`7DFx@ul%fm0#*tuw`mjFE zNBk$jP4iLSp^o=}TUQ%io#2w3*mZjvYHrNQmqF8<%*1GzBXHl3MC^<48Jt9Ql$qOo zLZBSLtRUe_5nQe0B8|W=Qv&RE0!e=Wt)fHfKNhLu&cm&^e%d_Wjzcn%NRIOZ!#!$)HNE-B&r*OyiWzo zN#C{+bH(B<&YUfRHiHhf96)XO_Nr+a!{f_K5jvB8!fiA^JYjoit*doi^gfoVz`1l< zBzE!wH)qNOg^S)dIdqcgr#*^k@JeVyd9LMo?aAv~D8+XQ0v~S8x=k9l5lVC|u<9!)l*4&JT zz?~4IweE0~x4^^($l7`&vUQ;R|;jg&>ywhjOF0?XdR1MESTv6A# zB~Dh!iU(7h(aP{ekH5KNf}7rkm6dLlo5lCJlx4%o%xsaqq9%I{<+1WSJU1Y8K@M$r)HWK4wGZ3O~J* zOjjZXp8W$YGaI?-vX&C^+Rcbm#{Uev655K5Lp%+~4KbZr^L0Qv4MxnyZ~HO6Kxdn% zQ^w5Oq%9MNupecd%U2qGY)YPJ=zR=dnUcJDE1gHBS}xNK!f4VoiL|{jb>)i+n&|zt?gF4wS3b-M!Vuujk`N@epdOia#jJiH-T&h$v1!EKRi8^J?&ow zTbK@X?fOWgeRD*fAm<1Dx&6|eYgLy9W3}dV3YuXu0Fb0;0n@kIs?cKPH669^el(l{%d&#W+;Z#6ZW$Ra13$ZV8s4!sIsf8YDl z;k?1hX5awz%hvqy(5aobCLzdr1HFoDcad8gB01s-k?ggGNVSGRr0SF)bn|c5CV89t zHefIMk3l0;KM0}Twf&@u6aSo-8mcFTPF>uD>bipw#9CD}r2 z&Fty|!g>=$xrkg+{K%HN>N!f=O&ep|-fBHeU{8nVwTY-B70cLKK(2Y>(wrt|Gh=ii zcw6QRGJ;-JT!EWBncT*9bWNUgK!)|B4g*Yp`W8QB&A?^|#Oj;D4A0;`eUMG98LX8M z&=!{&;~QpW8^1|uE$E0r3G!@!Hcw6Pd3zqoW+?jOoFT7vNF)uaH`CgT8M-H>N>H#+QPd{x7p@Yz3F-8GsEOq zl^Tt(h~i!>v1m&hLdiAoY+_m>Cq!^QR`wX%>C;pc*xTnb3)T8`h>p(8F2|Iwr@Zn# zYCmU2^4KtzJP7@(ms*m|P>qe4s4nXTiQ|>AgV&bNvr7F!Mx5quy_bB>rYQ&CmBl&HobS0z`YXy3EphqyN}yLF$j{@&J|BTS1)|nyZbHA1 z_(}vi>~*qFIjBZJ^g^Ki$jBACrvRAWo{%Tb;@Jr}Gn&Qr&WnHx)x?M_kD~mw& zVWG>b@4E|%i7xCKJ~LtUV;W+YcBczePqeOjk?laMNLuJ%Ert-5z;Y}LD{>NaHfX^z zp2341il8@Vy^^z~Da#C~(wj-i_Mr(kjk2r=yPXsy5bv@LQhM3^eMV`huU$1Z zGMhn`))yvJIWa_h1Ynf|hjQZ+lrDD58}nqHq*U3A?}CgTXd;kWde=J#R;@$@RtfW*@AylQaExrh;Mq?(~ewew&lJg1CyB7bh_B9MdYQ zLD=>lTLLHaatAEfb-#QiaqD*-E?FVBGtv6_IJNi(8fhFMTPxR*z1)f0#R-;dn51m> zn@Je{P@{9>jPW4vRk(tDNF_v00-6`5KHFzRHBZE)d_ZK<)vE-|1%853XKfhHo&z zw<0*hRs-1KJ*wo5Qu+nYq0d)?xa7=jJsUEiZHzKQ*%23l{-0&1GXh2!#-TuHs0L*>$ z;{4FyC~jBEL-5VMQSd0l?jzK2 zXI>K5CVvk+twP=Rq3! zqS?c^TiEzYrt)^9!sqe+Wcc#tJ?x?~KWl5DHeO1-nj~f1?OlW=0<6(CS*us5V=~@h zB^XKgu`cmB8ZT|is~jHETODnad1YQZK*lT<)SzjyRuXPnHu7NZc}vE8RDPLts0y5Y z?exf?B^Z!%VeDF6|3vD-YAx8EBoF6PxH*%hC81<@i!g(c5c%Jr=Ste9xX8N4bpg5^6_BWJ{df8%xF+ zk@jkhS_6e+rN=W=5+ymJ$df-%(dzAQK`i0&o~QLahYR@|{hReV80U7nY@Qu;yDdPG zB$g{)o=(}07D5t-b=6C775U=6_eJW{(#CGAOtV&Hcv@tFZCX(_V0-BH9KO#A~RIMUG-whf(LT<_3z-l+~3M& zg=(dG#wNJWQ;k0_s~CEOY&7l%M6wxM delta 6090 zcmai!RXiO4!-q{z#}S)0Ip^r^?siOfP7M>o;pp7d5l44ScQf4#M{}6&VVbe`z5L#* z@8-GsUp>Fy{}~-X9T-5Z3*dm#h$+V8v>ahF%#;9Rmt6TlVMtSxF}!a!*}F5X7A~o@ zEOlG3>xl*c#a;`)fAIZ;cSHXxC)bpL9;zJqHp?Q7K~imF0$-d~mxyoM+w`YVj>Owj zF9-MC&>$)5&JUD5xpeh!YWt^tmeFX*SZS=3Whp5CHg&5sk+L+&>Lj3kE}U-3;AFv>;9h_*oI z-*$G=sqTcu7R0{+KKDiqwvJ$AFvc+MAtICiB)=xjq1p6~5gAIVGew4*(je{Z&%lgK zAvsL7o|rl9=Gbh>yQ8vM_w;gmgczz)R^|l@-Y;jJRT)HFVetsYWx-N_Do``$0eV2> z`4vakIwdpR)Cdkk9WyAL6Y020XL-X!`j!l4dcDczFU!TkCjQuSa47O_^FDbGpS+D51=uDBn7Jdg(5h#x4c9~8r zo;5Wt`cg`3GF`;>N8n!it9wui*jbjJ_DzJ@0hpwKMHkY4pa)iwEhZyD-|PR>neNIY zMn;sBdje9G1C%_zdPy_S3CgNc59KM7ktnL%#gl;Dc;raL#!-uX(gdAC1c zJ(}h&z-T!|q1=*^IG?>d?W|mJzO0%h>o|vT6OC*>1MZf2VuwEX|ULD-+s1R0LU;5X6@(gq%fC4dMXjHPIWTGi*#vge!g~1hw z#DiDTUot<@vOWbbXddHAwl`{V%-$a3RPxn6s{{(|N1|1{F3e~9>;1LsHOPa%T6(R5 z1JjSc7WL0SS1d~EUkZmeC1#?R2xC?UNScz)IF8a~Bpb`B->RFqt92~1Jo}Jtp~Mic z)|$eDs>ZeKU<|pP3YetZPnB{uB)W@Lp21-WSPN8?ZlpljG0{S(>Nd`-g1Z zlQf+yaI28aUURmh zm5*C1bMNXQdp?`E67(#^!TKQe!?$m6+pGsFWzqj}iQ2f)co{ZKn1Yx9GNAh1-x~iG;1|nmib+?QY8S6;T zDEO06l}({ZRrU1XY8)6NHomOHpE}n(yljtvK#Oax6DAX7)UPI%LEL9$@Aq+}Gk`cZ zW_plM$4eY^vprfvF_!toe38*ePvXC{OULuKGZ>@+K3uQ&#kGZ%`FA;`8(dQ9KPZ(v z{rxQeL?@|KUg=WR{zddJ2O+=}7E_3)T0q?}L*hAuoL8LZ8jj@k=}%BJG=BSz{q(;U-80pcqyT$ltceiuoN7dF$&HUU9DnY88xSP&bwUJoZl@A**2$aTE5(kd?dw11E)$ryk30^W%qv65miu%T1~6&M`936xIiCWf{c z#sMQ-x#-L0x&NHa!T*S`y^!F)LYCQaVv}_H< zaO-tKbL(u1U)Pie7&9rfw*?L)l4KoBI&X@&iu^i<(H(HibgM!++3&y3QVeCPpOQ`k zve||8ASN-I-(LOVl2EY1xv&VTEaS1scNupDmw^Q`UxLDWPRh(~A29Sg+N|GhaRIPyu{=epkmiLT*u&2>aA;exsWPcLagu`R-o!zjHhU zIfW{}Ib#u@Lk!M85|u3qz&$PNdVniFsSMe+`cwX}T4??}U?S z_!dVUA)`Kxw*Cg%TkJ`8t!zHNL~-t7CdvEJj+eL__~M&ARj^pJ^C?W&c6rvDCK%8r zIllxhVSXIZdp3Ju7YB`8F!6kpOiBr9LeG%CePb)bF~j!<;7F#NmQMkm%@<&l!GUoI zP(fdZXJSe?f_W~!;~vVGeb!)^RQM4*llRJw-4Q=Dw;!F)%a&9-vc1;`lZ#}U_e+u~ zj5T`90iG`pWs%W<@Xa*ByS83?GF=1g?F|VfM?%8I|6+x0kQvCF$CCXTvR=eA=LOfF zV&J58OCvqc4;TO3=9|R1xxNaeNDXJ+bA6Y+=q+jwEvxGBli~~%j@uH}$l4Tr6srkb z(lXk5!SIPa%YYpUh(dY(C%W1bP6OOTT_+s3aNSTk5GTevHUdmoM}jbegORaVs3A!M zHKa^+j>u{Qe={bPNm&|yAI)sXteRzx3YEd;45`4f`?>F({gKf9&n0-$P7i9e1$x z3;v<}gnM%~vI;qyLb*jm{;!WIW7p4XZNx7d79m_YnP<_1PZJu$dLxn+>x1p{V*(1j#a>o*o=4TMFU1bp&-N1>%yRN(FkVsT4JrcOt2Tt!fx!wF zP{%3v%2Z+I|6K0O#lrwo#3{v44GT)7Di?}X;m|J3iKzEj0SChDjCq-hZQcry z6TzwFQZEeitW^6g-|jHGJAZ}sR`U>`QC3v|FlCAXEPOLXJcRTGE8I~YZ((9z5#b5q zu)!2IZoGp+d-~9)CIx3{t`U4{)!;Gx@9GW zO#;HkIP2A~sdwxx@qc*rmlYJF=Y#j&FN5O}>1bb$)}&onzRg!)H9EgB)7ZaC+SRqA ztN16Vp94D42@LSlTl6*p%@IybLC`jq`Q>rSFg52Q%K%hO>eYYc3ToZB4EDUJfTNk1 zE>Q9BwyFX0Y+0UzBz);gJKWaZZ@96vN{`x`tsnW^Z z0dZ{aw+5)3wH0{Q7DVysd@m^2EG~d8*nc!{&8kMG8$l23bz;?sx@^ldsBE1WYm)9d zIAs*8CdXgZB2HWrjR?IDJzMD!UOl@6%|LI5srDg_6{Vlz?6bQ?M5Z>#@-k0P?LEz? zTZgV%giq|BU-Yg9Obk2n5fS|O{mYZgqg_878lKk^-~P1_+m1Rmi}+4BZINSIIr^6);^Wc)JiEvuu3^=Xf7%2j+lEfym_h|Q5g)y>sKmWv0 z|L;MIDVfkv;di>=g%}U2R2CkIT|pj+U!3(4>yq^n%MA4(r|>f$^0jSRRH)Sr!LHf` z=7v~#eT4h|zvfRXn)XFvtC{LmYx7|1=h9NrHQABU?raJn?``9YyAFlzL7zDF>~%Mt z7(Iz>t_dTFMGwsmc5SO7ti#+HZ>m<4KL-{;b{oaY1tkFNS1ul6c?St4#`F%CVY*Wu z*90m_RkW`fe&`BidcOPPE@AJc^)AF9@2sjaspJ*^c=*6Sbi zE+E=K{fd50$4+V*$&7XtT+t41h-$uS+8x%y+{u6}z^jKV; zmHGSr5)$`OZe=pg!C+{C%x$gB!z9ILQ36w|uO0Da-ipg)RP%8wLUVi!XExq#1pSvl zylRKYdG1h4qwvZUW5=6|<(X+mnggQNz~FuzI>h*rtCkWQ-8)F*d&xPPf3(#HowJz& zI1RX|h^f;7zKVi30Z>5^kb!sO@P(6ZEe}0k`w+JR!-pmrgQg_I`<7_c!%89rYh7=h z`GqqacB;U4jAT4-AYnUJ`2BI3H-!ayAAvR!pGb!@d+gVDj@hrY!25y@vgitQMb-tr z(%{>V#;Gfj5xaDE?}#rard;$(54k zkSfv{(57`ir~pSp-tqRxn!P!vf4)=J>p$!k1yNl*DWA5hck=Wzgq?g=(6bsFCc%tC3FaB~hd{CysJGNBualia0mcMc5Ko*2L zKqikxsCO?K|NUso=jh64&LhB<*$7#3VZb`FOE#Lkt@vlowE`8*6295%yGsuTv`7C~ z(b1@}!cZb?LUpplN7eo&!JSnki@a|o5%Sv_l%0pTOAi&qi4%=NlFRB7Iv+&MhUU)F z`hFbrKvFq?y*wAnltj)j*Wc76EmVksNet?;a|}75mar8+)&zKn>JA<{QVwP;Gt8mC z4^J$nYc(>&wVbuw2_M`u9QOesw|jx^?Bw;shx(Pvp#najdI3(72p8OL5MKR4%ey{k z;CoZC{^ob>M6+<)-rGQv9kfeeD0lO zjjnG#sgSK^juN;5A!bQL#x37#ln;=!}?)iTThPG zhxz9-vu~BVE87*Z4lCY9LY19iMMmCGBc_#v>vqi6(Xhrhnl%mWYTO7s@Oj^WQpoZ8y0}7x3Ih{avfu?MbD}*}gkU!EMhg^2nL}DcVLL1Ve6x**_qGFaBtXL2`v~ zEO`$trr0x51z-7k`#821B|tM&0T6CQ6cNVeP7=M{!=i&(z4R^?$r!sC#wNFV#~cfl z2Q*6QZ2S-l@`bFUU3F#t?0KcuFQ|4XG{%+xy6~Jx3uvv8z&c9P2kQ;@@N)eADp#2! zVk^w{GLK=x4c84WE46%E9tyN3tT_euaOt2!2LYW@dhuGCK@THw0c_B}YY~9Qt@~Fd zlN1=KG7Xe8kG48qkLc)R8c?3g)UfgV=ST4D8n`|{+fMo4hD%by^7gI6MCFnL#pb_t zxhvOgoM-kvw2E*p9H1;ACOPwy=_SC>{XRyJ>cep*RvMD!v~lQDy~2gWa)aK(26#51 zeEY6!eD*SHhe9dX(;Nz_`1ICbFs%4PbeUaqTd(UVW63(1LAe%JMFq0cWA9CvgD;Sdr^KTb-y*YmRY^D!RB z`LyKu_|scp>N`=_V1E97GE-?j{utJ>00QW!a(?koa>@`lcsM(DFF$$#jVw2sRkSp~ z-->+wSGeSbv=m-1JzgL8bucc3n;-}t%t?vC^;HJTOBOGgC`v?Gnq0Szt*=SK<#385 z-k9+`4Zcb8ubwIB2Gb;2)u~!pu&9}1g`!%2th4!+8Cu!QqL8;(0HAC3^n5Px2EG$A zul8n_a?DYAFmaZ9?`xc{`!KhfQYQ6Ou%u=4@@k+J)C5(-QhB{f1ucgJ81kq$vZq+?7#B%}rs(kaE8sKrt*woT zL@Zcvf36z@%lkshvWw?B(e9Z*nRa0d?b5fYyk6RL9rftYta3T94fZQgUEbTeO&_g1 zstrhyc9r#uqq%Jh`p)JZ$JQFU*Q_ypbhmLL9DA{e)_xv88-M95M!NZ_Ogo5GWruId z5pzV!?(KvVn&%s`KorO>hn@w&A{!0)hji>%Wln5|hyskBRF?o1n#1Pl&6X_Da)ZXv z`Xd<(@ot$6w7eIBSwb4p5=$v?L0u-D5!26%S&Ywq@{ZvnOP>03&EGTQQi5}q)>awr2eCD!mV0GWYm(WV(9mxH7S8PJ4`F|<2*kIe z%TYB?=uxb!JO~L{S#r;JZ|$t!ko(TROw@Mn5~BRDf59N@9g_&oei+t~UJkc>CCVRz zGl%z6ezM6}8AE)ZC4Vtw`K8a_u0Qd*w(>O>&k`aHDIt^q<+ZYWWZ#9%$VCn0bI>O0 z@--))X$1sdj2x3}BhOn?W+u)?ud#ovHq!F`oZi6&S|RKMr1qRN?$wvbT8PEFueQ9d zT)r1nF^^nnR{i)L-S6?-q}GqFi+OKnu9()cU<S?&MAEq=^{5c@B<3EIv- zx)n%y@v1rTf>u$-Eb+O=cU5=r^9sg#_ zNZ>lvlQ5Iq85&j(6R}{~4W}(pIi>%}54U;&!BYI?GP?&U8(8cb-CnT ztI}yj2oZ;i6=&@+a-^d%g8?%(@Qr!7MBh6%9vi<-_NeMUJ$DFX4qDX7bv`DtH-;t_ zx4{B>7`>>GDMgb4E4TvvSs2GpvLJauGGa}Rx%;2}OQsBEG6qH(uIOF0;usbEX}a|_ z>N(5=J~GnwL6rt2iej7wlSEdv*p$lRo%*^(>7fMrk7z3VEQaW(WdwsA%*MASVwaz& zY_bgue(?+i97CSsOH}g@55uQtYobOVcbYy){R@wdod+TEU+|J%IWNfpR2Q!SN3(X% z(h(vIN+mS}R;}3S)4K)z8^o{a@?_6zaV+Z-NK5BQ{D&r?I8a;!%SzWJMZ`wl`HU3<> z8D^_`wd#I*XrVMcdDAhm-?Z?>v<~{^^%%8owS;0obhNidOUxrysP2K!TNoE^3-Eb)DIfs2Q77m^kS)9Jz zmF}wL-XTyq%UC9|X&bEIv>M(u}jqMsh>$z5~? zRGe?szG`YvLBkqwa;C+Tpq}}tm9M^dfd0iXSsos$L`5dRJncka^#-AA$}=g_da%X`c+AVs)o~&mHtJS)I_GKbi$?b&#$g3*)W+*($N^%I$x;-sW5UgFVg+drL-|3 zXjXTful+k@Ax{SBwi0c%&dPf)6+zhRU8Mn}ua4x$q`UV2R3qoDO(SPoNn4|tgi!i~ z(EOB)%XFU=?rm!kZ3W_!T*Fth`^kY@O}>|1aR3JO9AJcjBL&Qnxi@+ZBkEX0cJYF} zY+ER1nK$OA5NUDRRQ*lB_~}bXCix-#IX>P$lqUgn*PoFOJ}1ymzxzELLE6V$Kbg+im=D3 zcBFjrytLE24!O-w>LuOJM#KoOZ!Z0uUhOYZsg^Zc^_9JLFEzH8>ej= ziCQLnA@L-!^rOCFL!3UnDE<|~_$2gvRkoJdnX?r%t7u>4JI>W4QD=QBdy}P`7*xoR z)QO$S@KR4+WaCiv=>CU;6yo`Rof8W8`kyQc-#-EbI6X`J?qwoRnC?hs1ZN2fNsSTD z`BL3WvDn_9)fr998R!u_l7Ty%kQvmTpt!|08q0qJOB6M}b7O1d^vKh+X%nLJGCa}R zYidc(TrX~|#imHm-SGYe;;;X%yUwJGQx-wq`I#d$rC&U5%%u~{qz&}-e#*Hf!ipRkn_Q(h?*TR5x#?*(z-; z3NV4%YI<_GF;nVte#ZV$mwA5j{kBkLe$)K=vo|I6jG`|MM#v*c%0MeEUrjO!`Y}cx zW2l9|M4PaWXv#laQ6FRU%K;M(%h4G>k5_oy(#MjZ<*D>`fv{T)mPYJxCi$^7)b>v1 zhJ@*FP9}C}?>#&6BXyvmGfh2Ei*JsrvfnKAcjB**--(37Im)i`V|l4#FG<*sqrq{R zss#!RtVp5IUgl@sOzo7PDN=Vjp~$X7q546!&}?%09&;l4o?iqajv1lvTSX=(%fllZ za$46S-Kkh`CqG`o+P^HXvZm}ccusZATM_-TZ&_tI%Mm^{Td3Ua1)bdxyy7~DuB+i1 z)I^B?=QbN)Fi$L)E_m@3vZ=v)kaN$5cblJW*))03W<8ZGf;vixuRK{fS*bV!0eKag zq@-*up8iyzE?rLI@<)X07-OslwyqDaz$YXWKWfw?N}^N{~Yz4b^y z5#bRH>Vw~1>$EfWETTVY2ujdK2f%@VBFXYSMQ0LWI9;K9ruBdjl8jrJF?G^i$y5&$ zeP{bK8CelNhxjNSwIKA#uZLkGmTXv99B@a5D(68*3=$p2LV;x&D8PiH&?aEtL(2NPj3G$W}>d!L~$_05v*J zYe2NpQoM1f5xMPzH5IhJI49>rfNxn^0wx_b1dLlHx3@Gfwa*CW%OUWi{{TatXUnKt zAhOo`H(j96{@t#KD3#IZx!c6Lx56V#&-xnipk@u?eo10dNll%a9dSQTk(~7n8{=Vw zdg5U<%PE`v^;?9ndT>yN?x1q}26f6&-)-!zr{86%&)dXbEBU+(GRJrE0{8b@Evq{l z1s(G%_DDGDP$-1lwI3N#)6pLwNesAEU0@IvexW&F^60MANwmDdQgps(chdvkz^<5& z6qYys`mwG@yxNt50;rh_n`Fwp2-H;@w}Tk-6@)|_dV^@(yI65;PYnz?s4m6#fSNH7 zhWx9wP1uuzHnd==o&P)i*jTr+KY3fM_e-zZ1p{FCx4zkKHMYl9j5IHt#GlOFtrWUc za9;VUQiv;YS@SrUW_hRkVHk4^Sd4MejTG3e^bu98#C*xw&3tjcN@iIx%T199|#&U zA1a6QCHfbvYJe(ePye#*>o;81{D_o|o!qgK8uo5}o+_)i=w5jK{*4E@f&?5p=7V!t z6&=yzflDo_hHu-#u@bLf6~2q+CiF~P z_4})`#XfVh2ho8r{;#Y(6Ivow(XZ1X&4c@U;F*#c#>v<(eZTwlyTl%QjZbzJuWmnY zcFd32`r%GsY50nR#3D~fuWHVjNTV6`i7PL^cl_KLx>A|Et$Y(Hg`FR;t1*QRYvBXY z=7@_B&a)7^#4*s4mR7HJJBW=jKT!QTquV+f-8*OY5I>BEDffvp9ER9=J%kyYnPXj_ z87>xS4NR+>YYLi-Oh<@pS)a7q_eTesQ;rVw%~pWY^C`AfYB3SU&8E9gSB=p{(Jp~&sn=cO_G5DUJ%jbr|Y>PNZEhO$~ zN}_bIT45pEt*KdcP+er+wE2ryE7$|ov8v4-S`eF1+8rA%;>(e=0Cuk72}Rh|ZBFg!0{!@k)#56u{)|L({c1fd;mNA?){*y9?wmhMIxz z)!eo7)ZEoMcjxh-j#bBw4s|v}sQcHh6UKn_XPHv5>@i%|RH@-0hWT@FnKM+4|LQ3t zqwq|1MdznvESPEir?}eJk^tKc^NO_8u%x`v6BYS8-Rln!6+i6J}1+SM{j$@pa2!DRl5kzi@eiETX#%a z5E_l?jzI4Z#oF8ll?Grc^1y`mG#Iw0pn?OQOIvhIrOL78-%O{|4t9cJ#eTQ@SlwUl zoVDCiI9k#Ko3m0(d7Qb4vquACQ(4c^yK9!_Bx}xG3qnz41KqqLmkZsE`NuOl`YJP? z3KE`;xeb*rFWEO;TiOxdvZ)0ewJ5VP?x*;zw12zYog0ouKoA~=3)BujSQl4N5u5Jv zOHJ7+=-B)AK6ap4lr@ZBmMt@OklIhsPdS^Q$K|c)gM1mLsrAR8;dQjB{^8n&YysQ- zclE%vlKSJ9ipeWWtg?T6@v*XT;@7ULyDGAhFJskE>mTOb10e{;&ZkWWoBWkbK-HtN zw$#9KI~|Dj0>kpw6Ze(8QW4&(K49&RssCkG^r9k0_qiN1pGt9Ezo>&ua|RWUrhV?2 z$m?_Z$#1acq3naHxIb_RkG8FmbZea|T}qE=pAU4xAYr8KhsktS&J6tcQ-Eii#>JsA ze%r+;w&uRNb0tD&#+Q^yV=LptOVaOUdm2OhNJ3tfUkw}3LAPq*>Lz$M=E_g5Ch#E; z8vI1hx1jqJP$u~z;^$a*J9U^jxWXaA1! zBdVk$iKhLk#l{RtzT)NV5V~t3jh(ZcdnXMP+>C`qLRJerKPQzx2YolB*5o7?NE=APjd$x8tY0St=~D` zRn}i7E8#Iivsvg$)y=$DckJqpRJ0t=y|ygCuojebCE)jV*-Szm_}zkaeTrmX9)os|k~^+f{9aTIKmz=21uVZ}v=JJ2lbx zyo_W>T^3qUpmq-{P`l0jP9QuF>SF@JUFeX<*#uC>9mDJ>Nxs)&%Mh4+*X*hzKALL3}_@U_V?E&OKn zAB&)-$vkGIqqi~wI*K~m+%a$`pamm==06y^e*>1{L#&_La@?D7r*gu`Js4pp@k1lw z=m_ayL0nMQIuP`zCnjIU)97Xgqgqi>J~02ww4wgQ&RG9;rnKT_y0kPj(O8wluq>Tw t9okOi1ErRdx>+?;U4(kER^jm3-!Ks0OI9gVZ~!|gn4F7CSyzSpe*k-lEVTdt delta 4733 zcmai%Rag`X*M%jdL|Q^nnjxg5J7fq!2I(P$p}QMo=op$Ay1Q#g2|>D%kQf9G-8Fzn zf9LW)SO3krT336$&)$2LU1OA9W7MN)F$Cbtv`kP6%)@rpUuI;`HADkSI>Ie~^Oi@g>nbXVd=Ty`t>2f#XANIHG zUYh=JH{pY$UoHF?Iq1~x)+(9zM+Jw>P&3bolCjIZ+1{xy>T8K1mhW4Pe)r7Rir%3E zS%Ofi0QC1F_#WcK)RiaeA6qv2y2YZm!9XDCDC=A<9~ti0bMabDgH_^r>Wn2LR%HLFqqMaaZ8=dklHZm`3=EH{wtsX=ws? z4>wDQBY_XfIC;%Jl#gU+?STuu$PZ(CM9GGI0^r$8*xlh4pL@uElAnuwycQ7K@55C2 zyY)S-X6)vmrDVAg_f&SHMK!)f>e`0sKx$?}g85pmIvKiRaWj{%1(jH)11|o?CB+ z|K#PA?s*;d^NQ%I$LPGp#-HGBm?7m#{7wfX)eQewoX9w!jL%74ETW{zh${Sa6Mq@4 zEe@NeG`kdaL*OX;jAe$Eu*lMWD2(4A!EaRO{{s6~Ozpg46xNes?%&ECd%V|JV8(C7 zU1#56pIvj#uzn-ZG&4UoA+ySp+YC>8D-K7|5<^$)dif}ZPVXpoXZVZh?N;(BnsPKs zvl?%o*}5GN2;?kP&BAJt}Z&a$C&kfaAUTLVFxe zTCVtqe45)3A|)e%cgJc#S1EISGxg{R70uAyo% zI&m-Tdtn-OXi4G`j;hq$fr(`HXr;j;fk>%R45Jj@20JT?gqunsCd!yFKP-nQwd;`BKh@B{EUXphQ-#(YBTY!bz_R4nXoV4f?w7tFZO^)D}p3HZyX!`HH zy#0|raF=)}Fpdh2#UKt9{ttr)LuZsMiQ?MLtGo0{SWoK;10xGhfxIJ%lf}^_u1iDp zM+>3k*{S{MP5*}^S@Dk59Dz;9VAZjop&pgMKK+eu}=`Q)*Os%E($OeHvH#&hxbF9dCa3;auc2VyMXS>90jH zRPjQ#N|q94{>O|R0%|hl4Q?BNwXuR_4`mXp5RXWW)^qw5v)=AB83nlbdpd>PI5oux4X<{j#}t z11Q*5g%6tI^UWnqAzD{~2%m>$ejf+gi-Ogwms>}8LK#Tr)r&{9qGRdIfIZ@Dei5Ff z5)EEVS22ZJE3wjzKH{XgG+(XGJ(xW4UNs$%tmX{}6uhXGjJ7@>-@{Y|iIUDoGnwWL zKR=B9f}BG|Be2f1-AFia$N$xe`MKGOc_z2v@>*tG028hfuh^B)>F~?>EZ{aD&0M7V zSI8+>Rl(ZEI`NuTlL{t~i#>ckF}xJgX%kNvYL%XNjR7|5sHN9~#74-8j#pRyxmH;g zK6c>@c*1~3D8S(UOybb@{~<}^lYJi_Vb`7B-Xj0U4MH!1U~Xf^dZ!{M+q(FFZ@?|q zkl5DRl-CDha{c_WYYVAYj;)8W1RD1~T+}$!h@`gw3IE`qo8^QBuasGP4YuD2dKcRe z|NiP4r(wi<??GInRumzdOK{s5=aw#JQpL0Ogz7*I>^?vL!6@Y-I~m_-v4c3XQ~-rRI+2u^4!<_Pze8KPK4} zy-xRd<2^K`8{jCXzZ?1S}oR!6`Lbs<)%NwhpllBIF$IS~oyjg0uk%@VM z|8%wc;0Tw2RxiZa_L_DU%tDWi7S99IB0p+-D4d1sQoOTmCUdhWj4S7@n zCSQXNk}4h_c!g$&hS06`AACel`$AVUt2G z_VzH=0I3aTayjxD$5w%VHR;G@i1K7sNE;j1EW^no=EeyFjV$2yin48(BPxYoE~7Vq zY;=weo10+vqgS*&l9H{BC5@r2SeD!-Nes_`g1r6kSy&+HZo@^Cc6lK)iPcY0=7E#m zv{g3@xYQ!2i|E@36))d2El6Q{m1$wRgegv(yM~K*NM>d;UjwDbf+XprTv11EJJ*=E zds}DUjFeYxCwRFoXj!e*sPFAu#M`ldkxtxRm4UUMr8Od1$5f%B|2NKNu%XuSN3y^* z6Mp9y?-%wbXNziVS1PjAKFZE6pcI}#sll|p@ zSt-nQYU^YHh%5!1hi?Kw?qi<0sq_Ags8#xTrtL`et{T@)r!^-ZF~4oP2t%qb@dlVA z7dp}tLZs$OSucyv)<&Ynp~3$7^&CejA$htfnR&3vOQo>q;hlK(P8o@&oWyo~#A5p| zXVdkJ_>a?n(wTq2=-GS*!x*QIQY~+&uW9%#pJUNW6U9eL_dKj;Jv)U&FlzigM~!v- zdyG#+FX#&065W-9fP(c9*>EaCUBQ7^hIdeeMm$OI2i?RE!7PkSiVeyvL|`H15?NvH z9aEq`0fAjxWNRi(bhX$>ELy+T6o)O`auD2x`8rVqlKv{{X{m8*=THz74}c>-*3wJs z2)47cLD>M?4(U^x@s8VI^m{RoI=}2N*e%v_VdSpL-H%9G@7HB33(cRZz-!11H`Kal z>t@r#Xc7%>mC2KSwEugzbZ)u3$Np^;SBKMQ4`<8x^{VF&(@;mfpNfVnFXgS2l%S!v zvt$IdWe;*h)UIAB*(q>>3u{E3D2~SgRcLWPm6C0gGymD&2pUqVp8I96SleWkpzpGT zsuOyMQ>^_UJ-2mVN2d?Ifp&X>l)Pt`_Q=ETbW*0+RCu2>3DH$lfVdu@tBLl- zWxgH}QC-zz^#NVB-pyA%FXYyzCutM&kv$V`2Jx9{MI@FH6^VGjzEI0vO@W0di(DJ*Dxs@-ehvt7v#Z|&o|{Lbzvz^~3CyVI;@s9n}|5hCq+yN_i|?-+=jO z6Z#la`77>~x`%5q8}X)V%Sg%(kKo$~t^x9v0S>&5B~(Q1@`SGXCgHJ#>TXB5i^S5r z*Mx{J*J`sBz|d!@*-UQ{I>>aCqXJIwGq1ttZWWn%b2v;{%f|E3W?{@x-5J@i-WBj= zqTPVM+~{%_`hs3#>utRnStc+$ zdZcZ1ce~FaajGH@uJ1tV?AnqV5zrW*IouxW(n~ZkiClca~_kB?B zaPVvdEQX%0@XkOX+EQ$YpZX(PD|tb5qE6bs#IvV7#hn5WDM z4$~7V@Hi5&Hu)Z{G<=SYkaxFEB_mB?Ou!aGJb4WTwbzKL>=>pf>;)DWb{fk}E%y|r zQ9KS|_9c`bFHPLYxuGm$reYJXSAXU-2t2G=5SP%{ZWTV)3o?U{y1!h0{agQAoV)8= z$54a;_^>RC@wOPr1f!MvMYN=ZZ8PbTUMf&F8>Dr}_)Q=9Q4%yVDKD~=K;TN-uVqcr z@k?1QuQbd|Y1&aIvDu~E9L&zx)%ImbRAqYR90lb`=CzwDAaW_WE3=4+jxe+KL0Mp) za)xi__fT=*1>ei{$`e2fK@nqnb_&ETYZ+F=gk(P@5%=@;5B)CzO!kGdFM~A# zG3fH$zHh={|34!t&|IaH$XV}G-nvKM^V`bKQ&5;h=cp3#=3{JsUH7Ex8;QK%DGjCx zFkhiNKNrb>S*@$X#CKC06IU8ocfvlpt#6ZEHKdRFUXJ))#>Ljmn zORxL5spctF1-71fvF6cD;TDg$e{dB2P7f_yWnDPi$Eu}bSDH=p<-au=ql40{Q1%@N zxNH?xl5^fivLlZ){BCn&);v)~Ou&XAU!11BPbSuxwj+7ryKp(NeqolHZ%}dL zK4~NN3itVi?XA)P84iXVX5bpyOhUuYxppnjm?tNxn}|KN4r^fa6dUG<1r(8OuzJbf z@|>a>hXTR&?0Ftid;1y1wLmaZz@IB$e;Nm-}(GR4E{O>0r(ZCAZA5iQ(!yx>JH^b9_ZWIQ&Z3$lhK=)px9w`;fou4k0 z-zF7=)Vcc3P8xv4<_1mg6o2x$+vuBl_4wj2+C&MqDWeVY!W`K9i32MfB-a2nxyW?o z9*uu_!B+5+9PlS-gy-0w8R*0SJWfC)v@Xy)=e2g;2t6Md0q{-BXlsP6Lctmchs}O1 zIs6FxF>|@kGF|x&pnnKB)N2?Aj)D<2mDdr`t3lDL^lZGV4|m(YXpDS~)kjUhpx^MA z>bvhT@4lPs$|h+#RRLIL+lSG@>OW(MQ|cFM{!S5fykR?`-DWvb7_R-@%!l-f^Sm=M z_hw(CWzAYxe6IO~3)6y-!Vi4k$y=I>3W^C>@bL=@nh6R)EJQ>t#e{{;MIn6rR)QkJ zg3^-zp98M%r}*FRoO^QQGbV&2V%>>?N+A(Q0Wn>)Ncp5#4|_HoTp5iKPaO(yW;1Do=;zmGpdlP zhbIrZdnqzt@wbI&MfC}>FyMb~3#KSGWyH01!A(mHHx!M0(H4ohwT>~WG1ohPd;yD} zN$UfNRi4J9TH_q-H@hoQ{}w%4XNb=1KeKlEKUOPi-Pi1@FGQ*$cx0+EjUNr9?9%Hq iZK(}}Je%F8L~xU2oHpJt!=e5Rcwz*stU!<=!T$i<$tHvV diff --git a/ps/ps3.pdf b/ps/ps3.pdf index dc2bdb748b93ddc7b873dbae33d54a265be630f4..ab74929cad891cb33ffe3865326de757faa56f23 100644 GIT binary patch delta 5577 zcmai$Ra_H}!iE(P7%klmqZuVB(k0!aWF!96L{dOu0s}^a!%@;LAs{I|NI_K)Uf3KeR{k^{@x&$Y#1gF}O2OhXj-&;EQnqB1|AgXeH^c<4t>MfVY%dw~0 zareGpJPGhL-7>JmT# z?XOYsN|>!b_va_qmtHOtKOF09rAI~SoTo}0Xq?oPqW0F!aHBnKj^=+$c3TeV!;=-r zk!LkqSdS{lda6AVa*=Zk)r^k^r}_c3DYW&|LmM zkvf8ZPve?N$A6i8yma=oZEjd>Wjb2liWNIQ3E>Z^c#RW|xLK+juNqu_!51G}fC}c7 zg^SB7DiZnldOJa2L;-WQ9~Wc7K@<^3C)CaRwbT#wPoDbM+ah3%E;o-{#lNe>bL^=v zhv;bp)r8`;Z+uRN@ao&q7VY!y*_?K|r@3IY^B~O5!=X!g2i$+(3&<%}IvOlVsDHlg znMglV%JMlgn@GJlGLbsf4Z3d#^A7>F!P{&T?R1|=mbNR>7{WJeOLNxog84H;i}~{= zz#fY-yTHZw$W=1WtwT-X(JYVME%fwxarUPmC35e4kpXe0U){%-?DnQ{;m);BK7Jn> zbP2LC*HDUqA1#r=E+QYqVHwUC3B?Q9mVdYer>A~r>3s0gFqcunx@}fV zzXd)O34qnpY2;eT{#0w+N1Wa6^X;eZP~56!(QI!2UJVe>8imv?WM*1r|K^NShIE&m zb#tmoqz0bt5bVQs7U{Jp$va=fXWeu?x34kT3KK;g&GY+Kwqw>ln{AXGPAyfdK37^P z;;$zTly;>)H`nr2c{TPjY|L0IX3Utp{+f<_yY?j)JR)m7~^#mNDIL5vd7@&3y;JGV$j>YN4nhZR>4bNK|S1eCV_!(U#EoK zN2z6>x=1N@j55lbja}i^BjQO=f&6lC$^Y45imm&a5_RO!iQrc4kg@L8w=lwIMpKKG zwr)4LQ1S1f83-}E%OmOZi(U*qb@tW!UjqZ_oAc25XW?mg0NeVDaG7T?mfND1$idj~ z7gDf~+D6ioN-+-PkhDxmgvDgPCVv z-P~hyg5{BtV3Bn%hSM)q=c(-#@A>9Y2DH1ELcLH4#i5?YGx*6NV4kdh&y%5ebM~Z3 zLC>!tE=9AJ9nLQXR3`G_W*T@N&RqKFxw`o0%y5}TOga%_UKr3=PE(4MB-5~R$iatI zlNXBrWT2=J24v5Lt+;CcF|`30^h)gRDxGqVZ35KjlJp_*E>go=>>0^P2JtTaoombk z_GAnb$gYnR;Tm?5&}we06MrjIl5HD4M++K(q#mY#C-+#3Azhf`5J(}wQ~rrqWwrH4 zE#!}6iexo33FpVA5q5k49j<%71emIo+!*aBd*tSXvWfwXqxvNw5U*HAQ}KGvp% zrd;pVZnx@^arORen$QiHv!|oacBcj2#}glhR?KRIdMDx2uoGwWM|S@bug@;~XUW>x zj}+ozUsKaJCF569e0H%k|Msp<@c#bsW26=rs!N&*MIr=*|L?n@#yrD76n)EAg4az> zVoG_Hypd`~RCJ5zvUi#?EYY;af4d$+MvvCIwb-x5T5j6i-$B~#DqxGze}<$s6Xd+U zz&3ZL7HqZGw6^E)YpBg^{dMoH?P8x1p1{UJs4Xa0Z-ZN|W;AIu;=O&^y7qgdJQcgSJqZ01uqy}Ddd{$ui7$d2sl z*Td}M{JDxnZ5A?2=wuVxB$yfV8fNrSa6diFb>c;OTGcg>_=ow!lu1P0dfBb;0tM^H z%^yCU^`9n(+Tn#g8F>5#yaXHua3K2zkHjctAiqn5mSI?iRjBcXuo7b+qa($Dq%yY! z#1GfX9tME6)s#pcbLS=1UgJsP=M|*Tq*;cZY5HAXwEdw$ zPu}tBg1nmMH_?CEBoG0@W&Y zsvv&S8-yqgfwzR%u)RnGE=tf~VVg60#|NA5;YbOJ8Ihsw0d{V|s`E2RnSH@Yw;w(@ z6j{y(U9*5^Usa?pSKcj~=YIMUK3r6xTEZIN3`rb(%%lEvW388E8i$};#;RD3TQ;lw z!!5`{*)SS@;tP=yesG%dEE#Ckb3L~SIIXibWH4;_X*Zr*=iOR7vc~#^tuiYsq7yT` zM4oLLFf!q?4qm-`+|2s+>OWlcM3vzGcAP{jY#3ps{u}p3qj@&_^%^QS&M~{wWrLGQ zTOo?ofvfk=`}JZgk`8_>cvh-ubGciA-5DoTiJ=zX>&O4%nm0o09NY8DgZlLn%^tvh zJqw8*uUgsc$oo-y0=w^q+P+T@4ar_`nVaBFpL_$-T8h*j%f?O6)EWsTu=K$fuqlD` zXDDI+`tRdIx93If7$aNgEg`L#g--Mb8r%4ke>Jhe7+Va{9$ZvCAUSQB&AA!+q)X>A(e@YwJO~GoO^^ zj*;^Qj{jr)?Fxi6hl~IB_=dq?&v`A%kmXlhsl6HUpxKKW5Q*iKjK2pk^gvg9Xf>R3 zjO?wGQPK|k{l$%$34v2==qKD6cAXmVqF;!u?eH6_8m(%i51khq;45byr_+wuI9gLc zTtj1AtuIq+;K$fI+CUg*JkQVXS~o~ye$Gm`a*f(C@?~U^nwTNZ#0LRG@|ZxK4~&MS zZpv9m>J3*Bt|x(vsIIlSJuQqvw=dqO1ues$()iTg@dTfqHsVkp3-fkSzs`#1UXs7$ z`MaddhxXpK=p9lPPcNsyPDtA9*D+LZa+(?|zdQ7hoMkkKG2YPSU})heSx+cV~?Uy%QcCcy-zarYnK zkbp~Rt&YIYw7X-D#*@2e-6a_sbekI;u3t0aW59jvsLfs1AT+B(lFRKdmP<&!RsWTl zG>zoZ;(L*Mk(sezF~O*C0PFH_OrWd)Z+Aba^xlNTU4e8b=c0eQ%~9@qJT|0n990|1 zvP*+%iWY=NG-qiCXUlSDQ0cdGMT?H&uwdRN?(Z%h`1)S}Py~LqKY3j{E{(}z%Prb^ z5wXYZ72C6R9gK&sro+K3M_H4-!5F8{C7$>fz4!*kkyW%3&;e^M+Je1`sFlGf!ye=M zNO3g8LP0Da@+ATwOpGRRwN~+syRK)=u)Llc?Sp@Dsz90zSuzNY6@`>7z8I^5P97`u z>wMMZHFtg3l-UJ%e`I-gLX>W^zh=_D)b3xanahx>QIPel?EIY(Z|xn*p}yZG{J= zk;cmUgI6M{?pb0+b=s{eGA2yB#oGmL8OZYrwlbGIVyO0<*3>`(o*6N+^@W1uR+wAj zh-ULAHpAJXHt5)V`ncEjY*Oq4*deGXSa2EmaWRe^Ws}G3fyy)}hc`Gez+o z&{L(Th1YlnKz?TY6BcI_i^r8G3fEH{7_DWX+Heta+F{io?}j#9_?;H1{p0%}b?t5d z;-F{T=vZqKqvP8p--CXA5UnnKIUVLz_v0q+4i~=O_pK2N^`>U8F9l85MnJ62*Eg*v13w~vwfYuj z6?byLxMdyVfl+S;2A)^Ifuw)5Ap%zp60unNvf-jZp*fFAJ(0Y$h{mSUra#}B2J8=l zJWSKRypJR%tMRTB56|H9O1G+N@HGhY;9`{WElBZwvZYLDd*ImgPIfQTh)c3_?xB1I3}%Fq!6{nK0fwjL|RPad_@kyPb6I%JEa;lrh6&?M+EbK&|N19>6WFe z6>l+$AwjYeRg*sLB&M=M03H$1#c|FF`BFOA$5j;dF5y~<&>6adF^FYGwBrwc9rb4B!V_hC-YbHqtqhmDqR8-~xE~N4kc%!wOTiwl>Sl%SX z9s(w6Ya{PCj{pXYgO6S&I?8^A;>j8ZjwH6JlencX%C;cWHk^N>jkJm~(Ldg->yhcB zAS&i>av$(F(TePd;FsV1L0hzY{QGM`FM-e^b9K9QUB z1lO?ec~;q-eO<1Md)h(4cMdSTB@TtoDH##x(ziXP#^S^*JR;H&|A5(%00ujE7vD!K z?Z(*=^Hq^j_xnH6Nj#iM^usAFi-Vh=TR zvx0-V$nbS{1RIMIZEA6OF)Pj^ay4u1p`Q*O+u-&_ljfGPe5f@$t>RnVXV6`EJApcx z_S9dl(uH(XWy{>*@t;TSX#)-Gj$yf4asdi^(QD`1)2OPQdVAJEA1ps%8$_f4Tvsgs4*MM{4&mHytUE5VgV+VEp2piG7?`sJz+Sgs4}!OMh~fV*X; z!!Pq*(2h{5-qElP3f zHyb~2&Nmem@1xnGE}J~@Q}MUMMY9flRdZAFZyxB(+fV2!r3UtIW8>GhR&!*lE&3`N z3W7rDBeL|9kfZ%G^`W#^*s@FDUByFQ2yJ8AFMaixXHaH80VQC6QIZ_Z6;}S=WIP@_ z6%=iHq-Oh&Lu3?wZDk#=@~ zI)fxY4iZj^3jbdMMgCC}F7@AL$urU-t_q=!T&?Hi_E7sAEwiP?tvvHN8?C8J|)q83>kALD|0soo?(=P zjFC3dFn`3a+zj~v!}GcPt-~~{WjHM&bKw2nRE0n+lM0y8$#PCCv*4r>F-&x{y;Nhf^mVAt8!pCjCe&$bt^IPs` zBVY1E`rTgLd6s{{Pc+wLytRb6&^Nld1i06i9k`d9z~EM}-Nn?&h^6gHx4FPZw>{Mf iH*1odkVREbHqHk^dhS^i1jR}u0TL(T;ZZlxAo>@KU#Twu delta 5580 zcmai&Ra6uTw1!1O7-Hx~a7ZbKhM~I~hHeG~=~7S}8is}eX&Acm&-a26*IkP#Pm++n z>D~%d0$<#sN&zxN^xIL>|AVYm>IdOcs!JfdtIVnDV=@qL$pL5IoXTto7 z3No3q&EGbGmi<(SU9b0tmwYZeXyQ!N=ErkxRDauelN^$+-`Lf=>pwlAtFA<~ApKcH z;Cx^S34o`U2h7SD@M+mBaV;_!Oc;9ZA8G1i6#2Xygk9NK;ndbPdyO_N+!{zNI~&#U z$U=40hedpw=xG<0)-10N4btp=)J8A20&!8q1i7b@p?ieK;tl|ndY0xzcgXG8 zJKqC)*Zi}u`v&fhzEJtGKUn$)x_s4wH*}cAeU+hpukG)z!qC;NZzMT@^H-%FeTX-l z&|u|OxRllCEPac>b^oV{0IK@9iVK%%HmAz?!;lVC3PyWjywhh@@!4YwG1`&wHMNzcFSQ6i!8cm-3U_DG=8z(A&jUeKi*t>lk)ohAR-Z6(i?gLvy!B@b& zy|ET^z+ZY|-^u*>+gq6rr&{s{z+YZ5V5HAm2%P6X$tA5hZflV${7a?h5-;iHXANAC zVLn5BR&FCOo<)&W3rYAV&sB2Bh@MML$|`H4;iJ#>mv#;xXHT9>Nm+xl$3EKh#@7!m zZ3M|e^iPZR{|NJV>zc`JbwZLKojSg9-=|#cjy)7Eo@Pf{XiikGl{&$94u8(kSLf`x z-8?nMRF*yl&beB&3n4xkoOX^=z?U1s$2rOQZ1#5=_rFkejE^jbYp;d{ZeBRJ^rgM+J%je19Rb*1 z#smino{pfn?os@9+~GBiATd7NEeH8eMaBsGGQPbd2H(hC58x2yFsK3{h+INv;v>2n zjqt(a97TBS4_YeVLFMqMe1{^sB5f;IU|gWKB`i=IR8FpRGy=}HRkyk*cyZuFYn3{JDMYBrNC;e0RW>I~k2=GX4W2y)d_uKP{?L-|+ zDEF$Rew$ukAxmb&X%U2ELa%8jO;`2XDP(X6d7mlf?tVe~*4}n#D-Fq9{7xgJINvl6}X*_GLfw@|krA=hC^hVU4Z(-GUHV%=Y5a z{8U3{OIUB9Mi##H+vO^%w1=jaSy{sUHox1FcTC-1I=Q39F7I6yRwQE0JC3~Iq6@E6@>%HWDJxRpJkFMmkS}3+Uoz+s5>odp+7kV#Dt@T;2UBB{6fA(+vW@@i#{TpfU zr0Go)#%DGd3j)-QrbKO39htNec*|BbvI3w}SGLyr8k*$s&d-&_0Ecj!OOrMcv30dA z?WF#+;#DM;GC}}HCtHm`CR+n8HK;m|651_95htK91R7oz4su9Q`%>Ey4qZhmRdpX@ z=~BD(&S5%*{@KyntBQi%u`Q*COu6q`u2)V^@9PSs?0aw z)4g%sP%bX<=+AMmsJ1*%Px;kU%G`J9S6O=9+a;_cM8>CuAZ37zM^o;v%Rnk(eS15J z#Q?&rj-$_ym!BXDewOTro7vio-TRZhRQN7KjY-P8f25U!)PQ4R5ZaPseo{=dh-3NB zUOr)K8IxQRx9i1Z?+cHuK31krIBCnk<<%=4r2hpN`$W)Db*XU9hIN)Qul=wyXCJ@akf9rSdFL0K zvFpISlF+izS`DB!ni)2pncVP}Kb2X4LL}_HO=7-{HXW<%H4k;@<-in2#ZNCT?+(sv z;D>EGf7h)SHjev_Qt#fBmj<5v&W&&%@ii6;OK3F}6psRh!_}3GMte%=PhloE6nixb?D*BI#tZBOUW0#P_3K&fy{wQKl2P&uh@> zv|eXn_S&PZ$IbW2IQwl?!dGWfs#=Z_tSG$3ACrl}PzE%Bv|!YWrRIj*AN|f!Xzy{2 zgSFn(80irDnc!^TSANmLlA#Ufy28U@188eXX=69&1`{m92=`H4Y0ya_f%n^-xvc@l zRvc25HDo9#L-As&)ncba9L&7vfLs4#W${*e4BrPn&F17QQOQI&DKW!e-6PJ zQ22lTqcI~R*A*q=uNxL!DShc};H8^dY&=7#7*W4Szt8rN+bv3|KCG>dP05S!`Tnu2 zG`Ozs$1s``Vc}xMPe^A~m0dH!kjb#OC8{|t#2`T{f%l5iVT6&4ijmO)w4+{NyL%`e z^<6NbpV-c>qN|BYk>Jwv@!Y{#UKxxPUF8EUtg2ZBm zEZv>GnXIy{AVDfFOl(h|@#u0VWF_mKGMoPkue#gjA9@Lc4S zDa9%?M8Y*U1=O_()>adIN!q>Dx#>p0jg7WP2`CB0l+nSXErH9gf>}P|iF-VtZ(DjW z(S~#bx6)!b<;l^*Q64OyI;2fNvrk)OdXKriX$T65Z<|dhI31DK$Bq$~HfW3i ze7z+e>^DxxXWQf6i=i!B#DCF)2}_hGc;i9P6%a$~5E7&C!}!384->PBT8RY2OQ054 z8}6@^Or|703pc?@eyzu76omIysyFh$Z3g1Ol zR$gozmbZI~8`Z&4#+F%a4>BotWG>q9Vm_j;1hVp`xlVwzRhxHsUv|q)m)Z)WUTz$D=gV<#4^hyFEc_|jv0xOEr zA5o#_=v2HcX>H@M`@<_?{4x_JaL>xQez@W0z0*UBqj1}>7%Nw?=fh;Pwg|D%%PH!k zg9^^O6xcK@`J>}B!j`J{KBWzT!wJMe=>ol##W4H}`19>6_76eo@ajy>npbu=GY%c$ zALN1{8(qnZzQS0)GO$rm=9EM;sr!F&4tHDmQ zx~aWf{S;!_%=C%|$2hjCfvw_)kwFF5@VP{LWHYo3;fb zoNl8u+P5_)oTNhycw>>-GmPYkE^<<# z!?-eiPOpczKTN=xe6EE_;_7D(#BkHmHcZk>{x>@1rJ9Vuhd))X4iYS7!2E;oes>#gt!euC zKJDiXOYM}ZmIaL#oBpT3PWPRq6<;1v{CH!1O2-1gBfnv0!@=v zPij`HG%#spm2M=8p>EYyX?=Og5K`sCyV_3sXRL%Mef0h@ zb|*YK3_8f5Wc=(#*G(Eqvm}4=ru{vO9~r#6gIub}9jkMT7H27SjZGd)ju3M(40|&a z=Uq3(i%?tzB4=_4f(NWHKW%-E#+8CA^MZ8icH*!@*=H+~O3j%Ub<`b``35~OM5rZ^ zCX6>jaae?Cim(C3UtSOF{T1c}OJxZDQTG-72gWpK{g}*v(3d8nb-cB(9hxI^K{>c9 zl@)}wBy7Y(Mbh+sL~U5 zcN8}4##>Q({$c{gP6=<&qIuD*tU}=vSpL)`!G-Nxb=a$M2J|jik8rc^9Th*r_*~-N zMGZZAHKUFUfU5jY!As6$u?5O46%;_l;+1`(x&UmO67?*^lu3^_H$NKYVDqoSE5+U=k@RHN%-0VB3QqkvaVh zki|>(c`9RLg$sCUNNTzayFXGihT-S=c@@#^JVeA|`VS{u4PoCiYrZO(+&lCVOd}_% znxEOiV5aq;cN1%uQqIvI*~T<^o^m>v7#ik-khkV5@{Nx;1pcjsln{;QDUhw~T(QVc zjE?<=><&~!Y9$?gL^{+`I6pU(AD z{w4{urbp|OBw;JUCvA8cIVCE^!-X28L!p?^*`g&pr|kuuj_I+J#t$0;wwI;cPhX(C zlqE^oe&T4IZb{vpHEme5Kjg#Is?%fPVk)#7@(AQcF!AgxcPeF4IyqdeCK!$6@38X` zW6Eg&0^Gdxk48mvwIvMxqDji#`W4&p3)Cr@Kbwm1=a(cOh1J%tEC{$8`(hry`)yw_ z7A&5Zv1Un44L1dXjy8Xnutu)Xn`=Vy7lIH=fe6tTEN0U{g~(4qq;O%K@hZ)UdvU|d z)^vSIGPbW2Y}t~yY=`icCMhDBQ*^+XgEw0RKIvG#&5--s+~?0W%@b#Rh)00N7gmi{ z>!whIy>sd(_ZMoA5c+81Ka|Yh=wcJ2fWhl~1LSrY3{RhuZWaA27B+M>>W{ctMGsnRyn)< zJL$l0Px6zxh-Q-QZ!sZCK{YO2t=v2Kq+Y%lczh5*=x+wm_BN_F)M3V3$}( zd^j{TT&PXh8ns<5GffthGys(Qc@(OQ%XP)kRha{Q-&^y z^PTVNKUe?VzStLgKkvKtyVlOg!OY6RtRG~6NxXbX;N$zo9_miuzhIH7<2nE&ibmg1 z$@nFZ^SGmkm8`X&MYb$DEin1{xO0h$wRoo~Ki=EaoCCv%lh5}D-k5{(v}!Qcg&kEA zpNeT|$8^&bD70DE_@@3`P2)gQ$XVe*f+lXXoV5(b! z{g|k5PYWB2p03DK)j*rO??dFRP4>PjAl09oq?h0JEn8o5|G+NnLZdW&wOWC)S#OW0 zOPfs&^+~>zl;CZV)9ca3b#GxZ%?YYW=Y0$fmNg|S&U{?QFKyE12x&h%^%WVV`##l= zV&^g=>&7t?cq(Yc3nK5PaL24AFA*dYpD}Mxpd)N52hcs^(PZ7Pw~U6OU*9y`_plsL zm-?VK1}Y+(Z;5#mlVP8~-hT9Yc%lT1uSFDdQ6TV{xM8Be|G}4L;2HrW9bUbs3|OL7 zEv;=~RjQz3Jdkkt{>wHuU$;^-_IA=!9)Wl%kn% z6{e^Y{j^4zA%{Pz8jjQ{pa*j=$nYfFwNIA+z74 z46n+Ia$5se{XpC>POFSpWkCvThBh!|6+{viI|eH#O*rj1QCOSDsTj1AI>gqG|Dfhq z&hp>k?K86(OR*NzOo*f-H&P389iPHp_P?tom1MLScsyc& zt&6w+1)Rw&1?yKJ{jVE+nkYwO=plF*I(%3zkXLR&7Vmf@to^cdURVzpg|Dk>&~kA;>jb>_bBjP#JpM##mmGV zy^2ZkTcXdlptOoEv!Fiew6+rGPQCb>7apD6PuY(EH&9MNaZ_Cd4luzbkVI>o+>pd&clNgAPfUW(%JyaF^bt2 z!Xfr%98>$Kqzw+WJ@mKlH9irP=klDVmFlG@4^gwf8sOvdB<+dH|FXhlm4N+(mjWrH z(eL5rG|3SmcrVferi#GfHj_BATzKBOky7m#TQFp~s-Em1R}rQ+ zE8SWS=8(Udl3zQP^WijWL_rys-*XRa8yqtR(ppV1Tb$gmG*ZVA#dGtpT}>?oslUFI zy*`BD&D?!5iWcNxS8}K%HXZpGWrzB}Sc*ToGABA`Fwwb#H6l8`U&nQ+ zldTw;hK{VT^YiCBk5`i;9+{*lu-}b4j7DQ&vWC>Av=$jNw0YOFjrqhjZA1E{9*nxc)vn+UeTTdnd|7lzoIr?R9jJ%<5jZyQ>4g!@V6K)CxuRSnIhkf3)| zaU2B<41;l%dDV`mkNakRirJpuJxnY$BReO4EcW=XxV6^0Z#Z>SoZTCC>|M?A&6bo? zB=j;f#la#2EEt(vk)K?HT!w;~@yGykxezq7O=IOEhg@P1`V;ruug}@RJGi|*ZSR(& zKG_A4VViC*59ymUi!XOHvYRZ3VxKvIbF%BAGqCLrU^{7bOnNhVdunG=EMuh_!}A)N zo6k%`4fDB*a(FHX0x_tHOu*$un3iuQ2^^Xxi1JMr`8gb4z1V@bPA`ZhlfExJc(G-IsO?RBDOV zEOwZzD&h)z-i*b@5tTq`smc)!RZ_!27u^i$a}nF)G|wJa6^$LssQ^C`nGrS7I*A-R zQ*-xpw>@)&+>1MmK0p)5wn_(n{@wK;22^84Y#O`%lT4%t7FDbQS zxjZyL35mxcx3Y0_tMC{qG#>k{cL(;x{ ztSom(i%lU`yKcgIS#JIqLWwTy@*M1_RfLP9CtznRQp>{T<5!*G z(Phybmxs(uq*tR{i!VLo%(cDUG}FM5zp887FHRL5OqZE|>$oyhnKO8^e}3O1{_S$* zElpeaZk+9hs)kZ2yANt@Z@Vx@hwwiR8klIMDOprumxNRWO|%R&$oM!X4ez&Yv>CYY zD#KH04U>#mYl#e8Mtoo?=A7_NjsOid!_agN!BmjOD+F4FLa%oc(J(>?J`!vqm+C9J zX6AD<%B=jTHZ;aG*}(rrC^u0n^4l&&5s9GM&?}m@pGA?y1|`QcJaDVvbua08PsVM; z{a3oY*~Xc&%qkP^xP$5;m(zx?W6YUR$>QDaS~LS?*ole#T`(>*rpLk;(y)EM z1=q4rlK7Z~)yAn!-E?d11~uOT3qgv1ucA>L^O+R09YGx|7w5EVbdFO% zwx12B@t!HsM>=1kE8Vg_JO5N&2)L|LkSEjA(4$0z8e zOq6pZu;oyIPJC(W58U94xazmz0kR2jQyqc{(Q!Bf0_QTW@-14M_oYqZl?zcORV zW2ECoyo6~>Se+SI4HaPG?6@E5Ifwu|cVU7!Qb*7z zTsHv3h>|ivyuClwP>Wvacem~FtoIvY9MvbhoQw)7R$UranJ7d?_;VQQ^LfWL=)?VD zUz@)v0+D-uR$2=CGFi7;ZDKN0#D`ZlrD-!;TZcZHt%!P(FE51aG_53l^jl=xzDbug z#mDzAC|5bRrLEd7S_k#{eeOOgmU-KVjlS`a7>`CCwA6P02|#VNHI!G#wl9XrgI$&M zI>)`(tzAs`#9-b9Q*xA-OuA^BMV>m3R~*P?^MRl>vX)c1Gd#!w(zw8_%3sfyM_FnG z)!A|ZT)M{6jkj+uE9a!P&TU5;q1K0X*3TTro7b1Z7Yxp5UtsU(ZWyTViEP$QnQXny zLALbD@@++6zYN~T8wr!F+?wV(Y{LAoH2RhNC)C#Y9F|RWGx@Dk^&}%OtvTcKsEw(( z=-!QW`zc1uPo~UKv;9VJ=DT=kg9-&j``qddwo!6?MhJhN@n?9oDYtnI+$~f4tZB;- zZTX;b(CGeNu-i@ILvADW0XPu=8er8EhX1tlG7S0dv{3TbTTvp=QV6bSECh|DEmh*L;wCuI)lFxMa>-umgo57l;XN*RsD2-|_f zVn@SI?V?rcF>I;azsxHzOlIz2Fe7Yel}zJ#lS?63 z4N49_yC~wRX$wr1=h(FeuBGw?1{)he@ujB44Tjfg#k{RAIh=QPK^v*e6cuEg9kKoz zpFpYPaB@wFj!|Eu-KTPIr&Ac*C_BVpn6xRfzSevrX259(tHl=x=rvQTy4I}joCKLC zov|WyP^Ly~y*E3-6P&iM_CL16WK{vSLeaG#Qp|t^sVJg{6ulZ+34f81gfwxHR43iY zArs>XF(MO0{dQg;TYnGIyJk+RsTtFC7E{QDCsL(kaoym{9Hyn*??^#tIDiZWx6+Xe zS!;nz40+|yx2o9JGD3LDv}hWtT*PFUemc>MxzHGs)2y>WxJ)av?Vv8}joi-Cd)$T< z#1Jkm#IX<_m7hu1-82)~CRR@#qZ4>as69k~*Qn5!FZgfhn+e<5ysqp*(-q z5Mj@)?e%P7se_}hIm}l*nRA>Qg-KQEEJHe8q9pOoEwG@3XM5X=>U3ar%F1!zO{7>^ zZM%$lDTX<3m7^1V=)r07Jk}KU=j6Z!240qN-M_u79}7|^@F?Gt@Lt9#G`JV-;_%M8 zVGOl|mb8S1^8tz`lkLMoyp;oP;uQ80ED3oX2zfn7ihP|{X<)n$82aZA3O8r>I*+Fv zTVtBiOma`P?j*EcJd&mVuHOxhQ>}9D!*arUwYwv)E^k={$kyPGS}OpbBwO3Rdje6w zcQ5j9z}5GwUZLR%mk*7bXW>S_3~7_;1$Xur)*<}^NS@us>}&GqCji`wcxC)2Q&Be$ zr~^&Ma|z&y3yhH4fh7B%Nf@4;Vi+DiXatbWn%;JA zaN4ul+ac`D`i5GtdHr*fBx34~1jpNSPsmWsqI?zPye?OZ^_o~YwmY*JTS;42Nfq3;b z(=Q8HkT+oM6h6;H``ir{Zny0ZetV3yAryn?xc{oCdcX9wXE${pBM|iA$ny%dsC7AmCzq>wCM0@ z-p<&w9jP6ef>`a2=K=$mzI_sI{Qx5 z%1lsS=DR4L9T(6jj5MD{^qx_A$PG#Ub<+O8Q$vOAUXvScRiiLNQX@C>4OBx=SF03N z2cnu`sZBC|Y%qJhv9kXv(X-~QI6o-M<-t>Oy5j^_u1od`*LslK{J$^}-U*C&Vgy`V Js(NY!{{aN{N~r(< delta 5386 zcmaiXRZ!G{qqPDOD<$3C-Hmh#EFrZZ!qO>?FzE;Y3aWI z@0+~ZKn*v#M9dF?6> zX6lI6=CmE0+&0-LLvMRb{`6o;giKUNZGZbRpxytX^Ib*YN7gz+JXlmNpN7*n1W!CE zamuHFf09R^@j9WSLlNh=>+4t7rv@*918|cg&x!zePS?~SJ^fTbPs#e#-_v0xwt%Vu z;6~!ZoN4(q-2SPeR9U}GoupX{-trpalpdwXYm5VXWe^OOr}pV#n8r1L@3dL!Yuqo24tl@vCZ!`WN{Bf2=%rfH8)V_Qi3M>%1$i{O z*H;MI@CQ{QJAuMTG7b=7L_3 z7+c}yrt;yI{A6xk-siS&4Dmo~$ZXlhy>DB2t`<#I^_U*m0hqb8(6VL8S2{l^iHLq~ zs&-GKzRNq$l~T&^In?3QMoh3)2KPK%GWgOND?<-$=QrvaU<8IJ6yQjwv=53nYShS8 zVc}P-(6E1A`^Rzr7R2!-xNbsX;NY!8cfkQ7mf;l;rT8Bq z9}=ZYQ`a;|CL?hZB&H_n*f|aYm+6g|4+Scuw^mIZ+2RQIJ4IJ-tMbSL3$hsQ1Rd=X z{%GlG7Wi`q02weFWRdle@05#=T_%2U>fc5US8Ty#m)WYKO8)HgI75 zMI`@U1`I*nuWx=eYZZO_u3SiZ^TCP)AE{hR%Xh9{0i1a9RJdI8E@d$A$DT387o|F> z)L9bqvm~M7kzgI_NhG)9z-Oy?6XwaJ1kOr~LMbN5X-fuYWuVpFP#tUWbQyi0#)pa3 zv~<%QZ0=n^bIah9`sV#}RE;fpj>3w^M|Q+NMlC=Ww)p=n_dqV=!Tri@FogscyDy3P z+ahwA;7pY31nF`B{WymdFf{X?_2@W7J$@?X(g z>6PA-Nq=;nDpm1U;0;!5kkh_jrORIV12oEA3NWnU-vQIiwOt=ICgaNH?EOr$Kx;Gy z22*hu{N=tZR;t13$;X#v$MwpLmiin4fWW7PW%3&&`Uz6-563zYP&roSX3Hose;0U% z-UN*-b;DC(OcX)<3|po!=CGnxyeJt{kcZ+K8#_~W=*z^_M$8Tf!a65k0IyO!A$Ldv zOj4e;uk@vgV;!GF05qe87g^8I0om{%FPFu6-${*+=NNB*7bDs|^kk1}o?H$fSnU_E zemP}S5G*!?kLLhdF0ygZYzdgs>C)Ng6?+3$>QY-T6+c{t>x26PI*mkR<;ZW;igK~- zb`wTK&Q$$Xz$C7*Jk(P<1=`rTu~^_JLg9Vz)PmjqmWE_jPyG`Aqn&>SciV-h{bPOP zLx1{8V%J=r5Kl=N7Y=(qMlqkFD;(SX?#G&?8Frt@2?J(zo3!L7Mg@v{YYPYc#l^lY zx;U?df2#l{p{xhOmPH%*?{IHhqGIJ_gm6VjxHH=>7sTOclGZ=3)AO8!wcV3ELMcVr zj^}I7ULiYM%qY-*)ULJ{Tr^sx#L@n~u18fYUd2APQBp7bjLvl)-o6@BOv4ewoOj&81$RrHHr@Z}O-j~wkEz6mJcowrg|LIws1PZ_)GHdIyqO1seM;G;3& zaM-i4e>#7&+PLM~XLN4VG%h!SbILX0C1FSssQo*i%n{FNLyKS0XpE#T+#Uq!d_@S^ z)CtnI1;B<8B-z4RVD6Ao8YGRKb$|3O_ic99M7Y^=QjNU#DRiNrja{AivgABfG>x=o z(i8g>R-uAO?`O(~G{yShM(l#zu2N1r*9>Ln;gv*GRSj6y`=DM*M3l)c*-7#V2 zTYTdQeFuG4f;`Xw{;CRA*Fb)cA}oReGtc3WV-a@i)zac|4}O`st{kCoIeQ9oF!)Khw9&gwZ|1!Q)9{f zbeE!)7+5BJ7f!RVleEIG)@?{9<2~?xXSZj3CAk7dLF(Jp8R&sFtVzFE8cnuS9ub- zu(zqs`E{(aR+#Aea(Veh{R^@-QQ0^zxrM-f?Qz2Hu;e#yIur>}PHfrDu*!)2^*Io% zsn~hXhV8Tx+|sZg(s#=lh5-e_(pDL#yI2_^HG4F0XkT0&FwILR_|vj#Wfeq|f=xJg z>VFXNgEhUfDHlg7ZDTB%tp>No*dmx=D{aYMw;?VVCI4!BA(6E)c$By3b$Z)l3~79X zlu0IyY@*7HOSI3ZE8LQX|NXu)Nj8bLBvz!DDuH=h z^V!4>7Li zMFE%3LG~#>svwmdjIKlbHbz%wlZ&Ok^16_*Ij*5wj)_Pxy^k04sd3f17iPk6SbNzdM|1chOSocHi!odKvxAVxMvqk zvZXec_AMP^yKvMB$&LF9wF`~H760$IEn&1(# zUiI+K+z79j^Ik3NqnI8&Yh^PXp1B%|6;gs&^IgB=QRPpRvD9-!ox)%6zVW|cC-fmn zUbhlbRhp1cEDm=8R0*|hG-%8`;k8SmSFH4r>&&~24#yUFZUXI${1_YRHE8^9FTSWn zhJhF4-QmpSW1V!7-7K;_}I&90zju-;YrLx6e;c&O?MBFxHif$zkRTf^E97 zJ=-wE$95nVbA0nXBVmS=O!;8^Fm{*_5BE9T$-Y4#CSRet+$rubeKsWfREHx?2PuQQp_1{WER2++1y%kF?5IO=7SlaLM~?fFMxp zH;-;ZT1jLX>YBs+D%Ei+Y~W?puyH$2!?01Og~D4eC5a$C%AR~4%8G6y5SC%l-fOB+ zps`rY{DaCc%+~5BDOaV3VSK8*L~C)^mmr<|!kUSi7gWoHu%3>~@rM2o-_X;X`%tFF z)Fr%!fRw3244hD3Dx7;Ds$@)wwl3%R5Pk}5yfZx7KWNFI*BBFH-*7YB7PhM%G)NA! zqpB`bDTAoku@pP+PGDwl*qRCz1bF>`V0hfWq;_On)( zkE6(r-Jx3c8PE5Loen60u}N~#cG+cin`z#kb{~`oN~emzTNlOVcP7KrS2_8aC@Ij! zXGcV37K86Z&gYn*`h0A10#eUO8o}nnEO+PI2xp-gDB3aUL=KHHAqX%rM#*0Qh-cgV z_zBjSy*vxy-yBCq~9my{z&adaRnhFHYc=C>C}y z_7pc?#Gfx?I6;5EFiXyH`8s0(LQDa>?XThW)?XSFo5C9^OJzJhrB>!txwEW@lt=Pi z@Ff%5Y^sty!T0<)m^q9YkVw~SiEn{o!um($1){!V&NC5qJBGrk*hfG? zswxhfVaapl=ur7hlWH#~*Jm@8&n&0(a1&G;dWyoUpS3jg;aPNI(X)^NKQle5_u8lrlA6n)n^H{(!zN`y2aWmARG$v#k;6?*fz>LDx1diy~ArMwlP7Gfus z21vUKizmsm>L>$Cp$sB+1Y!MGuKYfC?stJA)u`eO9#{s&nYg+ni$bM&(%*H`@=CTTU=8hR0~m44Bh%yPusbNDIs zGJS_p@NGxd>~>{{Vg8(~;>jlr5O7mNpkJKY9Ih;aB8+K>fMUtMj+BwV$zxpF_uuH2 z4@Zn8Agbo-tvArtB>&-y``2nvcvR4R@F`_7UC`<-Y`$}Ul1_GAkCJu^OfRb(z#yxQ zR@(S8hZp;T`xcG>rDrDZH3Y9UB>nAD2zPbz4kp{?8(m@ z+U3me*+0*}x$zZeUvhTn0*imo4Cw6nOJV7WQkQ&{uFBVOUG`!7mQLZMOu9;@=AT00 z9LBJT5$%Pu3nRKlFsQgGF4y?$t>Ad`Vm_iakzWZaB5Z31g-T1=h)LPnI@k-_h=|xp z*hz{(rER64qA%s_{tpC#31$pP8!8(RtFPZ~MoZ;yMB}nVa@}}hmvN3%wMVPkl9XFM zO~6ZzP;bJ)3|PY$(m{D+I`qB=*Y}&2RUkZeEXzo}@ES1K&7uY7BKYTGKy>X4bREpu ziUcF;uLTb2wx@3Dc$6hAJ7S=3(Zqb**G{%r9M(?^g`PYAQY0EuqWBHi5XV|m1u+w{ z3W03l;ags8MAcL>4B3qBk{{5&CB*)MV@XRI_K zm$NV?zk@t04>3EafYkEvwxB`eZ4lJg6|{`WeV@ylkHHn9;%z92DL#ug<<~!#U+>K@ dPK5;=5*f^L{QrxA@cBT8D*|F?R|Bhq{tGhwRTcmM diff --git a/ps/ps5.pdf b/ps/ps5.pdf index e52f1b817db91bed674ccd298c67d6b06addb1e2..f34cf862286990b681feaa7baf6086324751417e 100644 GIT binary patch delta 5946 zcmai$Ra6@amxYTIFH%C$;#M@cI|;7E8r%gR1#KxMY zjm`%iBy;3TJaOw)K4EG|K>YjguU~PnMQd{9--ISD8gfm#uUVwjkKhd7!byl6?znS- z%I$(%>0;a(r10!o4o-1S?E;c1=E-0?rIb|pQ~7r3icmWZc>#S6%|XY;MISoREKPoq!Xep?6d&;3V_Zk-+760q#?z@ef;=ZRU>&(+Nf zDktHU1r!HJIigEFE?C_KPwXs7-N`2^!`LMch*<1>k6=W3_tXsZrigvI%4zXWX4;>^ z?3A{T3Q()~RK`D_xsIZ`3S``6GLhkxiy8X8TLC6fpt^g~&$o*0b6XLIpp)YIRd-pT zPs9GN@AAeH(}Ux~gI}e{VP8m0;sa#KDRAZ&sT=f$@s6ojJyZx&Bs-2wG@tBPLwPTx zQBsKP639A2OMiV=UZo4J#~P>iiAvlRE7oq#g(fl6o;1zi^2pXC2>4Fq1NWkI6lk9NotP0(uCoscO)JSfRP9aLo5_E3aqLs zvZBuFc4A6kJCd3D@m9?yI0FBumv7^t`dg}RSr+wUCtqDJ^#PbYBBxa(%56{u(FsJI z8xEpxvu3Z{wG}g$0>%7t?G`$f8n_%fF@xmy*>wr%3rM|>fdMhX1WKq+-_d2Zj8*tA zSQ;KIODFrhdN$u%y*|OOkK?rspa?Wo&v(3fZBtT7ui;gH6U})*jmoW7W<1=kgs7>P zEg-%RHhGJeCaaB?=2tq`?IWC;<9Nru`jwh9{E9Xbkzyy;U_{7CA72%j#;r;|llH-O zG-EwdkM3hnKR>Y^iWU`Nm9NPd@w_;9kTqobU?zT>0lZH$ip=r|0bEHjUX&^z&du@QCj@v-Ou#HRht3~S%=eiEG=umsw$rCOmm2bv|BNP-X z!Xt7oPs{iCoRxH6G^Xvw$rEhb)}6i80tIHI0u8Aj=co>g51xuTuZx6EpLK-b!aTah zCZNr3and!bwji!H(}AU|{T1`;2H3;hRfj8V0z|BE>kn%S1Ux+*A_bd@FBH_f1H9K6 zx7Plg`L^C)R*N**NPZ*Zd*6G2Y<%JWj4Thll^Sf^2;6yS6Uk^_xTdS5axuQ$oak!J zw5u}+J+%0p>b%2ktaaaIryj(c5N>P!XJzw~YQV>*=2M%{P8wu4NQyc^zwexNbPEK_ z^Pa^40Rpc_ta$$s3sPl`U_ekw<#+H&-Zi*$s)?g?nG% z(OEL1v~&2$4oN8=LQ-Ttg+{z#%;8fM1?`QUNiO2~7i4#mgkrMy%7IgP#SM~l*xPkt zO1*Bdv>e%cwW8~&2$tUUhLCL1?R3I+eaKKzH+!Pa5;fDQ2%>oNMCttF4~#w+EJw}K zjjC69h$0JSq9!wDKE2CPFU-vgewXvXb*6x?h^5jqd7^-?goWe%mVBvtD#~10L!M|! zJx8NAy+aPXX9R62M?7zABt&tpC>MWD-Rxq_U5zU<|5;bZRe&lEspIxY>&?j zb6EBR;JjS7QWSt%E5c3ZmdXZ(5)#rE(ody6w|5=F|31&<{QJM%FxUzsSm0!Mvw7v0 z4qJ!uscZUFPV-tVOdn}oIBNlQB%tTx}Rxo!w=c(Zujnw<1yJ$aEyt}S^X zq7emHn{Q|50=K#3bj3`)SV%dy_dm=+`6NKU+x|$kkGqLc9d@UFOJLEMITs#>(`%9s zOWR`Z7TeV(+7Trm z5q!B2)>IBRs*{qPO_!3v$RdP59VVtMLK)lpQ;uXE_sC^cl+*chfRtFA0pZB0fy!;wU_du^!@iQ zb|cqbpf4>B>T!T~TdoL$hE8wUH`s*#OsarU zcD1agZ`y@+^!7t^0kx6G_#lrEKy!<-rJfmIp4AFqsfUASK?7v*}1wR*`QS>`WwBY|qOHE{S9}utOf!*^+TX_p`13)#5kTrYA zvzPi&n>stWIOvXl_t3aD*@we1X$PKXwNzWP+o5Y0Wq@2wvU9#yxA8{J?~}2Y#Z6TH z+o=30b~QORr$yqDh|V(o-cQmArXj=!Rl8j4>G*x(E;a3%xCrYmyB(khIIhhe^ZHUz zu2Hh8peC;M{vK(6EhshZ`In?N2ELa2mTKIlljVubjgZh2} zEt!`sKQOsV_91#QzfB1vf5yRp<88xPxjx}6K#sJ2ur)sp-G>a{PoL(mCPfpwkdFiUNfbIpO*JqN0iAhMRQalL* z4S!O4PAV~R#QW7zvvcTt|JqO9t74T?2 zCb*f&SvtPfX7(psl;BTaqAg^vw&P2{k5KlasEd6iZ^Eo#YYSWKz+Kxy);-j%>y}ds zE)-yA+3@~}M`Od}8M!n|7ng<7nDfUXtjiUW2ZEr(%2FaSOwXMq1iCBdl=ZoRc?uKp zpWQRz97;D~3RgA$h@oVRr%|KqH2PeLvk}NQmIjibw$NmpZ($V9?{tM>O4Gh)|1JOj zz=gF-{Wn<)T^R5`YqJsJ?79Me`yA9A{3B5K5+~{5S^GD%X4_*sW(M6G>Vkaxg`THL zGXo$m8u{`hQD4Y7`tD;zR2&v^XKZ{$vYn41>@X4(3|hTcj8x<$#U`=mqTgibvtrb> z%81wb{WT7dkZqB4k<`pP&AgReF81}!}O?pR`8rjgVctN+oYk=0Occ3_p#RN`lCF|J@jpzR@3UnOy!bc>u?B>aJ zML@CJngx^3;(Bng^~sr|#f54x$DtU_jNlQ5BA{&4G+AeCtc)ExxRMkLB@GLPZDUEq z4*V29MW;j75y{bSzt!XDT#I3W7m%oF+bSo_BTC4yIV?~319T{|F1pAA@D^5k3U6ggDrY$GfuezX^5A{cHWnI65BQzlkG^3CRID0H z&L6a@Rua<8ukjeM#}w3kE-0E%-cIBeou0e{`5C0?!cA6#ob0>>*Hg*~W1xP3{qzhh>selK}Es0O#@6i7xjyECc-n-1`*%91g8za=1K-G4j8@;y{R!Q z9vawPDinU%FSQiTS{ut06%(#zM#R{3IBkW@6t^g~#9m(69xc67!^rcds^#w_k10A< zlkw^-i!3@-v#!K)-o1^riz?@napo`}a(i+K-7^?jpY2z^E*8|2umn~54ax>i=tX+T z4etGYMcSjLg_^elo8L32kHaH&?8;a&xW)6TJz3hfT+3`ttGLEL#&x2X4ba?H94o*0 zc4k)9jo6=$020@FXJy1TZj9+TK~zYQI3-ub@oJ8T5W$4F#UKU?27kTIOtnMRFx`|X z`$@Xolu5Cs%7Fv!?&ok7QJLJtWoAXUo#TMq(>&$eVVQE+q`1sIO#yF?w7UYyBlVIG z+=ovQJm1SCp&IV?yPS^qty6Hnm^^q(ZMDXJcqWQpF3281oag;d+wpuz<1))AKG?9Q z^1VDro?UmVpDXKg(zX6bvINAZIh8r7-Ktq1^eRZ6 z)}3k0wX;c_e1nW)Y%_5Ym2)e-OekXNc%00l=8r|I6UV{TXRaI}{c@bH=>xXqZVM1~ z8)V!5dZF#MpEX+YDrXvY+j}2>B+-MfmVPr*NelK-NX982=o{1SP&8cU;=43Sl5tMU zOZty zEG+y?#5h8JPc;~q!Z7&!1?GOZlm*hF{@0c1%OjN|a=%UVXXpb>NnKqiXAxLd!#T6x zh*l1cDW(92bGe~=-DE;BU8sxW?QE-Uj5=wCaBUOjBkcmgYQ?#=#$-s)J_a+-grjV zthY)typR7~e7-Qk86eJ3=kcOZjy4rf55|oxPqu@^vigA`in-(f8`<5m zUV13JI+q$oB`z0OAGXGJX%|OjppR627_Xv98vDx9HmQuM z?dr#op_HDiB9|@OU&Y}0w9kG#^nd9Y+@-m*Nm02~eTcrk0&p43gHUv%82irg-5C0a zSUoqaK5}J`D(3f6H6;aB>cRULsk73qLB}6vVS`#JV*rFno#5M6`h} zqYX+*S|n`YeHRWU@*ENw3E$loWKZ5}5Ahi_TkDceEFNgoS9R~^*m_zMC?p80+gzd! zfg4%@n;d3k%bQnzG1RR>2;E@??wOYqxP!7O`t^F_W@fy(ENFkFLFupYbyxP6cIhS9 zg8;iX+RpE@r*?hrg~3P`^yn^3z=LAmS4mABOaP+!9}7R}Btz*QW9&TO{eeD_=Fu(s zsM(dDG>@UQyZKe-GtATn+&5lJhRx!|N%-eD5AM&RJaAXXk z{u0^v;b{ManER9t1e*!u#R60X(6A=ICAf)3tURCGj{Lg_P|Fa=kYN$BjTT6dLJ|ZT|ZYIp~iPv^T0ei>NrLPT`t)E0^^mA zn7c)HFtJo5Jb&@pZ0jDF|BZkq0gkfo-zP8ePt+O>^Y?`Kc)z)~l#N=C&%pc*tY$Y^ zDeIuVxPqT&adoMF4TmlOrR$>*ZSQ{F-pQ~6rFr(rnr_c_yfA18#g)WL%~DDAd%|uZ zC+akImw%zv$K&y47m>@O@ex)%c{+3}eW_;6NiotB42qU?pd#TI_Fb*ftQkHvsX4+o zr{S=ngLN#cks0>niU(<%n#F{Ab_SK+2Rm_GTf9we5*3Y2oM}F(C#iZ)eJ%Dy;RK0!h;(VZ2lw`GD zW&8L{Ls=`H0q=nW;D-eiG9n@t0ygH>JQn=?g60C2B7A(-f>!3{f`UA}JVGKsNpX?? zX95mJD8PjNla4v3TD60D0MeUAt3IshSbq9r5ubY(wW6E{_<1|dv#5yp#OsI1HgF5IvlG(#DH+AqLx z(GX%D6Js2X5EXHkHODU`EukLRF!4hf_WnF{{kL)ZGL*5>1ggCNRXIXYT68`juE!*< z$(anxm5dbL4_dq)pywBwLAqDOzrJXZp$U1)k-amcrmw+?Ax!;SlA}p)q10ulZLV_v zGx@IWw`|&DoTLC}n10op4}zF!sasX^#VJ%(!70xTTiQ2{?-lQ4997tEAsj>6cT>uW z0;OJTa|%_3vzatrbd6$F&hEJl)dP3)%1UeN(k4N1cbMpooUHJsLRu^y9401^1{mkR E05`{OJpcdz delta 5953 zcmai2RZtuNk1mB4Sln5(xGwInxR#>DrMR=WQ@XhO;##b@yA+BQcU|1wr8vFb%{|=z zbPt)yBzel@%a^-H16 zMfa0vwsf%@a+IqsvIp2P9c zPVaIgJGHo6cU&ihFh`4ND)fM9GSXTpDOtK&x?Mao*jh!Jm!DPj_glzeNOV#??Id^q zd>&k1yAvXI{?yXHv-13aUz5b4C5BLR*~xELF#*`eVhoSUh7s~BK#lBzIPm+|U+}$G zi2J+d2&5MqxJ*`MA@DCVUha}HR7X->FFbOOHU0rcXRT{6U$bHvtw9g-KdEOj3N+>V zx;7+_3TUQ8wt6>MLRRhEL$ zpX==@c2Fud!Lw$Bu(T6lw=NPp&ehj}2HVr2=?06XJCi7d#MfGrChn~L>xVq^UQrf!b;Wub9R-8>U z`^FB#AZ-mFS2Zp34rdsTa>*HqC^36rtX{bHGLdbX=0`0XNL z8(%yvg})JIcKaIGq)Brx(y`!tgvl+Yv->1bbElMIeyvt!RcfP%sS(Ya-v-}9OWsX# zeNr*7TU{PAsEES$mYqV`VnOc}!2d%oiVBRf#Yne=3nG$^3O8fP-wkClTjJdPBsh~} zO~6H-=GDW^{2>xCpLZh4yC-A6pYoHUp5K!5*FMGY zo_}wY6Y0QZC`Pz?w}n6Ic6)Jrx1H;V1&vcVEwxUBk9`*SvL5-mx9goL@HqsrICs6> zSpCwW9x7(kW3xb}SFiIaIvvyG_F=j!lfAsm55s7nklZO6TyQgvy8Of#PHt6SL64VSF8pM<(6K9X zpbhzugw605fW=9K)`|F5*8Ikzpy*&JDB8 zQQKy(OU*7tI3?wcZc(jEdO1-;Ma(-qqr#{@;8Jv;U@^JZddVjv%rU`1WtC^rjYOqV zN!fb(LL+%IQIj@@tYOOR>ME)dVdK%whm+}3c*;UU`~Tvoa$#OLW!x6x?)0FNCbyJ= z`7hFyAYg|`{OSneaEC~^hT+kn8l<#Xu8}BjX-4sj-n+J-xA9+y^LHE1 z!x9MPAY>`q2IZL-oH6Z9qy<~&l!a7^$FLj`Ap}ksT!b2WY8<|-&XBrhZgUDSRjAkv z`s-KZ7fQxlvpID#+QD64Ak(UU-sYhZWz<=X<)BgAWY^LDhu=up-EF@qDp7+M0tSTq zeN@Mo%3sm$jXT0_i$$7mc8VwqKvtU&T?LFZY%F&K7w7Mw#_7#PhiK zMW}70O86FP+Zvsu_sQYZ{l@dQ#`dw`iLdMFkaU0ftb#&xSjx+_1*5mPy~RPpSp%QS zAh4~?amBsFcE9}WB;P^=9HQq?TpVt&@NBgJbQt*j(tLG&xnm(jG2pF^#Eb}J9Q%$x z9bk`Ic`@=KpO`BD-_%5bu#)KRZ5aFMleU{fS^1!dU=tW@BEAnFM2HKLOnhNdr9l2T z6tDQlD*W+^XwVo>Qc?L-P%k>(JT)!R&>92sp`p-CVFtB6WHNEeTWQ}NeTs-BcswbR zN9eQdB%|U`@sugu-g{ENDeI}x`N@Adt{CMYE*4sD>d1tXS>wT~^hgs~t#CFA+;(FE^g#sKE2u2PlcrOqhr!9DxM95UPonI79fL-`2J0QU^UU_XI*Z=T zOXGfi*rDIje)gPG#{0aXqAc&=qyaNQ0sWDbrG3Yi_7m0x7k9@`l`B(m6Au@|4Ey|W{p>AL z9`G@-!cNgKu(CQe()?E~T7Rl8O`&Ji-I$0TnJt{@zbRaleW`$#yJD>cYI`}f+lekm zV~$mB2fzQsZR6LAP!vMT3}Bb*IZxXXupORMg_|WYDB{O!E8^pPd_4!F8A1lWl{(KA z8kE6kX+jEaz?1Z<1lGqku=h*eO9|r1#V2z!Bl8m--FCk7{Q}LclI2vo9vc_w-}@#!se%n%>?&!7Q{yYHo-BLcl#r~FGAh1} zH|*V+Y6of=^|;;R)JEYQ6jB(wS1$)Qz#3IFO29jXWzzzWOEIJY1r}uFGl+>?SZWg9 zC~7#fTSibdcQBNLXfbI%z;MSca}kZged=_XH8%^bs~ARWb=rd|jieh>rlS5&C>Mif!8^S<4$t+!TO49( ztLs&oE=U0{pIr^nCEGSLnnrebX0-&q92bk;gtIfAn>KHZc1dzps zdQL1dHNkL&9OU*LVB!-B%!CZ|88MOd4@^qyVF;D^@*M0})|RuLetUuk0t<`w*(lf1 z3KCLG&1?px5MVsUvYE|0bgv_u&dF0EkE~}IzX}GB^Z!L~V zYZg2m4!7Eu0W9D|Aw{*^CSWI+v*9vN8%X!ag$~J=4RVo}4swz*Q3&=^@6tG06e{%z zYA2}S5P3n|`iceyX{u@r0;?M0@MACwVp{fy$z=y8RiWkNK~3SL<1`=+W*nP{gdM3W zi<IRg zasMK8a(W+BlTbj;(SjE{a8|2=#vuF&Jg?tES|_l=Rut{Ts@FR}%=$pN#y~V(1$@A3 zPo5-J2C@YSNy+sOk-D)>9+5|EJ0T);%MxlChb2Jgkg$yh^!z2#27UZL0f^;s)8eu& z8tfL|%ZvD?bhc*_9ZKkGDfu`*<>kT4Z62(AbU^g|)Moz7BQvRoHa>o#&DU( zkq0KfKUJBQ|6upSo!j=GkuMB|nO<6PvYaKFx@5+dXFfoi=ap*Z zp^jxPOB!hLTl*6;*QP7xlIhYMf!PXV%5T2OJ3UU<^$Ekops!d5qGgtITc)BUL@m^; zLMkd{b=^;RSI{)Y77-C^0YQ<;ghrJdGGNXiO#s%?I$PPL%#``z@;2HT~tz$HyJqTl_Hb zbhmr(!|U&B?glGEfvzUu({=k^8}r@qrKynPQ8IgxIx5YmZ?^R>qy;)oOl_EV^mBFA z!=zbzn+@`m0J?&cp9?6X<}ZK4Xtn4#kv8Qo5nKONE-;&@(m?-1RB!C&+>PZDbQZqm z%)8cQEgMC@CJc$p3?30sG1?yGv*vo!xE)?}IHto2aRihdE6K3>IWn8cSPOSqNh@k( zquN5c6a6pR3G>b9d_Uu{%w;a=+(r}B9s$vA!TMD57{G#UFj8PK{}oO1N*Yr=WDj-vFLv@FQlxO%#@*oW%)U$st~$cN8xA0}C!suaP`(Bg z_Z%A-!SCGrgGz;A8vRs}H2X~>;_Mg%tW)lnfAYPVV01}RL%KnoPT9WRn^YH?zzQ!2 z@szW{ZB_D6bdSC|_@K*D7%aiX+$sGCNr7DN`_bKwnTiZ~%-$~}9#yqgIf=eb5@*` zm+Az}lD(C(carG@+4Lj>**}I-8^Xi%*4QFdG&ZXmvOW>H&-_&Qr>TlCw0g_te84 zEnI}z2z_%Ia(_UthXyhyrYL;_QM*~KpncAEY&D)Dwvj)#&m|JGsHnV7lUA%wjP8k= zg?+i%hlAeCvXx|i$`tWBtH{7VWOu*ZYmxv1mGgrt}N+QUw|M0bac><=hRdt=pSLuyh=@yZ2DL67D@g9}=H&CXv=p-YG2U zOr@?x+z7vPlPL5`Kae>#4oBa#a!S)~#%8Y6j*=W;oBVVQkQE{uvf6}9;_WqMK1J`i zoEdkFwQg+sQ8bw$p<`pSDb)V$ zOeb|j9v~PmAiP;9EblhU@HAf_&={+zN`#KQZJtm{*LL&kWZ923B;H_LW03|`Pj~IG zA7wu(x)0&!Vh6UDNl(--r0-7Mb>nCyA8CQHp8ljjOu4)#`b?Ki4OsxeUQNu2zEGp^ z8>Mx~#vn>LPcXk5^gTHhw&vY@^`g7+VB^_!<$a6+CkY;LOxJ|Gs1dm73Sta6@Mh%U z=-KA+TEnz0_XzY5j&#tFdaM6!?b3NJrjjsFH0G3`P*Fh}Tb|z2?T&<-%hTJW$hwtk zun8^10;uJWqusOf!2I#^LetTYIORASWuz2}*duDy3AQ+8DD?TIUE6AZZmeX-<8Mbx zTXqa#-#2IN0ww{Fm==|tfzq+1bF(K$sVKhlW9X6#&x~&|-cGoUE0NU9*I3*fE)=OR z^bd?6Nn}IN!7&-Ycpz0npeXf5W~vQWN3VTP1?z}Y!R6EE0Nk8A9LpKB#T&6CmGHpp z)OWlW0s9*>(79c=X#>P+zHp(!uk~?@6sKq!vt)Ci4DEX%u*^+XEY@(jE>7)@IbWM( zMaYV9VXPHOvxO$n#Nw4oeN5N>>Nul0mT#PZ@{jtr3$hKZ_-CxfrAu{(b2#8D`#-5{ z>(KO9)!1@WKb%?Lg*H2`KvaSv#~J!f&pGy8{FQgZ&^315T5?8)uFeh-un~|Z2b=Mc zLiL0u!1PgEja)T)?M>t%f{eqPceS|UCO31{)ZZ5!QIjOjZ~aaA{I@*b49DLX25eEh zkvYXoqEwz(|Jiv0bk!N7SMkT~iol)ddQZow5Nd3SZR^8P8*02oQw8z^(jyyFM{i_c znD+Z2Kv|Z_Ti!=n`F;1QMO1ByTqB^%0{cg-k<%krvq8a65DS2Bo7dJZxbUDlk|9WZ zYWOaIn(1<*LZI!`kwWGD&xN3fCI1LG0=?9ThBJyjGgJU?$ZQ z@b`N;gm*F*B#e^b5OJ!UFCjgT;!`3^sbre1&_W~Ff#sb|=birDM@K5JC*5REGUtW9Sz#A7a>Z#X;c($wTc5niTD+#9Ul=Q2=i02~HuwO8I*nU2V z1Rn>lASaiJ8JCI4KTm*{gUigE(~N_U&&-sM+uTG{MDYKqfJqj}LHYi#ETWriRhE>% z_umw3K?Izkwd64}N0J7H+gB~DE?>SOOd+IC`kfJKT*%+)Um4LEkenQ6yidCEEI5`5 z-Q(dNAfB=J`7~M4OTsE4eA0u7{{h)oHCSp989DZs{B|$#5xR08(hXw`I>n{$ZPh!i z4cu`$=yOGomb^cjHm=IScZ|?HQhH!2?fTLJjVKnyfP}5F| zELZDG7HOrDb8f@OtydemBC=_r_^Y4uDH>-Dv7d{s3X&6TrH?hru=H$7-!-xodXzVU zZsjWNN}+ZZg(gNH*t>o|BcgMGME^ram^+%<{`*;)xVU+_SXcshIQcnv0d#aSDzbq8 E0`(|dQvd(} diff --git a/ps/ps6.pdf b/ps/ps6.pdf index 18379d89446b9ffef36b576244b766dbcb4726df..75ba3ba3f5100a4bb1d0075aaa83ebc917b4f187 100644 GIT binary patch delta 5018 zcmai2S2UarphQ@#USkn`)g`Pht3+A7MO~tX=skM*qKg)+7SVer5xqy$C4%Vj>p^tU z>;3OJ_vL@OFLUN$p61NV!;EF%3}@if@3Eja*~tJ+r#?aAsKYB7DW7;sE|)11`FwS5 z+?imJeOvKaL%6~#1}@3d<)=+*WK?rULv`jhJOaCwGma6L7|6s5LCxgkswF2ExxgIH z6DW@gbl_Q}>FF<|hyTiX_yxhS|An66^7vtN(N3#Y6NcSqDy@C<>Wel!X)fjZO#{JB z&Mr{dO2U5L=t{ETk4rvo>qxgmLi@}?;Bvz&PpeR`Exfy(#%YbF*zA!RjBXA>9o0qJQJKht|QAi0@y2Y zTqp~rDO#P4^W*UrQtXf;D9YrZj0kLLasuR;(!bc8b7Tq=Mc0v zM!#G2_39BF5SM~xWRnEok}@)2uQ#5yRxV(_1+x@gr*J{)$mKg4|2;Y-w8_3)gccvc zu2^O8+^ejYIXY%{73!!6i z-h|gN^$`SD&}*iAWB-=tvLv{GKhIdw)r;{ww}sAWHQIQM@%p~d9Qaj$FPhS$u6H3B zdmNE2NGH(aIUSaP`&Ui%#VFsI+%VuBAh%dU(fv!nGi&UGCC0vu9P{K8@y-S}aWYt+ zZJ%zEiAUacgZrhClbgz+iX`RzQ-tJ#UNw!%_M%n#9d<)L9L(5Hyc09>gt9bINWFts zNIibodKB*V{c+3hYBS@7FQXTd=0d1v;)`ZD;sYh?IbmDULJKf~i(71vY(Xlzid_{@ zkNYY}K6rRX?ezv#OxAcH&tu;%kFqe1>>RN9L+eaA8E~}d_Dm14Cx_GRzr`%ecOpOg z;VS@hh}O;ijgK!p;fIxo z*~j)Q^Y0%8L6J0Nxp}m#^sPe}yG1KNZ+k>8#SiPg;_#Qx0Ul>QXhSSr??Ghv+|3|4qb7@*ENf0`apKHcpnk#5F!#rG?TY3%xx zC8j9VUP&a6VvH@3Kjt^H24pIW$8O|9R}3xdRMa~z7*3%Ci^)kxCKt@#{C1c_q-=Wd zNwhNYWOr%?*;WA0HlVWc{JqKgE>X=V1)3b$-IxS^G%ZvL5c*FBQ+4Ck{x^d+d?ykZ z_?(DlAq!ra0r`xFUG^1rV}h%dvwAhz{f&zkDXGTuRdr7*KOw6-0_m_R#Zf2JOpTngRz}fcp z_TYPS-J~oK;#^G8_-yrbu+x`~)h({?PO1gdB<*e88Xu9FU^V7_&Va6_PlAU&$Z^V#u z$NG1CcW+*H#SgAaq@nT=N!U<;@SHN~E0aT{G~QSKs0h=aV0~Qr$B$|#GQz}S9`U@B zCXx41koOVAhh&Se#G#8NwcxJU)}YEJmP$@th_DWXJK>lVFGlJy%O8#smX1Jh&Tl;2 z9G+~If~lnPNXUw18>siWLYP)$5WUGL2$vsxHg%*i$im&gks$u{#$g-K@Z(B_4wV&* zo@mXaU#+Adwdq=HRCP)xs6ZpIcheD1K_=9H?2^oitU+fD96z;>X}R@A@onk7#g9f) z%BdhQ?|;$EvI4C*zLFBDMUnwSG;owRmABi@_Byp&12O?WX_xupEr7aw_LB+eOAZteYwbk;Op8n+> zKIj!3iuUJH1vKA(6eREb%NO*k(xg8xNJY2UGMw6>xh88#-3>PzHVGnUjlR8E5O&Uf z!6A3(|NAOo@LcV1wAP91^R?*t**{tvb3SACPw%{|`6cJ{7M0@Br8Q-c-ewsig)ftJ z(?>;8rk?|ZTCa+_yY=-JJI@oIw9h(T``x=}&yG6=g#Z8_N`pS|TXyrJC1Kw2_dU0a z;auOPPh7j+ibEqMjTsId$z?86%zNVFcZ@lXi#bD6?f*P#=qBwelU{UX6IpPUODq^( zK(!DXxC~Z4o440oYGB=+l>Rv6GiLqLYG&0K&}Ub)6Ucgz>x$(WqEy`>LF+E*aailzo z#_>sS-`wW2#iI1Ec&EcliyOdr`T;a(u&v1V!tz#O%5b7`P35PN#mdY?rJ+Ac!Wj>LIhDAqa&D-&!rFFBGUd2PAqd?V2ZmEi=O>--|X>T zrO@_SoWH3NISn5VyLf8hC{m3c=aB-0|1%>xnr`8O#GPA4mvau41RA@9teX01bI6P5 zl6!QNeQaqQ@YSAfW>sCXh7FYv|M7FL7lwl)_GH6csQLV8a|R`e`41fO(cxmSF*%2w zX5KDO>O`n5|1VTLf$Giq*|k?lP@R*L^T@@r#Hi8pn{!V!fGoehJNapx!cbG2lo)2i zqgum_bsQr1dE|7%A#^G3^Aj1|y>RLju}6CgpWmqnKlAOUNWlLd%hsgswy0X=?kUek zD9nPnqQ~1P69q>h?{1d$&7J~yc;|XepCx4x%I8t|W=Da|e0GK20SL8I8p9u;Ltn2M zJcChLO{8aO0|DQt!GBE{Lz%iQB56aGEri^hT&+9~GS~zym3LmvrWWe zp{O~eVjk{7>{L`!`P*nGu}t-IFp^%i%;VV}k#^at!x6+9^rI3}8ic*6HfpcvGkp>r zFtFKC``^X31A>Q=!eMx!uM+Y0KYU#g^aD60E;PL2jqO7xtRqO71bI1CAT~r+1!I=P zI41v=lz1&N!bU;NKYoQPF#kC>!8CSJxaZI;!-E$QbFrV#HmCoK&R(|mN{$_2vib{h_5M3(oSMr!Jn&%bi7<9l z#Mw{`+jy2QV>|tW%mhE5Lg5(cSl`J(ZKeTc!xBYYr7C&z7-qg}q36Bd$4H30Y+SCk zDeQY)LaDbXWsz!8+P?(Ag&G^3cG!eG1C=?Ejj%vr2){6mfBE;x{8xyI?_$oVJI%Y} z9pPAIGN`fXl%BJXY|igzF(Vf`K2qf)H%z=9kEhF->Sv#)J)iiYBuZ*;tP2Gck!JgN zi<&d_>b~g6y;j#%WSjXme3ZpRP66M0v6jnDbK7AS;uzs%gaAY+awesp`hMt$yFHm| zH2Zp%`iEoJGN9`tFSVKi7fLrvQ~W~}Q*EvQytl`jC}YwYeip%-MBH^kZ=u|6Y^d%i zYOPwD^HEil-+YO-xjEaZzqQ|9d2D>e$o;@QT*^C@Df~g|uBL-$D&z)Tye4D3qFzRn zOY9f-M=RrAiyt5#$#E8$kZ=}B$em)#bM$ZK?$V9>G~+3Sf>pidl4gb(iLDg24GN*W z%xEwtAwsZpji1@#WPfoA2m95!HpePZN}GICwFZ`AgC#-p9y~tG;!hE^rh~J;Fatc( z`W?n=e{LJ-Z~X&w<2l38CxOVt%e9H|q3P*PZjeE%F#yed@s@OITZ(4DOMpj{c2{R= zEo;^bCvsX7ocuYXG{c6@ppH=gi=kK(sOZM(lfvSb{}>wCBl!m5s5 zq2c%WyrkFNVa2#*wd5{kvKzN~m*N%D?b(kT)(Nso!?Lw15m9@Ej5IsZwjTZ#nq0z) zg64WZs{zVIN8bbi@Ch$OXs-@`+R`L(9nTuhuITDl+B%jqTD;mS{J{%;!8XOYVQy=9 zjW8C}3#MpY5mjKB)gT!ej-AL&uMu^Q@D=zh_wiHg zt%q0=r;9(6r3BAgz82;pJ^`<&Tylhwl$T#kelO=0=`x|;MD@+3m*H%266*9wu~1>7 zrl2gU#P(K~NRxxlX<{ffSBWJF}?wrcmdPdoWH)L`2ha3d*ks zd|}+4>0!Cfed)K(@D%zZv=^+0Kf}AMI5$&(?){-?C7(z04NH%)jIbapEc_e-yqASl z$6B<{ke=gda6<(MG(==bMaGL6uL|l2nYFqE=1v9F>JGd0KgDOCIcVbM<`SnX6^BUN z-?ly6XZ($;I&l@GIykDNTh_)WvhEHsF2tApxRid31JO>OURz!9(+}ZEIrS9crA5zQ zt~f{RTECFy_#u*46j+LhSfAklu39dmP3gMmr+ zS=c|fb|8OtZ2S-m+C&3WQ#+RNVMMKDEgI$pSmPtM$FX(yLP5KEMFYYB@7mG?d*ama z7dh8InB1=@{n6tmT6KD-Uw<*y%7if%bOV(%EPDTY6@NZNfc_mC^tc(Q$3Fp>w>N1f{oWtBe#mBmM?3k zdf|&xA{@YwFVTue-HERQv=tu`Q9s~vdjk0#t+(rc_Wd!LXiuqYLT;h=I%^@g(CwFP zmSI|TBvf{UoS}s4+vA#*k2ojQto*nPZ-tHLpY+h`?9uAnGRe*J1X!dJ+~aaO0+}zn zy8VDF9~bb$yE6`F_Wn5LO0FC^A*nB+h1+X{r-P=e_vZnVkWLan1L4;94I^zi2@Q5D z9^z7&xMAPy045)9uQR+lzP&z8HyITGP0!{?EdPaXY8-kIr4o-7*m66fJ}$~L!NfTW}K(t{V05$ z`Kku!e0uHnOM^{$GE3frmJfA zqBHUfPTKPz#$hPqXp*@IczhqNvjUzE3z1~zTgl951T|oZO>0FNTMvdBheM2`Xfneb z%H?rI=ORmam#qvNpOgoK3X%CT#3DfOM1={o?oq$C`BrRq2Wc90M{O!1@Ci@w-!|Rr zt!cxjCvAF%{kz6@iL>F?vgV&aOGiPO!Z-MLe^`dWwyrk+cOCu9+^ik{J(8?E-}rgj RqPycj1cG2FR7o2N{vYoGX7B(2 delta 5002 zcmai2MOc)9q6J2}5s>ad8XQ7Gq`MK2l`XLYFtm({;q3I-=>C%Y7-ii;yA zN3=2A^WSC`(SA|a41oTi`3Y9@TfKsjt}e--HdHR}KZs|99r(7sl9RD)HUr*`-HDV^ zNE+mNQakzY1kL^F$B#A*h^EHQ7daIZ)jn2!<~h~ym(qJhJN!*cDNJLwS5w&RV-)6v zA}#l@u|VHdWU`Q{K$q7Nh&w$gh592i7?b~Bdnu8-`}5Fd!ZWw6t@teyTBk3p?rc?- zpLvP`iY$t>pbaYBJoMDuj>(WFV_}uzuVj3TPG8vD3%nV#+5@N-NX|p!%|^y<$5F+- z-=p!lh?Ck)Y$JFPGD2yvd^jVU1RyN*U%aJQx<&|5_N_cnemJC2vaRxGk*}a)ICz{% z`;JOCx5(=!yUbq+&EyR+mhBO|W|00v|B}B>elYB3Zpj9is3=XHX18_nJqwkHzQoLP z`7>+QyKVmI#4JR5s3Skw_1)jhkcU^DIb#;Xl$UdkPW|+N@WBDNl$Bh!5bZrDdmja> zY`X?EM3qZXph#Gc?cfhq$99d+EUK2iR8>Vo?o(hLkPlsY$~c*>pYlwW2^ARoBAVgt!=iH0M)DGxZ{h%x^ZTHqWn^L1-lxs2<;Xk zi$1xd^x3OZC3RVf(w{K;-MtTOBqf7?GpkioJ&5w`JkmV1!KJv_&Xx%^+KloD`l~ZG zK=HARzKX$rdyS);$}{1r8_$6KE^arisuq#U7%2^q|5GW4n5L z)}FaPuXcN8c$!V|>9Y^Fhe2)tp#{Av1!CcIA;AjmAVksmL)sV-%S#okTi1Cg`M$CT~GwcnS1yYOo{#`7aTs>bWioQAVvi@LtTnfbtKg zP~e5lB+hlUB$h5myJHb>*ylopPp)HqWPkDUwAL9NaVJ*Oj_$v0 zl8TE-3T8ICnzO#~b(~9Ji~Du(J}0uR#q*DSnafVa@n(S`MB4Q`PpKngbM>v~3MZZv z!$Iel+@#^sy}%SZZ$>UsJ3!VM+D=(itFmCF6ZRg)x<=iINflPoCu`PgpGV=v0{&hq zH{hW*tpnk)5ShoaNxL#o+ zCwA@`qyo+tkzpEXPweTWt@B1wjJ0lT@|5yJaej;PIOxh~$UjhehC%v*bxZUDK;wLI zlF3N@=W&vg*lCh#7eoRkyEE?xyM;BvU0^L{{)}x+^jACWD(7G) z?zkfJPx5VihAbHo{19DCNS-|n#{JZep3r#h9bQGvvJMDL)FM>A{cXzs_%&fnXnJAy z3G-(Vgcu(9{6E(6C=?<>0hw*npA0#jn-~jf!p@f8 zyuW3#f?+U-JNkKyhf;UT0IE13zs8qB9l#k`oUHkWcJOmDf_y*`fu9_=L;*2n zQ-27Ik3Z9l*4)a#N*_Ss&BQ632w-vMzlO2117Cf>q{p>GGjo)KD9(C&I5;3`-VzPq z^w(}R!4yauOIM$B=aq(I4JpQ(5$eT02y{7)Y0$u|u+N|Tu)|9{A2ICP5!g12<*8rBSd(l`>w^Y0qu8hcjsTIzwQqr2uq$KZ*sGB zwptwh{$*rhUfGM}_PTy)qUQ-@_f}d>N6nDgwTgj**-jBgf>R7$bEBrflTK8x=fnLL zxmbP!jWbI#ApYSQEU((p0U#_}?&eBfEZFTmK1FSHJFn#E0KYLgNZy`(XvDiUnnU^L zFlsk45m91;RNFL16OHDj)uuNrx|Y@0xTU+RD;8oKV2#hRb){5IO#1zQ`$<`IjHMBB zRIAu2?H5L_Zx-5>ZFTCT6OmIAgBn==n@1Sx>E9>6S&AxDw6w-9KJ|a; z>ZaRw4|;dujoDO>!#iRKi2h$1Mor9g=+&r0S1k2BHBRsXmIQetpRhi0Na@1(Z61V1 zey2yJc$xV>1^)}}e6!TFxa#m7qAEqFV=?y4=K(##tsK~&k*Yb+RfLQe9!(|FR4WlF z33J{_ZyNC)jd##Cbs#!N2(ufLiEkTEAh?N0A{>EmpK%dXY!o#Ozw+)XW}!`Zs$3Hp z^#V~pOCGkEDw5@$s;ZbB9EBHJhf5b3uFEJS$rTekicAV2wq@0Ksz@(*_(UF*5D60Z zw5kn%y}|!x!ZFI0Ac!xOj?Vf=7z6z({igW5vz5+j_FCX4`dHXzk+yXM-K3R%mk$>; zrV5ZgL*~LAYvs2Jk%K5|OXkzwe->bIsRbDcF=((=<@8;^8*t=7Xy@SQCzy!(EZ4*2>f!zwq5U)4apYQr*TaE8pO$UAtL=sQ*dW% zwWX}?;%rl-ojcTE@74k^Xe*D|2ofJBCH;4rikZ}S&m4zH@Nhhzd@FQaQJ$_d4PUAr~?cHq&>594qn$9Wiy&a)!&-x^b;#%9o$oNy>?!(E{aOvOOt0 zw;;&Ai{QLRU9?3@7N6a?AK}ZXOZl(XIFnd(l4V)`(vq(7I;~^6?B`9R!m>hXu> z(M|Wwo!sgpit18>=O(z*=Sv3px^$9*&?&c;LG#P^m%hpPIMrr9Qyp@?sThRW{Vkrm zk<^ANHRc(yr`atu)4qkTuQvT`-!kVN<5!k)3&rUS_lcpO`POrRcR#n6bylLOvyjU{ zb^XX-DQWT7uIr;gK+DAsmoPy0*sd_HU#`&kCYAcuczCPrMw`E1eRwahqbxJ$!X5bD zfcb1zI&m=bpxik1`dse2Rl#=KtpqPikOAM8*>ZNKkwFJ!%@o@q#C&Ln@vF_9*ZIgs zi(hbGyHmElRv!#pJtk9u%q=~EuKfhe9|xT1EnlH&o7-A;lsb8KhJoHU1QXVD7Ny;R zW8mUJ8%7sX^Yb9>yxJBuR{g6#7b!%A`=VjcUqk)BtO8}Ywm-^5ub}GJj+RKT<`A1s z**U8KSj}DF{rjmJXgeH;D0-rzLLdLqKldD1Un#mev@z05VspwQ@RqDTqp1tns#TTY zgTn4yAPBbb)64<~;LChkE3dEK=dXjY*1&8H8tzRgNCu~6_3jZKbw9S_Bp%Nh%!W5V z%MfF|=Bc0BL564S!v?TxxWnlehW8>aGe};|al%RVkfTlwi8fyb6UHhDni~~4_LxFA z_l;-B%3ODvuai|xUpd$6F$K-XKUvemkYZG}OoAXdM*nsJ>Tp;5Z%`3BqW6lER*;6B zW}js2g?Piwr1~%s=izgV9yLsJLv1GWA^k35pHv=J`mv4_h@~Q)F#N-0 z7jsWtBKr(Nh`uf~;I0e*D3M`_Bop|IK>xyp&1WyUOlPrHv zqqiXHcB7=(kRIIlkarc`QJT&J{CNttEHd2VLiGTXy);&|$a<|Sr@ZJ>!}`^|=y@*h z7iA+VmTMTx9?iOKNWM&x4L)}a3HhmBM+9F_JrTK)2$JhsDZE9Vw> z$^}kudb-PR8eNa9EW0B0*TcIMH#TI?HHVU!(V2hOHY>ulP6{s2kFHe11Go59ab43J zj|H(a!-8MGc6OqVj#?|us%5A={*}`#>7adVIv<;5Nkd<^^KW27=mg7e$J-07ZyQyo zV7ImDNu`DD=d9H*uS?;ulHWv2!WNtiKLE}_RG&-*@ zRFcHCQE;s0Z-O5Q9Q9tHeu%fLPYV0gcM{z`nxKRJ(|D>13kwxJqK7?^DIVTtmcuW% zV)khMU>|RmR?d^8CZRfW?{7cY#|KN~iGRnZEJff`F5*BvKG}@o=(-Vdxq!I1sxb&~ zb+H~KG+H{Ja^*~#f%*oC^L815PgCHKw|O@zvdOKWt%u9ogL+ba+r;6)zu3AqOnJQz z1nLvbzR2E!nFgzMJ|u^bHN!)*nqQZqDL2tcFBHJyT?EZzL6l+mjZ8d zVsDxC+}4ymSNo4YWtI9J#xp(ONjn5w1f%b_Mwt!HmcLA&frsOPi52wD<&Qw3N^ADw zKSmC*rL!w%{uwW@%Hl|U(YZd|9;o<_1w)bABffpr*l7vJVuw>p{&*|2;H1_?u-!kx z{VYN$0(%d;|9y?@;W$thc~LQY2?sF|h>(boxR8kL3sF0933~}qYkLQX@C$JifkTikq1=fN#@(hO=+}6JnrSG)zuPC`&?RZAskRF zFuNDrsC_UaR=*}>tK^ClIPAj9?jrYHr4(3WsAdfl_|qj_RF_R_PuCWtG!z<$Psh`F zml1}Y9!*h+luWgz6tjBDQ;Thq9`+(VN@75c$$mQ@V?#I6-Z;!KJzU8+96vphXC@zK zal&9rYNWgg)Hp8NP9BQ@japjzRvHbu>YPT*WvmO1Tmy;c9$Mlu6y s^O*g3DrwNU;aB>Y^tMv&j8u-!mDf|_r9H{`~E%se_wSx02>czeou{7)Yb+D1_ih}`2fR~?C?mhIC+*E-$K!F zeuzFEJIkR*ZhPC`^=j!oNx8%btDqM}{#5z_qb*Z*NZeP>4Qv&XGt*E!T|RjqpEcW9 z$G~uj5^3gDZVc$0tg}c@J(Sfb=Ywd_T}rAu?q9iE@@G5~s0m!7!)?rR7i`}d+d)S_ z4_}&_JpM^6k2y{W%Ba}m5Spt=?8Ku$pacS3KC;M8;x^qsFmsI!cu_agG#2P7Rl0Mv% zNQ?a zHmr8parEJ{_Hd0qNK>1rvB(59YjAqckVRkMdDnb9u=FmIV7@Kpi0&4KmR20C`Y+9F z#IrbghQx!56tD&YL|0lEpQ-v(v^u!hu=_a=158Q~=tuq}uz*d18&-%%^J#>vG<=~a zq>o*;_eeRRg6pN}VRx;le*h8neCsWB<{O5cbw)fx<4n_$!%TU<{WXbqyGfIuYY(b8 zR)12B2Vp|c&!GqK0V%9O)StYLt0Qpx)g;y~-FdBtY)J~CAzn%N)y1vGop*EPkN z`}xG}sQ-+1I;5oOf3}#L2Iy*OR0HAZ(x`7!V^GYXs!_y0C;H^`i#4xj*;~C~pC7Nz zx}-YlKz_v0b=d>sD!aJT{RSV(RVq{B|B}r7bk-ZhKCe-I>aFv#@HFFcn{*Z!YC?(e zP<5!U&nrz|R;G>NGsp;il}YWZx))XC_Y_rACL{dfm!ku}cwo|EA=<<}GT=`+;i;2j ziq^-32;=87=P8sUblKRti$KZCSJE++ZeT_XXWGbsH8pY08dT1H{k>hTG-UsmoRU$A zY=fr3aYI1cmzNeLZ({6|?O%FUsNe@@q>kEF*xs_!>wy{ukVYf1J4CbcCt*^$GUC9N zeh6Fi59@%`upcpNX`C%_N6W{;p)=c}+8lK`Ay!uxfWx$m?tgsL(Q5zt4ukND(_u+G zxJuD#fBlWLFUAsYAzh{HV_+)L!(CjZYwPO5bapx$RRc9M37g9gcXxBY@XmSnNa^PF z=aAVqH2zN?`E0M)dBdHJhX%u(M0Toq@>V)LM_2!94PN||BTIL^N8J5Qyj&0N*syQ+ zm*wY@dnkVGS6lZo<7+F&vMAsR5)^%=xH8QZ|1*GduozhqRoqPRyM^M`&f`f8iav$% z3;pB$WnkH@m?7v88Z0tzJemoX<#5H=Hu~}u^hk4{#fDr}7|jP+olC??!xGRwDlyV9 z>q1kuajtMgL@1)Q0d0_7XmZKDweDeJ?3>hD@xg|`kgDimfv^H?oH?3vrY?SSvt1K4 z&r2A(T6YTI7DFUXF}#Ik$MZ#0-C$ziOH)l@ew?$Flyr@b?Z*VzTUd@BiQJwqvJ-PB zR`i{7b1i?+W%*A3fzJU9O@TZ;f0Bc#pF#}}&9rW+YpM>Dqc!Qp_xx6vDvjm?Lb@B= zAh-D;2rS32N0@}z`tkfm`Zqg&2j$>74O#KHdN*5@%T+AhDGWb}Vcz)N+$vgO!bh zU;OG|%QX*IVm=N2WGUGN=b4*453tmr$5kjUzw0uze5ztDt{~odv;njzIT>5 zeQ%9LobK~rnEtk}Y+HvNI4sPz?Y`Du{E_25&JtBdfPXZG5x#z6J0?3*i+xOB9J*DNP4*2zo#8QaG_Q;aAZ@^Q{c)izhgNo@---h#mYNT z07=6OZ*p-^&ZDSB2Cb@&6`cQWP4Zgpnh~@1>)6RYb+S&&7#&(bHkM)Zn`~@AgHN+} z!6wRnbzB5}T-b!@+DHO<>tj?0AY;D*olM@_#2AUsYG=&{pUm9(9v1q|e8Z^v$DCbm z;B~|9b8c8@ag$j2j+T)?3i0IURn{dQV{5oibzQmk%ubKTcnCo*zKWHF(^hZx1696V zto_>Uz;`}L#39GRX}4M{hFu)ja%VCl>ptV^@;T`TpdkY4@yXsjT>#p*&c(gxydsRh zQc127MNCjW!o2B`Tl|+lCaTsytmkj3LOiMw1vb2##on|SsCXbrD2_90U{~#-Pj%YF z;|#%ZYKXs7{CFD#nqqqXs77<1CL$|r&tDLajDCR|>g+|W z2;14~Mfc5g3dbjL-{CaXquD`Ih_y~<(g?FBn{$0L06%&oRrF(WRUiB({UDvf1vqH`EwX-3n;Bs%)^>itInxp-dN#6?f-rk58xwS#?{)+=vXB*#;_YD*&S&m4=LVCcJ& z)_1=0BJi;(TeZxHQBicdji(71k+oU*&bZDwSyh)49(%Fvz7iaMoWGyjjquFXhSrIT z38Ovd)D3o4TqNzO|vf@ybB)T^6RbvT_UxBUSYLA&sk zV^n;EwAlUBMNRpVqVb{D0;?}kx^e0{F?UAP!H0iO9u}`TkXGe&B9^X{#Bp%7_u!Jsz~rE%_ma(HZaA z?ihRE+Do-oH;4}{TB5w=@m4-k&f+4p`8eQ4X^itGlgWwni{lFNe1+cOGe;qNW(@N) z^!4#iXBXf#={#A%K@fM@82z48LceBN2N`r&Rdgj(^F2E3?jmoGY~BK2-Y?5uROyMO zFg!Z$_1~PU*8ow7BO9htM@CmgURBWzg+e)@po*@FicX3uPADZO6vPRt2$6@XL9{jh zUx8u0XNT5fVFmYcgZ;gALRED{eH%W%9bbrhpI>x)s5 zDUW9aJJPzV#ghL}d?3`1Wr5M#-YjQiRW759BFE4IM5YqNe#yTOtjg@8(%;b<_g{lh z$-&4=5DOma_N{_dA0L&=g%OqO#U(G~qEy?MH)ii1BNZIRB8j=Wcl#y;C@Km!)(8Fv Du=*HS delta 3197 zcmaizXE+;-qs42ls6A@erdEi(#cYvMl!`4hMwB;%qG_$#D{97QQTw&EVw9S#Jz~`= zv5OkD$Nzo*pYFY1?#FYU^Wi+d=bX|AqK^?oHFacYpoRwZUr0ZglQ(tXoZXn2XSfXg zon;hSk4I5|tQ60x`@Au%22t3g$(K+JH0!5rBp3@f%33WX$#lc9y?u)!_!Z-wi$8Ourk6Or~R~&7))0EeI)_5)9Y( z?-~aGBd2lPO|2(QpfAR|@*rg!RfqeDtZ(b*3&($Fw{H!ZM~?9Re2}G!IiO;~P*RAX z75^8^)TMu6_AOmMIGh2-_U3a(*T8AM;&>amy?lUqrlmmL&`q>v8%9GSIRdg5+&p0S z3pC<~PaM3FX_Iht^{)^o0c+l)s*Lz($%?YUfAO4#NurCbPNWnAwe7)TgX3et;5FAlsJ)k}0;0^f zV);f6S`FR?07=F#nA>gWcl1qy_E7 z+-S_2mHqi4vT~JK=|oU{kws^vxoqwdv^o=Quaie{=VxcD+R(+_I7PJ7$eC+ses&>G zHyLY5oqyiR$4M3gW;tDLv(FpO7n~R_5j=1*_p@WNK`(W`r;Fa}u-mwtcIaHQ`OJ25feE? z6|Lfcet3BtB&s9iq0)KrV#i2#Fee1v0+>V$mN5+!i$rMRjfVsNQNI6_0+NFPYOJ_z0809nHz#f z!9|XK2Z89q1dX|2iOiz0+6#h%M>ZDClW&9bc5yiMIq*(%O08{?8 zSUQC8pNt~vqu-;N8m3ZR4d3Xq-y;oEYrsj&9dF);kLe#cm5*PYouM5b2U(spKu+=4;J-F~_2MQYPs4t&)Bd#8O6Z}}zv z)ocbZjMGEw$~LB=ywxV61-Dd@QvU&ZLlNM;hCsoN_hQJ$$FEstpN1IRjPz6v>lVGQ zO902Sz4JCm@A*TRUeYM!d&~9C8BhPN1}e>ZF4tmy_ss!G3tGGpY(aQ$2iO#S8<5Mv zV^LAb{yn^s)ng*OrMCziIx6aYHhSqEAit|pY^FdVWGko3`y9ovZ&sR<{nAN-=2^2; zww@A>_R}YwXFtz==I8};rM*D8e>L!_2<2wPe!OrH0@MwQBJM~!&AsS#aR zSr}LHmY=ydg~s2PJhYYMEq?)NLsh__7XA}ef^3+VE+n`<4zJV>LFZ^^87Z*c-xcuE zE#~`mGXTv3KvOKVuiZin(2NZQT_Q+$7WA0VV?PE&a!t!a`Kxy}-R-&$_Btwr zbB$-KTBJ5^jHXXzqxMv6Az*xP<^o&A?Ct`|Wt&KI+C0@w@RucAG$>KnWHaH_*Ypo0 zPl|-s5FO`V4yJty4_>jPivTycGyC)~eI4zR?)T5F-=6NcnMp8g%!F>z3)c;8vvQ@y znh?97{f32yT+N25NXKCcmd&zR4E)n7n5c~?_v|0@MOg}Pgf|pf$8Efh9n6gb*y}P@ zuSzs#$*Mrqqd;u)-DUI&6N?1Vhg)*FhfvNT}*W$bExjzw%mZ^-GAg8~i z1=z%h8c%(bkUO*yAmm zpJvsI#Z0ARI8`3L%QAvb*68agLe~NA{lNfiZL?L1f=U$4yJAzfLKgz>N{3`hUtbWD zO%=bDtBKh_(SUJ0kXEJeLHbki-g~KzMd8b)x&X4;i@+*4c;(2#Ro_`F>hUf$Ob?yK zH=zu@le4Vy7qa%wlD-9i2c)amX@Uq7cB{(03`ol@5qtcGFa~Z$!VBG zQ}e-AdL>qONNbZI}GL6T^1 z55Fk%`46jv7g6L-BiylzA2+4f-D;@LmrV{mlqKB_!FBahQJxBKsDyIw?)Hc^OKD2c zJf?Vk5NxGJmv(zCd$IfLXo~%uj<_e3EbUmDxMz=852aP>Mkg9TA?iUVTB>`8Y>BaS z?5z1%;EkWmOI59d7A@cv8CpwO@VZ#*z=1^XD1&6}Xa}`AR;}8LO^=&RGKgWg6a{YY z7m$XY(i=-W8@g-!yA6=JY>QXANPI%!$X~@A#oHZwd)F$DZkg$E{5(nQE277FEZ62~ zlsB}--2+aC+Z!XRahNKxtNb02-jGP<(@z)QvbT-?2Lm2=h-W^hHCJ#5yrrlhbtuFXYf0$9oSl@MU7VGbm7J91U11;>Cs}z}XPAPE zqLaLWt0G84?f*+)n6&NCuJjCCFIc&Jy|hrQT0)3lUBzwIVffFrBv<=aL+a8wg8b&w ziGs4>V)31C;-Rc`OBH_^0%{do@TS+4TD|`xeXyR&oi+;}Y8h4l diff --git a/search.json b/search.json index 0e013ce..3e13652 100644 --- a/search.json +++ b/search.json @@ -1,74 +1,4 @@ [ - { - "objectID": "publish.html", - "href": "publish.html", - "title": "Statistics 243 Fall 2023", - "section": "", - "text": "We use Quarto with GitHub Actions to publish the materials as the course website. All commits should be done to main and not to gh-pages.\nThis documents creation of new pages that require Python or R computation. We render the source document locally, with ‘freeze’ set on the document so that results are stored in _freeze and don’t need to be re-run on every commit.\n\nCopy the preamble from an existing units/*.qmd file.\nUpdate _quarto.yml to reflect the new content (unless you don’t yet want it discoverable online).\nRun quarto render <new-Rmd-or-qmd> locally, which will store computations in the _freeze directory.\nCommit the new page and all changes to the _freeze dir including .json and figure (.png/pdf) files.\nPush to GitHub and the publish action should run fairly quickly via GitHub actions.\n\nTo set up the website for a new course/year, one needs to run quarto publish from within main initially.\nNotes:\n2023-09-07: GHA can fail with messages about nbformat. Can often fix by re-rendering the problematic qmd. I think this is happening when commits are made to a qmd without rendering that updates the freeze files.\n2023-08-29: when using knitr engine with unit3-bash.qmd (so that one can work with bash chunks), some GHA runs are complaining about missing rmarkdown. But then it sometimes works. Trying to install rmarkdown leads to a permission issue in the system directory that the R package is being installed into. If try to use jupyter engine with bash chunks, you probably need a Jupyter bash kernel, but I am still investigating." - }, - { - "objectID": "howtos/accessingPython.html", - "href": "howtos/accessingPython.html", - "title": "Accessing Python", - "section": "", - "text": "We recommend using using the Anaconda (Python 3.11 distribution) on your laptop. Click “Download” and then click 64-bit “Graphical Installer” for your current operating system.\nOnce you’ve installed Python, please install the following packages:\nAssuming you installed Anaconda Python, you should be able to do this from the command line:\nWhile you’re welcome to work with Python in a Jupyter notebook for exploration (e.g., using the campus DataHub, you’ll need to submit Quarto (.qmd) documents with Python chunks for your problem sets. So you’ll need Python set up on your laptop or to use it by logging in to an SCF machine." - }, - { - "objectID": "howtos/accessingPython.html#python-from-the-command-line", - "href": "howtos/accessingPython.html#python-from-the-command-line", - "title": "Accessing Python", - "section": "Python from the command line", - "text": "Python from the command line\nOnce you get your SCF account, you can access Python or IPython from the UNIX command line as soon as you login to an SCF server. Just SSH to an SCF Linux machine (e.g., gandalf.berkeley.edu or radagast.berkeley.edu) and run ‘python’ or ‘ipython’ from the command line.\nMore details on using SSH are here. Note that if you have the Ubuntu subsystem for Windows, you can use SSH directly from the Ubuntu terminal." - }, - { - "objectID": "howtos/accessingPython.html#python-via-jupyter-notebook", - "href": "howtos/accessingPython.html#python-via-jupyter-notebook", - "title": "Accessing Python", - "section": "Python via Jupyter notebook", - "text": "Python via Jupyter notebook\nYou can use a Jupyter notebook to run Python code from the SCF JupyterHub or the Berkeley DataHub.\nIf you’re on the SCF JupyterHub, select Start My Server. Then, unless you are running long or parallelized code, just click Spawn (in other words, accept the default ‘standalone’ partition). On the next page select ‘New’ and ‘Python 3’.\nTo finish your session, click on Control Panel and Stop My Server. Do not click Logout." - }, - { - "objectID": "howtos/gitInstall.html", - "href": "howtos/gitInstall.html", - "title": "Installing Git", - "section": "", - "text": "Here are some instructions for installing Git on your computer. Git is the version control software we’ll use in the course.\nYou can install Git by downloading and installing the correct binary from here.\nFor macOS, deeb recommends using the Homebrew option.\nGit comes installed on the SCF, so if you login to an SCF machine and want to use Git there, you don’t need to install Git.\n\nSidenotes on using Git with RStudio\nYou can work with Git through RStudio via RStudio projects.\nHere are some instructions. Here are some helpful guidelines from RStudio.\nYou may need to tell RStudio where the Git executable is located as follows.\n\nOn Windows, the git executable should be installed somewhere like: \"C:/Program Files (x86)/Git/bin/git.exe\"\nOn MacOS X, you can locate the executable by executing the following in Terminal: which git\nOnce you locate the executable, you may then need to confirm that RStudio is looking in the right place. Go to “Tools -> Options -> Git/SVN -> Git executable” and confirm it has the correct information about the location of the git executable." - }, - { - "objectID": "howtos/windowsInstall.html", - "href": "howtos/windowsInstall.html", - "title": "R/Rstudio on Windows", - "section": "", - "text": "While R was built/designed for UNIX systems, it has been well adapted for Windows. Here, we’ll start with the basics of installing R on Windows. Then, we’ll cover the recommended editor (Rstudio), and how to build pdf documents using MikTeX.\n\n\n\n\n\n\nNote\n\n\n\nThis tutorial installs Windows-only versions of everything. Modern Windows systems have an Ubuntu subsystem available that we highly recommend. See the Installing the Linux Subsystem on Windows tutorial for setting up that configuration.\n\n\n\nInstalling R\nThe first step in installing the R language. This is available on CRAN (Comprehensive R Archive Network).\n\nGo to the CRAN webpage, www.r-project.org\nIn the first paragraph, click the link download R\nYou’re now on a page titled CRAN Mirrors, choose the mirror located closest to your geographic location\n\nMirrors are different servers that all host copies of the same website. You get best performance from the location closest to you.\n\nYou’re now on a paged titled The Comprehensive R Archive Network. The first box is labeled Downloand and Install R, click Download R for Windows\nClick base or install R for the first time, these take you to the same place\n\nFor more advanced things, you may need the Rtools download later. It isn’t necessary now, but remember that for the future.\n\nAt the top is a large-font link, Download R X.X.X for Windows, click this. It will begin downloading the Windows installer for R.\nFollow the instructions for setup. If you are unsure of anything, leave the default settings\n\n\n\nInstalling RStudio\nRStudio is one of the best text editors for coding in R. It is our recommended option for beginning. After you are comfortable with the language, or if you use other languages as well, you may want to explore Atom or Sublime. More advanced options include Emacs with [ESS package][https://ess.r-project.org/] and vim with the Nvim-R plugin.\nTo install RStudio:\n\nGo to the RStudio Desktop download page, rstudio.com/products/rstudio/download/#download\nChoose the download for your OS, most likely the Windows 10/8/7 one\nFollow the instructions for setup. If you are unsure of anything, leave the default settings\nOpen RStudio (R will run automatically in the background)\n\nYou may have to allow RStudio to run if prompted (depends on security settings and anti-virus software)\n\n\nOnce RStudio is installed, you can install or update packages in one of two ways:\n\nVia the console, using install.packages() or update.packages()\n\nVia the gui:\n\nIn the top bar, click on tools\nSelect Install Packages… to install packages\nSelect Check for Package Updates… to update packages\n\n\n\n\nCompiling PDF Documents\nFor the purposes of this class, you will be submitting homeworks as PDF documents that blend written text, code, and code-generated output. These documents are RMarkdown documents, and are dynamic documents that provide a convenient method for documenting your work (more on this in one of the lab sections). To do this, you need a LaTeX renderer. We recommend MiKTeX for Windows.\n\nGo to Getting MiKTeX to download MiKTeX for Windows, miktex.org/download\nThe first page should be Install on Windows, click Download at the bottom of the page\n\nClick the download to begin\n\n\n\n\n\n\n\nImportant\n\n\n\nFOLLOW THESE INSTALL INSTRUCTIONS.\nThe default options are fine in most places, but there is one that will cause problems.\n\n\n\nAccept the Copying Conditions, click next\nInstall only for you, click next\nUse the default directory, click next\nThis should be the Settings page. Under Install missing packages on-the-fly, change the setting to Yes, click next\n\n\n\nBecause we are using MiKTeX as an external renderer, it can’t ask you to install missing packages, and will then fail, so we have to set that installation as automatic.\n\n\nClick start (Optional, but highly recommended) Open RStudio, select a new .Rmd document, d then choose knit. This may take some time, because MiKTeX is installing new braries, but it ensures that your pipeline is setup correctly" - }, - { - "objectID": "howtos/ps-submission.html", - "href": "howtos/ps-submission.html", - "title": "Problem Set Submissions", - "section": "", - "text": "Problem set solutions should be written in Quarto Markdown (.qmd) source files, interspersing explanatory text with Python (and in some cases bash) code chunks. Please do not use Jupyter notebook (.ipynb) files as your underlying source file for your solutions.\nWhy?\n\nFor one or two of the initial problem sets you’ll need to include both bash and Python code. This isn’t possible in a single notebook.\nThe underlying format of .ipynb files is JSON. While this is a plain text format, the key-value pair structure is much less well-suited for use with Git version control (which relies on diff) than Markdown-based formats.\nOne can run chunks in a Jupyter notebook in arbitrary order. What is printed to PDF depends on the order in which the chunks are run and the results can differ from what one would expect based on reading the notebook sequentially and running the chunks sequentially. For example, consider the following experiment and you’ll see what I mean: (1) Have one code chunk with a = 3 and run it; (2) Add a second chunk with print(a) and run it; and (3) Change the first chunk to a=4 and DO NOT rerun the second chunk. Save the notebook to PDF. You’ll see that your “report” makes no sense. Here’s the result of me doing that experiment.\n\nIf you really want to do your initial explorations of the problems in a Jupyter notebook, with content then copied to qmd, that is fine.\nFor problem sets later in the semester, we may allow the work to be done in a Jupyter notebook (committed to the repository as the source file) and then submitted as a PDF, but the initial problem sets must be provided as qmd source files." - }, - { - "objectID": "howtos/ps-submission.html#submission-format", - "href": "howtos/ps-submission.html#submission-format", - "title": "Problem Set Submissions", - "section": "", - "text": "Problem set solutions should be written in Quarto Markdown (.qmd) source files, interspersing explanatory text with Python (and in some cases bash) code chunks. Please do not use Jupyter notebook (.ipynb) files as your underlying source file for your solutions.\nWhy?\n\nFor one or two of the initial problem sets you’ll need to include both bash and Python code. This isn’t possible in a single notebook.\nThe underlying format of .ipynb files is JSON. While this is a plain text format, the key-value pair structure is much less well-suited for use with Git version control (which relies on diff) than Markdown-based formats.\nOne can run chunks in a Jupyter notebook in arbitrary order. What is printed to PDF depends on the order in which the chunks are run and the results can differ from what one would expect based on reading the notebook sequentially and running the chunks sequentially. For example, consider the following experiment and you’ll see what I mean: (1) Have one code chunk with a = 3 and run it; (2) Add a second chunk with print(a) and run it; and (3) Change the first chunk to a=4 and DO NOT rerun the second chunk. Save the notebook to PDF. You’ll see that your “report” makes no sense. Here’s the result of me doing that experiment.\n\nIf you really want to do your initial explorations of the problems in a Jupyter notebook, with content then copied to qmd, that is fine.\nFor problem sets later in the semester, we may allow the work to be done in a Jupyter notebook (committed to the repository as the source file) and then submitted as a PDF, but the initial problem sets must be provided as qmd source files." - }, - { - "objectID": "howtos/ps-submission.html#problem-set-solution-workflows", - "href": "howtos/ps-submission.html#problem-set-solution-workflows", - "title": "Problem Set Submissions", - "section": "Problem set solution workflows", - "text": "Problem set solution workflows\nHere we outline a few suggested workflows for developing your problem set solutions:\n\nOpen the qmd file in any editor you like (e.g., Emacs, Sublime, ….). From the command line (we think this will work from a Windows command line such as cmd.exe or PowerShell as well), run quarto preview FILE to show your rendered document live as you edit and save changes. You can put the preview window side by side with your editor, and the preview document should automatically render as you save your qmd file.\nUse VS Code with the following extensions: Python, Quarto, and Jupyter Notebooks. This allows you to execute and preview chunks (and whole document) inside VS Code. This is currently deeb’s favorite path due to how well it integrated with the Python debugger.\nUse RStudio (yes, RStudio), which can manage Python code and will display chunk output in the same way it does with R chunks. This path seems to work quite well and is recommended if you are already familiar with RStudio.\n\nLater in the semester, you may be allowed to work directly in Jupyter notebooks and use quarto to render from them directly. This has a few quirks and limitations, but may be allowed for some problem sets.\nPlease commit your work regularly to your repository as you develop your solutions." - }, - { - "objectID": "howtos/ps-submission.html#github-repository", - "href": "howtos/ps-submission.html#github-repository", - "title": "Problem Set Submissions", - "section": "GitHub repository", - "text": "GitHub repository\n\nSetting up your repository\nWe are creating repositories for everyone at github.berkeley.edu. Additionally, homeworks still need to be submitted as PDFs on Gradescope.\nSteps:\n\nLog into github.berkeley.edu using your Berkeley credentials. Because of how the system works, you will need to log in before your account is created. Nothing else needs to be done, just log in and log out.\nAfter accounts are created (may take a couple days after first login), when you log in again, you should see one private repository listed on the left side (e.g., stat243-fall-2022/ahv36). This is your class repository. Do not change the repository settings! They are set up for this class.\nClone the repo to your home directory (I would clone it into a directory just for repositories (e.g., I use ~/repos). In the top-level of your working directory, you should create a file named (exactly) .gitignore.\n\nThe .gitignore file causes Git to ignore transient or computer-specific files that Quarto generates. (more info at https://github.com/github/gitignore) In it, put (again, don’t put dashed lines):\n# cache directories\n/__pycache__\n\n# pickle files\n*.pkl\n*.pickle\n\n\nRepository Organization\nThe problem sets in your repository should be organized into folders with specific filenames.\nWhen we pull from your repository, our code will be assuming the following structure:\nyour_repo/\n├── ps1/\n│ ├── ps1.pdf\n│ ├── ps1.qmd \n│ ├── <possibly auxiliary code or other files>\n├── ps2/\n│ ├── ...\n├── ...\n├── ps8/\n├── .gitignore\n└── info.json\nThe file names are case-sensitive, so please keep everything lowercase." - }, { "objectID": "index.html", "href": "index.html", @@ -91,165 +21,417 @@ "text": "Course content\n\nThis site: most non-recorded course material\n\nFor html versions of the Units, see the navigation bar above.\nFor PDF versions of the Units, clone the GitHub repository, or if you’re not yet familiar with Git, download a zip file.\n\nVarious SCF tutorials: These include the various tutorials referred to in the class materials (e.g., the UNIX and bash shell tutorials, the dynamic documents tutorial, the Git tutorial, the string processing tutorial, etc.).\nbCourses: links to class course captures and any pre-recorded material.\n\nIf you’re not yet familiar with Git, go to the upper right of this page and click on ‘Clone or download’ and then ‘Download ZIP’." }, { - "objectID": "labs/lab5-codereview.html", - "href": "labs/lab5-codereview.html", - "title": "Lab 5: Code Reviews", - "section": "", - "text": "Pair up in teams of 2 (or 3 if necessary).\nOne member of the team will create a new repository on github and invite the other teammember(s) as collaborators on the repo.\nUsing the Github web UI, each team member will create a new file in the repo, and paste their code for presidential speaches into it.\nCommit you changes (from the github UI) as a pull request / separate branch.\nNow each team member will go to the pull request made by another team member and look at the code changes.\nRead eachother’s code carefully and leave some thoughtful comments.\nThe comments may be about implementation details, efficiency concerns, or style and readability\nRemember to be constructive and kind :)" - }, - { - "objectID": "labs/lab5-codereview.html#hands-on-steps-for-lab", - "href": "labs/lab5-codereview.html#hands-on-steps-for-lab", - "title": "Lab 5: Code Reviews", + "objectID": "units/unit4-goodPractices.html", + "href": "units/unit4-goodPractices.html", + "title": "Good practices: coding practices, debugging, and reproducible research", "section": "", - "text": "Pair up in teams of 2 (or 3 if necessary).\nOne member of the team will create a new repository on github and invite the other teammember(s) as collaborators on the repo.\nUsing the Github web UI, each team member will create a new file in the repo, and paste their code for presidential speaches into it.\nCommit you changes (from the github UI) as a pull request / separate branch.\nNow each team member will go to the pull request made by another team member and look at the code changes.\nRead eachother’s code carefully and leave some thoughtful comments.\nThe comments may be about implementation details, efficiency concerns, or style and readability\nRemember to be constructive and kind :)" + "text": "PDF\nSources:\nThis unit covers good coding/software development practices, debugging (and practices for avoiding bugs), and doing reproducible research. As in later units of the course, the material is generally not specific to Python, but some details and the examples are in Python." }, { - "objectID": "labs/06/scf.html", - "href": "labs/06/scf.html", - "title": "SCF Computing Cluster", - "section": "", - "text": "PDF" + "objectID": "units/unit4-goodPractices.html#editors", + "href": "units/unit4-goodPractices.html#editors", + "title": "Good practices: coding practices, debugging, and reproducible research", + "section": "Editors", + "text": "Editors\nUse an editor that supports the language you are using (e.g., Atom, Emacs/Aquamacs, Sublime, vim, VSCode, TextMate, WinEdt, or the built-in editor in RStudio [you can use Python from within RStudio]). Some advantages of this can include:\n\nhelpful color coding of different types of syntax,\nautomatic indentation and spacing,\nparenthesis matching,\nline numbering (good for finding bugs), and\ncode can often be run (or compiled) and debugged from within the editor.\n\nSee the problem set submission how-to document for more information about editors that interact nicely with Quarto documents." }, { - "objectID": "labs/06/scf.html#logging-in", - "href": "labs/06/scf.html#logging-in", - "title": "SCF Computing Cluster", - "section": "Logging in", - "text": "Logging in\nThe SCF has a number of login nodes which you can access via ssh.\n\n\n\n\n\n\nNote\n\n\n\nFor info on using ssh (including on Windows), see here.\n\n\nFor example, I’ll use ssh to connect to the dorothy node:\njames@pop-os:~$ ssh <scf-username>@dorothy.berkeley.edu\nThe authenticity of host 'dorothy.berkeley.edu (128.32.135.58)' can't be established.\nED25519 key fingerprint is SHA256:rOY7ED/iIiTgI++Y4XHmiEl+tC+OmSGBvWp03CSII5E.\nThis key is not known by any other names\nAre you sure you want to continue connecting (yes/no/[fingerprint])? yes\nWarning: Permanently added 'dorothy.berkeley.edu' (ED25519) to the list of known hosts.\nNotice that upon first connecting to a server you haven’t visited there is a warning that the “authenticity of the host … can’t be established”. So long as you have typed in the hostname correctly (dorothy.berkeley.edu, in this case), and trust the host (we trust the SCF!) then you can type yes to add the host to your known hosts file (found on your local machine at ~/.ssh/known_hosts).\nYou’ll then be asked to enter your password for the SCF cluster. For privacy, you won’t see anything happen in your terminal when you type it in, so type carefully (you can use Backspace if you make a mistake) and press Enter when you’re done. If you were successful, you should see a welcome message and your shell prompt, like:\n<scf-username>@dorothy:~$\nTo get your bearings, you can type pwd to see where your home directory is located on the SCF cluster filesystem:\n<scf-username>@dorothy:~$ pwd\n/accounts/grad/<scf-username>\nYour home directory is likely also in the /accounts/grad/ directory, as mine is.\n\nOther login nodes\n\n\n\n\n\n\nImportant\n\n\n\nDon’t run computationally intensive tasks on the login nodes!\nThey are shared by all the SCF users, and should only be used for non-intensive interactive work such as job submission and monitoring, basic compilation, managing your disk space, and transferring data to/from the server.\n\n\nIf for some reason dorothy is not working for you, the SCF has a number of nodes which can be accessed from your local machine with commands of the form ssh <scf-username>@<hostname>.berkeley.edu. Currently, these are:\n\naragorn\narwen\ndorothy\ngandalf\ngollum\nhermione\nquidditch\nradagast\nshelob" + "objectID": "units/unit4-goodPractices.html#coding-syntax", + "href": "units/unit4-goodPractices.html#coding-syntax", + "title": "Good practices: coding practices, debugging, and reproducible research", + "section": "Coding syntax", + "text": "Coding syntax\nThe PEP 8 style guide is your go-to reference for Python style. I’ve highlighted some details here as well as included some general suggestions of my own.\n\nHeader information: put metainfo on the code into the first few lines of the file as comments. Include who, when, what, how the code fits within a larger program (if appropriate), possibly the versions of Python and key packages that you used.\nWrite docstrings for public modules, classes, functions, and methods. For non-public items, a comment after the def line is sufficient to describe the purpose of the item.\nIndentation: Python is strict about indentation of course, which helps to enforce clear indentation more than in other languages. This helps you and others to read and understand the code and can help in detecting errors in your code because it can expose lack of symmetry.\n\nuse 4 spaces per indentation level (avoid tabs if possible).\n\nWhitespace: use it in a variety of places. Some places where it is good to have it are\n\naround operators (assignment and arithmetic);\nbetween function arguments;\nbetween list/tuple elements; and\nbetween matrix/array indices.\n\nUse blank lines to separate blocks of code with comments to say what the block does.\nUse whitespaces or parentheses for clarity even if not needed for order of operations. For example, a/y*x will work but is not easy to read and you can easily induce a bug if you forget the order of ops. Instead, use a/y * x.\nAvoid code lines longer than 79 characters and comment/docstring lines longer than 72 characters.\nComments: add lots of comments (but don’t belabor the obvious, such as x = x + 1 # increment x).\n\nRemember that in a few months, you may not follow your own code any better than a stranger.\nSome key things to document: (1) summarizing a block of code, (2) explaining a very complicated piece of code - recall our complicated regular expressions, and (3) explaining arbitrary constant values.\nComments should generally be complete sentences.\n\nYou can use parentheses to group operations such that they can be split up into lines and easily commented, e.g.,\n\nnewdf = (\n pd.read_csv('file.csv')\n .rename(columns = {'STATE': 'us_state'}) # adjust column names\n .dropna() # remove some rows\n )\n\nFor software development, break code into separate files (2000-3000 lines per file) with meaningful file names and related functions grouped within a file.\nBeing consistent about the naming style for objects and functions is hard, but try to be consistent. PEP8 suggests:\n\nClass names should be UpperCamelCase.\nFunction, method, and variable names should be snake_case, e.g., number_of_its or n_its.\nNon-public methods and variables should have a leading underscore.\n\nTry to have the names be informative without being overly long.\nDon’t overwrite names of objects/functions that already exist in Python. E.g., don’t use len. That said, the namespace system helps with the unavoidable cases where there are name conflicts.\nUse active names for functions (e.g., calc_loglik, calc_log_lik rather than loglik or loglik_calc). The idea is that a function in a programming language is like a verb in regular language (a function does something), so use a verb to name it.\nLearn from others’ code\n\nThis semester, someone will be reading your code - the GSI and and me when we look at your assignments. So to help us in understanding your code and develop good habits, put these ideas into practice in your assignments.\nWhile not Python examples, the files goodCode.R and badCode.R in the units directory of the class repository provide examples of code written such that it does and does not conform to the general ideas listed above (leaving aside the different syntax of Python and R)." }, { - "objectID": "labs/06/scf.html#disk-space", - "href": "labs/06/scf.html#disk-space", - "title": "SCF Computing Cluster", - "section": "Disk space", - "text": "Disk space\nYour home directory has a limited amount of disk space; you can check how much you have used and available using the quota command, which will show your home directory usage and quota on the line for the accounts filesystem.\n<scf-username>@dorothy:~$ quota\nFILESYSTEM USE QUOTA %\n accounts 5.29 G 20 G 26.5\nThe accounts filesystem is accessible from all the nodes on the SCF cluster. This means that regardless of which login or compute node you use, you will have access to the files in your home directory. Moreover, your home directory is backed up regularly, so you are protected from accidental data loss.\nIf you’re running out of space, you should try to selectively delete large files that you no longer need. See here for some tips on finding large files. If all else fails, you can request additional space or request access the /scratch filesystem. The latter is a good place for large datasets, but note that /scratch is not backed up, unlike your home directory. See the previous link for more info.\nFor temporary files (e.g., intermediate results of a computation that you don’t need to store for later), every machine has a /tmp filesystem. However, /tmp is always linked to the specific machine you are on, meaning that if you put something in /tmp on dorothy and then later go to aragorn, you won’t find your files in the /tmp directory there. If you go back to dorothy, you will likely find them again, but /tmp is automatically wiped when a machine reboots, so only use it for files you don’t care about preserving!" + "objectID": "units/unit4-goodPractices.html#coding-style", + "href": "units/unit4-goodPractices.html#coding-style", + "title": "Good practices: coding practices, debugging, and reproducible research", + "section": "Coding style", + "text": "Coding style\nThis is particularly focused on software development, but some of the ideas are useful for data analysis as well.\n\nBreak down tasks into core units\nWrite reusable code for core functionality and keep a single copy of the code (using version control or at least with a reasonable backup strategy) so you only need to make changes to a piece of code in one place\nSmaller functions are easier to debug, easier to understand, and can be combined in a modular fashion (like the UNIX utilities)\nWrite functions that take data as an argument and not lines of code that operate on specific data objects. Why? Functions allow us to reuse blocks of code easily for later use and for recreating an analysis (reproducible research). It’s more transparent than sourcing a file of code because the inputs and outputs are specified formally, so you don’t have to read through the code to figure out what it does.\nFunctions should:\n\nbe modular (having a single task);\nhave meaningful name; and\nhave a doc string describing their purpose, inputs and outputs.\n\nWrite tests for each function (i.e., unit tests)\nDon’t hard code numbers - use variables (e.g., number of iterations, parameter values in simulations), even if you don’t expect to change the value, as this makes the code more readable. For example, the speed of light is a constant in a scientific sense, but best to make it a variable in code: speed_of_light = 3e8\nUse lists or tuples to keep disparate parts of related data together\nPractice defensive programming (see also the discussion below on assertions)\n\ncheck function inputs and warn users if the code will do something they might not expect or makes particular choices;\ncheck inputs to if:\n\nNote that in Python, an expression used as the condition of an if will be equivalent to True unless it is one of False, 0, None, or an empty list/tuple/string.\n\nprovide reasonable default arguments;\ndocument the range of valid inputs;\ncheck that the output produced is valid; and\nstop execution based on checks and give an informative error message.\n\nTry to avoid system-dependent code that only runs on a specific version of an OS or specific OS\nLearn from others’ code\nConsider rewriting your code once you know all the settings and conditions; often analyses and projects meander as we do our work and the initial plan for the code no longer makes sense and the code is no longer designed specifically for the job being done." }, { - "objectID": "labs/06/scf.html#data-transfer-scp-sftp", - "href": "labs/06/scf.html#data-transfer-scp-sftp", - "title": "SCF Computing Cluster", - "section": "Data transfer: SCP / SFTP", - "text": "Data transfer: SCP / SFTP\nWe can use the scp and sftp protocols to transfer files to and from any login node on the SCF cluster. scp is a shell program that should be available by default on macOS and Linux, while on Windows you can use WinSCP. WinSCP can also do sftp transfers, or you can use FileZilla on any platform for sftp. Both WinSCP and FileZilla have a GUI that allows you to drag and drop files between your local machine and a remote host.\nThe syntax for scp is scp <from-place> <to-place>, and you’ll typically use scp on your local machine. For example, we’ll show how to transfer the file data.csv (you can find it here) :\n\nTo SCF while on your local machine\n# transfer to my home directory without renaming the file \nscp data.csv <scf-username>@dorothy.berkeley.edu:~/\n\n# transfer to the data/ subdirectory and rename the file\nscp data.csv <scf-username>@dorothy.berkeley.edu:~/data/new_name.csv\n\n# transfer to the dorothy-specific /tmp/ directory\nscp data.csv <scf-username>@dorothy.berkeley.edu:/tmp/\n\n\nFrom SCF while on your local machine\n# now <from-place> is a path in my home directory on the SCF \nscp <scf-username>@dorothy.berkeley.edu:~/data/new_name.csv ~/Desktop/data_scf.csv\nFor more information, see here. In particular, if you have a very large dataset to transfer, Globus is a better option than either scp or sftp." + "objectID": "units/unit4-goodPractices.html#assertions-exceptions-and-testing", + "href": "units/unit4-goodPractices.html#assertions-exceptions-and-testing", + "title": "Good practices: coding practices, debugging, and reproducible research", + "section": "Assertions, exceptions and testing", + "text": "Assertions, exceptions and testing\nAssertions, exceptions and testing are critically important for writing robust code that is less likely to contain bugs.\n\nExceptions\nYou’ve probably already seen exceptions in action whenever you’ve done something in Python that causes an error to occur and an error message to be printed. Syntax errors are different from exceptions in that exceptions occur when the syntax is correct but the execution of the code results in some sort of error.\nExceptions can be a valuable tool for making your code handle different modes of failure (missing file, URL unreachable, permission denied, invalid inputs, etc.). You use them when you are writing code that is supposed to perform a task (e.g., a function that does something with an input file) to indicate that the task failed and the reason for that failure (e.g., the file was not there in the first place). In such case, you want your code to raise an exception and make the error message informative as possible, typically by handling the exception thrown by another function you call and augmenting the message. Another possibility is that your code detects a situation where you need to throw an exception (e.g., an invalid input to a function).\nThe other side of the coin happens when you want to write a piece of code that handles failure in a specific way, instead of simply giving up. For example, if you are writing a script that reads and downloads hundreds of URLs, you don’t want your program to stop when any of them fails to respond (or do you?). You might want to continue with the rest of the URLs, and write out the failed URLs in a secondary output file.\n\nUsing try-except to continue execution\nIf you want some code to continue running even when it encounters an error, you can use try-except. This would often be done in code where you were running some workflow rather than in functions that you write for general purpose use (e.g., code in a package you are writing).\nSuppose we have a loop and we want to run all the iterations even if the code for some iterations fail. We can embed the code that might fail in a try block and then in the except block, run code that will handle the situation when the error occurs.\n\nfor i in range(n):\n try:\n <some code that might fail>\n result[i] = <actual result>\n except:\n <what to do if the code fails>\n result[i] = None \n\n\n\nStrategies for invoking and handling errors\nHere we’ll address situations that might arise when you are developing code for general purpose use (e.g., writing functions in a package) and need that code to invoke an error under certain circumstances or deal gracefully with an error occurring in some code that you are calling.\nA basic situation is when you want to detect a situation where you need to invoke an error (i.e., throw an exception).\nWith raise you can invoke an exception. Suppose we need an input to be a positive number. We’ll use Python’s built-in ValueError, one of the various exception types that Python provides and that you could use. You can also create your own exceptions by subclassing one of Python’s existing exception classes. (We haven’t yet discussed classes and object-oriented programming, so don’t worry if you’re not sure about what that means.)\n\ndef myfun(val):\n if val <= 0:\n raise ValueError(\"`val` should be positive\")\n\nmyfun(-3)\n\nValueError: `val` should be positive\n\n\nNext let’s consider cases where your function runs some code that might return an error.\nWe’d often want to catch the error using try-except. In some cases we would want to notify the user and then continue (perhaps falling back to a different way to do things or returning None from our function) while in others we might want to provide a more informative error message than if we had just let the error occur, but still have the exception be raised.\nFirst let’s see the case of continuing execution.\n\nimport os\n\ndef myfun(filename):\n try:\n with open(filename, \"r\") as file:\n text = file.read()\n except Exception as err:\n print(f\"{err}\\nCheck that the file `{filename}` can be found \"\\\n f\"in the current path: `{os.getcwd()}`.\")\n return None\n\n return(text.lower())\n\n\nmyfun('missing_file.txt')\n\n[Errno 2] No such file or directory: 'missing_file.txt'\nCheck that the file `missing_file.txt` can be found in the current path: `/accounts/vis/paciorek/teaching/243fall23/stat243-fall-2023/units`.\n\n\nFinally let’s see how we can intercept an error but then “re-raise” the error rather than continuing execution.\n\nimport requests\n\ndef myfun(url):\n try:\n requests.get(url)\n except Exception as err:\n print(f\"There was a problem accessing {url}. \"\\\n f\"Perhaps it doesn't exist or the URL has a typo?\")\n raise\n\nmyfun(\"http://missingurl.com\")\n\nThere was a problem accessing http://missingurl.com. Perhaps it doesn't exist or the URL has a typo?\n\n\nConnectionError: HTTPConnectionPool(host='missingurl.com', port=80): Max retries exceeded with url: / (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f348db44590>: Failed to establish a new connection: [Errno -2] Name or service not known'))\n\n\n\n\n\nAssertions\nAssertions are a quick way to raise a specific type of Exception (AssertionError). Assertions are useful for performing quick checks in your code that the state of the program is as you expect. They’re primarily intended for use during the development process to provide “sanity checks” that specific conditions are true, and there are ways to disable them when you are running your code for production purposes (to improve performance). A common use is for verifying preconditions and postconditions (especially preconditions). One would generally only expect such conditions not to be true if there is a bug in the code. Here’s an example of using the assert statement in Python, with a clear assertion message telling the developer what the problem is.\n\nnumber = -42\nassert number > 0, f\"number greater than 0 expected, got: {number}\"\n## Produces this error:\n## Traceback (most recent call last):\n## File \"<stdin>\", line 1, in <module>\n## AssertionError: number greater than 0 expected, got: -42\n\nVarious operators/functions are commonly used in assertions, including\n\nassert x in y\nassert x not in y\nassert x is y\nassert x is not y\nassert isinstance(x, <some_type>)\nassert all(x)\nassert any(x)\n\n\n\nTesting\nTesting is informally what you do after you write some code and want to check that it actually works. But when you are developing important code (e.g. functions that are going to be used by others) you typically want to encode your tests in code. There are many reasons to do that, including making sure that if someone else changes your code later on without fully understanding what it was supposed to do, the test suite should immediately indicate that.\nSome people even advocate for writing a preliminary test suite before writing the code itself(!) as it can be a good way to organize work and track progress, as well as act as a secondary form of documentation for clarity. This can include tests that your code provides correct and useful errors when something goes wrong (so that means that a test might be to see if problematic input correctly produces an error). Unit tests are intended to test the behavior of small pieces (units) of code, generally individual functions. Unit tests naturally work well with the ideas above of writing small, modular functions. I recommend the pytest package, which is designed to make it easier to write sets of good tests.\nIn lab, we’ll go over assertions, exceptions, and testing in detail." }, { - "objectID": "labs/lab3-debugging.html", - "href": "labs/lab3-debugging.html", - "title": "Lab 3: Debugging", - "section": "", - "text": "Today we’re going to explore the concept of debugging and some of the tooling that allows for debugging python code.\nIt’s widely recognized and accepted that any sizeable code base will have a non-trivial number of bugs (where does the term come from?). The main goal of testing is to make sure the main expected cases are behaving correctly.\nSometime you code doesn’t do what you expect or want it to do, and it’s not clear just by reading through it, what the problem is.\nIn interpreted languages (esp. those with interactive shells like python) one can sometimes run the code piece by piece and inspect the state of the variables.\nIf the code involves a loop or a function, a common practice is to judiciously place a few print statements that dump the state of some variables to the terminal so that you can spot the problem by tracing through it.\nWhen the code involves multiple functions and complex state, this strategy starts to break down. This is where you start rolling up your sleeves and invoking a debugger!\nDebugging can be a slow process, so you typically start a debugging session by deciding which line in your code you would like to start tracing the behavior from, and you place a breakpoint. Then you can have the debugger run the program up to that point and stop at it, allowing you to:\n1- inspect the current state of variables 2- step through the code line by line 3- step over or into functions as they are called 4- resume program execution" + "objectID": "units/unit4-goodPractices.html#version-control", + "href": "units/unit4-goodPractices.html#version-control", + "title": "Good practices: coding practices, debugging, and reproducible research", + "section": "Version control", + "text": "Version control\n\nUse it! Even for projects that only you are working on. It’s the closest thing you’ll get to having a time machine!\nUse an issues tracker (e.g., the GitHub issues tracker is quite nice), or at least a simple to-do file, noting changes you’d like to make in the future.\nIn addition to good commit messages, it’s a good idea to keep good running notes documenting your projects.\n\nWe’ve already seen Git some and will see it in a lot more detail later in the semester, so I don’t have more to say here." }, { - "objectID": "labs/lab3-debugging.html#debugging", - "href": "labs/lab3-debugging.html#debugging", - "title": "Lab 3: Debugging", - "section": "", - "text": "Today we’re going to explore the concept of debugging and some of the tooling that allows for debugging python code.\nIt’s widely recognized and accepted that any sizeable code base will have a non-trivial number of bugs (where does the term come from?). The main goal of testing is to make sure the main expected cases are behaving correctly.\nSometime you code doesn’t do what you expect or want it to do, and it’s not clear just by reading through it, what the problem is.\nIn interpreted languages (esp. those with interactive shells like python) one can sometimes run the code piece by piece and inspect the state of the variables.\nIf the code involves a loop or a function, a common practice is to judiciously place a few print statements that dump the state of some variables to the terminal so that you can spot the problem by tracing through it.\nWhen the code involves multiple functions and complex state, this strategy starts to break down. This is where you start rolling up your sleeves and invoking a debugger!\nDebugging can be a slow process, so you typically start a debugging session by deciding which line in your code you would like to start tracing the behavior from, and you place a breakpoint. Then you can have the debugger run the program up to that point and stop at it, allowing you to:\n1- inspect the current state of variables 2- step through the code line by line 3- step over or into functions as they are called 4- resume program execution" + "objectID": "units/unit4-goodPractices.html#basic-debugging-strategies", + "href": "units/unit4-goodPractices.html#basic-debugging-strategies", + "title": "Good practices: coding practices, debugging, and reproducible research", + "section": "Basic debugging strategies", + "text": "Basic debugging strategies\nHere we’ll discuss some basic strategies for finding and fixing bugs. Other useful locations for tips on debugging include:\n\nEfficient Debugging by Goldspink\nDebugging for Beginners by Brody\n\nRead and think about the error message (the traceback), starting from the bottom of the traceback. Sometimes it’s inscrutable, but often it just needs a bit of deciphering. Looking up a given error message by simply doing a web search with the exact message in double quotes can be a good strategy, or you could look specifically on Stack Overflow.\nBelow we’ll see how one can view the stack trace. Usually when an error occurs, it occurs in a function call that is nested in a series of function calls. This series of calls is the call stack and the traceback or stack trace shows that series of calls that led to the error. To debug, you’ll often need to focus on the function being executed at the time the error occurred (which will be at the top of the call stack but the bottom of the traceback) and the arguments passed into that function. However, if the error occurs in a function you didn’t write, the problem will often be with the arguments that your code provided at the last point in the call stack at which code that you wrote was run. Check the arguments that your code passed into that first function that is not a function of yours.\nWhen running code that produces multiple errors, fix errors from the top down - fix the first error that is reported, because later errors are often caused by the initial error. It’s common to have a string of many errors, which looks daunting, caused by a single initial error.\nIs the bug reproducible - does it always happen in the same way at at the same point? It can help to restart Python and see if the bug persists - this can sometimes help in figuring out if there is a scoping issue and we are using a global variable that we did not mean to.\nIf you can’t figure out where the error occurs based on the error messages, a basic strategy is to build up code in pieces (or tear it back in pieces to a simpler version). This allows you to isolate where the error is occurring. You might use a binary search strategy. Figure out which half of the code the error occurs in. Then split the ‘bad’ half in half and figure out which half the error occurs in. Repeat until you’ve isolated the problem.\nIf you’ve written your code modularly with lots of functions, you can test individual functions. Often the error will be in what gets passed into and out of each function.\nAt the beginning of time (the 1970s?), the standard debugging strategy was to insert print statements in one’s code to see the value of a variable and thereby decipher what could be going wrong. We have better tools nowadays. But sometimes we still need to fall back to inserting print statements.\nPython is a scripting language, so you can usually run your code line by line to figure out what is happening. This can be a decent approach, particularly for simple code. However, when you are trying to find errors that occur within a series of many nested function calls or when the errors involve variable scoping (how Python looks for variables that are not local to a function), or in other complicated situations, using formal debugging tools can be much more effective. Finally, if the error occurs inside of functions provided by Python, rather than ones you write, it can be hard to run the code in those functions line by line." }, { - "objectID": "labs/lab3-debugging.html#advanced-debugging", - "href": "labs/lab3-debugging.html#advanced-debugging", - "title": "Lab 3: Debugging", - "section": "advanced debugging", - "text": "advanced debugging\nSome bugs are really tricky to catch. Those are typically the bugs that happen very rarely, and in unclear circumstances. In statistical computing and data analysis settings these might be conditions that happen in iterative algorithms sometimes, but not very often, and can be hard to reproduce.\nOne way to deal with these bugs is to start the debugging session with placing one or more conditional breakpoints. These are similar to regular breakpoints but are not going to cause the debugger to stop the execution of the program and hand you the controls unless a specific condition (that you specify) evaluates to true as the program is executing that particular line of code. You may use some conditions that are similar to what you would place in an assert statement (conditions that shouldn’t happen) or any other conditions that you think may be associated with the occurrence of the anomalous behavior your are debugging." + "objectID": "units/unit4-goodPractices.html#using-pdb", + "href": "units/unit4-goodPractices.html#using-pdb", + "title": "Good practices: coding practices, debugging, and reproducible research", + "section": "Using pdb", + "text": "Using pdb\nWe can activate the debugger in various ways:\n\nby inserting breakpoint() (or equivalently import pdb; pdb.set_trace()) inside a function or module at a location of interest (and then running the function or module)\nby using pdb.pm() after an error (i.e., an exception) has occurred to invoke the browser at the point the error occurred\nby running a function under debugger control with pdb.run()\nby starting python with python -m pdb file.py and adding breakpoints\n\n\nUsing breakpoint\nLet’s define a function that will run a stratified analysis, in this case fitting a regression to each of the strata (groups/clusters) in some data. Our function is in stratified_with_break.py, and it contains breakpoint at the point where we want to invoke the debugger.\nNow I can call the function and will be put into debugging mode just before the next line is called:\n\nimport run_with_break as run\nrun.fit(run.data, run.n_cats)\n\nWhen I run this, I see this:\n>>> run.fit(data, n_cats)\n> /accounts/vis/paciorek/teaching/243fall22/stat243-fall-2023/units/run_with_break.py(10)fit()\n-> sub = data[data['cats'] == i]\n(Pdb) \nThis indicates I am debugging at line 10 of run_with_break.py, which is the line that creates sub, but I haven’t yet created sub.\nI can type n to run that line and go to the next one:\n(Pdb) n\n> /accounts/vis/paciorek/teaching/243fall22/stat243-fall-2023/units/run_with_break.py(11)fit()\n-> model = statsmodels.api.OLS(sub['y'], statsmodels.api.add_constant(sub['x']))\nat which point the debugger is about to execute line 11, which fits the regression.\nI can type c to continue until the next breakpoint:\n(Pdb) c\n> /accounts/vis/paciorek/teaching/243fall22/stat243-fall-2023/units/run_with_break.py(10)fit()\n-> sub = data[data['cats'] == i]\nNow if I print i, I see that it has incremented to 1.\n(Pdb) p i\n1\nWe could keep hitting n or c until hitting the stratum where an error occurs, but that would be tedious.\nLet’s hit q to quit out of the debugger.\n(Pdb) q\n>>>\nNext let’s see how we can enter debugging mode only at point an error occurs.\n\n\nPost-mortem debugging\nWe’ll use a version of the module without the breakpoint() command.\n\nimport pdb\nimport run_no_break as run \n\nrun.fit(run.data, run.n_cats)\npdb.pm()\n\nThat puts us into debugging mode at the point the error occurred:\n> /usr/local/linux/mambaforge-3.11/lib/python3.11/site-packages/numpy/core/fromnumeric.py(86)_wrapreduction()\n-> return ufunc.reduce(obj, axis, dtype, out, **passkwargs)\n(Pdb)\nwhich turns out to be in some internal Python function that calls a reduce function, which is where the error occurs (presumably the debugger doesn’t enter this function because it calls compiled code):\n(Pdb) l\n 81 if dtype is not None:\n 82 return reduction(axis=axis, dtype=dtype, out=out, **passkwargs)\n 83 else:\n 84 return reduction(axis=axis, out=out, **passkwargs)\n 85 \n 86 -> return ufunc.reduce(obj, axis, dtype, out, **passkwargs)\n 87 \n 88 \n 89 def _take_dispatcher(a, indices, axis=None, out=None, mode=None):\n 90 return (a, out)\n 91 \nWe can enter u multiple times (it’s only shown once below) to go up in the stack of function calls until we recognize code that we wrote:\n(Pdb) u\n> /accounts/vis/paciorek/teaching/243fall22/stat243-fall-2023/units/run_no_break.py(10)fit()\n-> model = statsmodels.api.OLS(sub['y'], statsmodels.api.add_constant(sub['x']))\nNow let’s use p to print variable values to understand the problem:\n(Pdb) p i\n29\n(Pdb) p sub\nEmpty DataFrame\nColumns: [y, x, cats]\nIndex: []\nAh, so in the 29th stratum there are no data!\nIn addition using the IPython magic %debug will put you into the debugger in in post-mortem mode when an error occurs.\n\n\npdb commands\nHere’s a list of useful pdb commands (some of which we saw above) that you can use once you’ve entered debugging mode.\n\nh or help: shows all the commands\nl or list: show the code around where the debugger is currently operating\nc or continue: continue running the code until the next breakpoint\np or print: print a variable\nn or next: run the current line and go to the next line in the current function\ns or step: jump (step) into the function called in the current line (if it’s a Python function)\nr or run: exit out of the current function (e.g., if you accidentally stepped into a function) (but note this stops at breakpoints)\nunt or until: run until the next line (or unt <number> to run until reaching line number ); this is useful for letting a loop run until completion\nb or break: set a breakpoint\ntbreak: one-time breakpoint\nwhere: shows call stack\nu (or up) and d (or down): move up and down the call stack\nq quit out of the debugger\n<return>: runs the previous pdb command again\n\n\n\nInvoking pdb on a function or block of code\nWe can use pdb.run() to run a function under the debugger. We need to make sure to use s as the first pdb command in order to actually step into the function. From there, we can debug as normal as if we had set a breakpoint at the start of the function.\n\nimport run_with_break as run\nimport pdb\npdb.run(\"run.fit(run.data, run.n_cats)\")\n(Pdb) s\n\n\n\nInvoking pdb on a module\nWe can also invoke pdb when we start Python, executing a file (module). Here we’ve added fit(data, n_cats) at the end of run_no_break2.py so that we can have that run under the debugger.\n#| eval: false\npython -m pdb run_no_break2.py\n> /accounts/vis/paciorek/teaching/243fall22/stat243-fall-2023/units/run_no_break2.py(1)<module>()\n-> import numpy as np\n(Pdb) \nLet’s set a breakpoint at the same place we did with breakpoint() but using a line number (this avoids having to actually modify our code):\n(Pdb) b 9\nBreakpoint 1 at /accounts/vis/paciorek/teaching/243fall22/stat243-fall-2023/units/run_no_break.py:9\n\n(Pdb) c\n> /accounts/vis/paciorek/teaching/243fall22/stat243-fall-2023/units/run_no_break2.py(9)fit()\n-> model = statsmodels.api.OLS(sub['y'], statsmodels.api.add_constant(sub['x']))\n\nSo we’ve broken at the same point where we manually added breakpoint() in run_with_break.py.\nOr we could have set a breakpoint at the start of the function:\n(Pdb) disable 1\nDisabled breakpoint 1 at /accounts/vis/paciorek/teaching/243fall22/stat243-fall-2023/units/run_no_break2.py:9\n(Pdb) b fit\nBreakpoint 1 at /accounts/vis/paciorek/teaching/243fall22/stat243-fall-2023/units/run_no_break.py:6" }, { - "objectID": "labs/lab3-debugging.html#integrated-gui-debugger-with-vs-code", - "href": "labs/lab3-debugging.html#integrated-gui-debugger-with-vs-code", - "title": "Lab 3: Debugging", - "section": "Integrated GUI debugger (with VS Code)", - "text": "Integrated GUI debugger (with VS Code)\nToday we will experiment with the visual debugging tools integrated with IDEs. We will do that in VS Code (unless you have another IDE with debugger integration). We will load a piece of code, go through it to understand what it does, then try to discover the problem with it and fix it.\nHere’s a piece of code that implements the binary search algorithm to locate the first occurence of a number in a list of numbers:\n\nimport math\ndef binary_search(lst, T):\n L = 0\n R = len(lst) - 1\n while L < R:\n m = math.floor((L + R) / 2)\n if lst[m] <= T:\n L = m + 1\n else:\n R = m - 1\n if lst[L] == T:\n return L\n return -1\n\nThere are a couple of things not quite right with this implementation, even though it will run and produce correct results for some cases.\nHere’s another piece of code implementing merge sort (also with some bugs in it):\n\ndef merge_sort(lst):\n n = len(lst)\n if n == 1:\n return lst\n return merge(merge_sort(lst[:n//2]), merge_sort(lst[n//2:]))\n\ndef merge(lst1, lst2):\n merged = []\n i, j = 0,0\n while i < len(lst1) and j < len(lst2):\n if i < len(lst1) and j < len(lst2) and lst1[i] < lst2[j]:\n merged.append(lst1[i])\n i += 1\n else:\n merged.append(lst1[j])\n j += 1\n while i < len(lst1):\n merged.append(lst1[i])\n i += 1\n while j < len(lst2):\n merged.append(lst2[j])\n j += 1\n return merged\n \nmerge_sort([3,1,5,1,6,3,9,12,8])\n\nYou can use this to practice stepping inside functions, and thinking about recursion.\nIncidentally, if you first sort, then find, you can get the quantile of a particular value within a collection (you’ll need to adjust the binary search a little to achieve this).\nAlternatively you could start by implementing (without using any existing functions) a function that inverst the order of the words in a string, and debug it until it works.\nHere’s a version of this function where I injected a couple of bugs, if you prefer to start from there:\n\ndef reverse_words(input):\n working = list(input)\n invert(working)\n start = 0\n for i, c in enumerate(working):\n if c == ' ' and i != start:\n invert(working, start, i)\n start = i+1\n return ''.join(working)\n\ndef invert(lst, start=None, end=None):\n if None == start:\n start = 0\n if None == end:\n end = len(lst)-1\n \n while start < end:\n tmp = lst[start]\n lst[start] = lst[start]\n lst[end] = tmp\n start += 1\n end -= 1\n\nreverse_words(\"These are my words. I have spoken!\")\n\nNext time we will touch briefly on how to do debugging without an IDE with debugger integration." + "objectID": "units/unit4-goodPractices.html#some-common-causes-of-bugs", + "href": "units/unit4-goodPractices.html#some-common-causes-of-bugs", + "title": "Good practices: coding practices, debugging, and reproducible research", + "section": "Some common causes of bugs", + "text": "Some common causes of bugs\nSome of these are Python-specific, while others are common to a variety of languages.\n\nParenthesis mis-matches\n== vs. =\nComparing real numbers exactly using == is dangerous because numbers on a computer are only represented to limited numerical precision. For example,\n::: {.cell execution_count=12} {.python .cell-code} 1/3 == 4*(4/12-3/12)\n::: {.cell-output .cell-output-display execution_count=12} False ::: :::\nYou expect a single value but execution of the code gives an array\nSilent type conversion when you don’t want it, or lack of coercion where you’re expecting it\nUsing the wrong function or variable name\nGiving unnamed arguments to a function in the wrong order\nForgetting to define a variable in the environment of a function and having Python, via lexical scoping, get that variable as a global variable from one of the enclosing scope. At best the types are not compatible and you get an error; at worst, you use a garbage value and the bug is hard to trace. In some cases your code may work fine when you develop the code (if the variable exists in the enclosing environment), but then may not work when you restart Python if the variable no longer exists or is different.\nPython (usually helpfully) drops matrix and array dimensions that are extraneous. This can sometimes confuse later code that expects an object of a certain dimension. More on this below." }, { - "objectID": "labs/py_vs_R.html", - "href": "labs/py_vs_R.html", - "title": "Lab 7: Python vs. R", - "section": "", - "text": "PDF" + "objectID": "units/unit4-goodPractices.html#tips-for-avoiding-bugs-and-catching-errors", + "href": "units/unit4-goodPractices.html#tips-for-avoiding-bugs-and-catching-errors", + "title": "Good practices: coding practices, debugging, and reproducible research", + "section": "Tips for avoiding bugs and catching errors", + "text": "Tips for avoiding bugs and catching errors\n\nPractice defensive programming\nWhen writing functions, and software more generally, you’ll want to warn the user or stop execution when there is an error and exit gracefully, giving the user some idea of what happened. Here are some things to consider:\n\ncheck function inputs and warn users if the code will do something they might not expect or makes particular choices;\ncheck inputs to if and the ranges in for loops;\nprovide reasonable default arguments;\ndocument the range of valid inputs;\ncheck that the output produced is valid; and\nstop execution based on assertions, try or raise with an informative error message.\n\nHere’s an example of building a robust square root function:\n\nimport warnings\n\ndef mysqrt(x):\n assert not isinstance(x, str), f\"what is the square root of '{x}'?\"\n if isinstance(x, int) or isinstance(x, float):\n if x < 0:\n warnings.warn(\"Input value is negative.\", UserWarning)\n return float('nan') # avoid complex number result\n else:\n return x**0.5\n else:\n raise ValueError(f\"Cannot take the square root of {x}\")\n\n\nmysqrt(3.1)\nmysqrt(-3)\ntry:\n mysqrt('hat')\nexcept Exception as error:\n print(error)\n\nwhat is the square root of 'hat'?\n\n\n/tmp/ipykernel_3887481/1815678236.py:7: UserWarning:\n\nInput value is negative.\n\n\n\n\n\nCatch run-time errors with try/except statements\nAlso, sometimes a function you call will fail, but you want to continue execution. For example, consider the stratified analysis show previously in which you take subsets of your data based on some categorical variable and fit a statistical model for each value of the categorical variable. If some of the subsets have no or very few observations, the statistical model fitting might fail. To do this, you might be using a for loop or apply. You want your code to continue and fit the model for the rest of the cases even if one (or more) of the cases cannot be fit. You can wrap the function call that may fail within the try statement and then your code won’t stop, even when an error occurs. Here’s a toy example.\n\nimport numpy as np\nimport pandas as pd\nimport random\nimport statsmodels.api\n\nnp.random.seed(2)\nn_cats = 30\nn = 80\ny = np.random.normal(size=n)\nx = np.random.normal(size=n)\ncats = [np.random.randint(0, n_cats-1) for _ in range(n)]\ndata = pd.DataFrame({'y': y, 'x': x, 'cats': cats})\n\nparams = np.full((n_cats, 2), np.nan)\nfor i in range(n_cats):\n sub = data[data['cats'] == i]\n try:\n model = statsmodels.api.OLS(sub['y'], statsmodels.api.add_constant(sub['x']))\n fit = model.fit()\n params[i, :] = fit.params.values\n except Exception as error:\n print(f\"Regression cannot be fit for stratum {i}.\")\n\nprint(params)\n\nRegression cannot be fit for stratum 7.\nRegression cannot be fit for stratum 20.\nRegression cannot be fit for stratum 24.\nRegression cannot be fit for stratum 29.\n[[ 5.52897442e-01 2.61511154e-01]\n [ 5.72564369e-01 4.37210543e-02]\n [-9.91086764e-01 2.84116572e-01]\n [-6.50606465e-01 4.26310060e-01]\n [-2.59058826e+00 -2.59058826e+00]\n [ 8.59455139e-01 -4.64514288e+00]\n [ 3.82737032e-06 3.82737032e-06]\n [ nan nan]\n [-5.55478634e-01 -1.17864561e-01]\n [-9.11601460e-02 -5.91519525e-01]\n [-7.30270153e-01 -1.99976841e-01]\n [-1.14495705e-01 -3.06421213e-02]\n [ 4.01648095e-01 9.30890661e-01]\n [ 7.88388728e-01 -1.45835443e+00]\n [ 4.08462508e+01 6.89262864e+01]\n [ 2.95467536e-01 8.80528901e-01]\n [ 1.04592517e+00 4.55379445e+00]\n [ 6.99549010e-01 -5.17503241e-01]\n [-1.75642254e+00 -8.07798224e-01]\n [-4.49033150e-02 3.53455362e-01]\n [ nan nan]\n [ 2.63097970e-01 2.63097970e-01]\n [ 1.13328314e+00 -1.39985074e-01]\n [ 1.17996663e+00 3.68770563e-01]\n [ nan nan]\n [-3.85101497e-03 -3.85101497e-03]\n [-8.04536124e-01 -5.19470059e-01]\n [-5.19200779e-01 -1.39952387e-01]\n [-9.16593858e-01 -2.67613324e-01]\n [ nan nan]]\n\n\nThe stratum with id 7 had no observations, so that call to do the regression failed, but the loop continued because we ‘caught’ the error with try. In this example, we could have checked the sample size for the subset before doing the regression, but in other contexts, we may not have an easy way to check in advance whether the function call will fail.\n\n\nMaintain dimensionality\nPython (usually helpfully) drops array dimensions that are extraneous. This can sometimes confuse later code that expects an object of a certain dimension. Here’s a work-around:\n\nimport numpy as np\nmat = np.array([[1, 2], [3, 4]])\nnp.sum(mat, axis=0) # This sums columns, as desired\n\nrow_subset = 1\nmat2 = mat[row_subset, :]\nnp.sum(mat2, axis=0) # This sums the elements, not the columns.\n\nif len(mat2.shape) != 2: # Fix dimensionality.\n mat2 = mat2.reshape(1, -1)\n\n\nnp.sum(mat2, axis=0) \n\narray([3, 4])\n\n\nIn this simple case it’s obvious that a dimension will be dropped, but in more complicated settings, this can easily occur for some inputs without the coder realizing that it may happen. Not dropping dimensions is much easier than putting checks in to see if dimensions have been dropped and having the code behave differently depending on the dimensionality.\n\n\nFind and avoid global variables\nIn general, using global variables (variables that are not created or passed into a function) results in code that is not robust. Results will change if you or a user modifies that global variable, usually without realizing/remembering that a function depends on it.\nOne ad hoc strategy is to remove objects you don’t need from Python’s global scope, to avoid accidentally using values from an old object via Python’s scoping rules. You can also run your function in a fresh session to see if it’s unable to find variables.\n\ndel x # Mimic having a fresh sesson (knowing in this case `x` is global).\n\ndef f(z):\n y = 3\n print(x + y + z)\n\ntry:\n f(2)\nexcept Exception as error:\n print(error)\n\nname 'x' is not defined\n\n\n\n\nMiscellaneous tips\n\nUse core Python functionality and algorithms already coded. Figure out if a functionality already exists in (or can be adapted from) an Python package (or potentially in a C/Fortran library/package): code that is part of standard mathematical/numerical packages will probably be more efficient and bug-free than anything you would write.\nCode in a modular fashion, making good use of functions, so that you don’t need to debug the same code multiple times. Smaller functions are easier to debug, easier to understand, and can be combined in a modular fashion (like the UNIX utilities).\nWrite code for clarity and accuracy first; then worry about efficiency. Write an initial version of the code in the simplest way, without trying to be efficient (e.g., you might use for loops even if you’re coding in Python); then make a second version that employs efficiency tricks and check that both produce the same output.\nPlan out your code in advance, including all special cases/possibilities.\nWrite tests for your code early in the process.\nBuild up code in pieces, testing along the way. Make big changes in small steps, sequentially checking to see if the code has broken on test case(s).\nBe careful that the conditions of if statements and the sequences of for loops are robust when they involve evaluating R code.\nDon’t hard code numbers - use variables (e.g., number of iterations, parameter values in simulations), even if you don’t expect to change the value, as this makes the code more readable and reduces bugs when you use the same number multiple times; e.g. speed_of_light = 3e8 or n_its = 1000.\n\nIn a future Lab, we’ll go over debugging in detail." }, { - "objectID": "labs/py_vs_R.html#instructions", - "href": "labs/py_vs_R.html#instructions", - "title": "Lab 7: Python vs. R", - "section": "Instructions", - "text": "Instructions\nPlease carefully read the full instructions before starting the assignment, as you will need to decide with your group how to organize and combine your efforts.\nYour group is expected to work on this for ~60 minutes\nIt’s OK if you don’t get through all of the questions in one hour (i.e., you can just submit what you have at the end of section), but it should be clear that you put some thought and effort into the questions your group worked on.\n\n\n\n\n\n\nTip\n\n\n\nDo your best to document your efforts and insights as you go.\n\n\n\nWe will separate into groups of (ideally) 2 or 3. I’ll try to make sure that each group has at least one person with some R experience.\nGroups should try to answer every question in the Main Questions section. Here are some options for working through these questions:\n\nOption A – Many minds, one task: The group works together, discussing and answering each question sequentially.\nOption B – Divide, consult, conquer: Individual group members work on different questions, consulting with each other as they go or as needed.\n\nAt the end of section (say, the last ~10-15 minutes), combine the solutions into one PDF. Each group member will then individually submit copies of this PDF on Gradescope.\n\nMake sure to include the name of each group member at the top of the PDF.\nHow you create the PDF is up to you but some options are:\n\n(recommended) Using Google Docs.\n\nSeparate documents, then combine: Each group member uses a separate Google Doc, then one person combines those documents into a final shared document.\nOne document: Everyone adds to a single document at the same time.\nYou can take screenshots of code / outputs that you run in Jupyter or RStudio (alternatively, the IPython or R consoles), or just copy paste code and outputs directly to the document.\nWhen your final shared document is ready, each group member will export it to PDF and make individual submission to Gradescope.\n\nCreate an R Markdown file with the code and render to PDF. This may be more of a headache to combine if each person is working separately.\n\n\n\n\nNo shows\nIf you miss lab this week, you can form a group of 2-3 students and submit this assignment on Gradescope by Monday October 17th at 10pm." + "objectID": "units/unit4-goodPractices.html#some-basic-strategies", + "href": "units/unit4-goodPractices.html#some-basic-strategies", + "title": "Good practices: coding practices, debugging, and reproducible research", + "section": "Some basic strategies", + "text": "Some basic strategies\n\nHave a directory for each project with subdirectories with meaningful and standardized names: e.g., code, data, paper. The Journal of the American Statistical Association (JASA) has a template GitHub repository with some suggestions.\nHave a file of code for pre-processing, one or more for analysis, and one for figure/table preparation.\n\nThe pre-processing may involve time-consuming steps. Save the output of the pre-processing as a file that can be read in to the analysis script.\nYou may want to name your files something like this, so there is an obvious ordering: “1-prep.py”, “2-analysis.py”, “3-figs.py”.\nHave the code file for the figures produce the exact manuscript/report figures, operating on a file (e.g., a pickle file) that contains all the objects necessary to run the figure-producing code; the code producing the pickle file should be in your analysis code file (or somewhere else sensible).\nAlternatively, use Quarto or Jupyter notebooks for your document preparation.\n\nKeep a document describing your running analysis with dates in a text file (i.e., a lab book).\nNote where data were obtained (and when, which can be helpful when publishing) and pre-processing steps in the lab book. Have data version numbers with a file describing the changes and dates (or in lab book). If possible, have all changes to data represented as code that processes the data relative to a fixed baseline dataset.\nNote what code files do what in the lab book.\nKeep track of the details of the system and software you are running your code under, e.g., operating system version, software (e.g., Python,\n\nversions, Python or R package versions, etc.\n\n\npip list and conda list will show you version numbers for installed packages." }, { - "objectID": "labs/py_vs_R.html#questions", - "href": "labs/py_vs_R.html#questions", - "title": "Lab 7: Python vs. R", - "section": "Questions", - "text": "Questions\n\nMain Questions\n\n\n\n\n\n\nTip\n\n\n\nIdeally, you will make it through all of these, although if you run out of time that is okay.\n\n\n\nDo R functions behave like pass-by-value or pass-by-reference? In other words, if you pass in an object and modify it, does that affect the value of the object in the environment from which the function was called? Check this for a scalar, a list, and an R vector.\nCan R lists and vectors be modified in place, without copying the object?\nFor this the function .Internal(inspect) will be helpful. Here’s an example for a list.\n\n#| eval: false\nx <- list(7, c('abc', 'def'), rnorm(5))\n.Internal(inspect(x))\n\n@5652776540f8 19 VECSXP g0c3 [REF(1)] (len=3, tl=0)\n@5652776b9740 14 REALSXP g0c1 [REF(3)] (len=1, tl=0) 7\n@56527580b0d8 16 STRSXP g0c2 [REF(1)] (len=2, tl=0)\n@5652776b97b0 09 CHARSXP g0c1 [REF(4),gp=0x60] [ASCII] [cached] \"abc\"\n@565275b60168 09 CHARSXP g0c1 [MARK,REF(14),gp=0x61] [ASCII] [cached] \"def\"\n@565275a97478 14 REALSXP g0c4 [REF(1)] (len=5, tl=0) -0.38248,-0.100364,-0.485605,1.15111,-0.111647\n\n#`5652776540f8` is the address of the overall list.\n#`5652776b9740` of the 1-element vector containing '7'.\n#`56527580b0d8` is the address of the character vector.\n#`565275a97478` is the address of the vector of random numbers.\n\nDoes R behave similarly to Python in terms of storing strings, as seen in PS4?\nIf you make a copy of an R vector does it use the same memory as the original vector and does changing an element of the original vector affect the copy of the vector?\nHow does variable scoping work in R - does it use lexical scoping and look for variables in the environment where a function was defined?\nCan you create a closure with embedded data, like we did in Python?\n\n\n\nAdditional Questions\n\n\n\n\n\n\nTip\n\n\n\nWork on these if you finish quickly/are curious\n\n\n\nConsider the relative efficiency of for loops versus vectorized calculations vs. apply for numeric vectors in R and see how it compares to the equivalent operations in python.\nCan you determine if the speed of looking up values in a named vector varies with the size of the dictionary (this will indicate if something like hashing is going on or if the lookup has to scan through all the elements)." + "objectID": "units/unit4-goodPractices.html#formal-tools", + "href": "units/unit4-goodPractices.html#formal-tools", + "title": "Good practices: coding practices, debugging, and reproducible research", + "section": "Formal tools", + "text": "Formal tools\n\nIn some cases you may be able to carry out your complete workflow in a Quarto document or in a Jupyter notebook.\nYou might consider workflow/pipeline management software such as Drake or other tools discussed in the CRAN Reproducible Research Task View. Alternatively, one can use the make tool, which is generally used for compiling code, as a tool for reproducible research: if interested, see the tutorial on Using make for workflows or this Journal of Statistical Software article for more details.\nYou might organize your workflow as a Python or R package as described (for the R case) in this article.\nPackage management:\n\nPython: You can manage the versions of Python packages (and dependent packages) used in your project using Conda environments (or virtualenvs).\nR: You can manage the versions of R packages (and dependent packages) used in your project using package management packages such as renv and packrat. Unfortunately, the useful checkpoint package relies on snapshots of CRAN that are not available after January 2023.\n\nIf your project uses multiple pieces of software (e.g., not just Python or R), you can set up a reproducible environment using containers, of which Docker containers are the best known. These provide something that is like a lightweight virtual machine in which you can install exactly the software (and versions) you want and then share with others. Docker container images are a key building block of various tools such as GitHub Actions and the Binder project. Alternatively Conda is a general package manager that can install lots of non-Python packages and can also be used in many circumstances." }, { - "objectID": "units/unit3-bash.html", - "href": "units/unit3-bash.html", - "title": "The bash shell and UNIX commands", + "objectID": "units/unit10-linalg.html", + "href": "units/unit10-linalg.html", + "title": "Numerical linear algebra", "section": "", - "text": "PDF\nReference:" + "text": "PDF\nReferences:\nVideos (optional):\nThere are various videos from 2020 in the bCourses Media Gallery that you can use for reference if you want to.\nIn working through how to compute something or understanding an algorithm, it can be very helpful to depict the matrices and vectors graphically. We’ll see this on the board in class." }, { - "objectID": "units/unit3-bash.html#first-challenge", - "href": "units/unit3-bash.html#first-challenge", - "title": "The bash shell and UNIX commands", - "section": "4.1 First challenge", - "text": "4.1 First challenge\nConsider the file cpds.csv. How would you write a shell command that returns “There are 8 occurrences of the word ‘Belgium’ in this file.”, where ‘8’ should instead be the correct number of times the word occurs.\nExtra: make your code into a function that can operate on any file indicated by the user and any word of interest." + "objectID": "units/unit10-linalg.html#context", + "href": "units/unit10-linalg.html#context", + "title": "Numerical linear algebra", + "section": "Context", + "text": "Context\nMany statistical and machine learning methods involve linear algebra of some sort - at the very least matrix multiplication and very often some sort of matrix decomposition to fit models and do analysis: linear regression, various more sophisticated forms of regression, deep neural networks, principle components analysis (PCA) and the wide varieties of generalizations and variations on PCA, etc., etc." }, { - "objectID": "units/unit3-bash.html#second-challenge", - "href": "units/unit3-bash.html#second-challenge", - "title": "The bash shell and UNIX commands", - "section": "4.2 Second challenge", - "text": "4.2 Second challenge\nConsider the data in the RTADataSub.csv file. This is a subset of data giving freeway travel times for segments of a freeway in an Australian city. The data are from a kaggle.com competition. We want to try to understand the kinds of data in each field of the file. The following would be particularly useful if the data were in many files or the data were many gigabytes in size.\n\nFirst, take the fourth column. Figure out the unique values in that column.\nNext, automate the process of determining if any of the values are non-numeric so that you don’t have to scan through all of the unique values looking for non-numbers. You’ll need to look for the following regular expression pattern [^0-9], which is interpreted as NOT any of the numbers 0 through 9.\n\nExtra: do it for all the fields, except the first one. Have your code print out the result in a human-readable way understandable by someone who didn’t write the code. For simplicity, you can assume you know the number of fields." + "objectID": "units/unit10-linalg.html#goals", + "href": "units/unit10-linalg.html#goals", + "title": "Numerical linear algebra", + "section": "Goals", + "text": "Goals\nHere’s what I’d like you to get out of this unit:\n\nHow to think about the computational order (number of computations involved) of a problem\nHow to choose a computational approach to a given linear algebra calculation you need to do.\nAn understanding of how issues with computer numbers (Unit 8) affect linear algebra calculations." }, { - "objectID": "units/unit3-bash.html#third-challenge", - "href": "units/unit3-bash.html#third-challenge", - "title": "The bash shell and UNIX commands", - "section": "4.3 Third challenge", - "text": "4.3 Third challenge\n\nFor Belgium, determine the minimum unemployment value (field #6) in cpds.csv in a programmatic way.\nHave what is printed out to the screen look like “Belgium 6.2”.\nNow store the unique values of the countries in a variable, first stripping out the quotation marks.\nFigure out how to automate step 1 to do the calculation for all the countries and print to the screen.\nHow would you instead store the results in a new file?" + "objectID": "units/unit10-linalg.html#key-principle", + "href": "units/unit10-linalg.html#key-principle", + "title": "Numerical linear algebra", + "section": "Key principle", + "text": "Key principle\nThe form of a mathematical expression and how it should be evaluated on a computer may be very different. Better computational approaches can increase speed and improve the numerical properties of the calculation.\n\nExample 1 (already seen in Unit 5): If \\(X\\) and \\(Y\\) are matrices and \\(z\\) is a vector, we should compute \\(X(Yz)\\) rather than \\((XY)z\\); the former is much more computationally efficient.\nExample 2: We do not compute \\((X^{\\top}X)^{-1}X^{\\top}Y\\) by computing \\(X^{\\top}X\\) and finding its inverse. In fact, perhaps more surprisingly, we may never actually form \\(X^{\\top}X\\) in some implementations.\nExample 3: Suppose I have a matrix \\(A\\), and I want to permute (switch) two rows. I can do this with a permutation matrix, \\(P\\), which is mostly zeroes. On a computer, in general I wouldn’t need to even change the values of \\(A\\) in memory in some cases (e.g., if I were to calculate \\(PAB\\)). Why not?" }, { - "objectID": "units/unit3-bash.html#fourth-challenge", - "href": "units/unit3-bash.html#fourth-challenge", - "title": "The bash shell and UNIX commands", - "section": "4.4 Fourth challenge", - "text": "4.4 Fourth challenge\nLet’s return to the RTADataSub.csv file and the issue of missing values.\n\nCreate a new file without any rows that have an ‘x’ (which indicate a missing value).\nTurn the code into a function that also prints out the number of rows that are being removed and that sends its output to stdout so that it can be used with piping.\nNow modify your function so that the user could provide the missing value string and the input filename." + "objectID": "units/unit10-linalg.html#computational-complexity", + "href": "units/unit10-linalg.html#computational-complexity", + "title": "Numerical linear algebra", + "section": "Computational complexity", + "text": "Computational complexity\nWe can assess the computational complexity of a linear algebra calculation by counting the number multiplys/divides and the number of adds/subtracts. Sidenote: addition is a bit faster than multiplication, so some algorithms attempt to trade multiplication for addition.\nIn general we do not try to count the actual number of calculations, but just their order, though in some cases in this unit we’ll actually get a more exact count. In general, we denote this as \\(O(f(n))\\) which means that the number of calculations approaches \\(cf(n)\\) as \\(n\\to\\infty\\) (i.e., we know the calculation is approximately proportional to \\(f(n)\\)). Consider matrix multiplication, \\(AB\\), with matrices of size \\(a\\times b\\) and \\(b\\times c\\). Each column of the second matrix is multiplied by all the rows of the first. For any given inner product of a row by a column, we have \\(b\\) multiplies. We repeat these operations for each column and then for each row, so we have \\(abc\\) multiplies so \\(O(abc)\\) operations. We could count the additions as well, but there’s usually an addition for each multiply, so we can usually just count the multiplys and then say there are such and such {multiply and add}s. This is Monahan’s approach, but you may see other counting approaches where one counts the multiplys and the adds separately.\nFor two symmetric, \\(n\\times n\\) matrices, this is \\(O(n^{3})\\). Similarly, matrix factorization (e.g., the Cholesky decomposition) is \\(O(n^{3})\\) unless the matrix has special structure, such as being sparse. As matrices get large, the speed of calculations decreases drastically because of the scaling as \\(n^{3}\\) and memory use increases drastically. In terms of memory use, to hold the result of the multiply indicated above, we need to hold \\(ab+bc+ac\\) total elements, which for symmetric matrices sums to \\(3n^{2}\\). So for a matrix with \\(n=10000\\), we have \\(3\\cdot10000^{2}\\cdot8/1e9=2.4\\)Gb.\nWhen we have \\(O(n^{q})\\) this is known as polynomial time. Much worse is \\(O(b^{n})\\) (exponential time), while much better is \\(O(\\log n\\)) (log time). Computer scientists talk about NP-complete problems; these are essentially problems for which there is not a polynomial time algorithm - it turns out all such problems can be rewritten such that they are equivalent to one another.\nIn real calculations, it’s possible to have the actual time ordering of two approaches differ from what the order approximations tell us. For example, something that involves \\(n^{2}\\) operations may be faster than one that involves \\(1000(n\\log n+n)\\) even though the former is \\(O(n^{2})\\) and the latter \\(O(n\\log n)\\). The problem is that the constant, \\(c=1000\\), can matter (depending on how big \\(n\\) is), as can the extra calculations from the lower order term(s), in this case \\(1000n\\).\nA note on terminology: flops stands for both floating point operations (the number of operations required) and floating point operations per second, the speed of calculation." }, { - "objectID": "units/unit3-bash.html#fifth-challenge", - "href": "units/unit3-bash.html#fifth-challenge", - "title": "The bash shell and UNIX commands", - "section": "4.5 Fifth challenge", - "text": "4.5 Fifth challenge\nConsider the coop.txt weather station file.\nFigure out how to use grep to tell you the starting position of the state field. Hints: search for a known state-country combination and figure out what flags you can use with grep to print out the “byte offset” for the matched state.\nUse that information to automate the first mission where we extracted the state field using cut. You’ll need to do a bit of arithmetic using shell commands." + "objectID": "units/unit10-linalg.html#notation-and-dimensions", + "href": "units/unit10-linalg.html#notation-and-dimensions", + "title": "Numerical linear algebra", + "section": "Notation and dimensions", + "text": "Notation and dimensions\nI’ll try to use capital letters for matrices, \\(A\\), and lower-case for vectors, \\(x\\). Then \\(x_{i}\\) is the ith element of \\(x\\), \\(A_{ij}\\) is the \\(i\\)th row, \\(j\\)th column element, and \\(A_{\\cdot j}\\) is the \\(j\\)th column and \\(A_{i\\cdot}\\) the \\(i\\)th row. By default, we’ll consider a vector, \\(x\\), to be a one-column matrix, and \\(x^{\\top}\\) to be a one-row matrix. Some of the references given at the start of this Unit also use \\(a_{ij}\\) for \\(A_{ij}\\) and \\(a_{j}\\) for the \\(j\\)th column.\nThroughout, we’ll need to be careful that the matrices involved in an operation are conformable: for \\(A+B\\) both matrices need to be of the same dimension, while for \\(AB\\) the number of columns of \\(A\\) must match the number of rows of \\(B\\). Note that this allows for \\(B\\) to be a column vector, with only one column, \\(Ab\\). Just checking dimensions is a good way to catch many errors. Example: is \\(\\mbox{Cov}(Ax)=A\\mbox{Cov}(x)A^{\\top}\\) or \\(\\mbox{Cov}(Ax)=A^{\\top}\\mbox{Cov}(x)A\\)? Well, if \\(A\\) is \\(m\\times n\\), it must be the former, as the latter is not conformable.\nThe inner product of two vectors is \\(\\sum_{i}x_{i}y_{i}=x^{\\top}y\\equiv\\langle x,y\\rangle\\equiv x\\cdot y\\).\nThe outer product is \\(xy^{\\top}\\), which comes from all pairwise products of the elements.\nWhen the indices of summation should be obvious, I’ll sometimes leave them implicit. Ask me if it’s not clear." }, { - "objectID": "units/unit3-bash.html#sixth-challenge", - "href": "units/unit3-bash.html#sixth-challenge", - "title": "The bash shell and UNIX commands", - "section": "4.6 Sixth challenge", - "text": "4.6 Sixth challenge\nHere’s an advanced one - you’ll probably need to use sed, but the brief examples of text substitution in the using bash tutorial (or in the demos above) should be sufficient to solve the problem.\nConsider a CSV file that has rows that look like this:\n1,\"America, United States of\",45,96.1,\"continental, coastal\" \n2,\"France\",33,807.1,\"continental, coastal\"\nWhile Pandas would be able to handle this using read_csv(), using cut in UNIX won’t work because of the commas embedded within the fields. The challenge is to convert this file to one that we can use cut on, as follows.\nFigure out a way to make this into a new delimited file in which the delimiter is not a comma. At least one solution that will work for this particular two-line dataset does not require you to use regular expressions, just simple replacement of fixed patterns." + "objectID": "units/unit10-linalg.html#norms", + "href": "units/unit10-linalg.html#norms", + "title": "Numerical linear algebra", + "section": "Norms", + "text": "Norms\nFor a vector, \\(\\|x\\|_{p}=(\\sum_{i}|x_{i}|^{p})^{1/p}\\) and the standard (Euclidean) norm is \\(\\|x\\|_{2}=\\sqrt{\\sum x_{i}^{2}}=\\sqrt{x^{\\top}x}\\), just the length of the vector in Euclidean space, which we’ll refer to as \\(\\|x\\|\\), unless noted otherwise.\nOne commonly used norm for a matrix is the Frobenius norm, \\(\\|A\\|_{F}=(\\sum_{i,j}a_{ij}^{2})^{1/2}\\).\nIn this Unit, we’ll often make use of the induced matrix norm, which is defined relative to a corresponding vector norm, \\(\\|\\cdot\\|\\), as: \\[\\|A\\|=\\sup_{x\\ne0}\\frac{\\|Ax\\|}{\\|x\\|}\\] So we have \\[\\|A\\|_{2}=\\sup_{x\\ne0}\\frac{\\|Ax\\|_{2}}{\\|x\\|_{2}}=\\sup_{\\|x\\|_{2}=1}\\|Ax\\|_{2}\\] If you’re not familiar with the supremum (“sup” above), you can just think of it as taking the maximum. In the case of the 2-norm, the norm turns out to be the largest singular value in the singular value decomposition (SVD) of the matrix.\nWe can interpret the norm of a matrix as the most that the matrix can stretch a vector when multiplying by the vector (relative to the length of the vector).\nA property of any legitimate matrix norm (including the induced norm) is that \\(\\|AB\\|\\leq\\|A\\|\\|B\\|\\). Also recall that norms must obey the triangle inequality, \\(\\|A+B\\|\\leq\\|A\\|+\\|B\\|\\).\nA normalized vector is one with “length”, i.e., Euclidean norm, of one. We can easily normalize a vector: \\(\\tilde{x}=x/\\|x\\|\\)\nThe angle between two vectors is \\[\\theta=\\cos^{-1}\\left(\\frac{\\langle x,y\\rangle}{\\sqrt{\\langle x,x\\rangle\\langle y,y\\rangle}}\\right)\\]" }, { - "objectID": "units/unit3-bash.html#versions-of-regular-expressions", - "href": "units/unit3-bash.html#versions-of-regular-expressions", - "title": "The bash shell and UNIX commands", - "section": "Versions of regular expressions", - "text": "Versions of regular expressions\nOne thing that can cause headaches is differences in version of regular expression syntax used. As discussed in man grep, extended regular expressions are standard, with basic regular expressions providing less functionality and Perl regular expressions additional functionality.\nThe bash shell tutorial provides a full documentation of the extended regular expressions syntax, which we’ll focus on here. This syntax should be sufficient for most usage and should be usable in Python and R, but if you notice something funny going on, it might be due to differences between the regular expressions versions.\n\nIn bash, grep -E (or egrep) enables use of the extended regular expressions.\nIn Python, the re package provides syntax “similar to” Perl.\nIn R, stringr provides ICU regular expressions (see help(regex)), which are based on Perl regular expressions.\n\nMore details about Perl regular expressions can be found in the regex Wikipedia page." + "objectID": "units/unit10-linalg.html#orthogonality", + "href": "units/unit10-linalg.html#orthogonality", + "title": "Numerical linear algebra", + "section": "Orthogonality", + "text": "Orthogonality\nTwo vectors are orthogonal if \\(x^{\\top}y=0\\), in which case we say \\(x\\perp y\\). An orthogonal matrix is a square matrix in which all of the columns are orthogonal to each other and normalized. The same holds for the rows. Orthogonal matrices can be shown to have full rank. Furthermore if \\(A\\) is orthogonal, \\(A^{\\top}A=I\\), so \\(A^{-1}=A^{\\top}\\). Given all this, the determinant of orthogonal \\(A\\) is either 1 or -1. Finally the product of two orthogonal matrices, \\(A\\) and \\(B\\), is also orthogonal since \\((AB)^{\\top}AB=B^{\\top}A^{\\top}AB=B^{\\top}B=I\\).\n\nPermutations\nSometimes we make use of matrices that permute two rows (or two columns) of another matrix when multiplied. Such a matrix is known as an elementary permutation matrix and is an orthogonal matrix with a determinant of -1. You can multiply such matrices to get more general permutation matrices that are also orthogonal. If you premultiply by \\(P\\), you permute rows, and if you postmultiply by \\(P\\) you permute columns. Note that on a computer, you wouldn’t need to actually do the multiply (and if you did, you should use a sparse matrix routine), but rather one can often just rework index values that indicate where relevant pieces of the matrix are stored (more in the next section)." }, { - "objectID": "units/unit3-bash.html#general-principles-for-working-with-regex", - "href": "units/unit3-bash.html#general-principles-for-working-with-regex", - "title": "The bash shell and UNIX commands", - "section": "General principles for working with regex", - "text": "General principles for working with regex\nThe syntax is very concise, so it’s helpful to break down individual regular expressions into the component parts to understand them. As Murrell notes, since regex are their own language, it’s a good idea to build up a regex in pieces as a way of avoiding errors just as we would with any computer code. re.findall in Python and str_detect in R’s stringr, as well as regex101.com are particularly useful in seeing what was matched to help in understanding and learning regular expression syntax and debugging your regex. As with many kinds of coding, I find that debugging my regex is usually what takes most of my time." + "objectID": "units/unit10-linalg.html#some-vector-and-matrix-properties", + "href": "units/unit10-linalg.html#some-vector-and-matrix-properties", + "title": "Numerical linear algebra", + "section": "Some vector and matrix properties", + "text": "Some vector and matrix properties\n\\(AB\\ne BA\\) but \\(A+B=B+A\\) and \\(A(BC)=(AB)C\\).\nIn Python, recall the syntax is\n\nA + B\n\n# Matrix multiplication\nnp.matmul(A, B) \nA @ B # alternative\nA.dot(B) # not recommended by the NumPy docs\n\nA * B # Hadamard (direct) product\n\nYou don’t need the spaces, but they’re nice for code readability." }, { - "objectID": "units/unit3-bash.html#challenge-problem", - "href": "units/unit3-bash.html#challenge-problem", - "title": "The bash shell and UNIX commands", - "section": "Challenge problem", - "text": "Challenge problem\nChallenge: Let’s think about what regex syntax we would need to detect any number, integer- or real-valued. Let’s start from a test-driven development perspective of writing out test cases including: - various cases we want to detect, - various tricky cases that are not numbers and we don’t want to detect, and - “corner cases” – tricky (perhaps unexpected) cases that might trip us up." + "objectID": "units/unit10-linalg.html#trace-and-determinant-of-square-matrices", + "href": "units/unit10-linalg.html#trace-and-determinant-of-square-matrices", + "title": "Numerical linear algebra", + "section": "Trace and determinant of square matrices", + "text": "Trace and determinant of square matrices\nThe trace of a matrix is the sum of the diagonal elements. For square matrices, \\(\\mbox{tr}(A+B)=\\mbox{tr}(A)+\\mbox{tr}(B)\\), \\(\\mbox{tr}(A)=\\mbox{tr}(A^{\\top})\\).\nWe also have \\(\\mbox{tr}(ABC)=\\mbox{tr}(CAB)=\\mbox{tr}(BCA)\\) - basically you can move a matrix from the beginning to the end or end to beginning, provided they are conformable for this operation. This is helpful for a couple reasons:\n\nWe can find the ordering that reduces computation the most if the individual matrices are not square.\n\\(x^{\\top}Ax=\\mbox{tr}(x^{\\top}Ax)\\) since the quadratic form, \\(x^{\\top}Ax\\), is a scalar, and this is equal to \\(\\mbox{tr}(xx^{\\top}A)\\) where \\(xx^{\\top}A\\) is a matrix. It can be helpful to be able to go back and forth between a scalar and a trace in some statistical calculations.\n\nFor square matrices, the determinant exists and we have \\(|AB|=|A||B|\\) and therefore, \\(|A^{-1}|=1/|A|\\) since \\(|I|=|AA^{-1}|=1\\). Also \\(|A|=|A^{\\top}|\\), which can be seen using the QR decomposition for \\(A\\) and understanding properties of determinants of triangular matrices (in this case \\(R\\)) and orthogonal matrices (in this case \\(Q\\)).\nFor square, invertible matrices, we have that \\((A^{-1})^{\\top}=(A^{\\top})^{-1}\\). Why? Since we have \\((AB)^{\\top}=B^{\\top}A^{\\top}\\), we have: \\[A^{\\top}(A^{-1})^{\\top}=(A^{-1}A)^{\\top}=I\\] so \\((A^{\\top})^{-1}=(A^{-1})^{\\top}\\).\n\nOther matrix multiplications\nThe Hadamard or direct product is simply multiplication of the correspoding elements of two matrices by each other. In R this is simplyA * B.\nChallenge: How can I find \\(\\mbox{tr}(AB)\\) without using A %*% B ?\nThe Kronecker product is the product of each element of one matrix with the entire other matrix”\n\\[A\\otimes B=\\left(\\begin{array}{ccc}\nA_{11}B & \\cdots & A_{1m}B\\\\\n\\vdots & \\ddots & \\vdots\\\\\nA_{n1}B & \\cdots & A_{nm}B\n\\end{array}\\right)\\]\nThe inverse of a Kronecker product is the Kronecker product of the inverses,\n\\[ B^{-1} \\otimes A^{-1} \\]\nwhich is obviously quite a bit faster because the inverse (i.e., solving a system of equations) in this special case is \\(O(n^{3}+m^{3})\\) rather than the naive approach being \\(O((nm)^{3})\\)." + }, + { + "objectID": "units/unit10-linalg.html#matrix-decompositions", + "href": "units/unit10-linalg.html#matrix-decompositions", + "title": "Numerical linear algebra", + "section": "Matrix decompositions", + "text": "Matrix decompositions\nA matrix decomposition is a re-expression of a matrix, \\(A\\), in terms of a product of two or three other, simpler matrices, where the decomposition reveals structure or relationships present in the original matrix, \\(A\\). The “simpler” matrices may be simpler in various ways, including\n\nhaving fewer rows or columns;\nbeing diagonal, triangular or sparse in some way,\nbeing orthogonal matrices.\n\nIn addition, once you have a decomposition, computation is generally easier, because of the special structure of the simpler matrices.\nWe’ll see this in great detail in Section 3." + }, + { + "objectID": "units/unit10-linalg.html#linear-independence-rank-and-basis-vectors", + "href": "units/unit10-linalg.html#linear-independence-rank-and-basis-vectors", + "title": "Numerical linear algebra", + "section": "Linear independence, rank, and basis vectors", + "text": "Linear independence, rank, and basis vectors\nA set of vectors, \\(v_{1},\\ldots v_{n}\\), is linearly independent (LIN) when none of the vectors can be represented as a linear combination, \\(\\sum c_{i}v_{i}\\), of the others for scalars, \\(c_{1},\\ldots,c_{n}\\). If we have vectors of length \\(n\\), we can have at most \\(n\\) linearly independent vectors. The rank of a matrix is the number of linearly independent rows (or columns - it’s the same), and is at most the minimum of the number of rows and number of columns. We’ll generally think about it in terms of the dimension of the column space - so we can just think about the number of linearly independent columns.\nAny set of linearly independent vectors (say \\(v_{1},\\ldots,v_{n}\\)) span a space made up of all linear combinations of those vectors (\\(\\sum_{i=1}^{n}c_{i}v_{i}\\)). The spanning vectors are known as basis vectors. We can express a vector \\(y\\) that is in the space with respect to (as a linear combination of) basis vectors as \\(y=\\sum_{i}c_{i}v_{i}\\), where if the basis vectors are normalized and orthogonal, we can find the weights as \\(c_{i}=\\langle y,v_{i}\\rangle\\).\nConsider a regression context. We have \\(p\\) covariates (\\(p\\) columns in the design matrix, \\(X\\)), of which \\(q\\leq p\\) are linearly independent covariates. This means that \\(p-q\\) of the vectors can be written as linear combos of the \\(q\\) vectors. The space spanned by the covariate vectors is of dimension \\(q\\), rather than \\(p\\), and \\(X^{\\top}X\\) has \\(p-q\\) eigenvalues that are zero. The \\(q\\) LIN vectors are basis vectors for the space - we can represent any point in the space as a linear combination of the basis vectors. You can think of the basis vectors as being like the axes of the space, except that the basis vectors are not orthogonal. So it’s like denoting a point in \\(\\Re^{q}\\) as a set of \\(q\\) numbers telling us where on each of the axes we are - this is the same as a linear combination of axis-oriented vectors.\nWhen fitting a regression, if \\(n=p=q\\), a vector of \\(n\\) observations can be represented exactly as a linear combination of the \\(p\\) basis vectors, so there is no residual and we have a single unique (and exact) solution (e.g., with \\(n=p=2\\), the observations fall exactly on the simple linear regression line). If \\(n<p\\), then we have at most \\(n\\) linearly independent covariates (the rank is at most \\(n\\)). In this case we have multiple possible solutions and the system is ill-determined (under-determined). Similarly, if \\(q<p\\) and \\(n\\geq p\\), the rank is again less than \\(p\\) and we have multiple possible solutions. Of course we usually have \\(n>p\\), so the system is overdetermined - there is no exact solution, but regression is all about finding solutions that minimize some criterion about the differences between the observations and linear combinations of the columns of the \\(X\\) matrix (such as least squares or penalized least squares). In standard regression, we project the observation vector onto the space spanned by the columns of the \\(X\\) matrix, so we find the point in the space closest to the observation vector." + }, + { + "objectID": "units/unit10-linalg.html#invertibility-singularity-rank-and-positive-definiteness", + "href": "units/unit10-linalg.html#invertibility-singularity-rank-and-positive-definiteness", + "title": "Numerical linear algebra", + "section": "Invertibility, singularity, rank, and positive definiteness", + "text": "Invertibility, singularity, rank, and positive definiteness\nFor square matrices, let’s consider how invertibility, singularity, rank and positive (or non-negative) definiteness relate.\nSquare matrices that are “regular” have an eigendecomposition, \\(A=\\Gamma\\Lambda\\Gamma^{-1}\\) where \\(\\Gamma\\) is a matrix with the eigenvectors as the columns and \\(\\Lambda\\) is a diagonal matrix of eigenvalues, \\(\\Lambda_{ii}=\\lambda_{i}\\). Symmetric matrices and matrices with unique eigenvalues are regular, as are some other matrices. The number of non-zero eigenvalues is the same as the rank of the matrix. Square matrices that have an inverse are also called nonsingular, and this is equivalent to having full rank. If the matrix is symmetric, the eigenvectors and eigenvalues are real and \\(\\Gamma\\) is orthogonal, so we have \\(A=\\Gamma\\Lambda\\Gamma^{\\top}\\). The determinant of the matrix is the product of the eigenvalues (why?), which is zero if it is less than full rank. Note that if none of the eigenvalues are zero then \\(A^{-1}=\\Gamma\\Lambda^{-1}\\Gamma^{\\top}\\).\nLet’s focus on symmetric matrices. The symmetric matrices that tend to arise in statistics are either positive definite (p.d.) or non-negative definite (n.n.d.). If a matrix is positive definite, then by definition \\(x^{\\top}Ax>0\\) for any \\(x\\). Note that if \\(\\mbox{Cov}(y)=A\\) then \\(x^{\\top}Ax=x^{\\top}\\mbox{Cov}(y)x=\\mbox{Cov}(x^{\\top}y)=\\mbox{Var}(x^{\\top}y)\\) if so positive definiteness amounts to having linear combinations of random variables (with the elements of \\(x\\) here being the weights) having positive variance. So we must have that positive definite matrices are equivalent to variance-covariance matrices (I’ll just refer to this as a variance matrix or as a covariance matrix). If \\(A\\) is p.d. then it has all positive eigenvalues and it must have an inverse, though as we’ll see, from a numerical perspective, we may not be able to compute it if some of the eigenvalues are very close to zero. In Python, numpy.linalg.eig(A)[1] is \\(\\Gamma\\), with each column a vector, and numpy.linalg.eig(A)[0] contains the (unordered) eigenvalues.\nTo summarize, here are some of the various connections between mathematical and statistical properties of positive definite matrices:\n\\(A\\) positive definite \\(\\Leftrightarrow\\) \\(A\\) is a covariance matrix \\(\\Leftrightarrow\\) \\(x^{\\top}Ax>0\\) \\(\\Leftrightarrow\\) \\(\\lambda_{i}>0\\) (positive eigenvalues) \\(\\Rightarrow\\)\\(|A|>0\\) \\(\\Rightarrow\\)\\(A\\) is invertible \\(\\Leftrightarrow\\) \\(A\\) is non singular \\(\\Leftrightarrow\\) \\(A\\) is full rank.\nAnd here are connections for positive semi-definite matrices:\n\\(A\\) positive semi-definite \\(\\Leftrightarrow\\) \\(A\\) is a constrained covariance matrix \\(\\Leftrightarrow\\) \\(x^{\\top}Ax\\geq0\\) and equal to 0 for some \\(x\\) \\(\\Leftrightarrow\\) \\(\\lambda_{i}\\geq 0\\) (non-negative eigenvalues), with at least one zero \\(\\Rightarrow\\) \\(|A|=0\\) \\(\\Leftrightarrow\\) \\(A\\) is not invertible \\(\\Leftrightarrow\\) \\(A\\) is singular \\(\\Leftrightarrow\\) \\(A\\) is not full rank." + }, + { + "objectID": "units/unit10-linalg.html#interpreting-an-eigendecomposition", + "href": "units/unit10-linalg.html#interpreting-an-eigendecomposition", + "title": "Numerical linear algebra", + "section": "Interpreting an eigendecomposition", + "text": "Interpreting an eigendecomposition\nLet’s interpret the eigendecomposition in a generative context as a way of generating random vectors. We can generate \\(y\\) s.t. \\(\\mbox{Cov}(y)=A\\) if we generate \\(y=\\Gamma\\Lambda^{1/2}z\\) where \\(\\mbox{Cov}(z)=I\\) and \\(\\Lambda^{1/2}\\) is formed by taking the square roots of the eigenvalues. So \\(\\sqrt{\\lambda_{i}}\\) is the standard deviation associated with the basis vector \\(\\Gamma_{\\cdot i}\\). That is, the \\(z\\)’s provide the weights on the basis vectors, with scaling based on the eigenvalues. So \\(y\\) is produced as a linear combination of eigenvectors as basis vectors, with the variance attributable to the basis vectors determined by the eigenvalues.\nTo go the other direction, we can project a vector \\(y\\) onto the space spanned by the eigenvectors: \\(z = (\\Gamma^{\\top}\\Gamma)^{-1}\\Gamma^{\\top}y = \\Gamma^{\\top}y\\), where the simplification of course comes from \\(\\Gamma\\) being orthogonal.\nIf \\(x^{\\top}Ax\\geq0\\) then \\(A\\) is nonnegative definite (also called positive semi-definite). In this case one or more eigenvalues can be zero. Let’s interpret this a bit more in the context of generating random vectors based on non-negative definite matrices, \\(y=\\Gamma\\Lambda^{1/2}z\\) where \\(\\mbox{Cov}(z)=I\\). Questions:\n\nWhat does it mean when one or more eigenvalue (i.e., \\(\\lambda_{i}=\\Lambda_{ii}\\)) is zero?\nSuppose I have an eigenvalue that is very small and I set it to zero? What will be the impact upon \\(y\\) and \\(\\mbox{Cov}(y)\\)?\nNow let’s consider the inverse of a covariance matrix, known as the precision matrix, \\(A^{-1}=\\Gamma\\Lambda^{-1}\\Gamma^{\\top}\\). What does it mean if a \\((\\Lambda^{-1})_{ii}\\) is very large? What if \\((\\Lambda^{-1})_{ii}\\) is very small?\n\nConsider an arbitrary \\(n\\times p\\) matrix, \\(X\\). Any crossproduct or sum of squares matrix, such as \\(X^{\\top}X\\) is positive definite (non-negative definite if \\(p>n\\)). This makes sense as it’s just a scaling of an empirical covariance matrix." + }, + { + "objectID": "units/unit10-linalg.html#generalized-inverses-optional", + "href": "units/unit10-linalg.html#generalized-inverses-optional", + "title": "Numerical linear algebra", + "section": "Generalized inverses (optional)", + "text": "Generalized inverses (optional)\nSuppose I want to find \\(x\\) such that \\(Ax=b\\). Mathematically the answer (provided \\(A\\) is invertible, i.e. of full rank) is \\(x=A^{-1}b\\).\nGeneralized inverses arise in solving equations when \\(A\\) is not full rank. A generalized inverse is a matrix, \\(A^{-}\\) s.t. \\(AA^{-}A=A\\). The Moore-Penrose inverse (the pseudo-inverse), \\(A^{+}\\), is a (unique) generalized inverse that also satisfies some additional properties. \\(x=A^{+}b\\) is the solution to the linear system, \\(Ax=b\\), that has the shortest length for \\(x\\).\nWe can find the pseudo-inverse based on an eigendecomposition (or an SVD) as \\(\\Gamma\\Lambda^{+}\\Gamma^{\\top}\\). We obtain \\(\\Lambda^{+}\\) from \\(\\Lambda\\) as follows. For values \\(\\lambda_{i}>0\\), compute \\(1/\\lambda_{i}\\). All other values are set to 0. Let’s interpret this statistically. Suppose we have a precision matrix with one or more zero eigenvalues and we want to find the covariance matrix. A zero eigenvalue means we have no precision, or infinite variance, for some linear combination (i.e., for some basis vector). We take the pseudo-inverse and assign that linear combination zero variance.\nLet’s consider a specific example. Autoregressive models are often used for smoothing (in time, in space, and in covariates). A first order autoregressive model for \\(y_{1},y_{2},\\ldots,y_{T}\\) has \\(E(y_{i}|y_{-i})=\\frac{1}{2}(y_{i-1}+y_{i+1})\\). Another way of writing the model is in time-order: \\(y_{i}=y_{i-1}+\\epsilon_{i}\\). A second order autoregressive model has \\(E(y_{i}|y_{-i})=\\frac{1}{6}(4y_{i-1}+4y_{i+1}-y_{i-2}-y_{i+2})\\). These constructions basically state that each value should be a smoothed version of its neighbors. One can figure out that the precision matrix for \\(y\\) in the first order model is \\[\\left(\\begin{array}{ccccc}\n\\ddots & & \\vdots\\\\\n-1 & 2 & -1 & 0\\\\\n\\cdots & -1 & 2 & -1 & \\dots\\\\\n& 0 & -1 & 2 & -1\\\\\n& & \\vdots & & \\ddots\n\\end{array}\\right)\\] and in the second order model is\n\\[\\left( \\begin{array}{ccccccc} \\ddots & & & \\vdots \\\\ 1 & -4 & 6 & -4 & 1 \\\\ \\cdots & 1 & -4 & 6 & -4 & 1 & \\cdots \\\\ & & 1 & -4 & 6 & -4 & 1 \\\\ & & & \\vdots \\end{array} \\right).\\]\nIf we look at the eigendecomposition of such matrices, we see that in the first order case, the eigenvalue corresponding to the constant eigenvector is zero.\n\nimport numpy as np\n\nprecMat = np.array([[1,-1,0,0,0],[-1,2,-1,0,0],[0,-1,2,-1,0],[0,0,-1,2,-1],[0,0,0,-1,1]])\ne = np.linalg.eig(precMat)\ne[0] # 4th eigenvalue is numerically zero\n\narray([3.61803399e+00, 2.61803399e+00, 1.38196601e+00, 4.97762256e-17,\n 3.81966011e-01])\n\ne[1][:,3] # constant eigenvector\n\narray([0.4472136, 0.4472136, 0.4472136, 0.4472136, 0.4472136])\n\n\nThis means we have no information about the overall level of \\(y\\). So how would we generate sample \\(y\\) vectors? We can’t put infinite variance on the constant basis vector and still generate samples. Instead we use the pseudo-inverse and assign ZERO variance to the constant basis vector. This corresponds to generating realizations under the constraint that \\(\\sum y_{i}\\) has no variation, i.e., \\(\\sum y_{i}=\\bar{y}=0\\) - you can see this by seeing that \\(\\mbox{Var}(\\Gamma_{\\cdot i}^{\\top}y)=0\\) when \\(\\lambda_{i}=0\\).\n\n# generate a realization\nevals = e[0]\nevals = 1/evals # variances\nevals[3] = 0 # generalized inverse\ny = e[1] @ ((evals ** 0.5) * np.random.normal(size = 5))\ny.sum()\n\n-2.220446049250313e-16\n\n\nIn the second order case, we have two non-identifiabilities: for the sum and for the linear component of the variation in \\(y\\) (linear in the indices of \\(y\\)).\nI could parameterize a statistical model as \\(\\mu+y\\) where \\(y\\) has covariance that is the generalized inverse discussed above. Then I allow for both a non-zero mean and for smooth variation governed by the autoregressive structure. In the second-order case, I would need to add a linear component as well, given the second non-identifiability." + }, + { + "objectID": "units/unit10-linalg.html#matrices-arising-in-regression", + "href": "units/unit10-linalg.html#matrices-arising-in-regression", + "title": "Numerical linear algebra", + "section": "Matrices arising in regression", + "text": "Matrices arising in regression\nIn regression, we work with \\(X^{\\top}X\\). Some properties of this matrix are that it is symmetric and non-negative definite (hence our use of \\((X^{\\top}X)^{-1}\\) in the OLS estimator). When is it not positive definite?\nFitted values are \\(X\\hat{\\beta}=X(X^{\\top}X)^{-1}X^{\\top}Y=HY\\). The “hat” matrix, \\(H\\), projects \\(Y\\) into the column space of \\(X\\). \\(H\\) is idempotent: \\(HH=H\\), which makes sense - once you’ve projected into the space, any subsequent projection just gives you the same thing back. \\(H\\) is singular. Why? Also, under what special circumstance would it not be singular?" + }, + { + "objectID": "units/unit10-linalg.html#storing-matrices", + "href": "units/unit10-linalg.html#storing-matrices", + "title": "Numerical linear algebra", + "section": "Storing matrices", + "text": "Storing matrices\nWe’ve discussed column-major and row-major storage of matrices. First, retrieval of matrix elements from memory is quickest when multiple elements are contiguous in memory. So in a column-major language (e.g., R, Fortran), it is best to work with values in a common column (or entire columns) while in a row-major language (e.g., Python, C) for values in a common row.\nIn some cases, one can save space (and potentially speed) by overwriting the output from a matrix calculation into the space occupied by an input. This occurs in some clever implementations of matrix factorizations." + }, + { + "objectID": "units/unit10-linalg.html#algorithms", + "href": "units/unit10-linalg.html#algorithms", + "title": "Numerical linear algebra", + "section": "Algorithms", + "text": "Algorithms\nGood algorithms can change the efficiency of an algorithm by one or more orders of magnitude, and many of the improvements in computational speed over recent decades have been in algorithms rather than in computer speed.\nMost matrix algebra calculations can be done in multiple ways. For example, we could compute \\(b=Ax\\) in either of the following ways, denoted here in pseudocode.\n\nStack the inner products of the rows of \\(A\\) with \\(x\\).\n\n\n for(i=1:n){ \n b_i = 0\n for(j=1:m){\n b_i = b_i + a_{ij} x_j\n }\n }\n\nTake the linear combination (based on \\(x\\)) of the columns of \\(A\\)\n\n\n for(i=1:n){ \n b_i = 0\n }\n for(j=1:m){\n for(i = 1:n){\n b_i = b_i + a_{ij} x_j \n }\n }\nIn this case the two approaches involve the same number of operations but the first might be better for row-major matrices (so might be how we would implement in C) and the second for column-major (so might be how we would implement in Fortran).\nChallenge: check whether the first approach is faster in Python. (Write the code just doing the outer loop and doing the inner loop using vectorized calculation.)\n\nGeneral computational issues\nThe same caveats we discussed in terms of computer arithmetic hold naturally for linear algebra, since this involves arithmetic with many elements. Good implementations of algorithms are aware of the danger of catastrophic cancellation and of the possibility of dividing by zero or by values that are near zero." + }, + { + "objectID": "units/unit10-linalg.html#ill-conditioned-problems", + "href": "units/unit10-linalg.html#ill-conditioned-problems", + "title": "Numerical linear algebra", + "section": "Ill-conditioned problems", + "text": "Ill-conditioned problems\n\nBasics\nA problem is ill-conditioned if small changes to values in the computation result in large changes in the result. This is quantified by something called the condition number of a calculation. For different operations there are different condition numbers.\nIll-conditionedness arises most often in terms of matrix inversion, so the standard condition number is the “condition number with respect to inversion”, which when using the \\(L_{2}\\) norm is the ratio of the absolute values of the largest to smallest eigenvalue. Here’s an example: \\[A=\\left(\\begin{array}{cccc}\n10 & 7 & 8 & 7\\\\\n7 & 5 & 6 & 5\\\\\n8 & 6 & 10 & 9\\\\\n7 & 5 & 9 & 10\n\\end{array}\\right).\\] The solution of \\(Ax=b\\) for \\(b=(32,23,33,31)\\) is \\(x=(1,1,1,1)\\), while the solution for \\(b+\\delta b=(32.1,22.9,33.1,30.9)\\) is \\(x+\\delta x=(9.2,-12.6,4.5,-1.1)\\), where \\(\\delta\\) is notation for a perturbation to the vector or matrix.\n\ndef norm2(x):\n return(np.sum(x**2) ** 0.5)\n\nA = np.array([[10,7,8,7],[7,5,6,5],[8,6,10,9],[7,5,9,10]])\nb = np.array([32,23,33,31])\nx = np.linalg.solve(A, b)\n\nbPerturbed = np.array([32.1, 22.9, 33.1, 30.9])\nxPerturbed = np.linalg.solve(A, bPerturbed)\n\nWhat’s going on? Some manipulations with inequalities involving the induced matrix norm (for any chosen vector norm, but we might as well just think about the Euclidean norm) (see Gentle-CS Sec. 5.1 or the derivation in class) give \\[\\frac{\\|\\delta x\\|}{\\|x\\|}\\leq\\|A\\|\\|A^{-1}\\|\\frac{\\|\\delta b\\|}{\\|b\\|}\\] where we define the condition number w.r.t. inversion as \\(\\mbox{cond}(A)\\equiv\\|A\\|\\|A^{-1}\\|\\). We’ll generally work with the \\(L_{2}\\) norm, and for a nonsingular square matrix the result is that the condition number is the ratio of the absolute values of the largest and smallest magnitude eigenvalues. This makes sense since \\(\\|A\\|_{2}\\) is the absolute value of the largest magnitude eigenvalue of \\(A\\) and \\(\\|A^{-1}\\|_{2}\\) that of the inverse of the absolute value of the smallest magnitude eigenvalue of \\(A\\).\nWe see in the code above that the large disparity in eigenvalues of \\(A\\) leads to an effect predictable from our inequality above, with the condition number helping us find an upper bound.\n\ne = np.linalg.eig(A)\nevals = e[0]\nnorm2(x - xPerturbed) ## delta x\n\n16.396950936073825\n\nnorm2(b - bPerturbed) ## delta b\n\n0.20000000000000284\n\nnorm2(x - xPerturbed)/norm2(x)\n\n8.19847546803699\n\n(evals[0]/evals[2])*norm2(b - bPerturbed)/norm2(b)\n\n9.942833687618297\n\n\nThe main use of these ideas for our purposes is in thinking about the numerical accuracy of a linear system solution (Gentle-NLA Sec 3.4). On a computer we have the system \\[(A+\\delta A)(x+\\delta x)=b+\\delta b\\] where the ‘perturbation’ is from the inaccuracy of computer numbers. Our exploration of computer numbers tells us that \\[\\frac{\\|\\delta b\\|}{\\|b\\|}\\approx10^{-p};\\,\\,\\,\\frac{\\|\\delta A\\|}{\\|A\\|}\\approx10^{-p}\\] where \\(p=16\\) for standard double precision floating points. Following Gentle, one gets the approximation\n\\[\\frac{\\|\\delta x\\|}{\\|x\\|}\\approx\\mbox{cond}(A)10^{-p},\\] so if \\(\\mbox{cond}(A)\\approx10^{t}\\), we have accuracy of order \\(10^{t-p}\\) instead of \\(10^{-p}\\). (Gentle cautions that this holds only if \\(10^{t-p}\\ll1\\)). So we can think of the condition number as giving us the number of digits of accuracy lost during a computation relative to the precision of numbers on the computer. E.g., a condition number of \\(10^{8}\\) means we lose 8 digits of accuracy relative to our original 16 on standard systems. One issue is that estimating the condition number is itself subject to numerical error and requires computation of \\(A^{-1}\\) (albeit not in the case of \\(L_{2}\\) norm with square, nonsingular \\(A\\)) but see Golub and van Loan (1996; p. 76-78) for an algorithm.\n\n\nImproving conditioning\nIll-conditioned problems in statistics often arise from collinearity of regressors. Often the best solution is not a numerical one, but re-thinking the modeling approach, as this generally indicates statistical issues beyond just the numerical difficulties.\nA general comment on improving conditioning is that we want to avoid large differences in the magnitudes of numbers involved in a calculation. In some contexts such as regression, we can center and scale the columns to avoid such differences - this will improve the condition of the problem. E.g., in simple quadratic regression with \\(x=\\{1990,\\ldots,2010\\}\\) (e.g., regressing on calendar years), we see that centering and scaling the matrix columns makes a huge difference on the condition number\n\nt1 = np.arange(1990, 2011) # naive covariate\nX1 = np.column_stack((np.ones(21), t1, t1 ** 2))\ne1 = np.linalg.eig(np.dot(X1.T, X1))\nnp.sort(e1[0])[::-1]\n\narray([3.36018564e+14, 7.69949736e+02, 2.24079720e-08])\n\nt2 = t1 - 2000 # centered\nX2 = np.column_stack((np.ones(21), t2, t2 ** 2))\ne2 = np.linalg.eig(np.dot(X2.T, X2))\nwith np.printoptions(suppress=True):\n np.sort(e2[0])[::-1]\n\narray([50677.70427505, 770. , 9.29572495])\n\nt3 = t2/10 # centered and scaled\nX3 = np.column_stack((np.ones(21), t3, t3 ** 2))\ne3 = np.linalg.eig(np.dot(X3.T, X3))\nwith np.printoptions(suppress=True):\n np.sort(e3[0])[::-1]\n\narray([24.11293487, 7.7 , 1.95366513])\n\n\nThe basic story is that simple strategies often solve the problem, and that you should be aware of the absolute and relative magnitudes involved in your calculations.\nOne rule of thumb is to try to work with numbers whose magnitude is around 1. We can often scale the values in our problem in order to do this. I.e., change the units of your variables. Instead of personal income in dollars, use personal income in thousands or hundreds of thousands of dollars." + }, + { + "objectID": "units/unit10-linalg.html#triangular-systems", + "href": "units/unit10-linalg.html#triangular-systems", + "title": "Numerical linear algebra", + "section": "Triangular systems", + "text": "Triangular systems\nAs a preface, let’s figure out how to solve \\(Ax=b\\) if \\(A\\) is upper triangular. The basic algorithm proceeds from the bottom up (and therefore is called a ‘backsolve’. We solve for \\(x_{n}\\) trivially, and then move upwards plugging in the known values of \\(x\\) and solving for the remaining unknown in each row (each equation).\n\n\\(x_{n}=b_{n}/A_{nn}\\)\nNow for \\(k<n\\), use the already computed \\(\\{x_{n},x_{n-1},\\ldots,x_{k+1}\\}\\) to calculate \\(x_{k}=\\frac{b_{k}-\\sum_{j=k+1}^{n}x_{j}A_{kj}}{A_{kk}}\\).\nRepeat for all rows.\n\nHow many multiplies and adds are done? Solving lower triangular systems is very similar and involves the same number of calculations.\nIn R, backsolve() solves upper triangular systems and forwardsolve() solves lower triangular systems:\n\nimport scipy as sp\nnp.random.seed(1)\nn = 20\nX = np.random.normal(size = (n,n))\n\n## R has the `crossprod` function, which would be more efficient\n## than having to transpose, but numpy doesn't seem to have an equivalent.\nX = X.T @ X\nb = np.random.normal(size = n)\nL = np.linalg.cholesky(X) # L is upper-triangular\nU = L.T\n\nout1 = sp.linalg.solve_triangular(L, b, lower=True)\nout2 = np.linalg.inv(L) @ b\nnp.allclose(out1, out2)\n\nTrue\n\nout3 = sp.linalg.solve_triangular(U, b, lower=False)\nout4 = np.linalg.inv(U) @ b\nnp.allclose(out1, out2)\n\nTrue\n\n\nTo reiterate the distinction between matrix inversion and solving a system of equations, when we write \\(U^{-1}b\\), what we mean on a computer is to carry out the above algorithm, not to find the inverse and then multiply.\nHere’s a good reason why.\n\nimport time\n\nnp.random.seed(1)\nn = 5000\nX = np.random.normal(size = (n,n))\n\n## R has the `crossprod` function, which would be more efficient\n## than having to transpose, but numpy doesn't seem to have an equivalent.\nX = X.T @ X\nb = np.random.normal(size = n)\nL = np.linalg.cholesky(X) # L is upper-triangular\n\nt0 = time.time()\nout1 = sp.linalg.solve_triangular(L, b, lower=True)\ntime.time() - t0\n\n0.031229019165039062\n\nt0 = time.time()\nout2 = np.linalg.inv(L) @ b\ntime.time() - t0\n\n3.1720523834228516\n\n\nThat assumes you have \\(L\\), but we’ll see in a bit that even when one accounts for the creation of \\(L\\), you don’t want to invert matrices in order to solve systems of equations." + }, + { + "objectID": "units/unit10-linalg.html#gaussian-elimination-lu-decomposition", + "href": "units/unit10-linalg.html#gaussian-elimination-lu-decomposition", + "title": "Numerical linear algebra", + "section": "Gaussian elimination (LU decomposition)", + "text": "Gaussian elimination (LU decomposition)\nGaussian elimination is a standard way of directly computing a solution for \\(Ax=b\\). It is equivalent to the LU decomposition. LU is primarily done with square matrices, but not always. Also LU decompositions do exist for some singular matrices.\nThe idea of Gaussian elimination is to convert the problem to a triangular system. In class, we’ll walk through Gaussian elimination in detail and see how it relates to the LU decomposition. I’ll describe it more briefly here. Following what we learned in algebra when we have multiple equations, we preserve the solution, \\(x\\), when we add multiples of rows (i.e., add multiples of equations) together. This amounts to doing \\(L_{1}Ax=L_{1}b\\) for a lower-triangular matrix \\(L_{1}\\) that produces all zeroes in the first column of \\(L_{1}A\\) except for the first row. We proceed to zero out values below the diagonal for the other columns of \\(A\\). The result is \\(L_{n-1}\\cdots L_{1}Ax\\equiv Ux=L_{n-1}\\cdots L_{1}b\\equiv b^{*}\\) where \\(U\\) is upper triangular. This is the forward reduction step of Gaussian elimination. Then the backward elimination step solves \\(Ux=b^{*}\\).\nIf we’re just looking for the solution of the system, we don’t need the lower-triangular factor \\(L=(L_{n-1}\\cdots L_{1})^{-1}\\) in \\(A=LU\\), but it turns out to have a simple form that is computed as we go along, it is unit lower triangular and the values below the diagonal are the negative of the values below the diagonals in \\(L_{1},\\ldots,L_{n-1}\\) (note that each \\(L_{j}\\) has non-zeroes below the diagonal only in the \\(j\\)th column). As a side note related to storage, it turns out that as we proceed, we can store the elements of \\(L\\) and \\(U\\) in the original \\(A\\) matrix, except for the implicit 1s on the diagonal of \\(L\\).\nIn class, we’ll work out the computational complexity of the LU and see that it is \\(O(n^{3})\\).\nIf we look at help(np.linalg.solve) in Python, we see that it uses *_gesv*. A Google search indicates that this is a Lapack routine that does the LU decomposition with partial pivoting and row interchanges (see below on what these are), so numpy is using the algorithm we’ve just discussed.\nWe can also explicitly get the LU decomposition in Python with scipy.linalg.lu(), though for most use cases, what we want to do is solve a system of equations. (In R, one can’t easily get the explicit LU decomposition, though solve() in R does use the LU.\nOne additional complexity is that we want to avoid dividing by very small values to avoid introducing numerical inaccuracy (we would get large values that might overwhelm whatever they are being added to, and small errors in the divisor will have large effects on the result). This can be done on the fly by interchanging equations to use the equation (row) that produces the largest value to divide by. For example in the first step, we would switch the first equation (first row) for whichever of the remaining equations has the largest value in the first column. This is called partial pivoting. The divisors are called pivots. Complete pivoting also considers interchanging columns, and while theoretically better, partial pivoting is generally sufficient and requires fewer computations. Partial pivoting can be expressed as multiplying along the way by permutation matrices, \\(P_{1},\\ldots P_{n-1}\\) that switch rows. One can show with some work that based on pivoting, we have \\(PA=LU\\), where \\(P=P_{n-1}\\cdots P_{1}\\). In the demo code, we’ll see a toy example of the impact of pivoting.\nFinally \\(|PA|=|P||A|=|L||U|=|U|\\) (why?) so \\(|A|=|U|/|P|\\) and since the determinant of each permutation matrix, \\(P_{j}\\) is -1 (except when \\(P_{j}=I\\) because we don’t need to switch rows), we just need to multiply by minus one if there is an odd number of permutations. Or if we know the matrix is non-negative definite, we just take the absolute value of \\(|U|\\). So Gaussian elimination provides a fast stable way to find the determinant." + }, + { + "objectID": "units/unit10-linalg.html#cholesky-decomposition", + "href": "units/unit10-linalg.html#cholesky-decomposition", + "title": "Numerical linear algebra", + "section": "Cholesky decomposition", + "text": "Cholesky decomposition\nWhen \\(A\\) is p.d., we can use the Cholesky decomposition to solve a system of equations. Positive definite matrices can be decomposed as \\(U^{\\top}U=A\\) where \\(U\\) is upper triangular. \\(U\\) is called a square root matrix and is unique (apart from the sign, which we fix by requiring the diagonals to be positive). One algorithm for computing \\(U\\) is:\n\n\\(U_{11}=\\sqrt{A_{11}}\\)\nFor \\(j=2,\\ldots,n\\), \\(U_{1j}=A_{1j}/U_{11}\\)\nFor \\(i=2,\\ldots,n\\),\n\n\\(U_{ii}=\\sqrt{A_{ii}-\\sum_{k=1}^{i-1}U_{ki}^{2}}\\)\nif \\(i<n\\), then for \\(j=i+1,\\ldots,n\\): \\(U_{ij}=(A_{ij}-\\sum_{k=1}^{i-1}U_{ki}U_{kj})/U_{ii}\\)\n\n\nWe can then solve a system of equations as: \\(U^{-1}(U^{\\top-1}b)\\).\nSince numpy’s cholesky gives \\(L = U^\\top\\), let’s instead solve the system as: \\(L^{\\top-1}(L^{-1}b)\\), which in Python can be done in either of the following ways:\n\nL = sp.linalg.cholesky(A)\nsp.linalg.solve_triangular(L.T, \n sp.linalg.solve_triangular(L, b, lower=True),\n lower=False)\n\nc, low = sp.linalg.cho_factor(A)\nsp.linalg.cho_solve((c, low), b)\n\nThe Cholesky has some nice advantages over the LU: (1) while both are \\(O(n^{3})\\), the Cholesky involves only half as many computations, \\(n^{3}/6+O(n^{2})\\) and (2) the Cholesky factorization has only \\((n^{2}+n)/2\\) unique values compared to \\(n^{2}+n\\) for the LU. Of course the LU is more broadly applicable. The Cholesky does require computation of square roots, but it turns out this is not too intensive. There is also a method for finding the Cholesky without square roots.\n\nUses of the Cholesky\nThe standard algorithm for generating \\(y\\sim\\mathcal{N}(0,A)\\) is:\n\nL = sp.linalg.cholesky(A)\ny = L @ np.random.normal(size = n)\n\nQuestion: where will most of the time in this two-step calculation be spent?\nIf a regression design matrix, \\(X\\), is full rank, then \\(X^{\\top}X\\) is positive definite, so we could find \\(\\hat{\\beta}=(X^{\\top}X)^{-1}X^{\\top}Y\\) using either the Cholesky or Gaussian elimination. Challenge: write efficient R code to carry out the OLS solution using either LU or Cholesky factorization.\nHowever, it turns out that the standard approach is to work with \\(X\\) using the QR decomposition rather than working with \\(X^{\\top}X\\); working with \\(X\\) is more numerically stable, though in most situations without extreme collinearity, either of the approaches will be fine.\n\n\nNumerical issues with eigendecompositions and Cholesky decompositions for positive definite matrices\nMonahan comments that in general Gaussian elimination and the Cholesky decomposition are very stable. However, in the Cholesky case, if the matrix is very ill-conditioned we can get \\(A_{ii}-\\sum_{k}U_{ki}^{2}\\) being negative and then the algorithm stops when we try to take the square root. In this case, the Cholesky decomposition does not exist numerically although it exists mathematically. It’s not all that hard to produce such a matrix, particularly when working with high-dimensional covariance matrices with large correlations.\n\nlocs = np.random.uniform(size = 100)\nrho = .1\ndists = np.abs(locs[:, np.newaxis] - locs)\nC = np.exp(-dists**2/rho**2)\ne = np.linalg.eig(C)\nnp.sort(e[0])[::-1][96:100]\n\narray([-4.16702596e-16+0.j, -5.38621518e-16+0.j, -6.26206506e-16+0.j,\n -6.98611709e-16+0.j])\n\ntry:\n L = np.linalg.cholesky(C)\nexcept Exception as error:\n print(error)\n \n\nMatrix is not positive definite\n\nvals = np.abs(e[0])\nnp.max(vals)/np.min(vals)\n\n3.135151918122864e+18\n\n\nI don’t see a way to use pivoting with the Cholesky in Python, but in R, one can do chol(C, pivot = TRUE).\nWe can think about the accuracy here as follows. Suppose we have a matrix whose diagonal elements (i.e., the variances) are order of magnitude 1 and that the true value of a \\(U_{ii}\\) is less than \\(1\\times10^{-16}\\). From the given \\(A_{ii}\\) we are subtracting \\(\\sum_{k}U_{ki}^{2}\\) and trying to calculate this very small number but we know that we can only represent the values \\(A_{ii}\\) and \\(\\sum_{k}U_{ki}^{2}\\) accurately to 16 places, so the difference is garbage starting in the 17th position and could well be negative. Now realize that \\(\\sum_{k}U_{ki}^{2}\\) is the result of a potentially large set of arithmetic operations, and is likely represented accurately to fewer than 16 places. Now if the true value of \\(U_{ii}\\) is smaller than the accuracy to which \\(\\sum_{k}U_{ki}^{2}\\) is represented, we can get a difference that is negative.\nNote that when the Cholesky fails, we can still compute an eigendecomposition, but we have negative numeric eigenvalues. Even if all the eigenvalues are numerically positive (or equivalently, we’re able to get the Cholesky), errors in small eigenvalues near machine precision could have large effects when we work with the inverse of the matrix. This is what happens when we have columns of the \\(X\\) matrix nearly collinear. We cannot statistically distinguish the effect of two (or more) covariates, and this plays out numerically in terms of unstable results.\nA strategy when working with mathematically but not numerically positive definite \\(A\\) is to set eigenvalues or singular values to zero when they get very small, which amounts to using a pseudo-inverse and setting to zero any linear combinations with very small variance. We can also use pivoting with the Cholesky and accumulate zeroes in the last \\(n-q\\) rows (for cases where we try to take the square root of a negative number), corresponding to the columns of \\(A\\) that are numerically linearly dependent. See the pivot argument to R’s chol()." + }, + { + "objectID": "units/unit10-linalg.html#qr-decomposition", + "href": "units/unit10-linalg.html#qr-decomposition", + "title": "Numerical linear algebra", + "section": "QR decomposition", + "text": "QR decomposition\n\nIntroduction\nThe QR decomposition is available for any matrix, \\(X=QR\\), with \\(Q\\) orthogonal and \\(R\\) upper triangular. If \\(X\\) is non-square, \\(n\\times p\\) with \\(n>p\\) then the leading \\(p\\) rows of \\(R\\) provide an upper triangular matrix (\\(R_{1}\\)) and the remaining rows are 0. (I’m using \\(p\\) because the QR is generally applied to design matrices in regression). In this case we really only need the first \\(p\\) columns of \\(Q\\), and we have \\(X=Q_{1}R_{1}\\), the ‘skinny’ QR (this is what R’s QR provides). For uniqueness, we can require the diagonals of \\(R\\) to be nonnegative, and then \\(R\\) will be the same as the upper-triangular Cholesky factor of \\(X^{\\top}X\\):\n\\[\\begin{aligned} X^{\\top}X & = & R^{\\top}Q^{\\top}QR \\\\ & = & R^{\\top}R\\end{aligned}.\\]\nThere are three standard approaches for computing the QR, using (1) reflections (Householder transformations), (2) rotations (Givens transformations), or (3) Gram-Schmidt orthogonalization (see below for details).\nFor \\(n\\times n\\) \\(X\\), the QR (for the Householder approach) requires \\(2n^{3}/3\\) flops, so QR is less efficient than LU or Cholesky.\nWe can also obtain the pseudo-inverse of \\(X\\) from the QR: \\(X^{+}=[R_{1}^{-1}\\,0]Q^{\\top}\\). In the case that \\(X\\) is not full-rank, there is a version of the QR that will work (involving pivoting) and we end up with some additional zeroes on the diagonal of \\(R_{1}\\).\n\n\nRegression and the QR\nOften QR is used to fit linear models, including in R. Consider the linear model in the form \\(Y=X\\beta+\\epsilon\\), finding \\(\\hat{\\beta}=(X^{\\top}X)^{-1}X^{\\top}Y\\). Let’s consider the skinny QR and note that \\(R^{\\top}\\) is invertible. Therefore, we can express the normal equations as\n\\[\n\\begin{aligned}\nX^{\\top}X\\beta & = & X^{\\top} Y \\\\\nR^{\\top}Q^{\\top}QR\\beta & = & R^{\\top}Q^{\\top} Y \\\\\nR \\beta & = & Q^{\\top} Y\n\\end{aligned}\n\\]\nand solving for \\(\\beta\\) is just a backsolve since \\(R\\) is upper-triangular. Furthermore the standard regression quantities, such as the hat matrix, the SSE, the residuals, etc. can be easily expressed in terms of \\(Q\\) and \\(R\\).\nWhy use the QR instead of the Cholesky on \\(X^{\\top}X\\)? The condition number of \\(X\\) is the square root of that of \\(X^{\\top}X\\), and the \\(QR\\) factorizes \\(X\\). Monahan has a discussion of the condition of the regression problem, but from a larger perspective, the situations where numerical accuracy is a concern are generally cases where the OLS estimators are not particularly helpful anyway (e.g., highly collinear predictors).\nWhat about computational order of the different approaches to least squares? The Cholesky is \\(np^{2}+\\frac{1}{3}p^{3}\\), an algorithm called sweeping is \\(np^{2}+p^{3}\\) , the Householder method for QR is \\(2np^{2}-\\frac{2}{3}p^{3}\\), and the modified Gram-Schmidt approach for QR is \\(2np^{2}\\). So if \\(n\\gg p\\) then Cholesky (and sweeping) are faster than the QR approaches. According to Monahan, modified Gram-Schmidt is most numerically stable and sweeping least. In general, regression is pretty quick unless \\(p\\) is large since it is linear in \\(n\\), so it may not be worth worrying too much about computational differences of the sort noted here.\n\n\nRegression and the QR in R and Python\nWe can get the Q and R matrices easily in Python.\n\nQ,R = np.linalg.qr(X)\n\nOne of the methods used by the statsmodel package in Python uses the QR to fit a regression.\nNote that by default in Python (and in R), you get the skinny QR, namely only the first \\(p\\) rows of \\(R\\) and the first \\(p\\) columns of \\(Q\\), where the latter form an orthonormal basis for the column space of \\(X\\). The remaining columns form an orthonormal basis for the null space of \\(X\\) (the space orthogonal to the column space of \\(X\\)). The analogy in regression is that we get the basis vectors for the regression, while adding the remaining columns gives us the full \\(n\\)-dimensional space of the observations.\nRegression in R uses the QR decomposition via qr(), which calls a Fortran function. qr() (and the Fortran functions that are called) is specifically designed to output quantities useful in fitting linear models.\nIn R, qr() returns the result as a list meant for use by other tools. R stores the \\(R\\) matrix in the upper triangle of $qr, while the lower triangle of $qr and $aux store the information for constructing \\(Q\\) (this relates to the Householder-related vectors \\(u\\) below). One can multiply by \\(Q\\) using qr.qy() and by \\(Q^{\\top}\\) using qr.qty(). If you want to extract \\(R\\) and \\(Q\\), the following will work:\n\nX.qr = qr(X)\nQ = qr.Q(X.qr)\nR = qr.R(X.qr) \n\nAs a side note, there are QR-based functions that provide regression-related quantities, such as qr.resid(), qr.fitted() and qr.coef(). These functions (and their Fortran counterparts) exist because one can work through the various regression quantities of interest and find their expressions in terms of \\(Q\\) and \\(R\\), with nice properties resulting from \\(Q\\) being orthogonal and \\(R\\) triangular.\n\n\nComputing the QR decomposition\nHere we’ll see some of the details of the different approaches to the QR, in part because they involve some concepts that may be useful in other contexts. I won’t expect you to see all of how this works, but please skim through this to get an idea of how things are done.\nOne approach involves reflections of vectors and a second rotations of vectors. Reflections and rotations are transformations that are performed by orthogonal matrices. The determinant of a reflection matrix is -1 and the determinant of a rotation matrix is 1. We’ll see some of the details in the demo code.\n\nQR Method 1: Reflections\nIf \\(u\\) and \\(v\\) are orthonormal vectors and \\(x\\) is in the space spanned by \\(u\\) and \\(v\\), \\(x=c_{1}u+c_{2}v\\), then \\(\\tilde{x}=-c_{1}u+c_{2}v\\) is a reflection (a Householder reflection) along the \\(u\\) dimension (since we are using the negative of that basis vector). We can think of this as reflecting across the plane perpendicular to \\(u\\). This extends simply to higher dimensions with orthonormal vectors, \\(u,v_{1},v_{2},\\ldots\\)\nSuppose we want to formulate the reflection in terms of a “Householder” matrix, \\(Q\\). It turns out that \\[Qx=\\tilde{x}\\] if \\(Q=I-2uu^{\\top}\\). \\(Q\\) has the following properties: (1) \\(Qu=-u\\), (2) \\(Qv=v\\) for \\(u^{\\top}v=0\\), (3) \\(Q\\) is orthogonal and symmetric.\nOne way to create the QR decomposition is by a series of Householder transformations that create an upper triangular \\(R\\) from \\(X\\):\n\\[\n\\begin{aligned}\nR & = & Q_{p}\\cdots Q_{1} X \\\\\nQ & = & (Q_{p}\\cdots Q_{1})^{\\top}\n\\end{aligned}\n\\]\nwhere we make use of the symmetry in defining \\(Q\\).\nBasically \\(Q_{1}\\) reflects the first column of \\(X\\) with respect to a carefully chosen \\(u\\), so that the result is all zeroes except for the first element. We want \\(Q_{1}x=\\tilde{x}=(||x||,0,\\ldots,0)\\). This can be achieved with \\(u=\\frac{x-\\tilde{x}}{||x-\\tilde{x}||}\\). Then \\(Q_{2}\\) makes the last \\(n-2\\) rows of the second column equal to zero. We’ll work through this a bit in class.\nIn the regression context, as we work through the individual transformations, \\(Q_{j}=I-2u_{j}u_{j}^{\\top}\\), we apply them to \\(X\\) and \\(Y\\) to create \\(R\\) (note this would not involve doing the full matrix multiplication - think about what calculations are actually needed) and \\(QY=Q^{\\top}Y\\), and then solve \\(R\\beta=Q^{\\top}Y\\). To find \\(\\mbox{Cov}(\\hat{\\beta})\\propto(X^{\\top}X)^{-1}=(R^{\\top}R)^{-1}=R^{-1}R^{-\\top}\\) we do need to invert \\(R\\), but it’s upper-triangular and of dimension \\(p\\times p\\). It turns out that \\(Q^{\\top}Y\\) can be partitioned into the first \\(p\\) and the last \\(n-p\\) elements, \\(z^{(1)}\\) and \\(z^{(2)}\\). The SSR is \\(\\|z^{(1)}\\|^{2}\\) and SSE is \\(\\|z^{(2)}\\|^{2}\\).\nFinal side note: if \\(X\\) is square (so \\(n=p)\\) you might wonder why we need \\(Q_{p}\\) since after \\(p-1\\) reflections, we don’t need to zero anything else out (since the last column of \\(R\\) has \\(n\\) non-zero elements). It turns out that if we go back to thinking about a Householder reflection in general, there is a lack of uniqueness in choosing \\(\\tilde{x}\\). It could either be \\((||x||,0,\\ldots,0)\\) or \\((-||x||,0,\\ldots,0)\\). For better numerical stability, one chooses from the two of those such that \\(x_{1}\\) is of the opposite sign to \\(\\tilde{x}_{1}\\), so that one avoids cancellation of numbers that may be of the same magnitude when doing \\(x-\\tilde{x}\\). The transformation \\(Q_{p}\\) is the last step of taking that approach of choosing the sign at each step. \\(Q_{p}\\) doesn’t zero anything out; it just basically just involves potentially setting \\(R_{pp}\\) to be \\(-R_{pp}\\). (To be honest, I’m not clear on why one would bother to do that last step, but that seems to be how it is presented in discussions of the Householder approach.) Of course in the case of \\(p<n\\), we definitely need \\(Q_{p}\\) so that the last \\(n-p\\) rows of \\(R\\) are zero and we can then discard them when just using the skinny QR.\n\n\nQR Method 2: Rotations\nA Givens rotation matrix rotates a vector in a two-dimensional subspace to be axis oriented with respect to one of the two dimensions by changing the value of the other dimension. E.g. we can create \\(\\tilde{x}=(x_{1},\\ldots,\\tilde{x}_{p},\\ldots,0,\\ldots x_{n})\\) from \\(x=(x_{1,}\\ldots,x_{p},\\ldots,x_{q},\\ldots,x_{n})\\) using a matrix multiplication: \\(\\tilde{x}=Qx\\). \\(Q\\) is orthogonal but not symmetric.\nWe can use a series of Givens rotations to do the QR but unless it is done carefully, more computations are needed than with Householder reflections. The basic story is that we apply a series of Givens rotations to \\(X\\) such that we zero out the lower triangular elements.\n\\[\n\\begin{aligned}\nR & = & Q_{pn}\\cdots Q_{23}Q_{1n}\\cdots Q_{13}Q_{12} X \\\\\nQ & = & (Q_{pn}\\cdots Q_{12})^{\\top}\\end{aligned}.\n\\]\nNote that we create the \\(n-p\\) zero rows in \\(R\\) (because the calculations affect the upper triangle of \\(R\\)), but we can then ignore those rows and the corresponding columns of \\(Q\\).\n\n\nQR Method 3: Gram-Schmidt Orthogonalization\nGram-Schmidt involves finding a set of orthonormal vectors to span the same space as a set of LIN vectors, \\(x_{1},\\ldots,x_{p}\\). If we take the LIN vectors to be the columns of \\(X\\), so that we are discussing the column space of \\(X\\), then G-S yields the QR decomposition. Here’s the algorithm:\n\n\\(\\tilde{x}_{1}=\\frac{x_{1}}{\\|x_{1}\\|}\\) (normalize the first vector)\nOrthogonalize the remaining vectors with respect to \\(\\tilde{x}_{1}\\):\n\n\\(\\tilde{x}_{2}=\\frac{x_{2}-\\tilde{x}_{1}^{\\top}x_{2}\\tilde{x}_{1}}{\\|x_{2}-\\tilde{x}_{1}^{\\top}x_{2}\\tilde{x}_{1}\\|}\\), which orthogonalizes with respect to \\(\\tilde{x}_{1}\\) and normalizes. Note that \\(\\tilde{x}_{1}^{\\top}x_{2}\\tilde{x}_{1}=\\langle\\tilde{x}_{1},x_{2}\\rangle\\tilde{x}_{1}\\). So we are finding a scaling, \\(c\\tilde{x}_{1}\\), where \\(c\\) is based on the inner product, to remove the variation in the \\(x_{1}\\) direction from \\(x_{2}\\).\nFor \\(k>2\\), find interim vectors, \\(x_{k}^{(2)}\\), by orthogonalizing with respect to \\(\\tilde{x}_{1}\\)\n\nProceed for \\(k=3,\\ldots\\), in turn orthogonalizing and normalizing the first of the remaining vectors w.r.t. \\(\\tilde{x}_{k-1}\\) and orthogonalizing the remaining vectors w.r.t. \\(\\tilde{x}_{k-1}\\) to get new interim vectors\n\nMathematically, we could instead orthogonalize \\(x_{2}\\) w.r.t. \\(\\tilde{x}_{1}\\), then orthogonalize \\(x_{3}\\) w.r.t. \\(\\{\\tilde{x}_{1},\\tilde{x}_{2}\\}\\), etc. The algorithm above is the modified G-S, and is known to be more numerically stable if the columns of \\(X\\) are close to collinear, giving vectors that are closer to orthogonal. The resulting \\(\\tilde{x}\\) vectors are the columns of \\(Q\\). The elements of \\(R\\) are obtained as we proceed: the diagonal values are the the normalization values in the denominators, while the off-diagonals are the inner products with the already-computed columns of \\(Q\\) that are computed as part of the numerators.\nAnother way to think about this is that \\(R=Q^{\\top}X\\), which is the same as regressing the columns of \\(X\\) on \\(Q,\\) since \\((Q^{\\top}Q)^{-1}Q^{\\top}X=Q^{\\top}X\\). By construction, the first column of \\(X\\) is a scaling of the first column of \\(Q\\), the second column of \\(X\\) is a linear combination of the first two columns of \\(Q\\), etc., so \\(R\\) being upper triangular makes sense.\n\n\n\nThe “tall-skinny” QR\nSuppose you have a very large regression problem, with \\(n\\) very large, and \\(n\\gg p\\). There is a variant of the QR, called the tall-skinny QR (see http://arxiv.org/pdf/0808.2664v1.pdf for details) that allows us to find the decomposition in a parallel fashion. The basic idea is to do a nested set of QR decompositions on blocks of rows of \\(X\\):\n\\[\nX = \\left( \\begin{array}{c}\nX_{0} \\\\\nX_{1} \\\\\nX_{2} \\\\\nX_{3}\n\\end{array}\n\\right) =\n\\left(\n\\begin{array}{c}\nQ_{0} R_{0} \\\\\nQ_{1} R_{1} \\\\\nQ_{2} R_{2} \\\\\nQ_{3} R_{3}\n\\end{array} \\right),\n\\]\nfollowed by ‘reduction’ steps (this can be done in a map-reduce context) that do the \\(QR\\) of pairs of the \\(R\\) factors: \\[\\left(\\begin{array}{c}\nR_{0}\\\\\nR_{1}\\\\\nR_{2}\\\\\nR_{3}\n\\end{array}\\right)=\\left(\\begin{array}{c}\n\\left(\\begin{array}{c}\nR_{0}\\\\\nR_{1}\n\\end{array}\\right)\\\\\n\\left(\\begin{array}{c}\nR_{2}\\\\\nR_{3}\n\\end{array}\\right)\n\\end{array}\\right)=\\left(\\begin{array}{c}\nQ_{01}R_{01}\\\\\nQ_{23}R_{23}\n\\end{array}\\right)\\] and \\[\\left(\\begin{array}{c}\nR_{01}\\\\\nR_{23}\n\\end{array}\\right)=Q_{0123}R_{0123}.\\]\nThe full decomposition is then\n\\[X=\\left( \\begin{array}{cccc} Q_{0} & 0 & 0 & 0 \\\\ 0 & Q_{1} & 0 & 0 \\\\ 0 & 0 & Q_{2} & 0 \\\\ 0 & 0 & 0 & Q_{3} \\end{array} \\right) \\left( \\begin{array}{cc} Q_{01} & 0 \\\\ 0 & Q_{23} \\end{array} \\right) Q_{0123} R_{0123} = QR.\\]\nThe computation can be done in parallel (in particular it can be done with map-reduce) and the \\(Q\\) matrix for big problems would generally not be computed explicitly but would be stored in its constituent pieces.\nAlternatively, there is a variant on the algorithm that processes the row-blocks of \\(X\\) serially, allowing you to do QR on a large tall-skinny matrix that you can’t fit in memory (or possibly even on disk). First you do \\(QR\\) on \\(X_{0}\\) to get \\(Q_{0}R_{0}\\). Then you stack \\(R_{0}\\) on top of \\(X_{1}\\) and do QR to get \\(R_{01}\\). Then stack \\(R_{01}\\) on top of \\(X_{2}\\) to get \\(R_{012}\\), etc." + }, + { + "objectID": "units/unit10-linalg.html#determinants", + "href": "units/unit10-linalg.html#determinants", + "title": "Numerical linear algebra", + "section": "Determinants", + "text": "Determinants\nThe absolute value of the determinant of a square matrix can be found from the product of the diagonals of the triangular matrix in any factorization that gives a triangular (including diagonal) matrix times an orthogonal matrix (or matrices) since the determinant of an orthogonal matrix is either one or minus one.\n\\(|A|=|QR|=|Q||R|=\\pm|R|\\)\n\\(|A^{\\top}A|=|(QR)^{\\top}QR|=|R^{\\top}R|=|R_{1}^{\\top}R_{1}|=|R_{1}|^{2}\\)\nIn R, the following will do it (on the log scale), since \\(R\\) is stored in the upper triangle of the $qr element.\n\nQ,R = qr(A)\nmagn = np.sum(np.log(np.abs(np.diag(R)))) \n\nAn alternative is the product of the diagonal elements of \\(D\\) (the singular values) in the SVD factorization, \\(A=UDV^{\\top}\\).\nFor non-negative definite matrices, we know the determinant is non-negative, so the uncertainty about the sign is not an issue. For positive definite matrices, a good approach is to use the product of the diagonal elements of the Cholesky decomposition.\nOne can also use the product of the eigenvalues: \\(|A|=|\\Gamma\\Lambda\\Gamma^{-1}|=|\\Gamma||\\Gamma^{-1}||\\Lambda|=|\\Lambda|\\)\n\nComputation\nComputing from any of these diagonal or triangular matrices as the product of the diagonals is prone to overflow and underflow, so we always work on the log scale as the sum of the log of the values. When some of these may be negative, we can always keep track of the number of negative values and take the log of the absolute values.\nOften we will have the factorization as a result of other parts of the computation, so we get the determinant for free.\nWe can use np.linalg.logdet() or (definitely not recommended) np.linalg.det() to calculate the determinant in Python. These functions use the LU decomposition." + }, + { + "objectID": "units/unit10-linalg.html#eigendecomposition", + "href": "units/unit10-linalg.html#eigendecomposition", + "title": "Numerical linear algebra", + "section": "Eigendecomposition", + "text": "Eigendecomposition\nThe eigendecomposition (spectral decomposition) is useful in considering convergence of algorithms and of course for statistical decompositions such as PCA. We think of decomposing the components of variation into orthogonal patterns (the eigenvectors) with variances (eigenvalues) associated with each pattern.\nSquare symmetric matrices have real eigenvectors and eigenvalues, with the factorization into orthogonal \\(\\Gamma\\) and diagonal \\(\\Lambda\\), \\(A=\\Gamma\\Lambda\\Gamma^{\\top}\\), where the eigenvalues on the diagonal of \\(\\Lambda\\) are ordered in decreasing value. Of course this is equivalent to the definition of an eigenvalue/eigenvector pair as a pair such that \\(Ax=\\lambda x\\) where \\(x\\) is the eigenvector and \\(\\lambda\\) is a scalar, the eigenvalue. The inverse of the eigendecomposition is simply \\(\\Gamma\\Lambda^{-1}\\Gamma^{\\top}\\). On a similar note, we can create a square root matrix, \\(\\Gamma\\Lambda^{1/2}\\), by taking the square roots of the eigenvalues.\nThe spectral radius of \\(A\\), denoted \\(\\rho(A)\\), is the maximum of the absolute values of the eigenvalues. As we saw when talking about ill-conditionedness, for symmetric matrices, this maximum is the induced norm, so we have \\(\\rho(A)=\\|A\\|_{2}\\). It turns out that \\(\\rho(A)\\leq\\|A\\|\\) for any induced matrix norm. The spectral radius comes up in determining the rate of convergence of some iterative algorithms.\n\nComputation\nThere are several methods for eigenvalues; a common one for doing the full eigendecomposition is the QR algorithm. The first step is to reduce \\(A\\) to upper Hessenburg form, which is an upper triangular matrix except that the first subdiagonal in the lower triangular part can be non-zero. For symmetric matrices, the result is actually tridiagonal. We can do the reduction using Householder reflections or Givens rotations. At this point the QR decomposition (using Givens rotations) is applied iteratively (to a version of the matrix in which the diagonals are shifted), and the result converges to a diagonal matrix, which provides the eigenvalues. It’s more work to get the eigenvectors, but they are obtained as a product of Householder matrices (required for the initial reduction) multiplied by the product of the \\(Q\\) matrices from the successive QR decompositions.\nWe won’t go into the algorithm in detail, but note that it involves manipulations and ideas we’ve seen already.\nIf only the largest (or the first few largest) eigenvalues and their eigenvectors are needed, which can come up in time series and Markov chain contexts, the problem is easier and can be solved by the power method. E.g., in a Markov chain context, steady state is reached through \\(x_{t}=A^{t}x_{0}\\). One can find the largest eigenvector by multiplying by \\(A\\) many times, normalizing at each step. \\(v^{(k)}=Az^{(k-1)}\\) and \\(z^{(k)}=v^{(k)}/\\|v^{(k)}\\|\\). There is an extension to find the \\(p\\) largest eigenvalues and their vectors. See the demo code in the qmd source file for an implementation (in R)." + }, + { + "objectID": "units/unit10-linalg.html#singular-value-decomposition", + "href": "units/unit10-linalg.html#singular-value-decomposition", + "title": "Numerical linear algebra", + "section": "Singular value decomposition", + "text": "Singular value decomposition\nLet’s consider an \\(n\\times m\\) matrix, \\(A\\), with \\(n\\geq m\\) (if \\(m>n\\), we can always work with \\(A^{\\top})\\). This often is a matrix representing \\(m\\) features of \\(n\\) observations. We could have \\(n\\) documents and \\(m\\) words, or \\(n\\) gene expression levels and \\(m\\) experimental conditions, etc. \\(A\\) can always be decomposed as \\[A=UDV^{\\top}\\] where \\(U\\) and \\(V\\) are matrices with orthonormal columns (left and right eigenvectors) and \\(D\\) is diagonal with non-negative values (which correspond to eigenvalues in the case of square \\(A\\) and to squared eigenvalues of \\(A^{\\top}A\\)).\nThe SVD can be represented in more than one way. One representation is \\[A_{n\\times m}=U_{n\\times k}D_{k\\times k}V_{k\\times m}^{\\top}=\\sum_{j=1}^{k}D_{jj}u_{j}v_{j}^{\\top}\\] where \\(u_{j}\\) and \\(v_{j}\\) are the columns of \\(U\\) and \\(V\\) and where \\(k\\) is the rank of \\(A\\) (which is at most the minimum of \\(n\\) and \\(m\\) of course). The diagonal elements of \\(D\\) are the singular values.\nThat representation is as the sum of rank-one matrices (since each term is the scaled outer product of two vectors).\nIf \\(A\\) is positive semi-definite, the eigendecomposition is an SVD. Furthermore, \\(A^{\\top}A=VD^{2}V^{\\top}\\) and \\(AA^{\\top}=UD^{2}U^{\\top}\\), so we can find the eigendecomposition of such matrices using the SVD of \\(A\\) (for \\(AA^{\\top}\\) we need to fill out \\(U\\) to have \\(n\\) columns). Note that the squares of the singular values of \\(A\\) are the eigenvalues of \\(A^{\\top}A\\) and \\(AA^{\\top}\\).\nWe can also fill out the matrices to get \\[A=U_{n\\times n}D_{n\\times m}V_{m\\times m}^{\\top}\\] where the added rows and columns of \\(D\\) are zero with the upper left block the \\(D_{k\\times k}\\) from above.\n\nUses\nThe SVD is an excellent way to determine a matrix rank and to construct a pseudo-inverse (\\(A^{+}=VD^{+}U^{\\top})\\).\nWe can use the SVD to approximate \\(A\\) by taking \\(A\\approx\\tilde{A}=\\sum_{j=1}^{p}D_{jj}u_{j}v_{j}^{\\top}\\) for \\(p<m\\). The Eckart-Minsky-Young theorem shows that the truncated SVD minimizes the Frobenius norm of \\(A-\\tilde{A}\\) over all possible rank-\\(p\\) approximations. As an example if we have a large image of dimension \\(n\\times m\\), we could hold a compressed version by a rank-\\(p\\) approximation using the SVD. The SVD is used a lot in clustering problems. For example, the Netflix prize was won based on a variant of SVD (in fact all of the top methods used variants on SVD, I believe).\nHere’s another way to think about the SVD in terms of transformations and bases. Applying the SVD to a vector, $ UDV^{top}x$, carries out the following steps:\n\n\\(V^{top}x\\) expresses \\(x\\) in terms of weights for the columns of \\(V\\).\nMultiplying the result by \\(D\\) scales/stretches the weights.\nMultiplying by \\(U\\) produces the result, which is a weighted combination of columns of \\(U\\), spanning the column-space of \\(U\\).\n\nSo applying the SVD transforms \\(x\\) from the column space of \\(V\\) to the column space of \\(U\\).\n\n\nComputation\nThe basic algorithm (Golub-Reinsch) is similar to the QR method for the eigendecomposition. We use a series of Householder transformations on the left and right to reduce \\(A\\) to an upper bidiagonal matrix, \\(A^{(0)}\\). The post-multiplications (the transformations on the right) generate the zeros in the upper triangle. (An upper bidiagonal matrix is one with non-zeroes only on the diagonal and first subdiagonal above the diagonal). Then the algorithm produces a series of upper bidiagonal matrices, \\(A^{(0)}\\), \\(A^{(1)},\\) etc. that converge to a diagonal matrix, \\(D\\) . Each step is carried out by a sequence of Givens transformations:\n\\[\n\\begin{aligned}\nA^{(j+1)} & = & R_{m-2}^{\\top} R_{m-3}^{\\top} \\cdots R_{0}^{\\top} A^{(j)} T_{0} T_{1} \\cdots T_{m-2} \\\\\n& = & RA^{(j)} T\n\\end{aligned}.\n\\]\nThis eventually gives \\(A^{(...)}=D\\) and by construction, \\(U\\) (the product of the pre-multiplied Householder matrices and the \\(R\\) matrices) and \\(V\\) (the product of the post-multiplied Householder matrices and the \\(T\\) matrices) are orthogonal. The result is then transformed by a diagonal matrix to make the elements of \\(D\\) non-negative and by permutation matrices to order the elements of \\(D\\) in nonincreasing order.\n\n\nComputation for large tall-skinny matrices\nThe SVD can also be generated from a QR decomposition. Take \\(X=QR\\) and then do an SVD on the \\(R\\) matrix to get \\(X=QUDV^{\\top}=U^{*}DV^{\\top}\\). This is particularly helpful for the case when \\(X\\) is tall and skinny (suppose \\(X\\) is \\(n\\times p\\) with \\(n\\gg p\\)), because we can do the tall-skinny QR, and the resulting SVD on \\(R\\) is easy computationally if \\(p\\) is manageable." + }, + { + "objectID": "units/unit10-linalg.html#linear-algebra-in-python", + "href": "units/unit10-linalg.html#linear-algebra-in-python", + "title": "Numerical linear algebra", + "section": "Linear algebra in Python", + "text": "Linear algebra in Python\nSpeedups and storage savings can be obtained by working with matrices stored in special formats when the matrices have special structure. E.g., we might store a symmetric matrix as a full matrix but only use the upper or lower triangle. Banded matrices and block diagonal matrices are other common formats. Banded matrices are all zero except for \\(A_{i,i+c_{k}}\\) for some small number of integers, \\(c_{k}\\). Viewed as an image, these have bands. The bands are known as co-diagonals.\nNote that for many matrix decompositions, you can change whether all of the aspects of the decomposition are returned, or just some, which may speed calculations.\nScipy provides functionality for working with matrices in various ways, including the scipy.sparse module, which provides support for structured sparse matrices such as triangular and diagonal matrices as well as unstructured sparse matrices using various standard representations.\nSome useful packages in R for matrices are Matrix, spam, and bdsmatrix. Matrix can represent a variety of rectangular matrices, including triangular, orthogonal, diagonal, etc. and provides methods for various matrix calculations that are specific to the matrix type. spam handles general sparse matrices with fast matrix calculations, in particular a fast Cholesky decomposition. bdsmatrix focuses on block-diagonal matrices, which arise frequently in contexts where there is clustering that induces within-cluster correlation and cross-cluster independence.\nIn general, matrix operations in Python and R go to compiled C or Fortran code without much intermediate Python or R code, so they can actually be pretty efficient and are based on the best algorithms developed by numerical experts. The core libraries that are used are LAPACK and BLAS (the Linear Algebra PACKage and the Basic Linear Algebra Subroutines). As we’ve discussed in the parallelization unit, one way to speed up code that relies heavily on linear algebra is to make sure you have a BLAS library tuned to your machine. These include OpenBLAS (open source), Intel’s MKL, AMD’s ACML, and Apple’s vecLib.\nIf you use Conda, numpy will generally be linked against MKL or OpenBLAS (this will depend on the locations online of the packages being installed, i.e., the channel(s) used). With pip, numpy will generally be linked against OpenBLAS. it’s possible to install numpy so that it uses OpenBLAS. R can be linked to the shared object library file (.so file or .dylib on a Mac) for a fast BLAS. These BLAS libraries are also available in threaded versions that farm out the calculations across multiple cores or processors that share memory. More details are available in this SCF documentation.\nBLAS routines do vector operations (level 1), matrix-vector operations (level 2), and dense matrix-matrix operations (level 3). Often the name of the routine has as its first letter “d”, “s”, “c” to indicate the routine is double precision, single precision, or complex. LAPACK builds on BLAS to implement standard linear algebra routines such as eigendecomposition, solutions of linear systems, a variety of factorizations, etc." + }, + { + "objectID": "units/unit10-linalg.html#sparse-matrices", + "href": "units/unit10-linalg.html#sparse-matrices", + "title": "Numerical linear algebra", + "section": "Sparse matrices", + "text": "Sparse matrices\nAs an example of exploiting sparsity, we can use a standard format (CSR = compressed sparse row) in the Scipy sparse module:\nConsider the matrix to be row-major and store the non-zero elements in order in an array called data. Then create a array called indptr that stores the position of the first element of each row. Finally, have a array, indices that tells the column identity of each element.\n\nimport scipy.sparse as sparse\nmat = np.array([[0,0,1,0,10],[0,0,0,100,0],[0,0,0,0,0],[1000,0,0,0,0]])\nmat = sparse.csr_array(mat)\nmat.data\n\narray([ 1, 10, 100, 1000])\n\nmat.indices # column indices\n\narray([2, 4, 3, 0], dtype=int32)\n\nmat.indptr # row pointers\n\n## Ideally don't first construct the dense matrix if it is large.\n\narray([0, 2, 3, 3, 4], dtype=int32)\n\nmat2 = sparse.csr_array((mat.data, mat.indices, mat.indptr))\nmat2.toarray()\n\narray([[ 0, 0, 1, 0, 10],\n [ 0, 0, 0, 100, 0],\n [ 0, 0, 0, 0, 0],\n [1000, 0, 0, 0, 0]])\n\n\nThat’s also how things are done in the spam package in R.```\nWe can do a fast matrix multiply, \\(x = Ab\\), as follows in pseudo-code:\n for(i in 1:nrows(A)){\n x[i] = 0\n # should also check that row is not empty...\n for(j in (rowpointers[i]:(rowpointers[i+1]-1)) {\n x[i] = x[i] + entries[j] * b[colindices[j]]\n } \n }\nHow many computations have we done? Only \\(k\\) multiplies and \\(O(k)\\) additions where \\(k\\) is the number of non-zero elements of \\(A\\). Compare this to the usual \\(O(n^{2})\\) for dense multiplication.\nNote that for the Cholesky of a sparse matrix, if the sparsity pattern is fixed, but the entries change, one can precompute an optimal re-ordering that retains as much sparsity in \\(U\\) as possible. Then multiple Cholesky decompositions can be done more quickly as the entries change.\n\nBanded matrices\nSuppose we have a banded matrix \\(A\\) where the lower bandwidth is \\(p\\), namely \\(A_{ij}=0\\) for \\(i>j+p\\) and the upper bandwidth is \\(q\\) (\\(A_{ij}=0\\) for \\(j>i+q\\)). An alternative to reducing to \\(Ux=b^{*}\\) is to compute \\(A=LU\\) and then do two solutions, \\(U^{-1}(L^{-1}b)\\). One can show that the computational complexity of the LU factorization is \\(O(npq)\\) for banded matrices, while solving the two triangular systems is \\(O(np+nq)\\), so for small \\(p\\) and \\(q\\), the speedup can be dramatic.\nBanded matrices come up in time series analysis. E.g., moving average (MA) models produce banded covariance structures because the covariance is zero after a certain number of lags." + }, + { + "objectID": "units/unit10-linalg.html#low-rank-updates-optional", + "href": "units/unit10-linalg.html#low-rank-updates-optional", + "title": "Numerical linear algebra", + "section": "Low rank updates (optional)", + "text": "Low rank updates (optional)\nA transformation of the form \\(A-uv^{\\top}\\) is a rank-one update because \\(uv^{\\top}\\) is of rank one.\nMore generally a low rank update of \\(A\\) is \\(\\tilde{A}=A-UV^{\\top}\\) where \\(U\\) and \\(V\\) are \\(n\\times m\\) with \\(n\\geq m\\). The Sherman-Morrison-Woodbury formula tells us that \\[\\tilde{A}^{-1}=A^{-1}+A^{-1}U(I_{m}-V^{\\top}A^{-1}U)^{-1}V^{\\top}A^{-1}\\] so if we know \\(x_{0}=A^{-1}b\\), then the solution to \\(\\tilde{A}x=b\\) is \\(x+A^{-1}U(I_{m}-V^{\\top}A^{-1}U)^{-1}V^{\\top}x\\). Provided \\(m\\) is not too large, and particularly if we already have a factorization of \\(A\\), then \\(A^{-1}U\\) is not too bad computationally, and \\(I_{m}-V^{\\top}A^{-1}U\\) is \\(m\\times m\\). As a result \\(A^{-1}(U(\\cdots)^{-1}V^{\\top}x)\\) isn’t too bad.\nThis also comes up in working with precision matrices in Bayesian problems where we may have \\(A^{-1}\\) but not \\(A\\) (we often add precision matrices to find conditional normal distributions). An alternative expression for the formula is \\(\\tilde{A}=A+UCV^{\\top}\\), and the identity tells us \\[\\tilde{A}^{-1}=A^{-1}-A^{-1}U(C^{-1}+V^{\\top}A^{-1}U)^{-1}V^{\\top}A^{-1}\\]\nBasically Sherman-Morrison-Woodbury gives us matrix identities that we can use in combination with our knowledge of smart ways of solving systems of equations." + }, + { + "objectID": "units/unit9-sim.html", + "href": "units/unit9-sim.html", + "title": "Simulation", + "section": "", + "text": "PDF\nReferences:\nMany (most?) statistical papers include a simulation (i.e., Monte Carlo) study. Many papers on machine learning methods also include a simulation study. The basic idea is that closed-form mathematical analysis of the properties of a statistical or machine learning method/model is often hard to do. Even if possible, it usually involves approximations or simplifications. A canonical situation in statistics is that we have an asymptotic result and we want to know what happens in finite samples, but often we do not even have the asymptotic result. Instead, we can estimate mathematical expressions using random numbers. So we design a simulation study to evaluate the method/model or compare multiple methods. The result is that the researcher carries out an experiment (on the computer, sometimes called in silico), generally varying different factors to see what has an effect on the outcome of interest.\nThe basic strategy generally involves simulating data and then using the method(s) on the simulated data, summarizing the results to assess/compare the method(s).\nMost simulation studies aim to approximate an integral, generally an expected value (mean, bias, variance, MSE, probability, etc.). In low dimensions, methods such as Gaussian quadrature are best for estimating an integral but these methods don’t scale well, so in higher dimensions (e.g., the usual situation with \\(n\\) observations) we often use Monte Carlo techniques.\nTo be more concrete:" + }, + { + "objectID": "units/unit9-sim.html#motivating-example", + "href": "units/unit9-sim.html#motivating-example", + "title": "Simulation", + "section": "Motivating example", + "text": "Motivating example\nLet’s consider linear regression, with observations \\(Y=(y_{1},y_{2},\\ldots,y_{n})\\) and an \\(n\\times p\\) matrix of predictors/covariates/features/variables \\(X\\), where \\(\\hat{\\beta}=(X^{\\top}X)^{-1}X^{\\top}Y\\). If we assume that we have \\(EY=X\\beta\\) and \\(\\mbox{Var}(Y)=\\sigma^{2}I\\), then we can determine analytically that we have \\[\\begin{aligned}\nE\\hat{\\beta} & = & \\beta\\\\\n\\mbox{Var}(\\hat{\\beta})=E((\\hat{\\beta}-E\\hat{\\beta})^{2}) & = & \\sigma^{2}(X^{\\top}X)^{-1}\\\\\n\\mbox{MSPE}(Y^{*})=E(Y^{*}-\\hat{Y})^{2}) & = & \\sigma^{2}(1+X^{*\\top}(X^{\\top}X)^{-1}X^{*}).\\end{aligned}\\] where \\(Y^{*}\\)is some new observation we’d like to predict given \\(X^{*}\\).\nBut suppose that we’re interested in the properties of standard regression estimation when in reality the mean is not linear in \\(X\\) or the properties of the errors are more complicated than having independent homoscedastic errors. (This is always the case, but the issue is how far from the truth the standard assumptions are.) Or suppose we have a modified procedure to produce \\(\\hat{\\beta}\\), such as a procedure that is robust to outliers. In those cases, we cannot compute the expectations above analytically.\nInstead we decide to use a Monte Carlo estimate. To keep the notation more simple, let’s just consider one element of the vector \\(\\beta\\) (i.e., one of the regression coefficients) and continue to call that \\(\\beta\\). If we randomly generate \\(m\\) different datasets from some distribution \\(f\\), and \\(\\hat{\\beta}_{i}\\) is the estimated coefficient based on the \\(i\\)th dataset: \\(Y_{i}=(y_{i1},y_{i2},\\ldots,y_{in})\\), then we can estimate \\(E\\hat{\\beta}\\) under that distribution \\(f\\) as \\[\\hat{E}(\\hat{\\beta})=\\bar{\\hat{\\beta}}=\\frac{1}{m}\\sum_{i=1}^{m}\\hat{\\beta}_{i}\\] Or to estimate the variance, we have \\[\\widehat{\\mbox{Var}}(\\hat{\\beta})=\\frac{1}{m}\\sum_{i=1}^{m}(\\hat{\\beta}_{i}-\\bar{\\hat{\\beta}})^{2}.\\] In evaluating the performance of regression under non-standard conditions or the performance of our robust regression procedure, what decisions do we have to make to be able to carry out our Monte Carlo procedure?\nNext let’s think about Monte Carlo methods in general." + }, + { + "objectID": "units/unit9-sim.html#monte-carlo-mc-basics", + "href": "units/unit9-sim.html#monte-carlo-mc-basics", + "title": "Simulation", + "section": "Monte Carlo (MC) basics", + "text": "Monte Carlo (MC) basics\n\nMonte Carlo overview\nThe basic idea is that we often want to estimate \\(\\phi\\equiv E_{f}(h(Y))\\) for \\(Y\\sim f\\). Note that if \\(h\\) is an indicator function, this includes estimation of probabilities, e.g., for a scalar \\(Y\\), we have \\(p=P(Y\\leq y)=F(y)=\\int_{-\\infty}^{y}f(t)dt=\\int I(t\\leq y)f(t)dt=E_{f}(I(Y\\leq y))\\). We would estimate variances or MSEs by having \\(h\\) involve squared terms.\nWe get an MC estimate of \\(\\phi\\) based on an iid sample of a large number of values of \\(Y\\) from \\(f\\): \\[\\hat{\\phi}=\\frac{1}{m}\\sum_{i=1}^{m}h(Y_{i}),\\] which is justified by the Law of Large Numbers: \\[\\lim_{m\\to\\infty}\\frac{1}{m}\\sum_{i=1}^{m}h(Y_{i})=E_{f}h(Y).\\]\nNote that in most simulation studies, \\(Y\\) is an entire dataset (predictors/covariates), and the “iid sample” means generating \\(m\\) different datasets from \\(f\\), i.e., \\(Y_{i}\\in\\{Y_{1},\\ldots,Y_{m}\\}\\) not \\(m\\) different scalar values. If the dataset has \\(n\\) observations, then \\(Y_{i}=(Y_{i1},\\ldots,Y_{in})\\).\n\nBack to the regression example\nLet’s relate that back to our regression example. In that particular case, if we’re interested in whether the regression estimator is biased, we want to know: \\[\\phi=E\\hat{\\beta},\\] where \\(h(Y) = \\hat{\\beta}(Y)\\). We can use the Monte Carlo estimate of \\(\\phi\\): \\[\\hat{\\phi}=\\frac{1}{m}\\sum_{i=1}^{m}h(Y_{i})=\\frac{1}{m}\\sum_{i=1}^{m}\\hat{\\beta}_{i}=\\widehat{E(\\hat{\\beta})}.\\]\nIf we are interested in the variance of the regression estimator, we have\n\\[\\phi=\\mbox{Var}(\\hat{\\beta})=E_{f}((\\hat{\\beta}-E\\hat{\\beta})^{2})\\] and we can use the Monte Carlo estimate of \\(\\phi\\): \\[\\hat{\\phi}=\\frac{1}{m}\\sum_{i=1}^{m}h(Y_{i})=\\frac{1}{m}\\sum_{i=1}^{m}(\\hat{\\beta}_{i}-E\\hat{\\beta})^{2}=\\widehat{\\mbox{Var}(\\hat{\\beta)}}\\] where \\[h(Y)=(\\hat{\\beta}-E\\hat{\\beta})^{2}.\\]\nFinally note that we also need to use the Monte Carlo estimate of \\(E\\hat{\\beta}\\) in the Monte Carlo estimation of the variance.\nWe might also be interested in the coverage of a confidence interval. In that case we have \\[h(Y)=1_{\\beta\\in CI(Y)}\\] and we can estimate the coverage as \\[\\hat{\\phi}=\\frac{1}{m}\\sum_{i=1}^{m}1_{\\beta\\in CI(y_{i})}.\\] Of course we want that \\(\\hat{\\phi}\\approx1-\\alpha\\) for a \\(100(1-\\alpha)\\) confidence interval. In the standard case of a 95% interval we want \\(\\hat{\\phi}\\approx0.95\\).\n\n\n\nSimulation uncertainty (i.e., Monte Carlo uncertainty)\nSince \\(\\hat{\\phi}\\) is simply an average of \\(m\\) identically-distributed values, \\(h(Y_{1}),\\ldots,h(Y_{m})\\), the simulation variance of \\(\\hat{\\phi}\\) is \\(\\mbox{Var}(\\hat{\\phi})=\\sigma^{2}/m\\), with \\(\\sigma^{2}=\\mbox{Var}(h(Y))\\). An estimator of \\(\\sigma^{2}=E_{f}((h(Y)-\\phi)^{2})\\) is \\[\\begin{aligned}\n\\hat{\\sigma}^{2} & = & \\frac{1}{m-1}\\sum_{i=1}^{m}(h(Y_{i})-\\hat{\\phi})^{2}\\end{aligned}\\] So our MC simulation error is based on \\[\\widehat{\\mbox{Var}}(\\hat{\\phi})=\\frac{\\hat{\\sigma}^{2}}{m}=\\frac{1}{m(m-1)}\\sum_{i=1}^{m}(h(Y_{i})-\\hat{\\phi})^{2}.\\] Note that this is particularly confusing if we have \\(\\hat{\\phi}=\\widehat{\\mbox{Var}(\\hat{\\beta})}\\) because then we have \\(\\widehat{\\mbox{Var}}(\\hat{\\phi})=\\widehat{\\mbox{Var}}(\\widehat{\\mbox{Var}(\\hat{\\beta})})\\)!\nThe simulation variance is \\(O(\\frac{1}{m})\\) because we have \\(m^{2}\\) in the denominator and a sum over \\(m\\) terms in the numerator.\nNote that in the simulation setting, the randomness in the system is very well-defined (as it is in survey sampling, but unlike in most other applications of statistics), because it comes from the RNG that we perform as part of our attempt to estimate \\(\\phi\\). Happily, we are in control of \\(m\\), so in principle we can reduce the simulation error to as little as we desire. Unhappily, as usual, the simulation standard error goes down with the square root of \\(m\\).\n\nImportant: This is the uncertainty in our simulation-based estimate of some quantity (expectation) of interest. It is NOT the statistical uncertainty in a problem.\n\n\nBack to the regression example\nSome examples of simulation variances we might be interested in in the regression example include:\n\nUncertainty in our estimate of bias: \\(\\widehat{\\mbox{Var}}(\\widehat{E(\\hat{\\beta})}-\\beta)\\).\nUncertainty in the estimated variance of the estimated coefficient: \\(\\widehat{\\mbox{Var}}(\\widehat{\\mbox{Var}(\\hat{\\beta})})\\).\nUncertainty in the estimated mean square prediction error: \\(\\widehat{\\mbox{Var}}(\\widehat{\\mbox{MSPE}(Y^{*})})\\).\n\nIn all cases we have to estimate the simulation variance, hence the \\(\\widehat{\\mbox{Var}}()\\) notation.\n\n\n\nFinal notes\nSometimes the \\(Y_{i}\\) are generated in a dependent fashion (e.g., sequential MC or MCMC), in which case this variance estimator, \\(\\widehat{\\mbox{Var}}(\\hat{\\phi})\\) does not hold because the samples are not IID, but the estimator \\(\\hat{\\phi}\\) is still a valid, unbiased estimator of \\(\\phi\\)." + }, + { + "objectID": "units/unit9-sim.html#variance-reduction-optional", + "href": "units/unit9-sim.html#variance-reduction-optional", + "title": "Simulation", + "section": "Variance reduction (optional)", + "text": "Variance reduction (optional)\nThere are some tools for variance reduction in MC settings. One is importance sampling (see Section 3). Others are the use of control variates and antithetic sampling. I haven’t personally run across these latter in practice, so I’m not sure how widely used they are and won’t go into them here.\nIn some cases we can set up natural strata, for which we know the probability of being in each stratum. Then we would estimate \\(\\mu\\) for each stratum and combine the estimates based on the probabilities. The intuition is that we remove the variability in sampling amongst the strata from our simulation.\nAnother strategy that comes up in MCMC contexts is Rao-Blackwellization. Suppose we want to know \\(E(h(X))\\) where \\(X=\\{X_{1},X_{2}\\}\\). Iterated expectation tells us that \\(E(h(X))=E(E(h(X)|X_{2})\\). If we can compute \\(E(h(X)|X_{2})=\\int h(x_{1},x_{2})f(x_{1}|x_{2})dx_{1}\\) then we should avoid introducing stochasticity related to the \\(X_{1}\\) draw (since we can analytically integrate over that) and only average over stochasticity from the \\(X_{2}\\) draw by estimating \\(E_{X_{2}}(E(h(X)|X_{2})\\). The estimator is \\[\\hat{\\mu}_{RB}=\\frac{1}{m}\\sum_{i=1}^{m}E(h(X)|X_{2,i})\\] where we either draw from the marginal distribution of \\(X_{2}\\), or equivalently, draw \\(X\\), but only use \\(X_{2}\\). Our MC estimator averages over the simulated values of \\(X_{2}\\). This is called Rao-Blackwellization because it relates to the idea of conditioning on a sufficient statistic. It has lower variance because the variance of each term in the sum of the Rao-Blackwellized estimator is \\(\\mbox{Var}(E(h(X)|X_{2})\\), which is less than the variance in the usual MC estimator, \\(\\mbox{Var}(h(X))\\), based on the usual iterated variance formula: \\(V(X)=E(V(X|Y))+V(E(X|Y))\\Rightarrow V(E(X|Y))<V(X)\\)." + }, + { + "objectID": "units/unit9-sim.html#basic-steps-of-a-simulation-study", + "href": "units/unit9-sim.html#basic-steps-of-a-simulation-study", + "title": "Simulation", + "section": "Basic steps of a simulation study", + "text": "Basic steps of a simulation study\n\nSpecify what makes up an individual experiment (i.e., the individual simulated dataset) given a specific set of inputs: sample size, distribution(s) to use, parameter values, statistic of interest, etc. In other words, exactly how would you generate one simulated dataset?\nOften you’ll want to see how your results will vary if you change some of the inputs; e.g., sample sizes, parameter values, data generating mechanisms. So determine what factors you’ll want to vary. Each unique combination of input values will be a scenario.\nWrite code to carry out the individual experiment and return the quantity of interest, with arguments to your code being the inputs that you want to vary.\nFor each combination of inputs you want to explore (each scenario), repeat the experiment \\(m\\) times. Note this is an easily parallel calculation (in both the data generating dimension and the inputs dimension(s)).\nSummarize the results for each scenario, quantifying simulation uncertainty.\nReport the results in graphical or tabular form.\n\nOften a simulation study will compare multiple methods, so you’ll need to do steps 3-6 for each method." + }, + { + "objectID": "units/unit9-sim.html#various-considerations", + "href": "units/unit9-sim.html#various-considerations", + "title": "Simulation", + "section": "Various considerations", + "text": "Various considerations\nSince a simulation study is an experiment, we should use the same principles of design and analysis we would recommend when advising a practicioner on setting up a scientific experiment.\nThese include efficiency, reporting of uncertainty, reproducibility and documentation.\nIn generating the data for a simulation study, we want to think about what structure real data would have that we want to mimic in the simulation study: distributional assumptions, parameter values, dependence structure, outliers, random effects, sample size (\\(n\\)), etc.\nAll of these may become input variables in a simulation study. Often we compare two or more statistical methods conditioning on the data context and then assess whether the differences between methods vary with the data context choices. E.g., if we compare an MLE to a robust estimator, which is better under a given set of choices about the data generating mechanism and how sensitive is the comparison to changing the features of the data generating mechanism? So the “treatment variable” is the choice of statistical method. We’re then interested in sensitivity to the conditions (different input values).\nOften we can have a large number of replicates (\\(m\\)) because the simulation is fast on a computer, so we can sometimes reduce the simulation error to essentially zero and thereby avoid reporting uncertainty. To do this, we need to calculate the simulation standard error, generally, \\(s/\\sqrt{m}\\) and see how it compares to the effect sizes. This is particularly important when reporting on the bias of a statistical method.\nWe might denote the data, which could be the statistical estimator under each of two methods as \\(Y_{ijklq}\\), where \\(q\\) indexes treatment, \\(j,k,l\\) index different additional input variables, and \\(i\\in\\{1,\\ldots,m\\}\\) indexes the replicate. E.g., \\(j\\) might index whether the data are from a t or normal, \\(k\\) the value of a parameter, and \\(l\\) the dataset sample size (i.e., different levels of \\(n\\)).\nOne can think about choosing \\(m\\) based on a basic power calculation, though since we can always generate more replicates, one might just proceed sequentially and stop when the precision of the results is sufficient.\nWhen comparing methods, it’s best to use the same simulated datasets for each level of the treatment variable and to do an analysis that controls for the dataset (i.e., for the random numbers used), thereby removing some variability from the error term. A simple example is to do a paired analysis, where we look at differences between the outcome for two statistical methods, pairing based on the simulated dataset.\nOne can even use the “same” random number generation for the replicates under different conditions. E.g., in assessing sensitivity to a \\(t\\) vs. normal data generating mechanism, we might generate the normal RVs and then for the \\(t\\) use the same random numbers, in the sense of using the same quantiles of the \\(t\\) as were generated for the normal - this is pretty easy, as seen below. This helps to control for random differences between the datasets.\n\nfrom scipy.stats import t, norm\n\ndevs = np.random.normal(size=100)\ntdevs = t.ppf(norm.cdf(devs), df=1)\n\nplt.scatter(devs, tdevs)\nplt.xlabel('devs'); plt.ylabel('tdevs')\nplt.plot([min(devs), max(devs)], [min(devs), max(devs)], color='red')\nplt.show()" + }, + { + "objectID": "units/unit9-sim.html#experimental-design-optional", + "href": "units/unit9-sim.html#experimental-design-optional", + "title": "Simulation", + "section": "Experimental Design (optional)", + "text": "Experimental Design (optional)\nA typical context is that one wants to know the effect of multiple input variables on some outcome. Often, scientists, and even statisticians doing simulation studies will vary one input variable at a time. As we know from standard experimental design, this is inefficient.\nThe standard strategy is to discretize the inputs, each into a small number of levels. If we have a small enough number of inputs and of levels, we can do a full factorial design (potentially with replication). For example if we have three inputs and three levels each, we have \\(3^{3}\\) different treatment combinations. Choosing the levels in a reasonable way is obviously important.\nAs the number of inputs and/or levels increases to the point that we can’t carry out the full factorial, a fractional factorial is an option. This carefully chooses which treatment combinations to omit. The goal is to achieve balance across the levels in a way that allows us to estimate lower level effects (in particular main effects) but not all high-order interactions. What happens is that high-order interactions are aliased to (confounded with) lower-order effects. For example you might choose a fractional factorial design so that you can estimate main effects and two-way interactions but not higher-order interactions.\nIn interpreting the results, I suggest focusing on the decomposition of sums of squares and not on statistical significance. In most cases, we expect the inputs to have at least some effect on the outcome, so the null hypothesis is a straw man. Better to assess the magnitude of the impacts of the different inputs.\nWhen one has a very large number of inputs, one can use the Latin hypercube approach to sample in the input space in a uniform way, spreading the points out so that each input is sampled uniformly. Assume that each input is \\(\\mathcal{U}(0,1)\\) (one can easily transform to whatever marginal distributions you want). Suppose that you can run \\(m\\) samples. Then for each input variable, we divide the unit interval into \\(m\\) bins and randomly choose the order of bins and the position within each bin. This is done independently for each variable and then combined to give \\(m\\) samples from the input space. We would then analyze main effects and perhaps two-way interactions to assess which inputs seem to be most important.\nEven amongst statisticians, taking an experimental design approach to a simulation study is not particularly common, but it’s worth considering." + }, + { + "objectID": "units/unit9-sim.html#computational-efficiency", + "href": "units/unit9-sim.html#computational-efficiency", + "title": "Simulation", + "section": "Computational efficiency", + "text": "Computational efficiency\nParallel processing is often helpful for simulation studies. The reason is that simulation studies are embarrassingly parallel - we can send each replicate to a different computer processor and then collect the results back, and the speedup should scale directly with the number of processors we used. Since we often need to some sort of looping, writing code in C/C++ and compiling and linking to the code from Python may also be a good strategy, albeit one not covered in this course.\nA handy function in Python is itertools.product to get all combinations of a set of vectors.\n\nimport itertools\n\nthetaLevels = [\"low\", \"med\", \"hi\"]\nn = [10, 100, 1000]\ntVsNorm = [\"t\", \"norm\"]\nlevels = list(itertools.product(thetaLevels, tVsNorm, n))" + }, + { + "objectID": "units/unit9-sim.html#analysis-and-reporting", + "href": "units/unit9-sim.html#analysis-and-reporting", + "title": "Simulation", + "section": "Analysis and reporting", + "text": "Analysis and reporting\nOften results are reported simply in tables, but it can be helpful to think through whether a graphical representation is more informative (sometimes it’s not or it’s worse, but in some cases it may be much better). Since you’ll often have a variety of scenarios to display, using trellis plots in ggplot2 via the facet_wrap function will often be a good approach to display how results vary as a function of multiple inputs in R. In Python, it looks like there are various ways (RPlot in pandas, seaborn, plotly), but I don’t know what the most standard way is.\nYou should set the seed when you start the experiment, so that it’s possible to replicate it. It’s also a good idea to save the current value of the seed whenever you save interim results, so that you can restart simulations (this is particularly helpful for MCMC) at the exact point you left off, including the random number sequence.\nTo enhance reproducibility, it’s good practice to post your simulation code (and potentially simulated data) on GitHub, on your website, or as supplementary material with the journal. Another person should be able to fully reproduce your results, including the exact random number generation that you did (e.g., you should provide code for how you set the random seed for your randon number generator).\nMany journals are requiring increasingly detailed documentation of the code and data used in your work, including code and data for simulations. Here are the American Statistical Association’s requirements on documenting computations in its journals:\n“The ASA strongly encourages authors to submit datasets, code, other programs, and/or extended appendices that are directly relevant to their submitted articles. These materials are valuable to users of the ASA’s journals and further the profession’s commitment to reproducible research. Whenever a dataset is used, its source should be fully documented and the data should be made available as on online supplement. Exceptions for reasons of security or confidentiality may be granted by the Editor. Whenever specific code has been used to implement or illustrate the results of a paper, that code should be made available if possible. [.…snip.…] Articles reporting results based on computation should provide enough information so that readers can evaluate the quality of the results. Such information includes estimated accuracy of results, as well as descriptions of pseudorandom-number generators, numerical algorithms, programming languages, and major software components used.”" + }, + { + "objectID": "units/unit9-sim.html#generating-random-uniforms-on-a-computer", + "href": "units/unit9-sim.html#generating-random-uniforms-on-a-computer", + "title": "Simulation", + "section": "Generating random uniforms on a computer", + "text": "Generating random uniforms on a computer\nGenerating a sequence of random standard uniforms is the basis for all generation of random variables, since random uniforms (either a single one or more than one) can be used to generate values from other distributions. Most random numbers on a computer are pseudo-random. The numbers are chosen from a deterministic stream of numbers that behave like random numbers but are actually a finite sequence (recall that both integers and real numbers on a computer are actually discrete and there are finitely many distinct values), so it’s actually possible to get repeats. The seed of a RNG is the place within that sequence where you start to use the pseudo-random numbers.\n\nSequential congruential generators\nMany RNG methods are sequential congruential methods. The basic idea is that the next value is \\[u_{k}=f(u_{k-1},\\ldots,u_{k-j})\\mbox{mod}\\,m\\] for some function, \\(f\\), and some positive integer \\(m\\) . Often \\(j=1\\). mod just means to take the remainder after dividing by \\(m\\). One then generates the random standard uniform value as \\(u_{k}/m\\), which by construction is in \\([0,1]\\). For our discussion below, it is important to distinguish the state (\\(u\\)) from the output of the RNG.\nGiven the construction, such sequences are periodic if the subsequence ever reappears, which is of course guaranteed because there is a finite number of possible subsequence values given that all the \\(u_{k}\\) values are remainders of divisions by a fixed number . One key to a good random number generator (RNG) is to have a very long period.\nAn example of a sequential congruential method is a basic linear congruential generator: \\[u_{k}=(au_{k-1}+c)\\mbox{mod}\\,m\\] with integer \\(a\\), \\(m\\), \\(c\\), and \\(u_{k}\\) values. (Note that in some cases \\(c=0\\), in which case the periodicity can’t exceed \\(m-1\\) as the method is then set up so that we never get \\(u_{k}=0\\) as this causes the algorithm to break.) The seed is the initial state, \\(u_{0}\\) - i.e., the point in the sequence at which we start. By setting the seed you guarantee reproducibility since given a starting value, the sequence is deterministic. In general \\(a\\), \\(c\\) and \\(m\\) are chosen to be ‘large’. The standard values of \\(m\\) are Mersenne primes, which have the form \\(2^{p}-1\\) (but these are not prime for all \\(p\\)). Here’s an example of a linear congruential sampler (with \\(c=0\\)):\n\nn = 100\na = 171\nm = 30269\n\nu = np.empty(n)\nu[0] = 7306\n\nfor i in range(1, n):\n u[i] = (a * u[i-1]) % m\n\nu = u / m\nuFromNP = np.random.uniform(size = n)\n\nplt.figure(figsize=(10, 8))\n\nplt.subplot(2, 2, 1)\nplt.plot(range(1, n+1), u)\nplt.title(\"manual\")\nplt.xlabel(\"Index\"); plt.ylabel(\"Value\")\n\nplt.subplot(2, 2, 2)\nplt.plot(range(1, n+1), uFromNP)\nplt.title(\"numpy\")\nplt.xlabel(\"Index\"); plt.ylabel(\"Value\")\n\nplt.subplot(2, 2, 3)\nplt.hist(u, bins=25)\n\n(array([6., 3., 6., 4., 1., 4., 6., 3., 4., 9., 4., 5., 1., 1., 2., 7., 1.,\n 2., 5., 2., 6., 5., 4., 4., 5.]), array([0.01833559, 0.05743434, 0.09653309, 0.13563183, 0.17473058,\n 0.21382933, 0.25292808, 0.29202683, 0.33112557, 0.37022432,\n 0.40932307, 0.44842182, 0.48752057, 0.52661931, 0.56571806,\n 0.60481681, 0.64391556, 0.68301431, 0.72211305, 0.7612118 ,\n 0.80031055, 0.8394093 , 0.87850804, 0.91760679, 0.95670554,\n 0.99580429]), <BarContainer object of 25 artists>)\n\nplt.xlabel(\"Value\"); plt.ylabel(\"Frequency\")\n\nplt.subplot(2, 2, 4)\nplt.hist(uFromNP, bins=25)\n\n(array([5., 9., 1., 4., 2., 6., 4., 4., 5., 3., 3., 2., 2., 4., 5., 8., 1.,\n 3., 2., 6., 4., 3., 5., 5., 4.]), array([0.00489673, 0.04443965, 0.08398257, 0.12352549, 0.16306841,\n 0.20261133, 0.24215425, 0.28169717, 0.32124009, 0.36078301,\n 0.40032593, 0.43986885, 0.47941177, 0.51895469, 0.55849761,\n 0.59804053, 0.63758345, 0.67712637, 0.71666929, 0.75621221,\n 0.79575513, 0.83529805, 0.87484097, 0.91438389, 0.95392681,\n 0.99346973]), <BarContainer object of 25 artists>)\n\nplt.xlabel(\"Value\"); plt.ylabel(\"Frequency\")\n\nplt.tight_layout()\nplt.show()\n\n\n\n\nA wide variety of different RNG have been proposed. Many have turned out to have substantial defects based on tests designed to assess if the behavior of the RNG mimics true randomness. Some of the behavior we want to ensure is uniformity of each individual random deviate, independence of sequences of deviates, and multivariate uniformity of subsequences. One test of a RNG that many RNGs don’t perform well on is to assess the properties of \\(k\\)-tuples - subsequences of length \\(k\\), which should be independently distributed in the \\(k\\)-dimensional unit hypercube. Unfortunately, linear congruential methods produce values that lie on a simple lattice in \\(k\\)-space, i.e., the points are not selected from \\(q^{k}\\) uniformly spaced points, where \\(q\\) is the the number of unique values. Instead, points often lie on parallel lines in the hypercube.\nCombining generators can yield better generators. The Wichmann-Hill is an option in R and is a combination of three linear congruential generators with \\(a=\\{171,172,170\\}\\), \\(m=\\{30269,30307,30323\\}\\), and \\(u_{i}=(x_{i}/30269+y_{i}/30307+z_{i}/30323)\\mbox{mod}\\,1\\) where \\(x\\), \\(y\\), and \\(z\\) are generated from the three individual generators. Let’s mimic the Wichmann-Hill manually:\n\nRNGkind(\"Wichmann-Hill\")\nset.seed(1)\nsaveSeed <- .Random.seed\nuFromR <- runif(10)\na <- c(171, 172, 170)\nm <- c(30269, 30307, 30323)\nxyz <- matrix(NA, nr = 10, nc = 3)\nxyz[1, ] <- (a * saveSeed[2:4]) %% m\nfor( i in 2:10)\n xyz[i, ] <- (a * xyz[i-1, ]) %% m\nfor(i in 1:10)\n print(c(uFromR[i],sum(xyz[i, ]/m)%%1))\n\n[1] 0.1297134 0.1297134\n[1] 0.9822407 0.9822407\n[1] 0.8267184 0.8267184\n[1] 0.242355 0.242355\n[1] 0.8568853 0.8568853\n[1] 0.8408788 0.8408788\n[1] 0.3421633 0.3421633\n[1] 0.7062672 0.7062672\n[1] 0.6212432 0.6212432\n[1] 0.6537663 0.6537663\n\n## we should be able to recover the current value of the seed\nxyz[10, ]\n\n[1] 24279 14851 10966\n\n.Random.seed[2:4]\n\n[1] 24279 14851 10966\n\n\n\n\nModern generators (PCG and Mersenne Twister)\nSomewhat recently O’Neal (2014) proposed a new approach to using the linear congruential generator in a way that gives much better performance than the basic versions of such generators described above. This approach is now the default random number generator in numpy (see numpy.random.default_rng()), called the PCG-64 generator. ‘PCG’ stands for permutation congruential generator and encompasses a family of such generators.\nThe idea of the PCG approach goes like this:\n\nLinear congruential generators (LCG) are simple and fast, but for small values of \\(m\\) don’t perform all that well statistically, in particular having values on a lattice as discussed above.\nUsing a large value of \\(m\\) can actually give good statistical performance.\nApplying a technique called permutation functions to the state of the LCG in order to produce the output at each step (the random value returned to the user) can improve the statistical performance even further.\n\nInstead of using relatively small values of \\(m\\) seen above, in the PCG approach one uses \\(m=2^k\\), for ‘large enough’ \\(k\\), usually 64 or 128. It turns out that if \\(m=2^k\\) then the period of the \\(b\\)th bit of the state is \\(2^b\\) where \\(b=1\\) is the right-most bit. Small periods are of course bad for RNG, so the bits with small period cause the LCG to not perform well. Thankfully, one simple fix is simply to discard some number of the right-most bits (this is one form of bit shift). Note that if one does this, the output of the RNG is based on a subset of the bits, which means that the number of unique values that can be generated is smaller than the period. This is not a problem given we start with a state with a large number of bits (64 or 128 as mentioned above).\nO’Neal then goes further; instead of simply discarding bits, she proposes to either shift bits by a random amount or rotate bits by a random amount, where the random amount is determined by a small number of the initial bits. This improves the statistical performance of the generator. The choice of how to do this gives the various members of the PCG family of generators. The details are fairly complicated (the PCG paper is 50-odd pages) and not important for our purposes here.\nBy default R uses something called the Mersenne twister, which is in the class of generalized feedback shift registers (GFSR). The basic idea of a GFSR is to come up with a deterministic generator of bits (i.e., a way to generate sequences of 0s and 1s), \\(B_{i}\\), \\(i=1,2,3,\\ldots\\). The pseudo-random numbers are then determined as sequential subsequences of length \\(L\\) from \\(\\{B_{i}\\}\\), considered as a base-2 number and dividing by \\(2^{L}\\) to get a number in \\((0,1)\\). In general the sequence of bits is generated by taking \\(B_{i}\\) to be the exclusive or [i.e., 0+0 = 0; 0 + 1 = 1; 1 + 0 = 1; 1 + 1 = 0] summation of two previous bits further back in the sequence where the lengths of the lags are carefully chosen.\nnumpy also provides access to the Mersenne Twister via the MT19937 generator; more on this below. It looks like PCG-64 only became available as of numpy version 1.17.\n\nAdditional notes\nGenerators should give you the same sequence of random numbers, starting at a given seed, whether you ask for a bunch of numbers at once, or sequentially ask for individual numbers.\nWhen one invokes a RNG without a seed, they generally have a method for choosing a seed, often based on the system clock.\nThere have been some attempts to generate truly random numbers based on physical randomness. One that is based on quantum physics is http://www.idquantique.com/true-random-number-generator/quantis-usb-pcie-pci.html. Another approach is based on lava lamps!" + }, + { + "objectID": "units/unit9-sim.html#rng-in-python", + "href": "units/unit9-sim.html#rng-in-python", + "title": "Simulation", + "section": "RNG in Python", + "text": "RNG in Python\nWe can change the RNG for numpy using np.random.<name_of_generator> (e.g., np.random.MT19937 for the Mersenne Twister). We can set the seed with np.random.seed() or with np.random.default_rng().\nIn numpy, the default_rng RNG is PCG-64. It has a period of \\(2^{128}\\) and supports advancing an arbitrary number of steps, as well as \\(2^{127}\\) streams (both useful for generating random numbers when parallelizing). The state of the PCG-64 RNG is represented by two 128-bit unsigned integers, one the actual state and one the value of \\(c\\) (the increment).\nStrangely, while the default is PCG-64, simply using the functions available via np.random to generate random numbers seems to actually use the Mersenne Twister, so the meaning of default is unclear.\nIn R, the default RNG is the Mersenne twister (?RNGkind), which is considered to be state-of-the-art (by some; O’Neal criticizes it). It has some theoretical support, has performed reasonably on standard tests of pseudorandom numbers and has been used without evidence of serious failure. Plus it’s fast (because bitwise operations are fast). The particular Mersenne twister used has a periodicity of \\(2^{19937}-1\\approx10^{6000}\\). Practically speaking this means that if we generated one random uniform per nanosecond for 10 billion years, then we would generate \\(10^{25}\\) numbers, well short of the period. So we don’t need to worry about the periodicity! The seed for the Mersenne twister is a set of 624 32-bit integers plus a position in the set, where the position is .Random.seed[2].\nFor the Mersenne Twister, we can set the seed by passing an integer to np.random.seed() in Python or set.seed() in R, which then sets as many actual seeds as required for the Mersenne Twister. Here I’ll refer to the single integer passed in as the seed. Ideally, nearby seeds generally should not correspond to getting sequences from the stream that are closer to each other than far away seeds. According to Gentle (CS, p. 327) the input to set.seed() in R should be an integer, \\(i\\in\\{0,\\ldots,1023\\}\\) , and each of these 1024 values produces positions in the RNG sequence that are “far away” from each other. I don’t see any mention of this in the R documentation for set.seed() and furthermore, you can pass integers larger than 1023 to set.seed(), so I’m not sure how much to trust Gentle’s claim. More on generating parallel streams of random numbers below.\nSo we get replicability by setting the seed to a specific value at the beginning of our simulation. We can then set the seed to that same value when we want to replicate the simulation.\n\nnp.random.seed(1)\nnp.random.normal(size = 5)\n\narray([ 1.62434536, -0.61175641, -0.52817175, -1.07296862, 0.86540763])\n\nnp.random.seed(1)\nnp.random.normal(size = 5)\n\narray([ 1.62434536, -0.61175641, -0.52817175, -1.07296862, 0.86540763])\n\n\nWe can also save the state of the RNG and pick up where we left off. So this code will pick where you had left off, ignoring what happened in between saving to saved_state and resetting.\n\nnp.random.seed(1)\nnp.random.normal(size = 5)\n\narray([ 1.62434536, -0.61175641, -0.52817175, -1.07296862, 0.86540763])\n\nsaved_state = np.random.get_state()\nnp.random.normal(size = 5)\n\narray([-2.3015387 , 1.74481176, -0.7612069 , 0.3190391 , -0.24937038])\n\n\nNow we’ll do some arbitrary work with random numbers, and see that if we use the saved state we can pick up where we left off above.\n\ntmp = np.random.choice(np.arange(1, 51), size=2000, replace=True) # arbitrary work\n\n## Restore the state.\nnp.random.set_state(saved_state)\nnp.random.normal(size = 5)\n\narray([-2.3015387 , 1.74481176, -0.7612069 , 0.3190391 , -0.24937038])\n\n\nIf we look at saved_state, we see it actually corresponds to the Mersenne Twister.\nTo do the equivalent with the PCG-64:\n\nrng = np.random.default_rng(1)\nrng.normal(size = 5)\n\narray([ 0.34558419, 0.82161814, 0.33043708, -1.30315723, 0.90535587])\n\nsaved_state = rng.bit_generator.state\nrng.normal(size = 5)\n\narray([ 0.44637457, -0.53695324, 0.5811181 , 0.3645724 , 0.2941325 ])\n\ntmp = rng.choice(np.arange(1, 51), size=2000, replace=True)\nrng.bit_generator.state = saved_state\nrng.normal(size = 5)\n\narray([ 0.44637457, -0.53695324, 0.5811181 , 0.3645724 , 0.2941325 ])\n\nsaved_state\n\n{'bit_generator': 'PCG64', 'state': {'state': 216676376075457487203159048251690499413, 'inc': 194290289479364712180083596243593368443}, 'has_uint32': 0, 'uinteger': 0}\n\nsaved_state['state']['state'] # actual state\n\n216676376075457487203159048251690499413\n\nsaved_state['state']['inc'] # increment ('c')\n\n194290289479364712180083596243593368443\n\n\nsaved_state contains the actual state and the value of c, the increment.\nThe output of the PCG-64 is 64 bits while for the Mersenne Twister the output is 32 bits. This means you could get duplicated values in long runs, but this does not violate the comment about the periodicity of PCG-64 and Mersenne-Twister being longer than \\(2^{64}\\) and \\(2^{32}\\), because the two values after the two duplicated numbers will not be duplicates of each other – as noted previously, there is a distinction between the output presented to the user and the state of the RNG algorithm." + }, + { + "objectID": "units/unit9-sim.html#rng-in-parallel", + "href": "units/unit9-sim.html#rng-in-parallel", + "title": "Simulation", + "section": "RNG in parallel", + "text": "RNG in parallel\nWe can generally rely on the RNG in Python and R to give a reasonable set of values. One time when we want to think harder is when doing work with RNG in parallel on multiple processors. The worst thing that could happen is that one sets things up in such a way that every process is using the same sequence of random numbers. This could happen if you mistakenly set the same seed in each process, e.g., using np.random.seed(1) on every process. More details on parallel RNG are given in Unit 6." + }, + { + "objectID": "units/unit9-sim.html#multivariate-distributions", + "href": "units/unit9-sim.html#multivariate-distributions", + "title": "Simulation", + "section": "Multivariate distributions", + "text": "Multivariate distributions\nThe mvtnorm package supplies code for working with the density and CDF of multivariate normal and t distributions.\nTo generate a multivariate normal, in Unit 10, we’ll see the standard method based on the Cholesky decomposition:\n\nL = np.linalg.cholesky(covMat) # L is lower-triangular\nx = L @ np.random.normal(size = covMat.shape[0])\n\nSide note: for a singular covariance matrix we can use the Cholesky with pivoting, setting as many rows to zero as the rank deficiency. Then when we generate the multivariate normals, they respect the constraints implicit in the rank deficiency. However, you’ll need to reorder the resulting vector because of the reordering involved in the pivoted Cholesky." + }, + { + "objectID": "units/unit9-sim.html#inverse-cdf", + "href": "units/unit9-sim.html#inverse-cdf", + "title": "Simulation", + "section": "Inverse CDF", + "text": "Inverse CDF\nMost of you know the inverse CDF method. To generate \\(X\\sim F\\) where \\(F\\) is a CDF and is an invertible function, first generate \\(Z\\sim\\mathcal{U}(0,1)\\), then \\(x=F^{-1}(z)\\). For discrete CDFs, one can work with a discretized version. For multivariate distributions, one can work with a univariate marginal and then a sequence of univariate conditionals: \\(f(x_{1})f(x_{2}|x_{1})\\cdots f(x_{k}|x_{k-1},\\ldots,x_{1})\\), when the distribution allows this analytic decomposition." + }, + { + "objectID": "units/unit9-sim.html#rejection-sampling", + "href": "units/unit9-sim.html#rejection-sampling", + "title": "Simulation", + "section": "Rejection sampling", + "text": "Rejection sampling\nThe basic idea of rejection sampling (RS) relies on the introduction of an auxiliary variable, \\(u\\). Suppose \\(X\\sim F\\). Then we can write \\(f(x)=\\int_{0}^{f(x)}du\\). Thus \\(f\\) is the marginal density of \\(X\\) in the joint density, \\((X,U)\\sim\\mathcal{U}\\{(x,u):0<u<f(x)\\}\\). Now we’d like to use this in a way that relies only on evaluating \\(f(x)\\) without having to draw from \\(f\\).\nTo implement this we draw from a larger set and then only keep draws for which \\(u<f(x)\\). We choose a density, \\(g\\), that is easy to draw from and that can majorize \\(f\\), which means there exists a constant \\(c\\) s.t. , \\(cg(x)\\geq f(x)\\) \\(\\forall x\\). In other words we have that \\(cg(x)\\) is an upper envelope for \\(f(x)\\). The algorithm is\n\ngenerate \\(x\\sim g\\)\ngenerate \\(u\\sim\\mathcal{U}(0,1)\\)\nif \\(u\\leq f(x)/cg(x)\\) then use \\(x\\); otherwise go back to step 1\n\nThe intuition here is graphical: we generate from under a curve that is always above \\(f(x)\\) and accept only when \\(u\\) puts us under \\(f(x)\\) relative to the majorizing density. A key here is that the majorizing density have fatter tails than the density of interest, so that the constant \\(c\\) can exist. So we could use a \\(t\\) to generate from a normal but not the reverse. We’d like \\(c\\) to be small to reduce the number of rejections because it turns out that \\(\\frac{1}{c}=\\frac{\\int f(x)dx}{\\int cg(x)dx}\\) is the acceptance probability. This approach works in principle for multivariate densities but as the dimension increases, the proportion of rejections grows, because more of the volume under \\(cg(x)\\) is above \\(f(x)\\).\nIf \\(f\\) is costly to evaluate, we can sometimes reduce calculation using a lower bound on \\(f\\). In this case we accept if \\(u\\leq f_{\\mbox{low}}(y)/cg_{Y}(y)\\). If it is not, then we need to evaluate the ratio in the usual rejection sampling algorithm. This is called squeezing.\nOne example of RS is to sample from a truncated normal. Of course we can just sample from the normal and then reject, but this can be inefficient, particularly if the truncation is far in the tail (a case in which inverse CDF suffers from numerical difficulties). Suppose the truncation point is greater than zero. Working with the standardized version of the normal, you can use an translated exponential with lower end point equal to the truncation point as the majorizing density (Robert 1995; Statistics and Computing). For truncation less than zero, just make the values negative." + }, + { + "objectID": "units/unit9-sim.html#adaptive-rejection-sampling-optional", + "href": "units/unit9-sim.html#adaptive-rejection-sampling-optional", + "title": "Simulation", + "section": "Adaptive rejection sampling (optional)", + "text": "Adaptive rejection sampling (optional)\nThe difficulty of RS is finding a good enveloping function. Adaptive rejection sampling refines the envelope as the draws occur, in the case of a continuous, differentiable, log-concave density. The basic idea considers the log of the density and involves using tangents or secants to define an upper envelope and secants to define a lower envelope for a set of points in the support of the distribution. The result is that we have piecewise exponentials (since we are exponentiating from straight lines on the log scale) as the bounds. We can sample from the upper envelope based on sampling from a discrete distribution and then the appropriate exponential. The lower envelope is used for squeezing. We add points to the set that defines the envelopes whenever we accept a point that requires us to evaluate \\(f(x)\\) (the points that are accepted based on squeezing are not added to the set)." + }, + { + "objectID": "units/unit9-sim.html#importance-sampling", + "href": "units/unit9-sim.html#importance-sampling", + "title": "Simulation", + "section": "Importance sampling", + "text": "Importance sampling\nImportance sampling (IS) allows us to estimate expected values. It’s an extension of the simple Monte Carlo sampling we saw at the beginning of the unit, with some commonalities with rejection sampling.\n\\[\\phi=E_{f}(h(Y))=\\int h(y)\\frac{f(y)}{g(y)}g(y)dy\\] so \\(\\hat{\\phi}=\\frac{1}{m}\\sum_{i}h(y_{i})\\frac{f(y_{i})}{g(y_{i})}\\) for \\(y_{i}\\) drawn from \\(g(y)\\), where \\(w_{i}=f(y_{i})/g(y_{i})\\) act as weights. (Often in Bayesian contexts, we know \\(f(y)\\) only up to a normalizing constant. In this case we need to use \\(w_{i}^{*}=w_{i}/\\sum_{j}w_{j}\\).\nHere we don’t require the majorizing property, just that the densities have common support, but things can be badly behaved if we sample from a density with lighter tails than the density of interest. So in general we want \\(g\\) to have heavier tails. More specifically for a low variance estimator of \\(\\phi\\), we would want that \\(f(y_{i})/g(y_{i})\\) is large only when \\(h(y_{i})\\) is very small, to avoid having overly influential points.\nThis suggests we can reduce variance in an IS context by oversampling \\(y\\) for which \\(h(y)\\) is large and undersampling when it is small, since \\(\\mbox{Var}(\\hat{\\phi})=\\frac{1}{m}\\mbox{Var}(h(Y)\\frac{f(Y)}{g(Y)})\\). An example is that if \\(h\\) is an indicator function that is 1 only for rare events, we should oversample rare events and then the IS estimator corrects for the oversampling.\nWhat if we actually want a sample from \\(f\\) as opposed to estimating the expected value above? We can draw \\(y\\) from the unweighted sample, \\(\\{y_{i}\\}\\), with weights \\(\\{w_{i}\\}\\). This is called sampling importance resampling (SIR)." + }, + { + "objectID": "units/unit9-sim.html#ratio-of-uniforms-optional", + "href": "units/unit9-sim.html#ratio-of-uniforms-optional", + "title": "Simulation", + "section": "Ratio of uniforms (optional)", + "text": "Ratio of uniforms (optional)\nIf \\(U\\) and \\(V\\) are uniform in \\(C=\\{(u,v):\\,0\\leq u\\leq\\sqrt{f(v/u)}\\) then \\(X=V/U\\) has density proportion to \\(f\\). The basic algorithm is to choose a rectangle that encloses \\(C\\) and sample until we find \\(u\\leq f(v/u)\\). Then we use \\(x=v/u\\) as our RV. The larger region enclosing \\(C\\) is the majorizing region and a simple approach (if \\(f(x)\\)and \\(x^{2}f(x)\\) are bounded in \\(C\\)) is to choose the rectangle, \\(0\\leq u\\leq\\sup_{x}\\sqrt{f(x)}\\), \\(\\inf_{x}x\\sqrt{f(x)}\\leq v\\leq\\sup_{x}x\\sqrt{f(x)}\\).\nOne can also consider truncating the rectangular region, depending on the features of \\(f\\).\nMonahan recommends the ratio of uniforms, particularly a version for discrete distributions (p. 323 of the 2nd edition)." }, { "objectID": "units/unit5-programming.html", @@ -401,373 +583,247 @@ { "objectID": "units/unit5-programming.html#decorators", "href": "units/unit5-programming.html#decorators", - "title": "Programming concepts", - "section": "Decorators", - "text": "Decorators\nNow that we’ve seen function generators, it’s straightforward to discuss decorators.\nA decorator is a wrapper around a function that extends the functionality of the function without actually modifying the function.\nWe can create a simple decorator “manually” like this:\n\ndef verbosity_wrapper(myfun):\n def wrapper(*args, **kwargs):\n print(f\"Starting {myfun.__name__}.\")\n output = myfun(*args, **kwargs)\n print(f\"Finishing {myfun.__name__}.\")\n return output\n return wrapper\n \nverbose_rnorm = verbosity_wrapper(np.random.normal)\n\nx = verbose_rnorm(size = 5)\n\nStarting normal.\nFinishing normal.\n\nx\n\narray([ 0.07202449, -0.67672674, 0.98248139, -1.65748762, 0.81795808])\n\n\nPython provides syntax that helps you create decorators with less work (this is an example of the general idea of syntactic sugar).\nWe can easily apply our decorator defined above to a function as follows. Now the function name refers to the wrapped version of the function.\n\n@verbosity_wrapper\ndef myfun(x):\n return x\n\ny = myfun(7)\n\nStarting myfun.\nFinishing myfun.\n\ny\n\n7\n\n\nOur decorator doesn’t do anything useful, but hopefully you can imagine that the idea of being able to have more control over the operation of functions could be useful. For example we could set up a timing wrapper so that when we run a function, we get a report on how long it took to run the function. Or using the idea of a closure, we could keep a running count of the number of times a function has been called.\nOne real-world example of using decorators is in setting up functions to run in parallel in dask, which we’ll discuss in Unit 7." - }, - { - "objectID": "units/unit5-programming.html#overview-2", - "href": "units/unit5-programming.html#overview-2", - "title": "Programming concepts", - "section": "Overview", - "text": "Overview\nThe main things to remember when thinking about memory use are: (1) numeric vectors take 8 bytes per element and (2) we need to keep track of when large objects are created, including local variables in the frames of functions.\n\nx = np.random.normal(size = 5)\nx.itemsize # 8 bytes\n\n8\n\nx.nbytes\n\n40\n\n\n\nAllocating and freeing memory\nUnlike compiled languages like C, in Python we do not need to explicitly allocate storage for objects. (However, we will see that there are times that we do want to allocate storage in advance, rather than successively concatenating onto a larger object.)\nPython automatically manages memory, releasing memory back to the operating system when it’s not needed via a process called garbage collection. Very occasionally you may want to remove large objects as soon as they are not needed. del does not actually free up memory, it just disassociates the name from the memory used to store the object. In general Python will quickly clean up such objects without a reference (i.e., a name), so there is generally no need to call gc.collect() to force the garbage collection.\nIn a language like C in which the user allocates and frees up memory, memory leaks are a major cause of bugs. Basically if you are looping and you allocate memory at each iteration and forget to free it, the memory use builds up inexorably and eventually the machine runs out of memory. In Python, with automatic garbage collection, this is generally not an issue, but occasionally memory leaks could occur.\n\n\nThe heap and the stack\nThe heap is the memory that is available for dynamically creating new objects while a program is executing, e.g., if you create a new object in Python or call new in C++. When more memory is needed the program can request more from the operating system. When objects are removed in Python, Python will handle the garbage collection of releasing that memory.\nThe stack is the memory used for local variables when a function is called.\nThere’s a nice discussion of this on this Stack Overflow thread." - }, - { - "objectID": "units/unit5-programming.html#monitoring-memory-use", - "href": "units/unit5-programming.html#monitoring-memory-use", - "title": "Programming concepts", - "section": "Monitoring memory use", - "text": "Monitoring memory use\n\nMonitoring overall memory use on a UNIX-style computer\nTo understand how much memory is available on your computer, one needs to have a clear understanding of disk caching. The operating system will generally cache files/data in memory when it reads from disk. Then if that information is still in memory the next time it is needed, it will be much faster to access it the second time around than if it had to read the information from disk. While the cached information is using memory, that same memory is immediately available to other processes, so the memory is available even though it is “in use”.\nWe can see this via free -h (the -h is for ‘human-readable’, i.e. show in GB (G)) on Linux machine.\n total used free shared buff/cache available \n Mem: 251G 998M 221G 2.6G 29G 247G \n Swap: 7.6G 210M 7.4G\nYou’ll generally be interested in the Mem row. (See below for some comments on Swap.) The shared column is complicated and probably won’t be of use to you. The buff/cache column shows how much space is used for disk caching and related purposes but is actually available. Hence the available column is the sum of the free and buff/cache columns (more or less). In this case only about 1 GB is in use (indicated in the used column).\ntop (Linux or Mac) and vmstat (on Linux) both show overall memory use, but remember that the amount actually available to you is the amount free plus any buff/cache usage. Here is some example output from vmstat:\n\n procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu----- \n r b swpd free buff cache si so bi bo in cs us sy id wa st \n 1 0 215140 231655120 677944 30660296 0 0 1 2 0 0 18 0 82 0 0\nIt shows 232 GB free and 31 GB used for cache and therefore available, for a total of 263 GB available.\nHere are some example lines from top:\n KiB Mem : 26413715+total, 23180236+free, 999704 used, 31335072 buff/cache \n KiB Swap: 7999484 total, 7784336 free, 215148 used. 25953483+avail Mem\nWe see that this machine has 264 GB RAM (the total column in the Mem row), with 259.5 GB available (232 GB free plus 31 GB buff/cache as seen in the Mem row). (I realize the numbers don’t quite add up for reasons I don’t fully understand, but we probably don’t need to worry about that degree of exactness.) Only 1 GB is in use.\nSwap is essentially the reverse of disk caching. It is disk space that is used for memory when the machine runs out of physical memory. You never want your machine to be using swap for memory because your jobs will slow to a crawl. As seen above, the swap line in both free and top shows 8 GB swap space, with very little in use, as desired.\n\n\nMonitoring memory use in Python\nThere are a number of ways to see how much memory is being used. When Python is actively executing statements, you can use top from the UNIX shell.\nIn Python, we can call out to the system to get the info we want:\n\nimport psutil\n\n# Get memory information\nmemory_info = psutil.Process().memory_info()\n\n# Print the memory usage\nprint(\"Memory usage:\", memory_info.rss/10**6, \" Mb.\")\n\n# Let's turn that into a function for later use:\n\nMemory usage: 470.765568 Mb.\n\ndef mem_used():\n print(\"Memory usage:\", psutil.Process().memory_info().rss/10**6, \" Mb.\")\n\nWe can see the size of an object (in bytes) with sys.getsizeof().\n\nmy_list = [1, 2, 3, 4, 5]\nsys.getsizeof(my_list)\n\n104\n\nx = np.random.normal(size = 10**7) # should use about 80 Mb\nsys.getsizeof(x)\n\n80000112\n\n\nHowever, we need to be careful about objects that refer to other objects:\n\ny = [3, x]\nsys.getsizeof(y) # Whoops!\n\n72\n\n\nHere’s a trick where we serialize the object, as if to export it, and then see how long the binary representation is.\n\nimport pickle\nser_object = pickle.dumps(y)\nsys.getsizeof(ser_object)\n\n80000201\n\n\nThere are also some flags that one can start python with that allow one to see information about memory use and allocation. See man python. You could also look into the memory_profiler or pympler packages." - }, - { - "objectID": "units/unit5-programming.html#how-memory-is-used-in-python", - "href": "units/unit5-programming.html#how-memory-is-used-in-python", - "title": "Programming concepts", - "section": "How memory is used in Python", - "text": "How memory is used in Python\n\nTwo key tools: id and is\nWe can use the id function to see where in memory an object is stored and is to see if two object are actually the same objects in memory. It’s particularly useful for understanding storage and memory use for complicated data structures. We’ll also see that they can be handy tools for seeing where copies are made and where they are not.\n\nx = np.random.normal(size = 10**7)\nid(x)\n\n140228665313968\n\nsys.getsizeof(x)\n\n80000112\n\ny = x\nid(y)\n\n140228665313968\n\nx is y\n\nTrue\n\nsys.getsizeof(y)\n\n80000112\n\nz = x.copy()\nid(z)\n\n140228667485872\n\nsys.getsizeof(z)\n\n80000112\n\n\n\n\nMemory use in specific circumstances\n\nHow lists are stored\nHere we can use id to determine how the overall list is stored as well as the elements of the list.\n\nnums = np.random.normal(size = 5)\nobj = [nums, nums, np.random.normal(size = 5), ['adfs']]\n\nid(nums)\n\n140228665322032\n\nid(obj)\n\n140228665946048\n\nid(obj[0])\n\n140228665322032\n\nid(obj[1])\n\n140228665322032\n\nid(obj[2])\n\n140228665675856\n\nid(obj[3])\n\n140228665966656\n\nid(obj[3][0])\n\n140228665970736\n\nobj[0] is obj[1]\n\nTrue\n\nobj[0] is obj[2]\n\nFalse\n\n\nWhat do we notice?\n\nThe list itself appears to be a array of references (pointers) to the component elements.\nEach element has its own address.\nTwo elements of a list can use the same memory (see the first two elements here, whose contents are at the same memory address).\nA list element can use the same memory as another object (or part of another object).\n\n\n\nHow character strings are stored.\nSimilar tricks are used for storing strings (and also integers). We’ll explore this in a problem on PS4.\n\n\nModifying elements in place\nWhat do this simple experiment tell us?\n\nx = np.random.normal(size = 5)\nid(x)\n\n140228665321552\n\nx[2] = 3.5\nid(x)\n\n140228665321552\n\n\nIt makes some sense that modifying elements of an object here doesn’t cause a copy – if it did, working with large objects would be very difficult.\n\n\n\nWhen are copies made?\nLet’s try to understand when Python uses additional memory for objects, and how it knows when it can delete memory. We’ll use large objects so that we can use free or top to see how memory use by the Python process changes.\n\nx = np.random.normal(size = 10**8)\nid(x)\n\n140228665675952\n\ny = x\nid(y)\n\n140228665675952\n\nx = np.random.normal(size = 10**8)\nid(x)\n\n140228665321552\n\n\nOnly if we re-assign x to reference a different object does additional memory get used.\n\nHow does Python know when it can free up memory?\nPython keeps track of how many names refer to an object and only removes memory when there are no remaining references to an object.\n\nimport sys\n\nx = np.random.normal(size = 10**8)\ny = x\nsys.getrefcount(y)\n\n3\n\ndel x\nsys.getrefcount(y)\n\n2\n\ndel y\n\nWe can see the number of references using sys.getrefcount. Confusingly, the number is one higher than we’d expect, because it includes the temporary reference from passing the object as the argument to getrefcount.\n\nx = np.random.normal(size = 5)\nsys.getrefcount(x) # 2 \n\n2\n\ny = x\nsys.getrefcount(x) # 3\n\n3\n\nsys.getrefcount(y) # 3\n\n3\n\ndel y\nsys.getrefcount(x) # 2\n\n2\n\ny = x\nx = np.random.normal(size = 5)\nsys.getrefcount(y) # 2\n\n2\n\nsys.getrefcount(x) # 2\n\n2\n\n\nThis notion of reference counting occurs in other contexts, such as shared pointers in C++ and in how R handles copying and garbage collection." - }, - { - "objectID": "units/unit5-programming.html#strategies-for-saving-memory", - "href": "units/unit5-programming.html#strategies-for-saving-memory", - "title": "Programming concepts", - "section": "Strategies for saving memory", - "text": "Strategies for saving memory\nA frew basic strategies for saving memory include:\n\nAvoiding unnecessary copies.\nRemoving objects that are not being used, at which point the Python garbage collector should free up the memory.\n\nIf you’re really trying to optimize memory use, you may also consider:\n\nUsing types that take up less memory (e.g., Bool, Int16, Float32) when possible.\n\nx = np.array(np.random.normal(size = 5), dtype = \"float32\")\nx.itemsize\n\n4\n\nx = np.array([3,4,2,-2], dtype = \"int16\")\nx.itemsize\n\n2\n\n\nReading data in from files in chunks rather than reading the entire dataset (more in Unit 7).\nExploring packages such as arrow for efficiently using memory, as discussed in Unit 2." - }, - { - "objectID": "units/unit5-programming.html#example", - "href": "units/unit5-programming.html#example", - "title": "Programming concepts", - "section": "Example", - "text": "Example\nLet’s work through a real example where we keep a running tally of current memory in use and maximum memory used in a function call. We’ll want to consider hidden uses of memory, when copies are made, and lazy evaluation. This code (translated from the original R code) comes from a PhD student’s research. For our purposes here, let’s assume that xvar and yvar are very long numpy arrays using a lot of memory.\n\ndef fastcount(xvar, yvar):\n naline = np.isnan(xvar)\n naline[np.isnan(yvar)] = True\n localx = xvar.copy()\n localy = yvar.copy()\n localx[naline] = 0\n localy[naline] = 0\n useline = ~naline\n ## We'll ignore the rest of the code.\n ## ...." - }, - { - "objectID": "units/unit5-programming.html#interpreters-and-compilation", - "href": "units/unit5-programming.html#interpreters-and-compilation", - "title": "Programming concepts", - "section": "Interpreters and compilation", - "text": "Interpreters and compilation\n\nWhy are interpreted languages slow?\nCompiled code runs quickly because the original code has been translated into instructions (machine language) that the processor can understand (i.e., zeros and ones). In the process of doing so, various checking and lookup steps are done once and don’t need to be redone when running the compiled code.\nIn contrast, when one runs code in an interpreted language such as Python or R, the interpreter needs to do all the checking and lookup each time the code is run. This is required because the types and locations in memory of the variables could have changed.\nWe’ll focus on Python in the following discussion, but most of the concepts apply to other interpreted languages.\nFor example, consider this code:\n\nx = 3\nabs(x)\nx*7\nx = 'hi'\nabs(x)\nx*3\n\nBecause of dynamic typing, when the interpreter sees abs(x) it needs to check if x is something to which the absolute value function can be applied, including dealing with the fact that x could be a list or array with many numbers in it. In addition it needs to (using scoping rules) look up the value of x. (Consider that x might not even exist at the point that abs(x) is called.) Only then can the absolute value calculation happen. For the multiplication, Python needs to lookup the version of * that can be used, depending on the type of x.\nLet’s consider writing a loop with some ridiculous code:\n\nx = np.random.normal(10)\nfor i in range(10):\n if np.random.normal(size = 1) > 0:\n x = 'hi'\n if np.random.normal(size = 1) > 0.5:\n del x\n x[i]= np.exp(x[i])\n\nThere is no way around the fact that because of how dynamic this is, the interpreter needs to check if x exists, if it is a vector of sufficient length, if it contains numeric values, and it needs to go retrieve the required value, EVERY TIME the np.exp() is executed. Now the code above is unusual, and in most cases, we wouldn’t have the if statements that modify x. So you could imagine a process by which the checking were done on the first iteration and then not needed after that – that gets into the idea of just-in-time compilation, discussed later.\nThe standard Python interpreter (CPython) is a C function so in some sense everything that happens is running as compiled code, but there are lots more things being done to accomplish a given task using interpreted code than if the task had been written directly in code that is compiled. By analogy, consider talking directly to a person in a language you both know compared to talking to a person via an interpreter who has to translate between two languages. Ultimately, the same information gets communicated (hopefully!) but the number of words spoken and time involved is much greater.\nWhen running more complicated functions, there is often a lot of checking that is part of the function itself. For example scipy’s solve_triangular function ultimately calls out to the trtrs Lapack function, but before doing so, there is a lot of checking that can take time. To that point, the documentation suggests you might set check_finite=False to improve performance at the expense of potential problems if the input matrices contain troublesome elements.\nWe can flip the question on its head and ask what operations in an interpreted language will execute quickly. In Python, these include:\n\noperations that call out to compiled C code,\nlinear algebra operations (these call out to compiled C or Fortran code provided by the BLAS and LAPACK software packages), and\nvectorized calls rather than loops:\n\nvectorized calls generally run loops in compiled C code rather than having the loop run in Python, and\nthat means that the interpreter doesn’t have to do all the checking discussed above for every iteration of the loop.\n\n\n\n\nCompilation\n\nOverview\nCompilation is the process of turning code in a given language (such a C++) into machine code. Machine code is the code that the processor actually executes. The machine code is stored in the executable file, which is a binary file. The history of programming has seen ever great levels of abstraction, so that humans can write code using syntax that is easier for us to understand, re-use, and develop building blocks that can be put together to do complicated tasks. For example assembly language is a step above machine code. Languages like C and Fortran provide additional abstraction beyond that. The Statistics 750 class at CMU has a nice overview if you want to see more details.\nNote that interpreters such as Python are themselves programs – the standard Python interpreter (CPython) is a C program that has been compiled. It happens to be a program that processes Python code. The interpreter doesn’t turn Python code into machine code, but the interpreter itself is machine code.\n\n\nJust-in-time (JIT) compilation\nStandard compilation (ahead-of-time or AOT compilation) happens before any code is executed and can involve a lot of optimization to produce the most efficient machine code possible.\nIn contrast, just-in-time (JIT) compilation happens at the time that the code is executing. JIT compilation is heavily used in Julia, which is very fast (in some cases as fast as C). JIT compilation involves translating to machine code as the code is running. One nice aspect is that the results are cached so that if code is rerun, the compilation process doesn’t have to be redone. So if you use a language like Julia, you’ll see that the speed can vary drastically between the first time and later times you run a given function during a given session.\nOne thing that needs to be dealt with is type checking. As discussed above, part of why an interpreter is slow is because the type of the variable(s) involved in execution of a piece of code is not known in advance, so the interpreter needs to check the type. In JIT systems, there are often type inference systems that determine variable types.\nJIT compilation can involve translation from the original code to machine code or translation of bytecode (see next section) to machine code.\n\n\nByte compiling (optional)\nFunctions in Python and Python packages may byte compiled. What does that mean? Byte-compiled code is a special representation that can be executed more efficiently because it is in the form of compact codes that encode the results of parsing and semantic analysis of scoping and other complexities of the Python source code. This byte code can be executed faster than the original Python code because it skips the stage of having to be interpreted by the Python interpreter.\nIf you look at the file names in the directory of an installed Python package you may see files with the .pyc extension. These files have been byte-compiled.\nWe can byte compile our own functions using either the py_compile or compileall modules. Here’s an example (silly since as experienced Python programmers, we would use vectorized calculation here rather than this unvectorized code.)\n\nimport time\n\ndef f(vals):\n x = np.zeros(len(vals))\n for i in range(len(vals)):\n x[i] = np.exp(vals[i])\n return(x)\n\nx = np.random.normal(size = 10**6)\nt0 = time.time()\nout = f(x)\ntime.time() - t0\n\n0.7474195957183838\n\nt0 = time.time()\nout = np.exp(x)\ntime.time() - t0\n\n0.011847734451293945\n\n\n\nimport py_compile\npy_compile.compile('vec.py')\n\n'__pycache__/vec.cpython-311.pyc'\n\n\n\ncp __pycache__/vec.cpython-311.pyc vec.pyc\nrm vec.py # make sure non-compiled module not loaded\n\n\nimport vec\nvec.__file__\n \n\n'/accounts/vis/paciorek/teaching/243fall23/stat243-fall-2023/units/vec.pyc'\n\nt0 = time.time()\nout = vec.f(x)\ntime.time() - t0\n\n0.714287519454956\n\n\nUnfortunately, as seen above byte compiling may not speed things up much. I’m not sure why." - }, - { - "objectID": "units/unit5-programming.html#benchmarking-and-profiling", - "href": "units/unit5-programming.html#benchmarking-and-profiling", - "title": "Programming concepts", - "section": "Benchmarking and profiling", - "text": "Benchmarking and profiling\nRecall that it’s a waste of time to optimize code before you determine (1) that the code is too slow for how it will be used and (2) which are the slow steps on which to focus your attempts to speed the code up. A 100x speedup in a step that takes 1% of the time will speed up the overall code by very little.\n\nTiming your code\nThere are a few ways to time code:\n\nimport time\nt0 = time.time()\nx = 3\nt1 = time.time()\n\nprint(f\"Execution time: {t1-t0} seconds.\")\n\nExecution time: 0.005488395690917969 seconds.\n\n\nIn general, it’s a good idea to repeat (replicate) your timing, as there is some stochasticity in how fast your computer will run a piece of code at any given moment.\nUsing time is fine for code that takes a little while to run, but for code that is really fast, it may not be very accurate. Measuring fast bits of code is tricky to do well. This next approach is better for benchmarking code (particularly faster bits of code).\n\nimport timeit\n\ntimeit.timeit('x = np.exp(3.)', setup = 'import numpy as np', number = 100)\n\n0.00019408203661441803\n\ncode = '''\nx = np.exp(3.)\n'''\n\ntimeit.timeit(code, setup = 'import numpy as np', number = 100)\n\n0.00018205121159553528\n\n\nThat reports the total time for the 100 replications.\nWe can run it from the command line.\n\npython -m timeit -s 'import numpy' -n 1000 'x = numpy.exp(3.)'\n\n1000 loops, best of 5: 987 nsec per loop\n\n\ntimeit ran the code 1000 times for 5 different repetitions, giving the average time for the 1000 samples for the best of the 5 repetitions.\n\n\nProfiling\nThe Cprofile module will show you how much time is spent in different functions, which can help you pinpoint bottlenecks in your code.\nI haven’t run this code when producing this document as the output of the profiling can be lengthy.\n\ndef lr_slow(y, x):\n xtx = x.T @ x\n xty = x.T @ y\n inv = np.linalg.inv(xtx)\n return inv @ xty\n\n## generate random observations and random matrix of predictors\ny = np.random.normal(size = 5000)\nx = np.random.normal(size = (5000,1000))\n\nt0 = time.time()\nregr = lr_slow(y, x)\nt1 = time.time()\nprint(f\"Execution time: {t1-t0} seconds.\")\n\nimport cProfile\ncProfile.run('lr_slow(y,x)')\n\nThe cumtime column includes the time spent in nested calls to functions while the tottime column excludes it.\nAs we’ll discuss in detail in Unit 10, we almost never want to explicitly invert a matrix. Instead we factorize the matrix and use the factorized result to do the computation of interest. In this case using the Cholesky decomposition is a standard approach, followed by solving triangular systems of equations.\n\nimport scipy as sp\n\ndef lr_fast(y, x):\n xtx = x.T @ x\n xty = x.T @ y\n L = sp.linalg.cholesky(xtx)\n out = sp.linalg.solve_triangular(L.T, \n sp.linalg.solve_triangular(L, xty, lower=True),\n lower=False)\n return(out)\n\nt0 = time.time()\nregr = lr_fast(y, x)\nt1 = time.time()\nprint(f\"Execution time: {t1-t0} seconds.\")\n\ncProfile.run('lr_fast(y,x)')\n\nThe Cholesky now dominates the computational time (but is much faster than inv), so there’s not much more we can do in this case.\nYou might wonder if it’s better to use x.T or np.transpose(x). Try using timeit to decide.\nThe Python profilers (cProfile and profile (not shown)) use deterministic profiling – calculating the interval between events (i.e., function calls and returns). However, there is some limit to accuracy – the underlying ‘clock’ measures in units of about 0.001 seconds.\n(In contrast, R’s profiler works by sampling (statistical profiling) - every little while during a calculation it finds out what function R is in and saves that information to a file. So if you try to profile code that finishes really quickly, there’s not enough opportunity for the sampling to represent the calculation accurately and you may get spurious results.)" - }, - { - "objectID": "units/unit5-programming.html#writing-efficient-python-code", - "href": "units/unit5-programming.html#writing-efficient-python-code", - "title": "Programming concepts", - "section": "Writing efficient Python code", - "text": "Writing efficient Python code\nWe’ll discuss a variety of these strategies, including:\n\nPre-allocating memory rather than growing objects iteratively\nVectorization and use of fast matrix algebra\nConsideration of loops vs. map operations\nSpeed of lookup operations, including hashing\n\n\nPre-allocating memory\nLet’s consider whether we should pre-allocate space for the output of an operation or if it’s ok to keep extending the length of an array or list.\n\nn = 100000\nz = np.random.normal(size = n)\n\n## Pre-allocation\n\ndef fun_prealloc(vals):\n n = len(vals)\n x = [0] * n\n for i in range(n):\n x[i] = np.exp(vals[i])\n return(x)\n\n## Appending to a list\n\ndef fun_append(vals):\n x = []\n for i in range(n):\n x.append(np.exp(vals[i]))\n return(x)\n\n## Appending to a numpy array\n\ndef fun_append_np(vals):\n x = np.array([])\n for i in range(n):\n x = np.append(x, np.exp(vals[i]))\n return(x)\n\n\nt0 = time.time()\nout1 = fun_prealloc(z)\ntime.time() - t0\n\n0.0720217227935791\n\nt0 = time.time()\nout2 = fun_append(z)\ntime.time() - t0\n\n0.0720512866973877\n\nt0 = time.time()\nout3 = fun_append_np(z)\ntime.time() - t0\n\n2.4477429389953613\n\n\nSo what’s going on? First let’s consider what is happening with the use of np.append. Note that it is a function, rather than a method, and we need to reassign to x. What must be happening in terms of memory use and copying when we append an element?\n\nx = np.random.normal(size = 5)\nid(x)\n\n140228665678064\n\nid(np.append(x, 3.34))\n\n140228665678160\n\n\nWe can avoid that large cost of copying and memory allocation by pre-allocating space for the entire output array. (This is equivalent to variable initialization in compiled languages.)\nOk, but how is it that we can append to the list at apparently no cost?\nIt’s not magic, just that Python is clever. Let’s get an idea of what is going on:\n\ndef fun_append2(vals):\n n = len(vals)\n x = []\n print(f\"Initial id: {id(x)}\")\n sz = sys.getsizeof(x)\n print(f\"iteration 0: size {sz}\")\n for i in range(n):\n x.append(np.exp(vals[i]))\n if sys.getsizeof(x) != sz:\n sz = sys.getsizeof(x)\n print(f\"iteration {i}: size {sz}\")\n print(f\"Final id: {id(x)}\")\n return(x)\n\nz = np.random.normal(size = 1000)\nout = fun_append2(z)\n\nInitial id: 140228665954048\niteration 0: size 56\niteration 0: size 88\niteration 4: size 120\niteration 8: size 184\niteration 16: size 248\niteration 24: size 312\niteration 32: size 376\niteration 40: size 472\niteration 52: size 568\niteration 64: size 664\niteration 76: size 792\niteration 92: size 920\niteration 108: size 1080\niteration 128: size 1240\niteration 148: size 1432\niteration 172: size 1656\niteration 200: size 1912\niteration 232: size 2200\niteration 268: size 2520\niteration 308: size 2872\niteration 352: size 3256\niteration 400: size 3704\niteration 456: size 4216\niteration 520: size 4792\niteration 592: size 5432\niteration 672: size 6136\niteration 760: size 6936\niteration 860: size 7832\niteration 972: size 8856\nFinal id: 140228665954048\n\n\nSurprisingly, the id of x doesn’t seem to change, even though we are allocating new memory at many of the iterations. What is happening is that x is an wrapper object that contains within it a reference to an array of references (pointers) to the list elements. The location of the wrapper object doesn’t change, but the underlying array of references/pointers is being reallocated.\nSide note: our assessment of size above does not include the actual size of the list elements.\n\nprint(sys.getsizeof(out))\n\n8856\n\nout[2] = np.random.normal(size = 100000)\nprint(sys.getsizeof(out))\n\n8856\n\n\nOne upshot of this is that if you need to grow an object use a Python list. Then once it is complete, you can always convert it to another type, such as a numpy array.\n\n\nVectorization and use of fast matrix algebra\nOne key way to write efficient Python code is to take advantage of numpy’s vectorized operations.\n\nn = 10**6\nz = np.random.normal(size = n)\nt0 = time.time()\nx = np.exp(z)\nprint(time.time() - t0)\n\n0.03270316123962402\n\nx = np.zeros(n) # Leave out pre-allocation timing to focus on computation.\nt0 = time.time()\nfor i in range(n):\n x[i] = np.exp(z[i])\n\n\nprint(time.time() - t0)\n\n0.808849573135376\n\n\nSo what is different in how Python handles the calculations above that explains the huge disparity in efficiency? The vectorized calculation is being done natively in C in a for loop. The explicit Python for loop involves executing the for loop in Python with repeated calls to C code at each iteration. This involves a lot of overhead because of the repeated processing of the Python code inside the loop. For example, in each iteration of the loop, Python is checking the types of the variables because it’s possible that the types might change, as discussed earlier.\nYou can usually get a sense for how quickly a Python call will pass things along to C or Fortran by looking at the body of the relevant function(s) being called.\nUnfortunately seeing the source code in Python often involves going and finding it in a file on disk, whereas in R, printing a function will show its source code. However you can use ?? in IPython to get the code for non-builtin functions. Consider numpy.linspace??.\nHere I found the source code for the scipy triangular_solve function, which calls out to a Fortran function trtrs, found in the LAPACK library.\n\n## On an SCF machine:\ncat /usr/local/linux/mambaforge-3.11/lib/python3.11/site-packages/scipy/linalg/_basic.py\n\nWith a bit more digging around we could verify that trtrs is a LAPACK funcion by doing some grepping:\n./linalg/_basic.py: trtrs, = get_lapack_funcs(('trtrs',), (a1, b1))\nMany numpy and scipy functions allow you to pass in arrays, and operate on those arrays in vectorized fashion. So before writing a for loop, look at the help information on the relevant function(s) to see if they operate in a vectorized fashion. Functions might take arrays for one or more of their arguments.\nOutside of the numerical packages, we often have to manually do the looping:\n\nx = [3.5, 2.7, 4.6]\ntry:\n math.cos(x)\nexcept Exception as error:\n print(error)\n\nmust be real number, not list\n\n[math.cos(val) for val in x]\n\n[-0.9364566872907963, -0.9040721420170612, -0.11215252693505487]\n\nlist(map(math.cos, x))\n\n[-0.9364566872907963, -0.9040721420170612, -0.11215252693505487]\n\n\nChallenge: Consider the chi-squared statistic involved in a test of independence in a contingency table:\n\\[\n\\chi^{2}=\\sum_{i}\\sum_{j}\\frac{(y_{ij}-e_{ij})^{2}}{e_{ij}},\\,\\,\\,\\, e_{ij}=\\frac{y_{i\\cdot}y_{\\cdot j}}{y_{\\cdot\\cdot}}\n\\]\nwhere \\(y_{i\\cdot}=\\sum_{j}y_{ij}\\) and \\(y_{\\cdot j} = \\sum_{i} y_{ij}\\) and \\(y_{\\cdot\\cdot} = \\sum_{i} \\sum_{j} y_{ij}\\). Write this in a vectorized way without any loops. Note that ‘vectorized’ calculations also work with matrices and arrays.\nSometimes we can exploit vectorized mathematical operations in surprising ways, though sometimes the code is uglier.\n\nx = np.random.normal(size = n)\n\n## List comprehension\ntimeit.timeit('truncx = [max(0,val) for val in x]', number = 10, globals = {'x':x})\n\n0.17521439492702484\n\n\n\n## Vectorized slice replacement\ntimeit.timeit('truncx = x.copy(); truncx[x < 0] = 0', number = 10, globals = {'x':x})\n\n0.005411941558122635\n\n\n\n## Vectorized math trick\ntimeit.timeit('truncx = x * x>0', number = 10, globals = {'x':x})\n\n0.0010863672941923141\n\n\nWe’ll discuss what has to happen (in terms of calculations, memory allocation, and copying) in the two vectorized approaches to try to understand which is more efficient.\nAdditional tips:\n\nIf you do need to loop over dimensions of a matrix or array, if possible loop over the smallest dimension and use the vectorized calculation on the larger dimension(s). For example if you have a 10000 by 10 matrix, try to set up your problem so you can loop over the 10 columns rather than the 10000 rows.\nIn general, in Python looping over rows is likely to be faster than looping over columns because of numpy’s row-major ordering (by default, matrices are stored in memory as a long array in which values in a row are adjacent to each other). However how numpy handles this is more complicated (see more in the Section on cache-aware programming), such that it may not matter for numpy calculations.\nYou can use direct arithmetic operations to add/subtract/multiply/divide a vector by each column of a matrix, e.g. A*b does element-wise multiplication of each column of A by a vector b. If you need to operate by row, you can do it by transposing the matrix.\n\nCaution: relying on Python’s broadcasting rule in the context of vectorized operations, such as is done when direct-multiplying a matrix by a vector to scale the columns relative to each other, can be dangerous as the code may not be easy for someone to read and poses greater dangers of bugs. In some cases you may want to first write the code more directly and then compare the more efficient code to make sure the results are the same. It’s also a good idea to comment your code in such cases.\n\n\nVectorization, mapping, and loops\nNext let’s consider when loops and mapping would be particularly slow and how mapping and loops might compare to each other.\nFirst, the potential for inefficiency of looping and map operations in interpreted languages will depend in part on whether a substantial part of the work is in the overhead involved in the looping or in the time required by the function evaluation on each of the elements.\nHere’s an example, where the core computation is very fast, so we might expect the overhead of looping (in its various forms seen here) to be important.\n\nimport time\nn = 10**6\nx = np.random.normal(size = n)\n\nt0 = time.time()\nout = np.exp(x)\ntime.time() - t0\n\n0.012585163116455078\n\nt0 = time.time()\nvals = np.zeros(n)\nfor i in range(n):\n vals[i] = np.exp(x[i])\n\n\ntime.time() - t0\n\n0.8506276607513428\n\nt0 = time.time()\nvals = [np.exp(v) for v in x]\ntime.time() - t0\n\n0.6357121467590332\n\nt0 = time.time()\nvals = list(map(np.exp, x))\ntime.time() - t0\n\n0.5772781372070312\n\n\nRegardless of how we do the looping (an explicit loop, list comprehension, or map), it looks like we can’t avoid the overhead unless we use the vectorized call, which is of course the recommended approach in this case, both for speed and readability (and conciseness).\nSecond, is it faster to use map than to use a loop? In the example above it is somewhat faster to use map. That might not be surprising. In the loop case, the interpreter needs to do the checking we discussed earlier in this section at each iteration of the loop. What about in the map case? For mapping over a numpy array, perhaps not, but what if mapping over a list? So without digging into how map works, it’s hard to say.\nHere’s an example where the bulk of time is in the actual computation and not in the looping itself. We’ll run a bunch of regressions on a matrix X (i.e., each column of X is a predictor) using each column of the matrix mat to do a separate regression.\n\nimport time\nimport statsmodels.api as sm\n\nn = 500000;\nnr = 10000\nnCalcs = int(n/nr)\n\nmat = np.random.normal(size = (nr, nCalcs))\n\nX = list(range(nr))\nX = sm.add_constant(X)\n\ndef regrFun(i):\n model = sm.OLS(mat[:,i], X)\n return(model.fit().params[1])\n\nt0 = time.time()\nout1 = list(map(regrFun, range(nCalcs)))\ntime.time() - t0\n\n0.05824542045593262\n\nt0 = time.time()\nout2 = np.zeros(nCalcs)\nfor i in range(nCalcs):\n out2[i] = regrFun(i)\n\n\ntime.time() - t0\n\n0.07360720634460449\n\n\nHere the looping is faster. I don’t have any particular explanation for that result.\n\n\nMatrix algebra efficiency\nOften calculations that are not explicitly linear algebra calculations can be done as matrix algebra. If our Python installation has a fast (and possibly parallelized) BLAS, this allows our calculation to take advantage of it.\nFor example, we can sum the rows of a matrix by multiplying by a vector of ones.\n\nmat = np.random.normal(size=(500,500))\n\ntimeit.timeit('mat.dot(np.ones(500))', setup = 'import numpy as np',\n number = 1000, globals = {'mat': mat})\n\n0.033969756215810776\n\ntimeit.timeit('np.sum(mat, axis = 1)', setup = 'import numpy as np',\n number = 1000, globals = {'mat': mat})\n\n0.1168225146830082\n\n\nGiven the extra computation involved in actually multiplying each number by one, it’s surprising that this is faster than numpy sum function. One thing we’d want to know is whether the BLAS matrix multiplication call is being done in parallel.\nOn the other hand, big matrix operations can be slow.\nChallenge: Suppose you want a new matrix that computes the differences between successive columns of a matrix of arbitrary size. How would you do this as matrix algebra operations? It’s possible to write it as multiplying the matrix by another matrix that contains 0s, 1s, and -1s in appropriate places. Here it turns out that the for loop is much faster than matrix multiplication. However, there is a way to do it faster as matrix direct subtraction.\n\n\nOrder of operations and efficiency\nWhen doing matrix algebra, the order in which you do operations can be critical for efficiency. How should I order the following calculation?\n\nn = 5000\nA = np.random.normal(size=(n, n))\nB = np.random.normal(size=(n, n))\nx = np.random.normal(size=n)\n\nt0 = time.time()\nres1 = (A @ B) @ x\nprint(time.time() - t0)\n\n1.7937490940093994\n\nt0 = time.time()\nres1 = A @ (B @ x)\nprint(time.time() - t0)\n\n0.08377599716186523\n\n\nWhy is the second order much faster?\n\n\nAvoiding unnecessary operations\nWe can use the matrix direct product (i.e., A*B) to do some manipulations much more quickly than using matrix multiplication. Challenge: How can I use the direct product to find the trace of a matrix, \\(XY\\)?\nFinally, when working with diagonal matrices, you can generally get much faster results by being smart. The following operations: \\(X+D\\), \\(DX\\), \\(XD\\) are mathematically the sum of two matrices and products of two matrices. But we can do the computation without using two full matrices. Challenge: How?\n\nn = 1000\nX = np.random.normal(size=(n, n))\ndiagvals = np.random.normal(size=n)\nD = np.diag(diagvals)\n\n# The following lines are very inefficient\nsummedMat = X + D\nprodMat1 = D @ X\nprodMat2 = X @ D\n\nMore generally, sparse matrices and structured matrices (such as block diagonal matrices) can generally be worked with MUCH more efficiently than treating them as arbitrary matrices. The scipy.sparse package (for both structured and arbitrary sparse matrices) can help, as can specialized code available in other languages, such as C and Fortran packages.\n\n\nSpeed of lookup operations\nThere are lots of situations in which we need to retrieve values for subsequent computations. In some cases we might be retrieving elements of an array or looking up values in a dictionary.\nLet’s compare the speed of some different approaches to lookup.\n\nn = 1000\nx = list(np.random.normal(size = n))\nkeys = [str(v) for v in range(n)]\nxD = dict(zip(keys, x))\n\ntimeit.timeit(\"x[500]\", number = 10**6, globals = {'x':x})\n\n0.014431843534111977\n\ntimeit.timeit(\"xD['500']\", number=10**6, globals = {'xD':xD})\n\n0.026302199810743332\n\n\nHow is it that Python can look up by key in the dictionary at essentially the same speed as jumping to an index position? It uses hashing, which allows O(1) lookup. In contrast, if one has to look through each key in turn, that is O(n), which is much slower:\n\ntimeit.timeit(\"x[keys.index('500')]\", number = 10**6, globals = {'x':x, 'keys':keys})\n\n5.733805952593684\n\n\nAs a further point of contrast, if we look up elements by name in R in named vectors or lists, that is much slower than looking up by index, because R doesn’t use hashing in that context and has to scan through the objects one by one until it finds the one with the name it is looking for. This stands in contrast to R and Python being able to directly go to the position of interest based on the index of an array, or to the hash-based lookup in a Python dictionary or an R environment.\n\n\nHashing (including name lookup)\nAbove I mentioned that Python uses hashing to store and lookup values by key in a dictionary. I’ll briefly describe what hashing is here, because it is a commonly-used strategy in programming in general.\nA hash function is a function that takes as input some data and maps it to a fixed-length output that can be used as a shortened reference to the data. (The function should be deterministic, always returing the same output for a given input.) We’ve seen this in the context of git commits where each commit was labeled with a long base-16 number. This also comes up when verifying files on the Internet. You can compute the hash value on the file you get and check that it is the same as the hash value associated with the legitimate copy of the file.\nWhile there are various uses of hashing, for our purposes here, hashing can allow one to look up values by their name via a hash table. The idea is that you have a set of key-value pairs (sometimes called a dictionary) where the key is the name associated with the value and the value is some arbitrary object. You want to be able to quickly find the value/object quickly.\nHashing allows one to quickly determine an index associated with the key and therefore quickly find the relevant value based on the index. For example, one approach is to compute the hash as a function of the key and then take the remainder when dividing by the number of possible results (here the fact that the result is a fixed-length output is important) to get the index. Here’s the procedure in pseudocode:\n hash = hashfunc(key) \n index = hash %% array_size \n ## %% is modulo operator - it gives the remainder\nIn general, there will be collisions – multiple keys will be assigned to the same index. However with a good hash function, usually there will be a small number of keys associated with a given bucket. So each bucket will contain a list of a small number of values and the associated keys. (The buckets might contain the actual values or they might contain the addresses of where the values are actually stored if the values are complicated objects.) Then determining the correct value (or the required address) within a given bucket is fast even with simple linear search through the items one by one. Put another way, the hash function distributes the keys amongst an array of buckets and allows one to look up the appropriate bucket quickly based on the computed index value. When the hash table is properly set up, the cost of looking up a value does not depend on the number of key-value pairs stored.\nPython uses hashing to look up the value based on the key in a given dictionary, and similarly when looking up variables in namespaces. This allows Python to retrieve objects very quickly." - }, - { - "objectID": "units/unit5-programming.html#additional-general-strategies-for-efficiency", - "href": "units/unit5-programming.html#additional-general-strategies-for-efficiency", - "title": "Programming concepts", - "section": "Additional general strategies for efficiency", - "text": "Additional general strategies for efficiency\nIt’s also useful to be aware of some other strategies for improving efficiency.\n\nCache-aware programming\nIn addition to main memory (what we usually mean when we talk about RAM), computers also have memory caches, which are small amounts of fast memory that can be accessed very quickly by the processor. For example your computer might have L1, L2, and L3 caches, with L1 the smallest and fastest and L3 the largest and slowest. The idea is to try to have the data that is most used by the processor in the cache.\nIf the next piece of data needed for computation is available in the cache, this is a cache hit and the data can be accessed very quickly. However, if the data is not available in the cache, this is a cache miss and the speed of access will be a lot slower. Cache-aware programming involves writing your code to minimize cache misses. Generally when data is read from memory it will be read in chunks, so values that are contiguous will be read together.\nHow does this inform one’s programming? For example, if you have a matrix of values stored in row-major order, computing on a row will be a lot faster than computing on a column, because the row can be read into the cache from main memory and then accessed in the cache. In contrast, if the matrix is large and therefore won’t fit in the cache, when you access the values of a column, you’ll have to go to main memory repeatedly to get the values for the row because the values are not stored contiguously.\nThere’s a nice example of the importance of the cache at the bottom of this blog post.\nIf you know the size of the cache, you can try to design your code so that in a given part of your code you access data structures that will fit in the cache. This sort of thing is generally more relevant if you’re coding in a language like C. But it can matter sometimes in interpreted languages too.\nLet’s see what happens in Python. By default, matrices in numpy are row-major, also called “C order”. I’ll create a long matrix with a small number of very long columns and a wide matrix with a small number of very long rows.\n\nnr = 800000\nnc = 100\n\nA = np.random.normal(size=(nr, nc)) # long matrix\ntA = np.random.normal(size=(nc, nr)) # wide matrix\n\n## Verify that A is row-major using `.flags` (notice the `C_CONTIGUOUS: True`).\nA.flags\n\n C_CONTIGUOUS : True\n F_CONTIGUOUS : False\n OWNDATA : True\n WRITEABLE : True\n ALIGNED : True\n WRITEBACKIFCOPY : False\n\n\nNote that I didn’t use A.T or np.transpose as that doesn’t make a copy in memory and so the transposed matrix doesn’t end up being row-major. You can use A.flags and A.T.flags` to see this.\nNow let’s time calculating the sum by column in the long matrix vs. the sum by row in the wide matrix. Exactly the same number of arithmetic operations needs to be done in an equivalent manner for the two cases. We want to use a large enough matrix so the entire matrix doesn’t fit in the cache, but not so large that the example takes a long time or a huge amount of memory. We’ll use a rectangular matrix, such that the summation for a single column of the long matrix or a single row of the wide matrix involves many numbers, but there are a limited number of such summations. This focuses the example on the efficiency of the column-wise vs. row-wise summation rather than any issues that might be involved in managing large numbers of such summations (e.g., doing many, many summations that involve just a few numbers).\n\n# Define the sum calculations as functions\ndef sum_by_column():\n return np.sum(A, axis=0)\n\ndef sum_by_row():\n return np.sum(tA, axis=1)\n\ntimeit.timeit(sum_by_column, number=10) # potentially slow\n\n0.5056534707546234\n\ntimeit.timeit(sum_by_row, number=10)\n\n0.408780699595809\n\n\nSuppose we instead do the looping manually.\n\ntimeit.timeit('[np.sum(A[:,col]) for col in range(A.shape[1])]',\n setup = 'import numpy as np', number=10, globals = {'A': A}) \n\n5.12814655713737\n\ntimeit.timeit('[np.sum(tA[row,:]) for row in range(tA.shape[0])]',\n setup = 'import numpy as np', number=10, globals = {'tA': tA}) \n\n0.41643139719963074\n\n\nIndeed, the row-wise calculations are much faster when done manually. However, when done with the axis argument in np.sum there is little difference. So that suggests numpy might be doing something clever in its implementation of sum with the axis argument.\n\nChallenge: suppose you were writing code for this kind of use case. How could you set up your calculations to do either row-wise or column-wise operations in a way that processes each number sequentially based on the order in which the numbers are stored. For example suppose the values are stored row-major but you want the column sums.\n\nWhen we define a numpy array, we can choose to use column-major order (i.e., “Fortran” order) with the order argument.\n\n\nLoop fusion\nLet’s consider this (vectorized) code:\n\nx = np.exp(x) + 3*np.sin(x)\n\nThis code has some downsides.\n\nThink about whether any additional memory has to be allocated.\nThink about how many for loops will have to get executed.\n\nContrast that to running directly as a for loop (e.g., here in Julia or in C/C++):\n\nfor i in 1:length(x)\n x[i] = exp(x[i]) + 3*sin(x[i])\nend\n\nHow does that affect the downsides mentioned above?\nCombining loops is called ‘fusing’ and is an important optimization that Julia can do, as shown in this demo. It’s also a key optimization done by XLA, a compiler used with Tensorflow, so one approach to getting loop fusion in Python is to use Tensorflow for such calculations within Python rather than simply using numpy.\n\n\nLazy evaluation\nWhat’s strange about this R code?\n\nf <- function(x) print(\"hi\")\nsystem.time(mean(rnorm(1000000)))\n\n user system elapsed \n 0.058 0.000 0.059 \n\nsystem.time(f(3))\n\n[1] \"hi\"\n\n\n user system elapsed \n 0 0 0 \n\nsystem.time(f(mean(rnorm(1000000)))) \n\n[1] \"hi\"\n\n\n user system elapsed \n 0.001 0.000 0.001 \n\n\nLazy evaluation is not just an R thing. It also occurs in Tensorflow (particularly version 1), the Python Dask package, and in Spark. The basic idea is to delay executation until it’s really needed, with the goal that if one does so, the system may be able to better optimize a series of multiple steps as a joint operation relative to executing them one by one.\nHowever, Python itself does not have lazy evaluation." - }, - { - "objectID": "units/unit9-sim.html", - "href": "units/unit9-sim.html", - "title": "Simulation", - "section": "", - "text": "PDF\nReferences:\nMany (most?) statistical papers include a simulation (i.e., Monte Carlo) study. Many papers on machine learning methods also include a simulation study. The basic idea is that closed-form mathematical analysis of the properties of a statistical or machine learning method/model is often hard to do. Even if possible, it usually involves approximations or simplifications. A canonical situation in statistics is that we have an asymptotic result and we want to know what happens in finite samples, but often we do not even have the asymptotic result. Instead, we can estimate mathematical expressions using random numbers. So we design a simulation study to evaluate the method/model or compare multiple methods. The result is that the researcher carries out an experiment (on the computer, sometimes called in silico), generally varying different factors to see what has an effect on the outcome of interest.\nThe basic strategy generally involves simulating data and then using the method(s) on the simulated data, summarizing the results to assess/compare the method(s).\nMost simulation studies aim to approximate an integral, generally an expected value (mean, bias, variance, MSE, probability, etc.). In low dimensions, methods such as Gaussian quadrature are best for estimating an integral but these methods don’t scale well, so in higher dimensions (e.g., the usual situation with \\(n\\) observations) we often use Monte Carlo techniques.\nTo be more concrete:" - }, - { - "objectID": "units/unit9-sim.html#motivating-example", - "href": "units/unit9-sim.html#motivating-example", - "title": "Simulation", - "section": "Motivating example", - "text": "Motivating example\nLet’s consider linear regression, with observations \\(Y=(y_{1},y_{2},\\ldots,y_{n})\\) and an \\(n\\times p\\) matrix of predictors/covariates/features/variables \\(X\\), where \\(\\hat{\\beta}=(X^{\\top}X)^{-1}X^{\\top}Y\\). If we assume that we have \\(EY=X\\beta\\) and \\(\\mbox{Var}(Y)=\\sigma^{2}I\\), then we can determine analytically that we have \\[\\begin{aligned}\nE\\hat{\\beta} & = & \\beta\\\\\n\\mbox{Var}(\\hat{\\beta})=E((\\hat{\\beta}-E\\hat{\\beta})^{2}) & = & \\sigma^{2}(X^{\\top}X)^{-1}\\\\\n\\mbox{MSPE}(Y^{*})=E(Y^{*}-\\hat{Y})^{2}) & = & \\sigma^{2}(1+X^{*\\top}(X^{\\top}X)^{-1}X^{*}).\\end{aligned}\\] where \\(Y^{*}\\)is some new observation we’d like to predict given \\(X^{*}\\).\nBut suppose that we’re interested in the properties of standard regression estimation when in reality the mean is not linear in \\(X\\) or the properties of the errors are more complicated than having independent homoscedastic errors. (This is always the case, but the issue is how far from the truth the standard assumptions are.) Or suppose we have a modified procedure to produce \\(\\hat{\\beta}\\), such as a procedure that is robust to outliers. In those cases, we cannot compute the expectations above analytically.\nInstead we decide to use a Monte Carlo estimate. To keep the notation more simple, let’s just consider one element of the vector \\(\\beta\\) (i.e., one of the regression coefficients) and continue to call that \\(\\beta\\). If we randomly generate \\(m\\) different datasets from some distribution \\(f\\), and \\(\\hat{\\beta}_{i}\\) is the estimated coefficient based on the \\(i\\)th dataset: \\(Y_{i}=(y_{i1},y_{i2},\\ldots,y_{in})\\), then we can estimate \\(E\\hat{\\beta}\\) under that distribution \\(f\\) as \\[\\hat{E}(\\hat{\\beta})=\\bar{\\hat{\\beta}}=\\frac{1}{m}\\sum_{i=1}^{m}\\hat{\\beta}_{i}\\] Or to estimate the variance, we have \\[\\widehat{\\mbox{Var}}(\\hat{\\beta})=\\frac{1}{m}\\sum_{i=1}^{m}(\\hat{\\beta}_{i}-\\bar{\\hat{\\beta}})^{2}.\\] In evaluating the performance of regression under non-standard conditions or the performance of our robust regression procedure, what decisions do we have to make to be able to carry out our Monte Carlo procedure?\nNext let’s think about Monte Carlo methods in general." - }, - { - "objectID": "units/unit9-sim.html#monte-carlo-mc-basics", - "href": "units/unit9-sim.html#monte-carlo-mc-basics", - "title": "Simulation", - "section": "Monte Carlo (MC) basics", - "text": "Monte Carlo (MC) basics\n\nMonte Carlo overview\nThe basic idea is that we often want to estimate \\(\\phi\\equiv E_{f}(h(Y))\\) for \\(Y\\sim f\\). Note that if \\(h\\) is an indicator function, this includes estimation of probabilities, e.g., for a scalar \\(Y\\), we have \\(p=P(Y\\leq y)=F(y)=\\int_{-\\infty}^{y}f(t)dt=\\int I(t\\leq y)f(t)dt=E_{f}(I(Y\\leq y))\\). We would estimate variances or MSEs by having \\(h\\) involve squared terms.\nWe get an MC estimate of \\(\\phi\\) based on an iid sample of a large number of values of \\(Y\\) from \\(f\\): \\[\\hat{\\phi}=\\frac{1}{m}\\sum_{i=1}^{m}h(Y_{i}),\\] which is justified by the Law of Large Numbers: \\[\\lim_{m\\to\\infty}\\frac{1}{m}\\sum_{i=1}^{m}h(Y_{i})=E_{f}h(Y).\\]\nNote that in most simulation studies, \\(Y\\) is an entire dataset (predictors/covariates), and the “iid sample” means generating \\(m\\) different datasets from \\(f\\), i.e., \\(Y_{i}\\in\\{Y_{1},\\ldots,Y_{m}\\}\\) not \\(m\\) different scalar values. If the dataset has \\(n\\) observations, then \\(Y_{i}=(Y_{i1},\\ldots,Y_{in})\\).\n\nBack to the regression example\nLet’s relate that back to our regression example. In that particular case, if we’re interested in whether the regression estimator is biased, we want to know: \\[\\phi=E\\hat{\\beta},\\] where \\(h(Y) = \\hat{\\beta}(Y)\\). We can use the Monte Carlo estimate of \\(\\phi\\): \\[\\hat{\\phi}=\\frac{1}{m}\\sum_{i=1}^{m}h(Y_{i})=\\frac{1}{m}\\sum_{i=1}^{m}\\hat{\\beta}_{i}=\\widehat{E(\\hat{\\beta})}.\\]\nIf we are interested in the variance of the regression estimator, we have\n\\[\\phi=\\mbox{Var}(\\hat{\\beta})=E_{f}((\\hat{\\beta}-E\\hat{\\beta})^{2})\\] and we can use the Monte Carlo estimate of \\(\\phi\\): \\[\\hat{\\phi}=\\frac{1}{m}\\sum_{i=1}^{m}h(Y_{i})=\\frac{1}{m}\\sum_{i=1}^{m}(\\hat{\\beta}_{i}-E\\hat{\\beta})^{2}=\\widehat{\\mbox{Var}(\\hat{\\beta)}}\\] where \\[h(Y)=(\\hat{\\beta}-E\\hat{\\beta})^{2}.\\]\nFinally note that we also need to use the Monte Carlo estimate of \\(E\\hat{\\beta}\\) in the Monte Carlo estimation of the variance.\nWe might also be interested in the coverage of a confidence interval. In that case we have \\[h(Y)=1_{\\beta\\in CI(Y)}\\] and we can estimate the coverage as \\[\\hat{\\phi}=\\frac{1}{m}\\sum_{i=1}^{m}1_{\\beta\\in CI(y_{i})}.\\] Of course we want that \\(\\hat{\\phi}\\approx1-\\alpha\\) for a \\(100(1-\\alpha)\\) confidence interval. In the standard case of a 95% interval we want \\(\\hat{\\phi}\\approx0.95\\).\n\n\n\nSimulation uncertainty (i.e., Monte Carlo uncertainty)\nSince \\(\\hat{\\phi}\\) is simply an average of \\(m\\) identically-distributed values, \\(h(Y_{1}),\\ldots,h(Y_{m})\\), the simulation variance of \\(\\hat{\\phi}\\) is \\(\\mbox{Var}(\\hat{\\phi})=\\sigma^{2}/m\\), with \\(\\sigma^{2}=\\mbox{Var}(h(Y))\\). An estimator of \\(\\sigma^{2}=E_{f}((h(Y)-\\phi)^{2})\\) is \\[\\begin{aligned}\n\\hat{\\sigma}^{2} & = & \\frac{1}{m-1}\\sum_{i=1}^{m}(h(Y_{i})-\\hat{\\phi})^{2}\\end{aligned}\\] So our MC simulation error is based on \\[\\widehat{\\mbox{Var}}(\\hat{\\phi})=\\frac{\\hat{\\sigma}^{2}}{m}=\\frac{1}{m(m-1)}\\sum_{i=1}^{m}(h(Y_{i})-\\hat{\\phi})^{2}.\\] Note that this is particularly confusing if we have \\(\\hat{\\phi}=\\widehat{\\mbox{Var}(\\hat{\\beta})}\\) because then we have \\(\\widehat{\\mbox{Var}}(\\hat{\\phi})=\\widehat{\\mbox{Var}}(\\widehat{\\mbox{Var}(\\hat{\\beta})})\\)!\nThe simulation variance is \\(O(\\frac{1}{m})\\) because we have \\(m^{2}\\) in the denominator and a sum over \\(m\\) terms in the numerator.\nNote that in the simulation setting, the randomness in the system is very well-defined (as it is in survey sampling, but unlike in most other applications of statistics), because it comes from the RNG that we perform as part of our attempt to estimate \\(\\phi\\). Happily, we are in control of \\(m\\), so in principle we can reduce the simulation error to as little as we desire. Unhappily, as usual, the simulation standard error goes down with the square root of \\(m\\).\n\nImportant: This is the uncertainty in our simulation-based estimate of some quantity (expectation) of interest. It is NOT the statistical uncertainty in a problem.\n\n\nBack to the regression example\nSome examples of simulation variances we might be interested in in the regression example include:\n\nUncertainty in our estimate of bias: \\(\\widehat{\\mbox{Var}}(\\widehat{E(\\hat{\\beta})}-\\beta)\\).\nUncertainty in the estimated variance of the estimated coefficient: \\(\\widehat{\\mbox{Var}}(\\widehat{\\mbox{Var}(\\hat{\\beta})})\\).\nUncertainty in the estimated mean square prediction error: \\(\\widehat{\\mbox{Var}}(\\widehat{\\mbox{MSPE}(Y^{*})})\\).\n\nIn all cases we have to estimate the simulation variance, hence the \\(\\widehat{\\mbox{Var}}()\\) notation.\n\n\n\nFinal notes\nSometimes the \\(Y_{i}\\) are generated in a dependent fashion (e.g., sequential MC or MCMC), in which case this variance estimator, \\(\\widehat{\\mbox{Var}}(\\hat{\\phi})\\) does not hold because the samples are not IID, but the estimator \\(\\hat{\\phi}\\) is still a valid, unbiased estimator of \\(\\phi\\)." - }, - { - "objectID": "units/unit9-sim.html#variance-reduction-optional", - "href": "units/unit9-sim.html#variance-reduction-optional", - "title": "Simulation", - "section": "Variance reduction (optional)", - "text": "Variance reduction (optional)\nThere are some tools for variance reduction in MC settings. One is importance sampling (see Section 3). Others are the use of control variates and antithetic sampling. I haven’t personally run across these latter in practice, so I’m not sure how widely used they are and won’t go into them here.\nIn some cases we can set up natural strata, for which we know the probability of being in each stratum. Then we would estimate \\(\\mu\\) for each stratum and combine the estimates based on the probabilities. The intuition is that we remove the variability in sampling amongst the strata from our simulation.\nAnother strategy that comes up in MCMC contexts is Rao-Blackwellization. Suppose we want to know \\(E(h(X))\\) where \\(X=\\{X_{1},X_{2}\\}\\). Iterated expectation tells us that \\(E(h(X))=E(E(h(X)|X_{2})\\). If we can compute \\(E(h(X)|X_{2})=\\int h(x_{1},x_{2})f(x_{1}|x_{2})dx_{1}\\) then we should avoid introducing stochasticity related to the \\(X_{1}\\) draw (since we can analytically integrate over that) and only average over stochasticity from the \\(X_{2}\\) draw by estimating \\(E_{X_{2}}(E(h(X)|X_{2})\\). The estimator is \\[\\hat{\\mu}_{RB}=\\frac{1}{m}\\sum_{i=1}^{m}E(h(X)|X_{2,i})\\] where we either draw from the marginal distribution of \\(X_{2}\\), or equivalently, draw \\(X\\), but only use \\(X_{2}\\). Our MC estimator averages over the simulated values of \\(X_{2}\\). This is called Rao-Blackwellization because it relates to the idea of conditioning on a sufficient statistic. It has lower variance because the variance of each term in the sum of the Rao-Blackwellized estimator is \\(\\mbox{Var}(E(h(X)|X_{2})\\), which is less than the variance in the usual MC estimator, \\(\\mbox{Var}(h(X))\\), based on the usual iterated variance formula: \\(V(X)=E(V(X|Y))+V(E(X|Y))\\Rightarrow V(E(X|Y))<V(X)\\)." - }, - { - "objectID": "units/unit9-sim.html#basic-steps-of-a-simulation-study", - "href": "units/unit9-sim.html#basic-steps-of-a-simulation-study", - "title": "Simulation", - "section": "Basic steps of a simulation study", - "text": "Basic steps of a simulation study\n\nSpecify what makes up an individual experiment (i.e., the individual simulated dataset) given a specific set of inputs: sample size, distribution(s) to use, parameter values, statistic of interest, etc. In other words, exactly how would you generate one simulated dataset?\nOften you’ll want to see how your results will vary if you change some of the inputs; e.g., sample sizes, parameter values, data generating mechanisms. So determine what factors you’ll want to vary. Each unique combination of input values will be a scenario.\nWrite code to carry out the individual experiment and return the quantity of interest, with arguments to your code being the inputs that you want to vary.\nFor each combination of inputs you want to explore (each scenario), repeat the experiment \\(m\\) times. Note this is an easily parallel calculation (in both the data generating dimension and the inputs dimension(s)).\nSummarize the results for each scenario, quantifying simulation uncertainty.\nReport the results in graphical or tabular form.\n\nOften a simulation study will compare multiple methods, so you’ll need to do steps 3-6 for each method." - }, - { - "objectID": "units/unit9-sim.html#various-considerations", - "href": "units/unit9-sim.html#various-considerations", - "title": "Simulation", - "section": "Various considerations", - "text": "Various considerations\nSince a simulation study is an experiment, we should use the same principles of design and analysis we would recommend when advising a practicioner on setting up a scientific experiment.\nThese include efficiency, reporting of uncertainty, reproducibility and documentation.\nIn generating the data for a simulation study, we want to think about what structure real data would have that we want to mimic in the simulation study: distributional assumptions, parameter values, dependence structure, outliers, random effects, sample size (\\(n\\)), etc.\nAll of these may become input variables in a simulation study. Often we compare two or more statistical methods conditioning on the data context and then assess whether the differences between methods vary with the data context choices. E.g., if we compare an MLE to a robust estimator, which is better under a given set of choices about the data generating mechanism and how sensitive is the comparison to changing the features of the data generating mechanism? So the “treatment variable” is the choice of statistical method. We’re then interested in sensitivity to the conditions (different input values).\nOften we can have a large number of replicates (\\(m\\)) because the simulation is fast on a computer, so we can sometimes reduce the simulation error to essentially zero and thereby avoid reporting uncertainty. To do this, we need to calculate the simulation standard error, generally, \\(s/\\sqrt{m}\\) and see how it compares to the effect sizes. This is particularly important when reporting on the bias of a statistical method.\nWe might denote the data, which could be the statistical estimator under each of two methods as \\(Y_{ijklq}\\), where \\(q\\) indexes treatment, \\(j,k,l\\) index different additional input variables, and \\(i\\in\\{1,\\ldots,m\\}\\) indexes the replicate. E.g., \\(j\\) might index whether the data are from a t or normal, \\(k\\) the value of a parameter, and \\(l\\) the dataset sample size (i.e., different levels of \\(n\\)).\nOne can think about choosing \\(m\\) based on a basic power calculation, though since we can always generate more replicates, one might just proceed sequentially and stop when the precision of the results is sufficient.\nWhen comparing methods, it’s best to use the same simulated datasets for each level of the treatment variable and to do an analysis that controls for the dataset (i.e., for the random numbers used), thereby removing some variability from the error term. A simple example is to do a paired analysis, where we look at differences between the outcome for two statistical methods, pairing based on the simulated dataset.\nOne can even use the “same” random number generation for the replicates under different conditions. E.g., in assessing sensitivity to a \\(t\\) vs. normal data generating mechanism, we might generate the normal RVs and then for the \\(t\\) use the same random numbers, in the sense of using the same quantiles of the \\(t\\) as were generated for the normal - this is pretty easy, as seen below. This helps to control for random differences between the datasets.\n\nfrom scipy.stats import t, norm\n\ndevs = np.random.normal(size=100)\ntdevs = t.ppf(norm.cdf(devs), df=1)\n\nplt.scatter(devs, tdevs)\nplt.xlabel('devs'); plt.ylabel('tdevs')\nplt.plot([min(devs), max(devs)], [min(devs), max(devs)], color='red')\nplt.show()" - }, - { - "objectID": "units/unit9-sim.html#experimental-design-optional", - "href": "units/unit9-sim.html#experimental-design-optional", - "title": "Simulation", - "section": "Experimental Design (optional)", - "text": "Experimental Design (optional)\nA typical context is that one wants to know the effect of multiple input variables on some outcome. Often, scientists, and even statisticians doing simulation studies will vary one input variable at a time. As we know from standard experimental design, this is inefficient.\nThe standard strategy is to discretize the inputs, each into a small number of levels. If we have a small enough number of inputs and of levels, we can do a full factorial design (potentially with replication). For example if we have three inputs and three levels each, we have \\(3^{3}\\) different treatment combinations. Choosing the levels in a reasonable way is obviously important.\nAs the number of inputs and/or levels increases to the point that we can’t carry out the full factorial, a fractional factorial is an option. This carefully chooses which treatment combinations to omit. The goal is to achieve balance across the levels in a way that allows us to estimate lower level effects (in particular main effects) but not all high-order interactions. What happens is that high-order interactions are aliased to (confounded with) lower-order effects. For example you might choose a fractional factorial design so that you can estimate main effects and two-way interactions but not higher-order interactions.\nIn interpreting the results, I suggest focusing on the decomposition of sums of squares and not on statistical significance. In most cases, we expect the inputs to have at least some effect on the outcome, so the null hypothesis is a straw man. Better to assess the magnitude of the impacts of the different inputs.\nWhen one has a very large number of inputs, one can use the Latin hypercube approach to sample in the input space in a uniform way, spreading the points out so that each input is sampled uniformly. Assume that each input is \\(\\mathcal{U}(0,1)\\) (one can easily transform to whatever marginal distributions you want). Suppose that you can run \\(m\\) samples. Then for each input variable, we divide the unit interval into \\(m\\) bins and randomly choose the order of bins and the position within each bin. This is done independently for each variable and then combined to give \\(m\\) samples from the input space. We would then analyze main effects and perhaps two-way interactions to assess which inputs seem to be most important.\nEven amongst statisticians, taking an experimental design approach to a simulation study is not particularly common, but it’s worth considering." - }, - { - "objectID": "units/unit9-sim.html#computational-efficiency", - "href": "units/unit9-sim.html#computational-efficiency", - "title": "Simulation", - "section": "Computational efficiency", - "text": "Computational efficiency\nParallel processing is often helpful for simulation studies. The reason is that simulation studies are embarrassingly parallel - we can send each replicate to a different computer processor and then collect the results back, and the speedup should scale directly with the number of processors we used. Since we often need to some sort of looping, writing code in C/C++ and compiling and linking to the code from Python may also be a good strategy, albeit one not covered in this course.\nA handy function in Python is itertools.product to get all combinations of a set of vectors.\n\nimport itertools\n\nthetaLevels = [\"low\", \"med\", \"hi\"]\nn = [10, 100, 1000]\ntVsNorm = [\"t\", \"norm\"]\nlevels = list(itertools.product(thetaLevels, tVsNorm, n))" - }, - { - "objectID": "units/unit9-sim.html#analysis-and-reporting", - "href": "units/unit9-sim.html#analysis-and-reporting", - "title": "Simulation", - "section": "Analysis and reporting", - "text": "Analysis and reporting\nOften results are reported simply in tables, but it can be helpful to think through whether a graphical representation is more informative (sometimes it’s not or it’s worse, but in some cases it may be much better). Since you’ll often have a variety of scenarios to display, using trellis plots in ggplot2 via the facet_wrap function will often be a good approach to display how results vary as a function of multiple inputs in R. In Python, it looks like there are various ways (RPlot in pandas, seaborn, plotly), but I don’t know what the most standard way is.\nYou should set the seed when you start the experiment, so that it’s possible to replicate it. It’s also a good idea to save the current value of the seed whenever you save interim results, so that you can restart simulations (this is particularly helpful for MCMC) at the exact point you left off, including the random number sequence.\nTo enhance reproducibility, it’s good practice to post your simulation code (and potentially simulated data) on GitHub, on your website, or as supplementary material with the journal. Another person should be able to fully reproduce your results, including the exact random number generation that you did (e.g., you should provide code for how you set the random seed for your randon number generator).\nMany journals are requiring increasingly detailed documentation of the code and data used in your work, including code and data for simulations. Here are the American Statistical Association’s requirements on documenting computations in its journals:\n“The ASA strongly encourages authors to submit datasets, code, other programs, and/or extended appendices that are directly relevant to their submitted articles. These materials are valuable to users of the ASA’s journals and further the profession’s commitment to reproducible research. Whenever a dataset is used, its source should be fully documented and the data should be made available as on online supplement. Exceptions for reasons of security or confidentiality may be granted by the Editor. Whenever specific code has been used to implement or illustrate the results of a paper, that code should be made available if possible. [.…snip.…] Articles reporting results based on computation should provide enough information so that readers can evaluate the quality of the results. Such information includes estimated accuracy of results, as well as descriptions of pseudorandom-number generators, numerical algorithms, programming languages, and major software components used.”" + "title": "Programming concepts", + "section": "Decorators", + "text": "Decorators\nNow that we’ve seen function generators, it’s straightforward to discuss decorators.\nA decorator is a wrapper around a function that extends the functionality of the function without actually modifying the function.\nWe can create a simple decorator “manually” like this:\n\ndef verbosity_wrapper(myfun):\n def wrapper(*args, **kwargs):\n print(f\"Starting {myfun.__name__}.\")\n output = myfun(*args, **kwargs)\n print(f\"Finishing {myfun.__name__}.\")\n return output\n return wrapper\n \nverbose_rnorm = verbosity_wrapper(np.random.normal)\n\nx = verbose_rnorm(size = 5)\n\nStarting normal.\nFinishing normal.\n\nx\n\narray([ 0.07202449, -0.67672674, 0.98248139, -1.65748762, 0.81795808])\n\n\nPython provides syntax that helps you create decorators with less work (this is an example of the general idea of syntactic sugar).\nWe can easily apply our decorator defined above to a function as follows. Now the function name refers to the wrapped version of the function.\n\n@verbosity_wrapper\ndef myfun(x):\n return x\n\ny = myfun(7)\n\nStarting myfun.\nFinishing myfun.\n\ny\n\n7\n\n\nOur decorator doesn’t do anything useful, but hopefully you can imagine that the idea of being able to have more control over the operation of functions could be useful. For example we could set up a timing wrapper so that when we run a function, we get a report on how long it took to run the function. Or using the idea of a closure, we could keep a running count of the number of times a function has been called.\nOne real-world example of using decorators is in setting up functions to run in parallel in dask, which we’ll discuss in Unit 7." }, { - "objectID": "units/unit9-sim.html#generating-random-uniforms-on-a-computer", - "href": "units/unit9-sim.html#generating-random-uniforms-on-a-computer", - "title": "Simulation", - "section": "Generating random uniforms on a computer", - "text": "Generating random uniforms on a computer\nGenerating a sequence of random standard uniforms is the basis for all generation of random variables, since random uniforms (either a single one or more than one) can be used to generate values from other distributions. Most random numbers on a computer are pseudo-random. The numbers are chosen from a deterministic stream of numbers that behave like random numbers but are actually a finite sequence (recall that both integers and real numbers on a computer are actually discrete and there are finitely many distinct values), so it’s actually possible to get repeats. The seed of a RNG is the place within that sequence where you start to use the pseudo-random numbers.\n\nSequential congruential generators\nMany RNG methods are sequential congruential methods. The basic idea is that the next value is \\[u_{k}=f(u_{k-1},\\ldots,u_{k-j})\\mbox{mod}\\,m\\] for some function, \\(f\\), and some positive integer \\(m\\) . Often \\(j=1\\). mod just means to take the remainder after dividing by \\(m\\). One then generates the random standard uniform value as \\(u_{k}/m\\), which by construction is in \\([0,1]\\). For our discussion below, it is important to distinguish the state (\\(u\\)) from the output of the RNG.\nGiven the construction, such sequences are periodic if the subsequence ever reappears, which is of course guaranteed because there is a finite number of possible subsequence values given that all the \\(u_{k}\\) values are remainders of divisions by a fixed number . One key to a good random number generator (RNG) is to have a very long period.\nAn example of a sequential congruential method is a basic linear congruential generator: \\[u_{k}=(au_{k-1}+c)\\mbox{mod}\\,m\\] with integer \\(a\\), \\(m\\), \\(c\\), and \\(u_{k}\\) values. (Note that in some cases \\(c=0\\), in which case the periodicity can’t exceed \\(m-1\\) as the method is then set up so that we never get \\(u_{k}=0\\) as this causes the algorithm to break.) The seed is the initial state, \\(u_{0}\\) - i.e., the point in the sequence at which we start. By setting the seed you guarantee reproducibility since given a starting value, the sequence is deterministic. In general \\(a\\), \\(c\\) and \\(m\\) are chosen to be ‘large’. The standard values of \\(m\\) are Mersenne primes, which have the form \\(2^{p}-1\\) (but these are not prime for all \\(p\\)). Here’s an example of a linear congruential sampler (with \\(c=0\\)):\n\nn = 100\na = 171\nm = 30269\n\nu = np.empty(n)\nu[0] = 7306\n\nfor i in range(1, n):\n u[i] = (a * u[i-1]) % m\n\nu = u / m\nuFromNP = np.random.uniform(size = n)\n\nplt.figure(figsize=(10, 8))\n\nplt.subplot(2, 2, 1)\nplt.plot(range(1, n+1), u)\nplt.title(\"manual\")\nplt.xlabel(\"Index\"); plt.ylabel(\"Value\")\n\nplt.subplot(2, 2, 2)\nplt.plot(range(1, n+1), uFromNP)\nplt.title(\"numpy\")\nplt.xlabel(\"Index\"); plt.ylabel(\"Value\")\n\nplt.subplot(2, 2, 3)\nplt.hist(u, bins=25)\n\n(array([6., 3., 6., 4., 1., 4., 6., 3., 4., 9., 4., 5., 1., 1., 2., 7., 1.,\n 2., 5., 2., 6., 5., 4., 4., 5.]), array([0.01833559, 0.05743434, 0.09653309, 0.13563183, 0.17473058,\n 0.21382933, 0.25292808, 0.29202683, 0.33112557, 0.37022432,\n 0.40932307, 0.44842182, 0.48752057, 0.52661931, 0.56571806,\n 0.60481681, 0.64391556, 0.68301431, 0.72211305, 0.7612118 ,\n 0.80031055, 0.8394093 , 0.87850804, 0.91760679, 0.95670554,\n 0.99580429]), <BarContainer object of 25 artists>)\n\nplt.xlabel(\"Value\"); plt.ylabel(\"Frequency\")\n\nplt.subplot(2, 2, 4)\nplt.hist(uFromNP, bins=25)\n\n(array([8., 7., 1., 5., 7., 3., 2., 1., 5., 7., 3., 2., 2., 3., 2., 0., 2.,\n 6., 4., 7., 6., 5., 4., 3., 5.]), array([0.00396122, 0.04306257, 0.08216392, 0.12126527, 0.16036663,\n 0.19946798, 0.23856933, 0.27767068, 0.31677204, 0.35587339,\n 0.39497474, 0.43407609, 0.47317745, 0.5122788 , 0.55138015,\n 0.5904815 , 0.62958286, 0.66868421, 0.70778556, 0.74688691,\n 0.78598827, 0.82508962, 0.86419097, 0.90329232, 0.94239368,\n 0.98149503]), <BarContainer object of 25 artists>)\n\nplt.xlabel(\"Value\"); plt.ylabel(\"Frequency\")\n\nplt.tight_layout()\nplt.show()\n\n\n\n\nA wide variety of different RNG have been proposed. Many have turned out to have substantial defects based on tests designed to assess if the behavior of the RNG mimics true randomness. Some of the behavior we want to ensure is uniformity of each individual random deviate, independence of sequences of deviates, and multivariate uniformity of subsequences. One test of a RNG that many RNGs don’t perform well on is to assess the properties of \\(k\\)-tuples - subsequences of length \\(k\\), which should be independently distributed in the \\(k\\)-dimensional unit hypercube. Unfortunately, linear congruential methods produce values that lie on a simple lattice in \\(k\\)-space, i.e., the points are not selected from \\(q^{k}\\) uniformly spaced points, where \\(q\\) is the the number of unique values. Instead, points often lie on parallel lines in the hypercube.\nCombining generators can yield better generators. The Wichmann-Hill is an option in R and is a combination of three linear congruential generators with \\(a=\\{171,172,170\\}\\), \\(m=\\{30269,30307,30323\\}\\), and \\(u_{i}=(x_{i}/30269+y_{i}/30307+z_{i}/30323)\\mbox{mod}\\,1\\) where \\(x\\), \\(y\\), and \\(z\\) are generated from the three individual generators. Let’s mimic the Wichmann-Hill manually:\n\nRNGkind(\"Wichmann-Hill\")\nset.seed(1)\nsaveSeed <- .Random.seed\nuFromR <- runif(10)\na <- c(171, 172, 170)\nm <- c(30269, 30307, 30323)\nxyz <- matrix(NA, nr = 10, nc = 3)\nxyz[1, ] <- (a * saveSeed[2:4]) %% m\nfor( i in 2:10)\n xyz[i, ] <- (a * xyz[i-1, ]) %% m\nfor(i in 1:10)\n print(c(uFromR[i],sum(xyz[i, ]/m)%%1))\n\n[1] 0.1297134 0.1297134\n[1] 0.9822407 0.9822407\n[1] 0.8267184 0.8267184\n[1] 0.242355 0.242355\n[1] 0.8568853 0.8568853\n[1] 0.8408788 0.8408788\n[1] 0.3421633 0.3421633\n[1] 0.7062672 0.7062672\n[1] 0.6212432 0.6212432\n[1] 0.6537663 0.6537663\n\n## we should be able to recover the current value of the seed\nxyz[10, ]\n\n[1] 24279 14851 10966\n\n.Random.seed[2:4]\n\n[1] 24279 14851 10966\n\n\n\n\nModern generators (PCG and Mersenne Twister)\nSomewhat recently O’Neal (2014) proposed a new approach to using the linear congruential generator in a way that gives much better performance than the basic versions of such generators described above. This approach is now the default random number generator in numpy (see numpy.random.default_rng()), called the PCG-64 generator. ‘PCG’ stands for permutation congruential generator and encompasses a family of such generators.\nThe idea of the PCG approach goes like this:\n\nLinear congruential generators (LCG) are simple and fast, but for small values of \\(m\\) don’t perform all that well statistically, in particular having values on a lattice as discussed above.\nUsing a large value of \\(m\\) can actually give good statistical performance.\nApplying a technique called permutation functions to the state of the LCG in order to produce the output at each step (the random value returned to the user) can improve the statistical performance even further.\n\nInstead of using relatively small values of \\(m\\) seen above, in the PCG approach one uses \\(m=2^k\\), for ‘large enough’ \\(k\\), usually 64 or 128. It turns out that if \\(m=2^k\\) then the period of the \\(b\\)th bit of the state is \\(2^b\\) where \\(b=1\\) is the right-most bit. Small periods are of course bad for RNG, so the bits with small period cause the LCG to not perform well. Thankfully, one simple fix is simply to discard some number of the right-most bits (this is one form of bit shift). Note that if one does this, the output of the RNG is based on a subset of the bits, which means that the number of unique values that can be generated is smaller than the period. This is not a problem given we start with a state with a large number of bits (64 or 128 as mentioned above).\nO’Neal then goes further; instead of simply discarding bits, she proposes to either shift bits by a random amount or rotate bits by a random amount, where the random amount is determined by a small number of the initial bits. This improves the statistical performance of the generator. The choice of how to do this gives the various members of the PCG family of generators. The details are fairly complicated (the PCG paper is 50-odd pages) and not important for our purposes here.\nBy default R uses something called the Mersenne twister, which is in the class of generalized feedback shift registers (GFSR). The basic idea of a GFSR is to come up with a deterministic generator of bits (i.e., a way to generate sequences of 0s and 1s), \\(B_{i}\\), \\(i=1,2,3,\\ldots\\). The pseudo-random numbers are then determined as sequential subsequences of length \\(L\\) from \\(\\{B_{i}\\}\\), considered as a base-2 number and dividing by \\(2^{L}\\) to get a number in \\((0,1)\\). In general the sequence of bits is generated by taking \\(B_{i}\\) to be the exclusive or [i.e., 0+0 = 0; 0 + 1 = 1; 1 + 0 = 1; 1 + 1 = 0] summation of two previous bits further back in the sequence where the lengths of the lags are carefully chosen.\nnumpy also provides access to the Mersenne Twister via the MT19937 generator; more on this below. It looks like PCG-64 only became available as of numpy version 1.17.\n\nAdditional notes\nGenerators should give you the same sequence of random numbers, starting at a given seed, whether you ask for a bunch of numbers at once, or sequentially ask for individual numbers.\nWhen one invokes a RNG without a seed, they generally have a method for choosing a seed, often based on the system clock.\nThere have been some attempts to generate truly random numbers based on physical randomness. One that is based on quantum physics is http://www.idquantique.com/true-random-number-generator/quantis-usb-pcie-pci.html. Another approach is based on lava lamps!" + "objectID": "units/unit5-programming.html#overview-2", + "href": "units/unit5-programming.html#overview-2", + "title": "Programming concepts", + "section": "Overview", + "text": "Overview\nThe main things to remember when thinking about memory use are: (1) numeric vectors take 8 bytes per element and (2) we need to keep track of when large objects are created, including local variables in the frames of functions.\n\nx = np.random.normal(size = 5)\nx.itemsize # 8 bytes\n\n8\n\nx.nbytes\n\n40\n\n\n\nAllocating and freeing memory\nUnlike compiled languages like C, in Python we do not need to explicitly allocate storage for objects. (However, we will see that there are times that we do want to allocate storage in advance, rather than successively concatenating onto a larger object.)\nPython automatically manages memory, releasing memory back to the operating system when it’s not needed via a process called garbage collection. Very occasionally you may want to remove large objects as soon as they are not needed. del does not actually free up memory, it just disassociates the name from the memory used to store the object. In general Python will quickly clean up such objects without a reference (i.e., a name), so there is generally no need to call gc.collect() to force the garbage collection.\nIn a language like C in which the user allocates and frees up memory, memory leaks are a major cause of bugs. Basically if you are looping and you allocate memory at each iteration and forget to free it, the memory use builds up inexorably and eventually the machine runs out of memory. In Python, with automatic garbage collection, this is generally not an issue, but occasionally memory leaks could occur.\n\n\nThe heap and the stack\nThe heap is the memory that is available for dynamically creating new objects while a program is executing, e.g., if you create a new object in Python or call new in C++. When more memory is needed the program can request more from the operating system. When objects are removed in Python, Python will handle the garbage collection of releasing that memory.\nThe stack is the memory used for local variables when a function is called.\nThere’s a nice discussion of this on this Stack Overflow thread." }, { - "objectID": "units/unit9-sim.html#rng-in-python", - "href": "units/unit9-sim.html#rng-in-python", - "title": "Simulation", - "section": "RNG in Python", - "text": "RNG in Python\nWe can change the RNG for numpy using np.random.<name_of_generator> (e.g., np.random.MT19937 for the Mersenne Twister). We can set the seed with np.random.seed() or with np.random.default_rng().\nIn numpy, the default_rng RNG is PCG-64. It has a period of \\(2^{128}\\) and supports advancing an arbitrary number of steps, as well as \\(2^{127}\\) streams (both useful for generating random numbers when parallelizing). The state of the PCG-64 RNG is represented by two 128-bit unsigned integers, one the actual state and one the value of \\(c\\) (the increment).\nStrangely, while the default is PCG-64, simply using the functions available via np.random to generate random numbers seems to actually use the Mersenne Twister, so the meaning of default is unclear.\nIn R, the default RNG is the Mersenne twister (?RNGkind), which is considered to be state-of-the-art (by some; O’Neal criticizes it). It has some theoretical support, has performed reasonably on standard tests of pseudorandom numbers and has been used without evidence of serious failure. Plus it’s fast (because bitwise operations are fast). The particular Mersenne twister used has a periodicity of \\(2^{19937}-1\\approx10^{6000}\\). Practically speaking this means that if we generated one random uniform per nanosecond for 10 billion years, then we would generate \\(10^{25}\\) numbers, well short of the period. So we don’t need to worry about the periodicity! The seed for the Mersenne twister is a set of 624 32-bit integers plus a position in the set, where the position is .Random.seed[2].\nFor the Mersenne Twister, we can set the seed by passing an integer to np.random.seed() in Python or set.seed() in R, which then sets as many actual seeds as required for the Mersenne Twister. Here I’ll refer to the single integer passed in as the seed. Ideally, nearby seeds generally should not correspond to getting sequences from the stream that are closer to each other than far away seeds. According to Gentle (CS, p. 327) the input to set.seed() in R should be an integer, \\(i\\in\\{0,\\ldots,1023\\}\\) , and each of these 1024 values produces positions in the RNG sequence that are “far away” from each other. I don’t see any mention of this in the R documentation for set.seed() and furthermore, you can pass integers larger than 1023 to set.seed(), so I’m not sure how much to trust Gentle’s claim. More on generating parallel streams of random numbers below.\nSo we get replicability by setting the seed to a specific value at the beginning of our simulation. We can then set the seed to that same value when we want to replicate the simulation.\n\nnp.random.seed(1)\nnp.random.normal(size = 5)\n\narray([ 1.62434536, -0.61175641, -0.52817175, -1.07296862, 0.86540763])\n\nnp.random.seed(1)\nnp.random.normal(size = 5)\n\narray([ 1.62434536, -0.61175641, -0.52817175, -1.07296862, 0.86540763])\n\n\nWe can also save the state of the RNG and pick up where we left off. So this code will pick where you had left off, ignoring what happened in between saving to saved_state and resetting.\n\nnp.random.seed(1)\nnp.random.normal(size = 5)\n\narray([ 1.62434536, -0.61175641, -0.52817175, -1.07296862, 0.86540763])\n\nsaved_state = np.random.get_state()\nnp.random.normal(size = 5)\n\narray([-2.3015387 , 1.74481176, -0.7612069 , 0.3190391 , -0.24937038])\n\n\nNow we’ll do some arbitrary work with random numbers, and see that if we use the saved state we can pick up where we left off above.\n\ntmp = np.random.choice(np.arange(1, 51), size=2000, replace=True) # arbitrary work\n\n## Restore the state.\nnp.random.set_state(saved_state)\nnp.random.normal(size = 5)\n\narray([-2.3015387 , 1.74481176, -0.7612069 , 0.3190391 , -0.24937038])\n\n\nIf we look at saved_state, we see it actually corresponds to the Mersenne Twister.\nTo do the equivalent with the PCG-64:\n\nrng = np.random.default_rng(1)\nrng.normal(size = 5)\n\narray([ 0.34558419, 0.82161814, 0.33043708, -1.30315723, 0.90535587])\n\nsaved_state = rng.bit_generator.state\nrng.normal(size = 5)\n\narray([ 0.44637457, -0.53695324, 0.5811181 , 0.3645724 , 0.2941325 ])\n\ntmp = rng.choice(np.arange(1, 51), size=2000, replace=True)\nrng.bit_generator.state = saved_state\nrng.normal(size = 5)\n\narray([ 0.44637457, -0.53695324, 0.5811181 , 0.3645724 , 0.2941325 ])\n\nsaved_state\n\n{'bit_generator': 'PCG64', 'state': {'state': 216676376075457487203159048251690499413, 'inc': 194290289479364712180083596243593368443}, 'has_uint32': 0, 'uinteger': 0}\n\nsaved_state['state']['state'] # actual state\n\n216676376075457487203159048251690499413\n\nsaved_state['state']['inc'] # increment ('c')\n\n194290289479364712180083596243593368443\n\n\nsaved_state contains the actual state and the value of c, the increment.\nThe output of the PCG-64 is 64 bits while for the Mersenne Twister the output is 32 bits. This means you could get duplicated values in long runs, but this does not violate the comment about the periodicity of PCG-64 and Mersenne-Twister being longer than \\(2^{64}\\) and \\(2^{32}\\), because the two values after the two duplicated numbers will not be duplicates of each other – as noted previously, there is a distinction between the output presented to the user and the state of the RNG algorithm." + "objectID": "units/unit5-programming.html#monitoring-memory-use", + "href": "units/unit5-programming.html#monitoring-memory-use", + "title": "Programming concepts", + "section": "Monitoring memory use", + "text": "Monitoring memory use\n\nMonitoring overall memory use on a UNIX-style computer\nTo understand how much memory is available on your computer, one needs to have a clear understanding of disk caching. The operating system will generally cache files/data in memory when it reads from disk. Then if that information is still in memory the next time it is needed, it will be much faster to access it the second time around than if it had to read the information from disk. While the cached information is using memory, that same memory is immediately available to other processes, so the memory is available even though it is “in use”.\nWe can see this via free -h (the -h is for ‘human-readable’, i.e. show in GB (G)) on Linux machine.\n total used free shared buff/cache available \n Mem: 251G 998M 221G 2.6G 29G 247G \n Swap: 7.6G 210M 7.4G\nYou’ll generally be interested in the Mem row. (See below for some comments on Swap.) The shared column is complicated and probably won’t be of use to you. The buff/cache column shows how much space is used for disk caching and related purposes but is actually available. Hence the available column is the sum of the free and buff/cache columns (more or less). In this case only about 1 GB is in use (indicated in the used column).\ntop (Linux or Mac) and vmstat (on Linux) both show overall memory use, but remember that the amount actually available to you is the amount free plus any buff/cache usage. Here is some example output from vmstat:\n\n procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu----- \n r b swpd free buff cache si so bi bo in cs us sy id wa st \n 1 0 215140 231655120 677944 30660296 0 0 1 2 0 0 18 0 82 0 0\nIt shows 232 GB free and 31 GB used for cache and therefore available, for a total of 263 GB available.\nHere are some example lines from top:\n KiB Mem : 26413715+total, 23180236+free, 999704 used, 31335072 buff/cache \n KiB Swap: 7999484 total, 7784336 free, 215148 used. 25953483+avail Mem\nWe see that this machine has 264 GB RAM (the total column in the Mem row), with 259.5 GB available (232 GB free plus 31 GB buff/cache as seen in the Mem row). (I realize the numbers don’t quite add up for reasons I don’t fully understand, but we probably don’t need to worry about that degree of exactness.) Only 1 GB is in use.\nSwap is essentially the reverse of disk caching. It is disk space that is used for memory when the machine runs out of physical memory. You never want your machine to be using swap for memory because your jobs will slow to a crawl. As seen above, the swap line in both free and top shows 8 GB swap space, with very little in use, as desired.\n\n\nMonitoring memory use in Python\nThere are a number of ways to see how much memory is being used. When Python is actively executing statements, you can use top from the UNIX shell.\nIn Python, we can call out to the system to get the info we want:\n\nimport psutil\n\n# Get memory information\nmemory_info = psutil.Process().memory_info()\n\n# Print the memory usage\nprint(\"Memory usage:\", memory_info.rss/10**6, \" Mb.\")\n\n# Let's turn that into a function for later use:\n\nMemory usage: 470.765568 Mb.\n\ndef mem_used():\n print(\"Memory usage:\", psutil.Process().memory_info().rss/10**6, \" Mb.\")\n\nWe can see the size of an object (in bytes) with sys.getsizeof().\n\nmy_list = [1, 2, 3, 4, 5]\nsys.getsizeof(my_list)\n\n104\n\nx = np.random.normal(size = 10**7) # should use about 80 Mb\nsys.getsizeof(x)\n\n80000112\n\n\nHowever, we need to be careful about objects that refer to other objects:\n\ny = [3, x]\nsys.getsizeof(y) # Whoops!\n\n72\n\n\nHere’s a trick where we serialize the object, as if to export it, and then see how long the binary representation is.\n\nimport pickle\nser_object = pickle.dumps(y)\nsys.getsizeof(ser_object)\n\n80000201\n\n\nThere are also some flags that one can start python with that allow one to see information about memory use and allocation. See man python. You could also look into the memory_profiler or pympler packages." }, { - "objectID": "units/unit9-sim.html#rng-in-parallel", - "href": "units/unit9-sim.html#rng-in-parallel", - "title": "Simulation", - "section": "RNG in parallel", - "text": "RNG in parallel\nWe can generally rely on the RNG in Python and R to give a reasonable set of values. One time when we want to think harder is when doing work with RNG in parallel on multiple processors. The worst thing that could happen is that one sets things up in such a way that every process is using the same sequence of random numbers. This could happen if you mistakenly set the same seed in each process, e.g., using np.random.seed(1) on every process. More details on parallel RNG are given in Unit 6." + "objectID": "units/unit5-programming.html#how-memory-is-used-in-python", + "href": "units/unit5-programming.html#how-memory-is-used-in-python", + "title": "Programming concepts", + "section": "How memory is used in Python", + "text": "How memory is used in Python\n\nTwo key tools: id and is\nWe can use the id function to see where in memory an object is stored and is to see if two object are actually the same objects in memory. It’s particularly useful for understanding storage and memory use for complicated data structures. We’ll also see that they can be handy tools for seeing where copies are made and where they are not.\n\nx = np.random.normal(size = 10**7)\nid(x)\n\n140228665313968\n\nsys.getsizeof(x)\n\n80000112\n\ny = x\nid(y)\n\n140228665313968\n\nx is y\n\nTrue\n\nsys.getsizeof(y)\n\n80000112\n\nz = x.copy()\nid(z)\n\n140228667485872\n\nsys.getsizeof(z)\n\n80000112\n\n\n\n\nMemory use in specific circumstances\n\nHow lists are stored\nHere we can use id to determine how the overall list is stored as well as the elements of the list.\n\nnums = np.random.normal(size = 5)\nobj = [nums, nums, np.random.normal(size = 5), ['adfs']]\n\nid(nums)\n\n140228665322032\n\nid(obj)\n\n140228665946048\n\nid(obj[0])\n\n140228665322032\n\nid(obj[1])\n\n140228665322032\n\nid(obj[2])\n\n140228665675856\n\nid(obj[3])\n\n140228665966656\n\nid(obj[3][0])\n\n140228665970736\n\nobj[0] is obj[1]\n\nTrue\n\nobj[0] is obj[2]\n\nFalse\n\n\nWhat do we notice?\n\nThe list itself appears to be a array of references (pointers) to the component elements.\nEach element has its own address.\nTwo elements of a list can use the same memory (see the first two elements here, whose contents are at the same memory address).\nA list element can use the same memory as another object (or part of another object).\n\n\n\nHow character strings are stored.\nSimilar tricks are used for storing strings (and also integers). We’ll explore this in a problem on PS4.\n\n\nModifying elements in place\nWhat do this simple experiment tell us?\n\nx = np.random.normal(size = 5)\nid(x)\n\n140228665321552\n\nx[2] = 3.5\nid(x)\n\n140228665321552\n\n\nIt makes some sense that modifying elements of an object here doesn’t cause a copy – if it did, working with large objects would be very difficult.\n\n\n\nWhen are copies made?\nLet’s try to understand when Python uses additional memory for objects, and how it knows when it can delete memory. We’ll use large objects so that we can use free or top to see how memory use by the Python process changes.\n\nx = np.random.normal(size = 10**8)\nid(x)\n\n140228665675952\n\ny = x\nid(y)\n\n140228665675952\n\nx = np.random.normal(size = 10**8)\nid(x)\n\n140228665321552\n\n\nOnly if we re-assign x to reference a different object does additional memory get used.\n\nHow does Python know when it can free up memory?\nPython keeps track of how many names refer to an object and only removes memory when there are no remaining references to an object.\n\nimport sys\n\nx = np.random.normal(size = 10**8)\ny = x\nsys.getrefcount(y)\n\n3\n\ndel x\nsys.getrefcount(y)\n\n2\n\ndel y\n\nWe can see the number of references using sys.getrefcount. Confusingly, the number is one higher than we’d expect, because it includes the temporary reference from passing the object as the argument to getrefcount.\n\nx = np.random.normal(size = 5)\nsys.getrefcount(x) # 2 \n\n2\n\ny = x\nsys.getrefcount(x) # 3\n\n3\n\nsys.getrefcount(y) # 3\n\n3\n\ndel y\nsys.getrefcount(x) # 2\n\n2\n\ny = x\nx = np.random.normal(size = 5)\nsys.getrefcount(y) # 2\n\n2\n\nsys.getrefcount(x) # 2\n\n2\n\n\nThis notion of reference counting occurs in other contexts, such as shared pointers in C++ and in how R handles copying and garbage collection." }, { - "objectID": "units/unit9-sim.html#multivariate-distributions", - "href": "units/unit9-sim.html#multivariate-distributions", - "title": "Simulation", - "section": "Multivariate distributions", - "text": "Multivariate distributions\nThe mvtnorm package supplies code for working with the density and CDF of multivariate normal and t distributions.\nTo generate a multivariate normal, in Unit 10, we’ll see the standard method based on the Cholesky decomposition:\n\nL = np.linalg.cholesky(covMat) # L is lower-triangular\nx = L @ np.random.normal(size = covMat.shape[0])\n\nSide note: for a singular covariance matrix we can use the Cholesky with pivoting, setting as many rows to zero as the rank deficiency. Then when we generate the multivariate normals, they respect the constraints implicit in the rank deficiency. However, you’ll need to reorder the resulting vector because of the reordering involved in the pivoted Cholesky." + "objectID": "units/unit5-programming.html#strategies-for-saving-memory", + "href": "units/unit5-programming.html#strategies-for-saving-memory", + "title": "Programming concepts", + "section": "Strategies for saving memory", + "text": "Strategies for saving memory\nA frew basic strategies for saving memory include:\n\nAvoiding unnecessary copies.\nRemoving objects that are not being used, at which point the Python garbage collector should free up the memory.\n\nIf you’re really trying to optimize memory use, you may also consider:\n\nUsing types that take up less memory (e.g., Bool, Int16, Float32) when possible.\n\nx = np.array(np.random.normal(size = 5), dtype = \"float32\")\nx.itemsize\n\n4\n\nx = np.array([3,4,2,-2], dtype = \"int16\")\nx.itemsize\n\n2\n\n\nReading data in from files in chunks rather than reading the entire dataset (more in Unit 7).\nExploring packages such as arrow for efficiently using memory, as discussed in Unit 2." }, { - "objectID": "units/unit9-sim.html#inverse-cdf", - "href": "units/unit9-sim.html#inverse-cdf", - "title": "Simulation", - "section": "Inverse CDF", - "text": "Inverse CDF\nMost of you know the inverse CDF method. To generate \\(X\\sim F\\) where \\(F\\) is a CDF and is an invertible function, first generate \\(Z\\sim\\mathcal{U}(0,1)\\), then \\(x=F^{-1}(z)\\). For discrete CDFs, one can work with a discretized version. For multivariate distributions, one can work with a univariate marginal and then a sequence of univariate conditionals: \\(f(x_{1})f(x_{2}|x_{1})\\cdots f(x_{k}|x_{k-1},\\ldots,x_{1})\\), when the distribution allows this analytic decomposition." + "objectID": "units/unit5-programming.html#example", + "href": "units/unit5-programming.html#example", + "title": "Programming concepts", + "section": "Example", + "text": "Example\nLet’s work through a real example where we keep a running tally of current memory in use and maximum memory used in a function call. We’ll want to consider hidden uses of memory, when copies are made, and lazy evaluation. This code (translated from the original R code) comes from a PhD student’s research. For our purposes here, let’s assume that xvar and yvar are very long numpy arrays using a lot of memory.\n\ndef fastcount(xvar, yvar):\n naline = np.isnan(xvar)\n naline[np.isnan(yvar)] = True\n localx = xvar.copy()\n localy = yvar.copy()\n localx[naline] = 0\n localy[naline] = 0\n useline = ~naline\n ## We'll ignore the rest of the code.\n ## ...." }, { - "objectID": "units/unit9-sim.html#rejection-sampling", - "href": "units/unit9-sim.html#rejection-sampling", - "title": "Simulation", - "section": "Rejection sampling", - "text": "Rejection sampling\nThe basic idea of rejection sampling (RS) relies on the introduction of an auxiliary variable, \\(u\\). Suppose \\(X\\sim F\\). Then we can write \\(f(x)=\\int_{0}^{f(x)}du\\). Thus \\(f\\) is the marginal density of \\(X\\) in the joint density, \\((X,U)\\sim\\mathcal{U}\\{(x,u):0<u<f(x)\\}\\). Now we’d like to use this in a way that relies only on evaluating \\(f(x)\\) without having to draw from \\(f\\).\nTo implement this we draw from a larger set and then only keep draws for which \\(u<f(x)\\). We choose a density, \\(g\\), that is easy to draw from and that can majorize \\(f\\), which means there exists a constant \\(c\\) s.t. , \\(cg(x)\\geq f(x)\\) \\(\\forall x\\). In other words we have that \\(cg(x)\\) is an upper envelope for \\(f(x)\\). The algorithm is\n\ngenerate \\(x\\sim g\\)\ngenerate \\(u\\sim\\mathcal{U}(0,1)\\)\nif \\(u\\leq f(x)/cg(x)\\) then use \\(x\\); otherwise go back to step 1\n\nThe intuition here is graphical: we generate from under a curve that is always above \\(f(x)\\) and accept only when \\(u\\) puts us under \\(f(x)\\) relative to the majorizing density. A key here is that the majorizing density have fatter tails than the density of interest, so that the constant \\(c\\) can exist. So we could use a \\(t\\) to generate from a normal but not the reverse. We’d like \\(c\\) to be small to reduce the number of rejections because it turns out that \\(\\frac{1}{c}=\\frac{\\int f(x)dx}{\\int cg(x)dx}\\) is the acceptance probability. This approach works in principle for multivariate densities but as the dimension increases, the proportion of rejections grows, because more of the volume under \\(cg(x)\\) is above \\(f(x)\\).\nIf \\(f\\) is costly to evaluate, we can sometimes reduce calculation using a lower bound on \\(f\\). In this case we accept if \\(u\\leq f_{\\mbox{low}}(y)/cg_{Y}(y)\\). If it is not, then we need to evaluate the ratio in the usual rejection sampling algorithm. This is called squeezing.\nOne example of RS is to sample from a truncated normal. Of course we can just sample from the normal and then reject, but this can be inefficient, particularly if the truncation is far in the tail (a case in which inverse CDF suffers from numerical difficulties). Suppose the truncation point is greater than zero. Working with the standardized version of the normal, you can use an translated exponential with lower end point equal to the truncation point as the majorizing density (Robert 1995; Statistics and Computing). For truncation less than zero, just make the values negative." + "objectID": "units/unit5-programming.html#interpreters-and-compilation", + "href": "units/unit5-programming.html#interpreters-and-compilation", + "title": "Programming concepts", + "section": "Interpreters and compilation", + "text": "Interpreters and compilation\n\nWhy are interpreted languages slow?\nCompiled code runs quickly because the original code has been translated into instructions (machine language) that the processor can understand (i.e., zeros and ones). In the process of doing so, various checking and lookup steps are done once and don’t need to be redone when running the compiled code.\nIn contrast, when one runs code in an interpreted language such as Python or R, the interpreter needs to do all the checking and lookup each time the code is run. This is required because the types and locations in memory of the variables could have changed.\nWe’ll focus on Python in the following discussion, but most of the concepts apply to other interpreted languages.\nFor example, consider this code:\n\nx = 3\nabs(x)\nx*7\nx = 'hi'\nabs(x)\nx*3\n\nBecause of dynamic typing, when the interpreter sees abs(x) it needs to check if x is something to which the absolute value function can be applied, including dealing with the fact that x could be a list or array with many numbers in it. In addition it needs to (using scoping rules) look up the value of x. (Consider that x might not even exist at the point that abs(x) is called.) Only then can the absolute value calculation happen. For the multiplication, Python needs to lookup the version of * that can be used, depending on the type of x.\nLet’s consider writing a loop with some ridiculous code:\n\nx = np.random.normal(10)\nfor i in range(10):\n if np.random.normal(size = 1) > 0:\n x = 'hi'\n if np.random.normal(size = 1) > 0.5:\n del x\n x[i]= np.exp(x[i])\n\nThere is no way around the fact that because of how dynamic this is, the interpreter needs to check if x exists, if it is a vector of sufficient length, if it contains numeric values, and it needs to go retrieve the required value, EVERY TIME the np.exp() is executed. Now the code above is unusual, and in most cases, we wouldn’t have the if statements that modify x. So you could imagine a process by which the checking were done on the first iteration and then not needed after that – that gets into the idea of just-in-time compilation, discussed later.\nThe standard Python interpreter (CPython) is a C function so in some sense everything that happens is running as compiled code, but there are lots more things being done to accomplish a given task using interpreted code than if the task had been written directly in code that is compiled. By analogy, consider talking directly to a person in a language you both know compared to talking to a person via an interpreter who has to translate between two languages. Ultimately, the same information gets communicated (hopefully!) but the number of words spoken and time involved is much greater.\nWhen running more complicated functions, there is often a lot of checking that is part of the function itself. For example scipy’s solve_triangular function ultimately calls out to the trtrs Lapack function, but before doing so, there is a lot of checking that can take time. To that point, the documentation suggests you might set check_finite=False to improve performance at the expense of potential problems if the input matrices contain troublesome elements.\nWe can flip the question on its head and ask what operations in an interpreted language will execute quickly. In Python, these include:\n\noperations that call out to compiled C code,\nlinear algebra operations (these call out to compiled C or Fortran code provided by the BLAS and LAPACK software packages), and\nvectorized calls rather than loops:\n\nvectorized calls generally run loops in compiled C code rather than having the loop run in Python, and\nthat means that the interpreter doesn’t have to do all the checking discussed above for every iteration of the loop.\n\n\n\n\nCompilation\n\nOverview\nCompilation is the process of turning code in a given language (such a C++) into machine code. Machine code is the code that the processor actually executes. The machine code is stored in the executable file, which is a binary file. The history of programming has seen ever great levels of abstraction, so that humans can write code using syntax that is easier for us to understand, re-use, and develop building blocks that can be put together to do complicated tasks. For example assembly language is a step above machine code. Languages like C and Fortran provide additional abstraction beyond that. The Statistics 750 class at CMU has a nice overview if you want to see more details.\nNote that interpreters such as Python are themselves programs – the standard Python interpreter (CPython) is a C program that has been compiled. It happens to be a program that processes Python code. The interpreter doesn’t turn Python code into machine code, but the interpreter itself is machine code.\n\n\nJust-in-time (JIT) compilation\nStandard compilation (ahead-of-time or AOT compilation) happens before any code is executed and can involve a lot of optimization to produce the most efficient machine code possible.\nIn contrast, just-in-time (JIT) compilation happens at the time that the code is executing. JIT compilation is heavily used in Julia, which is very fast (in some cases as fast as C). JIT compilation involves translating to machine code as the code is running. One nice aspect is that the results are cached so that if code is rerun, the compilation process doesn’t have to be redone. So if you use a language like Julia, you’ll see that the speed can vary drastically between the first time and later times you run a given function during a given session.\nOne thing that needs to be dealt with is type checking. As discussed above, part of why an interpreter is slow is because the type of the variable(s) involved in execution of a piece of code is not known in advance, so the interpreter needs to check the type. In JIT systems, there are often type inference systems that determine variable types.\nJIT compilation can involve translation from the original code to machine code or translation of bytecode (see next section) to machine code.\n\n\nByte compiling (optional)\nFunctions in Python and Python packages may byte compiled. What does that mean? Byte-compiled code is a special representation that can be executed more efficiently because it is in the form of compact codes that encode the results of parsing and semantic analysis of scoping and other complexities of the Python source code. This byte code can be executed faster than the original Python code because it skips the stage of having to be interpreted by the Python interpreter.\nIf you look at the file names in the directory of an installed Python package you may see files with the .pyc extension. These files have been byte-compiled.\nWe can byte compile our own functions using either the py_compile or compileall modules. Here’s an example (silly since as experienced Python programmers, we would use vectorized calculation here rather than this unvectorized code.)\n\nimport time\n\ndef f(vals):\n x = np.zeros(len(vals))\n for i in range(len(vals)):\n x[i] = np.exp(vals[i])\n return(x)\n\nx = np.random.normal(size = 10**6)\nt0 = time.time()\nout = f(x)\ntime.time() - t0\n\n0.7474195957183838\n\nt0 = time.time()\nout = np.exp(x)\ntime.time() - t0\n\n0.011847734451293945\n\n\n\nimport py_compile\npy_compile.compile('vec.py')\n\n'__pycache__/vec.cpython-311.pyc'\n\n\n\ncp __pycache__/vec.cpython-311.pyc vec.pyc\nrm vec.py # make sure non-compiled module not loaded\n\n\nimport vec\nvec.__file__\n \n\n'/accounts/vis/paciorek/teaching/243fall23/stat243-fall-2023/units/vec.pyc'\n\nt0 = time.time()\nout = vec.f(x)\ntime.time() - t0\n\n0.714287519454956\n\n\nUnfortunately, as seen above byte compiling may not speed things up much. I’m not sure why." }, { - "objectID": "units/unit9-sim.html#adaptive-rejection-sampling-optional", - "href": "units/unit9-sim.html#adaptive-rejection-sampling-optional", - "title": "Simulation", - "section": "Adaptive rejection sampling (optional)", - "text": "Adaptive rejection sampling (optional)\nThe difficulty of RS is finding a good enveloping function. Adaptive rejection sampling refines the envelope as the draws occur, in the case of a continuous, differentiable, log-concave density. The basic idea considers the log of the density and involves using tangents or secants to define an upper envelope and secants to define a lower envelope for a set of points in the support of the distribution. The result is that we have piecewise exponentials (since we are exponentiating from straight lines on the log scale) as the bounds. We can sample from the upper envelope based on sampling from a discrete distribution and then the appropriate exponential. The lower envelope is used for squeezing. We add points to the set that defines the envelopes whenever we accept a point that requires us to evaluate \\(f(x)\\) (the points that are accepted based on squeezing are not added to the set)." + "objectID": "units/unit5-programming.html#benchmarking-and-profiling", + "href": "units/unit5-programming.html#benchmarking-and-profiling", + "title": "Programming concepts", + "section": "Benchmarking and profiling", + "text": "Benchmarking and profiling\nRecall that it’s a waste of time to optimize code before you determine (1) that the code is too slow for how it will be used and (2) which are the slow steps on which to focus your attempts to speed the code up. A 100x speedup in a step that takes 1% of the time will speed up the overall code by very little.\n\nTiming your code\nThere are a few ways to time code:\n\nimport time\nt0 = time.time()\nx = 3\nt1 = time.time()\n\nprint(f\"Execution time: {t1-t0} seconds.\")\n\nExecution time: 0.005488395690917969 seconds.\n\n\nIn general, it’s a good idea to repeat (replicate) your timing, as there is some stochasticity in how fast your computer will run a piece of code at any given moment.\nUsing time is fine for code that takes a little while to run, but for code that is really fast, it may not be very accurate. Measuring fast bits of code is tricky to do well. This next approach is better for benchmarking code (particularly faster bits of code).\n\nimport timeit\n\ntimeit.timeit('x = np.exp(3.)', setup = 'import numpy as np', number = 100)\n\n0.00019408203661441803\n\ncode = '''\nx = np.exp(3.)\n'''\n\ntimeit.timeit(code, setup = 'import numpy as np', number = 100)\n\n0.00018205121159553528\n\n\nThat reports the total time for the 100 replications.\nWe can run it from the command line.\n\npython -m timeit -s 'import numpy' -n 1000 'x = numpy.exp(3.)'\n\n1000 loops, best of 5: 987 nsec per loop\n\n\ntimeit ran the code 1000 times for 5 different repetitions, giving the average time for the 1000 samples for the best of the 5 repetitions.\n\n\nProfiling\nThe Cprofile module will show you how much time is spent in different functions, which can help you pinpoint bottlenecks in your code.\nI haven’t run this code when producing this document as the output of the profiling can be lengthy.\n\ndef lr_slow(y, x):\n xtx = x.T @ x\n xty = x.T @ y\n inv = np.linalg.inv(xtx)\n return inv @ xty\n\n## generate random observations and random matrix of predictors\ny = np.random.normal(size = 5000)\nx = np.random.normal(size = (5000,1000))\n\nt0 = time.time()\nregr = lr_slow(y, x)\nt1 = time.time()\nprint(f\"Execution time: {t1-t0} seconds.\")\n\nimport cProfile\ncProfile.run('lr_slow(y,x)')\n\nThe cumtime column includes the time spent in nested calls to functions while the tottime column excludes it.\nAs we’ll discuss in detail in Unit 10, we almost never want to explicitly invert a matrix. Instead we factorize the matrix and use the factorized result to do the computation of interest. In this case using the Cholesky decomposition is a standard approach, followed by solving triangular systems of equations.\n\nimport scipy as sp\n\ndef lr_fast(y, x):\n xtx = x.T @ x\n xty = x.T @ y\n L = sp.linalg.cholesky(xtx)\n out = sp.linalg.solve_triangular(L.T, \n sp.linalg.solve_triangular(L, xty, lower=True),\n lower=False)\n return(out)\n\nt0 = time.time()\nregr = lr_fast(y, x)\nt1 = time.time()\nprint(f\"Execution time: {t1-t0} seconds.\")\n\ncProfile.run('lr_fast(y,x)')\n\nThe Cholesky now dominates the computational time (but is much faster than inv), so there’s not much more we can do in this case.\nYou might wonder if it’s better to use x.T or np.transpose(x). Try using timeit to decide.\nThe Python profilers (cProfile and profile (not shown)) use deterministic profiling – calculating the interval between events (i.e., function calls and returns). However, there is some limit to accuracy – the underlying ‘clock’ measures in units of about 0.001 seconds.\n(In contrast, R’s profiler works by sampling (statistical profiling) - every little while during a calculation it finds out what function R is in and saves that information to a file. So if you try to profile code that finishes really quickly, there’s not enough opportunity for the sampling to represent the calculation accurately and you may get spurious results.)" }, { - "objectID": "units/unit9-sim.html#importance-sampling", - "href": "units/unit9-sim.html#importance-sampling", - "title": "Simulation", - "section": "Importance sampling", - "text": "Importance sampling\nImportance sampling (IS) allows us to estimate expected values. It’s an extension of the simple Monte Carlo sampling we saw at the beginning of the unit, with some commonalities with rejection sampling.\n\\[\\phi=E_{f}(h(Y))=\\int h(y)\\frac{f(y)}{g(y)}g(y)dy\\] so \\(\\hat{\\phi}=\\frac{1}{m}\\sum_{i}h(y_{i})\\frac{f(y_{i})}{g(y_{i})}\\) for \\(y_{i}\\) drawn from \\(g(y)\\), where \\(w_{i}=f(y_{i})/g(y_{i})\\) act as weights. (Often in Bayesian contexts, we know \\(f(y)\\) only up to a normalizing constant. In this case we need to use \\(w_{i}^{*}=w_{i}/\\sum_{j}w_{j}\\).\nHere we don’t require the majorizing property, just that the densities have common support, but things can be badly behaved if we sample from a density with lighter tails than the density of interest. So in general we want \\(g\\) to have heavier tails. More specifically for a low variance estimator of \\(\\phi\\), we would want that \\(f(y_{i})/g(y_{i})\\) is large only when \\(h(y_{i})\\) is very small, to avoid having overly influential points.\nThis suggests we can reduce variance in an IS context by oversampling \\(y\\) for which \\(h(y)\\) is large and undersampling when it is small, since \\(\\mbox{Var}(\\hat{\\phi})=\\frac{1}{m}\\mbox{Var}(h(Y)\\frac{f(Y)}{g(Y)})\\). An example is that if \\(h\\) is an indicator function that is 1 only for rare events, we should oversample rare events and then the IS estimator corrects for the oversampling.\nWhat if we actually want a sample from \\(f\\) as opposed to estimating the expected value above? We can draw \\(y\\) from the unweighted sample, \\(\\{y_{i}\\}\\), with weights \\(\\{w_{i}\\}\\). This is called sampling importance resampling (SIR)." + "objectID": "units/unit5-programming.html#writing-efficient-python-code", + "href": "units/unit5-programming.html#writing-efficient-python-code", + "title": "Programming concepts", + "section": "Writing efficient Python code", + "text": "Writing efficient Python code\nWe’ll discuss a variety of these strategies, including:\n\nPre-allocating memory rather than growing objects iteratively\nVectorization and use of fast matrix algebra\nConsideration of loops vs. map operations\nSpeed of lookup operations, including hashing\n\n\nPre-allocating memory\nLet’s consider whether we should pre-allocate space for the output of an operation or if it’s ok to keep extending the length of an array or list.\n\nn = 100000\nz = np.random.normal(size = n)\n\n## Pre-allocation\n\ndef fun_prealloc(vals):\n n = len(vals)\n x = [0] * n\n for i in range(n):\n x[i] = np.exp(vals[i])\n return(x)\n\n## Appending to a list\n\ndef fun_append(vals):\n x = []\n for i in range(n):\n x.append(np.exp(vals[i]))\n return(x)\n\n## Appending to a numpy array\n\ndef fun_append_np(vals):\n x = np.array([])\n for i in range(n):\n x = np.append(x, np.exp(vals[i]))\n return(x)\n\n\nt0 = time.time()\nout1 = fun_prealloc(z)\ntime.time() - t0\n\n0.0720217227935791\n\nt0 = time.time()\nout2 = fun_append(z)\ntime.time() - t0\n\n0.0720512866973877\n\nt0 = time.time()\nout3 = fun_append_np(z)\ntime.time() - t0\n\n2.4477429389953613\n\n\nSo what’s going on? First let’s consider what is happening with the use of np.append. Note that it is a function, rather than a method, and we need to reassign to x. What must be happening in terms of memory use and copying when we append an element?\n\nx = np.random.normal(size = 5)\nid(x)\n\n140228665678064\n\nid(np.append(x, 3.34))\n\n140228665678160\n\n\nWe can avoid that large cost of copying and memory allocation by pre-allocating space for the entire output array. (This is equivalent to variable initialization in compiled languages.)\nOk, but how is it that we can append to the list at apparently no cost?\nIt’s not magic, just that Python is clever. Let’s get an idea of what is going on:\n\ndef fun_append2(vals):\n n = len(vals)\n x = []\n print(f\"Initial id: {id(x)}\")\n sz = sys.getsizeof(x)\n print(f\"iteration 0: size {sz}\")\n for i in range(n):\n x.append(np.exp(vals[i]))\n if sys.getsizeof(x) != sz:\n sz = sys.getsizeof(x)\n print(f\"iteration {i}: size {sz}\")\n print(f\"Final id: {id(x)}\")\n return(x)\n\nz = np.random.normal(size = 1000)\nout = fun_append2(z)\n\nInitial id: 140228665954048\niteration 0: size 56\niteration 0: size 88\niteration 4: size 120\niteration 8: size 184\niteration 16: size 248\niteration 24: size 312\niteration 32: size 376\niteration 40: size 472\niteration 52: size 568\niteration 64: size 664\niteration 76: size 792\niteration 92: size 920\niteration 108: size 1080\niteration 128: size 1240\niteration 148: size 1432\niteration 172: size 1656\niteration 200: size 1912\niteration 232: size 2200\niteration 268: size 2520\niteration 308: size 2872\niteration 352: size 3256\niteration 400: size 3704\niteration 456: size 4216\niteration 520: size 4792\niteration 592: size 5432\niteration 672: size 6136\niteration 760: size 6936\niteration 860: size 7832\niteration 972: size 8856\nFinal id: 140228665954048\n\n\nSurprisingly, the id of x doesn’t seem to change, even though we are allocating new memory at many of the iterations. What is happening is that x is an wrapper object that contains within it a reference to an array of references (pointers) to the list elements. The location of the wrapper object doesn’t change, but the underlying array of references/pointers is being reallocated.\nSide note: our assessment of size above does not include the actual size of the list elements.\n\nprint(sys.getsizeof(out))\n\n8856\n\nout[2] = np.random.normal(size = 100000)\nprint(sys.getsizeof(out))\n\n8856\n\n\nOne upshot of this is that if you need to grow an object use a Python list. Then once it is complete, you can always convert it to another type, such as a numpy array.\n\n\nVectorization and use of fast matrix algebra\nOne key way to write efficient Python code is to take advantage of numpy’s vectorized operations.\n\nn = 10**6\nz = np.random.normal(size = n)\nt0 = time.time()\nx = np.exp(z)\nprint(time.time() - t0)\n\n0.03270316123962402\n\nx = np.zeros(n) # Leave out pre-allocation timing to focus on computation.\nt0 = time.time()\nfor i in range(n):\n x[i] = np.exp(z[i])\n\n\nprint(time.time() - t0)\n\n0.808849573135376\n\n\nSo what is different in how Python handles the calculations above that explains the huge disparity in efficiency? The vectorized calculation is being done natively in C in a for loop. The explicit Python for loop involves executing the for loop in Python with repeated calls to C code at each iteration. This involves a lot of overhead because of the repeated processing of the Python code inside the loop. For example, in each iteration of the loop, Python is checking the types of the variables because it’s possible that the types might change, as discussed earlier.\nYou can usually get a sense for how quickly a Python call will pass things along to C or Fortran by looking at the body of the relevant function(s) being called.\nUnfortunately seeing the source code in Python often involves going and finding it in a file on disk, whereas in R, printing a function will show its source code. However you can use ?? in IPython to get the code for non-builtin functions. Consider numpy.linspace??.\nHere I found the source code for the scipy triangular_solve function, which calls out to a Fortran function trtrs, found in the LAPACK library.\n\n## On an SCF machine:\ncat /usr/local/linux/mambaforge-3.11/lib/python3.11/site-packages/scipy/linalg/_basic.py\n\nWith a bit more digging around we could verify that trtrs is a LAPACK funcion by doing some grepping:\n./linalg/_basic.py: trtrs, = get_lapack_funcs(('trtrs',), (a1, b1))\nMany numpy and scipy functions allow you to pass in arrays, and operate on those arrays in vectorized fashion. So before writing a for loop, look at the help information on the relevant function(s) to see if they operate in a vectorized fashion. Functions might take arrays for one or more of their arguments.\nOutside of the numerical packages, we often have to manually do the looping:\n\nx = [3.5, 2.7, 4.6]\ntry:\n math.cos(x)\nexcept Exception as error:\n print(error)\n\nmust be real number, not list\n\n[math.cos(val) for val in x]\n\n[-0.9364566872907963, -0.9040721420170612, -0.11215252693505487]\n\nlist(map(math.cos, x))\n\n[-0.9364566872907963, -0.9040721420170612, -0.11215252693505487]\n\n\nChallenge: Consider the chi-squared statistic involved in a test of independence in a contingency table:\n\\[\n\\chi^{2}=\\sum_{i}\\sum_{j}\\frac{(y_{ij}-e_{ij})^{2}}{e_{ij}},\\,\\,\\,\\, e_{ij}=\\frac{y_{i\\cdot}y_{\\cdot j}}{y_{\\cdot\\cdot}}\n\\]\nwhere \\(y_{i\\cdot}=\\sum_{j}y_{ij}\\) and \\(y_{\\cdot j} = \\sum_{i} y_{ij}\\) and \\(y_{\\cdot\\cdot} = \\sum_{i} \\sum_{j} y_{ij}\\). Write this in a vectorized way without any loops. Note that ‘vectorized’ calculations also work with matrices and arrays.\nSometimes we can exploit vectorized mathematical operations in surprising ways, though sometimes the code is uglier.\n\nx = np.random.normal(size = n)\n\n## List comprehension\ntimeit.timeit('truncx = [max(0,val) for val in x]', number = 10, globals = {'x':x})\n\n0.17521439492702484\n\n\n\n## Vectorized slice replacement\ntimeit.timeit('truncx = x.copy(); truncx[x < 0] = 0', number = 10, globals = {'x':x})\n\n0.005411941558122635\n\n\n\n## Vectorized math trick\ntimeit.timeit('truncx = x * x>0', number = 10, globals = {'x':x})\n\n0.0010863672941923141\n\n\nWe’ll discuss what has to happen (in terms of calculations, memory allocation, and copying) in the two vectorized approaches to try to understand which is more efficient.\nAdditional tips:\n\nIf you do need to loop over dimensions of a matrix or array, if possible loop over the smallest dimension and use the vectorized calculation on the larger dimension(s). For example if you have a 10000 by 10 matrix, try to set up your problem so you can loop over the 10 columns rather than the 10000 rows.\nIn general, in Python looping over rows is likely to be faster than looping over columns because of numpy’s row-major ordering (by default, matrices are stored in memory as a long array in which values in a row are adjacent to each other). However how numpy handles this is more complicated (see more in the Section on cache-aware programming), such that it may not matter for numpy calculations.\nYou can use direct arithmetic operations to add/subtract/multiply/divide a vector by each column of a matrix, e.g. A*b does element-wise multiplication of each column of A by a vector b. If you need to operate by row, you can do it by transposing the matrix.\n\nCaution: relying on Python’s broadcasting rule in the context of vectorized operations, such as is done when direct-multiplying a matrix by a vector to scale the columns relative to each other, can be dangerous as the code may not be easy for someone to read and poses greater dangers of bugs. In some cases you may want to first write the code more directly and then compare the more efficient code to make sure the results are the same. It’s also a good idea to comment your code in such cases.\n\n\nVectorization, mapping, and loops\nNext let’s consider when loops and mapping would be particularly slow and how mapping and loops might compare to each other.\nFirst, the potential for inefficiency of looping and map operations in interpreted languages will depend in part on whether a substantial part of the work is in the overhead involved in the looping or in the time required by the function evaluation on each of the elements.\nHere’s an example, where the core computation is very fast, so we might expect the overhead of looping (in its various forms seen here) to be important.\n\nimport time\nn = 10**6\nx = np.random.normal(size = n)\n\nt0 = time.time()\nout = np.exp(x)\ntime.time() - t0\n\n0.012585163116455078\n\nt0 = time.time()\nvals = np.zeros(n)\nfor i in range(n):\n vals[i] = np.exp(x[i])\n\n\ntime.time() - t0\n\n0.8506276607513428\n\nt0 = time.time()\nvals = [np.exp(v) for v in x]\ntime.time() - t0\n\n0.6357121467590332\n\nt0 = time.time()\nvals = list(map(np.exp, x))\ntime.time() - t0\n\n0.5772781372070312\n\n\nRegardless of how we do the looping (an explicit loop, list comprehension, or map), it looks like we can’t avoid the overhead unless we use the vectorized call, which is of course the recommended approach in this case, both for speed and readability (and conciseness).\nSecond, is it faster to use map than to use a loop? In the example above it is somewhat faster to use map. That might not be surprising. In the loop case, the interpreter needs to do the checking we discussed earlier in this section at each iteration of the loop. What about in the map case? For mapping over a numpy array, perhaps not, but what if mapping over a list? So without digging into how map works, it’s hard to say.\nHere’s an example where the bulk of time is in the actual computation and not in the looping itself. We’ll run a bunch of regressions on a matrix X (i.e., each column of X is a predictor) using each column of the matrix mat to do a separate regression.\n\nimport time\nimport statsmodels.api as sm\n\nn = 500000;\nnr = 10000\nnCalcs = int(n/nr)\n\nmat = np.random.normal(size = (nr, nCalcs))\n\nX = list(range(nr))\nX = sm.add_constant(X)\n\ndef regrFun(i):\n model = sm.OLS(mat[:,i], X)\n return(model.fit().params[1])\n\nt0 = time.time()\nout1 = list(map(regrFun, range(nCalcs)))\ntime.time() - t0\n\n0.05824542045593262\n\nt0 = time.time()\nout2 = np.zeros(nCalcs)\nfor i in range(nCalcs):\n out2[i] = regrFun(i)\n\n\ntime.time() - t0\n\n0.07360720634460449\n\n\nHere the looping is faster. I don’t have any particular explanation for that result.\n\n\nMatrix algebra efficiency\nOften calculations that are not explicitly linear algebra calculations can be done as matrix algebra. If our Python installation has a fast (and possibly parallelized) BLAS, this allows our calculation to take advantage of it.\nFor example, we can sum the rows of a matrix by multiplying by a vector of ones.\n\nmat = np.random.normal(size=(500,500))\n\ntimeit.timeit('mat.dot(np.ones(500))', setup = 'import numpy as np',\n number = 1000, globals = {'mat': mat})\n\n0.033969756215810776\n\ntimeit.timeit('np.sum(mat, axis = 1)', setup = 'import numpy as np',\n number = 1000, globals = {'mat': mat})\n\n0.1168225146830082\n\n\nGiven the extra computation involved in actually multiplying each number by one, it’s surprising that this is faster than numpy sum function. One thing we’d want to know is whether the BLAS matrix multiplication call is being done in parallel.\nOn the other hand, big matrix operations can be slow.\nChallenge: Suppose you want a new matrix that computes the differences between successive columns of a matrix of arbitrary size. How would you do this as matrix algebra operations? It’s possible to write it as multiplying the matrix by another matrix that contains 0s, 1s, and -1s in appropriate places. Here it turns out that the for loop is much faster than matrix multiplication. However, there is a way to do it faster as matrix direct subtraction.\n\n\nOrder of operations and efficiency\nWhen doing matrix algebra, the order in which you do operations can be critical for efficiency. How should I order the following calculation?\n\nn = 5000\nA = np.random.normal(size=(n, n))\nB = np.random.normal(size=(n, n))\nx = np.random.normal(size=n)\n\nt0 = time.time()\nres1 = (A @ B) @ x\nprint(time.time() - t0)\n\n1.7937490940093994\n\nt0 = time.time()\nres1 = A @ (B @ x)\nprint(time.time() - t0)\n\n0.08377599716186523\n\n\nWhy is the second order much faster?\n\n\nAvoiding unnecessary operations\nWe can use the matrix direct product (i.e., A*B) to do some manipulations much more quickly than using matrix multiplication. Challenge: How can I use the direct product to find the trace of a matrix, \\(XY\\)?\nFinally, when working with diagonal matrices, you can generally get much faster results by being smart. The following operations: \\(X+D\\), \\(DX\\), \\(XD\\) are mathematically the sum of two matrices and products of two matrices. But we can do the computation without using two full matrices. Challenge: How?\n\nn = 1000\nX = np.random.normal(size=(n, n))\ndiagvals = np.random.normal(size=n)\nD = np.diag(diagvals)\n\n# The following lines are very inefficient\nsummedMat = X + D\nprodMat1 = D @ X\nprodMat2 = X @ D\n\nMore generally, sparse matrices and structured matrices (such as block diagonal matrices) can generally be worked with MUCH more efficiently than treating them as arbitrary matrices. The scipy.sparse package (for both structured and arbitrary sparse matrices) can help, as can specialized code available in other languages, such as C and Fortran packages.\n\n\nSpeed of lookup operations\nThere are lots of situations in which we need to retrieve values for subsequent computations. In some cases we might be retrieving elements of an array or looking up values in a dictionary.\nLet’s compare the speed of some different approaches to lookup.\n\nn = 1000\nx = list(np.random.normal(size = n))\nkeys = [str(v) for v in range(n)]\nxD = dict(zip(keys, x))\n\ntimeit.timeit(\"x[500]\", number = 10**6, globals = {'x':x})\n\n0.014431843534111977\n\ntimeit.timeit(\"xD['500']\", number=10**6, globals = {'xD':xD})\n\n0.026302199810743332\n\n\nHow is it that Python can look up by key in the dictionary at essentially the same speed as jumping to an index position? It uses hashing, which allows O(1) lookup. In contrast, if one has to look through each key in turn, that is O(n), which is much slower:\n\ntimeit.timeit(\"x[keys.index('500')]\", number = 10**6, globals = {'x':x, 'keys':keys})\n\n5.733805952593684\n\n\nAs a further point of contrast, if we look up elements by name in R in named vectors or lists, that is much slower than looking up by index, because R doesn’t use hashing in that context and has to scan through the objects one by one until it finds the one with the name it is looking for. This stands in contrast to R and Python being able to directly go to the position of interest based on the index of an array, or to the hash-based lookup in a Python dictionary or an R environment.\n\n\nHashing (including name lookup)\nAbove I mentioned that Python uses hashing to store and lookup values by key in a dictionary. I’ll briefly describe what hashing is here, because it is a commonly-used strategy in programming in general.\nA hash function is a function that takes as input some data and maps it to a fixed-length output that can be used as a shortened reference to the data. (The function should be deterministic, always returing the same output for a given input.) We’ve seen this in the context of git commits where each commit was labeled with a long base-16 number. This also comes up when verifying files on the Internet. You can compute the hash value on the file you get and check that it is the same as the hash value associated with the legitimate copy of the file.\nWhile there are various uses of hashing, for our purposes here, hashing can allow one to look up values by their name via a hash table. The idea is that you have a set of key-value pairs (sometimes called a dictionary) where the key is the name associated with the value and the value is some arbitrary object. You want to be able to quickly find the value/object quickly.\nHashing allows one to quickly determine an index associated with the key and therefore quickly find the relevant value based on the index. For example, one approach is to compute the hash as a function of the key and then take the remainder when dividing by the number of possible results (here the fact that the result is a fixed-length output is important) to get the index. Here’s the procedure in pseudocode:\n hash = hashfunc(key) \n index = hash %% array_size \n ## %% is modulo operator - it gives the remainder\nIn general, there will be collisions – multiple keys will be assigned to the same index. However with a good hash function, usually there will be a small number of keys associated with a given bucket. So each bucket will contain a list of a small number of values and the associated keys. (The buckets might contain the actual values or they might contain the addresses of where the values are actually stored if the values are complicated objects.) Then determining the correct value (or the required address) within a given bucket is fast even with simple linear search through the items one by one. Put another way, the hash function distributes the keys amongst an array of buckets and allows one to look up the appropriate bucket quickly based on the computed index value. When the hash table is properly set up, the cost of looking up a value does not depend on the number of key-value pairs stored.\nPython uses hashing to look up the value based on the key in a given dictionary, and similarly when looking up variables in namespaces. This allows Python to retrieve objects very quickly." }, { - "objectID": "units/unit9-sim.html#ratio-of-uniforms-optional", - "href": "units/unit9-sim.html#ratio-of-uniforms-optional", - "title": "Simulation", - "section": "Ratio of uniforms (optional)", - "text": "Ratio of uniforms (optional)\nIf \\(U\\) and \\(V\\) are uniform in \\(C=\\{(u,v):\\,0\\leq u\\leq\\sqrt{f(v/u)}\\) then \\(X=V/U\\) has density proportion to \\(f\\). The basic algorithm is to choose a rectangle that encloses \\(C\\) and sample until we find \\(u\\leq f(v/u)\\). Then we use \\(x=v/u\\) as our RV. The larger region enclosing \\(C\\) is the majorizing region and a simple approach (if \\(f(x)\\)and \\(x^{2}f(x)\\) are bounded in \\(C\\)) is to choose the rectangle, \\(0\\leq u\\leq\\sup_{x}\\sqrt{f(x)}\\), \\(\\inf_{x}x\\sqrt{f(x)}\\leq v\\leq\\sup_{x}x\\sqrt{f(x)}\\).\nOne can also consider truncating the rectangular region, depending on the features of \\(f\\).\nMonahan recommends the ratio of uniforms, particularly a version for discrete distributions (p. 323 of the 2nd edition)." + "objectID": "units/unit5-programming.html#additional-general-strategies-for-efficiency", + "href": "units/unit5-programming.html#additional-general-strategies-for-efficiency", + "title": "Programming concepts", + "section": "Additional general strategies for efficiency", + "text": "Additional general strategies for efficiency\nIt’s also useful to be aware of some other strategies for improving efficiency.\n\nCache-aware programming\nIn addition to main memory (what we usually mean when we talk about RAM), computers also have memory caches, which are small amounts of fast memory that can be accessed very quickly by the processor. For example your computer might have L1, L2, and L3 caches, with L1 the smallest and fastest and L3 the largest and slowest. The idea is to try to have the data that is most used by the processor in the cache.\nIf the next piece of data needed for computation is available in the cache, this is a cache hit and the data can be accessed very quickly. However, if the data is not available in the cache, this is a cache miss and the speed of access will be a lot slower. Cache-aware programming involves writing your code to minimize cache misses. Generally when data is read from memory it will be read in chunks, so values that are contiguous will be read together.\nHow does this inform one’s programming? For example, if you have a matrix of values stored in row-major order, computing on a row will be a lot faster than computing on a column, because the row can be read into the cache from main memory and then accessed in the cache. In contrast, if the matrix is large and therefore won’t fit in the cache, when you access the values of a column, you’ll have to go to main memory repeatedly to get the values for the row because the values are not stored contiguously.\nThere’s a nice example of the importance of the cache at the bottom of this blog post.\nIf you know the size of the cache, you can try to design your code so that in a given part of your code you access data structures that will fit in the cache. This sort of thing is generally more relevant if you’re coding in a language like C. But it can matter sometimes in interpreted languages too.\nLet’s see what happens in Python. By default, matrices in numpy are row-major, also called “C order”. I’ll create a long matrix with a small number of very long columns and a wide matrix with a small number of very long rows.\n\nnr = 800000\nnc = 100\n\nA = np.random.normal(size=(nr, nc)) # long matrix\ntA = np.random.normal(size=(nc, nr)) # wide matrix\n\n## Verify that A is row-major using `.flags` (notice the `C_CONTIGUOUS: True`).\nA.flags\n\n C_CONTIGUOUS : True\n F_CONTIGUOUS : False\n OWNDATA : True\n WRITEABLE : True\n ALIGNED : True\n WRITEBACKIFCOPY : False\n\n\nNote that I didn’t use A.T or np.transpose as that doesn’t make a copy in memory and so the transposed matrix doesn’t end up being row-major. You can use A.flags and A.T.flags` to see this.\nNow let’s time calculating the sum by column in the long matrix vs. the sum by row in the wide matrix. Exactly the same number of arithmetic operations needs to be done in an equivalent manner for the two cases. We want to use a large enough matrix so the entire matrix doesn’t fit in the cache, but not so large that the example takes a long time or a huge amount of memory. We’ll use a rectangular matrix, such that the summation for a single column of the long matrix or a single row of the wide matrix involves many numbers, but there are a limited number of such summations. This focuses the example on the efficiency of the column-wise vs. row-wise summation rather than any issues that might be involved in managing large numbers of such summations (e.g., doing many, many summations that involve just a few numbers).\n\n# Define the sum calculations as functions\ndef sum_by_column():\n return np.sum(A, axis=0)\n\ndef sum_by_row():\n return np.sum(tA, axis=1)\n\ntimeit.timeit(sum_by_column, number=10) # potentially slow\n\n0.5056534707546234\n\ntimeit.timeit(sum_by_row, number=10)\n\n0.408780699595809\n\n\nSuppose we instead do the looping manually.\n\ntimeit.timeit('[np.sum(A[:,col]) for col in range(A.shape[1])]',\n setup = 'import numpy as np', number=10, globals = {'A': A}) \n\n5.12814655713737\n\ntimeit.timeit('[np.sum(tA[row,:]) for row in range(tA.shape[0])]',\n setup = 'import numpy as np', number=10, globals = {'tA': tA}) \n\n0.41643139719963074\n\n\nIndeed, the row-wise calculations are much faster when done manually. However, when done with the axis argument in np.sum there is little difference. So that suggests numpy might be doing something clever in its implementation of sum with the axis argument.\n\nChallenge: suppose you were writing code for this kind of use case. How could you set up your calculations to do either row-wise or column-wise operations in a way that processes each number sequentially based on the order in which the numbers are stored. For example suppose the values are stored row-major but you want the column sums.\n\nWhen we define a numpy array, we can choose to use column-major order (i.e., “Fortran” order) with the order argument.\n\n\nLoop fusion\nLet’s consider this (vectorized) code:\n\nx = np.exp(x) + 3*np.sin(x)\n\nThis code has some downsides.\n\nThink about whether any additional memory has to be allocated.\nThink about how many for loops will have to get executed.\n\nContrast that to running directly as a for loop (e.g., here in Julia or in C/C++):\n\nfor i in 1:length(x)\n x[i] = exp(x[i]) + 3*sin(x[i])\nend\n\nHow does that affect the downsides mentioned above?\nCombining loops is called ‘fusing’ and is an important optimization that Julia can do, as shown in this demo. It’s also a key optimization done by XLA, a compiler used with Tensorflow, so one approach to getting loop fusion in Python is to use Tensorflow for such calculations within Python rather than simply using numpy.\n\n\nLazy evaluation\nWhat’s strange about this R code?\n\nf <- function(x) print(\"hi\")\nsystem.time(mean(rnorm(1000000)))\n\n user system elapsed \n 0.058 0.000 0.059 \n\nsystem.time(f(3))\n\n[1] \"hi\"\n\n\n user system elapsed \n 0 0 0 \n\nsystem.time(f(mean(rnorm(1000000)))) \n\n[1] \"hi\"\n\n\n user system elapsed \n 0.001 0.000 0.001 \n\n\nLazy evaluation is not just an R thing. It also occurs in Tensorflow (particularly version 1), the Python Dask package, and in Spark. The basic idea is to delay executation until it’s really needed, with the goal that if one does so, the system may be able to better optimize a series of multiple steps as a joint operation relative to executing them one by one.\nHowever, Python itself does not have lazy evaluation." }, { - "objectID": "units/unit1-intro.html", - "href": "units/unit1-intro.html", - "title": "Introduction to UNIX, computers, and key tools", + "objectID": "units/unit2-dataTech.html", + "href": "units/unit2-dataTech.html", + "title": "Data technologies, formats, and structures", "section": "", - "text": "PDF" - }, - { - "objectID": "units/unit1-intro.html#some-useful-editors", - "href": "units/unit1-intro.html#some-useful-editors", - "title": "Introduction to UNIX, computers, and key tools", - "section": "Some useful editors", - "text": "Some useful editors\n\nvarious editors available on all operating systems:\n\ntraditional editors born in UNIX: emacs, vim\nsome newer editors: Atom, Sublime Text (Sublime is proprietary/not free)\n\nWindows-specific: WinEdt\nMac-specific: Aquamacs Emacs, TextMate, TextEdit\nRStudio provides a built-in editor for R code and Quarto/R Markdown files. One can actually edit and run Python code chunks quite nicely in RStudio. (Note: RStudio as a whole is an IDE (integrated development environment. The editor is just the editing window where you edit code (and Markdown) files.)\nVSCode has a powerful code editor that is customized to work with various languages, and it has a Quarto extension.\n\nAs you get started it’s ok to use a very simple text editor such as Notepad in Windows, but you should take the time in the next few weeks to try out more powerful editors such as one of those listed above. It will be well worth your time over the course of your graduate work and then your career.\nBe careful in Windows - file suffixes are often hidden." + "text": "PDF\nReferences (see syllabus for links):\n(Optional) Videos:\nThere are four videos from 2020 in the bCourses Media Gallery that you can use for reference if you want to:\nNote that the videos were prepared for a version of the course that used R, so there are some differences from the content in the current version of the unit that reflect translating between R and Python. I’m not sure how helpful they’ll be, but they are available." }, { - "objectID": "units/unit1-intro.html#optional-basic-emacs", - "href": "units/unit1-intro.html#optional-basic-emacs", - "title": "Introduction to UNIX, computers, and key tools", - "section": "(Optional) Basic emacs", - "text": "(Optional) Basic emacs\nEmacs is one option as an editor. I use Emacs a fair amount, so I’m including some tips here, but other editors listed above are just as good.\n\nEmacs has special modes for different types of files: Python code files, R code files, C code files, Latex files – it’s worth your time to figure out how to set this up on your machine for the kinds of files you often work on\n\nIf working with Python and R, one can start up a Python or R interpreter in an additional Emacs buffer and send code to that interpreter and see the results of running the code.\nFor working with R, ESS (emacs speaks statistics) mode is helpful. This is built into Aquamacs Emacs.\n\nTo open emacs in the terminal window rather than as a new window, which is handy when it’s too slow (or impossible) to pass (i.e., tunnel) the graphical emacs window through ssh: emacs -nw file.txt" + "objectID": "units/unit2-dataTech.html#text-and-binary-files", + "href": "units/unit2-dataTech.html#text-and-binary-files", + "title": "Data technologies, formats, and structures", + "section": "Text and binary files", + "text": "Text and binary files\nIn general, files can be divided into text files and binary files. In both cases, information is stored as a series of bits. Recall that a bit is a single value in base 2 (i.e., a 0 or a 1), while a byte is 8 bits.\nA text file is one in which the bits in the file encode individual characters. Note that the characters can include the digit characters 0-9, so one can include numbers in a text file by writing down the digits needed for the number of interest. Examples of text file formats include CSV, XML, HTML, and JSON.\nText files may be simple ASCII files (i.e., files encoded using ASCII) or files in other encodings such as UTF-8, both covered in Section 5. ASCII files have 8 bits (1 byte) per character and can represent 128 characters (the 52 lower and upper case letters in English, 10 digits, punctuation and a few other things – basically what you see on a standard US keyboard). UTF-8 files have between 1 and 4 bytes per character.\nSome text file formats, such as JSON or HTML, are not easily interpretable/manipulable on a line-by-line basis (unlike, e.g., CSV), so they are not as amenable to processing using shell commands.\nA binary file is one in which the bits in the file encode the information in a custom format and not simply individual characters. Binary formats are not (easily) human readable but can be more space-efficient and faster to work with (because it can allow random access into the data rather than requiring sequential reading). The meaning of the bytes in such files depends on the specific binary format being used and a program that uses the file needs to know how the format represents information. Examples of binary files include netCDF files, Python pickle files, R data (e.g., .Rda) files, , and compiled code files.\nNumbers in binary files are usually stored as 8 bytes per number. We’ll discuss this much more in Unit 8." }, { - "objectID": "units/unit1-intro.html#optional-emacs-keystroke-sequence-shortcuts.", - "href": "units/unit1-intro.html#optional-emacs-keystroke-sequence-shortcuts.", - "title": "Introduction to UNIX, computers, and key tools", - "section": "(Optional) Emacs keystroke sequence shortcuts.", - "text": "(Optional) Emacs keystroke sequence shortcuts.\n\nNote Several of these (Ctrl-a, Ctrl-e, Ctrl-k, Ctrl-y) work in the command line, interactive Python and R sessions, and other places as well.\n\n\n\n\n\n\n\n\nSequence\nResult\n\n\n\n\nCtrl-x,Ctrl-c\nClose the file\n\n\nCtrl-x,Ctrl-s\nSave the file\n\n\nCtrl-x,Ctrl-w\nSave with a new name\n\n\nCtrl-s\nSearch\n\n\nESC\nGet out of command buffer at bottom of screen\n\n\nCtrl-a\nGo to beginning of line\n\n\nCtrl-e\nGo to end of line\n\n\nCtrl-k\nDelete the rest of the line from cursor forward\n\n\nCtrl-space, then move to end of block\nHighlight a block of text\n\n\nCtrl-w\nRemove the highlighted block, putting it in the kill buffer\n\n\nCtrl-y (after using Ctrl-k or Ctrl-w)\nPaste from kill buffer (‘y’ is for ‘yank’)" + "objectID": "units/unit2-dataTech.html#common-file-types", + "href": "units/unit2-dataTech.html#common-file-types", + "title": "Data technologies, formats, and structures", + "section": "Common file types", + "text": "Common file types\nHere are some of the common file types, some of which are text formats and some of which are binary formats.\n\n‘Flat’ text files: data are often provided as simple text files. Often one has one record or observation per row and each column or field is a different variable or type of information about the record. Such files can either have a fixed number of characters in each field (fixed width format) or a special character (a delimiter) that separates the fields in each row. Common delimiters are tabs, commas, one or more spaces, and the pipe (|). Common file extensions are .txt and .csv. Metadata (information about the data) are often stored in a separate file. CSV files are quite common, but if you have files where the data contain commas, other delimiters might be preferable. Text can be put in quotes in CSV files, and this can allow use of commas within the data. This is difficult to deal with from the command line, but read_table() in Pandas handles this situation.\n\nOne occasionally tricky difficulty is as follows. If you have a text file created in Windows, the line endings are coded differently than in UNIX. Windows uses a newline (the ASCII character \\n) and a carriage return (the ASCII character \\r) whereas UNIX uses onlyl a newline in UNIX). There are UNIX utilities (fromdos in Ubuntu, including the SCF Linux machines and dos2unix in other Linux distributions) that can do the necessary conversion. If you see \\^M at the end of the lines in a file, that’s the tool you need. Alternatively, if you open a UNIX file in Windows, it may treat all the lines as a single line. You can fix this with todos or unix2dos.\n\nIn some contexts, such as textual data and bioinformatics data, the data may be in a text file with one piece of information per row, but without meaningful columns/fields.\nData may also be in text files in formats designed for data interchange between various languages, in particular XML or JSON. These formats are “self-describing”; namely the metadata is part of the file. The lxml and json packages are useful for reading and writing from these formats. More in Section 4.\nYou may be scraping information on the web, so dealing with text files in various formats, including HTML. The requests and BeautifulSoup packages are useful for reading HTML.\nIn scientific contexts, netCDF (.nc) (and the related HDF5) are popular format for gridded data that allows for highly-efficient storage and contains the metadata within the file. The basic structure of a netCDF file is that each variable is an array with multiple dimensions (e.g., latitude, longitude, and time), and one can also extract the values of and metadata about each dimension. The netCDF4 package in Python nicely handles working with netCDF files.\nData may already be in a database or in the data storage format of another statistical package (Stata, SAS, SPSS, etc.). The Pandas package in Python has capabilities for importing Stata (read_stata), SPSS (read_spss), and SAS (read_sas) files, among others.\nFor Excel, there are capabilities to read an Excel file (see the read_excel function in Pandas), but you can also just go into Excel and export as a CSV file or the like and then read that into Python. In general, it’s best not to pass around data files as Excel or other spreadsheet format files because (1) Excel is proprietary, so someone may not have Excel and the format is subject to change, (2) Excel imposes limits on the number of rows, (3) one can easily manipulate text files such as CSV using UNIX tools, but this is not possible with an Excel file, (4) Excel files often have more than one sheet, graphs, macros, etc., so they’re not a data storage format per se.\nPython can easily interact with databases (SQLite, PostgreSQL, MySQL, Oracle, etc.), querying the database using SQL and returning results to Python. More in the big data unit and in the large datasets tutorial mentioned above." }, { - "objectID": "units/unit7-bigData.html", - "href": "units/unit7-bigData.html", - "title": "Big data and databases", - "section": "", - "text": "PDF\nReferences:\nI’ve also pulled material from a variety of other sources, some mentioned in context below.\nNote that for a lot of the demo code I ran the code separately from rendering this document because of the time involved in working with large datasets.\nWe’ll focus on Dask and databases/SQL in this Unit. The material on using Spark is provided for reference, but you’re not responsible for that material. If you’re interested in working with big datasets in R or with tools other than Dask in Python, there is some material in the tutorial on working with large datasets." + "objectID": "units/unit2-dataTech.html#csv-vs.-specialized-formats-such-as-parquet", + "href": "units/unit2-dataTech.html#csv-vs.-specialized-formats-such-as-parquet", + "title": "Data technologies, formats, and structures", + "section": "CSV vs. specialized formats such as Parquet", + "text": "CSV vs. specialized formats such as Parquet\nCSV is a common format (particularly in some disciplines/contexts) and has the advantages of being simple to understand, human readable, and readily manipulable by line-based processing tools such as shell commands. However, it has various disadvantages:\n\nstorage is by row, which will often mix values of different types;\nextra space is taken up by explicitly storing commas and newlines; and\none must search through the document to find a given row or value – e.g., to find the 10th row, we must search for the 9th newline and then read until the 10th newline.\n\nA popular file format that has some advantages over plain text formats such as CSV is Parquet. The storage is by column (actually in chunks of columns). This works well with how datasets are often structured in that a given field/variable will generally have values of all the same type and there may be many repeated values, so there are opportunities for efficient storage including compression. Storage by column also allows retrieval only of the columns that a user needs. As a result data stored in the Parquet format often takes up much less space than stored as CSV and can be queried much faster. Also note that data stored in Parquet will often be stored as multiple files.\nHere’s a brief exploration using a data file not in the class repository.\n\nimport time\n\n## Read from CSV\nt0 = time.time()\ndata_from_csv = pd.read_csv(os.path.join('..', 'data', 'airline.csv'))\nprint(time.time() - t0)\n\n0.8456258773803711\n\n\n\n## Write out Parquet-formatted data\ndata_from_csv.to_parquet(os.path.join('..', 'data', 'airline.parquet'))\n\n## Read from Parquet\nt0 = time.time()\ndata_from_parquet = pd.read_parquet(os.path.join(\n '..', 'data', 'airline.parquet'))\nprint(time.time() - t0)\n\n0.10250258445739746\n\n\nThe CSV file is 51 MB while the Parquet file is 8 MB.\n\nimport subprocess\nsubprocess.run([\"ls\", \"-l\", os.path.join(\"..\", \"data\", \"airline.csv\")])\nsubprocess.run([\"ls\", \"-l\", os.path.join(\"..\", \"data\", \"airline.parquet\")])\n\n-rw-r--r-- 1 paciorek scfstaff 51480244 Aug 29 2022 ../data/airline.csv\n\n\nCompletedProcess(args=['ls', '-l', '../data/airline.csv'], returncode=0)\n\n\n-rw-r--r-- 1 paciorek scfstaff 8153160 Aug 31 08:01 ../data/airline.parquet\n\n\nCompletedProcess(args=['ls', '-l', '../data/airline.parquet'], returncode=0)" }, { - "objectID": "units/unit7-bigData.html#an-editorial-on-big-data", - "href": "units/unit7-bigData.html#an-editorial-on-big-data", - "title": "Big data and databases", - "section": "An editorial on ‘big data’", - "text": "An editorial on ‘big data’\n‘Big data’ was trendy these days, though I guess it’s not quite the buzzword/buzzphrase that it was a few years ago, given the AI/ML revolution, but of course that revolution is largely based on having massive datasets available online.\nPersonally, I think some of the hype around giant datasets is justified and some is hype. Large datasets allow us to address questions that we can’t with smaller datasets, and they allow us to consider more sophisticated (e.g., nonlinear) relationships than we might with a small dataset. But they do not directly help with the problem of correlation not being causation. Having medical data on every American still doesn’t tell me if higher salt intake causes hypertension. Internet transaction data does not tell me if one website feature causes increased viewership or sales. One either needs to carry out a designed experiment or think carefully about how to infer causation from observational data. Nor does big data help with the problem that an ad hoc ‘sample’ is not a statistical sample and does not provide the ability to directly infer properties of a population. Consider the immense difficulties we’ve seen in answering questions about Covid despite large amounts of data, because it is incomplete/non-representative. A well-chosen smaller dataset may be much more informative than a much larger, more ad hoc dataset. However, having big datasets might allow you to select from the dataset in a way that helps get at causation or in a way that allows you to construct a population-representative sample. Finally, having a big dataset also allows you to do a large number of statistical analyses and tests, so multiple testing is a big issue. With enough analyses, something will look interesting just by chance in the noise of the data, even if there is no underlying reality to it.\nDifferent people define the ‘big’ in big data differently. One definition involves the actual size of the data, and in some cases the speed with which it is collected. Our efforts here will focus on dataset sizes that are large for traditional statistical work but would probably not be thought of as large in some contexts such as Google or the US National Security Agency (NSA). Another definition of ‘big data’ has more to do with how pervasive data and empirical analyses backed by data are in society and not necessarily how large the actual dataset size is." + "objectID": "units/unit2-dataTech.html#core-python-functions", + "href": "units/unit2-dataTech.html#core-python-functions", + "title": "Data technologies, formats, and structures", + "section": "Core Python functions", + "text": "Core Python functions\nThe read_table and read_csv functions in the Pandas package are commonly used for reading in data. They read in delimited files (CSV specifically in the latter case). The key arguments are the delimiter (the sep argument) and whether the file contains a header, a line with the variable names. We can use read_fwf() to read from a fixed width text file into a data frame.\nThe most difficult part of reading in such files can be dealing with how Pandas determines the types of the fields that are read in. While Pandas will try to determine the types automatically, it can be safer (and faster) to tell Pandas what the types are, using the dtype argument to read_table().\nLet’s work through a couple examples. Before we do that, let’s look at the arguments to read_table. Note that sep='' can use regular expressions (which would be helpful if you want to separate on any amount of white space, as one example).\n\ndat = pd.read_table(os.path.join('..', 'data', 'RTADataSub.csv'),\n sep = ',', header = None)\ndat.dtypes.head() # 'object' is string or mixed type\ndat.loc[0,1] \ntype(dat.loc[0,1]) # string!\n## Whoops, there is an 'x', presumably indicating missingness:\ndat.loc[:,1].unique()\n\n/usr/local/linux/mambaforge-3.11/lib/python3.11/site-packages/IPython/core/formatters.py:342: FutureWarning:\n\nIn future versions `DataFrame.to_latex` is expected to utilise the base implementation of `Styler.to_latex` for formatting and rendering. The arguments signature may therefore change. It is recommended instead to use `DataFrame.style.to_latex` which also contains additional functionality.\n\n\n\n\n\n\n'2336'\n\n\nstr\n\n\narray(['2336', '2124', '1830', '1833', '1600', '1578', '1187', '1005',\n '918', '865', '871', '860', '883', '897', '898', '893', '913',\n '870', '962', '880', '875', '884', '894', '836', '848', '885',\n '851', '900', '861', '866', '867', '829', '853', '920', '877',\n '908', '855', '845', '859', '856', '825', '828', '854', '847',\n '840', '873', '822', '818', '838', '815', '813', '816', '849',\n '802', '805', '792', '823', '808', '798', '800', '842', '809',\n '807', '826', '810', '801', '794', '771', '796', '790', '787',\n '775', '751', '783', '811', '768', '779', '795', '770', '821',\n '830', '767', '772', '791', '781', '773', '777', '814', '778',\n '782', '837', '759', '846', '797', '835', '832', '793', '803',\n '834', '785', '831', '820', '812', '824', '728', '760', '762',\n '753', '758', '764', '741', '709', '735', '749', '752', '761',\n '750', '776', '766', '789', '763', '864', '858', '869', '886',\n '844', '863', '916', '890', '872', '907', '926', '935', '933',\n '906', '905', '912', '972', '996', '1009', '961', '952', '981',\n '917', '1011', '1071', '1920', '3245', '3805', '3926', '3284',\n '2700', '2347', '2078', '2935', '3040', '1860', '1437', '1512',\n '1720', '1493', '1026', '928', '874', '833', '850', nan, 'x'],\n dtype=object)\n\n\n\n## Let's treat 'x' as a missing value indicator.\ndat2 = pd.read_table(os.path.join('..', 'data', 'RTADataSub.csv'),\n sep = ',', header = None, na_values = 'x')\ndat2.dtypes.head()\ndat2.loc[:,1].unique()\n\n/usr/local/linux/mambaforge-3.11/lib/python3.11/site-packages/IPython/core/formatters.py:342: FutureWarning:\n\nIn future versions `DataFrame.to_latex` is expected to utilise the base implementation of `Styler.to_latex` for formatting and rendering. The arguments signature may therefore change. It is recommended instead to use `DataFrame.style.to_latex` which also contains additional functionality.\n\n\n\n\n\n\narray([2336., 2124., 1830., 1833., 1600., 1578., 1187., 1005., 918.,\n 865., 871., 860., 883., 897., 898., 893., 913., 870.,\n 962., 880., 875., 884., 894., 836., 848., 885., 851.,\n 900., 861., 866., 867., 829., 853., 920., 877., 908.,\n 855., 845., 859., 856., 825., 828., 854., 847., 840.,\n 873., 822., 818., 838., 815., 813., 816., 849., 802.,\n 805., 792., 823., 808., 798., 800., 842., 809., 807.,\n 826., 810., 801., 794., 771., 796., 790., 787., 775.,\n 751., 783., 811., 768., 779., 795., 770., 821., 830.,\n 767., 772., 791., 781., 773., 777., 814., 778., 782.,\n 837., 759., 846., 797., 835., 832., 793., 803., 834.,\n 785., 831., 820., 812., 824., 728., 760., 762., 753.,\n 758., 764., 741., 709., 735., 749., 752., 761., 750.,\n 776., 766., 789., 763., 864., 858., 869., 886., 844.,\n 863., 916., 890., 872., 907., 926., 935., 933., 906.,\n 905., 912., 972., 996., 1009., 961., 952., 981., 917.,\n 1011., 1071., 1920., 3245., 3805., 3926., 3284., 2700., 2347.,\n 2078., 2935., 3040., 1860., 1437., 1512., 1720., 1493., 1026.,\n 928., 874., 833., 850., nan])\n\n\nUsing dtype is a good way to control how data are read in.\n\ndat = pd.read_table(os.path.join('..', 'data', 'hivSequ.csv'),\n sep = ',', header = 0,\n dtype = {\n 'PatientID': int,\n 'Resp': int,\n 'PR Seq': str,\n 'RT Seq': str,\n 'VL-t0': float,\n 'CD4-t0': int})\ndat.dtypes\ndat.loc[0,'PR Seq']\n\n/usr/local/linux/mambaforge-3.11/lib/python3.11/site-packages/IPython/core/formatters.py:342: FutureWarning:\n\nIn future versions `DataFrame.to_latex` is expected to utilise the base implementation of `Styler.to_latex` for formatting and rendering. The arguments signature may therefore change. It is recommended instead to use `DataFrame.style.to_latex` which also contains additional functionality.\n\n\n\n\n\n\n'CCTCAAATCACTCTTTGGCAACGACCCCTCGTCCCAATAAGGATAGGGGGGCAACTAAAGGAAGCYCTATTAGATACAGGAGCAGATGATACAGTATTAGAAGACATGGAGTTGCCAGGAAGATGGAAACCAAAAATGATAGGGGGAATTGGAGGTTTTATCAAAGTAARACAGTATGATCAGRTACCCATAGAAATCTATGGACATAAAGCTGTAGGTACAGTATTAATAGGACCTACACCTGTCAACATAATTGGAAGAAATCTGTTGACTCAGCTTGGTTGCACTTTAAATTTY'\n\n\nNote that you can avoid reading in one or more columns by using the usecols argument. Also, specifying the dtype argument explicitly should make for faster file reading.\nIf possible, it’s a good idea to look through the input file in the shell or in an editor before reading into Python to catch such issues in advance. Using the UNIX command less on RTADataSub.csv would have revealed these various issues, but note that RTADataSub.csv is a 1000-line subset of a much larger file of data available from the kaggle.com website. So more sophisticated use of UNIX utilities (as we will see in Unit 3) is often useful before trying to read something into a program.\nIf the file is not nicely arranged by field (e.g., if it has ragged lines), we’ll need to do some more work. We can read each line as a separate string, after which we can process the lines using text manipulation. Here’s an example from some US meteorological data where I know from metadata (not provided here) that the 4-11th values are an identifier, the 17-20th are the year, the 22-23rd the month, etc.\n\nfile_path = os.path.join('..', 'data', 'precip.txt')\nwith open(file_path, 'r') as file:\n lines = file.readlines()\n\nid = [line[3:11] for line in lines]\nyear = [int(line[17:21]) for line in lines]\nmonth = [int(line[21:23]) for line in lines]\nnvalues = [int(line[27:30]) for line in lines]\nyear[0:5]\n\n[2010, 2010, 2010, 2010, 2010]\n\n\nActually, that file, precip.txt, is in a fixed-width format (i.e., every element in a given column has the exact same number of characters),so reading in using pandas.read_fwf() would be a good strategy." }, { - "objectID": "units/unit7-bigData.html#logistics-and-data-size", - "href": "units/unit7-bigData.html#logistics-and-data-size", - "title": "Big data and databases", - "section": "Logistics and data size", - "text": "Logistics and data size\nOne of the main drawbacks with Python (and R) in working with big data is that all objects are stored in memory, so you can’t directly work with datasets that are more than 1-20 Gb or so, depending on the memory on your machine.\nThe techniques and tools discussed in this Unit (apart from the section on MapReduce/Spark) are designed for datasets in the range of gigabytes to tens of gigabytes, though they may scale to larger if you have a machine with a lot of memory or simply have enough disk space and are willing to wait. If you have 10s of gigabytes of data, you’ll be better off if your machine has 10s of GBs of memory, as discussed in this Unit.\nIf you’re scaling to 100s of GBs, terabytes or petabytes, tools such as carefully-administered databases, cloud-based tools such as provided by AWS and Google Cloud Platform, and Spark or other such tools are probably your best bet.\nNote: in handling big data files, it’s best to have the data on the local disk of the machine you are using to reduce traffic and delays from moving data over the network." + "objectID": "units/unit2-dataTech.html#connections-and-streaming", + "href": "units/unit2-dataTech.html#connections-and-streaming", + "title": "Data technologies, formats, and structures", + "section": "Connections and streaming", + "text": "Connections and streaming\nPython allows you to read in not just from a file but from a more general construct called a connection. This can include reading in text from the output of running a shell command and from unzipping a file on the fly.\nHere are some examples of connections:\n\nimport gzip\nwith gzip.open('dat.csv.gz', 'r') as file:\n lines = file.readlines()\n\nimport zipfile\nwith zipfile.ZipFile('dat.zip', 'r') as archive:\n with archive.open('data.txt', 'r') as file:\n lines = file.readlines()\n\nimport subprocess\ncommand = \"ls -al\"\noutput = subprocess.check_output(command, shell = True)\n# `output` is a sequence of bytes.\nwith io.BytesIO(output) as stream: # Create a file-like object.\n content = stream.readlines()\n\ndf = pd.read_csv(\"https://download.bls.gov/pub/time.series/cu/cu.item\", sep=\"\\t\")\n\nIf a file is large, we may want to read it in in chunks (of lines), do some computations to reduce the size of things, and iterate. This is referred to as online processing, streaming, or chunking, and can be done using Pandas (among other tools).\n\nfile_path = os.path.join('..', 'data', 'RTADataSub.csv')\nchunksize = 50 # Obviously this would be much larger in any real application.\n\nwith pd.read_csv(file_path, chunksize = chunksize) as reader:\n for chunk in reader:\n # manipulate the lines and store the key stuff\n print(f'Read {len(chunk)} rows.')\n\nMore details on sequential (on-line) processing of large files can be found in the tutorial on large datasets mentioned in the reference list above.\nOne cool trick that can come in handy is to ‘read’ from a string as if it were a text file. Here’s an example:\n\nfile_path = os.path.join('..', 'data', 'precip.txt')\nwith open(file_path, 'r') as file:\n text = file.read()\n\nstringIOtext = io.StringIO(text)\ndf = pd.read_fwf(stringIOtext, header = None, widths = [3,8,4,2,4,2])\n\nWe can create connections for writing output too. Just make sure to open the connection first." }, { - "objectID": "units/unit7-bigData.html#what-we-already-know-about-handling-big-data", - "href": "units/unit7-bigData.html#what-we-already-know-about-handling-big-data", - "title": "Big data and databases", - "section": "What we already know about handling big data!", - "text": "What we already know about handling big data!\nUNIX operations are generally very fast, so if you can manipulate your data via UNIX commands and piping, that will allow you to do a lot. We’ve already seen UNIX commands for extracting columns. And various commands such as grep, head, tail, etc. allow you to pick out rows based on certain criteria. As some of you have done in problem sets, one can use awk to extract rows. So basic shell scripting may allow you to reduce your data to a more manageable size.\nThe tool GNU parallel allows you to parallelize operations from the command line and is commonly used in working on Linux clusters.\nAnd don’t forget simple things. If you have a dataset with 30 columns that takes up 10 Gb but you only need 5 of the columns, get rid of the rest and work with the smaller dataset. Or you might be able to get the same information from a random sample of your large dataset as you would from doing the analysis on the full dataset. Strategies like this will often allow you to stick with the tools you already know.\nAlso, remember that we can often store data more compactly in binary formats than in flat text (e.g., csv) files.\nFinally, for many applications, storing large datasets in a standard database will work well." + "objectID": "units/unit2-dataTech.html#file-paths", + "href": "units/unit2-dataTech.html#file-paths", + "title": "Data technologies, formats, and structures", + "section": "File paths", + "text": "File paths\nA few notes on file paths, related to ideas of reproducibility.\n\nIn general, you don’t want to hard-code absolute paths into your code files because those absolute paths won’t be available on the machines of anyone you share the code with. Instead, use paths relative to the directory the code file is in, or relative to a baseline directory for the project, e.g.:\n\n\ndat = pd.read_csv('../data/cpds.csv')\n\nUsing UNIX style directory separators will work in Windows, Mac or Linux, but using Windows style separators is not portable across operating systems.\n\n## good: will work on Windows\ndat = pd.read_csv('../data/cpds.csv')\n## bad: won't work on Mac or Linux\ndat = pd.read_csv('..\\data\\cpds.csv') \n\nEven better, use os.path.join so that paths are constructed specifically for the operating system the user is using:\n\n\n## good: operating-system independent\ndat = pd.read_csv(os.path.join('..', 'data', 'cpds.csv'))" }, { - "objectID": "units/unit7-bigData.html#overview", - "href": "units/unit7-bigData.html#overview", - "title": "Big data and databases", - "section": "Overview", - "text": "Overview\nA basic paradigm for working with big datasets is the MapReduce paradigm. The basic idea is to store the data in a distributed fashion across multiple nodes and try to do the computation in pieces on the data on each node. Results can also be stored in a distributed fashion.\nA key benefit of this is that if you can’t fit your dataset on disk on one machine you can on a cluster of machines. And your processing of the dataset can happen in parallel. This is the basic idea of MapReduce.\nThe basic steps of MapReduce are as follows:\n\nread individual data objects (e.g., records/lines from CSVs or individual data files)\nmap: create key-value pairs using the inputs (more formally, the map step takes a key-value pair and returns a new key-value pair)\nreduce: for each key, do an operation on the associated values and create a result - i.e., aggregate within the values assigned to each key\nwrite out the {key,result} pair\n\nA similar paradigm that is implemented in pandas and dplyr is the split-apply-combine strategy.\nA few additional comments. In our map function, we could exclude values or transform them in some way, including producing multiple records from a single record. And in our reduce function, we can do more complicated analysis. So one can actually do fairly sophisticated things within what may seem like a restrictive paradigm. But we are constrained such that in the map step, each record needs to be treated independently and in the reduce step each key needs to be treated independently. This allows for the parallelization.\nOne important note is that any operations that require moving a lot of data between the workers can take a long time. (This is sometimes called a shuffle.) This could happen if, for example, you computed the median value within each of many groups if the data for each group are spread across the workers. In contrast, if we compute the mean or sum, one can compute the partial sums on each worker and then just add up the partial sums.\nNote that as discussed in Unit 5 the concepts of map and reduce are core concepts in functional programming, and of course Python provides the map function.\nHadoop is an infrastructure for enabling MapReduce across a network of machines. The basic idea is to hide the complexity of distributing the calculations and collecting results. Hadoop includes a file system for distributed storage (HDFS), where each piece of information is stored redundantly (on multiple machines). Calculations can then be done in a parallel fashion, often on data in place on each machine thereby limiting the amount of communication that has to be done over the network. Hadoop also monitors completion of tasks and if a node fails, it will redo the relevant tasks on another node. Hadoop is based on Java. Given the popularity of Spark, I’m not sure how much usage these approaches currently see. Setting up a Hadoop cluster can be tricky. Hopefully if you’re in a position to need to use Hadoop, it will be set up for you and you will be interacting with it as a user/data analyst.\nOk, so what is Spark? You can think of Spark as in-memory Hadoop. Spark allows one to treat the memory across multiple nodes as a big pool of memory. Therefore, Spark should be faster than Hadoop when the data will fit in the collective memory of multiple nodes. In cases where it does not, Spark will make use of the HDFS (and generally, Spark will be reading the data initially from HDFS.) While Spark is more user-friendly than Hadoop, there are also some things that can make it hard to use. Setting up a Spark cluster also involves a bit of work, Spark can be hard to configure for optimal performance, and Spark calculations have a tendency to fail (often involving memory issues) in ways that are hard for users to debug." + "objectID": "units/unit2-dataTech.html#reading-data-quickly-arrow-and-polars", + "href": "units/unit2-dataTech.html#reading-data-quickly-arrow-and-polars", + "title": "Data technologies, formats, and structures", + "section": "Reading data quickly: Arrow and Polars", + "text": "Reading data quickly: Arrow and Polars\nApache Arrow provides efficient data structures for working with data in memory, usable in Python via the PyArrow package. Data are stored by column, with values in a column stored sequentially and in such a way that one can access a specific value without reading the other values in the column (O(1) lookup). Arrow is designed to read data from various file formats, including Parquet, native Arrow format, and text files. In general Arrow will only read data from disk as needed, avoiding keeping the entire dataset in memory.\nOther options for avoiding reading all your data into memory include the Dask package and using numpy.load with the mmap_mode argument.\npolars is designed to be a faster alternative to Pandas for working with data in-memory.\n\nimport polars\nimport time\nt0 = time.time()\ndat = pd.read_csv(os.path.join('..', 'data', 'airline.csv'))\nt1 = time.time()\ndat2 = polars.read_csv(os.path.join('..', 'data', 'airline.csv'), null_values = ['NA'])\nt2 = time.time()\nprint(f\"Timing for Pandas: {t1-t0}.\")\nprint(f\"Timing for Polars: {t2-t1}.\")\n\nTiming for Pandas: 0.8093833923339844.\nTiming for Polars: 0.12352108955383301." }, { - "objectID": "units/unit7-bigData.html#using-dask-for-big-data-processing", - "href": "units/unit7-bigData.html#using-dask-for-big-data-processing", - "title": "Big data and databases", - "section": "Using Dask for big data processing", - "text": "Using Dask for big data processing\nUnit 6 on parallelization gives an overview of using Dask for flexible parallelization on different kinds of computational resources (in particular, parallelizing across multiple cores on one machine versus parallelizing across multiple cores across multiple machines/nodes).\nHere we’ll see the use of Dask to work with distributed datasets. Dask can process datasets (potentially very large ones) by parallelizing operations across subsets of the data using multiple cores on one or more machines.\nLike Spark, Dask automatically reads data from files in parallel and operates on chunks (also called partitions or shards) of the full dataset in parallel. There are two big advantages of this:\n\nYou can do calculations (including reading from disk) in parallel because each worker will work on a piece of the data.\nWhen the data is split across machines, you can use the memory of multiple machines to handle much larger datasets than would be possible in memory on one machine. That said, Dask processes the data in chunks, so one often doesn’t need a lot of memory, even just on one machine.\n\nWhile reading from disk in parallel is a good goal, if all the data are on one hard drive, there are limitations on the speed of reading the data from disk because of having multiple processes all trying to access the disk at once. Supercomputing systems will generally have parallel file systems that support truly parallel reading (and writing, i.e., parallel I/O). Hadoop/Spark deal with this by distributing across multiple disks, generally one disk per machine/node.\nBecause computations are done in external compiled code (e.g., via numpy) it’s effective to use the threads scheduler when operating on one node to avoid having to copy and move the data.\n\nDask dataframes (pandas)\nDask dataframes are Pandas-like dataframes where each dataframe is split into groups of rows, stored as smaller Pandas dataframes.\nOne can do a lot of the kinds of computations that you would do on a Pandas dataframe on a Dask dataframe, but many operations are not possible. See here.\nBy default dataframes are handled by the threads scheduler. (Recall we discussed Dask’s various schedulers in Unit 6.)\nHere’s an example of reading from a dataset of flight delays (about 11 GB data). You can get the data here.\n\nimport dask\ndask.config.set(scheduler='threads', num_workers = 4) \nimport dask.dataframe as ddf\npath = '/scratch/users/paciorek/243/AirlineData/csvs/'\nair = ddf.read_csv(path + '*.csv.bz2',\n compression = 'bz2',\n encoding = 'latin1', # (unexpected) latin1 value(s) in TailNum field in 2001\n dtype = {'Distance': 'float64', 'CRSElapsedTime': 'float64',\n 'TailNum': 'object', 'CancellationCode': 'object', 'DepDelay': 'float64'})\n# specify some dtypes so Pandas doesn't complain about column type heterogeneity\nair\n\nDask will reads the data in parallel from the various .csv.bz2 files (unzipping on the fly), but note the caveat in the previous section about the possibilities for truly parallel I/O.\nHowever, recall that Dask uses delayed evaluation. In this case, the reading is delayed until compute() is called. For that matter, the various other calculations (max, groupby, mean) shown below are only done after compute() is called.\n\nimport time\n\nt0 = time.time()\nair.DepDelay.max().compute() # this takes a while\nprint(time.time() - t0)\n\nt0 = time.time()\nair.DepDelay.mean().compute() # this takes a while\nprint(time.time() - t0)\n\nair.DepDelay.median().compute() \n\nWe’ll discuss in class why Dask won’t do the median. Consider the discussion about moving data in the earlier section on MapReduce.\nNext let’s see a full split-apply-combine (aka MapReduce) type of analysis.\n\nsub = air[(air.UniqueCarrier == 'UA') & (air.Origin == 'SFO')]\nbyDest = sub.groupby('Dest').DepDelay.mean()\nresults = byDest.compute() # this takes a while too\nresults\n\nYou should see this:\n Dest \n ACV 26.200000 \n BFL 1.000000 \n BOI 12.855069 \n BOS 9.316795 \n CLE 4.000000\n ...\nNote: calling compute twice is a bad idea as Dask will read in the data twice - more on this in a bit.\n\nWarning Think carefully about the size of the result from calling compute. The result will be returned as a standard Python object, not distributed across multiple workers (and possibly machines), and with the object entirely in memory. It’s easy to accidentally return an entire giant dataset.\n\n\n\nDask bags\nBags are like lists but there is no particular ordering, so it doesn’t make sense to ask for the i’th element.\nYou can think of operations on Dask bags as being like parallel map operations on lists in Python or R.\nBy default bags are handled via the processes scheduler.\nLet’s see some basic operations on a large dataset of Wikipedia log files. You can get a subset of the Wikipedia data here.\nHere we again read the data in (which Dask will do in parallel):\n\nimport dask.multiprocessing\ndask.config.set(scheduler='processes', num_workers = 4) \nimport dask.bag as db\n## This is the full data\n## path = '/scratch/users/paciorek/wikistats/dated_2017/'\n## For demo we'll just use a small subset\npath = '/scratch/users/paciorek/wikistats/dated_2017_small/dated/'\nwiki = db.read_text(path + 'part-0*gz')\n\nHere we’ll just count the number of records.\n\nimport time\nt0 = time.time()\nwiki.count().compute()\ntime.time() - t0 # 136 sec. for full data\n\nAnd here is a more realistic example of filtering (subsetting).\n\nimport re\ndef find(line, regex = 'Armenia'):\n vals = line.split(' ')\n if len(vals) < 6:\n return(False)\n tmp = re.search(regex, vals[3])\n if tmp is None:\n return(False)\n else:\n return(True)\n \n\nwiki.filter(find).count().compute()\narmenia = wiki.filter(find)\nsmp = armenia.take(100) ## grab a handful as proof of concept\nsmp[0:5]\n\nNote that it is quite inefficient to do the find() (and implicitly reading the data in) and then compute on top of that intermediate result in two separate calls to compute(). Rather, we should set up the code so that all the operations are set up before a single call to compute(). This is discussed in detail in the Dask/future tutorial.\nSince the data are just treated as raw strings, we might want to introduce structure by converting each line to a tuple and then converting to a data frame.\n\ndef make_tuple(line):\n return(tuple(line.split(' ')))\n\ndtypes = {'date': 'object', 'time': 'object', 'language': 'object',\n'webpage': 'object', 'hits': 'float64', 'size': 'float64'}\n\n## Let's create a Dask dataframe. \n## This will take a while if done on full data.\ndf = armenia.map(make_tuple).to_dataframe(dtypes)\ntype(df)\n\n## Now let's actually do the computation, returning a Pandas df\nresult = df.compute() \ntype(result)\nresult[0:5]\n\n\n\nDask arrays (numpy)\nDask arrays are numpy-like arrays where each array is split up by both rows and columns into smaller numpy arrays.\nOne can do a lot of the kinds of computations that you would do on a numpy array on a Dask array, but many operations are not possible. See here.\nBy default arrays are handled via the threads scheduler.\n\nNon-distributed arrays\nLet’s first see operations on a single node, using a single 13 GB two-dimensional array. Again, Dask uses lazy evaluation, so creation of the array doesn’t happen until an operation requiring output is done.\n\nimport dask\ndask.config.set(scheduler = 'threads', num_workers = 4) \nimport dask.array as da\nx = da.random.normal(0, 1, size=(40000,40000), chunks=(10000, 10000))\n# square 10k x 10k chunks\nmycalc = da.mean(x, axis = 1) # by row\nimport time\nt0 = time.time()\nrs = mycalc.compute()\ntime.time() - t0 # 41 sec.\n\nFor a row-based operation, we would presumably only want to chunk things up by row, but this doesn’t seem to actually make a difference, presumably because the mean calculation can be done in pieces and only a small number of summary statistics moved between workers.\n\nimport dask\ndask.config.set(scheduler='threads', num_workers = 4) \nimport dask.array as da\n# x = da.from_array(x, chunks=(2500, 40000)) # adjust chunk size of existing array\nx = da.random.normal(0, 1, size=(40000,40000), chunks=(2500, 40000))\nmycalc = da.mean(x, axis = 1) # row means\nimport time\nt0 = time.time()\nrs = mycalc.compute()\ntime.time() - t0 # 42 sec.\n\nOf course, given the lazy evaluation, this timing comparison is not just timing the actual row mean calculations.\nBut this doesn’t really clarify the story…\n\nimport dask\ndask.config.set(scheduler='threads', num_workers = 4) \nimport dask.array as da\nimport numpy as np\nimport time\nt0 = time.time()\nx = np.random.normal(0, 1, size=(40000,40000))\ntime.time() - t0 # 110 sec.\n# for some reason the from_array and da.mean calculations are not done lazily here\nt0 = time.time()\ndx = da.from_array(x, chunks=(2500, 40000))\ntime.time() - t0 # 27 sec.\nt0 = time.time()\nmycalc = da.mean(x, axis = 1) # what is this doing given .compute() also takes time?\ntime.time() - t0 # 28 sec.\nt0 = time.time()\nrs = mycalc.compute()\ntime.time() - t0 # 21 sec.\n\nDask will avoid storing all the chunks in memory. (It appears to just generate them on the fly.) Here we have an 80 GB array but we never use more than a few GB of memory (based on top or free -h).\n\nimport dask\ndask.config.set(scheduler='threads', num_workers = 4) \nimport dask.array as da\nx = da.random.normal(0, 1, size=(100000,100000), chunks=(10000, 10000))\nmycalc = da.mean(x, axis = 1) # row means\nimport time\nt0 = time.time()\nrs = mycalc.compute()\ntime.time() - t0 # 205 sec.\nrs[0:5]\n\n\n\nDistributed arrays\nUsing arrays distributed across multiple machines should be straightforward based on using Dask distributed. However, one would want to be careful about creating arrays by distributing the data from a single Python process as that would involve copying between machines." + "objectID": "units/unit2-dataTech.html#writing-output-to-files", + "href": "units/unit2-dataTech.html#writing-output-to-files", + "title": "Data technologies, formats, and structures", + "section": "Writing output to files", + "text": "Writing output to files\nFunctions for text output are generally analogous to those for input.\n\nfile_path = os.path.join('/tmp', 'tmp.txt')\nwith open(file_path, 'w') as file:\n file.writelines(lines)\n\nWe can also use file.write() to write individual strings.\nIn Pandas, we can use DataFrame.to_csv and DataFrame.to_parquet.\nWe can use the json.dump function to output appropriate data objects (e.g., dictionaries or possibly lists) as JSON. One use of JSON as output from Python would be to ‘serialize’ the information in an Python object such that it could be read into another program.\nAnd of course you can always save to a Pickle data file (a binary file format) using pickle.dump() and pickle.load() from the pickle package. Happily this is platform-independent so can be used to transfer Python objects between different OS." }, { - "objectID": "units/unit7-bigData.html#overview-1", - "href": "units/unit7-bigData.html#overview-1", - "title": "Big data and databases", - "section": "Overview", - "text": "Overview\nBasically, standard SQL databases are relational databases that are a collection of rectangular format datasets (tables, also called relations), with each table similar to R or Pandas data frames, in that a table is made up of columns, which are called fields or attributes, each containing a single type (numeric, character, date, currency, enumerated (i.e., categorical), …) and rows or records containing the observations for one entity. Some of the tables in a given database will generally have fields in common so it makes sense to merge (i.e., join) information from multiple tables. E.g., you might have a database with a table of student information, a table of teacher information and a table of school information, and you might join student information with information about the teacher(s) who taught the students. Databases are set up to allow for fast querying and merging (called joins in database terminology).\n\nMemory and disk use\nFormally, databases are stored on disk, while Python and R store datasets in memory. This would suggest that databases will be slow to access their data but will be able to store more data than can be loaded into an Python or R session. However, databases can be quite fast due in part to disk caching by the operating system as well as careful implementation of good algorithms for database operations." + "objectID": "units/unit2-dataTech.html#formatting-output", + "href": "units/unit2-dataTech.html#formatting-output", + "title": "Data technologies, formats, and structures", + "section": "Formatting output", + "text": "Formatting output\nWe can use string formatting to control how output is printed to the screen.\nThe mini-language involved in the format specification can get fairly involved, but a few basic pieces of syntax can do most of what one generally needs to do.\nWe can format numbers to chosen number of digits and decimal places and handle alignment, using the format method of the string class.\nFor example:\n\n'{:>10}'.format(3.5) # right-aligned, using 10 characters\n'{:.10f}'.format(1/3) # force 10 decimal places\n'{:15.10f}'.format(1/3) # force 15 characters, with 10 decimal places\nformat(1/3, '15.10f') # alternative using a function\n\n' 3.5'\n\n\n'0.3333333333'\n\n\n' 0.3333333333'\n\n\n' 0.3333333333'\n\n\nWe can also “interpolate” variables into strings.\n\n\"The number pi is {}.\".format(np.pi)\n\"The number pi is {:.5f}.\".format(np.pi)\n\"The number pi is {:.12f}.\".format(np.pi)\n\n'The number pi is 3.141592653589793.'\n\n\n'The number pi is 3.14159.'\n\n\n'The number pi is 3.141592653590.'\n\n\n\nval1 = 1.5\nval2 = 2.5\n# As of Python 3.6, put the variable names in directly.\nprint(f\"Let's add {val1} and {val2}.\") \nnum1 = 1/3\nprint(\"Let's add the %s numbers %.5f and %15.7f.\"\n %('floating point', num1 ,32+1/7))\n\nLet's add 1.5 and 2.5.\nLet's add the floating point numbers 0.33333 and 32.1428571.\n\n\nOr to insert into a file:\n\nfile_path = os.path.join('/tmp', 'tmp.txt')\nwith open(file_path, 'a') as file:\n file.write(\"Let's add the %s numbers %.5f and %15.7f.\"\n %('floating point', num1 ,32+1/7))\n\nround is another option, but it’s often better to directly control the printing format." }, { - "objectID": "units/unit7-bigData.html#interacting-with-a-database", - "href": "units/unit7-bigData.html#interacting-with-a-database", - "title": "Big data and databases", - "section": "Interacting with a database", - "text": "Interacting with a database\nYou can interact with databases in a variety of database systems (DBMS=database management system). Some popular systems are SQLite, DuckDB, MySQL, PostgreSQL, Oracle and Microsoft Access. We’ll concentrate on accessing data in a database rather than management of databases. SQL is the Structured Query Language and is a special-purpose high-level language for managing databases and making queries. Variations on SQL are used in many different DBMS.\nQueries are the way that the user gets information (often simply subsets of tables or information merged across tables). The result of an SQL query is in general another table, though in some cases it might have only one row and/or one column.\nMany DBMS have a client-server model. Clients connect to the server, with some authentication, and make requests (i.e., queries).\nThere are often multiple ways to interact with a DBMS, including directly using command line tools provided by the DBMS or via Python or R, among others.\nWe’ll concentrate on SQLite (because it is simple to use on a single machine). SQLite is quite nice in terms of being self-contained - there is no server-client model, just a single file on your hard drive that stores the database and to which you can connect to using the SQLite shell, R, Python, etc. However, it does not have some useful functionality that other DBMS have. For example, you can’t use ALTER TABLE to modify column types or drop columns.\nA good alternative to SQLite that I encourage you to consider is DuckDB. DuckDB stores data column-wise, which can lead to big speedups when doing queries operating on large portions of tables (so-called “online analytical processing” (OLAP)). Another nice feature of DuckDB is that it can interact with data on disk without always having to read all the data into memory. In fact, ideally we’d use it for this class, but I haven’t had time to create a DuckDB version of the StackOverflow database." + "objectID": "units/unit2-dataTech.html#reading-html", + "href": "units/unit2-dataTech.html#reading-html", + "title": "Data technologies, formats, and structures", + "section": "Reading HTML", + "text": "Reading HTML\nHTML (Hypertext Markup Language) is the standard markup language used for displaying content in a web browser. In simple webpages (ignoring the more complicated pages that involve Javascript), what you see in your browser is simply a rendering (by the browser) of a text file containing HTML.\nHowever, instead of rendering the HTML in a browser, we might want to use code to extract information from the HTML.\nLet’s see a brief example of reading in HTML tables.\nNote that before doing any coding, it can be helpful to look at the raw HTML source code for a given page. We can explore the underlying HTML source in advance of writing our code by looking at the page source directly in the browser (e.g., in Firefox under the 3-lines (hamburger) “open menu” symbol, see Web Developer (or More Tools) -> Page Source and in Chrome View -> Developer -> View Source), or by downloading the webpage and looking at it in an editor, although in some cases (such as the nytimes.com case), what we might see is a lot of JavaScript.\nOne lesson here is not to write a lot of your own code to do something that someone else has probably already written a package for. We’ll use the BeautifulSoup4 package.\n\nimport requests\nfrom bs4 import BeautifulSoup as bs\n\nURL = \"https://en.wikipedia.org/wiki/List_of_countries_and_dependencies_by_population\"\nresponse = requests.get(URL)\nhtml = response.content\n\n# Create a BeautifulSoup object to parse the HTML\nsoup = bs(html, 'html.parser')\n\nhtml_tables = soup.find_all('table')\n\npd_tables = [pd.read_html(str(tbl))[0] for tbl in html_tables]\n\n[x.shape[0] for x in pd_tables]\n\npd_tables[0].head()\n\n[242, 13, 1]\n\n\n/usr/local/linux/mambaforge-3.11/lib/python3.11/site-packages/IPython/core/formatters.py:342: FutureWarning:\n\nIn future versions `DataFrame.to_latex` is expected to utilise the base implementation of `Styler.to_latex` for formatting and rendering. The arguments signature may therefore change. It is recommended instead to use `DataFrame.style.to_latex` which also contains additional functionality.\n\n\n\n\n\n\n\n\n\n\nUnnamed: 0\nCountry / Dependency\nPopulation\n% of world\nDate\nSource (official or from the United Nations)\nUnnamed: 6\n\n\n\n\n0\n–\nWorld\n8057107000\n100%\n31 Aug 2023\nUN projection[3]\nNaN\n\n\n1\n1\nChina\n1411750000\nNaN\n31 Dec 2022\nOfficial estimate[4]\n[b]\n\n\n2\n2\nIndia\n1392329000\nNaN\n1 Jul 2023\nOfficial projection[5]\n[c]\n\n\n3\n3\nUnited States\n335340000\nNaN\n31 Aug 2023\nNational population clock[7]\n[d]\n\n\n4\n4\nIndonesia\n277749853\nNaN\n31 Dec 2022\nOfficial estimate[8]\nNaN\n\n\n\n\n\n\n\nBeautiful Soup works by reading in the HTML as text and then parsing it to build up a tree containing the HTML elements. Then one can search by HTML tag or attribute for information you want using find_all.\nAs another example, it’s often useful to be able to extract the hyperlinks in an HTML document.\n\nURL = \"http://www1.ncdc.noaa.gov/pub/data/ghcn/daily/by_year\"\nresponse = requests.get(URL)\nsoup = bs(response.content, 'html.parser')\n\n## Approach 1: search for HTML 'a' tags.\na_elements = soup.find_all('a')\nlinks1 = [x.get('href') for x in a_elements]\n## Approach 2: search for 'a' elements with 'href' attribute\nhref_elements = soup.find_all('a', href = True)\nlinks2 = [x.get('href') for x in href_elements]\n## In either case, then use `get` to retrieve the `href` attribute value.\n\nlinks2[0:9]\n# help(bs.find_all)\n\n['?C=N;O=D',\n '?C=M;O=A',\n '?C=S;O=A',\n '?C=D;O=A',\n '/pub/data/ghcn/daily/',\n '1750.csv.gz',\n '1763.csv.gz',\n '1764.csv.gz',\n '1765.csv.gz']\n\n\nThe kwargs keyword arguments to find and find_all allow one to search for elements with particular characteristics, such as having a particular attribute (seen above) or having an attribute have a particular value (e.g., picking out an element with a particular id).\nHere’s another example of extracting specific components of information from a webpage (results not shown, since headlines will vary from day to day). We’ll use get_text to retrieve the element’s value.\n\nURL = \"https://www.nytimes.com\"\nresponseNYT = requests.get(URL)\nsoupNYT = bs(responseNYT.content, 'html.parser')\nh2_elements = soupNYT.find_all(\"h2\")\nheadlines2 = [x.get_text() for x in h2_elements]\nh3_elements = soupNYT.find_all(\"h3\")\nheadlines3 = [x.get_text() for x in h3_elements]\n\nMore generally, we may want to read an HTML document, parse it into its components (i.e., the HTML elements), and navigate through the tree structure of the HTML.\nWe can use CSS selectors with the select method for more powerful extraction capabilities. Going back to the climate data, let’s extract all the th elements nested within tr elements:\n\nsoup.select(\"tr th\")\n\n[<th><a href=\"?C=N;O=D\">Name</a></th>,\n <th><a href=\"?C=M;O=A\">Last modified</a></th>,\n <th><a href=\"?C=S;O=A\">Size</a></th>,\n <th><a href=\"?C=D;O=A\">Description</a></th>,\n <th colspan=\"4\"><hr/></th>,\n <th colspan=\"4\"><hr/></th>]\n\n\nOr we could extract the a elements whose parents are th elements:\n\nsoup.select(\"th > a\")\n\n[<a href=\"?C=N;O=D\">Name</a>,\n <a href=\"?C=M;O=A\">Last modified</a>,\n <a href=\"?C=S;O=A\">Size</a>,\n <a href=\"?C=D;O=A\">Description</a>]\n\n\nNext let’s use the XPath language to specify elements rather than CSS selectors. XPath can also be used for navigating through XML documents.\n\nimport lxml.html\n\n# Convert the BeautifulSoup object to a lxml object\nlxml_doc = lxml.html.fromstring(str(soup))\n\n# Use XPath to select elements\na_elements = lxml_doc.xpath('//a[@href]')\nlinks = [x.get('href') for x in a_elements]\nlinks[0:9]\n\n['?C=N;O=D',\n '?C=M;O=A',\n '?C=S;O=A',\n '?C=D;O=A',\n '/pub/data/ghcn/daily/',\n '1750.csv.gz',\n '1763.csv.gz',\n '1764.csv.gz',\n '1765.csv.gz']" }, { - "objectID": "units/unit7-bigData.html#database-schema-and-normalization", - "href": "units/unit7-bigData.html#database-schema-and-normalization", - "title": "Big data and databases", - "section": "Database schema and normalization", - "text": "Database schema and normalization\nTo truly leverage the conceptual and computational power of a database you’ll want to have your data in a normalized form, which means spreading your data across multiple tables in such a way that you don’t repeat information unnecessarily.\nThe schema is the metadata about the tables in the database and the fields (and their types) in those tables.\nLet’s consider this using an educational example. Suppose we have a school with multiple teachers teaching multiple classes and multiple students taking multiple classes. If we put this all in one table organized per student, the data might have the following fields:\n\nstudent ID\nstudent grade level\nstudent name\nclass 1\nclass 2\n…\nclass n\ngrade in class 1\ngrade in class 2\n…\ngrade in class n\nteacher ID 1\nteacher ID 2\n…\nteacher ID n\nteacher name 1\nteacher name 2\n…\nteacher name n\nteacher department 1\nteacher department 2\n…\nteacher department n\nteacher age 1\nteacher age 2\n…\nteacher age n\n\nThere are a lot of problems with this:\n\nA lot of information is repeated across rows (e.g., teacher age for students who have the same teacher) - this is a waste of space - it is hard/error-prone to update values in the database (e.g., after a teacher’s birthday), because a given value needs to be updated in multiple places\nThere are potentially a lot of empty cells (e.g., for a student who takes fewer than ‘n’ classes). This will generally result in a waste of space.\nIt’s hard to see the information that is not organized uniquely by row – i.e., it’s much easier to understand the information at the student level than the teacher level\nWe have to know in advance how big ‘n’ is. Then if a single student takes more than ‘n’ classes, the whole database needs to be restructured.\n\nIt would get even worse if there was a field related to teachers for which a given teacher could have multiple values (e.g., teachers could be in multiple departments). This would lead to even more redundancy - each student-class-teacher combination would be crossed with all of the departments for the teacher (so-called multivalued dependency in database theory).\nAn alternative organization of the data would be to have each row represent the enrollment of a student in a class.\n\nstudent ID\nstudent name\nclass\ngrade in class\nstudent grade level\nteacher ID\nteacher department\nteacher age\n\nThis has some advantages relative to our original organization in terms of not having empty cells, but it doesn’t solve the other three issues above.\nInstead, a natural way to order this database is with the following four tables.\n\nStudent\n\nID\nname\ngrade_level\n\nTeacher\n\nID\nname\ndepartment\nage\n\nClass\n\nID\ntopic\nclass_size\nteacher_ID\n\nClassAssignment\n\nstudent_ID\nclass_ID\ngrade\n\n\nThe ClassAssignment table has one row per student-class pair. Having a table like this handles “ragged” data where the number of observations per unit (in this case classes per student) varies. Using such tables is a common pattern when considering how to normalize a database. It’s also a core part of the idea of “tidy data” and data in long format, seen in the tidyr package.\nThen we do queries to pull information from multiple tables. We do the joins based on keys, which are the fields in each table that allow us to match rows from different tables.\n(That said, if all anticipated uses of a database will end up recombining the same set of tables, we may want to have a denormalized schema in which those tables are actually combined in the database. It is possible to be too pure about normalization! We can also create a virtual table, called a view, as discussed later.)\n\nKeys\nA key is a field or collection of fields that give(s) a unique value for every row/observation. A table in a database should then have a primary key that is the main unique identifier used by the DBMS. Foreign keys are columns in one table that give the value of the primary key in another table. When information from multiple tables is joined together, the matching of a row from one table to a row in another table is generally done by equating the primary key in one table with a foreign key in a different table.\nIn our educational example, the primary keys would presumably be: Student.ID, Teacher.ID, Class.ID, and for ClassAssignment a primary key made of two fields: {ClassAssignment.studentID, ClassAssignment.class_ID}.\nSome examples of foreign keys would be:\n\nstudent_ID as the foreign key in ClassAssignment for joining with Student on Student.ID\nteacher_ID as the foreign key in Class for joining with Teacher based on Teacher.ID\nclass_ID as the foreign key in ClassAssignment for joining with Class based on Class.ID\n\n\n\nQueries that join data across multiple tables\nSuppose we want a result that has the grades of all students in 9th grade. For this we need information from the Student table (to determine grade level) and information from the ClassAssignment table (to determine the class grade). More specifically we need a query that:\n\njoins Student with ClassAssignment based on matching rows in Student with rows in ClassAssignment where Student.ID is the same as ClassAssignment.student_ID and\nfilters the rows based on Student.grade_level:\n\n\nSELECT Student.ID, grade FROM Student, ClassAssignment WHERE \n Student.ID = ClassAssignment.student_ID and Student.grade_level = 9;\n\nNote that the query is a join (specifically an inner join), which is like merge() (or dplyr::join) in R. We don’t specifically use the JOIN keyword, but one could do these queries explicitly using JOIN, as we’ll see later." + "objectID": "units/unit2-dataTech.html#xml", + "href": "units/unit2-dataTech.html#xml", + "title": "Data technologies, formats, and structures", + "section": "XML", + "text": "XML\nXML is a markup language used to store data in self-describing (no metadata needed) format, often with a hierarchical structure. It consists of sets of elements (also known as nodes because they generally occur in a hierarchical structure and therefore have parents, children, etc.) with tags that identify/name the elements, with some similarity to HTML. Some examples of the use of XML include serving as the underlying format for Microsoft Office and Google Docs documents and for the KML language used for spatial information in Google Earth.\nHere’s a brief example. The book with id attribute bk101 is an element; the author of the book is also an element that is a child element of the book. The id attribute allows us to uniquely identify the element.\n <?xml version=\"1.0\"?>\n <catalog>\n <book id=\"bk101\">\n <author>Gambardella, Matthew</author>\n <title>XML Developer's Guide</title>\n <genre>Computer</genre>\n <price>44.95</price>\n <publish_date>2000-10-01</publish_date>\n <description>An in-depth look at creating applications with XML.</description>\n </book>\n <book id=\"bk102\">\n <author>Ralls, Kim</author>\n <title>Midnight Rain</title>\n <genre>Fantasy</genre>\n <price>5.95</price>\n <publish_date>2000-12-16</publish_date>\n <description>A former architect battles corporate zombies, an evil sorceress, and her own childhood to become queen of the world.</description>\n </book>\n </catalog>\nWe can read XML documents into Python using various packages, including lxml and then manipulate the resulting structured data object. Here’s an example of working with lending data from the Kiva lending non-profit. You can see the XML format in a browser at http://api.kivaws.org/v1/loans/newest.xml.\nXML documents have a tree structure with information at nodes. As above with HTML, one can use the XPath language for navigating the tree and finding and extracting information from the node(s) of interest.\nHere is some example code for extracting loan info from the Kiva data. We’ll first show the ‘brute force’ approach of working with the data as a list and then the better approach of using XPath.\n\nimport xmltodict\n\nURL = \"https://api.kivaws.org/v1/loans/newest.xml\"\nresponse = requests.get(URL)\ndata = xmltodict.parse(response.content)\ndata.keys()\ndata['response'].keys()\ndata['response']['loans'].keys()\nlen(data['response']['loans']['loan'])\ndata['response']['loans']['loan'][2]\ndata['response']['loans']['loan'][2]['activity']\n\ndict_keys(['response'])\n\n\ndict_keys(['paging', 'loans'])\n\n\ndict_keys(['@type', 'loan'])\n\n\n20\n\n\n{'id': '2635728',\n 'name': 'Gulmira',\n 'description': {'languages': {'@type': 'list', 'language': ['ru', 'en']}},\n 'status': 'fundraising',\n 'funded_amount': '0',\n 'basket_amount': '0',\n 'image': {'id': '5248685', 'template_id': '1'},\n 'activity': 'Education provider',\n 'sector': 'Education',\n 'use': 'to purchase a printer.',\n 'location': {'country_code': 'KG',\n 'country': 'Kyrgyzstan',\n 'town': 'Kochkor district, Naryn region',\n 'geo': {'level': 'town', 'pairs': '42.099103 75.52767', 'type': 'point'}},\n 'partner_id': '171',\n 'posted_date': '2023-08-31T14:50:10Z',\n 'planned_expiration_date': '2023-10-05T14:50:10Z',\n 'loan_amount': '375',\n 'borrower_count': '1',\n 'lender_count': '0',\n 'bonus_credit_eligibility': '1',\n 'tags': None}\n\n\n'Education provider'\n\n\n\nfrom lxml import etree\ndoc = etree.fromstring(response.content)\n\nloans = doc.xpath(\"//loan\")\n[loan.xpath(\"activity/text()\") for loan in loans]\n\n## suppose we only want the country locations of the loans (using XPath)\n[loan.xpath(\"location/country/text()\") for loan in loans]\n## or extract the geographic coordinates\n[loan.xpath(\"location/geo/pairs/text()\") for loan in loans]\n\n[['Personal Housing Expenses'],\n ['Education provider'],\n ['Education provider'],\n ['Agriculture'],\n ['Education provider'],\n ['Farming'],\n ['Farming'],\n ['Farming'],\n ['Farming'],\n ['Farming'],\n ['Farming'],\n ['Agriculture'],\n ['Agriculture'],\n ['Clothing'],\n ['Crafts'],\n ['Food Production/Sales'],\n ['Agriculture'],\n ['Agriculture'],\n ['Agriculture'],\n ['Food']]\n\n\n[['Nicaragua'],\n ['Kyrgyzstan'],\n ['Kyrgyzstan'],\n ['Kyrgyzstan'],\n ['Kyrgyzstan'],\n ['Kenya'],\n ['Kenya'],\n ['Kenya'],\n ['Kenya'],\n ['Kenya'],\n ['Kenya'],\n ['Samoa'],\n ['Samoa'],\n ['Samoa'],\n ['Samoa'],\n ['Samoa'],\n ['Samoa'],\n ['Samoa'],\n ['Samoa'],\n ['Samoa']]\n\n\n[['11.934407 -85.956001'],\n ['42.099103 75.52767'],\n ['42.099103 75.52767'],\n ['41.824168 71.097317'],\n ['42.874621 74.569762'],\n ['-0.333333 37.65'],\n ['-0.333333 37.65'],\n ['43.010557 -74.379162'],\n ['-0.538838 37.459641'],\n ['0.566667 34.566667'],\n ['-0.583333 35.183333'],\n ['-13.816667 -171.916667'],\n ['-13.583333 -172.333333'],\n ['-13.816667 -171.783333'],\n ['-14.045728 -171.414159'],\n ['-13.833333 -171.816667'],\n ['-14.045728 -171.414159'],\n ['-13.883333 -171.55'],\n ['-13.866667 -171.566667'],\n ['-13.816667 -171.783333']]" }, { - "objectID": "units/unit7-bigData.html#stack-overflow-metadata-example", - "href": "units/unit7-bigData.html#stack-overflow-metadata-example", - "title": "Big data and databases", - "section": "Stack Overflow metadata example", - "text": "Stack Overflow metadata example\nI’ve obtained data from Stack Overflow, the popular website for asking coding questions, and placed it into a normalized database. The SQLite version has metadata (i.e., it lacks the actual text of the questions and answers) on all of the questions and answers posted in 2021.\nWe’ll explore SQL functionality using this example database.\nNow let’s consider the Stack Overflow data. Each question may have multiple answers and each question may have multiple (topic) tags.\nIf we tried to put this into a single table, the fields could look like this if we have one row per question:\n\nquestion ID\nID of user submitting question\nquestion title\ntag 1\ntag 2\n…\ntag n\nanswer 1 ID\nID of user submitting answer 1\nage of user submitting answer 1\nname of user submitting answer 1\nanswer 2 ID\nID of user submitting answer 2\nage of user submitting answer 2\nname of user submitting answer 2\n…\n\nor like this if we have one row per question-answer pair:\n\nquestion ID\nID of user submitting question\nquestion title\ntag 1\ntag 2\n…\ntag n\nanswer ID\nID of user submitting answer\nage of user submitting answer\nname of user submitting answer\n\nAs we’ve discussed neither of those schema is particularly desirable.\nChallenge: How would you devise a schema to normalize the data. I.e., what set of tables do you think we should create?\nYou can view one reasonable schema. The lines between tables indicate the relationship of foreign keys in one table to primary keys in another table. The schema in the actual database of Stack Overflow data we’ll use in the examples here is similar to but not identical to that.\nYou can download a copy of the SQLite version of the Stack Overflow 2021 database." + "objectID": "units/unit2-dataTech.html#json", + "href": "units/unit2-dataTech.html#json", + "title": "Data technologies, formats, and structures", + "section": "JSON", + "text": "JSON\nJSON files are structured as “attribute-value” pairs (aka “key-value” pairs), often with a hierarchical structure. Here’s a brief example:\n {\n \"firstName\": \"John\",\n \"lastName\": \"Smith\",\n \"isAlive\": true,\n \"age\": 25,\n \"address\": {\n \"streetAddress\": \"21 2nd Street\",\n \"city\": \"New York\",\n \"state\": \"NY\",\n \"postalCode\": \"10021-3100\"\n },\n \"phoneNumbers\": [\n {\n \"type\": \"home\",\n \"number\": \"212 555-1234\"\n },\n {\n \"type\": \"office\",\n \"number\": \"646 555-4567\"\n }\n ],\n \"children\": [],\n \"spouse\": null\n }\nA set of key-value pairs is a named array and is placed inside braces (squiggly brackets). Note the nestedness of arrays within arrays (e.g., address within the overarching person array and the use of square brackets for unnamed arrays (i.e., vectors of information), as well as the use of different types: character strings, numbers, null, and (not shown) boolean/logical values. JSON and XML can be used in similar ways, but JSON is less verbose than XML.\nWe can read JSON into Python using the json package. Let’s play again with the Kiva data. The same data that we had worked with in XML format is also available in JSON format: https://api.kivaws.org/v1/loans/newest.json.\n\nURL = \"https://api.kivaws.org/v1/loans/newest.json\"\nresponse = requests.get(URL)\n\nimport json\ndata = json.loads(response.text)\ntype(data)\ndata.keys()\n\ntype(data['loans'])\ndata['loans'][0].keys()\n\ndata['loans'][0]['location']['country']\n[loan['location']['country'] for loan in data['loans']]\n\ndict\n\n\ndict_keys(['paging', 'loans'])\n\n\nlist\n\n\ndict_keys(['id', 'name', 'description', 'status', 'funded_amount', 'basket_amount', 'image', 'activity', 'sector', 'use', 'location', 'partner_id', 'posted_date', 'planned_expiration_date', 'loan_amount', 'borrower_count', 'lender_count', 'bonus_credit_eligibility', 'tags'])\n\n\n'Nicaragua'\n\n\n['Nicaragua',\n 'Kyrgyzstan',\n 'Kyrgyzstan',\n 'Kyrgyzstan',\n 'Kyrgyzstan',\n 'Kenya',\n 'Kenya',\n 'Kenya',\n 'Kenya',\n 'Kenya',\n 'Kenya',\n 'Samoa',\n 'Samoa',\n 'Samoa',\n 'Samoa',\n 'Samoa',\n 'Samoa',\n 'Samoa',\n 'Samoa',\n 'Samoa']\n\n\nOne disadvantage of JSON is that it is not set up to deal with missing values, infinity, etc." }, { - "objectID": "units/unit7-bigData.html#accessing-databases-in-python", - "href": "units/unit7-bigData.html#accessing-databases-in-python", - "title": "Big data and databases", - "section": "Accessing databases in Python", - "text": "Accessing databases in Python\nPython provides a variety of front-end packages for manipulating databases from a variety of DBMS (SQLite, DuckDB, MySQL, PostgreSQL, among others). Basically, you start with a bit of code that links to the actual database, and then you can easily query the database using SQL syntax regardless of the back-end. The Python function calls that wrap around the SQL syntax will also look the same regardless of the back-end (basically execute(\"SOME SQL STATEMENT\")).\nWith SQLite, Python processes make calls against the stand-alone SQLite database (.db) file, so there are no SQLite-specific processes. With a client-server DBMS like PostgreSQL, Python processes call out to separate Postgres processes; these are started from the overall Postgres background process\nYou can access and navigate an SQLite database from Python as follows.\n\nimport sqlite3 as sq\ndir_path = '../data' # Replace with the actual path\ndb_filename = 'stackoverflow-2021.db'\n## download from http://www.stat.berkeley.edu/share/paciorek/stackoverflow-2021.db\n\ncon = sq.connect(os.path.join(dir_path, db_filename))\ndb = con.cursor()\ndb.execute(\"select * from questions limit 5\") # simple query \n\n<sqlite3.Cursor object at 0x7ff8707acd40>\n\ndb.fetchall() # retrieve results\n\n[(65534165.0, '2021-01-01 22:15:54', 0.0, 112.0, 2.0, 0.0, None, \"Can't update a value in sqlite3\", 13189393.0), (65535296.0, '2021-01-02 01:33:13', 2.0, 1109.0, 0.0, 0.0, None, 'Install and run ROS on Google Colab', 14924336.0), (65535910.0, '2021-01-02 04:01:34', -1.0, 110.0, 1.0, 8.0, 0.0, 'Operators on date/time fields', 651174.0), (65535916.0, '2021-01-02 04:03:20', 1.0, 35.0, 1.0, 0.0, None, 'Plotting values normalised', 14695007.0), (65536749.0, '2021-01-02 07:03:04', 0.0, 108.0, 1.0, 5.0, None, 'Export C# to word with template', 14899717.0)]\n\n\nAlternatively, we could use DuckDB. However, I don’t have a DuckDB version of the StackOverflow database, so one can’t actually run this code.\n\nimport duckdb as dd\ndir_path = '../data' # Replace with the actual path\ndb_filename = 'stackoverflow-2021.duckdb' # This doesn't exist.\n\ncon = dd.connect(os.path.join(dir_path, db_filename))\ndb = con.cursor()\ndb.execute(\"select * from questions limit 5\") # simple query \ndb.fetchall() # retrieve results\n\nWe can (fairly) easily see the tables (this is easier from R):\n\ndef db_list_tables(db):\n db.execute(\"SELECT name FROM sqlite_master WHERE type='table';\")\n tables = db.fetchall()\n return [table[0] for table in tables]\n\ndb_list_tables(db)\n\n['questions', 'answers', 'questions_tags', 'users']\n\n\nTo see the fields in the table, if you’ve just queried the table, you can look at description:\n\n[item[0] for item in db.description]\n\n['name']\n\ndef get_fields():\n return [item[0] for item in db.description]\n\nHere’s how to make a basic SQL query. One can either make the query and get the results in one go or make the query and separately fetch the results. Here we’ve selected the first five rows (and all columns, based on the * wildcard) and brought them into Python as list of tuples.\n\nresults = db.execute(\"select * from questions limit 5\").fetchall() # simple query \ntype(results)\n\n<class 'list'>\n\ntype(results[0])\n\n<class 'tuple'>\n\nquery = db.execute(\"select * from questions\") # simple query \nresults2 = query.fetchmany(5)\nresults == results2\n\nTrue\n\n\nTo disconnect from the database:\n\ndb.close()\n\nIt’s convenient to get a Pandas dataframe back as the result. To that we can execute queries like this:\n\nimport pandas as pd\nresults = pd.read_sql(\"select * from questions limit 5\", con)" + "objectID": "units/unit2-dataTech.html#webscraping-and-web-apis", + "href": "units/unit2-dataTech.html#webscraping-and-web-apis", + "title": "Data technologies, formats, and structures", + "section": "Webscraping and web APIs", + "text": "Webscraping and web APIs\nHere we’ll see some examples of making requests over the Web to get data. We’ll use APIs to systematically query a website for information. Ideally, but not always, the API will be documented. In many cases that simply amounts to making an HTTP GET request, which is done by constructing a URL.\nThe requests package is useful for a wide variety of such functionality. Note that much of the functionality I describe below is also possible within the shell using either wget or curl.\n\nWebscraping ethics and best practices\nWebscraping is the process of extracting data from the web, either directly from a website or using a web API (application programming interface).\n\nShould you webscrape? In general, if we can avoid webscraping (particularly if there is not an API) and instead directly download a data file from a website, that is greatly preferred.\nMay you webscrape? Before you set up any automated downloading of materials/data from the web you should make sure that what you are about to do is consistent with the rules provided by the website.\n\nSome places to look for information on what the website allows are:\n\nlegal pages such as Terms of Service or Terms and Conditions on the website.\ncheck the robots.txt file (e.g., https://scholar.google.com/robots.txt) to see what a web crawler is allowed to do, and whether the site requires a particular delay between requests to the sites\npotentially contact the site owner if you plan to scrape a large amount of data\n\nHere are some links with useful information:\n\nBlog post on webscraping ethics\nSome information on how to understand a robots.txt file\n\nTips for when you make automated requests:\n\nWhen debugging code that processes the result of such a request, just run the request once, save (i.e., cache) the result, and then work on the processing code applied to the result. Don’t make the same request over and over again.\nIn many cases you will want to include a time delay between your automated requests to a site, including if you are not actually crawling a site but just want to automate a small number of queries.\n\n\n\nWhat is HTTP?\nHTTP (hypertext transfer protocol) is a system for communicating information from a server (i.e., the website of interest) to a client (e.g., your laptop). The client sends a request and the server sends a response.\nWhen you go to a website in a browser, your browser makes an HTTP GET request to the website. Similarly, when we did some downloading of html from webpages above, we used an HTTP GET request.\nAnytime the URL you enter includes parameter information after a question mark (www.somewebsite.com?param1=arg1¶m2=arg2), you are using an API.\nThe response to an HTTP request will include a status code, which can be interpreted based on this information.\nThe response will generally contain content in the form of text (e.g., HTML, XML, JSON) or raw bytes.\n\n\nAPIs: REST- and SOAP-based web services\nIdeally a web service documents their API (Applications Programming Interface) that serves data or allows other interactions. REST and SOAP are popular API standards/styles. Both REST and SOAP use HTTP requests; we’ll focus on REST as it is more common and simpler. When using REST, we access resources, which might be a Facebook account or a database of stock quotes. The API will (hopefully) document what information it expects from the user and will return the result in a standard format (often a particular file format rather than producing a webpage).\nOften the format of the request is a URL (aka an endpoint) plus a query string, passed as a GET request. Let’s search for plumbers near Berkeley, and we’ll see the GET request, in the form:\nhttps://www.yelp.com/search?find_desc=plumbers&find_loc=Berkeley+CA&ns=1\n\nthe query string begins with ?\nthere are one or more Parameter=Argument pairs\npairs are separated by &\n+ is used in place of each space\n\nLet’s see an example of accessing economic data from the World Bank, using the documentation for their API. Following the API call structure, we can download (for example), data on various countries. The documentation indicates that our REST-based query can use either a URL structure or an argument-based structure.\n\n## Queries based on the documentation\napi_url = \"http://api.worldbank.org/V2/incomeLevel/LIC/country\"\napi_args = \"http://api.worldbank.org/V2/country?incomeLevel=LIC\"\n\n## Generalizing a bit\nurl = \"http://api.worldbank.org/V2/country?incomeLevel=MIC&format=json\"\nresponse = requests.get(url)\n\ndata = json.loads(response.content)\n\n## Be careful of data truncation/pagination\nif False:\n url = \"http://api.worldbank.org/V2/country?incomeLevel=MIC&format=json&per_page=1000\"\n response = requests.get(url)\n data = json.loads(response.content)\n\n## Programmatic control\nbaseURL = \"http://api.worldbank.org/V2/country\"\ngroup = 'MIC'\nformat = 'json'\nargs = {'incomeLevel': group, 'format': format, 'per_page': 1000}\nurl = baseURL + '?' + '&'.join(['='.join(\n [key, str(args[key])]) for key in args])\nresponse = requests.get(url)\ndata = json.loads(response.content)\n \ntype(data)\nlen(data[1])\ntype(data[1][5])\ndata[1][5]\n\nlist\n\n\n108\n\n\ndict\n\n\n{'id': 'BEN',\n 'iso2Code': 'BJ',\n 'name': 'Benin',\n 'region': {'id': 'SSF', 'iso2code': 'ZG', 'value': 'Sub-Saharan Africa '},\n 'adminregion': {'id': 'SSA',\n 'iso2code': 'ZF',\n 'value': 'Sub-Saharan Africa (excluding high income)'},\n 'incomeLevel': {'id': 'LMC',\n 'iso2code': 'XN',\n 'value': 'Lower middle income'},\n 'lendingType': {'id': 'IDX', 'iso2code': 'XI', 'value': 'IDA'},\n 'capitalCity': 'Porto-Novo',\n 'longitude': '2.6323',\n 'latitude': '6.4779'}\n\n\nAPIs can change and disappear. A few years ago, the example above involved the World Bank’s Climate Data API, which I can no longer find!\nAs another example, here we can see the US Treasury Department API, which allows us to construct queries for federal financial data.\nThe Nolan and Temple Lang book provides a number of examples of different ways of authenticating with web services that control access to the service.\nFinally, some web services allow us to pass information to the service in addition to just getting data or information. E.g., you can programmatically interact with your Facebook, Dropbox, and Google Drive accounts using REST based on HTTP POST, PUT, and DELETE requests. Authentication is of course important in these contexts and some times you would first authenticate with your login and password and receive a “token”. This token would then be used in subsequent interactions in the same session.\nI created your github.berkeley.edu accounts from Python by interacting with the GitHub API using requests.\n\n\nHTTP requests by deconstructing an (undocumented) API\nIn some cases an API may not be documented or we might be lazy and not use the documentation. Instead we might deconstruct the queries a browser makes and then mimic that behavior, in some cases having to parse HTML output to get at data. Note that if the webpage changes even a little bit, our carefully constructed query syntax may fail.\nLet’s look at some UN data (agricultural crop data). By going to\nhttp://data.un.org/Explorer.aspx?d=FAO, and clicking on “Crops”, we’ll see a bunch of agricultural products with “View data” links. Click on “apricots” as an example and you’ll see a “Download” button that allows you to download a CSV of the data. Let’s select a range of years and then try to download “by hand”. Sometimes we can right-click on the link that will download the data and directly see the URL that is being accessed and then one can deconstruct it so that you can create URLs programmatically to download the data you want.\nIn this case, we can’t see the full URL that is being used because there’s some Javascript involved. Therefore, rather than looking at the URL associated with a link we need to view the actual HTTP request sent by our browser to the server. We can do this using features of the browser (e.g., in Firefox see Web Developer -> Network and in Chrome View -> Developer -> Developer tools and choose the Network tab) (or right-click on the webpage and select Inspect and then Network). Based on this we can see that an HTTP GET request is being used with a URL such as:\nhttp://data.un.org/Handlers/DownloadHandler.ashx?DataFilter=itemCode:526;year:2012,2013,2014,2015,2016,2017&DataMartId=FAO&Format=csv&c=2,4,5,6,7&s=countryName:asc,elementCode:asc,year:desc.\nWe’e now able to easily download the data using that URL, which we can fairly easily construct using string processing in bash, Python, or R, such as this (here I just paste it together directly, but using more structured syntax such as I used for the World Bank example would be better):\n\nimport zipfile\n\n## example URL:\n## http://data.un.org/Handlers/DownloadHandler.ashx?DataFilter=itemCode:526;\n##year:2012,2013,2014,2015,2016,2017&DataMartId=FAO&Format=csv&c=2,4,5,6,7&\n##s=countryName:asc,elementCode:asc,year:desc\nitemCode = 526\nbaseURL = \"http://data.un.org/Handlers/DownloadHandler.ashx\"\nyrs = ','.join([str(yr) for yr in range(2012,2018)])\nfilter = f\"?DataFilter=itemCode:{itemCode};year:{yrs}\"\nargs1 = \"&DataMartId=FAO&Format=csv&c=2,3,4,5,6,7&\"\nargs2 = \"s=countryName:asc,elementCode:asc,year:desc\"\nurl = baseURL + filter + args1 + args2\n## If the website provided a CSV, this would be easier, but it zips the file.\nresponse = requests.get(url)\n\nwith io.BytesIO(response.content) as stream: # create a file-like object\n with zipfile.ZipFile(stream, 'r') as archive: # treat the object as a zip file\n with archive.open(archive.filelist[0].filename, 'r') as file: # get a pointer to the embedded file\n dat = pd.read_csv(file)\n\ndat.head()\n\n/usr/local/linux/mambaforge-3.11/lib/python3.11/site-packages/IPython/core/formatters.py:342: FutureWarning:\n\nIn future versions `DataFrame.to_latex` is expected to utilise the base implementation of `Styler.to_latex` for formatting and rendering. The arguments signature may therefore change. It is recommended instead to use `DataFrame.style.to_latex` which also contains additional functionality.\n\n\n\n\n\n\n\n\n\n\nCountry or Area\nElement Code\nElement\nYear\nUnit\nValue\nValue Footnotes\n\n\n\n\n0\nAfghanistan\n432\nGross Production Index Number (2014-2016 = 100)\n2017.0\nindex\n202.19\nFc\n\n\n1\nAfghanistan\n432\nGross Production Index Number (2014-2016 = 100)\n2016.0\nindex\n27.45\nFc\n\n\n2\nAfghanistan\n432\nGross Production Index Number (2014-2016 = 100)\n2015.0\nindex\n134.50\nFc\n\n\n3\nAfghanistan\n432\nGross Production Index Number (2014-2016 = 100)\n2014.0\nindex\n138.05\nFc\n\n\n4\nAfghanistan\n432\nGross Production Index Number (2014-2016 = 100)\n2013.0\nindex\n138.05\nFc\n\n\n\n\n\n\n\nSo, what have we achieved?\n\nWe have a reproducible workflow we can share with others (perhaps ourself in the future).\nWe can automate the process of downloading many such files.\n\n\n\nMore details on HTTP requests\nA more sophisticated way to do the download is to pass the request in a structured way with named input parameters. This request is easier to construct programmatically. Here what is returned is a zip file, which is represented in Python as a sequence of “raw” bytes.\n\ndata = {\"DataFilter\": f\"itemCode:{itemCode};year:{yrs}\",\n \"DataMartID\": \"FAO\", \n \"Format\": \"csv\", \n \"c\": \"2,3,4,5,6,7\",\n \"s\": \"countryName:asc,elementCode:asc,year:desc\"\n } \n\nresponse = requests.get(baseURL, params = data)\n\nwith io.BytesIO(response.content) as stream: \n with zipfile.ZipFile(stream, 'r') as archive:\n with archive.open(archive.filelist[0].filename, 'r') as file: \n dat = pd.read_csv(file)\n\nIn some cases we may need to send a lot of information as part of the URL in a GET request. If it gets to be too long (e.g,, more than 2048 characters) many web servers will reject the request. Instead we may need to use an HTTP POST request (POST requests are often used for submitting web forms). A typical request would have syntax like this search (using requests):\n\nurl = 'http://www.wormbase.org/db/searches/advanced/dumper'\n\ndata = { \"specipes\":\"briggsae\",\n \"list\": \"\",\n \"flank3\": \"0\",\n \"flank5\": \"0\",\n \"feature\": \"Gene Models\",\n \"dump\": \"Plain TEXT\",\n \"orientation\": \"Relative to feature\",\n \"relative\": \"Chromsome\",\n \"DNA\":\"flanking sequences only\",\n \".cgifields\" : \"feature, orientation, DNA, dump, relative\"\n} \n\nresponse = requests.post(url, data = data)\nif response.status_code == 200:\n print(\"POST request successful\")\nelse:\n print(f\"POST request failed with status code: {response.status_code}\")\n\nUnfortunately that specific search doesn’t work because the server URL and/or API seem to have changed. But it gives you an idea of what the format would look like.\nrequests can handle other kinds of HTTP requests such as PUT and DELETE. Finally, some websites use cookies to keep track of users, and you may need to download a cookie in the first interaction with the HTTP server and then send that cookie with later interactions. More details are available in the Nolan and Temple Lang book.\n\n\nPackaged access to an API\nFor popular websites/data sources, a developer may have packaged up the API calls in a user-friendly fashion for use from Python, R, or other software. For example there are Python (twitter) and R (twitteR) packages for interfacing with Twitter via its API.\nHere’s some example code for Python. This looks up the US senators’ Twitter names and then downloads a portion of each of their timelines, i.e., the time series of their tweets. Note that Twitter has limits on how much one can download at once.\n\nimport json\nimport twitter\n\n# You will need to set the following variables with your\n# personal information. To do this you will need to create\n# a personal account on Twitter (if you don't already have\n# one). Once you've created an account, create a new\n# application here:\n# https://dev.twitter.com/apps\n#\n# You can manage your applications here:\n# https://apps.twitter.com/\n#\n# Select your application and then under the section labeled\n# \"Key and Access Tokens\", you will find the information needed\n# below. Keep this information private.\nCONSUMER_KEY = \"\"\nCONSUMER_SECRET = \"\"\nOAUTH_TOKEN = \"\"\nOAUTH_TOKEN_SECRET = \"\"\n\nauth = twitter.oauth.OAuth(OAUTH_TOKEN, OAUTH_TOKEN_SECRET,\n CONSUMER_KEY, CONSUMER_SECRET)\napi = twitter.Twitter(auth=auth)\n\n# get the list of senators\nsenators = api.lists.members(owner_screen_name=\"gov\",\n slug=\"us-senate\", count=100)\n\n# get all the senators' timelines\nnames = [d[\"screen_name\"] for d in senators[\"users\"]]\ntimelines = [api.statuses.user_timeline(screen_name=name, count = 500) \n for name in names]\n\n# save information out to JSON\nwith open(\"senators-list.json\", \"w\") as f:\n json.dump(senators, f, indent=4, sort_keys=True)\nwith open(\"timelines.json\", \"w\") as f:\n json.dump(timelines, f, indent=4, sort_keys=True)\n\n\n\nAccessing dynamic pages\nSome websites dynamically change in reaction to the user behavior. In these cases you need a tool that can mimic the behavior of a human interacting with a site. Some options are:\n\nselenium is a popular tool for doing this, and there is a Python package of the same name.\nUsing scrapy plus splash is another approach." }, { - "objectID": "units/unit7-bigData.html#basic-sql-for-choosing-rows-and-columns-from-a-table", - "href": "units/unit7-bigData.html#basic-sql-for-choosing-rows-and-columns-from-a-table", - "title": "Big data and databases", - "section": "Basic SQL for choosing rows and columns from a table", - "text": "Basic SQL for choosing rows and columns from a table\nSQL is a declarative language that tells the database system what results you want. The system then parses the SQL syntax and determines how to implement the query.\n\nNote: An imperative language is one where you provide the sequence of commands you want to be run, in order. A declarative language is one where you declare what result you want and rely on the system that interprets the commands to determine how to actually do it. Most of the languages we’re generally familiar with are imperative. (That said, even in languages like Python, function calls in many ways simply say what we want rather than exactly how the computer should carry out the granular operations.)\n\nHere are some examples using the Stack Overflow database of getting questions that have been viewed a lot (the viewcount field is large).\n\n## Get the questions (* indicates all fields) for which the viewcount field is large.\ndb.execute('select * from questions where viewcount > 100000').fetchall()\n\n## Find the 10 largest viewcounts (and associated titles) in the questions table,\n## by sorting in descending order based on viewcount and returning the first 10.\n\n[(65547199.0, '2021-01-03 06:22:52', 124.0, 110832.0, 7.0, 2.0, 0.0, 'Using Bootstrap 5 with Vue 3', 11232893.0), (65549858.0, '2021-01-03 12:30:19', 52.0, 130479.0, 11.0, 0.0, 0.0, '\"ERESOLVE unable to resolve dependency tree\" when installing npm react-facebook-login', 12425004.0), (65630743.0, '2021-01-08 14:20:57', 77.0, 107140.0, 19.0, 4.0, 0.0, 'How to solve flutter web api cors error only with dart code?', 12373446.0), (65632698.0, '2021-01-08 16:22:59', 74.0, 101044.0, 9.0, 1.0, 0.0, 'How to open a link in a new Tab in NextJS?', 9578961.0), (65896334.0, '2021-01-26 05:33:33', 111.0, 141899.0, 12.0, 7.0, 0.0, 'Python Pip broken with sys.stderr.write(f\"ERROR: {exc}\")', 202576.0), (65908987.0, '2021-01-26 20:42:25', 238.0, 215399.0, 9.0, 1.0, 0.0, \"How can I open Visual Studio Code's 'settings.json' file?\", 793320.0), (65980952.0, '2021-01-31 15:36:21', 22.0, 141857.0, 10.0, 1.0, 0.0, 'Python: Could not install packages due to an OSError: [Errno 2] No such file or directory', 14489450.0), (66020820.0, '2021-02-03 03:27:19', 161.0, 174829.0, 3.0, 0.0, 0.0, 'npm: When to use `--force` and `--legacy-peer-deps`', 7824245.0), (66029781.0, '2021-02-03 14:41:21', 20.0, 107446.0, 6.0, 1.0, 0.0, 'Appcenter iOS install error \"this app cannot be installed because its integrity could not be verified\"', 462440.0), (66060487.0, '2021-02-05 09:11:48', 188.0, 241216.0, 19.0, 0.0, 0.0, 'ValueError: numpy.ndarray size changed, may indicate binary incompatibility. Expected 88 from C header, got 80 from PyObject', 15150567.0), (66082397.0, '2021-02-06 21:58:06', 333.0, 315861.0, 17.0, 1.0, 0.0, 'TypeError: this.getOptions is not a function', 14337399.0), (66146088.0, '2021-02-10 22:32:45', 152.0, 175983.0, 17.0, 2.0, 0.0, 'Docker - failed to compute cache key: not found - runs fine in Visual Studio', 7419676.0), (66231282.0, '2021-02-16 19:54:53', 70.0, 116757.0, 10.0, 0.0, 0.0, 'How to add a GitHub personal access token to Visual Studio Code', 1186050.0), (66239691.0, '2021-02-17 10:03:55', 323.0, 261165.0, 6.0, 5.0, 0.0, \"What does npm install --legacy-peer-deps do exactly? When is it recommended / What's a potential use case?\", 15093141.0), (66252333.0, '2021-02-18 01:32:39', 21.0, 127736.0, 7.0, 0.0, 0.0, 'ERROR NullInjectorError: R3InjectorError(AppModule)', 14629851.0), (66366582.0, '2021-02-25 10:15:20', 90.0, 123630.0, 18.0, 5.0, 0.0, 'Github - unexpected disconnect while reading sideband packet', 11839478.0), (66597544.0, '2021-03-12 09:44:52', 29.0, 100293.0, 25.0, 4.0, 0.0, \"ENOENT: no such file or directory, lstat '/Users/Desktop/node_modules'\", 15124187.0), (66629862.0, '2021-03-14 21:43:50', 285.0, 117306.0, 9.0, 9.0, 0.0, \"Cannot determine the organization name for this 'dev.azure.com' remote url\", 11124332.0), (66662820.0, '2021-03-16 20:19:44', 104.0, 129848.0, 11.0, 0.0, 0.0, \"M1 docker preview and keycloak 'image's platform (linux/amd64) does not match the detected host platform (linux/arm64/v8)' Issue\", 4100000.0), (66666134.0, '2021-03-17 02:24:45', 159.0, 273201.0, 11.0, 2.0, 0.0, 'How to install homebrew on M1 mac', 15411878.0), (66801256.0, '2021-03-25 14:10:21', 143.0, 210144.0, 3.0, 8.0, 0.0, 'java.lang.IllegalAccessError: class lombok.javac.apt.LombokProcessor cannot access class com.sun.tools.javac.processing.JavacProcessingEnvironment', 12586904.0), (66835173.0, '2021-03-27 19:11:42', 158.0, 286886.0, 16.0, 0.0, 0.0, 'How to change background color of Elevated Button in Flutter from function?', 13628530.0), (66894200.0, '2021-03-31 19:33:26', 161.0, 263037.0, 13.0, 4.0, 0.0, 'Error message \"go: go.mod file not found in current directory or any parent directory; see \\'go help modules\\'\"', 4159198.0), (66964492.0, '2021-04-06 07:34:31', 40.0, 146006.0, 15.0, 2.0, 0.0, \"ImportError: cannot import name 'get_config' from 'tensorflow.python.eager.context'\", 5111234.0), (66980512.0, '2021-04-07 06:17:01', 739.0, 445775.0, 41.0, 0.0, 0.0, 'Android Studio Error \"Android Gradle plugin requires Java 11 to run. You are currently using Java 1.8\"', 11899911.0), (66989383.0, '2021-04-07 15:35:14', 54.0, 104476.0, 5.0, 0.0, 0.0, 'Could not resolve dependency: npm ERR! peer @angular/compiler@\"11.2.8\"', 12380096.0), (66992420.0, '2021-04-07 18:51:41', 86.0, 103193.0, 12.0, 1.0, 0.0, 'when I try to \"sync project with gradle files\" a warning pops up', 15576934.0), (67001968.0, '2021-04-08 10:19:41', 153.0, 141177.0, 17.0, 8.0, 0.0, 'How to disable maven blocking external HTTP repositories?', 5428154.0), (67045607.0, '2021-04-11 13:34:53', 204.0, 125367.0, 18.0, 1.0, 0.0, 'How to resolve \"Missing PendingIntent mutability flag\" lint warning in android api 30+?', 2652368.0), (67079327.0, '2021-04-13 17:02:54', 151.0, 278432.0, 13.0, 7.0, 0.0, 'How can I fix \"unsupported class file major version 60\" in IntelliJ IDEA?', 32914.0), (67191286.0, '2021-04-21 07:37:43', 142.0, 610026.0, 44.0, 8.0, 0.0, 'Crbug/1173575, non-JS module files deprecated. chromewebdata/(index)꞉5305:9:5551', 8732988.0), (67201708.0, '2021-04-21 18:39:28', 86.0, 105762.0, 2.0, 0.0, 0.0, 'Go update all modules', 1002260.0), (67246010.0, '2021-04-24 18:12:35', 41.0, 119121.0, 8.0, 0.0, 0.0, 'Error message \"The server selected protocol version TLS10 is not accepted by client preferences\"', 2153306.0), (67346232.0, '2021-05-01 12:23:44', 207.0, 276806.0, 60.0, 3.0, 0.0, 'Android Emulator issues in new versions - The emulator process has terminated', 13546747.0), (67352418.0, '2021-05-02 02:11:58', 74.0, 144925.0, 8.0, 1.0, 0.0, 'How to add SCSS styles to a React project?', 10836598.0), (67399785.0, '2021-05-05 10:45:12', 47.0, 151502.0, 12.0, 2.0, 0.0, 'How to solve npm install error “npm ERR! code 1”', 15841778.0), (67412084.0, '2021-05-06 05:00:27', 251.0, 286810.0, 14.0, 11.0, 0.0, 'Android Studio error: \"Manifest merger failed: Apps targeting Android 12\"', 15150212.0), (67440510.0, '2021-05-07 19:12:35', 10.0, 165243.0, 10.0, 2.0, 0.0, \"cv2.error: OpenCV(4.5.2) .error: (-215:Assertion failed) !_src.empty() in function 'cv::cvtColor'\", 15866017.0), (67448034.0, '2021-05-08 13:21:36', 180.0, 214970.0, 34.0, 4.0, 0.0, '\"Module was compiled with an incompatible version of Kotlin. The binary version of its metadata is 1.5.1, expected version is 1.1.16\"', 14099703.0), (67501093.0, '2021-05-12 09:42:06', 47.0, 149690.0, 5.0, 4.0, 0.0, 'Passthrough is not supported, GL is disabled', 15519827.0), (67505347.0, '2021-05-12 14:11:11', 165.0, 105852.0, 12.0, 5.0, 0.0, 'Non-nullable property must contain a non-null value when exiting constructor. Consider declaring the property as nullable', 1977871.0), (67507452.0, '2021-05-12 16:23:28', 89.0, 135460.0, 16.0, 0.0, 0.0, 'No spring.config.import property has been defined', 15816596.0), (67698176.0, '2021-05-26 03:33:11', 134.0, 119615.0, 14.0, 0.0, 0.0, 'Error loading webview: Error: Could not register service workers: TypeError: Failed to register a ServiceWorker for scope', 7148467.0), (67699823.0, '2021-05-26 06:45:07', 204.0, 178718.0, 21.0, 3.0, 0.0, 'Module was compiled with an incompatible version of Kotlin. The binary version of its metadata is 1.5.1, expected version is 1.1.15', 11576007.0), (67782975.0, '2021-06-01 05:03:24', 69.0, 114886.0, 15.0, 0.0, 0.0, 'How to fix the \\'\\'module java.base does not \"opens java.io\" to unnamed module \\'\\' error in Android Studio?', 14620854.0), (67899129.0, '2021-06-09 07:03:22', 35.0, 111003.0, 6.0, 0.0, 0.0, 'Postfix and OpenJDK 11: \"No appropriate protocol (protocol is disabled or cipher suites are inappropriate)\"', 1465758.0), (67900692.0, '2021-06-09 08:49:35', 259.0, 121306.0, 25.0, 3.0, 0.0, 'Latest version of Xcode stuck on installation (12.5)', 8612435.0), (68166721.0, '2021-06-28 16:12:46', 34.0, 116051.0, 8.0, 1.0, 0.0, 'CUDA error: device-side assert triggered on Colab', 5080195.0), (68191392.0, '2021-06-30 08:45:37', 242.0, 121764.0, 25.0, 22.0, 0.0, 'Password authentication is temporarily disabled as part of a brownout. Please use a personal access token instead', 15507251.0), (68236007.0, '2021-07-03 11:50:25', 318.0, 303399.0, 20.0, 1.0, 0.0, 'I am getting error \"cmdline-tools component is missing\" after installing Flutter and Android Studio... I added the Android SDK. How can I solve them?', 11993020.0), (68260784.0, '2021-07-05 18:40:58', 117.0, 219591.0, 8.0, 0.0, 0.0, 'npm WARN old lockfile The package-lock.json file was created with an old version of npm', 12530530.0), (68387270.0, '2021-07-15 02:45:41', 464.0, 426798.0, 34.0, 8.0, 0.0, 'Android Studio error \"Installed Build Tools revision 31.0.0 is corrupted\"', 11957368.0), (68397062.0, '2021-07-15 15:55:59', 50.0, 108364.0, 8.0, 1.0, 0.0, 'Could not initialize class org.apache.maven.plugin.war.util.WebappStructureSerializer\\t-Maven Configuration Problem Any solution?', 16456713.0), (68486207.0, '2021-07-22 14:02:27', 54.0, 130114.0, 8.0, 0.0, 0.0, 'Import could not be resolved/could not be resolved from source Pylance in VS Code using Python 3.9.2 on Windows 10', 14132348.0), (68554294.0, '2021-07-28 04:09:31', 188.0, 154478.0, 32.0, 24.0, 0.0, 'android:exported needs to be explicitly specified for <activity>. Apps targeting Android 12 and higher are required to specify', 14280831.0), (68673221.0, '2021-08-05 20:37:54', 62.0, 103200.0, 4.0, 4.0, 0.0, \"WARNING: Running pip as the 'root' user\", 15037284.0), (68775869.0, '2021-08-13 16:49:34', 1286.0, 1236876.0, 47.0, 18.0, 0.0, 'Message \"Support for password authentication was removed. Please use a personal access token instead.\"', 15573670.0), (68836551.0, '2021-08-18 17:01:31', 53.0, 117984.0, 9.0, 1.0, 0.0, \"Keras AttributeError: 'Sequential' object has no attribute 'predict_classes'\", 10377186.0), (68857411.0, '2021-08-20 05:33:34', 41.0, 122460.0, 4.0, 3.0, 0.0, 'npm WARN deprecated tar@2.2.2: This version of tar is no longer supported, and will not receive security updates. Please upgrade asap', 14930713.0), (68958221.0, '2021-08-27 19:00:39', 80.0, 111679.0, 22.0, 3.0, 0.0, 'MongoParseError: options useCreateIndex, useFindAndModify are not supported', 12459536.0), (68959632.0, '2021-08-27 21:43:39', 35.0, 505992.0, 4.0, 2.0, 0.0, \"TypeError: Cannot read properties of undefined (reading 'id')\", 16261380.0), (69033022.0, '2021-09-02 15:18:46', 96.0, 114598.0, 30.0, 10.0, 0.0, 'Message \"error: resource android:attr/lStar not found\"', 16813382.0), (69034879.0, '2021-09-02 17:36:44', 218.0, 307961.0, 8.0, 1.0, 0.0, 'How can I resolve the error \"The minCompileSdk (31) specified in a dependency\\'s AAR metadata\" in native Java or Kotlin?', 8359705.0), (69041454.0, '2021-09-03 08:01:10', 91.0, 102261.0, 9.0, 11.0, 0.0, 'Error: require() of ES modules is not supported when importing node-fetch', 16821219.0), (69080597.0, '2021-09-06 21:56:50', 58.0, 458856.0, 22.0, 4.0, 0.0, \"× TypeError: Cannot read properties of undefined (reading 'map')\", 16846583.0), (69081410.0, '2021-09-07 00:51:50', 134.0, 296364.0, 3.0, 2.0, 0.0, 'Error [ERR_REQUIRE_ESM]: require() of ES Module not supported', 16847125.0), (69139074.0, '2021-09-11 00:12:49', 8.0, 109316.0, 3.0, 0.0, 0.0, \"ERROR TypeError: Cannot read properties of undefined (reading 'title')\", 16723200.0), (69163511.0, '2021-09-13 13:25:07', 224.0, 136939.0, 13.0, 0.0, 0.0, \"Build was configured to prefer settings repositories over project repositories but repository 'maven' was added by build file 'build.gradle'\", 12886431.0), (69390676.0, '2021-09-30 10:34:46', 121.0, 121388.0, 14.0, 2.0, 0.0, 'How to use appsettings.json in Asp.net core 6 Program.cs file', 10336618.0), (69394001.0, '2021-09-30 14:28:21', 34.0, 117452.0, 10.0, 1.0, 0.0, 'How to fix? \"kex_exchange_identification: read: Connection reset by peer\"', 8939187.0), (69394632.0, '2021-09-30 15:07:50', 249.0, 242414.0, 12.0, 1.0, 0.0, 'Webpack build failing with ERR_OSSL_EVP_UNSUPPORTED', 17044429.0), (69564817.0, '2021-10-14 03:41:23', 48.0, 100499.0, 4.0, 0.0, 0.0, \"TypeError: load() missing 1 required positional argument: 'Loader' in Google Colab\", 17147261.0), (69567381.0, '2021-10-14 08:22:53', 102.0, 132221.0, 16.0, 1.0, 0.0, 'Getting \"Cannot read property \\'pickAlgorithm\\' of null\" error in react native', 15269749.0), (69665222.0, '2021-10-21 16:00:47', 90.0, 165387.0, 13.0, 0.0, 0.0, 'Node.js 17.0.1 Gatsby error - \"digital envelope routines::unsupported ... ERR_OSSL_EVP_UNSUPPORTED\"', 7002673.0), (69692842.0, '2021-10-23 23:39:57', 843.0, 816368.0, 43.0, 10.0, 0.0, 'Error message \"error:0308010C:digital envelope routines::unsupported\"', 14994086.0), (69722872.0, '2021-10-26 12:10:04', 184.0, 130738.0, 10.0, 1.0, 0.0, 'ASP.NET Core 6 how to access Configuration during startup', 1977871.0), (69773547.0, '2021-10-29 18:49:01', 62.0, 108232.0, 14.0, 1.0, 0.0, 'Visual Studio 2019 Not Showing .NET 6 Framework', 1407658.0), (69832748.0, '2021-11-03 23:06:48', 98.0, 140728.0, 16.0, 0.0, 0.0, 'Error \"Error: A <Route> is only ever to be used as the child of <Routes> element\"', 13149387.0), (69843615.0, '2021-11-04 17:44:31', 72.0, 102942.0, 6.0, 0.0, 0.0, \"Switch' is not exported from 'react-router-dom'\", 8467488.0), (69854011.0, '2021-11-05 13:26:59', 137.0, 107635.0, 7.0, 1.0, 0.0, 'Matched leaf route at location \"/\" does not have an element', 16102215.0), (69864165.0, '2021-11-06 12:55:13', 160.0, 149243.0, 18.0, 0.0, 0.0, 'Error: [PrivateRoute] is not a <Route> component. All component children of <Routes> must be a <Route> or <React.Fragment>', 16830299.0), (69868956.0, '2021-11-07 00:34:29', 145.0, 221716.0, 9.0, 4.0, 0.0, 'How can I redirect in React Router v6?', 2000548.0), (69875125.0, '2021-11-07 17:53:49', 78.0, 105208.0, 4.0, 1.0, 0.0, 'find_element_by_* commands are deprecated in Selenium', 17351258.0), (69875520.0, '2021-11-07 18:45:12', 150.0, 180541.0, 13.0, 1.0, 0.0, 'Unable to negotiate with 40.74.28.9 port 22: no matching host key type found. Their offer: ssh-rsa', 7122272.0), (70000324.0, '2021-11-17 07:23:39', 120.0, 112536.0, 3.0, 0.0, 0.0, 'What is \"crypt key missing\" error in Pgadmin4 and how to resolve it?', 10279487.0), (70036953.0, '2021-11-19 15:09:03', 90.0, 103677.0, 14.0, 5.0, 0.0, \"Spring Boot 2.6.0 / Spring fox 3 - Failed to start bean 'documentationPluginsBootstrapper'\", 306436.0), (70281346.0, '2021-12-08 20:29:31', 150.0, 155000.0, 12.0, 5.0, 0.0, 'Node.js Sass version 7.0.0 is incompatible with ^4.0.0 || ^5.0.0 || ^6.0.0', 13765920.0), (70319606.0, '2021-12-11 22:44:15', 115.0, 110984.0, 5.0, 0.0, 0.0, \"ImportError: cannot import name 'url' from 'django.conf.urls' after upgrading to Django 4.0\", 113962.0), (70358643.0, '2021-12-15 04:58:02', 233.0, 107966.0, 6.0, 4.0, 0.0, '\"You are running create-react-app 4.0.3 which is behind the latest release (5.0.0)\"', 14426381.0), (70368760.0, '2021-12-15 18:37:56', 163.0, 164918.0, 21.0, 2.0, 0.0, 'React Uncaught ReferenceError: process is not defined', 14880787.0), (70538793.0, '2021-12-31 03:41:36', 40.0, 103218.0, 9.0, 1.0, 0.0, 'remote: Write access to repository not granted. fatal: unable to access', 10781286.0)]\n\ndb.execute(\n'select title, viewcount from questions order by viewcount desc limit 10').fetchall()\n\n[('Message \"Support for password authentication was removed. Please use a personal access token instead.\"', 1236876.0), ('Error message \"error:0308010C:digital envelope routines::unsupported\"', 816368.0), ('Crbug/1173575, non-JS module files deprecated. chromewebdata/(index)꞉5305:9:5551', 610026.0), (\"TypeError: Cannot read properties of undefined (reading 'id')\", 505992.0), (\"× TypeError: Cannot read properties of undefined (reading 'map')\", 458856.0), ('Android Studio Error \"Android Gradle plugin requires Java 11 to run. You are currently using Java 1.8\"', 445775.0), ('Android Studio error \"Installed Build Tools revision 31.0.0 is corrupted\"', 426798.0), ('TypeError: this.getOptions is not a function', 315861.0), ('How can I resolve the error \"The minCompileSdk (31) specified in a dependency\\'s AAR metadata\" in native Java or Kotlin?', 307961.0), ('I am getting error \"cmdline-tools component is missing\" after installing Flutter and Android Studio... I added the Android SDK. How can I solve them?', 303399.0)]\n\n\nLet’s lay out the various verbs in SQL. Here’s the form of a standard query (though the ORDER BY is often omitted and sorting is computationally expensive):\nSELECT <column(s)> FROM <table> WHERE <condition(s) on column(s)> ORDER BY <column(s)>\nSQL keywords are often written in ALL CAPITALS, although I won’t necessarily do that in this document.\nAnd here is a table of some important keywords:\n\n\n\n\n\n\n\nKeyword\nUsage\n\n\n\n\nSELECT\nselect columns\n\n\nFROM\nwhich table to operate on\n\n\nWHERE\nfilter (choose) rows satisfying certain conditions\n\n\nLIKE, IN, <, >, ==, etc.\nused as part of conditions\n\n\nORDER BY\nsort based on columns\n\n\n\nFor logical comparisons in a WHERE clause, some common syntax for setting conditions includes LIKE (for patterns), =, >, <, >=, <=, !=.\nSome other keywords are: DISTINCT, ON, JOIN, GROUP BY, AS, USING, UNION, INTERSECT, SIMILAR TO.\nQuestion: how would we find the oldest users in the database?" + "objectID": "units/unit2-dataTech.html#standard-data-structures-in-python-and-r", + "href": "units/unit2-dataTech.html#standard-data-structures-in-python-and-r", + "title": "Data technologies, formats, and structures", + "section": "Standard data structures in Python and R", + "text": "Standard data structures in Python and R\n\nIn Python and R, one often ends up working with dataframes, lists, and arrays/vectors/matrices/tensors.\nIn Python we commonly work with data structures that are part of additional packages, in particular numpy arrays and pandas dataframes.\nDictionaries in Python allow for easy use of key-value pairs where one can access values based on their key/label. In R one can do something similar with named vectors or named lists or (more efficiently) by using environments.\nIn R, if we are not working with rectangular datasets or standard numerical objects, we often end up using lists or enhanced versions of lists, sometimes with deeply nested structures.\n\nIn Unit 7, we’ll talk about distributed data structures that allow one to easily work with data distributed across multiple computers." }, { - "objectID": "units/unit7-bigData.html#grouping-stratifying", - "href": "units/unit7-bigData.html#grouping-stratifying", - "title": "Big data and databases", - "section": "Grouping / stratifying", - "text": "Grouping / stratifying\nA common pattern of operation is to stratify the dataset, i.e., collect it into mutually exclusive and exhaustive subsets. One would then generally do some (reduction) operation on each subset (e.g., counting records, calculating the mean of a column, taking the max of a column). In SQL this is done with the GROUP BY keyword.\nThe basic syntax looks like this:\nSELECT <reduction_operation>(<column(s)>) FROM <table> GROUP BY <column(s)>\nHere’s a basic example where we count the occurrences of different tags. Note that we use as to define a name for the new column that is created based on the aggregation operation (count in this case).\n\ndb.execute(\"select tag, count(*) as n from questions_tags \\\n group by tag \\\n order by n desc limit 25\").fetchall()\n\n[('python', 255614), ('javascript', 182006), ('java', 89097), ('reactjs', 83180), ('html', 69401), ('c#', 67633), ('android', 55422), ('r', 51688), ('node.js', 50231), ('php', 48782), ('css', 48021), ('c++', 46267), ('pandas', 45862), ('sql', 43598), ('python-3.x', 42014), ('flutter', 39243), ('typescript', 33583), ('arrays', 29960), ('angular', 29783), ('django', 29228), ('mysql', 26562), ('dataframe', 25283), ('c', 24965), ('json', 24510), ('swift', 23008)]\n\n\nIn general GROUP BY statements will involve some aggregation operation on the subsets. Options include: COUNT, MIN, MAX, AVG, SUM. The number of results will be the same as the number of groups; in the example above there should be one result per tag.\nIf you filter after using GROUP BY, you need to use having instead of where.\nChallenge: Write a query that will count the number of answers for each question, returning the most answered questions." + "objectID": "units/unit2-dataTech.html#other-kinds-of-data-structures", + "href": "units/unit2-dataTech.html#other-kinds-of-data-structures", + "title": "Data technologies, formats, and structures", + "section": "Other kinds of data structures", + "text": "Other kinds of data structures\nYou may have heard of various other kinds of data structures, such as linked lists, trees, graphs, queues, and stacks. One of the key aspects that differentiate such data structures is how one navigates through the elements.\nSets are collections of elements that don’t have any duplicates (like a mathematical set).\nWith a linked list, with each element (or node) has a value and a pointer (reference) to the location of the next element. (With a doubly-linked list, there is also a pointer back to the previous element.) One big advantage of this is that one can insert an element by simply modifying the pointers involved at the site of the insertion, without copying any of the other elements in the list. A big disadvantage is that to get to an element you have to navigate through the list.\n\n\n\nLinked list (courtesy of computersciencewiki.org)\n\n\nBoth trees and graphs are collections of nodes (vertices) and links (edges). A tree involves a set of nodes and links to child nodes (also possibly containing information linking the child nodes to their parent nodes). With a graph, the links might not be directional, and there can be cycles.\n\n\n\nTree (courtesy of computersciencewiki.org)\n\n\n\n\n\nGraph (courtesy of computersciencewiki.org)\n\n\nA stack is a collection of elements that behave like a stack of lunch trays. You can only access the top element directly(“last in, first out”), so the operations are that you can push a new element onto the stack or pop the top element off the stack. In fact, nested function calls behave as stacks, and the memory used in the process of evaluating the function calls is called the ‘stack’.\nA queue is like the line at a grocery store, behaving as “first in, first out”.\nOne can use such data structures either directly or via add-on packages in Python and R, though I don’t think they’re all that commonly used in R. This is probably because statistical/data science/machine learning workflows often involve either ‘rectangular’ data (i.e., dataframe-style data) and/or mathematical computations with arrays. That said, trees and graphs are widely used.\nSome related concepts that we’ll discuss further in Unit 5 include:\n\ntypes: this refers to how a given piece of information is stored and what operations can be done with the information.\n\n‘primitive’ types are the most basic types that often relate directly to how data are stored in memory or on disk (e.g., booleans, integers, numeric (real-valued), character, pointer (address, reference).\n\npointers: references to other locations (addresses) in memory. One often uses pointers to avoid unnecessary copying of data.\nhashes: hashing involves fast lookup of the value associated with a key (a label), using a hash function, which allows one to convert the key to an address. This avoids having to find the value associated with a specific key by looking through all the keys until the key of interest is found (an O(n) operation)." }, { - "objectID": "units/unit7-bigData.html#getting-unique-results-distinct", - "href": "units/unit7-bigData.html#getting-unique-results-distinct", - "title": "Big data and databases", - "section": "Getting unique results (DISTINCT)", - "text": "Getting unique results (DISTINCT)\nA useful SQL keyword is DISTINCT, which allows you to eliminate duplicate rows from any table (or remove duplicate values when one only has a single column or set of values).\n\n## Get the unique tags from the questions_tags table.\ntag_names = db.execute(\"select distinct tag from questions_tags\").fetchall()\ntag_names[0:5]\n## Count the number of unique tags.\n\n[('sorting',), ('visual-c++',), ('mfc',), ('cgridctrl',), ('css',)]\n\ndb.execute(\"select count(distinct tag) from questions_tags\").fetchall()\n\n[(42137,)]" + "objectID": "schedule.html", + "href": "schedule.html", + "title": "Statistics 243 Fall 2023", + "section": "", + "text": "Submit your solutions on Gradescope and (for problem sets but not assignments) via your Git repository.\n\n\n\nProblem Set\nDate\nTime\n\n\n\n\nPS 1 (HTML) (PDF)\nThursday Sep. 7\n10 am\n\n\nPS 2 (HTML) (PDF)\nFriday Sep. 15\n10 am\n\n\nPS 3 (HTML) (PDF) (code)\nWednesday Sep. 27\n10 am\n\n\nPS 4 (HTML) (PDF)\nMonday Oct. 9\n10 am\n\n\nPS 5 (HTML) (PDF)\nFriday Oct. 27\n10 am\n\n\nPS 6 (HTML) (PDF)\nFriday Nov. 3\n10 am\n\n\n\nProblem set solutions need to follow the rules discussed in Lab 1 (Sep. 1) and documented here.\n\n\n\nAssignment\nDate\nTime\n\n\n\n\nbash shell tutorial and bash shell problems (first 10) (reading some of tutorial before Wed. Aug. 30 recommended); see Details below\nFriday Sep. 1\n10 am\n\n\nregular expression reading and problems; See Details below\nFriday Sep. 8\n10 am\n\n\n\nAssignments will generally be turned in on Gradescope but don’t need to follow the rules for problem set solutions and will be graded complete/incomplete." }, { - "objectID": "units/unit7-bigData.html#simple-sql-joins", - "href": "units/unit7-bigData.html#simple-sql-joins", - "title": "Big data and databases", - "section": "Simple SQL joins", - "text": "Simple SQL joins\nOften to get the information we need, we’ll need data from multiple tables. To do this we’ll need to do a database join, telling the database what columns should be used to match the rows in the different tables.\nThe syntax generally looks like this (again the WHERE and ORDER BY are optional):\nSELECT <column(s)> FROM <table1> JOIN <table2> ON <columns to match on>\nWHERE <condition(s) on column(s)> ORDER BY <column(s)>\nLet’s see some joins using the different syntax on the Stack Overflow database. In particular let’s select only the questions with the tag ‘python’. By selecting * we are selecting all columns from both the questions and questions_tags tables.\n\nresult1 = db.execute(\"select * from questions join questions_tags \\\n on questions.questionid = questions_tags.questionid \\\n where tag = 'python'\").fetchall()\nget_fields()\n\n['questionid', 'creationdate', 'score', 'viewcount', 'answercount', 'commentcount', 'favoritecount', 'title', 'ownerid', 'questionid', 'tag']\n\n\nIt turns out you can do it without using the JOIN keyword.\n\nresult2 = db.execute(\"select * from questions, questions_tags \\\n where questions.questionid = questions_tags.questionid and \\\n tag = 'python'\").fetchall()\n\nresult1[0:5]\n\n[(65526804.0, '2021-01-01 01:54:10', 0.0, 2087.0, 3.0, 3.0, None, 'How to play an audio file starting at a specific time', 14718094.0, 65526804.0, 'python'), (65527402.0, '2021-01-01 05:14:22', 1.0, 56.0, 1.0, 0.0, None, 'Join dataframe columns in python', 1492229.0, 65527402.0, 'python'), (65529525.0, '2021-01-01 12:06:43', 1.0, 175.0, 1.0, 0.0, None, 'Issues with pygame.time.get_ticks()', 13720770.0, 65529525.0, 'python'), (65529971.0, '2021-01-01 13:14:40', 1.0, 39.0, 0.0, 1.0, None, 'How to check if Windows prompts a notification box using python?', 13845215.0, 65529971.0, 'python'), (65532644.0, '2021-01-01 18:46:52', -2.0, 49.0, 1.0, 1.0, None, 'How I divide this text file in a Dataframe?', 14122166.0, 65532644.0, 'python')]\n\nresult1 == result2\n\nTrue\n\n\nHere’s a three-way join (using both types of syntax) with some additional use of aliases to abbreviate table names. What does this query ask for?\n\nresult1 = db.execute(\"select * from \\\n questions Q \\\n join questions_tags T on Q.questionid = T.questionid \\\n join users U on Q.ownerid = U.userid \\\n where tag = 'python' and \\\n viewcount > 1000\").fetchall()\n\nresult2 = db.execute(\"select * from \\\n questions Q, questions_tags T, users U where \\\n Q.questionid = T.questionid and \\\n Q.ownerid = U.userid and \\\n tag = 'python' and \\\n viewcount > 1000\").fetchall()\n\nresult1 == result2\n\nTrue\n\n\nChallenge: Write a query that would return all the answers to questions with the Python tag.\nChallenge: Write a query that would return the users who have answered a question with the Python tag." + "objectID": "schedule.html#problem-sets-and-assignments", + "href": "schedule.html#problem-sets-and-assignments", + "title": "Statistics 243 Fall 2023", + "section": "", + "text": "Submit your solutions on Gradescope and (for problem sets but not assignments) via your Git repository.\n\n\n\nProblem Set\nDate\nTime\n\n\n\n\nPS 1 (HTML) (PDF)\nThursday Sep. 7\n10 am\n\n\nPS 2 (HTML) (PDF)\nFriday Sep. 15\n10 am\n\n\nPS 3 (HTML) (PDF) (code)\nWednesday Sep. 27\n10 am\n\n\nPS 4 (HTML) (PDF)\nMonday Oct. 9\n10 am\n\n\nPS 5 (HTML) (PDF)\nFriday Oct. 27\n10 am\n\n\nPS 6 (HTML) (PDF)\nFriday Nov. 3\n10 am\n\n\n\nProblem set solutions need to follow the rules discussed in Lab 1 (Sep. 1) and documented here.\n\n\n\nAssignment\nDate\nTime\n\n\n\n\nbash shell tutorial and bash shell problems (first 10) (reading some of tutorial before Wed. Aug. 30 recommended); see Details below\nFriday Sep. 1\n10 am\n\n\nregular expression reading and problems; See Details below\nFriday Sep. 8\n10 am\n\n\n\nAssignments will generally be turned in on Gradescope but don’t need to follow the rules for problem set solutions and will be graded complete/incomplete." }, { - "objectID": "units/unit7-bigData.html#temporary-tables-and-views", - "href": "units/unit7-bigData.html#temporary-tables-and-views", - "title": "Big data and databases", - "section": "Temporary tables and views", - "text": "Temporary tables and views\nYou can think of a view as a temporary table that is the result of a query and can be used in subsequent queries. In any given query you can use both views and tables. The advantage is that they provide modularity in our querying. For example, if a given operation (portion of a query) is needed repeatedly, one could abstract that as a view and then make use of that view.\nSuppose we always want the age and displayname of owners of questions to be readily available. Once we have the view we can query it like a regular table.\n\ndb.execute(\"create view questionsAugment as select \\\n questionid, questions.creationdate, score, viewcount, \\\n title, ownerid, age, displayname \\\n from questions join users \\\n on questions.ownerid = users.userid\")\n## you'll see the return value is '0'\n \n\n<sqlite3.Cursor object at 0x7f8ac84b8d40>\n\ndb.execute(\"select * from questionsAugment where viewcount > 1000 limit 5\").fetchall()\n\n[(65535296.0, '2021-01-02 01:33:13', 2.0, 1109.0, 'Install and run ROS on Google Colab', 14924336.0, None, 'Gustavo Lima'), (65526407.0, '2021-01-01 00:03:01', 1.0, 2646.0, 'How to remove Branding WHMCS Ver 8.1 \"Powered by WHMcomplete solutions\"', 14920717.0, None, 'Blunch Restaurant'), (65526447.0, '2021-01-01 00:10:40', 7.0, 25536.0, 'React Router v5.2 - Blocking route change with createBrowserHistory and history.block', 10841085.0, None, 'user51462'), (65526500.0, '2021-01-01 00:22:41', 3.0, 2870.0, 'intellisense vscode not showing parameters nor documentation when hovering above with mouse', 13660865.0, None, 'albert chen'), (65526515.0, '2021-01-01 00:27:26', 2.0, 1568.0, 'How to identify time and space complexity of recursive backtracking algorithms with step-by-step analysis', 6801755.0, None, 'BlueTriangles')]\n\n\nOne use of a view would be to create a mega table that stores all the information from multiple tables in the (unnormalized) form you might have if you simply had one data frame in Python or R." + "objectID": "schedule.html#quizzes", + "href": "schedule.html#quizzes", + "title": "Statistics 243 Fall 2023", + "section": "Quizzes", + "text": "Quizzes\nQuizzes are in-person only.\n\nQuiz 1: Monday, October 23 in class.\n\nReview session Friday October 20 in section.\n\nQuiz 2: Monday, November 20 in class.\n\nReview session Friday November 17 in section." }, { - "objectID": "units/unit7-bigData.html#more-on-joins", - "href": "units/unit7-bigData.html#more-on-joins", - "title": "Big data and databases", - "section": "More on joins", - "text": "More on joins\nWe’ve seen a bunch of joins but haven’t discussed the full taxonomy of types of joins. There are various possibilities for how to do a join depending on whether there are rows in one table that do not match any rows in the other table.\nInner joins: In database terminology an inner join is when the result has a row for each match of a row in one table with the rows in the second table, where the matching is done on the columns you indicate. If a row in one table corresponds to more than one row in another table, you get all of the matching rows in the second table, with the information from the first table duplicated for each of the resulting rows. For example in the Stack Overflow data, an inner join of questions and answers would pair each question with each of the answers to that question. However, questions without any answers or (if this were possible) answers without a corresponding question would not be part of the result.\nOuter joins: Outer joins add additional rows from one table that do not match any rows from the other table as follows. A left outer join gives all the rows from the first table but only those from the second table that match a row in the first table. A right outer join is the converse, while a full outer join includes at least one copy of all rows from both tables. So a left outer join of the Stack Overflow questions and answers tables would, in addition to the matched questions and their answers, include a row for each question without any answers, as would a full outer join. In this case there should be no answers that do not correspond to question, so a right outer join should be the same as an inner join.\nCross joins: A cross join gives the Cartesian product of the two tables, namely the pairwise combination of every row from each table. I.e., take a row from the first table and pair it with each row from the second table, then repeat that for all rows from the first table. Since cross joins pair each row in one table with all the rows in another table, the resulting table can be quite large (the product of the number of rows in the two tables). In the Stack Overflow database, a cross join would pair each question with every answer in the database, regardless of whether the answer is an answer to that question.\nSimply listing two or more tables separated by commas as we saw earlier is the same as a cross join. Alternatively, listing two or more tables separated by commas, followed by conditions that equate rows in one table to rows in another is equivalent to an inner join.\nIn general, inner joins can be seen as a form of cross join followed by a condition that enforces matching between the rows of the table. More broadly, here are four equivalent joins that all perform the equivalent of an inner join:\n\n## explicit inner join:\nselect * from table1 join table2 on table1.id = table2.id \n## non-explicit join without JOIN\nselect * from table1, table2 where table1.id = table2.id \n## cross-join followed by matching\nselect * from table1 cross join table2 where table1.id = table2.id \n## explicit inner join with 'using'\nselect * from table1 join table2 using(id)\n\nChallenge: Create a view with one row for every question-tag pair, including questions without any tags.\nChallenge: Write a query that would return the displaynames of all of the users who have never posted a question. The NULL keyword will come in handy it’s like ‘NA’ in R. Hint: NULLs should be produced if you do an outer join." + "objectID": "schedule.html#project", + "href": "schedule.html#project", + "title": "Statistics 243 Fall 2023", + "section": "Project", + "text": "Project\nDue date: TBD." }, { - "objectID": "units/unit7-bigData.html#indexes", - "href": "units/unit7-bigData.html#indexes", - "title": "Big data and databases", - "section": "Indexes", - "text": "Indexes\nAn index is an ordering of rows based on one or more fields. DBMS use indexes to look up values quickly, either when filtering (if the index is involved in the WHERE condition) or when doing joins (if the index is involved in the JOIN condition). So in general you want your tables to have indexes.\nDBMS use indexing to provide sub-linear time lookup. Without indexes, a database needs to scan through every row sequentially, which is called linear time lookup if there are n rows, the lookup is O(n) in computational cost. With indexes, lookup may be logarithmic O(log(n)) (if using tree-based indexes) or constant time O(1) (if using hash-based indexes). A binary tree-based search is logarithmic; at each step through the tree you can eliminate half of the possibilities.\nHere’s how we create an index, with some time comparison for a simple query.\n\nt0 = time.time()\nresults = db.execute(\n \"select * from questions where viewcount > 10000\").fetchall()\nprint(time.time() - t0) # 10 seconds\nt0 = time.time()\ndb.execute(\n \"create index count_index on questions (viewcount)\")\nprint(time.time() - t0) # 19 seconds\nt0 = time.time()\ndb.execute(\n \"select * from questions where viewcount > 10000\").fetchall() \nprint(time.time() - t0) # 3 seconds\n\nIn other contexts, an index can save huge amounts of time. So if you’re working with a database and speed is important, check to see if there are indexes. That said, as seen above it takes time to create the index, so you’d only want to create it if you were doing multiple queries that could take advantage of the index. See the databases tutorial for more discussion of how using indexes in a lookup is not always advantageous." + "objectID": "schedule.html#first-few-weeks-activities-and-assignments", + "href": "schedule.html#first-few-weeks-activities-and-assignments", + "title": "Statistics 243 Fall 2023", + "section": "First few weeks activities and assignments", + "text": "First few weeks activities and assignments\n\n\n\nWeek\nDay\nDate\nTime\nActivity/Assignment\n\n\n\n\n1\nThursday\n2023-08-24\n4:00-5:30 pm\nOptional: Introduction to LaTeX session run by the library (see Details below)\n\n\n\nFriday\n2023-08-25\nnoon\nRequired (part of your class participation): class survey\n\n\n\n\n\nnoon\nRequired: office hour time survey\n\n\n\n\n\nnoon\nRequired: Go to github.berkeley.edu and login with your Calnet credentials (that’s all – it will allow me to create your repository)\n\n\n\n\n\nlab (1:00-4:30 pm)\nOptional: Lab 0 help session for software installation/setup and UNIX-style command line basics (see Details below)\n\n\n2\nMonday\n2022-08-28\n10 am\nRequired: read first three sections of Unit 2 (sections before ‘Webscraping’)\n\n\n\n\n\nnone\nOptional: work through the UNIX basics tutorial and answer (for yourself) the questions at the end\n\n\n\n\n\n7:00-8:15 pm\nOptional: Python workshop (see Details below); perhaps worthwhile if you have no Python background and if you couldn’t attend the Aug 16-17 workshop we held\n\n\n\nWednesday\n2023-08-30\n5:00-6:15 pm\nOptional: Python workshop (see Details below); perhaps worthwhile if you have no Python background and if you couldn’t attend the Aug 16-17 workshop we held\n\n\n\nFriday\n2022-09-01\n10 am\nRequired: Bash shell tutorial and exercises (see Details below)\n\n\n\n\n\nlab\nRequired: Lab 1 on using Git and Quarto and problem set submission (see Details below)\n\n\n3\nWednesday\n2022-09-06\n10 am\nRequired: PS1 due on Gradescope and via GitHub commit\n\n\n\n\n\n4:00-5:30 pm\nOptional: Introduction to LaTeX session run by the library (see Details below)\n\n\n\nFriday\n2022-09-08\n10 am\nRequired: Regular expression tutorial and exercises (see Details below)\n\n\n\n\n\nlab\nRequired: Lab 2 on exceptions and testing" }, { - "objectID": "units/unit7-bigData.html#set-operations-union-intersect-except", - "href": "units/unit7-bigData.html#set-operations-union-intersect-except", - "title": "Big data and databases", - "section": "Set operations: union, intersect, except", - "text": "Set operations: union, intersect, except\nYou can do set operations like union, intersection, and set difference using the UNION, INTERSECT, and EXCEPT keywords, respectively, on tables that have the same schema (same column names and types), though most often these would be used on single columns (i.e., single-column tables).\n\nNote: While one can often set up an equivalent query without using INTERSECT or UNION, set operations can be very handy. In the example below one could do it with a join, but the syntax is often more complicated.\n\nConsider the following example of using INTERSECT. What does it return?\n\nresult1 = db.execute(\"select displayname, userid from \\\n questions Q join users U on U.userid = Q.ownerid \\\n intersect \\\n select displayname, userid from \\\n answers A join users U on U.userid = A.ownerid\")\n\nChallenge: what if you wanted to find users who had neither asked nor answered a question?" + "objectID": "schedule.html#notes-on-assignments-and-activities", + "href": "schedule.html#notes-on-assignments-and-activities", + "title": "Statistics 243 Fall 2023", + "section": "Notes on assignments and activities", + "text": "Notes on assignments and activities\n\nOptional library LaTeX session: I highly recommend (in particular if you are a Statistics graduate student) that you know how to create equations in LaTeX. Even if you develop your documents using Quarto, Jupyter notebooks, R Markdown, etc. rather than LaTeX-based documents, LaTeX math syntax is the common tool for writing math syntax that will render beautifully.\nOptional Lab 0 software/command line help session: (August 25 in lab room) Help session for installing software, accessing a UNIX-style command line, and basic command line usage (e.g., the UNIX basics tutorial). You can show up at any time (unlike all remaining labs). You should have software installed, be able to accesss the command line, and have started to become familiar with basic command line usage before class on Wednesday August 30.\nBash shell tutorial and exercises: (by September 1) Read through this tutorial on using the bash shell. You can skip the pages on Regular Expressions and Managing Processes. Work through the first 10 problems in the exercises and submit your answers via Gradescope. This is not a formal problem set, so you don’t need to worry about formatting nor about explaining/commenting your answers, nor do you need to put your answers in your GitHub class repository. In fact it’s even fine with me if you hand-write the answers and scan them to an electronic document. I just want to make sure you’ve worked through the tutorial. I’ll be doing demonstrations on using the bash shell in class starting on Wednesday August 30, so that will be helpful as you work through the tutorial.\nLab 1: (September 1) First section/lab on using Git, setting up your GitHub repository for problem sets, and using Quarto to generate dynamic documents. Please come only to the section you are registered for given space limits in the room, unless you have talked with Chris and have his permission.\nRegular expression reading and exercises: (by September 8), read the regular expression material in the tutorial on using the bash shell. Then answer the regular expressions (regex) practice problems and submit your answers on Gradescope. This is not one of the graded problem sets but rather an assignment that will simply be noted as being completed or not." }, { - "objectID": "units/unit7-bigData.html#subqueries", - "href": "units/unit7-bigData.html#subqueries", - "title": "Big data and databases", - "section": "Subqueries", - "text": "Subqueries\nA subquery is a full query that is embedded in a larger query. These can be quite handy in building up complicated queries. One could instead use temporary tables, but it often is easier to write all in one query (and that let’s the database’s query optimizer operate on the entire query).\n\nSubqueries in the FROM statement\nWe can use subqueries in the FROM statement to create a temporary table to use in a query. Here we’ll do it in the context of a join.\n\nChallenge: What does the following do?\n\n\ndb.execute(\"select * from questions join answers A \\\n on questions.questionid = A.questionid \\\n join \\\n (select ownerid, count(*) as n_answered from answers \\\n group by ownerid order by n_answered desc limit 1000) most_responsive \\\n on A.ownerid = most_responsive.ownerid\")\n\nIt might be hard to just come up with that full query all at once. A good strategy is probably to think about creating a view that is the result of the inner query and then have the outer query use that. You can then piece together the complicated query in a modular way. For big databases, you are likely to want to submit this as a single query and not two queries so that the SQL optimizer can determine the best way to do the operations. But you want to start with code that you’re confident will give you the right answer!\nNote we could also have done that query using a subquery in the WHERE statement, as discussed in the next section.\n\n\nSubqueries in the WHERE statement\nInstead of a join, we can use subqueries as a way to combine information across tables, with the subquery involved in a WHERE statement. The subquery creates a set and we then can check for inclusion in (or exclusion from with not in) that set.\nFor example, suppose we want to know the average number of UpVotes for users who have posted a question with the tag “python”.\n\ndb.execute(\"select avg(upvotes) from users where userid in \\\n (select distinct ownerid from \\\n questions join questions_tags \\\n on questions.questionid = questions_tags.questionid \\\n where tag = 'python')\").fetchall()\n\n[(62.72529394895326,)]" + "objectID": "ps/ps4.html", + "href": "ps/ps4.html", + "title": "Problem Set 4", + "section": "", + "text": "This covers material in Unit 5, Sections 7-9.\nIt’s due at 10 am (Pacific) on October 9, both submitted as a PDF to Gradescope as well as committed to your GitHub repository.\nPlease see PS1 for formatting and attribution requirements." }, { - "objectID": "units/unit7-bigData.html#creating-database-tables", - "href": "units/unit7-bigData.html#creating-database-tables", - "title": "Big data and databases", - "section": "Creating database tables", - "text": "Creating database tables\nOne can create tables from within the ‘sqlite’ command line interfaces (discussed in the tutorial), but often one would do this from Python or R. Here’s the syntax from Python, creating the table from a Pandas dataframe.\n\n## create data frame 'student_data' in some fashion\ncon = sq.connect(db_path)\nstudent_data.to_sql('student', con, if_exists='replace', index=False)" + "objectID": "ps/ps4.html#comments", + "href": "ps/ps4.html#comments", + "title": "Problem Set 4", + "section": "", + "text": "This covers material in Unit 5, Sections 7-9.\nIt’s due at 10 am (Pacific) on October 9, both submitted as a PDF to Gradescope as well as committed to your GitHub repository.\nPlease see PS1 for formatting and attribution requirements." + }, + { + "objectID": "ps/ps4.html#problems", + "href": "ps/ps4.html#problems", + "title": "Problem Set 4", + "section": "Problems", + "text": "Problems\n\nThis problem will have you write a decorator that collects (and optionally reports) timing information on how long it takes the decorated function to run. Note that you can just present a unified solution; you don’t need to answer each part separately. Please set up the decorator in a module, but demonstrate use the decorator directly in your qmd file.\n\nWrite the decorator so that it times the function and prints timing information out for that single execution of the function, as well as returning the result of the function.\nModify the decorator so that when the function is called with a special “back-door” argument, called _REPORT, set to True, it returns a summary of the times for all previous executions of the function rather than invoking the actual computation. Note that your decorator should work regardless of how many arguments the decorated function takes. As an example of the desired behavior, myfun(_REPORT = True) rather than myfun(x,y) should cause the summary to be returned.\nHint: You’ll need to fool around with passing additional arguments into the wrapper function defined in the decorator function, and the order of how you do that relative to the *args and **kwargs will matter.\nNote: The use of the underscore in _REPORT is intended to avoid any conflicts in the event that the decorated function itself has a regular argument named REPORT. We want to set up the naming in situations like this such that our code won’t interact badly with code that would be written by users, which usually means having our naming be “special” in some way.\nSet things up so that if one sets a global variable called TIMING to be False, then no timing is done. I.e., by setting TIMING to True vs. False, we can toggle whether the decorator timing functionality is operating at all.\n\nThis problem explores how Python stores strings.\n\nLet’s consider the following lists of strings. Determine what storage is reused (if any) in storing ‘abc’ in the two lists.\n\na = ['abc', 'xyz', 'def', 'ghi']\nb = ['abc']*4\n\nNext, let’s dig into how much memory is used to store the information in a list of strings. Determine (i) how much memory is used to store a simple string (and how does this vary with the length of the string), including any metadata, (ii) how much memory is used for metadata of the list object, and (iii) how much is used for any references to the actual elements of the list (i.e., how the size of the list grows with the number of elements). Experimenting with lists of different lengths and strings of different lengths should allow you to work this out from examples without having to try to find technical documentation for Python’s internals.\n\nConsider multiplying an arbitrary \\(n \\times n\\) matrix \\(X\\) and a diagonal \\(n \\times n\\) matrix \\(D\\).\n\nHow many multiplications are done if you simply matrix multiply \\(D\\) and \\(X\\)?\nIn principle, how many multiplications need to be done to obtain the result without doing unnecessary calculations?\nHow can I use numpy functions/methods to compute \\(XD\\) efficiently?\nHow can I use numpy functions/methods to compute \\(DX\\) efficiently?\n\nThis problem asks you to efficiently compute a somewhat complicated log likelihood, arising from a computation from a student’s PhD research. The following is the probability mass function for an overdispersed binomial random variable: \\[\nP(Y =y;n,p,\\phi) = \\frac{f(y;n,p,\\phi)}{\\sum_{k=0}^{n}f(k;n,p,\\phi)} \\\\\n\\] \\[\nf(k;n,p,\\phi) = {n \\choose k}\\frac{k^{k}(n-k)^{n-k}}{n^{n}}\\left(\\frac{n^{n}}{k^{k}(n-k)^{n-k}}\\right)^{\\phi}p^{k\\phi}(1-p)^{(n-k)\\phi}\n\\]\nwhere the denominator of \\(P(Y =y;n,p,\\phi)\\) serves as a normalizing constant to ensure this is a valid probability mass function.\nWe’ll explore how would one efficiently code the computation of the denominator. For our purposes here you can take \\(n=10000\\), \\(p=0.3\\) and \\(\\phi=0.5\\) when you need to actually run your code. Recall that \\(0^0=1\\).\n\nWrite a basic version using map/apply style operations, where you have a function that carries out a single calculation of \\(f\\) for a value of \\(k\\) and then use map/apply to execute it for all the elements of the sum. Make sure to do all calculations on the log scale and only exponentiate before doing the summation. This avoids the possibility of numerical overflow or underflow that we’ll discuss in Unit 8.\nNow create a vectorized version using numpy arrays. Compare timing to the basic non-vectorized version.\nUse timing and profiling tools to understand what steps are slow and try to improve your efficiency. Keep an eye out for repeated calculations and calculations/operations that don’t need to be done. Compare timing to your initial vectorized version in (b)." }, { "objectID": "ps/ps2.html", @@ -812,1046 +868,990 @@ "text": "Problems\n\nIn class and in the Unit 8 notes, I mentioned that integers as large as \\(2^{53}\\) can be stored exactly in the double precision floating point representation. (Note that for this problem, you don’t need to write out e in base 2; you can use base 10).\n\nDemonstrate how the integers 1, 2, 3, …,\\(2^{53}-2\\), \\(2^{53}-1\\) can be stored exactly in the \\((-1)^{S}\\times1.d\\times2^{e-1023}\\) format where d is represented as 52 bits. I’m not expecting anything particularly formal - just write out for a few numbers and show the pattern.\nThen show that \\(2^{53}\\) and \\(2^{53}+2\\) can be represented exactly but \\(2^{53}+1\\) cannot, so the spacing of numbers of this magnitude is 2. Finally show that for numbers starting with \\(2^{54}\\) that the spacing between integers that can be represented exactly is 4. Then confirm that what you’ve shown is consistent with the result of executing \\(2.0^{53}-1\\), \\(2.0^{53}\\), and \\(2.0^{53}+1\\) in Python (you can use base Python floats or numpy).\nFinally, calculate the relative error in representing numbers of magnitude \\(2^{53}\\) in base 10. (This should, of course, look very familiar, and should be the same as the relative error for numbers of magnitude \\(2^{54}\\) or any other magnitude…)\n\nIf we want to estimate a derivative of a function on a computer (often because it is hard to calculate the derivative analytically), a standard way to approximate this is to compute: \\[f'(x)\\approx\\frac{f(x+\\epsilon)-f(x)}{\\epsilon}\\] for some small \\(\\epsilon\\). Since the limit of the right-hand side of the expression as \\(\\epsilon\\to0\\) is exactly \\(f'(x)\\) by the definition of the derivative, we presumably want to use \\(\\epsilon\\) very small from the perspective of using a difference to approximate the derivative.\n\nConsidering the numerator, if we try to do this on a computer, in what ways (there are more than one) do the limitations of arithmetic on a computer affect our choice of \\(\\epsilon\\)? (Note that I am ignoring the denominator because that just scales the magnitude of the result and itself has 16 digits of accuracy.)\nWrite a Python function that calculates the approximation and explore how the error in the estimated derivative for some \\(x\\) varies as a function of \\(\\epsilon\\) for a (non-linear) function that you choose such that you can calculate the derivative analytically (so that you know the truth).\n\nConsider multiclass logistic regression, where you have quantities like this: \\[p_{j}=\\text{Prob}(y=j)=\\frac{\\exp(x\\beta_{j})}{\\sum_{k=1}^{K}\\exp(x\\beta_{k})}=\\frac{\\exp(z_{j})}{\\sum_{k=1}^{K}\\exp(z_{k})}\\] for \\(z_{k}=x\\beta_{k}\\). Here \\(p_j\\) is the probability that the observation, \\(y\\), is in class \\(j\\).\n\nWhat will happen if the \\(z\\) values are very large in magnitude (either positive or negative)?\nHow can we reexpress the equation so as to be able to do the calculation even when either of those situations occurs?\n\nLet’s consider importance sampling and explore the need to have the sampling density have heavier tails than the density of interest. Assume that we want to estimate \\(\\phi=EX\\) and \\(\\phi=E(X^{2})\\) with respect to a density, \\(f\\). We’ll make use of the Pareto distribution, which has the pdf \\(p(x)=\\frac{\\beta\\alpha^{\\beta}}{x^{\\beta+1}}\\) for \\(\\alpha<x<\\infty\\), \\(\\alpha>0\\), \\(\\beta>0\\). The mean is \\(\\frac{\\beta\\alpha}{\\beta-1}\\) for \\(\\beta>1\\) and non-existent for \\(\\beta\\leq1\\) and the variance is \\(\\frac{\\beta\\alpha^{2}}{(\\beta-1)^{2}(\\beta-2)}\\) for \\(\\beta>2\\) and non-existent otherwise.\n\nDoes the tail of the Pareto decay more quickly or more slowly than that of an exponential distribution?\nSuppose \\(f\\) is an exponential density with parameter value equal to 1, shifted by two to the right so that \\(f(x)=0\\) for \\(x<2\\). Pretend that you can’t sample from \\(f\\) and use importance sampling where our sampling density, \\(g\\), is a Pareto distribution with \\(\\alpha=2\\) and \\(\\beta=3\\). Use \\(m=10000\\) to estimate \\(EX\\) and \\(E(X^{2})\\) and compare to the known expectations for the shifted exponential. Recall that \\(\\mbox{Var}(\\hat{\\phi})\\propto\\mbox{Var}(h(X)f(X)/g(X))\\). Create histograms of \\(h(x)f(x)/g(x)\\) and of the weights \\(f(x)/g(x)\\) to get an idea for whether \\(\\mbox{Var}(\\hat{\\phi})\\) is large. Note if there are any extreme weights that would have a very strong influence on \\(\\hat{\\phi}\\).\nNow suppose \\(f\\) is the Pareto distribution described above and pretend you can’t sample from \\(f\\) and use importance sampling where our sampling density, \\(g\\), is the exponential described above. Respond to the same questions as for part (b), comparing to the known values for the Pareto.\n\nExtra credit: This problem explores the smallest positive number that base Python or numpy can represent and how numbers just larger than the smallest positive number that can be represented.\n\nBy trial and error, find the base 10 representation of the smallest positive number that can be represented in Python. Hint: it’s rather smaller than \\(1\\times10^{-308}\\).\nExplain how it can be that we can store a number smaller than \\(1\\times2^{-1022}\\), which is the value of the smallest positive number that we discussed in class. Start by looking at the bit-wise representation of \\(1\\times2^{-1022}\\). What happens if you then figure out the natural representation of \\(1\\times2^{-1023}\\)? You should see that what you get is actually a very “well-known” number that is not equal to \\(1\\times2^{-1023}\\). Given the actual bit-wise representation of \\(1\\times2^{-1023}\\), show the progression of numbers smaller than that that can be represented exactly and show the smallest number that can be represented in Python written in both base 2 and base 10.\nHint: you’ll be working with numbers that are not normalized (i.e., denormalized); numbers that do not have 1 as the fixed number before the radix point in the floating point representation we discussed in Unit 8." }, { - "objectID": "ps/ps3.html", - "href": "ps/ps3.html", - "title": "Problem Set 3", + "objectID": "office_hours.html", + "href": "office_hours.html", + "title": "Office hours", "section": "", - "text": "This covers material in Unit 5..\nIt’s due at 10 am (Pacific) on September 27, both submitted as a PDF to Gradescope as well as committed to your GitHub repository.\nPlease see PS1 for formatting and attribution requirements." + "text": "Chris (Evans 495 or Zoom (see Ed Discussion post for link))\n\nTuesday 11 am - noon\nWednesday 3 pm - 4 pm\nThursday 12:30 - 1:30 pm\ngenerally available immediately after class\nfeel free to schedule an appointment or to drop by if my door is open\n\nDeeb:\n\nMonday 4-5:30 pm (Evans 444)\nFridays during unused section time, generally 2-3 pm and 4-4:30 pm (Evans 340)" }, { - "objectID": "ps/ps3.html#comments", - "href": "ps/ps3.html#comments", - "title": "Problem Set 3", + "objectID": "howtos/gitInstall.html", + "href": "howtos/gitInstall.html", + "title": "Installing Git", "section": "", - "text": "This covers material in Unit 5..\nIt’s due at 10 am (Pacific) on September 27, both submitted as a PDF to Gradescope as well as committed to your GitHub repository.\nPlease see PS1 for formatting and attribution requirements." - }, - { - "objectID": "ps/ps3.html#problems", - "href": "ps/ps3.html#problems", - "title": "Problem Set 3", - "section": "Problems", - "text": "Problems\n\nIn this problem we’ll create a little time-saving hack as a way to get practice with Python classes.\nSuppose I want to be lazy and when I type \"q\" in Python, Python should quit (i.e., I don’t want to have to type quit()). Write a bit of Python code to achieve this.\nHints: (a) What specifically happens if I type the name of an object? (b) You will probably want this code to be in an #| eval:false chunk, since it if it successful, its operation will cause Python to quit and presumably your document will not render.\nLet’s investigate the structure of the pandas package to get some experience with the structure of a large Python package and with how import and the __init__.py file(s) are used. You’ll need to go into the Pandas source code (see Unit 5). Note that the main __init__.py and the __init__.py files in the subpackages/submodules are complicated, and I’m not expecting you to understand everything about them. Also note that the following cases involve functions, classes, and class methods. Be sure to be clear to say which of those it is and in the method case(s), make sure you’re clear on what class the method is part of and any class inheritance structure. Import pandas and then consider the following questions:\n\nConsider pandas.core.config_init.is_terminal. What namespace is it (or its class) in? What file/module is is_terminal in? Is it a function, class, or a class method (and if a class method what is the class)? Describe how it is imported by discussing the relevant statement(s) in the relevant __init__.py file(s).\nConsider pandas.read_csv. What namespace is it (or its class) in? What file/module is read_csv in? Is it a function, class, or a class method (and if a class method what is the class)? Describe how it is imported by discussing the relevant statement(s) in the relevant __init__.py file(s).\nConsider pandas.arrays.BooleanArray. What namespace is it (or its class) in? What file/module is BooleanArray in? Is it a function, class, or a class method (and if a class method what is the class)? Describe how it is imported by discussing the relevant statement(s) in the relevant __init__.py file(s).\nConsider pandas.DataFrame.to_csv. What namespace is it (or its class) in? What file/module is to_csv in? Is it a function, class, or a class method (and if a class method what is the class)? Describe how it is imported by discussing the relevant statement(s) in the relevant __init__.py file(s).\n\nHints: (1) grep -R <pattern> <directory> will search all files within a directory recursively. (2) As you work on this, you may want to be able to modify one or more of the __init__.py files to better understand what is happening (e.g., by commenting out a line of code or adding a print statement). A good way to do this is to create a Conda environment in which pandas is installed, so you isolate any changes you make, e.g., conda create -n test_env python=3.11 pandas. Then you can edit code files in the environment and when you start Python and import pandas, you should see the effects of your changes. Alternatively, you could use the debugger to set breakpoint(s) in an __init__.py file. (3) Or you might create your own small toy package to experiment and see how things work with nested __init__.py files and various ways to use import.\nThe goal of this problem is two-fold: first to give you practice with regular expressions and string processing and the second to have you thinking about writing well-structured, readable code (similar to question 4 of PS1). The website https://www.presidency.ucsb.edu/documents/presidential-documents-archive-guidebook/annual-messages-congress-the-state-the-union#axzz265cEKp1a has the text from all of the State of the Union speeches by US presidents. (These are the annual speeches in which the president speaks to Congress to “report” on the situation in the country.) Your task is to process the information and produce data on the speeches for the years 1900 to present. Note that while I present the problem below as subparts (a)-(i), your solution does NOT need to be divided into subparts in the same way. Your solution should do all of the downloading and processing from within Python so that your operations are self-contained and reproducible.\nYou can choose to use either a functional programming approach or an object-oriented approach, or possibly something that mixes the two. I strongly recommend that you use the approach that you are less familiar with so as to gain more experience. Please think about writing short, modular functions or methods, and operating in a vectorized manner (e.g., using map or possibly list comprehension). Think carefully about how to structure your objects to store the speech information so that the structure works well with your functions/methods.\nGiven you’ve already worked on webscraping, I’m providing some initial code for processing the main landing page in ps/ps3start.py in the course repository, and an example for getting the text of a speech. Also note that you will want to distinguish regular dashes (i.e., the hyphen in a hyphenated work like “team-building”) from a long (“em”) dash that separates clauses (unicode U+2014) – syntax from Unit 2 Section 5 may be helpful.\n\nExtract the URLs for the individual speeches from the URL above. Then use that information to read each speech into Python.\nFor each speech, extract the body of the speech.\nConvert the text so that all text that was not spoken by the president (e.g., Laughter and Applause) is stripped out.\nExtract the words and sentences from each speech as lists of strings, one element per sentence and one element per word. Note that there are probably some special cases here. Try to deal with as much as you can, but we’re not expecting perfection.\nFor each speech count the number of words and characters and compute the average word length.\nCount the following words or word stems: I, we, America{,n}, democra{cy,tic}, republic, Democrat{,ic}, Republican, free{,dom}, war, God [not including God bless], God {B,b}bless, {Jesus, Christ, Christian}, and any others that you think would be interesting.\nThe result of all of this activity should be well-structured data object(s) containing the information about the speeches.\nMake some basic plots that show how the variables have changed over time. Your response here does not have to be extensive but should illustrate what you would do if you were to proceed on to do extensive exploratory data analysis. If you’re not comfortable plotting in Python (that is the case for me) and you prefer to move the relevant data to R to make the plots, that is fine.\nExtra credit: Do some additional research and/or additional thinking to come up with additional variables that quantify speech in interesting ways. Do some plotting that illustrates how the speeches have changed over time.\n\nNow sketch out a design for a functional programming (FP) approach (if your solution to problem 3 used OOP) or an OOP approach (if your solution to problem 3 used functional programming). If you’re designing an OOP approach, decide what the classes would be and the fields and methods of those classes. If you’re designing a FP approach, decide what the functions would be and what inputs/output they would use. To be clear, you do not have to write any of the code for the methods/classes/functions; the idea is just to design the code. As your response in the OOP case, for each class, please provide a bulleted list of methods and bulleted list of fields and for each item briefly comment what the purpose is. Or in the FP case, for each function, provide a bulleted list of inputs and output and briefly comment on the purpose of each function." + "text": "Here are some instructions for installing Git on your computer. Git is the version control software we’ll use in the course.\nYou can install Git by downloading and installing the correct binary from here.\nFor macOS, deeb recommends using the Homebrew option.\nGit comes installed on the SCF, so if you login to an SCF machine and want to use Git there, you don’t need to install Git.\n\nSidenotes on using Git with RStudio\nYou can work with Git through RStudio via RStudio projects.\nHere are some instructions. Here are some helpful guidelines from RStudio.\nYou may need to tell RStudio where the Git executable is located as follows.\n\nOn Windows, the git executable should be installed somewhere like: \"C:/Program Files (x86)/Git/bin/git.exe\"\nOn MacOS X, you can locate the executable by executing the following in Terminal: which git\nOnce you locate the executable, you may then need to confirm that RStudio is looking in the right place. Go to “Tools -> Options -> Git/SVN -> Git executable” and confirm it has the correct information about the location of the git executable." }, { - "objectID": "schedule.html", - "href": "schedule.html", - "title": "Statistics 243 Fall 2023", + "objectID": "howtos/ps-submission.html", + "href": "howtos/ps-submission.html", + "title": "Problem Set Submissions", "section": "", - "text": "Submit your solutions on Gradescope and (for problem sets but not assignments) via your Git repository.\n\n\n\nProblem Set\nDate\nTime\n\n\n\n\nPS 1 (HTML) (PDF)\nThursday Sep. 7\n10 am\n\n\nPS 2 (HTML) (PDF)\nFriday Sep. 15\n10 am\n\n\nPS 3 (HTML) (PDF) (code)\nWednesday Sep. 27\n10 am\n\n\nPS 4 (HTML) (PDF)\nMonday Oct. 9\n10 am\n\n\nPS 5 (HTML) (PDF)\nFriday Oct. 27\n10 am\n\n\nPS 6 (HTML) (PDF)\nFriday Nov. 3\n10 am\n\n\n\nProblem set solutions need to follow the rules discussed in Lab 1 (Sep. 1) and documented here.\n\n\n\nAssignment\nDate\nTime\n\n\n\n\nbash shell tutorial and bash shell problems (first 10) (reading some of tutorial before Wed. Aug. 30 recommended); see Details below\nFriday Sep. 1\n10 am\n\n\nregular expression reading and problems; See Details below\nFriday Sep. 8\n10 am\n\n\n\nAssignments will generally be turned in on Gradescope but don’t need to follow the rules for problem set solutions and will be graded complete/incomplete." + "text": "Problem set solutions should be written in Quarto Markdown (.qmd) source files, interspersing explanatory text with Python (and in some cases bash) code chunks. Please do not use Jupyter notebook (.ipynb) files as your underlying source file for your solutions.\nWhy?\n\nFor one or two of the initial problem sets you’ll need to include both bash and Python code. This isn’t possible in a single notebook.\nThe underlying format of .ipynb files is JSON. While this is a plain text format, the key-value pair structure is much less well-suited for use with Git version control (which relies on diff) than Markdown-based formats.\nOne can run chunks in a Jupyter notebook in arbitrary order. What is printed to PDF depends on the order in which the chunks are run and the results can differ from what one would expect based on reading the notebook sequentially and running the chunks sequentially. For example, consider the following experiment and you’ll see what I mean: (1) Have one code chunk with a = 3 and run it; (2) Add a second chunk with print(a) and run it; and (3) Change the first chunk to a=4 and DO NOT rerun the second chunk. Save the notebook to PDF. You’ll see that your “report” makes no sense. Here’s the result of me doing that experiment.\n\nIf you really want to do your initial explorations of the problems in a Jupyter notebook, with content then copied to qmd, that is fine.\nFor problem sets later in the semester, we may allow the work to be done in a Jupyter notebook (committed to the repository as the source file) and then submitted as a PDF, but the initial problem sets must be provided as qmd source files." }, { - "objectID": "schedule.html#problem-sets-and-assignments", - "href": "schedule.html#problem-sets-and-assignments", - "title": "Statistics 243 Fall 2023", + "objectID": "howtos/ps-submission.html#submission-format", + "href": "howtos/ps-submission.html#submission-format", + "title": "Problem Set Submissions", "section": "", - "text": "Submit your solutions on Gradescope and (for problem sets but not assignments) via your Git repository.\n\n\n\nProblem Set\nDate\nTime\n\n\n\n\nPS 1 (HTML) (PDF)\nThursday Sep. 7\n10 am\n\n\nPS 2 (HTML) (PDF)\nFriday Sep. 15\n10 am\n\n\nPS 3 (HTML) (PDF) (code)\nWednesday Sep. 27\n10 am\n\n\nPS 4 (HTML) (PDF)\nMonday Oct. 9\n10 am\n\n\nPS 5 (HTML) (PDF)\nFriday Oct. 27\n10 am\n\n\nPS 6 (HTML) (PDF)\nFriday Nov. 3\n10 am\n\n\n\nProblem set solutions need to follow the rules discussed in Lab 1 (Sep. 1) and documented here.\n\n\n\nAssignment\nDate\nTime\n\n\n\n\nbash shell tutorial and bash shell problems (first 10) (reading some of tutorial before Wed. Aug. 30 recommended); see Details below\nFriday Sep. 1\n10 am\n\n\nregular expression reading and problems; See Details below\nFriday Sep. 8\n10 am\n\n\n\nAssignments will generally be turned in on Gradescope but don’t need to follow the rules for problem set solutions and will be graded complete/incomplete." - }, - { - "objectID": "schedule.html#quizzes", - "href": "schedule.html#quizzes", - "title": "Statistics 243 Fall 2023", - "section": "Quizzes", - "text": "Quizzes\nQuizzes are in-person only.\n\nQuiz 1: Monday, October 23 in class.\n\nReview session Friday October 20 in section.\n\nQuiz 2: Monday, November 20 in class.\n\nReview session Friday November 17 in section." - }, - { - "objectID": "schedule.html#project", - "href": "schedule.html#project", - "title": "Statistics 243 Fall 2023", - "section": "Project", - "text": "Project\nDue date: TBD." - }, - { - "objectID": "schedule.html#first-few-weeks-activities-and-assignments", - "href": "schedule.html#first-few-weeks-activities-and-assignments", - "title": "Statistics 243 Fall 2023", - "section": "First few weeks activities and assignments", - "text": "First few weeks activities and assignments\n\n\n\nWeek\nDay\nDate\nTime\nActivity/Assignment\n\n\n\n\n1\nThursday\n2023-08-24\n4:00-5:30 pm\nOptional: Introduction to LaTeX session run by the library (see Details below)\n\n\n\nFriday\n2023-08-25\nnoon\nRequired (part of your class participation): class survey\n\n\n\n\n\nnoon\nRequired: office hour time survey\n\n\n\n\n\nnoon\nRequired: Go to github.berkeley.edu and login with your Calnet credentials (that’s all – it will allow me to create your repository)\n\n\n\n\n\nlab (1:00-4:30 pm)\nOptional: Lab 0 help session for software installation/setup and UNIX-style command line basics (see Details below)\n\n\n2\nMonday\n2022-08-28\n10 am\nRequired: read first three sections of Unit 2 (sections before ‘Webscraping’)\n\n\n\n\n\nnone\nOptional: work through the UNIX basics tutorial and answer (for yourself) the questions at the end\n\n\n\n\n\n7:00-8:15 pm\nOptional: Python workshop (see Details below); perhaps worthwhile if you have no Python background and if you couldn’t attend the Aug 16-17 workshop we held\n\n\n\nWednesday\n2023-08-30\n5:00-6:15 pm\nOptional: Python workshop (see Details below); perhaps worthwhile if you have no Python background and if you couldn’t attend the Aug 16-17 workshop we held\n\n\n\nFriday\n2022-09-01\n10 am\nRequired: Bash shell tutorial and exercises (see Details below)\n\n\n\n\n\nlab\nRequired: Lab 1 on using Git and Quarto and problem set submission (see Details below)\n\n\n3\nWednesday\n2022-09-06\n10 am\nRequired: PS1 due on Gradescope and via GitHub commit\n\n\n\n\n\n4:00-5:30 pm\nOptional: Introduction to LaTeX session run by the library (see Details below)\n\n\n\nFriday\n2022-09-08\n10 am\nRequired: Regular expression tutorial and exercises (see Details below)\n\n\n\n\n\nlab\nRequired: Lab 2 on exceptions and testing" - }, - { - "objectID": "schedule.html#notes-on-assignments-and-activities", - "href": "schedule.html#notes-on-assignments-and-activities", - "title": "Statistics 243 Fall 2023", - "section": "Notes on assignments and activities", - "text": "Notes on assignments and activities\n\nOptional library LaTeX session: I highly recommend (in particular if you are a Statistics graduate student) that you know how to create equations in LaTeX. Even if you develop your documents using Quarto, Jupyter notebooks, R Markdown, etc. rather than LaTeX-based documents, LaTeX math syntax is the common tool for writing math syntax that will render beautifully.\nOptional Lab 0 software/command line help session: (August 25 in lab room) Help session for installing software, accessing a UNIX-style command line, and basic command line usage (e.g., the UNIX basics tutorial). You can show up at any time (unlike all remaining labs). You should have software installed, be able to accesss the command line, and have started to become familiar with basic command line usage before class on Wednesday August 30.\nBash shell tutorial and exercises: (by September 1) Read through this tutorial on using the bash shell. You can skip the pages on Regular Expressions and Managing Processes. Work through the first 10 problems in the exercises and submit your answers via Gradescope. This is not a formal problem set, so you don’t need to worry about formatting nor about explaining/commenting your answers, nor do you need to put your answers in your GitHub class repository. In fact it’s even fine with me if you hand-write the answers and scan them to an electronic document. I just want to make sure you’ve worked through the tutorial. I’ll be doing demonstrations on using the bash shell in class starting on Wednesday August 30, so that will be helpful as you work through the tutorial.\nLab 1: (September 1) First section/lab on using Git, setting up your GitHub repository for problem sets, and using Quarto to generate dynamic documents. Please come only to the section you are registered for given space limits in the room, unless you have talked with Chris and have his permission.\nRegular expression reading and exercises: (by September 8), read the regular expression material in the tutorial on using the bash shell. Then answer the regular expressions (regex) practice problems and submit your answers on Gradescope. This is not one of the graded problem sets but rather an assignment that will simply be noted as being completed or not." + "text": "Problem set solutions should be written in Quarto Markdown (.qmd) source files, interspersing explanatory text with Python (and in some cases bash) code chunks. Please do not use Jupyter notebook (.ipynb) files as your underlying source file for your solutions.\nWhy?\n\nFor one or two of the initial problem sets you’ll need to include both bash and Python code. This isn’t possible in a single notebook.\nThe underlying format of .ipynb files is JSON. While this is a plain text format, the key-value pair structure is much less well-suited for use with Git version control (which relies on diff) than Markdown-based formats.\nOne can run chunks in a Jupyter notebook in arbitrary order. What is printed to PDF depends on the order in which the chunks are run and the results can differ from what one would expect based on reading the notebook sequentially and running the chunks sequentially. For example, consider the following experiment and you’ll see what I mean: (1) Have one code chunk with a = 3 and run it; (2) Add a second chunk with print(a) and run it; and (3) Change the first chunk to a=4 and DO NOT rerun the second chunk. Save the notebook to PDF. You’ll see that your “report” makes no sense. Here’s the result of me doing that experiment.\n\nIf you really want to do your initial explorations of the problems in a Jupyter notebook, with content then copied to qmd, that is fine.\nFor problem sets later in the semester, we may allow the work to be done in a Jupyter notebook (committed to the repository as the source file) and then submitted as a PDF, but the initial problem sets must be provided as qmd source files." }, { - "objectID": "office_hours.html", - "href": "office_hours.html", - "title": "Office hours", - "section": "", - "text": "Chris (Evans 495 or Zoom (see Ed Discussion post for link))\n\nTuesday 11 am - noon\nWednesday 3 pm - 4 pm\nThursday 12:30 - 1:30 pm\ngenerally available immediately after class\nfeel free to schedule an appointment or to drop by if my door is open\n\nDeeb:\n\nMonday 4-5:30 pm (Evans 444)\nFridays during unused section time, generally 2-3 pm and 4-4:30 pm (Evans 340)" + "objectID": "howtos/ps-submission.html#problem-set-solution-workflows", + "href": "howtos/ps-submission.html#problem-set-solution-workflows", + "title": "Problem Set Submissions", + "section": "Problem set solution workflows", + "text": "Problem set solution workflows\nHere we outline a few suggested workflows for developing your problem set solutions:\n\nOpen the qmd file in any editor you like (e.g., Emacs, Sublime, ….). From the command line (we think this will work from a Windows command line such as cmd.exe or PowerShell as well), run quarto preview FILE to show your rendered document live as you edit and save changes. You can put the preview window side by side with your editor, and the preview document should automatically render as you save your qmd file.\nUse VS Code with the following extensions: Python, Quarto, and Jupyter Notebooks. This allows you to execute and preview chunks (and whole document) inside VS Code. This is currently deeb’s favorite path due to how well it integrated with the Python debugger.\nUse RStudio (yes, RStudio), which can manage Python code and will display chunk output in the same way it does with R chunks. This path seems to work quite well and is recommended if you are already familiar with RStudio.\n\nLater in the semester, you may be allowed to work directly in Jupyter notebooks and use quarto to render from them directly. This has a few quirks and limitations, but may be allowed for some problem sets.\nPlease commit your work regularly to your repository as you develop your solutions." }, { - "objectID": "ps/ps5.html", - "href": "ps/ps5.html", - "title": "Problem Set 5", - "section": "", - "text": "This covers material in Units 6 and 7 (and a bit into Unit 9).\nIt’s due at 10 am (Pacific) on October 27, both submitted as a PDF to Gradescope as well as committed to your GitHub repository.\nPlease see PS1 for formatting and attribution requirements.\nRunning parallel code, working with large datasets, and using shared computers can result in delays and unexpected problems. Don’t wait until the last minute. The lengthy time period to complete this problem set is simply a logistical result of when the quiz is scheduled – you should start on the problem set before the quiz. We are happy to help troubleshoot problems you may have with accessing the SCF, running parallel Python code, submitting jobs to the SCF cluster, etc. We will not be happy to do so if it is clear that you started at the last minute.\nGiven that you’ll be running batch jobs and operating on remote servers, for problems 1 and 2 you’ll need to provide your code in chunks with #| eval: false. You can paste in any output you need to demonstrate your work. Remember that you can use “```” to delineate blocks of text you want printed verbatim.\nTo monitor an sbatch job, see this SCF doc." + "objectID": "howtos/ps-submission.html#github-repository", + "href": "howtos/ps-submission.html#github-repository", + "title": "Problem Set Submissions", + "section": "GitHub repository", + "text": "GitHub repository\n\nSetting up your repository\nWe are creating repositories for everyone at github.berkeley.edu. Additionally, homeworks still need to be submitted as PDFs on Gradescope.\nSteps:\n\nLog into github.berkeley.edu using your Berkeley credentials. Because of how the system works, you will need to log in before your account is created. Nothing else needs to be done, just log in and log out.\nAfter accounts are created (may take a couple days after first login), when you log in again, you should see one private repository listed on the left side (e.g., stat243-fall-2022/ahv36). This is your class repository. Do not change the repository settings! They are set up for this class.\nClone the repo to your home directory (I would clone it into a directory just for repositories (e.g., I use ~/repos). In the top-level of your working directory, you should create a file named (exactly) .gitignore.\n\nThe .gitignore file causes Git to ignore transient or computer-specific files that Quarto generates. (more info at https://github.com/github/gitignore) In it, put (again, don’t put dashed lines):\n# cache directories\n/__pycache__\n\n# pickle files\n*.pkl\n*.pickle\n\n\nRepository Organization\nThe problem sets in your repository should be organized into folders with specific filenames.\nWhen we pull from your repository, our code will be assuming the following structure:\nyour_repo/\n├── ps1/\n│ ├── ps1.pdf\n│ ├── ps1.qmd \n│ ├── <possibly auxiliary code or other files>\n├── ps2/\n│ ├── ...\n├── ...\n├── ps8/\n├── .gitignore\n└── info.json\nThe file names are case-sensitive, so please keep everything lowercase." }, { - "objectID": "ps/ps5.html#comments", - "href": "ps/ps5.html#comments", - "title": "Problem Set 5", + "objectID": "howtos/accessingUnixCommandLine.html", + "href": "howtos/accessingUnixCommandLine.html", + "title": "Accessing the Unix Command Line", "section": "", - "text": "This covers material in Units 6 and 7 (and a bit into Unit 9).\nIt’s due at 10 am (Pacific) on October 27, both submitted as a PDF to Gradescope as well as committed to your GitHub repository.\nPlease see PS1 for formatting and attribution requirements.\nRunning parallel code, working with large datasets, and using shared computers can result in delays and unexpected problems. Don’t wait until the last minute. The lengthy time period to complete this problem set is simply a logistical result of when the quiz is scheduled – you should start on the problem set before the quiz. We are happy to help troubleshoot problems you may have with accessing the SCF, running parallel Python code, submitting jobs to the SCF cluster, etc. We will not be happy to do so if it is clear that you started at the last minute.\nGiven that you’ll be running batch jobs and operating on remote servers, for problems 1 and 2 you’ll need to provide your code in chunks with #| eval: false. You can paste in any output you need to demonstrate your work. Remember that you can use “```” to delineate blocks of text you want printed verbatim.\nTo monitor an sbatch job, see this SCF doc." - }, - { - "objectID": "ps/ps5.html#problems", - "href": "ps/ps5.html#problems", - "title": "Problem Set 5", - "section": "Problems", - "text": "Problems\n\nThis problem asks you to use Dask to process some Wikipedia traffic data and makes use of tools discussed in Section on October 6. The files in /scratch/users/paciorek/wikistats/dated_2017_small/dated (on the SCF) contain data on the number of visits to different Wikipedia pages on November 4, 2008 (which was the date of the US election in 2008 in which Barack Obama was elected). The columns are: date, time, language, webpage, number of hits, and page size. (Note that the Unit 7 Dask bag example and Question 2 below use a larger set of the same data.)\n\nIn an interactive session on the SCF Linux cluster (ideally on the low partition, but if necessary on the high partition), started using srun (with four cores requested) as discussed in Section:\n\nCopy the data files to a subdirectory of the /tmp directory of the machine your interactive session is running on. (Keep your code files in your home directory.) Putting the files on the local hard drive of the machine you are computing on reduces the amount of copying data across the network (in the situation where you read the data into your program multiple times) and should speed things up in step ii.\nWrite efficient Python code to do the following: Using the dask package as seen in Unit 6, with either map or a list of delayed tasks, write code that, in parallel, reads in the space-delimited files and filters to only the rows that refer to pages where “Barack_Obama” appears in the page title (column 4). You can use the code from Unit 6 as a template. Collect all the results into a single data frame. In your srun invocation and in your code, please use four cores in your parallelization so that other cores are saved for use by other users/students. IMPORTANT: before running the code on the full set of data, please test your code on a small subset first (and test your function on a single input file serially).\nTabulate the number of hits for each hour of the day. (I don’t care how you do this - using either Python or R is fine.) Make a (time-series) plot showing how the number of visits varied over the day. Note that the time zone is UTC/GMT, so you won’t actually see the evening times when Obama’s victory was announced - we’ll see that in Question 2. Feel free to do this step outside the context of the parallelization. You’ll want to use datetime functions from Pandas or the datetime package to manipulate the timing information (i.e., don’t use string manipulations).\nRemove the files from /tmp.\n\nTips:\n\nIn general, keep in mind various ideas from Unit 2 about reading data from files. A couple things that posed problems when I was prototyping this in using pandas.read_csv were that there are lines with fewer than six fields and that there are lines that have quotes that should be treated as part of the text of the fields and not as separators. To get things to work ok, I needed to set the dtype to str for the first two fields (for ease of dealing with the date/time info later) but NOT set the dtype for the other fields, and to use the quoting argument to handle the literal quotes.\nWhen starting your srun session, please include the flag --mem-per-cpu=5G when submitting the Slurm job. In general one doesn’t need to request memory when submitting jobs to the SCF cluster but there is a weird interaction between Dask and Slurm that I don’t quite understand that requires this.\n\nNow replicate steps (i) and (ii) but using sbatch to submit your job as a batch job to the SCF Linux cluster, where step (ii) involves running Python from the command line. You don’t need to make the plot again. As discussed here in the Dask documentation, put your Python/Dask code inside an if __name__ == '__main__' block.\nNote that you need to copy the files to /tmp in your submission script, so that the files are copied to /tmp on whichever node of the SCF cluster your job gets run on. Make sure that as part of your sbatch script you remove the files in /tmp at the end of the script. (Why? In general /tmp is cleaned out when a machine is rebooted, but this might take a while to happen and many of you will be copying files to the same hard drive so otherwise /tmp could run out of space.)\n\nConsider the Wikipedia traffic data for October 15-November 2008 (already available in /var/local/s243/wikistats/dated_2017_sub on any of the SCF cluster nodes in the low or high partitions). Again, explore the variation over time in the number of visits to Barack Obama-related Wikipedia sites, based on searching for “Barack_Obama” on English language Wikipedia pages. You should use Dask with distributed data structures to do the reading and filtering, as seen in Unit 7. Then group by day-hour (it’s fine to do the grouping/counting in Python in a way that doesn’t use Dask data structures). You can do this either in an interactive session using srun or a batch job using sbatch. And if you use srun, you can run Python itself either interactively or as a background job. Time how long it takes to do the Dask part of the computations to get a sense for how much time is involved working with this much data. Once you have done the filtering and gotten the counts for each day-hour, you can simply use standard Python or R code on your laptop to do some plotting to show how the traffic varied over the days of the full month-long time period and particularly over the hours of November 3-5, 2008 (election day was November 4 and Obama’s victory was declared at 11 pm Eastern time on November 4).\nNotes:\n\nThere are various ways to do this using Dask bags or Dask data frames, but I think the easiest in terms of using code that you’ve seen in Unit 7 is to read the data in and do the filtering using a Dask bag and then convert the Dask bag to a Dask dataframe to do the grouping and summarization. Alternatively you should be able to use foldby() from dask.bag, but figuring out what arguments to pass to foldby() is a bit involved.\nMake sure to test your code on a portion of the data before doing computation on the full dataset. Reading and filtering the whole dataset will take something like 30 minutes with 16 cores. You MUST test on a small number of files on your laptop or on one of the stand-alone SCF machines (e.g., radagast, gandalf, arwen) before trying to run the code on the full 40 GB (zipped) of data. For testing, the files are also available in /scratch/users/paciorek/wikistats/dated_2017_sub.\nWhen doing the full computation via your Slurm job submission:\n\nDon’t copy the data (unlike in Question 1) to avoid overloading our disks with each student having their own copy. Just use the data from /var/local/s243/wikistats/dated_2017_sub.\nPlease do not use more than 16 cores in your Slurm job submissions so that cores are available for your classmates. If your job is stuck in the queue you may want to run it with 8 rather than 16 cores.\nAs discussed in Section, when you use sbatch to submit a job to the SCF cluster or srun to run interactively, you should be using the --cpus-per-task flag to specify the number of cores that your computation will use. In your Python code, you can then either hard-code that same number of cores as the number of workers or (better) you can use the SLURM_CPUS_PER_TASK shell environment variable to tell Dask how many workers to start.\n\n\nSQL practice.\n\nUsing the Stack Overflow database, write SQL code that will determine how many users have asked both R- and Python-related questions (not necessarily, but possibly, about R and Python in the same question). There are various ways to do this; you might do this in a single query (for which there are various options), but it’s perfectly fine to create one or more views and then use those views to get the result as a subsequent query. (Hint: if you do this using joins, you’ll probably need to do the equivalent of a self join at some point.)\nExtend your query to include only users who have asked questions about R (that are not also about Python) and questions about Python (that are not also about R). How many such users are there?\n\n(Please wait to work on this until the week of Oct. 23 if you can.) This question prepares for the discussion of a simulation study in section on Friday October 27. The goal of the problem is to think carefully about the design and interpretation of simulation studies, which we’ll talk about in Unit 9, in particular in Section on Friday October 27. In particular, we’ll work with Cao et al. (2015), an article in the Journal of the Royal Statistical Society, Series B, which is a leading statistics journal. The article is available as cao_etal_2015.pdf under the ps directory on GitHub. Read Sections 1, 2.1, and 4 of the article. Also read Section 2 of Unit 9.\nYou don’t need to understand their method for fitting the regression (i.e., you can treat it as some black box algorithm) or the theoretical development. In particular, you don’t need to know what an estimating equation is - you can think of it as an alternative to maximum likelihood or to least squares for estimating the parameters of the statistical model. Equation 3 on page 759 is analogous to taking the sum of squares for a regression model, differentiating with respect to \\(\\beta\\), and setting equal to zero to solve to get \\(\\hat{\\beta}\\). In Equation 3, to find \\(\\hat{\\beta}\\) one sets the equation equal to zero and solves for \\(\\beta\\). As far as the kernel, its role is to weight each pair of observation and covariate value. This downweights pairs where the covariate is measured at a very different time than the observation.\nBriefly (a few sentences for each of the three questions below) answer the following questions.\n\nWhat are the goals of their simulation study, and what are the metrics that they consider in assessing their method?\nWhat choices did the authors have to make in designing their simulation study? What are the key aspects of the data generating mechanism that might affect their assessment of their method?\nConsider their Tables reporting the simulation results. For a method to be a good method, what would one want to see numerically in these columns?" + "text": "You have several options for UNIX command-line access. You’ll need to choose one of these and get it working.\n\nMac OS (on your personal machine):\nOpen a Terminal by going to Applications -> Utilities -> Terminal\n\n\nWindows (on your personal machine):\n\nYou may be able to use the Ubuntu bash shell available in Windows.\nYour PC must be running a 64-bit version of Windows 10 Anniversary Update or later (build 1607+).\nPlease see these links for more information:\n\nhttp://blog.revolutionanalytics.com/2017/12/r-in-the-windows-subsystem-for-linux.html\nhttps://msdn.microsoft.com/en-us/commandline/wsl/install_guide\n\nFor more detailed instructions, see the Installing the Linux Subsystem on Windows tutorial.\n(Not recommended) There’s an older program called cygwin that provides a UNIX command-line interface.\n\nNote that when you install Git on Windows, you will get Git Bash. While you can use this to control Git, the functionality is limited so this will not be enough for general UNIX command-line access for the course.\n\n\nLinux (on your personal machine):\nIf you have access to a Linux machine, you very likely know how to access a terminal.\n\n\nAccess via DataHub (provided by UC Berkeley’s Data Science Education Program)\n\nGo to https://datahub.berkeley.edu\nClick on Sign in with bCourses, sign in via CalNet, and authorize DataHub to have access to your account.\nIn the mid-upper right, click on New and Terminal.\nTo end your session, click on Control Panel and Stop My Server. Note that Logout will not end your running session, it will just log you out of it.\n\n\n\nAccess via the Statistical Computing Facility (SCF)\nWith an SCF account (available here), you can access a bash shell in the ways listed below.\nThose of you in the Statistics Department should be in the process of getting an SCF account. Everyone else will need an SCF account when we get to the unit on parallel computing, but you can request an account now if you prefer.\n\nYou can login to our various Linux servers and access a bash shell that way. Please see http://statistics.berkeley.edu/computing/access.\nYou can also access a bash shell via the SCF JupyterHub interface; please see the Accessing Python instructions but when you click on New, choose Terminal. This is very similar to the DataHub functionality discussed above." }, { - "objectID": "ps/ps1.html", - "href": "ps/ps1.html", - "title": "Problem Set 1", + "objectID": "howtos/windowsInstall.html", + "href": "howtos/windowsInstall.html", + "title": "R/Rstudio on Windows", "section": "", - "text": "This covers material in Units 2 and 4 as well as practice with Quarto.\nIt’s due at 10 am (Pacific) on September 6, both submitted as a PDF to Gradescope as well as committed to your GitHub repository.\nPlease note my comments in the syllabus about when to ask for help and about working together. In particular, please give the names of any other students that you worked with on the problem set and indicate in the text or in code comments any specific ideas or code you borrowed from another student or any online reference (including ChatGPT or the like)." + "text": "While R was built/designed for UNIX systems, it has been well adapted for Windows. Here, we’ll start with the basics of installing R on Windows. Then, we’ll cover the recommended editor (Rstudio), and how to build pdf documents using MikTeX.\n\n\n\n\n\n\nNote\n\n\n\nThis tutorial installs Windows-only versions of everything. Modern Windows systems have an Ubuntu subsystem available that we highly recommend. See the Installing the Linux Subsystem on Windows tutorial for setting up that configuration.\n\n\n\nInstalling R\nThe first step in installing the R language. This is available on CRAN (Comprehensive R Archive Network).\n\nGo to the CRAN webpage, www.r-project.org\nIn the first paragraph, click the link download R\nYou’re now on a page titled CRAN Mirrors, choose the mirror located closest to your geographic location\n\nMirrors are different servers that all host copies of the same website. You get best performance from the location closest to you.\n\nYou’re now on a paged titled The Comprehensive R Archive Network. The first box is labeled Downloand and Install R, click Download R for Windows\nClick base or install R for the first time, these take you to the same place\n\nFor more advanced things, you may need the Rtools download later. It isn’t necessary now, but remember that for the future.\n\nAt the top is a large-font link, Download R X.X.X for Windows, click this. It will begin downloading the Windows installer for R.\nFollow the instructions for setup. If you are unsure of anything, leave the default settings\n\n\n\nInstalling RStudio\nRStudio is one of the best text editors for coding in R. It is our recommended option for beginning. After you are comfortable with the language, or if you use other languages as well, you may want to explore Atom or Sublime. More advanced options include Emacs with [ESS package][https://ess.r-project.org/] and vim with the Nvim-R plugin.\nTo install RStudio:\n\nGo to the RStudio Desktop download page, rstudio.com/products/rstudio/download/#download\nChoose the download for your OS, most likely the Windows 10/8/7 one\nFollow the instructions for setup. If you are unsure of anything, leave the default settings\nOpen RStudio (R will run automatically in the background)\n\nYou may have to allow RStudio to run if prompted (depends on security settings and anti-virus software)\n\n\nOnce RStudio is installed, you can install or update packages in one of two ways:\n\nVia the console, using install.packages() or update.packages()\n\nVia the gui:\n\nIn the top bar, click on tools\nSelect Install Packages… to install packages\nSelect Check for Package Updates… to update packages\n\n\n\n\nCompiling PDF Documents\nFor the purposes of this class, you will be submitting homeworks as PDF documents that blend written text, code, and code-generated output. These documents are RMarkdown documents, and are dynamic documents that provide a convenient method for documenting your work (more on this in one of the lab sections). To do this, you need a LaTeX renderer. We recommend MiKTeX for Windows.\n\nGo to Getting MiKTeX to download MiKTeX for Windows, miktex.org/download\nThe first page should be Install on Windows, click Download at the bottom of the page\n\nClick the download to begin\n\n\n\n\n\n\n\nImportant\n\n\n\nFOLLOW THESE INSTALL INSTRUCTIONS.\nThe default options are fine in most places, but there is one that will cause problems.\n\n\n\nAccept the Copying Conditions, click next\nInstall only for you, click next\nUse the default directory, click next\nThis should be the Settings page. Under Install missing packages on-the-fly, change the setting to Yes, click next\n\n\n\nBecause we are using MiKTeX as an external renderer, it can’t ask you to install missing packages, and will then fail, so we have to set that installation as automatic.\n\n\nClick start (Optional, but highly recommended) Open RStudio, select a new .Rmd document, d then choose knit. This may take some time, because MiKTeX is installing new braries, but it ensures that your pipeline is setup correctly" }, { - "objectID": "ps/ps1.html#comments", - "href": "ps/ps1.html#comments", - "title": "Problem Set 1", + "objectID": "labs/lab3-debugging.html", + "href": "labs/lab3-debugging.html", + "title": "Lab 3: Debugging", "section": "", - "text": "This covers material in Units 2 and 4 as well as practice with Quarto.\nIt’s due at 10 am (Pacific) on September 6, both submitted as a PDF to Gradescope as well as committed to your GitHub repository.\nPlease note my comments in the syllabus about when to ask for help and about working together. In particular, please give the names of any other students that you worked with on the problem set and indicate in the text or in code comments any specific ideas or code you borrowed from another student or any online reference (including ChatGPT or the like)." - }, - { - "objectID": "ps/ps1.html#formatting-requirements", - "href": "ps/ps1.html#formatting-requirements", - "title": "Problem Set 1", - "section": "Formatting requirements", - "text": "Formatting requirements\n\nYour electronic solution should be in the form of an Quarto file named ps1.qmd, with Python code chunks included in the file. Please see the Lab 1 and the dynamic documents tutorial for more information on how to do this.\nYour PDF submission should be the PDF produced from your qmd. Your GitHub submission should include the qmd file, any Python code files containing chunks that you read into your qmd file, and the final PDF, all named according to the submission guidelines.\nYour solution should not just be code - you should have text describing how you approached the problem and what the various steps were. Your code should have comments indicating what each function or block of code does, and for any lines of code or code constructs that may be hard to understand, a comment indicating what that code does.\nYou do not need to (and should not) show exhaustive output, but in general you should show short examples of what your code does to demonstrate its functionality. Please see the grading rubric, and note that the output should be produced as a result of the code chunks being run during the rendering process, not by copy-pasting of output from running the code separately." - }, - { - "objectID": "ps/ps1.html#problems", - "href": "ps/ps1.html#problems", - "title": "Problem Set 1", - "section": "Problems", - "text": "Problems\n\nPlease read these lecture notes about how computers work, used in a class on statistical computing at CMU. Briefly (a few sentences) describe the difference between disk and memory based on that reference and/or other resources you find.\nThis problem uses the ideas and tools in Unit 2, Sections 1-3 to explore approaches to reading and writing data from files and to consider file sizes in ASCII plain text vs. binary formats in light of the fact that numbers are (generally) stored as 8 bytes per number in binary formats.\n\nGenerate a numpy array (named x) of random numbers from a standard normal distribution with 10 columns and as many rows as needed that the data takes up about 100 Mb in size. As part of your answer, show the arithmetic (formatted using LaTeX math syntax) you did to determine the number of rows.\nExplain the sizes of the two files created below. In discussing the CSV text file, how many characters do you expect to be in the file (i.e., you should be able to estimate this very accurately from first principles without using wc or any explicit program that counts characters). Hint: what do we know about numbers drawn from a standard normal distribution?\n\nimport os\nimport pandas as pd\nx = x.round(decimals = 10)\n\npd.DataFrame(x).to_csv('x.csv', header = False, index = False)\nprint(f\"{str(os.path.getsize('x.csv')/1e6)} MB\")\n\npd.DataFrame(x).to_pickle('x.pkl', compression = None) \nprint(f\"{str(os.path.getsize('x.pkl')/1e6)} MB\")\n\n167.358888 MB\n\n\n100.000572 MB\n\n\nSuppose we had rounded each number to three decimal places. Would using CSV have saved disk space relative to the pickle file?\nNow consider saving out the numbers one number per row in a CSV file. Given we no longer have to save all the commas, why is the file size unchanged (or perhaps even greater if you are on Windows)?\nRead the CSV file into Python using pandas.read_csv. Compare the speed of reading with and without providing the dtype argument and using the python vs c engines. Repeat the timing of your first attempt (without dtype and with the default engine) a few times. In some cases you might find that the first time is slower; if so this has to do with the operating system caching the file in memory (we’ll discuss this further in Unit 8).\nFinally, let’s consider reading the CSV file in chunks as discussed in Unit 2. Time how long it takes to read the first 100,000 rows.\nNow experiment with the skiprows to see if you can read in a large chunk of data from the middle of the file as quickly as the same size chunk from the start of the file. What does this indicate regarding whether Pandas/Python has to read in all the data up to the point where the chunk in the middle starts or can skip over it in some fashion? Is there any savings relative to reading all the initial rows and the chunk in the middle all at once?\nNow read the data sequentially in equal-sized chunks and determine if reading in the large chunk in the middle (after having already read the earlier chunks) takes the same amount of time as it did in part (f). Comment on what you’ve learned.\n\nPlease read Section 1 of Unit 4 on good programming/project practices and incorporate what you’ve learned from that reading into your solution for Problem 4. (You can skip the section on Assertions and Testing, as we’ll cover that in Lab.) As your response to this question, briefly (a few sentences) note what you did in your code for Problem 4 that reflects what you read. Please also note anything in Unit 4 that you disagree with, if you have a different stylistic perspective.\nWe’ll experiment with webscraping and manipulating HTML by getting song lyrics from the web. Go to http://mldb.org/search and (in the search bar in the middle, not at the left) enter the name of a song and choose to search by ‘Title’ and ‘All words’. In some cases the search goes directly to the lyrics of the song (presumably when there is no ambiguity) and in others it goes to a table of potential songs with that or similar name. (For example, compare ‘Dance in the Dark’ (or ‘Dancing in the Dark’) to ‘Leaving Las Vegas’.)\n\nBased on the GET request being sent to the MLDb server (in the cases like ‘Dance in the Dark’ where you get a table back rather than a single song’s lyrics), determine how to programmatically search for a song by ‘Title’ and ‘All words’ using Python, based on our explorations in Unit 2. Side question: what does the si parameter control?\nWarning: It’s possible that if you repeatedly query the site too quickly, it will start returning “503” errors because it detects automated usage (see problem 5 below). So, if you are going to run code from a script such that multiple queries would get done in quick succession, please put something like time.sleep(2) in between the calls that do the HTTP requests. Also when developing your code, once you have the code working to download the HTML, use the downloaded HTML to develop the remainder of your code that manipulates the HTML and don’t repeatedly re-download the HTML as you work on the remainder of the code.\nWrite an overall Python function (and modular helper functions to do particular pieces of the work needed) that takes as input a title and artist, searches by the title, and then (based on an exact match to the title and artist in the resulting set of song results) finds the URL of the page for the lyrics for that particular song. Then use that URL and return the lyrics, the artist, and the album(s). You can assume that the song you want is on the first page of results. If no exact match is found, just return None. Make sure to explain how your code extracts the HTML elements you need. Hint: you will need to use some string processing functions to do basic manipulations. We’ll see this more in Unit 5, but for now, you can find information in the https://berkeley-scf.github.io/tutorial-string-processing/text-manipulation#2-basic-text-manipulation-in-python. You should NOT need to use regular expressions (which we’ll cover in Units 3 and 5) or the re package, though you can if you want to.\nModify your function so it works either when the lyrics are returned directly from the initial search or when multiple songs are returned. Include checks in your code so that it fails gracefully if the user provides invalid input or MLDb doesn’t return a result.\n(Extra credit) Modify your code to handle cases (e.g., searching for “Dance with me”) that return more than one page of results.\n\nLook at the robots.txt file for MLDb and for Google Scholar (scholar.google.com) and the references in Unit 2 on the ethics of webscraping. Does it seem like it’s ok to scrape data from MLDb? What about Google Scholar?" + "text": "Today we’re going to explore the concept of debugging and some of the tooling that allows for debugging python code.\nIt’s widely recognized and accepted that any sizeable code base will have a non-trivial number of bugs (where does the term come from?). The main goal of testing is to make sure the main expected cases are behaving correctly.\nSometime you code doesn’t do what you expect or want it to do, and it’s not clear just by reading through it, what the problem is.\nIn interpreted languages (esp. those with interactive shells like python) one can sometimes run the code piece by piece and inspect the state of the variables.\nIf the code involves a loop or a function, a common practice is to judiciously place a few print statements that dump the state of some variables to the terminal so that you can spot the problem by tracing through it.\nWhen the code involves multiple functions and complex state, this strategy starts to break down. This is where you start rolling up your sleeves and invoking a debugger!\nDebugging can be a slow process, so you typically start a debugging session by deciding which line in your code you would like to start tracing the behavior from, and you place a breakpoint. Then you can have the debugger run the program up to that point and stop at it, allowing you to:\n1- inspect the current state of variables 2- step through the code line by line 3- step over or into functions as they are called 4- resume program execution" }, { - "objectID": "ps/ps4.html", - "href": "ps/ps4.html", - "title": "Problem Set 4", + "objectID": "labs/lab3-debugging.html#debugging", + "href": "labs/lab3-debugging.html#debugging", + "title": "Lab 3: Debugging", "section": "", - "text": "This covers material in Unit 5, Sections 7-9.\nIt’s due at 10 am (Pacific) on October 9, both submitted as a PDF to Gradescope as well as committed to your GitHub repository.\nPlease see PS1 for formatting and attribution requirements." + "text": "Today we’re going to explore the concept of debugging and some of the tooling that allows for debugging python code.\nIt’s widely recognized and accepted that any sizeable code base will have a non-trivial number of bugs (where does the term come from?). The main goal of testing is to make sure the main expected cases are behaving correctly.\nSometime you code doesn’t do what you expect or want it to do, and it’s not clear just by reading through it, what the problem is.\nIn interpreted languages (esp. those with interactive shells like python) one can sometimes run the code piece by piece and inspect the state of the variables.\nIf the code involves a loop or a function, a common practice is to judiciously place a few print statements that dump the state of some variables to the terminal so that you can spot the problem by tracing through it.\nWhen the code involves multiple functions and complex state, this strategy starts to break down. This is where you start rolling up your sleeves and invoking a debugger!\nDebugging can be a slow process, so you typically start a debugging session by deciding which line in your code you would like to start tracing the behavior from, and you place a breakpoint. Then you can have the debugger run the program up to that point and stop at it, allowing you to:\n1- inspect the current state of variables 2- step through the code line by line 3- step over or into functions as they are called 4- resume program execution" }, { - "objectID": "ps/ps4.html#comments", - "href": "ps/ps4.html#comments", - "title": "Problem Set 4", - "section": "", - "text": "This covers material in Unit 5, Sections 7-9.\nIt’s due at 10 am (Pacific) on October 9, both submitted as a PDF to Gradescope as well as committed to your GitHub repository.\nPlease see PS1 for formatting and attribution requirements." + "objectID": "labs/lab3-debugging.html#advanced-debugging", + "href": "labs/lab3-debugging.html#advanced-debugging", + "title": "Lab 3: Debugging", + "section": "advanced debugging", + "text": "advanced debugging\nSome bugs are really tricky to catch. Those are typically the bugs that happen very rarely, and in unclear circumstances. In statistical computing and data analysis settings these might be conditions that happen in iterative algorithms sometimes, but not very often, and can be hard to reproduce.\nOne way to deal with these bugs is to start the debugging session with placing one or more conditional breakpoints. These are similar to regular breakpoints but are not going to cause the debugger to stop the execution of the program and hand you the controls unless a specific condition (that you specify) evaluates to true as the program is executing that particular line of code. You may use some conditions that are similar to what you would place in an assert statement (conditions that shouldn’t happen) or any other conditions that you think may be associated with the occurrence of the anomalous behavior your are debugging." }, { - "objectID": "ps/ps4.html#problems", - "href": "ps/ps4.html#problems", - "title": "Problem Set 4", - "section": "Problems", - "text": "Problems\n\nThis problem will have you write a decorator that collects (and optionally reports) timing information on how long it takes the decorated function to run. Note that you can just present a unified solution; you don’t need to answer each part separately. Please set up the decorator in a module, but demonstrate use the decorator directly in your qmd file.\n\nWrite the decorator so that it times the function and prints timing information out for that single execution of the function, as well as returning the result of the function.\nModify the decorator so that when the function is called with a special “back-door” argument, called _REPORT, set to True, it returns a summary of the times for all previous executions of the function rather than invoking the actual computation. Note that your decorator should work regardless of how many arguments the decorated function takes. As an example of the desired behavior, myfun(_REPORT = True) rather than myfun(x,y) should cause the summary to be returned.\nHint: You’ll need to fool around with passing additional arguments into the wrapper function defined in the decorator function, and the order of how you do that relative to the *args and **kwargs will matter.\nNote: The use of the underscore in _REPORT is intended to avoid any conflicts in the event that the decorated function itself has a regular argument named REPORT. We want to set up the naming in situations like this such that our code won’t interact badly with code that would be written by users, which usually means having our naming be “special” in some way.\nSet things up so that if one sets a global variable called TIMING to be False, then no timing is done. I.e., by setting TIMING to True vs. False, we can toggle whether the decorator timing functionality is operating at all.\n\nThis problem explores how Python stores strings.\n\nLet’s consider the following lists of strings. Determine what storage is reused (if any) in storing ‘abc’ in the two lists.\n\na = ['abc', 'xyz', 'def', 'ghi']\nb = ['abc']*4\n\nNext, let’s dig into how much memory is used to store the information in a list of strings. Determine (i) how much memory is used to store a simple string (and how does this vary with the length of the string), including any metadata, (ii) how much memory is used for metadata of the list object, and (iii) how much is used for any references to the actual elements of the list (i.e., how the size of the list grows with the number of elements). Experimenting with lists of different lengths and strings of different lengths should allow you to work this out from examples without having to try to find technical documentation for Python’s internals.\n\nConsider multiplying an arbitrary \\(n \\times n\\) matrix \\(X\\) and a diagonal \\(n \\times n\\) matrix \\(D\\).\n\nHow many multiplications are done if you simply matrix multiply \\(D\\) and \\(X\\)?\nIn principle, how many multiplications need to be done to obtain the result without doing unnecessary calculations?\nHow can I use numpy functions/methods to compute \\(XD\\) efficiently?\nHow can I use numpy functions/methods to compute \\(DX\\) efficiently?\n\nThis problem asks you to efficiently compute a somewhat complicated log likelihood, arising from a computation from a student’s PhD research. The following is the probability mass function for an overdispersed binomial random variable: \\[\nP(Y =y;n,p,\\phi) = \\frac{f(y;n,p,\\phi)}{\\sum_{k=0}^{n}f(k;n,p,\\phi)} \\\\\n\\] \\[\nf(k;n,p,\\phi) = {n \\choose k}\\frac{k^{k}(n-k)^{n-k}}{n^{n}}\\left(\\frac{n^{n}}{k^{k}(n-k)^{n-k}}\\right)^{\\phi}p^{k\\phi}(1-p)^{(n-k)\\phi}\n\\]\nwhere the denominator of \\(P(Y =y;n,p,\\phi)\\) serves as a normalizing constant to ensure this is a valid probability mass function.\nWe’ll explore how would one efficiently code the computation of the denominator. For our purposes here you can take \\(n=10000\\), \\(p=0.3\\) and \\(\\phi=0.5\\) when you need to actually run your code. Recall that \\(0^0=1\\).\n\nWrite a basic version using map/apply style operations, where you have a function that carries out a single calculation of \\(f\\) for a value of \\(k\\) and then use map/apply to execute it for all the elements of the sum. Make sure to do all calculations on the log scale and only exponentiate before doing the summation. This avoids the possibility of numerical overflow or underflow that we’ll discuss in Unit 8.\nNow create a vectorized version using numpy arrays. Compare timing to the basic non-vectorized version.\nUse timing and profiling tools to understand what steps are slow and try to improve your efficiency. Keep an eye out for repeated calculations and calculations/operations that don’t need to be done. Compare timing to your initial vectorized version in (b)." + "objectID": "labs/lab3-debugging.html#integrated-gui-debugger-with-vs-code", + "href": "labs/lab3-debugging.html#integrated-gui-debugger-with-vs-code", + "title": "Lab 3: Debugging", + "section": "Integrated GUI debugger (with VS Code)", + "text": "Integrated GUI debugger (with VS Code)\nToday we will experiment with the visual debugging tools integrated with IDEs. We will do that in VS Code (unless you have another IDE with debugger integration). We will load a piece of code, go through it to understand what it does, then try to discover the problem with it and fix it.\nHere’s a piece of code that implements the binary search algorithm to locate the first occurence of a number in a list of numbers:\n\nimport math\ndef binary_search(lst, T):\n L = 0\n R = len(lst) - 1\n while L < R:\n m = math.floor((L + R) / 2)\n if lst[m] <= T:\n L = m + 1\n else:\n R = m - 1\n if lst[L] == T:\n return L\n return -1\n\nThere are a couple of things not quite right with this implementation, even though it will run and produce correct results for some cases.\nHere’s another piece of code implementing merge sort (also with some bugs in it):\n\ndef merge_sort(lst):\n n = len(lst)\n if n == 1:\n return lst\n return merge(merge_sort(lst[:n//2]), merge_sort(lst[n//2:]))\n\ndef merge(lst1, lst2):\n merged = []\n i, j = 0,0\n while i < len(lst1) and j < len(lst2):\n if i < len(lst1) and j < len(lst2) and lst1[i] < lst2[j]:\n merged.append(lst1[i])\n i += 1\n else:\n merged.append(lst1[j])\n j += 1\n while i < len(lst1):\n merged.append(lst1[i])\n i += 1\n while j < len(lst2):\n merged.append(lst2[j])\n j += 1\n return merged\n \nmerge_sort([3,1,5,1,6,3,9,12,8])\n\nYou can use this to practice stepping inside functions, and thinking about recursion.\nIncidentally, if you first sort, then find, you can get the quantile of a particular value within a collection (you’ll need to adjust the binary search a little to achieve this).\nAlternatively you could start by implementing (without using any existing functions) a function that inverst the order of the words in a string, and debug it until it works.\nHere’s a version of this function where I injected a couple of bugs, if you prefer to start from there:\n\ndef reverse_words(input):\n working = list(input)\n invert(working)\n start = 0\n for i, c in enumerate(working):\n if c == ' ' and i != start:\n invert(working, start, i)\n start = i+1\n return ''.join(working)\n\ndef invert(lst, start=None, end=None):\n if None == start:\n start = 0\n if None == end:\n end = len(lst)-1\n \n while start < end:\n tmp = lst[start]\n lst[start] = lst[start]\n lst[end] = tmp\n start += 1\n end -= 1\n\nreverse_words(\"These are my words. I have spoken!\")\n\nNext time we will touch briefly on how to do debugging without an IDE with debugger integration." }, { - "objectID": "ps/regex.html", - "href": "ps/regex.html", - "title": "Assignment: regex problems", + "objectID": "labs/lab0-setup.html", + "href": "labs/lab0-setup.html", + "title": "Lab 0: Setup", "section": "", - "text": "Overview\nRead the regular expression section of the bash shell tutorial and provide regular expression syntax to match the strings in the following scenarios. Any reasonable syntax si fine, and even better, challenge yourself to figure out multiple ways to answer each question.\nThis is an assignment, graded credit/no credit, and it does not need to follow the requirements for problem set submissions. Any format is fine (including hand-written and scanned).\n\n\nProblems\n\nMatch the strings “dog”, “Dog”, “dOg”, “doG”, “DOg”, etc. (The word ‘dog’ in any combination of lower and upper case.)\nMatch the strings “cat”, “at”, and “t”.\nMatch the strings “cat”, “caat”, “caaat”, etc.\nMatch two words separated by any amount of whitespace.\nMatch exactly two words separated by ay amount of whitespace and with or without whitespace at the beginning and end. (I.e., you shouldn’t match if there are three words.)" + "text": "You will need to set up (or make sure you have access to) the following:\n\nUnix shell\nGit\nQuarto\nPython\nA text editor or IDE of your choice (deeb recommends VS Code or sublime, but if you are already familiar with a specific editor, stick with it)\n\nAfter making sure you have access to all these 5 tools, it may be a good idea to go through some of the following tutorials on unix bash and unix commands. You can do this on your own or in the Lab section on Friday 8/25.\nAttending lab 0 is optional. If you successfully set up your environment to have all the listed tools, you don’t need to attend. You are welcome to attend to ask for help with the setup or to help your classmates with setting up their environments.\nYou are also welcome to come and ask for help on using any of these 5 tools/systems (esp. bash and unix commands this week), but priority will be given to environment setup questions.\nReach out to deeb @ deeb@berkeley.edu with any unresolved problems or if you discover something that needs to be changed with our howtos and instructions.\nDeeb’s unsolicited advice on languages and tools:\n\nWhichever editor you pick, make sure to spend some time every week learning a few keyboard shortcuts for it. The same goes for the bash shell (ctrl+a and ctrl+e are among my favorites) and for your OS of choice in general (and even your web browser!). Not only do keyboard shortcuts make you more efficient, but they dramatically reduce the cognitive load after a while, and so make your life less painful in the long run. They can be the difference between hating computers and loving them.\nProgramming languages come and go, but Unix is forever! Well, maybe not forever, but close enough. Invest more of your time in getting familiar with durable and proven paradigms. Different programming languages are suitable in different situations and change dramatically from one decade to the next, but the unix shell and commands are as pristine, long lived, and as widely applicable as you’ll find in the computing world. I have a much more mixed view of git, Python, R, and C++. Another example of a very durable computing paradigm is SQL, which we’ll get to in a few weeks." }, { - "objectID": "units/unit6-parallel.html", - "href": "units/unit6-parallel.html", - "title": "Parallel processing", + "objectID": "labs/lab0-setup.html#setting-up-your-environment-082523", + "href": "labs/lab0-setup.html#setting-up-your-environment-082523", + "title": "Lab 0: Setup", "section": "", - "text": "PDF\nReferences:\nThis unit will be fairly Linux-focused as most serious parallel computation is done on systems where some variant of Linux is running. The single-machine parallelization discussed here should work on Macs and Windows, but some of the details of what is happening under the hood are different for Windows.\nAs context, let’s consider some ways we might be able to achieve faster computation:" + "text": "You will need to set up (or make sure you have access to) the following:\n\nUnix shell\nGit\nQuarto\nPython\nA text editor or IDE of your choice (deeb recommends VS Code or sublime, but if you are already familiar with a specific editor, stick with it)\n\nAfter making sure you have access to all these 5 tools, it may be a good idea to go through some of the following tutorials on unix bash and unix commands. You can do this on your own or in the Lab section on Friday 8/25.\nAttending lab 0 is optional. If you successfully set up your environment to have all the listed tools, you don’t need to attend. You are welcome to attend to ask for help with the setup or to help your classmates with setting up their environments.\nYou are also welcome to come and ask for help on using any of these 5 tools/systems (esp. bash and unix commands this week), but priority will be given to environment setup questions.\nReach out to deeb @ deeb@berkeley.edu with any unresolved problems or if you discover something that needs to be changed with our howtos and instructions.\nDeeb’s unsolicited advice on languages and tools:\n\nWhichever editor you pick, make sure to spend some time every week learning a few keyboard shortcuts for it. The same goes for the bash shell (ctrl+a and ctrl+e are among my favorites) and for your OS of choice in general (and even your web browser!). Not only do keyboard shortcuts make you more efficient, but they dramatically reduce the cognitive load after a while, and so make your life less painful in the long run. They can be the difference between hating computers and loving them.\nProgramming languages come and go, but Unix is forever! Well, maybe not forever, but close enough. Invest more of your time in getting familiar with durable and proven paradigms. Different programming languages are suitable in different situations and change dramatically from one decade to the next, but the unix shell and commands are as pristine, long lived, and as widely applicable as you’ll find in the computing world. I have a much more mixed view of git, Python, R, and C++. Another example of a very durable computing paradigm is SQL, which we’ll get to in a few weeks." }, { - "objectID": "units/unit6-parallel.html#embarrassingly-parallel-ep-problems", - "href": "units/unit6-parallel.html#embarrassingly-parallel-ep-problems", - "title": "Parallel processing", - "section": "Embarrassingly parallel (EP) problems", - "text": "Embarrassingly parallel (EP) problems\nAn EP problem is one that can be solved by doing independent computations in separate processes without communication between the processes. You can get the answer by doing separate tasks and then collecting the results. Examples in statistics include\n\nsimulations with many independent replicates\nbootstrapping\nstratified analyses\nrandom forests\ncross-validation.\n\nThe standard setup is that we have the same code running on different datasets. (Note that different processes may need different random number streams, as we will discuss in the Simulation Unit.)\nTo do parallel processing in this context, you need to have control of multiple processes. Note that on a shared system with queueing/scheduling software set up, this will generally mean requesting access to a certain number of processors and then running your job in such a way that you use multiple processors.\nIn general, except for some modest overhead, an EP problem can ideally be solved with \\(1/p\\) the amount of time for the non-parallel implementation, given \\(p\\) CPUs. This gives us a speedup of \\(p\\), which is called linear speedup (basically anytime the speedup is of the form \\(kp\\) for some constant \\(k\\))." + "objectID": "labs/lab2-testing.html", + "href": "labs/lab2-testing.html", + "title": "Lab 2: Assertions, Exceptions, and Tesing", + "section": "", + "text": "Today we will spend some time getting familiar with some of the programming tools that can help make your code more robust and resilient to errors and boundary conditions. Tools like unit tests, exceptions, and asserts (and next week we will spend some time on debugging misbehaving code).\nTesting is what you do when you finish implementing a piece of code and want to try it out to see if it works. Running your code manually and seeing if it works is a workable strategy for simple one-time scripts that do simple tasks, but there are situations (like writing a function that others will repeatedly use, or like running the same piece of code on hundreds of files or URLs) where it is prudent to test your code ahead of its actual use or deployment. Today we will use the pytest package to do that. We will also use some error handling techniques to make sure our code can handle malformed inputs." }, { - "objectID": "units/unit6-parallel.html#computer-architecture", - "href": "units/unit6-parallel.html#computer-architecture", - "title": "Parallel processing", - "section": "Computer architecture", - "text": "Computer architecture\nComputers now come with multiple processors for doing computation. Basically, physical constraints have made it harder to keep increasing the speed of individual processors, so the chip industry is now putting multiple processing units in a given computer and trying/hoping to rely on implementing computations in a way that takes advantage of the multiple processors.\nEveryday personal computers usually have more than one processor (more than one chip) and on a given processor, often have more than one core (multi-core). A multi-core processor has multiple processors on a single computer chip. On personal computers, all the processors and cores share the same memory.\nSupercomputers and computer clusters generally have tens, hundreds, or thousands of ‘nodes’, linked by a fast local network. Each node is essentially a computer with its own processor(s) and memory. Memory is local to each node (distributed memory). One basic principle is that communication between a processor and its memory is much faster than communication between processors with different memory. An example of a modern supercomputer is the Perlmutter supercomputer at Lawrence Berkeley National Lab, which has 3072 CPU-only nodes and 1792 nodes with GPUs, and a total of about 500,000 CPU cores. Each node has 512 GB of memory for a total of 2.3 PB of memory.\nFor our purposes, there is little practical distinction between multi-processor and multi-core situations. The main issue is whether processes share memory or not. In general, I won’t distinguish between cores and processors. We’ll just focus on the number of cores on given personal computer or a given node in a cluster." + "objectID": "labs/lab2-testing.html#lab-exercise", + "href": "labs/lab2-testing.html#lab-exercise", + "title": "Lab 2: Assertions, Exceptions, and Tesing", + "section": "Lab Exercise", + "text": "Lab Exercise\n1- Imagine a function that:\n[option 1] takes in a string and return another string where the space separated words (or tokens) are the same as the input string but sorted according to a specific ordering. The ordering should be determined by the second argument to the function. The ordering could be specified as (1) lexicographic (alphabetic) according to the words (2) lexicographic according to the key produced by sorting the letters of the original word, or (3) numeric.\n[option 2] (in case you want to practice regex and keep stick with the theme of this week’s lecture) takes in a string and returns the first number in that string, returns None if there are no numeric values in the string.\n2- Write an interface for that function (a function name and arguments), but do not implement the function yet (you can have it return an None, or an empty string for now). We will do this in a good old fashioned .py file (not a notebook or a quarto file).\n3- Build a test suite using the pytest package to test that your function works as intended. Add at least 8 test cases with justification for each. Try to cover the main use cases, and as many potential corner cases or boundary conditions as possible.\n4- Now run the test suite. It should fail for all your tests (unless one of them was passing an empty string).\n5- Implement the function. You can do this at one go, or case by case. As you implement a case, you can rerun the test suite and see some of the tests relevant to that cases stopping to fail. When all the tests pass, you are done. This is called test-driven development.\n6- If some cases are still failing, that’s alright, we can use that failing code next week for demonstrating debugging functionality\n7- Make sure you have assertions to check for invalid input types and combinations.\n8- Make your function throw an exception for invalid input combinations.\n9- Now write a loop that calls your function on a variety of inputs including invalid inputs.\n10- In that loop, handle the thown exceptions and print something appropriate, but let the loop continue for later inputs." }, { - "objectID": "units/unit6-parallel.html#some-useful-terminology", - "href": "units/unit6-parallel.html#some-useful-terminology", - "title": "Parallel processing", - "section": "Some useful terminology:", - "text": "Some useful terminology:\n\ncores: We’ll use this term to mean the different processing units available on a single machine or node of a cluster. A given CPU will have multiple cores. (E.g, the AMD EPYC 7763 has 64 cores per CPU.)\nnodes: We’ll use this term to mean the different computers, each with their own distinct memory, that make up a cluster or supercomputer.\nprocesses: instances of a program(s) executing on a machine; multiple processes may be executing at once. A given program may start up multiple processes at once. Ideally we have no more processes than cores on a node.\nworkers: the individual processes that are carrying out the (parallelized) computation. We’ll use worker and process interchangeably.\ntasks: individual units of computation; one or more tasks will be executed by a given process on a given core.\nthreads: multiple paths of execution within a single process; the operating system sees the threads as a single process, but one can think of them as ‘lightweight’ processes. Ideally when considering the processes and their threads, we would the same number of cores as we have processes and threads combined.\nforking: child processes are spawned that are identical to the parent, but with different process IDs and their own memory. In some cases if objects are not changed, the objects in the child process may refer back to the original objects in the original process, avoiding making copies.\nscheduler: a program that manages users’ jobs on a cluster. Slurm is a commonly used scheduler.\nload-balanced: when all the cores that are part of a computation are busy for the entire period of time the computation is running.\nsockets: some of R’s parallel functionality involves creating new R processes (e.g., starting processes via Rscript) and communicating with them via a communication technology called sockets." + "objectID": "labs/py_vs_R.html", + "href": "labs/py_vs_R.html", + "title": "Lab 7: Python vs. R", + "section": "", + "text": "PDF" }, { - "objectID": "units/unit6-parallel.html#distributed-vs.-shared-memory", - "href": "units/unit6-parallel.html#distributed-vs.-shared-memory", - "title": "Parallel processing", - "section": "Distributed vs. shared memory", - "text": "Distributed vs. shared memory\nThere are two basic flavors of parallel processing (leaving aside GPUs): distributed memory and shared memory. With shared memory, multiple processors (which I’ll call cores for the rest of this document) share the same memory. With distributed memory, you have multiple nodes, each with their own memory. You can think of each node as a separate computer connected by a fast network.\n\nShared memory\nFor shared memory parallelism, each core is accessing the same memory so there is no need to pass information (in the form of messages) between different machines. However, unless one is using threading (or in some cases when one has processes created by forking), objects will still be copied when creating new processes to do the work in parallel. With threaded computations, multiple threads can access object(s) without making explicit copies. But in some programming contexts one needs to be careful that the threads on different cores doesn’t mistakenly overwrite places in memory that are used by other cores (this is generally not an issue in Python or R).\nWe’ll cover two types of shared memory parallelism approaches in this unit:\n\nthreaded linear algebra\nmulticore functionality\n\n\nThreading\nThreads are multiple paths of execution within a single process. If you are monitoring CPU usage (such as with top in Linux or Mac) and watching a job that is executing threaded code, you’ll see the process using more than 100% of CPU. When this occurs, the process is using multiple cores, although it appears as a single process rather than as multiple processes.\nNote that this is a different notion than a processor that is hyperthreaded. With hyperthreading a single core appears as two cores to the operating system.\n\n\n\nDistributed memory\nParallel programming for distributed memory parallelism requires passing messages between the different nodes. The standard protocol for doing this is MPI, of which there are various versions, including openMPI.\nWhile there are various Python and R that use MPI behind the scenes, we’ll only cover distributed memory parallelization via Dask, which doesn’t use MPI." + "objectID": "labs/py_vs_R.html#instructions", + "href": "labs/py_vs_R.html#instructions", + "title": "Lab 7: Python vs. R", + "section": "Instructions", + "text": "Instructions\nPlease carefully read the full instructions before starting the assignment, as you will need to decide with your group how to organize and combine your efforts.\nYour group is expected to work on this for ~60 minutes\nIt’s OK if you don’t get through all of the questions in one hour (i.e., you can just submit what you have at the end of section), but it should be clear that you put some thought and effort into the questions your group worked on.\n\n\n\n\n\n\nTip\n\n\n\nDo your best to document your efforts and insights as you go.\n\n\n\nWe will separate into groups of (ideally) 2 or 3. I’ll try to make sure that each group has at least one person with some R experience.\nGroups should try to answer every question in the Main Questions section. Here are some options for working through these questions:\n\nOption A – Many minds, one task: The group works together, discussing and answering each question sequentially.\nOption B – Divide, consult, conquer: Individual group members work on different questions, consulting with each other as they go or as needed.\n\nAt the end of section (say, the last ~10-15 minutes), combine the solutions into one PDF. Each group member will then individually submit copies of this PDF on Gradescope.\n\nMake sure to include the name of each group member at the top of the PDF.\nHow you create the PDF is up to you but some options are:\n\n(recommended) Using Google Docs.\n\nSeparate documents, then combine: Each group member uses a separate Google Doc, then one person combines those documents into a final shared document.\nOne document: Everyone adds to a single document at the same time.\nYou can take screenshots of code / outputs that you run in Jupyter or RStudio (alternatively, the IPython or R consoles), or just copy paste code and outputs directly to the document.\nWhen your final shared document is ready, each group member will export it to PDF and make individual submission to Gradescope.\n\nCreate an R Markdown file with the code and render to PDF. This may be more of a headache to combine if each person is working separately.\n\n\n\n\nNo shows\nIf you miss lab this week, you can form a group of 2-3 students and submit this assignment on Gradescope by Monday October 17th at 10pm." }, { - "objectID": "units/unit6-parallel.html#gpus", - "href": "units/unit6-parallel.html#gpus", - "title": "Parallel processing", - "section": "GPUs", - "text": "GPUs\nGPUs (Graphics Processing Units) are processing units originally designed for rendering graphics on a computer quickly. This is done by having a large number of simple processing units for massively parallel calculation. The idea of general purpose GPU (GPGPU) computing is to exploit this capability for general computation.\nMost researchers don’t program for a GPU directly but rather use software (often machine learning software such as Tensorflow or PyTorch, or other software that automatically uses the GPU such as JAX) that has been programmed to take advantage of a GPU if one is available. The computations that run on the GPU are run in GPU kernels, which are functions that are launched on the GPU. The overall workflow runs on the CPU and then particular (usually computationally-intensive tasks for which parallelization is helpful) tasks are handed off to the GPU. GPUs and similar devices (e.g., TPUs) are often called “co-processors” in recognition of this style of workflow.\nThe memory on a GPU is distinct from main memory on the computer, so when writing code that will use the GPU, one generally wants to avoid having large amounts of data needing to be transferred back and forth between main (CPU) memory and GPU memory. Also, since there is overhead in launching a GPU kernel, one wants to avoid launching a lot of kernels relative to the amount of work being done by each kernel." + "objectID": "labs/py_vs_R.html#questions", + "href": "labs/py_vs_R.html#questions", + "title": "Lab 7: Python vs. R", + "section": "Questions", + "text": "Questions\n\nMain Questions\n\n\n\n\n\n\nTip\n\n\n\nIdeally, you will make it through all of these, although if you run out of time that is okay.\n\n\n\nDo R functions behave like pass-by-value or pass-by-reference? In other words, if you pass in an object and modify it, does that affect the value of the object in the environment from which the function was called? Check this for a scalar, a list, and an R vector.\nCan R lists and vectors be modified in place, without copying the object?\nFor this the function .Internal(inspect) will be helpful. Here’s an example for a list.\n\n#| eval: false\nx <- list(7, c('abc', 'def'), rnorm(5))\n.Internal(inspect(x))\n\n@5652776540f8 19 VECSXP g0c3 [REF(1)] (len=3, tl=0)\n@5652776b9740 14 REALSXP g0c1 [REF(3)] (len=1, tl=0) 7\n@56527580b0d8 16 STRSXP g0c2 [REF(1)] (len=2, tl=0)\n@5652776b97b0 09 CHARSXP g0c1 [REF(4),gp=0x60] [ASCII] [cached] \"abc\"\n@565275b60168 09 CHARSXP g0c1 [MARK,REF(14),gp=0x61] [ASCII] [cached] \"def\"\n@565275a97478 14 REALSXP g0c4 [REF(1)] (len=5, tl=0) -0.38248,-0.100364,-0.485605,1.15111,-0.111647\n\n#`5652776540f8` is the address of the overall list.\n#`5652776b9740` of the 1-element vector containing '7'.\n#`56527580b0d8` is the address of the character vector.\n#`565275a97478` is the address of the vector of random numbers.\n\nDoes R behave similarly to Python in terms of storing strings, as seen in PS4?\nIf you make a copy of an R vector does it use the same memory as the original vector and does changing an element of the original vector affect the copy of the vector?\nHow does variable scoping work in R - does it use lexical scoping and look for variables in the environment where a function was defined?\nCan you create a closure with embedded data, like we did in Python?\n\n\n\nAdditional Questions\n\n\n\n\n\n\nTip\n\n\n\nWork on these if you finish quickly/are curious\n\n\n\nConsider the relative efficiency of for loops versus vectorized calculations vs. apply for numeric vectors in R and see how it compares to the equivalent operations in python.\nCan you determine if the speed of looking up values in a named vector varies with the size of the dictionary (this will indicate if something like hashing is going on or if the lookup has to scan through all the elements)." }, { - "objectID": "units/unit6-parallel.html#some-other-approaches-to-parallel-processing", - "href": "units/unit6-parallel.html#some-other-approaches-to-parallel-processing", - "title": "Parallel processing", - "section": "Some other approaches to parallel processing", - "text": "Some other approaches to parallel processing\n\nSpark and Hadoop\nSpark and Hadoop are systems for implementing computations in a distributed memory environment, using the MapReduce approach, as discussed in Unit 7.\n\n\nCloud computing\nAmazon (Amazon Web Services’ EC2 service), Google (Google Cloud Platform’s Compute Engine service) and Microsoft (Azure) offer computing through the cloud. The basic idea is that they rent out their servers on a pay-as-you-go basis. You get access to a virtual machine that can run various versions of Linux or Microsoft Windows server and where you choose the number of processing cores you want. You configure the virtual machine with the applications, libraries, and data you need and then treat the virtual machine as if it were a physical machine that you log into as usual. You can also assemble multiple virtual machines into your own virtual cluster and use platforms such as databases and Spark on the cloud provider’s virtual machines." + "objectID": "publish.html", + "href": "publish.html", + "title": "Statistics 243 Fall 2023", + "section": "", + "text": "We use Quarto with GitHub Actions to publish the materials as the course website. All commits should be done to main and not to gh-pages.\nThis documents creation of new pages that require Python or R computation. We render the source document locally, with ‘freeze’ set on the document so that results are stored in _freeze and don’t need to be re-run on every commit.\n\nCopy the preamble from an existing units/*.qmd file.\nUpdate _quarto.yml to reflect the new content (unless you don’t yet want it discoverable online).\nRun quarto render <new-Rmd-or-qmd> locally, which will store computations in the _freeze directory.\nCommit the new page and all changes to the _freeze dir including .json and figure (.png/pdf) files.\nPush to GitHub and the publish action should run fairly quickly via GitHub actions.\n\nTo set up the website for a new course/year, one needs to run quarto publish from within main initially.\nNotes:\n2023-09-07: GHA can fail with messages about nbformat. Can often fix by re-rendering the problematic qmd. I think this is happening when commits are made to a qmd without rendering that updates the freeze files.\n2023-08-29: when using knitr engine with unit3-bash.qmd (so that one can work with bash chunks), some GHA runs are complaining about missing rmarkdown. But then it sometimes works. Trying to install rmarkdown leads to a permission issue in the system directory that the R package is being installed into. If try to use jupyter engine with bash chunks, you probably need a Jupyter bash kernel, but I am still investigating." }, { - "objectID": "units/unit6-parallel.html#overview-key-idea", - "href": "units/unit6-parallel.html#overview-key-idea", - "title": "Parallel processing", - "section": "Overview: Key idea", - "text": "Overview: Key idea\nA key idea in Dask (and in R’s future package and the ray package for Python) involve abstracting (i.e, divorcing/separating) the specification of the parallelization in the code away from the computational resources that the code will be run on. We want to:\n\nSeparate what to parallelize from how and where the parallelization is actually carried out.\nAllow different users to run the same code on different computational resources (without touching the actual code that does the computation).\n\nThe computational resources on which the code is run is sometimes called the backend." + "objectID": "labs/06/scf.html", + "href": "labs/06/scf.html", + "title": "SCF Computing Cluster", + "section": "", + "text": "PDF" }, { - "objectID": "units/unit6-parallel.html#overview-of-parallel-backends", - "href": "units/unit6-parallel.html#overview-of-parallel-backends", - "title": "Parallel processing", - "section": "Overview of parallel backends", - "text": "Overview of parallel backends\nOne sets the scheduler to control how parallelization is done, whether to run code on multiple machines, and how many cores on each machine to use.\nFor example to parallelize across multiple cores via separate Python processes, we’d do this.\n\nimport dask\ndask.config.set(scheduler='processes', num_workers = 4) \n\nThis table shows the different types of schedulers.\n\n\n\n\n\n\n\n\n\nType\nDescription\nMulti-node\nCopies of objects made?\n\n\n\n\nsynchronous\nnot in parallel (serial)\nno\nno\n\n\nthreads (1)\nthreads within current Python session\nno\nno\n\n\nprocesses\nbackground Python sessions\nno\nyes\n\n\ndistributed (2)\nPython sessions across multiple nodes\nyes\nyes\n\n\n\nComments:\n\nNote that because of Python’s Global Interpreter Lock (GIL) (which prevents threading of Python code), many computations done in pure Python code won’t be parallelized using the ‘threads’ scheduler; however computations on numeric data in numpy arrays, Pandas dataframes and other C/C++/Cython-based code will parallelize.\nIt’s fine to use the distributed scheduler on one machine, such as your laptop. According to the Dask documentation, it has advantages over multiprocessing, including the diagnostic dashboard (see the tutorial) and better handling of when copies need to be made. In addition, one needs to use it for parallel map operations (see next section)." + "objectID": "labs/06/scf.html#logging-in", + "href": "labs/06/scf.html#logging-in", + "title": "SCF Computing Cluster", + "section": "Logging in", + "text": "Logging in\nThe SCF has a number of login nodes which you can access via ssh.\n\n\n\n\n\n\nNote\n\n\n\nFor info on using ssh (including on Windows), see here.\n\n\nFor example, I’ll use ssh to connect to the dorothy node:\njames@pop-os:~$ ssh <scf-username>@dorothy.berkeley.edu\nThe authenticity of host 'dorothy.berkeley.edu (128.32.135.58)' can't be established.\nED25519 key fingerprint is SHA256:rOY7ED/iIiTgI++Y4XHmiEl+tC+OmSGBvWp03CSII5E.\nThis key is not known by any other names\nAre you sure you want to continue connecting (yes/no/[fingerprint])? yes\nWarning: Permanently added 'dorothy.berkeley.edu' (ED25519) to the list of known hosts.\nNotice that upon first connecting to a server you haven’t visited there is a warning that the “authenticity of the host … can’t be established”. So long as you have typed in the hostname correctly (dorothy.berkeley.edu, in this case), and trust the host (we trust the SCF!) then you can type yes to add the host to your known hosts file (found on your local machine at ~/.ssh/known_hosts).\nYou’ll then be asked to enter your password for the SCF cluster. For privacy, you won’t see anything happen in your terminal when you type it in, so type carefully (you can use Backspace if you make a mistake) and press Enter when you’re done. If you were successful, you should see a welcome message and your shell prompt, like:\n<scf-username>@dorothy:~$\nTo get your bearings, you can type pwd to see where your home directory is located on the SCF cluster filesystem:\n<scf-username>@dorothy:~$ pwd\n/accounts/grad/<scf-username>\nYour home directory is likely also in the /accounts/grad/ directory, as mine is.\n\nOther login nodes\n\n\n\n\n\n\nImportant\n\n\n\nDon’t run computationally intensive tasks on the login nodes!\nThey are shared by all the SCF users, and should only be used for non-intensive interactive work such as job submission and monitoring, basic compilation, managing your disk space, and transferring data to/from the server.\n\n\nIf for some reason dorothy is not working for you, the SCF has a number of nodes which can be accessed from your local machine with commands of the form ssh <scf-username>@<hostname>.berkeley.edu. Currently, these are:\n\naragorn\narwen\ndorothy\ngandalf\ngollum\nhermione\nquidditch\nradagast\nshelob" }, { - "objectID": "units/unit6-parallel.html#accessing-variables-and-workers-in-the-worker-processes", - "href": "units/unit6-parallel.html#accessing-variables-and-workers-in-the-worker-processes", - "title": "Parallel processing", - "section": "Accessing variables and workers in the worker processes", - "text": "Accessing variables and workers in the worker processes\nDask usually does a good job of identifying the packages and (global) variables you use in your parallelized code and importing those packages on the workers and copying necessary variables to the workers.\nHere’s a toy example that shows that the numpy package and a global variable n are automatically available in the worker processes without any action on our part.\nNote the use of the @delayed decorator to flag the function so that it operates in a lazy manner for use with Dask’s parallelization capabilities.\n\nimport dask\ndask.config.set(scheduler='processes', num_workers = 4, chunksize = 1) \n\n<dask.config.set object at 0x7fcd37ff9d10>\n\nimport numpy as np\nn = 10\n\n@dask.delayed\ndef myfun(idx):\n return np.random.normal(size = n)\n\n\ntasks = []\np = 8\nfor i in range(p):\n tasks.append(myfun(i)) # add lazy task\n\ntasks\n\n[Delayed('myfun-fb6d5933-5a6e-455f-87c2-aee3ca9aadd3'), Delayed('myfun-62938a41-9c82-4e80-b9ee-09aba30dcbfb'), Delayed('myfun-91faca5a-ff4b-4eef-8736-acf786d1cbe3'), Delayed('myfun-e85f3b84-3ce1-477f-91a4-9069abb33748'), Delayed('myfun-1d342e40-ff5d-41a1-b10c-7f686d8714c0'), Delayed('myfun-73e39dc6-32fb-4cc5-9e8a-03e8ace1db71'), Delayed('myfun-e984bae0-6a28-480e-9165-86a980f5198e'), Delayed('myfun-b57c623a-d447-4d68-ae93-479e96cc5546')]\n\nresults = dask.compute(tasks) # compute all in parallel\n\nIn other contexts (in various languages) you may need to explicitly copy objects to the workers (or load packages on the workers). This is sometimes called exporting variables.\nWe don’t have to flag the function in advanced with @delayed. We could also have directly called the decorator like this, which as the advantage of allowing us to run the function in the normal way if we simply invoke it.\n\ntasks.append(dask.delayed(myfun)(i))" + "objectID": "labs/06/scf.html#disk-space", + "href": "labs/06/scf.html#disk-space", + "title": "SCF Computing Cluster", + "section": "Disk space", + "text": "Disk space\nYour home directory has a limited amount of disk space; you can check how much you have used and available using the quota command, which will show your home directory usage and quota on the line for the accounts filesystem.\n<scf-username>@dorothy:~$ quota\nFILESYSTEM USE QUOTA %\n accounts 5.29 G 20 G 26.5\nThe accounts filesystem is accessible from all the nodes on the SCF cluster. This means that regardless of which login or compute node you use, you will have access to the files in your home directory. Moreover, your home directory is backed up regularly, so you are protected from accidental data loss.\nIf you’re running out of space, you should try to selectively delete large files that you no longer need. See here for some tips on finding large files. If all else fails, you can request additional space or request access the /scratch filesystem. The latter is a good place for large datasets, but note that /scratch is not backed up, unlike your home directory. See the previous link for more info.\nFor temporary files (e.g., intermediate results of a computation that you don’t need to store for later), every machine has a /tmp filesystem. However, /tmp is always linked to the specific machine you are on, meaning that if you put something in /tmp on dorothy and then later go to aragorn, you won’t find your files in the /tmp directory there. If you go back to dorothy, you will likely find them again, but /tmp is automatically wiped when a machine reboots, so only use it for files you don’t care about preserving!" }, { - "objectID": "units/unit6-parallel.html#scenario-1-one-model-fit", - "href": "units/unit6-parallel.html#scenario-1-one-model-fit", - "title": "Parallel processing", - "section": "Scenario 1: one model fit", - "text": "Scenario 1: one model fit\nSpecific scenario: You need to fit a single statistical/machine learning model, such as a random forest or regression model, to your data.\nGeneral scenario: Parallelizing a single task.\n\nScenario 1A:\nA given method may have been written to use parallelization and you simply need to figure out how to invoke the method for it to use multiple cores.\nFor example the documentation for the RandomForestClassifier in scikit-learn’s ensemble module indicates it can use multiple cores – note the n_jobs argument (not shown here because the help info is very long).\n\nimport sklearn.ensemble\nhelp(sklearn.ensemble.RandomForestClassifier)\n\nYou’ll usually need to look for an argument with one of the words threads, processes, cores, cpus, jobs, etc. in the argument name.\n\n\nScenario 1B:\nIf a method does linear algebra computations on large matrices/vectors, Python (and R) can call out to parallelized linear algebra packages (the BLAS and LAPACK).\nThe BLAS is the library of basic linear algebra operations (written in Fortran or C). A fast BLAS can greatly speed up linear algebra in R relative to the default BLAS that comes with R. Some fast BLAS libraries are\n\nIntel’s MKL; available for educational use for free\nOpenBLAS; open source and free\nApple’s Accelerate framework BLAS (vecLib) for Macs; provided with your Mac\n\nIn addition to being fast when used on a single core, all of these BLAS libraries are threaded - if your computer has multiple cores and there are free resources, your linear algebra will use multiple cores, provided your program is linked against the threaded BLAS installed on your machine and provided the shell environment variable OMP_NUM_THREADS is not set to one. (Macs make use of VECLIB_MAXIMUM_THREADS rather than OMP_NUM_THREADS and if MKL is being used, then one needs MKL_NUM_THREADS)\nFor parallel (threaded) linear algebra in Python, one can use an optimized BLAS with the numpy and (therefore) scipy packages, on Linux or using the Mac’s vecLib BLAS. Details will depend on how you install Python, numpy, and scipy. More details on figuring out what BLAS is being used and how to install a fast threaded BLAS on your own computer are here.\nDask and some other packages also provide threading, but pure Python code is not threaded.\nHere’s some code that illustrates the speed of using a threaded BLAS:\n\nimport numpy as np\nimport time\n\nx = np.random.normal(size = (6000, 6000))\n\nstart_time = time.time()\nx = np.dot(x.T, x) \nU = np.linalg.cholesky(x)\nelapsed_time = time.time() - start_time\nprint(\"Elapsed Time (8 threads):\", elapsed_time)\n\nWe’d need to restart Python after setting OMP_NUM_THREADS to 1 in order to compare the time when run in parallel vs. on a single core. That’s hard to demonstrate in this generated document, but when I ran it, it took 6.6 seconds, compared to 3 seconds using 8 cores.\nNote that for smaller linear algebra problems, we may not see any speed-up or even that the threaded calculation might be slower because of overhead in setting up the parallelization and because the parallelized linear algebra calculation involves more actual operations than when done serially.\n\n\nGPUs and linear algebra\nLinear algebra with large matrices is often a very good use case for GPUs.\nHere’s an example of using the GPU to multiply large matrices using PyTorch. We could do this similarly with Tensorflow or JAX. I’ve just inserted the timing from running this on an SCF machine with a powerful GPU.\n\nimport torch\n\nstart = torch.cuda.Event(enable_timing=True)\nend = torch.cuda.Event(enable_timing=True)\n\ngpu = torch.device(\"cuda:0\")\n\nn = 10000\nx = torch.randn(n,n, device = gpu)\ny = torch.randn(n,n, device = gpu)\n\n## Time the matrix multiplication on GPU:\nstart.record()\nz = torch.matmul(x, y)\nend.record()\ntorch.cuda.synchronize()\nprint(start.elapsed_time(end)) # 120 ms.\n\n## Compare to CPU:\ncpu = torch.device(\"cpu\")\n\nx = torch.randn(n,n, device = cpu)\ny = torch.randn(n,n, device = cpu)\n\n## Time the matrix multiplication on CPU:\nstart.record()\nz = torch.matmul(x, y)\nend.record()\ntorch.cuda.synchronize()\nprint(start.elapsed_time(end)) # 18 sec.\n\nThe GPU calculation takes 100-200 milliseconds (ms), while the CPU calculation took 18 seconds using two CPU cores. That’s a speed-up of more than 100x!\nFor a careful comparison between GPU and CPU, we’d want to consider the effect of using 4-byte floating point numbers for the GPU calculation.\nWe’d also want to think about how many CPU cores should be used for the comparison." + "objectID": "labs/06/scf.html#data-transfer-scp-sftp", + "href": "labs/06/scf.html#data-transfer-scp-sftp", + "title": "SCF Computing Cluster", + "section": "Data transfer: SCP / SFTP", + "text": "Data transfer: SCP / SFTP\nWe can use the scp and sftp protocols to transfer files to and from any login node on the SCF cluster. scp is a shell program that should be available by default on macOS and Linux, while on Windows you can use WinSCP. WinSCP can also do sftp transfers, or you can use FileZilla on any platform for sftp. Both WinSCP and FileZilla have a GUI that allows you to drag and drop files between your local machine and a remote host.\nThe syntax for scp is scp <from-place> <to-place>, and you’ll typically use scp on your local machine. For example, we’ll show how to transfer the file data.csv (you can find it here) :\n\nTo SCF while on your local machine\n# transfer to my home directory without renaming the file \nscp data.csv <scf-username>@dorothy.berkeley.edu:~/\n\n# transfer to the data/ subdirectory and rename the file\nscp data.csv <scf-username>@dorothy.berkeley.edu:~/data/new_name.csv\n\n# transfer to the dorothy-specific /tmp/ directory\nscp data.csv <scf-username>@dorothy.berkeley.edu:/tmp/\n\n\nFrom SCF while on your local machine\n# now <from-place> is a path in my home directory on the SCF \nscp <scf-username>@dorothy.berkeley.edu:~/data/new_name.csv ~/Desktop/data_scf.csv\nFor more information, see here. In particular, if you have a very large dataset to transfer, Globus is a better option than either scp or sftp." }, { - "objectID": "units/unit6-parallel.html#scenario-2-three-different-prediction-methods-on-your-data", - "href": "units/unit6-parallel.html#scenario-2-three-different-prediction-methods-on-your-data", - "title": "Parallel processing", - "section": "Scenario 2: three different prediction methods on your data", - "text": "Scenario 2: three different prediction methods on your data\nSpecific scenario: You need to fit three different statistical/machine learning models to your data.\nGeneral scenario: Parallelizing a small number of tasks.\nWhat are some options?\n\nuse one core per model\nif you have rather more than three cores, apply the ideas here combined with Scenario 1 above - with access to a cluster and parallelized implementations of each model, you might use one node per model\n\nHere we’ll use the processes scheduler. In principal given this relies on numpy code, we could have also used the threads scheduler, but I’m not seeing effective parallelization when I try that.\n\nimport dask\nimport time\nimport numpy as np\n\ndef gen_and_mean(func, n, par1, par2):\n return np.mean(func(par1, par2, size = n))\n\ndask.config.set(scheduler='processes', num_workers = 3, chunksize = 1) \n\n<dask.config.set object at 0x7fcd37ffe810>\n\nn = 100000000\nt0 = time.time()\ntasks = []\ntasks.append(dask.delayed(gen_and_mean)(np.random.normal, n, 0, 1))\ntasks.append(dask.delayed(gen_and_mean)(np.random.gamma, n, 1, 1))\ntasks.append(dask.delayed(gen_and_mean)(np.random.uniform, n, 0, 1))\nresults = dask.compute(tasks)\nprint(time.time() - t0) \n\n3.6668434143066406\n\nt0 = time.time()\np = gen_and_mean(np.random.normal, n, 0, 1)\nq = gen_and_mean(np.random.gamma, n, 1, 1)\ns = gen_and_mean(np.random.uniform, n, 0, 1)\nprint(time.time() - t0) \n\n5.191020488739014\n\n\nQuestion: Why might this not have shown a perfect three-fold speedup?\nYou could also have used tools like a parallel map here as well, as we’ll discuss in the next scenario.\n\nLazy evaluation, synchronicity, and blocking\nIf we look at the delayed objects, we see that each one is a representation of the computation that needs to be done and that execution happens lazily. Also note that dask.compute executes synchronously, which means the main process waits until the dask.compute call is complete before allowing other commands to be run. This synchronous evaluation is also called a blocking call because execution of the task in the worker processes blocks the main process. In contrast, if control returns to the user before the worker processes are done, that would be asynchronous evaluation (aka, a non-blocking call).\nNote: the use of chunksize = 1 forces Dask to immediately start one task on each worker. Without that argument, by default it groups tasks so as to reduce the overhead of starting each task individually, but when we have few tasks, that prevents effective parallelization. We’ll discuss this in much more detail in Scenario 4." + "objectID": "labs/lab1-submission.html", + "href": "labs/lab1-submission.html", + "title": "Lab 1: Submitting problem set solutions", + "section": "", + "text": "By now you should already have access to the following 5 basic tools:\n\nUnix shell\nGit\nQuarto\nPython\nA text editor of your choice\n\nToday we will use all these tools together to submit a solution for Problem Set 0 (not a real problem set) to make sure you know how to submit solutions to upcoming (real) problem sets.\nHere is a selection of some basic reference tutorials and documentation for unix, bash and unix commands, git & GitHub, quarto, python and VS Code\nSome books to learn more about Unix." }, { - "objectID": "units/unit6-parallel.html#scenario-3-10-fold-cv-and-10-or-fewer-cores", - "href": "units/unit6-parallel.html#scenario-3-10-fold-cv-and-10-or-fewer-cores", - "title": "Parallel processing", - "section": "Scenario 3: 10-fold CV and 10 or fewer cores", - "text": "Scenario 3: 10-fold CV and 10 or fewer cores\nSpecific scenario: You are running a prediction method on 10 cross-validation folds.\nGeneral scenario: Parallelizing tasks via a parallel map.\nThis illustrates the idea of running some number of tasks using the cores available on a single machine.\nHere I’ll illustrate using a parallel map, using this simulated dataset and basic use of RandomForestRegressor().\nFirst, let’s set up our fit function and simulate some data.\nIn this case our fit function uses global variables. The reason for this is that we’ll use Dask’s map function, which allows us to pass only a single argument. We could bundle the input data with the fold_idx value and pass as a larger object, but here we’ll stick with the simplicity of global variables.\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import KFold\n\ndef cv_fit(fold_idx):\n train_idx = folds != fold_idx\n test_idx = folds == fold_idx\n X_train = X.iloc[train_idx]\n X_test = X.iloc[test_idx]\n Y_train = Y[train_idx]\n model = RandomForestRegressor()\n model.fit(X_train, Y_train)\n predictions = model.predict(X_test)\n return predictions\n\n\nnp.random.seed(1)\n\n# Generate data\nn = 1000\np = 50\nX = pd.DataFrame(np.random.normal(size = (n, p)),\\\n columns=[f\"X{i}\" for i in range(1, p + 1)])\nY = X['X1'] + np.sqrt(np.abs(X['X2'] * X['X3'])) +\\\n X['X2'] - X['X3'] + np.random.normal(size = n)\n\nn_folds = 10\nseq = np.arange(n_folds)\nfolds = np.random.permutation(np.repeat(seq, 100))\n\nTo do a parallel map, we need to use the distributed scheduler, but it’s fine to do that with multiple cores on a single machine (such as a laptop).\n\nn_cores = 2\nfrom dask.distributed import Client, LocalCluster\ncluster = LocalCluster(n_workers = n_cores)\nc = Client(cluster)\n\ntasks = c.map(cv_fit, range(n_folds))\nresults = c.gather(tasks)\n# We'd need to sort the results appropriately to align them with the observations.\n\nNow suppose you have 4 cores (and therefore won’t have an equal number of tasks per core with the 10 tasks). The approach in the next scenario should work better." + "objectID": "labs/lab1-submission.html#submitting-problem-set-solutions-090123", + "href": "labs/lab1-submission.html#submitting-problem-set-solutions-090123", + "title": "Lab 1: Submitting problem set solutions", + "section": "", + "text": "By now you should already have access to the following 5 basic tools:\n\nUnix shell\nGit\nQuarto\nPython\nA text editor of your choice\n\nToday we will use all these tools together to submit a solution for Problem Set 0 (not a real problem set) to make sure you know how to submit solutions to upcoming (real) problem sets.\nHere is a selection of some basic reference tutorials and documentation for unix, bash and unix commands, git & GitHub, quarto, python and VS Code\nSome books to learn more about Unix." }, { - "objectID": "units/unit6-parallel.html#scenario-4-parallelizing-over-prediction-methods", - "href": "units/unit6-parallel.html#scenario-4-parallelizing-over-prediction-methods", - "title": "Parallel processing", - "section": "Scenario 4: parallelizing over prediction methods", - "text": "Scenario 4: parallelizing over prediction methods\nScenario: parallelizing over prediction methods or other cases where execution time varies.\nIf you need to parallelize over prediction methods or in other contexts in which the computation time for the different tasks varies widely, you want to avoid having the parallelization group the tasks into batches in advance, because some cores may finish a lot more quickly than others. Starting the tasks one by one (not in batches) is called dynamic allocation.\nIn contrast, if the computation time is about the same for the different tasks (or if you have so many tasks that the effect of averaging helps with load-balancing) then you want to group the tasks into batches. This is called static allocation or prescheduling. This avoids the extra overhead (~1 millisecond per task) of scheduling many tasks.\n\nDynamic allocation\nWith Dask’s distributed scheduler, Dask starts up each delayed evaluation separately (i.e., dynamic allocation).\nWe’ll set up an artificial example with four slow tasks and 12 fast tasks and see the speed of running with the default of dynamic allocation under Dask’s distributed scheduler. Then in the next section, we’ll compare to the worst-case scenario with all four slow tasks in a single batch.\n\nimport scipy.special\n\nn_cores = 4\nfrom dask.distributed import Client, LocalCluster\ncluster = LocalCluster(n_workers = n_cores)\nc = Client(cluster)\n\n## 4 slow tasks and 12 fast ones.\nn = np.repeat([10**7, 10**5, 10**5, 10**5], 4)\n\ndef fun(i):\n print(f\"Working on {i}.\")\n return np.mean(scipy.special.gammaln(np.exp(np.random.normal(size = n[i]))))\n \n\nt0 = time.time()\nout = fun(1)\n\nWorking on 1.\n\nprint(time.time() - t0)\n\n0.5563864707946777\n\nt0 = time.time()\nout = fun(5)\n\nWorking on 5.\n\nprint(time.time() - t0)\n\n0.01022791862487793\n\nt0 = time.time()\ntasks = c.map(fun, range(len(n)))\nresults = c.gather(tasks) \nprint(time.time() - t0) # 0.8 sec. \n\n0.8854873180389404\n\ncluster.close()\n\nNote that with relatively few tasks per core here, we could have gotten unlucky if the tasks were in a random order and multiple slow tasks happen to be done by a single worker.\n\n\nStatic allocation\nNext, note that by default the ‘processes’ scheduler sets up tasks in batches, with a default chunksize of 6. In this case that means that the first 4 (slow) tasks are all allocated to a single worker.\n\ndask.config.set(scheduler='processes', num_workers = 4)\n\n<dask.config.set object at 0x7fcd2391ac50>\n\ntasks = []\np = len(n)\nfor i in range(p):\n tasks.append(dask.delayed(fun)(i)) # add lazy task\n\nt0 = time.time()\nresults = dask.compute(tasks) # compute all in parallel\nprint(time.time() - t0) # 2.6 sec.\n\n1.7515089511871338\n\n\nTo force dynamic allocation, we can set chunksize = 1 (as was shown in our original example of using the processes scheduler).\n\ndask.config.set(scheduler='processes', num_workers = 4, chunksize = 1)\n\ntasks = []\np = len(n)\nfor i in range(p):\n tasks.append(dask.delayed(fun)(i)) # add lazy task\n\nresults = dask.compute(tasks) # compute all in parallel\n\n\n\nChoosing static vs. dynamic allocation in Dask\nWith the distributed scheduler, Dask starts up each delayed evaluation separately (i.e., dynamic allocation). And even with a distributed map() it doesn’t appear possible to ask that the tasks be broken up into batches. Therefore if you want static allocation, you could use the processes scheduler if using a single machine, or if you need to use the distributed schduler you could break up the tasks into batches manually.\nWith the processes scheduler, static allocation is the default, with a default chunksize of 6 tasks per batch. You can force dynamic allocation by setting chunksize = 1.\n(Note that in R, static allocation is the default when using the future package.)" + "objectID": "labs/lab1-submission.html#quick-intro-to-git-and-github", + "href": "labs/lab1-submission.html#quick-intro-to-git-and-github", + "title": "Lab 1: Submitting problem set solutions", + "section": "Quick Intro to git and GitHub", + "text": "Quick Intro to git and GitHub\n\nCreating a enw repository\nMaking changes\n\n\nEditing and saving files\nStaging changes\nCommitting changes locally\nPushing changes to remote repository\n\n\nUndoing changes:\n\n\nLocal changes\nLocal staged changes\nLocal commited changes\nPushed changes\n\n\nMerging divergent versions\nWorking with branches\nGUI options (sourcetree)\nGetting help\n\nDiscussion: - Why is git so damn complicated? - What do you need to remember when working with collaborators on the same repository?" }, { - "objectID": "units/unit6-parallel.html#scenario-5-10-fold-cv-across-multiple-methods-with-many-more-than-10-cores", - "href": "units/unit6-parallel.html#scenario-5-10-fold-cv-across-multiple-methods-with-many-more-than-10-cores", - "title": "Parallel processing", - "section": "Scenario 5: 10-fold CV across multiple methods with many more than 10 cores", - "text": "Scenario 5: 10-fold CV across multiple methods with many more than 10 cores\nSpecific scenario: You are running an ensemble prediction method such as SuperLearner or Bayesian model averaging on 10 cross-validation folds, with many statistical/machine learning methods.\nGeneral scenario: parallelizing nested tasks or a large number of tasks, ideally across multiple machines.\nHere you want to take advantage of all the cores you have available, so you can’t just parallelize over folds.\nFirst we’ll discuss how to deal with the nestedness of the problem and then we’ll talk about how to make use of many cores across multiple nodes to parallelize over a large number of tasks.\n\nScenario 5A: nested parallelization\nOne can always flatten the looping, either in a for loop or in similar ways when using apply-style statements.\n\n## original code: multiple loops \nfor fold in range(n):\n for method in range(M):\n ### code here \n \n## revised code: flatten the loops \nfor idx in range(n*M): \n fold = idx // M \n method = idx % M \n print(idx, fold, method)### code here \n\nRather than flattening the loops at the loop level (which you’d need to do to use map), one could just generate a list of delayed tasks within the nested loops.\n\nfor fold in range(n):\n for method in range(M):\n tasks.append(dask.delayed(myfun)(fold,method))\n\nThe future package in R has some nice functionality for easily parallelizing with nested loops.\n\n\nScenario 5B: Parallelizing across multiple nodes\nIf you have access to multiple machines networked together, including a Linux cluster, you can use Dask to start workers across multiple nodes (either in a nested parallelization situation with many total tasks or just when you have lots of unnested tasks to parallelize over). Here we’ll just illustrate how to use multiple nodes, but if you had a nested parallelization case you can combine the ideas just above with the use of multiple nodes.\nSimply start Python as you usually would. Then the following code will parallelize on workers across the machines specified.\n\nfrom dask.distributed import Client, SSHCluster\n# First host is the scheduler.\ncluster = SSHCluster(\n [\"gandalf.berkeley.edu\", \"radagast.berkeley.edu\", \"radagast.berkeley.edu\",\n \"arwen.berkeley.edu\", \"arwen.berkeley.edu\"]\n)\nc = Client(cluster)\n\n## On the SCF, Savio and other clusters using the SLURM scheduler,\n## you can figure out the machine names like this, repeating the\n## first machine for the scheduler:\n## \n## machines = subprocess.check_output(\"srun hostname\", shell = True,\n## universal_newlines = True).strip().split('\\n')\n## machines = [machines[0]] + machines\n\ndef fun(i, n=10**6):\n return np.mean(np.random.normal(size = n))\n\nn_tasks = 120\n\ntasks = c.map(fun, range(n_tasks))\nresults = c.gather(tasks)\n\n## And just to check we are actually using the various machines:\nimport subprocess\n\nc.gather(c.map(lambda x: subprocess.check_output(\"hostname\", shell = True), \\\n range(4)))\n\ncluster.close()" + "objectID": "labs/lab1-submission.html#hands-on-lab-instructions", + "href": "labs/lab1-submission.html#hands-on-lab-instructions", + "title": "Lab 1: Submitting problem set solutions", + "section": "Hands-on Lab Instructions", + "text": "Hands-on Lab Instructions\nProblem Set submission instructions:\n\nOpen the qmd file in any editor you like (e.g., Emacs, Sublime, ….). Use quarto preview FILE to show your rendered document live as you edit and save changes. You can put the preview window side by side with your editor, and the preview document should automatically render as you save your qmd file.\nUse VS Code with the following extensions: Python, Quarto, and Jupyter Notebooks. This allows you to execute and preview chunks (and whole document) inside VS Code. This is currently deeb’s favorite path due to how well it integrated with the Python debugger.\nUse RStudio (yes, RStudio), which can manage Python code and will display chunk output in the same way it does with R chunks. This path seems to work quite well and is recommended if you are already familiar with RStudio.\n\n\nSteps to perform today:\n\nClone your github repository to your development environment\nCreate a subdirectory in your github repository with the name ps0\nIn that subdirectory, create a quarto document (ps0.qmd) that has some simple code that creates a simple plot (you can follow this example/tutorial here)\nUse the quarto command line to render it into a pdf document (quarto render FILE –to pdf)\nCommit the changes to your repository (git add FILES; git commit -m MESSAGE; git push)\nAdd another section to your quarto document (use your imagination), then preview and commit the changes\nUse the quarto command line to render the updated document into a pdf document\nAdd the pdf document to the repository as well\nMake sure that you can log into gradescope and upload a pdf document\n[optional] Undo your last set of changes and regenerate the pdf file\n\nIf we finish early, We will also take today’s lab as an opportunity to get familiar with the basic use of all the 5 basic tools listed above.\nFor git and quarto, very basic knowledge should be sufficient for now, but for unix commands and python, the more you learn the more effective you will be at solving the problem sets (and at any computational task you take on after that). You will need to learn more advanced use of git and github towards the end of the semester when you start working with other team members on the same project.\n\n\nChunk options\nLike RMarkdown, quarto allows for several execution options to be set per document and per chunk. Spend some time getting familiar with the various options, and keep this link handy when you are working on the first few problem sets.\nDepending on what’s required in the problem sets, you may need to set eval to false (just print out code) or error to true (print errors and don’t halt rendering of the document). Some of the other options may be useful for controlling how the code gets printed." }, { - "objectID": "units/unit6-parallel.html#scenario-6-stratified-analysis-on-a-very-large-dataset", - "href": "units/unit6-parallel.html#scenario-6-stratified-analysis-on-a-very-large-dataset", - "title": "Parallel processing", - "section": "Scenario 6: Stratified analysis on a very large dataset", - "text": "Scenario 6: Stratified analysis on a very large dataset\nSpecific scenario: You are doing stratified analysis on a very large dataset and want to avoid unnecessary copies.\nGeneral scenario: Avoiding copies when working with large data in parallel.\nIn many parallelization tools, if you try to parallelize this case on a single node, you end up making copies of the original dataset, which both takes up time and eats up memory.\nHere when we use the processes scheduler, we make copies for each task.\n\ndef do_analysis(i,x):\n # A fake \"analysis\", identical for each task.\n print(id(x)) # Check the number of copies.\n return np.mean(x)\n\nn_cores = 4\n\nx = np.random.normal(size = 5*10**7) # our big \"dataset\"\n\ndask.config.set(scheduler='processes', num_workers = n_cores, chunksize = 1)\n\n<dask.config.set object at 0x7fcd2310e810>\n\ntasks = []\np = 8\nfor i in range(p):\n tasks.append(dask.delayed(do_analysis)(i,x))\n\nt0 = time.time()\nresults = dask.compute(tasks)\nprint(time.time() - t0)\n\n6.334888458251953\n\n\nA better approach is to use the distributed scheduler (which is fine to use on a single machine or multiple machines), which makes one copy per worker instead of one per task, provided you apply delayed() to the global data object.\n\nfrom dask.distributed import Client, LocalCluster\ncluster = LocalCluster(n_workers = n_cores)\nc = Client(cluster)\n\nx = dask.delayed(x)\n\ntasks = []\np = 8\nfor i in range(p):\n tasks.append(dask.delayed(do_analysis)(i,x))\n\nt0 = time.time()\nresults = dask.compute(tasks)\n\n/system/linux/mambaforge-3.11/lib/python3.11/site-packages/distributed/client.py:3141: UserWarning: Sending large graph of size 47.69 MiB.\nThis may cause some slowdown.\nConsider scattering data ahead of time and using futures.\n warnings.warn(\n\nprint(time.time() - t0)\n\n1.1176700592041016\n\ncluster.close()\n\nThat seems to work, though Dask suggests sending the data to the workers in advance. I’m not sure of the distinction between what it is recommending and use of dask.delayed(x).\nFurthermore, the id of x seems to be the same for all the tasks, even though we have four workers. The documentation indicates one copy per worker, but perhaps that’s not actually the case.\nEven better would be to use the threads scheduler, in which case all workers can access the same data objects with no copying (but of course we cannot modify the data in that case without potentially causing problems for the other tasks). Without the copying, this is really fast.\n\ndask.config.set(scheduler='threads', num_workers = n_cores)\n\n<dask.config.set object at 0x7fcd2311e810>\n\ntasks = []\np = 8\nfor i in range(p):\n tasks.append(dask.delayed(do_analysis)(i,x))\n\nt0 = time.time()\nresults = dask.compute(tasks)\n\n140519032960464\n140519032960464\n140519032960464\n140519032960464\n140519032960464\n140519032960464\n140519032960464\n140519032960464\n\nprint(time.time() - t0)\n\n0.06301546096801758" + "objectID": "labs/lab5-codereview.html", + "href": "labs/lab5-codereview.html", + "title": "Lab 5: Code Reviews", + "section": "", + "text": "Pair up in teams of 2 (or 3 if necessary).\nOne member of the team will create a new repository on github and invite the other teammember(s) as collaborators on the repo.\nUsing the Github web UI, each team member will create a new file in the repo, and paste their code for presidential speaches into it.\nCommit you changes (from the github UI) as a pull request / separate branch.\nNow each team member will go to the pull request made by another team member and look at the code changes.\nRead eachother’s code carefully and leave some thoughtful comments.\nThe comments may be about implementation details, efficiency concerns, or style and readability\nRemember to be constructive and kind :)" }, { - "objectID": "units/unit6-parallel.html#scenario-7-simulation-study-with-n1000-replicates-parallel-random-number-generation", - "href": "units/unit6-parallel.html#scenario-7-simulation-study-with-n1000-replicates-parallel-random-number-generation", - "title": "Parallel processing", - "section": "Scenario 7: Simulation study with n=1000 replicates: parallel random number generation", - "text": "Scenario 7: Simulation study with n=1000 replicates: parallel random number generation\nWe’ll probably skip this for now and come back to it when we discuss random number generation in the Simulation Unit.\nThe key thing when thinking about random numbers in a parallel context is that you want to avoid having the same ‘random’ numbers occur on multiple processes. On a computer, random numbers are not actually random but are generated as a sequence of pseudo-random numbers designed to mimic true random numbers. The sequence is finite (but very long) and eventually repeats itself. When one sets a seed, one is choosing a position in that sequence to start from. Subsequent random numbers are based on that subsequence. All random numbers can be generated from one or more random uniform numbers, so we can just think about a sequence of values between 0 and 1.\nSpecific scenario: You are running a simulation study with n=1000 replicates.\nGeneral scenario: Safely handling random number generation in parallel.\nEach replicate involves fitting two statistical/machine learning methods.\nHere, unless you really have access to multiple hundreds of cores, you might as well just parallelize across replicates.\nHowever, you need to think about random number generation. One option is to set the random number seed to different values for each replicate. One danger in setting the seed like that is that the random numbers in the different replicate could overlap somewhat. This is probably somewhat unlikely if you are not generating a huge number of random numbers, but it’s unclear how safe it is.\nWe can use functionality with numpy’s PCG64 or MT19937 generators to be completely safe in our parallel random number generation. Each provide a jumped() function that moves the RNG ahead as if one had generated a very large number of random variables (\\(2^{128}\\)) for the Mersenne Twister and nearly that for the PCG64).\nHere’s how we can set up the use of the PCG64 generator:\n\nbitGen = np.random.PCG64(1)\nrng = np.random.Generator(bitGen)\nrng.random(size = 3)\n\narray([0.51182162, 0.9504637 , 0.14415961])\n\n\nNow let’s see how to jump forward. And then verify that jumping forward two increments is the same as making two separate jumps.\n\nbitGen = np.random.PCG64(1)\nbitGen = bitGen.jumped(1)\nrng = np.random.Generator(bitGen)\nrng.normal(size = 3)\n\narray([ 1.23362391, 0.42793616, -1.90447637])\n\nbitGen = np.random.PCG64(1)\nbitGen = bitGen.jumped(2)\nrng = np.random.Generator(bitGen)\nrng.normal(size = 3)\n\narray([-0.31752967, 1.22269493, 0.28254622])\n\nbitGen = np.random.PCG64(1)\nbitGen = bitGen.jumped(1)\nbitGen = bitGen.jumped(1)\nrng = np.random.Generator(bitGen)\nrng.normal(size = 3)\n\narray([-0.31752967, 1.22269493, 0.28254622])\n\n\nWe can also use jumped() with the Mersenne Twister.\n\nbitGen = np.random.MT19937(1)\nbitGen = bitGen.jumped(1)\nrng = np.random.Generator(bitGen)\nrng.normal(size = 3)\n\narray([ 0.12667829, -2.1031878 , -1.53950735])\n\n\nSo the strategy to parallelize across tasks (or potentially workers if random number generation is done sequentially for tasks done by a single worker) is to give each task the same seed and use jumped(i) where i indexes the tasks (or workers).\n\ndef myrandomfun(i):\n bitGen = np.random.PCG(1)\n bitGen = bitGen.jumped(i)\n # insert code with random number generation\n\nOne caution is that it appears that the period for PCG64 is \\(2^{128}\\) and that jumped(1) jumps forward by nearly that many random numbers. That seems quite strange, and I don’t understand it.\nAlternatively as recommended in the docs:\n\nn_tasks = 10\nsg = np.random.SeedSequence(1)\nrngs = [Generator(PCG64(s)) for s in sg.spawn(n_tasks)]\n## Now pass elements of rng into your function that is being computed in parallel\n\ndef myrandomfun(rng):\n # insert code with random number generation, such as:\n z = rng.normal(size = 5)\n\nIn R, the rlecuyer package deals with this. The L’Ecuyer algorithm has a period of \\(2^{191}\\), which it divides into subsequences of length \\(2^{127}\\)." + "objectID": "labs/lab5-codereview.html#hands-on-steps-for-lab", + "href": "labs/lab5-codereview.html#hands-on-steps-for-lab", + "title": "Lab 5: Code Reviews", + "section": "", + "text": "Pair up in teams of 2 (or 3 if necessary).\nOne member of the team will create a new repository on github and invite the other teammember(s) as collaborators on the repo.\nUsing the Github web UI, each team member will create a new file in the repo, and paste their code for presidential speaches into it.\nCommit you changes (from the github UI) as a pull request / separate branch.\nNow each team member will go to the pull request made by another team member and look at the code changes.\nRead eachother’s code carefully and leave some thoughtful comments.\nThe comments may be about implementation details, efficiency concerns, or style and readability\nRemember to be constructive and kind :)" }, { - "objectID": "units/unit6-parallel.html#avoiding-repeated-calculations-by-calling-compute-once", - "href": "units/unit6-parallel.html#avoiding-repeated-calculations-by-calling-compute-once", - "title": "Parallel processing", - "section": "Avoiding repeated calculations by calling compute once", - "text": "Avoiding repeated calculations by calling compute once\nAs far as I can tell, Dask avoids keeping all the pieces of a distributed object or computation in memory. However, in many cases this can mean repeating computations or re-reading data if you need to do multiple operations on a dataset.\nFor example, if you are create a Dask distributed dataset from data on disk, I think this means that every distinct set of computations (each computational graph) will involve reading the data from disk again.\nOne implication is that if you can include all computations on a large dataset within a single computational graph (i.e., a call to compute) that may be much more efficient than making separate calls.\nHere’s an example with Dask dataframe on some air traffic delay data, where we make sure to do all our computations as part of one graph:\n\nimport dask\ndask.config.set(scheduler='processes', num_workers = 6) \nimport dask.dataframe as ddf\nair = ddf.read_csv('/scratch/users/paciorek/243/AirlineData/csvs/*.csv.bz2',\n compression = 'bz2',\n encoding = 'latin1', # (unexpected) latin1 value(s) 2001 file TailNum field\n dtype = {'Distance': 'float64', 'CRSElapsedTime': 'float64',\n 'TailNum': 'object', 'CancellationCode': 'object'})\n# specify dtypes so Pandas doesn't complain about column type heterogeneity\n\nimport time\nt0 = time.time()\nair.DepDelay.min().compute() # about 200 seconds.\nprint(time.time()-t0)\nt0 = time.time()\nair.DepDelay.max().compute() # about 200 seconds.\nprint(time.time()-t0)\nt0 = time.time()\n(mn, mx) = dask.compute(air.DepDelay.max(), air.DepDelay.min()) # about 200 seconds\nprint(time.time()-t0)" + "objectID": "howtos/RandRStudioInstall.html", + "href": "howtos/RandRStudioInstall.html", + "title": "Installing R & RStudio", + "section": "", + "text": "On your laptop\nIf your version of R is older than 4.0.0, please install the latest version.\nTo install R, see:\n\nMacOS: install the R-4.2.1.pkg from https://cran.rstudio.com/bin/macosx\nWindows: https://cran.rstudio.com/bin/windows/base/\nLinux: https://cran.rstudio.com/bin/linux/\n\nThen install RStudio. To do so, see https://www.rstudio.com/ide/download/desktop, scrolling down to the “Installers for Supported Platforms” section and selecting the Installer for your operating system.\nVerify that you can install add-on R packages by installing the ‘fields’ package. In RStudio, go to ‘Tools->Install Packages’. In the resulting dialog box, enter ‘fields’ (without quotes) in the ‘Packages’ field. Depending on the location specified in the ‘Install to Library’ field, you may need to enter your administrator password. To be able to install packages to the directory of an individual user, you may need to do the following:\n\nIn R, enter the command Sys.getenv()['R_LIBS_USER'].\nCreate the directory specified in the result that R returns, e.g., on a Mac, this might be ~/Library/R/4.0/library.\n\nFor more detailed installation instructions for Windows, see Using R, RStudio, and LaTeX on Windows file.\n\n\nVia DataHub\nSee the instructions in Accessing the Unix Command Line for how to login to Datahub. Then in the mid-upper right, click on New and RStudio. Alternatively, to go directly to RStudio, go to https://r.datahub.berkeley.edu." }, { - "objectID": "units/unit6-parallel.html#setting-the-number-of-threads-cores-used-in-threaded-code-including-parallel-linear-algebra-in-python-and-r", - "href": "units/unit6-parallel.html#setting-the-number-of-threads-cores-used-in-threaded-code-including-parallel-linear-algebra-in-python-and-r", - "title": "Parallel processing", - "section": "Setting the number of threads (cores used) in threaded code (including parallel linear algebra in Python and R)", - "text": "Setting the number of threads (cores used) in threaded code (including parallel linear algebra in Python and R)\nIn general, threaded code will detect the number of cores available on a machine and make use of them. However, you can also explicitly control the number of threads available to a process.\nFor most threaded code (that based on the openMP protocol), the number of threads can be set by setting the OMP_NUM_THREADS environment variable (VECLIB_MAXIMUM_THREADS on a Mac). E.g., to set it for four threads in the bash shell:\n\nexport OMP_NUM_THREADS=4\n\nDo this before starting your R or Python session or before running your compiled executable.\nAlternatively, you can set OMP_NUM_THREADS as you invoke your job, e.g., here with R:\n\nOMP_NUM_THREADS=4 R CMD BATCH --no-save job.R job.out\n\n\nSpeed and threaded BLAS\nIn many cases, using multiple threads for linear algebra operations will outperform using a single thread, but there is no guarantee that this will be the case, in particular for operations with small matrices and vectors. You can compare speeds by setting OMP_NUM_THREADS to different values. In cases where threaded linear algebra is slower than unthreaded, you would want to set OMP_NUM_THREADS to 1.\nMore generally, if you are using the parallel tools in Section 4 to simultaneously carry out many independent calculations (tasks), it is likely to be more effective to use the fixed number of cores available on your machine so as to split up the tasks, one per core, without taking advantage of the threaded BLAS (i.e., restricting each process to a single thread)." + "objectID": "howtos/accessingPython.html", + "href": "howtos/accessingPython.html", + "title": "Accessing Python", + "section": "", + "text": "We recommend using using the Anaconda (Python 3.11 distribution) on your laptop. Click “Download” and then click 64-bit “Graphical Installer” for your current operating system.\nOnce you’ve installed Python, please install the following packages:\nAssuming you installed Anaconda Python, you should be able to do this from the command line:\nWhile you’re welcome to work with Python in a Jupyter notebook for exploration (e.g., using the campus DataHub, you’ll need to submit Quarto (.qmd) documents with Python chunks for your problem sets. So you’ll need Python set up on your laptop or to use it by logging in to an SCF machine." }, { - "objectID": "units/unit6-parallel.html#overview-futures-and-the-r-future-package", - "href": "units/unit6-parallel.html#overview-futures-and-the-r-future-package", - "title": "Parallel processing", - "section": "Overview: Futures and the R future package", - "text": "Overview: Futures and the R future package\nWhat is a future? It’s basically a flag used to tag a given operation such that when and where that operation is carried out is controlled at a higher level. If there are multiple operations tagged then this allows for parallelization across those operations.\nAccording to Henrik Bengtsson (the future package developer) and those who developed the concept:\n\na future is an abstraction for a value that will be available later\nthe value is the result of an evaluated expression\nthe state of a future is either unresolved or resolved\n\nWhy use futures? The future package allows one to write one’s computational code without hard-coding whether or how parallelization would be done. Instead one writes the code in a generic way and at the beginning of one’s code sets the ‘plan’ for how the parallel computation should be done given the computational resources available. Simply changing the ‘plan’ changes how parallelization is done for any given run of the code.\nMore concisely, the key ideas are:\n\nSeparate what to parallelize from how and where the parallelization is actually carried out.\nDifferent users can run the same code on different computational resources (without touching the actual code that does the computation)." + "objectID": "howtos/accessingPython.html#python-from-the-command-line", + "href": "howtos/accessingPython.html#python-from-the-command-line", + "title": "Accessing Python", + "section": "Python from the command line", + "text": "Python from the command line\nOnce you get your SCF account, you can access Python or IPython from the UNIX command line as soon as you login to an SCF server. Just SSH to an SCF Linux machine (e.g., gandalf.berkeley.edu or radagast.berkeley.edu) and run ‘python’ or ‘ipython’ from the command line.\nMore details on using SSH are here. Note that if you have the Ubuntu subsystem for Windows, you can use SSH directly from the Ubuntu terminal." }, { - "objectID": "units/unit6-parallel.html#overview-of-parallel-backends-1", - "href": "units/unit6-parallel.html#overview-of-parallel-backends-1", - "title": "Parallel processing", - "section": "Overview of parallel backends", - "text": "Overview of parallel backends\nOne uses plan() to control how parallelization is done, including what machine(s) to use and how many cores on each machine to use.\nFor example,\n\nplan(multiprocess)\n## spreads work across multiple cores\n# alternatively, one can also control number of workers\nplan(multiprocess, workers = 4)\n\nThis table gives an overview of the different plans.\n\n\n\n\n\n\n\n\n\nType\nDescription\nMulti-node\nCopies of objects made?\n\n\n\n\nmultisession\nuses additional R sessions as the workers\nno\nyes\n\n\nmulticore\nuses forked R processes as the workers\nno\nnot if object not modified\n\n\ncluster\nuses R sessions on other machine(s)\nyes\nyes" + "objectID": "howtos/accessingPython.html#python-via-jupyter-notebook", + "href": "howtos/accessingPython.html#python-via-jupyter-notebook", + "title": "Accessing Python", + "section": "Python via Jupyter notebook", + "text": "Python via Jupyter notebook\nYou can use a Jupyter notebook to run Python code from the SCF JupyterHub or the Berkeley DataHub.\nIf you’re on the SCF JupyterHub, select Start My Server. Then, unless you are running long or parallelized code, just click Spawn (in other words, accept the default ‘standalone’ partition). On the next page select ‘New’ and ‘Python 3’.\nTo finish your session, click on Control Panel and Stop My Server. Do not click Logout." }, { - "objectID": "units/unit6-parallel.html#accessing-variables-and-workers-in-the-worker-processes-1", - "href": "units/unit6-parallel.html#accessing-variables-and-workers-in-the-worker-processes-1", - "title": "Parallel processing", - "section": "Accessing variables and workers in the worker processes", - "text": "Accessing variables and workers in the worker processes\nThe future package usually does a good job of identifying the packages and (global) variables you use in your parallelized code and loading those packages on the workers and copying necessary variables to the workers. It uses the globals package to do this.\nHere’s a toy example that shows that n and MASS::geyser are automatically available in the worker processes.\n\nlibrary(future)\nlibrary(future.apply)\n\nplan(multisession)\n\nlibrary(MASS)\nn <- nrow(geyser)\n\nmyfun <- function(idx) {\n # geyser is in MASS package\n return(sum(geyser$duration) / n)\n}\n\nfuture_sapply(1:5, myfun)\n\n[1] 3.460814 3.460814 3.460814 3.460814 3.460814\n\n\nIn other contexts in R (or other languages) you may need to explicitly copy objects to the workers (or load packages on the workers). This is sometimes called exporting variables." + "objectID": "howtos/windowsAndLinux.html", + "href": "howtos/windowsAndLinux.html", + "title": "Windows 10 and the Ubuntu Subsystem", + "section": "", + "text": "Windows 10 has a powerful new feature that allows a full Linux system to be installed and run from within Windows. This is incredibly useful for building/testing code in Linux, without having a dedicated Linux machine, but it poses strange new behaviors as two very different operating systems coexist in one place. Initially, this document mirrors the Windows Install tutorial, showing you how to install Ubuntu and setting up R, RStudio, and LaTex. Then, we cover some of the issues of running two systems together, starting with finding files, finding the Ubuntu subsystem, and file modifications." }, { - "objectID": "units/unit10-linalg.html", - "href": "units/unit10-linalg.html", - "title": "Numerical linear algebra", - "section": "", - "text": "PDF\nReferences:\nVideos (optional):\nThere are various videos from 2020 in the bCourses Media Gallery that you can use for reference if you want to.\nIn working through how to compute something or understanding an algorithm, it can be very helpful to depict the matrices and vectors graphically. We’ll see this on the board in class." + "objectID": "howtos/windowsAndLinux.html#installing-ubuntu", + "href": "howtos/windowsAndLinux.html#installing-ubuntu", + "title": "Windows 10 and the Ubuntu Subsystem", + "section": "Installing Ubuntu", + "text": "Installing Ubuntu\nThere are 2 parts to installing a Linux subsystem in Windows. I will write this using Ubuntu as the example, as it is my preferred Linux distro, but several others are provided by Windows.\nSources:\n\nOfficial Windows Instructions\nUbuntu Update Instructions\n\n\n1) Enable Linux Subsystem\nBy default, the Linux subsystem is an optional addition in Windows. This feature has to be enabled prior to installing Linux. There are two ways to do it.\n\nCMD Line\nThe simplest way to enable the Linux subsystem is through PowerShell.\n\nOpen PowerShell as Administrator\nRun the following (on one line):\nEnable-WindowsOptionalFeature -Online -FeatureName Microsoft-Windows-Subsystem-Linux\nRestart the computer\n\nGUI\nIf you don’t wish to use PowerShell, you can work your way through the control panel and turn on the Linux subsystem.\n\nOpen the settings page through the search bar\nGo to Programs and Features\nFind Turn Windows Features on or off on the right side\nEnable the Windows Subsystem for Linux option\nRestart the computer\n\n\n\n\n2) Install Linux Subsystem\nOnce the Linux subsystem feature has been enabled, there are multiple methods to download and install the Linux distro you want. I highly recommend installing Ubuntu from the Microsoft store. There are several other flavors available as well, but Ubuntu is generally the easiest to learn and the most well supported.\n\nOpen the Microsoft Store\nSearch for Ubuntu\n\nYou’re looking for the highest number followed by LTS, currently 20.04 LTS (or 18.04 LTS is fine too). This is the current long-term-release, meaning it will be supported for the next 5 years.\n\nClick on the tile, then click Get, and this should start the installation.\nFollow the prompts to install Ubuntu.\n\nAfter installing Ubuntu, it is advisable to update it. This is something you should do on a regular basis.\n\nOpen a Bash terminal.\nType sudo apt update to update your local package database.\nType sudo apt upgrade to upgrade your installed packages." }, { - "objectID": "units/unit10-linalg.html#context", - "href": "units/unit10-linalg.html#context", - "title": "Numerical linear algebra", - "section": "Context", - "text": "Context\nMany statistical and machine learning methods involve linear algebra of some sort - at the very least matrix multiplication and very often some sort of matrix decomposition to fit models and do analysis: linear regression, various more sophisticated forms of regression, deep neural networks, principle components analysis (PCA) and the wide varieties of generalizations and variations on PCA, etc., etc." + "objectID": "howtos/windowsAndLinux.html#using-the-linux-terminal-from-r-in-windows", + "href": "howtos/windowsAndLinux.html#using-the-linux-terminal-from-r-in-windows", + "title": "Windows 10 and the Ubuntu Subsystem", + "section": "Using the Linux Terminal from R in Windows", + "text": "Using the Linux Terminal from R in Windows\nTo get all the functionality of a UNIX-style commandline from within R (e.g., for bash code chunks), you should set the terminal under R in Windows to be the Linux subsystem." }, { - "objectID": "units/unit10-linalg.html#goals", - "href": "units/unit10-linalg.html#goals", - "title": "Numerical linear algebra", - "section": "Goals", - "text": "Goals\nHere’s what I’d like you to get out of this unit:\n\nHow to think about the computational order (number of computations involved) of a problem\nHow to choose a computational approach to a given linear algebra calculation you need to do.\nAn understanding of how issues with computer numbers (Unit 8) affect linear algebra calculations." + "objectID": "howtos/windowsAndLinux.html#a-note-on-file-modification", + "href": "howtos/windowsAndLinux.html#a-note-on-file-modification", + "title": "Windows 10 and the Ubuntu Subsystem", + "section": "A Note on File Modification", + "text": "A Note on File Modification\nDO NOT MODIFY LINUX FILES FROM WINDOWS\nIt is highly recommended that you never modify Linux files from Windows because of metadata corruption issues. Any files created under the Linux subsystem, only modify them with Linux tools. In contrast, you can create files in the Windows system and modify them with both Windows or Linux tools. There could be file permission issues because Windows doesn’t have the same concept of file permissions as Linux. So, if you intend to work on files using both Linux and Windows, create the files in the C drive under Windows, and you should be safe to edit them with either OS." }, { - "objectID": "units/unit10-linalg.html#key-principle", - "href": "units/unit10-linalg.html#key-principle", - "title": "Numerical linear algebra", - "section": "Key principle", - "text": "Key principle\nThe form of a mathematical expression and how it should be evaluated on a computer may be very different. Better computational approaches can increase speed and improve the numerical properties of the calculation.\n\nExample 1 (already seen in Unit 5): If \\(X\\) and \\(Y\\) are matrices and \\(z\\) is a vector, we should compute \\(X(Yz)\\) rather than \\((XY)z\\); the former is much more computationally efficient.\nExample 2: We do not compute \\((X^{\\top}X)^{-1}X^{\\top}Y\\) by computing \\(X^{\\top}X\\) and finding its inverse. In fact, perhaps more surprisingly, we may never actually form \\(X^{\\top}X\\) in some implementations.\nExample 3: Suppose I have a matrix \\(A\\), and I want to permute (switch) two rows. I can do this with a permutation matrix, \\(P\\), which is mostly zeroes. On a computer, in general I wouldn’t need to even change the values of \\(A\\) in memory in some cases (e.g., if I were to calculate \\(PAB\\)). Why not?" + "objectID": "howtos/windowsAndLinux.html#finding-windows-from-linux", + "href": "howtos/windowsAndLinux.html#finding-windows-from-linux", + "title": "Windows 10 and the Ubuntu Subsystem", + "section": "Finding Windows from Linux", + "text": "Finding Windows from Linux\nOnce you have some flavor of Linux installed, you need to be able to navigate from your Linux home directory to wherever Windows stores files. This is relatively simple, as the Windows Subsystem shows Windows to Linux as a mounted drive.\n\nSource\n\n\nOpen a Bash terminal.\nType cd / to get to the root directory.\nIn root, type cd /mnt. This gets you to the mount point for your drives.\nType ls to see what drives are available (you should see a c, and maybe d as well).\nType cd c to get into the Windows C drive. This is the root of the C directory for Windows.\nTo find your files, change directy into the users folder, then into your username.\n\ncd Users/<your-user-name>\nThis is your home directory in Windows. If you type ls here, you should see things like\n\nDocuments\nDownloads\nPictures\nVideos\netc…" }, { - "objectID": "units/unit10-linalg.html#computational-complexity", - "href": "units/unit10-linalg.html#computational-complexity", - "title": "Numerical linear algebra", - "section": "Computational complexity", - "text": "Computational complexity\nWe can assess the computational complexity of a linear algebra calculation by counting the number multiplys/divides and the number of adds/subtracts. Sidenote: addition is a bit faster than multiplication, so some algorithms attempt to trade multiplication for addition.\nIn general we do not try to count the actual number of calculations, but just their order, though in some cases in this unit we’ll actually get a more exact count. In general, we denote this as \\(O(f(n))\\) which means that the number of calculations approaches \\(cf(n)\\) as \\(n\\to\\infty\\) (i.e., we know the calculation is approximately proportional to \\(f(n)\\)). Consider matrix multiplication, \\(AB\\), with matrices of size \\(a\\times b\\) and \\(b\\times c\\). Each column of the second matrix is multiplied by all the rows of the first. For any given inner product of a row by a column, we have \\(b\\) multiplies. We repeat these operations for each column and then for each row, so we have \\(abc\\) multiplies so \\(O(abc)\\) operations. We could count the additions as well, but there’s usually an addition for each multiply, so we can usually just count the multiplys and then say there are such and such {multiply and add}s. This is Monahan’s approach, but you may see other counting approaches where one counts the multiplys and the adds separately.\nFor two symmetric, \\(n\\times n\\) matrices, this is \\(O(n^{3})\\). Similarly, matrix factorization (e.g., the Cholesky decomposition) is \\(O(n^{3})\\) unless the matrix has special structure, such as being sparse. As matrices get large, the speed of calculations decreases drastically because of the scaling as \\(n^{3}\\) and memory use increases drastically. In terms of memory use, to hold the result of the multiply indicated above, we need to hold \\(ab+bc+ac\\) total elements, which for symmetric matrices sums to \\(3n^{2}\\). So for a matrix with \\(n=10000\\), we have \\(3\\cdot10000^{2}\\cdot8/1e9=2.4\\)Gb.\nWhen we have \\(O(n^{q})\\) this is known as polynomial time. Much worse is \\(O(b^{n})\\) (exponential time), while much better is \\(O(\\log n\\)) (log time). Computer scientists talk about NP-complete problems; these are essentially problems for which there is not a polynomial time algorithm - it turns out all such problems can be rewritten such that they are equivalent to one another.\nIn real calculations, it’s possible to have the actual time ordering of two approaches differ from what the order approximations tell us. For example, something that involves \\(n^{2}\\) operations may be faster than one that involves \\(1000(n\\log n+n)\\) even though the former is \\(O(n^{2})\\) and the latter \\(O(n\\log n)\\). The problem is that the constant, \\(c=1000\\), can matter (depending on how big \\(n\\) is), as can the extra calculations from the lower order term(s), in this case \\(1000n\\).\nA note on terminology: flops stands for both floating point operations (the number of operations required) and floating point operations per second, the speed of calculation." + "objectID": "howtos/windowsAndLinux.html#finding-linux-from-windows", + "href": "howtos/windowsAndLinux.html#finding-linux-from-windows", + "title": "Windows 10 and the Ubuntu Subsystem", + "section": "Finding Linux from Windows", + "text": "Finding Linux from Windows\nThis is slightly more tricky than getting from Linux to Windows. Windows stores the Linux files in a hidden subfolder so that you don’t mess with them from Windows. However, you can find them, and then the easiest way (note, do not read as safest or smartest) to find those files in the future is by creating a desktop shortcut.\n\nSource\n\n\nOpen File Explorer\nIn the address bar, type %userprofile%\\AppData\\Local\\Packages\n\n%userprofile% will expand to something like C:\\Users\\<your-user-name>\n\nLook for a folder related to the Linux distro that you installed\n\nThese names will change slightly over time, but look for something similar-ish.\nFor Ubuntu, look for something with CanonicalGroupLimited.UbuntuonWindows in it.\n\nCanonical is the creator/distributor of Ubuntu.\n\n\nClick LocalState\nClick rootfs\n\nThis is the root of your Linux distro.\n\nClick into home and then into your user name.\n\nThis is your home directory under Linux.\n\nDO NOT MODIFY THESE FILES FROM WINDOWS\n\nData corruption is a possibility.\n\n\nSo, the final path to find your home directory from windows will look like:\n%userprofile%\\AppData\\Local\\Packages\\<Distro-Folder>\\LocalStat\\rootfs\\home\\<your-user-name>\\" }, { - "objectID": "units/unit10-linalg.html#notation-and-dimensions", - "href": "units/unit10-linalg.html#notation-and-dimensions", - "title": "Numerical linear algebra", - "section": "Notation and dimensions", - "text": "Notation and dimensions\nI’ll try to use capital letters for matrices, \\(A\\), and lower-case for vectors, \\(x\\). Then \\(x_{i}\\) is the ith element of \\(x\\), \\(A_{ij}\\) is the \\(i\\)th row, \\(j\\)th column element, and \\(A_{\\cdot j}\\) is the \\(j\\)th column and \\(A_{i\\cdot}\\) the \\(i\\)th row. By default, we’ll consider a vector, \\(x\\), to be a one-column matrix, and \\(x^{\\top}\\) to be a one-row matrix. Some of the references given at the start of this Unit also use \\(a_{ij}\\) for \\(A_{ij}\\) and \\(a_{j}\\) for the \\(j\\)th column.\nThroughout, we’ll need to be careful that the matrices involved in an operation are conformable: for \\(A+B\\) both matrices need to be of the same dimension, while for \\(AB\\) the number of columns of \\(A\\) must match the number of rows of \\(B\\). Note that this allows for \\(B\\) to be a column vector, with only one column, \\(Ab\\). Just checking dimensions is a good way to catch many errors. Example: is \\(\\mbox{Cov}(Ax)=A\\mbox{Cov}(x)A^{\\top}\\) or \\(\\mbox{Cov}(Ax)=A^{\\top}\\mbox{Cov}(x)A\\)? Well, if \\(A\\) is \\(m\\times n\\), it must be the former, as the latter is not conformable.\nThe inner product of two vectors is \\(\\sum_{i}x_{i}y_{i}=x^{\\top}y\\equiv\\langle x,y\\rangle\\equiv x\\cdot y\\).\nThe outer product is \\(xy^{\\top}\\), which comes from all pairwise products of the elements.\nWhen the indices of summation should be obvious, I’ll sometimes leave them implicit. Ask me if it’s not clear." + "objectID": "howtos/windowsAndLinux.html#installing-r-on-the-linux-subsystem", + "href": "howtos/windowsAndLinux.html#installing-r-on-the-linux-subsystem", + "title": "Windows 10 and the Ubuntu Subsystem", + "section": "Installing R on the Linux Subsystem", + "text": "Installing R on the Linux Subsystem\nIMPORTANT: This section is only if you’d like to try using R under Linux. For class, using R under Windows should be fine.\nThe Linux Subsystem behaves exactly like a regular Linux installation, but for completeness, I will provide instructions here for people new to Linux. These instructions are written from the perspective of Ubuntu, but will be similar for other repos.\nR is not a part of the standard Ubuntu installation. So, we have to add the repository manually to our repository list. This is relatively straightforward, and R supports several versions of Ubuntu.\nSources:\n\nCRAN guide for Ubuntu\nDigital Ocean quick tutorial\n\n\nIn a bash window, type:\nsudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9\n\nThis adds the key to “sign”, or validate, the R repository\n\nThen, type:\nsudo add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/'\n\ncloud.r-project.org is the default mirror, however, it is prudent to connect to the mirror closest to you geographically. Berkeley has it’s own mirror, so the command with the Berkeley mirror would look like\n\nsudo add-apt-repository 'deb https://cran.r-project.org/bin/linux/ubuntu/bionic-cran40/'\nFinally, type sudo apt install r-base, and press y to confirm installation\nTo test that it worked, type R into the console, and an R session should begin\n\nType q() to quit the R session" }, { - "objectID": "units/unit10-linalg.html#norms", - "href": "units/unit10-linalg.html#norms", - "title": "Numerical linear algebra", - "section": "Norms", - "text": "Norms\nFor a vector, \\(\\|x\\|_{p}=(\\sum_{i}|x_{i}|^{p})^{1/p}\\) and the standard (Euclidean) norm is \\(\\|x\\|_{2}=\\sqrt{\\sum x_{i}^{2}}=\\sqrt{x^{\\top}x}\\), just the length of the vector in Euclidean space, which we’ll refer to as \\(\\|x\\|\\), unless noted otherwise.\nOne commonly used norm for a matrix is the Frobenius norm, \\(\\|A\\|_{F}=(\\sum_{i,j}a_{ij}^{2})^{1/2}\\).\nIn this Unit, we’ll often make use of the induced matrix norm, which is defined relative to a corresponding vector norm, \\(\\|\\cdot\\|\\), as: \\[\\|A\\|=\\sup_{x\\ne0}\\frac{\\|Ax\\|}{\\|x\\|}\\] So we have \\[\\|A\\|_{2}=\\sup_{x\\ne0}\\frac{\\|Ax\\|_{2}}{\\|x\\|_{2}}=\\sup_{\\|x\\|_{2}=1}\\|Ax\\|_{2}\\] If you’re not familiar with the supremum (“sup” above), you can just think of it as taking the maximum. In the case of the 2-norm, the norm turns out to be the largest singular value in the singular value decomposition (SVD) of the matrix.\nWe can interpret the norm of a matrix as the most that the matrix can stretch a vector when multiplying by the vector (relative to the length of the vector).\nA property of any legitimate matrix norm (including the induced norm) is that \\(\\|AB\\|\\leq\\|A\\|\\|B\\|\\). Also recall that norms must obey the triangle inequality, \\(\\|A+B\\|\\leq\\|A\\|+\\|B\\|\\).\nA normalized vector is one with “length”, i.e., Euclidean norm, of one. We can easily normalize a vector: \\(\\tilde{x}=x/\\|x\\|\\)\nThe angle between two vectors is \\[\\theta=\\cos^{-1}\\left(\\frac{\\langle x,y\\rangle}{\\sqrt{\\langle x,x\\rangle\\langle y,y\\rangle}}\\right)\\]" + "objectID": "howtos/windowsAndLinux.html#installing-rstudio-on-the-linux-subsystem", + "href": "howtos/windowsAndLinux.html#installing-rstudio-on-the-linux-subsystem", + "title": "Windows 10 and the Ubuntu Subsystem", + "section": "Installing Rstudio on the Linux Subsystem", + "text": "Installing Rstudio on the Linux Subsystem\n\n\n\n\n\n\nWarning\n\n\n\nTHIS NO LONGER WORKS\n\n\nAs of Rstudio 1.5.x, it does not run on WSL. Link\nAlso possible issues, WSL has no GUI, and therefore can’t support anything that uses a GUI.\nThese instructions work, but Rstudio doesn’t run.\nSources:\n\nRstudio\nSource\n\n\nGo the the Rstudio website (link above) and download the appropriate Rstudio Desktop version.\n\nFor most people, this is the Ubuntu 18 (64-bit) installer.\nSave it somewhere that you can find it.\nYou should have a file similar to rstudio-<version number>-amd64.deb\n\nOpen a terminal window and navigate to wherever you saved the rstudio install file.\nType the command sudo dpkg -i ./rstudio-<version number>-amd64.deb\n\nThis tells the package installer (dpkg) to install (-i) the file specified (./thing.deb)\n\nType the command sudo apt-get install -f\n\nThis tells the package manager (apt-get) to fix (-f) any dependency issues that may have arisen when installing the package.\n\nType the command which rstudio to make sure the system can find it.\n\nOutput should be similar to /usr/bin/rstudio\n\nRun rstudio from linux by typing rstudio &\n\nThe & runs it in the background, allowing you to close the terminal window." }, { - "objectID": "units/unit10-linalg.html#orthogonality", - "href": "units/unit10-linalg.html#orthogonality", - "title": "Numerical linear algebra", - "section": "Orthogonality", - "text": "Orthogonality\nTwo vectors are orthogonal if \\(x^{\\top}y=0\\), in which case we say \\(x\\perp y\\). An orthogonal matrix is a square matrix in which all of the columns are orthogonal to each other and normalized. The same holds for the rows. Orthogonal matrices can be shown to have full rank. Furthermore if \\(A\\) is orthogonal, \\(A^{\\top}A=I\\), so \\(A^{-1}=A^{\\top}\\). Given all this, the determinant of orthogonal \\(A\\) is either 1 or -1. Finally the product of two orthogonal matrices, \\(A\\) and \\(B\\), is also orthogonal since \\((AB)^{\\top}AB=B^{\\top}A^{\\top}AB=B^{\\top}B=I\\).\n\nPermutations\nSometimes we make use of matrices that permute two rows (or two columns) of another matrix when multiplied. Such a matrix is known as an elementary permutation matrix and is an orthogonal matrix with a determinant of -1. You can multiply such matrices to get more general permutation matrices that are also orthogonal. If you premultiply by \\(P\\), you permute rows, and if you postmultiply by \\(P\\) you permute columns. Note that on a computer, you wouldn’t need to actually do the multiply (and if you did, you should use a sparse matrix routine), but rather one can often just rework index values that indicate where relevant pieces of the matrix are stored (more in the next section)." + "objectID": "howtos/windowsAndLinux.html#installing-latex-on-the-linux-subsystem", + "href": "howtos/windowsAndLinux.html#installing-latex-on-the-linux-subsystem", + "title": "Windows 10 and the Ubuntu Subsystem", + "section": "Installing LaTeX on the Linux Subsystem", + "text": "Installing LaTeX on the Linux Subsystem\n\n\n\n\n\n\nImportant\n\n\n\nThis section is only if you’d like to try using LaTeX under Linux. For class, using LaTeX (or R Markdown) under Windows should be fine.\n\n\nLaTeX is a text-markup language used when generating documents from .Rmd files.\nSource LaTeX\n\nType sudo apt-get install texlive-full, press y to confirm installation\n\nGenerally, if you want to create and edit R Markdown documents you will also need a text editor to go with your LaTeX installation, but we won’t go into that here." }, { - "objectID": "units/unit10-linalg.html#some-vector-and-matrix-properties", - "href": "units/unit10-linalg.html#some-vector-and-matrix-properties", - "title": "Numerical linear algebra", - "section": "Some vector and matrix properties", - "text": "Some vector and matrix properties\n\\(AB\\ne BA\\) but \\(A+B=B+A\\) and \\(A(BC)=(AB)C\\).\nIn Python, recall the syntax is\n\nA + B\n\n# Matrix multiplication\nnp.matmul(A, B) \nA @ B # alternative\nA.dot(B) # not recommended by the NumPy docs\n\nA * B # Hadamard (direct) product\n\nYou don’t need the spaces, but they’re nice for code readability." + "objectID": "howtos/quartoInstall.html", + "href": "howtos/quartoInstall.html", + "title": "Installing and Using Quarto", + "section": "", + "text": "Unless you plan to generate your problem set solutions on the SCF, you’ll need to install Quarto.\nOnce installed, you should be able to run commands such as quarto render FILE and quarto preview FILE from the command line.\nQuarto also runs from the Windows Command shell or PowerShell. We’ll add details/troubleshooting tips here as needed." }, { - "objectID": "units/unit10-linalg.html#trace-and-determinant-of-square-matrices", - "href": "units/unit10-linalg.html#trace-and-determinant-of-square-matrices", - "title": "Numerical linear algebra", - "section": "Trace and determinant of square matrices", - "text": "Trace and determinant of square matrices\nThe trace of a matrix is the sum of the diagonal elements. For square matrices, \\(\\mbox{tr}(A+B)=\\mbox{tr}(A)+\\mbox{tr}(B)\\), \\(\\mbox{tr}(A)=\\mbox{tr}(A^{\\top})\\).\nWe also have \\(\\mbox{tr}(ABC)=\\mbox{tr}(CAB)=\\mbox{tr}(BCA)\\) - basically you can move a matrix from the beginning to the end or end to beginning, provided they are conformable for this operation. This is helpful for a couple reasons:\n\nWe can find the ordering that reduces computation the most if the individual matrices are not square.\n\\(x^{\\top}Ax=\\mbox{tr}(x^{\\top}Ax)\\) since the quadratic form, \\(x^{\\top}Ax\\), is a scalar, and this is equal to \\(\\mbox{tr}(xx^{\\top}A)\\) where \\(xx^{\\top}A\\) is a matrix. It can be helpful to be able to go back and forth between a scalar and a trace in some statistical calculations.\n\nFor square matrices, the determinant exists and we have \\(|AB|=|A||B|\\) and therefore, \\(|A^{-1}|=1/|A|\\) since \\(|I|=|AA^{-1}|=1\\). Also \\(|A|=|A^{\\top}|\\), which can be seen using the QR decomposition for \\(A\\) and understanding properties of determinants of triangular matrices (in this case \\(R\\)) and orthogonal matrices (in this case \\(Q\\)).\nFor square, invertible matrices, we have that \\((A^{-1})^{\\top}=(A^{\\top})^{-1}\\). Why? Since we have \\((AB)^{\\top}=B^{\\top}A^{\\top}\\), we have: \\[A^{\\top}(A^{-1})^{\\top}=(A^{-1}A)^{\\top}=I\\] so \\((A^{\\top})^{-1}=(A^{-1})^{\\top}\\).\n\nOther matrix multiplications\nThe Hadamard or direct product is simply multiplication of the correspoding elements of two matrices by each other. In R this is simplyA * B.\nChallenge: How can I find \\(\\mbox{tr}(AB)\\) without using A %*% B ?\nThe Kronecker product is the product of each element of one matrix with the entire other matrix”\n\\[A\\otimes B=\\left(\\begin{array}{ccc}\nA_{11}B & \\cdots & A_{1m}B\\\\\n\\vdots & \\ddots & \\vdots\\\\\nA_{n1}B & \\cdots & A_{nm}B\n\\end{array}\\right)\\]\nThe inverse of a Kronecker product is the Kronecker product of the inverses,\n\\[ B^{-1} \\otimes A^{-1} \\]\nwhich is obviously quite a bit faster because the inverse (i.e., solving a system of equations) in this special case is \\(O(n^{3}+m^{3})\\) rather than the naive approach being \\(O((nm)^{3})\\)." + "objectID": "syllabus.html", + "href": "syllabus.html", + "title": "Statistics 243 Fall 2023", + "section": "", + "text": "Statistics 243 is an introduction to statistical computing, taught using Python. The course will cover both programming concepts and statistical computing concepts. Programming concepts will include data and text manipulation, regular expressions, data structures, functions and variable scope, memory use, efficiency, debugging, testing, and parallel processing. Statistical computing topics will include working with large datasets, numerical linear algebra, computer arithmetic/precision, simulation studies and Monte Carlo methods, numerical optimization, and numerical integration/differentiation. A goal is that coverage of these topics complement the models/methods discussed in the rest of the statistics/biostatistics graduate curriculum. We will also cover the basics of UNIX/Linux, in particular shell scripting and operating on remote servers, as well as a bit of R.\n\n\n\nWhile the course is taught using Python and you will learn a lot about using Python at an advanced level, this is not a course about learning Python. Rather the focus of the course is computing for statistics and data science more generally, using Python to illustrate the concepts.\nThis is not a course that will cover specific statistical/machine learning/data analysis methods.\n\n\n\n\nInformal prerequisites: If you are not a statistics or biostatistics graduate student, please chat with me if you’re not sure if this course makes sense for you. A background in calculus, linear algebra, probability and statistics is expected, as well as a basic ability to operate on a computer (but I do not assume familiarity with the UNIX-style command line/terminal/shell). Furthermore, I’m expecting you will know the basics of Python, at the level of the Python short course offered Aug. 16-17, 2023. If you don’t have that background you’ll need to spend time in the initial couple weeks getting up to speed. In addition, we may have an optional hands-on practice session during the second or third week of class, and the GSI can also provide assistance." }, { - "objectID": "units/unit10-linalg.html#matrix-decompositions", - "href": "units/unit10-linalg.html#matrix-decompositions", - "title": "Numerical linear algebra", - "section": "Matrix decompositions", - "text": "Matrix decompositions\nA matrix decomposition is a re-expression of a matrix, \\(A\\), in terms of a product of two or three other, simpler matrices, where the decomposition reveals structure or relationships present in the original matrix, \\(A\\). The “simpler” matrices may be simpler in various ways, including\n\nhaving fewer rows or columns;\nbeing diagonal, triangular or sparse in some way,\nbeing orthogonal matrices.\n\nIn addition, once you have a decomposition, computation is generally easier, because of the special structure of the simpler matrices.\nWe’ll see this in great detail in Section 3." + "objectID": "syllabus.html#course-description", + "href": "syllabus.html#course-description", + "title": "Statistics 243 Fall 2023", + "section": "", + "text": "Statistics 243 is an introduction to statistical computing, taught using Python. The course will cover both programming concepts and statistical computing concepts. Programming concepts will include data and text manipulation, regular expressions, data structures, functions and variable scope, memory use, efficiency, debugging, testing, and parallel processing. Statistical computing topics will include working with large datasets, numerical linear algebra, computer arithmetic/precision, simulation studies and Monte Carlo methods, numerical optimization, and numerical integration/differentiation. A goal is that coverage of these topics complement the models/methods discussed in the rest of the statistics/biostatistics graduate curriculum. We will also cover the basics of UNIX/Linux, in particular shell scripting and operating on remote servers, as well as a bit of R.\n\n\n\nWhile the course is taught using Python and you will learn a lot about using Python at an advanced level, this is not a course about learning Python. Rather the focus of the course is computing for statistics and data science more generally, using Python to illustrate the concepts.\nThis is not a course that will cover specific statistical/machine learning/data analysis methods.\n\n\n\n\nInformal prerequisites: If you are not a statistics or biostatistics graduate student, please chat with me if you’re not sure if this course makes sense for you. A background in calculus, linear algebra, probability and statistics is expected, as well as a basic ability to operate on a computer (but I do not assume familiarity with the UNIX-style command line/terminal/shell). Furthermore, I’m expecting you will know the basics of Python, at the level of the Python short course offered Aug. 16-17, 2023. If you don’t have that background you’ll need to spend time in the initial couple weeks getting up to speed. In addition, we may have an optional hands-on practice session during the second or third week of class, and the GSI can also provide assistance." }, { - "objectID": "units/unit10-linalg.html#linear-independence-rank-and-basis-vectors", - "href": "units/unit10-linalg.html#linear-independence-rank-and-basis-vectors", - "title": "Numerical linear algebra", - "section": "Linear independence, rank, and basis vectors", - "text": "Linear independence, rank, and basis vectors\nA set of vectors, \\(v_{1},\\ldots v_{n}\\), is linearly independent (LIN) when none of the vectors can be represented as a linear combination, \\(\\sum c_{i}v_{i}\\), of the others for scalars, \\(c_{1},\\ldots,c_{n}\\). If we have vectors of length \\(n\\), we can have at most \\(n\\) linearly independent vectors. The rank of a matrix is the number of linearly independent rows (or columns - it’s the same), and is at most the minimum of the number of rows and number of columns. We’ll generally think about it in terms of the dimension of the column space - so we can just think about the number of linearly independent columns.\nAny set of linearly independent vectors (say \\(v_{1},\\ldots,v_{n}\\)) span a space made up of all linear combinations of those vectors (\\(\\sum_{i=1}^{n}c_{i}v_{i}\\)). The spanning vectors are known as basis vectors. We can express a vector \\(y\\) that is in the space with respect to (as a linear combination of) basis vectors as \\(y=\\sum_{i}c_{i}v_{i}\\), where if the basis vectors are normalized and orthogonal, we can find the weights as \\(c_{i}=\\langle y,v_{i}\\rangle\\).\nConsider a regression context. We have \\(p\\) covariates (\\(p\\) columns in the design matrix, \\(X\\)), of which \\(q\\leq p\\) are linearly independent covariates. This means that \\(p-q\\) of the vectors can be written as linear combos of the \\(q\\) vectors. The space spanned by the covariate vectors is of dimension \\(q\\), rather than \\(p\\), and \\(X^{\\top}X\\) has \\(p-q\\) eigenvalues that are zero. The \\(q\\) LIN vectors are basis vectors for the space - we can represent any point in the space as a linear combination of the basis vectors. You can think of the basis vectors as being like the axes of the space, except that the basis vectors are not orthogonal. So it’s like denoting a point in \\(\\Re^{q}\\) as a set of \\(q\\) numbers telling us where on each of the axes we are - this is the same as a linear combination of axis-oriented vectors.\nWhen fitting a regression, if \\(n=p=q\\), a vector of \\(n\\) observations can be represented exactly as a linear combination of the \\(p\\) basis vectors, so there is no residual and we have a single unique (and exact) solution (e.g., with \\(n=p=2\\), the observations fall exactly on the simple linear regression line). If \\(n<p\\), then we have at most \\(n\\) linearly independent covariates (the rank is at most \\(n\\)). In this case we have multiple possible solutions and the system is ill-determined (under-determined). Similarly, if \\(q<p\\) and \\(n\\geq p\\), the rank is again less than \\(p\\) and we have multiple possible solutions. Of course we usually have \\(n>p\\), so the system is overdetermined - there is no exact solution, but regression is all about finding solutions that minimize some criterion about the differences between the observations and linear combinations of the columns of the \\(X\\) matrix (such as least squares or penalized least squares). In standard regression, we project the observation vector onto the space spanned by the columns of the \\(X\\) matrix, so we find the point in the space closest to the observation vector." + "objectID": "syllabus.html#objectives-of-the-course", + "href": "syllabus.html#objectives-of-the-course", + "title": "Statistics 243 Fall 2023", + "section": "Objectives of the course", + "text": "Objectives of the course\nThe goals of the course are that, by the end of the course, students be able to:\n\noperate effectively in a UNIX environment and on remote servers and compute clusters;\nhave a solid understanding of general programming concepts and principles, and be able to program effectively (including having an advanced knowledge of Python functionality);\nbe familiar with concepts and tools for reproducible research and good scientific computing practices; and\nunderstand in depth and be able to make use of principles of numerical linear algebra, optimization, and simulation for statistics- and data science-related analyses and research." }, { - "objectID": "units/unit10-linalg.html#invertibility-singularity-rank-and-positive-definiteness", - "href": "units/unit10-linalg.html#invertibility-singularity-rank-and-positive-definiteness", - "title": "Numerical linear algebra", - "section": "Invertibility, singularity, rank, and positive definiteness", - "text": "Invertibility, singularity, rank, and positive definiteness\nFor square matrices, let’s consider how invertibility, singularity, rank and positive (or non-negative) definiteness relate.\nSquare matrices that are “regular” have an eigendecomposition, \\(A=\\Gamma\\Lambda\\Gamma^{-1}\\) where \\(\\Gamma\\) is a matrix with the eigenvectors as the columns and \\(\\Lambda\\) is a diagonal matrix of eigenvalues, \\(\\Lambda_{ii}=\\lambda_{i}\\). Symmetric matrices and matrices with unique eigenvalues are regular, as are some other matrices. The number of non-zero eigenvalues is the same as the rank of the matrix. Square matrices that have an inverse are also called nonsingular, and this is equivalent to having full rank. If the matrix is symmetric, the eigenvectors and eigenvalues are real and \\(\\Gamma\\) is orthogonal, so we have \\(A=\\Gamma\\Lambda\\Gamma^{\\top}\\). The determinant of the matrix is the product of the eigenvalues (why?), which is zero if it is less than full rank. Note that if none of the eigenvalues are zero then \\(A^{-1}=\\Gamma\\Lambda^{-1}\\Gamma^{\\top}\\).\nLet’s focus on symmetric matrices. The symmetric matrices that tend to arise in statistics are either positive definite (p.d.) or non-negative definite (n.n.d.). If a matrix is positive definite, then by definition \\(x^{\\top}Ax>0\\) for any \\(x\\). Note that if \\(\\mbox{Cov}(y)=A\\) then \\(x^{\\top}Ax=x^{\\top}\\mbox{Cov}(y)x=\\mbox{Cov}(x^{\\top}y)=\\mbox{Var}(x^{\\top}y)\\) if so positive definiteness amounts to having linear combinations of random variables (with the elements of \\(x\\) here being the weights) having positive variance. So we must have that positive definite matrices are equivalent to variance-covariance matrices (I’ll just refer to this as a variance matrix or as a covariance matrix). If \\(A\\) is p.d. then it has all positive eigenvalues and it must have an inverse, though as we’ll see, from a numerical perspective, we may not be able to compute it if some of the eigenvalues are very close to zero. In Python, numpy.linalg.eig(A)[1] is \\(\\Gamma\\), with each column a vector, and numpy.linalg.eig(A)[0] contains the (unordered) eigenvalues.\nTo summarize, here are some of the various connections between mathematical and statistical properties of positive definite matrices:\n\\(A\\) positive definite \\(\\Leftrightarrow\\) \\(A\\) is a covariance matrix \\(\\Leftrightarrow\\) \\(x^{\\top}Ax>0\\) \\(\\Leftrightarrow\\) \\(\\lambda_{i}>0\\) (positive eigenvalues) \\(\\Rightarrow\\)\\(|A|>0\\) \\(\\Rightarrow\\)\\(A\\) is invertible \\(\\Leftrightarrow\\) \\(A\\) is non singular \\(\\Leftrightarrow\\) \\(A\\) is full rank.\nAnd here are connections for positive semi-definite matrices:\n\\(A\\) positive semi-definite \\(\\Leftrightarrow\\) \\(A\\) is a constrained covariance matrix \\(\\Leftrightarrow\\) \\(x^{\\top}Ax\\geq0\\) and equal to 0 for some \\(x\\) \\(\\Leftrightarrow\\) \\(\\lambda_{i}\\geq 0\\) (non-negative eigenvalues), with at least one zero \\(\\Rightarrow\\) \\(|A|=0\\) \\(\\Leftrightarrow\\) \\(A\\) is not invertible \\(\\Leftrightarrow\\) \\(A\\) is singular \\(\\Leftrightarrow\\) \\(A\\) is not full rank." + "objectID": "syllabus.html#topics-in-order-with-rough-timing", + "href": "syllabus.html#topics-in-order-with-rough-timing", + "title": "Statistics 243 Fall 2023", + "section": "Topics (in order with rough timing)", + "text": "Topics (in order with rough timing)\nThe ‘days’ here are (roughly) class sessions, as general guidance.\n\nIntroduction to UNIX, operating on a compute server (1 day)\nData formats, data access, webscraping, data structures (2 days)\nDebugging, good programming practices, reproducible research (1 day)\nThe bash shell and shell scripting, version control (3 days)\nProgramming concepts and advanced Python programming: text processing and regular expressions, object-oriented programming, functions and variable scope, memory use, efficient programming (9 days)\nParallel processing (2 days)\nWorking with databases, hashing, and big data (3 days)\nComputer arithmetic/representation of numbers on a computer (3 days)\nSimulation studies and Monte Carlo (2 days)\nNumerical linear algebra (5 days)\nOptimization (5 days)\nGraphics (1 day)" }, { - "objectID": "units/unit10-linalg.html#interpreting-an-eigendecomposition", - "href": "units/unit10-linalg.html#interpreting-an-eigendecomposition", - "title": "Numerical linear algebra", - "section": "Interpreting an eigendecomposition", - "text": "Interpreting an eigendecomposition\nLet’s interpret the eigendecomposition in a generative context as a way of generating random vectors. We can generate \\(y\\) s.t. \\(\\mbox{Cov}(y)=A\\) if we generate \\(y=\\Gamma\\Lambda^{1/2}z\\) where \\(\\mbox{Cov}(z)=I\\) and \\(\\Lambda^{1/2}\\) is formed by taking the square roots of the eigenvalues. So \\(\\sqrt{\\lambda_{i}}\\) is the standard deviation associated with the basis vector \\(\\Gamma_{\\cdot i}\\). That is, the \\(z\\)’s provide the weights on the basis vectors, with scaling based on the eigenvalues. So \\(y\\) is produced as a linear combination of eigenvectors as basis vectors, with the variance attributable to the basis vectors determined by the eigenvalues.\nTo go the other direction, we can project a vector \\(y\\) onto the space spanned by the eigenvectors: \\(z = (\\Gamma^{\\top}\\Gamma)^{-1}\\Gamma^{\\top}y = \\Gamma^{\\top}y\\), where the simplification of course comes from \\(\\Gamma\\) being orthogonal.\nIf \\(x^{\\top}Ax\\geq0\\) then \\(A\\) is nonnegative definite (also called positive semi-definite). In this case one or more eigenvalues can be zero. Let’s interpret this a bit more in the context of generating random vectors based on non-negative definite matrices, \\(y=\\Gamma\\Lambda^{1/2}z\\) where \\(\\mbox{Cov}(z)=I\\). Questions:\n\nWhat does it mean when one or more eigenvalue (i.e., \\(\\lambda_{i}=\\Lambda_{ii}\\)) is zero?\nSuppose I have an eigenvalue that is very small and I set it to zero? What will be the impact upon \\(y\\) and \\(\\mbox{Cov}(y)\\)?\nNow let’s consider the inverse of a covariance matrix, known as the precision matrix, \\(A^{-1}=\\Gamma\\Lambda^{-1}\\Gamma^{\\top}\\). What does it mean if a \\((\\Lambda^{-1})_{ii}\\) is very large? What if \\((\\Lambda^{-1})_{ii}\\) is very small?\n\nConsider an arbitrary \\(n\\times p\\) matrix, \\(X\\). Any crossproduct or sum of squares matrix, such as \\(X^{\\top}X\\) is positive definite (non-negative definite if \\(p>n\\)). This makes sense as it’s just a scaling of an empirical covariance matrix." + "objectID": "syllabus.html#personnel", + "href": "syllabus.html#personnel", + "title": "Statistics 243 Fall 2023", + "section": "Personnel", + "text": "Personnel\n\nInstructor:\n\nChris Paciorek (paciorek@stat.berkeley.edu)\n\nGSI\n\nAhmed Eldeeb (Deeb) (deeb@berkeley.edu)\n\nOffice hours can be found here.\nWhen to see us about an assignment: We’re here to help, including providing guidance on assignments. You don’t want to be futilely spinning your wheels for a long time getting nowhere. That said, before coming to see us about a difficulty, you should try something a few different ways and define/summarize for yourself what is going wrong or where you are getting stuck." }, { - "objectID": "units/unit10-linalg.html#generalized-inverses-optional", - "href": "units/unit10-linalg.html#generalized-inverses-optional", - "title": "Numerical linear algebra", - "section": "Generalized inverses (optional)", - "text": "Generalized inverses (optional)\nSuppose I want to find \\(x\\) such that \\(Ax=b\\). Mathematically the answer (provided \\(A\\) is invertible, i.e. of full rank) is \\(x=A^{-1}b\\).\nGeneralized inverses arise in solving equations when \\(A\\) is not full rank. A generalized inverse is a matrix, \\(A^{-}\\) s.t. \\(AA^{-}A=A\\). The Moore-Penrose inverse (the pseudo-inverse), \\(A^{+}\\), is a (unique) generalized inverse that also satisfies some additional properties. \\(x=A^{+}b\\) is the solution to the linear system, \\(Ax=b\\), that has the shortest length for \\(x\\).\nWe can find the pseudo-inverse based on an eigendecomposition (or an SVD) as \\(\\Gamma\\Lambda^{+}\\Gamma^{\\top}\\). We obtain \\(\\Lambda^{+}\\) from \\(\\Lambda\\) as follows. For values \\(\\lambda_{i}>0\\), compute \\(1/\\lambda_{i}\\). All other values are set to 0. Let’s interpret this statistically. Suppose we have a precision matrix with one or more zero eigenvalues and we want to find the covariance matrix. A zero eigenvalue means we have no precision, or infinite variance, for some linear combination (i.e., for some basis vector). We take the pseudo-inverse and assign that linear combination zero variance.\nLet’s consider a specific example. Autoregressive models are often used for smoothing (in time, in space, and in covariates). A first order autoregressive model for \\(y_{1},y_{2},\\ldots,y_{T}\\) has \\(E(y_{i}|y_{-i})=\\frac{1}{2}(y_{i-1}+y_{i+1})\\). Another way of writing the model is in time-order: \\(y_{i}=y_{i-1}+\\epsilon_{i}\\). A second order autoregressive model has \\(E(y_{i}|y_{-i})=\\frac{1}{6}(4y_{i-1}+4y_{i+1}-y_{i-2}-y_{i+2})\\). These constructions basically state that each value should be a smoothed version of its neighbors. One can figure out that the precision matrix for \\(y\\) in the first order model is \\[\\left(\\begin{array}{ccccc}\n\\ddots & & \\vdots\\\\\n-1 & 2 & -1 & 0\\\\\n\\cdots & -1 & 2 & -1 & \\dots\\\\\n& 0 & -1 & 2 & -1\\\\\n& & \\vdots & & \\ddots\n\\end{array}\\right)\\] and in the second order model is\n\\[\\left( \\begin{array}{ccccccc} \\ddots & & & \\vdots \\\\ 1 & -4 & 6 & -4 & 1 \\\\ \\cdots & 1 & -4 & 6 & -4 & 1 & \\cdots \\\\ & & 1 & -4 & 6 & -4 & 1 \\\\ & & & \\vdots \\end{array} \\right).\\]\nIf we look at the eigendecomposition of such matrices, we see that in the first order case, the eigenvalue corresponding to the constant eigenvector is zero.\n\nimport numpy as np\n\nprecMat = np.array([[1,-1,0,0,0],[-1,2,-1,0,0],[0,-1,2,-1,0],[0,0,-1,2,-1],[0,0,0,-1,1]])\ne = np.linalg.eig(precMat)\ne[0] # 4th eigenvalue is numerically zero\n\narray([3.61803399e+00, 2.61803399e+00, 1.38196601e+00, 4.97762256e-17,\n 3.81966011e-01])\n\ne[1][:,3] # constant eigenvector\n\narray([0.4472136, 0.4472136, 0.4472136, 0.4472136, 0.4472136])\n\n\nThis means we have no information about the overall level of \\(y\\). So how would we generate sample \\(y\\) vectors? We can’t put infinite variance on the constant basis vector and still generate samples. Instead we use the pseudo-inverse and assign ZERO variance to the constant basis vector. This corresponds to generating realizations under the constraint that \\(\\sum y_{i}\\) has no variation, i.e., \\(\\sum y_{i}=\\bar{y}=0\\) - you can see this by seeing that \\(\\mbox{Var}(\\Gamma_{\\cdot i}^{\\top}y)=0\\) when \\(\\lambda_{i}=0\\).\n\n# generate a realization\nevals = e[0]\nevals = 1/evals # variances\nevals[3] = 0 # generalized inverse\ny = e[1] @ ((evals ** 0.5) * np.random.normal(size = 5))\ny.sum()\n\n-2.220446049250313e-16\n\n\nIn the second order case, we have two non-identifiabilities: for the sum and for the linear component of the variation in \\(y\\) (linear in the indices of \\(y\\)).\nI could parameterize a statistical model as \\(\\mu+y\\) where \\(y\\) has covariance that is the generalized inverse discussed above. Then I allow for both a non-zero mean and for smooth variation governed by the autoregressive structure. In the second-order case, I would need to add a linear component as well, given the second non-identifiability." + "objectID": "syllabus.html#course-websites-github-piazza-gradescope-and-bcourses", + "href": "syllabus.html#course-websites-github-piazza-gradescope-and-bcourses", + "title": "Statistics 243 Fall 2023", + "section": "Course websites: GitHub, Ed Discussion, Gradescope, and bCourses", + "text": "Course websites: GitHub, Ed Discussion, Gradescope, and bCourses\nKey websites for the course are:\n\nThis course website, which is hosted on GitHub pages, and the GitHub repository containing the source materials: https://github.com/berkeley-stat243/stat243-fall-2023\nSCF tutorials for additional content: https://statistics.berkeley.edu/computing/training/tutorials\nEd Discussion site for discussions/Q&A: https://edstem.org/us/courses/42474/discussion/\nbCourses site for course capture recordings (see Media Gallery) and possibly some other materials: https://bcourses.berkeley.edu/courses/1527498.\nGradescope for assignments (also linked from bCourses): https://www.gradescope.com/courses/569739\n\nAll course materials will be posted on here on the website (and on GitHub) except for video content, which will be in bCourses.\n\nCourse discussion\nWe will use the course Ed Discussion site for communication (announcements, questions, and discussion). You should ask questions about class material and problem sets through the site. Please use this site for your questions so that either James or I can respond and so that everyone can benefit from the discussion. I suggest you to modify your settings on Ed Discussion so you are informed by email of postings. In particular you are responsible for keeping track of all course announcements, which we’ll make on the Discussion forum. I strongly encourage you to respond to or comment on each other’s questions as well (this will help your class participation grade), although of course you should not provide a solution to a problem set problem. If you have a specific administrative question you need to direct just to me, it’s fine to email me directly or post privately on the Discussion site. But if you simply want to privately ask a question about content, then just come to an office hour or see me after class or James during/after section.\nIf you’re enrolled in the class you should be a member of the group and be able to access it. If you’re auditing or not yet enrolled and would like access, make sure to fill out the course survey and I will add you. In addition, we will use Gradescope for viewing grades." }, { - "objectID": "units/unit10-linalg.html#matrices-arising-in-regression", - "href": "units/unit10-linalg.html#matrices-arising-in-regression", - "title": "Numerical linear algebra", - "section": "Matrices arising in regression", - "text": "Matrices arising in regression\nIn regression, we work with \\(X^{\\top}X\\). Some properties of this matrix are that it is symmetric and non-negative definite (hence our use of \\((X^{\\top}X)^{-1}\\) in the OLS estimator). When is it not positive definite?\nFitted values are \\(X\\hat{\\beta}=X(X^{\\top}X)^{-1}X^{\\top}Y=HY\\). The “hat” matrix, \\(H\\), projects \\(Y\\) into the column space of \\(X\\). \\(H\\) is idempotent: \\(HH=H\\), which makes sense - once you’ve projected into the space, any subsequent projection just gives you the same thing back. \\(H\\) is singular. Why? Also, under what special circumstance would it not be singular?" + "objectID": "syllabus.html#course-material", + "href": "syllabus.html#course-material", + "title": "Statistics 243 Fall 2023", + "section": "Course material", + "text": "Course material\n\nPrimary materials: Course notes on course webpage/GitHub, SCF tutorials, and potentially pre-recorded videos on bCourses.\nBack-up textbooks (generally available via UC Library via links below):\n\nFor bash: Newham, Cameron and Rosenblatt, Bill. Learning the bash Shell available electronically through UC Library\nFor Quarto: The Quarto reference guide\nFor statistical computing topics:\n\nGentle, James. Computational Statistics\nGentle, James. Matrix Algebra or Numerical Linear Algebra with Applications in Statistics\n\nOther resources with more detail on particular aspects of statistical computing concepts:\n\nLange, Kenneth; Numerical Analysis for Statisticians, 2nd ed. First edition available through UC library\nMonahan, John; Numerical Methods of Statistics" }, { - "objectID": "units/unit10-linalg.html#storing-matrices", - "href": "units/unit10-linalg.html#storing-matrices", - "title": "Numerical linear algebra", - "section": "Storing matrices", - "text": "Storing matrices\nWe’ve discussed column-major and row-major storage of matrices. First, retrieval of matrix elements from memory is quickest when multiple elements are contiguous in memory. So in a column-major language (e.g., R, Fortran), it is best to work with values in a common column (or entire columns) while in a row-major language (e.g., Python, C) for values in a common row.\nIn some cases, one can save space (and potentially speed) by overwriting the output from a matrix calculation into the space occupied by an input. This occurs in some clever implementations of matrix factorizations." + "objectID": "syllabus.html#section", + "href": "syllabus.html#section", + "title": "Statistics 243 Fall 2023", + "section": "Section", + "text": "Section\nThe GSI will lead a two-hour discussion section each week (there are two sections). By and large, these will only last for about one hour of actual content, but the second hour may be used as an office hour with the GSI or for troubleshooting software during the early weeks. The discussion sections will vary in format and topic, but material will include demonstrations on various topics (version control, debugging, testing, etc.), group work on these topics, discussion of relevant papers, and discussion of problem set solutions. The first section (1-3 pm) generally has more demand, so to avoid having too many people in the room, you should go to your assigned section unless you talk to me first." }, { - "objectID": "units/unit10-linalg.html#algorithms", - "href": "units/unit10-linalg.html#algorithms", - "title": "Numerical linear algebra", - "section": "Algorithms", - "text": "Algorithms\nGood algorithms can change the efficiency of an algorithm by one or more orders of magnitude, and many of the improvements in computational speed over recent decades have been in algorithms rather than in computer speed.\nMost matrix algebra calculations can be done in multiple ways. For example, we could compute \\(b=Ax\\) in either of the following ways, denoted here in pseudocode.\n\nStack the inner products of the rows of \\(A\\) with \\(x\\).\n\n\n for(i=1:n){ \n b_i = 0\n for(j=1:m){\n b_i = b_i + a_{ij} x_j\n }\n }\n\nTake the linear combination (based on \\(x\\)) of the columns of \\(A\\)\n\n\n for(i=1:n){ \n b_i = 0\n }\n for(j=1:m){\n for(i = 1:n){\n b_i = b_i + a_{ij} x_j \n }\n }\nIn this case the two approaches involve the same number of operations but the first might be better for row-major matrices (so might be how we would implement in C) and the second for column-major (so might be how we would implement in Fortran).\nChallenge: check whether the first approach is faster in Python. (Write the code just doing the outer loop and doing the inner loop using vectorized calculation.)\n\nGeneral computational issues\nThe same caveats we discussed in terms of computer arithmetic hold naturally for linear algebra, since this involves arithmetic with many elements. Good implementations of algorithms are aware of the danger of catastrophic cancellation and of the possibility of dividing by zero or by values that are near zero." + "objectID": "syllabus.html#computing-resources", + "href": "syllabus.html#computing-resources", + "title": "Statistics 243 Fall 2023", + "section": "Computing Resources", + "text": "Computing Resources\nMost work for the course can be done on your laptop. Later in the course we’ll also use the Statistics Department Linux cluster. You can also use the SCF JupyterHub or the campus DataHub to access a bash shell or run an IPython notebook.\nThe software needed for the course is as follows:\n\nAccess to the UNIX command line (bash shell)\nGit\nPython (Anaconda/Miniconda is recommended but by no means required)\n\nWe have some tips for software installation (and access to DataHub), including suggestions for how to access a UNIX shell, which you’ll need to be able to do by the second week of class." }, { - "objectID": "units/unit10-linalg.html#ill-conditioned-problems", - "href": "units/unit10-linalg.html#ill-conditioned-problems", - "title": "Numerical linear algebra", - "section": "Ill-conditioned problems", - "text": "Ill-conditioned problems\n\nBasics\nA problem is ill-conditioned if small changes to values in the computation result in large changes in the result. This is quantified by something called the condition number of a calculation. For different operations there are different condition numbers.\nIll-conditionedness arises most often in terms of matrix inversion, so the standard condition number is the “condition number with respect to inversion”, which when using the \\(L_{2}\\) norm is the ratio of the absolute values of the largest to smallest eigenvalue. Here’s an example: \\[A=\\left(\\begin{array}{cccc}\n10 & 7 & 8 & 7\\\\\n7 & 5 & 6 & 5\\\\\n8 & 6 & 10 & 9\\\\\n7 & 5 & 9 & 10\n\\end{array}\\right).\\] The solution of \\(Ax=b\\) for \\(b=(32,23,33,31)\\) is \\(x=(1,1,1,1)\\), while the solution for \\(b+\\delta b=(32.1,22.9,33.1,30.9)\\) is \\(x+\\delta x=(9.2,-12.6,4.5,-1.1)\\), where \\(\\delta\\) is notation for a perturbation to the vector or matrix.\n\ndef norm2(x):\n return(np.sum(x**2) ** 0.5)\n\nA = np.array([[10,7,8,7],[7,5,6,5],[8,6,10,9],[7,5,9,10]])\nb = np.array([32,23,33,31])\nx = np.linalg.solve(A, b)\n\nbPerturbed = np.array([32.1, 22.9, 33.1, 30.9])\nxPerturbed = np.linalg.solve(A, bPerturbed)\n\nWhat’s going on? Some manipulations with inequalities involving the induced matrix norm (for any chosen vector norm, but we might as well just think about the Euclidean norm) (see Gentle-CS Sec. 5.1 or the derivation in class) give \\[\\frac{\\|\\delta x\\|}{\\|x\\|}\\leq\\|A\\|\\|A^{-1}\\|\\frac{\\|\\delta b\\|}{\\|b\\|}\\] where we define the condition number w.r.t. inversion as \\(\\mbox{cond}(A)\\equiv\\|A\\|\\|A^{-1}\\|\\). We’ll generally work with the \\(L_{2}\\) norm, and for a nonsingular square matrix the result is that the condition number is the ratio of the absolute values of the largest and smallest magnitude eigenvalues. This makes sense since \\(\\|A\\|_{2}\\) is the absolute value of the largest magnitude eigenvalue of \\(A\\) and \\(\\|A^{-1}\\|_{2}\\) that of the inverse of the absolute value of the smallest magnitude eigenvalue of \\(A\\).\nWe see in the code above that the large disparity in eigenvalues of \\(A\\) leads to an effect predictable from our inequality above, with the condition number helping us find an upper bound.\n\ne = np.linalg.eig(A)\nevals = e[0]\nnorm2(x - xPerturbed) ## delta x\n\n16.396950936073825\n\nnorm2(b - bPerturbed) ## delta b\n\n0.20000000000000284\n\nnorm2(x - xPerturbed)/norm2(x)\n\n8.19847546803699\n\n(evals[0]/evals[2])*norm2(b - bPerturbed)/norm2(b)\n\n9.942833687618297\n\n\nThe main use of these ideas for our purposes is in thinking about the numerical accuracy of a linear system solution (Gentle-NLA Sec 3.4). On a computer we have the system \\[(A+\\delta A)(x+\\delta x)=b+\\delta b\\] where the ‘perturbation’ is from the inaccuracy of computer numbers. Our exploration of computer numbers tells us that \\[\\frac{\\|\\delta b\\|}{\\|b\\|}\\approx10^{-p};\\,\\,\\,\\frac{\\|\\delta A\\|}{\\|A\\|}\\approx10^{-p}\\] where \\(p=16\\) for standard double precision floating points. Following Gentle, one gets the approximation\n\\[\\frac{\\|\\delta x\\|}{\\|x\\|}\\approx\\mbox{cond}(A)10^{-p},\\] so if \\(\\mbox{cond}(A)\\approx10^{t}\\), we have accuracy of order \\(10^{t-p}\\) instead of \\(10^{-p}\\). (Gentle cautions that this holds only if \\(10^{t-p}\\ll1\\)). So we can think of the condition number as giving us the number of digits of accuracy lost during a computation relative to the precision of numbers on the computer. E.g., a condition number of \\(10^{8}\\) means we lose 8 digits of accuracy relative to our original 16 on standard systems. One issue is that estimating the condition number is itself subject to numerical error and requires computation of \\(A^{-1}\\) (albeit not in the case of \\(L_{2}\\) norm with square, nonsingular \\(A\\)) but see Golub and van Loan (1996; p. 76-78) for an algorithm.\n\n\nImproving conditioning\nIll-conditioned problems in statistics often arise from collinearity of regressors. Often the best solution is not a numerical one, but re-thinking the modeling approach, as this generally indicates statistical issues beyond just the numerical difficulties.\nA general comment on improving conditioning is that we want to avoid large differences in the magnitudes of numbers involved in a calculation. In some contexts such as regression, we can center and scale the columns to avoid such differences - this will improve the condition of the problem. E.g., in simple quadratic regression with \\(x=\\{1990,\\ldots,2010\\}\\) (e.g., regressing on calendar years), we see that centering and scaling the matrix columns makes a huge difference on the condition number\n\nt1 = np.arange(1990, 2011) # naive covariate\nX1 = np.column_stack((np.ones(21), t1, t1 ** 2))\ne1 = np.linalg.eig(np.dot(X1.T, X1))\nnp.sort(e1[0])[::-1]\n\narray([3.36018564e+14, 7.69949736e+02, 2.24079720e-08])\n\nt2 = t1 - 2000 # centered\nX2 = np.column_stack((np.ones(21), t2, t2 ** 2))\ne2 = np.linalg.eig(np.dot(X2.T, X2))\nwith np.printoptions(suppress=True):\n np.sort(e2[0])[::-1]\n\narray([50677.70427505, 770. , 9.29572495])\n\nt3 = t2/10 # centered and scaled\nX3 = np.column_stack((np.ones(21), t3, t3 ** 2))\ne3 = np.linalg.eig(np.dot(X3.T, X3))\nwith np.printoptions(suppress=True):\n np.sort(e3[0])[::-1]\n\narray([24.11293487, 7.7 , 1.95366513])\n\n\nThe basic story is that simple strategies often solve the problem, and that you should be aware of the absolute and relative magnitudes involved in your calculations.\nOne rule of thumb is to try to work with numbers whose magnitude is around 1. We can often scale the values in our problem in order to do this. I.e., change the units of your variables. Instead of personal income in dollars, use personal income in thousands or hundreds of thousands of dollars." + "objectID": "syllabus.html#class-time", + "href": "syllabus.html#class-time", + "title": "Statistics 243 Fall 2023", + "section": "Class time", + "text": "Class time\nMy goal is to have classes be an interactive environment. This is both more interesting for all of us and more effective in learning the material. I encourage you to ask questions and will pose questions to the class to think about, respond to via online polling or Google forms, and discuss. To increase time for discussion and assimilation of the material in class, before some classes I may ask that you read material or work through tutorials in advance of class. Occasionally, I will ask you to submit answers to questions in advance of class as well.\nPlease do not use phones during class and limit laptop use to the material being covered.\nStudent backgrounds with computing will vary. For those of you with limited background on a topic, I encourage you to ask questions during class so I know what you find confusing. For those of you with extensive background on a topic (there will invariably be some topics where one of you will know more about it than I do or have more real-world experience), I encourage you to pitch in with your perspective. In general, there are many ways to do things on a computer, particularly in a UNIX environment and in Python, so it will help everyone (including me) if we hear multiple perspectives/ideas.\nFinally, class recordings for review or to make up for absence will be available through the bCourses Media Gallery, available on the Media Gallery tab on the bCourses page for the class." }, { - "objectID": "units/unit10-linalg.html#triangular-systems", - "href": "units/unit10-linalg.html#triangular-systems", - "title": "Numerical linear algebra", - "section": "Triangular systems", - "text": "Triangular systems\nAs a preface, let’s figure out how to solve \\(Ax=b\\) if \\(A\\) is upper triangular. The basic algorithm proceeds from the bottom up (and therefore is called a ‘backsolve’. We solve for \\(x_{n}\\) trivially, and then move upwards plugging in the known values of \\(x\\) and solving for the remaining unknown in each row (each equation).\n\n\\(x_{n}=b_{n}/A_{nn}\\)\nNow for \\(k<n\\), use the already computed \\(\\{x_{n},x_{n-1},\\ldots,x_{k+1}\\}\\) to calculate \\(x_{k}=\\frac{b_{k}-\\sum_{j=k+1}^{n}x_{j}A_{kj}}{A_{kk}}\\).\nRepeat for all rows.\n\nHow many multiplies and adds are done? Solving lower triangular systems is very similar and involves the same number of calculations.\nIn R, backsolve() solves upper triangular systems and forwardsolve() solves lower triangular systems:\n\nimport scipy as sp\nnp.random.seed(1)\nn = 20\nX = np.random.normal(size = (n,n))\n\n## R has the `crossprod` function, which would be more efficient\n## than having to transpose, but numpy doesn't seem to have an equivalent.\nX = X.T @ X\nb = np.random.normal(size = n)\nL = np.linalg.cholesky(X) # L is upper-triangular\nU = L.T\n\nout1 = sp.linalg.solve_triangular(L, b, lower=True)\nout2 = np.linalg.inv(L) @ b\nnp.allclose(out1, out2)\n\nTrue\n\nout3 = sp.linalg.solve_triangular(U, b, lower=False)\nout4 = np.linalg.inv(U) @ b\nnp.allclose(out1, out2)\n\nTrue\n\n\nTo reiterate the distinction between matrix inversion and solving a system of equations, when we write \\(U^{-1}b\\), what we mean on a computer is to carry out the above algorithm, not to find the inverse and then multiply.\nHere’s a good reason why.\n\nimport time\n\nnp.random.seed(1)\nn = 5000\nX = np.random.normal(size = (n,n))\n\n## R has the `crossprod` function, which would be more efficient\n## than having to transpose, but numpy doesn't seem to have an equivalent.\nX = X.T @ X\nb = np.random.normal(size = n)\nL = np.linalg.cholesky(X) # L is upper-triangular\n\nt0 = time.time()\nout1 = sp.linalg.solve_triangular(L, b, lower=True)\ntime.time() - t0\n\n0.031229019165039062\n\nt0 = time.time()\nout2 = np.linalg.inv(L) @ b\ntime.time() - t0\n\n3.1720523834228516\n\n\nThat assumes you have \\(L\\), but we’ll see in a bit that even when one accounts for the creation of \\(L\\), you don’t want to invert matrices in order to solve systems of equations." + "objectID": "syllabus.html#course-requirements-and-grading", + "href": "syllabus.html#course-requirements-and-grading", + "title": "Statistics 243 Fall 2023", + "section": "Course requirements and grading", + "text": "Course requirements and grading\n\nScheduling Conflicts\nCampus asks that I include this information about conflicts: Please notify me in writing by the second week of the term about any known or potential extracurricular conflicts (such as religious observances, graduate or medical school interviews, or team activities). I will try my best to help you with making accommodations, but I cannot promise them in all cases. In the event there is no mutually-workable solution, you may be dropped from the class.\nThe main conflict that would be a problem would be the quizzes, whose dates I will determine in late August / early September.\nQuizzes are in-person. There is no remote option, and the only make-up accommodations I will make are for illness or serious personal issues. Do not schedule any travel that may conflict with a quiz.\n\n\nCourse grades\nThe grade for this course is primarily based on assignments due every 1-2 weeks, two quizzes (likely in early-mid October and mid-late November), and a final group project. I will also provide extra credit questions on some problem sets. There is no final exam. 50% of the grade is based on the problem sets, 25% on the quizzes, 15% on the project, and 10% on your participation in discussions on Ed, your responses to the in-class Google forms questions, as well as occasional brief questions that I will ask you to answer in advance of the next class.\nGrades will generally be As and Bs. An A involves doing all the work, getting full credit on most of the problem sets, doing well on the quizzes, and doing a thorough job on the final project.\n\n\nProblem sets\nWe will be less willing to help you if you come to our office hours or post a question online at the last minute. Working with computers can be unpredictable, so give yourself plenty of time for the assignments.\nThere are several rules for submitting your assignments.\n\nYou should prepare your assignments using Quarto.\nProblem set submission consists of both of the following:\n\nA PDF submitted electronically through Gradescope, by the start of class (10 am) on the due date, and\nAn electronic copy of the PDF, code file, and Quarto document pushed to your class GitHub repository, following the instructions to be provided by the GSI.\n\nOn-time submission will be determined based on the time stamp of when the PDF is submitted to Gradescope.\nAnswers should consist of textual response or mathematical expressions as appropriate, with key chunks of code embedded within the document. Extensive additional code can be provided as an appendix. Before diving into the code for a problem, you should say what the goal of the code is and your strategy for solving the problem. Raw code without explanation is not an appropriate solution. Please see our qualitative grading rubric for guidance. In general the rubric is meant to reinforce good coding practices and high-quality scientific communication.\nAny mathematical derivations may be done by hand and scanned with your phone if you prefer that to writing up LaTeX equations.\n\nNote: Quarto Markdown is an extension to the Markdown markup language that allows one to embed Python and R code within an HTML document. Please see the SCF dynamics document tutorial; there will be additional information in the first section and on the first problem set.\n\n\nSubmitting assignments\nIn the first section (September 1), we’ll discuss how to submit your problem sets both on Gradescope and via your class GitHub repository, located at https://github.berkeley.edu/<your_calnet_username>.\n\n\nProblem set grading\nThe grading scheme for problem sets is as follows. Each problem set will receive a numeric score for (1) presentation and explanation of results, (2) technical accuracy of code or mathematical derivation, and (3) code quality/style and creativity. For each of these three components, the possible scores are:\n\n0 = no credit,\n1 = partial credit (you did some of the problems but not all),\n2 = satisfactory (you tried everything but there were pieces of what you did that didn’t solve or present/explain one or more problems in a complete way), and\n3 = full credit.\n\nAgain, the qualitative grading rubric provides guidance on what we want to see for full credit.\nFor components #1 and #3, many of you will get a score of 2 for some problem sets as you develop good coding practices. You can still get an A in the class despite this.\nYour total score for the PS is a weighted sum of the scores for the three components. If you turn in a PS late, I’ll bump you down by two points (out of the available). If you turn it in really late (e.g., after we start grading them), I will bump you down by four points. No credit after solutions are distributed.\n\n\nFinal project\nThe final project will be a joint coding project in groups of 3-4. I’ll assign an overall task, and you’ll be responsible for dividing up the work, coding, debugging, testing, and documentation. You’ll need to use the Git version control system for working in your group.\n\n\nRules for working together and the campus honor code\nI encourage you to work together and help each other out. However, the problem set solutions you submit must be your own. What do I mean by that?\n\nYou should first try to figure out a given problem on your own. After that, if you’re stuck or want to explore alternative approaches or check what you’ve done, feel free to consult with your fellow students and with the GSI and me.\nWhat does “consult with a fellow student mean”? You can discuss a problem with another student, brainstorm approaches, and share code syntax (generally not more than one line) on how to do small individual coding tasks within a problem.\n\nYou should not ask another student for complete code or solutions, or look at their code/solution.\nYou should not share complete code or solutions with another student or on Ed Discussion.\n\nYou may use ChatGPT (or similar chatbots) for help with small sections of a problem (e.g., how to do some specific Python or bash task). You should not use ChatGPT to try to answer an entire question. You should carefully verify that the result is correct.\nYou must provide attribution for ideas obtained elsewhere, including other students and ChatGPT or similar chatbots.\n\nIf you got a specific idea for how to do part of a problem from a fellow student (or some other resource, including ChatGPT), you should note that in your solution in the appropriate place (for specific syntax ideas, note this in a code comment), just as you would cite a book or URL.\nYou MUST note on your problem set solution any fellow students who you worked/consulted with.\nYou do not need to cite any Ed Discussion posts nor any discussions with Chris or Deeb.\n\nUltimately, your solution to a problem set (writeup and code) must be your own, and you’ll hear from me if either look too similar to someone else’s.\n\nPlease see the last section of this document for more information on the Campus Honor Code, which I expect you to follow." }, { - "objectID": "units/unit10-linalg.html#gaussian-elimination-lu-decomposition", - "href": "units/unit10-linalg.html#gaussian-elimination-lu-decomposition", - "title": "Numerical linear algebra", - "section": "Gaussian elimination (LU decomposition)", - "text": "Gaussian elimination (LU decomposition)\nGaussian elimination is a standard way of directly computing a solution for \\(Ax=b\\). It is equivalent to the LU decomposition. LU is primarily done with square matrices, but not always. Also LU decompositions do exist for some singular matrices.\nThe idea of Gaussian elimination is to convert the problem to a triangular system. In class, we’ll walk through Gaussian elimination in detail and see how it relates to the LU decomposition. I’ll describe it more briefly here. Following what we learned in algebra when we have multiple equations, we preserve the solution, \\(x\\), when we add multiples of rows (i.e., add multiples of equations) together. This amounts to doing \\(L_{1}Ax=L_{1}b\\) for a lower-triangular matrix \\(L_{1}\\) that produces all zeroes in the first column of \\(L_{1}A\\) except for the first row. We proceed to zero out values below the diagonal for the other columns of \\(A\\). The result is \\(L_{n-1}\\cdots L_{1}Ax\\equiv Ux=L_{n-1}\\cdots L_{1}b\\equiv b^{*}\\) where \\(U\\) is upper triangular. This is the forward reduction step of Gaussian elimination. Then the backward elimination step solves \\(Ux=b^{*}\\).\nIf we’re just looking for the solution of the system, we don’t need the lower-triangular factor \\(L=(L_{n-1}\\cdots L_{1})^{-1}\\) in \\(A=LU\\), but it turns out to have a simple form that is computed as we go along, it is unit lower triangular and the values below the diagonal are the negative of the values below the diagonals in \\(L_{1},\\ldots,L_{n-1}\\) (note that each \\(L_{j}\\) has non-zeroes below the diagonal only in the \\(j\\)th column). As a side note related to storage, it turns out that as we proceed, we can store the elements of \\(L\\) and \\(U\\) in the original \\(A\\) matrix, except for the implicit 1s on the diagonal of \\(L\\).\nIn class, we’ll work out the computational complexity of the LU and see that it is \\(O(n^{3})\\).\nIf we look at help(np.linalg.solve) in Python, we see that it uses *_gesv*. A Google search indicates that this is a Lapack routine that does the LU decomposition with partial pivoting and row interchanges (see below on what these are), so numpy is using the algorithm we’ve just discussed.\nWe can also explicitly get the LU decomposition in Python with scipy.linalg.lu(), though for most use cases, what we want to do is solve a system of equations. (In R, one can’t easily get the explicit LU decomposition, though solve() in R does use the LU.\nOne additional complexity is that we want to avoid dividing by very small values to avoid introducing numerical inaccuracy (we would get large values that might overwhelm whatever they are being added to, and small errors in the divisor will have large effects on the result). This can be done on the fly by interchanging equations to use the equation (row) that produces the largest value to divide by. For example in the first step, we would switch the first equation (first row) for whichever of the remaining equations has the largest value in the first column. This is called partial pivoting. The divisors are called pivots. Complete pivoting also considers interchanging columns, and while theoretically better, partial pivoting is generally sufficient and requires fewer computations. Partial pivoting can be expressed as multiplying along the way by permutation matrices, \\(P_{1},\\ldots P_{n-1}\\) that switch rows. One can show with some work that based on pivoting, we have \\(PA=LU\\), where \\(P=P_{n-1}\\cdots P_{1}\\). In the demo code, we’ll see a toy example of the impact of pivoting.\nFinally \\(|PA|=|P||A|=|L||U|=|U|\\) (why?) so \\(|A|=|U|/|P|\\) and since the determinant of each permutation matrix, \\(P_{j}\\) is -1 (except when \\(P_{j}=I\\) because we don’t need to switch rows), we just need to multiply by minus one if there is an odd number of permutations. Or if we know the matrix is non-negative definite, we just take the absolute value of \\(|U|\\). So Gaussian elimination provides a fast stable way to find the determinant." + "objectID": "syllabus.html#feedback", + "href": "syllabus.html#feedback", + "title": "Statistics 243 Fall 2023", + "section": "Feedback", + "text": "Feedback\nI welcome comments and suggestions and concerns. Particularly good suggestions will count towards your class participation grade." }, { - "objectID": "units/unit10-linalg.html#cholesky-decomposition", - "href": "units/unit10-linalg.html#cholesky-decomposition", - "title": "Numerical linear algebra", - "section": "Cholesky decomposition", - "text": "Cholesky decomposition\nWhen \\(A\\) is p.d., we can use the Cholesky decomposition to solve a system of equations. Positive definite matrices can be decomposed as \\(U^{\\top}U=A\\) where \\(U\\) is upper triangular. \\(U\\) is called a square root matrix and is unique (apart from the sign, which we fix by requiring the diagonals to be positive). One algorithm for computing \\(U\\) is:\n\n\\(U_{11}=\\sqrt{A_{11}}\\)\nFor \\(j=2,\\ldots,n\\), \\(U_{1j}=A_{1j}/U_{11}\\)\nFor \\(i=2,\\ldots,n\\),\n\n\\(U_{ii}=\\sqrt{A_{ii}-\\sum_{k=1}^{i-1}U_{ki}^{2}}\\)\nif \\(i<n\\), then for \\(j=i+1,\\ldots,n\\): \\(U_{ij}=(A_{ij}-\\sum_{k=1}^{i-1}U_{ki}U_{kj})/U_{ii}\\)\n\n\nWe can then solve a system of equations as: \\(U^{-1}(U^{\\top-1}b)\\).\nSince numpy’s cholesky gives \\(L = U^\\top\\), let’s instead solve the system as: \\(L^{\\top-1}(L^{-1}b)\\), which in Python can be done in either of the following ways:\n\nL = sp.linalg.cholesky(A)\nsp.linalg.solve_triangular(L.T, \n sp.linalg.solve_triangular(L, b, lower=True),\n lower=False)\n\nc, low = sp.linalg.cho_factor(A)\nsp.linalg.cho_solve((c, low), b)\n\nThe Cholesky has some nice advantages over the LU: (1) while both are \\(O(n^{3})\\), the Cholesky involves only half as many computations, \\(n^{3}/6+O(n^{2})\\) and (2) the Cholesky factorization has only \\((n^{2}+n)/2\\) unique values compared to \\(n^{2}+n\\) for the LU. Of course the LU is more broadly applicable. The Cholesky does require computation of square roots, but it turns out this is not too intensive. There is also a method for finding the Cholesky without square roots.\n\nUses of the Cholesky\nThe standard algorithm for generating \\(y\\sim\\mathcal{N}(0,A)\\) is:\n\nL = sp.linalg.cholesky(A)\ny = L @ np.random.normal(size = n)\n\nQuestion: where will most of the time in this two-step calculation be spent?\nIf a regression design matrix, \\(X\\), is full rank, then \\(X^{\\top}X\\) is positive definite, so we could find \\(\\hat{\\beta}=(X^{\\top}X)^{-1}X^{\\top}Y\\) using either the Cholesky or Gaussian elimination. Challenge: write efficient R code to carry out the OLS solution using either LU or Cholesky factorization.\nHowever, it turns out that the standard approach is to work with \\(X\\) using the QR decomposition rather than working with \\(X^{\\top}X\\); working with \\(X\\) is more numerically stable, though in most situations without extreme collinearity, either of the approaches will be fine.\n\n\nNumerical issues with eigendecompositions and Cholesky decompositions for positive definite matrices\nMonahan comments that in general Gaussian elimination and the Cholesky decomposition are very stable. However, in the Cholesky case, if the matrix is very ill-conditioned we can get \\(A_{ii}-\\sum_{k}U_{ki}^{2}\\) being negative and then the algorithm stops when we try to take the square root. In this case, the Cholesky decomposition does not exist numerically although it exists mathematically. It’s not all that hard to produce such a matrix, particularly when working with high-dimensional covariance matrices with large correlations.\n\nlocs = np.random.uniform(size = 100)\nrho = .1\ndists = np.abs(locs[:, np.newaxis] - locs)\nC = np.exp(-dists**2/rho**2)\ne = np.linalg.eig(C)\nnp.sort(e[0])[::-1][96:100]\n\narray([-4.16702596e-16+0.j, -5.38621518e-16+0.j, -6.26206506e-16+0.j,\n -6.98611709e-16+0.j])\n\ntry:\n L = np.linalg.cholesky(C)\nexcept Exception as error:\n print(error)\n \n\nMatrix is not positive definite\n\nvals = np.abs(e[0])\nnp.max(vals)/np.min(vals)\n\n3.135151918122864e+18\n\n\nI don’t see a way to use pivoting with the Cholesky in Python, but in R, one can do chol(C, pivot = TRUE).\nWe can think about the accuracy here as follows. Suppose we have a matrix whose diagonal elements (i.e., the variances) are order of magnitude 1 and that the true value of a \\(U_{ii}\\) is less than \\(1\\times10^{-16}\\). From the given \\(A_{ii}\\) we are subtracting \\(\\sum_{k}U_{ki}^{2}\\) and trying to calculate this very small number but we know that we can only represent the values \\(A_{ii}\\) and \\(\\sum_{k}U_{ki}^{2}\\) accurately to 16 places, so the difference is garbage starting in the 17th position and could well be negative. Now realize that \\(\\sum_{k}U_{ki}^{2}\\) is the result of a potentially large set of arithmetic operations, and is likely represented accurately to fewer than 16 places. Now if the true value of \\(U_{ii}\\) is smaller than the accuracy to which \\(\\sum_{k}U_{ki}^{2}\\) is represented, we can get a difference that is negative.\nNote that when the Cholesky fails, we can still compute an eigendecomposition, but we have negative numeric eigenvalues. Even if all the eigenvalues are numerically positive (or equivalently, we’re able to get the Cholesky), errors in small eigenvalues near machine precision could have large effects when we work with the inverse of the matrix. This is what happens when we have columns of the \\(X\\) matrix nearly collinear. We cannot statistically distinguish the effect of two (or more) covariates, and this plays out numerically in terms of unstable results.\nA strategy when working with mathematically but not numerically positive definite \\(A\\) is to set eigenvalues or singular values to zero when they get very small, which amounts to using a pseudo-inverse and setting to zero any linear combinations with very small variance. We can also use pivoting with the Cholesky and accumulate zeroes in the last \\(n-q\\) rows (for cases where we try to take the square root of a negative number), corresponding to the columns of \\(A\\) that are numerically linearly dependent. See the pivot argument to R’s chol()." + "objectID": "syllabus.html#accomodations-for-students-with-disabilities", + "href": "syllabus.html#accomodations-for-students-with-disabilities", + "title": "Statistics 243 Fall 2023", + "section": "Accomodations for Students with Disabilities", + "text": "Accomodations for Students with Disabilities\nPlease see me as soon as possible if you need particular accommodations, and we will work out the necessary arrangements." }, { - "objectID": "units/unit10-linalg.html#qr-decomposition", - "href": "units/unit10-linalg.html#qr-decomposition", - "title": "Numerical linear algebra", - "section": "QR decomposition", - "text": "QR decomposition\n\nIntroduction\nThe QR decomposition is available for any matrix, \\(X=QR\\), with \\(Q\\) orthogonal and \\(R\\) upper triangular. If \\(X\\) is non-square, \\(n\\times p\\) with \\(n>p\\) then the leading \\(p\\) rows of \\(R\\) provide an upper triangular matrix (\\(R_{1}\\)) and the remaining rows are 0. (I’m using \\(p\\) because the QR is generally applied to design matrices in regression). In this case we really only need the first \\(p\\) columns of \\(Q\\), and we have \\(X=Q_{1}R_{1}\\), the ‘skinny’ QR (this is what R’s QR provides). For uniqueness, we can require the diagonals of \\(R\\) to be nonnegative, and then \\(R\\) will be the same as the upper-triangular Cholesky factor of \\(X^{\\top}X\\):\n\\[\\begin{aligned} X^{\\top}X & = & R^{\\top}Q^{\\top}QR \\\\ & = & R^{\\top}R\\end{aligned}.\\]\nThere are three standard approaches for computing the QR, using (1) reflections (Householder transformations), (2) rotations (Givens transformations), or (3) Gram-Schmidt orthogonalization (see below for details).\nFor \\(n\\times n\\) \\(X\\), the QR (for the Householder approach) requires \\(2n^{3}/3\\) flops, so QR is less efficient than LU or Cholesky.\nWe can also obtain the pseudo-inverse of \\(X\\) from the QR: \\(X^{+}=[R_{1}^{-1}\\,0]Q^{\\top}\\). In the case that \\(X\\) is not full-rank, there is a version of the QR that will work (involving pivoting) and we end up with some additional zeroes on the diagonal of \\(R_{1}\\).\n\n\nRegression and the QR\nOften QR is used to fit linear models, including in R. Consider the linear model in the form \\(Y=X\\beta+\\epsilon\\), finding \\(\\hat{\\beta}=(X^{\\top}X)^{-1}X^{\\top}Y\\). Let’s consider the skinny QR and note that \\(R^{\\top}\\) is invertible. Therefore, we can express the normal equations as\n\\[\n\\begin{aligned}\nX^{\\top}X\\beta & = & X^{\\top} Y \\\\\nR^{\\top}Q^{\\top}QR\\beta & = & R^{\\top}Q^{\\top} Y \\\\\nR \\beta & = & Q^{\\top} Y\n\\end{aligned}\n\\]\nand solving for \\(\\beta\\) is just a backsolve since \\(R\\) is upper-triangular. Furthermore the standard regression quantities, such as the hat matrix, the SSE, the residuals, etc. can be easily expressed in terms of \\(Q\\) and \\(R\\).\nWhy use the QR instead of the Cholesky on \\(X^{\\top}X\\)? The condition number of \\(X\\) is the square root of that of \\(X^{\\top}X\\), and the \\(QR\\) factorizes \\(X\\). Monahan has a discussion of the condition of the regression problem, but from a larger perspective, the situations where numerical accuracy is a concern are generally cases where the OLS estimators are not particularly helpful anyway (e.g., highly collinear predictors).\nWhat about computational order of the different approaches to least squares? The Cholesky is \\(np^{2}+\\frac{1}{3}p^{3}\\), an algorithm called sweeping is \\(np^{2}+p^{3}\\) , the Householder method for QR is \\(2np^{2}-\\frac{2}{3}p^{3}\\), and the modified Gram-Schmidt approach for QR is \\(2np^{2}\\). So if \\(n\\gg p\\) then Cholesky (and sweeping) are faster than the QR approaches. According to Monahan, modified Gram-Schmidt is most numerically stable and sweeping least. In general, regression is pretty quick unless \\(p\\) is large since it is linear in \\(n\\), so it may not be worth worrying too much about computational differences of the sort noted here.\n\n\nRegression and the QR in R and Python\nWe can get the Q and R matrices easily in Python.\n\nQ,R = np.linalg.qr(X)\n\nOne of the methods used by the statsmodel package in Python uses the QR to fit a regression.\nNote that by default in Python (and in R), you get the skinny QR, namely only the first \\(p\\) rows of \\(R\\) and the first \\(p\\) columns of \\(Q\\), where the latter form an orthonormal basis for the column space of \\(X\\). The remaining columns form an orthonormal basis for the null space of \\(X\\) (the space orthogonal to the column space of \\(X\\)). The analogy in regression is that we get the basis vectors for the regression, while adding the remaining columns gives us the full \\(n\\)-dimensional space of the observations.\nRegression in R uses the QR decomposition via qr(), which calls a Fortran function. qr() (and the Fortran functions that are called) is specifically designed to output quantities useful in fitting linear models.\nIn R, qr() returns the result as a list meant for use by other tools. R stores the \\(R\\) matrix in the upper triangle of $qr, while the lower triangle of $qr and $aux store the information for constructing \\(Q\\) (this relates to the Householder-related vectors \\(u\\) below). One can multiply by \\(Q\\) using qr.qy() and by \\(Q^{\\top}\\) using qr.qty(). If you want to extract \\(R\\) and \\(Q\\), the following will work:\n\nX.qr = qr(X)\nQ = qr.Q(X.qr)\nR = qr.R(X.qr) \n\nAs a side note, there are QR-based functions that provide regression-related quantities, such as qr.resid(), qr.fitted() and qr.coef(). These functions (and their Fortran counterparts) exist because one can work through the various regression quantities of interest and find their expressions in terms of \\(Q\\) and \\(R\\), with nice properties resulting from \\(Q\\) being orthogonal and \\(R\\) triangular.\n\n\nComputing the QR decomposition\nHere we’ll see some of the details of the different approaches to the QR, in part because they involve some concepts that may be useful in other contexts. I won’t expect you to see all of how this works, but please skim through this to get an idea of how things are done.\nOne approach involves reflections of vectors and a second rotations of vectors. Reflections and rotations are transformations that are performed by orthogonal matrices. The determinant of a reflection matrix is -1 and the determinant of a rotation matrix is 1. We’ll see some of the details in the demo code.\n\nQR Method 1: Reflections\nIf \\(u\\) and \\(v\\) are orthonormal vectors and \\(x\\) is in the space spanned by \\(u\\) and \\(v\\), \\(x=c_{1}u+c_{2}v\\), then \\(\\tilde{x}=-c_{1}u+c_{2}v\\) is a reflection (a Householder reflection) along the \\(u\\) dimension (since we are using the negative of that basis vector). We can think of this as reflecting across the plane perpendicular to \\(u\\). This extends simply to higher dimensions with orthonormal vectors, \\(u,v_{1},v_{2},\\ldots\\)\nSuppose we want to formulate the reflection in terms of a “Householder” matrix, \\(Q\\). It turns out that \\[Qx=\\tilde{x}\\] if \\(Q=I-2uu^{\\top}\\). \\(Q\\) has the following properties: (1) \\(Qu=-u\\), (2) \\(Qv=v\\) for \\(u^{\\top}v=0\\), (3) \\(Q\\) is orthogonal and symmetric.\nOne way to create the QR decomposition is by a series of Householder transformations that create an upper triangular \\(R\\) from \\(X\\):\n\\[\n\\begin{aligned}\nR & = & Q_{p}\\cdots Q_{1} X \\\\\nQ & = & (Q_{p}\\cdots Q_{1})^{\\top}\n\\end{aligned}\n\\]\nwhere we make use of the symmetry in defining \\(Q\\).\nBasically \\(Q_{1}\\) reflects the first column of \\(X\\) with respect to a carefully chosen \\(u\\), so that the result is all zeroes except for the first element. We want \\(Q_{1}x=\\tilde{x}=(||x||,0,\\ldots,0)\\). This can be achieved with \\(u=\\frac{x-\\tilde{x}}{||x-\\tilde{x}||}\\). Then \\(Q_{2}\\) makes the last \\(n-2\\) rows of the second column equal to zero. We’ll work through this a bit in class.\nIn the regression context, as we work through the individual transformations, \\(Q_{j}=I-2u_{j}u_{j}^{\\top}\\), we apply them to \\(X\\) and \\(Y\\) to create \\(R\\) (note this would not involve doing the full matrix multiplication - think about what calculations are actually needed) and \\(QY=Q^{\\top}Y\\), and then solve \\(R\\beta=Q^{\\top}Y\\). To find \\(\\mbox{Cov}(\\hat{\\beta})\\propto(X^{\\top}X)^{-1}=(R^{\\top}R)^{-1}=R^{-1}R^{-\\top}\\) we do need to invert \\(R\\), but it’s upper-triangular and of dimension \\(p\\times p\\). It turns out that \\(Q^{\\top}Y\\) can be partitioned into the first \\(p\\) and the last \\(n-p\\) elements, \\(z^{(1)}\\) and \\(z^{(2)}\\). The SSR is \\(\\|z^{(1)}\\|^{2}\\) and SSE is \\(\\|z^{(2)}\\|^{2}\\).\nFinal side note: if \\(X\\) is square (so \\(n=p)\\) you might wonder why we need \\(Q_{p}\\) since after \\(p-1\\) reflections, we don’t need to zero anything else out (since the last column of \\(R\\) has \\(n\\) non-zero elements). It turns out that if we go back to thinking about a Householder reflection in general, there is a lack of uniqueness in choosing \\(\\tilde{x}\\). It could either be \\((||x||,0,\\ldots,0)\\) or \\((-||x||,0,\\ldots,0)\\). For better numerical stability, one chooses from the two of those such that \\(x_{1}\\) is of the opposite sign to \\(\\tilde{x}_{1}\\), so that one avoids cancellation of numbers that may be of the same magnitude when doing \\(x-\\tilde{x}\\). The transformation \\(Q_{p}\\) is the last step of taking that approach of choosing the sign at each step. \\(Q_{p}\\) doesn’t zero anything out; it just basically just involves potentially setting \\(R_{pp}\\) to be \\(-R_{pp}\\). (To be honest, I’m not clear on why one would bother to do that last step, but that seems to be how it is presented in discussions of the Householder approach.) Of course in the case of \\(p<n\\), we definitely need \\(Q_{p}\\) so that the last \\(n-p\\) rows of \\(R\\) are zero and we can then discard them when just using the skinny QR.\n\n\nQR Method 2: Rotations\nA Givens rotation matrix rotates a vector in a two-dimensional subspace to be axis oriented with respect to one of the two dimensions by changing the value of the other dimension. E.g. we can create \\(\\tilde{x}=(x_{1},\\ldots,\\tilde{x}_{p},\\ldots,0,\\ldots x_{n})\\) from \\(x=(x_{1,}\\ldots,x_{p},\\ldots,x_{q},\\ldots,x_{n})\\) using a matrix multiplication: \\(\\tilde{x}=Qx\\). \\(Q\\) is orthogonal but not symmetric.\nWe can use a series of Givens rotations to do the QR but unless it is done carefully, more computations are needed than with Householder reflections. The basic story is that we apply a series of Givens rotations to \\(X\\) such that we zero out the lower triangular elements.\n\\[\n\\begin{aligned}\nR & = & Q_{pn}\\cdots Q_{23}Q_{1n}\\cdots Q_{13}Q_{12} X \\\\\nQ & = & (Q_{pn}\\cdots Q_{12})^{\\top}\\end{aligned}.\n\\]\nNote that we create the \\(n-p\\) zero rows in \\(R\\) (because the calculations affect the upper triangle of \\(R\\)), but we can then ignore those rows and the corresponding columns of \\(Q\\).\n\n\nQR Method 3: Gram-Schmidt Orthogonalization\nGram-Schmidt involves finding a set of orthonormal vectors to span the same space as a set of LIN vectors, \\(x_{1},\\ldots,x_{p}\\). If we take the LIN vectors to be the columns of \\(X\\), so that we are discussing the column space of \\(X\\), then G-S yields the QR decomposition. Here’s the algorithm:\n\n\\(\\tilde{x}_{1}=\\frac{x_{1}}{\\|x_{1}\\|}\\) (normalize the first vector)\nOrthogonalize the remaining vectors with respect to \\(\\tilde{x}_{1}\\):\n\n\\(\\tilde{x}_{2}=\\frac{x_{2}-\\tilde{x}_{1}^{\\top}x_{2}\\tilde{x}_{1}}{\\|x_{2}-\\tilde{x}_{1}^{\\top}x_{2}\\tilde{x}_{1}\\|}\\), which orthogonalizes with respect to \\(\\tilde{x}_{1}\\) and normalizes. Note that \\(\\tilde{x}_{1}^{\\top}x_{2}\\tilde{x}_{1}=\\langle\\tilde{x}_{1},x_{2}\\rangle\\tilde{x}_{1}\\). So we are finding a scaling, \\(c\\tilde{x}_{1}\\), where \\(c\\) is based on the inner product, to remove the variation in the \\(x_{1}\\) direction from \\(x_{2}\\).\nFor \\(k>2\\), find interim vectors, \\(x_{k}^{(2)}\\), by orthogonalizing with respect to \\(\\tilde{x}_{1}\\)\n\nProceed for \\(k=3,\\ldots\\), in turn orthogonalizing and normalizing the first of the remaining vectors w.r.t. \\(\\tilde{x}_{k-1}\\) and orthogonalizing the remaining vectors w.r.t. \\(\\tilde{x}_{k-1}\\) to get new interim vectors\n\nMathematically, we could instead orthogonalize \\(x_{2}\\) w.r.t. \\(\\tilde{x}_{1}\\), then orthogonalize \\(x_{3}\\) w.r.t. \\(\\{\\tilde{x}_{1},\\tilde{x}_{2}\\}\\), etc. The algorithm above is the modified G-S, and is known to be more numerically stable if the columns of \\(X\\) are close to collinear, giving vectors that are closer to orthogonal. The resulting \\(\\tilde{x}\\) vectors are the columns of \\(Q\\). The elements of \\(R\\) are obtained as we proceed: the diagonal values are the the normalization values in the denominators, while the off-diagonals are the inner products with the already-computed columns of \\(Q\\) that are computed as part of the numerators.\nAnother way to think about this is that \\(R=Q^{\\top}X\\), which is the same as regressing the columns of \\(X\\) on \\(Q,\\) since \\((Q^{\\top}Q)^{-1}Q^{\\top}X=Q^{\\top}X\\). By construction, the first column of \\(X\\) is a scaling of the first column of \\(Q\\), the second column of \\(X\\) is a linear combination of the first two columns of \\(Q\\), etc., so \\(R\\) being upper triangular makes sense.\n\n\n\nThe “tall-skinny” QR\nSuppose you have a very large regression problem, with \\(n\\) very large, and \\(n\\gg p\\). There is a variant of the QR, called the tall-skinny QR (see http://arxiv.org/pdf/0808.2664v1.pdf for details) that allows us to find the decomposition in a parallel fashion. The basic idea is to do a nested set of QR decompositions on blocks of rows of \\(X\\):\n\\[\nX = \\left( \\begin{array}{c}\nX_{0} \\\\\nX_{1} \\\\\nX_{2} \\\\\nX_{3}\n\\end{array}\n\\right) =\n\\left(\n\\begin{array}{c}\nQ_{0} R_{0} \\\\\nQ_{1} R_{1} \\\\\nQ_{2} R_{2} \\\\\nQ_{3} R_{3}\n\\end{array} \\right),\n\\]\nfollowed by ‘reduction’ steps (this can be done in a map-reduce context) that do the \\(QR\\) of pairs of the \\(R\\) factors: \\[\\left(\\begin{array}{c}\nR_{0}\\\\\nR_{1}\\\\\nR_{2}\\\\\nR_{3}\n\\end{array}\\right)=\\left(\\begin{array}{c}\n\\left(\\begin{array}{c}\nR_{0}\\\\\nR_{1}\n\\end{array}\\right)\\\\\n\\left(\\begin{array}{c}\nR_{2}\\\\\nR_{3}\n\\end{array}\\right)\n\\end{array}\\right)=\\left(\\begin{array}{c}\nQ_{01}R_{01}\\\\\nQ_{23}R_{23}\n\\end{array}\\right)\\] and \\[\\left(\\begin{array}{c}\nR_{01}\\\\\nR_{23}\n\\end{array}\\right)=Q_{0123}R_{0123}.\\]\nThe full decomposition is then\n\\[X=\\left( \\begin{array}{cccc} Q_{0} & 0 & 0 & 0 \\\\ 0 & Q_{1} & 0 & 0 \\\\ 0 & 0 & Q_{2} & 0 \\\\ 0 & 0 & 0 & Q_{3} \\end{array} \\right) \\left( \\begin{array}{cc} Q_{01} & 0 \\\\ 0 & Q_{23} \\end{array} \\right) Q_{0123} R_{0123} = QR.\\]\nThe computation can be done in parallel (in particular it can be done with map-reduce) and the \\(Q\\) matrix for big problems would generally not be computed explicitly but would be stored in its constituent pieces.\nAlternatively, there is a variant on the algorithm that processes the row-blocks of \\(X\\) serially, allowing you to do QR on a large tall-skinny matrix that you can’t fit in memory (or possibly even on disk). First you do \\(QR\\) on \\(X_{0}\\) to get \\(Q_{0}R_{0}\\). Then you stack \\(R_{0}\\) on top of \\(X_{1}\\) and do QR to get \\(R_{01}\\). Then stack \\(R_{01}\\) on top of \\(X_{2}\\) to get \\(R_{012}\\), etc." + "objectID": "syllabus.html#campus-honor-code", + "href": "syllabus.html#campus-honor-code", + "title": "Statistics 243 Fall 2023", + "section": "Campus Honor Code", + "text": "Campus Honor Code\nThe following is the Campus Honor Code. With regard to collaboration and independence, please see my rules regarding problem sets above – Chris.\nThe student community at UC Berkeley has adopted the following Honor Code: “As a member of the UC Berkeley community, I act with honesty, integrity, and respect for others.” The hope and expectation is that you will adhere to this code.\nCollaboration and Independence: Reviewing lecture and reading materials and studying for exams can be enjoyable and enriching things to do with fellow students. This is recommended. However, unless otherwise instructed, homework assignments are to be completed independently and materials submitted as homework should be the result of one’s own independent work.\nCheating: A good lifetime strategy is always to act in such a way that no one would ever imagine that you would even consider cheating. Anyone caught cheating on a quiz or exam in this course will receive a failing grade in the course and will also be reported to the University Center for Student Conduct. In order to guarantee that you are not suspected of cheating, please keep your eyes on your own materials and do not converse with others during the quizzes and exams.\nPlagiarism: To copy text or ideas from another source without appropriate reference is plagiarism and will result in a failing grade for your assignment and usually further disciplinary action. For additional information on plagiarism and how to avoid it, see, for example: http://gsi.berkeley.edu/teachingguide/misconduct/prevent-plag.html\nAcademic Integrity and Ethics: Cheating on exams and plagiarism are two common examples of dishonest, unethical behavior. Honesty and integrity are of great importance in all facets of life. They help to build a sense of self-confidence, and are key to building trust within relationships, whether personal or professional. There is no tolerance for dishonesty in the academic world, for it undermines what we are dedicated to doing – furthering knowledge for the benefit of humanity.\nYour experience as a student at UC Berkeley is hopefully fueled by passion for learning and replete with fulfilling activities. And we also appreciate that being a student may be stressful. There may be times when there is temptation to engage in some kind of cheating in order to improve a grade or otherwise advance your career. This could be as blatant as having someone else sit for you in an exam, or submitting a written assignment that has been copied from another source. And it could be as subtle as glancing at a fellow student’s exam when you are unsure of an answer to a question and are looking for some confirmation. One might do any of these things and potentially not get caught. However, if you cheat, no matter how much you may have learned in this class, you have failed to learn perhaps the most important lesson of all." }, { - "objectID": "units/unit10-linalg.html#determinants", - "href": "units/unit10-linalg.html#determinants", - "title": "Numerical linear algebra", - "section": "Determinants", - "text": "Determinants\nThe absolute value of the determinant of a square matrix can be found from the product of the diagonals of the triangular matrix in any factorization that gives a triangular (including diagonal) matrix times an orthogonal matrix (or matrices) since the determinant of an orthogonal matrix is either one or minus one.\n\\(|A|=|QR|=|Q||R|=\\pm|R|\\)\n\\(|A^{\\top}A|=|(QR)^{\\top}QR|=|R^{\\top}R|=|R_{1}^{\\top}R_{1}|=|R_{1}|^{2}\\)\nIn R, the following will do it (on the log scale), since \\(R\\) is stored in the upper triangle of the $qr element.\n\nQ,R = qr(A)\nmagn = np.sum(np.log(np.abs(np.diag(R)))) \n\nAn alternative is the product of the diagonal elements of \\(D\\) (the singular values) in the SVD factorization, \\(A=UDV^{\\top}\\).\nFor non-negative definite matrices, we know the determinant is non-negative, so the uncertainty about the sign is not an issue. For positive definite matrices, a good approach is to use the product of the diagonal elements of the Cholesky decomposition.\nOne can also use the product of the eigenvalues: \\(|A|=|\\Gamma\\Lambda\\Gamma^{-1}|=|\\Gamma||\\Gamma^{-1}||\\Lambda|=|\\Lambda|\\)\n\nComputation\nComputing from any of these diagonal or triangular matrices as the product of the diagonals is prone to overflow and underflow, so we always work on the log scale as the sum of the log of the values. When some of these may be negative, we can always keep track of the number of negative values and take the log of the absolute values.\nOften we will have the factorization as a result of other parts of the computation, so we get the determinant for free.\nWe can use np.linalg.logdet() or (definitely not recommended) np.linalg.det() to calculate the determinant in Python. These functions use the LU decomposition." + "objectID": "ps/regex.html", + "href": "ps/regex.html", + "title": "Assignment: regex problems", + "section": "", + "text": "Overview\nRead the regular expression section of the bash shell tutorial and provide regular expression syntax to match the strings in the following scenarios. Any reasonable syntax si fine, and even better, challenge yourself to figure out multiple ways to answer each question.\nThis is an assignment, graded credit/no credit, and it does not need to follow the requirements for problem set submissions. Any format is fine (including hand-written and scanned).\n\n\nProblems\n\nMatch the strings “dog”, “Dog”, “dOg”, “doG”, “DOg”, etc. (The word ‘dog’ in any combination of lower and upper case.)\nMatch the strings “cat”, “at”, and “t”.\nMatch the strings “cat”, “caat”, “caaat”, etc.\nMatch two words separated by any amount of whitespace.\nMatch exactly two words separated by ay amount of whitespace and with or without whitespace at the beginning and end. (I.e., you shouldn’t match if there are three words.)" }, { - "objectID": "units/unit10-linalg.html#eigendecomposition", - "href": "units/unit10-linalg.html#eigendecomposition", - "title": "Numerical linear algebra", - "section": "Eigendecomposition", - "text": "Eigendecomposition\nThe eigendecomposition (spectral decomposition) is useful in considering convergence of algorithms and of course for statistical decompositions such as PCA. We think of decomposing the components of variation into orthogonal patterns (the eigenvectors) with variances (eigenvalues) associated with each pattern.\nSquare symmetric matrices have real eigenvectors and eigenvalues, with the factorization into orthogonal \\(\\Gamma\\) and diagonal \\(\\Lambda\\), \\(A=\\Gamma\\Lambda\\Gamma^{\\top}\\), where the eigenvalues on the diagonal of \\(\\Lambda\\) are ordered in decreasing value. Of course this is equivalent to the definition of an eigenvalue/eigenvector pair as a pair such that \\(Ax=\\lambda x\\) where \\(x\\) is the eigenvector and \\(\\lambda\\) is a scalar, the eigenvalue. The inverse of the eigendecomposition is simply \\(\\Gamma\\Lambda^{-1}\\Gamma^{\\top}\\). On a similar note, we can create a square root matrix, \\(\\Gamma\\Lambda^{1/2}\\), by taking the square roots of the eigenvalues.\nThe spectral radius of \\(A\\), denoted \\(\\rho(A)\\), is the maximum of the absolute values of the eigenvalues. As we saw when talking about ill-conditionedness, for symmetric matrices, this maximum is the induced norm, so we have \\(\\rho(A)=\\|A\\|_{2}\\). It turns out that \\(\\rho(A)\\leq\\|A\\|\\) for any induced matrix norm. The spectral radius comes up in determining the rate of convergence of some iterative algorithms.\n\nComputation\nThere are several methods for eigenvalues; a common one for doing the full eigendecomposition is the QR algorithm. The first step is to reduce \\(A\\) to upper Hessenburg form, which is an upper triangular matrix except that the first subdiagonal in the lower triangular part can be non-zero. For symmetric matrices, the result is actually tridiagonal. We can do the reduction using Householder reflections or Givens rotations. At this point the QR decomposition (using Givens rotations) is applied iteratively (to a version of the matrix in which the diagonals are shifted), and the result converges to a diagonal matrix, which provides the eigenvalues. It’s more work to get the eigenvectors, but they are obtained as a product of Householder matrices (required for the initial reduction) multiplied by the product of the \\(Q\\) matrices from the successive QR decompositions.\nWe won’t go into the algorithm in detail, but note that it involves manipulations and ideas we’ve seen already.\nIf only the largest (or the first few largest) eigenvalues and their eigenvectors are needed, which can come up in time series and Markov chain contexts, the problem is easier and can be solved by the power method. E.g., in a Markov chain context, steady state is reached through \\(x_{t}=A^{t}x_{0}\\). One can find the largest eigenvector by multiplying by \\(A\\) many times, normalizing at each step. \\(v^{(k)}=Az^{(k-1)}\\) and \\(z^{(k)}=v^{(k)}/\\|v^{(k)}\\|\\). There is an extension to find the \\(p\\) largest eigenvalues and their vectors. See the demo code in the qmd source file for an implementation (in R)." + "objectID": "ps/ps1.html", + "href": "ps/ps1.html", + "title": "Problem Set 1", + "section": "", + "text": "This covers material in Units 2 and 4 as well as practice with Quarto.\nIt’s due at 10 am (Pacific) on September 6, both submitted as a PDF to Gradescope as well as committed to your GitHub repository.\nPlease note my comments in the syllabus about when to ask for help and about working together. In particular, please give the names of any other students that you worked with on the problem set and indicate in the text or in code comments any specific ideas or code you borrowed from another student or any online reference (including ChatGPT or the like)." }, { - "objectID": "units/unit10-linalg.html#singular-value-decomposition", - "href": "units/unit10-linalg.html#singular-value-decomposition", - "title": "Numerical linear algebra", - "section": "Singular value decomposition", - "text": "Singular value decomposition\nLet’s consider an \\(n\\times m\\) matrix, \\(A\\), with \\(n\\geq m\\) (if \\(m>n\\), we can always work with \\(A^{\\top})\\). This often is a matrix representing \\(m\\) features of \\(n\\) observations. We could have \\(n\\) documents and \\(m\\) words, or \\(n\\) gene expression levels and \\(m\\) experimental conditions, etc. \\(A\\) can always be decomposed as \\[A=UDV^{\\top}\\] where \\(U\\) and \\(V\\) are matrices with orthonormal columns (left and right eigenvectors) and \\(D\\) is diagonal with non-negative values (which correspond to eigenvalues in the case of square \\(A\\) and to squared eigenvalues of \\(A^{\\top}A\\)).\nThe SVD can be represented in more than one way. One representation is \\[A_{n\\times m}=U_{n\\times k}D_{k\\times k}V_{k\\times m}^{\\top}=\\sum_{j=1}^{k}D_{jj}u_{j}v_{j}^{\\top}\\] where \\(u_{j}\\) and \\(v_{j}\\) are the columns of \\(U\\) and \\(V\\) and where \\(k\\) is the rank of \\(A\\) (which is at most the minimum of \\(n\\) and \\(m\\) of course). The diagonal elements of \\(D\\) are the singular values.\nThat representation is as the sum of rank-one matrices (since each term is the scaled outer product of two vectors).\nIf \\(A\\) is positive semi-definite, the eigendecomposition is an SVD. Furthermore, \\(A^{\\top}A=VD^{2}V^{\\top}\\) and \\(AA^{\\top}=UD^{2}U^{\\top}\\), so we can find the eigendecomposition of such matrices using the SVD of \\(A\\) (for \\(AA^{\\top}\\) we need to fill out \\(U\\) to have \\(n\\) columns). Note that the squares of the singular values of \\(A\\) are the eigenvalues of \\(A^{\\top}A\\) and \\(AA^{\\top}\\).\nWe can also fill out the matrices to get \\[A=U_{n\\times n}D_{n\\times m}V_{m\\times m}^{\\top}\\] where the added rows and columns of \\(D\\) are zero with the upper left block the \\(D_{k\\times k}\\) from above.\n\nUses\nThe SVD is an excellent way to determine a matrix rank and to construct a pseudo-inverse (\\(A^{+}=VD^{+}U^{\\top})\\).\nWe can use the SVD to approximate \\(A\\) by taking \\(A\\approx\\tilde{A}=\\sum_{j=1}^{p}D_{jj}u_{j}v_{j}^{\\top}\\) for \\(p<m\\). The Eckart-Minsky-Young theorem shows that the truncated SVD minimizes the Frobenius norm of \\(A-\\tilde{A}\\) over all possible rank-\\(p\\) approximations. As an example if we have a large image of dimension \\(n\\times m\\), we could hold a compressed version by a rank-\\(p\\) approximation using the SVD. The SVD is used a lot in clustering problems. For example, the Netflix prize was won based on a variant of SVD (in fact all of the top methods used variants on SVD, I believe).\nHere’s another way to think about the SVD in terms of transformations and bases. Applying the SVD to a vector, $ UDV^{top}x$, carries out the following steps:\n\n\\(V^{top}x\\) expresses \\(x\\) in terms of weights for the columns of \\(V\\).\nMultiplying the result by \\(D\\) scales/stretches the weights.\nMultiplying by \\(U\\) produces the result, which is a weighted combination of columns of \\(U\\), spanning the column-space of \\(U\\).\n\nSo applying the SVD transforms \\(x\\) from the column space of \\(V\\) to the column space of \\(U\\).\n\n\nComputation\nThe basic algorithm (Golub-Reinsch) is similar to the QR method for the eigendecomposition. We use a series of Householder transformations on the left and right to reduce \\(A\\) to an upper bidiagonal matrix, \\(A^{(0)}\\). The post-multiplications (the transformations on the right) generate the zeros in the upper triangle. (An upper bidiagonal matrix is one with non-zeroes only on the diagonal and first subdiagonal above the diagonal). Then the algorithm produces a series of upper bidiagonal matrices, \\(A^{(0)}\\), \\(A^{(1)},\\) etc. that converge to a diagonal matrix, \\(D\\) . Each step is carried out by a sequence of Givens transformations:\n\\[\n\\begin{aligned}\nA^{(j+1)} & = & R_{m-2}^{\\top} R_{m-3}^{\\top} \\cdots R_{0}^{\\top} A^{(j)} T_{0} T_{1} \\cdots T_{m-2} \\\\\n& = & RA^{(j)} T\n\\end{aligned}.\n\\]\nThis eventually gives \\(A^{(...)}=D\\) and by construction, \\(U\\) (the product of the pre-multiplied Householder matrices and the \\(R\\) matrices) and \\(V\\) (the product of the post-multiplied Householder matrices and the \\(T\\) matrices) are orthogonal. The result is then transformed by a diagonal matrix to make the elements of \\(D\\) non-negative and by permutation matrices to order the elements of \\(D\\) in nonincreasing order.\n\n\nComputation for large tall-skinny matrices\nThe SVD can also be generated from a QR decomposition. Take \\(X=QR\\) and then do an SVD on the \\(R\\) matrix to get \\(X=QUDV^{\\top}=U^{*}DV^{\\top}\\). This is particularly helpful for the case when \\(X\\) is tall and skinny (suppose \\(X\\) is \\(n\\times p\\) with \\(n\\gg p\\)), because we can do the tall-skinny QR, and the resulting SVD on \\(R\\) is easy computationally if \\(p\\) is manageable." + "objectID": "ps/ps1.html#comments", + "href": "ps/ps1.html#comments", + "title": "Problem Set 1", + "section": "", + "text": "This covers material in Units 2 and 4 as well as practice with Quarto.\nIt’s due at 10 am (Pacific) on September 6, both submitted as a PDF to Gradescope as well as committed to your GitHub repository.\nPlease note my comments in the syllabus about when to ask for help and about working together. In particular, please give the names of any other students that you worked with on the problem set and indicate in the text or in code comments any specific ideas or code you borrowed from another student or any online reference (including ChatGPT or the like)." }, { - "objectID": "units/unit10-linalg.html#linear-algebra-in-python", - "href": "units/unit10-linalg.html#linear-algebra-in-python", - "title": "Numerical linear algebra", - "section": "Linear algebra in Python", - "text": "Linear algebra in Python\nSpeedups and storage savings can be obtained by working with matrices stored in special formats when the matrices have special structure. E.g., we might store a symmetric matrix as a full matrix but only use the upper or lower triangle. Banded matrices and block diagonal matrices are other common formats. Banded matrices are all zero except for \\(A_{i,i+c_{k}}\\) for some small number of integers, \\(c_{k}\\). Viewed as an image, these have bands. The bands are known as co-diagonals.\nNote that for many matrix decompositions, you can change whether all of the aspects of the decomposition are returned, or just some, which may speed calculations.\nScipy provides functionality for working with matrices in various ways, including the scipy.sparse module, which provides support for structured sparse matrices such as triangular and diagonal matrices as well as unstructured sparse matrices using various standard representations.\nSome useful packages in R for matrices are Matrix, spam, and bdsmatrix. Matrix can represent a variety of rectangular matrices, including triangular, orthogonal, diagonal, etc. and provides methods for various matrix calculations that are specific to the matrix type. spam handles general sparse matrices with fast matrix calculations, in particular a fast Cholesky decomposition. bdsmatrix focuses on block-diagonal matrices, which arise frequently in contexts where there is clustering that induces within-cluster correlation and cross-cluster independence.\nIn general, matrix operations in Python and R go to compiled C or Fortran code without much intermediate Python or R code, so they can actually be pretty efficient and are based on the best algorithms developed by numerical experts. The core libraries that are used are LAPACK and BLAS (the Linear Algebra PACKage and the Basic Linear Algebra Subroutines). As we’ve discussed in the parallelization unit, one way to speed up code that relies heavily on linear algebra is to make sure you have a BLAS library tuned to your machine. These include OpenBLAS (open source), Intel’s MKL, AMD’s ACML, and Apple’s vecLib.\nIf you use Conda, numpy will generally be linked against MKL or OpenBLAS (this will depend on the locations online of the packages being installed, i.e., the channel(s) used). With pip, numpy will generally be linked against OpenBLAS. it’s possible to install numpy so that it uses OpenBLAS. R can be linked to the shared object library file (.so file or .dylib on a Mac) for a fast BLAS. These BLAS libraries are also available in threaded versions that farm out the calculations across multiple cores or processors that share memory. More details are available in this SCF documentation.\nBLAS routines do vector operations (level 1), matrix-vector operations (level 2), and dense matrix-matrix operations (level 3). Often the name of the routine has as its first letter “d”, “s”, “c” to indicate the routine is double precision, single precision, or complex. LAPACK builds on BLAS to implement standard linear algebra routines such as eigendecomposition, solutions of linear systems, a variety of factorizations, etc." + "objectID": "ps/ps1.html#formatting-requirements", + "href": "ps/ps1.html#formatting-requirements", + "title": "Problem Set 1", + "section": "Formatting requirements", + "text": "Formatting requirements\n\nYour electronic solution should be in the form of an Quarto file named ps1.qmd, with Python code chunks included in the file. Please see the Lab 1 and the dynamic documents tutorial for more information on how to do this.\nYour PDF submission should be the PDF produced from your qmd. Your GitHub submission should include the qmd file, any Python code files containing chunks that you read into your qmd file, and the final PDF, all named according to the submission guidelines.\nYour solution should not just be code - you should have text describing how you approached the problem and what the various steps were. Your code should have comments indicating what each function or block of code does, and for any lines of code or code constructs that may be hard to understand, a comment indicating what that code does.\nYou do not need to (and should not) show exhaustive output, but in general you should show short examples of what your code does to demonstrate its functionality. Please see the grading rubric, and note that the output should be produced as a result of the code chunks being run during the rendering process, not by copy-pasting of output from running the code separately." }, { - "objectID": "units/unit10-linalg.html#sparse-matrices", - "href": "units/unit10-linalg.html#sparse-matrices", - "title": "Numerical linear algebra", - "section": "Sparse matrices", - "text": "Sparse matrices\nAs an example of exploiting sparsity, we can use a standard format (CSR = compressed sparse row) in the Scipy sparse module:\nConsider the matrix to be row-major and store the non-zero elements in order in an array called data. Then create a array called indptr that stores the position of the first element of each row. Finally, have a array, indices that tells the column identity of each element.\n\nimport scipy.sparse as sparse\nmat = np.array([[0,0,1,0,10],[0,0,0,100,0],[0,0,0,0,0],[1000,0,0,0,0]])\nmat = sparse.csr_array(mat)\nmat.data\n\narray([ 1, 10, 100, 1000])\n\nmat.indices # column indices\n\narray([2, 4, 3, 0], dtype=int32)\n\nmat.indptr # row pointers\n\n## Ideally don't first construct the dense matrix if it is large.\n\narray([0, 2, 3, 3, 4], dtype=int32)\n\nmat2 = sparse.csr_array((mat.data, mat.indices, mat.indptr))\nmat2.toarray()\n\narray([[ 0, 0, 1, 0, 10],\n [ 0, 0, 0, 100, 0],\n [ 0, 0, 0, 0, 0],\n [1000, 0, 0, 0, 0]])\n\n\nThat’s also how things are done in the spam package in R.```\nWe can do a fast matrix multiply, \\(x = Ab\\), as follows in pseudo-code:\n for(i in 1:nrows(A)){\n x[i] = 0\n # should also check that row is not empty...\n for(j in (rowpointers[i]:(rowpointers[i+1]-1)) {\n x[i] = x[i] + entries[j] * b[colindices[j]]\n } \n }\nHow many computations have we done? Only \\(k\\) multiplies and \\(O(k)\\) additions where \\(k\\) is the number of non-zero elements of \\(A\\). Compare this to the usual \\(O(n^{2})\\) for dense multiplication.\nNote that for the Cholesky of a sparse matrix, if the sparsity pattern is fixed, but the entries change, one can precompute an optimal re-ordering that retains as much sparsity in \\(U\\) as possible. Then multiple Cholesky decompositions can be done more quickly as the entries change.\n\nBanded matrices\nSuppose we have a banded matrix \\(A\\) where the lower bandwidth is \\(p\\), namely \\(A_{ij}=0\\) for \\(i>j+p\\) and the upper bandwidth is \\(q\\) (\\(A_{ij}=0\\) for \\(j>i+q\\)). An alternative to reducing to \\(Ux=b^{*}\\) is to compute \\(A=LU\\) and then do two solutions, \\(U^{-1}(L^{-1}b)\\). One can show that the computational complexity of the LU factorization is \\(O(npq)\\) for banded matrices, while solving the two triangular systems is \\(O(np+nq)\\), so for small \\(p\\) and \\(q\\), the speedup can be dramatic.\nBanded matrices come up in time series analysis. E.g., moving average (MA) models produce banded covariance structures because the covariance is zero after a certain number of lags." + "objectID": "ps/ps1.html#problems", + "href": "ps/ps1.html#problems", + "title": "Problem Set 1", + "section": "Problems", + "text": "Problems\n\nPlease read these lecture notes about how computers work, used in a class on statistical computing at CMU. Briefly (a few sentences) describe the difference between disk and memory based on that reference and/or other resources you find.\nThis problem uses the ideas and tools in Unit 2, Sections 1-3 to explore approaches to reading and writing data from files and to consider file sizes in ASCII plain text vs. binary formats in light of the fact that numbers are (generally) stored as 8 bytes per number in binary formats.\n\nGenerate a numpy array (named x) of random numbers from a standard normal distribution with 10 columns and as many rows as needed that the data takes up about 100 Mb in size. As part of your answer, show the arithmetic (formatted using LaTeX math syntax) you did to determine the number of rows.\nExplain the sizes of the two files created below. In discussing the CSV text file, how many characters do you expect to be in the file (i.e., you should be able to estimate this very accurately from first principles without using wc or any explicit program that counts characters). Hint: what do we know about numbers drawn from a standard normal distribution?\n\nimport os\nimport pandas as pd\nx = x.round(decimals = 10)\n\npd.DataFrame(x).to_csv('x.csv', header = False, index = False)\nprint(f\"{str(os.path.getsize('x.csv')/1e6)} MB\")\n\npd.DataFrame(x).to_pickle('x.pkl', compression = None) \nprint(f\"{str(os.path.getsize('x.pkl')/1e6)} MB\")\n\n167.358888 MB\n\n\n100.000572 MB\n\n\nSuppose we had rounded each number to three decimal places. Would using CSV have saved disk space relative to the pickle file?\nNow consider saving out the numbers one number per row in a CSV file. Given we no longer have to save all the commas, why is the file size unchanged (or perhaps even greater if you are on Windows)?\nRead the CSV file into Python using pandas.read_csv. Compare the speed of reading with and without providing the dtype argument and using the python vs c engines. Repeat the timing of your first attempt (without dtype and with the default engine) a few times. In some cases you might find that the first time is slower; if so this has to do with the operating system caching the file in memory (we’ll discuss this further in Unit 8).\nFinally, let’s consider reading the CSV file in chunks as discussed in Unit 2. Time how long it takes to read the first 100,000 rows.\nNow experiment with the skiprows to see if you can read in a large chunk of data from the middle of the file as quickly as the same size chunk from the start of the file. What does this indicate regarding whether Pandas/Python has to read in all the data up to the point where the chunk in the middle starts or can skip over it in some fashion? Is there any savings relative to reading all the initial rows and the chunk in the middle all at once?\nNow read the data sequentially in equal-sized chunks and determine if reading in the large chunk in the middle (after having already read the earlier chunks) takes the same amount of time as it did in part (f). Comment on what you’ve learned.\n\nPlease read Section 1 of Unit 4 on good programming/project practices and incorporate what you’ve learned from that reading into your solution for Problem 4. (You can skip the section on Assertions and Testing, as we’ll cover that in Lab.) As your response to this question, briefly (a few sentences) note what you did in your code for Problem 4 that reflects what you read. Please also note anything in Unit 4 that you disagree with, if you have a different stylistic perspective.\nWe’ll experiment with webscraping and manipulating HTML by getting song lyrics from the web. Go to http://mldb.org/search and (in the search bar in the middle, not at the left) enter the name of a song and choose to search by ‘Title’ and ‘All words’. In some cases the search goes directly to the lyrics of the song (presumably when there is no ambiguity) and in others it goes to a table of potential songs with that or similar name. (For example, compare ‘Dance in the Dark’ (or ‘Dancing in the Dark’) to ‘Leaving Las Vegas’.)\n\nBased on the GET request being sent to the MLDb server (in the cases like ‘Dance in the Dark’ where you get a table back rather than a single song’s lyrics), determine how to programmatically search for a song by ‘Title’ and ‘All words’ using Python, based on our explorations in Unit 2. Side question: what does the si parameter control?\nWarning: It’s possible that if you repeatedly query the site too quickly, it will start returning “503” errors because it detects automated usage (see problem 5 below). So, if you are going to run code from a script such that multiple queries would get done in quick succession, please put something like time.sleep(2) in between the calls that do the HTTP requests. Also when developing your code, once you have the code working to download the HTML, use the downloaded HTML to develop the remainder of your code that manipulates the HTML and don’t repeatedly re-download the HTML as you work on the remainder of the code.\nWrite an overall Python function (and modular helper functions to do particular pieces of the work needed) that takes as input a title and artist, searches by the title, and then (based on an exact match to the title and artist in the resulting set of song results) finds the URL of the page for the lyrics for that particular song. Then use that URL and return the lyrics, the artist, and the album(s). You can assume that the song you want is on the first page of results. If no exact match is found, just return None. Make sure to explain how your code extracts the HTML elements you need. Hint: you will need to use some string processing functions to do basic manipulations. We’ll see this more in Unit 5, but for now, you can find information in the https://berkeley-scf.github.io/tutorial-string-processing/text-manipulation#2-basic-text-manipulation-in-python. You should NOT need to use regular expressions (which we’ll cover in Units 3 and 5) or the re package, though you can if you want to.\nModify your function so it works either when the lyrics are returned directly from the initial search or when multiple songs are returned. Include checks in your code so that it fails gracefully if the user provides invalid input or MLDb doesn’t return a result.\n(Extra credit) Modify your code to handle cases (e.g., searching for “Dance with me”) that return more than one page of results.\n\nLook at the robots.txt file for MLDb and for Google Scholar (scholar.google.com) and the references in Unit 2 on the ethics of webscraping. Does it seem like it’s ok to scrape data from MLDb? What about Google Scholar?" }, { - "objectID": "units/unit10-linalg.html#low-rank-updates-optional", - "href": "units/unit10-linalg.html#low-rank-updates-optional", - "title": "Numerical linear algebra", - "section": "Low rank updates (optional)", - "text": "Low rank updates (optional)\nA transformation of the form \\(A-uv^{\\top}\\) is a rank-one update because \\(uv^{\\top}\\) is of rank one.\nMore generally a low rank update of \\(A\\) is \\(\\tilde{A}=A-UV^{\\top}\\) where \\(U\\) and \\(V\\) are \\(n\\times m\\) with \\(n\\geq m\\). The Sherman-Morrison-Woodbury formula tells us that \\[\\tilde{A}^{-1}=A^{-1}+A^{-1}U(I_{m}-V^{\\top}A^{-1}U)^{-1}V^{\\top}A^{-1}\\] so if we know \\(x_{0}=A^{-1}b\\), then the solution to \\(\\tilde{A}x=b\\) is \\(x+A^{-1}U(I_{m}-V^{\\top}A^{-1}U)^{-1}V^{\\top}x\\). Provided \\(m\\) is not too large, and particularly if we already have a factorization of \\(A\\), then \\(A^{-1}U\\) is not too bad computationally, and \\(I_{m}-V^{\\top}A^{-1}U\\) is \\(m\\times m\\). As a result \\(A^{-1}(U(\\cdots)^{-1}V^{\\top}x)\\) isn’t too bad.\nThis also comes up in working with precision matrices in Bayesian problems where we may have \\(A^{-1}\\) but not \\(A\\) (we often add precision matrices to find conditional normal distributions). An alternative expression for the formula is \\(\\tilde{A}=A+UCV^{\\top}\\), and the identity tells us \\[\\tilde{A}^{-1}=A^{-1}-A^{-1}U(C^{-1}+V^{\\top}A^{-1}U)^{-1}V^{\\top}A^{-1}\\]\nBasically Sherman-Morrison-Woodbury gives us matrix identities that we can use in combination with our knowledge of smart ways of solving systems of equations." + "objectID": "ps/ps5.html", + "href": "ps/ps5.html", + "title": "Problem Set 5", + "section": "", + "text": "This covers material in Units 6 and 7 (and a bit into Unit 9).\nIt’s due at 10 am (Pacific) on October 27, both submitted as a PDF to Gradescope as well as committed to your GitHub repository.\nPlease see PS1 for formatting and attribution requirements.\nRunning parallel code, working with large datasets, and using shared computers can result in delays and unexpected problems. Don’t wait until the last minute. The lengthy time period to complete this problem set is simply a logistical result of when the quiz is scheduled – you should start on the problem set before the quiz. We are happy to help troubleshoot problems you may have with accessing the SCF, running parallel Python code, submitting jobs to the SCF cluster, etc. We will not be happy to do so if it is clear that you started at the last minute.\nGiven that you’ll be running batch jobs and operating on remote servers, for problems 1 and 2 you’ll need to provide your code in chunks with #| eval: false. You can paste in any output you need to demonstrate your work. Remember that you can use “```” to delineate blocks of text you want printed verbatim.\nTo monitor an sbatch job, see this SCF doc." }, - { - "objectID": "units/unit2-dataTech.html", - "href": "units/unit2-dataTech.html", - "title": "Data technologies, formats, and structures", + { + "objectID": "ps/ps5.html#comments", + "href": "ps/ps5.html#comments", + "title": "Problem Set 5", "section": "", - "text": "PDF\nReferences (see syllabus for links):\n(Optional) Videos:\nThere are four videos from 2020 in the bCourses Media Gallery that you can use for reference if you want to:\nNote that the videos were prepared for a version of the course that used R, so there are some differences from the content in the current version of the unit that reflect translating between R and Python. I’m not sure how helpful they’ll be, but they are available." + "text": "This covers material in Units 6 and 7 (and a bit into Unit 9).\nIt’s due at 10 am (Pacific) on October 27, both submitted as a PDF to Gradescope as well as committed to your GitHub repository.\nPlease see PS1 for formatting and attribution requirements.\nRunning parallel code, working with large datasets, and using shared computers can result in delays and unexpected problems. Don’t wait until the last minute. The lengthy time period to complete this problem set is simply a logistical result of when the quiz is scheduled – you should start on the problem set before the quiz. We are happy to help troubleshoot problems you may have with accessing the SCF, running parallel Python code, submitting jobs to the SCF cluster, etc. We will not be happy to do so if it is clear that you started at the last minute.\nGiven that you’ll be running batch jobs and operating on remote servers, for problems 1 and 2 you’ll need to provide your code in chunks with #| eval: false. You can paste in any output you need to demonstrate your work. Remember that you can use “```” to delineate blocks of text you want printed verbatim.\nTo monitor an sbatch job, see this SCF doc." }, { - "objectID": "units/unit2-dataTech.html#text-and-binary-files", - "href": "units/unit2-dataTech.html#text-and-binary-files", - "title": "Data technologies, formats, and structures", - "section": "Text and binary files", - "text": "Text and binary files\nIn general, files can be divided into text files and binary files. In both cases, information is stored as a series of bits. Recall that a bit is a single value in base 2 (i.e., a 0 or a 1), while a byte is 8 bits.\nA text file is one in which the bits in the file encode individual characters. Note that the characters can include the digit characters 0-9, so one can include numbers in a text file by writing down the digits needed for the number of interest. Examples of text file formats include CSV, XML, HTML, and JSON.\nText files may be simple ASCII files (i.e., files encoded using ASCII) or files in other encodings such as UTF-8, both covered in Section 5. ASCII files have 8 bits (1 byte) per character and can represent 128 characters (the 52 lower and upper case letters in English, 10 digits, punctuation and a few other things – basically what you see on a standard US keyboard). UTF-8 files have between 1 and 4 bytes per character.\nSome text file formats, such as JSON or HTML, are not easily interpretable/manipulable on a line-by-line basis (unlike, e.g., CSV), so they are not as amenable to processing using shell commands.\nA binary file is one in which the bits in the file encode the information in a custom format and not simply individual characters. Binary formats are not (easily) human readable but can be more space-efficient and faster to work with (because it can allow random access into the data rather than requiring sequential reading). The meaning of the bytes in such files depends on the specific binary format being used and a program that uses the file needs to know how the format represents information. Examples of binary files include netCDF files, Python pickle files, R data (e.g., .Rda) files, , and compiled code files.\nNumbers in binary files are usually stored as 8 bytes per number. We’ll discuss this much more in Unit 8." + "objectID": "ps/ps5.html#problems", + "href": "ps/ps5.html#problems", + "title": "Problem Set 5", + "section": "Problems", + "text": "Problems\n\nThis problem asks you to use Dask to process some Wikipedia traffic data and makes use of tools discussed in Section on October 6. The files in /scratch/users/paciorek/wikistats/dated_2017_small/dated (on the SCF) contain data on the number of visits to different Wikipedia pages on November 4, 2008 (which was the date of the US election in 2008 in which Barack Obama was elected). The columns are: date, time, language, webpage, number of hits, and page size. (Note that the Unit 7 Dask bag example and Question 2 below use a larger set of the same data.)\n\nIn an interactive session on the SCF Linux cluster (ideally on the low partition, but if necessary on the high partition), started using srun (with four cores requested) as discussed in Section:\n\nCopy the data files to a subdirectory of the /tmp directory of the machine your interactive session is running on. (Keep your code files in your home directory.) Putting the files on the local hard drive of the machine you are computing on reduces the amount of copying data across the network (in the situation where you read the data into your program multiple times) and should speed things up in step ii.\nWrite efficient Python code to do the following: Using the dask package as seen in Unit 6, with either map or a list of delayed tasks, write code that, in parallel, reads in the space-delimited files and filters to only the rows that refer to pages where “Barack_Obama” appears in the page title (column 4). You can use the code from Unit 6 as a template. Collect all the results into a single data frame. In your srun invocation and in your code, please use four cores in your parallelization so that other cores are saved for use by other users/students. IMPORTANT: before running the code on the full set of data, please test your code on a small subset first (and test your function on a single input file serially).\nTabulate the number of hits for each hour of the day. (I don’t care how you do this - using either Python or R is fine.) Make a (time-series) plot showing how the number of visits varied over the day. Note that the time zone is UTC/GMT, so you won’t actually see the evening times when Obama’s victory was announced - we’ll see that in Question 2. Feel free to do this step outside the context of the parallelization. You’ll want to use datetime functions from Pandas or the datetime package to manipulate the timing information (i.e., don’t use string manipulations).\nRemove the files from /tmp.\n\nTips:\n\nIn general, keep in mind various ideas from Unit 2 about reading data from files. A couple things that posed problems when I was prototyping this in using pandas.read_csv were that there are lines with fewer than six fields and that there are lines that have quotes that should be treated as part of the text of the fields and not as separators. To get things to work ok, I needed to set the dtype to str for the first two fields (for ease of dealing with the date/time info later) but NOT set the dtype for the other fields, and to use the quoting argument to handle the literal quotes.\nWhen starting your srun session, please include the flag --mem-per-cpu=5G when submitting the Slurm job. In general one doesn’t need to request memory when submitting jobs to the SCF cluster but there is a weird interaction between Dask and Slurm that I don’t quite understand that requires this.\n\nNow replicate steps (i) and (ii) but using sbatch to submit your job as a batch job to the SCF Linux cluster, where step (ii) involves running Python from the command line. You don’t need to make the plot again. As discussed here in the Dask documentation, put your Python/Dask code inside an if __name__ == '__main__' block.\nNote that you need to copy the files to /tmp in your submission script, so that the files are copied to /tmp on whichever node of the SCF cluster your job gets run on. Make sure that as part of your sbatch script you remove the files in /tmp at the end of the script. (Why? In general /tmp is cleaned out when a machine is rebooted, but this might take a while to happen and many of you will be copying files to the same hard drive so otherwise /tmp could run out of space.)\n\nConsider the Wikipedia traffic data for October 15-November 2008 (already available in /var/local/s243/wikistats/dated_2017_sub on any of the SCF cluster nodes in the low or high partitions). Again, explore the variation over time in the number of visits to Barack Obama-related Wikipedia sites, based on searching for “Barack_Obama” on English language Wikipedia pages. You should use Dask with distributed data structures to do the reading and filtering, as seen in Unit 7. Then group by day-hour (it’s fine to do the grouping/counting in Python in a way that doesn’t use Dask data structures). You can do this either in an interactive session using srun or a batch job using sbatch. And if you use srun, you can run Python itself either interactively or as a background job. Time how long it takes to do the Dask part of the computations to get a sense for how much time is involved working with this much data. Once you have done the filtering and gotten the counts for each day-hour, you can simply use standard Python or R code on your laptop to do some plotting to show how the traffic varied over the days of the full month-long time period and particularly over the hours of November 3-5, 2008 (election day was November 4 and Obama’s victory was declared at 11 pm Eastern time on November 4).\nNotes:\n\nThere are various ways to do this using Dask bags or Dask data frames, but I think the easiest in terms of using code that you’ve seen in Unit 7 is to read the data in and do the filtering using a Dask bag and then convert the Dask bag to a Dask dataframe to do the grouping and summarization. Alternatively you should be able to use foldby() from dask.bag, but figuring out what arguments to pass to foldby() is a bit involved.\nMake sure to test your code on a portion of the data before doing computation on the full dataset. Reading and filtering the whole dataset will take something like 30 minutes with 16 cores. You MUST test on a small number of files on your laptop or on one of the stand-alone SCF machines (e.g., radagast, gandalf, arwen) before trying to run the code on the full 40 GB (zipped) of data. For testing, the files are also available in /scratch/users/paciorek/wikistats/dated_2017_sub.\nWhen doing the full computation via your Slurm job submission:\n\nDon’t copy the data (unlike in Question 1) to avoid overloading our disks with each student having their own copy. Just use the data from /var/local/s243/wikistats/dated_2017_sub.\nPlease do not use more than 16 cores in your Slurm job submissions so that cores are available for your classmates. If your job is stuck in the queue you may want to run it with 8 rather than 16 cores.\nAs discussed in Section, when you use sbatch to submit a job to the SCF cluster or srun to run interactively, you should be using the --cpus-per-task flag to specify the number of cores that your computation will use. In your Python code, you can then either hard-code that same number of cores as the number of workers or (better) you can use the SLURM_CPUS_PER_TASK shell environment variable to tell Dask how many workers to start.\n\n\nSQL practice.\n\nUsing the Stack Overflow database, write SQL code that will determine how many users have asked both R- and Python-related questions (not necessarily, but possibly, about R and Python in the same question). There are various ways to do this; you might do this in a single query (for which there are various options), but it’s perfectly fine to create one or more views and then use those views to get the result as a subsequent query. (Hint: if you do this using joins, you’ll probably need to do the equivalent of a self join at some point.)\nExtend your query to include only users who have asked questions about R (that are not also about Python) and questions about Python (that are not also about R). How many such users are there?\n\n(Please wait to work on this until the week of Oct. 23 if you can.) This question prepares for the discussion of a simulation study in section on Friday October 27. The goal of the problem is to think carefully about the design and interpretation of simulation studies, which we’ll talk about in Unit 9, in particular in Section on Friday October 27. In particular, we’ll work with Cao et al. (2015), an article in the Journal of the Royal Statistical Society, Series B, which is a leading statistics journal. The article is available as cao_etal_2015.pdf under the ps directory on GitHub. Read Sections 1, 2.1, and 4 of the article. Also read Section 2 of Unit 9.\nYou don’t need to understand their method for fitting the regression (i.e., you can treat it as some black box algorithm) or the theoretical development. In particular, you don’t need to know what an estimating equation is - you can think of it as an alternative to maximum likelihood or to least squares for estimating the parameters of the statistical model. Equation 3 on page 759 is analogous to taking the sum of squares for a regression model, differentiating with respect to \\(\\beta\\), and setting equal to zero to solve to get \\(\\hat{\\beta}\\). In Equation 3, to find \\(\\hat{\\beta}\\) one sets the equation equal to zero and solves for \\(\\beta\\). As far as the kernel, its role is to weight each pair of observation and covariate value. This downweights pairs where the covariate is measured at a very different time than the observation.\nBriefly (a few sentences for each of the three questions below) answer the following questions.\n\nWhat are the goals of their simulation study, and what are the metrics that they consider in assessing their method?\nWhat choices did the authors have to make in designing their simulation study? What are the key aspects of the data generating mechanism that might affect their assessment of their method?\nConsider their Tables reporting the simulation results. For a method to be a good method, what would one want to see numerically in these columns?" }, { - "objectID": "units/unit2-dataTech.html#common-file-types", - "href": "units/unit2-dataTech.html#common-file-types", - "title": "Data technologies, formats, and structures", - "section": "Common file types", - "text": "Common file types\nHere are some of the common file types, some of which are text formats and some of which are binary formats.\n\n‘Flat’ text files: data are often provided as simple text files. Often one has one record or observation per row and each column or field is a different variable or type of information about the record. Such files can either have a fixed number of characters in each field (fixed width format) or a special character (a delimiter) that separates the fields in each row. Common delimiters are tabs, commas, one or more spaces, and the pipe (|). Common file extensions are .txt and .csv. Metadata (information about the data) are often stored in a separate file. CSV files are quite common, but if you have files where the data contain commas, other delimiters might be preferable. Text can be put in quotes in CSV files, and this can allow use of commas within the data. This is difficult to deal with from the command line, but read_table() in Pandas handles this situation.\n\nOne occasionally tricky difficulty is as follows. If you have a text file created in Windows, the line endings are coded differently than in UNIX. Windows uses a newline (the ASCII character \\n) and a carriage return (the ASCII character \\r) whereas UNIX uses onlyl a newline in UNIX). There are UNIX utilities (fromdos in Ubuntu, including the SCF Linux machines and dos2unix in other Linux distributions) that can do the necessary conversion. If you see \\^M at the end of the lines in a file, that’s the tool you need. Alternatively, if you open a UNIX file in Windows, it may treat all the lines as a single line. You can fix this with todos or unix2dos.\n\nIn some contexts, such as textual data and bioinformatics data, the data may be in a text file with one piece of information per row, but without meaningful columns/fields.\nData may also be in text files in formats designed for data interchange between various languages, in particular XML or JSON. These formats are “self-describing”; namely the metadata is part of the file. The lxml and json packages are useful for reading and writing from these formats. More in Section 4.\nYou may be scraping information on the web, so dealing with text files in various formats, including HTML. The requests and BeautifulSoup packages are useful for reading HTML.\nIn scientific contexts, netCDF (.nc) (and the related HDF5) are popular format for gridded data that allows for highly-efficient storage and contains the metadata within the file. The basic structure of a netCDF file is that each variable is an array with multiple dimensions (e.g., latitude, longitude, and time), and one can also extract the values of and metadata about each dimension. The netCDF4 package in Python nicely handles working with netCDF files.\nData may already be in a database or in the data storage format of another statistical package (Stata, SAS, SPSS, etc.). The Pandas package in Python has capabilities for importing Stata (read_stata), SPSS (read_spss), and SAS (read_sas) files, among others.\nFor Excel, there are capabilities to read an Excel file (see the read_excel function in Pandas), but you can also just go into Excel and export as a CSV file or the like and then read that into Python. In general, it’s best not to pass around data files as Excel or other spreadsheet format files because (1) Excel is proprietary, so someone may not have Excel and the format is subject to change, (2) Excel imposes limits on the number of rows, (3) one can easily manipulate text files such as CSV using UNIX tools, but this is not possible with an Excel file, (4) Excel files often have more than one sheet, graphs, macros, etc., so they’re not a data storage format per se.\nPython can easily interact with databases (SQLite, PostgreSQL, MySQL, Oracle, etc.), querying the database using SQL and returning results to Python. More in the big data unit and in the large datasets tutorial mentioned above." + "objectID": "ps/ps3.html", + "href": "ps/ps3.html", + "title": "Problem Set 3", + "section": "", + "text": "This covers material in Unit 5..\nIt’s due at 10 am (Pacific) on September 27, both submitted as a PDF to Gradescope as well as committed to your GitHub repository.\nPlease see PS1 for formatting and attribution requirements." }, { - "objectID": "units/unit2-dataTech.html#csv-vs.-specialized-formats-such-as-parquet", - "href": "units/unit2-dataTech.html#csv-vs.-specialized-formats-such-as-parquet", - "title": "Data technologies, formats, and structures", - "section": "CSV vs. specialized formats such as Parquet", - "text": "CSV vs. specialized formats such as Parquet\nCSV is a common format (particularly in some disciplines/contexts) and has the advantages of being simple to understand, human readable, and readily manipulable by line-based processing tools such as shell commands. However, it has various disadvantages:\n\nstorage is by row, which will often mix values of different types;\nextra space is taken up by explicitly storing commas and newlines; and\none must search through the document to find a given row or value – e.g., to find the 10th row, we must search for the 9th newline and then read until the 10th newline.\n\nA popular file format that has some advantages over plain text formats such as CSV is Parquet. The storage is by column (actually in chunks of columns). This works well with how datasets are often structured in that a given field/variable will generally have values of all the same type and there may be many repeated values, so there are opportunities for efficient storage including compression. Storage by column also allows retrieval only of the columns that a user needs. As a result data stored in the Parquet format often takes up much less space than stored as CSV and can be queried much faster. Also note that data stored in Parquet will often be stored as multiple files.\nHere’s a brief exploration using a data file not in the class repository.\n\nimport time\n\n## Read from CSV\nt0 = time.time()\ndata_from_csv = pd.read_csv(os.path.join('..', 'data', 'airline.csv'))\nprint(time.time() - t0)\n\n0.8456258773803711\n\n\n\n## Write out Parquet-formatted data\ndata_from_csv.to_parquet(os.path.join('..', 'data', 'airline.parquet'))\n\n## Read from Parquet\nt0 = time.time()\ndata_from_parquet = pd.read_parquet(os.path.join(\n '..', 'data', 'airline.parquet'))\nprint(time.time() - t0)\n\n0.10250258445739746\n\n\nThe CSV file is 51 MB while the Parquet file is 8 MB.\n\nimport subprocess\nsubprocess.run([\"ls\", \"-l\", os.path.join(\"..\", \"data\", \"airline.csv\")])\nsubprocess.run([\"ls\", \"-l\", os.path.join(\"..\", \"data\", \"airline.parquet\")])\n\n-rw-r--r-- 1 paciorek scfstaff 51480244 Aug 29 2022 ../data/airline.csv\n\n\nCompletedProcess(args=['ls', '-l', '../data/airline.csv'], returncode=0)\n\n\n-rw-r--r-- 1 paciorek scfstaff 8153160 Aug 31 08:01 ../data/airline.parquet\n\n\nCompletedProcess(args=['ls', '-l', '../data/airline.parquet'], returncode=0)" + "objectID": "ps/ps3.html#comments", + "href": "ps/ps3.html#comments", + "title": "Problem Set 3", + "section": "", + "text": "This covers material in Unit 5..\nIt’s due at 10 am (Pacific) on September 27, both submitted as a PDF to Gradescope as well as committed to your GitHub repository.\nPlease see PS1 for formatting and attribution requirements." }, { - "objectID": "units/unit2-dataTech.html#core-python-functions", - "href": "units/unit2-dataTech.html#core-python-functions", - "title": "Data technologies, formats, and structures", - "section": "Core Python functions", - "text": "Core Python functions\nThe read_table and read_csv functions in the Pandas package are commonly used for reading in data. They read in delimited files (CSV specifically in the latter case). The key arguments are the delimiter (the sep argument) and whether the file contains a header, a line with the variable names. We can use read_fwf() to read from a fixed width text file into a data frame.\nThe most difficult part of reading in such files can be dealing with how Pandas determines the types of the fields that are read in. While Pandas will try to determine the types automatically, it can be safer (and faster) to tell Pandas what the types are, using the dtype argument to read_table().\nLet’s work through a couple examples. Before we do that, let’s look at the arguments to read_table. Note that sep='' can use regular expressions (which would be helpful if you want to separate on any amount of white space, as one example).\n\ndat = pd.read_table(os.path.join('..', 'data', 'RTADataSub.csv'),\n sep = ',', header = None)\ndat.dtypes.head() # 'object' is string or mixed type\ndat.loc[0,1] \ntype(dat.loc[0,1]) # string!\n## Whoops, there is an 'x', presumably indicating missingness:\ndat.loc[:,1].unique()\n\n/usr/local/linux/mambaforge-3.11/lib/python3.11/site-packages/IPython/core/formatters.py:342: FutureWarning:\n\nIn future versions `DataFrame.to_latex` is expected to utilise the base implementation of `Styler.to_latex` for formatting and rendering. The arguments signature may therefore change. It is recommended instead to use `DataFrame.style.to_latex` which also contains additional functionality.\n\n\n\n\n\n\n'2336'\n\n\nstr\n\n\narray(['2336', '2124', '1830', '1833', '1600', '1578', '1187', '1005',\n '918', '865', '871', '860', '883', '897', '898', '893', '913',\n '870', '962', '880', '875', '884', '894', '836', '848', '885',\n '851', '900', '861', '866', '867', '829', '853', '920', '877',\n '908', '855', '845', '859', '856', '825', '828', '854', '847',\n '840', '873', '822', '818', '838', '815', '813', '816', '849',\n '802', '805', '792', '823', '808', '798', '800', '842', '809',\n '807', '826', '810', '801', '794', '771', '796', '790', '787',\n '775', '751', '783', '811', '768', '779', '795', '770', '821',\n '830', '767', '772', '791', '781', '773', '777', '814', '778',\n '782', '837', '759', '846', '797', '835', '832', '793', '803',\n '834', '785', '831', '820', '812', '824', '728', '760', '762',\n '753', '758', '764', '741', '709', '735', '749', '752', '761',\n '750', '776', '766', '789', '763', '864', '858', '869', '886',\n '844', '863', '916', '890', '872', '907', '926', '935', '933',\n '906', '905', '912', '972', '996', '1009', '961', '952', '981',\n '917', '1011', '1071', '1920', '3245', '3805', '3926', '3284',\n '2700', '2347', '2078', '2935', '3040', '1860', '1437', '1512',\n '1720', '1493', '1026', '928', '874', '833', '850', nan, 'x'],\n dtype=object)\n\n\n\n## Let's treat 'x' as a missing value indicator.\ndat2 = pd.read_table(os.path.join('..', 'data', 'RTADataSub.csv'),\n sep = ',', header = None, na_values = 'x')\ndat2.dtypes.head()\ndat2.loc[:,1].unique()\n\n/usr/local/linux/mambaforge-3.11/lib/python3.11/site-packages/IPython/core/formatters.py:342: FutureWarning:\n\nIn future versions `DataFrame.to_latex` is expected to utilise the base implementation of `Styler.to_latex` for formatting and rendering. The arguments signature may therefore change. It is recommended instead to use `DataFrame.style.to_latex` which also contains additional functionality.\n\n\n\n\n\n\narray([2336., 2124., 1830., 1833., 1600., 1578., 1187., 1005., 918.,\n 865., 871., 860., 883., 897., 898., 893., 913., 870.,\n 962., 880., 875., 884., 894., 836., 848., 885., 851.,\n 900., 861., 866., 867., 829., 853., 920., 877., 908.,\n 855., 845., 859., 856., 825., 828., 854., 847., 840.,\n 873., 822., 818., 838., 815., 813., 816., 849., 802.,\n 805., 792., 823., 808., 798., 800., 842., 809., 807.,\n 826., 810., 801., 794., 771., 796., 790., 787., 775.,\n 751., 783., 811., 768., 779., 795., 770., 821., 830.,\n 767., 772., 791., 781., 773., 777., 814., 778., 782.,\n 837., 759., 846., 797., 835., 832., 793., 803., 834.,\n 785., 831., 820., 812., 824., 728., 760., 762., 753.,\n 758., 764., 741., 709., 735., 749., 752., 761., 750.,\n 776., 766., 789., 763., 864., 858., 869., 886., 844.,\n 863., 916., 890., 872., 907., 926., 935., 933., 906.,\n 905., 912., 972., 996., 1009., 961., 952., 981., 917.,\n 1011., 1071., 1920., 3245., 3805., 3926., 3284., 2700., 2347.,\n 2078., 2935., 3040., 1860., 1437., 1512., 1720., 1493., 1026.,\n 928., 874., 833., 850., nan])\n\n\nUsing dtype is a good way to control how data are read in.\n\ndat = pd.read_table(os.path.join('..', 'data', 'hivSequ.csv'),\n sep = ',', header = 0,\n dtype = {\n 'PatientID': int,\n 'Resp': int,\n 'PR Seq': str,\n 'RT Seq': str,\n 'VL-t0': float,\n 'CD4-t0': int})\ndat.dtypes\ndat.loc[0,'PR Seq']\n\n/usr/local/linux/mambaforge-3.11/lib/python3.11/site-packages/IPython/core/formatters.py:342: FutureWarning:\n\nIn future versions `DataFrame.to_latex` is expected to utilise the base implementation of `Styler.to_latex` for formatting and rendering. The arguments signature may therefore change. It is recommended instead to use `DataFrame.style.to_latex` which also contains additional functionality.\n\n\n\n\n\n\n'CCTCAAATCACTCTTTGGCAACGACCCCTCGTCCCAATAAGGATAGGGGGGCAACTAAAGGAAGCYCTATTAGATACAGGAGCAGATGATACAGTATTAGAAGACATGGAGTTGCCAGGAAGATGGAAACCAAAAATGATAGGGGGAATTGGAGGTTTTATCAAAGTAARACAGTATGATCAGRTACCCATAGAAATCTATGGACATAAAGCTGTAGGTACAGTATTAATAGGACCTACACCTGTCAACATAATTGGAAGAAATCTGTTGACTCAGCTTGGTTGCACTTTAAATTTY'\n\n\nNote that you can avoid reading in one or more columns by using the usecols argument. Also, specifying the dtype argument explicitly should make for faster file reading.\nIf possible, it’s a good idea to look through the input file in the shell or in an editor before reading into Python to catch such issues in advance. Using the UNIX command less on RTADataSub.csv would have revealed these various issues, but note that RTADataSub.csv is a 1000-line subset of a much larger file of data available from the kaggle.com website. So more sophisticated use of UNIX utilities (as we will see in Unit 3) is often useful before trying to read something into a program.\nIf the file is not nicely arranged by field (e.g., if it has ragged lines), we’ll need to do some more work. We can read each line as a separate string, after which we can process the lines using text manipulation. Here’s an example from some US meteorological data where I know from metadata (not provided here) that the 4-11th values are an identifier, the 17-20th are the year, the 22-23rd the month, etc.\n\nfile_path = os.path.join('..', 'data', 'precip.txt')\nwith open(file_path, 'r') as file:\n lines = file.readlines()\n\nid = [line[3:11] for line in lines]\nyear = [int(line[17:21]) for line in lines]\nmonth = [int(line[21:23]) for line in lines]\nnvalues = [int(line[27:30]) for line in lines]\nyear[0:5]\n\n[2010, 2010, 2010, 2010, 2010]\n\n\nActually, that file, precip.txt, is in a fixed-width format (i.e., every element in a given column has the exact same number of characters),so reading in using pandas.read_fwf() would be a good strategy." + "objectID": "ps/ps3.html#problems", + "href": "ps/ps3.html#problems", + "title": "Problem Set 3", + "section": "Problems", + "text": "Problems\n\nIn this problem we’ll create a little time-saving hack as a way to get practice with Python classes.\nSuppose I want to be lazy and when I type \"q\" in Python, Python should quit (i.e., I don’t want to have to type quit()). Write a bit of Python code to achieve this.\nHints: (a) What specifically happens if I type the name of an object? (b) You will probably want this code to be in an #| eval:false chunk, since it if it successful, its operation will cause Python to quit and presumably your document will not render.\nLet’s investigate the structure of the pandas package to get some experience with the structure of a large Python package and with how import and the __init__.py file(s) are used. You’ll need to go into the Pandas source code (see Unit 5). Note that the main __init__.py and the __init__.py files in the subpackages/submodules are complicated, and I’m not expecting you to understand everything about them. Also note that the following cases involve functions, classes, and class methods. Be sure to be clear to say which of those it is and in the method case(s), make sure you’re clear on what class the method is part of and any class inheritance structure. Import pandas and then consider the following questions:\n\nConsider pandas.core.config_init.is_terminal. What namespace is it (or its class) in? What file/module is is_terminal in? Is it a function, class, or a class method (and if a class method what is the class)? Describe how it is imported by discussing the relevant statement(s) in the relevant __init__.py file(s).\nConsider pandas.read_csv. What namespace is it (or its class) in? What file/module is read_csv in? Is it a function, class, or a class method (and if a class method what is the class)? Describe how it is imported by discussing the relevant statement(s) in the relevant __init__.py file(s).\nConsider pandas.arrays.BooleanArray. What namespace is it (or its class) in? What file/module is BooleanArray in? Is it a function, class, or a class method (and if a class method what is the class)? Describe how it is imported by discussing the relevant statement(s) in the relevant __init__.py file(s).\nConsider pandas.DataFrame.to_csv. What namespace is it (or its class) in? What file/module is to_csv in? Is it a function, class, or a class method (and if a class method what is the class)? Describe how it is imported by discussing the relevant statement(s) in the relevant __init__.py file(s).\n\nHints: (1) grep -R <pattern> <directory> will search all files within a directory recursively. (2) As you work on this, you may want to be able to modify one or more of the __init__.py files to better understand what is happening (e.g., by commenting out a line of code or adding a print statement). A good way to do this is to create a Conda environment in which pandas is installed, so you isolate any changes you make, e.g., conda create -n test_env python=3.11 pandas. Then you can edit code files in the environment and when you start Python and import pandas, you should see the effects of your changes. Alternatively, you could use the debugger to set breakpoint(s) in an __init__.py file. (3) Or you might create your own small toy package to experiment and see how things work with nested __init__.py files and various ways to use import.\nThe goal of this problem is two-fold: first to give you practice with regular expressions and string processing and the second to have you thinking about writing well-structured, readable code (similar to question 4 of PS1). The website https://www.presidency.ucsb.edu/documents/presidential-documents-archive-guidebook/annual-messages-congress-the-state-the-union#axzz265cEKp1a has the text from all of the State of the Union speeches by US presidents. (These are the annual speeches in which the president speaks to Congress to “report” on the situation in the country.) Your task is to process the information and produce data on the speeches for the years 1900 to present. Note that while I present the problem below as subparts (a)-(i), your solution does NOT need to be divided into subparts in the same way. Your solution should do all of the downloading and processing from within Python so that your operations are self-contained and reproducible.\nYou can choose to use either a functional programming approach or an object-oriented approach, or possibly something that mixes the two. I strongly recommend that you use the approach that you are less familiar with so as to gain more experience. Please think about writing short, modular functions or methods, and operating in a vectorized manner (e.g., using map or possibly list comprehension). Think carefully about how to structure your objects to store the speech information so that the structure works well with your functions/methods.\nGiven you’ve already worked on webscraping, I’m providing some initial code for processing the main landing page in ps/ps3start.py in the course repository, and an example for getting the text of a speech. Also note that you will want to distinguish regular dashes (i.e., the hyphen in a hyphenated work like “team-building”) from a long (“em”) dash that separates clauses (unicode U+2014) – syntax from Unit 2 Section 5 may be helpful.\n\nExtract the URLs for the individual speeches from the URL above. Then use that information to read each speech into Python.\nFor each speech, extract the body of the speech.\nConvert the text so that all text that was not spoken by the president (e.g., Laughter and Applause) is stripped out.\nExtract the words and sentences from each speech as lists of strings, one element per sentence and one element per word. Note that there are probably some special cases here. Try to deal with as much as you can, but we’re not expecting perfection.\nFor each speech count the number of words and characters and compute the average word length.\nCount the following words or word stems: I, we, America{,n}, democra{cy,tic}, republic, Democrat{,ic}, Republican, free{,dom}, war, God [not including God bless], God {B,b}bless, {Jesus, Christ, Christian}, and any others that you think would be interesting.\nThe result of all of this activity should be well-structured data object(s) containing the information about the speeches.\nMake some basic plots that show how the variables have changed over time. Your response here does not have to be extensive but should illustrate what you would do if you were to proceed on to do extensive exploratory data analysis. If you’re not comfortable plotting in Python (that is the case for me) and you prefer to move the relevant data to R to make the plots, that is fine.\nExtra credit: Do some additional research and/or additional thinking to come up with additional variables that quantify speech in interesting ways. Do some plotting that illustrates how the speeches have changed over time.\n\nNow sketch out a design for a functional programming (FP) approach (if your solution to problem 3 used OOP) or an OOP approach (if your solution to problem 3 used functional programming). If you’re designing an OOP approach, decide what the classes would be and the fields and methods of those classes. If you’re designing a FP approach, decide what the functions would be and what inputs/output they would use. To be clear, you do not have to write any of the code for the methods/classes/functions; the idea is just to design the code. As your response in the OOP case, for each class, please provide a bulleted list of methods and bulleted list of fields and for each item briefly comment what the purpose is. Or in the FP case, for each function, provide a bulleted list of inputs and output and briefly comment on the purpose of each function." }, { - "objectID": "units/unit2-dataTech.html#connections-and-streaming", - "href": "units/unit2-dataTech.html#connections-and-streaming", - "title": "Data technologies, formats, and structures", - "section": "Connections and streaming", - "text": "Connections and streaming\nPython allows you to read in not just from a file but from a more general construct called a connection. This can include reading in text from the output of running a shell command and from unzipping a file on the fly.\nHere are some examples of connections:\n\nimport gzip\nwith gzip.open('dat.csv.gz', 'r') as file:\n lines = file.readlines()\n\nimport zipfile\nwith zipfile.ZipFile('dat.zip', 'r') as archive:\n with archive.open('data.txt', 'r') as file:\n lines = file.readlines()\n\nimport subprocess\ncommand = \"ls -al\"\noutput = subprocess.check_output(command, shell = True)\n# `output` is a sequence of bytes.\nwith io.BytesIO(output) as stream: # Create a file-like object.\n content = stream.readlines()\n\ndf = pd.read_csv(\"https://download.bls.gov/pub/time.series/cu/cu.item\", sep=\"\\t\")\n\nIf a file is large, we may want to read it in in chunks (of lines), do some computations to reduce the size of things, and iterate. This is referred to as online processing, streaming, or chunking, and can be done using Pandas (among other tools).\n\nfile_path = os.path.join('..', 'data', 'RTADataSub.csv')\nchunksize = 50 # Obviously this would be much larger in any real application.\n\nwith pd.read_csv(file_path, chunksize = chunksize) as reader:\n for chunk in reader:\n # manipulate the lines and store the key stuff\n print(f'Read {len(chunk)} rows.')\n\nMore details on sequential (on-line) processing of large files can be found in the tutorial on large datasets mentioned in the reference list above.\nOne cool trick that can come in handy is to ‘read’ from a string as if it were a text file. Here’s an example:\n\nfile_path = os.path.join('..', 'data', 'precip.txt')\nwith open(file_path, 'r') as file:\n text = file.read()\n\nstringIOtext = io.StringIO(text)\ndf = pd.read_fwf(stringIOtext, header = None, widths = [3,8,4,2,4,2])\n\nWe can create connections for writing output too. Just make sure to open the connection first." + "objectID": "rubric.html", + "href": "rubric.html", + "title": "Statistics 243 Fall 2023", + "section": "", + "text": "This document provides guidance for submitting high-quality problem set (and project) solutions. This guidance is based on general good practices for scientific communication, reproducible research, and software development." }, { - "objectID": "units/unit2-dataTech.html#file-paths", - "href": "units/unit2-dataTech.html#file-paths", - "title": "Data technologies, formats, and structures", - "section": "File paths", - "text": "File paths\nA few notes on file paths, related to ideas of reproducibility.\n\nIn general, you don’t want to hard-code absolute paths into your code files because those absolute paths won’t be available on the machines of anyone you share the code with. Instead, use paths relative to the directory the code file is in, or relative to a baseline directory for the project, e.g.:\n\n\ndat = pd.read_csv('../data/cpds.csv')\n\nUsing UNIX style directory separators will work in Windows, Mac or Linux, but using Windows style separators is not portable across operating systems.\n\n## good: will work on Windows\ndat = pd.read_csv('../data/cpds.csv')\n## bad: won't work on Mac or Linux\ndat = pd.read_csv('..\\data\\cpds.csv') \n\nEven better, use os.path.join so that paths are constructed specifically for the operating system the user is using:\n\n\n## good: operating-system independent\ndat = pd.read_csv(os.path.join('..', 'data', 'cpds.csv'))" + "objectID": "rubric.html#general-presentation", + "href": "rubric.html#general-presentation", + "title": "Statistics 243 Fall 2023", + "section": "General presentation", + "text": "General presentation\n\nSimply presenting code or derivations is not sufficient.\nBriefly describe the overall goal or strategy before providing code/derivations.\nAs needed describe what the pieces of your code/derivation are doing to make it easier for a reader to follow the steps.\nKeep your output focused on illustrating what you did, without distracting from the flow of your solution by showing voluminous output. The output should illustrate and demonstrate, not overwhelm or obscure. If you need to show longer output, you can add it at the end as supplemental material.\nOutput should be produced by your code (i.e., from the code chunks running when the document is rendered), not by copying and pasting results into the document." }, { - "objectID": "units/unit2-dataTech.html#reading-data-quickly-arrow-and-polars", - "href": "units/unit2-dataTech.html#reading-data-quickly-arrow-and-polars", - "title": "Data technologies, formats, and structures", - "section": "Reading data quickly: Arrow and Polars", - "text": "Reading data quickly: Arrow and Polars\nApache Arrow provides efficient data structures for working with data in memory, usable in Python via the PyArrow package. Data are stored by column, with values in a column stored sequentially and in such a way that one can access a specific value without reading the other values in the column (O(1) lookup). Arrow is designed to read data from various file formats, including Parquet, native Arrow format, and text files. In general Arrow will only read data from disk as needed, avoiding keeping the entire dataset in memory.\nOther options for avoiding reading all your data into memory include the Dask package and using numpy.load with the mmap_mode argument.\npolars is designed to be a faster alternative to Pandas for working with data in-memory.\n\nimport polars\nimport time\nt0 = time.time()\ndat = pd.read_csv(os.path.join('..', 'data', 'airline.csv'))\nt1 = time.time()\ndat2 = polars.read_csv(os.path.join('..', 'data', 'airline.csv'), null_values = ['NA'])\nt2 = time.time()\nprint(f\"Timing for Pandas: {t1-t0}.\")\nprint(f\"Timing for Polars: {t2-t1}.\")\n\nTiming for Pandas: 0.8093833923339844.\nTiming for Polars: 0.12352108955383301." + "objectID": "rubric.html#coding-practice", + "href": "rubric.html#coding-practice", + "title": "Statistics 243 Fall 2023", + "section": "Coding practice", + "text": "Coding practice\n\nMinimize (or eliminate) use of global variables.\nBreak down work into core tasks and develop small, modular, self-contained functions (or class methods) to carry out those tasks.\nDon’t repeat code. As needed refactor code to create new functions (or class methods).\nFunctions and classes should be “weakly coupled”, interacting via their interfaces and not by having to know the internals of how they work.\nUse data structures appropriate for the computations that need to be done.\nDon’t hard code ‘magic’ numbers. Assign such numbers to variables with clear names, e.g., speed_of_light = 3e8.\nProvide reasonable default arguments to functions (or class methods) when possible.\nProvide tests (including unit tests) when requested (this is good general practice but we won’t require it in all cases).\nAvoid overly complicated syntax – try to use the clearest syntax you can to solve the problem.\nIn terms of speed, don’t worry about it too much so long as the code finishes real-world tasks in a reasonable amount of time. When optimizing, focus on the parts of the code that are the bottlenecks.\nUse functions already available in the language rather than recreating yourself." }, { - "objectID": "units/unit2-dataTech.html#writing-output-to-files", - "href": "units/unit2-dataTech.html#writing-output-to-files", - "title": "Data technologies, formats, and structures", - "section": "Writing output to files", - "text": "Writing output to files\nFunctions for text output are generally analogous to those for input.\n\nfile_path = os.path.join('/tmp', 'tmp.txt')\nwith open(file_path, 'w') as file:\n file.writelines(lines)\n\nWe can also use file.write() to write individual strings.\nIn Pandas, we can use DataFrame.to_csv and DataFrame.to_parquet.\nWe can use the json.dump function to output appropriate data objects (e.g., dictionaries or possibly lists) as JSON. One use of JSON as output from Python would be to ‘serialize’ the information in an Python object such that it could be read into another program.\nAnd of course you can always save to a Pickle data file (a binary file format) using pickle.dump() and pickle.load() from the pickle package. Happily this is platform-independent so can be used to transfer Python objects between different OS." + "objectID": "rubric.html#code-style", + "href": "rubric.html#code-style", + "title": "Statistics 243 Fall 2023", + "section": "Code style", + "text": "Code style\n\nFollow a consistent style. While you don’t have to follow Python’s PEP8 style guide exactly, please look at it and follow it generally.\nUse informative variable and function names and have a consistent naming style.\nUse whitespace (spaces, newlines) and parentheses to make the structure of the code easy to understand and the individual syntax pieces clear.\nUse consistent indentation to make the structure of the code easy to understand.\nProvide comments that give the goal of a given piece of code and why it does things, but don’t use comments to restate what the code does when it should be obvious from reading the code.\n\nProvide summaries for blocks of code.\nFor particularly complicated syntax, say what a given piece of code does." }, { - "objectID": "units/unit2-dataTech.html#formatting-output", - "href": "units/unit2-dataTech.html#formatting-output", - "title": "Data technologies, formats, and structures", - "section": "Formatting output", - "text": "Formatting output\nWe can use string formatting to control how output is printed to the screen.\nThe mini-language involved in the format specification can get fairly involved, but a few basic pieces of syntax can do most of what one generally needs to do.\nWe can format numbers to chosen number of digits and decimal places and handle alignment, using the format method of the string class.\nFor example:\n\n'{:>10}'.format(3.5) # right-aligned, using 10 characters\n'{:.10f}'.format(1/3) # force 10 decimal places\n'{:15.10f}'.format(1/3) # force 15 characters, with 10 decimal places\nformat(1/3, '15.10f') # alternative using a function\n\n' 3.5'\n\n\n'0.3333333333'\n\n\n' 0.3333333333'\n\n\n' 0.3333333333'\n\n\nWe can also “interpolate” variables into strings.\n\n\"The number pi is {}.\".format(np.pi)\n\"The number pi is {:.5f}.\".format(np.pi)\n\"The number pi is {:.12f}.\".format(np.pi)\n\n'The number pi is 3.141592653589793.'\n\n\n'The number pi is 3.14159.'\n\n\n'The number pi is 3.141592653590.'\n\n\n\nval1 = 1.5\nval2 = 2.5\n# As of Python 3.6, put the variable names in directly.\nprint(f\"Let's add {val1} and {val2}.\") \nnum1 = 1/3\nprint(\"Let's add the %s numbers %.5f and %15.7f.\"\n %('floating point', num1 ,32+1/7))\n\nLet's add 1.5 and 2.5.\nLet's add the floating point numbers 0.33333 and 32.1428571.\n\n\nOr to insert into a file:\n\nfile_path = os.path.join('/tmp', 'tmp.txt')\nwith open(file_path, 'a') as file:\n file.write(\"Let's add the %s numbers %.5f and %15.7f.\"\n %('floating point', num1 ,32+1/7))\n\nround is another option, but it’s often better to directly control the printing format." + "objectID": "units/unit7-bigData.html", + "href": "units/unit7-bigData.html", + "title": "Big data and databases", + "section": "", + "text": "PDF\nReferences:\nI’ve also pulled material from a variety of other sources, some mentioned in context below.\nNote that for a lot of the demo code I ran the code separately from rendering this document because of the time involved in working with large datasets.\nWe’ll focus on Dask and databases/SQL in this Unit. The material on using Spark is provided for reference, but you’re not responsible for that material. If you’re interested in working with big datasets in R or with tools other than Dask in Python, there is some material in the tutorial on working with large datasets." }, { - "objectID": "units/unit2-dataTech.html#reading-html", - "href": "units/unit2-dataTech.html#reading-html", - "title": "Data technologies, formats, and structures", - "section": "Reading HTML", - "text": "Reading HTML\nHTML (Hypertext Markup Language) is the standard markup language used for displaying content in a web browser. In simple webpages (ignoring the more complicated pages that involve Javascript), what you see in your browser is simply a rendering (by the browser) of a text file containing HTML.\nHowever, instead of rendering the HTML in a browser, we might want to use code to extract information from the HTML.\nLet’s see a brief example of reading in HTML tables.\nNote that before doing any coding, it can be helpful to look at the raw HTML source code for a given page. We can explore the underlying HTML source in advance of writing our code by looking at the page source directly in the browser (e.g., in Firefox under the 3-lines (hamburger) “open menu” symbol, see Web Developer (or More Tools) -> Page Source and in Chrome View -> Developer -> View Source), or by downloading the webpage and looking at it in an editor, although in some cases (such as the nytimes.com case), what we might see is a lot of JavaScript.\nOne lesson here is not to write a lot of your own code to do something that someone else has probably already written a package for. We’ll use the BeautifulSoup4 package.\n\nimport requests\nfrom bs4 import BeautifulSoup as bs\n\nURL = \"https://en.wikipedia.org/wiki/List_of_countries_and_dependencies_by_population\"\nresponse = requests.get(URL)\nhtml = response.content\n\n# Create a BeautifulSoup object to parse the HTML\nsoup = bs(html, 'html.parser')\n\nhtml_tables = soup.find_all('table')\n\npd_tables = [pd.read_html(str(tbl))[0] for tbl in html_tables]\n\n[x.shape[0] for x in pd_tables]\n\npd_tables[0].head()\n\n[242, 13, 1]\n\n\n/usr/local/linux/mambaforge-3.11/lib/python3.11/site-packages/IPython/core/formatters.py:342: FutureWarning:\n\nIn future versions `DataFrame.to_latex` is expected to utilise the base implementation of `Styler.to_latex` for formatting and rendering. The arguments signature may therefore change. It is recommended instead to use `DataFrame.style.to_latex` which also contains additional functionality.\n\n\n\n\n\n\n\n\n\n\nUnnamed: 0\nCountry / Dependency\nPopulation\n% of world\nDate\nSource (official or from the United Nations)\nUnnamed: 6\n\n\n\n\n0\n–\nWorld\n8057107000\n100%\n31 Aug 2023\nUN projection[3]\nNaN\n\n\n1\n1\nChina\n1411750000\nNaN\n31 Dec 2022\nOfficial estimate[4]\n[b]\n\n\n2\n2\nIndia\n1392329000\nNaN\n1 Jul 2023\nOfficial projection[5]\n[c]\n\n\n3\n3\nUnited States\n335340000\nNaN\n31 Aug 2023\nNational population clock[7]\n[d]\n\n\n4\n4\nIndonesia\n277749853\nNaN\n31 Dec 2022\nOfficial estimate[8]\nNaN\n\n\n\n\n\n\n\nBeautiful Soup works by reading in the HTML as text and then parsing it to build up a tree containing the HTML elements. Then one can search by HTML tag or attribute for information you want using find_all.\nAs another example, it’s often useful to be able to extract the hyperlinks in an HTML document.\n\nURL = \"http://www1.ncdc.noaa.gov/pub/data/ghcn/daily/by_year\"\nresponse = requests.get(URL)\nsoup = bs(response.content, 'html.parser')\n\n## Approach 1: search for HTML 'a' tags.\na_elements = soup.find_all('a')\nlinks1 = [x.get('href') for x in a_elements]\n## Approach 2: search for 'a' elements with 'href' attribute\nhref_elements = soup.find_all('a', href = True)\nlinks2 = [x.get('href') for x in href_elements]\n## In either case, then use `get` to retrieve the `href` attribute value.\n\nlinks2[0:9]\n# help(bs.find_all)\n\n['?C=N;O=D',\n '?C=M;O=A',\n '?C=S;O=A',\n '?C=D;O=A',\n '/pub/data/ghcn/daily/',\n '1750.csv.gz',\n '1763.csv.gz',\n '1764.csv.gz',\n '1765.csv.gz']\n\n\nThe kwargs keyword arguments to find and find_all allow one to search for elements with particular characteristics, such as having a particular attribute (seen above) or having an attribute have a particular value (e.g., picking out an element with a particular id).\nHere’s another example of extracting specific components of information from a webpage (results not shown, since headlines will vary from day to day). We’ll use get_text to retrieve the element’s value.\n\nURL = \"https://www.nytimes.com\"\nresponseNYT = requests.get(URL)\nsoupNYT = bs(responseNYT.content, 'html.parser')\nh2_elements = soupNYT.find_all(\"h2\")\nheadlines2 = [x.get_text() for x in h2_elements]\nh3_elements = soupNYT.find_all(\"h3\")\nheadlines3 = [x.get_text() for x in h3_elements]\n\nMore generally, we may want to read an HTML document, parse it into its components (i.e., the HTML elements), and navigate through the tree structure of the HTML.\nWe can use CSS selectors with the select method for more powerful extraction capabilities. Going back to the climate data, let’s extract all the th elements nested within tr elements:\n\nsoup.select(\"tr th\")\n\n[<th><a href=\"?C=N;O=D\">Name</a></th>,\n <th><a href=\"?C=M;O=A\">Last modified</a></th>,\n <th><a href=\"?C=S;O=A\">Size</a></th>,\n <th><a href=\"?C=D;O=A\">Description</a></th>,\n <th colspan=\"4\"><hr/></th>,\n <th colspan=\"4\"><hr/></th>]\n\n\nOr we could extract the a elements whose parents are th elements:\n\nsoup.select(\"th > a\")\n\n[<a href=\"?C=N;O=D\">Name</a>,\n <a href=\"?C=M;O=A\">Last modified</a>,\n <a href=\"?C=S;O=A\">Size</a>,\n <a href=\"?C=D;O=A\">Description</a>]\n\n\nNext let’s use the XPath language to specify elements rather than CSS selectors. XPath can also be used for navigating through XML documents.\n\nimport lxml.html\n\n# Convert the BeautifulSoup object to a lxml object\nlxml_doc = lxml.html.fromstring(str(soup))\n\n# Use XPath to select elements\na_elements = lxml_doc.xpath('//a[@href]')\nlinks = [x.get('href') for x in a_elements]\nlinks[0:9]\n\n['?C=N;O=D',\n '?C=M;O=A',\n '?C=S;O=A',\n '?C=D;O=A',\n '/pub/data/ghcn/daily/',\n '1750.csv.gz',\n '1763.csv.gz',\n '1764.csv.gz',\n '1765.csv.gz']" + "objectID": "units/unit7-bigData.html#an-editorial-on-big-data", + "href": "units/unit7-bigData.html#an-editorial-on-big-data", + "title": "Big data and databases", + "section": "An editorial on ‘big data’", + "text": "An editorial on ‘big data’\n‘Big data’ was trendy these days, though I guess it’s not quite the buzzword/buzzphrase that it was a few years ago, given the AI/ML revolution, but of course that revolution is largely based on having massive datasets available online.\nPersonally, I think some of the hype around giant datasets is justified and some is hype. Large datasets allow us to address questions that we can’t with smaller datasets, and they allow us to consider more sophisticated (e.g., nonlinear) relationships than we might with a small dataset. But they do not directly help with the problem of correlation not being causation. Having medical data on every American still doesn’t tell me if higher salt intake causes hypertension. Internet transaction data does not tell me if one website feature causes increased viewership or sales. One either needs to carry out a designed experiment or think carefully about how to infer causation from observational data. Nor does big data help with the problem that an ad hoc ‘sample’ is not a statistical sample and does not provide the ability to directly infer properties of a population. Consider the immense difficulties we’ve seen in answering questions about Covid despite large amounts of data, because it is incomplete/non-representative. A well-chosen smaller dataset may be much more informative than a much larger, more ad hoc dataset. However, having big datasets might allow you to select from the dataset in a way that helps get at causation or in a way that allows you to construct a population-representative sample. Finally, having a big dataset also allows you to do a large number of statistical analyses and tests, so multiple testing is a big issue. With enough analyses, something will look interesting just by chance in the noise of the data, even if there is no underlying reality to it.\nDifferent people define the ‘big’ in big data differently. One definition involves the actual size of the data, and in some cases the speed with which it is collected. Our efforts here will focus on dataset sizes that are large for traditional statistical work but would probably not be thought of as large in some contexts such as Google or the US National Security Agency (NSA). Another definition of ‘big data’ has more to do with how pervasive data and empirical analyses backed by data are in society and not necessarily how large the actual dataset size is." }, { - "objectID": "units/unit2-dataTech.html#xml", - "href": "units/unit2-dataTech.html#xml", - "title": "Data technologies, formats, and structures", - "section": "XML", - "text": "XML\nXML is a markup language used to store data in self-describing (no metadata needed) format, often with a hierarchical structure. It consists of sets of elements (also known as nodes because they generally occur in a hierarchical structure and therefore have parents, children, etc.) with tags that identify/name the elements, with some similarity to HTML. Some examples of the use of XML include serving as the underlying format for Microsoft Office and Google Docs documents and for the KML language used for spatial information in Google Earth.\nHere’s a brief example. The book with id attribute bk101 is an element; the author of the book is also an element that is a child element of the book. The id attribute allows us to uniquely identify the element.\n <?xml version=\"1.0\"?>\n <catalog>\n <book id=\"bk101\">\n <author>Gambardella, Matthew</author>\n <title>XML Developer's Guide</title>\n <genre>Computer</genre>\n <price>44.95</price>\n <publish_date>2000-10-01</publish_date>\n <description>An in-depth look at creating applications with XML.</description>\n </book>\n <book id=\"bk102\">\n <author>Ralls, Kim</author>\n <title>Midnight Rain</title>\n <genre>Fantasy</genre>\n <price>5.95</price>\n <publish_date>2000-12-16</publish_date>\n <description>A former architect battles corporate zombies, an evil sorceress, and her own childhood to become queen of the world.</description>\n </book>\n </catalog>\nWe can read XML documents into Python using various packages, including lxml and then manipulate the resulting structured data object. Here’s an example of working with lending data from the Kiva lending non-profit. You can see the XML format in a browser at http://api.kivaws.org/v1/loans/newest.xml.\nXML documents have a tree structure with information at nodes. As above with HTML, one can use the XPath language for navigating the tree and finding and extracting information from the node(s) of interest.\nHere is some example code for extracting loan info from the Kiva data. We’ll first show the ‘brute force’ approach of working with the data as a list and then the better approach of using XPath.\n\nimport xmltodict\n\nURL = \"https://api.kivaws.org/v1/loans/newest.xml\"\nresponse = requests.get(URL)\ndata = xmltodict.parse(response.content)\ndata.keys()\ndata['response'].keys()\ndata['response']['loans'].keys()\nlen(data['response']['loans']['loan'])\ndata['response']['loans']['loan'][2]\ndata['response']['loans']['loan'][2]['activity']\n\ndict_keys(['response'])\n\n\ndict_keys(['paging', 'loans'])\n\n\ndict_keys(['@type', 'loan'])\n\n\n20\n\n\n{'id': '2635728',\n 'name': 'Gulmira',\n 'description': {'languages': {'@type': 'list', 'language': ['ru', 'en']}},\n 'status': 'fundraising',\n 'funded_amount': '0',\n 'basket_amount': '0',\n 'image': {'id': '5248685', 'template_id': '1'},\n 'activity': 'Education provider',\n 'sector': 'Education',\n 'use': 'to purchase a printer.',\n 'location': {'country_code': 'KG',\n 'country': 'Kyrgyzstan',\n 'town': 'Kochkor district, Naryn region',\n 'geo': {'level': 'town', 'pairs': '42.099103 75.52767', 'type': 'point'}},\n 'partner_id': '171',\n 'posted_date': '2023-08-31T14:50:10Z',\n 'planned_expiration_date': '2023-10-05T14:50:10Z',\n 'loan_amount': '375',\n 'borrower_count': '1',\n 'lender_count': '0',\n 'bonus_credit_eligibility': '1',\n 'tags': None}\n\n\n'Education provider'\n\n\n\nfrom lxml import etree\ndoc = etree.fromstring(response.content)\n\nloans = doc.xpath(\"//loan\")\n[loan.xpath(\"activity/text()\") for loan in loans]\n\n## suppose we only want the country locations of the loans (using XPath)\n[loan.xpath(\"location/country/text()\") for loan in loans]\n## or extract the geographic coordinates\n[loan.xpath(\"location/geo/pairs/text()\") for loan in loans]\n\n[['Personal Housing Expenses'],\n ['Education provider'],\n ['Education provider'],\n ['Agriculture'],\n ['Education provider'],\n ['Farming'],\n ['Farming'],\n ['Farming'],\n ['Farming'],\n ['Farming'],\n ['Farming'],\n ['Agriculture'],\n ['Agriculture'],\n ['Clothing'],\n ['Crafts'],\n ['Food Production/Sales'],\n ['Agriculture'],\n ['Agriculture'],\n ['Agriculture'],\n ['Food']]\n\n\n[['Nicaragua'],\n ['Kyrgyzstan'],\n ['Kyrgyzstan'],\n ['Kyrgyzstan'],\n ['Kyrgyzstan'],\n ['Kenya'],\n ['Kenya'],\n ['Kenya'],\n ['Kenya'],\n ['Kenya'],\n ['Kenya'],\n ['Samoa'],\n ['Samoa'],\n ['Samoa'],\n ['Samoa'],\n ['Samoa'],\n ['Samoa'],\n ['Samoa'],\n ['Samoa'],\n ['Samoa']]\n\n\n[['11.934407 -85.956001'],\n ['42.099103 75.52767'],\n ['42.099103 75.52767'],\n ['41.824168 71.097317'],\n ['42.874621 74.569762'],\n ['-0.333333 37.65'],\n ['-0.333333 37.65'],\n ['43.010557 -74.379162'],\n ['-0.538838 37.459641'],\n ['0.566667 34.566667'],\n ['-0.583333 35.183333'],\n ['-13.816667 -171.916667'],\n ['-13.583333 -172.333333'],\n ['-13.816667 -171.783333'],\n ['-14.045728 -171.414159'],\n ['-13.833333 -171.816667'],\n ['-14.045728 -171.414159'],\n ['-13.883333 -171.55'],\n ['-13.866667 -171.566667'],\n ['-13.816667 -171.783333']]" + "objectID": "units/unit7-bigData.html#logistics-and-data-size", + "href": "units/unit7-bigData.html#logistics-and-data-size", + "title": "Big data and databases", + "section": "Logistics and data size", + "text": "Logistics and data size\nOne of the main drawbacks with Python (and R) in working with big data is that all objects are stored in memory, so you can’t directly work with datasets that are more than 1-20 Gb or so, depending on the memory on your machine.\nThe techniques and tools discussed in this Unit (apart from the section on MapReduce/Spark) are designed for datasets in the range of gigabytes to tens of gigabytes, though they may scale to larger if you have a machine with a lot of memory or simply have enough disk space and are willing to wait. If you have 10s of gigabytes of data, you’ll be better off if your machine has 10s of GBs of memory, as discussed in this Unit.\nIf you’re scaling to 100s of GBs, terabytes or petabytes, tools such as carefully-administered databases, cloud-based tools such as provided by AWS and Google Cloud Platform, and Spark or other such tools are probably your best bet.\nNote: in handling big data files, it’s best to have the data on the local disk of the machine you are using to reduce traffic and delays from moving data over the network." }, { - "objectID": "units/unit2-dataTech.html#json", - "href": "units/unit2-dataTech.html#json", - "title": "Data technologies, formats, and structures", - "section": "JSON", - "text": "JSON\nJSON files are structured as “attribute-value” pairs (aka “key-value” pairs), often with a hierarchical structure. Here’s a brief example:\n {\n \"firstName\": \"John\",\n \"lastName\": \"Smith\",\n \"isAlive\": true,\n \"age\": 25,\n \"address\": {\n \"streetAddress\": \"21 2nd Street\",\n \"city\": \"New York\",\n \"state\": \"NY\",\n \"postalCode\": \"10021-3100\"\n },\n \"phoneNumbers\": [\n {\n \"type\": \"home\",\n \"number\": \"212 555-1234\"\n },\n {\n \"type\": \"office\",\n \"number\": \"646 555-4567\"\n }\n ],\n \"children\": [],\n \"spouse\": null\n }\nA set of key-value pairs is a named array and is placed inside braces (squiggly brackets). Note the nestedness of arrays within arrays (e.g., address within the overarching person array and the use of square brackets for unnamed arrays (i.e., vectors of information), as well as the use of different types: character strings, numbers, null, and (not shown) boolean/logical values. JSON and XML can be used in similar ways, but JSON is less verbose than XML.\nWe can read JSON into Python using the json package. Let’s play again with the Kiva data. The same data that we had worked with in XML format is also available in JSON format: https://api.kivaws.org/v1/loans/newest.json.\n\nURL = \"https://api.kivaws.org/v1/loans/newest.json\"\nresponse = requests.get(URL)\n\nimport json\ndata = json.loads(response.text)\ntype(data)\ndata.keys()\n\ntype(data['loans'])\ndata['loans'][0].keys()\n\ndata['loans'][0]['location']['country']\n[loan['location']['country'] for loan in data['loans']]\n\ndict\n\n\ndict_keys(['paging', 'loans'])\n\n\nlist\n\n\ndict_keys(['id', 'name', 'description', 'status', 'funded_amount', 'basket_amount', 'image', 'activity', 'sector', 'use', 'location', 'partner_id', 'posted_date', 'planned_expiration_date', 'loan_amount', 'borrower_count', 'lender_count', 'bonus_credit_eligibility', 'tags'])\n\n\n'Nicaragua'\n\n\n['Nicaragua',\n 'Kyrgyzstan',\n 'Kyrgyzstan',\n 'Kyrgyzstan',\n 'Kyrgyzstan',\n 'Kenya',\n 'Kenya',\n 'Kenya',\n 'Kenya',\n 'Kenya',\n 'Kenya',\n 'Samoa',\n 'Samoa',\n 'Samoa',\n 'Samoa',\n 'Samoa',\n 'Samoa',\n 'Samoa',\n 'Samoa',\n 'Samoa']\n\n\nOne disadvantage of JSON is that it is not set up to deal with missing values, infinity, etc." + "objectID": "units/unit7-bigData.html#what-we-already-know-about-handling-big-data", + "href": "units/unit7-bigData.html#what-we-already-know-about-handling-big-data", + "title": "Big data and databases", + "section": "What we already know about handling big data!", + "text": "What we already know about handling big data!\nUNIX operations are generally very fast, so if you can manipulate your data via UNIX commands and piping, that will allow you to do a lot. We’ve already seen UNIX commands for extracting columns. And various commands such as grep, head, tail, etc. allow you to pick out rows based on certain criteria. As some of you have done in problem sets, one can use awk to extract rows. So basic shell scripting may allow you to reduce your data to a more manageable size.\nThe tool GNU parallel allows you to parallelize operations from the command line and is commonly used in working on Linux clusters.\nAnd don’t forget simple things. If you have a dataset with 30 columns that takes up 10 Gb but you only need 5 of the columns, get rid of the rest and work with the smaller dataset. Or you might be able to get the same information from a random sample of your large dataset as you would from doing the analysis on the full dataset. Strategies like this will often allow you to stick with the tools you already know.\nAlso, remember that we can often store data more compactly in binary formats than in flat text (e.g., csv) files.\nFinally, for many applications, storing large datasets in a standard database will work well." }, { - "objectID": "units/unit2-dataTech.html#webscraping-and-web-apis", - "href": "units/unit2-dataTech.html#webscraping-and-web-apis", - "title": "Data technologies, formats, and structures", - "section": "Webscraping and web APIs", - "text": "Webscraping and web APIs\nHere we’ll see some examples of making requests over the Web to get data. We’ll use APIs to systematically query a website for information. Ideally, but not always, the API will be documented. In many cases that simply amounts to making an HTTP GET request, which is done by constructing a URL.\nThe requests package is useful for a wide variety of such functionality. Note that much of the functionality I describe below is also possible within the shell using either wget or curl.\n\nWebscraping ethics and best practices\nWebscraping is the process of extracting data from the web, either directly from a website or using a web API (application programming interface).\n\nShould you webscrape? In general, if we can avoid webscraping (particularly if there is not an API) and instead directly download a data file from a website, that is greatly preferred.\nMay you webscrape? Before you set up any automated downloading of materials/data from the web you should make sure that what you are about to do is consistent with the rules provided by the website.\n\nSome places to look for information on what the website allows are:\n\nlegal pages such as Terms of Service or Terms and Conditions on the website.\ncheck the robots.txt file (e.g., https://scholar.google.com/robots.txt) to see what a web crawler is allowed to do, and whether the site requires a particular delay between requests to the sites\npotentially contact the site owner if you plan to scrape a large amount of data\n\nHere are some links with useful information:\n\nBlog post on webscraping ethics\nSome information on how to understand a robots.txt file\n\nTips for when you make automated requests:\n\nWhen debugging code that processes the result of such a request, just run the request once, save (i.e., cache) the result, and then work on the processing code applied to the result. Don’t make the same request over and over again.\nIn many cases you will want to include a time delay between your automated requests to a site, including if you are not actually crawling a site but just want to automate a small number of queries.\n\n\n\nWhat is HTTP?\nHTTP (hypertext transfer protocol) is a system for communicating information from a server (i.e., the website of interest) to a client (e.g., your laptop). The client sends a request and the server sends a response.\nWhen you go to a website in a browser, your browser makes an HTTP GET request to the website. Similarly, when we did some downloading of html from webpages above, we used an HTTP GET request.\nAnytime the URL you enter includes parameter information after a question mark (www.somewebsite.com?param1=arg1¶m2=arg2), you are using an API.\nThe response to an HTTP request will include a status code, which can be interpreted based on this information.\nThe response will generally contain content in the form of text (e.g., HTML, XML, JSON) or raw bytes.\n\n\nAPIs: REST- and SOAP-based web services\nIdeally a web service documents their API (Applications Programming Interface) that serves data or allows other interactions. REST and SOAP are popular API standards/styles. Both REST and SOAP use HTTP requests; we’ll focus on REST as it is more common and simpler. When using REST, we access resources, which might be a Facebook account or a database of stock quotes. The API will (hopefully) document what information it expects from the user and will return the result in a standard format (often a particular file format rather than producing a webpage).\nOften the format of the request is a URL (aka an endpoint) plus a query string, passed as a GET request. Let’s search for plumbers near Berkeley, and we’ll see the GET request, in the form:\nhttps://www.yelp.com/search?find_desc=plumbers&find_loc=Berkeley+CA&ns=1\n\nthe query string begins with ?\nthere are one or more Parameter=Argument pairs\npairs are separated by &\n+ is used in place of each space\n\nLet’s see an example of accessing economic data from the World Bank, using the documentation for their API. Following the API call structure, we can download (for example), data on various countries. The documentation indicates that our REST-based query can use either a URL structure or an argument-based structure.\n\n## Queries based on the documentation\napi_url = \"http://api.worldbank.org/V2/incomeLevel/LIC/country\"\napi_args = \"http://api.worldbank.org/V2/country?incomeLevel=LIC\"\n\n## Generalizing a bit\nurl = \"http://api.worldbank.org/V2/country?incomeLevel=MIC&format=json\"\nresponse = requests.get(url)\n\ndata = json.loads(response.content)\n\n## Be careful of data truncation/pagination\nif False:\n url = \"http://api.worldbank.org/V2/country?incomeLevel=MIC&format=json&per_page=1000\"\n response = requests.get(url)\n data = json.loads(response.content)\n\n## Programmatic control\nbaseURL = \"http://api.worldbank.org/V2/country\"\ngroup = 'MIC'\nformat = 'json'\nargs = {'incomeLevel': group, 'format': format, 'per_page': 1000}\nurl = baseURL + '?' + '&'.join(['='.join(\n [key, str(args[key])]) for key in args])\nresponse = requests.get(url)\ndata = json.loads(response.content)\n \ntype(data)\nlen(data[1])\ntype(data[1][5])\ndata[1][5]\n\nlist\n\n\n108\n\n\ndict\n\n\n{'id': 'BEN',\n 'iso2Code': 'BJ',\n 'name': 'Benin',\n 'region': {'id': 'SSF', 'iso2code': 'ZG', 'value': 'Sub-Saharan Africa '},\n 'adminregion': {'id': 'SSA',\n 'iso2code': 'ZF',\n 'value': 'Sub-Saharan Africa (excluding high income)'},\n 'incomeLevel': {'id': 'LMC',\n 'iso2code': 'XN',\n 'value': 'Lower middle income'},\n 'lendingType': {'id': 'IDX', 'iso2code': 'XI', 'value': 'IDA'},\n 'capitalCity': 'Porto-Novo',\n 'longitude': '2.6323',\n 'latitude': '6.4779'}\n\n\nAPIs can change and disappear. A few years ago, the example above involved the World Bank’s Climate Data API, which I can no longer find!\nAs another example, here we can see the US Treasury Department API, which allows us to construct queries for federal financial data.\nThe Nolan and Temple Lang book provides a number of examples of different ways of authenticating with web services that control access to the service.\nFinally, some web services allow us to pass information to the service in addition to just getting data or information. E.g., you can programmatically interact with your Facebook, Dropbox, and Google Drive accounts using REST based on HTTP POST, PUT, and DELETE requests. Authentication is of course important in these contexts and some times you would first authenticate with your login and password and receive a “token”. This token would then be used in subsequent interactions in the same session.\nI created your github.berkeley.edu accounts from Python by interacting with the GitHub API using requests.\n\n\nHTTP requests by deconstructing an (undocumented) API\nIn some cases an API may not be documented or we might be lazy and not use the documentation. Instead we might deconstruct the queries a browser makes and then mimic that behavior, in some cases having to parse HTML output to get at data. Note that if the webpage changes even a little bit, our carefully constructed query syntax may fail.\nLet’s look at some UN data (agricultural crop data). By going to\nhttp://data.un.org/Explorer.aspx?d=FAO, and clicking on “Crops”, we’ll see a bunch of agricultural products with “View data” links. Click on “apricots” as an example and you’ll see a “Download” button that allows you to download a CSV of the data. Let’s select a range of years and then try to download “by hand”. Sometimes we can right-click on the link that will download the data and directly see the URL that is being accessed and then one can deconstruct it so that you can create URLs programmatically to download the data you want.\nIn this case, we can’t see the full URL that is being used because there’s some Javascript involved. Therefore, rather than looking at the URL associated with a link we need to view the actual HTTP request sent by our browser to the server. We can do this using features of the browser (e.g., in Firefox see Web Developer -> Network and in Chrome View -> Developer -> Developer tools and choose the Network tab) (or right-click on the webpage and select Inspect and then Network). Based on this we can see that an HTTP GET request is being used with a URL such as:\nhttp://data.un.org/Handlers/DownloadHandler.ashx?DataFilter=itemCode:526;year:2012,2013,2014,2015,2016,2017&DataMartId=FAO&Format=csv&c=2,4,5,6,7&s=countryName:asc,elementCode:asc,year:desc.\nWe’e now able to easily download the data using that URL, which we can fairly easily construct using string processing in bash, Python, or R, such as this (here I just paste it together directly, but using more structured syntax such as I used for the World Bank example would be better):\n\nimport zipfile\n\n## example URL:\n## http://data.un.org/Handlers/DownloadHandler.ashx?DataFilter=itemCode:526;\n##year:2012,2013,2014,2015,2016,2017&DataMartId=FAO&Format=csv&c=2,4,5,6,7&\n##s=countryName:asc,elementCode:asc,year:desc\nitemCode = 526\nbaseURL = \"http://data.un.org/Handlers/DownloadHandler.ashx\"\nyrs = ','.join([str(yr) for yr in range(2012,2018)])\nfilter = f\"?DataFilter=itemCode:{itemCode};year:{yrs}\"\nargs1 = \"&DataMartId=FAO&Format=csv&c=2,3,4,5,6,7&\"\nargs2 = \"s=countryName:asc,elementCode:asc,year:desc\"\nurl = baseURL + filter + args1 + args2\n## If the website provided a CSV, this would be easier, but it zips the file.\nresponse = requests.get(url)\n\nwith io.BytesIO(response.content) as stream: # create a file-like object\n with zipfile.ZipFile(stream, 'r') as archive: # treat the object as a zip file\n with archive.open(archive.filelist[0].filename, 'r') as file: # get a pointer to the embedded file\n dat = pd.read_csv(file)\n\ndat.head()\n\n/usr/local/linux/mambaforge-3.11/lib/python3.11/site-packages/IPython/core/formatters.py:342: FutureWarning:\n\nIn future versions `DataFrame.to_latex` is expected to utilise the base implementation of `Styler.to_latex` for formatting and rendering. The arguments signature may therefore change. It is recommended instead to use `DataFrame.style.to_latex` which also contains additional functionality.\n\n\n\n\n\n\n\n\n\n\nCountry or Area\nElement Code\nElement\nYear\nUnit\nValue\nValue Footnotes\n\n\n\n\n0\nAfghanistan\n432\nGross Production Index Number (2014-2016 = 100)\n2017.0\nindex\n202.19\nFc\n\n\n1\nAfghanistan\n432\nGross Production Index Number (2014-2016 = 100)\n2016.0\nindex\n27.45\nFc\n\n\n2\nAfghanistan\n432\nGross Production Index Number (2014-2016 = 100)\n2015.0\nindex\n134.50\nFc\n\n\n3\nAfghanistan\n432\nGross Production Index Number (2014-2016 = 100)\n2014.0\nindex\n138.05\nFc\n\n\n4\nAfghanistan\n432\nGross Production Index Number (2014-2016 = 100)\n2013.0\nindex\n138.05\nFc\n\n\n\n\n\n\n\nSo, what have we achieved?\n\nWe have a reproducible workflow we can share with others (perhaps ourself in the future).\nWe can automate the process of downloading many such files.\n\n\n\nMore details on HTTP requests\nA more sophisticated way to do the download is to pass the request in a structured way with named input parameters. This request is easier to construct programmatically. Here what is returned is a zip file, which is represented in Python as a sequence of “raw” bytes.\n\ndata = {\"DataFilter\": f\"itemCode:{itemCode};year:{yrs}\",\n \"DataMartID\": \"FAO\", \n \"Format\": \"csv\", \n \"c\": \"2,3,4,5,6,7\",\n \"s\": \"countryName:asc,elementCode:asc,year:desc\"\n } \n\nresponse = requests.get(baseURL, params = data)\n\nwith io.BytesIO(response.content) as stream: \n with zipfile.ZipFile(stream, 'r') as archive:\n with archive.open(archive.filelist[0].filename, 'r') as file: \n dat = pd.read_csv(file)\n\nIn some cases we may need to send a lot of information as part of the URL in a GET request. If it gets to be too long (e.g,, more than 2048 characters) many web servers will reject the request. Instead we may need to use an HTTP POST request (POST requests are often used for submitting web forms). A typical request would have syntax like this search (using requests):\n\nurl = 'http://www.wormbase.org/db/searches/advanced/dumper'\n\ndata = { \"specipes\":\"briggsae\",\n \"list\": \"\",\n \"flank3\": \"0\",\n \"flank5\": \"0\",\n \"feature\": \"Gene Models\",\n \"dump\": \"Plain TEXT\",\n \"orientation\": \"Relative to feature\",\n \"relative\": \"Chromsome\",\n \"DNA\":\"flanking sequences only\",\n \".cgifields\" : \"feature, orientation, DNA, dump, relative\"\n} \n\nresponse = requests.post(url, data = data)\nif response.status_code == 200:\n print(\"POST request successful\")\nelse:\n print(f\"POST request failed with status code: {response.status_code}\")\n\nUnfortunately that specific search doesn’t work because the server URL and/or API seem to have changed. But it gives you an idea of what the format would look like.\nrequests can handle other kinds of HTTP requests such as PUT and DELETE. Finally, some websites use cookies to keep track of users, and you may need to download a cookie in the first interaction with the HTTP server and then send that cookie with later interactions. More details are available in the Nolan and Temple Lang book.\n\n\nPackaged access to an API\nFor popular websites/data sources, a developer may have packaged up the API calls in a user-friendly fashion for use from Python, R, or other software. For example there are Python (twitter) and R (twitteR) packages for interfacing with Twitter via its API.\nHere’s some example code for Python. This looks up the US senators’ Twitter names and then downloads a portion of each of their timelines, i.e., the time series of their tweets. Note that Twitter has limits on how much one can download at once.\n\nimport json\nimport twitter\n\n# You will need to set the following variables with your\n# personal information. To do this you will need to create\n# a personal account on Twitter (if you don't already have\n# one). Once you've created an account, create a new\n# application here:\n# https://dev.twitter.com/apps\n#\n# You can manage your applications here:\n# https://apps.twitter.com/\n#\n# Select your application and then under the section labeled\n# \"Key and Access Tokens\", you will find the information needed\n# below. Keep this information private.\nCONSUMER_KEY = \"\"\nCONSUMER_SECRET = \"\"\nOAUTH_TOKEN = \"\"\nOAUTH_TOKEN_SECRET = \"\"\n\nauth = twitter.oauth.OAuth(OAUTH_TOKEN, OAUTH_TOKEN_SECRET,\n CONSUMER_KEY, CONSUMER_SECRET)\napi = twitter.Twitter(auth=auth)\n\n# get the list of senators\nsenators = api.lists.members(owner_screen_name=\"gov\",\n slug=\"us-senate\", count=100)\n\n# get all the senators' timelines\nnames = [d[\"screen_name\"] for d in senators[\"users\"]]\ntimelines = [api.statuses.user_timeline(screen_name=name, count = 500) \n for name in names]\n\n# save information out to JSON\nwith open(\"senators-list.json\", \"w\") as f:\n json.dump(senators, f, indent=4, sort_keys=True)\nwith open(\"timelines.json\", \"w\") as f:\n json.dump(timelines, f, indent=4, sort_keys=True)\n\n\n\nAccessing dynamic pages\nSome websites dynamically change in reaction to the user behavior. In these cases you need a tool that can mimic the behavior of a human interacting with a site. Some options are:\n\nselenium is a popular tool for doing this, and there is a Python package of the same name.\nUsing scrapy plus splash is another approach." + "objectID": "units/unit7-bigData.html#overview", + "href": "units/unit7-bigData.html#overview", + "title": "Big data and databases", + "section": "Overview", + "text": "Overview\nA basic paradigm for working with big datasets is the MapReduce paradigm. The basic idea is to store the data in a distributed fashion across multiple nodes and try to do the computation in pieces on the data on each node. Results can also be stored in a distributed fashion.\nA key benefit of this is that if you can’t fit your dataset on disk on one machine you can on a cluster of machines. And your processing of the dataset can happen in parallel. This is the basic idea of MapReduce.\nThe basic steps of MapReduce are as follows:\n\nread individual data objects (e.g., records/lines from CSVs or individual data files)\nmap: create key-value pairs using the inputs (more formally, the map step takes a key-value pair and returns a new key-value pair)\nreduce: for each key, do an operation on the associated values and create a result - i.e., aggregate within the values assigned to each key\nwrite out the {key,result} pair\n\nA similar paradigm that is implemented in pandas and dplyr is the split-apply-combine strategy.\nA few additional comments. In our map function, we could exclude values or transform them in some way, including producing multiple records from a single record. And in our reduce function, we can do more complicated analysis. So one can actually do fairly sophisticated things within what may seem like a restrictive paradigm. But we are constrained such that in the map step, each record needs to be treated independently and in the reduce step each key needs to be treated independently. This allows for the parallelization.\nOne important note is that any operations that require moving a lot of data between the workers can take a long time. (This is sometimes called a shuffle.) This could happen if, for example, you computed the median value within each of many groups if the data for each group are spread across the workers. In contrast, if we compute the mean or sum, one can compute the partial sums on each worker and then just add up the partial sums.\nNote that as discussed in Unit 5 the concepts of map and reduce are core concepts in functional programming, and of course Python provides the map function.\nHadoop is an infrastructure for enabling MapReduce across a network of machines. The basic idea is to hide the complexity of distributing the calculations and collecting results. Hadoop includes a file system for distributed storage (HDFS), where each piece of information is stored redundantly (on multiple machines). Calculations can then be done in a parallel fashion, often on data in place on each machine thereby limiting the amount of communication that has to be done over the network. Hadoop also monitors completion of tasks and if a node fails, it will redo the relevant tasks on another node. Hadoop is based on Java. Given the popularity of Spark, I’m not sure how much usage these approaches currently see. Setting up a Hadoop cluster can be tricky. Hopefully if you’re in a position to need to use Hadoop, it will be set up for you and you will be interacting with it as a user/data analyst.\nOk, so what is Spark? You can think of Spark as in-memory Hadoop. Spark allows one to treat the memory across multiple nodes as a big pool of memory. Therefore, Spark should be faster than Hadoop when the data will fit in the collective memory of multiple nodes. In cases where it does not, Spark will make use of the HDFS (and generally, Spark will be reading the data initially from HDFS.) While Spark is more user-friendly than Hadoop, there are also some things that can make it hard to use. Setting up a Spark cluster also involves a bit of work, Spark can be hard to configure for optimal performance, and Spark calculations have a tendency to fail (often involving memory issues) in ways that are hard for users to debug." }, { - "objectID": "units/unit2-dataTech.html#standard-data-structures-in-python-and-r", - "href": "units/unit2-dataTech.html#standard-data-structures-in-python-and-r", - "title": "Data technologies, formats, and structures", - "section": "Standard data structures in Python and R", - "text": "Standard data structures in Python and R\n\nIn Python and R, one often ends up working with dataframes, lists, and arrays/vectors/matrices/tensors.\nIn Python we commonly work with data structures that are part of additional packages, in particular numpy arrays and pandas dataframes.\nDictionaries in Python allow for easy use of key-value pairs where one can access values based on their key/label. In R one can do something similar with named vectors or named lists or (more efficiently) by using environments.\nIn R, if we are not working with rectangular datasets or standard numerical objects, we often end up using lists or enhanced versions of lists, sometimes with deeply nested structures.\n\nIn Unit 7, we’ll talk about distributed data structures that allow one to easily work with data distributed across multiple computers." + "objectID": "units/unit7-bigData.html#using-dask-for-big-data-processing", + "href": "units/unit7-bigData.html#using-dask-for-big-data-processing", + "title": "Big data and databases", + "section": "Using Dask for big data processing", + "text": "Using Dask for big data processing\nUnit 6 on parallelization gives an overview of using Dask for flexible parallelization on different kinds of computational resources (in particular, parallelizing across multiple cores on one machine versus parallelizing across multiple cores across multiple machines/nodes).\nHere we’ll see the use of Dask to work with distributed datasets. Dask can process datasets (potentially very large ones) by parallelizing operations across subsets of the data using multiple cores on one or more machines.\nLike Spark, Dask automatically reads data from files in parallel and operates on chunks (also called partitions or shards) of the full dataset in parallel. There are two big advantages of this:\n\nYou can do calculations (including reading from disk) in parallel because each worker will work on a piece of the data.\nWhen the data is split across machines, you can use the memory of multiple machines to handle much larger datasets than would be possible in memory on one machine. That said, Dask processes the data in chunks, so one often doesn’t need a lot of memory, even just on one machine.\n\nWhile reading from disk in parallel is a good goal, if all the data are on one hard drive, there are limitations on the speed of reading the data from disk because of having multiple processes all trying to access the disk at once. Supercomputing systems will generally have parallel file systems that support truly parallel reading (and writing, i.e., parallel I/O). Hadoop/Spark deal with this by distributing across multiple disks, generally one disk per machine/node.\nBecause computations are done in external compiled code (e.g., via numpy) it’s effective to use the threads scheduler when operating on one node to avoid having to copy and move the data.\n\nDask dataframes (pandas)\nDask dataframes are Pandas-like dataframes where each dataframe is split into groups of rows, stored as smaller Pandas dataframes.\nOne can do a lot of the kinds of computations that you would do on a Pandas dataframe on a Dask dataframe, but many operations are not possible. See here.\nBy default dataframes are handled by the threads scheduler. (Recall we discussed Dask’s various schedulers in Unit 6.)\nHere’s an example of reading from a dataset of flight delays (about 11 GB data). You can get the data here.\n\nimport dask\ndask.config.set(scheduler='threads', num_workers = 4) \nimport dask.dataframe as ddf\npath = '/scratch/users/paciorek/243/AirlineData/csvs/'\nair = ddf.read_csv(path + '*.csv.bz2',\n compression = 'bz2',\n encoding = 'latin1', # (unexpected) latin1 value(s) in TailNum field in 2001\n dtype = {'Distance': 'float64', 'CRSElapsedTime': 'float64',\n 'TailNum': 'object', 'CancellationCode': 'object', 'DepDelay': 'float64'})\n# specify some dtypes so Pandas doesn't complain about column type heterogeneity\nair\n\nDask will reads the data in parallel from the various .csv.bz2 files (unzipping on the fly), but note the caveat in the previous section about the possibilities for truly parallel I/O.\nHowever, recall that Dask uses delayed evaluation. In this case, the reading is delayed until compute() is called. For that matter, the various other calculations (max, groupby, mean) shown below are only done after compute() is called.\n\nimport time\n\nt0 = time.time()\nair.DepDelay.max().compute() # this takes a while\nprint(time.time() - t0)\n\nt0 = time.time()\nair.DepDelay.mean().compute() # this takes a while\nprint(time.time() - t0)\n\nair.DepDelay.median().compute() \n\nWe’ll discuss in class why Dask won’t do the median. Consider the discussion about moving data in the earlier section on MapReduce.\nNext let’s see a full split-apply-combine (aka MapReduce) type of analysis.\n\nsub = air[(air.UniqueCarrier == 'UA') & (air.Origin == 'SFO')]\nbyDest = sub.groupby('Dest').DepDelay.mean()\nresults = byDest.compute() # this takes a while too\nresults\n\nYou should see this:\n Dest \n ACV 26.200000 \n BFL 1.000000 \n BOI 12.855069 \n BOS 9.316795 \n CLE 4.000000\n ...\nNote: calling compute twice is a bad idea as Dask will read in the data twice - more on this in a bit.\n\nWarning Think carefully about the size of the result from calling compute. The result will be returned as a standard Python object, not distributed across multiple workers (and possibly machines), and with the object entirely in memory. It’s easy to accidentally return an entire giant dataset.\n\n\n\nDask bags\nBags are like lists but there is no particular ordering, so it doesn’t make sense to ask for the i’th element.\nYou can think of operations on Dask bags as being like parallel map operations on lists in Python or R.\nBy default bags are handled via the processes scheduler.\nLet’s see some basic operations on a large dataset of Wikipedia log files. You can get a subset of the Wikipedia data here.\nHere we again read the data in (which Dask will do in parallel):\n\nimport dask.multiprocessing\ndask.config.set(scheduler='processes', num_workers = 4) \nimport dask.bag as db\n## This is the full data\n## path = '/scratch/users/paciorek/wikistats/dated_2017/'\n## For demo we'll just use a small subset\npath = '/scratch/users/paciorek/wikistats/dated_2017_small/dated/'\nwiki = db.read_text(path + 'part-0*gz')\n\nHere we’ll just count the number of records.\n\nimport time\nt0 = time.time()\nwiki.count().compute()\ntime.time() - t0 # 136 sec. for full data\n\nAnd here is a more realistic example of filtering (subsetting).\n\nimport re\ndef find(line, regex = 'Armenia'):\n vals = line.split(' ')\n if len(vals) < 6:\n return(False)\n tmp = re.search(regex, vals[3])\n if tmp is None:\n return(False)\n else:\n return(True)\n \n\nwiki.filter(find).count().compute()\narmenia = wiki.filter(find)\nsmp = armenia.take(100) ## grab a handful as proof of concept\nsmp[0:5]\n\nNote that it is quite inefficient to do the find() (and implicitly reading the data in) and then compute on top of that intermediate result in two separate calls to compute(). Rather, we should set up the code so that all the operations are set up before a single call to compute(). This is discussed in detail in the Dask/future tutorial.\nSince the data are just treated as raw strings, we might want to introduce structure by converting each line to a tuple and then converting to a data frame.\n\ndef make_tuple(line):\n return(tuple(line.split(' ')))\n\ndtypes = {'date': 'object', 'time': 'object', 'language': 'object',\n'webpage': 'object', 'hits': 'float64', 'size': 'float64'}\n\n## Let's create a Dask dataframe. \n## This will take a while if done on full data.\ndf = armenia.map(make_tuple).to_dataframe(dtypes)\ntype(df)\n\n## Now let's actually do the computation, returning a Pandas df\nresult = df.compute() \ntype(result)\nresult[0:5]\n\n\n\nDask arrays (numpy)\nDask arrays are numpy-like arrays where each array is split up by both rows and columns into smaller numpy arrays.\nOne can do a lot of the kinds of computations that you would do on a numpy array on a Dask array, but many operations are not possible. See here.\nBy default arrays are handled via the threads scheduler.\n\nNon-distributed arrays\nLet’s first see operations on a single node, using a single 13 GB two-dimensional array. Again, Dask uses lazy evaluation, so creation of the array doesn’t happen until an operation requiring output is done.\n\nimport dask\ndask.config.set(scheduler = 'threads', num_workers = 4) \nimport dask.array as da\nx = da.random.normal(0, 1, size=(40000,40000), chunks=(10000, 10000))\n# square 10k x 10k chunks\nmycalc = da.mean(x, axis = 1) # by row\nimport time\nt0 = time.time()\nrs = mycalc.compute()\ntime.time() - t0 # 41 sec.\n\nFor a row-based operation, we would presumably only want to chunk things up by row, but this doesn’t seem to actually make a difference, presumably because the mean calculation can be done in pieces and only a small number of summary statistics moved between workers.\n\nimport dask\ndask.config.set(scheduler='threads', num_workers = 4) \nimport dask.array as da\n# x = da.from_array(x, chunks=(2500, 40000)) # adjust chunk size of existing array\nx = da.random.normal(0, 1, size=(40000,40000), chunks=(2500, 40000))\nmycalc = da.mean(x, axis = 1) # row means\nimport time\nt0 = time.time()\nrs = mycalc.compute()\ntime.time() - t0 # 42 sec.\n\nOf course, given the lazy evaluation, this timing comparison is not just timing the actual row mean calculations.\nBut this doesn’t really clarify the story…\n\nimport dask\ndask.config.set(scheduler='threads', num_workers = 4) \nimport dask.array as da\nimport numpy as np\nimport time\nt0 = time.time()\nx = np.random.normal(0, 1, size=(40000,40000))\ntime.time() - t0 # 110 sec.\n# for some reason the from_array and da.mean calculations are not done lazily here\nt0 = time.time()\ndx = da.from_array(x, chunks=(2500, 40000))\ntime.time() - t0 # 27 sec.\nt0 = time.time()\nmycalc = da.mean(x, axis = 1) # what is this doing given .compute() also takes time?\ntime.time() - t0 # 28 sec.\nt0 = time.time()\nrs = mycalc.compute()\ntime.time() - t0 # 21 sec.\n\nDask will avoid storing all the chunks in memory. (It appears to just generate them on the fly.) Here we have an 80 GB array but we never use more than a few GB of memory (based on top or free -h).\n\nimport dask\ndask.config.set(scheduler='threads', num_workers = 4) \nimport dask.array as da\nx = da.random.normal(0, 1, size=(100000,100000), chunks=(10000, 10000))\nmycalc = da.mean(x, axis = 1) # row means\nimport time\nt0 = time.time()\nrs = mycalc.compute()\ntime.time() - t0 # 205 sec.\nrs[0:5]\n\n\n\nDistributed arrays\nUsing arrays distributed across multiple machines should be straightforward based on using Dask distributed. However, one would want to be careful about creating arrays by distributing the data from a single Python process as that would involve copying between machines." }, { - "objectID": "units/unit2-dataTech.html#other-kinds-of-data-structures", - "href": "units/unit2-dataTech.html#other-kinds-of-data-structures", - "title": "Data technologies, formats, and structures", - "section": "Other kinds of data structures", - "text": "Other kinds of data structures\nYou may have heard of various other kinds of data structures, such as linked lists, trees, graphs, queues, and stacks. One of the key aspects that differentiate such data structures is how one navigates through the elements.\nSets are collections of elements that don’t have any duplicates (like a mathematical set).\nWith a linked list, with each element (or node) has a value and a pointer (reference) to the location of the next element. (With a doubly-linked list, there is also a pointer back to the previous element.) One big advantage of this is that one can insert an element by simply modifying the pointers involved at the site of the insertion, without copying any of the other elements in the list. A big disadvantage is that to get to an element you have to navigate through the list.\n\n\n\nLinked list (courtesy of computersciencewiki.org)\n\n\nBoth trees and graphs are collections of nodes (vertices) and links (edges). A tree involves a set of nodes and links to child nodes (also possibly containing information linking the child nodes to their parent nodes). With a graph, the links might not be directional, and there can be cycles.\n\n\n\nTree (courtesy of computersciencewiki.org)\n\n\n\n\n\nGraph (courtesy of computersciencewiki.org)\n\n\nA stack is a collection of elements that behave like a stack of lunch trays. You can only access the top element directly(“last in, first out”), so the operations are that you can push a new element onto the stack or pop the top element off the stack. In fact, nested function calls behave as stacks, and the memory used in the process of evaluating the function calls is called the ‘stack’.\nA queue is like the line at a grocery store, behaving as “first in, first out”.\nOne can use such data structures either directly or via add-on packages in Python and R, though I don’t think they’re all that commonly used in R. This is probably because statistical/data science/machine learning workflows often involve either ‘rectangular’ data (i.e., dataframe-style data) and/or mathematical computations with arrays. That said, trees and graphs are widely used.\nSome related concepts that we’ll discuss further in Unit 5 include:\n\ntypes: this refers to how a given piece of information is stored and what operations can be done with the information.\n\n‘primitive’ types are the most basic types that often relate directly to how data are stored in memory or on disk (e.g., booleans, integers, numeric (real-valued), character, pointer (address, reference).\n\npointers: references to other locations (addresses) in memory. One often uses pointers to avoid unnecessary copying of data.\nhashes: hashing involves fast lookup of the value associated with a key (a label), using a hash function, which allows one to convert the key to an address. This avoids having to find the value associated with a specific key by looking through all the keys until the key of interest is found (an O(n) operation)." + "objectID": "units/unit7-bigData.html#overview-1", + "href": "units/unit7-bigData.html#overview-1", + "title": "Big data and databases", + "section": "Overview", + "text": "Overview\nBasically, standard SQL databases are relational databases that are a collection of rectangular format datasets (tables, also called relations), with each table similar to R or Pandas data frames, in that a table is made up of columns, which are called fields or attributes, each containing a single type (numeric, character, date, currency, enumerated (i.e., categorical), …) and rows or records containing the observations for one entity. Some of the tables in a given database will generally have fields in common so it makes sense to merge (i.e., join) information from multiple tables. E.g., you might have a database with a table of student information, a table of teacher information and a table of school information, and you might join student information with information about the teacher(s) who taught the students. Databases are set up to allow for fast querying and merging (called joins in database terminology).\n\nMemory and disk use\nFormally, databases are stored on disk, while Python and R store datasets in memory. This would suggest that databases will be slow to access their data but will be able to store more data than can be loaded into an Python or R session. However, databases can be quite fast due in part to disk caching by the operating system as well as careful implementation of good algorithms for database operations." }, { - "objectID": "units/unit4-goodPractices.html", - "href": "units/unit4-goodPractices.html", - "title": "Good practices: coding practices, debugging, and reproducible research", - "section": "", - "text": "PDF\nSources:\nThis unit covers good coding/software development practices, debugging (and practices for avoiding bugs), and doing reproducible research. As in later units of the course, the material is generally not specific to Python, but some details and the examples are in Python." + "objectID": "units/unit7-bigData.html#interacting-with-a-database", + "href": "units/unit7-bigData.html#interacting-with-a-database", + "title": "Big data and databases", + "section": "Interacting with a database", + "text": "Interacting with a database\nYou can interact with databases in a variety of database systems (DBMS=database management system). Some popular systems are SQLite, DuckDB, MySQL, PostgreSQL, Oracle and Microsoft Access. We’ll concentrate on accessing data in a database rather than management of databases. SQL is the Structured Query Language and is a special-purpose high-level language for managing databases and making queries. Variations on SQL are used in many different DBMS.\nQueries are the way that the user gets information (often simply subsets of tables or information merged across tables). The result of an SQL query is in general another table, though in some cases it might have only one row and/or one column.\nMany DBMS have a client-server model. Clients connect to the server, with some authentication, and make requests (i.e., queries).\nThere are often multiple ways to interact with a DBMS, including directly using command line tools provided by the DBMS or via Python or R, among others.\nWe’ll concentrate on SQLite (because it is simple to use on a single machine). SQLite is quite nice in terms of being self-contained - there is no server-client model, just a single file on your hard drive that stores the database and to which you can connect to using the SQLite shell, R, Python, etc. However, it does not have some useful functionality that other DBMS have. For example, you can’t use ALTER TABLE to modify column types or drop columns.\nA good alternative to SQLite that I encourage you to consider is DuckDB. DuckDB stores data column-wise, which can lead to big speedups when doing queries operating on large portions of tables (so-called “online analytical processing” (OLAP)). Another nice feature of DuckDB is that it can interact with data on disk without always having to read all the data into memory. In fact, ideally we’d use it for this class, but I haven’t had time to create a DuckDB version of the StackOverflow database." }, { - "objectID": "units/unit4-goodPractices.html#editors", - "href": "units/unit4-goodPractices.html#editors", - "title": "Good practices: coding practices, debugging, and reproducible research", - "section": "Editors", - "text": "Editors\nUse an editor that supports the language you are using (e.g., Atom, Emacs/Aquamacs, Sublime, vim, VSCode, TextMate, WinEdt, or the built-in editor in RStudio [you can use Python from within RStudio]). Some advantages of this can include:\n\nhelpful color coding of different types of syntax,\nautomatic indentation and spacing,\nparenthesis matching,\nline numbering (good for finding bugs), and\ncode can often be run (or compiled) and debugged from within the editor.\n\nSee the problem set submission how-to document for more information about editors that interact nicely with Quarto documents." + "objectID": "units/unit7-bigData.html#database-schema-and-normalization", + "href": "units/unit7-bigData.html#database-schema-and-normalization", + "title": "Big data and databases", + "section": "Database schema and normalization", + "text": "Database schema and normalization\nTo truly leverage the conceptual and computational power of a database you’ll want to have your data in a normalized form, which means spreading your data across multiple tables in such a way that you don’t repeat information unnecessarily.\nThe schema is the metadata about the tables in the database and the fields (and their types) in those tables.\nLet’s consider this using an educational example. Suppose we have a school with multiple teachers teaching multiple classes and multiple students taking multiple classes. If we put this all in one table organized per student, the data might have the following fields:\n\nstudent ID\nstudent grade level\nstudent name\nclass 1\nclass 2\n…\nclass n\ngrade in class 1\ngrade in class 2\n…\ngrade in class n\nteacher ID 1\nteacher ID 2\n…\nteacher ID n\nteacher name 1\nteacher name 2\n…\nteacher name n\nteacher department 1\nteacher department 2\n…\nteacher department n\nteacher age 1\nteacher age 2\n…\nteacher age n\n\nThere are a lot of problems with this:\n\nA lot of information is repeated across rows (e.g., teacher age for students who have the same teacher) - this is a waste of space - it is hard/error-prone to update values in the database (e.g., after a teacher’s birthday), because a given value needs to be updated in multiple places\nThere are potentially a lot of empty cells (e.g., for a student who takes fewer than ‘n’ classes). This will generally result in a waste of space.\nIt’s hard to see the information that is not organized uniquely by row – i.e., it’s much easier to understand the information at the student level than the teacher level\nWe have to know in advance how big ‘n’ is. Then if a single student takes more than ‘n’ classes, the whole database needs to be restructured.\n\nIt would get even worse if there was a field related to teachers for which a given teacher could have multiple values (e.g., teachers could be in multiple departments). This would lead to even more redundancy - each student-class-teacher combination would be crossed with all of the departments for the teacher (so-called multivalued dependency in database theory).\nAn alternative organization of the data would be to have each row represent the enrollment of a student in a class.\n\nstudent ID\nstudent name\nclass\ngrade in class\nstudent grade level\nteacher ID\nteacher department\nteacher age\n\nThis has some advantages relative to our original organization in terms of not having empty cells, but it doesn’t solve the other three issues above.\nInstead, a natural way to order this database is with the following four tables.\n\nStudent\n\nID\nname\ngrade_level\n\nTeacher\n\nID\nname\ndepartment\nage\n\nClass\n\nID\ntopic\nclass_size\nteacher_ID\n\nClassAssignment\n\nstudent_ID\nclass_ID\ngrade\n\n\nThe ClassAssignment table has one row per student-class pair. Having a table like this handles “ragged” data where the number of observations per unit (in this case classes per student) varies. Using such tables is a common pattern when considering how to normalize a database. It’s also a core part of the idea of “tidy data” and data in long format, seen in the tidyr package.\nThen we do queries to pull information from multiple tables. We do the joins based on keys, which are the fields in each table that allow us to match rows from different tables.\n(That said, if all anticipated uses of a database will end up recombining the same set of tables, we may want to have a denormalized schema in which those tables are actually combined in the database. It is possible to be too pure about normalization! We can also create a virtual table, called a view, as discussed later.)\n\nKeys\nA key is a field or collection of fields that give(s) a unique value for every row/observation. A table in a database should then have a primary key that is the main unique identifier used by the DBMS. Foreign keys are columns in one table that give the value of the primary key in another table. When information from multiple tables is joined together, the matching of a row from one table to a row in another table is generally done by equating the primary key in one table with a foreign key in a different table.\nIn our educational example, the primary keys would presumably be: Student.ID, Teacher.ID, Class.ID, and for ClassAssignment a primary key made of two fields: {ClassAssignment.studentID, ClassAssignment.class_ID}.\nSome examples of foreign keys would be:\n\nstudent_ID as the foreign key in ClassAssignment for joining with Student on Student.ID\nteacher_ID as the foreign key in Class for joining with Teacher based on Teacher.ID\nclass_ID as the foreign key in ClassAssignment for joining with Class based on Class.ID\n\n\n\nQueries that join data across multiple tables\nSuppose we want a result that has the grades of all students in 9th grade. For this we need information from the Student table (to determine grade level) and information from the ClassAssignment table (to determine the class grade). More specifically we need a query that:\n\njoins Student with ClassAssignment based on matching rows in Student with rows in ClassAssignment where Student.ID is the same as ClassAssignment.student_ID and\nfilters the rows based on Student.grade_level:\n\n\nSELECT Student.ID, grade FROM Student, ClassAssignment WHERE \n Student.ID = ClassAssignment.student_ID and Student.grade_level = 9;\n\nNote that the query is a join (specifically an inner join), which is like merge() (or dplyr::join) in R. We don’t specifically use the JOIN keyword, but one could do these queries explicitly using JOIN, as we’ll see later." }, { - "objectID": "units/unit4-goodPractices.html#coding-syntax", - "href": "units/unit4-goodPractices.html#coding-syntax", - "title": "Good practices: coding practices, debugging, and reproducible research", - "section": "Coding syntax", - "text": "Coding syntax\nThe PEP 8 style guide is your go-to reference for Python style. I’ve highlighted some details here as well as included some general suggestions of my own.\n\nHeader information: put metainfo on the code into the first few lines of the file as comments. Include who, when, what, how the code fits within a larger program (if appropriate), possibly the versions of Python and key packages that you used.\nWrite docstrings for public modules, classes, functions, and methods. For non-public items, a comment after the def line is sufficient to describe the purpose of the item.\nIndentation: Python is strict about indentation of course, which helps to enforce clear indentation more than in other languages. This helps you and others to read and understand the code and can help in detecting errors in your code because it can expose lack of symmetry.\n\nuse 4 spaces per indentation level (avoid tabs if possible).\n\nWhitespace: use it in a variety of places. Some places where it is good to have it are\n\naround operators (assignment and arithmetic);\nbetween function arguments;\nbetween list/tuple elements; and\nbetween matrix/array indices.\n\nUse blank lines to separate blocks of code with comments to say what the block does.\nUse whitespaces or parentheses for clarity even if not needed for order of operations. For example, a/y*x will work but is not easy to read and you can easily induce a bug if you forget the order of ops. Instead, use a/y * x.\nAvoid code lines longer than 79 characters and comment/docstring lines longer than 72 characters.\nComments: add lots of comments (but don’t belabor the obvious, such as x = x + 1 # increment x).\n\nRemember that in a few months, you may not follow your own code any better than a stranger.\nSome key things to document: (1) summarizing a block of code, (2) explaining a very complicated piece of code - recall our complicated regular expressions, and (3) explaining arbitrary constant values.\nComments should generally be complete sentences.\n\nYou can use parentheses to group operations such that they can be split up into lines and easily commented, e.g.,\n\nnewdf = (\n pd.read_csv('file.csv')\n .rename(columns = {'STATE': 'us_state'}) # adjust column names\n .dropna() # remove some rows\n )\n\nFor software development, break code into separate files (2000-3000 lines per file) with meaningful file names and related functions grouped within a file.\nBeing consistent about the naming style for objects and functions is hard, but try to be consistent. PEP8 suggests:\n\nClass names should be UpperCamelCase.\nFunction, method, and variable names should be snake_case, e.g., number_of_its or n_its.\nNon-public methods and variables should have a leading underscore.\n\nTry to have the names be informative without being overly long.\nDon’t overwrite names of objects/functions that already exist in Python. E.g., don’t use len. That said, the namespace system helps with the unavoidable cases where there are name conflicts.\nUse active names for functions (e.g., calc_loglik, calc_log_lik rather than loglik or loglik_calc). The idea is that a function in a programming language is like a verb in regular language (a function does something), so use a verb to name it.\nLearn from others’ code\n\nThis semester, someone will be reading your code - the GSI and and me when we look at your assignments. So to help us in understanding your code and develop good habits, put these ideas into practice in your assignments.\nWhile not Python examples, the files goodCode.R and badCode.R in the units directory of the class repository provide examples of code written such that it does and does not conform to the general ideas listed above (leaving aside the different syntax of Python and R)." + "objectID": "units/unit7-bigData.html#stack-overflow-metadata-example", + "href": "units/unit7-bigData.html#stack-overflow-metadata-example", + "title": "Big data and databases", + "section": "Stack Overflow metadata example", + "text": "Stack Overflow metadata example\nI’ve obtained data from Stack Overflow, the popular website for asking coding questions, and placed it into a normalized database. The SQLite version has metadata (i.e., it lacks the actual text of the questions and answers) on all of the questions and answers posted in 2021.\nWe’ll explore SQL functionality using this example database.\nNow let’s consider the Stack Overflow data. Each question may have multiple answers and each question may have multiple (topic) tags.\nIf we tried to put this into a single table, the fields could look like this if we have one row per question:\n\nquestion ID\nID of user submitting question\nquestion title\ntag 1\ntag 2\n…\ntag n\nanswer 1 ID\nID of user submitting answer 1\nage of user submitting answer 1\nname of user submitting answer 1\nanswer 2 ID\nID of user submitting answer 2\nage of user submitting answer 2\nname of user submitting answer 2\n…\n\nor like this if we have one row per question-answer pair:\n\nquestion ID\nID of user submitting question\nquestion title\ntag 1\ntag 2\n…\ntag n\nanswer ID\nID of user submitting answer\nage of user submitting answer\nname of user submitting answer\n\nAs we’ve discussed neither of those schema is particularly desirable.\nChallenge: How would you devise a schema to normalize the data. I.e., what set of tables do you think we should create?\nYou can view one reasonable schema. The lines between tables indicate the relationship of foreign keys in one table to primary keys in another table. The schema in the actual database of Stack Overflow data we’ll use in the examples here is similar to but not identical to that.\nYou can download a copy of the SQLite version of the Stack Overflow 2021 database." }, { - "objectID": "units/unit4-goodPractices.html#coding-style", - "href": "units/unit4-goodPractices.html#coding-style", - "title": "Good practices: coding practices, debugging, and reproducible research", - "section": "Coding style", - "text": "Coding style\nThis is particularly focused on software development, but some of the ideas are useful for data analysis as well.\n\nBreak down tasks into core units\nWrite reusable code for core functionality and keep a single copy of the code (using version control or at least with a reasonable backup strategy) so you only need to make changes to a piece of code in one place\nSmaller functions are easier to debug, easier to understand, and can be combined in a modular fashion (like the UNIX utilities)\nWrite functions that take data as an argument and not lines of code that operate on specific data objects. Why? Functions allow us to reuse blocks of code easily for later use and for recreating an analysis (reproducible research). It’s more transparent than sourcing a file of code because the inputs and outputs are specified formally, so you don’t have to read through the code to figure out what it does.\nFunctions should:\n\nbe modular (having a single task);\nhave meaningful name; and\nhave a doc string describing their purpose, inputs and outputs.\n\nWrite tests for each function (i.e., unit tests)\nDon’t hard code numbers - use variables (e.g., number of iterations, parameter values in simulations), even if you don’t expect to change the value, as this makes the code more readable. For example, the speed of light is a constant in a scientific sense, but best to make it a variable in code: speed_of_light = 3e8\nUse lists or tuples to keep disparate parts of related data together\nPractice defensive programming (see also the discussion below on assertions)\n\ncheck function inputs and warn users if the code will do something they might not expect or makes particular choices;\ncheck inputs to if:\n\nNote that in Python, an expression used as the condition of an if will be equivalent to True unless it is one of False, 0, None, or an empty list/tuple/string.\n\nprovide reasonable default arguments;\ndocument the range of valid inputs;\ncheck that the output produced is valid; and\nstop execution based on checks and give an informative error message.\n\nTry to avoid system-dependent code that only runs on a specific version of an OS or specific OS\nLearn from others’ code\nConsider rewriting your code once you know all the settings and conditions; often analyses and projects meander as we do our work and the initial plan for the code no longer makes sense and the code is no longer designed specifically for the job being done." + "objectID": "units/unit7-bigData.html#accessing-databases-in-python", + "href": "units/unit7-bigData.html#accessing-databases-in-python", + "title": "Big data and databases", + "section": "Accessing databases in Python", + "text": "Accessing databases in Python\nPython provides a variety of front-end packages for manipulating databases from a variety of DBMS (SQLite, DuckDB, MySQL, PostgreSQL, among others). Basically, you start with a bit of code that links to the actual database, and then you can easily query the database using SQL syntax regardless of the back-end. The Python function calls that wrap around the SQL syntax will also look the same regardless of the back-end (basically execute(\"SOME SQL STATEMENT\")).\nWith SQLite, Python processes make calls against the stand-alone SQLite database (.db) file, so there are no SQLite-specific processes. With a client-server DBMS like PostgreSQL, Python processes call out to separate Postgres processes; these are started from the overall Postgres background process\nYou can access and navigate an SQLite database from Python as follows.\n\nimport sqlite3 as sq\ndir_path = '../data' # Replace with the actual path\ndb_filename = 'stackoverflow-2021.db'\n## download from http://www.stat.berkeley.edu/share/paciorek/stackoverflow-2021.db\n\ncon = sq.connect(os.path.join(dir_path, db_filename))\ndb = con.cursor()\ndb.execute(\"select * from questions limit 5\") # simple query \n\n<sqlite3.Cursor object at 0x7ff8707acd40>\n\ndb.fetchall() # retrieve results\n\n[(65534165.0, '2021-01-01 22:15:54', 0.0, 112.0, 2.0, 0.0, None, \"Can't update a value in sqlite3\", 13189393.0), (65535296.0, '2021-01-02 01:33:13', 2.0, 1109.0, 0.0, 0.0, None, 'Install and run ROS on Google Colab', 14924336.0), (65535910.0, '2021-01-02 04:01:34', -1.0, 110.0, 1.0, 8.0, 0.0, 'Operators on date/time fields', 651174.0), (65535916.0, '2021-01-02 04:03:20', 1.0, 35.0, 1.0, 0.0, None, 'Plotting values normalised', 14695007.0), (65536749.0, '2021-01-02 07:03:04', 0.0, 108.0, 1.0, 5.0, None, 'Export C# to word with template', 14899717.0)]\n\n\nAlternatively, we could use DuckDB. However, I don’t have a DuckDB version of the StackOverflow database, so one can’t actually run this code.\n\nimport duckdb as dd\ndir_path = '../data' # Replace with the actual path\ndb_filename = 'stackoverflow-2021.duckdb' # This doesn't exist.\n\ncon = dd.connect(os.path.join(dir_path, db_filename))\ndb = con.cursor()\ndb.execute(\"select * from questions limit 5\") # simple query \ndb.fetchall() # retrieve results\n\nWe can (fairly) easily see the tables (this is easier from R):\n\ndef db_list_tables(db):\n db.execute(\"SELECT name FROM sqlite_master WHERE type='table';\")\n tables = db.fetchall()\n return [table[0] for table in tables]\n\ndb_list_tables(db)\n\n['questions', 'answers', 'questions_tags', 'users']\n\n\nTo see the fields in the table, if you’ve just queried the table, you can look at description:\n\n[item[0] for item in db.description]\n\n['name']\n\ndef get_fields():\n return [item[0] for item in db.description]\n\nHere’s how to make a basic SQL query. One can either make the query and get the results in one go or make the query and separately fetch the results. Here we’ve selected the first five rows (and all columns, based on the * wildcard) and brought them into Python as list of tuples.\n\nresults = db.execute(\"select * from questions limit 5\").fetchall() # simple query \ntype(results)\n\n<class 'list'>\n\ntype(results[0])\n\n<class 'tuple'>\n\nquery = db.execute(\"select * from questions\") # simple query \nresults2 = query.fetchmany(5)\nresults == results2\n\nTrue\n\n\nTo disconnect from the database:\n\ndb.close()\n\nIt’s convenient to get a Pandas dataframe back as the result. To that we can execute queries like this:\n\nimport pandas as pd\nresults = pd.read_sql(\"select * from questions limit 5\", con)" }, { - "objectID": "units/unit4-goodPractices.html#assertions-exceptions-and-testing", - "href": "units/unit4-goodPractices.html#assertions-exceptions-and-testing", - "title": "Good practices: coding practices, debugging, and reproducible research", - "section": "Assertions, exceptions and testing", - "text": "Assertions, exceptions and testing\nAssertions, exceptions and testing are critically important for writing robust code that is less likely to contain bugs.\n\nExceptions\nYou’ve probably already seen exceptions in action whenever you’ve done something in Python that causes an error to occur and an error message to be printed. Syntax errors are different from exceptions in that exceptions occur when the syntax is correct but the execution of the code results in some sort of error.\nExceptions can be a valuable tool for making your code handle different modes of failure (missing file, URL unreachable, permission denied, invalid inputs, etc.). You use them when you are writing code that is supposed to perform a task (e.g., a function that does something with an input file) to indicate that the task failed and the reason for that failure (e.g., the file was not there in the first place). In such case, you want your code to raise an exception and make the error message informative as possible, typically by handling the exception thrown by another function you call and augmenting the message. Another possibility is that your code detects a situation where you need to throw an exception (e.g., an invalid input to a function).\nThe other side of the coin happens when you want to write a piece of code that handles failure in a specific way, instead of simply giving up. For example, if you are writing a script that reads and downloads hundreds of URLs, you don’t want your program to stop when any of them fails to respond (or do you?). You might want to continue with the rest of the URLs, and write out the failed URLs in a secondary output file.\n\nUsing try-except to continue execution\nIf you want some code to continue running even when it encounters an error, you can use try-except. This would often be done in code where you were running some workflow rather than in functions that you write for general purpose use (e.g., code in a package you are writing).\nSuppose we have a loop and we want to run all the iterations even if the code for some iterations fail. We can embed the code that might fail in a try block and then in the except block, run code that will handle the situation when the error occurs.\n\nfor i in range(n):\n try:\n <some code that might fail>\n result[i] = <actual result>\n except:\n <what to do if the code fails>\n result[i] = None \n\n\n\nStrategies for invoking and handling errors\nHere we’ll address situations that might arise when you are developing code for general purpose use (e.g., writing functions in a package) and need that code to invoke an error under certain circumstances or deal gracefully with an error occurring in some code that you are calling.\nA basic situation is when you want to detect a situation where you need to invoke an error (i.e., throw an exception).\nWith raise you can invoke an exception. Suppose we need an input to be a positive number. We’ll use Python’s built-in ValueError, one of the various exception types that Python provides and that you could use. You can also create your own exceptions by subclassing one of Python’s existing exception classes. (We haven’t yet discussed classes and object-oriented programming, so don’t worry if you’re not sure about what that means.)\n\ndef myfun(val):\n if val <= 0:\n raise ValueError(\"`val` should be positive\")\n\nmyfun(-3)\n\nValueError: `val` should be positive\n\n\nNext let’s consider cases where your function runs some code that might return an error.\nWe’d often want to catch the error using try-except. In some cases we would want to notify the user and then continue (perhaps falling back to a different way to do things or returning None from our function) while in others we might want to provide a more informative error message than if we had just let the error occur, but still have the exception be raised.\nFirst let’s see the case of continuing execution.\n\nimport os\n\ndef myfun(filename):\n try:\n with open(filename, \"r\") as file:\n text = file.read()\n except Exception as err:\n print(f\"{err}\\nCheck that the file `{filename}` can be found \"\\\n f\"in the current path: `{os.getcwd()}`.\")\n return None\n\n return(text.lower())\n\n\nmyfun('missing_file.txt')\n\n[Errno 2] No such file or directory: 'missing_file.txt'\nCheck that the file `missing_file.txt` can be found in the current path: `/accounts/vis/paciorek/teaching/243fall23/stat243-fall-2023/units`.\n\n\nFinally let’s see how we can intercept an error but then “re-raise” the error rather than continuing execution.\n\nimport requests\n\ndef myfun(url):\n try:\n requests.get(url)\n except Exception as err:\n print(f\"There was a problem accessing {url}. \"\\\n f\"Perhaps it doesn't exist or the URL has a typo?\")\n raise\n\nmyfun(\"http://missingurl.com\")\n\nThere was a problem accessing http://missingurl.com. Perhaps it doesn't exist or the URL has a typo?\n\n\nConnectionError: HTTPConnectionPool(host='missingurl.com', port=80): Max retries exceeded with url: / (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f348db44590>: Failed to establish a new connection: [Errno -2] Name or service not known'))\n\n\n\n\n\nAssertions\nAssertions are a quick way to raise a specific type of Exception (AssertionError). Assertions are useful for performing quick checks in your code that the state of the program is as you expect. They’re primarily intended for use during the development process to provide “sanity checks” that specific conditions are true, and there are ways to disable them when you are running your code for production purposes (to improve performance). A common use is for verifying preconditions and postconditions (especially preconditions). One would generally only expect such conditions not to be true if there is a bug in the code. Here’s an example of using the assert statement in Python, with a clear assertion message telling the developer what the problem is.\n\nnumber = -42\nassert number > 0, f\"number greater than 0 expected, got: {number}\"\n## Produces this error:\n## Traceback (most recent call last):\n## File \"<stdin>\", line 1, in <module>\n## AssertionError: number greater than 0 expected, got: -42\n\nVarious operators/functions are commonly used in assertions, including\n\nassert x in y\nassert x not in y\nassert x is y\nassert x is not y\nassert isinstance(x, <some_type>)\nassert all(x)\nassert any(x)\n\n\n\nTesting\nTesting is informally what you do after you write some code and want to check that it actually works. But when you are developing important code (e.g. functions that are going to be used by others) you typically want to encode your tests in code. There are many reasons to do that, including making sure that if someone else changes your code later on without fully understanding what it was supposed to do, the test suite should immediately indicate that.\nSome people even advocate for writing a preliminary test suite before writing the code itself(!) as it can be a good way to organize work and track progress, as well as act as a secondary form of documentation for clarity. This can include tests that your code provides correct and useful errors when something goes wrong (so that means that a test might be to see if problematic input correctly produces an error). Unit tests are intended to test the behavior of small pieces (units) of code, generally individual functions. Unit tests naturally work well with the ideas above of writing small, modular functions. I recommend the pytest package, which is designed to make it easier to write sets of good tests.\nIn lab, we’ll go over assertions, exceptions, and testing in detail." + "objectID": "units/unit7-bigData.html#basic-sql-for-choosing-rows-and-columns-from-a-table", + "href": "units/unit7-bigData.html#basic-sql-for-choosing-rows-and-columns-from-a-table", + "title": "Big data and databases", + "section": "Basic SQL for choosing rows and columns from a table", + "text": "Basic SQL for choosing rows and columns from a table\nSQL is a declarative language that tells the database system what results you want. The system then parses the SQL syntax and determines how to implement the query.\n\nNote: An imperative language is one where you provide the sequence of commands you want to be run, in order. A declarative language is one where you declare what result you want and rely on the system that interprets the commands to determine how to actually do it. Most of the languages we’re generally familiar with are imperative. (That said, even in languages like Python, function calls in many ways simply say what we want rather than exactly how the computer should carry out the granular operations.)\n\nHere are some examples using the Stack Overflow database of getting questions that have been viewed a lot (the viewcount field is large).\n\n## Get the questions (* indicates all fields) for which the viewcount field is large.\ndb.execute('select * from questions where viewcount > 100000').fetchall()\n\n## Find the 10 largest viewcounts (and associated titles) in the questions table,\n## by sorting in descending order based on viewcount and returning the first 10.\n\n[(65547199.0, '2021-01-03 06:22:52', 124.0, 110832.0, 7.0, 2.0, 0.0, 'Using Bootstrap 5 with Vue 3', 11232893.0), (65549858.0, '2021-01-03 12:30:19', 52.0, 130479.0, 11.0, 0.0, 0.0, '\"ERESOLVE unable to resolve dependency tree\" when installing npm react-facebook-login', 12425004.0), (65630743.0, '2021-01-08 14:20:57', 77.0, 107140.0, 19.0, 4.0, 0.0, 'How to solve flutter web api cors error only with dart code?', 12373446.0), (65632698.0, '2021-01-08 16:22:59', 74.0, 101044.0, 9.0, 1.0, 0.0, 'How to open a link in a new Tab in NextJS?', 9578961.0), (65896334.0, '2021-01-26 05:33:33', 111.0, 141899.0, 12.0, 7.0, 0.0, 'Python Pip broken with sys.stderr.write(f\"ERROR: {exc}\")', 202576.0), (65908987.0, '2021-01-26 20:42:25', 238.0, 215399.0, 9.0, 1.0, 0.0, \"How can I open Visual Studio Code's 'settings.json' file?\", 793320.0), (65980952.0, '2021-01-31 15:36:21', 22.0, 141857.0, 10.0, 1.0, 0.0, 'Python: Could not install packages due to an OSError: [Errno 2] No such file or directory', 14489450.0), (66020820.0, '2021-02-03 03:27:19', 161.0, 174829.0, 3.0, 0.0, 0.0, 'npm: When to use `--force` and `--legacy-peer-deps`', 7824245.0), (66029781.0, '2021-02-03 14:41:21', 20.0, 107446.0, 6.0, 1.0, 0.0, 'Appcenter iOS install error \"this app cannot be installed because its integrity could not be verified\"', 462440.0), (66060487.0, '2021-02-05 09:11:48', 188.0, 241216.0, 19.0, 0.0, 0.0, 'ValueError: numpy.ndarray size changed, may indicate binary incompatibility. Expected 88 from C header, got 80 from PyObject', 15150567.0), (66082397.0, '2021-02-06 21:58:06', 333.0, 315861.0, 17.0, 1.0, 0.0, 'TypeError: this.getOptions is not a function', 14337399.0), (66146088.0, '2021-02-10 22:32:45', 152.0, 175983.0, 17.0, 2.0, 0.0, 'Docker - failed to compute cache key: not found - runs fine in Visual Studio', 7419676.0), (66231282.0, '2021-02-16 19:54:53', 70.0, 116757.0, 10.0, 0.0, 0.0, 'How to add a GitHub personal access token to Visual Studio Code', 1186050.0), (66239691.0, '2021-02-17 10:03:55', 323.0, 261165.0, 6.0, 5.0, 0.0, \"What does npm install --legacy-peer-deps do exactly? When is it recommended / What's a potential use case?\", 15093141.0), (66252333.0, '2021-02-18 01:32:39', 21.0, 127736.0, 7.0, 0.0, 0.0, 'ERROR NullInjectorError: R3InjectorError(AppModule)', 14629851.0), (66366582.0, '2021-02-25 10:15:20', 90.0, 123630.0, 18.0, 5.0, 0.0, 'Github - unexpected disconnect while reading sideband packet', 11839478.0), (66597544.0, '2021-03-12 09:44:52', 29.0, 100293.0, 25.0, 4.0, 0.0, \"ENOENT: no such file or directory, lstat '/Users/Desktop/node_modules'\", 15124187.0), (66629862.0, '2021-03-14 21:43:50', 285.0, 117306.0, 9.0, 9.0, 0.0, \"Cannot determine the organization name for this 'dev.azure.com' remote url\", 11124332.0), (66662820.0, '2021-03-16 20:19:44', 104.0, 129848.0, 11.0, 0.0, 0.0, \"M1 docker preview and keycloak 'image's platform (linux/amd64) does not match the detected host platform (linux/arm64/v8)' Issue\", 4100000.0), (66666134.0, '2021-03-17 02:24:45', 159.0, 273201.0, 11.0, 2.0, 0.0, 'How to install homebrew on M1 mac', 15411878.0), (66801256.0, '2021-03-25 14:10:21', 143.0, 210144.0, 3.0, 8.0, 0.0, 'java.lang.IllegalAccessError: class lombok.javac.apt.LombokProcessor cannot access class com.sun.tools.javac.processing.JavacProcessingEnvironment', 12586904.0), (66835173.0, '2021-03-27 19:11:42', 158.0, 286886.0, 16.0, 0.0, 0.0, 'How to change background color of Elevated Button in Flutter from function?', 13628530.0), (66894200.0, '2021-03-31 19:33:26', 161.0, 263037.0, 13.0, 4.0, 0.0, 'Error message \"go: go.mod file not found in current directory or any parent directory; see \\'go help modules\\'\"', 4159198.0), (66964492.0, '2021-04-06 07:34:31', 40.0, 146006.0, 15.0, 2.0, 0.0, \"ImportError: cannot import name 'get_config' from 'tensorflow.python.eager.context'\", 5111234.0), (66980512.0, '2021-04-07 06:17:01', 739.0, 445775.0, 41.0, 0.0, 0.0, 'Android Studio Error \"Android Gradle plugin requires Java 11 to run. You are currently using Java 1.8\"', 11899911.0), (66989383.0, '2021-04-07 15:35:14', 54.0, 104476.0, 5.0, 0.0, 0.0, 'Could not resolve dependency: npm ERR! peer @angular/compiler@\"11.2.8\"', 12380096.0), (66992420.0, '2021-04-07 18:51:41', 86.0, 103193.0, 12.0, 1.0, 0.0, 'when I try to \"sync project with gradle files\" a warning pops up', 15576934.0), (67001968.0, '2021-04-08 10:19:41', 153.0, 141177.0, 17.0, 8.0, 0.0, 'How to disable maven blocking external HTTP repositories?', 5428154.0), (67045607.0, '2021-04-11 13:34:53', 204.0, 125367.0, 18.0, 1.0, 0.0, 'How to resolve \"Missing PendingIntent mutability flag\" lint warning in android api 30+?', 2652368.0), (67079327.0, '2021-04-13 17:02:54', 151.0, 278432.0, 13.0, 7.0, 0.0, 'How can I fix \"unsupported class file major version 60\" in IntelliJ IDEA?', 32914.0), (67191286.0, '2021-04-21 07:37:43', 142.0, 610026.0, 44.0, 8.0, 0.0, 'Crbug/1173575, non-JS module files deprecated. chromewebdata/(index)꞉5305:9:5551', 8732988.0), (67201708.0, '2021-04-21 18:39:28', 86.0, 105762.0, 2.0, 0.0, 0.0, 'Go update all modules', 1002260.0), (67246010.0, '2021-04-24 18:12:35', 41.0, 119121.0, 8.0, 0.0, 0.0, 'Error message \"The server selected protocol version TLS10 is not accepted by client preferences\"', 2153306.0), (67346232.0, '2021-05-01 12:23:44', 207.0, 276806.0, 60.0, 3.0, 0.0, 'Android Emulator issues in new versions - The emulator process has terminated', 13546747.0), (67352418.0, '2021-05-02 02:11:58', 74.0, 144925.0, 8.0, 1.0, 0.0, 'How to add SCSS styles to a React project?', 10836598.0), (67399785.0, '2021-05-05 10:45:12', 47.0, 151502.0, 12.0, 2.0, 0.0, 'How to solve npm install error “npm ERR! code 1”', 15841778.0), (67412084.0, '2021-05-06 05:00:27', 251.0, 286810.0, 14.0, 11.0, 0.0, 'Android Studio error: \"Manifest merger failed: Apps targeting Android 12\"', 15150212.0), (67440510.0, '2021-05-07 19:12:35', 10.0, 165243.0, 10.0, 2.0, 0.0, \"cv2.error: OpenCV(4.5.2) .error: (-215:Assertion failed) !_src.empty() in function 'cv::cvtColor'\", 15866017.0), (67448034.0, '2021-05-08 13:21:36', 180.0, 214970.0, 34.0, 4.0, 0.0, '\"Module was compiled with an incompatible version of Kotlin. The binary version of its metadata is 1.5.1, expected version is 1.1.16\"', 14099703.0), (67501093.0, '2021-05-12 09:42:06', 47.0, 149690.0, 5.0, 4.0, 0.0, 'Passthrough is not supported, GL is disabled', 15519827.0), (67505347.0, '2021-05-12 14:11:11', 165.0, 105852.0, 12.0, 5.0, 0.0, 'Non-nullable property must contain a non-null value when exiting constructor. Consider declaring the property as nullable', 1977871.0), (67507452.0, '2021-05-12 16:23:28', 89.0, 135460.0, 16.0, 0.0, 0.0, 'No spring.config.import property has been defined', 15816596.0), (67698176.0, '2021-05-26 03:33:11', 134.0, 119615.0, 14.0, 0.0, 0.0, 'Error loading webview: Error: Could not register service workers: TypeError: Failed to register a ServiceWorker for scope', 7148467.0), (67699823.0, '2021-05-26 06:45:07', 204.0, 178718.0, 21.0, 3.0, 0.0, 'Module was compiled with an incompatible version of Kotlin. The binary version of its metadata is 1.5.1, expected version is 1.1.15', 11576007.0), (67782975.0, '2021-06-01 05:03:24', 69.0, 114886.0, 15.0, 0.0, 0.0, 'How to fix the \\'\\'module java.base does not \"opens java.io\" to unnamed module \\'\\' error in Android Studio?', 14620854.0), (67899129.0, '2021-06-09 07:03:22', 35.0, 111003.0, 6.0, 0.0, 0.0, 'Postfix and OpenJDK 11: \"No appropriate protocol (protocol is disabled or cipher suites are inappropriate)\"', 1465758.0), (67900692.0, '2021-06-09 08:49:35', 259.0, 121306.0, 25.0, 3.0, 0.0, 'Latest version of Xcode stuck on installation (12.5)', 8612435.0), (68166721.0, '2021-06-28 16:12:46', 34.0, 116051.0, 8.0, 1.0, 0.0, 'CUDA error: device-side assert triggered on Colab', 5080195.0), (68191392.0, '2021-06-30 08:45:37', 242.0, 121764.0, 25.0, 22.0, 0.0, 'Password authentication is temporarily disabled as part of a brownout. Please use a personal access token instead', 15507251.0), (68236007.0, '2021-07-03 11:50:25', 318.0, 303399.0, 20.0, 1.0, 0.0, 'I am getting error \"cmdline-tools component is missing\" after installing Flutter and Android Studio... I added the Android SDK. How can I solve them?', 11993020.0), (68260784.0, '2021-07-05 18:40:58', 117.0, 219591.0, 8.0, 0.0, 0.0, 'npm WARN old lockfile The package-lock.json file was created with an old version of npm', 12530530.0), (68387270.0, '2021-07-15 02:45:41', 464.0, 426798.0, 34.0, 8.0, 0.0, 'Android Studio error \"Installed Build Tools revision 31.0.0 is corrupted\"', 11957368.0), (68397062.0, '2021-07-15 15:55:59', 50.0, 108364.0, 8.0, 1.0, 0.0, 'Could not initialize class org.apache.maven.plugin.war.util.WebappStructureSerializer\\t-Maven Configuration Problem Any solution?', 16456713.0), (68486207.0, '2021-07-22 14:02:27', 54.0, 130114.0, 8.0, 0.0, 0.0, 'Import could not be resolved/could not be resolved from source Pylance in VS Code using Python 3.9.2 on Windows 10', 14132348.0), (68554294.0, '2021-07-28 04:09:31', 188.0, 154478.0, 32.0, 24.0, 0.0, 'android:exported needs to be explicitly specified for <activity>. Apps targeting Android 12 and higher are required to specify', 14280831.0), (68673221.0, '2021-08-05 20:37:54', 62.0, 103200.0, 4.0, 4.0, 0.0, \"WARNING: Running pip as the 'root' user\", 15037284.0), (68775869.0, '2021-08-13 16:49:34', 1286.0, 1236876.0, 47.0, 18.0, 0.0, 'Message \"Support for password authentication was removed. Please use a personal access token instead.\"', 15573670.0), (68836551.0, '2021-08-18 17:01:31', 53.0, 117984.0, 9.0, 1.0, 0.0, \"Keras AttributeError: 'Sequential' object has no attribute 'predict_classes'\", 10377186.0), (68857411.0, '2021-08-20 05:33:34', 41.0, 122460.0, 4.0, 3.0, 0.0, 'npm WARN deprecated tar@2.2.2: This version of tar is no longer supported, and will not receive security updates. Please upgrade asap', 14930713.0), (68958221.0, '2021-08-27 19:00:39', 80.0, 111679.0, 22.0, 3.0, 0.0, 'MongoParseError: options useCreateIndex, useFindAndModify are not supported', 12459536.0), (68959632.0, '2021-08-27 21:43:39', 35.0, 505992.0, 4.0, 2.0, 0.0, \"TypeError: Cannot read properties of undefined (reading 'id')\", 16261380.0), (69033022.0, '2021-09-02 15:18:46', 96.0, 114598.0, 30.0, 10.0, 0.0, 'Message \"error: resource android:attr/lStar not found\"', 16813382.0), (69034879.0, '2021-09-02 17:36:44', 218.0, 307961.0, 8.0, 1.0, 0.0, 'How can I resolve the error \"The minCompileSdk (31) specified in a dependency\\'s AAR metadata\" in native Java or Kotlin?', 8359705.0), (69041454.0, '2021-09-03 08:01:10', 91.0, 102261.0, 9.0, 11.0, 0.0, 'Error: require() of ES modules is not supported when importing node-fetch', 16821219.0), (69080597.0, '2021-09-06 21:56:50', 58.0, 458856.0, 22.0, 4.0, 0.0, \"× TypeError: Cannot read properties of undefined (reading 'map')\", 16846583.0), (69081410.0, '2021-09-07 00:51:50', 134.0, 296364.0, 3.0, 2.0, 0.0, 'Error [ERR_REQUIRE_ESM]: require() of ES Module not supported', 16847125.0), (69139074.0, '2021-09-11 00:12:49', 8.0, 109316.0, 3.0, 0.0, 0.0, \"ERROR TypeError: Cannot read properties of undefined (reading 'title')\", 16723200.0), (69163511.0, '2021-09-13 13:25:07', 224.0, 136939.0, 13.0, 0.0, 0.0, \"Build was configured to prefer settings repositories over project repositories but repository 'maven' was added by build file 'build.gradle'\", 12886431.0), (69390676.0, '2021-09-30 10:34:46', 121.0, 121388.0, 14.0, 2.0, 0.0, 'How to use appsettings.json in Asp.net core 6 Program.cs file', 10336618.0), (69394001.0, '2021-09-30 14:28:21', 34.0, 117452.0, 10.0, 1.0, 0.0, 'How to fix? \"kex_exchange_identification: read: Connection reset by peer\"', 8939187.0), (69394632.0, '2021-09-30 15:07:50', 249.0, 242414.0, 12.0, 1.0, 0.0, 'Webpack build failing with ERR_OSSL_EVP_UNSUPPORTED', 17044429.0), (69564817.0, '2021-10-14 03:41:23', 48.0, 100499.0, 4.0, 0.0, 0.0, \"TypeError: load() missing 1 required positional argument: 'Loader' in Google Colab\", 17147261.0), (69567381.0, '2021-10-14 08:22:53', 102.0, 132221.0, 16.0, 1.0, 0.0, 'Getting \"Cannot read property \\'pickAlgorithm\\' of null\" error in react native', 15269749.0), (69665222.0, '2021-10-21 16:00:47', 90.0, 165387.0, 13.0, 0.0, 0.0, 'Node.js 17.0.1 Gatsby error - \"digital envelope routines::unsupported ... ERR_OSSL_EVP_UNSUPPORTED\"', 7002673.0), (69692842.0, '2021-10-23 23:39:57', 843.0, 816368.0, 43.0, 10.0, 0.0, 'Error message \"error:0308010C:digital envelope routines::unsupported\"', 14994086.0), (69722872.0, '2021-10-26 12:10:04', 184.0, 130738.0, 10.0, 1.0, 0.0, 'ASP.NET Core 6 how to access Configuration during startup', 1977871.0), (69773547.0, '2021-10-29 18:49:01', 62.0, 108232.0, 14.0, 1.0, 0.0, 'Visual Studio 2019 Not Showing .NET 6 Framework', 1407658.0), (69832748.0, '2021-11-03 23:06:48', 98.0, 140728.0, 16.0, 0.0, 0.0, 'Error \"Error: A <Route> is only ever to be used as the child of <Routes> element\"', 13149387.0), (69843615.0, '2021-11-04 17:44:31', 72.0, 102942.0, 6.0, 0.0, 0.0, \"Switch' is not exported from 'react-router-dom'\", 8467488.0), (69854011.0, '2021-11-05 13:26:59', 137.0, 107635.0, 7.0, 1.0, 0.0, 'Matched leaf route at location \"/\" does not have an element', 16102215.0), (69864165.0, '2021-11-06 12:55:13', 160.0, 149243.0, 18.0, 0.0, 0.0, 'Error: [PrivateRoute] is not a <Route> component. All component children of <Routes> must be a <Route> or <React.Fragment>', 16830299.0), (69868956.0, '2021-11-07 00:34:29', 145.0, 221716.0, 9.0, 4.0, 0.0, 'How can I redirect in React Router v6?', 2000548.0), (69875125.0, '2021-11-07 17:53:49', 78.0, 105208.0, 4.0, 1.0, 0.0, 'find_element_by_* commands are deprecated in Selenium', 17351258.0), (69875520.0, '2021-11-07 18:45:12', 150.0, 180541.0, 13.0, 1.0, 0.0, 'Unable to negotiate with 40.74.28.9 port 22: no matching host key type found. Their offer: ssh-rsa', 7122272.0), (70000324.0, '2021-11-17 07:23:39', 120.0, 112536.0, 3.0, 0.0, 0.0, 'What is \"crypt key missing\" error in Pgadmin4 and how to resolve it?', 10279487.0), (70036953.0, '2021-11-19 15:09:03', 90.0, 103677.0, 14.0, 5.0, 0.0, \"Spring Boot 2.6.0 / Spring fox 3 - Failed to start bean 'documentationPluginsBootstrapper'\", 306436.0), (70281346.0, '2021-12-08 20:29:31', 150.0, 155000.0, 12.0, 5.0, 0.0, 'Node.js Sass version 7.0.0 is incompatible with ^4.0.0 || ^5.0.0 || ^6.0.0', 13765920.0), (70319606.0, '2021-12-11 22:44:15', 115.0, 110984.0, 5.0, 0.0, 0.0, \"ImportError: cannot import name 'url' from 'django.conf.urls' after upgrading to Django 4.0\", 113962.0), (70358643.0, '2021-12-15 04:58:02', 233.0, 107966.0, 6.0, 4.0, 0.0, '\"You are running create-react-app 4.0.3 which is behind the latest release (5.0.0)\"', 14426381.0), (70368760.0, '2021-12-15 18:37:56', 163.0, 164918.0, 21.0, 2.0, 0.0, 'React Uncaught ReferenceError: process is not defined', 14880787.0), (70538793.0, '2021-12-31 03:41:36', 40.0, 103218.0, 9.0, 1.0, 0.0, 'remote: Write access to repository not granted. fatal: unable to access', 10781286.0)]\n\ndb.execute(\n'select title, viewcount from questions order by viewcount desc limit 10').fetchall()\n\n[('Message \"Support for password authentication was removed. Please use a personal access token instead.\"', 1236876.0), ('Error message \"error:0308010C:digital envelope routines::unsupported\"', 816368.0), ('Crbug/1173575, non-JS module files deprecated. chromewebdata/(index)꞉5305:9:5551', 610026.0), (\"TypeError: Cannot read properties of undefined (reading 'id')\", 505992.0), (\"× TypeError: Cannot read properties of undefined (reading 'map')\", 458856.0), ('Android Studio Error \"Android Gradle plugin requires Java 11 to run. You are currently using Java 1.8\"', 445775.0), ('Android Studio error \"Installed Build Tools revision 31.0.0 is corrupted\"', 426798.0), ('TypeError: this.getOptions is not a function', 315861.0), ('How can I resolve the error \"The minCompileSdk (31) specified in a dependency\\'s AAR metadata\" in native Java or Kotlin?', 307961.0), ('I am getting error \"cmdline-tools component is missing\" after installing Flutter and Android Studio... I added the Android SDK. How can I solve them?', 303399.0)]\n\n\nLet’s lay out the various verbs in SQL. Here’s the form of a standard query (though the ORDER BY is often omitted and sorting is computationally expensive):\nSELECT <column(s)> FROM <table> WHERE <condition(s) on column(s)> ORDER BY <column(s)>\nSQL keywords are often written in ALL CAPITALS, although I won’t necessarily do that in this document.\nAnd here is a table of some important keywords:\n\n\n\n\n\n\n\nKeyword\nUsage\n\n\n\n\nSELECT\nselect columns\n\n\nFROM\nwhich table to operate on\n\n\nWHERE\nfilter (choose) rows satisfying certain conditions\n\n\nLIKE, IN, <, >, ==, etc.\nused as part of conditions\n\n\nORDER BY\nsort based on columns\n\n\n\nFor logical comparisons in a WHERE clause, some common syntax for setting conditions includes LIKE (for patterns), =, >, <, >=, <=, !=.\nSome other keywords are: DISTINCT, ON, JOIN, GROUP BY, AS, USING, UNION, INTERSECT, SIMILAR TO.\nQuestion: how would we find the oldest users in the database?" }, { - "objectID": "units/unit4-goodPractices.html#version-control", - "href": "units/unit4-goodPractices.html#version-control", - "title": "Good practices: coding practices, debugging, and reproducible research", - "section": "Version control", - "text": "Version control\n\nUse it! Even for projects that only you are working on. It’s the closest thing you’ll get to having a time machine!\nUse an issues tracker (e.g., the GitHub issues tracker is quite nice), or at least a simple to-do file, noting changes you’d like to make in the future.\nIn addition to good commit messages, it’s a good idea to keep good running notes documenting your projects.\n\nWe’ve already seen Git some and will see it in a lot more detail later in the semester, so I don’t have more to say here." + "objectID": "units/unit7-bigData.html#grouping-stratifying", + "href": "units/unit7-bigData.html#grouping-stratifying", + "title": "Big data and databases", + "section": "Grouping / stratifying", + "text": "Grouping / stratifying\nA common pattern of operation is to stratify the dataset, i.e., collect it into mutually exclusive and exhaustive subsets. One would then generally do some (reduction) operation on each subset (e.g., counting records, calculating the mean of a column, taking the max of a column). In SQL this is done with the GROUP BY keyword.\nThe basic syntax looks like this:\nSELECT <reduction_operation>(<column(s)>) FROM <table> GROUP BY <column(s)>\nHere’s a basic example where we count the occurrences of different tags. Note that we use as to define a name for the new column that is created based on the aggregation operation (count in this case).\n\ndb.execute(\"select tag, count(*) as n from questions_tags \\\n group by tag \\\n order by n desc limit 25\").fetchall()\n\n[('python', 255614), ('javascript', 182006), ('java', 89097), ('reactjs', 83180), ('html', 69401), ('c#', 67633), ('android', 55422), ('r', 51688), ('node.js', 50231), ('php', 48782), ('css', 48021), ('c++', 46267), ('pandas', 45862), ('sql', 43598), ('python-3.x', 42014), ('flutter', 39243), ('typescript', 33583), ('arrays', 29960), ('angular', 29783), ('django', 29228), ('mysql', 26562), ('dataframe', 25283), ('c', 24965), ('json', 24510), ('swift', 23008)]\n\n\nIn general GROUP BY statements will involve some aggregation operation on the subsets. Options include: COUNT, MIN, MAX, AVG, SUM. The number of results will be the same as the number of groups; in the example above there should be one result per tag.\nIf you filter after using GROUP BY, you need to use having instead of where.\nChallenge: Write a query that will count the number of answers for each question, returning the most answered questions." }, { - "objectID": "units/unit4-goodPractices.html#basic-debugging-strategies", - "href": "units/unit4-goodPractices.html#basic-debugging-strategies", - "title": "Good practices: coding practices, debugging, and reproducible research", - "section": "Basic debugging strategies", - "text": "Basic debugging strategies\nHere we’ll discuss some basic strategies for finding and fixing bugs. Other useful locations for tips on debugging include:\n\nEfficient Debugging by Goldspink\nDebugging for Beginners by Brody\n\nRead and think about the error message (the traceback), starting from the bottom of the traceback. Sometimes it’s inscrutable, but often it just needs a bit of deciphering. Looking up a given error message by simply doing a web search with the exact message in double quotes can be a good strategy, or you could look specifically on Stack Overflow.\nBelow we’ll see how one can view the stack trace. Usually when an error occurs, it occurs in a function call that is nested in a series of function calls. This series of calls is the call stack and the traceback or stack trace shows that series of calls that led to the error. To debug, you’ll often need to focus on the function being executed at the time the error occurred (which will be at the top of the call stack but the bottom of the traceback) and the arguments passed into that function. However, if the error occurs in a function you didn’t write, the problem will often be with the arguments that your code provided at the last point in the call stack at which code that you wrote was run. Check the arguments that your code passed into that first function that is not a function of yours.\nWhen running code that produces multiple errors, fix errors from the top down - fix the first error that is reported, because later errors are often caused by the initial error. It’s common to have a string of many errors, which looks daunting, caused by a single initial error.\nIs the bug reproducible - does it always happen in the same way at at the same point? It can help to restart Python and see if the bug persists - this can sometimes help in figuring out if there is a scoping issue and we are using a global variable that we did not mean to.\nIf you can’t figure out where the error occurs based on the error messages, a basic strategy is to build up code in pieces (or tear it back in pieces to a simpler version). This allows you to isolate where the error is occurring. You might use a binary search strategy. Figure out which half of the code the error occurs in. Then split the ‘bad’ half in half and figure out which half the error occurs in. Repeat until you’ve isolated the problem.\nIf you’ve written your code modularly with lots of functions, you can test individual functions. Often the error will be in what gets passed into and out of each function.\nAt the beginning of time (the 1970s?), the standard debugging strategy was to insert print statements in one’s code to see the value of a variable and thereby decipher what could be going wrong. We have better tools nowadays. But sometimes we still need to fall back to inserting print statements.\nPython is a scripting language, so you can usually run your code line by line to figure out what is happening. This can be a decent approach, particularly for simple code. However, when you are trying to find errors that occur within a series of many nested function calls or when the errors involve variable scoping (how Python looks for variables that are not local to a function), or in other complicated situations, using formal debugging tools can be much more effective. Finally, if the error occurs inside of functions provided by Python, rather than ones you write, it can be hard to run the code in those functions line by line." + "objectID": "units/unit7-bigData.html#getting-unique-results-distinct", + "href": "units/unit7-bigData.html#getting-unique-results-distinct", + "title": "Big data and databases", + "section": "Getting unique results (DISTINCT)", + "text": "Getting unique results (DISTINCT)\nA useful SQL keyword is DISTINCT, which allows you to eliminate duplicate rows from any table (or remove duplicate values when one only has a single column or set of values).\n\n## Get the unique tags from the questions_tags table.\ntag_names = db.execute(\"select distinct tag from questions_tags\").fetchall()\ntag_names[0:5]\n## Count the number of unique tags.\n\n[('sorting',), ('visual-c++',), ('mfc',), ('cgridctrl',), ('css',)]\n\ndb.execute(\"select count(distinct tag) from questions_tags\").fetchall()\n\n[(42137,)]" }, { - "objectID": "units/unit4-goodPractices.html#using-pdb", - "href": "units/unit4-goodPractices.html#using-pdb", - "title": "Good practices: coding practices, debugging, and reproducible research", - "section": "Using pdb", - "text": "Using pdb\nWe can activate the debugger in various ways:\n\nby inserting breakpoint() (or equivalently import pdb; pdb.set_trace()) inside a function or module at a location of interest (and then running the function or module)\nby using pdb.pm() after an error (i.e., an exception) has occurred to invoke the browser at the point the error occurred\nby running a function under debugger control with pdb.run()\nby starting python with python -m pdb file.py and adding breakpoints\n\n\nUsing breakpoint\nLet’s define a function that will run a stratified analysis, in this case fitting a regression to each of the strata (groups/clusters) in some data. Our function is in stratified_with_break.py, and it contains breakpoint at the point where we want to invoke the debugger.\nNow I can call the function and will be put into debugging mode just before the next line is called:\n\nimport run_with_break as run\nrun.fit(run.data, run.n_cats)\n\nWhen I run this, I see this:\n>>> run.fit(data, n_cats)\n> /accounts/vis/paciorek/teaching/243fall22/stat243-fall-2023/units/run_with_break.py(10)fit()\n-> sub = data[data['cats'] == i]\n(Pdb) \nThis indicates I am debugging at line 10 of run_with_break.py, which is the line that creates sub, but I haven’t yet created sub.\nI can type n to run that line and go to the next one:\n(Pdb) n\n> /accounts/vis/paciorek/teaching/243fall22/stat243-fall-2023/units/run_with_break.py(11)fit()\n-> model = statsmodels.api.OLS(sub['y'], statsmodels.api.add_constant(sub['x']))\nat which point the debugger is about to execute line 11, which fits the regression.\nI can type c to continue until the next breakpoint:\n(Pdb) c\n> /accounts/vis/paciorek/teaching/243fall22/stat243-fall-2023/units/run_with_break.py(10)fit()\n-> sub = data[data['cats'] == i]\nNow if I print i, I see that it has incremented to 1.\n(Pdb) p i\n1\nWe could keep hitting n or c until hitting the stratum where an error occurs, but that would be tedious.\nLet’s hit q to quit out of the debugger.\n(Pdb) q\n>>>\nNext let’s see how we can enter debugging mode only at point an error occurs.\n\n\nPost-mortem debugging\nWe’ll use a version of the module without the breakpoint() command.\n\nimport pdb\nimport run_no_break as run \n\nrun.fit(run.data, run.n_cats)\npdb.pm()\n\nThat puts us into debugging mode at the point the error occurred:\n> /usr/local/linux/mambaforge-3.11/lib/python3.11/site-packages/numpy/core/fromnumeric.py(86)_wrapreduction()\n-> return ufunc.reduce(obj, axis, dtype, out, **passkwargs)\n(Pdb)\nwhich turns out to be in some internal Python function that calls a reduce function, which is where the error occurs (presumably the debugger doesn’t enter this function because it calls compiled code):\n(Pdb) l\n 81 if dtype is not None:\n 82 return reduction(axis=axis, dtype=dtype, out=out, **passkwargs)\n 83 else:\n 84 return reduction(axis=axis, out=out, **passkwargs)\n 85 \n 86 -> return ufunc.reduce(obj, axis, dtype, out, **passkwargs)\n 87 \n 88 \n 89 def _take_dispatcher(a, indices, axis=None, out=None, mode=None):\n 90 return (a, out)\n 91 \nWe can enter u multiple times (it’s only shown once below) to go up in the stack of function calls until we recognize code that we wrote:\n(Pdb) u\n> /accounts/vis/paciorek/teaching/243fall22/stat243-fall-2023/units/run_no_break.py(10)fit()\n-> model = statsmodels.api.OLS(sub['y'], statsmodels.api.add_constant(sub['x']))\nNow let’s use p to print variable values to understand the problem:\n(Pdb) p i\n29\n(Pdb) p sub\nEmpty DataFrame\nColumns: [y, x, cats]\nIndex: []\nAh, so in the 29th stratum there are no data!\nIn addition using the IPython magic %debug will put you into the debugger in in post-mortem mode when an error occurs.\n\n\npdb commands\nHere’s a list of useful pdb commands (some of which we saw above) that you can use once you’ve entered debugging mode.\n\nh or help: shows all the commands\nl or list: show the code around where the debugger is currently operating\nc or continue: continue running the code until the next breakpoint\np or print: print a variable\nn or next: run the current line and go to the next line in the current function\ns or step: jump (step) into the function called in the current line (if it’s a Python function)\nr or run: exit out of the current function (e.g., if you accidentally stepped into a function) (but note this stops at breakpoints)\nunt or until: run until the next line (or unt <number> to run until reaching line number ); this is useful for letting a loop run until completion\nb or break: set a breakpoint\ntbreak: one-time breakpoint\nwhere: shows call stack\nu (or up) and d (or down): move up and down the call stack\nq quit out of the debugger\n<return>: runs the previous pdb command again\n\n\n\nInvoking pdb on a function or block of code\nWe can use pdb.run() to run a function under the debugger. We need to make sure to use s as the first pdb command in order to actually step into the function. From there, we can debug as normal as if we had set a breakpoint at the start of the function.\n\nimport run_with_break as run\nimport pdb\npdb.run(\"run.fit(run.data, run.n_cats)\")\n(Pdb) s\n\n\n\nInvoking pdb on a module\nWe can also invoke pdb when we start Python, executing a file (module). Here we’ve added fit(data, n_cats) at the end of run_no_break2.py so that we can have that run under the debugger.\n#| eval: false\npython -m pdb run_no_break2.py\n> /accounts/vis/paciorek/teaching/243fall22/stat243-fall-2023/units/run_no_break2.py(1)<module>()\n-> import numpy as np\n(Pdb) \nLet’s set a breakpoint at the same place we did with breakpoint() but using a line number (this avoids having to actually modify our code):\n(Pdb) b 9\nBreakpoint 1 at /accounts/vis/paciorek/teaching/243fall22/stat243-fall-2023/units/run_no_break.py:9\n\n(Pdb) c\n> /accounts/vis/paciorek/teaching/243fall22/stat243-fall-2023/units/run_no_break2.py(9)fit()\n-> model = statsmodels.api.OLS(sub['y'], statsmodels.api.add_constant(sub['x']))\n\nSo we’ve broken at the same point where we manually added breakpoint() in run_with_break.py.\nOr we could have set a breakpoint at the start of the function:\n(Pdb) disable 1\nDisabled breakpoint 1 at /accounts/vis/paciorek/teaching/243fall22/stat243-fall-2023/units/run_no_break2.py:9\n(Pdb) b fit\nBreakpoint 1 at /accounts/vis/paciorek/teaching/243fall22/stat243-fall-2023/units/run_no_break.py:6" + "objectID": "units/unit7-bigData.html#simple-sql-joins", + "href": "units/unit7-bigData.html#simple-sql-joins", + "title": "Big data and databases", + "section": "Simple SQL joins", + "text": "Simple SQL joins\nOften to get the information we need, we’ll need data from multiple tables. To do this we’ll need to do a database join, telling the database what columns should be used to match the rows in the different tables.\nThe syntax generally looks like this (again the WHERE and ORDER BY are optional):\nSELECT <column(s)> FROM <table1> JOIN <table2> ON <columns to match on>\nWHERE <condition(s) on column(s)> ORDER BY <column(s)>\nLet’s see some joins using the different syntax on the Stack Overflow database. In particular let’s select only the questions with the tag ‘python’. By selecting * we are selecting all columns from both the questions and questions_tags tables.\n\nresult1 = db.execute(\"select * from questions join questions_tags \\\n on questions.questionid = questions_tags.questionid \\\n where tag = 'python'\").fetchall()\nget_fields()\n\n['questionid', 'creationdate', 'score', 'viewcount', 'answercount', 'commentcount', 'favoritecount', 'title', 'ownerid', 'questionid', 'tag']\n\n\nIt turns out you can do it without using the JOIN keyword.\n\nresult2 = db.execute(\"select * from questions, questions_tags \\\n where questions.questionid = questions_tags.questionid and \\\n tag = 'python'\").fetchall()\n\nresult1[0:5]\n\n[(65526804.0, '2021-01-01 01:54:10', 0.0, 2087.0, 3.0, 3.0, None, 'How to play an audio file starting at a specific time', 14718094.0, 65526804.0, 'python'), (65527402.0, '2021-01-01 05:14:22', 1.0, 56.0, 1.0, 0.0, None, 'Join dataframe columns in python', 1492229.0, 65527402.0, 'python'), (65529525.0, '2021-01-01 12:06:43', 1.0, 175.0, 1.0, 0.0, None, 'Issues with pygame.time.get_ticks()', 13720770.0, 65529525.0, 'python'), (65529971.0, '2021-01-01 13:14:40', 1.0, 39.0, 0.0, 1.0, None, 'How to check if Windows prompts a notification box using python?', 13845215.0, 65529971.0, 'python'), (65532644.0, '2021-01-01 18:46:52', -2.0, 49.0, 1.0, 1.0, None, 'How I divide this text file in a Dataframe?', 14122166.0, 65532644.0, 'python')]\n\nresult1 == result2\n\nTrue\n\n\nHere’s a three-way join (using both types of syntax) with some additional use of aliases to abbreviate table names. What does this query ask for?\n\nresult1 = db.execute(\"select * from \\\n questions Q \\\n join questions_tags T on Q.questionid = T.questionid \\\n join users U on Q.ownerid = U.userid \\\n where tag = 'python' and \\\n viewcount > 1000\").fetchall()\n\nresult2 = db.execute(\"select * from \\\n questions Q, questions_tags T, users U where \\\n Q.questionid = T.questionid and \\\n Q.ownerid = U.userid and \\\n tag = 'python' and \\\n viewcount > 1000\").fetchall()\n\nresult1 == result2\n\nTrue\n\n\nChallenge: Write a query that would return all the answers to questions with the Python tag.\nChallenge: Write a query that would return the users who have answered a question with the Python tag." }, { - "objectID": "units/unit4-goodPractices.html#some-common-causes-of-bugs", - "href": "units/unit4-goodPractices.html#some-common-causes-of-bugs", - "title": "Good practices: coding practices, debugging, and reproducible research", - "section": "Some common causes of bugs", - "text": "Some common causes of bugs\nSome of these are Python-specific, while others are common to a variety of languages.\n\nParenthesis mis-matches\n== vs. =\nComparing real numbers exactly using == is dangerous because numbers on a computer are only represented to limited numerical precision. For example,\n::: {.cell execution_count=12} {.python .cell-code} 1/3 == 4*(4/12-3/12)\n::: {.cell-output .cell-output-display execution_count=12} False ::: :::\nYou expect a single value but execution of the code gives an array\nSilent type conversion when you don’t want it, or lack of coercion where you’re expecting it\nUsing the wrong function or variable name\nGiving unnamed arguments to a function in the wrong order\nForgetting to define a variable in the environment of a function and having Python, via lexical scoping, get that variable as a global variable from one of the enclosing scope. At best the types are not compatible and you get an error; at worst, you use a garbage value and the bug is hard to trace. In some cases your code may work fine when you develop the code (if the variable exists in the enclosing environment), but then may not work when you restart Python if the variable no longer exists or is different.\nPython (usually helpfully) drops matrix and array dimensions that are extraneous. This can sometimes confuse later code that expects an object of a certain dimension. More on this below." + "objectID": "units/unit7-bigData.html#temporary-tables-and-views", + "href": "units/unit7-bigData.html#temporary-tables-and-views", + "title": "Big data and databases", + "section": "Temporary tables and views", + "text": "Temporary tables and views\nYou can think of a view as a temporary table that is the result of a query and can be used in subsequent queries. In any given query you can use both views and tables. The advantage is that they provide modularity in our querying. For example, if a given operation (portion of a query) is needed repeatedly, one could abstract that as a view and then make use of that view.\nSuppose we always want the age and displayname of owners of questions to be readily available. Once we have the view we can query it like a regular table.\n\ndb.execute(\"create view questionsAugment as select \\\n questionid, questions.creationdate, score, viewcount, \\\n title, ownerid, age, displayname \\\n from questions join users \\\n on questions.ownerid = users.userid\")\n## you'll see the return value is '0'\n \n\n<sqlite3.Cursor object at 0x7f8ac84b8d40>\n\ndb.execute(\"select * from questionsAugment where viewcount > 1000 limit 5\").fetchall()\n\n[(65535296.0, '2021-01-02 01:33:13', 2.0, 1109.0, 'Install and run ROS on Google Colab', 14924336.0, None, 'Gustavo Lima'), (65526407.0, '2021-01-01 00:03:01', 1.0, 2646.0, 'How to remove Branding WHMCS Ver 8.1 \"Powered by WHMcomplete solutions\"', 14920717.0, None, 'Blunch Restaurant'), (65526447.0, '2021-01-01 00:10:40', 7.0, 25536.0, 'React Router v5.2 - Blocking route change with createBrowserHistory and history.block', 10841085.0, None, 'user51462'), (65526500.0, '2021-01-01 00:22:41', 3.0, 2870.0, 'intellisense vscode not showing parameters nor documentation when hovering above with mouse', 13660865.0, None, 'albert chen'), (65526515.0, '2021-01-01 00:27:26', 2.0, 1568.0, 'How to identify time and space complexity of recursive backtracking algorithms with step-by-step analysis', 6801755.0, None, 'BlueTriangles')]\n\n\nOne use of a view would be to create a mega table that stores all the information from multiple tables in the (unnormalized) form you might have if you simply had one data frame in Python or R." }, { - "objectID": "units/unit4-goodPractices.html#tips-for-avoiding-bugs-and-catching-errors", - "href": "units/unit4-goodPractices.html#tips-for-avoiding-bugs-and-catching-errors", - "title": "Good practices: coding practices, debugging, and reproducible research", - "section": "Tips for avoiding bugs and catching errors", - "text": "Tips for avoiding bugs and catching errors\n\nPractice defensive programming\nWhen writing functions, and software more generally, you’ll want to warn the user or stop execution when there is an error and exit gracefully, giving the user some idea of what happened. Here are some things to consider:\n\ncheck function inputs and warn users if the code will do something they might not expect or makes particular choices;\ncheck inputs to if and the ranges in for loops;\nprovide reasonable default arguments;\ndocument the range of valid inputs;\ncheck that the output produced is valid; and\nstop execution based on assertions, try or raise with an informative error message.\n\nHere’s an example of building a robust square root function:\n\nimport warnings\n\ndef mysqrt(x):\n assert not isinstance(x, str), f\"what is the square root of '{x}'?\"\n if isinstance(x, int) or isinstance(x, float):\n if x < 0:\n warnings.warn(\"Input value is negative.\", UserWarning)\n return float('nan') # avoid complex number result\n else:\n return x**0.5\n else:\n raise ValueError(f\"Cannot take the square root of {x}\")\n\n\nmysqrt(3.1)\nmysqrt(-3)\ntry:\n mysqrt('hat')\nexcept Exception as error:\n print(error)\n\nwhat is the square root of 'hat'?\n\n\n/tmp/ipykernel_3887481/1815678236.py:7: UserWarning:\n\nInput value is negative.\n\n\n\n\n\nCatch run-time errors with try/except statements\nAlso, sometimes a function you call will fail, but you want to continue execution. For example, consider the stratified analysis show previously in which you take subsets of your data based on some categorical variable and fit a statistical model for each value of the categorical variable. If some of the subsets have no or very few observations, the statistical model fitting might fail. To do this, you might be using a for loop or apply. You want your code to continue and fit the model for the rest of the cases even if one (or more) of the cases cannot be fit. You can wrap the function call that may fail within the try statement and then your code won’t stop, even when an error occurs. Here’s a toy example.\n\nimport numpy as np\nimport pandas as pd\nimport random\nimport statsmodels.api\n\nnp.random.seed(2)\nn_cats = 30\nn = 80\ny = np.random.normal(size=n)\nx = np.random.normal(size=n)\ncats = [np.random.randint(0, n_cats-1) for _ in range(n)]\ndata = pd.DataFrame({'y': y, 'x': x, 'cats': cats})\n\nparams = np.full((n_cats, 2), np.nan)\nfor i in range(n_cats):\n sub = data[data['cats'] == i]\n try:\n model = statsmodels.api.OLS(sub['y'], statsmodels.api.add_constant(sub['x']))\n fit = model.fit()\n params[i, :] = fit.params.values\n except Exception as error:\n print(f\"Regression cannot be fit for stratum {i}.\")\n\nprint(params)\n\nRegression cannot be fit for stratum 7.\nRegression cannot be fit for stratum 20.\nRegression cannot be fit for stratum 24.\nRegression cannot be fit for stratum 29.\n[[ 5.52897442e-01 2.61511154e-01]\n [ 5.72564369e-01 4.37210543e-02]\n [-9.91086764e-01 2.84116572e-01]\n [-6.50606465e-01 4.26310060e-01]\n [-2.59058826e+00 -2.59058826e+00]\n [ 8.59455139e-01 -4.64514288e+00]\n [ 3.82737032e-06 3.82737032e-06]\n [ nan nan]\n [-5.55478634e-01 -1.17864561e-01]\n [-9.11601460e-02 -5.91519525e-01]\n [-7.30270153e-01 -1.99976841e-01]\n [-1.14495705e-01 -3.06421213e-02]\n [ 4.01648095e-01 9.30890661e-01]\n [ 7.88388728e-01 -1.45835443e+00]\n [ 4.08462508e+01 6.89262864e+01]\n [ 2.95467536e-01 8.80528901e-01]\n [ 1.04592517e+00 4.55379445e+00]\n [ 6.99549010e-01 -5.17503241e-01]\n [-1.75642254e+00 -8.07798224e-01]\n [-4.49033150e-02 3.53455362e-01]\n [ nan nan]\n [ 2.63097970e-01 2.63097970e-01]\n [ 1.13328314e+00 -1.39985074e-01]\n [ 1.17996663e+00 3.68770563e-01]\n [ nan nan]\n [-3.85101497e-03 -3.85101497e-03]\n [-8.04536124e-01 -5.19470059e-01]\n [-5.19200779e-01 -1.39952387e-01]\n [-9.16593858e-01 -2.67613324e-01]\n [ nan nan]]\n\n\nThe stratum with id 7 had no observations, so that call to do the regression failed, but the loop continued because we ‘caught’ the error with try. In this example, we could have checked the sample size for the subset before doing the regression, but in other contexts, we may not have an easy way to check in advance whether the function call will fail.\n\n\nMaintain dimensionality\nPython (usually helpfully) drops array dimensions that are extraneous. This can sometimes confuse later code that expects an object of a certain dimension. Here’s a work-around:\n\nimport numpy as np\nmat = np.array([[1, 2], [3, 4]])\nnp.sum(mat, axis=0) # This sums columns, as desired\n\nrow_subset = 1\nmat2 = mat[row_subset, :]\nnp.sum(mat2, axis=0) # This sums the elements, not the columns.\n\nif len(mat2.shape) != 2: # Fix dimensionality.\n mat2 = mat2.reshape(1, -1)\n\n\nnp.sum(mat2, axis=0) \n\narray([3, 4])\n\n\nIn this simple case it’s obvious that a dimension will be dropped, but in more complicated settings, this can easily occur for some inputs without the coder realizing that it may happen. Not dropping dimensions is much easier than putting checks in to see if dimensions have been dropped and having the code behave differently depending on the dimensionality.\n\n\nFind and avoid global variables\nIn general, using global variables (variables that are not created or passed into a function) results in code that is not robust. Results will change if you or a user modifies that global variable, usually without realizing/remembering that a function depends on it.\nOne ad hoc strategy is to remove objects you don’t need from Python’s global scope, to avoid accidentally using values from an old object via Python’s scoping rules. You can also run your function in a fresh session to see if it’s unable to find variables.\n\ndel x # Mimic having a fresh sesson (knowing in this case `x` is global).\n\ndef f(z):\n y = 3\n print(x + y + z)\n\ntry:\n f(2)\nexcept Exception as error:\n print(error)\n\nname 'x' is not defined\n\n\n\n\nMiscellaneous tips\n\nUse core Python functionality and algorithms already coded. Figure out if a functionality already exists in (or can be adapted from) an Python package (or potentially in a C/Fortran library/package): code that is part of standard mathematical/numerical packages will probably be more efficient and bug-free than anything you would write.\nCode in a modular fashion, making good use of functions, so that you don’t need to debug the same code multiple times. Smaller functions are easier to debug, easier to understand, and can be combined in a modular fashion (like the UNIX utilities).\nWrite code for clarity and accuracy first; then worry about efficiency. Write an initial version of the code in the simplest way, without trying to be efficient (e.g., you might use for loops even if you’re coding in Python); then make a second version that employs efficiency tricks and check that both produce the same output.\nPlan out your code in advance, including all special cases/possibilities.\nWrite tests for your code early in the process.\nBuild up code in pieces, testing along the way. Make big changes in small steps, sequentially checking to see if the code has broken on test case(s).\nBe careful that the conditions of if statements and the sequences of for loops are robust when they involve evaluating R code.\nDon’t hard code numbers - use variables (e.g., number of iterations, parameter values in simulations), even if you don’t expect to change the value, as this makes the code more readable and reduces bugs when you use the same number multiple times; e.g. speed_of_light = 3e8 or n_its = 1000.\n\nIn a future Lab, we’ll go over debugging in detail." + "objectID": "units/unit7-bigData.html#more-on-joins", + "href": "units/unit7-bigData.html#more-on-joins", + "title": "Big data and databases", + "section": "More on joins", + "text": "More on joins\nWe’ve seen a bunch of joins but haven’t discussed the full taxonomy of types of joins. There are various possibilities for how to do a join depending on whether there are rows in one table that do not match any rows in the other table.\nInner joins: In database terminology an inner join is when the result has a row for each match of a row in one table with the rows in the second table, where the matching is done on the columns you indicate. If a row in one table corresponds to more than one row in another table, you get all of the matching rows in the second table, with the information from the first table duplicated for each of the resulting rows. For example in the Stack Overflow data, an inner join of questions and answers would pair each question with each of the answers to that question. However, questions without any answers or (if this were possible) answers without a corresponding question would not be part of the result.\nOuter joins: Outer joins add additional rows from one table that do not match any rows from the other table as follows. A left outer join gives all the rows from the first table but only those from the second table that match a row in the first table. A right outer join is the converse, while a full outer join includes at least one copy of all rows from both tables. So a left outer join of the Stack Overflow questions and answers tables would, in addition to the matched questions and their answers, include a row for each question without any answers, as would a full outer join. In this case there should be no answers that do not correspond to question, so a right outer join should be the same as an inner join.\nCross joins: A cross join gives the Cartesian product of the two tables, namely the pairwise combination of every row from each table. I.e., take a row from the first table and pair it with each row from the second table, then repeat that for all rows from the first table. Since cross joins pair each row in one table with all the rows in another table, the resulting table can be quite large (the product of the number of rows in the two tables). In the Stack Overflow database, a cross join would pair each question with every answer in the database, regardless of whether the answer is an answer to that question.\nSimply listing two or more tables separated by commas as we saw earlier is the same as a cross join. Alternatively, listing two or more tables separated by commas, followed by conditions that equate rows in one table to rows in another is equivalent to an inner join.\nIn general, inner joins can be seen as a form of cross join followed by a condition that enforces matching between the rows of the table. More broadly, here are four equivalent joins that all perform the equivalent of an inner join:\n\n## explicit inner join:\nselect * from table1 join table2 on table1.id = table2.id \n## non-explicit join without JOIN\nselect * from table1, table2 where table1.id = table2.id \n## cross-join followed by matching\nselect * from table1 cross join table2 where table1.id = table2.id \n## explicit inner join with 'using'\nselect * from table1 join table2 using(id)\n\nChallenge: Create a view with one row for every question-tag pair, including questions without any tags.\nChallenge: Write a query that would return the displaynames of all of the users who have never posted a question. The NULL keyword will come in handy it’s like ‘NA’ in R. Hint: NULLs should be produced if you do an outer join." }, { - "objectID": "units/unit4-goodPractices.html#some-basic-strategies", - "href": "units/unit4-goodPractices.html#some-basic-strategies", - "title": "Good practices: coding practices, debugging, and reproducible research", - "section": "Some basic strategies", - "text": "Some basic strategies\n\nHave a directory for each project with subdirectories with meaningful and standardized names: e.g., code, data, paper. The Journal of the American Statistical Association (JASA) has a template GitHub repository with some suggestions.\nHave a file of code for pre-processing, one or more for analysis, and one for figure/table preparation.\n\nThe pre-processing may involve time-consuming steps. Save the output of the pre-processing as a file that can be read in to the analysis script.\nYou may want to name your files something like this, so there is an obvious ordering: “1-prep.py”, “2-analysis.py”, “3-figs.py”.\nHave the code file for the figures produce the exact manuscript/report figures, operating on a file (e.g., a pickle file) that contains all the objects necessary to run the figure-producing code; the code producing the pickle file should be in your analysis code file (or somewhere else sensible).\nAlternatively, use Quarto or Jupyter notebooks for your document preparation.\n\nKeep a document describing your running analysis with dates in a text file (i.e., a lab book).\nNote where data were obtained (and when, which can be helpful when publishing) and pre-processing steps in the lab book. Have data version numbers with a file describing the changes and dates (or in lab book). If possible, have all changes to data represented as code that processes the data relative to a fixed baseline dataset.\nNote what code files do what in the lab book.\nKeep track of the details of the system and software you are running your code under, e.g., operating system version, software (e.g., Python,\n\nversions, Python or R package versions, etc.\n\n\npip list and conda list will show you version numbers for installed packages." + "objectID": "units/unit7-bigData.html#indexes", + "href": "units/unit7-bigData.html#indexes", + "title": "Big data and databases", + "section": "Indexes", + "text": "Indexes\nAn index is an ordering of rows based on one or more fields. DBMS use indexes to look up values quickly, either when filtering (if the index is involved in the WHERE condition) or when doing joins (if the index is involved in the JOIN condition). So in general you want your tables to have indexes.\nDBMS use indexing to provide sub-linear time lookup. Without indexes, a database needs to scan through every row sequentially, which is called linear time lookup if there are n rows, the lookup is O(n) in computational cost. With indexes, lookup may be logarithmic O(log(n)) (if using tree-based indexes) or constant time O(1) (if using hash-based indexes). A binary tree-based search is logarithmic; at each step through the tree you can eliminate half of the possibilities.\nHere’s how we create an index, with some time comparison for a simple query.\n\nt0 = time.time()\nresults = db.execute(\n \"select * from questions where viewcount > 10000\").fetchall()\nprint(time.time() - t0) # 10 seconds\nt0 = time.time()\ndb.execute(\n \"create index count_index on questions (viewcount)\")\nprint(time.time() - t0) # 19 seconds\nt0 = time.time()\ndb.execute(\n \"select * from questions where viewcount > 10000\").fetchall() \nprint(time.time() - t0) # 3 seconds\n\nIn other contexts, an index can save huge amounts of time. So if you’re working with a database and speed is important, check to see if there are indexes. That said, as seen above it takes time to create the index, so you’d only want to create it if you were doing multiple queries that could take advantage of the index. See the databases tutorial for more discussion of how using indexes in a lookup is not always advantageous." }, { - "objectID": "units/unit4-goodPractices.html#formal-tools", - "href": "units/unit4-goodPractices.html#formal-tools", - "title": "Good practices: coding practices, debugging, and reproducible research", - "section": "Formal tools", - "text": "Formal tools\n\nIn some cases you may be able to carry out your complete workflow in a Quarto document or in a Jupyter notebook.\nYou might consider workflow/pipeline management software such as Drake or other tools discussed in the CRAN Reproducible Research Task View. Alternatively, one can use the make tool, which is generally used for compiling code, as a tool for reproducible research: if interested, see the tutorial on Using make for workflows or this Journal of Statistical Software article for more details.\nYou might organize your workflow as a Python or R package as described (for the R case) in this article.\nPackage management:\n\nPython: You can manage the versions of Python packages (and dependent packages) used in your project using Conda environments (or virtualenvs).\nR: You can manage the versions of R packages (and dependent packages) used in your project using package management packages such as renv and packrat. Unfortunately, the useful checkpoint package relies on snapshots of CRAN that are not available after January 2023.\n\nIf your project uses multiple pieces of software (e.g., not just Python or R), you can set up a reproducible environment using containers, of which Docker containers are the best known. These provide something that is like a lightweight virtual machine in which you can install exactly the software (and versions) you want and then share with others. Docker container images are a key building block of various tools such as GitHub Actions and the Binder project. Alternatively Conda is a general package manager that can install lots of non-Python packages and can also be used in many circumstances." + "objectID": "units/unit7-bigData.html#set-operations-union-intersect-except", + "href": "units/unit7-bigData.html#set-operations-union-intersect-except", + "title": "Big data and databases", + "section": "Set operations: union, intersect, except", + "text": "Set operations: union, intersect, except\nYou can do set operations like union, intersection, and set difference using the UNION, INTERSECT, and EXCEPT keywords, respectively, on tables that have the same schema (same column names and types), though most often these would be used on single columns (i.e., single-column tables).\n\nNote: While one can often set up an equivalent query without using INTERSECT or UNION, set operations can be very handy. In the example below one could do it with a join, but the syntax is often more complicated.\n\nConsider the following example of using INTERSECT. What does it return?\n\nresult1 = db.execute(\"select displayname, userid from \\\n questions Q join users U on U.userid = Q.ownerid \\\n intersect \\\n select displayname, userid from \\\n answers A join users U on U.userid = A.ownerid\")\n\nChallenge: what if you wanted to find users who had neither asked nor answered a question?" }, { - "objectID": "units/unit8-numbers.html", - "href": "units/unit8-numbers.html", - "title": "Numbers on a computer", - "section": "", - "text": "PDF\nReferences:\nA quick note that, as we’ve already seen, Python’s version of scientific notation is XeY, which means \\(X\\cdot10^{Y}\\).\nA second note is that the concepts developed here apply outside of Python, but we’ll illustrate the principles of computer numbers using Python. Python usually makes use of the double type (8 bytes) in C for the underlying representation of real-valued numbers in C variables, so what we’ll really be seeing is how such types behave in C on most modern machines. It’s actually a bit more complicated in that one can use real-valued numbers that use something other than 8 bytes in numpy by specifying a dtype.\nThe handling of integers is even more complicated. In numpy, the default is 8 byte integers, but other integer dtypes are available. And in Python itself, integers can be arbitrarily large." + "objectID": "units/unit7-bigData.html#subqueries", + "href": "units/unit7-bigData.html#subqueries", + "title": "Big data and databases", + "section": "Subqueries", + "text": "Subqueries\nA subquery is a full query that is embedded in a larger query. These can be quite handy in building up complicated queries. One could instead use temporary tables, but it often is easier to write all in one query (and that let’s the database’s query optimizer operate on the entire query).\n\nSubqueries in the FROM statement\nWe can use subqueries in the FROM statement to create a temporary table to use in a query. Here we’ll do it in the context of a join.\n\nChallenge: What does the following do?\n\n\ndb.execute(\"select * from questions join answers A \\\n on questions.questionid = A.questionid \\\n join \\\n (select ownerid, count(*) as n_answered from answers \\\n group by ownerid order by n_answered desc limit 1000) most_responsive \\\n on A.ownerid = most_responsive.ownerid\")\n\nIt might be hard to just come up with that full query all at once. A good strategy is probably to think about creating a view that is the result of the inner query and then have the outer query use that. You can then piece together the complicated query in a modular way. For big databases, you are likely to want to submit this as a single query and not two queries so that the SQL optimizer can determine the best way to do the operations. But you want to start with code that you’re confident will give you the right answer!\nNote we could also have done that query using a subquery in the WHERE statement, as discussed in the next section.\n\n\nSubqueries in the WHERE statement\nInstead of a join, we can use subqueries as a way to combine information across tables, with the subquery involved in a WHERE statement. The subquery creates a set and we then can check for inclusion in (or exclusion from with not in) that set.\nFor example, suppose we want to know the average number of UpVotes for users who have posted a question with the tag “python”.\n\ndb.execute(\"select avg(upvotes) from users where userid in \\\n (select distinct ownerid from \\\n questions join questions_tags \\\n on questions.questionid = questions_tags.questionid \\\n where tag = 'python')\").fetchall()\n\n[(62.72529394895326,)]" }, { - "objectID": "units/unit8-numbers.html#representing-real-numbers", - "href": "units/unit8-numbers.html#representing-real-numbers", - "title": "Numbers on a computer", - "section": "Representing real numbers", - "text": "Representing real numbers\n\nInitial exploration\nReals (also called floating points) are stored on the computer as an approximation, albeit a very precise approximation. As an example, if we represent the distance from the earth to the sun using a double, the error is around a millimeter. However, we need to be very careful if we’re trying to do a calculation that produces a very small (or very large number) and particularly when we want to see if numbers are equal to each other.\nIf you run the code here, the results may surprise you.\n\n0.3 - 0.2 == 0.1\n0.3\n0.2\n0.1 # Hmmm...\n\nnp.float64(0.3) - np.float64(0.2) == np.float64(0.1)\n\n0.75 - 0.5 == 0.25\n0.6 - 0.4 == 0.2\n## any ideas what is different about those two comparisons?\n\nNext, let’s consider the number of digits of accuracy we have for a variety of numbers. We’ll use format within a handy wrapper function, dg, defined earlier, to view as many digits as we want:\n\na = 0.3\nb = 0.2\ndg(a)\n\n0.29999999999999998890\n\ndg(b)\n\n0.20000000000000001110\n\ndg(a-b)\n\n0.09999999999999997780\n\ndg(0.1)\n\n0.10000000000000000555\n\ndg(1/3)\n\n0.33333333333333331483\n\n\nSo empirically, it looks like we’re accurate up to the 16th decimal place\nBut actually, the key is the number of digits, not decimal places.\n\ndg(1234.1234)\n\n1234.12339999999994688551\n\ndg(1234.123412341234)\n\n1234.12341234123391586763\n\n\nNotice that we can represent the result accurately only up to 16 significant digits. This suggests no need to show more than 16 significant digits and no need to print out any more when writing to a file (except that if the number is bigger than \\(10^{16}\\) then we need extra digits to correctly show the magnitude of the number if not using scientific notation). And of course, often we don’t need anywhere near that many.\nLet’s return to our comparison, 0.75-0.5 == 0.25.\n\ndg(0.75)\n\n0.75000000000000000000\n\ndg(0.50)\n\n0.50000000000000000000\n\n\nWhat’s different about the numbers 0.75 and 0.5 compared to 0.3, 0.2, 0.1?\n\n\nMachine epsilon\nMachine epsilon is the term used for indicating the (relative) accuracy of real numbers and it is defined as the smallest float, \\(x\\), such that \\(1+x\\ne1\\):\n\n1e-16 + 1.0\n\n1.0\n\nnp.array(1e-16) + np.array(1.0)\n\n1.0\n\n1e-15 + 1.0\n\n1.000000000000001\n\nnp.array(1e-15) + np.array(1.0)\n\n1.000000000000001\n\n2e-16 + 1.0\n\n1.0000000000000002\n\nnp.finfo(np.float64).eps\n\n2.220446049250313e-16\n\ndg(2e-16 + 1.0)\n\n1.00000000000000022204\n\n\n\n## What about in single precision, e.g. on a GPU?\nnp.finfo(np.float32).eps\n\n1.1920929e-07\n\n\n\n\nFloating point representation\nFloating point refers to the decimal point (or radix point since we’ll be working with base 2 and decimal relates to 10).\nTo proceed further we need to consider scientific notation, such as in writing Avogadro’s number as \\(+6.023\\times10^{23}\\). As a baseline for what is about to follow note that we can express a decimal number in the following expansion \\[6.037=6\\times10^{0}+0\\times10^{-1}+3\\times10^{-2}+7\\times10^{-3}\\] A real number on a computer is stored in what is basically scientific notation: \\[\\pm d_{0}.d_{1}d_{2}\\ldots d_{p}\\times b^{e}\\label{eq:floatRep}\\] where \\(b\\) is the base, \\(e\\) is an integer and \\(d_{i}\\in\\{0,\\ldots,b-1\\}\\). \\(e\\) is called the exponent and \\(d=d_{1}d_{2}\\ldots d_{p}\\) is called the mantissa.\nLet’s consider the choices that the computer pioneers needed to make in using this system to represent numbers on a computer using base 2 (\\(b=2\\)). First, we need to choose the number of bits to represent \\(e\\) so that we can represent sufficiently large and small numbers. Second we need to choose the number of bits, \\(p\\), to allocate to \\(d=d_{1}d_{2}\\ldots d_{p}\\), which determines the accuracy of any computer representation of a real.\nThe great thing about floating points is that we can represent numbers that range from incredibly small to very large while maintaining good precision. The floating point floats to adjust to the size of the number. Suppose we had only three digits to use and were in base 10. In floating point notation we can express \\(0.12\\times0.12=0.0144\\) as \\((1.20\\times10^{-1})\\times(1.20\\times10^{-1})=1.44\\times10^{-2}\\), but if we had fixed the decimal point, we’d have \\(0.120\\times0.120=0.014\\) and we’d have lost a digit of accuracy. (Furthermore, we wouldn’t be able to represent numbers bigger than \\(0.99\\).)\nMore specifically, the actual storage of a number on a computer these days is generally as a double in the form: \\[(-1)^{S}\\times1.d\\times2^{e-1023}=(-1)^{S}\\times1.d_{1}d_{2}\\ldots d_{52}\\times2^{e-1023}\\] where the computer uses base 2, \\(b=2\\), (so \\(d_{i}\\in\\{0,1\\}\\)) because base-2 arithmetic is faster than base-10 arithmetic. The leading 1 normalizes the number; i.e., ensures there is a unique representation for a given computer number. This avoids representing any number in multiple ways, e.g., either \\(1=1.0\\times2^{0}=0.1\\times2^{1}=0.01\\times2^{2}\\). For a double, we have 8 bytes=64 bits. Consider our representation as (\\(S,d,e\\)) where \\(S\\) is the sign. The leading 1 is the hidden bit and doesn’t need to be stored because it is always present. In general \\(e\\) is represented using 11 bits (\\(2^{11}=2048\\)), and the subtraction takes the place of having a sign bit for the exponent. (Note that in our discussion we’ll just think of \\(e\\) in terms of its base 10 representation, although it is of course represented in base 2.) This leaves \\(p=52 = 64-1-11\\) bits for \\(d\\).\nIn this code I force storage as a double by tacking on a decimal place, .0.\n\nbits(2.0**(-1)) # 1/2\n\n'0011111111100000000000000000000000000000000000000000000000000000'\n\nbits(2.0**0) # 1\n\n'0011111111110000000000000000000000000000000000000000000000000000'\n\nbits(2.0**1) # 2\n\n'0100000000000000000000000000000000000000000000000000000000000000'\n\nbits(2.0**1 + 2.0**0) # 3\n\n'0100000000001000000000000000000000000000000000000000000000000000'\n\nbits(2.0**2) # 4\n\n'0100000000010000000000000000000000000000000000000000000000000000'\n\nbits(-2)\n\n'1100000000000000000000000000000000000000000000000000000000000000'\n\n\nLet’s see that we can manually work out the bit-wise representation of 3.25:\n\nbits(3.25)\n\n'0100000000001010000000000000000000000000000000000000000000000000'\n\n\nQuestion: Given a fixed number of bits for a number, what is the tradeoff between using bits for the \\(d\\) part vs. bits for the \\(e\\) part?\nLet’s consider what can be represented exactly:\n\ndg(.1)\n\n0.10000000000000000555\n\ndg(.5)\n\n0.50000000000000000000\n\ndg(.25)\n\n0.25000000000000000000\n\ndg(.26)\n\n0.26000000000000000888\n\ndg(1/32)\n\n0.03125000000000000000\n\ndg(1/33)\n\n0.03030303030303030387\n\n\nSo why is 0.5 stored exactly and 0.1 not stored exactly? By analogy, consider the difficulty with representing 1/3 in base 10." + "objectID": "units/unit7-bigData.html#creating-database-tables", + "href": "units/unit7-bigData.html#creating-database-tables", + "title": "Big data and databases", + "section": "Creating database tables", + "text": "Creating database tables\nOne can create tables from within the ‘sqlite’ command line interfaces (discussed in the tutorial), but often one would do this from Python or R. Here’s the syntax from Python, creating the table from a Pandas dataframe.\n\n## create data frame 'student_data' in some fashion\ncon = sq.connect(db_path)\nstudent_data.to_sql('student', con, if_exists='replace', index=False)" }, { - "objectID": "units/unit8-numbers.html#overflow-and-underflow", - "href": "units/unit8-numbers.html#overflow-and-underflow", - "title": "Numbers on a computer", - "section": "Overflow and underflow", - "text": "Overflow and underflow\nThe largest and smallest numbers we can represent are \\(2^{e_{\\max}}\\) and \\(2^{e_{\\min}}\\) where \\(e_{\\max}\\) and \\(e_{\\min}\\) are the smallest and largest possible values of the exponent. Let’s consider the exponent and what we can infer about the range of possible numbers. With 11 bits for \\(e\\), we can represent \\(\\pm2^{10}=\\pm1024\\) different exponent values (see np.finfo(np.float64).maxexp) (why is np.finfo(np.float64).minexp only -1022?). So the largest number we could represent is \\(2^{1024}\\). What is this in base 10?\n\nx = np.float64(10)\nx**308\n\n1e+308\n\nx**309\n\ninf\n\n<string>:1: RuntimeWarning: overflow encountered in double_scalars\n\nnp.log10(2.0**1024)\n\nError: OverflowError: (34, 'Numerical result out of range')\n\nnp.log10(2.0**1023)\n\n307.95368556425274\n\nnp.finfo(np.float64)\n\nfinfo(resolution=1e-15, min=-1.7976931348623157e+308, max=1.7976931348623157e+308, dtype=float64)\n\n\nWe could have been smarter about that calculation: \\(\\log_{10}2^{1024}=\\log_{2}2^{1024}/\\log_{2}10=1024/3.32\\approx308\\). The result is analogous for the smallest number, so we have that floating points can range between \\(1\\times10^{-308}\\) and \\(1\\times10^{308}\\), consistent with what numpy reports above. Producing something larger or smaller in magnitude than these values is called overflow and underflow respectively.\nLet’s see what happens when we underflow in numpy. Note that there is no warning.\n\nx**(-308)\n\n1e-308\n\nx**(-330)\n\n0.0\n\n\nSomething subtle happens for numbers like \\(10^{-309}\\) through \\(10^{-323}\\). They can actually be represented despite what I said above. Investigating that may be an extra credit problem on a problem set." + "objectID": "units/unit1-intro.html", + "href": "units/unit1-intro.html", + "title": "Introduction to UNIX, computers, and key tools", + "section": "", + "text": "PDF" }, { - "objectID": "units/unit8-numbers.html#integers-or-floats", - "href": "units/unit8-numbers.html#integers-or-floats", - "title": "Numbers on a computer", - "section": "Integers or floats?", - "text": "Integers or floats?\nValues stored as integers should overflow if they exceed the maximum integer.\nShould \\(2^{65}\\) overflow?\n\nnp.log2(np.iinfo(np.int64).max)\n\n63.0\n\nx = np.int64(2)\n# Yikes!\nx**64\n\n0\n\n\nPython’s int type doesn’t overflow.\n\n# Interesting:\nprint(2**64)\n\n18446744073709551616\n\nprint(2**100)\n\n1267650600228229401496703205376\n\n\nOf course, doubles won’t overflow until much larger values than 4- or 8-byte integers because we know they can be as big as \\(10^308\\).\n\nx = np.float64(2)\ndg(x**64, '.2f')\n\n18446744073709551616.00\n\ndg(x**100, '.2f')\n\n1267650600228229401496703205376.00\n\n\nHowever we need to think about what integer-valued numbers can and can’t be stored exactly in our base 2 representation of floating point numbers. It turns out that integer-valued numbers can be stored exactly as doubles when their absolute value is less than \\(2^{53}\\).\n\nChallenge: Why \\(2^{53}\\)? Write out what integers can be stored exactly in our base 2 representation of floating point numbers.\n\nYou can force storage as integers or doubles in a few ways.\n\nx = 3; type(x)\n\n<class 'int'>\n\nx = np.float64(x); type(x)\n\n<class 'numpy.float64'>\n\nx = 3.0; type(x)\n\n<class 'float'>\n\nx = np.float64(3); type(x)\n\n<class 'numpy.float64'>" + "objectID": "units/unit1-intro.html#some-useful-editors", + "href": "units/unit1-intro.html#some-useful-editors", + "title": "Introduction to UNIX, computers, and key tools", + "section": "Some useful editors", + "text": "Some useful editors\n\nvarious editors available on all operating systems:\n\ntraditional editors born in UNIX: emacs, vim\nsome newer editors: Atom, Sublime Text (Sublime is proprietary/not free)\n\nWindows-specific: WinEdt\nMac-specific: Aquamacs Emacs, TextMate, TextEdit\nRStudio provides a built-in editor for R code and Quarto/R Markdown files. One can actually edit and run Python code chunks quite nicely in RStudio. (Note: RStudio as a whole is an IDE (integrated development environment. The editor is just the editing window where you edit code (and Markdown) files.)\nVSCode has a powerful code editor that is customized to work with various languages, and it has a Quarto extension.\n\nAs you get started it’s ok to use a very simple text editor such as Notepad in Windows, but you should take the time in the next few weeks to try out more powerful editors such as one of those listed above. It will be well worth your time over the course of your graduate work and then your career.\nBe careful in Windows - file suffixes are often hidden." }, { - "objectID": "units/unit8-numbers.html#precision", - "href": "units/unit8-numbers.html#precision", - "title": "Numbers on a computer", - "section": "Precision", - "text": "Precision\nConsider our representation as (S, d, e) where we have \\(p=52\\) bits for \\(d\\). Since we have \\(2^{52}\\approx0.5\\times10^{16}\\), we can represent about that many discrete values, which means we can accurately represent about 16 digits (in base 10). The result is that floats on a computer are actually discrete (we have a finite number of bits), and if we get a number that is in one of the gaps (there are uncountably many reals), it’s approximated by the nearest discrete value. The accuracy of our representation is to within 1/2 of the gap between the two discrete values bracketing the true number. Let’s consider the implications for accuracy in working with large and small numbers. By changing \\(e\\) we can change the magnitude of a number. So regardless of whether we have a very large or small number, we have about 16 digits of accuracy, since the absolute spacing depends on what value is represented by the least significant digit (the ulp, or unit in the last place) in \\(d\\), i.e., the \\(p=52\\)nd one, or in terms of base 10, the 16th digit. Let’s explore this:\n\n# large vs. small numbers\ndg(.1234123412341234)\n\n0.12341234123412339607\n\ndg(1234.1234123412341234) # not accurate to 16 decimal places \n\n1234.12341234123414324131\n\ndg(123412341234.123412341234) # only accurate to 4 places \n\n123412341234.12341308593750000000\n\ndg(1234123412341234.123412341234) # no places! \n\n1234123412341234.00000000000000000000\n\ndg(12341234123412341234) # fewer than no places! \n\n12341234123412340736.00000000000000000000\n\n\nWe can see the implications of this in the context of calculations:\n\ndg(1234567812345678.0 - 1234567812345677.0)\n\n1.00000000000000000000\n\ndg(12345678123456788888.0 - 12345678123456788887.0)\n\n0.00000000000000000000\n\ndg(12345678123456780000.0 - 12345678123456770000.0)\n\n10240.00000000000000000000\n\n\nThe spacing of possible computer numbers that have a magnitude of about 1 leads us to another definition of machine epsilon (an alternative, but essentially equivalent definition to that given previously. Machine epsilon tells us also about the relative spacing of numbers. First let’s consider numbers of magnitude one. The difference between \\(1=1.00...00\\times2^{0}\\) and \\(1.000...01\\times2^{0}\\) is \\(\\epsilon=1\\times2^{-52}\\approx2.2\\times10^{-16}\\). Machine epsilon gives the absolute spacing for numbers near 1 and the relative spacing for numbers with a different order of magnitude and therefore a different absolute magnitude of the error in representing a real. The relative spacing at \\(x\\) is \\[\\frac{(1+\\epsilon)x-x}{x}=\\epsilon\\] since the next largest number from \\(x\\) is given by \\((1+\\epsilon)x\\).\nSuppose \\(x=1\\times10^{6}\\). Then the absolute error in representing a number of this magnitude is \\(x\\epsilon\\approx2\\times10^{-10}\\). (Actually the error would be one-half of the spacing, but that’s a minor distinction.) We can see by looking at the numbers in decimal form, where we are accurate to the order \\(10^{-10}\\) but not \\(10^{-11}\\). This is equivalent to our discussion that we have only 16 digits of accuracy.\n\ndg(1000000.1)\n\n1000000.09999999997671693563\n\n\nLet’s see what arithmetic we can do exactly with integer-valued numbers stored as doubles and how that relates to the absolute spacing of numbers we’ve just seen:\n\n2.0**52\n\n4503599627370496.0\n\n2.0**52+1\n\n4503599627370497.0\n\n2.0**53\n\n9007199254740992.0\n\n2.0**53+1\n\n9007199254740992.0\n\n2.0**53+2\n\n9007199254740994.0\n\ndg(2.0**54)\n\n18014398509481984.00000000000000000000\n\ndg(2.0**54+2)\n\n18014398509481984.00000000000000000000\n\ndg(2.0**54+4)\n\n18014398509481988.00000000000000000000\n\nbits(2**53)\n\n'0100001101000000000000000000000000000000000000000000000000000000'\n\nbits(2**53+1)\n\n'0100001101000000000000000000000000000000000000000000000000000000'\n\nbits(2**53+2)\n\n'0100001101000000000000000000000000000000000000000000000000000001'\n\nbits(2**54)\n\n'0100001101010000000000000000000000000000000000000000000000000000'\n\nbits(2**54+2)\n\n'0100001101010000000000000000000000000000000000000000000000000000'\n\nbits(2**54+4)\n\n'0100001101010000000000000000000000000000000000000000000000000001'\n\n\nThe absolute spacing is \\(x\\epsilon\\), so we have spacings of \\(2^{52}\\times2^{-52}=1\\), \\(2^{53}\\times2^{-52}=2\\), \\(2^{54}\\times2^{-52}=4\\) for numbers of magnitude \\(2^{52}\\), \\(2^{53}\\), and \\(2^{54}\\), respectively.\nWith a bit more work (e.g., using Mathematica), one can demonstrate that doubles in Python in general are represented as the nearest number that can stored with the 64-bit structure we have discussed and that the spacing is as we have discussed. The results below show the spacing that results, in base 10, for numbers around 0.1. The numbers Python reports are spaced in increments of individual bits in the base 2 representation.\n\ndg(0.1234567812345678)\n\n0.12345678123456779729\n\ndg(0.12345678123456781)\n\n0.12345678123456781117\n\ndg(0.12345678123456782)\n\n0.12345678123456782505\n\ndg(0.12345678123456783)\n\n0.12345678123456782505\n\ndg(0.12345678123456784)\n\n0.12345678123456783892\n\nbits(0.1234567812345678)\n\n'0011111110111111100110101101110100010101110111110011010010000110'\n\nbits(0.12345678123456781)\n\n'0011111110111111100110101101110100010101110111110011010010000111'\n\nbits(0.12345678123456782)\n\n'0011111110111111100110101101110100010101110111110011010010001000'\n\nbits(0.12345678123456783)\n\n'0011111110111111100110101101110100010101110111110011010010001000'\n\nbits(0.12345678123456784)\n\n'0011111110111111100110101101110100010101110111110011010010001001'" + "objectID": "units/unit1-intro.html#optional-basic-emacs", + "href": "units/unit1-intro.html#optional-basic-emacs", + "title": "Introduction to UNIX, computers, and key tools", + "section": "(Optional) Basic emacs", + "text": "(Optional) Basic emacs\nEmacs is one option as an editor. I use Emacs a fair amount, so I’m including some tips here, but other editors listed above are just as good.\n\nEmacs has special modes for different types of files: Python code files, R code files, C code files, Latex files – it’s worth your time to figure out how to set this up on your machine for the kinds of files you often work on\n\nIf working with Python and R, one can start up a Python or R interpreter in an additional Emacs buffer and send code to that interpreter and see the results of running the code.\nFor working with R, ESS (emacs speaks statistics) mode is helpful. This is built into Aquamacs Emacs.\n\nTo open emacs in the terminal window rather than as a new window, which is handy when it’s too slow (or impossible) to pass (i.e., tunnel) the graphical emacs window through ssh: emacs -nw file.txt" }, { - "objectID": "units/unit8-numbers.html#working-with-higher-precision-numbers", - "href": "units/unit8-numbers.html#working-with-higher-precision-numbers", - "title": "Numbers on a computer", - "section": "Working with higher precision numbers", - "text": "Working with higher precision numbers\nAs we’ve seen, Python will automatically work with integers in arbitrary precision. (Note that R does not do this – R uses 4-byte integers, and for many calculations it’s best to use R’s numeric type because integers that aren’t really large can be expressed exactly.)\nFor higher precision floating point numbers you can make use of the gmpy2 package.\n\nimport gmpy2\ngmpy2.get_context().precision=200\ngmpy2.const_pi()\n\n## not sure why this shows ...00004\ngmpy2.mpfr(\".1234567812345678\")" + "objectID": "units/unit1-intro.html#optional-emacs-keystroke-sequence-shortcuts.", + "href": "units/unit1-intro.html#optional-emacs-keystroke-sequence-shortcuts.", + "title": "Introduction to UNIX, computers, and key tools", + "section": "(Optional) Emacs keystroke sequence shortcuts.", + "text": "(Optional) Emacs keystroke sequence shortcuts.\n\nNote Several of these (Ctrl-a, Ctrl-e, Ctrl-k, Ctrl-y) work in the command line, interactive Python and R sessions, and other places as well.\n\n\n\n\n\n\n\n\nSequence\nResult\n\n\n\n\nCtrl-x,Ctrl-c\nClose the file\n\n\nCtrl-x,Ctrl-s\nSave the file\n\n\nCtrl-x,Ctrl-w\nSave with a new name\n\n\nCtrl-s\nSearch\n\n\nESC\nGet out of command buffer at bottom of screen\n\n\nCtrl-a\nGo to beginning of line\n\n\nCtrl-e\nGo to end of line\n\n\nCtrl-k\nDelete the rest of the line from cursor forward\n\n\nCtrl-space, then move to end of block\nHighlight a block of text\n\n\nCtrl-w\nRemove the highlighted block, putting it in the kill buffer\n\n\nCtrl-y (after using Ctrl-k or Ctrl-w)\nPaste from kill buffer (‘y’ is for ‘yank’)" }, { - "objectID": "units/unit8-numbers.html#computer-arithmetic-is-not-mathematical-arithmetic", - "href": "units/unit8-numbers.html#computer-arithmetic-is-not-mathematical-arithmetic", - "title": "Numbers on a computer", - "section": "Computer arithmetic is not mathematical arithmetic!", - "text": "Computer arithmetic is not mathematical arithmetic!\nAs mentioned for integers, computer number arithmetic is not closed, unlike real arithmetic. For example, if we multiply two computer floating points, we can overflow and not get back another computer floating point.\nAnother mathematical concept we should consider here is that computer arithmetic does not obey the associative and distributive laws, i.e., \\((a+b)+c\\) may not equal \\(a+(b+c)\\) on a computer and \\(a(b+c)\\) may not be the same as \\(ab+ac\\). Here’s an example with multiplication:\n\nval1 = 1/10; val2 = 0.31; val3 = 0.57\nres1 = val1*val2*val3\nres2 = val3*val2*val1\nres1 == res2\n\nFalse\n\ndg(res1)\n\n0.01766999999999999821\n\ndg(res2)\n\n0.01767000000000000168" + "objectID": "units/unit6-parallel.html", + "href": "units/unit6-parallel.html", + "title": "Parallel processing", + "section": "", + "text": "PDF\nReferences:\nThis unit will be fairly Linux-focused as most serious parallel computation is done on systems where some variant of Linux is running. The single-machine parallelization discussed here should work on Macs and Windows, but some of the details of what is happening under the hood are different for Windows.\nAs context, let’s consider some ways we might be able to achieve faster computation:" }, { - "objectID": "units/unit8-numbers.html#calculating-with-integers-vs.-floating-points", - "href": "units/unit8-numbers.html#calculating-with-integers-vs.-floating-points", - "title": "Numbers on a computer", - "section": "Calculating with integers vs. floating points", - "text": "Calculating with integers vs. floating points\nIt’s important to note that operations with integers are fast and exact (but can easily overflow – albeit not with Python’s base int) while operations with floating points are slower and approximate. Because of this slowness, floating point operations (flops) dominate calculation intensity and are used as the metric for the amount of work being done - a multiplication (or division) combined with an addition (or subtraction) is one flop. We’ll talk a lot about flops in the unit on linear algebra." + "objectID": "units/unit6-parallel.html#embarrassingly-parallel-ep-problems", + "href": "units/unit6-parallel.html#embarrassingly-parallel-ep-problems", + "title": "Parallel processing", + "section": "Embarrassingly parallel (EP) problems", + "text": "Embarrassingly parallel (EP) problems\nAn EP problem is one that can be solved by doing independent computations in separate processes without communication between the processes. You can get the answer by doing separate tasks and then collecting the results. Examples in statistics include\n\nsimulations with many independent replicates\nbootstrapping\nstratified analyses\nrandom forests\ncross-validation.\n\nThe standard setup is that we have the same code running on different datasets. (Note that different processes may need different random number streams, as we will discuss in the Simulation Unit.)\nTo do parallel processing in this context, you need to have control of multiple processes. Note that on a shared system with queueing/scheduling software set up, this will generally mean requesting access to a certain number of processors and then running your job in such a way that you use multiple processors.\nIn general, except for some modest overhead, an EP problem can ideally be solved with \\(1/p\\) the amount of time for the non-parallel implementation, given \\(p\\) CPUs. This gives us a speedup of \\(p\\), which is called linear speedup (basically anytime the speedup is of the form \\(kp\\) for some constant \\(k\\))." }, { - "objectID": "units/unit8-numbers.html#comparisons", - "href": "units/unit8-numbers.html#comparisons", - "title": "Numbers on a computer", - "section": "Comparisons", - "text": "Comparisons\nAs we saw, we should never test x == y unless:\n\nx and y are represented as integers,\nthey are integer-valued but stored as doubles that are small enough that they can be stored exactly), or\nthey are decimal numbers that have been created in the same way (e.g., 0.4-0.3 == 0.4-0.3 returns TRUE but 0.1 == 0.4-0.3 does not).\n\nSimilarly we should be careful about testing x == 0. And be careful of greater than/less than comparisons. For example, be careful of x[ x < 0 ] = np.nan if what you are looking for is values that might be mathematically less than zero, rather than whatever is numerically less than zero.\n\n4 - 3 == 1\n\nTrue\n\n4.0 - 3.0 == 1.0\n\nTrue\n\n4.1 - 3.1 == 1.0\n\nFalse\n\n0.4-0.3 == 0.1\n\nFalse\n\n0.4-0.3 == 0.4-0.3\n\nTrue\n\n\nOne nice approach to checking for approximate equality is to make use of machine epsilon. If the relative spacing of two numbers is less than machine epsilon, then for our computer approximation, we say they are the same. Here’s an implementation that relies on the absolute spacing being \\(x\\epsilon\\) (see above).\n\nx = 12345678123456781000\ny = 12345678123456782000\n\ndef approx_equal(a,b):\n if abs(a - b) < np.finfo(np.float64).eps * abs(a + b):\n print(\"approximately equal\")\n else:\n print (\"not equal\")\n\n\napprox_equal(a,b)\n\nnot equal\n\nx = 1234567812345678\ny = 1234567812345677\n\napprox_equal(a,b) \n\nnot equal\n\n\nActually, we probably want to use a number slightly larger than machine epsilon to be safe.\nFinally, sometimes we encounter the use of an unusual integer as a symbol for missing values. E.g., a datafile might store missing values as -9999. Testing for this using == with floats should generally be ok:x [ x == -9999 ] = np.nan, because integers of this magnitude are stored exactly as floating point values. But to be really careful, you can read in as an integer or character type and do the assessment before converting to a float." + "objectID": "units/unit6-parallel.html#computer-architecture", + "href": "units/unit6-parallel.html#computer-architecture", + "title": "Parallel processing", + "section": "Computer architecture", + "text": "Computer architecture\nComputers now come with multiple processors for doing computation. Basically, physical constraints have made it harder to keep increasing the speed of individual processors, so the chip industry is now putting multiple processing units in a given computer and trying/hoping to rely on implementing computations in a way that takes advantage of the multiple processors.\nEveryday personal computers usually have more than one processor (more than one chip) and on a given processor, often have more than one core (multi-core). A multi-core processor has multiple processors on a single computer chip. On personal computers, all the processors and cores share the same memory.\nSupercomputers and computer clusters generally have tens, hundreds, or thousands of ‘nodes’, linked by a fast local network. Each node is essentially a computer with its own processor(s) and memory. Memory is local to each node (distributed memory). One basic principle is that communication between a processor and its memory is much faster than communication between processors with different memory. An example of a modern supercomputer is the Perlmutter supercomputer at Lawrence Berkeley National Lab, which has 3072 CPU-only nodes and 1792 nodes with GPUs, and a total of about 500,000 CPU cores. Each node has 512 GB of memory for a total of 2.3 PB of memory.\nFor our purposes, there is little practical distinction between multi-processor and multi-core situations. The main issue is whether processes share memory or not. In general, I won’t distinguish between cores and processors. We’ll just focus on the number of cores on given personal computer or a given node in a cluster." }, { - "objectID": "units/unit8-numbers.html#calculations", - "href": "units/unit8-numbers.html#calculations", - "title": "Numbers on a computer", - "section": "Calculations", - "text": "Calculations\nGiven the limited precision of computer numbers, we need to be careful when in the following two situations.\n\nSubtracting large numbers that are nearly equal (or adding negative and positive numbers of the same magnitude). You won’t have the precision in the answer that you would like. How many decimal places of accuracy do we have here?\n\n# catastrophic cancellation w/ large numbers\ndg(123456781234.56 - 123456781234.00)\n\n0.55999755859375000000\n\n\nThe absolute error in the original numbers here is of the order \\(\\epsilon x=2.2\\times10^{-16}\\cdot1\\times10^{11}\\approx1\\times10^{-5}=.00001\\). While we might think that the result is close to the value 1 and should have error of about machine epsilon, the relevant absolute error is in the original numbers, so we actually only have about five significant digits in our result because we cancel out the other digits.\nThis is called catastrophic cancellation, because most of the digits that are left represent rounding error – many of the significant digits have cancelled with each other.\nHere’s catastrophic cancellation with small numbers. The right answer here is exactly 0.000000000000000000001234.\n\n# catastrophic cancellation w/ small numbers\nx = .000000000000123412341234\ny = .000000000000123412340000\n\n# So we know the right answer is .000000000000000000001234 exactly. \n\ndg(x-y, '.35f')\n## [1] \"0.00000000000000000000123399999315140\"\n\n0.00000000000000000000123399999315140\n\n\nBut the result is accurate only to 8 places + 20 = 28 decimal places, as expected from a machine precision-based calculation, since the “1” is in the 13th position, after 12 zeroes (12+16=28). Ideally, we would have accuracy to 36 places (16 digits + the 20 zeroes), but we’ve lost 8 digits to catastrophic cancellation.\nIt’s best to do any subtraction on numbers that are not too large. For example, if we compute the sum of squares in a naive way, we can lose all of the information in the calculation because the information is in digits that are not computed or stored accurately: \\[s^{2}=\\sum x_{i}^{2}-n\\bar{x}^{2}\\]\n\n## No problem here:\nx = np.array([-1.0, 0.0, 1.0])\nn = len(x)\nnp.sum(x**2)-n*np.mean(x)**2 \n\n2.0\n\nnp.sum((x - np.mean(x))**2)\n\n## Adding/subtracting a constant shouldn't change the result:\n\n2.0\n\nx = x + 1e8\nnp.sum(x**2)-n*np.mean(x)**2 ## YIKES!\n\n0.0\n\nnp.sum((x - np.mean(x))**2)\n\n2.0\n\n\nA good principle to take away is to subtract off a number similar in magnitude to the values (in this case \\(\\bar{x}\\) is obviously ideal) and adjust your calculation accordingly. In general, you can sometimes rearrange your calculation to avoid catastrophic cancellation. Another example involves the quadratic formula for finding a root (p. 101 of Gentle).\nAdding or subtracting numbers that are very different in magnitude. The precision will be that of the large magnitude number, since we can only represent that number to a certain absolute accuracy, which is much less than the absolute accuracy of the smaller number:\n\ndg(123456781234.2)\n\n123456781234.19999694824218750000\n\ndg(123456781234.2 - 0.1) # truth: 123456781234.1\n\n123456781234.09999084472656250000\n\ndg(123456781234.2 - 0.01) # truth: 123456781234.19\n\n123456781234.19000244140625000000\n\ndg(123456781234.2 - 0.001) # truth: 123456781234.199\n\n123456781234.19898986816406250000\n\ndg(123456781234.2 - 0.0001) # truth: 123456781234.1999\n\n123456781234.19989013671875000000\n\ndg(123456781234.2 - 0.00001) # truth: 123456781234.19999\n\n123456781234.19998168945312500000\n\ndg(123456781234.2 - 0.000001) # truth: 123456781234.199999\n\n123456781234.19999694824218750000\n\n123456781234.2 - 0.000001 == 123456781234.2\n\nTrue\n\n\nThe larger number in the calculations above is of magnitude \\(10^{11}\\), so the absolute error in representing the larger number is around \\(1\\times10^{^{-5}}\\). Thus in the calculations above we can only expect the answers to be accurate to about \\(1\\times10^{-5}\\). In the last calculation above, the smaller number is smaller than \\(1\\times10^{-5}\\) and so doing the subtraction has had no effect. This is analogous to trying to do \\(1+1\\times10^{-16}\\) and seeing that the result is still 1.\nA work-around when we are adding numbers of very different magnitudes is to add a set of numbers in increasing order. However, if the numbers are all of similar magnitude, then by the time you add ones later in the summation, the partial sum will be much larger than the new term. A (second) work-around to that problem is to add the numbers in a tree-like fashion, so that each addition involves a summation of numbers of similar size.\n\nGiven the limited range of computer numbers, be careful when you are:\n\nMultiplying or dividing many numbers, particularly large or small ones. Never take the product of many large or small numbers as this can cause over- or under-flow. Rather compute on the log scale and only at the end of your computations should you exponentiate. E.g., \\[\\prod_{i}x_{i}/\\prod_{j}y_{j}=\\exp(\\sum_{i}\\log x_{i}-\\sum_{j}\\log y_{j})\\]\n\nLet’s consider some challenges that illustrate that last concern.\n\nChallenge: consider multiclass logistic regression, where you have quantities like this: \\[p_{j}=\\text{Prob}(y=j)=\\frac{\\exp(x\\beta_{j})}{\\sum_{k=1}^{K}\\exp(x\\beta_{k})}=\\frac{\\exp(z_{j})}{\\sum_{k=1}^{K}\\exp(z_{k})}\\] for \\(z_{k}=x\\beta_{k}\\). What will happen if the \\(z\\) values are very large in magnitude (either positive or negative)? How can we reexpress the equation so as to be able to do the calculation? Hint: think about multiplying by \\(\\frac{c}{c}\\) for a carefully chosen \\(c\\).\nSecond challenge: The same issue arises in the following calculation. Suppose I want to calculate a predictive density (e.g., in a model comparison in a Bayesian context): \\[\\begin{aligned}\nf(y^{*}|y,x) & = & \\int f(y^{*}|y,x,\\theta)\\pi(\\theta|y,x)d\\theta\\\\\n& \\approx & \\frac{1}{m}\\sum_{j=1}^{m}\\prod_{i=1}^{n}f(y_{i}^{*}|x,\\theta_{j})\\\\\n& = & \\frac{1}{m}\\sum_{j=1}^{m}\\exp\\sum_{i=1}^{n}\\log f(y_{i}^{*}|x,\\theta_{j})\\\\\n& \\equiv & \\frac{1}{m}\\sum_{j=1}^{m}\\exp(v_{j})\\end{aligned}\\] First, why do I use the log conditional predictive density? Second, let’s work with an estimate of the unconditional predictive density on the log scale, \\(\\log f(y^{*}|y,x)\\approx\\log\\frac{1}{m}\\sum_{j=1}^{m}\\exp(v_{j})\\). Now note that \\(e^{v_{j}}\\) may be quite small as \\(v_{j}\\) is the sum of log likelihoods. So what happens if we have terms something like \\(e^{-1000}\\)? So we can’t exponentiate each individual \\(v_{j}\\). This is what is known as the “log sum of exponentials” problem (and the solution as the “log-sum-exp trick”). Thoughts?\n\nNumerical issues come up frequently in linear algebra. For example, they come up in working with positive definite and semi-positive-definite matrices, such as covariance matrices. You can easily get negative numerical eigenvalues even if all the eigenvalues are positive or non-negative. Here’s an example where we use an squared exponential correlation as a function of time (or distance in 1-d), which is mathematically positive definite (i.e., all the eigenvalues are positive) but not numerically positive definite:\n\nxs = np.arange(100)\ndists = np.abs(xs[:, np.newaxis] - xs)\ncorr_matrix = np.exp(-(dists/10)**2) # This is a p.d. matrix (mathematically).\nscipy.linalg.eigvals(corr_matrix)[80:99] # But not numerically!\n\narray([-2.10937946e-16+9.49526594e-17j, -2.10937946e-16-9.49526594e-17j,\n -1.77590164e-16+1.30160558e-16j, -1.77590164e-16-1.30160558e-16j,\n -2.09305049e-16+0.00000000e+00j, 2.23869166e-16+3.21640840e-17j,\n 2.23869166e-16-3.21640840e-17j, 1.98271873e-16+9.08175827e-17j,\n 1.98271873e-16-9.08175827e-17j, -1.49116518e-16+0.00000000e+00j,\n -1.23773149e-16+6.06467275e-17j, -1.23773149e-16-6.06467275e-17j,\n -2.48071368e-18+1.51188749e-16j, -2.48071368e-18-1.51188749e-16j,\n -4.08131705e-17+6.79669911e-17j, -4.08131705e-17-6.79669911e-17j,\n 1.27901871e-16+2.34695655e-17j, 1.27901871e-16-2.34695655e-17j,\n 5.23476667e-17+4.08642121e-17j])" + "objectID": "units/unit6-parallel.html#some-useful-terminology", + "href": "units/unit6-parallel.html#some-useful-terminology", + "title": "Parallel processing", + "section": "Some useful terminology:", + "text": "Some useful terminology:\n\ncores: We’ll use this term to mean the different processing units available on a single machine or node of a cluster. A given CPU will have multiple cores. (E.g, the AMD EPYC 7763 has 64 cores per CPU.)\nnodes: We’ll use this term to mean the different computers, each with their own distinct memory, that make up a cluster or supercomputer.\nprocesses: instances of a program(s) executing on a machine; multiple processes may be executing at once. A given program may start up multiple processes at once. Ideally we have no more processes than cores on a node.\nworkers: the individual processes that are carrying out the (parallelized) computation. We’ll use worker and process interchangeably.\ntasks: individual units of computation; one or more tasks will be executed by a given process on a given core.\nthreads: multiple paths of execution within a single process; the operating system sees the threads as a single process, but one can think of them as ‘lightweight’ processes. Ideally when considering the processes and their threads, we would the same number of cores as we have processes and threads combined.\nforking: child processes are spawned that are identical to the parent, but with different process IDs and their own memory. In some cases if objects are not changed, the objects in the child process may refer back to the original objects in the original process, avoiding making copies.\nscheduler: a program that manages users’ jobs on a cluster. Slurm is a commonly used scheduler.\nload-balanced: when all the cores that are part of a computation are busy for the entire period of time the computation is running.\nsockets: some of R’s parallel functionality involves creating new R processes (e.g., starting processes via Rscript) and communicating with them via a communication technology called sockets." }, { - "objectID": "units/unit8-numbers.html#final-note", - "href": "units/unit8-numbers.html#final-note", - "title": "Numbers on a computer", - "section": "Final note", - "text": "Final note\nHow the computer actually does arithmetic with the floating point representation in base 2 gets pretty complicated, and we won’t go into the details. These rules of thumb should be enough for our practical purposes. Monahan and the URL reference have many of the gory details." + "objectID": "units/unit6-parallel.html#distributed-vs.-shared-memory", + "href": "units/unit6-parallel.html#distributed-vs.-shared-memory", + "title": "Parallel processing", + "section": "Distributed vs. shared memory", + "text": "Distributed vs. shared memory\nThere are two basic flavors of parallel processing (leaving aside GPUs): distributed memory and shared memory. With shared memory, multiple processors (which I’ll call cores for the rest of this document) share the same memory. With distributed memory, you have multiple nodes, each with their own memory. You can think of each node as a separate computer connected by a fast network.\n\nShared memory\nFor shared memory parallelism, each core is accessing the same memory so there is no need to pass information (in the form of messages) between different machines. However, unless one is using threading (or in some cases when one has processes created by forking), objects will still be copied when creating new processes to do the work in parallel. With threaded computations, multiple threads can access object(s) without making explicit copies. But in some programming contexts one needs to be careful that the threads on different cores doesn’t mistakenly overwrite places in memory that are used by other cores (this is generally not an issue in Python or R).\nWe’ll cover two types of shared memory parallelism approaches in this unit:\n\nthreaded linear algebra\nmulticore functionality\n\n\nThreading\nThreads are multiple paths of execution within a single process. If you are monitoring CPU usage (such as with top in Linux or Mac) and watching a job that is executing threaded code, you’ll see the process using more than 100% of CPU. When this occurs, the process is using multiple cores, although it appears as a single process rather than as multiple processes.\nNote that this is a different notion than a processor that is hyperthreaded. With hyperthreading a single core appears as two cores to the operating system.\n\n\n\nDistributed memory\nParallel programming for distributed memory parallelism requires passing messages between the different nodes. The standard protocol for doing this is MPI, of which there are various versions, including openMPI.\nWhile there are various Python and R that use MPI behind the scenes, we’ll only cover distributed memory parallelization via Dask, which doesn’t use MPI." }, { - "objectID": "labs/lab1-submission.html", - "href": "labs/lab1-submission.html", - "title": "Lab 1: Submitting problem set solutions", - "section": "", - "text": "By now you should already have access to the following 5 basic tools:\n\nUnix shell\nGit\nQuarto\nPython\nA text editor of your choice\n\nToday we will use all these tools together to submit a solution for Problem Set 0 (not a real problem set) to make sure you know how to submit solutions to upcoming (real) problem sets.\nHere is a selection of some basic reference tutorials and documentation for unix, bash and unix commands, git & GitHub, quarto, python and VS Code\nSome books to learn more about Unix." + "objectID": "units/unit6-parallel.html#gpus", + "href": "units/unit6-parallel.html#gpus", + "title": "Parallel processing", + "section": "GPUs", + "text": "GPUs\nGPUs (Graphics Processing Units) are processing units originally designed for rendering graphics on a computer quickly. This is done by having a large number of simple processing units for massively parallel calculation. The idea of general purpose GPU (GPGPU) computing is to exploit this capability for general computation.\nMost researchers don’t program for a GPU directly but rather use software (often machine learning software such as Tensorflow or PyTorch, or other software that automatically uses the GPU such as JAX) that has been programmed to take advantage of a GPU if one is available. The computations that run on the GPU are run in GPU kernels, which are functions that are launched on the GPU. The overall workflow runs on the CPU and then particular (usually computationally-intensive tasks for which parallelization is helpful) tasks are handed off to the GPU. GPUs and similar devices (e.g., TPUs) are often called “co-processors” in recognition of this style of workflow.\nThe memory on a GPU is distinct from main memory on the computer, so when writing code that will use the GPU, one generally wants to avoid having large amounts of data needing to be transferred back and forth between main (CPU) memory and GPU memory. Also, since there is overhead in launching a GPU kernel, one wants to avoid launching a lot of kernels relative to the amount of work being done by each kernel." }, { - "objectID": "labs/lab1-submission.html#submitting-problem-set-solutions-090123", - "href": "labs/lab1-submission.html#submitting-problem-set-solutions-090123", - "title": "Lab 1: Submitting problem set solutions", - "section": "", - "text": "By now you should already have access to the following 5 basic tools:\n\nUnix shell\nGit\nQuarto\nPython\nA text editor of your choice\n\nToday we will use all these tools together to submit a solution for Problem Set 0 (not a real problem set) to make sure you know how to submit solutions to upcoming (real) problem sets.\nHere is a selection of some basic reference tutorials and documentation for unix, bash and unix commands, git & GitHub, quarto, python and VS Code\nSome books to learn more about Unix." + "objectID": "units/unit6-parallel.html#some-other-approaches-to-parallel-processing", + "href": "units/unit6-parallel.html#some-other-approaches-to-parallel-processing", + "title": "Parallel processing", + "section": "Some other approaches to parallel processing", + "text": "Some other approaches to parallel processing\n\nSpark and Hadoop\nSpark and Hadoop are systems for implementing computations in a distributed memory environment, using the MapReduce approach, as discussed in Unit 7.\n\n\nCloud computing\nAmazon (Amazon Web Services’ EC2 service), Google (Google Cloud Platform’s Compute Engine service) and Microsoft (Azure) offer computing through the cloud. The basic idea is that they rent out their servers on a pay-as-you-go basis. You get access to a virtual machine that can run various versions of Linux or Microsoft Windows server and where you choose the number of processing cores you want. You configure the virtual machine with the applications, libraries, and data you need and then treat the virtual machine as if it were a physical machine that you log into as usual. You can also assemble multiple virtual machines into your own virtual cluster and use platforms such as databases and Spark on the cloud provider’s virtual machines." }, { - "objectID": "labs/lab1-submission.html#quick-intro-to-git-and-github", - "href": "labs/lab1-submission.html#quick-intro-to-git-and-github", - "title": "Lab 1: Submitting problem set solutions", - "section": "Quick Intro to git and GitHub", - "text": "Quick Intro to git and GitHub\n\nCreating a enw repository\nMaking changes\n\n\nEditing and saving files\nStaging changes\nCommitting changes locally\nPushing changes to remote repository\n\n\nUndoing changes:\n\n\nLocal changes\nLocal staged changes\nLocal commited changes\nPushed changes\n\n\nMerging divergent versions\nWorking with branches\nGUI options (sourcetree)\nGetting help\n\nDiscussion: - Why is git so damn complicated? - What do you need to remember when working with collaborators on the same repository?" + "objectID": "units/unit6-parallel.html#overview-key-idea", + "href": "units/unit6-parallel.html#overview-key-idea", + "title": "Parallel processing", + "section": "Overview: Key idea", + "text": "Overview: Key idea\nA key idea in Dask (and in R’s future package and the ray package for Python) involve abstracting (i.e, divorcing/separating) the specification of the parallelization in the code away from the computational resources that the code will be run on. We want to:\n\nSeparate what to parallelize from how and where the parallelization is actually carried out.\nAllow different users to run the same code on different computational resources (without touching the actual code that does the computation).\n\nThe computational resources on which the code is run is sometimes called the backend." }, { - "objectID": "labs/lab1-submission.html#hands-on-lab-instructions", - "href": "labs/lab1-submission.html#hands-on-lab-instructions", - "title": "Lab 1: Submitting problem set solutions", - "section": "Hands-on Lab Instructions", - "text": "Hands-on Lab Instructions\nProblem Set submission instructions:\n\nOpen the qmd file in any editor you like (e.g., Emacs, Sublime, ….). Use quarto preview FILE to show your rendered document live as you edit and save changes. You can put the preview window side by side with your editor, and the preview document should automatically render as you save your qmd file.\nUse VS Code with the following extensions: Python, Quarto, and Jupyter Notebooks. This allows you to execute and preview chunks (and whole document) inside VS Code. This is currently deeb’s favorite path due to how well it integrated with the Python debugger.\nUse RStudio (yes, RStudio), which can manage Python code and will display chunk output in the same way it does with R chunks. This path seems to work quite well and is recommended if you are already familiar with RStudio.\n\n\nSteps to perform today:\n\nClone your github repository to your development environment\nCreate a subdirectory in your github repository with the name ps0\nIn that subdirectory, create a quarto document (ps0.qmd) that has some simple code that creates a simple plot (you can follow this example/tutorial here)\nUse the quarto command line to render it into a pdf document (quarto render FILE –to pdf)\nCommit the changes to your repository (git add FILES; git commit -m MESSAGE; git push)\nAdd another section to your quarto document (use your imagination), then preview and commit the changes\nUse the quarto command line to render the updated document into a pdf document\nAdd the pdf document to the repository as well\nMake sure that you can log into gradescope and upload a pdf document\n[optional] Undo your last set of changes and regenerate the pdf file\n\nIf we finish early, We will also take today’s lab as an opportunity to get familiar with the basic use of all the 5 basic tools listed above.\nFor git and quarto, very basic knowledge should be sufficient for now, but for unix commands and python, the more you learn the more effective you will be at solving the problem sets (and at any computational task you take on after that). You will need to learn more advanced use of git and github towards the end of the semester when you start working with other team members on the same project.\n\n\nChunk options\nLike RMarkdown, quarto allows for several execution options to be set per document and per chunk. Spend some time getting familiar with the various options, and keep this link handy when you are working on the first few problem sets.\nDepending on what’s required in the problem sets, you may need to set eval to false (just print out code) or error to true (print errors and don’t halt rendering of the document). Some of the other options may be useful for controlling how the code gets printed." + "objectID": "units/unit6-parallel.html#overview-of-parallel-backends", + "href": "units/unit6-parallel.html#overview-of-parallel-backends", + "title": "Parallel processing", + "section": "Overview of parallel backends", + "text": "Overview of parallel backends\nOne sets the scheduler to control how parallelization is done, whether to run code on multiple machines, and how many cores on each machine to use.\nFor example to parallelize across multiple cores via separate Python processes, we’d do this.\n\nimport dask\ndask.config.set(scheduler='processes', num_workers = 4) \n\nThis table shows the different types of schedulers.\n\n\n\n\n\n\n\n\n\nType\nDescription\nMulti-node\nCopies of objects made?\n\n\n\n\nsynchronous\nnot in parallel (serial)\nno\nno\n\n\nthreads (1)\nthreads within current Python session\nno\nno\n\n\nprocesses\nbackground Python sessions\nno\nyes\n\n\ndistributed (2)\nPython sessions across multiple nodes\nyes\nyes\n\n\n\nComments:\n\nNote that because of Python’s Global Interpreter Lock (GIL) (which prevents threading of Python code), many computations done in pure Python code won’t be parallelized using the ‘threads’ scheduler; however computations on numeric data in numpy arrays, Pandas dataframes and other C/C++/Cython-based code will parallelize.\nIt’s fine to use the distributed scheduler on one machine, such as your laptop. According to the Dask documentation, it has advantages over multiprocessing, including the diagnostic dashboard (see the tutorial) and better handling of when copies need to be made. In addition, one needs to use it for parallel map operations (see next section)." }, { - "objectID": "labs/lab0-setup.html", - "href": "labs/lab0-setup.html", - "title": "Lab 0: Setup", - "section": "", - "text": "You will need to set up (or make sure you have access to) the following:\n\nUnix shell\nGit\nQuarto\nPython\nA text editor or IDE of your choice (deeb recommends VS Code or sublime, but if you are already familiar with a specific editor, stick with it)\n\nAfter making sure you have access to all these 5 tools, it may be a good idea to go through some of the following tutorials on unix bash and unix commands. You can do this on your own or in the Lab section on Friday 8/25.\nAttending lab 0 is optional. If you successfully set up your environment to have all the listed tools, you don’t need to attend. You are welcome to attend to ask for help with the setup or to help your classmates with setting up their environments.\nYou are also welcome to come and ask for help on using any of these 5 tools/systems (esp. bash and unix commands this week), but priority will be given to environment setup questions.\nReach out to deeb @ deeb@berkeley.edu with any unresolved problems or if you discover something that needs to be changed with our howtos and instructions.\nDeeb’s unsolicited advice on languages and tools:\n\nWhichever editor you pick, make sure to spend some time every week learning a few keyboard shortcuts for it. The same goes for the bash shell (ctrl+a and ctrl+e are among my favorites) and for your OS of choice in general (and even your web browser!). Not only do keyboard shortcuts make you more efficient, but they dramatically reduce the cognitive load after a while, and so make your life less painful in the long run. They can be the difference between hating computers and loving them.\nProgramming languages come and go, but Unix is forever! Well, maybe not forever, but close enough. Invest more of your time in getting familiar with durable and proven paradigms. Different programming languages are suitable in different situations and change dramatically from one decade to the next, but the unix shell and commands are as pristine, long lived, and as widely applicable as you’ll find in the computing world. I have a much more mixed view of git, Python, R, and C++. Another example of a very durable computing paradigm is SQL, which we’ll get to in a few weeks." + "objectID": "units/unit6-parallel.html#accessing-variables-and-workers-in-the-worker-processes", + "href": "units/unit6-parallel.html#accessing-variables-and-workers-in-the-worker-processes", + "title": "Parallel processing", + "section": "Accessing variables and workers in the worker processes", + "text": "Accessing variables and workers in the worker processes\nDask usually does a good job of identifying the packages and (global) variables you use in your parallelized code and importing those packages on the workers and copying necessary variables to the workers.\nHere’s a toy example that shows that the numpy package and a global variable n are automatically available in the worker processes without any action on our part.\nNote the use of the @delayed decorator to flag the function so that it operates in a lazy manner for use with Dask’s parallelization capabilities.\n\nimport dask\ndask.config.set(scheduler='processes', num_workers = 4, chunksize = 1) \n\n<dask.config.set object at 0x7fcd37ff9d10>\n\nimport numpy as np\nn = 10\n\n@dask.delayed\ndef myfun(idx):\n return np.random.normal(size = n)\n\n\ntasks = []\np = 8\nfor i in range(p):\n tasks.append(myfun(i)) # add lazy task\n\ntasks\n\n[Delayed('myfun-fb6d5933-5a6e-455f-87c2-aee3ca9aadd3'), Delayed('myfun-62938a41-9c82-4e80-b9ee-09aba30dcbfb'), Delayed('myfun-91faca5a-ff4b-4eef-8736-acf786d1cbe3'), Delayed('myfun-e85f3b84-3ce1-477f-91a4-9069abb33748'), Delayed('myfun-1d342e40-ff5d-41a1-b10c-7f686d8714c0'), Delayed('myfun-73e39dc6-32fb-4cc5-9e8a-03e8ace1db71'), Delayed('myfun-e984bae0-6a28-480e-9165-86a980f5198e'), Delayed('myfun-b57c623a-d447-4d68-ae93-479e96cc5546')]\n\nresults = dask.compute(tasks) # compute all in parallel\n\nIn other contexts (in various languages) you may need to explicitly copy objects to the workers (or load packages on the workers). This is sometimes called exporting variables.\nWe don’t have to flag the function in advanced with @delayed. We could also have directly called the decorator like this, which as the advantage of allowing us to run the function in the normal way if we simply invoke it.\n\ntasks.append(dask.delayed(myfun)(i))" }, { - "objectID": "labs/lab0-setup.html#setting-up-your-environment-082523", - "href": "labs/lab0-setup.html#setting-up-your-environment-082523", - "title": "Lab 0: Setup", - "section": "", - "text": "You will need to set up (or make sure you have access to) the following:\n\nUnix shell\nGit\nQuarto\nPython\nA text editor or IDE of your choice (deeb recommends VS Code or sublime, but if you are already familiar with a specific editor, stick with it)\n\nAfter making sure you have access to all these 5 tools, it may be a good idea to go through some of the following tutorials on unix bash and unix commands. You can do this on your own or in the Lab section on Friday 8/25.\nAttending lab 0 is optional. If you successfully set up your environment to have all the listed tools, you don’t need to attend. You are welcome to attend to ask for help with the setup or to help your classmates with setting up their environments.\nYou are also welcome to come and ask for help on using any of these 5 tools/systems (esp. bash and unix commands this week), but priority will be given to environment setup questions.\nReach out to deeb @ deeb@berkeley.edu with any unresolved problems or if you discover something that needs to be changed with our howtos and instructions.\nDeeb’s unsolicited advice on languages and tools:\n\nWhichever editor you pick, make sure to spend some time every week learning a few keyboard shortcuts for it. The same goes for the bash shell (ctrl+a and ctrl+e are among my favorites) and for your OS of choice in general (and even your web browser!). Not only do keyboard shortcuts make you more efficient, but they dramatically reduce the cognitive load after a while, and so make your life less painful in the long run. They can be the difference between hating computers and loving them.\nProgramming languages come and go, but Unix is forever! Well, maybe not forever, but close enough. Invest more of your time in getting familiar with durable and proven paradigms. Different programming languages are suitable in different situations and change dramatically from one decade to the next, but the unix shell and commands are as pristine, long lived, and as widely applicable as you’ll find in the computing world. I have a much more mixed view of git, Python, R, and C++. Another example of a very durable computing paradigm is SQL, which we’ll get to in a few weeks." + "objectID": "units/unit6-parallel.html#scenario-1-one-model-fit", + "href": "units/unit6-parallel.html#scenario-1-one-model-fit", + "title": "Parallel processing", + "section": "Scenario 1: one model fit", + "text": "Scenario 1: one model fit\nSpecific scenario: You need to fit a single statistical/machine learning model, such as a random forest or regression model, to your data.\nGeneral scenario: Parallelizing a single task.\n\nScenario 1A:\nA given method may have been written to use parallelization and you simply need to figure out how to invoke the method for it to use multiple cores.\nFor example the documentation for the RandomForestClassifier in scikit-learn’s ensemble module indicates it can use multiple cores – note the n_jobs argument (not shown here because the help info is very long).\n\nimport sklearn.ensemble\nhelp(sklearn.ensemble.RandomForestClassifier)\n\nYou’ll usually need to look for an argument with one of the words threads, processes, cores, cpus, jobs, etc. in the argument name.\n\n\nScenario 1B:\nIf a method does linear algebra computations on large matrices/vectors, Python (and R) can call out to parallelized linear algebra packages (the BLAS and LAPACK).\nThe BLAS is the library of basic linear algebra operations (written in Fortran or C). A fast BLAS can greatly speed up linear algebra in R relative to the default BLAS that comes with R. Some fast BLAS libraries are\n\nIntel’s MKL; available for educational use for free\nOpenBLAS; open source and free\nApple’s Accelerate framework BLAS (vecLib) for Macs; provided with your Mac\n\nIn addition to being fast when used on a single core, all of these BLAS libraries are threaded - if your computer has multiple cores and there are free resources, your linear algebra will use multiple cores, provided your program is linked against the threaded BLAS installed on your machine and provided the shell environment variable OMP_NUM_THREADS is not set to one. (Macs make use of VECLIB_MAXIMUM_THREADS rather than OMP_NUM_THREADS and if MKL is being used, then one needs MKL_NUM_THREADS)\nFor parallel (threaded) linear algebra in Python, one can use an optimized BLAS with the numpy and (therefore) scipy packages, on Linux or using the Mac’s vecLib BLAS. Details will depend on how you install Python, numpy, and scipy. More details on figuring out what BLAS is being used and how to install a fast threaded BLAS on your own computer are here.\nDask and some other packages also provide threading, but pure Python code is not threaded.\nHere’s some code that illustrates the speed of using a threaded BLAS:\n\nimport numpy as np\nimport time\n\nx = np.random.normal(size = (6000, 6000))\n\nstart_time = time.time()\nx = np.dot(x.T, x) \nU = np.linalg.cholesky(x)\nelapsed_time = time.time() - start_time\nprint(\"Elapsed Time (8 threads):\", elapsed_time)\n\nWe’d need to restart Python after setting OMP_NUM_THREADS to 1 in order to compare the time when run in parallel vs. on a single core. That’s hard to demonstrate in this generated document, but when I ran it, it took 6.6 seconds, compared to 3 seconds using 8 cores.\nNote that for smaller linear algebra problems, we may not see any speed-up or even that the threaded calculation might be slower because of overhead in setting up the parallelization and because the parallelized linear algebra calculation involves more actual operations than when done serially.\n\n\nGPUs and linear algebra\nLinear algebra with large matrices is often a very good use case for GPUs.\nHere’s an example of using the GPU to multiply large matrices using PyTorch. We could do this similarly with Tensorflow or JAX. I’ve just inserted the timing from running this on an SCF machine with a powerful GPU.\n\nimport torch\n\nstart = torch.cuda.Event(enable_timing=True)\nend = torch.cuda.Event(enable_timing=True)\n\ngpu = torch.device(\"cuda:0\")\n\nn = 10000\nx = torch.randn(n,n, device = gpu)\ny = torch.randn(n,n, device = gpu)\n\n## Time the matrix multiplication on GPU:\nstart.record()\nz = torch.matmul(x, y)\nend.record()\ntorch.cuda.synchronize()\nprint(start.elapsed_time(end)) # 120 ms.\n\n## Compare to CPU:\ncpu = torch.device(\"cpu\")\n\nx = torch.randn(n,n, device = cpu)\ny = torch.randn(n,n, device = cpu)\n\n## Time the matrix multiplication on CPU:\nstart.record()\nz = torch.matmul(x, y)\nend.record()\ntorch.cuda.synchronize()\nprint(start.elapsed_time(end)) # 18 sec.\n\nThe GPU calculation takes 100-200 milliseconds (ms), while the CPU calculation took 18 seconds using two CPU cores. That’s a speed-up of more than 100x!\nFor a careful comparison between GPU and CPU, we’d want to consider the effect of using 4-byte floating point numbers for the GPU calculation.\nWe’d also want to think about how many CPU cores should be used for the comparison." }, { - "objectID": "labs/lab2-testing.html", - "href": "labs/lab2-testing.html", - "title": "Lab 2: Assertions, Exceptions, and Tesing", - "section": "", - "text": "Today we will spend some time getting familiar with some of the programming tools that can help make your code more robust and resilient to errors and boundary conditions. Tools like unit tests, exceptions, and asserts (and next week we will spend some time on debugging misbehaving code).\nTesting is what you do when you finish implementing a piece of code and want to try it out to see if it works. Running your code manually and seeing if it works is a workable strategy for simple one-time scripts that do simple tasks, but there are situations (like writing a function that others will repeatedly use, or like running the same piece of code on hundreds of files or URLs) where it is prudent to test your code ahead of its actual use or deployment. Today we will use the pytest package to do that. We will also use some error handling techniques to make sure our code can handle malformed inputs." + "objectID": "units/unit6-parallel.html#scenario-2-three-different-prediction-methods-on-your-data", + "href": "units/unit6-parallel.html#scenario-2-three-different-prediction-methods-on-your-data", + "title": "Parallel processing", + "section": "Scenario 2: three different prediction methods on your data", + "text": "Scenario 2: three different prediction methods on your data\nSpecific scenario: You need to fit three different statistical/machine learning models to your data.\nGeneral scenario: Parallelizing a small number of tasks.\nWhat are some options?\n\nuse one core per model\nif you have rather more than three cores, apply the ideas here combined with Scenario 1 above - with access to a cluster and parallelized implementations of each model, you might use one node per model\n\nHere we’ll use the processes scheduler. In principal given this relies on numpy code, we could have also used the threads scheduler, but I’m not seeing effective parallelization when I try that.\n\nimport dask\nimport time\nimport numpy as np\n\ndef gen_and_mean(func, n, par1, par2):\n return np.mean(func(par1, par2, size = n))\n\ndask.config.set(scheduler='processes', num_workers = 3, chunksize = 1) \n\n<dask.config.set object at 0x7fcd37ffe810>\n\nn = 100000000\nt0 = time.time()\ntasks = []\ntasks.append(dask.delayed(gen_and_mean)(np.random.normal, n, 0, 1))\ntasks.append(dask.delayed(gen_and_mean)(np.random.gamma, n, 1, 1))\ntasks.append(dask.delayed(gen_and_mean)(np.random.uniform, n, 0, 1))\nresults = dask.compute(tasks)\nprint(time.time() - t0) \n\n3.6668434143066406\n\nt0 = time.time()\np = gen_and_mean(np.random.normal, n, 0, 1)\nq = gen_and_mean(np.random.gamma, n, 1, 1)\ns = gen_and_mean(np.random.uniform, n, 0, 1)\nprint(time.time() - t0) \n\n5.191020488739014\n\n\nQuestion: Why might this not have shown a perfect three-fold speedup?\nYou could also have used tools like a parallel map here as well, as we’ll discuss in the next scenario.\n\nLazy evaluation, synchronicity, and blocking\nIf we look at the delayed objects, we see that each one is a representation of the computation that needs to be done and that execution happens lazily. Also note that dask.compute executes synchronously, which means the main process waits until the dask.compute call is complete before allowing other commands to be run. This synchronous evaluation is also called a blocking call because execution of the task in the worker processes blocks the main process. In contrast, if control returns to the user before the worker processes are done, that would be asynchronous evaluation (aka, a non-blocking call).\nNote: the use of chunksize = 1 forces Dask to immediately start one task on each worker. Without that argument, by default it groups tasks so as to reduce the overhead of starting each task individually, but when we have few tasks, that prevents effective parallelization. We’ll discuss this in much more detail in Scenario 4." }, { - "objectID": "labs/lab2-testing.html#lab-exercise", - "href": "labs/lab2-testing.html#lab-exercise", - "title": "Lab 2: Assertions, Exceptions, and Tesing", - "section": "Lab Exercise", - "text": "Lab Exercise\n1- Imagine a function that:\n[option 1] takes in a string and return another string where the space separated words (or tokens) are the same as the input string but sorted according to a specific ordering. The ordering should be determined by the second argument to the function. The ordering could be specified as (1) lexicographic (alphabetic) according to the words (2) lexicographic according to the key produced by sorting the letters of the original word, or (3) numeric.\n[option 2] (in case you want to practice regex and keep stick with the theme of this week’s lecture) takes in a string and returns the first number in that string, returns None if there are no numeric values in the string.\n2- Write an interface for that function (a function name and arguments), but do not implement the function yet (you can have it return an None, or an empty string for now). We will do this in a good old fashioned .py file (not a notebook or a quarto file).\n3- Build a test suite using the pytest package to test that your function works as intended. Add at least 8 test cases with justification for each. Try to cover the main use cases, and as many potential corner cases or boundary conditions as possible.\n4- Now run the test suite. It should fail for all your tests (unless one of them was passing an empty string).\n5- Implement the function. You can do this at one go, or case by case. As you implement a case, you can rerun the test suite and see some of the tests relevant to that cases stopping to fail. When all the tests pass, you are done. This is called test-driven development.\n6- If some cases are still failing, that’s alright, we can use that failing code next week for demonstrating debugging functionality\n7- Make sure you have assertions to check for invalid input types and combinations.\n8- Make your function throw an exception for invalid input combinations.\n9- Now write a loop that calls your function on a variety of inputs including invalid inputs.\n10- In that loop, handle the thown exceptions and print something appropriate, but let the loop continue for later inputs." + "objectID": "units/unit6-parallel.html#scenario-3-10-fold-cv-and-10-or-fewer-cores", + "href": "units/unit6-parallel.html#scenario-3-10-fold-cv-and-10-or-fewer-cores", + "title": "Parallel processing", + "section": "Scenario 3: 10-fold CV and 10 or fewer cores", + "text": "Scenario 3: 10-fold CV and 10 or fewer cores\nSpecific scenario: You are running a prediction method on 10 cross-validation folds.\nGeneral scenario: Parallelizing tasks via a parallel map.\nThis illustrates the idea of running some number of tasks using the cores available on a single machine.\nHere I’ll illustrate using a parallel map, using this simulated dataset and basic use of RandomForestRegressor().\nFirst, let’s set up our fit function and simulate some data.\nIn this case our fit function uses global variables. The reason for this is that we’ll use Dask’s map function, which allows us to pass only a single argument. We could bundle the input data with the fold_idx value and pass as a larger object, but here we’ll stick with the simplicity of global variables.\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import KFold\n\ndef cv_fit(fold_idx):\n train_idx = folds != fold_idx\n test_idx = folds == fold_idx\n X_train = X.iloc[train_idx]\n X_test = X.iloc[test_idx]\n Y_train = Y[train_idx]\n model = RandomForestRegressor()\n model.fit(X_train, Y_train)\n predictions = model.predict(X_test)\n return predictions\n\n\nnp.random.seed(1)\n\n# Generate data\nn = 1000\np = 50\nX = pd.DataFrame(np.random.normal(size = (n, p)),\\\n columns=[f\"X{i}\" for i in range(1, p + 1)])\nY = X['X1'] + np.sqrt(np.abs(X['X2'] * X['X3'])) +\\\n X['X2'] - X['X3'] + np.random.normal(size = n)\n\nn_folds = 10\nseq = np.arange(n_folds)\nfolds = np.random.permutation(np.repeat(seq, 100))\n\nTo do a parallel map, we need to use the distributed scheduler, but it’s fine to do that with multiple cores on a single machine (such as a laptop).\n\nn_cores = 2\nfrom dask.distributed import Client, LocalCluster\ncluster = LocalCluster(n_workers = n_cores)\nc = Client(cluster)\n\ntasks = c.map(cv_fit, range(n_folds))\nresults = c.gather(tasks)\n# We'd need to sort the results appropriately to align them with the observations.\n\nNow suppose you have 4 cores (and therefore won’t have an equal number of tasks per core with the 10 tasks). The approach in the next scenario should work better." }, { - "objectID": "syllabus.html", - "href": "syllabus.html", - "title": "Statistics 243 Fall 2023", - "section": "", - "text": "Statistics 243 is an introduction to statistical computing, taught using Python. The course will cover both programming concepts and statistical computing concepts. Programming concepts will include data and text manipulation, regular expressions, data structures, functions and variable scope, memory use, efficiency, debugging, testing, and parallel processing. Statistical computing topics will include working with large datasets, numerical linear algebra, computer arithmetic/precision, simulation studies and Monte Carlo methods, numerical optimization, and numerical integration/differentiation. A goal is that coverage of these topics complement the models/methods discussed in the rest of the statistics/biostatistics graduate curriculum. We will also cover the basics of UNIX/Linux, in particular shell scripting and operating on remote servers, as well as a bit of R.\n\n\n\nWhile the course is taught using Python and you will learn a lot about using Python at an advanced level, this is not a course about learning Python. Rather the focus of the course is computing for statistics and data science more generally, using Python to illustrate the concepts.\nThis is not a course that will cover specific statistical/machine learning/data analysis methods.\n\n\n\n\nInformal prerequisites: If you are not a statistics or biostatistics graduate student, please chat with me if you’re not sure if this course makes sense for you. A background in calculus, linear algebra, probability and statistics is expected, as well as a basic ability to operate on a computer (but I do not assume familiarity with the UNIX-style command line/terminal/shell). Furthermore, I’m expecting you will know the basics of Python, at the level of the Python short course offered Aug. 16-17, 2023. If you don’t have that background you’ll need to spend time in the initial couple weeks getting up to speed. In addition, we may have an optional hands-on practice session during the second or third week of class, and the GSI can also provide assistance." + "objectID": "units/unit6-parallel.html#scenario-4-parallelizing-over-prediction-methods", + "href": "units/unit6-parallel.html#scenario-4-parallelizing-over-prediction-methods", + "title": "Parallel processing", + "section": "Scenario 4: parallelizing over prediction methods", + "text": "Scenario 4: parallelizing over prediction methods\nScenario: parallelizing over prediction methods or other cases where execution time varies.\nIf you need to parallelize over prediction methods or in other contexts in which the computation time for the different tasks varies widely, you want to avoid having the parallelization group the tasks into batches in advance, because some cores may finish a lot more quickly than others. Starting the tasks one by one (not in batches) is called dynamic allocation.\nIn contrast, if the computation time is about the same for the different tasks (or if you have so many tasks that the effect of averaging helps with load-balancing) then you want to group the tasks into batches. This is called static allocation or prescheduling. This avoids the extra overhead (~1 millisecond per task) of scheduling many tasks.\n\nDynamic allocation\nWith Dask’s distributed scheduler, Dask starts up each delayed evaluation separately (i.e., dynamic allocation).\nWe’ll set up an artificial example with four slow tasks and 12 fast tasks and see the speed of running with the default of dynamic allocation under Dask’s distributed scheduler. Then in the next section, we’ll compare to the worst-case scenario with all four slow tasks in a single batch.\n\nimport scipy.special\n\nn_cores = 4\nfrom dask.distributed import Client, LocalCluster\ncluster = LocalCluster(n_workers = n_cores)\nc = Client(cluster)\n\n## 4 slow tasks and 12 fast ones.\nn = np.repeat([10**7, 10**5, 10**5, 10**5], 4)\n\ndef fun(i):\n print(f\"Working on {i}.\")\n return np.mean(scipy.special.gammaln(np.exp(np.random.normal(size = n[i]))))\n \n\nt0 = time.time()\nout = fun(1)\n\nWorking on 1.\n\nprint(time.time() - t0)\n\n0.5563864707946777\n\nt0 = time.time()\nout = fun(5)\n\nWorking on 5.\n\nprint(time.time() - t0)\n\n0.01022791862487793\n\nt0 = time.time()\ntasks = c.map(fun, range(len(n)))\nresults = c.gather(tasks) \nprint(time.time() - t0) # 0.8 sec. \n\n0.8854873180389404\n\ncluster.close()\n\nNote that with relatively few tasks per core here, we could have gotten unlucky if the tasks were in a random order and multiple slow tasks happen to be done by a single worker.\n\n\nStatic allocation\nNext, note that by default the ‘processes’ scheduler sets up tasks in batches, with a default chunksize of 6. In this case that means that the first 4 (slow) tasks are all allocated to a single worker.\n\ndask.config.set(scheduler='processes', num_workers = 4)\n\n<dask.config.set object at 0x7fcd2391ac50>\n\ntasks = []\np = len(n)\nfor i in range(p):\n tasks.append(dask.delayed(fun)(i)) # add lazy task\n\nt0 = time.time()\nresults = dask.compute(tasks) # compute all in parallel\nprint(time.time() - t0) # 2.6 sec.\n\n1.7515089511871338\n\n\nTo force dynamic allocation, we can set chunksize = 1 (as was shown in our original example of using the processes scheduler).\n\ndask.config.set(scheduler='processes', num_workers = 4, chunksize = 1)\n\ntasks = []\np = len(n)\nfor i in range(p):\n tasks.append(dask.delayed(fun)(i)) # add lazy task\n\nresults = dask.compute(tasks) # compute all in parallel\n\n\n\nChoosing static vs. dynamic allocation in Dask\nWith the distributed scheduler, Dask starts up each delayed evaluation separately (i.e., dynamic allocation). And even with a distributed map() it doesn’t appear possible to ask that the tasks be broken up into batches. Therefore if you want static allocation, you could use the processes scheduler if using a single machine, or if you need to use the distributed schduler you could break up the tasks into batches manually.\nWith the processes scheduler, static allocation is the default, with a default chunksize of 6 tasks per batch. You can force dynamic allocation by setting chunksize = 1.\n(Note that in R, static allocation is the default when using the future package.)" }, { - "objectID": "syllabus.html#course-description", - "href": "syllabus.html#course-description", - "title": "Statistics 243 Fall 2023", - "section": "", - "text": "Statistics 243 is an introduction to statistical computing, taught using Python. The course will cover both programming concepts and statistical computing concepts. Programming concepts will include data and text manipulation, regular expressions, data structures, functions and variable scope, memory use, efficiency, debugging, testing, and parallel processing. Statistical computing topics will include working with large datasets, numerical linear algebra, computer arithmetic/precision, simulation studies and Monte Carlo methods, numerical optimization, and numerical integration/differentiation. A goal is that coverage of these topics complement the models/methods discussed in the rest of the statistics/biostatistics graduate curriculum. We will also cover the basics of UNIX/Linux, in particular shell scripting and operating on remote servers, as well as a bit of R.\n\n\n\nWhile the course is taught using Python and you will learn a lot about using Python at an advanced level, this is not a course about learning Python. Rather the focus of the course is computing for statistics and data science more generally, using Python to illustrate the concepts.\nThis is not a course that will cover specific statistical/machine learning/data analysis methods.\n\n\n\n\nInformal prerequisites: If you are not a statistics or biostatistics graduate student, please chat with me if you’re not sure if this course makes sense for you. A background in calculus, linear algebra, probability and statistics is expected, as well as a basic ability to operate on a computer (but I do not assume familiarity with the UNIX-style command line/terminal/shell). Furthermore, I’m expecting you will know the basics of Python, at the level of the Python short course offered Aug. 16-17, 2023. If you don’t have that background you’ll need to spend time in the initial couple weeks getting up to speed. In addition, we may have an optional hands-on practice session during the second or third week of class, and the GSI can also provide assistance." + "objectID": "units/unit6-parallel.html#scenario-5-10-fold-cv-across-multiple-methods-with-many-more-than-10-cores", + "href": "units/unit6-parallel.html#scenario-5-10-fold-cv-across-multiple-methods-with-many-more-than-10-cores", + "title": "Parallel processing", + "section": "Scenario 5: 10-fold CV across multiple methods with many more than 10 cores", + "text": "Scenario 5: 10-fold CV across multiple methods with many more than 10 cores\nSpecific scenario: You are running an ensemble prediction method such as SuperLearner or Bayesian model averaging on 10 cross-validation folds, with many statistical/machine learning methods.\nGeneral scenario: parallelizing nested tasks or a large number of tasks, ideally across multiple machines.\nHere you want to take advantage of all the cores you have available, so you can’t just parallelize over folds.\nFirst we’ll discuss how to deal with the nestedness of the problem and then we’ll talk about how to make use of many cores across multiple nodes to parallelize over a large number of tasks.\n\nScenario 5A: nested parallelization\nOne can always flatten the looping, either in a for loop or in similar ways when using apply-style statements.\n\n## original code: multiple loops \nfor fold in range(n):\n for method in range(M):\n ### code here \n \n## revised code: flatten the loops \nfor idx in range(n*M): \n fold = idx // M \n method = idx % M \n print(idx, fold, method)### code here \n\nRather than flattening the loops at the loop level (which you’d need to do to use map), one could just generate a list of delayed tasks within the nested loops.\n\nfor fold in range(n):\n for method in range(M):\n tasks.append(dask.delayed(myfun)(fold,method))\n\nThe future package in R has some nice functionality for easily parallelizing with nested loops.\n\n\nScenario 5B: Parallelizing across multiple nodes\nIf you have access to multiple machines networked together, including a Linux cluster, you can use Dask to start workers across multiple nodes (either in a nested parallelization situation with many total tasks or just when you have lots of unnested tasks to parallelize over). Here we’ll just illustrate how to use multiple nodes, but if you had a nested parallelization case you can combine the ideas just above with the use of multiple nodes.\nSimply start Python as you usually would. Then the following code will parallelize on workers across the machines specified.\n\nfrom dask.distributed import Client, SSHCluster\n# First host is the scheduler.\ncluster = SSHCluster(\n [\"gandalf.berkeley.edu\", \"radagast.berkeley.edu\", \"radagast.berkeley.edu\",\n \"arwen.berkeley.edu\", \"arwen.berkeley.edu\"]\n)\nc = Client(cluster)\n\n## On the SCF, Savio and other clusters using the SLURM scheduler,\n## you can figure out the machine names like this, repeating the\n## first machine for the scheduler:\n## \n## machines = subprocess.check_output(\"srun hostname\", shell = True,\n## universal_newlines = True).strip().split('\\n')\n## machines = [machines[0]] + machines\n\ndef fun(i, n=10**6):\n return np.mean(np.random.normal(size = n))\n\nn_tasks = 120\n\ntasks = c.map(fun, range(n_tasks))\nresults = c.gather(tasks)\n\n## And just to check we are actually using the various machines:\nimport subprocess\n\nc.gather(c.map(lambda x: subprocess.check_output(\"hostname\", shell = True), \\\n range(4)))\n\ncluster.close()" }, { - "objectID": "syllabus.html#objectives-of-the-course", - "href": "syllabus.html#objectives-of-the-course", - "title": "Statistics 243 Fall 2023", - "section": "Objectives of the course", - "text": "Objectives of the course\nThe goals of the course are that, by the end of the course, students be able to:\n\noperate effectively in a UNIX environment and on remote servers and compute clusters;\nhave a solid understanding of general programming concepts and principles, and be able to program effectively (including having an advanced knowledge of Python functionality);\nbe familiar with concepts and tools for reproducible research and good scientific computing practices; and\nunderstand in depth and be able to make use of principles of numerical linear algebra, optimization, and simulation for statistics- and data science-related analyses and research." + "objectID": "units/unit6-parallel.html#scenario-6-stratified-analysis-on-a-very-large-dataset", + "href": "units/unit6-parallel.html#scenario-6-stratified-analysis-on-a-very-large-dataset", + "title": "Parallel processing", + "section": "Scenario 6: Stratified analysis on a very large dataset", + "text": "Scenario 6: Stratified analysis on a very large dataset\nSpecific scenario: You are doing stratified analysis on a very large dataset and want to avoid unnecessary copies.\nGeneral scenario: Avoiding copies when working with large data in parallel.\nIn many parallelization tools, if you try to parallelize this case on a single node, you end up making copies of the original dataset, which both takes up time and eats up memory.\nHere when we use the processes scheduler, we make copies for each task.\n\ndef do_analysis(i,x):\n # A fake \"analysis\", identical for each task.\n print(id(x)) # Check the number of copies.\n return np.mean(x)\n\nn_cores = 4\n\nx = np.random.normal(size = 5*10**7) # our big \"dataset\"\n\ndask.config.set(scheduler='processes', num_workers = n_cores, chunksize = 1)\n\n<dask.config.set object at 0x7fcd2310e810>\n\ntasks = []\np = 8\nfor i in range(p):\n tasks.append(dask.delayed(do_analysis)(i,x))\n\nt0 = time.time()\nresults = dask.compute(tasks)\nprint(time.time() - t0)\n\n6.334888458251953\n\n\nA better approach is to use the distributed scheduler (which is fine to use on a single machine or multiple machines), which makes one copy per worker instead of one per task, provided you apply delayed() to the global data object.\n\nfrom dask.distributed import Client, LocalCluster\ncluster = LocalCluster(n_workers = n_cores)\nc = Client(cluster)\n\nx = dask.delayed(x)\n\ntasks = []\np = 8\nfor i in range(p):\n tasks.append(dask.delayed(do_analysis)(i,x))\n\nt0 = time.time()\nresults = dask.compute(tasks)\n\n/system/linux/mambaforge-3.11/lib/python3.11/site-packages/distributed/client.py:3141: UserWarning: Sending large graph of size 47.69 MiB.\nThis may cause some slowdown.\nConsider scattering data ahead of time and using futures.\n warnings.warn(\n\nprint(time.time() - t0)\n\n1.1176700592041016\n\ncluster.close()\n\nThat seems to work, though Dask suggests sending the data to the workers in advance. I’m not sure of the distinction between what it is recommending and use of dask.delayed(x).\nFurthermore, the id of x seems to be the same for all the tasks, even though we have four workers. The documentation indicates one copy per worker, but perhaps that’s not actually the case.\nEven better would be to use the threads scheduler, in which case all workers can access the same data objects with no copying (but of course we cannot modify the data in that case without potentially causing problems for the other tasks). Without the copying, this is really fast.\n\ndask.config.set(scheduler='threads', num_workers = n_cores)\n\n<dask.config.set object at 0x7fcd2311e810>\n\ntasks = []\np = 8\nfor i in range(p):\n tasks.append(dask.delayed(do_analysis)(i,x))\n\nt0 = time.time()\nresults = dask.compute(tasks)\n\n140519032960464\n140519032960464\n140519032960464\n140519032960464\n140519032960464\n140519032960464\n140519032960464\n140519032960464\n\nprint(time.time() - t0)\n\n0.06301546096801758" }, { - "objectID": "syllabus.html#topics-in-order-with-rough-timing", - "href": "syllabus.html#topics-in-order-with-rough-timing", - "title": "Statistics 243 Fall 2023", - "section": "Topics (in order with rough timing)", - "text": "Topics (in order with rough timing)\nThe ‘days’ here are (roughly) class sessions, as general guidance.\n\nIntroduction to UNIX, operating on a compute server (1 day)\nData formats, data access, webscraping, data structures (2 days)\nDebugging, good programming practices, reproducible research (1 day)\nThe bash shell and shell scripting, version control (3 days)\nProgramming concepts and advanced Python programming: text processing and regular expressions, object-oriented programming, functions and variable scope, memory use, efficient programming (9 days)\nParallel processing (2 days)\nWorking with databases, hashing, and big data (3 days)\nComputer arithmetic/representation of numbers on a computer (3 days)\nSimulation studies and Monte Carlo (2 days)\nNumerical linear algebra (5 days)\nOptimization (5 days)\nGraphics (1 day)" + "objectID": "units/unit6-parallel.html#scenario-7-simulation-study-with-n1000-replicates-parallel-random-number-generation", + "href": "units/unit6-parallel.html#scenario-7-simulation-study-with-n1000-replicates-parallel-random-number-generation", + "title": "Parallel processing", + "section": "Scenario 7: Simulation study with n=1000 replicates: parallel random number generation", + "text": "Scenario 7: Simulation study with n=1000 replicates: parallel random number generation\nWe’ll probably skip this for now and come back to it when we discuss random number generation in the Simulation Unit.\nThe key thing when thinking about random numbers in a parallel context is that you want to avoid having the same ‘random’ numbers occur on multiple processes. On a computer, random numbers are not actually random but are generated as a sequence of pseudo-random numbers designed to mimic true random numbers. The sequence is finite (but very long) and eventually repeats itself. When one sets a seed, one is choosing a position in that sequence to start from. Subsequent random numbers are based on that subsequence. All random numbers can be generated from one or more random uniform numbers, so we can just think about a sequence of values between 0 and 1.\nSpecific scenario: You are running a simulation study with n=1000 replicates.\nGeneral scenario: Safely handling random number generation in parallel.\nEach replicate involves fitting two statistical/machine learning methods.\nHere, unless you really have access to multiple hundreds of cores, you might as well just parallelize across replicates.\nHowever, you need to think about random number generation. One option is to set the random number seed to different values for each replicate. One danger in setting the seed like that is that the random numbers in the different replicate could overlap somewhat. This is probably somewhat unlikely if you are not generating a huge number of random numbers, but it’s unclear how safe it is.\nWe can use functionality with numpy’s PCG64 or MT19937 generators to be completely safe in our parallel random number generation. Each provide a jumped() function that moves the RNG ahead as if one had generated a very large number of random variables (\\(2^{128}\\)) for the Mersenne Twister and nearly that for the PCG64).\nHere’s how we can set up the use of the PCG64 generator:\n\nbitGen = np.random.PCG64(1)\nrng = np.random.Generator(bitGen)\nrng.random(size = 3)\n\narray([0.51182162, 0.9504637 , 0.14415961])\n\n\nNow let’s see how to jump forward. And then verify that jumping forward two increments is the same as making two separate jumps.\n\nbitGen = np.random.PCG64(1)\nbitGen = bitGen.jumped(1)\nrng = np.random.Generator(bitGen)\nrng.normal(size = 3)\n\narray([ 1.23362391, 0.42793616, -1.90447637])\n\nbitGen = np.random.PCG64(1)\nbitGen = bitGen.jumped(2)\nrng = np.random.Generator(bitGen)\nrng.normal(size = 3)\n\narray([-0.31752967, 1.22269493, 0.28254622])\n\nbitGen = np.random.PCG64(1)\nbitGen = bitGen.jumped(1)\nbitGen = bitGen.jumped(1)\nrng = np.random.Generator(bitGen)\nrng.normal(size = 3)\n\narray([-0.31752967, 1.22269493, 0.28254622])\n\n\nWe can also use jumped() with the Mersenne Twister.\n\nbitGen = np.random.MT19937(1)\nbitGen = bitGen.jumped(1)\nrng = np.random.Generator(bitGen)\nrng.normal(size = 3)\n\narray([ 0.12667829, -2.1031878 , -1.53950735])\n\n\nSo the strategy to parallelize across tasks (or potentially workers if random number generation is done sequentially for tasks done by a single worker) is to give each task the same seed and use jumped(i) where i indexes the tasks (or workers).\n\ndef myrandomfun(i):\n bitGen = np.random.PCG(1)\n bitGen = bitGen.jumped(i)\n # insert code with random number generation\n\nOne caution is that it appears that the period for PCG64 is \\(2^{128}\\) and that jumped(1) jumps forward by nearly that many random numbers. That seems quite strange, and I don’t understand it.\nAlternatively as recommended in the docs:\n\nn_tasks = 10\nsg = np.random.SeedSequence(1)\nrngs = [Generator(PCG64(s)) for s in sg.spawn(n_tasks)]\n## Now pass elements of rng into your function that is being computed in parallel\n\ndef myrandomfun(rng):\n # insert code with random number generation, such as:\n z = rng.normal(size = 5)\n\nIn R, the rlecuyer package deals with this. The L’Ecuyer algorithm has a period of \\(2^{191}\\), which it divides into subsequences of length \\(2^{127}\\)." }, - { - "objectID": "syllabus.html#personnel", - "href": "syllabus.html#personnel", - "title": "Statistics 243 Fall 2023", - "section": "Personnel", - "text": "Personnel\n\nInstructor:\n\nChris Paciorek (paciorek@stat.berkeley.edu)\n\nGSI\n\nAhmed Eldeeb (Deeb) (deeb@berkeley.edu)\n\nOffice hours can be found here.\nWhen to see us about an assignment: We’re here to help, including providing guidance on assignments. You don’t want to be futilely spinning your wheels for a long time getting nowhere. That said, before coming to see us about a difficulty, you should try something a few different ways and define/summarize for yourself what is going wrong or where you are getting stuck." + { + "objectID": "units/unit6-parallel.html#avoiding-repeated-calculations-by-calling-compute-once", + "href": "units/unit6-parallel.html#avoiding-repeated-calculations-by-calling-compute-once", + "title": "Parallel processing", + "section": "Avoiding repeated calculations by calling compute once", + "text": "Avoiding repeated calculations by calling compute once\nAs far as I can tell, Dask avoids keeping all the pieces of a distributed object or computation in memory. However, in many cases this can mean repeating computations or re-reading data if you need to do multiple operations on a dataset.\nFor example, if you are create a Dask distributed dataset from data on disk, I think this means that every distinct set of computations (each computational graph) will involve reading the data from disk again.\nOne implication is that if you can include all computations on a large dataset within a single computational graph (i.e., a call to compute) that may be much more efficient than making separate calls.\nHere’s an example with Dask dataframe on some air traffic delay data, where we make sure to do all our computations as part of one graph:\n\nimport dask\ndask.config.set(scheduler='processes', num_workers = 6) \nimport dask.dataframe as ddf\nair = ddf.read_csv('/scratch/users/paciorek/243/AirlineData/csvs/*.csv.bz2',\n compression = 'bz2',\n encoding = 'latin1', # (unexpected) latin1 value(s) 2001 file TailNum field\n dtype = {'Distance': 'float64', 'CRSElapsedTime': 'float64',\n 'TailNum': 'object', 'CancellationCode': 'object'})\n# specify dtypes so Pandas doesn't complain about column type heterogeneity\n\nimport time\nt0 = time.time()\nair.DepDelay.min().compute() # about 200 seconds.\nprint(time.time()-t0)\nt0 = time.time()\nair.DepDelay.max().compute() # about 200 seconds.\nprint(time.time()-t0)\nt0 = time.time()\n(mn, mx) = dask.compute(air.DepDelay.max(), air.DepDelay.min()) # about 200 seconds\nprint(time.time()-t0)" }, { - "objectID": "syllabus.html#course-websites-github-piazza-gradescope-and-bcourses", - "href": "syllabus.html#course-websites-github-piazza-gradescope-and-bcourses", - "title": "Statistics 243 Fall 2023", - "section": "Course websites: GitHub, Ed Discussion, Gradescope, and bCourses", - "text": "Course websites: GitHub, Ed Discussion, Gradescope, and bCourses\nKey websites for the course are:\n\nThis course website, which is hosted on GitHub pages, and the GitHub repository containing the source materials: https://github.com/berkeley-stat243/stat243-fall-2023\nSCF tutorials for additional content: https://statistics.berkeley.edu/computing/training/tutorials\nEd Discussion site for discussions/Q&A: https://edstem.org/us/courses/42474/discussion/\nbCourses site for course capture recordings (see Media Gallery) and possibly some other materials: https://bcourses.berkeley.edu/courses/1527498.\nGradescope for assignments (also linked from bCourses): https://www.gradescope.com/courses/569739\n\nAll course materials will be posted on here on the website (and on GitHub) except for video content, which will be in bCourses.\n\nCourse discussion\nWe will use the course Ed Discussion site for communication (announcements, questions, and discussion). You should ask questions about class material and problem sets through the site. Please use this site for your questions so that either James or I can respond and so that everyone can benefit from the discussion. I suggest you to modify your settings on Ed Discussion so you are informed by email of postings. In particular you are responsible for keeping track of all course announcements, which we’ll make on the Discussion forum. I strongly encourage you to respond to or comment on each other’s questions as well (this will help your class participation grade), although of course you should not provide a solution to a problem set problem. If you have a specific administrative question you need to direct just to me, it’s fine to email me directly or post privately on the Discussion site. But if you simply want to privately ask a question about content, then just come to an office hour or see me after class or James during/after section.\nIf you’re enrolled in the class you should be a member of the group and be able to access it. If you’re auditing or not yet enrolled and would like access, make sure to fill out the course survey and I will add you. In addition, we will use Gradescope for viewing grades." + "objectID": "units/unit6-parallel.html#setting-the-number-of-threads-cores-used-in-threaded-code-including-parallel-linear-algebra-in-python-and-r", + "href": "units/unit6-parallel.html#setting-the-number-of-threads-cores-used-in-threaded-code-including-parallel-linear-algebra-in-python-and-r", + "title": "Parallel processing", + "section": "Setting the number of threads (cores used) in threaded code (including parallel linear algebra in Python and R)", + "text": "Setting the number of threads (cores used) in threaded code (including parallel linear algebra in Python and R)\nIn general, threaded code will detect the number of cores available on a machine and make use of them. However, you can also explicitly control the number of threads available to a process.\nFor most threaded code (that based on the openMP protocol), the number of threads can be set by setting the OMP_NUM_THREADS environment variable (VECLIB_MAXIMUM_THREADS on a Mac). E.g., to set it for four threads in the bash shell:\n\nexport OMP_NUM_THREADS=4\n\nDo this before starting your R or Python session or before running your compiled executable.\nAlternatively, you can set OMP_NUM_THREADS as you invoke your job, e.g., here with R:\n\nOMP_NUM_THREADS=4 R CMD BATCH --no-save job.R job.out\n\n\nSpeed and threaded BLAS\nIn many cases, using multiple threads for linear algebra operations will outperform using a single thread, but there is no guarantee that this will be the case, in particular for operations with small matrices and vectors. You can compare speeds by setting OMP_NUM_THREADS to different values. In cases where threaded linear algebra is slower than unthreaded, you would want to set OMP_NUM_THREADS to 1.\nMore generally, if you are using the parallel tools in Section 4 to simultaneously carry out many independent calculations (tasks), it is likely to be more effective to use the fixed number of cores available on your machine so as to split up the tasks, one per core, without taking advantage of the threaded BLAS (i.e., restricting each process to a single thread)." }, { - "objectID": "syllabus.html#course-material", - "href": "syllabus.html#course-material", - "title": "Statistics 243 Fall 2023", - "section": "Course material", - "text": "Course material\n\nPrimary materials: Course notes on course webpage/GitHub, SCF tutorials, and potentially pre-recorded videos on bCourses.\nBack-up textbooks (generally available via UC Library via links below):\n\nFor bash: Newham, Cameron and Rosenblatt, Bill. Learning the bash Shell available electronically through UC Library\nFor Quarto: The Quarto reference guide\nFor statistical computing topics:\n\nGentle, James. Computational Statistics\nGentle, James. Matrix Algebra or Numerical Linear Algebra with Applications in Statistics\n\nOther resources with more detail on particular aspects of statistical computing concepts:\n\nLange, Kenneth; Numerical Analysis for Statisticians, 2nd ed. First edition available through UC library\nMonahan, John; Numerical Methods of Statistics" + "objectID": "units/unit6-parallel.html#overview-futures-and-the-r-future-package", + "href": "units/unit6-parallel.html#overview-futures-and-the-r-future-package", + "title": "Parallel processing", + "section": "Overview: Futures and the R future package", + "text": "Overview: Futures and the R future package\nWhat is a future? It’s basically a flag used to tag a given operation such that when and where that operation is carried out is controlled at a higher level. If there are multiple operations tagged then this allows for parallelization across those operations.\nAccording to Henrik Bengtsson (the future package developer) and those who developed the concept:\n\na future is an abstraction for a value that will be available later\nthe value is the result of an evaluated expression\nthe state of a future is either unresolved or resolved\n\nWhy use futures? The future package allows one to write one’s computational code without hard-coding whether or how parallelization would be done. Instead one writes the code in a generic way and at the beginning of one’s code sets the ‘plan’ for how the parallel computation should be done given the computational resources available. Simply changing the ‘plan’ changes how parallelization is done for any given run of the code.\nMore concisely, the key ideas are:\n\nSeparate what to parallelize from how and where the parallelization is actually carried out.\nDifferent users can run the same code on different computational resources (without touching the actual code that does the computation)." }, { - "objectID": "syllabus.html#section", - "href": "syllabus.html#section", - "title": "Statistics 243 Fall 2023", - "section": "Section", - "text": "Section\nThe GSI will lead a two-hour discussion section each week (there are two sections). By and large, these will only last for about one hour of actual content, but the second hour may be used as an office hour with the GSI or for troubleshooting software during the early weeks. The discussion sections will vary in format and topic, but material will include demonstrations on various topics (version control, debugging, testing, etc.), group work on these topics, discussion of relevant papers, and discussion of problem set solutions. The first section (1-3 pm) generally has more demand, so to avoid having too many people in the room, you should go to your assigned section unless you talk to me first." + "objectID": "units/unit6-parallel.html#overview-of-parallel-backends-1", + "href": "units/unit6-parallel.html#overview-of-parallel-backends-1", + "title": "Parallel processing", + "section": "Overview of parallel backends", + "text": "Overview of parallel backends\nOne uses plan() to control how parallelization is done, including what machine(s) to use and how many cores on each machine to use.\nFor example,\n\nplan(multiprocess)\n## spreads work across multiple cores\n# alternatively, one can also control number of workers\nplan(multiprocess, workers = 4)\n\nThis table gives an overview of the different plans.\n\n\n\n\n\n\n\n\n\nType\nDescription\nMulti-node\nCopies of objects made?\n\n\n\n\nmultisession\nuses additional R sessions as the workers\nno\nyes\n\n\nmulticore\nuses forked R processes as the workers\nno\nnot if object not modified\n\n\ncluster\nuses R sessions on other machine(s)\nyes\nyes" }, { - "objectID": "syllabus.html#computing-resources", - "href": "syllabus.html#computing-resources", - "title": "Statistics 243 Fall 2023", - "section": "Computing Resources", - "text": "Computing Resources\nMost work for the course can be done on your laptop. Later in the course we’ll also use the Statistics Department Linux cluster. You can also use the SCF JupyterHub or the campus DataHub to access a bash shell or run an IPython notebook.\nThe software needed for the course is as follows:\n\nAccess to the UNIX command line (bash shell)\nGit\nPython (Anaconda/Miniconda is recommended but by no means required)\n\nWe have some tips for software installation (and access to DataHub), including suggestions for how to access a UNIX shell, which you’ll need to be able to do by the second week of class." + "objectID": "units/unit6-parallel.html#accessing-variables-and-workers-in-the-worker-processes-1", + "href": "units/unit6-parallel.html#accessing-variables-and-workers-in-the-worker-processes-1", + "title": "Parallel processing", + "section": "Accessing variables and workers in the worker processes", + "text": "Accessing variables and workers in the worker processes\nThe future package usually does a good job of identifying the packages and (global) variables you use in your parallelized code and loading those packages on the workers and copying necessary variables to the workers. It uses the globals package to do this.\nHere’s a toy example that shows that n and MASS::geyser are automatically available in the worker processes.\n\nlibrary(future)\nlibrary(future.apply)\n\nplan(multisession)\n\nlibrary(MASS)\nn <- nrow(geyser)\n\nmyfun <- function(idx) {\n # geyser is in MASS package\n return(sum(geyser$duration) / n)\n}\n\nfuture_sapply(1:5, myfun)\n\n[1] 3.460814 3.460814 3.460814 3.460814 3.460814\n\n\nIn other contexts in R (or other languages) you may need to explicitly copy objects to the workers (or load packages on the workers). This is sometimes called exporting variables." }, { - "objectID": "syllabus.html#class-time", - "href": "syllabus.html#class-time", - "title": "Statistics 243 Fall 2023", - "section": "Class time", - "text": "Class time\nMy goal is to have classes be an interactive environment. This is both more interesting for all of us and more effective in learning the material. I encourage you to ask questions and will pose questions to the class to think about, respond to via online polling or Google forms, and discuss. To increase time for discussion and assimilation of the material in class, before some classes I may ask that you read material or work through tutorials in advance of class. Occasionally, I will ask you to submit answers to questions in advance of class as well.\nPlease do not use phones during class and limit laptop use to the material being covered.\nStudent backgrounds with computing will vary. For those of you with limited background on a topic, I encourage you to ask questions during class so I know what you find confusing. For those of you with extensive background on a topic (there will invariably be some topics where one of you will know more about it than I do or have more real-world experience), I encourage you to pitch in with your perspective. In general, there are many ways to do things on a computer, particularly in a UNIX environment and in Python, so it will help everyone (including me) if we hear multiple perspectives/ideas.\nFinally, class recordings for review or to make up for absence will be available through the bCourses Media Gallery, available on the Media Gallery tab on the bCourses page for the class." + "objectID": "units/unit8-numbers.html", + "href": "units/unit8-numbers.html", + "title": "Numbers on a computer", + "section": "", + "text": "PDF\nReferences:\nA quick note that, as we’ve already seen, Python’s version of scientific notation is XeY, which means \\(X\\cdot10^{Y}\\).\nA second note is that the concepts developed here apply outside of Python, but we’ll illustrate the principles of computer numbers using Python. Python usually makes use of the double type (8 bytes) in C for the underlying representation of real-valued numbers in C variables, so what we’ll really be seeing is how such types behave in C on most modern machines. It’s actually a bit more complicated in that one can use real-valued numbers that use something other than 8 bytes in numpy by specifying a dtype.\nThe handling of integers is even more complicated. In numpy, the default is 8 byte integers, but other integer dtypes are available. And in Python itself, integers can be arbitrarily large." }, { - "objectID": "syllabus.html#course-requirements-and-grading", - "href": "syllabus.html#course-requirements-and-grading", - "title": "Statistics 243 Fall 2023", - "section": "Course requirements and grading", - "text": "Course requirements and grading\n\nScheduling Conflicts\nCampus asks that I include this information about conflicts: Please notify me in writing by the second week of the term about any known or potential extracurricular conflicts (such as religious observances, graduate or medical school interviews, or team activities). I will try my best to help you with making accommodations, but I cannot promise them in all cases. In the event there is no mutually-workable solution, you may be dropped from the class.\nThe main conflict that would be a problem would be the quizzes, whose dates I will determine in late August / early September.\nQuizzes are in-person. There is no remote option, and the only make-up accommodations I will make are for illness or serious personal issues. Do not schedule any travel that may conflict with a quiz.\n\n\nCourse grades\nThe grade for this course is primarily based on assignments due every 1-2 weeks, two quizzes (likely in early-mid October and mid-late November), and a final group project. I will also provide extra credit questions on some problem sets. There is no final exam. 50% of the grade is based on the problem sets, 25% on the quizzes, 15% on the project, and 10% on your participation in discussions on Ed, your responses to the in-class Google forms questions, as well as occasional brief questions that I will ask you to answer in advance of the next class.\nGrades will generally be As and Bs. An A involves doing all the work, getting full credit on most of the problem sets, doing well on the quizzes, and doing a thorough job on the final project.\n\n\nProblem sets\nWe will be less willing to help you if you come to our office hours or post a question online at the last minute. Working with computers can be unpredictable, so give yourself plenty of time for the assignments.\nThere are several rules for submitting your assignments.\n\nYou should prepare your assignments using Quarto.\nProblem set submission consists of both of the following:\n\nA PDF submitted electronically through Gradescope, by the start of class (10 am) on the due date, and\nAn electronic copy of the PDF, code file, and Quarto document pushed to your class GitHub repository, following the instructions to be provided by the GSI.\n\nOn-time submission will be determined based on the time stamp of when the PDF is submitted to Gradescope.\nAnswers should consist of textual response or mathematical expressions as appropriate, with key chunks of code embedded within the document. Extensive additional code can be provided as an appendix. Before diving into the code for a problem, you should say what the goal of the code is and your strategy for solving the problem. Raw code without explanation is not an appropriate solution. Please see our qualitative grading rubric for guidance. In general the rubric is meant to reinforce good coding practices and high-quality scientific communication.\nAny mathematical derivations may be done by hand and scanned with your phone if you prefer that to writing up LaTeX equations.\n\nNote: Quarto Markdown is an extension to the Markdown markup language that allows one to embed Python and R code within an HTML document. Please see the SCF dynamics document tutorial; there will be additional information in the first section and on the first problem set.\n\n\nSubmitting assignments\nIn the first section (September 1), we’ll discuss how to submit your problem sets both on Gradescope and via your class GitHub repository, located at https://github.berkeley.edu/<your_calnet_username>.\n\n\nProblem set grading\nThe grading scheme for problem sets is as follows. Each problem set will receive a numeric score for (1) presentation and explanation of results, (2) technical accuracy of code or mathematical derivation, and (3) code quality/style and creativity. For each of these three components, the possible scores are:\n\n0 = no credit,\n1 = partial credit (you did some of the problems but not all),\n2 = satisfactory (you tried everything but there were pieces of what you did that didn’t solve or present/explain one or more problems in a complete way), and\n3 = full credit.\n\nAgain, the qualitative grading rubric provides guidance on what we want to see for full credit.\nFor components #1 and #3, many of you will get a score of 2 for some problem sets as you develop good coding practices. You can still get an A in the class despite this.\nYour total score for the PS is a weighted sum of the scores for the three components. If you turn in a PS late, I’ll bump you down by two points (out of the available). If you turn it in really late (e.g., after we start grading them), I will bump you down by four points. No credit after solutions are distributed.\n\n\nFinal project\nThe final project will be a joint coding project in groups of 3-4. I’ll assign an overall task, and you’ll be responsible for dividing up the work, coding, debugging, testing, and documentation. You’ll need to use the Git version control system for working in your group.\n\n\nRules for working together and the campus honor code\nI encourage you to work together and help each other out. However, the problem set solutions you submit must be your own. What do I mean by that?\n\nYou should first try to figure out a given problem on your own. After that, if you’re stuck or want to explore alternative approaches or check what you’ve done, feel free to consult with your fellow students and with the GSI and me.\nWhat does “consult with a fellow student mean”? You can discuss a problem with another student, brainstorm approaches, and share code syntax (generally not more than one line) on how to do small individual coding tasks within a problem.\n\nYou should not ask another student for complete code or solutions, or look at their code/solution.\nYou should not share complete code or solutions with another student or on Ed Discussion.\n\nYou may use ChatGPT (or similar chatbots) for help with small sections of a problem (e.g., how to do some specific Python or bash task). You should not use ChatGPT to try to answer an entire question. You should carefully verify that the result is correct.\nYou must provide attribution for ideas obtained elsewhere, including other students and ChatGPT or similar chatbots.\n\nIf you got a specific idea for how to do part of a problem from a fellow student (or some other resource, including ChatGPT), you should note that in your solution in the appropriate place (for specific syntax ideas, note this in a code comment), just as you would cite a book or URL.\nYou MUST note on your problem set solution any fellow students who you worked/consulted with.\nYou do not need to cite any Ed Discussion posts nor any discussions with Chris or Deeb.\n\nUltimately, your solution to a problem set (writeup and code) must be your own, and you’ll hear from me if either look too similar to someone else’s.\n\nPlease see the last section of this document for more information on the Campus Honor Code, which I expect you to follow." + "objectID": "units/unit8-numbers.html#representing-real-numbers", + "href": "units/unit8-numbers.html#representing-real-numbers", + "title": "Numbers on a computer", + "section": "Representing real numbers", + "text": "Representing real numbers\n\nInitial exploration\nReals (also called floating points) are stored on the computer as an approximation, albeit a very precise approximation. As an example, if we represent the distance from the earth to the sun using a double, the error is around a millimeter. However, we need to be very careful if we’re trying to do a calculation that produces a very small (or very large number) and particularly when we want to see if numbers are equal to each other.\nIf you run the code here, the results may surprise you.\n\n0.3 - 0.2 == 0.1\n0.3\n0.2\n0.1 # Hmmm...\n\nnp.float64(0.3) - np.float64(0.2) == np.float64(0.1)\n\n0.75 - 0.5 == 0.25\n0.6 - 0.4 == 0.2\n## any ideas what is different about those two comparisons?\n\nNext, let’s consider the number of digits of accuracy we have for a variety of numbers. We’ll use format within a handy wrapper function, dg, defined earlier, to view as many digits as we want:\n\na = 0.3\nb = 0.2\ndg(a)\n\n0.29999999999999998890\n\ndg(b)\n\n0.20000000000000001110\n\ndg(a-b)\n\n0.09999999999999997780\n\ndg(0.1)\n\n0.10000000000000000555\n\ndg(1/3)\n\n0.33333333333333331483\n\n\nSo empirically, it looks like we’re accurate up to the 16th decimal place\nBut actually, the key is the number of digits, not decimal places.\n\ndg(1234.1234)\n\n1234.12339999999994688551\n\ndg(1234.123412341234)\n\n1234.12341234123391586763\n\n\nNotice that we can represent the result accurately only up to 16 significant digits. This suggests no need to show more than 16 significant digits and no need to print out any more when writing to a file (except that if the number is bigger than \\(10^{16}\\) then we need extra digits to correctly show the magnitude of the number if not using scientific notation). And of course, often we don’t need anywhere near that many.\nLet’s return to our comparison, 0.75-0.5 == 0.25.\n\ndg(0.75)\n\n0.75000000000000000000\n\ndg(0.50)\n\n0.50000000000000000000\n\n\nWhat’s different about the numbers 0.75 and 0.5 compared to 0.3, 0.2, 0.1?\n\n\nMachine epsilon\nMachine epsilon is the term used for indicating the (relative) accuracy of real numbers and it is defined as the smallest float, \\(x\\), such that \\(1+x\\ne1\\):\n\n1e-16 + 1.0\n\n1.0\n\nnp.array(1e-16) + np.array(1.0)\n\n1.0\n\n1e-15 + 1.0\n\n1.000000000000001\n\nnp.array(1e-15) + np.array(1.0)\n\n1.000000000000001\n\n2e-16 + 1.0\n\n1.0000000000000002\n\nnp.finfo(np.float64).eps\n\n2.220446049250313e-16\n\ndg(2e-16 + 1.0)\n\n1.00000000000000022204\n\n\n\n## What about in single precision, e.g. on a GPU?\nnp.finfo(np.float32).eps\n\n1.1920929e-07\n\n\n\n\nFloating point representation\nFloating point refers to the decimal point (or radix point since we’ll be working with base 2 and decimal relates to 10).\nTo proceed further we need to consider scientific notation, such as in writing Avogadro’s number as \\(+6.023\\times10^{23}\\). As a baseline for what is about to follow note that we can express a decimal number in the following expansion \\[6.037=6\\times10^{0}+0\\times10^{-1}+3\\times10^{-2}+7\\times10^{-3}\\] A real number on a computer is stored in what is basically scientific notation: \\[\\pm d_{0}.d_{1}d_{2}\\ldots d_{p}\\times b^{e}\\label{eq:floatRep}\\] where \\(b\\) is the base, \\(e\\) is an integer and \\(d_{i}\\in\\{0,\\ldots,b-1\\}\\). \\(e\\) is called the exponent and \\(d=d_{1}d_{2}\\ldots d_{p}\\) is called the mantissa.\nLet’s consider the choices that the computer pioneers needed to make in using this system to represent numbers on a computer using base 2 (\\(b=2\\)). First, we need to choose the number of bits to represent \\(e\\) so that we can represent sufficiently large and small numbers. Second we need to choose the number of bits, \\(p\\), to allocate to \\(d=d_{1}d_{2}\\ldots d_{p}\\), which determines the accuracy of any computer representation of a real.\nThe great thing about floating points is that we can represent numbers that range from incredibly small to very large while maintaining good precision. The floating point floats to adjust to the size of the number. Suppose we had only three digits to use and were in base 10. In floating point notation we can express \\(0.12\\times0.12=0.0144\\) as \\((1.20\\times10^{-1})\\times(1.20\\times10^{-1})=1.44\\times10^{-2}\\), but if we had fixed the decimal point, we’d have \\(0.120\\times0.120=0.014\\) and we’d have lost a digit of accuracy. (Furthermore, we wouldn’t be able to represent numbers bigger than \\(0.99\\).)\nMore specifically, the actual storage of a number on a computer these days is generally as a double in the form: \\[(-1)^{S}\\times1.d\\times2^{e-1023}=(-1)^{S}\\times1.d_{1}d_{2}\\ldots d_{52}\\times2^{e-1023}\\] where the computer uses base 2, \\(b=2\\), (so \\(d_{i}\\in\\{0,1\\}\\)) because base-2 arithmetic is faster than base-10 arithmetic. The leading 1 normalizes the number; i.e., ensures there is a unique representation for a given computer number. This avoids representing any number in multiple ways, e.g., either \\(1=1.0\\times2^{0}=0.1\\times2^{1}=0.01\\times2^{2}\\). For a double, we have 8 bytes=64 bits. Consider our representation as (\\(S,d,e\\)) where \\(S\\) is the sign. The leading 1 is the hidden bit and doesn’t need to be stored because it is always present. In general \\(e\\) is represented using 11 bits (\\(2^{11}=2048\\)), and the subtraction takes the place of having a sign bit for the exponent. (Note that in our discussion we’ll just think of \\(e\\) in terms of its base 10 representation, although it is of course represented in base 2.) This leaves \\(p=52 = 64-1-11\\) bits for \\(d\\).\nIn this code I force storage as a double by tacking on a decimal place, .0.\n\nbits(2.0**(-1)) # 1/2\n\n'0011111111100000000000000000000000000000000000000000000000000000'\n\nbits(2.0**0) # 1\n\n'0011111111110000000000000000000000000000000000000000000000000000'\n\nbits(2.0**1) # 2\n\n'0100000000000000000000000000000000000000000000000000000000000000'\n\nbits(2.0**1 + 2.0**0) # 3\n\n'0100000000001000000000000000000000000000000000000000000000000000'\n\nbits(2.0**2) # 4\n\n'0100000000010000000000000000000000000000000000000000000000000000'\n\nbits(-2)\n\n'1100000000000000000000000000000000000000000000000000000000000000'\n\n\nLet’s see that we can manually work out the bit-wise representation of 3.25:\n\nbits(3.25)\n\n'0100000000001010000000000000000000000000000000000000000000000000'\n\n\nQuestion: Given a fixed number of bits for a number, what is the tradeoff between using bits for the \\(d\\) part vs. bits for the \\(e\\) part?\nLet’s consider what can be represented exactly:\n\ndg(.1)\n\n0.10000000000000000555\n\ndg(.5)\n\n0.50000000000000000000\n\ndg(.25)\n\n0.25000000000000000000\n\ndg(.26)\n\n0.26000000000000000888\n\ndg(1/32)\n\n0.03125000000000000000\n\ndg(1/33)\n\n0.03030303030303030387\n\n\nSo why is 0.5 stored exactly and 0.1 not stored exactly? By analogy, consider the difficulty with representing 1/3 in base 10." }, { - "objectID": "syllabus.html#feedback", - "href": "syllabus.html#feedback", - "title": "Statistics 243 Fall 2023", - "section": "Feedback", - "text": "Feedback\nI welcome comments and suggestions and concerns. Particularly good suggestions will count towards your class participation grade." + "objectID": "units/unit8-numbers.html#overflow-and-underflow", + "href": "units/unit8-numbers.html#overflow-and-underflow", + "title": "Numbers on a computer", + "section": "Overflow and underflow", + "text": "Overflow and underflow\nThe largest and smallest numbers we can represent are \\(2^{e_{\\max}}\\) and \\(2^{e_{\\min}}\\) where \\(e_{\\max}\\) and \\(e_{\\min}\\) are the smallest and largest possible values of the exponent. Let’s consider the exponent and what we can infer about the range of possible numbers. With 11 bits for \\(e\\), we can represent \\(\\pm2^{10}=\\pm1024\\) different exponent values (see np.finfo(np.float64).maxexp) (why is np.finfo(np.float64).minexp only -1022?). So the largest number we could represent is \\(2^{1024}\\). What is this in base 10?\n\nx = np.float64(10)\nx**308\n\n1e+308\n\nx**309\n\ninf\n\n<string>:1: RuntimeWarning: overflow encountered in double_scalars\n\nnp.log10(2.0**1024)\n\nError: OverflowError: (34, 'Numerical result out of range')\n\nnp.log10(2.0**1023)\n\n307.95368556425274\n\nnp.finfo(np.float64)\n\nfinfo(resolution=1e-15, min=-1.7976931348623157e+308, max=1.7976931348623157e+308, dtype=float64)\n\n\nWe could have been smarter about that calculation: \\(\\log_{10}2^{1024}=\\log_{2}2^{1024}/\\log_{2}10=1024/3.32\\approx308\\). The result is analogous for the smallest number, so we have that floating points can range between \\(1\\times10^{-308}\\) and \\(1\\times10^{308}\\), consistent with what numpy reports above. Producing something larger or smaller in magnitude than these values is called overflow and underflow respectively.\nLet’s see what happens when we underflow in numpy. Note that there is no warning.\n\nx**(-308)\n\n1e-308\n\nx**(-330)\n\n0.0\n\n\nSomething subtle happens for numbers like \\(10^{-309}\\) through \\(10^{-323}\\). They can actually be represented despite what I said above. Investigating that may be an extra credit problem on a problem set." }, { - "objectID": "syllabus.html#accomodations-for-students-with-disabilities", - "href": "syllabus.html#accomodations-for-students-with-disabilities", - "title": "Statistics 243 Fall 2023", - "section": "Accomodations for Students with Disabilities", - "text": "Accomodations for Students with Disabilities\nPlease see me as soon as possible if you need particular accommodations, and we will work out the necessary arrangements." + "objectID": "units/unit8-numbers.html#integers-or-floats", + "href": "units/unit8-numbers.html#integers-or-floats", + "title": "Numbers on a computer", + "section": "Integers or floats?", + "text": "Integers or floats?\nValues stored as integers should overflow if they exceed the maximum integer.\nShould \\(2^{65}\\) overflow?\n\nnp.log2(np.iinfo(np.int64).max)\n\n63.0\n\nx = np.int64(2)\n# Yikes!\nx**64\n\n0\n\n\nPython’s int type doesn’t overflow.\n\n# Interesting:\nprint(2**64)\n\n18446744073709551616\n\nprint(2**100)\n\n1267650600228229401496703205376\n\n\nOf course, doubles won’t overflow until much larger values than 4- or 8-byte integers because we know they can be as big as \\(10^308\\).\n\nx = np.float64(2)\ndg(x**64, '.2f')\n\n18446744073709551616.00\n\ndg(x**100, '.2f')\n\n1267650600228229401496703205376.00\n\n\nHowever we need to think about what integer-valued numbers can and can’t be stored exactly in our base 2 representation of floating point numbers. It turns out that integer-valued numbers can be stored exactly as doubles when their absolute value is less than \\(2^{53}\\).\n\nChallenge: Why \\(2^{53}\\)? Write out what integers can be stored exactly in our base 2 representation of floating point numbers.\n\nYou can force storage as integers or doubles in a few ways.\n\nx = 3; type(x)\n\n<class 'int'>\n\nx = np.float64(x); type(x)\n\n<class 'numpy.float64'>\n\nx = 3.0; type(x)\n\n<class 'float'>\n\nx = np.float64(3); type(x)\n\n<class 'numpy.float64'>" }, { - "objectID": "syllabus.html#campus-honor-code", - "href": "syllabus.html#campus-honor-code", - "title": "Statistics 243 Fall 2023", - "section": "Campus Honor Code", - "text": "Campus Honor Code\nThe following is the Campus Honor Code. With regard to collaboration and independence, please see my rules regarding problem sets above – Chris.\nThe student community at UC Berkeley has adopted the following Honor Code: “As a member of the UC Berkeley community, I act with honesty, integrity, and respect for others.” The hope and expectation is that you will adhere to this code.\nCollaboration and Independence: Reviewing lecture and reading materials and studying for exams can be enjoyable and enriching things to do with fellow students. This is recommended. However, unless otherwise instructed, homework assignments are to be completed independently and materials submitted as homework should be the result of one’s own independent work.\nCheating: A good lifetime strategy is always to act in such a way that no one would ever imagine that you would even consider cheating. Anyone caught cheating on a quiz or exam in this course will receive a failing grade in the course and will also be reported to the University Center for Student Conduct. In order to guarantee that you are not suspected of cheating, please keep your eyes on your own materials and do not converse with others during the quizzes and exams.\nPlagiarism: To copy text or ideas from another source without appropriate reference is plagiarism and will result in a failing grade for your assignment and usually further disciplinary action. For additional information on plagiarism and how to avoid it, see, for example: http://gsi.berkeley.edu/teachingguide/misconduct/prevent-plag.html\nAcademic Integrity and Ethics: Cheating on exams and plagiarism are two common examples of dishonest, unethical behavior. Honesty and integrity are of great importance in all facets of life. They help to build a sense of self-confidence, and are key to building trust within relationships, whether personal or professional. There is no tolerance for dishonesty in the academic world, for it undermines what we are dedicated to doing – furthering knowledge for the benefit of humanity.\nYour experience as a student at UC Berkeley is hopefully fueled by passion for learning and replete with fulfilling activities. And we also appreciate that being a student may be stressful. There may be times when there is temptation to engage in some kind of cheating in order to improve a grade or otherwise advance your career. This could be as blatant as having someone else sit for you in an exam, or submitting a written assignment that has been copied from another source. And it could be as subtle as glancing at a fellow student’s exam when you are unsure of an answer to a question and are looking for some confirmation. One might do any of these things and potentially not get caught. However, if you cheat, no matter how much you may have learned in this class, you have failed to learn perhaps the most important lesson of all." + "objectID": "units/unit8-numbers.html#precision", + "href": "units/unit8-numbers.html#precision", + "title": "Numbers on a computer", + "section": "Precision", + "text": "Precision\nConsider our representation as (S, d, e) where we have \\(p=52\\) bits for \\(d\\). Since we have \\(2^{52}\\approx0.5\\times10^{16}\\), we can represent about that many discrete values, which means we can accurately represent about 16 digits (in base 10). The result is that floats on a computer are actually discrete (we have a finite number of bits), and if we get a number that is in one of the gaps (there are uncountably many reals), it’s approximated by the nearest discrete value. The accuracy of our representation is to within 1/2 of the gap between the two discrete values bracketing the true number. Let’s consider the implications for accuracy in working with large and small numbers. By changing \\(e\\) we can change the magnitude of a number. So regardless of whether we have a very large or small number, we have about 16 digits of accuracy, since the absolute spacing depends on what value is represented by the least significant digit (the ulp, or unit in the last place) in \\(d\\), i.e., the \\(p=52\\)nd one, or in terms of base 10, the 16th digit. Let’s explore this:\n\n# large vs. small numbers\ndg(.1234123412341234)\n\n0.12341234123412339607\n\ndg(1234.1234123412341234) # not accurate to 16 decimal places \n\n1234.12341234123414324131\n\ndg(123412341234.123412341234) # only accurate to 4 places \n\n123412341234.12341308593750000000\n\ndg(1234123412341234.123412341234) # no places! \n\n1234123412341234.00000000000000000000\n\ndg(12341234123412341234) # fewer than no places! \n\n12341234123412340736.00000000000000000000\n\n\nWe can see the implications of this in the context of calculations:\n\ndg(1234567812345678.0 - 1234567812345677.0)\n\n1.00000000000000000000\n\ndg(12345678123456788888.0 - 12345678123456788887.0)\n\n0.00000000000000000000\n\ndg(12345678123456780000.0 - 12345678123456770000.0)\n\n10240.00000000000000000000\n\n\nThe spacing of possible computer numbers that have a magnitude of about 1 leads us to another definition of machine epsilon (an alternative, but essentially equivalent definition to that given previously. Machine epsilon tells us also about the relative spacing of numbers. First let’s consider numbers of magnitude one. The difference between \\(1=1.00...00\\times2^{0}\\) and \\(1.000...01\\times2^{0}\\) is \\(\\epsilon=1\\times2^{-52}\\approx2.2\\times10^{-16}\\). Machine epsilon gives the absolute spacing for numbers near 1 and the relative spacing for numbers with a different order of magnitude and therefore a different absolute magnitude of the error in representing a real. The relative spacing at \\(x\\) is \\[\\frac{(1+\\epsilon)x-x}{x}=\\epsilon\\] since the next largest number from \\(x\\) is given by \\((1+\\epsilon)x\\).\nSuppose \\(x=1\\times10^{6}\\). Then the absolute error in representing a number of this magnitude is \\(x\\epsilon\\approx2\\times10^{-10}\\). (Actually the error would be one-half of the spacing, but that’s a minor distinction.) We can see by looking at the numbers in decimal form, where we are accurate to the order \\(10^{-10}\\) but not \\(10^{-11}\\). This is equivalent to our discussion that we have only 16 digits of accuracy.\n\ndg(1000000.1)\n\n1000000.09999999997671693563\n\n\nLet’s see what arithmetic we can do exactly with integer-valued numbers stored as doubles and how that relates to the absolute spacing of numbers we’ve just seen:\n\n2.0**52\n\n4503599627370496.0\n\n2.0**52+1\n\n4503599627370497.0\n\n2.0**53\n\n9007199254740992.0\n\n2.0**53+1\n\n9007199254740992.0\n\n2.0**53+2\n\n9007199254740994.0\n\ndg(2.0**54)\n\n18014398509481984.00000000000000000000\n\ndg(2.0**54+2)\n\n18014398509481984.00000000000000000000\n\ndg(2.0**54+4)\n\n18014398509481988.00000000000000000000\n\nbits(2**53)\n\n'0100001101000000000000000000000000000000000000000000000000000000'\n\nbits(2**53+1)\n\n'0100001101000000000000000000000000000000000000000000000000000000'\n\nbits(2**53+2)\n\n'0100001101000000000000000000000000000000000000000000000000000001'\n\nbits(2**54)\n\n'0100001101010000000000000000000000000000000000000000000000000000'\n\nbits(2**54+2)\n\n'0100001101010000000000000000000000000000000000000000000000000000'\n\nbits(2**54+4)\n\n'0100001101010000000000000000000000000000000000000000000000000001'\n\n\nThe absolute spacing is \\(x\\epsilon\\), so we have spacings of \\(2^{52}\\times2^{-52}=1\\), \\(2^{53}\\times2^{-52}=2\\), \\(2^{54}\\times2^{-52}=4\\) for numbers of magnitude \\(2^{52}\\), \\(2^{53}\\), and \\(2^{54}\\), respectively.\nWith a bit more work (e.g., using Mathematica), one can demonstrate that doubles in Python in general are represented as the nearest number that can stored with the 64-bit structure we have discussed and that the spacing is as we have discussed. The results below show the spacing that results, in base 10, for numbers around 0.1. The numbers Python reports are spaced in increments of individual bits in the base 2 representation.\n\ndg(0.1234567812345678)\n\n0.12345678123456779729\n\ndg(0.12345678123456781)\n\n0.12345678123456781117\n\ndg(0.12345678123456782)\n\n0.12345678123456782505\n\ndg(0.12345678123456783)\n\n0.12345678123456782505\n\ndg(0.12345678123456784)\n\n0.12345678123456783892\n\nbits(0.1234567812345678)\n\n'0011111110111111100110101101110100010101110111110011010010000110'\n\nbits(0.12345678123456781)\n\n'0011111110111111100110101101110100010101110111110011010010000111'\n\nbits(0.12345678123456782)\n\n'0011111110111111100110101101110100010101110111110011010010001000'\n\nbits(0.12345678123456783)\n\n'0011111110111111100110101101110100010101110111110011010010001000'\n\nbits(0.12345678123456784)\n\n'0011111110111111100110101101110100010101110111110011010010001001'" }, { - "objectID": "rubric.html", - "href": "rubric.html", - "title": "Statistics 243 Fall 2023", - "section": "", - "text": "This document provides guidance for submitting high-quality problem set (and project) solutions. This guidance is based on general good practices for scientific communication, reproducible research, and software development." + "objectID": "units/unit8-numbers.html#working-with-higher-precision-numbers", + "href": "units/unit8-numbers.html#working-with-higher-precision-numbers", + "title": "Numbers on a computer", + "section": "Working with higher precision numbers", + "text": "Working with higher precision numbers\nAs we’ve seen, Python will automatically work with integers in arbitrary precision. (Note that R does not do this – R uses 4-byte integers, and for many calculations it’s best to use R’s numeric type because integers that aren’t really large can be expressed exactly.)\nFor higher precision floating point numbers you can make use of the gmpy2 package.\n\nimport gmpy2\ngmpy2.get_context().precision=200\ngmpy2.const_pi()\n\n## not sure why this shows ...00004\ngmpy2.mpfr(\".1234567812345678\")" }, { - "objectID": "rubric.html#general-presentation", - "href": "rubric.html#general-presentation", - "title": "Statistics 243 Fall 2023", - "section": "General presentation", - "text": "General presentation\n\nSimply presenting code or derivations is not sufficient.\nBriefly describe the overall goal or strategy before providing code/derivations.\nAs needed describe what the pieces of your code/derivation are doing to make it easier for a reader to follow the steps.\nKeep your output focused on illustrating what you did, without distracting from the flow of your solution by showing voluminous output. The output should illustrate and demonstrate, not overwhelm or obscure. If you need to show longer output, you can add it at the end as supplemental material.\nOutput should be produced by your code (i.e., from the code chunks running when the document is rendered), not by copying and pasting results into the document." + "objectID": "units/unit8-numbers.html#computer-arithmetic-is-not-mathematical-arithmetic", + "href": "units/unit8-numbers.html#computer-arithmetic-is-not-mathematical-arithmetic", + "title": "Numbers on a computer", + "section": "Computer arithmetic is not mathematical arithmetic!", + "text": "Computer arithmetic is not mathematical arithmetic!\nAs mentioned for integers, computer number arithmetic is not closed, unlike real arithmetic. For example, if we multiply two computer floating points, we can overflow and not get back another computer floating point.\nAnother mathematical concept we should consider here is that computer arithmetic does not obey the associative and distributive laws, i.e., \\((a+b)+c\\) may not equal \\(a+(b+c)\\) on a computer and \\(a(b+c)\\) may not be the same as \\(ab+ac\\). Here’s an example with multiplication:\n\nval1 = 1/10; val2 = 0.31; val3 = 0.57\nres1 = val1*val2*val3\nres2 = val3*val2*val1\nres1 == res2\n\nFalse\n\ndg(res1)\n\n0.01766999999999999821\n\ndg(res2)\n\n0.01767000000000000168" }, { - "objectID": "rubric.html#coding-practice", - "href": "rubric.html#coding-practice", - "title": "Statistics 243 Fall 2023", - "section": "Coding practice", - "text": "Coding practice\n\nMinimize (or eliminate) use of global variables.\nBreak down work into core tasks and develop small, modular, self-contained functions (or class methods) to carry out those tasks.\nDon’t repeat code. As needed refactor code to create new functions (or class methods).\nFunctions and classes should be “weakly coupled”, interacting via their interfaces and not by having to know the internals of how they work.\nUse data structures appropriate for the computations that need to be done.\nDon’t hard code ‘magic’ numbers. Assign such numbers to variables with clear names, e.g., speed_of_light = 3e8.\nProvide reasonable default arguments to functions (or class methods) when possible.\nProvide tests (including unit tests) when requested (this is good general practice but we won’t require it in all cases).\nAvoid overly complicated syntax – try to use the clearest syntax you can to solve the problem.\nIn terms of speed, don’t worry about it too much so long as the code finishes real-world tasks in a reasonable amount of time. When optimizing, focus on the parts of the code that are the bottlenecks.\nUse functions already available in the language rather than recreating yourself." + "objectID": "units/unit8-numbers.html#calculating-with-integers-vs.-floating-points", + "href": "units/unit8-numbers.html#calculating-with-integers-vs.-floating-points", + "title": "Numbers on a computer", + "section": "Calculating with integers vs. floating points", + "text": "Calculating with integers vs. floating points\nIt’s important to note that operations with integers are fast and exact (but can easily overflow – albeit not with Python’s base int) while operations with floating points are slower and approximate. Because of this slowness, floating point operations (flops) dominate calculation intensity and are used as the metric for the amount of work being done - a multiplication (or division) combined with an addition (or subtraction) is one flop. We’ll talk a lot about flops in the unit on linear algebra." }, { - "objectID": "rubric.html#code-style", - "href": "rubric.html#code-style", - "title": "Statistics 243 Fall 2023", - "section": "Code style", - "text": "Code style\n\nFollow a consistent style. While you don’t have to follow Python’s PEP8 style guide exactly, please look at it and follow it generally.\nUse informative variable and function names and have a consistent naming style.\nUse whitespace (spaces, newlines) and parentheses to make the structure of the code easy to understand and the individual syntax pieces clear.\nUse consistent indentation to make the structure of the code easy to understand.\nProvide comments that give the goal of a given piece of code and why it does things, but don’t use comments to restate what the code does when it should be obvious from reading the code.\n\nProvide summaries for blocks of code.\nFor particularly complicated syntax, say what a given piece of code does." + "objectID": "units/unit8-numbers.html#comparisons", + "href": "units/unit8-numbers.html#comparisons", + "title": "Numbers on a computer", + "section": "Comparisons", + "text": "Comparisons\nAs we saw, we should never test x == y unless:\n\nx and y are represented as integers,\nthey are integer-valued but stored as doubles that are small enough that they can be stored exactly), or\nthey are decimal numbers that have been created in the same way (e.g., 0.4-0.3 == 0.4-0.3 returns TRUE but 0.1 == 0.4-0.3 does not).\n\nSimilarly we should be careful about testing x == 0. And be careful of greater than/less than comparisons. For example, be careful of x[ x < 0 ] = np.nan if what you are looking for is values that might be mathematically less than zero, rather than whatever is numerically less than zero.\n\n4 - 3 == 1\n\nTrue\n\n4.0 - 3.0 == 1.0\n\nTrue\n\n4.1 - 3.1 == 1.0\n\nFalse\n\n0.4-0.3 == 0.1\n\nFalse\n\n0.4-0.3 == 0.4-0.3\n\nTrue\n\n\nOne nice approach to checking for approximate equality is to make use of machine epsilon. If the relative spacing of two numbers is less than machine epsilon, then for our computer approximation, we say they are the same. Here’s an implementation that relies on the absolute spacing being \\(x\\epsilon\\) (see above).\n\nx = 12345678123456781000\ny = 12345678123456782000\n\ndef approx_equal(a,b):\n if abs(a - b) < np.finfo(np.float64).eps * abs(a + b):\n print(\"approximately equal\")\n else:\n print (\"not equal\")\n\n\napprox_equal(a,b)\n\nnot equal\n\nx = 1234567812345678\ny = 1234567812345677\n\napprox_equal(a,b) \n\nnot equal\n\n\nActually, we probably want to use a number slightly larger than machine epsilon to be safe.\nFinally, sometimes we encounter the use of an unusual integer as a symbol for missing values. E.g., a datafile might store missing values as -9999. Testing for this using == with floats should generally be ok:x [ x == -9999 ] = np.nan, because integers of this magnitude are stored exactly as floating point values. But to be really careful, you can read in as an integer or character type and do the assessment before converting to a float." }, { - "objectID": "howtos/windowsAndLinux.html", - "href": "howtos/windowsAndLinux.html", - "title": "Windows 10 and the Ubuntu Subsystem", - "section": "", - "text": "Windows 10 has a powerful new feature that allows a full Linux system to be installed and run from within Windows. This is incredibly useful for building/testing code in Linux, without having a dedicated Linux machine, but it poses strange new behaviors as two very different operating systems coexist in one place. Initially, this document mirrors the Windows Install tutorial, showing you how to install Ubuntu and setting up R, RStudio, and LaTex. Then, we cover some of the issues of running two systems together, starting with finding files, finding the Ubuntu subsystem, and file modifications." + "objectID": "units/unit8-numbers.html#calculations", + "href": "units/unit8-numbers.html#calculations", + "title": "Numbers on a computer", + "section": "Calculations", + "text": "Calculations\nGiven the limited precision of computer numbers, we need to be careful when in the following two situations.\n\nSubtracting large numbers that are nearly equal (or adding negative and positive numbers of the same magnitude). You won’t have the precision in the answer that you would like. How many decimal places of accuracy do we have here?\n\n# catastrophic cancellation w/ large numbers\ndg(123456781234.56 - 123456781234.00)\n\n0.55999755859375000000\n\n\nThe absolute error in the original numbers here is of the order \\(\\epsilon x=2.2\\times10^{-16}\\cdot1\\times10^{11}\\approx1\\times10^{-5}=.00001\\). While we might think that the result is close to the value 1 and should have error of about machine epsilon, the relevant absolute error is in the original numbers, so we actually only have about five significant digits in our result because we cancel out the other digits.\nThis is called catastrophic cancellation, because most of the digits that are left represent rounding error – many of the significant digits have cancelled with each other.\nHere’s catastrophic cancellation with small numbers. The right answer here is exactly 0.000000000000000000001234.\n\n# catastrophic cancellation w/ small numbers\nx = .000000000000123412341234\ny = .000000000000123412340000\n\n# So we know the right answer is .000000000000000000001234 exactly. \n\ndg(x-y, '.35f')\n## [1] \"0.00000000000000000000123399999315140\"\n\n0.00000000000000000000123399999315140\n\n\nBut the result is accurate only to 8 places + 20 = 28 decimal places, as expected from a machine precision-based calculation, since the “1” is in the 13th position, after 12 zeroes (12+16=28). Ideally, we would have accuracy to 36 places (16 digits + the 20 zeroes), but we’ve lost 8 digits to catastrophic cancellation.\nIt’s best to do any subtraction on numbers that are not too large. For example, if we compute the sum of squares in a naive way, we can lose all of the information in the calculation because the information is in digits that are not computed or stored accurately: \\[s^{2}=\\sum x_{i}^{2}-n\\bar{x}^{2}\\]\n\n## No problem here:\nx = np.array([-1.0, 0.0, 1.0])\nn = len(x)\nnp.sum(x**2)-n*np.mean(x)**2 \n\n2.0\n\nnp.sum((x - np.mean(x))**2)\n\n## Adding/subtracting a constant shouldn't change the result:\n\n2.0\n\nx = x + 1e8\nnp.sum(x**2)-n*np.mean(x)**2 ## YIKES!\n\n0.0\n\nnp.sum((x - np.mean(x))**2)\n\n2.0\n\n\nA good principle to take away is to subtract off a number similar in magnitude to the values (in this case \\(\\bar{x}\\) is obviously ideal) and adjust your calculation accordingly. In general, you can sometimes rearrange your calculation to avoid catastrophic cancellation. Another example involves the quadratic formula for finding a root (p. 101 of Gentle).\nAdding or subtracting numbers that are very different in magnitude. The precision will be that of the large magnitude number, since we can only represent that number to a certain absolute accuracy, which is much less than the absolute accuracy of the smaller number:\n\ndg(123456781234.2)\n\n123456781234.19999694824218750000\n\ndg(123456781234.2 - 0.1) # truth: 123456781234.1\n\n123456781234.09999084472656250000\n\ndg(123456781234.2 - 0.01) # truth: 123456781234.19\n\n123456781234.19000244140625000000\n\ndg(123456781234.2 - 0.001) # truth: 123456781234.199\n\n123456781234.19898986816406250000\n\ndg(123456781234.2 - 0.0001) # truth: 123456781234.1999\n\n123456781234.19989013671875000000\n\ndg(123456781234.2 - 0.00001) # truth: 123456781234.19999\n\n123456781234.19998168945312500000\n\ndg(123456781234.2 - 0.000001) # truth: 123456781234.199999\n\n123456781234.19999694824218750000\n\n123456781234.2 - 0.000001 == 123456781234.2\n\nTrue\n\n\nThe larger number in the calculations above is of magnitude \\(10^{11}\\), so the absolute error in representing the larger number is around \\(1\\times10^{^{-5}}\\). Thus in the calculations above we can only expect the answers to be accurate to about \\(1\\times10^{-5}\\). In the last calculation above, the smaller number is smaller than \\(1\\times10^{-5}\\) and so doing the subtraction has had no effect. This is analogous to trying to do \\(1+1\\times10^{-16}\\) and seeing that the result is still 1.\nA work-around when we are adding numbers of very different magnitudes is to add a set of numbers in increasing order. However, if the numbers are all of similar magnitude, then by the time you add ones later in the summation, the partial sum will be much larger than the new term. A (second) work-around to that problem is to add the numbers in a tree-like fashion, so that each addition involves a summation of numbers of similar size.\n\nGiven the limited range of computer numbers, be careful when you are:\n\nMultiplying or dividing many numbers, particularly large or small ones. Never take the product of many large or small numbers as this can cause over- or under-flow. Rather compute on the log scale and only at the end of your computations should you exponentiate. E.g., \\[\\prod_{i}x_{i}/\\prod_{j}y_{j}=\\exp(\\sum_{i}\\log x_{i}-\\sum_{j}\\log y_{j})\\]\n\nLet’s consider some challenges that illustrate that last concern.\n\nChallenge: consider multiclass logistic regression, where you have quantities like this: \\[p_{j}=\\text{Prob}(y=j)=\\frac{\\exp(x\\beta_{j})}{\\sum_{k=1}^{K}\\exp(x\\beta_{k})}=\\frac{\\exp(z_{j})}{\\sum_{k=1}^{K}\\exp(z_{k})}\\] for \\(z_{k}=x\\beta_{k}\\). What will happen if the \\(z\\) values are very large in magnitude (either positive or negative)? How can we reexpress the equation so as to be able to do the calculation? Hint: think about multiplying by \\(\\frac{c}{c}\\) for a carefully chosen \\(c\\).\nSecond challenge: The same issue arises in the following calculation. Suppose I want to calculate a predictive density (e.g., in a model comparison in a Bayesian context): \\[\\begin{aligned}\nf(y^{*}|y,x) & = & \\int f(y^{*}|y,x,\\theta)\\pi(\\theta|y,x)d\\theta\\\\\n& \\approx & \\frac{1}{m}\\sum_{j=1}^{m}\\prod_{i=1}^{n}f(y_{i}^{*}|x,\\theta_{j})\\\\\n& = & \\frac{1}{m}\\sum_{j=1}^{m}\\exp\\sum_{i=1}^{n}\\log f(y_{i}^{*}|x,\\theta_{j})\\\\\n& \\equiv & \\frac{1}{m}\\sum_{j=1}^{m}\\exp(v_{j})\\end{aligned}\\] First, why do I use the log conditional predictive density? Second, let’s work with an estimate of the unconditional predictive density on the log scale, \\(\\log f(y^{*}|y,x)\\approx\\log\\frac{1}{m}\\sum_{j=1}^{m}\\exp(v_{j})\\). Now note that \\(e^{v_{j}}\\) may be quite small as \\(v_{j}\\) is the sum of log likelihoods. So what happens if we have terms something like \\(e^{-1000}\\)? So we can’t exponentiate each individual \\(v_{j}\\). This is what is known as the “log sum of exponentials” problem (and the solution as the “log-sum-exp trick”). Thoughts?\n\nNumerical issues come up frequently in linear algebra. For example, they come up in working with positive definite and semi-positive-definite matrices, such as covariance matrices. You can easily get negative numerical eigenvalues even if all the eigenvalues are positive or non-negative. Here’s an example where we use an squared exponential correlation as a function of time (or distance in 1-d), which is mathematically positive definite (i.e., all the eigenvalues are positive) but not numerically positive definite:\n\nxs = np.arange(100)\ndists = np.abs(xs[:, np.newaxis] - xs)\ncorr_matrix = np.exp(-(dists/10)**2) # This is a p.d. matrix (mathematically).\nscipy.linalg.eigvals(corr_matrix)[80:99] # But not numerically!\n\narray([-2.10937946e-16+9.49526594e-17j, -2.10937946e-16-9.49526594e-17j,\n -1.77590164e-16+1.30160558e-16j, -1.77590164e-16-1.30160558e-16j,\n -2.09305049e-16+0.00000000e+00j, 2.23869166e-16+3.21640840e-17j,\n 2.23869166e-16-3.21640840e-17j, 1.98271873e-16+9.08175827e-17j,\n 1.98271873e-16-9.08175827e-17j, -1.49116518e-16+0.00000000e+00j,\n -1.23773149e-16+6.06467275e-17j, -1.23773149e-16-6.06467275e-17j,\n -2.48071368e-18+1.51188749e-16j, -2.48071368e-18-1.51188749e-16j,\n -4.08131705e-17+6.79669911e-17j, -4.08131705e-17-6.79669911e-17j,\n 1.27901871e-16+2.34695655e-17j, 1.27901871e-16-2.34695655e-17j,\n 5.23476667e-17+4.08642121e-17j])" }, { - "objectID": "howtos/windowsAndLinux.html#installing-ubuntu", - "href": "howtos/windowsAndLinux.html#installing-ubuntu", - "title": "Windows 10 and the Ubuntu Subsystem", - "section": "Installing Ubuntu", - "text": "Installing Ubuntu\nThere are 2 parts to installing a Linux subsystem in Windows. I will write this using Ubuntu as the example, as it is my preferred Linux distro, but several others are provided by Windows.\nSources:\n\nOfficial Windows Instructions\nUbuntu Update Instructions\n\n\n1) Enable Linux Subsystem\nBy default, the Linux subsystem is an optional addition in Windows. This feature has to be enabled prior to installing Linux. There are two ways to do it.\n\nCMD Line\nThe simplest way to enable the Linux subsystem is through PowerShell.\n\nOpen PowerShell as Administrator\nRun the following (on one line):\nEnable-WindowsOptionalFeature -Online -FeatureName Microsoft-Windows-Subsystem-Linux\nRestart the computer\n\nGUI\nIf you don’t wish to use PowerShell, you can work your way through the control panel and turn on the Linux subsystem.\n\nOpen the settings page through the search bar\nGo to Programs and Features\nFind Turn Windows Features on or off on the right side\nEnable the Windows Subsystem for Linux option\nRestart the computer\n\n\n\n\n2) Install Linux Subsystem\nOnce the Linux subsystem feature has been enabled, there are multiple methods to download and install the Linux distro you want. I highly recommend installing Ubuntu from the Microsoft store. There are several other flavors available as well, but Ubuntu is generally the easiest to learn and the most well supported.\n\nOpen the Microsoft Store\nSearch for Ubuntu\n\nYou’re looking for the highest number followed by LTS, currently 20.04 LTS (or 18.04 LTS is fine too). This is the current long-term-release, meaning it will be supported for the next 5 years.\n\nClick on the tile, then click Get, and this should start the installation.\nFollow the prompts to install Ubuntu.\n\nAfter installing Ubuntu, it is advisable to update it. This is something you should do on a regular basis.\n\nOpen a Bash terminal.\nType sudo apt update to update your local package database.\nType sudo apt upgrade to upgrade your installed packages." + "objectID": "units/unit8-numbers.html#final-note", + "href": "units/unit8-numbers.html#final-note", + "title": "Numbers on a computer", + "section": "Final note", + "text": "Final note\nHow the computer actually does arithmetic with the floating point representation in base 2 gets pretty complicated, and we won’t go into the details. These rules of thumb should be enough for our practical purposes. Monahan and the URL reference have many of the gory details." }, { - "objectID": "howtos/windowsAndLinux.html#using-the-linux-terminal-from-r-in-windows", - "href": "howtos/windowsAndLinux.html#using-the-linux-terminal-from-r-in-windows", - "title": "Windows 10 and the Ubuntu Subsystem", - "section": "Using the Linux Terminal from R in Windows", - "text": "Using the Linux Terminal from R in Windows\nTo get all the functionality of a UNIX-style commandline from within R (e.g., for bash code chunks), you should set the terminal under R in Windows to be the Linux subsystem." + "objectID": "units/unit3-bash.html", + "href": "units/unit3-bash.html", + "title": "The bash shell and UNIX commands", + "section": "", + "text": "PDF\nReference:" }, { - "objectID": "howtos/windowsAndLinux.html#a-note-on-file-modification", - "href": "howtos/windowsAndLinux.html#a-note-on-file-modification", - "title": "Windows 10 and the Ubuntu Subsystem", - "section": "A Note on File Modification", - "text": "A Note on File Modification\nDO NOT MODIFY LINUX FILES FROM WINDOWS\nIt is highly recommended that you never modify Linux files from Windows because of metadata corruption issues. Any files created under the Linux subsystem, only modify them with Linux tools. In contrast, you can create files in the Windows system and modify them with both Windows or Linux tools. There could be file permission issues because Windows doesn’t have the same concept of file permissions as Linux. So, if you intend to work on files using both Linux and Windows, create the files in the C drive under Windows, and you should be safe to edit them with either OS." + "objectID": "units/unit3-bash.html#first-challenge", + "href": "units/unit3-bash.html#first-challenge", + "title": "The bash shell and UNIX commands", + "section": "4.1 First challenge", + "text": "4.1 First challenge\nConsider the file cpds.csv. How would you write a shell command that returns “There are 8 occurrences of the word ‘Belgium’ in this file.”, where ‘8’ should instead be the correct number of times the word occurs.\nExtra: make your code into a function that can operate on any file indicated by the user and any word of interest." }, { - "objectID": "howtos/windowsAndLinux.html#finding-windows-from-linux", - "href": "howtos/windowsAndLinux.html#finding-windows-from-linux", - "title": "Windows 10 and the Ubuntu Subsystem", - "section": "Finding Windows from Linux", - "text": "Finding Windows from Linux\nOnce you have some flavor of Linux installed, you need to be able to navigate from your Linux home directory to wherever Windows stores files. This is relatively simple, as the Windows Subsystem shows Windows to Linux as a mounted drive.\n\nSource\n\n\nOpen a Bash terminal.\nType cd / to get to the root directory.\nIn root, type cd /mnt. This gets you to the mount point for your drives.\nType ls to see what drives are available (you should see a c, and maybe d as well).\nType cd c to get into the Windows C drive. This is the root of the C directory for Windows.\nTo find your files, change directy into the users folder, then into your username.\n\ncd Users/<your-user-name>\nThis is your home directory in Windows. If you type ls here, you should see things like\n\nDocuments\nDownloads\nPictures\nVideos\netc…" + "objectID": "units/unit3-bash.html#second-challenge", + "href": "units/unit3-bash.html#second-challenge", + "title": "The bash shell and UNIX commands", + "section": "4.2 Second challenge", + "text": "4.2 Second challenge\nConsider the data in the RTADataSub.csv file. This is a subset of data giving freeway travel times for segments of a freeway in an Australian city. The data are from a kaggle.com competition. We want to try to understand the kinds of data in each field of the file. The following would be particularly useful if the data were in many files or the data were many gigabytes in size.\n\nFirst, take the fourth column. Figure out the unique values in that column.\nNext, automate the process of determining if any of the values are non-numeric so that you don’t have to scan through all of the unique values looking for non-numbers. You’ll need to look for the following regular expression pattern [^0-9], which is interpreted as NOT any of the numbers 0 through 9.\n\nExtra: do it for all the fields, except the first one. Have your code print out the result in a human-readable way understandable by someone who didn’t write the code. For simplicity, you can assume you know the number of fields." }, { - "objectID": "howtos/windowsAndLinux.html#finding-linux-from-windows", - "href": "howtos/windowsAndLinux.html#finding-linux-from-windows", - "title": "Windows 10 and the Ubuntu Subsystem", - "section": "Finding Linux from Windows", - "text": "Finding Linux from Windows\nThis is slightly more tricky than getting from Linux to Windows. Windows stores the Linux files in a hidden subfolder so that you don’t mess with them from Windows. However, you can find them, and then the easiest way (note, do not read as safest or smartest) to find those files in the future is by creating a desktop shortcut.\n\nSource\n\n\nOpen File Explorer\nIn the address bar, type %userprofile%\\AppData\\Local\\Packages\n\n%userprofile% will expand to something like C:\\Users\\<your-user-name>\n\nLook for a folder related to the Linux distro that you installed\n\nThese names will change slightly over time, but look for something similar-ish.\nFor Ubuntu, look for something with CanonicalGroupLimited.UbuntuonWindows in it.\n\nCanonical is the creator/distributor of Ubuntu.\n\n\nClick LocalState\nClick rootfs\n\nThis is the root of your Linux distro.\n\nClick into home and then into your user name.\n\nThis is your home directory under Linux.\n\nDO NOT MODIFY THESE FILES FROM WINDOWS\n\nData corruption is a possibility.\n\n\nSo, the final path to find your home directory from windows will look like:\n%userprofile%\\AppData\\Local\\Packages\\<Distro-Folder>\\LocalStat\\rootfs\\home\\<your-user-name>\\" + "objectID": "units/unit3-bash.html#third-challenge", + "href": "units/unit3-bash.html#third-challenge", + "title": "The bash shell and UNIX commands", + "section": "4.3 Third challenge", + "text": "4.3 Third challenge\n\nFor Belgium, determine the minimum unemployment value (field #6) in cpds.csv in a programmatic way.\nHave what is printed out to the screen look like “Belgium 6.2”.\nNow store the unique values of the countries in a variable, first stripping out the quotation marks.\nFigure out how to automate step 1 to do the calculation for all the countries and print to the screen.\nHow would you instead store the results in a new file?" }, { - "objectID": "howtos/windowsAndLinux.html#installing-r-on-the-linux-subsystem", - "href": "howtos/windowsAndLinux.html#installing-r-on-the-linux-subsystem", - "title": "Windows 10 and the Ubuntu Subsystem", - "section": "Installing R on the Linux Subsystem", - "text": "Installing R on the Linux Subsystem\nIMPORTANT: This section is only if you’d like to try using R under Linux. For class, using R under Windows should be fine.\nThe Linux Subsystem behaves exactly like a regular Linux installation, but for completeness, I will provide instructions here for people new to Linux. These instructions are written from the perspective of Ubuntu, but will be similar for other repos.\nR is not a part of the standard Ubuntu installation. So, we have to add the repository manually to our repository list. This is relatively straightforward, and R supports several versions of Ubuntu.\nSources:\n\nCRAN guide for Ubuntu\nDigital Ocean quick tutorial\n\n\nIn a bash window, type:\nsudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9\n\nThis adds the key to “sign”, or validate, the R repository\n\nThen, type:\nsudo add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/'\n\ncloud.r-project.org is the default mirror, however, it is prudent to connect to the mirror closest to you geographically. Berkeley has it’s own mirror, so the command with the Berkeley mirror would look like\n\nsudo add-apt-repository 'deb https://cran.r-project.org/bin/linux/ubuntu/bionic-cran40/'\nFinally, type sudo apt install r-base, and press y to confirm installation\nTo test that it worked, type R into the console, and an R session should begin\n\nType q() to quit the R session" + "objectID": "units/unit3-bash.html#fourth-challenge", + "href": "units/unit3-bash.html#fourth-challenge", + "title": "The bash shell and UNIX commands", + "section": "4.4 Fourth challenge", + "text": "4.4 Fourth challenge\nLet’s return to the RTADataSub.csv file and the issue of missing values.\n\nCreate a new file without any rows that have an ‘x’ (which indicate a missing value).\nTurn the code into a function that also prints out the number of rows that are being removed and that sends its output to stdout so that it can be used with piping.\nNow modify your function so that the user could provide the missing value string and the input filename." }, { - "objectID": "howtos/windowsAndLinux.html#installing-rstudio-on-the-linux-subsystem", - "href": "howtos/windowsAndLinux.html#installing-rstudio-on-the-linux-subsystem", - "title": "Windows 10 and the Ubuntu Subsystem", - "section": "Installing Rstudio on the Linux Subsystem", - "text": "Installing Rstudio on the Linux Subsystem\n\n\n\n\n\n\nWarning\n\n\n\nTHIS NO LONGER WORKS\n\n\nAs of Rstudio 1.5.x, it does not run on WSL. Link\nAlso possible issues, WSL has no GUI, and therefore can’t support anything that uses a GUI.\nThese instructions work, but Rstudio doesn’t run.\nSources:\n\nRstudio\nSource\n\n\nGo the the Rstudio website (link above) and download the appropriate Rstudio Desktop version.\n\nFor most people, this is the Ubuntu 18 (64-bit) installer.\nSave it somewhere that you can find it.\nYou should have a file similar to rstudio-<version number>-amd64.deb\n\nOpen a terminal window and navigate to wherever you saved the rstudio install file.\nType the command sudo dpkg -i ./rstudio-<version number>-amd64.deb\n\nThis tells the package installer (dpkg) to install (-i) the file specified (./thing.deb)\n\nType the command sudo apt-get install -f\n\nThis tells the package manager (apt-get) to fix (-f) any dependency issues that may have arisen when installing the package.\n\nType the command which rstudio to make sure the system can find it.\n\nOutput should be similar to /usr/bin/rstudio\n\nRun rstudio from linux by typing rstudio &\n\nThe & runs it in the background, allowing you to close the terminal window." + "objectID": "units/unit3-bash.html#fifth-challenge", + "href": "units/unit3-bash.html#fifth-challenge", + "title": "The bash shell and UNIX commands", + "section": "4.5 Fifth challenge", + "text": "4.5 Fifth challenge\nConsider the coop.txt weather station file.\nFigure out how to use grep to tell you the starting position of the state field. Hints: search for a known state-country combination and figure out what flags you can use with grep to print out the “byte offset” for the matched state.\nUse that information to automate the first mission where we extracted the state field using cut. You’ll need to do a bit of arithmetic using shell commands." }, { - "objectID": "howtos/windowsAndLinux.html#installing-latex-on-the-linux-subsystem", - "href": "howtos/windowsAndLinux.html#installing-latex-on-the-linux-subsystem", - "title": "Windows 10 and the Ubuntu Subsystem", - "section": "Installing LaTeX on the Linux Subsystem", - "text": "Installing LaTeX on the Linux Subsystem\n\n\n\n\n\n\nImportant\n\n\n\nThis section is only if you’d like to try using LaTeX under Linux. For class, using LaTeX (or R Markdown) under Windows should be fine.\n\n\nLaTeX is a text-markup language used when generating documents from .Rmd files.\nSource LaTeX\n\nType sudo apt-get install texlive-full, press y to confirm installation\n\nGenerally, if you want to create and edit R Markdown documents you will also need a text editor to go with your LaTeX installation, but we won’t go into that here." + "objectID": "units/unit3-bash.html#sixth-challenge", + "href": "units/unit3-bash.html#sixth-challenge", + "title": "The bash shell and UNIX commands", + "section": "4.6 Sixth challenge", + "text": "4.6 Sixth challenge\nHere’s an advanced one - you’ll probably need to use sed, but the brief examples of text substitution in the using bash tutorial (or in the demos above) should be sufficient to solve the problem.\nConsider a CSV file that has rows that look like this:\n1,\"America, United States of\",45,96.1,\"continental, coastal\" \n2,\"France\",33,807.1,\"continental, coastal\"\nWhile Pandas would be able to handle this using read_csv(), using cut in UNIX won’t work because of the commas embedded within the fields. The challenge is to convert this file to one that we can use cut on, as follows.\nFigure out a way to make this into a new delimited file in which the delimiter is not a comma. At least one solution that will work for this particular two-line dataset does not require you to use regular expressions, just simple replacement of fixed patterns." }, { - "objectID": "howtos/RandRStudioInstall.html", - "href": "howtos/RandRStudioInstall.html", - "title": "Installing R & RStudio", - "section": "", - "text": "On your laptop\nIf your version of R is older than 4.0.0, please install the latest version.\nTo install R, see:\n\nMacOS: install the R-4.2.1.pkg from https://cran.rstudio.com/bin/macosx\nWindows: https://cran.rstudio.com/bin/windows/base/\nLinux: https://cran.rstudio.com/bin/linux/\n\nThen install RStudio. To do so, see https://www.rstudio.com/ide/download/desktop, scrolling down to the “Installers for Supported Platforms” section and selecting the Installer for your operating system.\nVerify that you can install add-on R packages by installing the ‘fields’ package. In RStudio, go to ‘Tools->Install Packages’. In the resulting dialog box, enter ‘fields’ (without quotes) in the ‘Packages’ field. Depending on the location specified in the ‘Install to Library’ field, you may need to enter your administrator password. To be able to install packages to the directory of an individual user, you may need to do the following:\n\nIn R, enter the command Sys.getenv()['R_LIBS_USER'].\nCreate the directory specified in the result that R returns, e.g., on a Mac, this might be ~/Library/R/4.0/library.\n\nFor more detailed installation instructions for Windows, see Using R, RStudio, and LaTeX on Windows file.\n\n\nVia DataHub\nSee the instructions in Accessing the Unix Command Line for how to login to Datahub. Then in the mid-upper right, click on New and RStudio. Alternatively, to go directly to RStudio, go to https://r.datahub.berkeley.edu." + "objectID": "units/unit3-bash.html#versions-of-regular-expressions", + "href": "units/unit3-bash.html#versions-of-regular-expressions", + "title": "The bash shell and UNIX commands", + "section": "Versions of regular expressions", + "text": "Versions of regular expressions\nOne thing that can cause headaches is differences in version of regular expression syntax used. As discussed in man grep, extended regular expressions are standard, with basic regular expressions providing less functionality and Perl regular expressions additional functionality.\nThe bash shell tutorial provides a full documentation of the extended regular expressions syntax, which we’ll focus on here. This syntax should be sufficient for most usage and should be usable in Python and R, but if you notice something funny going on, it might be due to differences between the regular expressions versions.\n\nIn bash, grep -E (or egrep) enables use of the extended regular expressions.\nIn Python, the re package provides syntax “similar to” Perl.\nIn R, stringr provides ICU regular expressions (see help(regex)), which are based on Perl regular expressions.\n\nMore details about Perl regular expressions can be found in the regex Wikipedia page." }, { - "objectID": "howtos/quartoInstall.html", - "href": "howtos/quartoInstall.html", - "title": "Installing and Using Quarto", - "section": "", - "text": "Unless you plan to generate your problem set solutions on the SCF, you’ll need to install Quarto.\nOnce installed, you should be able to run commands such as quarto render FILE and quarto preview FILE from the command line.\nQuarto also runs from the Windows Command shell or PowerShell. We’ll add details/troubleshooting tips here as needed." + "objectID": "units/unit3-bash.html#general-principles-for-working-with-regex", + "href": "units/unit3-bash.html#general-principles-for-working-with-regex", + "title": "The bash shell and UNIX commands", + "section": "General principles for working with regex", + "text": "General principles for working with regex\nThe syntax is very concise, so it’s helpful to break down individual regular expressions into the component parts to understand them. As Murrell notes, since regex are their own language, it’s a good idea to build up a regex in pieces as a way of avoiding errors just as we would with any computer code. re.findall in Python and str_detect in R’s stringr, as well as regex101.com are particularly useful in seeing what was matched to help in understanding and learning regular expression syntax and debugging your regex. As with many kinds of coding, I find that debugging my regex is usually what takes most of my time." }, { - "objectID": "howtos/accessingUnixCommandLine.html", - "href": "howtos/accessingUnixCommandLine.html", - "title": "Accessing the Unix Command Line", - "section": "", - "text": "You have several options for UNIX command-line access. You’ll need to choose one of these and get it working.\n\nMac OS (on your personal machine):\nOpen a Terminal by going to Applications -> Utilities -> Terminal\n\n\nWindows (on your personal machine):\n\nYou may be able to use the Ubuntu bash shell available in Windows.\nYour PC must be running a 64-bit version of Windows 10 Anniversary Update or later (build 1607+).\nPlease see these links for more information:\n\nhttp://blog.revolutionanalytics.com/2017/12/r-in-the-windows-subsystem-for-linux.html\nhttps://msdn.microsoft.com/en-us/commandline/wsl/install_guide\n\nFor more detailed instructions, see the Installing the Linux Subsystem on Windows tutorial.\n(Not recommended) There’s an older program called cygwin that provides a UNIX command-line interface.\n\nNote that when you install Git on Windows, you will get Git Bash. While you can use this to control Git, the functionality is limited so this will not be enough for general UNIX command-line access for the course.\n\n\nLinux (on your personal machine):\nIf you have access to a Linux machine, you very likely know how to access a terminal.\n\n\nAccess via DataHub (provided by UC Berkeley’s Data Science Education Program)\n\nGo to https://datahub.berkeley.edu\nClick on Sign in with bCourses, sign in via CalNet, and authorize DataHub to have access to your account.\nIn the mid-upper right, click on New and Terminal.\nTo end your session, click on Control Panel and Stop My Server. Note that Logout will not end your running session, it will just log you out of it.\n\n\n\nAccess via the Statistical Computing Facility (SCF)\nWith an SCF account (available here), you can access a bash shell in the ways listed below.\nThose of you in the Statistics Department should be in the process of getting an SCF account. Everyone else will need an SCF account when we get to the unit on parallel computing, but you can request an account now if you prefer.\n\nYou can login to our various Linux servers and access a bash shell that way. Please see http://statistics.berkeley.edu/computing/access.\nYou can also access a bash shell via the SCF JupyterHub interface; please see the Accessing Python instructions but when you click on New, choose Terminal. This is very similar to the DataHub functionality discussed above." + "objectID": "units/unit3-bash.html#challenge-problem", + "href": "units/unit3-bash.html#challenge-problem", + "title": "The bash shell and UNIX commands", + "section": "Challenge problem", + "text": "Challenge problem\nChallenge: Let’s think about what regex syntax we would need to detect any number, integer- or real-valued. Let’s start from a test-driven development perspective of writing out test cases including: - various cases we want to detect, - various tricky cases that are not numbers and we don’t want to detect, and - “corner cases” – tricky (perhaps unexpected) cases that might trip us up." } ] \ No newline at end of file diff --git a/sitemap.xml b/sitemap.xml index f9befee..716d0ee 100644 --- a/sitemap.xml +++ b/sitemap.xml @@ -1,155 +1,155 @@ - https://github.com/berkeley-stat243/stat243-fall-2023/publish.html - 2023-10-25T20:02:02.890Z + https://github.com/berkeley-stat243/stat243-fall-2023/index.html + 2023-10-25T20:19:22.221Z - https://github.com/berkeley-stat243/stat243-fall-2023/howtos/accessingPython.html - 2023-10-25T20:02:02.086Z + https://github.com/berkeley-stat243/stat243-fall-2023/units/unit4-goodPractices.html + 2023-10-25T20:19:06.736Z - https://github.com/berkeley-stat243/stat243-fall-2023/howtos/gitInstall.html - 2023-10-25T20:02:01.346Z + https://github.com/berkeley-stat243/stat243-fall-2023/units/unit10-linalg.html + 2023-10-25T20:18:38.228Z - https://github.com/berkeley-stat243/stat243-fall-2023/howtos/windowsInstall.html - 2023-10-25T20:02:00.542Z + https://github.com/berkeley-stat243/stat243-fall-2023/units/unit9-sim.html + 2023-10-25T20:18:07.460Z - https://github.com/berkeley-stat243/stat243-fall-2023/howtos/ps-submission.html - 2023-10-25T20:01:59.206Z + https://github.com/berkeley-stat243/stat243-fall-2023/units/unit5-programming.html + 2023-10-25T20:17:33.224Z - https://github.com/berkeley-stat243/stat243-fall-2023/index.html - 2023-10-25T20:01:58.242Z + https://github.com/berkeley-stat243/stat243-fall-2023/units/unit2-dataTech.html + 2023-10-25T20:16:50.995Z - https://github.com/berkeley-stat243/stat243-fall-2023/labs/lab5-codereview.html - 2023-10-25T20:01:53.890Z + https://github.com/berkeley-stat243/stat243-fall-2023/schedule.html + 2023-10-25T20:16:44.215Z - https://github.com/berkeley-stat243/stat243-fall-2023/labs/06/scf.html - 2023-10-25T20:01:40.066Z + https://github.com/berkeley-stat243/stat243-fall-2023/ps/ps4.html + 2023-10-25T20:16:35.123Z - https://github.com/berkeley-stat243/stat243-fall-2023/labs/lab3-debugging.html - 2023-10-25T20:01:29.721Z + https://github.com/berkeley-stat243/stat243-fall-2023/ps/ps2.html + 2023-10-25T20:16:24.915Z - https://github.com/berkeley-stat243/stat243-fall-2023/labs/py_vs_R.html - 2023-10-25T20:01:19.541Z + https://github.com/berkeley-stat243/stat243-fall-2023/ps/ps6.html + 2023-10-25T20:16:14.823Z - https://github.com/berkeley-stat243/stat243-fall-2023/units/unit3-bash.html - 2023-10-25T20:01:00.153Z + https://github.com/berkeley-stat243/stat243-fall-2023/office_hours.html + 2023-10-25T20:16:09.275Z - https://github.com/berkeley-stat243/stat243-fall-2023/units/unit5-programming.html - 2023-10-25T20:00:30.144Z + https://github.com/berkeley-stat243/stat243-fall-2023/howtos/gitInstall.html + 2023-10-25T20:16:07.559Z - https://github.com/berkeley-stat243/stat243-fall-2023/units/unit9-sim.html - 2023-10-25T19:59:56.056Z + https://github.com/berkeley-stat243/stat243-fall-2023/howtos/ps-submission.html + 2023-10-25T20:16:06.611Z - https://github.com/berkeley-stat243/stat243-fall-2023/units/unit1-intro.html - 2023-10-25T19:59:33.931Z + https://github.com/berkeley-stat243/stat243-fall-2023/howtos/accessingUnixCommandLine.html + 2023-10-25T20:16:04.971Z - https://github.com/berkeley-stat243/stat243-fall-2023/units/unit7-bigData.html - 2023-10-25T19:59:12.939Z + https://github.com/berkeley-stat243/stat243-fall-2023/howtos/windowsInstall.html + 2023-10-25T20:16:03.847Z - https://github.com/berkeley-stat243/stat243-fall-2023/ps/ps2.html - 2023-10-25T19:59:01.355Z + https://github.com/berkeley-stat243/stat243-fall-2023/labs/lab3-debugging.html + 2023-10-25T20:15:57.215Z - https://github.com/berkeley-stat243/stat243-fall-2023/ps/ps6.html - 2023-10-25T19:58:53.314Z + https://github.com/berkeley-stat243/stat243-fall-2023/labs/lab0-setup.html + 2023-10-25T20:15:48.243Z - https://github.com/berkeley-stat243/stat243-fall-2023/ps/ps3.html - 2023-10-25T19:58:45.050Z + https://github.com/berkeley-stat243/stat243-fall-2023/labs/lab2-testing.html + 2023-10-25T20:15:38.847Z - https://github.com/berkeley-stat243/stat243-fall-2023/schedule.html - 2023-10-25T19:58:39.430Z + https://github.com/berkeley-stat243/stat243-fall-2023/labs/py_vs_R.html + 2023-10-25T20:15:17.543Z - https://github.com/berkeley-stat243/stat243-fall-2023/office_hours.html - 2023-10-25T19:58:38.446Z + https://github.com/berkeley-stat243/stat243-fall-2023/publish.html + 2023-10-25T20:15:15.819Z - https://github.com/berkeley-stat243/stat243-fall-2023/ps/ps5.html - 2023-10-25T19:58:40.946Z + https://github.com/berkeley-stat243/stat243-fall-2023/labs/06/scf.html + 2023-10-25T20:15:26.379Z - https://github.com/berkeley-stat243/stat243-fall-2023/ps/ps1.html - 2023-10-25T19:58:49.282Z + https://github.com/berkeley-stat243/stat243-fall-2023/labs/lab1-submission.html + 2023-10-25T20:15:43.555Z - https://github.com/berkeley-stat243/stat243-fall-2023/ps/ps4.html - 2023-10-25T19:58:57.331Z + https://github.com/berkeley-stat243/stat243-fall-2023/labs/lab5-codereview.html + 2023-10-25T20:15:52.635Z - https://github.com/berkeley-stat243/stat243-fall-2023/ps/regex.html - 2023-10-25T19:59:04.975Z + https://github.com/berkeley-stat243/stat243-fall-2023/howtos/RandRStudioInstall.html + 2023-10-25T20:16:03.163Z - https://github.com/berkeley-stat243/stat243-fall-2023/units/unit6-parallel.html - 2023-10-25T19:59:25.815Z + https://github.com/berkeley-stat243/stat243-fall-2023/howtos/accessingPython.html + 2023-10-25T20:16:04.407Z - https://github.com/berkeley-stat243/stat243-fall-2023/units/unit10-linalg.html - 2023-10-25T19:59:45.851Z + https://github.com/berkeley-stat243/stat243-fall-2023/howtos/windowsAndLinux.html + 2023-10-25T20:16:05.955Z - https://github.com/berkeley-stat243/stat243-fall-2023/units/unit2-dataTech.html - 2023-10-25T20:00:06.664Z + https://github.com/berkeley-stat243/stat243-fall-2023/howtos/quartoInstall.html + 2023-10-25T20:16:07.047Z - https://github.com/berkeley-stat243/stat243-fall-2023/units/unit4-goodPractices.html - 2023-10-25T20:00:52.789Z + https://github.com/berkeley-stat243/stat243-fall-2023/syllabus.html + 2023-10-25T20:16:08.831Z - https://github.com/berkeley-stat243/stat243-fall-2023/units/unit8-numbers.html - 2023-10-25T20:01:08.769Z + https://github.com/berkeley-stat243/stat243-fall-2023/ps/regex.html + 2023-10-25T20:16:10.051Z - https://github.com/berkeley-stat243/stat243-fall-2023/labs/lab1-submission.html - 2023-10-25T20:01:25.821Z + https://github.com/berkeley-stat243/stat243-fall-2023/ps/ps1.html + 2023-10-25T20:16:20.059Z - https://github.com/berkeley-stat243/stat243-fall-2023/labs/lab0-setup.html - 2023-10-25T20:01:35.106Z + https://github.com/berkeley-stat243/stat243-fall-2023/ps/ps5.html + 2023-10-25T20:16:30.359Z - https://github.com/berkeley-stat243/stat243-fall-2023/labs/lab2-testing.html - 2023-10-25T20:01:50.230Z + https://github.com/berkeley-stat243/stat243-fall-2023/ps/ps3.html + 2023-10-25T20:16:39.875Z - https://github.com/berkeley-stat243/stat243-fall-2023/syllabus.html - 2023-10-25T20:01:57.842Z + https://github.com/berkeley-stat243/stat243-fall-2023/rubric.html + 2023-10-25T20:16:44.791Z - https://github.com/berkeley-stat243/stat243-fall-2023/rubric.html - 2023-10-25T20:01:58.694Z + https://github.com/berkeley-stat243/stat243-fall-2023/units/unit7-bigData.html + 2023-10-25T20:17:05.243Z - https://github.com/berkeley-stat243/stat243-fall-2023/howtos/windowsAndLinux.html - 2023-10-25T20:02:00.010Z + https://github.com/berkeley-stat243/stat243-fall-2023/units/unit1-intro.html + 2023-10-25T20:17:57.120Z - https://github.com/berkeley-stat243/stat243-fall-2023/howtos/RandRStudioInstall.html - 2023-10-25T20:02:00.918Z + https://github.com/berkeley-stat243/stat243-fall-2023/units/unit6-parallel.html + 2023-10-25T20:18:20.620Z - https://github.com/berkeley-stat243/stat243-fall-2023/howtos/quartoInstall.html - 2023-10-25T20:02:01.690Z + https://github.com/berkeley-stat243/stat243-fall-2023/units/unit8-numbers.html + 2023-10-25T20:18:50.204Z - https://github.com/berkeley-stat243/stat243-fall-2023/howtos/accessingUnixCommandLine.html - 2023-10-25T20:02:02.514Z + https://github.com/berkeley-stat243/stat243-fall-2023/units/unit3-bash.html + 2023-10-25T20:19:15.585Z diff --git a/units/unit1-intro.pdf b/units/unit1-intro.pdf index 7dbbc0321cb4e74dc898d7045dfbdb007facfe0b..e292f6877f34ae95e13d55f8a5848abd56aca576 100644 GIT binary patch delta 7311 zcmai&Ra_JRv-YK7S){wWyE|log{1^Zsijltc7dfEmXeZAN$HgCZjhFgkQS-W@4Y(j zxjFaG+&s_Be||IYN930WU(F*t`M z9A1X|bT=n?)Q1|zJF++V^9Pfr`#Ny^sXm9p-27dk8g`(l5!wm5XCd|3w|eb9hQ|s$ ziXrv**=#H9ap^$XPqQbdAEmsj=B3`}f$Va3l!Ks_<92Y0kKx-rs-zrMZD}5z|qz z0W+1IT$pwTtu%$Wbh&@)G=%{u5a>p+T~b1LO>I&O39cbsx~Fop^Njswvp>Z?W2{hR zZlm*hosJ2a7ab4pXD+!E=Qeh|!F6B=PPaZ2i{6XM57u@^3a@9LkN~MGwe^N(ox9#X@T9$tqv#TP)BkvH0t5t=4^Nc8<+CbkBo|*x1)lwo_)G z*6IF!6ui!JPHTU8)qs;5`_EP|%-9hwhZnZ_MEtP)HoDE#+Zd0YA4mQC=d5?3B}e}| zk$w7?t~T4lH)vrWY;VWIEHi4-9oK*ik8UQd#q#rURa80cF`Qtk!{ueVv;v3G;1SqZ z(;=J>#``y+fEoWR*G#6{?0jQG)Wmq%V1u_!$FqC03d+qtlT@;xLD6&eftn@SA zL&gsKg;hohYm^MJU(z8YpLB$HI;O`<_?Y}er!cn{U3gy*qNTljAl8t;Z@~~2=m4?G zsFe>prN~7Jq`0=H>EvHVLV#E-U+&vF#6&43+Bc1T+>9-c&M~l$+CR1JZJGuBo&o=- z&5OBMkOjmLj~rw$2)n%gx^NgBlM(%N19J(kxkoaf?e4XI@_4_( zv-#PJ=(+UtSF)0;i@N2}>ZX(Wj$-kffTz&X|FcBh71} zmxo%MmBsh{JDi;BUoJ&}vhzvgpsnI^Y*yFiwhxCAy22}??}v>s<`B`I{LD97{O^r5DQd*Jf*TL>T2)cHNY6iVU7YK5J3Z2SgOiV^aHRWi8I9Z zN43ZK&E%-55MX!{9a}%S9U3L!uRXd&O9kc~eXZA&D)J_z2xK01BwYj-4X{$p?k4=YUgzc`Yz=CI30 z1o}4vekWSdy!A!s3Uqb58s-u?;8om@PRy2Nylj(ioH=e54Rp=-lI3HFZSyt;7m<4Xm11l_pl*tYkI5q?+4DeQS&`q^*& z&UpnkFf+H2Tl1yZ|E~`(l-t3-_xNtj9Hg+Z7qMpDa9S;S18oI{VwO5jSbKN-LX`l@ePC7sM|#9z3gr{qtfc|+@|I9 z5Woq_2$?$|bWIqmdCWF(P!&hO0BmH=JXKVRG=oqi%>|QcQS?Y)%s3$xkh7!FuM*k# zp@bMlFPFwVQbpNx1lurK@vVm>%HzD{Q^Up^7s9e3I4@27W)Elep-5H;h}7$mkcTM( zauX!ub|9Dgo+N;6BjKlaL8sQg}?SqeDj(eGlXkk^$a` z2Xg2pD52v-Ew)TeXDT}CM&=7g-78K4ufPgG0J%_PPa+;%J}g7H^2W5zMub=7s(CC^ z;C_iB-lOH4~*9xx@uEfHB7E@Zd8hW60=Lu^0Z`-9KI zLTmZcpQb~~wIA@z$qmVaY1}KGmpj^)1W7%Fyets{z|RU3_^%d1{1-;=)=5GV?a+== zS(vY9zW#5Fdh^(Z7k<$PS+{}z(0WZ9XDY6J9-Q@UF;L%;8=Ca7V9M0cgq&4A_+39f zHqvY!=Z1E3-c4;jw5Kmk<=3c_{m|D5JceCAEmZ@H?aDyE&m@u~Gb*9}(wg)7CDi>6 zdsy4B$;K%}XLOdQAl0FlxPLRw$|TjmgnpMN>)M!DZW*g!)b}v~eV$MWX(109NA)HV zUlZHZWSwQE`y&C%B|n8Bxwy=0no*`jY^1 z>}|3^)4U3IM8d>PO!+z%Y6QYMCm@AJXvied6&O;PkteaQ;t|=(qRV*#fwr@Im9$6_ zdh65WVhXa2=kdXJOfV){P9g^A_+n;W0;+izC!5@1aKY=H&dKRbT*247$$J}S&i(Tn z{|RbWKm-Bc1pFgu&&xWhqDn)4a0w2-@2f(Du8Q8uA!y^@Y8wWjg=yMOW=qGqG)*+O z9fh@vI=|E6<%z4tkzVc5Wt#MGZrH^raH+^0k*V}w5qD|gY(Kx$=ry6fHGIB*Zd_|@ zyXm|SuI{T6x|(y{RnwkB8yK=xW-#B_AGn&C3#{a=f~gQFV|qI|(scSpuT;29U5j)+ z@C$!dG8;HaA-+}T_gIfREArnKw|CeJXsKMyq%ko}V!uhv?x(ELWy-oxrJlX%KF?QJ zyMOzXX2Ws!t?tKF(HEmtgSWJ4$eV9}1}o^I=MBHj4%RCJ$~)=`SSYk0@1-H1^+}N% z0(?z@0NC$1a4?e{mipPQFQ7jYTP-QVS3FWUoes+gPC_RZV}N+rGGa(4li5QLWg>bT zollVW7TTs@LlDUCmh`(IV~aVcr2n31k~PU?IQ1h*JxexwQIP>5n9VDkiHLhT!XHI7>LknN@*2_+u=JHPR|c zcGyCc-qx^|L&_}EYMzkDOrnZ{G~5d<2e_K56y*zm1pvXrflw*cVf43$`%!3SeIw>< z^fFw+AVlQU{A@|aVW6V0dWxofD`XUDF**zCuj+K}+0(CHiK~Ch ztNun1B5xEqO5VTI(1bVi(2r#M3e|SHSKv!f`;rh42K2l5&7|U&Qj5+uYxk9BNc9cL z_l!^lN;zf1VPvTWRVZ0M%GW6D=1fIov@`7pSX9eau2vWZxfV_0_*47W_jh|)%CTPr%+M(S@Hsymd zI;zExwe)?*rGQRN=~mpWQN*1W$3n=D9F@-myc0{O7uHt#CvP&c()4RnT1oYn&g(%! zD&!&AsIlZ7(NLbfOXGQKaK**aen5^1EVpRIL8i(FCQ*|HYRmX>TOWfo##6fR(RN_l zJx}1C%>da|(=UyI1q2hS4UZ#ZdY9JAn;04!#+} zOang0Uo~32Z%x6&_~ODYdR4CU(}eWrY*po+1h37&Wgk>%GU8%(c;_DPNR*ye1&p{iNw!4! zSSyuMY(>BAu=tauzUu-G#>dB#If(?>uv@ni&22dnzp=AOnH$s1AYD?=>_Y2X+~(2flh55T zB0MAsY!A(Ih8u3YKrlx^>8t<_D+;Qn#4&l+PULUyyj8mLu$m8O?^q9T`Wk^4%Z>fw zX;(XEUUePqJzLsg(&AG*ZUD%VYzM3wqa#NKQt9^5O*{61wqmCKIEM&(bbBixWcTO* zm$84@;`Zv0ev_gM{*hJx-E-n)E=gAAp*^|7E>xy81rJ+~_^F^R!uVCIzZJQ)CWdkL zPXv25bp`z>^?o({Qj(!YIh_A#>fRD7;g9%v`u$7QO@+W|4i}8hZH3q6 zq43jgzKAL{2w)L|YVv)&sCjcYYg2Jo z#?L>}gQg2U=Fa_cX`XXVbe4a5y*P+aJ3n4P|Fr2Ht)8Un8SPBYYz)g;C%7f5*5S-N zZEI)mY;Id)iKcHq%Z+RC(&8mHPy=E4m&&{Lnqw?{nsf?#G|PCC{FJpa#lU0eFMJ27 zHp#j>bDChXj<2J9_{qRqnY^(RzD%g_)R4JX<8rlTJlJ3Z+56)wB{_&{Ij-_JZOd|# zup7azwD4s47_eUQO9X~LWN3H?_LB)bCSHG-Az3;mVdeL;^scq8xq&f`X@W0 zPS$8F5{xL;;WifHB+`4+s5hZ>q_B3X^jFK9(m3K49jX4z5mps&2l&N<=1y~K4SFms0;0*%a31X8li||c&qWnFLg=9)|lYFj{xhi%H<&u&%m@MqWXR~hrha!rDDI> z*q>E4@R<51JxtBxD?)+-YuoyA&%;nJ&UbSsz%+~0@CA_la7mYJ{7}O}CyMG|&xoX6 zj)~iOoyteVLc$N*4i*k%28Km~vgx2Mj5|{J2T@|0{h^VHEt6x=MydnmKwhTBZMi!d zUq{A{gGQArz=9hcG?NH7kU4ND`Y+JxCmW+Oxi#V$_KI`h%aCfFvtGSLaX~zXo9Oha=q0&z!i9L0xEw*>1>XbK=kr6)3t_74K3m9U z2dbFaJ|oirj@6JW%CQ8SkoGpfBmT(l?bDGCmIY!hg#w1J{lNNONo4&9DLy( zV$c}{8_N3LoXXXZ4bCzHPTefNe|JX-pQmAD+Y&V`k7_F+e(587d&WXl!CL9sC3P=x zFJJ<3q3vRMk-~>yUTVIa+!UH(e%C5e?}`5XHr`t0W#wZVOPZ}=O} z(+FHMgM#{rgT?n&?Q^&BI}daAUF|3tj=9+JF9IBIsYEvp$gK@sBZXf? zFRu7p{^(~)p)2VeiaxeD3>bgyI0M>(Ji6ahj2wS`PPg(6UCAN!a`x@K z0^Poa|MtZ}8S#3}>cKD~PAkTRw8EDLUpRlQ!d0NS(SG+Y(s^!Sna9B92mnf!gy_$C$_%&;go_k{h(74p`OVc>E4 zjX%l&9ou?HEfOJzx%_l?9wT>12*TM;tPLBPLSFMtF_oP1mqdYldBrExo(TdCnBtZa zdaYJ`SXlq1QJYhN`u7yfFV;s2s(%dsCd!$7GCx#6U+3*%szO&%Q{)qzU&c%vpKSzQ zli{=Xc_i}?ll@it8IB$e<+BK?PQ#=&vS3aaKIW^_UdgwH6N zB@o2N+SucCM4|-Z(U9!IGgTBZVML4!INRa0hP+a9+%!r{+J|SF>sslxOig6+>gM89 zCV=Fgwthami`<`4X(bcEZ3Pc}e-_f5zE4 zuIpnnpP4x;QF0R@Bgl@UcbC9*cQYUbZrjD3!<5LKhBQZ$pSJ?~)sIzyw|XeQ9(IQ_@(gT)<~2AdpP9aK<=TTYAcMBu>l z+~*4V{^SV~=QD~mx6vN!;BOHHIJ50AQL?tOy(-rLrHp2YT(xv6lzXo-Do?&M$!v?Y z3TX@Z;fnB_Hr~H^_;Xc>4Xe<>7&A>6mAZa@JUabAnhyK)Fz0!}m+8LNJ$bJ8Z6-M! z=EGI_xb5)CIF#$q#Yk^#2=NW@@67;gWZl#{xfC?cp{L*MKE1?Ha<7^K_H@CzjN zzGI}+`_6TICB0p94(yqbV}(+_lp=qV#LpCnv@}>;M5$b>qL@7YC zHp8U0#WrN1p{Ky1maG$vWDYsz2Qv11-0!`wNLrjUHsek4xN>eC;R7!40a~Oqp1(sO z1E;|QRJdsgAD$1^Yh;^3eNDz$Hj&t~PwB#9WJ8DyIN*i4&|<$|AIkT`_UKG;5C6P$-heZ0UF%YB~n{jV~zMg(Xuj=vJ6>N z;%bR}1`s7jd>(Sb7(3;VzzAu?s0CqDenE3SG$CkIeni8aI`8!=(HE;hkE_ zIcdm_VLu39Caken$S$*CK4A^w3YZcAqbS^QtO09pZ$~3fXTYWf5iEVOVUouui`mj7p@1E zEsxLjVU%?_2ho<7KVDxB^(u{;EPot~ADURXdAN!ZP1M*gg_;KFvod;Jy}#&?BE@*X zaS6qX$OgvQQ%5h8xcJPbU%!M-Pk0YbSkPL235q@sRZYKH)moW!8{ju|!MHEXhu-DS z`n18276!jc|2yId5qIqr3IYN)Ha7ege1d{P!Zw0pVpdQaei43tYf&M78*3{YsEpMA zr9kL(D8mH*=P!+Wrd0s0O%}9$#T?0v1>J-u^mm;{nJ_?cpqs+F6ry=r_PRE@MmT`D zO1?sOjKNq}w2SU5`&{3tZ0pHv%UCdr^Psg2J@T#=(A6N^(;!S;<@;-=;G`j3OEx6R zWqPz*3(~jrI8=$R$rG^*6S)VgV9Mea1o2z1-fxBLh=o%P6A$IMjKS@PF>r|@4-{;% zEtEPiB)CHS%20Xd<+$hNv91+38{V*b$m6Ccp!~_hf5Z#2VS|&{!C%{jxRix+utfk4 z;o650NMxb(m_2;4a(KOb=!AUCU8KKkAGKq?w}Yi1mUiolSF)Qkz+00m0*f z5kd>%sL`uwL`mT4Fu>$m*vK8lTjeYMVAjlffM$g47Fg`bwzO#!<3_tAk@R3&#q8VB cz;k_TIZpc)M<^4q)=7uS4`5~nX(|E!A1SiqvH$=8 delta 7282 zcmai&MN}IM(5_qD-HJnTcXuhFXtCn%p5jml?jGDJ7K&@pP@JN{2@tG6k>U;o?)Trj z`d4=~o7v3sob%3^d3{2P_=nVZ$O@W@kKiMCv(>GYW|LiUuS;b@P`XGOdE#4rVwK`V zZB{x$a0C0XDfV8vw&a{ScQPk2pUkLqHLY{AGMhQE)?MGu@__Yy7^2iE^EY~}^Iiu= zUo-J4a2^b^_=dbsdwD$baQ+g|bzP_((h8Lml#VpotnZvLLKUuq$_O`F0zqk+GgguH z+CDQoD^`G%~NvAE6j%j>a)nxpqmriWF6v-XnZCta;HGVq=iT)J(VAl8j z3n zP{+R72{%~LC+Q6+G^k93nU-dbmoX5g<0q_x{U-fGfVTS)K63wNmM>49{yaSU&(Y`# znHHDOcudyw{z+$1l8c!A)BTo+rylR7m#(BnuS_*7AQE)1oA(zzh_$8=_SrvgS*Gp{ zr@op&_BUV78^=jp32kJ|9x1=kPh2T3Z+oq%xpcYxjrh#Oe~llGOKb0J%4qPPte#ug z|M4_d-eKqLtT2+EbpO$_jAT498IzZLmp>-w}i$Y&W|Dntb>D5*L#EkX+eJ%5d z=unt$xI?7~1J--pTCV=@Wy*#=k!goyq#eFoo6@58MF%z|Ut4{h8>qUe*4J&Sw=Ntg z-1fQrUf<5P73Ke5h;tkwAcS*-96{7saD1;TyRsPqK@tvgtYCkBUx{z&osQ*`nbbM9 z_W2H{7gBP5GFkMnO|idb?b59pS?un}kvG)TQdf563B-*WObof8AId(`S_>X0K!Rf* zdZmv1%o%WZeEz7f80_TB?Y8lkLg~2thMr5-MLTK`gjF(*ZA^fdyzGa)+CZBjn*o4w5pID7jQIt-R zi-n>wWh+NH#DNAH;^5R`uaOEhj8csdDIsM;Z?bv>Nx@sW=^-7L8qm$|o*lMYC+=h2 zc^!F9J#_@23)2@N(j&(a!I1ah^tcss%iGqI3R*!+1Wo5C!fqSzPo!oQQXI zn%?D(xKJGypetT@ulw3E%DFzCRgzgIhE!)|6Bx}cj<(kr z#)z4Z6xGPjOpj|Fg))StPVIHdwx5GdFlswjB8-6rM9Ui-849E#IOZ=Ux)5|krUJ;D z{QXI~h_JWe520Dt*5j$L$w>HPr zIyrLK#UXt`KEdR0o=u-AHZqW@#mNbqrp9Njx<{ifw4RWO>NWL@E!>hgtUO!_H zcg&`p%ka+m-QJ(=<=+#>L7Si~GZLBJwX2&NZ>^XV#<_3pthFy4xCTSJX3##f4{N7g zy1WK8aD%CvC-7O24=wKU*c=DS(3I(-HesxjYD%*h#^}uVQSY z8SWX+vQ5&7wpUasuVwOAO%C-o%r;USb>Ji~a&1o#U{itiu9k_rDcxaiKJ-BK8}>{& zHXEi@VT5Bl%4}jrD1jT2VDH<0F8ROn$|1>0f*W#zL;^Xk@?#}v>^aP+81{K6f^m1c zWAN5~Vkulz1{tLqOB&F7M%0;{yMEmZb$?2tFO}NFl=jgmf>C)eK0ziZOLO@)<3&gw zPb^?id0SX`V=2{=C4gm;$wLK=^t;OYh-f93bq7a{Z^9fyu)LwKj1JKcOblpIIFv({ zfY|;Lg~`0dzy?FXeDeHr{x1O5_Bd61GVK1g=yAZ3;=}cXmi&;``v6k47by694AM zcx}V1a6E|JYFLwFwB}h}JQ2|OoM*$^JV`tN+SF*{E$fYov74tm0LnZy;=-2-h6bUK zbyXjlXh9#_4}|^!r{huDRZr18igmx1_La=Z)U|k;r>=KtbN4rcYg2v2jy6}Px3`_K z@WYuVX-Jcq*y8009plC*BBg${`jU{A;GPnx0AqVdNVbzz!AkfKS^qR?vK(YIB9UEF@mKGGpr%ffAnT zh*IjL2!^%H{y#IR9u^4EU2Xa8DnXA8nGd4e%?hFso_CcsctWviWiE<$r&b(94qYOv z7s#JP7k}Co>zmvr=bw(hw79Bvz_p6p@6PDD&8j3f^Fi}BK+dUu-jl)X&Rr$<_kUGU zCPB)+{u}7d*ujTG&U+Z!$!=4@d|Lb_%ZS{KNneW! zAPFcT$b8GzOj1=M%%KQF%T&)HsF%;J2n9XyX?@62C?bzQ`f8&?k7=v`e)2CuYT6&r z*|jo9vz}rGgo~7*VoWnG?aFjyjWI`SaShJYzjBg%o{D|8c%3OKKf#X|%Raozg5Kb?p>gyCjms&!~~$XOAU zYOg3!lJI;=W)5eKE+vQ?CSV|Dm5ydMFZ(IAZ^nqT?^1}7@2RW@q6Qcj0~lEX7)c@Q z3;Y{FSpfBI3iXWg=C~dD;ec->0M%SJOdY&`@^fY34C#W|j_RcJ--+1U077{c^KtTK zzh%h^94UdfADL3kfv_R%;X-vGHq2yT376bjcuS+iEEe^mvmER;m55C>R& z30S^+j*VX{EQJAt*{PMNf(e(yLP#}jxrn{PVIGR9$Y|P9%i}{0`WDFzE9Z)(Ja5`y zLHkW}Z+H&OvE|$7H?^#mT6$Za86%*r9KZ+EPEs__ksCCRi#UisYRm}BEr^MeO9c2w z`C5myc_Woq0HiWY-do9^U$O}=(P7&QYOXH9-DnH4cfYDtORBpk@_Lx1v|RbB+>5n(+sW z{c2JuDDbh$<}x$?>)RGRWb^fNy*KT74-3E$S18Cip&_%nKY1(pe%`mS`s#A!>kXL& zZa2r}hb^@=#KD3|uq)I)p&`_)Y4c5so=q`(g-+#3)59vW#hI?wDRRfp1eL75j>($3lmyMt`b0qcb$*L4?}jgz|9z5+A~P zr!UO;nq_Av3>mQ#rp-Fkka;c8x*Qdv<8HRP;4{(teQ0Kob*CoC06}`;SV2!Cbw|Ht z+=kU(^Pgnc-wk`{f3lC*=HpevxCBbPJc642{VCK{O0v#GXtR0rJzodBGTtPMRF_Mk z8YrR2wBJyIeI~Zu0!8aj$wd!4#=a}-PmV^Yk|#X31qrfQZB*c*Dy6NAduCTFe4tMX zJyxLMQJ3_(MtyM;%bEGIb<)Ia6;Q0jf^fR9lbXy<2kz-8P=D>JSf8w-OtVN;`(ad{ zb!^AazQ5ek&!_Tbn5b5tzYZ4ii!e^njPW)aU1|T`lD#pSRm+M*YwO^C22^q_8O>(# zPUmg$46S_Qhjtv3vp(QTm;S~!@dRs~r5EmEKlH2Z=Zrbev6|fJ)t&LmYY?=HXX; zq49}Eae8b;+IZb$RcTE%v8?hr_-m)*p!IERbC6`wF8AF)YB^&d2lu?EfZTa`Sy2;v zGR7v-chYp@`$q%qlf546jtndG?kusGFGUE%^uC`cg%qTbwF<7R8Fu)b@&|Miy8Jr^ zsDG=;6K{3bMLAhBt;%0UjW^9IR^w@e@yKhFj6i zP`xUH$h!agHxs=(Pi%U>6t1$r2bKxouR_r+io!~rh;<+B5XIfSF}}(taS+!wq3j=o z!0QZSG2Vrz#bnL)H;(z~UpOGfY{X8xhPcJy3d z!zU5qvKCI!sn};HFeQ0%HKB80&==S4QfQrIT|`32i|1?IlpN$wG@*hgnpZc?4s!M&?L z&z^d%b<1^vhT9X<>p;B-qFWO81_PnBle5a}jiYCgPdM4z2xH278`ZGk;EbVV0*n+u@DLyJ%sGg%(7dTJBS!qkm*xg{%5|*e72Wr z5_tLT4pRP8mJsdG?5{h!#;&b$!k}A*rH1uFUHor_VE@O3i7p?^4TfJ;KL$9Ru4$bN z>uMmKzl^bV9FH8JZfXf#i$KkV{p;$_f!*c0+=r9yhk6raQ&>@p+I9LZpSoba-RUw5 zi1DS;Uir}4Cy+5`A1LZZ2G?5+xqB4q_HXeX%NN`x!m@-JBdv2+09G@GeInILY6MbT@};s*^*Y9h_iIsT{|r#esG($7MiFY!3miV>5#1JN^>*1 zwI$yL4Hu20o5mhCeq$z7+tLlnc?OUyX~Ow({{zlLNrQzd*7qv! z#K+n4LD!Oo>@QLV@pUcfxSXUudTC{I-To^Mmwd-k#-+B(>az19<39wZ_70lOwMfA; zY(a@Ls*`WiDUzSyRJPEFs~s0Hv6z`0XaMHUB646we5nRY|2@J)`Dv#y=!=#*>x8#a$LuFhAFYFOpmW!!4GpeZUK`6?LX-&bwLLBk8Mo$L z9mZhosS))r9A6?_KdIC&9O}1~tZhd@F#~tv^QGgt$HKqGm#-7&P`^L3C82|6TP7J% z7v<3cXA%tF2jxLsF=A~42e61#yr?J!!8z@Cpc5~`4@(8D&B4`yhI>O88tODvXJzuQ zu$>^XWEC&?GJ!=k^j6x9Nmb#QFL8%9NDEE1hjB16@6<|od*Z#_Xds?>l)VJJMU1+P zbXENErV!gp3ZI0&M}P}Y+m?3`yLg!|)%B030&9A~RnZ_(Dz44pz_qcUq3J-!9D3ar zhe{MxjGcPqkv6{a6ik~?DDT~yVBI4YqTN%ftRlhj&$R~r-Nho`^z8)wE`a)^zL z%^Ft-(f4zPku8cvXYx+ERa6@%e2yD~imw!j3}p&aVvdnb3?3a_)=ghRcrBziPV0S* z$xckEw4g<>&b6(C;%@r}d(Y4iz|~LznqXN)QI9(}%Hk=HEUv)ET-XbW=COr66RFRA zcW6~Y5e!aBdh?6RY(g%DFik+6^-w3Q)I+J>ql68O812A04pvHPa2Z>RhkUGxg5q*` zs+O8uZ?0qv!Nth{NW zPG1&qeK6Im69^YjvIb`>=-<_<;UE*9sTMgptx`icyYM3{+--v&a?B8U%)*zH! zRS`hJ`4#_Ziz&E30*>oaIc!DQW?^>n*GbsjGE2B^aP~TBR!?+9w)(+ zHO`E{GU+e7ri&HT695(n=NOHs5qhtkIWz?n;3MyIWC?#LnN&Q3@5O+$JZ4@VAE%yn ztoZiYpJe(>)X}^sL!J!{9tIb$+cAt_h>qI*6jq0_9hNg&kHg^?|3JSl!V1%Z^|X>( zLirk(=;N^zd&4E5wCuqL|L90&Q_gR4z8Ba;T3OFT?woa(=N~o}6aUsyN4MrW1>zDG zb;tgI3zt8>GvXcC{;Lm?fp(|ZOheb71!(o}R#Zh&C8}_~CnPc)Vs(h{FeAjmqnbG* zXD`lO14VxkXPIfD`Po^^-4XtRVGkjkKJD`wUTL+ofvVcvNxJXWQa+d_vx=J)#2dXa z5^6YD>c?x#p17X8ITW!!gN}#f;3W_SQ#rLMMvE=7&920K*dGm$~$1-eU|N7)0))Ma0 zCt~Mt?Hl2A}q&P22*;y#bv`Ff}Fr7RW7ZjfZji(Bg z)CGg1s2;xR2?l~h^&$yPTYCM0T3dEMFikA0W${HUw=dr`{$&bb(-B(>kC&$AST5f9 z_jvoPLvo8K`6g0o{X(GEv5HN~zN!sGA@w)ago|aV($a8rYpg`ao5#dUzu}OKqr)L> zLM8PR)025QzDc12ImcZ-D`oz1vw?5^n3Yru)I!%fo+vu8!%9=advf&>Lf?b`2o1NMvre5q8`d+e~!*jO>Gm zkM7;x2_&OtyoL2Z8@T9Ohhlb<4#MMrdNHOMgyM_#3ZN5{YI&DF-nSUlPYpXr{?{+5!pLbI#h}+l*TU*g z@c&oE?FB^y1Vrs*r2c;$VD%mq(EoaIbI*zk4$4!6UIZf?!aE{5RFw#T;EuSC-?1Gt z(H|Q`odrh{b+R?(`+vX=zap2FB)$qZ?2FD!)3&s@B&xFLCQePtQ;d{SFIkd`Skded zM3}9hy^q7UVGAPxL!1i63w`IoIgvi&C8}Qe_Ma9~}IN{4npSuN5fw9_`;4&Yk oOM+#I%oy@IQdLtU^1x^Qef$VTVLb41FFl4B7BjQ5)(5Qr16yVRBme*a diff --git a/units/unit10-linalg.pdf b/units/unit10-linalg.pdf index 3f0ded9e8a42b0bf605eebd8e977cae6e117ee7e..ad2ea23b29422d97291a34c729c0a1cb668a05d0 100644 GIT binary patch delta 8572 zcmaiaMOdBNvTble(BSTF3k|Yx2_D>a1()D1Kkn`>!6mqBa9I#M!QCae+uiS;(|G%P zjn}H#s7{ShRb$So8C>uou;7D#BaHnLzy$&c3!^x@IGLH)p?EAABq-YSgHS`y9&iQR zq7&nqM-eHfXqinLmo1n2y`1f6xwvB;6D8hWAk5dGAhbBc(Z|cHdWIB|pKrXRH4gr| zOeVEw(*0P~3o^SQSjeMc87ok|5+M;0(95VU{9FCoqMv^5UD($@oVavQYTj+Fn2gff77XYzCqYv~^Rfcvpw6tiaAKsSNiM^vW001g*Qu>q5%2yb@9<3U zNUKwZ#ZGHqhG76W{}uF%icJs*^WBC!AVnr9%`siU96q)5zHcdJ>D50yTd7Po{Ba5p znUE#NQ@&5^a9BIs-z`^Pm3bzwltQ4k5};*8^{R)JL9;6b<6$mFIHq_<%TQqQfx7-Vz`nuSfTUIR&6Hq!1 zofaP5qi{vDhpx4x7z@rPP6*YkDDezjlB3pBWhwAm@=sC+KN(%&cUg3TvSbZ?9LveL-$BjmR^;! zd_43QvE{l|?fj7|npZy6`}JLFR<7TFkQH0~2U*^KA)BmdUkJhsJ$ptW*ZaZr)3_sr zswkNIy?WOd8yuGl#JwLu&^IkUr^1G|%_ts~^ zYwqQEKZGEMPp*yPiIu0XUb(hLZQB%@l``S?cES(fhssu+mj2!~wn;-D8{d7Ut-ZpP zF#0DbpWotIyDW{0O7OO7E02WDY1;0g zxnVAir!G7boK^g|=*LE2Xbd&jH?t9qMt+J~b1$D%Gq4_q10gXy0aYgGFPvGlkiQ3t z0ctoK&0YsaGCgQHF#QE);Djb*0kH^+x|e16X%{q0(++|&kSVixMHWxtmg!|#4=^r~ z1wp5fLWIYOVRr*i-~aw-Uun#GA&ruo_^?s#K?g>A&!cwG7@A>C*{e`6OJp7jpN5+9 z$6<6zB<2K#boL;RvX^$7(P14LOXb>&# z*Y{Ub4g#qcI-#!?ds$I;SD2yf5(H;v9|bJ{VX|!yTSwmuS}6zez<)@Gti#B|64a;F ztiyZ(;(YF|CZTs}t9ssSnG8PqO?mU%!`@_s8g0}y-RlCSkWIVh9s@-8HQr&(iTwc8 zh8B)Xvjh7h6P+fcJ3AupQ|3Q^CS-+ts_y(ufz$Rc?rszLea3a?n^Y@f%}#063v!y8LLsGBU5PFDeEICq9XE*)E!Gfk9vFE~7r zrC@xP@4%|R9cN4ji5AP5h^hmN%KH*tXAkvyAiMJcRaW7mG2R13)(uiZnM+8+HW}ZF zT#Wf7kJcFIz;VKxa`I8F{)0-f3!DlqoR$6zN#7vyX5tjJqQMYCE~r5~leA7G#$p^G z@QiOM}7|u~TTCsYGmT z9x{-GstFDmK*eX)_|BHvu6+dIXt@5(wky~H*zi6PCE#Y&V6ESbijnnwdlBR_Q1AcE6O@!|g7a=(1I;%60oLY(~$tGyAmO9QV^YY^WK{ zfbGg}eMA5oVG; zJU5m(&z}+l5HLa|=c|anm$wS1a#QK9btcVVhM}iBK=D5be zoF?&@@`2W)D9Wtm2o*euW>YruTOFe)F2P5su`+-Qe9k}Sgs0tRQ>?Hf)t=+M=d?^} z2(>(9?_%I2(|-5`tuCn&!Zkrx@_TRH#>4E%7lCoSB=V5kQd$m$kYQ95B{_)O5}h+& zioieHWtJh^{N6WMOY3YnfC$l-V18EJPS^d7{h-5d3Ag+0olK@_I*Qph)Bj#D%+k7) zVUg10cm5vN8QA|wsy$xZ(U5CK3#kH9J`l$$b1R=HYimma`Fp%`fXL- zTldF{o%iGK?jp#`{(jnzw;SNl##`8>W!U$qWBK;^`9aIe>uLEm?+o~Ci1&1joVR{f zsvaKEvibMrCfl%ib-S*$O$T?-%R#U+_vvmcXqlXRW?1EE4MPQERqbMPZ9T#1&Q0J& zk-c0H-?nK{nJ<>gUE+OBsax7(v$tF&G4C_)pdyK1md4TThmMdx{oBCbwo|JnQcW(p z@z9X~mXRQ9<#(P`@3TS`dGkfpHQ?UU$x+!zf9#}0A;BqYFGqjqspyQ5fq78J@ZooI zTA)!qF2__?o>$YCB>Hvb)g!L(%f|&TDGi5Owoq*D!y$B0ZZZE$J-ng{jmlgDLnI+4 zb)-`>9PWHo7*i}Mn)DH{QAdcIRdw$fc3&^Sr0U7keo|0xeCwjlhQ9`a-bY9sB zA+lm_&Xr+FY}AksV4agLr@mV8VC;e+PzQzq85%V9>hOR>LL~{r!zdh~rvbB4|^V z$>~8?EfYjCVwjMnVtI)paO5uG%nwiG>?EC#-uO{HaOg;dkDVPuA6p0~FXdhFs9dm< zrz+J=`qhzBnq=mL7-IwHkMep|vGN8dSNb^nKNs5@mv*(@$4{H2ibj~%x8z21VQdD% zoJzy92U5V>JC<;G-~V*^nf-rii|=1eK1t6eNE9<<{RQ_aT{SSxbQLc!K3zMwBu$^e zm;0CocZ-SmDjqOe+#bvUE{gU&Hu&8!pj z(X3dI?(Vniw3lBy+fvJSy3HOR^Yt^MxSC@{HGDeMAx}Unn^|b>tR93%OX}$9ZX+u> z{@~=@;iA63w-CNKj@`F@-Nv5N6*1jui+&9{=~_;-HB*8#erfq;;cd6fHE$|_cnAAB!sj&%nA@f-U1OaWcpK0!E(cx z<(C)AZU79BNC!GG6r7;F*buxyFOZ{8inw=Fp91yQyY_tR-D8`(+qI+F75x!|Vcz7)cx%j7_J1Fw^g~qk-pF7!0}U!!~4^+8PmG( zVY3Ad1JY_=osa_CNy?1UxJmJePjQn}@Bfiv=bA+~utwf4h;?#}96HNAiI)&51E9}X z(R2lKYL>)@a+@yD?@&Fsk4P{d@Fb>K*vEtQoJs;f&f@8WjyfiqqQshe3DF+9$Y1D# zOFcuJ!jBM-SH!dhDXU_#1Dg`Owhk#}70+4k)*&xuZs-$4_Rmbm;L-;jr2=<(j7*eN zGVViNuc=n;`rj@#ha+-A0+-_2HfuqHOHS0JI^L93LZtj{2q(@SL~9+`U)o)r+&$0G zx!8j<61X_HRO6~xU-;{9#kKXAJT8jb@xP3B7Rd)1&fq2*k$8H6h3*ES%yx>~qVYzr z+(ab_w?fku%OEhKLbkV~i=tYIRsWsk$^Q}2w^x|+JZ{hb&6*bE6BzrMf2U28u8lpm zEZ$3Z&-ZRt%Vftca%LPx`1*e8{_}LPsabM`6?pmHScnDuo8M%)^|IY?P7|CMw}C;N zUdt~b*Rvm-vM7wU$$feW(aKBVJ0X!UDY}-Wl5%|vjLCC-@Nw4ERrB|5eX`w!4#MBp zQ0%K}DS`KS*~)Sem{ZkN;i0F7G*%a+Ah;zSGop}CfAOv0_dnRoX_GHm@@1EU>3j%6 zR1gd6iDciW+<*>jbEMHSNxE$%MbcOVQFh@3zcX$?og!46;6G?L6jwy3vZm&+;0(&J z^3a7S>6(|zS<>n_Fi0MtMmIKNVz@RXkfbOhrvr0^4G}E)4TB=Q1##PQ*qAr*Ukj3M zVYWih$!svC#kr8uV*)9V#-oS8N>V1M>JCH)-tTs}f3pQd1^xO>H22QMf=7T@7uM1< zKo(qu9imnHJ;7%+M;)Xnd z5vG_`Cr%t0+8TGm;my1bR}fC^^~A>N9n}1M_&obfpC`$aj*U3BG>O040#`fW?D72E zh@j_0SR%~E}47vPDR z=W5Z(%i39E>;22pyCU|Y|Et>U>7>Bkt%Khi16J-Y3*~8zh_A7ZXs4ZnKAL7_FRMGg zq_$|gvK>@cVO{rmNLL*rckGO%_cT=6g_Jv)$+D~)p{V+BhO>F}S0qNqQ}GCi zS&WRi5QHxMEb%yd(KH`pgAh4{u^ue5;xD-HU?3B-d;*;A{NkJq?OIj3HPkDx%dmZs6De&un#b>$DQR&w zm#C?Ajcc!M5CxEy+kX2Bw5+;yjjLd^B|zit!{hk0-3o#rRQniX4|<~BDDyk5MCfMF>Z~Ay}2<;C|*VVJl4{0csFJ+F&842nbGPVGoMppF4`B10^ z`0UXqUd{C~#j#o1Fau#u3X<3~uOSMtiGgn2qqOyAjp>j*`=pY%kkSl=3NSVAm+f>- zCQ{Rddw42r;Q!#M&t-{73SsKw*sdL<^*H6TmqdA=H1}HraS(YuUQZR*!%e-!%`yNT zy(9NabeqEb6xKeFZ4Tg`!6~Oyj9@eICJlRLYxHtr4PTJ;QPtp32#%r9>7$_Xp{OSA z#`s6LM$Z)~+0~dH?^J|?Y`R!TSvla*rQVG-ql~2637<6U_7OX<(d3&=V+PSALkE|& zb_G*gLRn9|o{J90=L(?QpL8KcH)3 zv$OP_R}Wy4#M9Q(D;}9qSZHTp$%ApWMWeY0EGQsrY*L6j49R9#!526Ks?w-k3{l}Q z2*kFB2f721{?Xby#X)HD1sv^QE$L{wfb{5e6~ekpQfO9qK(9k7P>y^iK!FW)jHg_w?<&}|nrfz2oRY4h#_ipav z2T}>FeoJyWI9Pj&k!`^#Vp-a!bGs{pJ-2hQC!>)k{-XZ$cnnvX7cyPR)kk z?=39rNAZt`IX6J42Rqy~#1x``J?TyN;GeQGy|tQXW;up32{f& zads}_C9{f1^r|=>V#mFVvTFlEVv0-QKQuz5s;G?5gL{pk{G)>EF7CF)zE-6N_X%T2 zLhTkIg=CGw*PqUr6Rxt}MW|zwA=*NEaB2lrfZB9Y81VYUOhDU!+yW8WMj)g2Dn#|C zrPK!EmN3*kD)mpK51)PvB}v1wA3S=f?=L}#er=KXD|-`#b|yL|f5E!ZaqR*d zi;#{w!}0#ec+R!~<4V?=$MB_)f zUzf&3Dm&kCP>qD>;C`y)&5Qml>&~l(srXHu)f=nOZp$Z^gWf7<$@uAdWMs7(*wVdg zq>G@qOp)*|&Aiy*WrT0*{4=;49B$v0tn~*9H}yW&CaC34-X1@*-)-=^ZG6yhzfO{# zNcXeD_4776`rQrbs(%BUT z?J+*mF*V}n2ClKl88mI^Wq{nbb3EM_T6M|qz=Ix_@SEx*$&2%=A2<`}RiKRW8vnWi z6RW@V?$tj{9(g4HtB^-1T;KhPrIX(@GCyA%LGEO5*vc7Sf>2)qgfj==iQCzVOR@pb zmzyq(XX41cv!mIG_KtQB#3i#l-S$7Fb#c$e-tuez^gCbPINY%n2xLUeX@^~~M5ZkP z*QRYhZmeEUjuLcbWy7p|A98C33q8D?j*hAl`^MQawyYa8FOV*7b=cOi>UHXUjXJ9x ztg1thC0*!Sr+-+cPK*KaMpF%+1#GPV3hnyA)816!ho^elHPfi2Jl4rr;b+(9_QHwn z!@&dLvaL_$AAw*dWLsbT3kLzx>Wu-^YMM1h#Mk=Se%vC#u~Eam0X^gf(mpo5J^{TM zGTTKaAEOE!`tD!2T_Re!dD-0_Z%*Y=o$S@geOL6zx{yRk&gC!FC#AD9)mG2XmxjC1~*RK$M^G}@I zy!>3893aS72(OumF!=xDk;V#zl>xZ^HI3l;ZKbSXMlyV{4~Qy@;z{7(V&STwnjuNK znX2J%{rMO);uOWnrH@&xiiym^Y+Im!BQb%ij+4BiYs5Xlm!vp5#mCRvRgqDWky+>6 z^H%yP?_uIB+k1a)9}MLfN)N*zr4BMj9BKudu<;MwXQ^faI_ZJV4!q_NfjwiqJWeY7 z<>-QpsJ81sbgz7RbTRhpp#nZs-zoeT2foLz?+_uFu|Ge92tGrQWBaBFBELGY#P)+Y z=zhLD8F+#V9WzK`u|cr(TQu3x3492J4ophe+d0-fT$yJlUrkXsj1)GIy)cMhYkSU7zzd{L`-$mdEqk*h&s!?aXIZFdHB#b`Yy?Q9Aop zS$y4#;pMNtc`vwQ(;wOGl0>&DOK#YgX=YHdW?zHs5-mFiTzHn~GR(`;bIq3zPKfBt zDQ0-fx&0vB)eSMf`Zpne_ZFr1P5jf>`|OgWu^IM&Go0mv==q0uHcpq&Q?7#`DRIBL zm|y#t2_tyRrqc(uqj<79_TD+CU!^X48Wh7EyNwp;!P(=zoag7RfVoT3aa|dxf&?mossve-P z4Le&@^h5xgKs-7HA&IT->GbcvERb& zj3I7o$n_yYI`@2pph~RXRh%PaBG0X!NnO%JORm1~Cj7&$^lXo0Zd^pVLJy%U;;+6% zZoLsJJ|wOKS95a+p82d+5`Mn0m@k{2Bk{M=8cAO^0eg8BtpUg*(d9?|Pyu=OE!s@L z{F1e5(BS&D%wqOi{cTijY~q-87>h{0DwAVfJ784t z)%VAvy##??oV**tnaXoXbqQef8-+RFst7RtjcsIsJ>79zp@kyL3-LnWk84MQFAaIw z{<7)fZ*=_RuK2ogZ+M>k)sm~U=o`xbcf0&S?m4Xt7 zJx^Xdv^HT+fNTgI9QawP`ytG|8T+rrj;i}y&_dv2=QZIW_Vk5_zrd0}p8$|6;PuGv zQsv^~!hPM-vTZU4xFg@B9WPD?d_HG5`nD1=tn7Zn29Bk_K+Wx&U5K%`Ajusj`z4pt z-*S#}M7A8IrQ&{{wQ?3kr36I!Zr@1SU-xnx#EJ2J{A|n++jmG1ibW8cz=u`HA-D6b z;7Rdm*sw)pxAmoM=G3@!*&8VScDC;Lln;GvR!>-evCUrmU9p6mtA=dx=tVpw;f<;5-L7}CU`XY_;KN7r` Ay8r+H delta 8562 zcmaiZML?Wu(`C@$jk~*RLvV-S79=<{?%D(kFB%9C+=9EiJHg$8(?|#|!Gp`(@Bd~o zcV;oGszq&1Rh>Fd)f1eD9-4<9*a+vc3h)XEiHoCoc)DAeJE3{68pm%sgm7aIZ$7h~ z)>+^KjSHBv2c8u5UKCAM({}E{d5kniGOv%{Gq25q5XEo1#-A^*>Y1%X0$-i*Yg_}{ zll#nSXgXT9=1$lS6_~Pr)!;( zdc_$nWv05Z-yt7i%^7vH0a{M3*NrQt43r3KdNw<)T}FcrGC7El1{Eh`vUB?EvM-eW z#S}?GKr~5dJCkOOWee)ECdW*~^tI#8!uYxF$qOn7C$AUW?M?&`gBuEj)bCTQp*?URmG&AgjBH# zJ||4(&GL70_z}SZlgsHc{7l-i?0NU##7g7;1NoRag~<~kWDA~MkHXuIyC%ADu#*?0 zeRY<}n5NtEI~B%|x#!`HJpeWwz*6hLh7Ry)k7(f7pW&ZsAZ33R`~%PAr6yhFlSek#kMs|)Mh z=0~^s)G4_#e5WOq!gJX5U*f@CL$U6qE`p24x`*X1@t1%{@`u$T;L?uIzeRnxhIJvo z$%g&8JyuuYGQh9ZlnHjW?t@*SLofRzl;fZEWGq?<+p+{WreE$BAEt~4`s+kPnZjX$ z0kDmBMS4xm&~2?&ei?@vi}jDD9Z}zONBs#PI+J=h`b#fFdZaM5{XLgIsk?{PCfW2p z28gT(c5xGuxQ#=p_8V|$02Z5vMp$w#yPQf8J%k?NXYRfvi3JyRwlon!KKj@DY=m9Z ziDqhR0FFmQk)|2WZa{&1{31YT0{oS8LXrsg==+&uX#z4!+AaZ#bvj}L#%GO#{!SVV z%8@uTj*DirMO>G{Uo782*^xGVb8#AcVQLO(B)Aj!!d2$h3a+CXraqXd7|(Mf)iXTb zUPy3_c(dVCqwoKT>vou*h&%X-@gh z0Ia*)d?Y~Tcf6q&CX96+H~ZT>Sljh%Yp-v<04SaREw|MsoHD$3Xlnf?oH{T*J()w* zamLlra$^M6ylb_Ws2GmD73D2=e{b@+2dXyM2Xh2EFitz2Vye=o5sIu`EOy3cHs|~6 z?Lnt7@Xe}TbxlU1@7>3<=aH9HXwKQ&%WUF7UHQ8yG(^#{;+g6P;EquE{taiJoQY4++0hFza?nTtMbS7xj;D|ZBx2Zo;NQ~1}eP+Sh6=MO-(jC zJb)F(i@6RF;~wSL>w8Ul`T5g`1*(>fuSuQbw=LtpraEqxzwYwlLIj*0I752AhC|mW z)^ZM4GitJ2RujGNCT_OIURG~6Pfu2*24?LEVw9-*FMGz93p9EO^+CO3swwm;MlsE40@NP9{YJeL)1dV`)({C%P75pu}LH z#?Y$=(T)0gT1t7!=G!k8*vIADYwKxuz0u@HEhl)$&s~iJpXM#0QDUG{;tQAl>ILFT z(-+FpL-%vzP{a1k6kO@^RQf|$CC|+i&Z*=1FhSd2{_z#hli z7g7eUYlVHQ;&ldp3(f^%(P>ja)rc~T0w5d|SG(U~-C4x=ab z{OIUeP@_K=RN$Llu6lg>iu8F&y`%3|BC^iX8_mku@_)Z7{_$K=XU51{Uo1xqJf#3`$q^$;qr-dhSwQEh9V}oK2MOy=P`M2DZL(p{>Lq7 z|HoFuPl2`J#~1Ii=2wi8XNzv_>ERddotC-B+UbQ40YW!qoaOy)!1u2&XQmxj)?}3J zSElO@)yO9xr^(^GtF8P+|CxP66RC@x>i#~5*uc09z+cSlCe6BVlV zr8T7f?6*dMv-kf5gr#NO@332Qu`nj+hLjTVrVT2som!wqRB=V8ZCZH6=V_GO<}+U_kK3eq8%&d>IlnFGw>g_sy%NrNQ3-i87>Oc{?4r;Uo~ z9>yh5tSI*?{ry@6=k3XR4o2A;rhysRm;&Xj>V&G!CE@QZEfPNPG}m^nzTvH%&2336&2?#rjCz| z6Kn|_N@;l1igGt?G>2UkkgHtU)_2q&>)fD^sCdXucSlm>CcdF!SM-F&} zlaoXY{O_1w+W&`K@BT%u#C7{VN$mejc|SXr1giB)BvjjZ+;S(Lhc|9~tAQ(_{3drd zsy7&f?QuU^I{0k%EPt1*u=43JJ-NURGKJl$RqPHI*QyxZ@|CSRxpA$EL>O-up|idL z?QE;l`HWjVF0oXwzsFuJE=PT#y@JP)fJ4#na6eyt5*7>ql z*xg}l!)nbbLwJY>G>{-G8R;l9f z|5-nhG+kD?@#QzK*q)Z{$`wwQof$$A^AuDpvgA8%LO=h~lXk&SXJ$~eCg~tk014QqpXUs#R zfeJ9ygZ+M-UaxilI@muy;i)afx_kQrFiW+Ct&!g?hEz#}8`b}aG4T}aX1VNMRMXyT zbJ1vMtC%tc`Uz#T_{=0+7(>-h3v@Op9?;Np^ZZ^g+)LZ8{^P(l?@4(zR%&-;pGI>@ z+{E0YP2OI{qfLBh^ULL$Lo-FpLXUe|icQ7iC3vzyu8B)LIt zQwTuvvBeaVtG_%UNw86#0%KdNq43dhI!vJ1_K%4s;HaZ-9fqER)itm**Uhp-P#l;- zo8mvz3TUZ|iqWGcB=~Hplrd#%a2{%j5gpqoREC{81r$XVy^jotFPcjts{?B!V4u|WIKVj@8GaW^5RgsiI>7E>7wnpsw4(&~z zmPM2lJ~Wm5XT1=ow)mLJF6_nZn9Mc!8zE85v-Ph`>%wgvou9(L6ZK7ud`}}Od%u|S zLay&eQ^`Tl>8-UM#bj;rlR=8>5N(tzMpP_2`|&sXZga64^#~V0iaDq7m}$EJB|b%9 zoaz#6FJ_r*-0}e57)$(#{D<5<;$WBmX`2?z6F8Tj|CY`s1DHFWBJStSn$niAwDa=Q zRLnk{JqRIpKL*D6%p8ccf+vzYp+AA?OV6f9xq+7Q>5YQ5%8Vz}xV~O3du)O=zA&y& zo~4su6s72!f=MZ{Mp>AbQXzeGg0v!s%6X|kpTRd5rpz4?w0*jxs`DWShhj}z6lQw0w3#D_t~ARue0rm*ZC|)1u@yIKMSPNmJ}6{r z(QZC>bZcDx+GDi7KtsHiE*-v`J(ZZ@eQhAWVR<_90%T=WS~GfH%R=j*&?==oR5t-B zTsm~>hk^t8g@zv4s3vMyAfPBFoG}6~Z1`k>_aImehSLNI5enN;QJWn6szWxX2YUpr zFr&Etn6RDL*2W)_L@EjDek-YF@dvk6@W8Q33RCivZb3sHLtqL~+CaasrZ*CsFfq|_ zFHt(p2Xv}Hg}^M{8(QXwPh(NJaPM(@^+^>Fsi<+kpy6Qu$kkNr>;fXv6YW1PCXlMh zp24%=BP`_=M6j3{#GxLTQM#fVxyncFydiVUphXE4XOSYXdD9(wfrOJZqsW!gxs=bjAM>;sMM6*l$O_Y zm(^4XCz`xz7z_h_I}o;I)|t_eL06EO3tvC9=s~QyP0n5TOsD~f!pVFPAr2mccoC4u z@}9D4#gS)CSYPeZz7_J{G|^u*H0vB5YqZlNdwJ95Tw4I_UMlmGai}txqd%9x#wpwQ8D#FP$A8)ISJrYGoa5M#rMvOQr=gPh9vIBA zYid9BW1Gy#nPknX1Y0x9j$g z|CvoE_I_-aeFN)^xhO>ox%f|6fJNW)JFqk1^{?!{l%9xogJGxS+M^qhZmD6=+Bs%DV}h- zdH2{Vd4F2|Ju9=pLq~M)&{N+gq$^83eRf#x>eSjtD62O>BDN~tZ7Vy&g|~8W`FuDK2j!pKPG5M zS2F>BM9VX}dc;ay*ED41<{!59OLdnW6d{ihzo>5C=A2H(DNv+lq_ftRuk;_jWQ3S;C!CGF)rspDC9f&i6_rF{2MtIy?VtmV z2(ALeC;m>J@)X2xH;C#O>%V*s#p02GI}|(Vx@W{d%osUe44aS@3jHWz=o9Khr|eN3 z!>N7<{Z2uLSufp>PO@Qfu7$KN(>)jQ-MF*_6BqY2qPoI737&5!yUKMche1A_e-?v0 zbJZj+{7&)%;wl;AWY!1$2_*$aO4JDeMsBiK33l1v6-lMRdf}qeBA#%E{aQdqeH$!w z;h0z{7^qW=gp;pI+;G?U&yH4Rz#(;5DRP`#;dTAg@wRwcrf;vTYa;E+bV*|70^0A9!A6t`wuwm= zQT&XX@%m>`cmlPxb=OH|LAsR@I*1tA+{BJTOg`a}r!iHHTz~K>B(&~x-``fc&W9|9 zzaR%NxCh`-$J_Vy2dbW=!h10RXoL-Pf%_f$!rZbN+I@?*Z-`V@jtLXt<=NohVc+U^ z8#Z?8%2vp?@}(mwG^Gi>!2-(?8E((^H!_qODGiAAfK3U_JotX~z8xVx^v2S2*Jk@iAVqSn^i4<7;pWed%L%}^Qh7(W^V>A_7!*vXFQ z!b=gCThM*FhfYCF-Ly(+vOOt-1aw#CfuI+apWpe6@?_KNaPVQ#0#hOZw6y(}^dFqe zL&){Tq!XLgP&+Z@;@*mx@j80!k=;qgy`A3b-%_XhW_!2g9xyY+ePLEkg8U^e#9-9DtCi;;PM&Vt1}fe@8MoyrN7@0 zWRt5YX;xPJc3wD5ykI_@az@V|tty9dLFSpJ_DMVv7RxZb6k?;-fOS@9sJdYS3azP7 zm`Q$>+jHSU+G;g(k{(%wi2E#^^W7 zOJJ%p%1Ew-m~R+D-}RHFm)i=8kZ>2dDSK%6+L>UySZ_Q85HJO+?;OP@8Uit}rVBIP zXez%Hfqum2(*$gYM{g%`&fw}A(A%ot4>z>)Dm3x2q57EL>_baIE7n?1Vw0&T0o8>yQ*F zmisoD?qPI5!Z#p+BjgUc{PRV!);dZgkVEDYWTo6WNeC4*40!Q=7>`P&aLS(IVggAK z^e89a(P>MWk^PPgpF4C#VzPE+xKQIcPtAg>re4=n!@_0UkV*?c>2O4aIV!%bEz-No z(`Hfg&tQ9StqCrq+Q*X>s?U2^s6CkUCM2ek z15iYaSbB=9+Oz^xR}kLhd*2(c z&L`j@u5d-w^V<{7&yPX}kj_7G$TlcDoTB3pg?V@;9Ul~@F_$;iaRNYyMXD$awMK}3 ztIIDXWX{AdL;BPetxNRrZtp}cd=_cp?XQ>>RliA4QP&hmB(q9;XmD|hI6KLiA|WQ} zITmcffM=|nPkrWNb`)MOL)DN4NA_vPr_5CEskc+H#b+C|KY6Tvmfh`(?6fgubM+Qe z+xp#Gfydn+Gy|9}jfvz`t+_SPr+vHNt1*n(2bJsKn3JKAH#+?T@Q2~g#kWLFt%6C~r~q1ZPk1gvB^1xRH0 zs5}mkD6S>d$%9P}zKILsf1U+}Z@FF%zrh%w#u&mO4yNP5f4^mm(a$4D*v}&!jp_;j zdX&R+ttv4C7sO$BuC!`>wwci%Tv>-AF4P)c`BGk2h7(7JMR|gHZ;rs#H*;JA@I2e& zz{$dJtM9kYq>89I1(m+s=jiUjN>f&E-VJanUG{v4xPPn$tQDx_is-D#bOfL;`{fo!Wr+cb~P=NDx_3AVwlad|D1;>{@TI{ zXEE@uRKGcSQeY8zyjv1z$3hr*qO{8p0>D9#7saM;)gO&PhqU4s&C|Ic9MtDQf%a?} zxqJ|eF-l%fxmn>z9(HU`#g_1CSU?E8Ma$6LJX%D;q($gJwvOH&KxmddlA`9PqZeFV z1RsAS;bf{Hp;(x=CeY5x7zKW|&!8nWwu_QorCc^yl8;un<%F?saU2``85p($nzz_m zv${i{SapAjN^Jz`v?d0SaF(ouEr8k1J<^xls3w&}L-uRYl`lk1emEP2sY5`oHeB~C zmsdMnBf=!Q+jgQ~OI$GnhM49wnI;VU)8+RSMN(!@m_Eg˯O0HKmT~Gf?1@n|n zcarA}{=F8CcmCz!{r%<5(e>OTa8$AqBD^ACMrBecAzS_98~AEO&kb*5fX$oi-PNe@ zrG94&*9;K^@n{Ve%=J??Ppv0)1doRl`ryQ%-djej{L0&glIX_xEHFL5gOu1 zUtY=|Az<@TCJQkk^1mwcLS7M~RDT^K3tXs=+X`${;h!_SiOCM}c)sqT_#hXXu|6B_ zh|aS#X1+LGDj9_dGsFPDMZm%go{XK!CQep204|^-O*=DXC3|VCKb6L&+ctl{$}h=$z=wzDXQKw^hSaX znebie*4!QqE1sS0XaR4SsPRtCn~eQd_pnJ++SkR^%&oCoCpCPV z0hx@3K~$C|A;rvA)(j8h`|L@+0TvUdD|kK1S|R<+A(IUBL81ETJ$yHVc1II>wd2=K qr((g+n;oZp_p4B@S2*vzA*cWAzXOtLHv^A@!iUDdAg3;m_P+q+eU8`w diff --git a/units/unit2-dataTech.pdf b/units/unit2-dataTech.pdf index 18f464965c762cdcf26e27eef64ca9ea0b887fcb..f793bb8503939272cba3cca19322fae0053c9e82 100644 GIT binary patch delta 9364 zcmajERZtxY6eJiRxVr>*cXzwEySuwPT-*aeZ*VRW9D=*&;x560ySv-I-JRLmsri_X z{y0^Ae!9D=Pt`eDLONPP3T=W4JqDUOMzCXix$+eQkP6An>Q#}%{S z+fPrUE0O7={}ChUoqj!eKXg+UN#oWxEzBXEsCHdS9V!XKE3Q*Tr8Y9cy$5;{fa|QP zY}1#yR$3# zY?by4bEanAjtKvl?7j4gv470`{|PK(0c5DSZ>2}lS7&XBO_b)j6>#Xtx6}$K z;ydM>J@+JmREH)eeL#IA#pi85t~%%p>~r?>#rWmGQ=tA6C=(!mWHU)~RQIoIuP`F1 zg_w_5dzBY}oa5z%Tl{HfM@{q?;{ z1E`?+ZdPE`Tm|e9>Y8-4H4M}4pr|s`>B#WlK7e953R7cKUUx?sPcw$?#>A?gLG?q- zt4*hV_9`VkI@uIW;5auKH#I+W%T(S@mtoF>_^RZ5X&x5Q>Qkp+bVCwV6Sx5LF2s?k zDQ@85f*({Ur1t3(QGRD)Y+`7XAxtLOXGY?co0GRc>LsPEt0LDU@m-*kgRPT7y z1Pp|lm%gu~x?I%HUA=HA*&Z-4UHkIeuaX39WJ*35c?>fns-fWB?jlw~so4~iASAv(-Un}ztPYwl1 zG0v6ROtz6g(u?Bq#ICio%O&}jJP~w<`q=b%vA4;k(e=f_1`?GU{@&Mp1NXe{@|fKK zvW+v>-6xvvWqBHP?Wj`?KgVdDTFdp=s&haQV5dNVrRAf7vi=-+Xr0tF-~MdaDIje8 zd_~PL$2cSS1I}829)<-oE8<6){Kz<{P?`LbbB0Ob25AP*zfc$^gZ#T3Dkndch*Ve# z8~KJL8=Q2vEylfg&xG3pbdez9B3?N-6u%p~Uok8m$#0hW!5XFrAL~V+ z6-Dy&@~}2{M)F%VP9Jyp&kVrhOS}m!N`LB0MkuCUNT_*gR$Ll{*}CE!p(xXC9QpHI zF8I<4GYnPMIN4CU(xmz^r`uJw{)yz}WlP50AsG8#X>asuT(MBA=%~kR)4g2$r?)QZ zsKvVxDx3D*T<5#s z)M`e~5bLzcQc^x?`I?QoBTpc*~s}!qb&#~RN%Kyd4L|H>csKzQG07S@?2{&WV zEmST!wk~0@V4Oh3#OGLU+Qp!gyZ_A^Y7CDwo-%|M(n>I5}* zEMHtpCY&?M0XIDh>#H?VGZQ^m3GUR-urax#!qxB@379#xUq0mYT-5YwKU1O`Mk!L` zt>lUB!3L;{0*rGrSzKi4`oN(*rz~`l`6nclbbYLJ`ygz%F=oUS8d{qls+KZv*p1X) z@^k10)LeyZcxf#YNk21JIPAyIs2^aP4IiQ+RrI_D8g36hn;P3-{j*2gB>!djXB{wY zox+vZQ7ADHUdFYZ-I}y)1RK*xiA|VXKcbr#A)6styIB1X&q1A!P>7&{tN)U~@xLS? zW7c6u94GoRINr!dKVF`eOH<gXlF=h19z=Dvmj}m#S9|j)ydHk8!yr$U zq1Ow~JaSlGH~x{+2Xm)ojaPOvvYb#R=W~O~pw;m8RUU zwWUMQ{mo%r84z@#)vO=ziG{Y5?9c`bT(@GiAIM*p>|PU2izv7)-uixdbB)){9X4y6&@zt{A_5P~}Pn%Fm5jU?z{t)m>z}T~rC8 zXa^#s`3Jl*r#>G%=qpLCy*<7&Td%$Ld>Eqc-ke>x0ELwkoG>dA_oS=j+7r?bt0yQ1 z6;i(MDIQUz21nY=AQ^bXhs9nMv}8YFG_2~xEP{Coep zAQW`?iX^GF&nViLFnH%2xewb*PNnZSZDFJbQD&5khBxung?jGd%^RME) z{0$G34ZQK^FCX zORwTQ?;5UCOKZEbRg=5?ISaEP_uVq8e>YW+W<08cM3eHqS-W%Y+v8m_tnJMZajGHW z&LB0+!0Z+36DMVd@BSn{d|mRZs~dFt)^aUNo^-p2It4#S>ozxiJx zL5+`4m>|vD{~BSZYwa~f^CgEn#&Pes;H15Zgpr7W{Xn~OMfIDzbDxIGY+3- zcF)o$W@W{_k&?}e3*m9t>qIW1{F^ahhc9}uoYB#$-7vcJVtJVLr?%lkQqK8Uh-nM; zzuP6=HCwW^b+Du2d0?Xjz3cbvJk@Vvn7E62K>-ic&_2KvvVzC3|N1pgkC~X_-{8+z z(Vu00hC1s}8)d$mzbNOAs{Dj(WnT}Tj$Xetm%wie*gMQWm#cW=uD@!eFx+S4M)8SO z=s6A1dCTd3?wH3qgACp?Z454KcRi*R-@G(1B%zwZCGDJX0W)wM^eBY{scO*fc!<|M z?b?4{MZv(Y1`;D!)-%K$f3fO`ci4V={6~GAfg5%SELV%nh?#{y!)1@Bg=U$WU}$6~~cze2dp_sibt8W|Bi-*jA_D zan?>%_DvE|Q|xa+VZOOU7AW_}&@de(Xn8@J?~KfF8z-EX>zugAGbYd)J<&p{RwcL9c;g6NkA<5KJ0zv@~)Lr}emvLDQP zx|n@;#X1!?drK`HrfGAS@pgR#RvoTw~7iP^6KDvXT06EF>z3EWISt zQ9e^MZ$#6B_*dj@3~7EnYg%~7V84LfVDOKOh!{5nO5!t-$Oszz(>U=-W|^dH{lstt zswxqVhIrOugi*PYS~cT@lu~pZnK31qADYP{^P{eCqd?Wjf?^xAUh;wIS@;-JjP1kn z+CjG1J-XBakCXxgv6z(qrmMy=?A~ydz&GNwvuk9^H)iAOCx6DH>jcXCj>w-alwO`O#^C%-cw3cXKR--&_y9 z=6@xF6gTf%JBKcT#*|((|FfV#ZI4h~P#mDMM<`0*LX(PO9`kA*b25npGorr9(`8l6 zC?yP5Zpy{Pel$Q^C)XaKH=_{M;fU}ON5w)12Mh_jCRZu6)T-Na5yT>l1Ss#NBqD6P zdcHjc0^g{!uMND){gm&lUn2&CXdD7XNp@_Hem`ccX<~<;MZO?Y#4W8oTdP~f|JbqxUzdhMdZ`#ZOb_&Lj%-^0KikI1YXt{h0xD2 zn7#G&*EHQ`k!}oY;BeeC3BW#ANI`@q-$dkG_*^6-bMAd=K4Gn4s`1sysI}>+I^F%8 z)7xL~G;x>3hoMBf{T7{cE%fkzhyiMSghFj4c!rA0`CpH=MI%GcZ+s0Qo2Fr{&C{83 z|JcfmYExa~nCf$%JQ@%|=uP|@dNgUQDJqc9kX|@;Y8(DSxs?$9ihT?2FGTC6Ks#<5 zxXdZ8sS+L~M4Tl3sDVZ^wXd`#mUHg3N@jD&w9p3Xig5*l)Aax5%em;~V*x<>tU0t& z%Q5xitb-qe0`p_VI7(72d#Qv*irZQ01er`W%vSD`C{}X+8nMo4Y%_=O!&dsTQd0%w zX>6xP$`fvpEa4AeUy75L7B<<@Z%gKw5ggd`X6!BnkkQU(bllKNX;Ai&RvF`vVeUDa zen{>>JL7<1ilg?S@U18obmDIXhw}BIJaMa=6;i_@Iq7hzS>^C;NEi@h^w>C%3<472NVb9ScaUb(J&dhg z{^(XWgw_Q~E;{Z}3RQ5stUSHJIB`Gf3ID!3J}QqA#X>H{ntU`xo_T0vdHCX#h1~CH z0ZH@)cT!??v<%Wge}wC&h&|XZU(bFXhFjSg@q36V%;AK0hkltEN0wzT?&OXu7j3|2 zzE_xo6~uPUpb`ax`^bPMX_v$u-(($%#|B2Bjjs1NDJYK%N%Y{REtQfh$;(lR;CG1n zzDATICGKJT5KE?gcH16n(%F zgzZF3;!6yL4DO%hjv9`LNyBAEgi4E|^+hnNhnD_rD*bmB*9imY_N}%&dd~yOO}o&a zQr;B5j(sqy*k{il*UrTLDh^`^cL-B-iunsmVfPjuM!C31HJzn=0}NdTm`U{(-ntKk z?jKG)`U1|DmKo6ql+ALDJsdJ}x>uxV=rhr*gqh%6292<}{Q@9B(KF_N3zmN&>;}%` z7|jMFg0~PSC^8jjS1M4;J$x52M6^zbTFo;M8{HYj%Z_v6kgSj8`t^q#c1$9B2^A|h z-IkjKrb?Q~&)7XONs|1&iIU&YBs7tF@qN*(i6smP33em`qgaebVOb>Wl!!qivp7w} z04yjNKrRxHf&SO#In@HnOhzG(eIr#+=ycW|0YX~v)#&BkiYptr0&jY_{}u^jTJ zTHCTC`JWCiY)t-6f6@RrAePVY9|l6_qS+%tq+=(Q&7Bt?J=!9Z5NBtA`HM@64bDj$ zp}Jt{CSfDcYsr6;8~07lDQ_ouvk61S+_C-QQRZV7s>PKsb2(*wb}_QY+D(ZdAI;v6 zPG*%@9PI|vq;gmBLPkN?p8|%M?&QQAcRlOzX!c3JaicoSE*C$UY`(@fVN&__sIt97 zaw?QQ3KDLq4{;b3;N)?^ymO8)edYT=U(xM|5v&HD{);lQ^<1~kEGA8!<``PLMz8X% zx^9F&J@9DMWBIf#Ar%b~{;>90W9H*1rEF%)?&agyC0anO&oNB*$p`An$AI0BNsmMm0{>TEZ{v7;pcwbw^J3R!Lx$zjkht_5}wrKs+ z#UaZ+?jkP~`>IurelgZBMI)T~_hfE0ac)M*&em+??CnQ79bUP9zQnmxMiz$)f%kmF zN5ju;ZA;iXLqiFCq2u$Wp5*$J0S`XdHU6&FUuQMg&mpA6s3i9cnp~@ynk$)_0_mDS z{l_wop;Dt*?z-Q7)>Kj)rV3`YKJI0S5{B%xD#pxF%pw)zTgCff{+u|%$vSP7i9~9& z{h+yPC7}Vm6sm`0}kV?%3K5Ua}TzWL@OGBl-?X0q8 zGmI`C&h90S1AL^-=8?x&L-R@IQqDafwK4q4qU`aPqZ4d;X!`kyWMowpEMl^)GKPNz zgLcm54IcRFEon*k@^bAX6+(0-w*52vMBqWF-8YwkX7d{DHf!0pZC7vYNrk7rEkYYp zI}1klcpY3KiNo2o#ptIh!$c6Gdms#Z3(y#H?D z4J@8PMp-Y^`rg@NG!$H)m@}0$p{)ZRyZPmnvJrwjxJr0u-Wy!ac9Z!&@#T@Vku~Jg z1lUCdvN47(8-E)nN%{2AP}G!{N@r+j2M51`yLbBLHZFWEgN^}Rff%94aJ(#PuKe57`ZA!^)JwcX@pj`XyWb)U?&Bqwt$i&+;+N}(*b zimQs)W7ITZTuNL{+zMdCLIr!p?+Q5$t5nLg^b&H#fZuoG38i%Y%p$WE_Qiv9Xj!E( zWXvMCX7*(h8q8+)*pRu0xY7z-GMZldzg5$%pvmn0lFxOJDyn#gON*R-%6C(jtH*#; z=sB;T9;4bLjwa{EH2HWH=BR+oRFO>1tPL}N`|iNs=^{808B9Q`3|;!Yo?O7=@c!`#O~s;b@+j=tm9j@NZIKU#v(J_O0xk5bmA)C*o>4Ai5R<{1g>$( zqh;Yd)$a@((?0L9_U61&0A|>6B&1lvZ46ws5xWsiR=Xk8Z4~X>dhI=Qk0Uc0dbuWx z^YA=1knjYkysvfYqxVITTMcCy-)@Sx9_+$?sj=2|2?J3W82;3A!d@~Qql=fpfs0kY zr?LuyE--c_X*Bj0#bk5Nc6D~UW9g)w*GIv{?CRRoZN)UpyT5)gp%kfBP!bhg8K|U3 z5|5!qR@1=u@RdfLS;L30OC#7P;gw%nnjaKu2_btb&{-LRSf=#z)FRRp7HKmsC%wP{ zX=AQZvA_|1@Q6Kl#2>6Cg`7C^c0>`Vm?*~D;Z>MgqY?nGt)c)i(_yT%ueem*4ly}d z$4kxq8K0XKpPL=u|4&ORZN8i0b*17l;*Cpo%?#~E6*VF|EfCDrCbLd;@zk|`C2;XX z>>EULu@}79qYNhFnP45p^uf4txagv*DiDTu2jaj7BZBPBk)lj|)aoj~H`IYnk4xd= zw*J&ZlZdfmcr%ZWHXl(R>|bf;dIO)v47UHxVx5VgxC!8R&D~n~nL?)Tq93~``$<{A zMN<3*F@4E76UGQ0u{VTBzyfN@z0usp#j`veNTtlLNeK_%95R}=+_XKMfVavlHXBtP zAW*eD4+4D|J|S7K!P_`_(h}^DA?9w^E5{Y`=&t<=rG2>Vd&o1sdx8BM93!*uNLujP zTA*XJA9D=$%k}mDBP$Q`75gIc6nb)dFirABzy2Hq*In1yVw#HU>V6h8yld*Bdpix9 zJdfx-1hR-u)`@;he%y+CFdTPJ07*rGO8jg($fiPY?^P?@7SsY`)#c(}VlN$w>=DQlkk`-c!HO@YUv4U-a=z?^_! z2R(^rRLI{|{&;+M99N3Y<1D^d$e9?eRh1>HWsB|36WICwj>7dPO|7o8Kcy0 zdBv2U_ee0%!4CLM`t(iIR$acJ<5}+L_o@EO}@Bnb{YR zSG}nb%`aSNS*n292nGk2-``6P7S_$CC6y>(L+voU(&5g=*5#1Izq2&DX^<;S#E6yq zFK~Ry6%AFxG$nPx<3kQ8vgYIClqeMjw6Gc%MUs`QyMEnb8__BOSHHCfe^yHR9*T)i5f{d>K{PYlN?b+$aaaN*tTD5V@mTC0V)qTw@z}tFe;*K{pU(N2ExW z$MJpGfR^QWRF_^WSDT=Ks&bV*&!w>me}rM!RSdZNhvspg>m`Tr>1oty{m<$WzS#TC z+kS6I$i22mvyG496*W1 zr=E$JpMWf7`*M(fY>H@bhrOtV$>?!2(Vm#&Ur=j;s@(YwCX>Nu!Mltt+i)+W(P-UR zuXXq-C|JHhn9s|L!lWS1Y{(V7_(=f);9VaN+ql2mI(U98-Mc0{AE=t`7M0VbJLb=> zR5kj!B=iIUVXr5>-mq4NcW#3GgXh^rL)^JKO}LQTO@Hu;O7=7u)#S)_y-#0F0h1e@ z)9)9qi}Bd-_)zo6#|cF-oO!P?&U#i1`ACin8u1<)$%dyvpeg|)4%Q2rz5_1$`gk^D z4Wtj3`;d0=Q#kbnQVq+WEcl-x!PkRnHM}>04Uk8bWy+@%r+OMgFPj7&l9uIgmdZiA zuTxxt-wa?q!YpOSitgy}k5J{tX>jK={fM zSqP2x-4aN(6q*zWkHCdu-j4_FaN?l*ZLRiHp=G}rnr?Jm=DGi9S|H}%Fdqfvt6dh= zebaRZ95L#qGq>-=F;V2-em7Fz%w-zH$TV^{u8qWQXjrmu z=Z|kbh6@Tzgjp8k*&*)VxKUpT?Q<=n|xz;p2prrwF%CfNqib( zV0tO%_&gxy<(15Du9qbh;`c^gqvdV#T{JZstqKjPx)KGxdF?W*4%}-xO~|<>V1p{q zKk#~V@ru5zPQ#->>KxwV+4Sl)=nLrf7c`3`ucbK;J3ogx zCocydJ3k*kmlcN<2M4dUjkz_yfDI?FsPO-Z0kSBAmILzq|9?t6f7dj%tqF#&AN7k$ zFc6@(2}{xd;Wl9u`F-RpIM_TYcn;x2a623TX%0+SOiX1d99*Ggf%82u^Eb9{-B;*W zzkZF{WdGVOva7kgJjUWvc8b;SbM(0au(2p#aJJAD+jx6DysBQO!TzQKI+U!G!*mZ5uRy#iUGG zFD%M%{iTc#{quyejyae$9lV|YZY6)fg)?4 zq){j`W4B^wSTmiBLmgTj_ev#AjaH+nZ)IZFIjg(ocdw>FbwNa(am{khkb+-NuF?hcGS-#gFTbC-G#&G z)K%*>{fHc|9N*x2fso59B$L5Q_04y6Km~Rjv4F}t*d{M#8 zyRI2cTG)G~+0i=Nn0)*I871kwhXA~iqAA0?XZxQ?S>^K~NGFj6@q&xqskVhG$4(2+ z!X%FtPzQx7yDL9q1YNx|PbR9x^Wu@!=yFs5d5Vhahd)&ifq9BAViPh(QQGB~W<0h1 z-g+^o^>8`=sX0W_hWgDdd8F>S-|F?;XX}zC47dRIu|o2sYreI49rpR}w~(J+iP)@1 zqn}CmzKYnA-Uge=ACmFmKMA=`?SQU5AWH#;t7P0h%#N~R>%a3upK*8y)ARI={k4Lw zp$l$ZKTjYcMN1#FfWKl%RV|LA|yBy6Fvch#N`shR=^I zz~ApcLXj0Cy2u)qor zN&4T{*U<$yf`G>@V-ZVTpH31=0ZnMC8DPEXf z3x%9U96F%%r&M=q>`*I4$CKS9S1;hgQieONtDa-BU;*V*JO@G-W8(p>LtGpb^$@Q$ zwZPS!2@#%LrJDt@?DeI9kjZNEL9b$_$!26w(NQPjzEx_FK{aAJuwu z4~x(+v3mBIb3C4PooHP-&_C=1bo)gsq#2O>c=QnVQQZoKh9hnF>qdnv8DfsCXiR$~ zkp@5bh$n6i-fmn^^z!kHjq`{}`fI)|Wl2ejh0ZROLiU3|;eS`qW^g)AWfMI^9yRpl z(eCB61%Snk9ebmf7DT(kuMHiR#rkwI=EO5W#PD}>3q&Jzf-7{#-{Va}Yx)$9U5hEl zN*V4%D}fir;U*s{AT)bGmpfDgzupyRi_acvf%ricQYN(QC*7myL0kB$DviG=lb60` z$f`gna!Mg9`%gV_UkTZ4azTzm+L)O72KY);97fYcNav`wu!)QJff&XfT-#4&eZ0n2nw-?=^-i>8qOYNggcjcPB z$>suO=M(#6aK)ds46L5D{EPxmAM29bNmBZEr!wOupr@C3b2z`==DrP2q;n+h!LUM$ z#wrBp*`iUy^D-v6T+$8e7K*cNj!KT&dYxQa*;pLtAW->h@AWX@w(ISqhDNex?&;E{ zG+&BrW|bnTgI$ilm>;3eN`ZLrhrla`uwck6)*!rp~uWR7fo9` zhcu%wPBqg#s8w?KHubcK_(GiOzEZn6zGU3bsjQ&=_zHcX2a*3@>bqV*(g0k-!iX-e z&X%V3h@Pv4$@>3kM(EZZ!#!IujNyH10LHR246_nJ0n9VK!dD&i>JUbe$1R;zmmlHq zc8^Uid>n-?leHV5d`Gf%XZAApmglb;nDhXYioQV;&-;mvpYfd=xDf~Tj#GDk;i#b?o8Q; zN4H4djMB`uwU&WdKmDz@96cVlcgBm&X!@@5n!U#z?<1=ONiX95;+fyb)txbJdnK#6 zhD~y*G@wjpufL;D_Y>_cjsiJ9dheQkly(jY%4z<9qldy%r53}32U4YjOz2wk{PGSg z^JoS&^FI5NX6z~kvu7p5s|W`mGop<4C*DFRJPQS-2l6vT=;0>miP8t23Pp?)i)l&+ zv4`4XCub1Bk7L>DsDfqnv)G_h5@y8iV8r22WF<1_#r`E#so5w{ETbhzu}2d*PbHf1 zD>yWZ@;S_fX&YmJu@!a_?~-e5T-ruT+WALnmDn* z^wtf->#`+XJ(GehDU7pW!`b%|%ZEj`c~X8S+s@x~Tq6ym82Pn_GX zEu>kzR(`8AY@-bCSb5l4H+rF`WZC(`Q^=@t@8*Uv^s!ieCcSs?(D0&`BhGH*`_%Dx z^K#P)T*jT3+cn(2vk}(TS^^)hO=AWPXmHvh-EL%KLu-o{=e`J~_Ab^*C$7nqduuJv zz9(CWoliY}2)SWaA}@Z&e-6{vg;+eU$1YagYIm@HC?v*$8$AuAy|p^gj%upJjk!ly zf0VRNGiEqA0bdSWihZ0NJu5yxOux{fJ>#@YbxvtY$x4;*q88gi8z2yFCr+=i)ou2_va zt{e?XT)gba1T|MYFIj1F%j)BCe0ueQBn7f?1WGszRM-dg?QKb3taZ1*V9Xs=$6#0h zRk^4N+fX;UOrk!lEEv<&V)WOpsvRj|7ChX*xgMbd0r}3hR0Z^dnbRmXWO!C)EARfk zS-C`P_0T9iv9wXqlsb5!Z>|#+j-|hEF5%y$&Vn&ro>ta0&^N=Pj5_GzO>zv9`E4Zm zTN-0@l@`>{(k5`aEw#``9~2XDVF^$sVt9GuMQUO@Y*x~f$1u8G;d)_6=Mdj|la!U@ z{+aXwd|pkOb&(t8h|YCLzTR5AU5B6=qNYrT*Si!lDwhaO?h09E66#5O(i4Ym3Szv4 z;IsZ5URbitN*(_6F1%U!b!#ik{uLv_+&8_=4RaHq>1ZKvAL<3Tscg0pEz=qwAU~&R z-25ZLq(hJ>I^30jO1(_x@YUpMNIb2rxcXOyXNb0#R7sqM-M&_qGhjEL^2clXhN-*z zde;|F2VTgFJ-^l*Df9Oll#R}CHOeVh`=&j(0*8g44IdTN-XoJ9S|3`?lL!#py zgeGApV7&0nvgE*DnE9b=aGDyH3`>-OHiTPS4aOcLxE~^l41o&WDgpgc|BF=DCA2({ zhaJMGhRTF)&V->a@e2DJfA!0da8syb&vGz1!in-rpfKD#i!?nL}FB10+To=Wz1QofZ$H0shN`qts^N1| zX|Xx~J`?k|e~`Id;V(I~YnZj;wo6MgT>O2qkfL=u0nU}4KYPx|I5~aJ_^z%2=;wMm_-tu? zjK8lP=#1yqBms}{ zu!>1hdCVRaA0v5+xY&%V`)p!a^veQg0|+kdP=Dw~X) z|1e`C!Whej{N&|x;UYRqZbVq@fr7i&uI%E9mw>@Hme5)cakrC~KNvz?d|?A0?#d)T zZtt8=RxjL_uHMiXL|lpfH=xow9w6BvS<}uRAjts}y@!LM=raqOGk7!I(ZBtGv5+DT zUi@~9s}(a3V@1Pf1Bu6K>D9cjSfmn1d@Csoy|Z@sT^QU_Zz_-Zz)@00TXK=H0Q8)# ziQb#(J-Ate1qul&+X9v~Tk+&KBAP=FtYW7N+Ph>=@4Kd(rUy+u`3C_N7KaJ`>5E3Ejk$tG_2K4; zg#*)Sm%!HL1KR5GY=FEq;PijQNNaz9L~g}@f{e@r`2J5+FB%vot#LQ__4Y!2Tk6rL z*m>DYozU7->>{Aqg)sykKm^=^!zT^!Ql0kqp*AIhJ&IYbhowe^ik|N0q{wcvZzL`2HW4fAK4Ro;lF;wtOHO4?X0WcLWJX>X znGr{fL~2qCL5;ylg)@i`Rn-UR*vC{^=F~LIY0DWmAib%#W5c|NmuV{wKgnAwM-_Db zUV!}0Re{2)I-{#Aw!MP}x-{v6C`S~6%r6Z4sX9Z1fDXH%rugCsS108X(2Ws043#&Y z;5iUOk)2<29frUIaPqDEf7uC zCwv4iq&ni|D9NYkjeJBlxQCVqf)cGcX}imB*&qoiI9k*V0RNd!8kUdil97hYr=@H z5SYjX^sI>z>3@qT;Hg+0(NR&MY=U5LgO(Yf5hQDml`emp7vxim6I2u+Ak85>(&GMT zy;7=ka$R?VVGQ{sGmk}#My#w|M+u{d^JASXUGu&W;#6uX*mq?PAY+(#phtAi`^$lD0f6mTD;FPaB+SC#i{D6Faxpb=<83YTu$T7t*Os5*!73~~5X5-#vWRwwl;q*x7f!Zbgkh zv{)cr40$y`{R#t+FGx9TnwOz%7eG6E$w}K@nCg_!pxq-d^CBxzIT5&=5PiM#=P6^T)W##H*;$qN)LkB zUTh^6MZcNdcf0lKA##ZYY$}M@3!$I8w~i}GcH{f!m<0W`~6HYnVmIiTP@8* z4X(zY_uIX1pyR#8LUaGDyeQGNvKJ=0T>Yt&hSXq*e_yhg#=1Q8%c37Fa`l%M@(hf8{DW zAiy?z2;&?dcco5b?98LqF^+_{&8wz^R(aglMWa%pGwI$6Ti|3Q=qKOmZ*=aENy9Ta ziQ@c8l8zCFaDrZ2VJe~ADLU^P!Ru=8H89#UC$s>V8Xht4^Jo64X(YUJ^^RoQRQ%|9 z^5W?BZnCfOsGNKTs86rwOXqc>JHg%2blBx@(}%T&zuG>zNt_+x6dN9HDb1N$8+6D# zCmF?Vl|~cK94s2%SSdmy4Y?Y!Nr+^^wQ%CrG&;L((3@&$Yg$;!^-0ythqCb?EL)e2 z4wnJqMhp`qSqr}jjCiH@TVMieAWcpoG_&#t`-H2}_|@B&)Ijd>-51+g6zV9vSHgt+ zFr#~`L6jlw9J*-qmXrQ+_aep{_8vZIA9c$YhQe>)8;ntdGHB_p{ZKxo<+5JBN9sg% znyouILcc{y0kcd;YYm#ZKabPN9qm5U8c6_9!hKlRtM}2;GW2YTfoq_oBa-*^@COt2 z1NKljn4bmySOpUMG`Q3GN);ilxVfW=yO5I;a}^JlU0{p)AQIL#!Aey?-x3@{AidF4 z)yoX_T>R4a=ePFEkg~T(HHn#BQJtV-A2%vVHJ!{K$)4O1YY7>@@r9U{1I>R26*$oR z{Z^jkLhBzkE1*AZxpmNIoXzgW|gA6-8i>Q;L20c#s zsLn4eT4~0$Kr@3^Fzs~#D!2kRi6}p4^7K9?qCO^;%NGX<0`BP&=ITU^b-2zMJ8Y`l zx5tCbUc1H517^+$3}a0gs&tDqfv~KiKct<{v_ueweo#^nmOq}HcE;p__Ml%mLtk8Q ztC|1-(00(ShbMJ_eh%sw!?Kj0Qqms{^tnMi*_Jwx&rm^{JGbrnf?6wY zGLFu!B8mPO8lF52rgpbt!+QTQxdux^9`~Bln_@X-1+Rt69B9QL_PJ$pJ;nTqAn;m9^g7LFxt~<+YBSoBJiS5?Xy~^T>9)9(j1) z`)=^V4Izv9hJ=Uw(EXfOD=cp zD}VU94Lk8q3R?MvE6K#5_?s3Pv#7FTyD-mV8ULeg%zRRng=r>Td_^^ zQFHJ%OfCm80Wl*ASP(naOxIl2q@-h)OOTUSkEas);+;1%mn4{)V_n}gYibkzYi<#me4>>(7U;FdCmx1 z^mOaoy*XoUy#}Uf#LE7Hc*jWP!)VFg-S>CS`@@9JhxQW#pwrg!!0mImz47Y4`?BaP zpb%fYXp!TM6BY7qxJm;(@z%rVMOpKRk z4~%c#|M^hT6yJUJ6D zAn9q)=H<}~@(X}u4i8fHHbTCve9YLq%F&@*p8ZVEA2Q1iBkaKLK{iS@Zbkblkr)wA z<1(e)=%I&QiJ!p@H@0E#pSOx{idONJqJ>&K`*Qk7r zlz5T)0bGa2Rj#6y@uT$$)avHA=LVn%anc*#Jg)b?HSJQ1D54GRb^XC0p)%b1fVka< zu88XtL4~jwx;MtZ?YUS5c8bG{{vAfNo{7Q48ZhZ;P+*->m&N@C}CfH&L`tT z&_B!3$o{uZwBgP1DWFyyZ#}Od#=HP~^zr=z5K?A-x|V3{iJ2aoD~vz6S#%&K`fzTv zEBRRE7V%%-sQJC!Cjz?6v=;2An02W6^-EqN*qd_6kw`;)h{P{mMJsGr&84tV@0M}v3qMXFJQO|SGr6{! z{#Wpio<$n9nIKK%+sWEueV5YuzMt$80Lgu;kl+aFViS3W#Clw7zKA!B7}R0kFF|!4 zdvlT!t{v*#?d9@cS!vg}DwQBpdGR_AdH1kKwH^x#vz|bC5=)A}+&;Txgh*txTUJNR zzCv=c?IO<{nw*oqfNS@3zGpMyj0Pua@sP7jBhZ{zf+rGsK7?dC*#Zri%2hK6*q?d) zISQqtVC5nyi@U>5$~R_6F0I|lei9;=Ty!D{X@Fag7=n$!t|!%X+!iojoFE8E2ih9qd29t;2dK-d)*oILbRjlYJTU%Hcl0qR#HpeHTGx*uFf4SYc?AB$n z+~n3}8N1KJ?t=#}G&Nw>Ot+{4V5HI3=U)kC;@XIQsLx6$g#Z(>=SCI4X)LUp5XeoI zclPCkri&v&%#t3{qAaI$CSV2N>w|ZSM3d9e0M(l^^Y;YF@49f-L|$mjQIitvrP!Dm zlF1?El;Bz);*4=mr6Za)#q=f~XFR%&a#(j zjwl71!A+tW=AvhW;Cx9B6Ns=ud@oK1dtuBUmqaT;l$w7IhE^tlD7|9q(UrrX4ddf# zoJ9#LlXZq?;3tB{M9KErAv2Y*0+JQ;RoZ0*@X286zE9cM@UIjnjm77F!HLTy^mn0p zX`QndarHTtX(Cv;{a6E7`d^DJ~1bB*5qp0`ea+z(7FWj;hVJA!I6JxIQRJQiLj zOV4Y(J~p1gm>Me{mFE{1^Mw-!hxRv~&wCpU?fTVQLeCD7lFmT$%PlU2-=pJNS7}$` z{8z5gUfY%>k4iMz^IZ%Ey^mV|;>NUj<2~J)3+AddbMN+UYTX4nN{*vqnVY;Xitae9$(n&N#AFphPcJMgXazV zfydjY^WBY9-BI8zr0E*rZ^|8bO>pNN2=jRN`C(sJRT@ju?Ag;{e+;8Lbropc7#(gB zH@VqqQv@*UEwj9{y}A2!>-J;M?;1hbwFi98e4L{y-WK`x-fWcb7EEiaQqJ}(b<4QR z^XBxyeWvsPmy!=6q0aLDi$C>vAOfz1aw=Q&HHR+ z-&QvuHk_R&U#4ChW+a#5P*OQH=E|@;T@ocw66O1|=lG@4*#G0?n+T(OC6^FUuYRw= z+52NujM`hWj-U2;r59IpeTPZRHvSsL(1|g{j_`Y@*lSh_UH5?M-P^1UA<{+Jzoq`v zn2$=F^E*9N93R30bJ_E?9D6=w0!hq%B`4EpTvm zKjRjKn(aabg{tR$ifm&bkBUSpi)f23FpEC7ApmQ0zkj(81_F8)I9C+sITa^o_%4sy zd%F%faxWh?+aGdVaxUGb+Ixhj<$BhFzwIDTx8t_PtcQhP0kp(@d&_~PS1`5CS)E|wORFAGUR_xWHG*9JHus)RnuQ!*Z zgF8ooGt1w}GiT^@r|>z~bGCod=6IqsTVVVo^i0gNs#6;}G>UA@XFQD1y5`L*Ej=-o z8eK?H`)OmO{Z5UkMobt{`#cv){h?1;G-5LyIZkWqv0F`rYL&=YMhuq~Nx~!31NeFH zonW<1FfXLD3n<3;u{~yveSN+Y_RsQ0y$h`a!AsZ3r`;QoNA*Z7+f--66JH{~?Fo27 z*KWT&Li$ILm}1uq#$98+Vu=h?dL~U*-S}d!4EQgTP^i*8c{~c$43k{pypJk(liXV1 zMSGJUyLg~H>#S5|1Q#BHm#ML?1b#_y-yT zyE>0t_uD*=Os9=o0hfP?WVTregT2M=N3z>+{y1iP%0!*pEzZjvW#)jq$BXiE==h)R z&Sq}Jry~%`=8dLZ_x&}Dz;&n?|`VEjqAU?Yfm zqk2?5rJkK}P^RI;d}=XYUglzhaY{H3(B-?xeGF6$AXbU{eX4Y#su||5>f>4t$#jHv z7?@lR=#=!IM5A8&<^ct;AiNfGUro;k0@$D4v(X=?l{$mMrMxkjc%qL&8Czf4hyNN~ z7x=gMDs~-nPP_~%yf6KgyB=n$?=tS9!(__6cBwpo-c5P6ddl)tU|?*G(8;zXc?!zS vWNM9C!uOwc5PA3_e7qHi@!opX;r9WTT@8bh)>{ch2G55`MJ1&ojrhL+H$f@P7g20wUtqwXKcj22>ojVoYS(0-&OH0fy`24a8g zMRRK{HNB=1W-9)r5=6w#&_VW)0nA}phpUrI#<>GjW15E19&gASF&QVDzemAnoiErj@mf2|) z2aWXRA=p=Lyu;jOW{D-S1%FO@H4}xL6G#?T*N}uo5v)+F7pBu3ujVf8n16 zJia`fNxS*I321bfMunXruU$`_Kf5qn$-{D_`S}x&?e-tk``^NVi)}c7YVi4booch0+nB9~YW(mz_d zgCMONrA}r%X}?1|Ob?6*(cs3uWlwi%9g#Wa1DPE2iRm|pBkS@1@R*KGr+Fr(C9Te0 zI~%@)=YG(p_k1Um%G5cyC;d5_lm1`-LM><^oI*m#u3#5)V|!%JHT@(d#~=~Rp)21g z!8MLBNQ0G$3=S1%`BKDv94iQWl?sLA^*+=ui9{B9EI!-I(ZjRt#Gz5mjb)>7)?Bf> zZLxxV;iQxfsCQz(FoGeWkt&$v_d{Wp&!K~5O?>vd69#Gc{=@y7D`ZX~|JA5tDc%O+ z&1>0%gS)Bh?cYoZeYi;taTDa^cbhk92ZuN*1-mR~rlwWV`xUodZ>E-yE}Sw`r*gD9 z-dL^+l>rt}rh(WDN2ZR&HD$KZiSqfEzZtuCWe3@bY6bxsZ@L&%Yc;JmrL3myx#$Nu zG;d}_WUz2~DaAU7Ak$bnm`TP^C|U?K@D`{X!&%Z^{G9=xOw+iRV9;OwoV%67v=7+>XxUIT#oHzf(6XN zLkorwTb*Rj0_g`exDLr34L2cc(af~I{NFy#iTmwRYV9_ooi5f~ru@5Dv1o|kwH2fS?a%j3;kk&&jYIKX4PFk2AIe&8Tc7pUV^!W? zpb$sL5&wgS%!V7H61~TY6T1DvaKkgfzCx`~Kn7=z8#I^tJr4@NQ7HSzni?U?E6 zWQ2@tn3(A^b#ak>&}p*F6`a=nQp49Knw8mdf4Q{jcQ81npTQ+tRF-K&7W#1tx}Rl) zFuZIZEJm(ULq2>qK3?xyu4>FP2Wx#i+95yQ?;h(FV#*+!gr&kH-JcV?7|`R_cWOHC zHzE(%u}!$bI9vOK2PY~n6)#7c?njoVxGyBTi2sUf^KQJn&+F^+NAQd{IO#)sX=ruo z8fs3ZUJOZ2c>1!}Z^S9#wYR*t2wSe4=)Ez#fdzEpNA`IX8brLv#Oa}dpJimTlF34o zig5375IrDYurJjaFt2`z5)BsTG3|-a_vOF?Ist&rj9`w-5F2w$gvD|4I1XuIN1cGQ zcoN145}AcD-vuODQ!^G3_3lq9oOE#c*mU3wPDDPn1*%X*@d9>>Ja+7gOb3y&8h~iC ziF?@(PlhiQBSRv}_A<)V%1W`Ic&NagzC>*=buq|5f`kwn1~!O8z-kK#W1w<%8dL>tvq-ZP3!(Z} z;xW_{qu}>rsPTVV+je2TVTsanrmQ9uK`sa(daB{vwu_>35C6|A^7(Jm&KwIk{&_q@ zOWC%DCy*%qp<4L3HEK+92oSJOsZPC0uZ%7$3MUhMNCpE6xgO-@K&!Cs7ecUed+;ar zFh&g*<(up(ahnEidc5J_`hP&t606YsH&S(Ww2*KAXW^44ZT4Alq#nGGnfOtuTFik= zAtIT%0M-lCBs`0=HQC{rm0=*B-wO~d)M1quYAV-p)Y(1@uGewsxHl{59qHK`PFii& z|2Aw2Avv?!+UvQX_A<_B5sIAqcvePBI@v4#l2a+pG2gp>G@Pyy~u47 z?$z4AIGf;mD2e39>-9yQ#o3#5d2tpP8Yo%6v*5}k_||P_8((@oupz+lru*vdcW-OA zx8m;O`nO}Hgl7FOYuAYQxgvGuucMPT;?54oJ?FAh9?eZ5MT^ygZ`_S{2j6lBYo2a| zU|YH`P9ck)`Z>Y|KP+`1>C>N?u)Sv4Rd(|=$XkP!BB9mIs(b?&OL3IPdf;W z;N($(M})dN!*4I4Y*7;WqDPGm3Q#Mg!x}e$DPoPUXCg&_ZjOeZyM`=pWY<| zmskWnfj#lSTe~NoN7tz20$&LQazu;L^KzEuok~X5*x?M8itaL8N`9Jt{g;@2;c1_Gq4@oS z4B1x8j!(578tX&TDDKDMXEnDEq)m5X@&Anyt>b_6NP*LxOsw=Dx#Y9qcemZI|7?@I zU~s#w^f%0a@%zuDtXl4)HtPi3im&M}#XdT7qZyiu88vSTawarf3cmAbqj_R{l; zKIeIRPG8NhY!5GjAFIXFr0ag$8siT?hP3rT?E)k&hIJWV2uLqs*c{qN$6wrvYLpeG z_aH0Azc!WXYYTfChe|G$ZaKCuzzQ82mMuIb#qD&C15WvI3!?%wKzSb0#j&1}GiBgh z3SZJ^pLtpBEF}hKnZ=K-_JH;Bgx-{#$u}_jgTOA$w`<&pV6Lzr++$qFA1=y9RzsqI zQIHgwZ95l^XP77aO~NjmS)n#4SoM3XwcZ>1OlCf&iUfL4*bEF@7`a@;*e;E-9Bec6 zD#dYT6f3MBbT5uruE;$ij*f6ytewz5$%d{d30uj{)%_N~U|R57;jFyR;_}z=FMS7G zK`Gbc@YRWP;iZt)pZ}VS)41$J=m+8W-wV5_XYEQf;{KLn$hIv;87g`;!GO<@NUKxf zsRC>{C%{Mv`U!|+N38oalXwH@6+8lC1oW|E(uuM}%Kc&7bX|6J+`c@?UkIfwVk+wI z!yVt*LAj6!B%+v$&PBUjcy# z7*5#O@SiLjAZQhJ*wg`onn-D4JBasKC36WSbXC_id4r0wD>CB0F|Dim;hhqUY$z5I z$j)Fd$jjmFq@{V_b&q(!?PyM98z6WdC66lEI%a9ubk2q-uLYRL#)bTIO=AmL6 z*vMQsg-9t6G8m&g_A%Iq`WZ(CAgB*w=^G7^YZDuCNM{EIBCtJgMO9o! zNfMEMlELN#h*HW4;g6{X|Fg$mV0DlYnMSeQ(V*#N0nXTcL}5rAg*eBI)@qL|GVI0H zIO)g`@0Fm?0IXvzI#K0sGCRNWj1MB1*MjT%&|qr4*DwwPl6FJ2CDyQ9WWKD4N&FnhK*~A{R#k2}BG%ysENF8`bCY_AY@$daxE6d{1AF78jx#+=mXR+R zd@OM%xQ=g=>D)oCF+`CLxWlb0!mOLaFQ(?q88hQSc63b|4 zuk*^g*SDk6Td8Dzwc+9Z{Vh>TIHnW=xzCw8OkM0^VWo@^y)wVjxY98^jHdBCx4(K> zWog_RS}pj(G}3s|n_S`QzvUBL>0ej;38k&4DS|1W)=c2jGgZUH&g*Ng;+WRe4d?6+ zynJLhxe{sy6T%_^PQ%8L<+C}g+u2lb1yT>HN^VB&=Dw}57J9r`3M$FIT_h#M)27E^ zl6yFDl#+j>TGsK^|ooHktV7H;*x5qif)QnmfmLFh*p0U9~_wQdwLAs z!O7~dU<r`Ely{QTZld{JUo=P9O9m=wc?c48>PvJ%)8n$)Tgk&-)%m81u--3}emn|Hidwf;!_sU7)4mwfSmUZ07sQ8~ScCRm5^B8h1p5JE==5D_nbzPh z(s-3`{~T3r@hopP5c`m>tv6>{%CXYC_u+X@@+$Rozv1E3-BVw1BE0nJ&uO$g6sKm` zj`XSF`AWiB&Wz3big=HtUC*^Qr*C@hBhi<+i@qR*j)4<% z4`($6YW=3Z2vQ&0BjZDeicN9@!LF<#07uI8M{n9h3VnrI^lMMXhmJyp(&o@{MHdN+ zq~R-_wO^4t(C7pWfCFoOGPY zrLUU!o&rA5pLN*P{B5sDqUtKI7jH`VI|vo27jHY(()vUkH4SSZzhO0C)^i%wQMJ+l z0&D1K7u?i3T-;<~UNv*I zFEL1kl2Ha_dAms8c>9#~(4`MS-pnu=z}1v!bJ# z!8D}NM*|qmU$5{NeM17OOD5E%e&J8N$gSWZ5;G^zRvZ*~M0XUV+IOvRQJ)G9P^rhv zFNO4rX){k)TeJ4neR!DQYXDOBabhx;Y=caS$XxWSQ-qqcQX8?(Z>_z)W&UVjL{?(<|${tFM-R zT=*DAR}fvz70tjCgx!R&x6ztHFOb#}#Zm>@#EzYLv7}5V!J~;kHgL)PX>a%B^7+)k z|BWO&e1Z!3YD@BN*;Bpy>HVZ+pf@(G0f&ba4aLhJl5_RJ+VhnK|E)24wG0LAaAIn; z(%fxg>?QAE{!h!A&0p&ust?DW%Wc?Sb1j&I?K{d<79}RbEc~`d(2zEfOkB6_bP0W5GvC7@c_qM5xGpuwj^N^RHTI%MF>dsYf%}%p#mt)L;G0}@nXeG*? zs;RW*l$PxhpCgGKEA!#Z^br1a2w|7@7r8kDYa(cK@0)CIBB6a`T-|u7&+BXytxnK5evXdMX^Xpcs zKoC5kC}b~Z;9!&2+7XCp{RY+6?Ggu`RJq%=54(P9bv#+cAyxBD;-kHgi`yQw6c=Li z!xy6O5)iFhGdEo|2KIyO=3yW2;;I8AtK;fL$ES<~5eeh*5D1`py%8jS)=i1F-c>Dx zi#f8rx23pGy;mF}HE*tJJYc4Alb;KWLAbNcI=Tp_TSamZ-Wv6|m>nxFsWsgq6K!kHo#Bt%WE7WG#|t zN^Kp^160tZ$H`y6&S<*ofP)K;1aB`@iQCO=LDqts9W0rMk++sC;Px!+IaoY$xxNrq zHm}l0bF3r7hn}eD$@Q$C282pgZfdNL!`u3OLZNSIrazOl*98*EFN@h7#n5%nRfA3t z$(k9e_3a3Jx2dgu+yv}|RU2eN*qbP*c1MY^Uja(G0HVb7{jOMHMLl9i|27kZj7{e~ z1p@ytk9$PJm1e0=%fu!=W<=)WD>+wfQI>i>uiC^QiU94Ii}$^kO$x#Z*-gaPx0QvQ z@FjvBhUXMB0!-$LA(w=f;9_grjOzH*q|?sI(^FZ*STT^sBNBj7{;&-aKEB62F1+RO zm@6(keIJo~`1Dk-0H_WBYttdmA4$8VltxmDrVwtfj7De|jB&}Ed*#!ZV6Rl7{IHtz z@wx(^_?6Iby0UlfvNG&vy`)h$)}wLIf?v7b=-VdMl;oyEJmEqU3elJT!%;!9suMD{HflaoB`?#R&}CCh$6{`Y$aNfp$JZ!IVMsIhe{l4;cO>{Zn~e3{HTCGt}Ptrz&vXl za*iALcwlyxIMW}(l7gJO zBgsOs#pwg8KyM~M?QId?wL){+=Ct%;!DuD>Q})!^u3IxibEF*9f6L`GJB@-B{JLkE zz!hzY_Hs8TVW71N$W3J+5$Q4DS^e&a9$Z<9-`uwlCO@TX4WL{Xuf?Z+X-C~k8Epod zc~Ca+V{=L+!?!;wyG(|5AlM{ZdW2fnkT`0uYpy@rFKLqaZwcLc{q>x6{e;s}k9dIL zIna{9az3hz>E2a8j{Wwlc{G5YoiQ3Umz+O;w1%OmZB=5mpnU z+bw?dDNVWv?>jVUtD0hd52OCIh7-m`m~xJMJ^i>mR`UK-lOY78Q0spE`Bh{rS9prK za;rA_^yuZlOd!-}zQN6Z@HQeh ztB}4 z{q?*n%KD@zvsrL~knPSwW-F>?nz(WaZEK z>ZmCF%+x+z%1{yF{nOhDq12dlN80Cv{oX52>Nu4nN+%^vyE_p6_(Cn#7lBBJ1wfqzHusu7s4TH)~<&6fh^dd_plRpLzaS^l}avGREc07v+Bn7X<$)tX$yXTq=cs zq-s10F;osQAcL|APvDy(qRi3jjl!S5GX7RN+hy@}bYiqb1$BYje@@MgD0w3XWzria zBYpLV90+c0%$L_N@ThtI$ervu&6`3{o4R%Dy%@{`7t7aefN^~CORXrREj)8dmGiVO zCwZh55dE9QS@-hiC#%!K%6I_jA}CY?hr+?wpePL1Lmp=Cgc@ca8Ga8yW8%LEUFAgD zHodyS<{m6xZ<{S^9l=YeqF8pFEb<_qB}GTANMVAkQLQmPM-p1PC^UAF1#}@IL8ACh zTqTEmSp>R*_PIX|TDHUNE*+3e0YQB4ZsMaMr611&@s1X)1Le1y-of~ty>Ct0nys}Z zT8-))`tkS^Po<4Csf`aBr4Qf!k-ySq%UWiY)0os;z8$a!ce#Vk6Qbg@$VUMv=5U`M zhx~*>A}B&zR7eie<#H?t%!gC}*95+h>6^8kU;g);tcKvaz8T9*q6nV-lt==It9et2T%^2C(4=aG-y3s0{)$W_HdnGLVmMWOll#Pb0ixBy|&R#5~p zfS^9deLAc|%Cth#wTj%svf`C_tydl++HR>gA&E~BW;Xb^U%#@|8EoyB-HM<8DZ5}8 zifaw3FVOhR5U34=@QYAZ9VeL{)FBaRN7@iE9Aatz%P_00@}gZIKQjK;vUdCJxdbrL zN)&n%H%qPesue3%oj#PY!YmPOF&Sj^BMcI8I9RA$`Ie#WysnKD|5KLeO?rUpa$rc1 zgt@o(?PT%q6<{hpwFL@F3K=4xzSZq}{W3kB3I`1(2|JB)D3_`T4aFp8W?{y|!^>;V z^Nr1zi<`&Hl!MclhsT1Om(7IRg45JgNZ|h$k=hLnB@N;E-={v#g)>zJXM&OIqtyi0 zd>4zth;wyC;3$A^Ml>SXw3(_n@jE8HWGJ>IWn%I?O6v~%F9MbrLS_*UBWhc}qwB+O z+fS1j%}dcAAaZh3Q<4O-lR@;ZxQRc6LPPkE#D>Pg=zE`{#*b;RK(oP6k9gZVJP_je zKprTXJ<6;YH79z-0W#n40eWmZCDdV}HsaEK)YphB4QT0$=k%2T06hR=hcP6D>)6d? z?5R7n>@dM-rwO!bGUwVF;-w!Gw$F{W=fODIht`XTo{5fjj+E8x*)*ox8gNPpl31r; zMd3DOliI`Ehyu1jx9+-c1Tr|&K&T5z3!QK+_H_h;g%}4_1!8AC3F3{QYK99vQR7Wf zhxd>KB1>7(Jxr-D2RPHBtkdFboiQ$lQ1!yk*bpA}p*8{#*zhro1rCjce(Xre_vveg zNu~v4vSLkK)8`BzR}IQ(haS~`VKG888071UY4(I8-BIWYyIToqt%JMThkUgga!=|1 z@-V^le0J*S!ZhZ_gyzBYat_*UR9mi8eHwVqm%aFyUpWG|x~0mBSKP^!B2<%>gT1N^ z`SmT|w9WI@yhl$Qmp17Q$E&5Y-W@gzZ_X<35K^|;mXxwSl;ZxjoL)!2!rZi2N+6p% gnEk)s_&?9l#0J?l^#c|P6^WA-nTkqMNecOY0H3p?Hvj+t delta 8635 zcmai3RZtyJkH+1-xVyW%yA_wh#T|;<#hn&+ceuD~aVb#TwG=PT1uo8Z|JjG#r+qj% zdC1I3zHgF~Npj=h(&FG63jolZsf7RvpmZg!)fs);83mXam+8s!07=MW;V~en z*1TWcmXA{jvdMD?&d{1 zy4Ihs+?d-q3IEExqg1rBVcMdKRJ>9YDZ^p%Xhlu7EaUTb!_!ijCs}|72rbkYQtn;M zBWj0ol@WbTSoDiW7FqRL1ZA|IgCEkN&=>3Nt;0IeBRv5pih#AOj% zo-baGJp%nC3ZB*N#`GENDsvk_Q(a%Us*4BD!P`)KsF~4!#S`YzyHAyJfCv#0Evo3H zeW{hE+va=g`9T}RwF_*1-3RFs)${H(j#ar1k&O9rs-cVxNy3^We|0L)68^Au5ul}by+16m>-usw7S?b zlkk=hTm_Wx-PmUB2&V4>pREhp|ASw&4Fix@TpZ=Ar-!wLGm7tqX{v@}xFpu-RdAeG zMcVHVK4@&5L@c`v99$26;R_d7= z&*LXP0qcwgFK!plS3!zLbStKb1WY+eEkbfsWsworua~MF3dW9k%UW}_X8h96hZO9A z8Wyf9RhtsZJvXCy66S%$8>C8&h9-xBltT`&;Gn9YCDY}W;u%S!<)x{>!pMqL@|iD}cLclaF5d9D8tWMD@R6cqYD45g_3H$zcZ?{Q-7X+>va+X!JvIYwoL zyvTPBo+&6^>4`!IyZVs)qdE9UTic`j{O_DBE0N9!*ukCk8*`KxiFIW8-7O%+^V798 z$*~fbn$Vvgp!V}{&H%QTj%GS+@k8_3+0pH(Guzb+J^7Ys;ALgOMSIs$Nd>G)M`nTS zv`MW%=Axvt1!lehJ1o`Etkx56XMCsO@^*nsWsi@3s^huV)_HSc|9albd$dt=cuW}Y zy$6mfs5?hJ$S|4ZB00>Ow?LN_d{YL!{#~l)*lbv{}Rc0-$RV&GL0$nC4 zWOu1FS_BsX6!gO6ZnU2hb2{&WYC^H)q@00D7qT@7DJtQ?8n2UboGY!Qz2%*i!r`a4 zW7f@`7RyG8K;r`aW(?E2^m3<=LFC!KUA%nyUg*{BcyYFEZ<2r90x8NI35i*SwR#U`D*iYKniaCp@q!C5?!DH?mg4@6l>U@j z3}7t=MwO&KD6M-O+2t~{2xY4|`*zBcY0XSDVf5I^tk6ltDwlTH;_fK8<+$}?qd#?a zRRsjFQf*XHZPH?V4r1_dVTdZ0hg!%$bOoFzUQtE6`RgJ-22leXk%;wTozfOGPa_;p z2Vc%Pd!=*arA5igp%N)r*;pAD#N_j$}y9hd|HcNymy(O4s;yx=)Ka~>1gO)qV z5=6#-(y~rVW6CJ?w^V620XZu&IjitgbHmNI^C5g_;fN%ORMmkmz5s4G7dbMGAH6v1|5WU96~2SvcP z@(UCd7pt~X2^3+_OK87&ACWdoQ0xWxEjqk1e%WKWje+M)?BSliV7mSyhl~6LY@pDR z_|l$x5NW^hOso8HCc~#%5%0zKT7xmKYHpSZ46g0}Flhk3)X%nTu69ZQb@wk`Hosn~ z;rg%!zmS$Njeri1z`k+OpF7TeIq`GZ_^+>!YwTBc`Pn@?zj_BfaPzx6_4IZ5nL90{ z{_-Od|3l@?kg9eB_B8osx-D=csOv=va-m>oIls-$T?ZfX6pqwCT?HXIZC?KEU-mUE zkTA=zHOx-kbVdPI%3dP-tK$_>0}f%WgvvvdAJ{0z#gn7Si=o(xMIKJK#mamz4UA*Z z%w!Q*2(2~6xYOul5ikv{u?}lJH3yZkpU%l-#pAQ-w3KF9V|!y{^(vDlY6`gF6nAA1 zk_Q(YSsZIJn=Aa8W&@CBQ`FOZ{jm!QE;D>HJgY|*;}3vL+Q*+YR;WeA3h4UUJqEjf zi+C8Alhc>1{(ElcGv$jL*E@1LXq;-u{OMx9&CL0$wi9gV;@RBf^!7!EJNW64g&jx z#?|6$H7`cCT(_rE=vP{6R(KzL*kVmjHkijT4YntLrU+T@x0Cuey3!m-)i zKX>NgGskzTXSzNm#P&l(DVoIYVSb-@=?_<9mOI(04qi`=pB z4`keyO*i5oAkY8K)MXRLuOtfIZw2ODyVA6gQvDRh?0U5G&7U)zawMnH8y zihzGix_U^3)DzoNvV+=)^g8cauv=Og=zSZx{=?o}VB57@9vPvqRJ0V10 zD~cZ(mJbacm=EEOAx5%fdkio8+SF(VDRN~~{Nop|5a|uCq;ibHfLa&Elm~D@iY zA2;-=G3;Q5Kp5o?78SZHA5q2bFN{H)rf*|25eIeH(;U^gwO!D8fv}jN>Kr0em z3p!P(*g71}$N}tg8r5obew8rG9391EyNc7*-gG=&NG*Txu5dO-u_wt6rOY^LMj@MqY=7~oDwX;~+q;ysv4@mc6r(>x za2>9{6!tuX1C3b~L(@GFC)X%A1Z*DMM_PtC2ZX>N^B1h7<7i|_a?M$}%XRgYPp9~Z z-b>permBdZl98#y2^pBc$m|UAYloF0^3UxCWJ#1nM`P1npM;kjC5h4EhDafaLWe76 z1hS`;BmBGP_>h|Mu#A%E&WYi4)1?2@nuX%=xpuS;5W zQ&SYbP<$zK$>9HzU`+!DoOH0SMoOm*r9Cw}JuyW-09P>W`CFV#faZ z-LCSoki=Er;Ae?7?GShqbE6T00ex-d&&r^<_O1x%d7@bZkeiNE@ zK~BT^{+ZV)(({Rm@G_ivRF%fB)ytc)Nqc~i&b^XROP{|IA&`>vHiZla%aY5 zUVtQTlO8LuO;{+Wd`D+>pq8|-mEBOXWQ+a&v?ZN4Afb&3e@X9-BaX;fy87ajuRS0k zomqzo9`UQ}3h{pUBTk~6meH$YbMRt7u-?OE<$n3)4+?gd$(|{Y`c(;s|LpkGVEuSx z=w$T{i{Y8wNQQfH#=*K1cW!?l$v0-aj}oZEbFrWaEG+uBOtk6f`@6qwC0O7sv4xhoeN*xhz*nf3dg@IRPGDp(09dnhr)o7P)hD)B{Os?Gmtd zhQaa$U+?3i?2*`+<^97chO75)pk@&1&W>8mD1r^aGM|2^peO*PMk}a$Vl+#WnNWk+ z^}{h1yk}S?&`=Nhp6g^EXiptazl3KpDgB@uqdeR+o9|J}5)ePH=H(|*je+Skv1|=Y z$OBWkBC_u=Ev44dXxSq9IKb4cZw#@l}FQ@kIgbm&4I=#umbAQpr9c;;@ndC1(^)EAIybMQU9>O$fV_Yfp zYjEOa4r>X$DO~kmML|+`;txlA#x#M^)af2dkxcFKSQ`t2T~!7uKDSN2;;)*UH^ExJ z6p+>&x>c=50-JjKBb-AA_HbfLjM)7hW~FQ=KV4A(BD$!VG~Hd!bx&h^H%eumr`7gcPv$*2Yg{ z-FKTRp>rafDI734N-B3qM^nUI&^hu{iN~+NrlU56xxYK^vwY~Ar8>-#_2)U5JMV*# zCgVmVmwyO9#-rVhfR1vEeqY-;E(hOFp=XiE&N-8Vyx_9F;qqwpYTmEYq~!Y2R2P__ zu$)U#;3E7wZpWw+3Z8FFItCs;y-(d-_;#HXnoi4`+HsnD&zP}-jMaA#`wF8wmi2Nq zTGF}|#(T!nPab){esWq=ORS-H;O9Mi5qD8MQOC+5*x`i#XCrOyepboNubX=aCcAu?t^kzlO1kRL5Z&eHBT5vcvZlk#5H$jI*;aH4u2` zjPsl$y_G!Y^Qi2=rhRffy7bKLT_%c`1kMxb$=yOm7Es|MA&(ZmHKh?qhLKO(Dn>@U)@2|M#lTz{knK+b@Qb> z^QBs66&5{Z)rFd8xQY=9>H_MMw>nN+A({Q}b-yR8n5=Z^qGnIuthfc2NM&P_?Uv8I zbJ!|!#@Yo+of0qVs@6Fj+V}w|u~a2(hEF)_{dL*!7C<`&7eNhY!Y%vQTaV@Q5to(( z>mDuLIl6@);}(o2seliA%$4r^HaA9J<6Qih+_w4Nt)LA%tJ)PC5bJ^y3lUz0f;-!j z3ka5gxXxSOnT4eG9-6gkZKbA`rOlbnnbQ=tDW9j}PPxOOoF-(czSXc&vcM|9OugqLDLuA4tv|zsvaHySoR(coW^E3C=k_9_Nqj zC7%JEa$nGv_(}l$^Z0qmGNkyd8qvi&j{|>P{gQ7N%a-1@)p!R3NEE%T0>v8M6?+

T`l>O{l2xl-p2qZMs$ z7NfPGNX4wGs03Go zHa#2b8`?YNtYyabuqC~r27CiK=ei_Ft|R1qK=H)U2;Gi9F%8`#*i&cjTQeUXFt;?; z@srv#>~))UD2TWp6r6di;Db(hPHZulJY^4Q+`W*Kn~ zebJD>KDTD7T~Y*n4GxF!YV_>9js)rYU2dRdYkQztGJt$b)55*X1vZi)|b zP`xmh7-uh07EQ!<8r$pI?>B4fL|=P@PQXVuCC5;VP5|slit8w6KJ&F?O{#NWKsw!X zwqr8%XDb7OPvUak(#fX!R(&WwNL6N{nn}~XG%@CK{<|O<&tI^o&-stHw~A0F!~6}) z(-n~0tX0dt8H3!qsIy53%MC)Gek#+e3ozk|y}?C-04Sj&&9qnk7*Lsd5j}Wh*;zZo z{vJU;y@40a&*Y>Z103yF=fCIhS-w*u&i;FdcnPKa9>~{I>)+2_V}6;LdV_bVHnnq# zO7md6y@RGPGC<6aOfVZYwP)q6h6D;}B}Ckyww(x_52D!`c=WWE>-G>f71~^72BP); zk#7p^2*P~qsY2yP$etLPTNPBac%FeLg)LFQENeX#oOmrEB3g!o?P9Nwz%%9f{6uET z>uJqDV`)Gz3+d1}m?!6*=$7)_@=M8rN2Kr4LKz28(|-Pu`N?R%x=1G@F#G_KHca{U z8y%upBRz5%`YNN>b7ayC6TXNvH8Am88Y6KImFar$5~p$nJl9z?;^2V@l~;T_h37_J zNG-f+qLO6`jq~Vs-f2J%zU`&&z@qkKa7a@*FCK|PX4a~Mb-xhVX~%(hVSN?OU+A-9 zvb?Cl5Q>ZPskP%!;$rg!Mw`!;+M_m~=Llpa1Y3fW{$9H%npR_s@D{L~yaG+i^$^LR ztrCWU?~xgl(1MrvRNFJ@MX_P~p&_&~*GR*Atn2z8R+DW20nZLGdZsvH9BM zVI|Ex9p?T%PJW$h{-H%-p%bCWFKYqYzqkBzM#%#wHksQx7x$@?e~a^Ed9uEZf>qV+ zcjm(IUd8x3kzGKCC0K6~2Bg!moI>iARqIM9?&Mxps8^}=uqY*=+XFlRm0oDygXfnN z^eJS-YuZpijPhGUy^83_mstp9i)sjC_-G?C>Laf;@CJM0ZiOvPfOCXw%T?j@1+1^! zC3%|rOre+Af{EIydGaH8HAbMcaxoMQP*BRIQ0f7{D2_a(kU=KimQvde{KNSue8W1W zzeaRCbKCpf(a$r62H?lrAxNNQb5Hl#rEf5PdpFpK6|_mX2r^{OsUMF&d|g~momzjX zS$v7y{5iIG)^3}Ty@-` zpLk=py~$qvw3IcHGGNcI1lK3DLkP6rK79u`^}aL_o9`Zo12x_|-<<=ydZnD6WdAa; zl(N@El}x#Alpoo)HZBxFU;ux8@2vdAs!`?x#6OvFpdfY^Y90heYxxQr>+Z%b1NEHV ze9X?qntHc`&u;=ns~>+Fi7`aVf#?Ky$u$EmDiEJ5d$W<$buiEU?J`eD!-5!8AzUV6Chy4(MBVJeeT z+4A$SBP4D&Qu&p*N)dar@lTQ`vBAwZgT3SZS5!D{V8<6ZA-Xy{cvV4|tYLbm5;~`1 zx;TZRL&}?RX*$=y&9OK?QN3W__sqgz|E3JNK$hBrH$n7T*|*OI$rbo4Mtn^;G3j?5 zu(CsO;zD{Dxs9ReRqaidmmYYYnlc7p^hq^L+6f{)NNhLc+3#ZSXG~yUvlI&d%}S!%P9fk0%<$QT@yw zXlQxl516Xnz^A{iu9>JX&~k9bOh=Tdk{5WHb}-PNqK zBSih?4Ax8546tN08?+Y0wcR5m)j}riI2&$69sWG@u^~?K#=L`JeDvEZJ%Zf$$g)GZ zW;esn7xfrq_1-K7bM=LmeIT+EiLjI~^S5^TAMG9)yq!_I^k&{oKqHaBK61h^TPOaa zyO7K9MJKHO7X%xiCEwV-@>V2669H}`&RHXZ{+^U*f^{eH_upEgQI4G0Q#XQozm2Ac zsJ9Z&I1#^cVWx6Ypf+Nqx{F;7_8EjRbz(yHp|?V!8}WiVv0L03Rsi@Ff9O?*b$maO zG-Bsoix!OW_!7)o@U;J?>kX;%#l8YCv|AvW4TF4PulCe{y@_JX>Ax;BjqhjMHm?4% zxYvQZSAbs52`xGh>GIRcdzthVyfb}uh!LdP)7qfl-GAUYS=qeoUZm2vmI8#vZvgas zj=slDcNIUQsvNH^ta*KR5v#dn*m=$de=vFO=!qRte?WIk+WbMWcCq@OUHCuE)zabL cW*pDL!_(Kp+6IN6pNp3lg^o^6LmuV702O+*CIA2c diff --git a/units/unit4-goodPractices.pdf b/units/unit4-goodPractices.pdf index f9751510a33f7d00910dd09830f189b5898031c0..a7688c5e00f08dffb8935cdc778fb03bed4f11e2 100644 GIT binary patch delta 7540 zcmai2Rag`Nm!(TuItGxI?xDMe?igAax;sZ=K$;PxLAs=yp#-HtO1is2N+ezP+y7zr zX&>&nFZb!(IOn?y2#gH`Mm^mb)H#xu5NXpNst6EQny#25Q#d?DQ|?S(mOAAdoW&N3 zt{`~6n-xF$hJs_+)yYJD#w2cAMVT4Omp5*0Wl#)44Hconu6~cuA@es`8n!0tvEyjA zV`VMuReUur^-^M)as7Ddb7oo?(B2~_DHUb1;m%HAj7_Szkx!M`2^8OjPKLb~)-xC? z8nzkFpUDp6)oNYNFh_R3UR_GFqB3ZjlN|A6p=$|~dF5o4L^@=gH2d4uNUjcs#;A{H zxU0?x(Gg2rL@{`I{NbNwsDEXYDVocw*S>=$k-rzjlyM__r9yIari%GfFc!o0-qQn8A!4AYb-Rg^Q7WixEc_A)}> z@LHXYMpm2eTP-uVl;4%j@9>0Yh~4$-{CaTqDL>#0c|l6BDG5%6oQp#RrKE7YeZ1_f zU2y`I%|G_V{bM)!LP{~oJiREk#!WVAzLvgurL?v4K#Nyk~ zYz8yGv^Q)C=JZF>7gaL1`ZnXh{VEt>xQ>ZwjG;hN^rS_AqfXK)y`qYEiIy^hlXIMD z5{Z~8X40E#N&%*7%HoWXs4v6}g(n{KQ@$SpFEQ9X&N4S#so$|a&{R+)W-Xr#o@3Yt zud)Ta;I295?ERL$g6DUqo?oa#`(^%-cuyMKOfqs;i;V4{Pv`_rkXr$(D^c_2)DK4f z?HTEbjKLrd@Pn-9WsSEl`L)dX@~mpjrizaHA%bN1U+&p|qqd&OBu5+K||LYlV~6yU>zvVR7B3tbIzYJFIZ zyI&>hUn3Z8TbKB5tJA_7L^&y?u2?ot&L;}#kQ})%_b-k1;gMj~sL4%U9c?WF8>QLe zgfiHl<7vI!&)HI?otcvk&ZIx};fB5vKs=81tzEy7Zb>uwSU~@t3kg=w$7S_ORvWI) zp(cxA`b_}EbrTyWbkPP%c%G_GER3yN&&QlDXSiiX^1<7B`w^1n#IQRfUV;}QxtOVre5sBPqnd3rqVktZM}0oePk zXDv}muhh1cQAYi>>#Qn}&85}@BYgeAaNu+)Up}tlnm%gkqoM0ka9AtHV{#+&Rxj=K z&epH1p2zd$nPIEe7r?lI@kppO zh7g~=zPXs4x25(b(u%3}fW`%HSXMF%eE6!SN3Qv%t?J9sd@J$bR0%)EYp4gl&xHw<*#IsUW+pDC@$oGU6N%8 zLT3+nn-a}BOOsCnP4%9p{{f4gBND);fjJW%@MHetQrn1Qih~Wc0ibLxStjJL!xkPf zDy#irY7%XZ1)0sSB&&eji{xiY!8MMM0}C6|VOK;1L@o`@T=y^wB8Nv5TonD`-S+zH z)i^mtsuC00y8;t}?s+rRpqgpfvj_GyPFQ%tD*e@W4MtHqu9*#hfxq)mpWn*9Atd{J zaB4&Dx4Cql*9FfY(g8|p_q7<5kN5u^rZhF@9bOXH-94#=9|g!pEnOEYT!IQqUEFon z4RWvy8k_OL1WGVbw5g2_Is|JYK^!EgFbrp)<{_*CU+q^dF$Vjtxx@pV6|@eBNHo5 zpH=DDKc}DI0?x`P_`fO2L2pj_NuY$V z{|pm8jZ_~s!!KNp)pjjQflb+Xz1f`K92Ub;){J5-7VxwsF;9|UQhrlu=u;Ve8g=N< zaOHtH5{X5S-eoKSBm!`Dv<7&*kuc_WxZG5L>ed|Ua%}@nCBI7oiIQ+fy(AX#qA*$e zBN;kH0S|caUz>^u%&U((40>q#_6n0ac_-@n!SE{O=@DPe7BQb7A~El(ABD5=`aHH% zl@{$9F`Dukz$2!7y$QY7NjR5p^r;nPTHs{*2m0xZ3l<8s-ij%VMZu!PBSd1|G_h&W zz{ZnLfka7OUT3#uiomO%Sw<7y@4^ZfE^pv|U)c3*;{-kRZeHgYN+ zIaMsUIEYJ@Qom+5mnAcm7Zrrj_>hD1gI1=VqHCY$n#Y_MZn*?oe=p|7NSTUkeR*sv z_gf@|{zd9(u|C1EbF=-A656??di>^w3jN;^TyxvzB~-cy#If{isZu#zbYqTE2nU_E zJ}N&VIod3$Fvp^p-JFqoN!{AOWyBitvIO2i5UbU~uHO8!mhwifvm>Y3t$>*ZaRLK= zcgKz9)YWW6rDoA(n^@YjEh2>bk#3J6>*&Gp`th-?PHS=!YQD0!boRMfx;Nw7*~!>D zfO?*vOW6HkYQy$~)1#+E>9?{qUx;uHg+Morb5i-%w^h#LbFR=1sYlVDjt9S=)4TDQ|(`u8{rIpYwlfHcXPb&#HR& zJl681&06hEpzulMe~L(-lW0qxVpk5M5my{g3B$gJ7qQ6@Xh#`N(H=DW5jeWaN?^k6 z3E|~>5RNPTE};XQL%4U!n8`Jvl57;bGm_V9cbm8I=YX|78dfoV|L{>Bg{WeNe3bqP zZhv15gD4xhs0<7t2Z1G=Gf>8QVCh%VpUBj_YUGAin8?n{mQ*VqM{bYEM#;<0qvOj) zMe1<_jUNs?+a+kQW9a+)bey$}Bn+-76yXRtbBVC@0D!a_-M>8kFF*kaMcYRZcJeh& zw~nFV8R=xWJ(m@UB1NDJs*+Agu+mLI@Sr%VQc@?0tpc~~U($2kuMBaAa6PnyT(3UO zVZGS*IwaP+q_Rp1$^674A0&zs6pnF1KRoo##;DB0#VLH78a;uIIOan?eY?Um7mg)x zGpt2^Y!?9<~R}$-PrTL!bG`3>CBi+*Q(#@JhQz#dhp6ygK%Sy-wVgWhL4^6K zOn#ztoqDc{p+;}{OM5;x{T^f`I&ldhj`6v546M#Lg@U33UNjHFMm`FGa zke2DakgfA=?X(?Qei2d1A`cq#G1bTcz))G@WjQrA81NB)`?Q+&213fvIZpqRFi_ip zw5cAH;KcZou+a3#IsqocsL#=Gk_ih{stDyANNVuNCvwVIv>)FN0mf!tHvlbhFKaUh zHQ6neUhU`P)&@CE+s9OO^1b`EDAig6^wp?2S~@mg(P;Xo{Y&OU%>P;L)cOYlX#Fvc zTxU%aZRhgt61EUFpXAMo)$F!OBElM;w>5M8 z{#BXj5$hJ%uLZ9^#B${hfl3DGAsCoE3T)e;cJ<3wzE13@x<6{)L}hxnzP5ANDQXa40pbTO=t$f#OEfKIeZEv<0Ck~`Q?Ub zQlWR(Ft?jp(yJ6(+i|YpbG(xI0`=@EYV>A=I*XKDho~eMW!e{ik6==vo`Tj1PkTt$=FLkW`H)5GW%8k2~(wz z;@wuU{-j=|^49L}bH1CEN=m+t$_fb9k6Si1*8Fm4>oRWh}X{i))%Um;Q$>w!UME!jw-6~KpvQK7mQAc>H{9PWs^{w zF>$>?%Z*Z=+Is^@GNriSr6$QKD0!cPO{g|SYpoYig(gnQ$%{RRf@WP>YLZtEV&SJI zM~lGi|5%O_SKUWfipxE`M*(Z`&|*)1HC|*L>5Bw%JoH2ph2k~>ASYN^@7?fXT+#@Y zBFjO*c=4|cG#%Df@z*Sby#45yJl0ienNwDPIw8g=87jHqIC@7QAvR7DGA};9y z_GSbhivM(~*Cz9p2B8k4Oo~ECfk_aNIp|F+g)G)M#p^AVLv~Cmc6hbINqEtgRV>Lb zWl=b>XIupn^!2zQk&yzm2Dv7BK=`YRVGxR>lTRQ@T0ahBTC}v1UWV!}Aw!&lL;!Dq zfVsI40e5!%D|TAmD~D$^Y0Nt2WrVwKZO$=e0!|7ZOLAfQXLv5pg4oy}nMpeQB)*H2 zKCNq!>%Nh0JMW(CANGImWxqTU(BWGokWaV_oG8Q@K_kiI_uW?mdhT29o3~4&a$*Sn zwm3^daKyk0>FDvoq#R&(mmZd}cGtMQlviPlpY@aGti2R#DJB+}Nf<~lT}x-lC~r>U`K8>x;wVPjfqB6zs}MJg zsdXwt=nAp|K1ZWH40I@Kr(2+vvvmCEYdo7|z+BB`9XNiE5t|ak<@ZeHPQ>@y_>?1p z1UK+^UNFn5BHZ;+zTxT(PtLqkQY6og4`p%OcwpyEkmS`f0JdE0qlT;7*UTGbFIgwy z4GT0NcE}oaId0>=0WY97tmv8*TvH$4$#$7Q+sv~nS@mhhu3kyzjnyq!Yv6y@Bp;g# zB9frySwN$o{Y8c5l(Q(W}RGsJ(Gvi_6Qbo%~8<)XTksLl$!-0f;+E_s zRPL5Dsn)e?_oJ_&;7UjL+UERy_ugntI+Y1H`^+&>pp9t=rSS796A+Bs%jR*bRV~Ky zgAmp;XZ>aI*eS!gEf8$8^hV!qKFF^55^QgyiQV0xPU_s`2y3y9Z$XH!3+coYgY|j? z@;@05;^vh{ktR8@H2|JHy#-FZy=n9?VJP?V$hQt=-hlX6=Bal@xCRk2ufzsfuFwtg- zdj}7RoCzZ}DM7nm(Zb-(Ms)7~ffS>34|B5i=5loIb7So}L26%S%jYI2BKJ^vS9YCQ zp;p>&d@RQ{qo=)?=QJ||G5!jozp+jDs>PlAvme;9tfmn0y*8(Z}%hu?|7wnGdz;ZX2ep*-zL}P^J~8$>{&^vQ}K2Gq@kYr zx`>~}1sTlZ*mA0C#eEkweP@&-Z)t8!W7C#if5qYLLd`hWJ(~SBD|shu);SN}?95%5nXx@v_;ZV$Z{Q)4^k2y=+JM zT1!6YTsAGmN9Q46I>G2#5!{^aUCS}o5I+~&mHT~VLFqO&?gT{A07mzyh%@WFMpPcm zk$j(^uea~d%D-QjN^exZEF)R`SUUTX^c!h^cd+C4pAMt~#D3u*6u0|MLQ7vl>t^aRoef8k;jfk-p@Q}@=8QVNG4KX?1=E1D;iwNnNmfam z^V{yo3j zSf{PoBZvNsq6_d>BxAAP!VNWjQ!{U8&yG9a$o>?6!+>2}A9JO@xG{|F2i#D06XL^D zf()TT`y#NY;$EkChuI<}>{`?=9+B5yn7jZC&OQ0*b?CQsHm4bh3HBPwCk2hkYqF7M z9f<3$Ly)F76`T&`Rq6AM$;!9zsq+sm!FAKs$R==8_CbG-cK4E(+^&}MKOzH!J=FHa363Llf9Ma^r;K3~vk?L&p{Fek}VA$)n}>%si89irx;J;_Xv`k|m8Bg|*kiK`I4 zZYAX5jX=i*sX#5?-AzfT@_KDVn=)=v#z6@ONTk35A*DNlXYsAdUrzD#c|jT~GCS7W zIDWa2aZ#Q+XKB0>ZXj6{cY`&T1}p*@P`XGDe{~ZKh{(1M3X|yUa*CqKwMgX2=$F5+ zC&?htwj19c+Yfj7>+N;Y8#5qMW$0)vi z;0Y-HS&J}Eb}j=sC&BuA?uIdRPwsi*>C6S^2*Yq>W&98_R(x==AsO7#s?(6Yka&b%U+ZY%a}MMjHt4%>~n^3l_C@7-wY$ zXXxRa*;1|hEyJyA@`HS3(QJmkXN=z7#L=uW-H*5X5Sokk(6>QtU2_}y>tY+9jaxxv zMS4gLcaEFXg|0NZl>C9lI zTPGuv2Ctdl7x}B1cUx9c7gh2n;J}0rYxr@`g9H*3qh(EK6H4oo)LXq3y_DC5If&vz|ZD;f@EBj6Qng*BKwPkg4Q&`K$#6MTBhl?Rc$) z?L>s_gl+8vg!uX13EA@g3qdJ~|F0tBJPTPFD)7JGJzR4}9vV6%gO@GbB1Oy*71*(8 z3~+i`ES+L>hM zN_pQc87Y)B|74_M<7z<12vmqD;pjjuvtm_O6j`S1d?#{F{OKbnQMSb*G}2-^(ypE} zh!1JZnm&C9#~2>?VC9|-GNxQ}ifq7%wlb)Wp5r5Nw$iAEwKU+*8^EppK7@)Uup&gi zH2~&^uoAvwPz>BsE*h1ZrA)lm*fE^bU@rOk5PjnYORR4I{c)#Gv~CuI*g!u4lM{st zJx|=LKGHaaa)!Eos_m?X#uir=e}uo&5Xk2Bl3=Mw2(c}mpu{uEdKsZ=<9MUrh;O{( zr&cl=XX8xlNLAsIZ>vG9kp|LC6RL42GDzZTEE7t+sNpFA&qlK-F-w2Tw#{aFOQ2Tm zsPs&rm`@ z;>J8gYrlTVj79Z?lGgUMv;#!8t!biyc`>~&u>b?_fD~Gr?3`aI+sWOx2)hy%JoqiX zhtR~!ds5&Jb)MS#V@M4I2hh=Nz7F}2{ELswvrgRc^A>eT63;K@p8f^1iyzlWbEi$| zq>{!MV@2es(|Id<4Je4FQVhdEE8Kx9V>)LW$?EvTsk|&~C`GacpXS~P`QP5jw$iZP z;HQ^Ar*{XKy}jv=&w9sq?(lBK;7`!X^w$R6?p*%*cpn;bOQ@cHc#+S&HShME(^Cz|yV_Q@ggDl{z;lVPbj%7joQBmOcN~BA fbtWX)uIT>Ip=8fyaOWY@V+#r5Ff%J^DdGGV>c@5j delta 7590 zcmai&Ra6{6m#vZD5VUa#?(WjKJHb5!cZWthXyZ;A2<}es-~ocW)401^f@U)F&%>Rk z`%eQ-rzFl?ps?B!frFP`+ROevVP%a#}E#F`{DpNv4q<3nl(K}IAB7!qinq$3+ z#e7JVFTcY?clxs=Z_+y)0|wL^f=W+vVVkAt!gtlxx)%A4D6}E#1(oRorh4-Yc@eHK zRA<jNp@lZv6oJh&uySQ?MjrrELOW$RJ_v!OB>&EaDxl z4wBwp9IcI(NWwCL?#uI}V44r-ejS&#kj9|f5E@8q22^OENgiv|=mC z-w$YUxEPslI&LwCvZGiIF79%MKt=9*^sb&9y{gLC0^i{D$crtY$#L-bseTtsU>o#nVo85k^*$i1ziinAfIA4;Xt0_Lf`S}A^p0AaNerPyer$=Mi&m&kq|E3{p>u9JC0EYvFHE+6G{O>Yi6WZvd}mQ zae6yu8*B++4Q?|WHMALHjyGu6bn|G!NS-XGj)0$~tdJBDTG3LFN@&uf)JUc&=ADHU z;WPtfY9!Gq&MS&gfxB>MK+EyZGD1*PG;K&#WQELGY@*_PyyfeKHP@KDIR>DS+o#jz z+{Jw?EgLWZ@=tL+ebgfh4MKZQlS5>6$(K?iUBc0r(_V+ffpRl9}nBBju#u5I9k zygZ+D%e)5=-2D`p>mbR5E#H6#M2z7nc! z2)ndQbWC+pZwL2Gfqlwa6VJPyJkwUGUIaUUtu z#C%w(b6ZKSfiicV+Vrgv!W*7bwIe84)X_t{2IRcaR>`DCLq}Lf&_pGhBX7ZZ2N-E; zBCyH_h|x}y=H^dB*pauzN9Zh4KOm9KfyX}%vt-92xO1omZq9%FB{(jrjdK$xa(gu8!;HtNrPT7}LSsuebuN?-!r6t~qW~#xPxt z*LrwGEFg#Z{0GeTH4|9y|0q6n)geq0C-O2Vrpt+0YN_7^oraE-njWaRiozwL6!Bwp z&}8qt>KVJMJpW3#@pfu_^3P-U@;4W}HwZ;m41@cLs^hW9nv=|q;^8EShroGBRrO-^ z?*8cU(Gl3a07UzH^|8xUPy@0wRZtg?skgcfo~`ZfQLd0xVf+!iu7F2<+g*OK6#djC zj76gcq<)zazVUAGz5Z?wp#^gM-EdZE;kCWK%F(~j@Fp9=xVK`!1sNXnDeWy{vYQUI zQyYKd^|R-a$W$DA91-I=mk1!=UA~@wnfGyruEKz*pEHF=DVQ~hL!_sXb8%4*3Fh8| zTZnb~kYnha(3ryG(a>i|i?=EyNpn@FuJ$srW0C){s!qso-(q5u%pT*7^0WwTUkP2z zC;#P79&w+6K0T?0fTSCmk5iC`R;h-dd8WuK#kt2+PAzo^aF|xrRGjuhLYfF?LTfLc z=A8&9rFoFW!X5h6#rOjd5yPk0C~_ez)kx`I(}sL!R1~&yuib8lj!4obwHIwH?_pTn z19Wg%oD4f}nW!1(SITGGj-gNB#69x1+<+t z5UAPkSW3O)A4OmFsW``0+&x0)N7Lebx)jVjEQ!Ek61(-@_&uJSZ0G<7tQT{@Ac$4w601p8JI?jbohJQ$##C8W0%Y_fKQ?{ZO**FbT4Ll!+?I`uxvs_G2tw)u;RZ5BTjSR zTdnfneQ0wQ1V4c=N@R70DggF##84|1b@M-_(aRgBS*XD zeU4f9WOk+J(H+eK28mZCk;`1coF=}yNc5r3?Rg%-`B9yeQiwpvxXNLCpI=_jWDA{Kz}73rFI|3LE6DsPcZ` zH)(uzk3xCh{&&`~k7>1${u7ZcbK33P$+G~f0)){5=DmE#^3D8e*~$)JmU4J^Dwnfoq2q_38h-S z|DC4+)-YY&hn?p&N%TPb&riohuuIiN1$qMSMhvcPnqP%-XD#L~M0oQw;xf$=_=o1M zEL<*5FW7@R#Gm=+?d?{&+Wbsih8R~pH+#xt95fidu@UZ`dNfqEi%>i_D9gYu=B#o} z#W8rz&BbGt98Nh;dB^HoP4DZ-Ygca}V-IX#)|&yvv1A0R#^f`g70jM`NL`CN#aueL zmp$YC0g_^H>xM*TS*bv%!&1Fd>)! z+Ab6q17|@XODx=lPg{265tZbSBPH}hLGk^l+4#@17}+^$vZ>_XL~CRwZGtuC>(V_W7AV=&cmxMc06er6=GsQ%k+%8R3_;z_m zo#MUsUVgpWiCjHPdWqYcqAqW)Sp>V^dO4mKL|(KMz0`EoV3HYBE zF6B`Wto7ec7v;)V{J)(pZ9c(2r`xCM*3MqY8PM+`gvVUG6Sf&V{nr|sHe^p>wZkl+ zkxqrz^!MinYol~+&PpR_vZP)?%eCtj-~3GLs%s_{>C#*SPEQ_q97< zohrdu+iohyCdj>N;Vy8+|6`ht#mzA+`3ej(%#yeS@@Li>;mRct_hHswcKOAOv^#O$ z@w`3r(PtW&5A0>{J!rfgwUhA|2W*SDJN5f5=Qyae`$o`cAF&E2i;C4pwe;hQ7LS2< zJ*mbx(w~^eh1ORw=S+RK2lS%T!(W8=WE{t{WegY%jqNP>e{=DE35=_kRy1M=4-t-tC+*|AHAZ-`M~@bgW62Q@a&J=nhM#;VdARQ%g3_4iIB-BkHi$pC&~FA(sa+ z4J6dN<-$8amu-i40)?WJZtI22QLk&EIO@zMFpPt5w}z(-OsX& z=FF>nods(fE;*n63~~SOMTcg@!DFWWd{P2)c_S5=4mrV=)lzL4|JmPwqt?7`Mj%`M z)!1{<$ZaDHrO)(|XtzEEm&B`Z z@Oe+(Z%EjTjl|Wb%&aP-Tn$l59we;&+zY~t4AZ^+E~~{B zA&@g#A#nNfIwg915GmwC*!R7PP#VW81?Dge z4`F!=_BHL%U4%xv4Yh=ei+eXJ`Q8)Z$EUcTrgz2UASDpK z>^v$C9WIsKS2I~?iXx3X4g&9>tPr06Xo$Qr9qWvLzMYT(4bqF2da-Oa4VY!*U8C+V zz>NG{gf_g{R-b4?iew_GX)avJ_de8;?ZlaRYdm4uC>qj4O03DUs?&oRYfnGs5=42* zn!K&Ut!7!{-)f`Y+Q2X^_=es+?)e+ul-qqHTN0|H$K77;gCM~_moI_hLP6i1IP~To z&&(z4@2{?RdpURIc#W?`lEFLYTTr=MA{VjEP8aby5cBrI^)C3#C^0j1k}%o2w@__h zbdl^^-3X7%x6i_Rp_{`IdP~;z?F*}gNjC;m{$1S7g0O={_|5Y#&f84`M+R~B+qBD! z>6YFZCdsVwx87)Sr~^{Q$4zp`uTKqeiDciUlPzF`e8G_mK5)tXF=vJa zc>bE11V?i=El&jGPyyAxoEzIR;zEC z)t%#GrOjEpTVq0|gXJ`yGaOxJiGe)|1rJQ?xM4b6?3mkzoGeo&7G~-1j7oXFJ=I{m zF5XL?wGa>;y+l3*;ecHio%L-_&~%3yUX(QsZ%(}pQu{T^V)q@oP$vh+yy1{F4qCtL z413$kZ~#Rlx%WVZxGkn->0#nt#NE4WqFW-f-K)bjz2$mKi5SvUcB4A^ZS2#kSXriF zHjdLyIga1qeQ50jP%mWaMR;%Pu-6-6WGV$m&HR7)1D*V|Kw!;IO<-)1BN=C#RSsan z$Hi6fhj*}Rn@3rR{Nh)e0HfV;ijjKBBSulQNy2_k43k&*#cMZWtNuqmHidx zjjskRsJfZWW+cupi&L>eqW#!BIU?RI(Vui9vQ>Ys^`OpOIuS`E0T|J<&~cW@Bfee2~VeGbz9=Tfpj7 zkSL~gA=D_7Ti*Vix#&StU(_I8Fd!lE+w>6m(=y;`ndnKZd$tw-$*z4(#X-Yc>4NBK z2TVA`c`!ZXiW<~SD7;Mb^#$O~I5_ekY~h>DwZ7Zmy&H50I>UHoQ3Rs<8#bhVNSg^E_5griBoP!0nD}J z9(PJbvsvyOxGeR#SO?VVo1jfR*qme<-y)3Ow<0GX*JX`(1@inAD?X%@MN=Xw(IA`v z@tZfvISL=Pf28Oz$Qd3}>I`z$$t(X+D~sEq3kRtjZWgWk=5J=IHDNy}%DE?^4*vLj zG(#j>*LNX8ZMGJq{8Dg#dww#7wY8tU-hE<+{Yl*=+c7ckl=hTL2$$_3bv4%Jdh2$t z?s#JFX3qBd8IisDtJKIQmH4>3oOQ5>89%oA-!KJarTO{`5Z%7S_1}Kg5JAC-vy8Dn zCjR+w=J6SJbqA?7SHZB(Hd9j#TkzKU=`~|DTTU!YYPG9kKRVmdf!91QPJj3G5x}lZ z0a+|bc9$ueAI>UBziTm{b+tnLEuz##OQhxHC$+P2g?I^}BlL1<@Y1Q~HtJRhw#%j{ ziDHOPx_TtcOwUVbfJf6lw#fIr9=!IlvwN>%jv_XNf5)VViC&bXFfBHp0WSERdl(&a z7)|+fE8`?LkKVo5mUw_!?nqa8!NubfrHD~Sk%?13h&!Gfk@+L8JmgV{6q%ecn}^mG zrfzvY+-t5g$9+Q<)Bv@zV2#9MSQB!(OU^H_<;s*#;@jHJ9sWFmbsUS8P{v0UMKl^6oOF6Co_*9awlD(iXo)_*-+gom}4)uOkwX0J4ltJMY1{=0T}|>kqkH z=2tT}Cg_NKNk6ib+tQTGmEbXyp<=-s=o?E~|CJ-x{%a%4JA4NE05-spl5t1ym`ud{ zlDsws5Pd~>0FG@IjRMt?>Ck1x(8c^^cM8NF?lO^23w7~7k*e)@=VE^#%~r$;VnX|c zKeofSX3>B9V#Cy+ zS`~iB6Sl>~Y-DV90517j_sE(s8P!BlG7!_N%%(ZL#6g@OZiUW6ZdXAZDW;`tMK-M> zvn>D6Jp8p>=);%I;Uj9nmUH1d1>6;d@zDd&e! z=>>6C4ESgG8ec28V!&kG9mQN-qli0&R3RnA3z9Bd5xdy52gc1_0459UXyO6toDTWx zoJMaWhdfjC#X^ND>hq=OhtXKI5AqB53*0Fhbs5}{iyTKm%=8aKPuT-(c%Tj94Aum) zGgWeZAJ08!mq38DKpmZ-LxUoVkdmp9;8~w*Q|*3kAsj_#$E)Q4+5W9|dl`y{8;_a5 z%iVsXt!+;(_~rR*VfC@>S~a4mbzy^ zc4fnDPdEo#2s^}2`!=@#g=8Eb9JD{mE|eAfdQXE_*f9sAN$-6K!#tA@J1i$DB^qk1I+Y*pojbi=_y{_B4c)oHt&D>O!o*U}!Rjur zzkmFZZ&sEFrn~m?Undqa>z0F?u|aCpg{+=(os;Ah5k3^zhUM83-J(fzmp}1*O688r zxFxFso0R8GZBCUYxqM3R9E)>l=>!IOyqzqpkOqOBxF03W8V4>$2+bP@2wOm{1BB-F z15Wiw7;jON!Y@HoPG}M->|KBU^|?{eSbpO*c&FnzFq84ll9LvsaeoOq)bo1N-Ed$P z3~a-${9{`A>#_j7vPZlYM3)o8+KaEBqL~1Gvrgavn#`E7G6taGUw%*(4CgoQ6EL3r zx0g;ZlkN>!gmdjtf1Ag{&ic31-DdWi?jI==H9rdPeBI?>X0^nop7eP7c(09gBPQGu z*{th0dL~*9v8#^5!=|`ZI8qr_V4oz1ZcLDH% z6Sq#|Cy1V21XVZ2{f@rpfU_t4zhv=Ol(~{3yL!mn{{7G`^fI?ETmKgJ5)5lF-adI{ zF(DTcbeb{pt&gy(MAu(*a#tqDMqLHJ#JHb$;PFy`4IX zrN3O$E^LFPf2&ro-@uAEZK2oTsWR(=$!wltKn0w2EcM-H%%|kv!Mr-~nyHk(zPN_B zjLcrd{zb{z_;U!wMTWrL6?{cr2`~`M3pbg!#Fx_&*EVSo2x)3JZSbvk~AG z;1w79|E+-DX2L6gx&GhJANB=(RUIjU!D~-FDK-KOCUk^QELTP0^5uZbWkCJ! z4M4zUm!;O8QD{LBtOd#Ks`MhhzZ#*61@Ng!{Dx!7w^JJs*PvLEV{ zD@58_$(BAku((v>%UTV&*QEik*dfYSS#X{eq-@pCMDn@3UbEO2ED9E4>yNBCsUfI= z)zUj@97*S7lkmBftgnD%&V6>p%5Nx+Yqe2&%**!itL)q?7#qf^s@e+H_r#7L?LObU6-G4v%<2^HTpvHmD$y}^ zzA?D>z;a1wvgjl!hvle&Ju^eISMymcZ zkQ@Tb2=jCkOXwW{k*qJd<3P93O(Lq?u?}`bOcKf5XkGgHmjS+8T4Iqy*Y9$Fxhzv? zY}<>)>bnaZY^$wBr@HZ|_f{+;mhEC@M&%o2@~fui+jBWI*p?-DzC<9hb4Ew&loUZLiSZ54{P&q3p2xG>9TnZ98{{ z`Ch|+-2!$dVz({jG$mfsi}#lO!OYvmsP#Q;>p2{sZe}t3r(Y znJe-$Ag=GkfXvYX=w)kz$M6{}#!w#kw)gz|k!R^p|N8N~XzJ26@Qvy2@icL2lm3CQ zM(54OKwa-Xz^1LH5}dYAu&@RFDq0@-%TQf6KG?CX$U!w_|DbY2f!(&Mw0xhi@|B1E z(ErwO%)XM>-1wr_2**{})Z4m#Kew_9f!!7aM!FQM4Cp$11P?o8`Jh7ubKt4a_=M5v K>E$%!(fR#2gd+oLN?k6vSxFCV}O9WB`@^=)mVXsi5Rqc|25kp+kw<85Kr=piMW`$&>Xccz$~wT+I8X`&o1>J0;Jcq67bi_em)-;5FuY` zbrM*N$Owou92;qma=F$(k==dZ?1vWQYCUcv)41hnSP|d*J*n4B*cYp=iVb45Xjf70 z^=i!owzB|_)Z@|=jDi8(9F|NUms{GV&m!EV+uGj?+jYC-na!2sjf-{+51OGiuH_>7 z-&A1Hi_mXAE&T(<$~?EY?cZKhVv2(!=E@WmiXe7 zoxRZrjXy}YoG7|bUU4Us5j){()850Uw~Fy`n9kZy|7?4ZOHLx%ozOQxSsG-#$(7nA zq1{vos2-6W+BBQR&Re``AM7~4@0MW{O3L93fK++ll8{|Qe*-YmI>?OKoN1K!ak8WTfDe<%w+rcPFQq(MlP6FQAeI>}f2l%I6du zP6m@IhcLaP?!T}#kg1DAWz>6Zu2zFV9zZC0uA;NcEz|$7CW9^DHsfNs8tN+2@5Boe?ZnqN z5>(;Y_f?`?#S1CHnr%W_UE*{1YzjO8_D*$Fom4r;?nQsk1-!~!zOH~U-{g(_$F{_0 z|7`pIzij(;+-XyaKpt@)XU>8vond^S@e}hghC-G`HfMt2gOO7p>MV+p`R%FVfvzoK z`0FxSZ`Y6QqN_(Ooh>vUZyol=eo*~t_*~VS+c~Z!>g`t6@%o$LM{g1AJ9s1eJ>9Ap z`IG9bFloqhNbvo`#o_*44G0(~6nDDt$G{tLo~Nkr@nmEc0X)^?>RLIe+np8cyjiN) z=GWo~%4WzPUa$PD|Iyd2^>X=8tG)2E=E06l6N2+W6%J}2YA!NHHQMQy_v7pw%YFFf zB|DpET5qbBuf~13pN3Xj6LM6R0gSe}tE1T2oDC4|He}M?aoil00=Z9^<2v3%=pQ-8 zuqni_1)#ZztAq1s+96UR2s-Kph)g*ZJ?vWCR@jklT)8k%W9$cp63IYwFfp1t zIx~*PlFY^>5>dVkmfQ)J#N?a4tU{t#zm&`k`50`LBmz=^%q?uML|D}W(?J2#0s6UA z&#mJxR=z$Pvt*+>$ekLDTx$b+9gqPEmF6iqMHlCc2rKPCM^jJ2<#3(=8Gp2YK6$ zqO{@4HInw$U+8clSduk(ak}*pA^GhQR~6gtv+aKz3i5#n6&oUT@sA07|ILIIr%hfG zdH54)z>$`U^LVNbW)xGcW9F#Wr)7LMXJ$cIqjpa8 z_25R*>0vnaV1;)=E9ChA$eDPv+QFtQKfl1p{o~0Y^zqK$uQ4lX*gaz_v?Ub4ralV+g)!YL0k4z^^k85Ro|W?&S1XqWt2 zvHo;f1)8A*W@!Xw@1)-_3XWJZImV{sAaJcwjj0G~lmRxV(9lMTKsKqXAoSTZ)M?zn z{1(c4rTPUbf@*;+=I~~g*|hULWf`9)a)QPx<)!claG1A#m}QM}eTJ|j+M%Niogoz{ zwQns2D^k2T{?#GLEa`MEVX|~&z^oe|QyUu$90jJQ?-;1~BnJ^TN$|rit8j0LipWu} z4a-p;3eS>J|FkucycdfZmB$p&H#n0vkVDTCCBjIrif+VTFI0})A1CKC32hFnvk=IJ zWZ$tgm8zY)s(n)K6Ib7qi&C9h5wl=G+@*B%XkY(&*q_DZG4e~?ffRY)JTJX|;+@9s z>7g{?Wc!b7WM82uV;&AOIQv4D=O9N{C{!S!arNLX`+mve_T?d9i{wJ5koi<=OWbc0 zvA2iP=C++S-lQcjzbc&3`v+})KXPdDQFTAx{s=G&dAYmwl1TTK=m5nk+H- z%s<}9>&%ygW|D49dOWzd56zdZuYQc zzHW7XShNO(5XLa3pmo@Q_Qp1%*n)5wm zd`o3nwbv-!3vVOgK*EDJYry%L{Dq zicYK<;=3>zWoxJv{)EXY1p|rE765826?I`&erUYVD9d&cd zF-wvqSpR!=DU6yjMfBkR7B?6nOrenkL3!~1fuBR8c|ga|Jd0N>G$!K|-C4#d&4h7- z<3TVQYAcP$a2;xlaV;GYUUY#JP1i^7h916#aZkPz;3)ymn+dF@z5iWIOr2qY*c&@I z#AfxHMRe?8s4MzI)dtx#(}hMC(T%0ZjAiiN$kITI-OJk-@rOdbf$**l7Fz!JsoO}t zJ_#A3IlR3apUKG%`FQ_;Q77y0=09{&NBq+s!T-{pG!t+iFF_>yf%9UoG3LjToHVv^ z`f=m?uAhyrg1H@N*%^kkkBF5WTnXobNS-Va0g+sdIp^H0O5!x0l|6CqZ`Ep6`^&4< zEbiYt?YihlZQcp5GIXxQobGRd&@ge=g(7H+>as5)(yu78!7D1xr%&q8YRz@#{@Lh_ipff^w=U; z3Th-F8!9&;WoR;M7O175qKHk(78l!ns91_mYX@f&Gf#W*ZdnW^y7FbBR9f_{P64{R)YG3!|a)Fkt7Q8AcB zbn5%$V5}O5#>q}Bs<`JTYvn=C>lwH zmG80{fzPjU#Anq0b4S#7|DP-@{eLrs=QwkFiZJGDP9z(p$5;Kr0t2HMY&t89G0gNU z;D6yk6vCxo**{xUrSY)}VCI~W-6E2`pI=zzv+sPPZi2rLGmGNLO8OU6l+80cDyF`j z_lRaZ*>)^tJgNGt+21#=e%3#g1i7@DfmXJEKLJHeCC1mEpImY=bj!rw_XJ;$lKfK% zgtutfX6YYad!cL!fgchJIaB?R_Q1-{q^nJz6NFY-p(FmzNpJ(z^wGQEm#oAJ@oZ=Q z{G{=1TLW)z(QCW(s7CSPQLauJPXQB)q&ueOYtSMscT(_CRnN9tr#Ps4Ezj;9C_m{t z1$#^eBPo(`!j{2IKrtpMrLTmv2+&>&i8PrxY@}fa`J$Q~dq#kykooGTXU7fZ{xb17 zSA+$@xqw!cx2uB`KM?9~j9cye2k$W!<`qbO(>Q!8H;P>({97{`INIQ$Hd0^fKua<>?U z09#Td#wt_Z3q6K^i=p&!bd28=DPwKr^0XBgu45$Ru9YHE$~d(JM z3kwO-x}sbO-S+H;cvD6lQqc6^ZGP#Q+$dFt(Sgpo|&jF7lbC_l)9%jm11(=n4-EP$Ln6wA?8!c>9`W*f>Q|TyPk4B=SMI0pK~cP zYKvE<@|3)|cr)|kXiMd~yFb*0+6V7GK5M6D|fUi{#x~?&3rhZ~x z_F?@sM=ih4(t4HQw=BB+HBaJgNlKB~#^|DtN$WP>^a*v7xp>XDriJleQ(C*x;<-3M z0ky@A9Ej&v7O9~BRZxm26k!lJoR_3)*PzP=qVxF4-H5;n$cgFQS$%Ak#OL1;3XPz^ ze|~_ni64cd?%3I9swG3pbg*%HTeYhdaB7lQ{}w3p6TCT!VA94lAU|{>1$&45<4k>w;)e2T(Fm*c36@WER_t7L zscW=~yb0q;!UkwFZoiKEr+y)?A`y`DXo!x zDC^APd-eMXzv+7vU}&af^dG*v5O_nw`a>;hFT%O&U6PjOdzOq{T@{1k2$kI&(1|!f z;11X5X+NgRP*oyosB{ebJk*NBw=rhy1ld*Pfn=_EUC=0J^KE+2!2{LzpNiI$Rxv($ zdgd#TOCiXBTksYwzU}j#@R#TjMB`jN@pfl+xx>!+8bWZ|n-+ zOqf`;w~F&vOj5ErrU!hxn0zF@+H6?W>aQl~xA34*iyugk;s=Wv!DNkUw9a1!CgKn} zXEjO^%E&9FdxY&I>^CGXItFlU`3Ae57#(smR_I<5^(d`qm+(wPV|Mkyi98}FZ8vyVB+d;QYOg!h6$U@*2g{_g$v5r#+H^)aQ zODlTYFO->sn;4p0VO3$dDmfMF@Kaxn*0)Cb4kc1~88YR+Aa&E*heh+;#5Uz}CF

    BMag)~ zdsenm^vQ-WC4JsZJxVmK$T!5)*n76Qr5ZH$zl!SsZ1Nu|U#mJaSeF=bCA9pdYedqE z0LM3^mr~-=e8+^D^oq{a7nJL2t^NcX_d20}hHdd*&7JX0*pEb^#&dYEqVgU2X<{am zBu0RW&q{qlO~FFb44fAP;A$;$hDq+fgP2)G%qvzxd(vd7Ozftt0Na)$opQ?@X1YE^ zqH#@mwHtsh%9s&rHEIl9{;$SI4T@A)`&bHd$*U|ni-J^XESZAQhJJtw5&@Z4xNzYG z{~Qy+FtCo(!hg6e9g#us>bM9USVX0KOc2#Hd9)(9Z?_(;CW6ZAp8+BwyV}0J*aCiFBuZh`wSrPqO zv3zsLLsGiz;+F_;plk7kG13`lf*l z#s`1t$&!eD!Ou8p>hJHdI_tc53J&%+wg@E&`{26X!KS%b^*RrWos7Us5*Z`I40EZ4 zw(oWHr$as%rVtepNta!HC#AD-xcA>)z9{EWFIU^cSr)QpZ8H}c6uw{YdcA+<(BST* zdb1`OR9)|I47%zajqa|xjUWR}jpXM+k)JB|0zX~_@MgkwXV=ON?fdncwzPNCSbwx7 z;I-uA3E}%Nz}00j=3O~0gjx?P4R~95vO7Cfg7=~^?Rn*9!ZViipKZzWkBQ`@Ur8?E z_Fl`%X-8n1#mLmh1@Ox0`SQt*Xfup`hh*b7JmziJ$P+EoB~@wX*N6WFTCJ1he^Zie(3 z2i-=5lPxII%$#(-u59a&w6K3-P9)6Fgp@)L4=x68JHnH3PM(84W-Naw_!5d-fOY*3atz68lh8L zIg@+lM3f>Ft4Ar2;3iZWhIZAf!eiG9l|-vjt571e;5aU=+~81lXKporU||Q+ojnlp z;~yl*2RUdY?c|5?L*v39g58_%EkDLfaxPA#?+YE564yQSKX-Mhb~R1{`G(myTS#~C z{0%R$;po4P8<*qEWN;j~2WMT4l&9G#th>ynHjLs(WcIRtPc;|NBbUzaUv1?)L!!aLcXsU}bpjbBAsnrD zrlow~ph2-UR<$xuxG2O0I*bBGc{sn`8BZvH(c8`_Q4__upw9b;O}FE}S*49m%ja1H zAI{%j^jd%=A>`m`i{q8vF+Z+35QW%~iw>s2GulUuN`Jan0 z>L;)6to*v^Ps){2^fvZ@5~u*@9Jfb_r>@=L#7YIiY3B*1B7&2;5|$_0l~%zb>Z~!z zg}%C>hCQCt6hS>WQC^`q1pICn~vzeC>7Q@Tiu}{?<^$&hh>W386$K;KU0}^tq^1_!|0Es%c&TXsG3~ z*OpaPNzf+d`3fuEVAeh>RqoZ~+bzB*=A^a?_KyMKkr=LWQt!ht8DkuIeQ-d>giYnK z9_Xe`1bzTOiQ6>y7xX`+{-|RFylgb?y5wcWabM$X%ZnpE&ybTQMyB!?ch2c5MRaZ> z{Wd4)>jT>I$f1`puHf*f`L)bd+eeUb2=qVGuSeaofpFb^u%AOm~Bs4DF#3X=zOeuo7fO;^P7lbMcMyOR~Rhpuyp z+Hi<=m1qc2@w~R*BxL^NhC)$4$^4`Et25Rx9KAl06=e%shz?GAE>Lkmutx99&+#f1+!%17FRH~~}-Rh%JgCBscU zqn8luUlL>+-7yydl7x1m)!y{8w~{! zGMT5^`f8p;zO)6Z9RPTl=!hW_Sg-6kgP>5CX*>Dd9gg3{6(hba!9ogly{sdBEe^l2 zn8LEO5}u@3h6H~6+y{-dxCDiBWrk(6e4h{&Qod&iwT^SO4Tvvz16D#G)&+R6Th2lY zHoXW^jHPUxdOHjVkbPdy&WB14tCX=txsSkbM+biPwaV)!WQ%f3v@T!#Q`Cp?=*^$Y_d)BgthK&gr8Ys~5G7o; zJd;9?oDB8%xIayj2Jz;_#568-!gXUOCyco>m%L6;RJ79;=YOg)cpx20PciWm%^-Wt zeAn4){uCn|Gods;g>v1~wm%KA2yzw% z`uWRE!{}HvcR&V$ts#Dw2gfU{l0PIuTAxKNltE{;5}gwH7CHz7yjJLX5mGe`oa&p- zFvuQv#i!v}0KxECSs~zik9N_(Gmp>#@AkVJ0s%AAoJ9-vB+}?;u?Ov7?|}9$DY7hy zer-I@s|FouHHlK!FP+)hfB4dd4ay?OB6793)A4P;gVGSmhqBBlI)z~J>x(1RgUj3L zrlj$~9USBYhjP}hOEfb`8=>Dcov*afRp+Kycv57zHEHXnrVdrq^75yrYm*mx;6bv9 z_n69yPQf1~i6&_|eV*UiAG3LLnj{B^S>!EVS?TU>l2A04x`Zc+$GMsb&M0Ce?BkoO z<<)2`f^gbw<|;?MN*JXmwaN#SXaRP{_(J;i90Mjkr{;3nKp=P!sU$*c}yfMssN+dhGu@|K=67Ho^H3^qxouNI#uk4e& z8T^IO{2^j?56q=djbsFYkl55zECrP}8s+nFC0S)E3hjwJ#4%TabfCm*ncL z$t@8$i2ZlnB>AFp=$}lIHs0Pn+%}%2^7Ej(QifYeh`q0btmvCG~WzRC)U7?~Af{HYXPQKW4?zDL3w5gt_Rl78r=~C8d zjXaR#gC+~&a8jIQtrzF5Qhsd2vi@1GF5fK$gO1c0Uy!ATY5A-^UEAhvb?y3hHf{+Seq@2=-hKkoQiyxYxr$h#3u+T zM-i1?&#&KGY52OoK~%w#Y2hc-5C>>OT@Ajj?-tf*b8$hr38ojpE^43`>Kj@@&9)YB zo|4e$4#GeVZT(8QKtr z-3idJT4o5Z54)(cG&1M&x)u#o&;p;B?wL0LKOEJCCi&f$&6-%EOSN*^sus59xX4*W zenG{c4DL~iCIVUZ%pxG#vKWE;++fjk&Bd6s;9006063UMX+P-&_1++P?t@YSs7BC) zi5+5__Hn2LviY({JO_SfTRoZHTe~OX*4xue>bp-pjzjMNE2B6ASSPhNIQzbFo|1h~ zc|hmr{lmLn%>nj@=7hVie1W5?gZ>tup7`&>H-g%ip)yd}4N%Y%47v@79=y|y6O;!q zZv-9LPhvwEHy&M+1UVfO!4ZIZd(ug)l$g$i>QWK*Io=hhAfRlNN;~Py*p!thji$>< z`7r;s)tp>9{=VA`e&;iNkx&%7TNi*DM4Whc^O)dT z#}h=%xze&bLw*O;{R-`R8VRfwgYpU#S{8_2A^nXdecGy1!2mPLhdIIp}LQEtAr{~28@5WyP-_3}T-f7-lA23Wd{hhq_ zT`>Iy@_+Z)@Gf&4z%!7*)$_)*ke&E_%%Z-Q(67*7oB7J5otnNING z4+*!s)rk7i{P1AibD1*ZI!*n_0q{uuP>G}T%|KPp!u_jY&U2__eJQefykCQCH5RDQ zk=q<12b6JKj)e%1LgN6(4X?`P4f=Sixz-Ajgu1b9L}T6yO+zaI)Pv*C%TOmk?f~bW zy}Gcv!8uu(p>A$zwqI3uvjS*X*rV9*!y6-u@a||o#mkxs<@q#IoI^{OYRUx3`j2R(w#${u=Z#A5`n?y8&fcq8?jKIOkoL#2Gh6yc=i^jAmXH zQ4nh`+`%!aifFRd;`gObK;cikl)7%OY>4VvC#Bv-gu7$r-!S<8Y{-ly<^}V`Zp8{? zHSuA1QqE`RMBjra6vZvb1X!@Vvmt+VOrs*(a`=Kr<=c=y6y^1pP!eASZ=m6P9| zR|J>S%C5T4Vo*Q0bgCuB<178}rbV%P*Ul4^l=4&dYbFl+^mY zU{7NV4h5fkK9@a{KcR~+8n5GW{mc>u^98U>4T%Vo?-o9f{Vw0FSR=nmtbcQzn5AkS zEeaO;s{jiu&s)rHptznY@r_7}pAAH?!zc~o9JF@R*M47v@+F#*l|TLn#fvbck1-VJ577^gg{BIqVXjLg)=@ z1&Alvr*NiSV@cOh1)yG?NK$FetTMYQJSC00;Y|(t3tM#M^IYWXf!b3$b!ZS@#Y|Gl zBfT4h7NkLDE>+Rj4I&Cw0HMC9C0r^SV&}MqCFJXO@c$=x+Tb63$<@kN==ipl+iCgM zr~iZ^sh<~%xSsWLD>oo${LZOM^1oKn8HMktIu&r3aQ zjh;BfCyXcHj(6~88FfO&jB;KOVD^3!YGUpqJSvoX!$|J?i3 zA&Lb2P_C&>;o$3=fiNl6E zeqcE>LR|~4|HcN;GMG^goOP+k8M1abN;qGlCgzjw>QT6NIB%s2eo=T{8b;7Y2fySV zJebMCU%Jnpx@%AIf43<1Gym0V$>9o}!Oxgy(&GE=5r@h!_JE5echkCRfn2j_OhYvR zH>9BpeV)x2#<)}&M`7j_pTCeEune*DPe#qiZOUN`?ZeK%P%U7BUu;>1*!#zzYLFP_ zy_vAfh*l{8OkKoVu33qEL^en zg_>=rxU1o$%Z$R47wAoQ3(o*v(&Jb-YnKPG<+p+8%KC0(%KH>s&e8gB@y?1#{nk$A z42A^iea&P3V~w;tlb+g2PFyrDj}MUi-kyAc`BbSaW= z7@xZweT_d4dM4Tr8q?FHW@8T(ZxFxU$eJ$TG!3ZMOk((nGS*>e6h0d2Me*fh)HsB$ zd6$$Kghr`&i$*m~!1_46HKo36!WdXx#t`fzHQlj;AHiQ*TM`j2iMiHV89v`3z!OR5 zX{>P2SZR@$P_!U6K|usJcUKD&2L$gmgG3d(5EiuI?RPAn4ZJ}>b9*qHCJo#fpDm)F z!rloaXZT+fALq00O@g2UpL#moG=WPa_p zCh}X*{5Z0bGU*nF9Z+H}y{7@a0&`y#t%ta+tV=s;vp{Dh#^mJ%B3I-6|LoAG`8?oB z9?Ndt17Zb4-+1Mbivr@)8ZVu$+gB8uzf`)*8CL5YFf5u+5cADY78oYBx+9Gz1t5o@ zV^qyBdb8$eiSC*jM#8p#ls3TiJI1_Wd%9Syo-5jnB!+pPS`(5HD~loU%%RXZV|7wu zmytV^Q!3?B{&MjvMmza-AEcwa&bojA=roZc4ohd>l}=~>JtkVkrww0;GQ++`sj?jV zuqkJj8Yu|-mB#MqvAwRNk5NFAAXljW-ZIkl6}5#bH}79$v9A9j%k^K#ChOU)3!_PH zKhQt27C;+lt||25`A7Da*vMr!PS-bjLf^=a+=hGxln98C*w*fxTXZ{F0tuZw*K?et zus=!KeXk@v>;u?a<-CY(OHnQtUT7!P=a-66S&+TyZsVM*VPSx~ zBgaRnhIgDlQ1;w_b|2Xn)}n}&Wb)SBU%X~a9u@Z(i? zMV+Jbg9&X%7c~$P4Nv+~pN-|aU%cTLEH0w2C#{bd$*zKCrJNhzkAsn?6#tAhM(#R& zW3z-dg_JTf7(IZ+EUL}|3NOn?|6M9s)2m}B%EK{M;xC15R)V0-ecY zIp-8O&NRFsBKg7ua3AZCV{t5U(pvkPf7hA87K^Jw^O(g5Yqy>)!bsiHy%CkCH=JqM zm5{Ed5|Rr{eLJsGu09$~Aki&vF|i<^%t-UeKeYODhFw)@hEEf{{U01lHn3Y~!3w{6r*~gvixM)Ip@SB~SdOoA zq;wp1B-WEbet?Z~O8^P!UCAM^1FnzHfP8$at?Ngx-8tgV>&PB9G%mKUL#yM?9!=O% zevYqXisJK2n4_P|2Ap4ZE}IBInmO1eIv?PVr|XO1Gn3;AkYH?XbHCNy{fFj&F2`eN zfm6$DRA}e)__{^?X15TsX%|W;9O~Uz+UVv$|4;nbM;Dvd1A?=)4+fGguM#J^n!RTm z{&gh^g!&0)-2PuA1b+CHYwZ<6>&C>BM$}7#Zx%rpS)VTt2=lR}FGa57*WUc7afKXH zzzd*_a$-I+2twvQf$KPltg28(Tf$Met+Z`+9`UdwAM#F^O)?LXyq%i7T|bCJ(l)+N zdJch-_X>Xiu91sOH_3VkjsvO${V_h|7>ra%1D{LxDCji*lwka;g-pA2L=M4Dp^#>8 zHYxP{Co`G3R#Gi2JTA;x<3D}6NydyWa|la!&`PyjAV63bB_@KOG#@vK`OsX(}aO2ws<}uoJ zF}PA(b>|zYH)%O{?rtPqo%HwfjoyB->sfQKM|}|N0Qnf$&!tQ@*=+5xE~VLXq)*Q6 zx$bI|IsmyXu-EIihWp$X0xVbE7XvIqw4FX*1A!*1>(wuM&F>dr1Fu(W^E;hF2?}Oi z>aRfeypc_suLw)n@g_MVu1AmrVqP=Qx}t8mreyWr=sFWbsyG5k3`?T)OCr=q{ICi0 z(IYlPp!mIv6@qC#dj;bhIM|}nKgNxDM34a31Xo+3gV6Kuo8_Szr|`uK6sme(P0Fir zcZ|!c2m7L(YZoSV|47LG+Dy?EKVZyhR^N(dVUrD45g8_u+#D3g^Ffyk*X94w4im|O zK;aSgmG=k2qZ=OfeV>VX{KRYuq*p}VeBd)o0!T46apws>eGX zyq8NCmTVGc*vC6xQXYZa1rc%EB4TNWNT(O@F+Q>Z4C(-1gV zsRJk!kx;QeNME{8@0lka4*k0Y3ZAq#9~xw6_!xRFXH#Bl=(A@ew$BPlIfxHz*szO$ z8l2Qd_|_0Q*6_U6NIuqyLj{M1LU^x{AG|Y_cI%#doH6elt*`igWqU`=Vs5rC8Yfqz z4vtnyBFR{33QtgUAol+6%eKGsS_o%SR6*txTb2-(Q$tSVV~_8MnKhs z8}PcAZNkize@(@1MLj$_9o5O=!Tjd5eoFbUtD7}8R}`SDuq)cK=4}F~(0!7>{FPOI z_hxV8nP(yPdpgR%bE^wRLUb2Ep4tqZP`!^4%lecnMLu6n^- z&W+LuABi4&{GwJhPX6J_Kb7plby285}A0F(BQ|`n3nh9>_#zIFQu~ik<{gauy1Gi<{5iS$X49&lECjy&|f%tA&ib7#bsz)@DBOezOkOmTo0ADB|rkc)Aq0fFV7cmcz?_ zcH`b=_-p;p`X{rPmc!hveO5EcTme6_2HTa*a%0lp=Tjq=*aI~&zmB?t^&f@C0@&t- zDvjdel(zG7^FSTV;|*uqHeN5Z5oayS5x{glhc@s-gVIm58m+A9e%6cSi)Qva@Id+r zxbD!INN{q|VY92$Da}nNf)81tgI&vMeA3&}?esOSQR!mVJZaEWpNXcUqH?}~xsoIK zf=S_U6|9{O?gwkjtLlAv1z7r=2k5;sMvPy8H^Gk#AUCjS@qx5RBSAUb%)9gli7y_V zsKiiCi}F+^zPd1sc_f)@Ra+>~E(+2eR>B%)^E$hFlFXokDmUyJk>aiDHaTy1=ZGlO z=v&pB`0X89ph#$5SI~8Xx#bOgS@qzwib1py+KyEdb)e?#0WLW>azW^;(@n$u$e2>b zD5~o%h*F8B2Ol7W9vF+j_ZAE%B&9|4pFN6}P7_K499j?hYBk2jA00zROT;oX$&c~`VttE64lN1GS z`ThVdLrSWICJb{KV@a$fh@-A0RN=EY)StJLQM$tW4zGAXM9>yWX`-{#kHJtXQT5#?{+BYuD>#L)AcbGOA&~Mb zV3XRNz>qtz0Gd6r^1IG)F#0J(Fv9M8dq`!JQ5%w_<1Ut)w`$AAb)BJhT2dP&pagK` z_ZXZqDcpIS(FfpJ6elGwOEFbMw6h~$0GOiBYMr=~>wCj(37F_Eji3a|N@RRq(54Zy zMmA*s79vx$s5DpgxMv2kV$(5|xz0amTxV~q$1tAL@}AGeZbq2}ho;{)o2(6fcF}R} zhW-}D+HWJoeMKxc=b;x>!DKa#m$2K9*Rb1%KzOVPysxM(?GazUEY|HPq7~Rr&BaiX zl8Ts{uPowlb(aN#*G`U`?eES7iOv=`zxvMmcCF!`zSWkx1$j-UsB0c!b1$|2xh}edOtca%wMp;vQZ$a$XRS9g6c6>|HIvrwI0X98!ZKYx zUQ2>1EbPxX?NWZ2qO_y4dXfno&G?>GVNUql%u^*kGH8z*w~tnVyi;8n zk9w6>kd{|>kKq_CMC|5R*8~Cima_)OowZ9D$h?$H0bcmJOFM39qSkM(VvweVpDaTLY=+pdw?qvI6}>O7cd5c2&;UD*dc}MqapLLV^L!URGETN>PVr!x6f49l z|46Yhv{`?=2Oh;!#z~5K?GGsFsS~3xlTIJ(?T`=0u&Ni%QL8Izu8qlljVmoTA5co` zse#6UP)b{>f#!$kNyDgxW&lO9s>-NG4X~A|&IudY(3eO{k`v3$^pAWKC95k-s4`|L zXG6$G;H2imni=I&$N^4N$QaouM^&&9q)wR*%GPNUIP{5_LzThd;+xScu;X=FqnEV= zdlZ|9sFdcn2H`dZFCuM}janR+?3krb#v{YyY3wV)nXcUAqLI-N!GmOW6fPj!a2#?a zH22AIbPAZnGxd6-(b=Trn)~f%LKpxts;!FFBlfE$%6&X?!l75hO}Z*NJJzO1O_CU` z$SKHCy*U-xEP!&i6iPW+2sV*!VX6>}P~jxA2%BU`GKS+XE}{n(#WG~E%Z6Irx3Mnt zSB&KUtVtm+sf%Dt_kv^@Z7^1F%zFpw5lH5KOOmtjGOAO-?>v@?{1aVjf?mFFjAud%#nO$J$!%m#^88hj4xi@MvELURnsMJyB zx%HDHR=HSUE8v0(OA4eE0J#;$i(tJ zzx-D>sw)lqHoEOEFTXw%kW!@gp^2dCni7JO4E&qr&|o0mjM<@=QPCyPDcs2s`o5m_ zp{WQAAdyQ&(vqUtzJ<%husW!rUt%9JT^{`D_hwxqYWD|06=eo@=%tivd1(w4_7orM zj(KR7BnJdVB#!mPzkjfcqK2`U?g=Hos&f=p6`lrPl&uiz#bEW5?+FK5$a|K{sDwAH zkk>`uweSa1HMDf z|3zhk!9bJ7+W<{YmE|ePWl~!Ab4~p@SoFYSD{@}4WvNXuXlg%u^4JgqpBHcAJU$K7 z08K*SBYj`6=X5I@Yx2`12m>Y{ia|NYEr^bYQ66kv&<+84ffR;~W#7V&7P;xbJ` z3z;t07jztO$Fja1iusc?t0R!5l*jx@peEIhbEjglu$SfAZny<})dszNHyeYg7vke3 zCw0S_{?@2WcHr(ER~`e}miXF1AaojCr$`#RHMy+iF;u^xbikE2&S9AfVvT!(pFi1X znXK*EwPPmpg(4A@i$T`K81pGzvp$K0yzV*Q&riB~`l)OK!FG>FZ@!N= zcSeg>N`DOjNEZlt;aD~6*3Xt-en?6J70F<#!LNrxk$*&j@<7z{U2WC4Qn(8~*|e{w z-MO_9Q>hKU_v00TZXMm3%ztW0OY=l7;+N*i$|WTb^3NBMnOA0q=O6BEePrZRVG!^! zi|7`!wG&7Z=yM3^>8ZD3yRwHzO$BF`)fJ+DZU-O$EQS&h z7P1%+94_(0c&WhMf+6)2WE_%CbcjtPiXUV{eJjX~QlN=K>M{K5tkB+~0=h6iM+lB% zt|?08@NXhF-#glQs5Oe!0b*2C?jhY}&J4-PH zmYr4EhD>zbvIS^A6Qc%cqF*ic%r}W(`;LKV?ko>BV=LA1A; z+_(MuNc#k9kOd_ple`TNk4=_D`sZF84-0WBqv9c*zwO}Jn(g#iX1lBn8XPjS-QHY1 z7Zw6`s*iPJ1D5TF5XgK!AOelKXYt+{^baeo?Am_S;X6ho#Go6!dhC4tNjSvHgZ4Sg zRR10iGbc~V%0$&-aHPVMU_KV7j{r(1fC9IcWM7_i7FG^Ag#2ad&FzO)9k zu9C3x=^CKvqIrrpsSC!>#!iugmS1JR6pJo1pMeib7mkjn$Nw7L3npZYRoPMU3ds!2 zZcw3;{k_T;xj9A2nt0Jk=2)CGEe|S5=IWnm^u-THXUroidE_U$3|bHDvu|dDOF|Jk z{IZ*4IK?E)hgzHgD+^#g7|CBJEBR(|yM$WE_p>5qFCl3~SGc(2_#3Ps+eNCR$)9Lu zGFFBDxL`-{xM}Q`$6y}TRYFl7o7B}7(YtRwx|9I5#hid_7Td|S&^NYf1^gtsy>lE3 z{b)AdXtwVs`9Jc7DD$i2!A>$6g)r;Mqb&-AV|ZP#7P9Q;av3vX_&q`vh5|zG*Si67 z*+=7*t3}s?=huS)$VO=Os!~;TGXp6dRfari z_nz|Qncuqo6AB|(OhCW0^Lm8dA1);EZoEU`4SpAo&%k0zW>T0m+G0|pnH*0m&(*=u z-Lz24_n^1HVYa}V9gu{cgl%WBR79}cKxEBLlPZFMw(xh~vsBwb>|47(z!eJRcxF|R z4rA71W)xghFFcQv+$D)H>!G9EXI`OoulBNG_>YG_whw;byt*AeFOKgS88+|B4EbY? zofmg=`6F9Y(W_b7>HX%2WOF~>R0u#V2IJnk;(OF)2p9st3sRXm)|Do~tOwNp`3?P= zIQ(edr+cZC`$QuI%FFEz=dl_eymi@&S(gdktOej%4e{U7mb~mDKiK4C4dD2gd*qi? zcBZ}&cM5Gh`#obs`_rw~OyVxf1Z@h^IhzD`vH$5dE_yr~cN!KV>Jhj&PmRNM-cY1C z@BWG&Tb;d>j$>6N-doTEN=f~Q)gt(8fq=!2O*|CAp9+u!ZBnSxuzJ1%hMwf>nKRr! z)~6k~PTp7DJ#V+UORw15jA}?(MLhZ8>q%3|Ui0<(q+a9!3(di(TTy+(XbF(PaXW}f zesg9%s6J!pWybxCTkf}JJxYNaaXs7?qDtXalOM_88=(IA3{;O9> z%-cfK-QsNR%y%~XVA8%F4ds77q<~In0#PmwUN$yU3kw!Yb1n{cc2hH6PHuKn6H5+u zGd4>zHeNyC|1Tr$TPL&(1W(#nCo~ZVW^k*HJeYQ+77C$ITN#c2u{&zTPwRXIdwI+6 zsy$0SdqUBYE8~1%L4j>lakuya(2?cWJ>4&q=X2;q%5;?d+I=K&4G{mQkdu9^glI_gDMmrS%;laVx8;%zeQ_ITL_jBck z8&O;vV6qOTc|dc8VBJ{HKB8`ggx^@|JT!+DrNbELIS4I3;Fun|SqDpPhVVFGxRPa- z`^^UsP|N(&ROmboY5?BHNy{EYre%-*Kt}XKj0Jg~6)IwvWnzGVD0Y8Z;m|*pYWYOwThaTZjNO-krPl_&RuA@)6}Y1tf6^b zP4x@3zkaB2tKsxy&5|Na#7mxmMsiTz}W< z7U;fK$hDSgC3)c9{GsPuDLwYJ$4!~)Y-6Y`yI?D(!Tn7i`*hEM2KsCuH5$T<4QCO))y!5 zCrZl%Q%OlPMJIy}^<0zpzdo=6V`*An6@Eh|G+0%A0^YgB9UGhWu|^dVV6zx_6vsd-dAG`ALBa)MNC%(rK^vwpGrbt6XAdak+qkt3k7D=l2<+%LGx z+q$j^`x&h!2Lfr*xzU6=f!wX~x3U|@rgGKsodw-r9UDM-+JP}|zglzt){#R`x%>FG zmPiPwZiQ0PaZMV)778%FzMKBkkF;a*8;3}i|5kw zIi)m^*RU0m8Av>l^&OXQ&^4gm_N$?IGZmAJo2^pkiyb3sd<$01YJl9=_(lw7&sr*S zz^}n*xn@9eagO)s9%H%m90{Tefe#f@|JW|_8_4=yjNFr$o|{tuREHDb<$3%aKj$#C`0oevdM%wYV3Q2l&(VH;r%Y$jt}y5v^(a z2;GC8_gZ`N!+?k~${V#rZGTbMq!;QYQ0xVTy;7l;jnHY^aXXR7CIr7J%}dP=hrd|q zwT;y&!u}B}@`Kl`3YOHbqi0B?B%ZSE`LMOYe@?x^jcEmLR#J*7n)@@$*eQbAnU=(I zU{vrFlqi~FKk@8CeJxavPb4=HUyY6A2S4rsCSHjUJZP{x#)21r&vA3+h#k}i`6iYR z?|4u2`)3O)A$UNXGP9w2bTbr|(wwp~_m#9FMLY8fbd^WDz<5gL^sFDxHf#lTIHRWi z5%j&!JRc4zGgAdkj^7s-#(sY6Yu)f|_3HUoEMIeN4f=VLlFk%qX%F}64dq6YYbWH2 zESPA+)0<&`d`%#E_Xv+K!@my&C8e1Xr2g^sbuZyj*i7dmexNJ+^5DPl)T+b+mA1e*ngHD={tDu=tKH21{`4U5A{v+j+(yaMHx(H$J$oslePyJryfjuhx81pH+Q(uNLK-Vm)PRA@; zE{H8eZbkP%Ui9h0Zz-(SZ>PvEmRi14E2F$dN-c<`H(2#h==t{d0;jfYVSG3qC4Z|` zmKsq%#eK#w+OZpI6EpxFck~DvF!cQbES@YJ1)XSr_XSp7!$nqC_8cF`og$BI_b4~TN@hElXG2Q2>gSrzn%D-`z^m+np zO$@4HJe0lGPfY^s!)I6iF{>r#8kAhY%Q9x*k}^2ZGnTUt`&9=*ZXSGGLFPB6Stw0l zDywuOqW+I}ipvk@{$KCxGc4<IIdQ`Dxy%JmtHm$#{9Vw@5vOHUF{Hk0Vs^!tmVn+=%F?gC^Z!T@Zi>xLRuQGl(0z z<~!wY?Z!RRyAxX|o#B7}#y;orfngMXyq-E0y^PVS`@vOuW%2T**h%jtP#yow7_c;y zyMp)NP=e0Y+q?ql!Q75KWWwM%ge5;voeugTv-{3Dmp8a=fBT-*bF>#AfAc;}z70aX zdFL8?>Y4WMsl0f7=ix6xg6A>Od`b~MzWOzlEIhcSSt?6b2%Jt8HaDt#mC1jwt#k{1 zfUSI$%Qs41Gz<2qTZ96zFb<~Pc^`}&A0$ET$Y? zk}0%?B-YWgywGgaUp1|WqJ#Y*$R!P5$|o>Xd&%%(>eNpsT$v2p$I(y)$-D-MGZ-Vd&W_Ltmz`VtxF}kxKsYh(}LMJDoGkbD3~$ z^aQQ~$|Uc~Cs}7qrbFdYE@1}UYN@z;h{@Gl#Dmt4Ilqv zrD=IFkkn!IiJf5*ac5;`wmfbUp}rTFblp2loEO)hwHWt2A&q%L-1YUHvC-f;y$#P` zWojG&Cx-La5Bj;*ia-0w`?GtO!8IOr%kqA~#Y=Zql#5!NrkZAj^}q62xer9X1bJH> zO|vp6Tf%p4ePq!7;t8LrCHK=;w-iq9ZAK@`XhHYInZLc>*14F&_)wu|eTqtgFDC>O zRt8issrnG`D?g+mHzt658ogOOA`V%mIt8Xtj8U9Xx|g77RyBwh@&VrE&o`YQ%}_+6 zIa2RG{1ivawH-e8qXJWdA_JDEvj*Jm+%2txl>vhjEX$*+_U};Zms$1yNv1SA6b-=g zUv&H>Y6D?G4L*Ov;&Z2^OtO{z@=4ry;k#wKd_tX%+llPF%7~N`^5cEuZ3sgJssBV; z>OIG_XQH;`Zkucc_%U~BhC(URMo>Sb+(Cx$aJ5y2ZpwYEAM-7}>ubK|n^x?(o3hK@ z`H|zp^~2L0>rG<j{^~~9T|&(r&NE!6qSOmwZ*avs4)Hx5$F$pi9O^I1 z$Yvm0cx$fWoQ^5pb2v?QY)$Wj@FeQw2*g2^eku?FNl=jhJ9tz(l%PpOGIf=385OeH zB2fez1==!9l0X^gx-e63DT@6{(Td+fjA4FkwaNP;R)jL5%w`rgsQf0=-&XuLM_GCz zheRs|IT0u2NxpP8d=i~t5oH$L=j0Qn{v>9|fYCv7BH9-jp!|h7;M zBMj{}(!c-1J`3-F@}R>%K&tu+pF`^8V^Dbx2K@0>h-F5K=)cp?K>=pj3MWVg4DL3y zAVg;D=J6{=k>r!4^O4u_#qEvsIOdVUw1{24D1?cAHja^!4eHKTZ{b*jhp$1o27A>m zuJ5K%eJ!TJ8{EeqVJH3ryaVSy;9370_!0Ymbx88wH#oVn3?Zs^O$JTvt7P=-O?(5f z$oHRQSb>Cfe>ZS^i}f*S<6}Zo`B6|Vt?SoY%uz4X&&veAY%jJf&$PBbMD+VBMMs*_ z9TnlH^a689ewfuwX76fE6x*lAGtXzQXI|~c%hN+UU|`8_;Xa!y1WU4v{^^w^G-X3+Krj`tLI5GC z)vz*t2h>R2Ffh0vN5B zk#Pedh`4D*{jwWaR;LKYvVZTy-~0*%iD_r!e#g6e?43lJx2-$eDf%fEbw4!;1lc%O zq?e&2-_0kJdZ84zr55O@&FyMpz9;YJr7J8bOWxP>gql9J0ne`;uWyh;pimG;k*^(q zFJ6(J+UV{)Q%ly^`g^o6!9)dmx5ys zINy)4J0HAKo{Alr88mtXY|W?53fw5DmnE-OWOV)vBMs}&>RvL1^2~(uup5e|%k=fh zfbQ_ZJduah*LA`Ulq7C^k?|~{b;vg3_?4XE_A#e2{P?b+)LikrWTeQqW>Gk z+ic^6%O%m<6R81rHn1H>`wnq;P`$q{UWTG3`!DK=k3&{C(eKRG&Z$#L0PH@gePA ze`5DI6T^ZD)l`%Xlv+{bdGM|rqUkYHr&O&|b+IXH{ z3dLm{*^^-IYgY#StQ?h$im^Id-??w&0b}-fAf_Rig zGq_zMpo=ZPTr#=PgaApCn2D7n;kbzV1GF6CQH>H&%A301Y%C!hl8l$w2xum`*jPNW z6&eST{lm0RWS3teI725X!>=*h3OsRKWu3mZ6+8&I%5K@rt?ILE>^RQ#g$Yq>Y*e1Y z{TZtz7%;M?6Q59ZNO%A1+%0^)F(`~+W^ej`?d89fLlOY>DfyQFpZ?kZOaJjB)_wn5 z@82SHihgb4DCx6=%vP1=+RJ6SPGyqC5{WV(>KN>tmrKit!%^_$xAyH0GauFMDfqwB zQZMHPWL+w43o!m9BXs({ib#HdtEpYbO%#6dl1I8781?z>zUtBI+1Hmk;?n#`_P_3EI=tVTQ&SOULK(C=wPX8OTXbBt?t~WHNyji3D}Zezr5pw#;c1IjLdG9`Vf? z7B>`CpGEm4yfON=$&`xM7@9&jWm#gH4KWi~=zPFXCuni+|KYQx;6L*z|F3RhO5ML< z)hp2U{M<|Y!Ab%CAwQGEqvKjCW1@{JFV=+f&mXx7)rIwhkdKXa?yp~FGI5P4u4Ihf zOiXI7xU#huRff8rjEg6Zf|j{YDVP=3G^nQ@Kt#h}HhqD_*XpadQP;4Td-&SDwt6Du*S+GGj)vWj{@^b8pP^>g&W1~ zKB-GT`}OM)2gm99Y6nvjB;mC6d}AhAR|twwV?KGp_viA{$TR<7Q?veUHDT4r_29@A zgUh3D!GH`DU0r+x9F2t;ES3tC&RzJb3>Ydnew!RKm(q1muCqP#=FI;yHrWITnR5AP z(oU%-{T;96Sn{wC6di!@Gu}BKZO2r?TuJIaE=q#ull7jXqsICcsssa*Q-^_rT$`wpJf4zq&{O$@63!$OG;r}xB9-}1B)xytA4_yddTE@=^X;{`keL)q zjz71@tWOMqGDEl6J`X+$#4H|j*e>7j_C8m2pW%1ELnU=+a{Y%7=feMF#rmSayRa~DN-y(+b3pwx{79IFaT?Z!@g|q-F;Vlm?}wk@b6|Pm?9i^6eVL`r zk{YaP>;wpKFi@j=M+vSn#M!#HpV~t{iI@9`53`X{yk34O0m9rHy$Is$YlLl#ptICaQpH1< zpm)p`=0c#qF3qp0Gmc@{h8Z=)uOf53SQl|}8#lDHb+OfK(Jb70qyA>nSM~4$r5c{D z_kWVw>?8_hoP&fa@q^XIeask|%V45;V+(5KBlOjTVRGjVR0gos2PuA?+e46}t-3!z z+wgX2;yowsI^XEV`M4dVzme2y1a#41^vkg4IA0!j(LCFFr$*iB!R^ zW7oAzs`YC}?3~GMv~7(lP-thUg_Zq@=}Z+;8>PXF-x7FG!gS_}E$PDd^Y~aqAf-Id z6`d7jh9_t}f1R2O<2S=HUsxj9Ku;&Tw7t+jIJ>t%)6zizYenG`GHAN^z=xhqimq6B z^S2q$px<_vh!H5$J5AP-SW-ezq#TVx$HQ+2!om}pzA`Zkm!cCfWrXMv$G0&{CeqFm%U2)$X{VKN6U(-K6`d`v z*O@fdn+VQ0-I)jm$~P}l zt)zf@Q@&!*4u28PYB%dH^_?{rn4>70$?#>>Z^A*}RaKS7ptSd|`>GPt7u*bJYxQ+W znu_S~<;0x`fST+EJiASfvAAX1+&BYIGZJTM;PLWR;9MwIY{pJg;^vTlWJ#2#9Y3FU zMeQkQR?p7HEDfZ}QhVe@VY3fYl>92=Wam$^@6yE?FXZQM$Yz>3%@hFyiI$nwC+njB z*inJi2MYagP%uzWDdfA_ap!=Y((3K#GHyO}ytD)@*oyHKthJAVbXMw31=Vb`hgB1q z=ZQzE<^@NlGOZwdO+{^2b=@6G>C`&;Wdl6}F3!kWB_!wO?^?WLF@lcl&=T&AokXbI zS&H^14>)_a`(S=J7^cuO&2)&&pxqQE4GS&qt zno8#VMU-jyGBG>^!U!vmZ>vJ1yc_UJj%}sNnFUmGr7$_78-daaF05g_;_0U>(c+?& zF&e*(LhaP!yWHpTTD|jrt`iz12;9ZOep_mBwYjH=aC2xYVsT|XJsiC-URiNMuEt8& zg#A^tdsWqnxYw)jFz=IB4C;u3P~dDX{+X_JpZ@V<=MRNVj#W1shj}fh`X6>>;l=o= z0~e>53VoQiF2Jary=nRtX02|+Y{8J(n9$1aS!IiaR2bYj&o|A%o~fpEu4dl=AHWJuJ=X*p-+aB64c(}gUiT)H1#6xr zQBLSefxQ9P1#LtOe#5WAT`k`nsl*&|)BYcHJ^1~+jm11T)yZD{iAUP09M%z#jR`I7 zN5l&Kr?UMk-35DPZ65xW^~j_n?`fgY<^`Xod*H}%M&G%5BRq{L?J3s^4>}yNO7F;v zT-TS&-1yU26N)0L)in5Pf6l!v@!coZW1{$raA70FZL;9Tp}5auq(;RxbOE#BBABk-JI*Y%jzi&Pm$K_O%28{rKOD5^)a)R2O{2dq#P6nev}7- zf#zC5qCc-0N<(XCZ}nDxwZVs^(y@hSNC5TmGhv-DAq3x&u6tj$N8G+EVl2vTCKaIx zL~nos+Wn!tlqRa@39$vtl*-uU5sD-mi>SCPuy{?zr#hl~@;0v?B)j?ez3SdfdW0H2 zXE2p8+@z1Znbvn@__~B%{ege9Uc)QfCowi3B(mw8I$PHVci&y_HH!{C9gECC698ok zRz@A2gA?;Wi(a)*_FeM*+qYNnoz?9dgm|s{j5ha|quR^y+iMUEwwbar*K23-dn8$O zDXzTaVIn}%4?xbr0jImgZ`)3KJ%t&_Hq~QneES35bHwkibKNf;yu$52s^6ThYP%z< zSDibZeZKvv6>oM86MEY-b#d-`1Aga&yehPMrw@YB+-1p##5$}_#p1v5B_i94sxfs& znHexM1i);-yF&>v1`d)kB+Htk%I9Mvlg?X4IHO5DRLw;`=$6%~cB#u#lPogoTMt}4 zet4XhS~8GYT4##Sb~O4G*h;Il^G3*~;g6L7vJTMg*Y_EXco{EF^0qqZ1x)lY<$HUr z=0VKxiG;4~s)~DDIz+yrzLMEXcB1Yd}% z8bJ9cRYMN1aF#AFNEIK1CHqw#OAB_0pgazBnvD#z8jsYGi!9K47$8VWMnk|O9ehBj zopl!dp~^1437^4M9B=d6=y1);myn=W(#QyMd`WdRzH2Pc*A<8Q+Iyc%Is)rlOHk)@ zuDf!>B_xYc?&8w>!Nhy>Z8}v_Bq_=!*i{_#wC_o*yCC*@Qk0B8U14+RV;oZ5)s!=0ST1X zRw{%p6c2;Tz5(v3QseQ1C=zHBV@jFSX>WDe>Mi~f#~K3Q=5Y0g>tf$AnC*w5Ax&cW z?ANQkfrRe2-DjNYd17Uz0-uYr_lw7uouB31Z`al{fG_wGOf)l(VxgTrB781rP`Pn? zA8Tk(^)yl+iE^J+9Vd`w_tp`cCFI9j{l_1!Ui87bja}$BhqUBX0U`AA;Ug z;~~wLI0zy7w|VE2+Ygs>v?Esm53dKFZz8aK;4hWnvjSnhakuf z+^6ECI*T#ROniZYBe0oG9Dg~8j;fM0I1xXVktx*m;JEIw{jl^n6iUzVXyEWBQTBlP zI5F@9lmRru!sz)xY@J<(WaA|Se`0*x2YJ?OfiUb&rrs`v)#tcK@bx<9_uFPW#DQ69 zfxczCr7igbC{rZdZ8clb>OTEe8ONezjeNLO-+HuucVS=n_2J!8eE{iRp_oN>k7ENo zAh752%UHT0on2N%vg0+BC~$ezPHgzvGAHTFS@i&~-O+-jU0~%bS)N9p)!Hk}YBCdx ze8|5)ED*%0-5W-4@b!11&K8`Dq&_49E7WCo>^d6^ezYqbSx66_?rTY7+S966VXupW z=LB)=GrX)wO11H42d6z~>J7!xH|ym>w@m*8SoEu?8MVGDB^!Qy2To6$nd#ine;Nid z&LAy3uhazFrZeVyeNJ#OmVcqGu)#ZnwuzV9>a5?;#Czra4)9f(XAb1-CG46+&~uur_gpUOz9C_M5K<3&Et z@uNbPeK;_hhXN%l8SH3oVy>3{)A*I3E?&AZ`P(fNqZlW*nUN_kk1>y#5gV5ghdGNm zHwzCByOEI@I}bZA3#*X8|1TnW@D@rM;P`*PS2*XTRc1YK|6JY&OK{ZolW=L+k_5n* zkk$r7d>2KNkgSD6L&^&ys_ln0u0FKPqhh_Y&s)sAj7wl2_jJm*{2j%cB5pq(_X=zz zk62`Rxi7xfw|DP%laYP-Vw6{_7GP*3mm(vNr)&^vQ}b~FLbVzOfSlL?%CbxX3CkwP z!cBw`ZVd5e=;9~x8d|Qt*b5We7{(gPre5V5PH;s0B5ZyQV^iPVA{4(dhr3$5R!?H6 zvqhCJM=Ro=@7gCJYqfB9{X9_v2Dq{R`q&t?D0jP>?pw%-VPwE9n8U=8TdP!p>M{S- zYTA)`zNDYyUOEA={8LKNb!q-Vu}#jogFAalK?RS!C*7X|V<=bXxbQykfVy-i`_7zY z4D$*#krIBLdd8ykm$v6MuVRy&St+;SR_XIfx+=#b-9(m8f*8B>~a0V_MLFrN!#_rW8YAEym4fb|=bZ{KdKD!>-uaL#$H@C8=jEy3m67iPwXu)`*Wm61cM0z9F2mq16B68gaCdjN;4oNlC%6Y4Tu#ot_hs*w z`_QYa*6OFO|NE;}_xd&B)-~eyMmP^PFt4yMimRK8rKtmo*NWk9Rr{Z8nD730f(L~> zqVj(?gUH287?Do+?VjLgcHIIACFVk3pVBgKOyPoAA+u|SIvSQm^GI8K}}te5UdV4ULgzJcbJp5cKB!TTE5Gh7lUV6m68>1pDOHCpR#;7_G3tn zvGU2MUJmdLm2a>S`w{f2^-=3>jQA`p?%k2oEOD2oc$E zZS2J6Lh2MBhm<9OqiBsdxtWqho|Jg>!U21tcpf8qTfBHP>=z0}KMH8H;WyCdDb=vi zl{WSOlkUB00l6L5;$mJFs0cVHAf5tJ!44+}{y&#By!is~KE;>9(Ki$eT2OM*Z>ess zUF71+Qc1*w<)8G`ue zLu;23o3_j6)7}YM$H04$;hnqHr|aRZLj7tw{p?&)&>9QtL8i2%c<{zS2V9>O3;1@g z>m-j^qv%_LSJ1t=&MfR!>9c-W%>~zTcjbrG(sU*k-6N(wlW{aX6(QtKF83q46xzn1 zNhi3J3c^k*d~LY|KZ7zvxz}0SDAPr`XBUCCS)d!nkFa8#x~c-ruto_qdJ9?m!fDm} zA4vrvjVQ!*LNmlIVi9p3vNfp)P2${A%rbpM3xNelu0yEEu0uMyL7JhQrt=g&LqvtW zP38f2bE&rZv4Ii4R_eMyrywWwSR#bj9Mi_M{Iys3i>B;O2sbC30GQ)HG?*mjSipuE zeEy6=)}g0q^7BkOA`Y8Qt^5qsY>VMb#WqTb`;*vZq{sMzsAz_gdM4C*Uno{+V6b|v z4TOEUfiV$C<9uvza>QZaFR?9qFe#-L-fT}(e6e{0!yg>97kaJq#lFmeyg<)i?#OB! zC5*uWWuz_r)=&4BM&G{Z@l)Kq@Fb{=cU(DjD9oL&*1emA2=v;c+$Fj_tRogipLfj+CzMI!?cekl z32lefuUl^ATh}JHz9ozbs$h+L*FaIrMWY0(V_u=+T+N^z62udfic`@g>&%h|a~Gp( z%EgHl!s$wK2vH}Sx4BzVze+;z27Q*Qh8_q?^=V4&GjjslEw{n0 zqEo0hYHk~%Vg-vIvsx4m{l+w>!;@f65?o5-xvVMS{^Gytm1#UavVQJpTB-`hi`2J% z_?#>6(SNrWU=4hL(drf{+j_s*h}T63cGy^S zx9%|VhCg!MHi3}D9S7o`GFsknsV1 zAEO1;wM4Dx=eWz~Q&a!d{zv=J0O}kw&JDnd-zK+&4G|t8dXi}T7l$c&L_scV1k8z+ zYu|dB`cr)-0f6b?y?f*%({Y1JORH{*uR}mFP5+A`4gp785nNRt!$p>Hika@-rXmQm zF;#(kE5vkGZ=6-IsoeK9DpLFY1>UIooh!tP_aE^8*>sZh?N>!Hr8i#~Z&KBMxUU)# zMg6)&Fi2<~7?s}y*q))7eH*!zc=az5B14{w>y7vD+&_)7YTt0YRrHT8?s;k!ytj9& z%qYKu+%7#ttV*Dsggqc$NtMeI$H zr`fZv!*gR}kML*)umub%b5Nksxk)U{q7B~Q9p=$vqEY1{X`YAJzhSLcmETT)iE>_! z88;Uv=-Yh_$@&uet4=mvmU$QHvY8-TtwA?q%~z0WI)7s>S`poWen$dkkA6qKOGVh` zQ<~-KYTdK`&Cb=XsScQQ*t;IJ5#2(nLli_rr>L7IPQWO|xM7KDt%qf8#b|?LALz^ewJ_ql#MlH8v6>6}9L3;*i3I@T3$2p={(&oUQ-6 zl|jO+=jAn4w&Q-YYM~D_SU4Qw$uB5LE~HV$xfdIZto#9${lrm4EI{I*Y)_1rmEJwF zpi%>Omb{CaHD@}O5hyIMo*cIZ>- z$B&*En84;Z^A-6n>6F6;ZEyM`$24mFv>+;tga z(0RTBTEl&Ph0|*`y8AD?F|&g(@xw`i*YIb^79{fkjSPT&p31&ko!MDjpq`okMUQdD z`Bpy)N{$gro6+Rdl{@Iz=-#+_9s#y(kTEt>)upevEeToiGNmflyJNbm$*8}625Nhd zSxWT1ZM*>^c2=K2n>{ z$f^IynM5a3#jRw`PKUC7eAR?hCvihx7yVyD&!P|o*7vN6>mC&4vkp=%)pHi|t`w}u zA+)%o9$sp(3NTjT=r|Allg*hJX#aezAksFQ`&cFwKq^dX8o>VfY8S8~UN;NycKJvz zE1p?GKA9mf7^vPD;srqfNPy*SLdLY>B%^|_w^7r-4Czr0(K<0#ayL4s=cRak3J$AN ziA2ey(qg!gGTTG-kH7yos3&@uFyM`1sMh(Zz3?lBp_(6Qd*MZs-4;u#qf6(pvt$lV z`S}7m{LcPT^!8Woro4kvPX}JBwoNjf=wb8!Dg)$9;GgTW|3~YOH~cq#0z%GTa1ZLu z0X?;@;8nXJSyg4m-%l24|ImhfgiFHuStzI?xhSDGU+dRb|v*Sra3ph zWtK>(1>vAa`I>$pDL9tCE&hU)ZOkQkIvV!p+*9%C_4-*1uro22w~MBF0OLi6f`dn3 zmny+Thqa#R{hH*Gyw%WKenw*Z4dpmX((g5vK&>m33J2b-i_sh5qK&cpcNONpSCl%C zoPS}p{pf|dcCfh7lr^(`5?93*&V?YfyNoaxwuN>+-I0D&@;E+9ljS5$za$U4Y0Vo^ zBj%4FkqX)7a!%4)NscjM#?3%9sM#?Vi2swh*~DP&5Rb1&~vvQ^w%BY2{D z@x6D8R|_@i85y!qFd;tgp`$U?xP5*E}(r($=3QB(=jNc+aTwK_?V09 zY-6pHg?A@$qV0Th2BISjN34;}lJh{-1S`4ojB!{sgw+sNL$5=jS4^%CeuaCoa18bF z5eP{Q96wP?zcW2af~b0e`eZNRBBoK=&5JhHC45@Qe*}+khK>58aAF8^_PMm+zDxJeCtnXXFo-llhfSn7N(>N=BU{3J@R}Rbi`0q zOjC zHk_AhPKr3y@m&J2_2=@_SIKzL=aLWg&EU1{+)8JR#~LT!_ukT2{nGl(0ZO8JJj5e94t+gJ_WIuC(-eC6L%4U1br=gQSZrv4tO_ z-x4$C+MBp;(RQiv$UNxI5WLPE@CLt^ttt6mWSQBaaPn}ILoP(%sKGq{0oaW4{{r^z zq9ccLI7U?gSV0t4tM|clUF6Jkm`k`rR8d?3!T2STh>GaNbA@N_Qj zAaD&vPWr-CR-PJT>MV}IM#R$V;vQI77@$!Id@jXS>F;qj)?f1=9mHs%$eb6>oXOM_ zTPAH1<1TfWqmekvZ;s4X<+F?R5uY!m{&oNt%*QEcI_y*>=1_n_n5?c5(szxrb`Trg zH|BRtS6Qh;8R-QFg@jZfkv!Yyw(6d+ay9ccGpvlGKfmYl&QNSI6oYfm zMg1D~?q`?Ruc7aY(8-pKEx6V9@H=NBkj`DWOmK!{OwNsV1|wZj>qdX5-^s-Ce&^f1 z|J!R9CMNAE3R!S0ps(yE{D9w|DdhXvrSpB}Hfgmg0x{{zMLcvnbSD19Bqi1#Ge`T( zEaR?ibH-}$MMyB<1Bo#ixF?`NIKbt+Tx317iuLtCb`yMmalbce39buwy|3g$@)d`b zNf6DnMQUP`b?p1{doF4phiX8eZKPIhZdJFO$*UcS2E(XVz)cRBI=;;;Ln8+>t3D;a z05wCuj#pF(vXNgF5qIEI2;n~5H!u|XdYAg9DF-EydqUyTnzkU(9xWJQy}s6;QRX+FH6Kt^u`h!}Aq^`k}M%aGA9j_wwKn?)<2wa<`N9;%Nbmrhp!3s2!Tv3YFIJ&{ygnez$(lY`3Xp3X91uq^TVYQvd zN_rNE%!3E1Y@RJ){E4d_h5S-**Pi#?Fgr}dys_+{L1k5;X@VmA0UTUYKHW34l!-Ps)7@C7(q0L2yO+X zS-Xp>aWILor9gDEF|mKgDE>+q_ga-Au3UZJ>hBBTj#4?qn&@mWL-tT1dClk3j^c#o zi16ijdSNP=l>1z38BKKLL730*Y2BALYX|SZUVtral-#_NoVq8_ak*|~xi0(Y>r-9! zucB32(==;Gsu(TxqAp=w&09L)@zE(83{WB>)tWyp zmE2&R+!%`yJaXI;}tT5fA)fn^MPyK|x2ON|n3S_i3W`!K{n?gq*0k@TFk*)&s+_Mm0ekT@mUb;J*S518KHVNK zOcYR+jwG&Mv>)5_*(x%vhDlq~;3zv*HqgU}rf6E&pFg=ayUzb@ ztXbLB1!g!_4K>WzoKc5f@}{5NW{ZQ_ zHZWAEWRqIabWqzq`=3NvwtG`DAl}XYkR1oE1ch@BUU=DaDY)UHGOSF-|ABV= zkeiEyS4WKaz1_=w`}H`QpLu0fzdC47#a8Q!_G}EzQ-&4LSyJho6kA{8L^PKckTn@D zKq?ut0P|UtkQQ1&opa|Yn8zb6Rr!LFKC}YX(8@rutX8-6pUPFAJ+n&cKA))=cYm7p zKhUZ6G$Zl=lw88!pSZ4%&uViD=H{&SuLaL8EGPd=&Ox1@9aSlJ*tF(-kQUHhJj?5R zCEFu@5tIqr)Flh)?4K`m>ichjE=vLvJ-qO-}z&+s+gfKELJCR>g?u^)YvDU$F504 z>FQ$l2x?ii$TQ=#h`LR6?_w-GdxOz(Nl($`@EKZzR$kt4(xXyszbrgbCU$hta>e(9 zp7)CT+x+n2f6e(`v~T=e1jUw!x~qXTNVykJ*bUFazs~6w>dX|8ys|h5de~Mp%V&Hs z)XQLJerX!l#lFYVZLimy6~VcIZ045L~%7laWYJQ*IuX$K|M>T=&-Pb zbl)r;Q4LbjAL(ajeN$6NNa*gt?-{=<@6iKy3!c^RC!PjeAC!>l_#YyABx``L&3>L# z1czs_#*!HtRPt(HGA<$a2Y(|>Erg>+qq7zL1}{gbQvWl7BN3v|0}{(PR^}?aUSR%*aKk;I%?LfCgNoOVYM1c z6UJpLf?+o8+l=A+BNY}Sdq0p#WCLsBf!H-UysKhS;Ex5X0-!0CHV zjmr-&A-iJ^wv(;(mCx;;pV}lmzrs7$k6veL@Q*p6LG(!c(qS8jrB@6NDx*h3uE*U_ zTdJm#U5*p=m1pzq(+v)PeieiC{1u+<+Qj?8rELs}yHmABO_lUYqq!wOwsJ&%q`@qt zS7C7XsvLJD064lm_`XOr5$NmH{g|6edC-+P)^Z~kO2#C_zUjuk$rj^e%po83PMMQf zO7dX=$*q6rSHMk^RLTc0kJt`uyzfV2f_j?fjT&J#Ur!w;qah-b{lLa($VAo2`#Ik@D@)MmtCY<3Kd<{7EC*#>S5I2`lSj z#Z<}5UYCegPmaG{*W10!U5&3C9d#z)tsN|48Ys1#n5YOoAAQT0HD{n9ptFIAhP@4K zzI<3!&UuWKaYlf+@K<*6Ko2wVPBg*KP|dg5gNcV_1V7l$=cG|i}cx#b_wCW z5tTHw)_^8Wj44|0YmiO>{g@Q~R)-erwtd-{^n(&})_0Q@w=!Jl0sO8$k)lUso{I7F*C91?6CxAYDkt=;T9-ds`dLg;@NqyY`#iXrkIgtTJz=F?_q2ourBAZTQ$%idvj^`1(xa&iJ0 zKI$m&N#6Q1oUWeF9yhl+{X33@@jl%^x_IO3q0kB$lIu`vLR3ETlO=5@l}geCJ}FfB z7~+5YF$^b024h9g1hbF0*AGW5k0%xX+myD%y{*a7__2%5`N>l8clJ)A?w| zG=OidDLLQ1M9T;<%iL;t#gdA8*Y%-&1_eA^OMXVV-l|-j(Im*GDng1ihuL~xrAkz_ z8=sDa{{E(3!4(ptk{J7^b%a)}1c&W?0cPVo{tMNZ?0ybuuXULwUzLdb*#T&C(Ast{ zU-t&)gI2d_IXy9H?uT!fv1*gJEhVhXNt8p7h5j3!UK~3y3WLRvFBv@vor2VFW)qxY zA#I15pir#AU)-)xG-p3L-YxO(>GJWqLou^2M`+8Q1#>u7x%yG(@o-eRr_&n-Ibb!a z63I4@z5Ap2wsrk5O05lkIhZegyFcB2;V-Tto2=%6fH!;_^?FXu$==-Bj#;Nzgx7qV z#jm`YAFhRc9&e61-$vRB|DainfWb4ySw`9Z;{14Emhac$v^`g`5yWqV;5?HhP1Z+r z<`^==ta>Kau+7+*1B|!_;+_yK@Ohz(*40tX%W~+ZT9!-?_#OsqdO(vHlv0i5>+$Oq zF}U~+4v-uolNS}$B^8n2arJmS-0t_@4Bt7QR2DWc9li6prd1nsjC(Vv1lK$|Nmx61Q_geG+bxHyKitK<_paLiLAyq+HIBJ1nFu;?oTdfBWGbK)nBXsQuk`n zs+f`7S*x$SSiL|83GD8|TtE=QYv98$6e99=(Ne{8%ve7dj;x=#lE~a_6*&s5<1&k^ zTFRFJij4=^* zq%{!GHJljucDtlxDhKF}WgN{I(1*(k8pO3tH~3w?A7Nd6p*+5HQy}p;*~WokI+4t_ z9RWdfvedf&WuUs)YBsoK@UV9@OP2lCFtc|uA%0|Ppor&Mp<7CzI}()6obLMh^^Ee% zde)S<;xj|_?r^ex^)#G;?Wjm9%VOeZM#s3uPM;_64U9FJA9=LqSBh=O$&161c=JTc zZ^58ZzB=kxz(N!1T_Bgt`)Pz$cAVkxj6GH~I{WRYL_@G_{VFEdsak%~y8!)QA~x2z z^Y!%{>>W_g*YSsW4jPF^o3Yt#R*O8F#Jk3@$DPl|t*ibUk^8&hS|nPo9u6_jCmTrxLiI;>c0r`Ikmo zIM2RQx!=?2L`Kdg`ifuEHwFa`lP?3{AbJW0&cWdv;w%nJP)q`^nIim1I#+?IXgEe z2Ok@Uu;Bmi1Y`&XC-a|w7~BiGD)a96Lzj05INXThYu=i-dwG!|g7tsTud@bJ0k96Dj0?LFx<`Nq_9 z5E~{dFQ01DSvlnCWR@yRo2Il8L{$5c*PH|l8YF3r?!lfHW~SuoaH8!p!w)eh@6m}()#z?Fex&mf z#?#I_;;^4=q=@G4(SJ!Q^yS?X*3{#9R!An@S6~#R{dlSqqh?qpBT>5MAsvuTUW$1I z+i)nR%gvl`6izFse^b2JcTXo^P`C2p{9#4IH)s92{UZCQ%kTi{B9P(?71S&G`uj=a zQ6c%5&~YPoRIRM)VqY;Gm>jaT5tWx#L4F=YIdot zPP_4%`;+|d$=0l2RXSqkw1>Xqy-hauCffETPejjwM!%J|fsMw4iff_KUYHnLU{k?} zG%wWH9c01qoE<4YGLeBcF9MN1Mlam+KX_hvJp)Y7DCav8)tk2mG6p|2Z<+;{zhW>R zIe;TZnt4tx0MS({=tli^2C+m=UqF2mtq2F^XbWZ}M#E>V@F&iAEWPWkh<9f2e~00{ z;FWj8#|JsRK2L3R+>;vh(v0`ZgYV%2qV6v2{k~7!BjIh)-qQz!v|RAyWHF>JI1?`!6G8p&?s8Tyqt z?4?!&U37KyL)5U?S9~02B$~XcN0r8>AL|If3Ws@zQHKhLGzV3gg0rf#S}=2ARRwc- zl?}m4*7|1`h865$b9Ade2AR>yH*(1^7GU$)HH`cG!;@{v9S$>y68Xw>&kT_7Nsil;qQT_+%!H$~% diff --git a/units/unit7-bigData.pdf b/units/unit7-bigData.pdf index da8ff43a5fb9f8ec01b9c93c5df56ceb3cdffff0..bd9062256af3d364d307f7ff157c528f77b91cd0 100644 GIT binary patch delta 7087 zcmai(WmME(zr_Va2}x;b1nKUS&Y?R629RzLiQk`&Aq0UDiIEUwXoNu+Lb^mix+SEB za6lMfi0gfx7xzB*&3$vuT6^smXYJ4W?sd*OH=P96odo|h;iS0$!itJSfk6Q-j^0GC z7VR>PJY$8)2iI<>F!LfQ1G2ixabvZrmMz8JYg1c6Dj=2V#3*cD!5_!y`%$5Ha)Nin zkN^j`Pu%Z852H;2P(@k4vVl3PZNs7ISEo0HbhIaRyRC#h8$)BX#}qyLe~j$EkHZFa zFnK2wpx7$U&K&F6Q$m2h1+950x$;U!eYRIFF&@t~k#C5t;vp?#I8PyZ18` z=vN3ybt;*uWmWN{U_6xci0>>%2dSj7?W^A1h(C+8e7>$LI2=}y4<6K$!26<9{#J(6 zPy#MKA6lA85L*4@mmk@P1}Wsg2B9M76F_sN+eC158-nH6Kdw0a=o!DUdKGhaLGB9J z*_tq`BvY$9z!RYAl(LnR!!n-r}$9g+U{4@9pR3|NeKb zW-oEL0C5@V{|GjFG-cyC)pN{ciuJj$__moD5&I|Y=kJQ>t+9Ac#gb?z%)2XLkpvWOCd7g*^_5Y)x}93Fm4gexct0t9J(^ycXZDx7`N9aeHfS%+?&u%81M&apj;?{@+uDThm+hW zmE0&xAUo_mgTk+Gy2UYJhAN7sY-94al)A9!T=1JrKCP0Zy!bvntwtfXVbYImm!jNG zTSSy%#Z*p_mC+&OMLbm?w77~B7k!^z()*7xYGaSeM0AM>#5m%tRC2__#X-O0DyXww zH(I9!e@hWBYS`*EfnK@e=t$ds|JlWqmJ(*>Pba{JS93k{R!-8P=W_4jwG4bvSkGdi zOXxQ9-ErVlTivry$5&ywcQ^Qa8#Lg5bIkC-;RgN>MG$0D_))^~CAH%B#z)DA^GDAK zo7{4#1cw7h{Q-FdnVklibzf5^94{}3)ylwr-L90itr6J(w@?N4&iIZiB6 zhn}OEYauUhPC+u$hOE}iPR;GS*iF1Ny}o%UUwc>kZ`c)HIMRUU-x1YVyP*y{XruzU zik7m*q!IOmgjSxrto@K~i+&&XiR=dBxgF6|uR{Fn`&QJ>wksp;(hQ=qZulZ}%^c|m zN)Ha)?fuwsW~Q~;Q|Y7Op3Atu&zo{r6}XT#&>y&P!#~bO@7sb>!row5#)S@By?zXD z>PoMyj^3h!y8vyw00K?(W2Lq?;Eh8MjDKswDMAJ@9HFR_g^sj|Y&~Zg?me04jQ8zr zcDP~t6C9|+Y!9((Z$t*Kgi9xGzt_`%sJGkcw8ItRbS=z_h4M}4#-3ru+4FplVEx?U ziFcha(ENEw^=oyTX z9IKc3zb2>xCit1tW$_c81EklKGBEqI7)-C^EkU1igtR}3E8&ITQrfpQ8+)7ENZk{l5mzk-)Tr+uvu4Av z6nM*{i)aatFWFF->(5@_>$RL#Cg z$Mp5f!l9bSIWnGPhTS|erYf9ZX%n!F3t_98@>VfE)d%XJLB*OCRqJrpqG;h0_SDpq z?92^Eu)v=o#i5}gWx6m5jcwANp*N83Pg@U2G;*Ja|F@*!<^-m))4<8f)RNe!CDBNL z)6w&Y(JPx%a~SXeUEx|$?AQFEuISUhTX6kF7)4a>Qu9Sv=C#_1iQviObvKd5{}wgE zHY|+Y6Ja0gn4;|CV8y(oN`oY_*D+dWo8K9uIMIw6oUPKSI?GHN`V_)M>}MYWjL1#^ zKf0XhS;sBQQ#yPK4N{v7Bl1d);r=QPnJH)RuB;FT5P-h2m<(nvW$hW_#?{bheX+G` z_t)jF&mEJR#0-9#MG^LZKG~FGpWKv#LL(i1A4Ec-5$!{=-)6eBkzQ@VqtejHkTaJF zeMwmJgM6l`J$(x0{PUE@5H^Sw&7S(qo`yw%eUKNI()3sBZG0Vfc#-`6)~i;ix~wvR;eV>{l*Y7N|cD>P$4e__ zuV78qBLs|-L>kI*8@38rROEtZGA{;}BC-C#iCX&8hmxO= z4PU;?=!d-6r9y1x`oeBv1m8Xm-Eq%HDE)#tbluIl|Enfi zmI84ob3u~-yC<4$-2<02C~rEuaMxy61u_F!Yy>S98VY9y^o%rF7WHRWo<3ynOKUrqZ^|2_Em86^dg>`#fwe8OpREq>x67aaa**yNO2Gdh15YUl~w4WCay%INM)ZQwV>LCmW)rp1(}BuOfzxmN^4C(yr>QLw+ian zvQH>lGU6qIkcwSioSG@ORMna!%o?Za_sw>nbgj7{A*s5zB+puL-XZ*e@6dvN&*!_m zEI)~B&ii{R5!jKyxV!0^#2ncHIsr^;S{OL*T%vI0}=H93V!412-^(YI6V#LxCx zM|{#6Y`by7N?Buh1xC5pDq%G2Tj;{CUJ1_!t?W-J2Y8+y1Zh2mbLG2mep9E|MwN+u zZ{GFPxu>wlXHKnm5dx1TDS2VU)a3Yp*;L5E+E5pJoV6Y0tHXEdmX(8VaniocBONs9MI9!u zYIshz&5xfT#L$oofG3p;!s8p2tt)~pI*3)sYpzzo_t5*0vs@4z3v2lx9@gZdOb8+f z;vx@tf!2B6tUlLvlwValx{1H0LThtY#)qez8*Qjf%~@g03Z6rypj;foGI0lrpX~Q} zOyh0`)TP+Ugea|qMutxyTlQ7#Az2i8V?`@*%NBX0$?HH&8Gd&#@0{K9gEJU;ux^mM zc1Bl;L(w*5i5na_dRD@-d+d{q^ArP#rf=8=SaMrLv+H& z=X0s^^2nA-_C&>)XnQ4H>&|CVU@yKcLKM=L!;UX)3=~NRCubyhQ1_u&{W^j^Nt4@Y~&$vC7w`tiz8DS^INwTj>=vyK;qk;i_p8%sH2C-h6U>dCDwZQm3Z<@ zP434T?0!SKe3Qe=(UFm9v$8%)myK0TPMVyBI#qp)gnWrnGrStCjqyslK_pCk=37kV zDSY%JR8m8q?yW8O$Vo3)+5V5efAaL8fB18jK{LY88lsW)O=RVw6X{-1hDVD_2g57@ zAn4Lybmv5D?ww}V3OVO2^QqUIo?3dTH1s^jTEMK%1H-7COT{NFC-nLw2keftD6m4) z+zuht68gAd+agjlLPNH`t$MTM{HrOgc}9^(kU>>%?>hWJdAV|otfyh+!VfA2(;}A@ zqPygs-?7KUUNJt&ja`%7RYu6Y6cm%9FD);BXMs{%=js7s9d$wQOGTX|X!^P4|15GrO{$u5 zP5FkS_3y1;f66`QXbvd0rjy{DW8|LO&0KDLzq7E^Mhi)5R@ui>OCk1If zVe0aD>7-Q8I7s7W)_Kt&2xsqR;_p6!#JHcy=>AGMBw>@IR}q93QR%iW4=E-|!-6R4 zTD2N7jUu+#bL5zoQzimypFuxO^5?b%BvXuM7wEjTt&hdO>SfGKO-mn&#H$n59#Un4 z2)=(~$mgO1q;c-1+z0CKW2ni?Xp)G-BV-MPR>8jFdrW23i{3WioS4#{@0@j7pZXCD zJM>mle^0aibzXl&Q_I*kz`Y!5(X3N*@G`8QmvXwBa%+z6%}G0Dq)PI{L{*@PQ62nI zuyeddyy>N2Ls@!uMQ%2#QvV;%$X(;t)5N>I423uV69862wg{&+C z1?-%nEMZ-D)7Sn6!fd;Fb1ax+P22j*hyaA=vBD*pkMG1A3v8Zy>a*tLLAid?p{v+wm)D(diUS?~8}fC^!3tOI;B(|#Tww-{05 zI^g1%LmOyuA;6=;)$5tTP^6)e0c^spt$RCT9O3;1dvb=U=xV>joYD$O9go+xDugZA zN3_c#i`VNEeM1kiQ>atWOJO0w7G(Z7hPg)ScoBOF^O-#y`*qgeF(q-;HoCHdwcP7X ztU6a{4omH9i@tt#7!n+KiBo8P(M^+0+-~NRL#|EaH40=VxD7P_}FEt0HTS<^@ z)l73o$X^|{+;xPlx^r~0*`(5aR%aTbF={O_ghF{QFt= zsBD#cBUQRC36$J_*_wZ0PyKrSMTf2>QFw|x*Q-GlaltV!%5!ZAFWdQqS39)Xdh+Lz z%ojL5WyG4>COguebHe+IsPXBM8{a=T)3jNTE4i z%6y?JzJ*p-z;j`r@?j^;oBp}aI{0iIS$w^>vrJQK=6x(Ht0POMB7-J>AEv%Q*wvR8 zK7HsX&_xE^r#KGKI|f|(`(}UoRn9yRVL)~zzMKS`#F%QQo+=#w zsHiG)eLV#MTnWM`%bk34{)n%gGKVK*Vn3z6-i>s)qS%V!?MAJ38NjrU>au`IEZMH+ z+lQa1*3eO`k&-eb4=YXEPiBJ52jSJDr>?@FbRF-Bd^U0A6T=+4=2bvTDYGED*EC%L zJ+oe|Z!B{g2TskYLHw*)}b+s#u0RO`z$+)$hI4o25ld~UCpkF2w&Z!b>b21 z>9rW{7Yd9n%icPbm5Gh*_6B3Zozz~n)6&JFRyKc%LH_BJfXz&yIg|bSRtYV$3 ziT(#bewccfiu98`>K5{2IU0<8ML@RsNIpgr%@qY%TgP1BZEuFP|3izmXiw`}XDUek z{5!NAf_VkxYmK5QzYj>f4D=N+FD=BJS;huVe7|^b%_!t7;4>g#`HzkN6Q4n*FONev z?F%61hj!hYuP6;R>8}}x(fmcdzQ1{Ila$th;HKmA2df)W4at!te0~xgpbPfpq>$Ko z+FR;{qj(d`90ElXyvT>EBhxS>P65` zx9H{5*E34Q^3W>9$gQy0{m+labye5QNX&P=@z6Z%bvRXyy-27j3oa|6s3Z|5r@Da# G(SHFw74TyK delta 7084 zcmaiZWmME%+r1!2NQuNvC@Ip?ATbD1(%s!LfPgf^PfB3ua-_SZdxnr0x{)sF?jGvz z{XQSw_xbeya;~%Xx;~tBU2E^P*FJZx*!Qj2Uu)5#tNn#qM<<_dX!_@`Y6yggDa+K+L^1vb6Q%-?3r(m zh28d(jFdpNL0*6V!WOdwGgTXt{n;$kj0uQ{PNiihSnxY0pj+^FR%sI-ISS1^Y&757 zc#(S24;-BO&DD@FPH95m6v1Abt~!e-G=%yjWi|T~?85qbntgn-3U!m?z70^B)O^V^ zvaOucWP83%=&KU69NLk4x5)07`sU7!9|p97B*>KFKGtW_6RP%MrAzTPA>6(DDPLSttjLm8Aa63@tyc$a_Osu_t+I~#DHrpy*m>E7o zESlGPpqfpf4U_LjS(}fBY}ep-H&}d85mm4_Fd7ML*cy!q;NkkO;f<>~1aT2P`Qr&7 z#XKT&#_V;7vb8XJ;7N0bX3ln^>8ciCs8K*XApyjmb^Hdl-aMfc@iX=S`|#6MPS=ZF zZEea>|8ew{Z%S;hHa|{6UE&}S(QG|RF@}&+f_BK`4af>M9kbSS)M)vDiX8f~1^7T( z*4YDb9iXkEB!O*>z5P90<~qIbP$7>e3*H7IktnoM$mH@dauBkE>&`}{Ycw%FoS2~Q zxvv`i=lv)17>mS=Z8mBW@}eg2Xn2b^Bhl8G9ChMk+r@i9#DR8%Nx71c!q(@J2wn5_ zqcQ=soevIqzut!ArJY;Xo&3CuRIZgelzXvbO3I_IoYVW&iSTLZXbp6GuuLVVmt%ui zEu)>jpEs-bAbNcko!!J<-puD3oc*IV!4vgzq@{5%bDbDDIe zVCENzjC20@sH(D8BG#m$F@M$S3RyJ#Ju!c|Z4iA`?(3;DjmL_o%|^r7Q3hu7+=Gr^ zNV~bT^H8@X>2Izz;2LV&Pdi)A?HE3=pzOSADMCZdD9c10#V7#g=$(k{U{E$c1V@U5 zK`PM_-zuY9Mnz;ZUv*fNnJ#lI*7`MPc>c%-H=f>JOh&b?K}mBa#@Ve{%h2&yKME3V zSG4eCRotr9-yf*3M-nFsyeR3o6Y$KLq=v>{Ne$uUaVxlC^D+eMNhb06@+lt&7m~*N zRO?5%4?_7es@7p=c4&%1>1pkN2<65^$tqXM$7lN2rEfjK@ZgDwlM~m(t5%wdYQW@< zfqQWa?n0xYI_)vR{&@QwD}?O(fA$z*kM#m?v3x`3)qbdEcEJ(NOq{CC9Aa6_)#vZM)(ElJ^&U;yVm z|8K_VFHR8pB-=S3OX#D!4*n|IMQbS@J#JI;8AZlD>P<@g%j=ElHBloOKmjZzmGfI@~p_t9!1qO-m(*(46O*` zE&qxvVLY)wi zjmcLb5$2IBx2NVDF|<=1i9y=5Pe=H%g*jd(>iy1eg9$xlvyk*WdrRbFE%B`Xlmok; z{biz^BJxA>zz5X=)yx2%5K6p6mf-*P`BTS&t2m>jc_+@=x{?RgxvLQIy^KjM&@ z)CJ~oDm8CU?V`h#|52RCc}cJ?Bw{W+96c*MJREHQs>I#zt+|P1b02Gha5>uYl{eic z|HJGmin60t+huyKV;C{%gSw%{yXVS;wmo82K4$&9Y6T!-?gNR;@;#|Hm?eUVH(8BH zVog8T$zUo8WRo!VZ0esJ(0*G*@>$h$Qje9c!pDgz zNMo8agfxEoGk8ud<&C?Bk@N}R&Wado^kPM1Zm=+}O`=8~ZXf1j3b)RR!4rL*ZbC~- z{SRi3)xaks_0XrKCQFD|%hdy)EB~f%6ZK+g2{(R;jBB^`o301T3GW6_ca*IJ)BYFk z+o_j7{AS4!URIua@P^O02mHyAWD#D7M6byv0$BM^Nt5~>t`wESOlh9w1@Qx)3Suz!(*FVGBqoq#6#>({_rqzPQg<3Lp zw2Ml@-a4T*8}-buVU-Tq3#IaP`rkLn=)4xM;0i%Jhu{N&IWGImWe!TnceQqZ&oon%TNNx>xXtQO2|H^#~75ZR-|?=$yOReVuz- zQD4K0Hl16`Zt+d=N6_>#AO?Uu9CgU1f$_`pR?2b{lh^&GQ+Ay(CLok0>q(BWKXtTS zo2_7avZ73P;seK?2#e}@!=T8Vda}bt4Kngfcs=F6_8Q6S=n4BoSjmQtZEw+*3iH(NjcE9-|h5=;>ucmr=`mt;xA7@j&=G z7dqz?De8ch%iPgxLB$27cV}$EaA&l67RhczZYc$DN<6Z{IZa8V|Iw5F)yqmADcbUH z*AR|$?2ED7NGTrr5oU%sd#P7B_KY398c9XPbREp&Wokw+2ZN-zTKC=|PH)A7=v)kNt(+|XV>w}| zU^HA9nk$+BVBpa4fhc78jO}tvosrzVArOO)`;DsOiDc%QbuC&Aj)ULvvfwPQ?zSZQ z^6-B0Y(jYEx}8&r^0i6*nD1^^6Hk7=fm=gLS2$9l-IOelL|P9ASOVa#_apewELwx=vtSa3^*oh*asp?td4 z{gO^*p!)sTK;@O4LUmG8*ckULBK^L_u|hLpN9qrs~T}*sH!QdCbvt3i!d{Nd6g{*aFYjR5~s6V7wf(xVAT!?!ezy6WqzoS zr}4@fZ(D4lqR5Cc^ZJ$z3^%tL%*SNi@-BQPCUn!K@Ocn*oFFJ%IC~zuHm(0ZGSM*S zhW6sSlFcT4Fi&X1w_N z=<3|?D_U^&CpKkiH0;6iFPR_+MS~D7dH_0ryqjM3$5OE-vFjKHyWOws+TT=!&&$d3 znx-5+%^Y(avZ-WU{8rsLufLmo7;-k|*gk{jHV(0nZNNX{9m9Fs8tJ~qyS!7-4M$6* zpnq2is>joFdKP8h5XWpI^P3ztJYooOiM87FotX84oBJ{Cl`_-}C;nVz#n^KPj!g;t z=bStZ)*ERzS3bhecVRO4=mLQ2!_elWwZD(^e7OfKb?TtbFSszuzi=GaC~&uhP>5r_iZ{v|$WvyI#^)LCy-bO=m+aXRJTS=~B*Q(k3*l$k zMIk#o1OgXX;PM~#&MDSZ*;elx`eM(7DpDE7`42c?I)tU` zO7EKr-ceDbqb*B!B-&Z9NwtU15F?m{D>R-D?vqoz=h|TF;IXipqKFaFZhSowP53F} zm|yJvgNoL!UGymL9$zqr>l_e3mdc^agsbNRSfO=3`mRz{t-^vj z5>gQZ-2|%ZxVqKm7>jZ6=)$dKxOkrLi<0s(j=BqXN4G&m?N{lbmjlpR_I?g1b-_?| zQyy3bNaFm=m|pt*wGwjVLPld7Ze<18_?hu3kbiIf(;bb8C-O;j%tJX2={iU)`0xr3qcz-c3n#4;;8(Oi+_nUN(V3 zAV{7Vx4KZ3ji8X<0kD?&%4nIGzMA>C-)8jy<>q-c`JlJ>RnX!LxNM(>NLLGz(zUBS zFun`9(3O2Xr&4QWWwpzyZXV9#a+FIP(Z4@US9nYLdaOuaOS?MHSD?#Z@8k9?sbhDd z^0OTgDu%a_pE!9uxeMU5?@*16QoC!7Vp%b3CBX=7%#GudMNLQU&@XV=y7_8jdYgl5>x;T%jHM?DRJD)66L25sYmSD63|qK(BcvD@_JTV zjngQV285uOk-9maHC>>eDqz~oW$IRanLa6Hr^`vswk!}G>86^dwIX%liF&KH~m76xY%+wSYDfY z8*MUsH@0HWc7jv!u_Kj>*u=u+_9sAv;@ zzA75sXp4v%R+Bqja;>;O1m8`sq}yvzY5A_m%WtvicyurY6==SplO>H*nhHE@+AwdX zml@<5*Pl+{B)$E9Izi#LEt!H*Q&*UcymJ00&Y5=fd$VY{ZW-D2)&QL!jO*F4(&S2V zMM*sQKsWIh=|FMaz*{_Q_2*q7VxaUwa_4Hnlyd@g?Sp!bYWa3Gv#od?#^2@!QD>1q zcmBTSvnlxP1_L7tXA|GVK&gJdwnuHVIVsMrDbA@Io}9OFI4E1nc6+j^xJXCq8@x&~ zSF*$prmHU7uc6R?q51Y#9q0>-*e>R|S=8_o9_Te)Gl}ZBxL^gD?LQ3sBiO)uf#OIE8>D4=YAKKMS}hs=-MOA^Z|3-B(8TJju@|;->diw zCtWqbuZ0DU6c9Et&Q-AH*`MM+=Td!vibuF#8wj}aJnIo_56-eN1{T_AT-rC!&cV}5 zBKI^-pUyc_d=D_i&GwhCul$3L3$G3?aw|2W#Mmyg5mujn*9^ha+m`CZAu~os#+}aM z87;&P4J$|&O4z$A1ASK~*{DB96%8t+=qBiB>aYrO^aubWed%r^v;m>2mYFeb18?7q z8L4+{yuIG)sqhERmT59tW|~D1o0sd-z9%cY``^B`-8C-O9c372I!gw5yEH8>`ZZtK zws_AkWj=nXadPIYcy}mflv{yVfHcP>Ew#1TFRP{YY?o5?iaR!xrVdP1p%}ci(VptBRdM&ILS7zh95TQgv210_ z1?Ml(1|Dq%uD8spqf%0Y{rOh=Hng6fZl8a^0K)1Kv3$;{i%mNp9@8vmR9e#kNXy*M z6{xoh2YNjyh&c6_M98D7LZG7J9XA}`-8F`C0(^6LWs;HEx?B6gB$h)aayEy+GJ<6` zhu9$UbtRGn9QU-I7Q6lX@fw%C*{JSa5V&|%rxMvqXK?-kT-+!Ao#Rlzpz8$eeYUy< z9QliJ)Ot5J=>s?=Ke)9PjEVvc4c4F%<1CGV-)@8b&tCYC_c#6gEd5WwflCsX6QNlF z9s8_#DELIS(n|fhO7qi*8kS*PkK4vpg}>M3U4t_DWqu=ZTO(_uIWoPmLo10*J<-sm zixfrbuzxzH`!Hdc{H^j2bmM8ct~Z7bz!X9=MCy{tw(C_c3WA)WQl~>WP0DeR6=VwV zA0k@W_SR$lFmT-Ono(yL(uPMb(vnpjS82i~^Ozky*3+1R0jY@`@m#wb)U8RTMv8BJ z(~_TFeWZw?r*3Ejht&ufT^EUgt>V~9bv%ddhVph9ufiiCcJ~mU-u~nD@p7RFM6WNA_Qvt2P6V02=y|?16H&v!iT_?|BQeIB+U={a#TR*?K8Mbw8-6*rlPSsGO8+miSX@xOLFR>XolvZwsV*TqT^CUcejY+{eW%&CY% z{7%XRnPr1FsN|!YFc7Bass{?hdcxgW_zqGc@0ud`iH`y3Yx)s+uc@x#rvh9?(h;gA zUw7RGoUio;X;xQwi~j&ciAueBtiJlV^EY48gdX~er-6df0vR+}m#*zuOEom>H@wfd zhcuJsz|Jz`mI<=O`xO@tFHzK|w$!<A>cLG5(a9p8dAZo_Z+jnq1(6vMxgbcYiT_7|8!v(pjJ(!xDlRjMfi z(C}nMTuNQYqS80&8gWPduo%W@{Rpi&;jN_@$2b5YO?vKyun1SpL<40{gfe~!r$v~o zOnbcqe$kxJ8+Z8K;Q-wOURXz~V$CsUpFyBsq1Rzo4dJ!VfOgDA3dcjo_A=Y87D|WD zbaNET0f=LrpU4pxLC%hMp9%fLeO+i?|N7YY&%yjHzRH^39ogQADA}s#wV3g*4Fyw-(~H zH-?b{Ct#j=!#X=~jVDI5WNJ>b5T#@EtY$6HWOLS2kUYee`reJ}xy_$1X;IQbG~_Gl zhLIs9A(+_+|2`AC2v^DP`#5^~G}Y{amQAR~QuVsk@GbSplC6yC5~tqtX2UwBF*hwP z9be;YeVr|D4X(Bm!-6@YLvIBxa$n5JM9g; w_+3ol%}>epI*$K01NrynY;OPe6VA-_iyG!4cYmQ)`x ze+#AW@u`;O^mO zY3hjJvt_t66dT5h3bxQ=oNS(1o;7Nxk6PRYC*HHY_R=`PnNY8qiu$}TU_2{}LZIq1 zMTuwqwEUS=TS4=^X{RfyQX`#?%`c<_e+=$fDmX5;IT@7UKcj^^uc9&>5Q7~J;463) z$(%nruhB2YH8v_HE*NG8+;;quDho#_b!LGKpPxe+tGz6q)hw4I^bh}g)LwCU9mnWZ z9yB9I=LGMFHKLaOA0sxtK#d6srb?rQ%3dHZPvyUNrsR z+QiOIy9^Z|B7azyKk_GjlLh)d9}r@NcwH4-CZY2RS)?YW=g%fz z?97JeIOs5pPxASy^nxVJnKYQ0Ekxs<_4nMWi%pZ}Q)L?y&+||tdugeyYcq5-c`p5a zST;>_k*&*G4Fm*LZubVx+oON4zdV1?1O{#2{trqsdmG#-lltKbsM(qLYh43#~%Kq(c$t+0d_G-MOeK zS-LPVZUm$k zPp~X=U%O220>bH5mi~vTkP{jJ&;O=sJ0X-6HS*Rs)~I_EcMeD$DJj-swJzl20~S6+ zl`Kwg^!Ekvi^j+SL$=@Rv~X_zPsO%F?WIxYeg)HymAIO<@?w2|9_SbjL%Zw0A+(nz z*3GviQvwr8-oCHdJd``n6Z6|QU(eZ*BzF7F6?hb{6-hQlb+iCxS-@$0!9(JBqiAfEmhN zG^rj95X1|k)S#3-_!Xmx8LerRK|ZgZ6cSMNTg)H?9_tu^6SBOB>Om*slC5~#26@^A zCRRZYVgo;yQP6CC@ z{%{S!h@l=9YwR_=5P`Y6iwOGr^hotr>1WxG0zO?$s@bi?g<-Iq7W zoc_7cR56eMfcJm#lV)sxD2gew|AqOuVM%v-s|bz8SxxHDqu?kan)A!4XdLyqct0qW zPee~g?BKV>X@S3c-=mrKP4|o4DIn6l?^WB~rpM|o^U<-Y&-ZnrJ$Ed|cI%xUVc@y? z!us*B`l7HlmFKY%q@;Ee^zG-zR|u$KPPj4tynj(cesylx-pCmz@p*r4d8J0Xo&Exy zT&VQixNy&${LXswsp~U{QgQEan!Ej;Xnpaw`CRalaevji8kzoHdnNfk?x~Hjs@3dd zmTnN{`Cig)BdNJLv!1c}to!XI5TwPi&-L%R2YO-;<$2#Oi#U5WOrzJ5DZXPSwUbg`%)ZcxdzS2u*Lp&jHhy)2x(JLuJBLgYqe{a7zg%&Piy-WX zCg!aaz)31$pn#>LNUp;&TvNqHBg$0dVMS;FUQ^~Rs&3wffJA+jb7xdIXmudvY6nP;b(lSN|P)TTXdoPl8e>W z%GSDq`Ec`jbIm^d$$p`5T0%nvEj|^!-V7>-x`OU|J!8u;Rh~6)I+boi{FVNUZC>Oj zqJou#lGETE|H&HWXAd3F%ChOTq>e!MM76gsgAOY0RN&oKLPe^#8S40wpN|+Z4)nO>x8~r^HyIwI6wecWJl^dBiQp z-VroV;R?8^{CV(!B6zQm`#+JtcTNM~{@z$Ab~)ov-r6fT;>iE3kA zq4WV>543=Ycg(X`93YVaEsf!#^R}1y-!301W#+C{2<6Q_%`dEO`*8P2Y$M&T7)J8v zBzB6lruMSV77NEM0RrpZr##*ao(F2oIhWTfJ630w=PN+~Q2q78)eF}ROW~@^$9MZ& zpY377n6q&pXM|77R#5)jeW$H=e)cT2m7ta2hX-;0@7a=b;lsqso_z_7D2W~(p4%Bm zzm~qIFKfZ^7l-DlfP#b>S8Z=4!DmgPsG8l0DMo_Au593hwQ!8%2~9bvgR3 zgghmASS2$Rp8SPw%21rkb+sA~N6?_c&} z2-r;z`UfW4Dq{aXKBRud4RDStWkns|fBkuULOikb$);9}$%#*E;6LG&>F>yOji8TZ zZ~U}voOiFdF0LP>!v3gu$ZOv>aOy@M=>?sHA@USi z)&A&GP#dnJBv_i+8$<$itGJoy8zv6;-7G&u_NLpAw&OGhfI)h;fIC-P!RI{Tf0iI?Y>i-!D!o zZciP?Pg3t}bs*_Qg-o*>DzJq4)J{M21b z4H}}BOlJ6?#Ug-4w*)Zq$4y61EVBZn!0kPWbbUTZ8pU? zDFk^d4!Gh`VM&y7A&6GD$Hv_1N+l%~?{NumYCj8ww+`Q0oK|E{g0}y#?pJLZv48-_ zV1yqF0lJ*xexMqM%&9}2DI>8AG#yK=>!}=QZkl@j0Jk>hkwrMZ!Bf4~{-rs+eHTev zGHTzOlN1-CpM*3@c=+_Nl)AmrKQ?{i^NNoDeZQL4TyC;()eF$A-S7%ic9k09pK&ed z7qP42o$W5Wg3kXQcI-Uw2!obKsJU?nT8yq|pMpA((VderrRc`shC0;hhSBuM zx`xD*BFQ*6;N?niJ57${Z!CpYDJNAl087MVy#Mm{FGr3h&P)90Q?H?W&ZTEx^hipD zTpR-3h3vz9PMWJuO-K9FfT5gJU_;~>BQ^l5jb69aKViT?yaegVAKdG`Zs~SQ?~y{& zXyPfBP6o#f=CMabbP^JlQIyD;lcYk-$!KYx`~On4xtKo6$Gv3UmWU!hXj@w2g!+8= zedqo#gaAKhEzAft&G~Y0&KUxO&^3}{$9_u*9QSPC%djnL=}7soR&E6%4~FbQQ$W6V>H zP>+~UIVZ<8P(&koXH|jHAOam-AXWp|BqPM}!|b7KtPzllm^Z`~{-D+`Ajq=nm`A%5 zvx1dV-r6$wGK!becqkRZl=qq;)Fg(xG+gEzV$6tV7sM~DfC+7U<{sJ)sD zmJ*B7Y_N&TH-e>^P;?H{wnL_)^+%s|@MEHV%@y4s9uZ}sr%6F&isNWewi^OW%gw

    8?hO27vK>{VK2qcuPlM0bes6w6C64L@i*Zx2lZZ!^K(?(=qAi2`XXFLU`nZTivhGOM8sVyX& zhJ>1fw>VIGdGP7Q^3d`wAz}WIlsjWuxS(8sa^?cULRYC!0~ehj8QE3M8Y_c^TfvmW z^->}!kn!o^lCK5_OM;8#6{ADd%QK)AHRRiqyzj8(8p6%UEg6B5WWfN94O0zF2-%p} z*p{q%QZ-vP1%(>;7S0nSfdu?ISvUv{)j!zK^vdQL_a3~Mbw%1H-Q|u@iX92vsI-Hq z2efx>k|!O;Sjh;cS!JI2~Xsm7HH-V zN~Gl?3O2OTq-jqYaIpY-4MyE9E0Y+Xsj#9^N*Oo=HFg-d8fEo5@nlnSG$guM!C~h= z^AJhlA!)wcmuXSIR5Dr9Xlby177|WM9H%X|$msReJzNI>m2UJuEXF zaO)2?OD4^mA(2^QQ+R&yc!)s>`QTq~4U1~T4m`Qi&+uwsNpL9s5FS1#7#}g^OQCP242F=1A+2Vvr9-Ky_z~^_D*?XHjD`W8b!tU~ zSxsrZ@M{xvj*MiX;r>FLlx~1@Tq@bbuav%JF%vEt=NYoYhU!nRrDnA!9vFL1vw({g zk3NLxvG5a3Li4~U;&<8s?dbVo&#m-8fM1pL{ORME-_t4J{yL|P#QJexdLC!y(vA;X zhi=T`Y2?i9$&Atc1h1kGFL2Q%1 zzp5QOT|X=u5a{1sn$NSa(SM04I_j<#Jp;-=-ZJT-`aTE7`4?3QGPZ&tzY3N9N>|dy zlFaLhk7F1B=f)_q)!=8Ng-*s=5~}|2eF1)xfTRNjCjoA~LX;=wj4@tlpxx*WqUFgOkAf5)HWkiGJm$+h9Ho#e*L#h4e>i}J3) z+TwMJ{vM3ki?}E92L*Y?bgyXkMtJY0o8W3FG4^kOCt^kvacr&nlx*g;VT%R)NZ{47 zf*uL$72%|qc!L7 z)=M)1)+cAK!unAMpZDL6Z3gKm>pWqLV>-W#=GIaJC2&eses?u;%N*O_|A>6Ptc@1) zYa%;gX<&I1@b54Z`LyMHzdMldl$%@z=Cn#?o*h z3*+JKC_WuKbZQECWbLbQ@uov)?9OUf{6?<^Sk_+-gCK6W+(LXknAx(YsantQgUcpX zrF{HN&?NixnCtg6F7MA)$7U8;Rwtg%9)kSz`zA9%Z*FgC;(B88tG%DBAC51ohVvCC zX1)Suj!##!aWga0>hzH*bU=`gfGxxXSAF({-iiDsGfm2~!f*W^g!$ZXZ_v@ikCF z7&r87zba{&@NIkZIHPW$!PD!{b=h4aMa77SSNu_;wzBn|ot%}zZ|;(};ka&G#D;$0 z^NPh60KPgNJ7If{1kb9+t?lD{wPzF1i?4-JySZ!Dk6a1fvCOAD3KUy+*)g|Ufvcu!+y@JYGgp1)Y_8G&%1vlwtQ^*aS8Y67$5_NhTMdB zdknN+%kdSZ0R?;70*36%1u~Q1uX{N)_%5(NR;QT%tW?8VJ6G0juDa+7G1h8wMb){m z^ly<@N4Y-MgWJOF4BOXu4aN&<^_aRof>8ghKHXg0|ByKC7(r=y?m2MiuXl3A5Dpli zk2)XEG(X2 zy4g{TrYYc+htMYUD>m2NRKr`n^)flKU&aV4ra~x98x**<{F|L)$z6T zNju%;AH3cOsEI;+n0oxYJ$e3cGs*SRy< zE=DD6mqbB{klGD;U)s;iUjA^pM7ODr1@{Jac)na}5g9ovvS9a#9lb0LF2~ysh13%4 z*Nm(y*{tpcZ|?{C1jGYC!2}|ItlqE{ab4yE^b;YS@_9U6%vH49!r1iV(Kh9O&1&nf z1FtFcV~~>qyB9~|XPFF9dmLjVk{66ztKv7Vg>__i+eU}%6z~^9?BOBlUHWP~xnet` zApG2z`ozEM$6!Zq^&CXs>r>+_J-PI$$U3}5+p}_lh|fqnsq+A=!;B1(6`y6Eq>sp! zhzr56q+YqCPRv`WPlUD}HTc;i%%eLPYLHw(3CEh%DW?6!V{dK;_dx@UKfP6`GSfAU+~x zB|ae_I~*~4({BNA4&?6kkEm~DshhJ#Nro(T(xBqH?WDr~;8Bk8v z?z&?ML&VkXsgc=+$D_rB)4)S9lHK<0YZI&*`PItTrXOH?S1WTtiNtH-3#jCr5bU@W zPCJX;A$XNZl~I9w7udvy*5$vUfd9H>aO*(eSRLTZ^J)}!b-gx3lG+XFwB5jAFK%l% zpx(NqqxN38ts}k3!b4=XStZB<*|pxdZojGES9It-tVONZ-6G zedAn*mF_*(J!5T#Yi}2<>$!df)u7##>-WzM-C^xzs+4OF#dOBcuJ=x58e@TTPPP%s zPXG*xOLwiJc?q2b{-!j3??uVw=SEjgB=v5^)XRd0;j*Z1IKE7}t*s3@ldn-=0?fMD zNUi0wa6YRO+OZ#;8xy7DxLjI8I^D&0r{>-|!`tUZa=(+bU%0Kx(7?QckkbSzZ-%4B z983bg)vxn~m6=VqHmA|gBahW+-zT$sH~@2`-%O(22-MC34htQj>Q!Sc@G$+%+rYC=YrZ!9B9su7p zTCEoF?4Lkr)TIvJJN&9+)G||4t^3&4!PPf{+9Ub4UaAlb&|bX3K4u%}$|KEauhiO_ zS>~w!+xeGBJ7IARliA3pr-DgayZ4qIuYlTx#?Hg%8QyYy_;G^j6I$_#Bqj1@_fF+C z^Rh6RI_p)r+bRi@a#I~eg%f|)Jm8dUdq_8D`M0?Y(R&(_^i~kU7Mk|fesSFV&x7}B z8ychso#DpC(LM5RHdpKU#Db9P@x76Dr)()bk9vJN{TY=XQVZ-d@ySF?r ztFbn1;A?2J`RI#!e7Dq$)7+?=5G8LWOu0H_mohHJK@StR{+L`dMmYC z=@X?9a%T)|U+c!no&c#nwe;7c*xq}sJt(OxB#8jB@CVw+G z?=N-!R(X9m)BO|t`f;_N3$XRuXw|Ct{A*X-ex{D5Kl*($CV~3)JKE>HB(|_I8>@x+1(Qhx&{izr0&D5>qFz&$s`Y;^Ymw zy<%!Rjrs}tXfP3)fVfRi^2|Rq=fO^;NT`*FZ_YOe@5e3KVa5^o0H_1sDxCZGbN6jr z^E!B6@;JhOg1-rYkZAaCZ!Cr>je$V*8LV2@8`RCBuZ#@d< zRdt+&pNo;Gqy=N{0v2_r`|t9WL$RUg{7E-b$+8+yL+ZR|%fH?HV?DP@dCck{Gqq&k z7k$0@ww#7f$ESIol%z%OcN0KyiRYcoVU&S8y|a&Ix=QPDC)8LCEn52fI-8As*;|5F z4o+!$cn;Ib$=Nq$&wcYh+7sP!KuFC3hfS|)oi=h?4`LuczrK>D5frz7hmyO?qyo1wNf4nREflm_W`sx`=?FDG!dDr1be);U*rxTcjZBFMF7y9<5 z*s@EU<>x0Sfa{E>IJ8BUYY7v(03RE^Z(XiVGV`99q2{NX+|V59@`B0I!rAy%jvP|w zD14rP=0Re$cM6#d8=Kv4XIxxI3>||!>5Iu(H6B-|y51b05IJEo(O%FzUnVP5HY*hD zq|>rMGJSkqlg<&%mdX9Gwk0Fl!-?IQh1q8oQ^ucX!01I2cQ+~5R#psjxpFFgf~%Lq zUo4_u6kLmnz4(Hcxnt(wI)8hR?^ zNH)1P+(j|MLr1maWO6iUaA0cGJ}~1**Z%oN{c~aFR+fkWE)X%vX)2ic=?TI7HWYybU2FV}Nnk-3K<=?w zfU}w8vO=x>YnkuLjZ)QQ16x%%~Hav;-ULMjB-U5Ig;MRCMh-f3MrVZ5-7Md)D$*~3hlh+Vf8&gYVZWB zKNkkmb8H<5mWs$fpNcT-K}RhWpH`3oz_E>ucKRGLW)$I%Bq+knSk?#}v{9HE<`g#R?-&d~3eViM- zA6#C%+>2-|-Y`U0!^<1+~a-zF#}E83pnCd-eI~BTI`sQuSdOt6#tMQDzPCo9p$9CTzp48Ej}ce^O%-z%wNG z&ovav-QUxU!0qeJ-K$UH{2tH+h{t{sJ@mbqX$I18|Et@0^EP@MuWK?6BD{C=(zd2@$JrX$Wx0GzAlJ$VyC3 z*%&m4X*ZLUP&EWbRe3DcNEK5y#t?)^8O*Aszx5&bp>mYaQ5Yyhi4s#AHepUik2 zgZC}m5S=hqbQS7O`r;uBe+&-e^v#j$-PmSB%&}+GaG}Li5tq@SFmOpp!`XuE!_Bgc z^-=K$VnZp>Am-*$7VQeCGA-4pI*bQ2ZU&`fmy|O%`2(I`Mr;HFmPvNfP;A9KkEnBl z^2_6-y;HC>jez*tQvfMfZRH2E9G@oT_THITIXw;L*@x6&@x1_L=@SvDgjxMB3oBxKkmMM7EA!=0_sRwh_;?YPQwUxhhHYYkm z8RT5txYLkztYcM_VkWPaFoMWsxXh))SKuotAqec#zFd{jr|{le;+ zi4~5;q7SWeIc+f`WWWZcklSnhjcal13>RjonTw4vIRThF*l2ACGtd{L4|#s{E5JnCl9~3x*XaJ(NX4k`X10>))h{~l5=Tc#+gv1iQwb(CB`L8oK%Qh zAbrqUt74zX*{TI(V!!FP+Bg#1iW&PwV|Sb;V~Ypqxqxx=>gmOEWA@eXBgF+lU=xH> zX$Tc!sR68|q#Ds>N#drhj#t#!?yC&End>KjJYTKTao|?Em|?BmB2;WW%-TdJZqV?` z6jXv{v}iW&d_%@vq>%@XN#txqk-*{uF`FPYnZo?;?Nv6X*K@-XH2wug2SM@c8RgsC zf!@=O>I?Xd#h!L;5=;Mbg!zh`OR4?FY$ZY=WWXG?E={G_SYbGWU03q-s9ZJT1;9QmvX~UgY_4+%d zSh26H8h&*?s{97v`RgBYSz?6cV=iz+ywKHJi1$Ttmkc?aYwZQG{0 zp~^ur`|p1Q>%Igq0yPb4GQ0R*AN{*8`fj83SC~N`?;qs0UQT+?*M>|5}dPY_U(#lTe;4@NVC70UaJ`3 zQ$*ue%%e5@z|pw?JCpb@R&y5hjT^kmnp6u_M4~lervn+LK{@!MMf7UU;PLOGX2X}2 zSQTErNgeW0V_2as{0~`OHM*`{A2zY3t-!cEE%DFu_eFWf-`eNvQ}Vb4sxh%LqF-^#k4LCb<;-yA6gd&eCfwTS{Fyq4n@s3wa<}0a#Xd4h1|n4 z^qb5KXg*R@uf6HAy%i~i7UgV4+Fcua@)*6v>KnuIKYPm`$|mJG7&Uwarsdsxfe)rJ z>Dau`A4gyg@sMmVum^q)VG^v!6K0(4W5u2L=B|+KBQKo@paVUFC?Zz)i6baLUNn&_ zy7G86ZeqR*zVZZ=L1=yx;@n_1FJjN3gF)m}R_Lw^U;UtlLCmfzYyA)(FQLDQ3So#V zFS?8=3t_yu3*p?@9xn-oD>(D0a3>tjkvd^yU=xY-f%HyTu8V-vkjKt1ysQ}APOJ{o zA2);fS@AtCJbNP)ojC6Iti+=L4j)1lb8w^4M<1-)L#5u*hpxe|g0vF2i9^JM{UxG- zM0duw;99*MI~r_iBukfFfGR z2~9U{^OZmmwRjtg-F2Z#T|f=Krq!-7mtX9&bQ%@3`-gt1J23=c8c~umzRNkSX5{P72S%hd@Oor6P^+ Fe*onWYJdO$ delta 11464 zcmajFQ*b3t)IA#8wkCEmv2EM7jT4;MC$??dnAo;8aWe6QbAR7=U+(`t-22eo)m>e+ ztJYp?*Y4f>&lAk!6HMdA0XTcg<^eH~E^6TdO1B5Jbt|M8&#Yn|1y}?Mr~K&CAqzh9 zFX#eT97?mOoW@PNZXnAD;FH#Vf&f{kbU3#{H+W$ zN~m~T_$j%3gWvr&H}e9hvJr!7#S8GYAv>J%+!Pptf7fe+oMcZx1InfxTaW-P+|h=S zjcRYE;aJr?E!>$1tZl98?iNC`We!fe7^GbVgy8Tvk4-W!2d6_$Pbh2BQ~U}?goQz@ zH){{ZRt`K&>3_&%Y@F!g62(fnOD9yZC~k;GYZ@ee(ckl{MGKV@OI9bEN!uRK!Y4Kw zhY+h>XIv+lQPyVN?&w{8dWjO(%&6lD5w`fJq|C#c{F9^ip|6E?Y}xsO+*3_?PH{cO zN})cY0dfloA-H?ES(-W`_-q+2soICKqJbTF10_kgrP%D+7@`*Rpo$Eo9bfaQD=_Vl zR7v0OaPLY{F|qnjnDf2geSK*?`Pu(%zchy=t=4JbLhZ_DOzwGA4-9|wdbJ}s)DXUSO*0r&QJa%21e4&uHX|h@nWsQy_zF`GWp1U{WHSC?u1m!K%O z))RSD^tkeOeLI2CzmSXB-0)Y@zr;DO{|jl&F%9tN{{d-=zI~_&TKN7K_D7~#NXF_e zR!CwdTyQy47uLItM>2wI8pwBIpy3w(K}Gb?-D=(yrKG@7i%|Xb*S}`Rt_q`Edv)FJ zHgEl8p2adQBfRKpW5EPX~PuQc3Y&#z0bF`}X?lO}!%mrqr?}(66q1T%3en zbrHWVQd*kAY_IkuL0@@xELWecz)$RNkLcUlc4Q`F^@{|{agVZoQr)biLj2}l-{yYL ztsHWe5tLCjUl-cGRy+1?cQK`YCZg}7Nv9gyd?ds3gvP&1wd0KogKr!!2h^4#K{$=f zcr3~IBREP`bG<@pQTdfjpqt9IjF==<;fN{}Y*k^g2tl4gZIx&(uRB&G&nt`>EVWHF zl?}|4NNpEHort}(EZ8g&6znJzY;=QC;)u(j>b1>m6FOb!pOi8xL&`sgM1P1v zr$ND^gmw{NCTHwKM^o0AP=IPV`N@<)(HJLHn+dkFlRv6VHGOEt$jk7xzq&3?cehNR zcm5ru19k?s79KM$wwwe`Z-vw2RyO|g6hDyX|HDhKyHFvCHW0zoOkE30+$HApBbr>l z$^Z3tTc3;LV)gH4{p25CRP5N)TJLiaR?sJ^!qu#k7k_;*MZm!AUfa5dR9%ytx7e3V z@s7{AeQ+w^rreqL===8#2-qgxqF(RkHf#fSxB-M5{psYY(EtE1N{HW%UjBQ2cJ1AE zqh^P2ha@nke$M%3&5YZKeQ|zw<&^4IHJQ1~CA+*r-M3Q67e+ox$vV8G-V9sGw}wWE zlh0|n;oP+nj9kNVL`$awkE~K#W%Pk=KO;Ua&~AN~l%)_Lx>wQ1;wlvy~J= zw9D{6RPl^-s9^x`vm&E7nP?I$ffNkaahhj7)LV$Ydf3b>+d>i%cHSNWp zQY94GTD!;HiX;C5FtEw}%$fatx&3g^u-N2Oy=T0oZ~T5fTX1QWT&k*>p{@FtX5$`FP z8cOE>%$`*h!&@rKk!_f!H|Hc3d#81sW!;}G%eyxH`f8v@k=-*c<9Qp)k2bN2x^ZTO zap7+f*-RMQ+%P@L9wa?+-qv8GCoo+b9H~UBS)RBtx_4fQY5jfFA53vutc>GbTHN36 z1m2F^`3U0~{)U@lv7hu!Y7&5aU! zhOD9{P>R6QKFgx_`7jc^e|xF>-0GHv908~~ESU=l*p6}%E(>#9ueaiM)9<7fv z(2Kf=b+qdC705I8RlWUU#g&8iQ_$HSb^LzUV3+gD{?o(vHNfF$!;9a&O}BPA=h+r- z??}i?Iz{pKqlZC!Pq*`tY2_u4)-JP>)#h7P&gSu3ma^IU=c}Mkd!N(Qz^adK{(eD* zHBdKe9+L^fw{ijl+goLB3W>fDqWKzgm(uNZfvGiY;Son{Ud}Xn4+3CenKdD|D_ZG? ze)$4JPySSUtbn0APe+1~OYy61HLJ5-ZNS_(C{;Af{sT;~QOzh7oL*BoH%-n1kuE6; zDZe|_g0{+97KY_lTZL7bLF|b-V5NJ+8W?T=-$@!@J>u=e;rk~$dVL?F5HibND;$Y%z5>4WgYNZbl z{ZkO1@#`oJoW)2`dBBGq3VqkSOoW1gh0@I-uEt#Q#~fp=!c_pZoP%X~qDP^kJ(l!`=XvCG{nH05 z6ozoZ|FTH!nhWTdP|S)pzW+h@?#7Y<3MhoeFoy`IHk$L(Sw{_0F-*pWJ3<`zO2d!u z&a8a<+Rpmdiqx{!6 zoyafU_H#^|4q9QghB^U+@36kI)h`wUpsl+-%eovyU*1bd+DO0kfUO!w7%}Z_rc!eq ziwn6Wew6*GRX@+9G-)iS!gz6l>ByA{brqrWfIj=1s^+sxB$}>qDuUI`4RWy# zvVc&W3E@Xx3u!1W|Fl(957ym5959^`KHjmYoC!FP!lXUi5sX04z-TC)S2iOeKZeSM zHSxXk>_!UK1p}M=_o9yMtEj9E?3caJtnX-5*5Bx6y$BMZ7!tT1p8%@|Mg+}MA zBdfjA(%+~9jVo2E7WbVG6#Pc+(INYl^7sab&pENJ&ra$p=uUwK)jDaE*`AZ=Ve~B3 zsm7BwRwtEEJ*GKs@;kZFek3$g{kAfm^bYh75YFvx*XZw<8GqQDI6cjM%hl8ve*3(R zhE9KM1^S@;XzQxx$B{%azqOqbjC-HN>8b8nn*Q`!!M>2%-8f3d`d#Mp^p&Tl_Tw>^ zhwH-MgQh1YqCd4%-tz~|Lpf#(ZI04)V)geXWPGDN6HrdD=VS8Z)^{ht8bPZNto0{W zy@+$8t{gHnJZ&fwM=OGeJ(@HQO0zE1#F!&exFwep;#QCfd23%OS{GkEoNkM!d0=_qi!Z}RxK}_>y zzclU==}#ho9Er{nfdd=Nf(m5AKd#b72(dMFWlhQ7*<(0uiZGK2iB>J}#A5prDP@BZ zt(;Gcxl|O3i_70rP~cR*+#UM&Nrf*{2r75#l{b{N?bb3L$v45rnK=rqL5X{v^f|t* zJQ~gFiIz}gdKPA_K&yyVL~SOa&aFik4zZWUZY)p(N6MNuk&0fFZ2pIuJ1T!Sxo^GwOAhvE1!hwy-pRq0Old63b=#r{y`#7Fz}ZUD^dF962l@}eageWI$s3wo|9O#D?@N-7C6+ zTbVRpPPwc?S^8p5_5HHF_VHTiYI&HN&$iFtX2v6MW4UWEa$gRfzQ*khRv{$dY`|gWFL*MOk0VO2pXOIqS7rYifSg-)1{;k=no~<5MLe zBtJa7x+_t`g;0ttGi&YKScP|ZNNN-aP%M|UBb0QLWT&$!w|#4 zX{mze!N3m)U>6{6&(22>;!cqpB%t+)smnW7}~&5ZPK@J=1l6tNOId43rEFO zmWrtgjrCe3@+99a$FRJy54kcL?x{%3Vk10jJp zL6k_TWj46KLUbhg!nKLUyU<^%58~A@BIWua2Usu^c%a4>cgXxlpzSbUf@LT!^u(IA zmtn{asE{4!n=`cFV8x<6hI>`zges%8OD9HMq`zqKVyS5kfFf+{c97HTSaI-32u~Vt zE6vmBV09Y&=fPUlf`3(>TyF45Y}pBfApTYS5y6@@kRB|p2ARl|eGrae10stUdMt)r zkmqqENm7Vyf>cv`O?!(FDb9zMZPsyMiq$^L64Z6aimLIlP2K*Wqr7s!prB@3jX}uJ z(ChS_x?~Hk0O93LgUzeN{maDyD&SA{ZE zyrEVzLdwo2WF^@xuR*mZW@MyPF{`%KE7bbbP>P<21k2sg=al*l4l{Tqx zMunmIF?iJo;|vz%NYdm=xcdRz?{CZEoB*x@mbDUYV1n~{ZC~~*kXX=|v7zQv$D|A^ zWsy`mqXgGHi(#JiX7>DJbUt=*bhEl!Zyx6n$gq0w>HkE~=Gs^m)S+`BKMk$6u_S(~ zHN8~B!z=tRqhh4y^~(o%FFnl;p&D*(jR@Poh<{y)+9fw5EZQ7&G32W)(fOW2yH@}45*Y=Gl;gxK zUmAQre6?Ir25}SF=!iGQVKp+25lIxXztfy?lN@Mv#~D!dj$^-Nqm-#AgS28Gdoj!` zw@S<>7d9;X&ir%|E+G6vJYidPd~wB|;)v>A^k9O}?DRtL)hqowyC z(=!ni41giK=$dG%+y~9LA6iRmxJeRI6)oCNUsNo)9U=ik!Fh1M=Et@3!E}7 zWfI0Revw*oJu5G*j9#{(KJ}`vO$zhB2qVR`Oy3=<10;Pw z0S4)W$4x$@vxHaY{~lZZo&ET6%wuLxZteNkgxzRmR&(KPycE*QF08f2#*#=&vB-2l z-+;;e&7SA?L)4PRNL13Po{OGKb>Fs5(%l`V@~em4_dvqACiZcU{TjKP`oa)I{NAB1 zWKI?w@7U9v$P3Nz9(mQE)lq-YONI=P$=wiF2-l?{32~;xJ$h1uxkz#yj8d?k`PVjN zEZJqn>CvTP8%aW7w-4`o*Lx?4;hwT{4er~6fH8A{nhb+XRdxoc)xaD{_0?vp0i&~Y zEhxXw*VT7ZMs8Gvy=+y=q^M<5S@MKf_-PZW^519Y_xCKIyWZ!Y;w?aOoh}V11pq1N zCBLiP=PJp^hT!Nb7Z&Zy{raae{kNz)um7vHp5T~OrWLxGf-upAqNd|lMsJ_z1gtpg zKSFG`>>oh>oSo$7`L;Qf-aM@Dz!8tXJzX=XIXxDoQ_W`(pIoiXQrkp+cGaZK<2VPrcJvP)dINXB%T8>HV(bROl z)orKxb&dN|Fa23V0qziiz>F)eXY;pRVZW^NiNSw&_(%6qJ%&~Nd4C4nzI;3Te0BEv zM-MIlhD>KAv6o-h)qtTt83ruziK>ro9; zG60YJjv&!$kg2!a4t8h4yp0ft8;Ejmtaw*Xa>r^B$p6WzKR`J5`|1*h`*8TR#x2vy zfAhIf|16vNvi0gCa5n}LsVukq^shZ%7yE5k zJ#GSER5eIR5AL&aT1G$Y;3I@7u@WAy47TMO?2~5>&ug3T6LZ3(gj&F_ttHvV6NEf= za0Ds;f~!LI>l^Dk8F{b%=1Dei}2|5yB@|_G2Q4DtfY7zH4 ziPhY~?0nx2AKL>3V0Zq4b5D1e{0j2>y$v98x$3P7CG=}fCXCy(i<{6gbLijdc5)=$ zlyeBWUIvjkIXGQ}Y&gmj#e}`Yh&zrry$*?8)>ENq=`RJL7|7$P1Yc>^)u@gLVlAZD zUtm9YGYOFeuA4zFdFL50*3@j$!*}&THjl16J&M2%aUKH0(5-aYI8JHmAhVol^@Csp zz?!z9T>ab?6niWz!h7_F>a{u>uKMK(=mLLw=Wt?z#uxfuV#;)_wkejM-Vfd>p5u@u;wc^P*^ z*JCvgiOPt)Ave@p}yZ5y?ww68g?e6#?H+Sg60u?S%Ivoe*YT?T~zvLF> z%q1@hoXqhp2X;K7EK44Gkxq<;jT#i?k2i%rdJ^WH5c_JjNpAKkwue%W*Vf`o;>T6T zk1K)O>^J9osg3_MJF`Ab^9@!(Ubo#xd&H-T_} zex<6udhy@s52E&;r^YRpxSnRxT@5kb;xJFD)Y7_>qIl~S z{0uT)cs5^nT)c@oKjIX2COcc2O{Dd_RP%w|NkQ#mmxm7d`-xxG>GiGU#)fMdW3Std zi)8{B>}`@;Yov3Uv2KiOxT|)3a@P6Fm_<(}0qy-Y{XJDqPGJiI==z5=VXS|TRbC0! zqYCD3@^5g~y>!oNh7FuAearK`)jO^p&8^{q+001}73S3zhx4oBo_udxvK7(z)Q3QB zpP*$!O9D(q^5qGxr2|b|!?W?ZT~3Y*F2(l9@~do^DT)=IY6W@m6r0J%OLvY=6oNoK zt-EFTV;s@?NV0XYC97PCH@&Rl+XVg7fWGAU1mcS;t5WIA7p%{Z-j!o7q+s(+sZ)o8 zO7pQy6ve(=ChaS!nTGNjshJYoSZSaO+lqdmhCXh;G%klo$LO72_dVxVw9`Y&V({)- z&siQR|5T^6G1DEfJj!_A^m-KCLEV9!nSFxXDT??pPk?*i^^7@paKEC-byIwMyS=yF z&SfRlZ}o)TOmRf_IdwkP4=q;TZPU{KzPd_~qjo|{gRXTsBy>sziuR-3?C+XS69 zJIR0hXaWY~Q;W)#PW-t}yass;Ybe&Q+ELlKn`_zEY&M3N07Lm5Cr}TYnXY|tSs$h2 zwppP0XVWR1rXxDx?*@H{nzp`Wt>MjRwTh$kGwJzECrV21u;K)`GHMFjiM3N9NAIi` z_@oe1mb9908`l_1Vm8m^A%7qj>L$wEQmz#V|K&o31?$p_`a*o?(l?Bue?G#xvLwH0 z;y6YGi24O*5fshu7x;wa!2Lqf?m5l4RP4Shx2YHCMjMh&*M?inf?@tzeHGZlR;eXX zgWMQzbDJ8qHs#B0s@fhrfrB;+Yri>yper1L_?*(=F_}{!x%b5QUMaML>z z;P~b=PVk3H?wvARf~tE-wNF4D6nC}Q&sz8w{)Tlp3DY%pnkRMn{(5H>7mjV5Q7S2d z#wGm(XR{z#p`{i%mNLus@#oul5(95~gC0cfHmwJoP9OTt7pF6?z`ciy%!ZTO&tC1+ zeBc&ye=R@uf{^izKCoo=TGJ87uAty4v3*at`G#pOv`W8bM*rT|jw{pOT`zM>BK105 zvA`j1&%8>%yeeIJH{Zx8N@*GE2K$EgbpR!4t($zVz!}QECSN(SnCUVUvAh6W% z`r0VZMyKEC^iq?-p}HvrQJ1AP(!r)_T#6KI7uJNuV7fQXTf~J%ys;v>xm@sWI2sVE zlu7b*F99#J2qexcyGESnWwH@XY=#XH#1DBxK%W4t8(gw%=-(%b*IHu<2f@SP*;X*t;Cx3cI@xboIF176YO-*9A!)Y=H5gX&;uD1STh z#ru5uXfn+U84VN&8tm@?8NXd4KDFUlMulNQ88J^*aF;zjz3yQ>Yy-`}$x4yL$-lAx z!Rq=83u!!B(}6d1$p0lUP+LNcy|R1C(0o2Ves|)EoLjQ;e|Li%cg*h4<-y+8kz9I? zy#N3@fSsp&MG)*WJc~G4dAWEXfi=Zi8C>V4diubwQWNV$TN_qe8xM2GSxOj#BdEE& z>ObQv{F8|#nRuMPxsl@9;Tf5get4ezuFT=#)-qD;9i%KkBinmC%az9qQNRm>DIL?fOs8z@!kMr3k#+h}g z-X{a?t(j*l??qwtuC6E6Tg)aLhRG_LBp7cutx?$zaJz*6j;DHwPnSd4fk|rf6d3ie#J2^w_;bQq>*2Q*A{J2bG;a`!>ip~leRFtSR_bhfJqJAn1d|K_X&n$RgYzTd0s|W$26$GK9EqhhP^)KB}@< zENdj92xDHcR_|JL+CK#EQR51vM>-0rDO?}XDQ&}K6e`@b=PDgKIsP)xK+9wpb$g3M zFA`|^K0De8_}`uJRG=_IQ+?K{C1om60!FyRYIE|27TQiqv=JqpJZCKq)=2CuL^#M| z4_hWS9kU_AJk&1(g~h0a1h|RD`RwT$$ihpaOYC#-^Td9XbS<_k4TL9yg`6~j02`R; z1oUUMdQGFLHf0#)jW|&lU=5gg=V1sEyo!33g&t`Zy6xzQfwD!EwrzgKtQ(AzY<{)& zTBep3W43Q}Aru!r-9D`C9>~wtcZiRF*)cD`mDry7n>vGrA@cQb_ z@vg_@p|eok+UnQor&Ut(^Y*4)->1jRHQ>ITATIDk-mB&9X7iL2m@rZ=j2v`vcv#|% z%JaGyjD;yGu-2;M?jh6q&Ka0x*sy<}YXb3{>-&txYsaP$Y;3%IQKIL?-YxOM)dS!f z=&M0s1Nd_H8Iphf1h#yKVZHI6_&ttRzbiR>>oz^!_5KFwnh$*P9Gza*FCDZ5PC-R$ zU3uWqw0!N))Gn|7_c?}{5!DPX0p$FD7EjwU?VY#e&>{w2wP(h)u%Gu>)WRf~a~H&9 zWXKh(16@K2hIe^&!mBs8?ZYCAMbC$ z9S~%eWy*Cq!b8^wxko!YfjB#-Mmb>Pbs42iv7khm6y(tqOoD>Y#=z>)V@v}bMQuK5 z5t1d~c=%$3Sgd7N=2|Gazwi58aGAx5kA0GAf9*P5srxF&h}HhRe?fEziIDpq>lJ^ zY`Jo%6MW!HHw|uXCFYlPljgNM2u~#|xCfD<{}A6@0qgfFlqHzF4m7JDb@4=Fw~KiAhIO3H)zDcrTlgcX3Xe3Al+clw(a z*GF{53j6EWD|m$fSyjWaRT&1Mifz$!(O8H8<1$vzVV&s&(UoaFRjMRMB? z7gdi2x|175wGDQrTpm3a1bAyhBR_<4fp4+-`P15rs(+i>1SDs2a@uB)I=02n~hb7c+ z855fU1usIon)r9%Fy{WNhyVjV=A}RaCi9EiI68OM!O^%xv(t*wM<#>$!@+j1EcjC^ z_Spo+UsH`5Ukxbt1Nl!yjkNf( zocJRW)z}P2oVxdE4I?pC@5Y!%|6^G9u&36+;uF>nH?sSGBgSVtIDwd%1t+T|JF7XD zIR~2+&rc3ZGgcmRc5bepY`i?ImTcxi0{?$EGITn?rT@?F4$f6=2PIRy!TYnft%o8# zj|Ec=eLn`fWj#*$n?C8fEqzeB9%r@HO?6*Y9T)e}5;ShHM0OM!dUOuo+5Y$}i}LrK zpO%|je3XoBl{BTu?XloatfD|MI>ii2R)9(q(YqNm`>vNl4K6)IGNa^bd);C8tDycS^Y$cgt_1I5&i z5dy10AzvGIL|$8u@v#g*7ARgat4w1RsK)rWC&xpgez7+u2bHgKk>N;%q3m|X$%^%Z zRl)LvZ(A2rxk>?{Sq%|UL+uWzTLrS#z;WzQgPk``oywkHyE=kStNo`;0K(O5?$a{+ zlPq_u%h2SbdAcFM@cbux>P<`Q^4K#?A5_ONm_sjFP|Go*Pu~!!V;dGuf45Z2G1jyJ z>1X!yI=rt5wSM&X2JW&e9<>?bbpk^ZY5j2e27>m1f&s|Bk#lYsod*LbZAuhB#>Rs( zZcdmRgz1V=ZmRDPnc|9|H0qNZf#{0PJoeHFm2t#w5KqhsuYKr26h+*GiSo;WC{Dcb zXZe8&QT)$F#JOJ@xghsOhPlzVPB2&#?(2B?PLkIHn4O3mmmgLp)J|iJJ0be6+)m@f znIL{w+`UnOnIQv#IM{n22g$g9Z^+p_qtO_@52SOAfA5h$&8d2;y=(AQhR1pXT|E`4 z!2nfHAdA;m)AqAdh0Hn^q5b`TlO5~4C!ugGI*Y!2vk&(|269UUykK1 ADgXcg diff --git a/units/unit9-sim.html b/units/unit9-sim.html index 0e07f11..bd48de4 100644 --- a/units/unit9-sim.html +++ b/units/unit9-sim.html @@ -643,13 +643,13 @@

    Sequent plt.subplot(2, 2, 4) plt.hist(uFromNP, bins=25)
    -
    (array([8., 7., 1., 5., 7., 3., 2., 1., 5., 7., 3., 2., 2., 3., 2., 0., 2.,
    -       6., 4., 7., 6., 5., 4., 3., 5.]), array([0.00396122, 0.04306257, 0.08216392, 0.12126527, 0.16036663,
    -       0.19946798, 0.23856933, 0.27767068, 0.31677204, 0.35587339,
    -       0.39497474, 0.43407609, 0.47317745, 0.5122788 , 0.55138015,
    -       0.5904815 , 0.62958286, 0.66868421, 0.70778556, 0.74688691,
    -       0.78598827, 0.82508962, 0.86419097, 0.90329232, 0.94239368,
    -       0.98149503]), <BarContainer object of 25 artists>)
    +
    (array([5., 9., 1., 4., 2., 6., 4., 4., 5., 3., 3., 2., 2., 4., 5., 8., 1.,
    +       3., 2., 6., 4., 3., 5., 5., 4.]), array([0.00489673, 0.04443965, 0.08398257, 0.12352549, 0.16306841,
    +       0.20261133, 0.24215425, 0.28169717, 0.32124009, 0.36078301,
    +       0.40032593, 0.43986885, 0.47941177, 0.51895469, 0.55849761,
    +       0.59804053, 0.63758345, 0.67712637, 0.71666929, 0.75621221,
    +       0.79575513, 0.83529805, 0.87484097, 0.91438389, 0.95392681,
    +       0.99346973]), <BarContainer object of 25 artists>)
    plt.xlabel("Value"); plt.ylabel("Frequency")
     
    diff --git a/units/unit9-sim.pdf b/units/unit9-sim.pdf
    index c6cb91d1385e377e1871ce36bcbcf58ad9985f6c..5ec4240f30c0939535d24a864e00323be80fa342 100644
    GIT binary patch
    delta 10156
    zcmaiaRZty3*CY}&cyM>;;_mM5?jGFr;%*mr*AU!2I0T2_?h@SLBAb7|eb{eb_F<}~
    zrsiR)PuJI&@2yZrU_sI+jJZb5AK8-aWFVT8J?13VZl^vHWJFHL27}h{iZ>a8RF>BOF}HW~^hPke8k+_SCgXt?t6?+)Kn1N=yXp
    z?Qep2ib7L22%4Os0mH}R_pB`-Lv>O$5iUG`7fRj+amZ*AUKYL@d(0X!>s{2Bl!)hz
    zB9?F(Tg(fVF2`R~wQNAf*R9m!h;79i_8-_VS~2dlh3(vG&rBt=XfdbF6bZvVM2PB-
    zD5{XNzkA-30*>g1WeoHwv%-`zugc{tVx^7+IZ`o;(SFj64O@^7V}W^=%L`3ZLXxm_
    zN75;ha&`XvP*q|uu(t2V&w|A|4A7w(VDD6pxKTd=bJS{{L1zQWqcF6qNX7qhl&~E2
    z=EF$$XrNb0Hzo<+w$cllf2H}ug8y3|t&`YZc2YnToHFYPzNH9e!s)3d%?PYw+;&cgpoZem2Fv
    zuEmV&Apa@UvS5WH0_8KWVSB|w5~CprJXJ!*nSqjtxgonA$cgarMS&KtN5!MB*+_?)
    zsq9dy$^e9v-cEMNsYRUu&IIdl)|;7Z)+B%-ICu@~di3cO2gu81`ithCB8QwKQhaafQEA
    zYbl1|%0pt6Gmpm27xL~cG|P}y4&FB^_||h=9)f}`Hq&*}&QCQnPe`2M(z|QQKlCSl
    zdTQo^lEoj0=1aGkkfFyJwg$t{dv@=yD2T1-7NFR3sI+Qtr-u4~9j4PG`D^viU-0c!PFf=$|@A
    zICJs=KZE)EJY6Zg?;iQ;1b_#T`KbX(rpcz_BM?PEq;`aIFE>!W`S5U#X>V%gst3JM
    z>k)-7CmTTe*Ht3NGhqSy{Ek3AYh!W(_YcF7}(9W4eItnQFfs
    zt&LO9(;u;Hh7kzScIJ@opAv{-sMNo#&ecSAbwlYdUEE{rEqBkj|{
    zyXEj2vr+{5lsN4_;*(;wgADzL(_XEw9wuB`HbLXx(KS%oK;MgP!A|wEb9}6`jC2;`i@cx`kRveB0fp4cO>*;=I<+tBn
    zKbXNUvtoV81z(vY*8Pc;4?B{A+mLM$VvLX%7EP+a7pR{Quhq|`84Xa7-T845NU3b(UPeJ5!`3M$X1dAZgG8kS{?5I1e?#UqgTWo3g~aZ04Q
    z4VrvrUz6mJxWhHwH>6w|jvrIJ;BRIgpFMqnZ!07UqnzuLs4EFLmq!)*2{oc-;&Y?b
    zk~$j(X%8(V_0{g?q!Z*p-g4^iJGDR-KhRMeYT(s`th-zr3qO{}N|m%d^GXphY4ojt
    z!Dyv`Ddl7OS9gOpT+49|*D=q9F^WI8+dgu-$5L;u$0fR~k~HJlhei`HbX0;Vi1Q3p
    zlxd|7voRn~QTzF`=u}vTbjsiBKgB%S`26#F`=35y<~mMWc5%c=-c!;>z|x4i<^fun
    zzzd{kl#CliNX!fyWk?zz+y5;*hV!cSp)6<+J6=FZVty&sXia^zq86So4}E(u^t^Yc
    zNIcn@_~$zZO+aShHI>&*_rgq!dC{WC8_D`O2CN!`q3%i7oE)i_P3j{qO|+Gj3_63Ow?%tD!cxRrlEu$4c2S1-`e4I6}L7uPF@S
    zojgP_R4}h0yh5XUQGJQxbhp-4^NW5nX_1LxQ;rw^E(rTxq%x!Yk|*MDyf$PD9zp8_
    zwmVEoTQ1qZQw`Pq!Kq+4)3ffduHKDjqTE@IB7J(#uR<}1Z-ab-x#S0mx*(w@~6f8<+6kI
    zPQn;Et)xZWZ*ygH=ZKgAxlVY{-D=wPiKGl8Km0$)-l#yz!hJRW(ac=4o*}9Vi3hA{
    zZ7q;|I~wtnZsNIbb#up`sikVDFn21^h^ck5F28n&w2Ilg?Z`694wf(wPB&iMpIW#B
    z^KfvbwdLGYbiKb-sx$HN{d(AE10_wa+j;R$aeTip#jMt(lz8iv5SqFquU*<~udx5T
    zTJ$*Z^~2OII0U?{=2e_V!ZtR^6a$OlchwG-5{UB3%c+)EC?u;bd=MLcWK=|rd48UE
    z3^cqKy*uW5DRamAB_G(LS0a^)str(!;(b@JUtvQlwkd~OaaBPkpp`dOC#AKaM$Uo7
    zTgkkHvOM=e-OJ34g>(TZ{0zjQwSMqJt7bneV+KQg&j^IjtcMB4ymv>=
    zFjRw-XK;G+e+^_V8D~F<{8P8IZPj9ZoUV!r#z`NW@pC$LlB{sKtr!4*Ew4X|{tueR
    zYLJqE)dBKzyEexA#5JLy>L&dnsqs!6@Z(}`5e^on5PKH8TXJ-4K!?{R<(`)?2Udoj
    zIwT>G`QcgbiE?kx_D)O7YlfLb+_+0(%PF$p@#C*s2JYNEdQrhLNaA-lbM;@J;=rqf
    zP@Osovb{v!JD*!^mnPdt=uN=qmcM+j8;#4IrpB(a2i_>j*|?&Zln?PdRb>sH9$%t|
    zc&{f-%#v%MYs1!*GBL>hv&KH!A=3QL5KILWSe@QgvyvsQgtri}jcLIwtsiOb&xRt4
    zHKm6xgE>J5;n_sWB!z^`kPoQHppAoxMRHRZnn-Koq8E!;xQWO~L=3)u
    z`!j|8MKCmhO^a6a=dUktP10z&EDFwIqN3uP=OQZF&PIa>OrpA^6Q#QV-`Ps`!`w<6
    z_V9cu%XGJ5ulPH5410x2$^ADZy9R|>b>}S+(LsCYW2SHZs@4jIT}p^K1r=y~7in#u0Z;Tsg5%clBWiDCX@73dbi-W22@9p5ae8zZagU
    z{!^xFjeqTg9JC@IeC0(|H)kp|D7d`yCXVnF&Ov)UV9Y(Rx{k9X^
    zc2PJotv`EK$8oN~W%Ejn*VJjY!soq13+nQLv)!4H!J58t_<DV-56ZZElTV<(-uvtOGjKkCf;w-rCv5+sHP>vr+C6T6+*mFW<{Uj3)o(~-0P8OYgt^(&j(=}7
    z82PJqpy@yaFw_n2{hnajn29{Yru#k8nioOwefwj>`TSkwoe6yNaBu1z0OXojuq-7{
    z4sDFtUeRV90x3=Ru4pQ+HvsRDU8i{Cg)=sFufXqc_Kk48{&wR~OU7$6)sz*D@RM0C7c
    zgUyx4VLi-S*Fw%Y*=$)GMansvCM8JOzltR-ByhLV|E+!G}P$y8I_l
    zry8fAJVqqz6OZk-^t*-A-_E{SaJSbi=c$(A>N;=rq|yU!1c67X)Df?(-GRS54LJKU
    zG^sSaGdab3K~7gqfx$y&@pD={$J6$pZq}{6Kyf!82Yjj^YC?-BLN?9lAVO9NAsHcd
    zFf5M-H%yu{D$3<#NqdU~!wVey;U7Xii;o$~Rft6ky^5$s1s8y)_gV<`Me#DY5MxEg
    zy)|qdH_t`>ByJG=r;yA$Q8jT8tdTA{DN5!Rk+Y)(!`<(|+>U?(B;4ZV+rOT!DZ%w(
    z7pxBGdkH@7_n#{Bw-Jnw&N5;Ty8GE2V+1X9KbULTD}3TRoQ!DcXp!-t2#J=VpbJR@
    z$&ul%3rPkv;#x(ci4>&2T|pmeOiB~~2rL8CNP~Vw{K&5joV=gpTR-e7zYKj)_o{lV
    zMNRNaLGZib+mIR&70pmGg#6R&CXz?Rl<8
    zESqONCcQew4y&-21+Q?Hg{!cOr?wR5HxdF=<(5KVS!2hDODrR25lS!C$@XJLW?pNE
    zRy8@{aMps*b*fk>hPgvpqBRaK?`#3ZZrPMz_smi2E;-y9vG8DgsAPs2S;L?X_oU{SO
    ziUiS#WQc~hORW~DhRr(TsF=npzZPLMFSF)V$13%`$tLJWslussl7zExOjQL<(UMl0M5?jB%zY{S(?N#A
    zhhrNCpPA4cM18rKYUMWugZ`bT$x=aU@;jd5RuD#^cZBQT8EHPN>5h)PrU5GxwwZ&W
    zu(q6`Ml^+^ET~C`JEp|w@fB_&AoeJnwLi1djOD|s$6%9WBE4YL{<O^*hUb0p
    zw(x4~Brs#n@}Fb%G59AjT>p>6IIXc@N^QSj1)g(gQo4L^!w@T0p)B?ID^)F65p4>R
    z!+`NV=_7geKNAP(ouZ8jZ9g0;W#lZD+bdhW3Oqh=e~-Ko?x2GnX5%_@N46AkL_N<@IQ0G
    z5$@fz7D%XkQ-A8&lQl&>qQG3&GP>|=>WHoQiQQ+|2j=8*`QFvj;bZ8yz`cIQ(mg7E
    zu0)+Pd2HK7*wyK@=TdpbqqQldWVL$8&fWBI$X+m1`*JJbZREGU`1#|eQvle`IMWDZ
    zE1=P%zEuHH(POJj|3;!Oglg=jxs4ux#w^5g&lo#h6)d&>`q+$SWfYc`Hi6hg)of*%
    zHD-216qeN!Di?Lfm|qroeNQe>N3UaMs1}W3&z#&Mv&5&oME(P)Z)TX>A3J7pC>9Z-@1g_d3~MR#BTVzUkEIYt
    z4*@1R|{u>yA5$P&=_bYg*NJs
    zeD39<{Q0S@#VK)}M7#Hcz~=rqn#4xTA_8TjE17s9Mm~~~A3gLjj3pW!v=zT7oT=usHgFL^
    zxadDDx-k03B9{MRQS!FkASvd^)g7jhXIye-YeOj7$~+9z#0TXCw2t5@RD4VWP5@X^
    z`DFM@ctPOtvrmGHE{Po{4+c=5iO;!|B(<64GzUv$+8IwnLY*9L7aOq}w6+*bMGPI|$n%(k(Xs1L;~fDfp}L4o1>hxiIsrqem#aG%rWYX&z`qKPx(ZH=mA^^g{2Ai1&K52W+_&%*xQ(7lc`@^m++o4u#GN(kt5#I5udZxm%_pW6>Vr3y@NGHNs==qyHA}QodSu}sCE6I;
    zc$SEQ5>mV{RkbgNH-ury5;iJgc>5}IV!DZ!^X8QY+FF$&xIc4&v-2jYfYBr`4$H$kwmD&7PdC_1CJ>;%t8_pD^@#b
    zfi8liMoE+nJYWm=W!2gI=P%r^=hyk<)M06WdOC;!7Y
    z$&*+~;`FiOknzdU_%Wj4h@>!;Ur2W)BXFdk70NIb!mwyp2ZQA4Mu=<}NHJk%Dxd@`
    zcUawF4{_OR5QvZIm{OKYa3H=yHLLRl5zs$`s0Oe5Cy9a0Xbko?g@6VPD=3yW{F+*fJghL8rc-^8-VJ?_l7%riKHna4P$Gw>)g_dcAx9<4aTZTZ2S*Zb
    z(LyBhjKhz593OWyphNJXiNML5-G;IOlxVFzp=rHzvTD)cIdHPO_ia*mo(PSwaY@Aa
    zPvyoDSkrE!<&Uu`0)PLvJg+`M87I2^yG5x+NTTGJ1>t45OMHDVtz}2j@Y&^*zk7f
    zxu!SogfEc~lf7t=NQ6?@Yp+@o>4LP3CF%mvpR(Ch{^TWQNU(-8-uE_j9fjf)vA3~U
    z|Lx{d!s5p%NuvdDA3>r&X%*PDSN?vZN>UpQWWKK1bTJAqxf6}IQ7IwgiQ60!@(R6L
    zpuaRLs*QTB8E*RacYd!5nPkh>?BY`eY`pxz-NWkWXRl_(L+L)(v9P2L6Y!IfZlX1I
    z_HX6p!vaa-KSFDmSHJC?>bvir$YJWbS&?
    zwG^rkVw{d02>j)jaX~Z`8f-5jD6$OYQVDZ@OS>WeeGc`#1fU
    z{6|ej4UgW8;43pSq(77H@IGrg!PU>MK6y=&~45H=nq=y)KQ7l;0zRq+XG?QrfW
    zG_auKlb_R#8CO$CH*3U9Da&LWwxFYyXZ=4%O_(`lD~!X`bmpy(KMeswhIwuBgus;e
    zy1u~gkn@sV<>IZ-BQx6GMqFHqLc~Nf$FYlSc{j^D4nw>>6+9XFqst?}tLlWN_x-_H
    z?Of@OBg3(Im0kO?ifQiB8%nLX^2o~kQL%v
    z0q>{0>s4Oqbxr^;N>F!uJoGbGFc;&khYdM>m{b?4gP8QmB#g~U;C+P$t{xXFx|x#k4a-~u}J4xZMEs6vI%
    zqZd_jRYo)B1YeR%%ficvsbUf)Dy^S>zcslj+qG7Ei`Ky@LakdHw{uyv{C#+r!vM$o
    zfzhKhzIUV^o*{TjFyClTGSd5Ei(0i&u?UWbq6Vs+xb^`WJT}9u*u4WpUcSoyT{L?=
    z?e6KkvwZb=ba@3{4&vM#6EU?MUAY3cdU7r#lD*kMRONhrVsBk{u)|hjc4Td4EWP9#6qZUFm{E3!iM69QnNcfX|R8Rl4f`4Q(Rt
    z!Q7mnw~X|{`!+}dHy>Bu4vHsN&wL*g#%DP)O?@%H2rCBvDJJ8gmBOE5|BUhl?=iD5
    zx&;YN*k=Oh2f{%w78|?k4^fJfMk4FI>$iU!AuCm#YQ
    zHrbH9L
    zPdi}2W?3PxEK`aWwsnRI&l`h|tw%z{RFkXNmDA;7flMh72Y94|f^U#W-keqzL{{cj
    z(Sh@s)kb`J5&GyP2TbD1ZD5pz;xtgG)po@
    z4+x0LLeP%Yelv_~{Kq3yM=A+fc_I4fG^HDDA+d`FnGEu|K_s>e8cEO(E>Nb&ytOQ4
    zAmm&Bb3Xf~J=XBuP_#)^?TfDhjR@O9d%iJ*%Bs6eyNM1~a|gBUw3wO2;waA^H;-gV
    zv!b4>O=QAPYdB{Fq{;CEv>dGvPDOfG#$6ywFb&-FKn|ds1_eK+?imUWIS};g0SVBj
    z%o@WZ75qUb1yWKtP_c9a!LgY6XCaFF!2He(&6wl$-c35Ik)<^6P+%+Weq}sdwB{vb
    zr4~qc`K3|Mz%{IG^BvKAlx(H4m-8MK(xVoF_Th4h
    zX<2nt5RrC+kj8Woh6%5AHY|b}Jjt5zsZoV9scm{ikE%5|sI7Ob+fneO(gS|ottsQ&
    z4v@1uk5O?b83f~yQ<#VtMM8(S3;|JFW!uJwKyg=n=z*AreDV~u$gjT!uZwS3hx3DM
    zO6z+>P$u9Zgs=Z%W}8HW1JF}R894=hEz5*tMFIF6Axx9f_aot~kiOyDV&CM8(QnG{
    zC&|5G8~Q-nMYToAzb${6Hup@FHB~SXmZOHaa_=Y?b;MjLBM1koh@YsDVF1U94UF8r
    z=<;bIH6<8lsTlEQ&>hxHF|AnRZ|Ql=T6&ZA9OWeJkb@vf
    z>eyA9LdUGQQeJ)OPAiJY8*}!$CS5fN0*1bfVy{BQ9SK*7c(>%YK@4JgqhoTf8{0iIK!rmUExYWp
    z##&i>_Jl>hcejv`zzCW3{n>(Xb!x1T;I4HbnKUZ(1Xs`d``lj|Ctx7>aP@R`Z_uLl
    zZBEzk=G4`7c+)t*uyzeq`S=R^^t%VEer;{<7H>Y_!|p1<$Ut$udi8B@dVx-fr)hDy
    zEAw=Jbm9@3#G4YB750YHuh+&}-D4H8b0#
    zD7%Pkhg5%f6S_dtSJyJvHjce!DZi4!q|r2t-I}y+_EqyctNwRUEx1;hda7FzLEBID
    zRkkqWDeS(fbZ>Zl*aMQ^>CSmfkItE(=$WB79!
    zPzwnAczjxxB)9Oj&jH#ek?qCjq}p+{9T=Gmpic#D>e|eL3{!3>T;R!E?85PA8lbK1
    zYg%;}vZ2h!EPw-7H#eSBAL*Ep$80xgANUI3h%%l^Yahl^gLAf&`RI2)mDPLOb?1G#
    zYQnLeVenu}Zu^D3Hsw720k2Pp++=A_0wvF`k5tjI%(g*c2~2Y~<><%S^L{a$#0zO8
    zM{%I(a+hi3C;d>_0+mmGVAzTUO4qu
    zkGGu@RC5xV!t84Fff!6CdY4X%iopFcGJ=$RM8LDOv8ohsn)K)CNEf9n>F>DR-Y%x^
    z_2c2hHZMj#;3!sqv+~Um<@{+PxCfQs(xwK?MSrPIRC}m(Vy#+;0@cEOxsn9UW7Hu$}?NJ4$U1ciodU{e&v5KD@2FW^{)_@
    z-(RSnb?LA}SSV&+2&|AK!#&9h^E8cDygJOg(=B~MpjB_SXSjTU!pykifx;7GH)H2D
    z=P~7EF}393VCUtuik^-{)SFne3QC|hyl5q6q
    zxQ~>cw6aj9GD^%xN3~t<<>bJ~^_R@5buL>LVls
    zj7i*uG6?(1ikmf~t3-KU;Km_epJX&U$bvKqi5m>W6P8ww*Y3@>iY_tvhzB25l4x;z~Ey1RYqrerSKxU7%pv1eJuwF|~C
    zDHP$+T&iSoCeD^d@%Ij9S8?o*v%A154Ssi-GJWluid@9LtUat;<1=v3FU;Qr7U$b*
    zGoNd%ga0JpziD15sC5n8WcuD%Kd2S?cF10A1$|a(d;Sn~v15J;bYR_5uX|d1tJr7E
    zjVGH#Uw+C-c(W{t&mGJwH>0X8Er`7vdRKL#2*tt?IDE-uxU2*4(uCytA6Tz`?)lJAo<)5lrhU8nmRtg1Wgt#VeWu+LOpa6mCewF!v8u8I{u3H
    z#>m*g+K8Lkb89E6+f@_zH(9aW2;1~Vh;_j;8fIFF>hZ>vuZL{g)$)g}GiONOgY+lZ
    z9u9pbCLTC^h761%*&X%9PBh1>{dMOJx!wTX_sjKs2;dJX+#mZ)4(p@@L=8;fzoGKS
    z^D>3G9MQhk^hY)x{iYLLU61gzL(q=1Jrtl54!&XXhg05FeZwsHHFJ%dKVtR0pS>M_
    z{hClH(RdH#=dhhYsC`4C7tacM?{M8UL3c3sjeU2x*EN54{I(krYG}`-u@0U!BkBMo
    zZp7|CRb<`B$?+)-A!J7MXRtpPD`C*})_|R4JN4Ge
    z(p!EU=Gj>W>tR)CMgD0yzDRJDw5!m|Cu_V+Jrodrx;kQ)#7|}dZ^uU%rdvmIIJ6n;
    z9CxK##}ZaG8yp-vPLP4}IVOe!g=~1G52ght%E*Non+2r1>jTndw4-qHt$k{}$Mu=C
    zpI@XW?%rt1w%wTj>~4JG{+b{9pApLciQWC%Ihoo03*ni%x_i4?S|PG>aB}jnB2rOF
    IDoY{$FV(V^KL7v#
    
    delta 10221
    zcmai(RZtzuwyuK)hv4q+?(S|OxVyVEVdCyG@!;+R2rzMXcXyW{!R4-f*UQ;er|MK!
    zzl`dB`N!Ym>mHr-0=Mx3SD&8?m~M&5xmY}D*2wAr%SlX$`@+tU9Z`o4YWu`-#tz`QwEsiWtx+Jq)b#i
    zQLOZr@wUfxt5ls&KVD4M&DHmW`~Yo=uoJa%MQ?s_CgTK)bO9Mc5yf$VETOr3KL*dm
    z-~t7gNbYcfi#NjrMHAwXfZwiT$xnoJX7IXEZr;tMD+KGbOS8WY3>q>o&XnLcPi3N1
    zkIE=-c0UXxPo(9EV|9806>hr1g%9o2c{n4{RmH<+0x8jWC`r#LzF=5qi_OwQBLdY#
    zd?FtS&`VD#6VwSUabWd=vg$zlYd
    zg|ytt5J7vzgcT9QhZARXO`z{7_`2Vv14hc|zG+BofDO;jYxPVB}{
    zrOB3yo&xY~-M|B|GfYv9Ap~Lj?eG25e81SxOx}khYX5`8V8|meT1D%W2QUC^Fa?MT
    za3yNeoEYlF3I2V7QRN2rL=yWxWsA8w(08RZ9loH?Z>;W2bNMt+xd-1`U@l%j5P;`j
    zufbAsC|xm^gw(>D_sHc*tu@s8*kETo+|$6;v|Q!SJevcQULiX!$iscvf#y-J1N0C-
    zX#3*6sQYH{)!Kh7XN2vu9|3J49})RIeI77*iSPMV{jM};*u+=Fc{;D+@pOE{>&EW^f%SiF
    z&Z(Wp-@SntxL09@y1(a3)Sg$s#wA)rdDk{Qr8c$KHuW2Heo89q(s>o3&Mp-TOu}_(
    z=ZnTZHnDbS#K(DtL#dOR+{Q1^0|`Sb5EB;}8N$zFpr$kVcV!>uFe85Cm*9cHJB}WC
    zi)XY>Of%$-Fs_l1gUPz(=Y!{88}>S@GHf${L^V+_Tr$xW5y+wsF#f
    zJ{~B7nU659sVIG=L^e@_a!$5v{Aek#x^n6}I1}P!Cd%VzE$)3K6r4$n^;JQCDw<*)
    z2M~corw~W@2>opQt||N{Dwh%Q)0s>86Yv;wsuKL1;-wKjdQeypY;8k`TG%!Ifrh_~XRy^mv>^IHcie1E
    zKrNyT)V|ax^FnzxWa`;2i|
    z*AWU+@qE1|a(l>(7iQ;H2p3h)Z_4Tqznyycb2vjuuWd4471f%>08=2h
    z@%z!Ly6QJM)VfiSC-o=Eg30j=VnfNc{K)oKJ>Nz94qUy;jzjgAx-k$lz}u(A-Q>@j
    z{i*&t*6LiYc_)fq`_vM{uM+U8-;riuD|nC3EryAyPAG%jF3r%NgkX!zFQ-7e4_UopQ
    zs0xTBZmgsUnyoFeS^c%BK+Q={?Q`=c1ZqCFC
    zmIjS^g_fuRPnPNh+()nLSj4;S_ByB+>izhO4*(Fpevp}Woz$GC;O0~CIBjK6}RoAYzy#4DWAepA9o-KEE->8=`9FR
    zs)54JanopG2c}$u;?H4=IPqj)Pdy2BLUlg`jJ_|GV0UjkTZ0m2X1mjTkcDTZ=<
    zHWKu*FsDM+y|U~1R5mx_x?gLKD97}e#62GQ0l8hh2_MnDZ8y!kIN~JlDQQE)G6*_m
    z{+b!UotY@ubhZ%)jC48`2wEV=|3sF-oRM7!E83SoULsP&0@AYsl?;(Hx`ci0BF&pW
    z=e^rSDh1ltoxjOOh#WM3h+Qapfp}I2fPMPz351q>UBDE~v-G
    zYJjhQBAdFY#7{E*>8bbg~GIro%3q?KIbps5Dhz++}B#hQIglog|9&YOE)R}bodp&>za*@biK)xtvzK6|;>?
    zLLt4uab)p9ah5zYPn@yxnU1{iQpB&;`NG?_%kFq{e(~ZbrH?K&J(NviYQ3B)EWtc_
    z3sNX;)+tO2p++z4nEDDP972{{f2kNOcWId@o}X|CFZHphC{+$3MwJN6n9e)iz?mYS
    zHGJ?-X0hse*ak=pUr=}8QhgBSy*o@gu{wkTL(LoDC1AQ}l;b2~p>Ap0l1uCONDcKB
    z2Q)J6XLaf%HEeQU*7u5tG;mh>ufS#M5Yo&xCg0kx~b9ztJX|HyE)Ez7JRO1^|6hB+GPIR@>l3`qjCAg
    z;<-D0#U3xRkdhr5dn;J1psL5w8%Jg`wuna1e9uW9@~9fkF28
    z9-Lph(-DrL8{8-glIu6P&sVb=<5RP?LJ*1@V?36;#y_^AT*^?58G6QBFwM*_+vx}j
    z4qt|y(xZMV^d?!rV(5X7!T41jp9A{qEDZun0kPk8?ya7SEm}7KG3j<)NXOgu+&I_A
    zCh!=9L(Z8QQ?uKN3H_}X2-hs1-H#mqrm0@zpAy6MUnM40&2F6)Gi>{g;cbOY-gG=!
    z9WH3WM%#E(e!kIaS(9C2AD71Gq&u(XdZrHv9*|t(H+6HU9CPWr&Yf=lvXED4&`SAQ
    zTJ!2GZ174?>9jl3l)|ZSR@xr92lQYBk2ZNTf{hN@DCaNt6Bg(5u0=?Ic`Cz=N>;z8
    zcb)xNUe}k?mClm+FMM7PixB0gw`#pejum(v*Wc-M7^&shiQ>7A6+V+Ts?X;2p15n@
    zug+(9F3H+_m^@u9ye``xosll7GL=B%-UiVfXBrW1TABqc>k=7^mi@mJv9ot|l*RQv
    ze~a)C`mJjL#{&RDog{|9yvbX4yo$(!d`2^t5G21sA4)&eO))#`K|Uz8-l%e8%^Ci=
    zdfd|>H@>D_LUZ*8#!z+|@ytvKzEEtlia?6}){my|=l+DdrDOaY+M^)w*@
    zbuaux=(jf#QY1Q0mW{5h4|natx)_G8Gvg>s^Eq=RQ0U1^*fQPC@*&3xxXcUq4)Vdg
    z?VLqJWZcTz)4O^GuK~DWE|SA!0#p5^$a&v81U&XDo?gH1Kx(G^n{in!2p-^nD1te-
    z{f_B#fjijEMHD%620h#!zt(5y(BUZ_m%J7c@H&_?69I!6Fxq*%m7>qXaHsBx;nuxK
    zN}r{@J}N!y(8?fop_s1BAJCJftO%Mgy8nC9x>`%zefBdW`?~6VdiebKiuJzvzSDwl
    zaaoyBWwm3rBeJYX^*WPQ0_Ol;p^6VA~&;pexQcy_xoRRJ;osZiD
    zQzM2qqFGPRcd|J%_7#iN+7%T#tw*}>7uLqmNvxi@72%dgNcfJBv=GX5J7Y*ji1-l_
    zPj3x^WAM76GX~)YZ-<-v&~C1t4BM5oNR@y0bI@@wV5+2*p=5YKu@0}tN0VP}8iFLk
    zWr}^9Wbb@15y8UIJe_L<0yP~`2Z9!YBb`$Rf(#gL?iH2NPmJYr0cWZ+a!Us6R!?6l
    zk~Nn2p`s;X>1&yLd28_fvG+yAtKy*+HQvMRWaF>vIXzAX`1@vcD;$c)qDtgPmD_6V
    z(yH6)2mLg&_I2JZ&gd0T
    zN4a>4QCr#FHBj35*wKL|0XJaX*et>c>^f;;a-6dM+VDavpi7K8j)yS|Qia5XvJj-ErpA%ic!kshhkxlSVjgJjf?@ov?)Z(|`+ys05me5ql-dxxiT?f+~rUI6B*
    zLB>ABLhqYZz{Mt%(1MP(6#8R4?M*Rp`H^$5M3>f2o2+{UL%7DY@+%x24hnhJ%B7r|
    zNg7murw(_iT
    zi!NN0$vcbrP|}b;Soq-xWLPgXsz7TM$*<}%H#`-q?s5$o%w}>5>}TTT(3c*HR`kiy
    z{|JYpEk}D~A0L`kAQ|4u7MMyIYM`tkSza%iahY?(I*T13N~cMks*}8rZMqPYdL=`K
    zq~u0#%z*$$zf!QE7{CcZ=ixXKHYJ5m23bnMlpp2CRkBL#Dh5q3Wsir!@k38Kg(n}H
    z_GiJy%Ux1@rkmr{adkaj#smi++wMDc10!^bPw5x8ZNRvhDf@%caNgieFW&a}tDDYd
    zyQ6o@iCW257XbKdXmfk>VJ_bA{`IT0=Y-wA1&2@nAH?wfFNl%;2Ql~mAjWRTo{V+YNFi8t<$eEvxRVgrf}Zw(oX)
    zE*lY@)(}@sOH~N0#6zJZ70j!GFH8m5lFHMLWI|vw
    z6-1LGsSKSqMm9ZB22D^Dqve)Z#wtO1DDU*1jxU@FOW(XLj$n;)qE+AlUEhSTC~Eb>
    z>20X|51e%M#m=x)vCw83zo_Q2V5%jmG}?v-a-A6SnmcnQU}5_|5xkB_Bu!>>+E%QH
    zW+O3ez4l=>ADM1mcIYHC9Qs{Z4d-AAHL*jYC9|
    z-PuqTHi=!x7&}$0*f0N9tXqSgWHqN?RE7p{5%d%y@T-n~71`
    zE4B=ih{Jj=^28{=XQF?O`gw~y&*Gw9@ylXF-k7>y3%i^h>;n-!Q1E+^IX)g6#z
    zAXiRBQyOb7Q$Eb1rCCGwM@wQaCD4qWLQs)Ah)PKhz1lBU+=?7g5a;9b+5JxVpZR#0
    zv2}^C|MF~K)=IzuTISj0nOba3d3wXqwZS<&CK+PMUH$*U{$?|VXd4Bx|4(H*b=O@9
    zc+mZ~>w4DLvehH)pjKSl{xdo%;7M>@7=OuA{~%9m{1h&OK!R82V7JlBMnFb_
    zwRX=Fw&sLopF^OFbr)BYvTVQ=
    z=`%hAhJs*=FyU(=84Rt9d^FJ3zDZZmHmNX4pG4aHK@}1GTI2JDUUksVglvr;9c5Zs
    zA*j-&7a@Y!yr5_Z5!E9$9QrzqbRncUm{eHYjWhG4Ig+xPlhPDZQ;s(JxyH1r2}^Td
    zPA(2%i{XnYvWo)R629z=e$>wz?4@4mBC=sIV7V)PVWXi{yL)=3uEojK_wJ$&<?e
    z?^fWa5jeWifl842&O)6zMcsfTJFjyJp~1u+TBI&+FCaj*ccy5kF8y-byZ#*kAE;S4
    ziLu@)mYSwiuE=wOL{=SB7;DR(FX53Tlwyt?yWh3(^JO)Trvr~Od1XZA*Z5&2huyt>
    zqrQO2y|W9*)-Go#46wPm7-(9P{#0^`UEj(|
    z#Y2HNaKCE$t06|hpfJ{WNq*07>-FwFcrkWHV7juTS<-?@yrzIClbkGYBjv>gkD!g8
    z!}*}7%WG99)N~+jW>~8fV;H%E9YQEz+z~97X^2#}Daf@;KWh`-w
    z344evw@f-1hu;FSwd+T=&v-DGvrQSRsrldso|qv2E_CP!ViG9W(XDJ6Zn=%)%{F0x
    z&F88egYRDcSjEs$XB2u@?<(`hLi~|?ZLtH^mW67l*JaX)jBm#_1=5qDyk~8YHzr8I
    z${n;h8&8Fe{c~L*&UaweXQsu`iJnq-2bnfvrj%qb4w}KprLahz|P_S
    zYW_l$$(N$Hm3R)}
    ziYiS@;ib5apw7bp|55ujpddx1;G}gjstp
    z46JwuN?Y7L9x>!xAz)IGmZN`*whhS|X~a8Wc1;b=P@&k`Jsno_v=pg`fc>iY^$$i;4cw0g!gkbZ`
    z1F{dV&LR1-Z#LZ9&;zbk?<*_ePIR&YKLts@h0_dyvc8~{Ev)1xqxz84B|Xsnf~Q*Elo49K`c{Pa$zKjVjdC`}3Re_EDTZ$oNdMTJr(>MQ4^Zm9&*MI$gtEon^tuYvSQD{
    zaiT)5g}Jd{|JP4~2Tn9k&K7i*;GMkT)i26u=T;ZI6^Nw>^qqaiyW@6yeqxlQ8=$oi
    zg+94spC|UHy7v0J0zdNDVo9e1Ak+k`Ba9<}Z3xF>lN9vLyno)k(C3AcW#E4QN=`!d
    zO911H>FF1V>8&6V_3I3W)8hIaN!G4o#YbJK8$lOx?Mk~Y;(7SQ3C`MMoG%*OMCi=8
    znlKKva#_C;adrN(#Xlh{&(b=ifC$$LD9%?Q^dX_#*!c!Z(~Ol;q6rmvfr>k$LBIZ#
    z#+*ilgzqtQ^@xQSQ928Sj9-6`C68Cj2S%$EV_@)Utm+>?#Nry2_Nf{Mu^;VNxT6^I
    z7W>+vk7_fg#M0IYEBO4?L6M1tPp}|l#kz=Ay1~NF?#pO!os1V*3YcwQ^(}_ZQr4#C
    z`MyV&5)+dF#<5YQ41PBT0}9Q%o|SauQ6`i^wa2;h)LTj`kA$5HDZxt$<*F&p_Eiv@M`tJ60Inu{v>&w;KFcZb
    znI0={6z>VIRw6vfg5*!f!+cmr{Rk4FD|{S=)=t<69B$(BuBZvq4KT!E&Vk*9a{tvS
    zltij+Y7S05MEDPDhc!U4frb>H&YmT)pB=IX^^HG_I>V$u0h$U<0U7lVSF$m6VN5)M
    zC~ooL({2!6f2eepR7Y8&{No&NF?zR@mSG_N9@8HBCs)v2#-^D%F2+hiGJKDKL9I<9
    z0gUIV^e)K)TDwLhG_a92eVI@Uq>R;Hy{|=`sx@UH?_=kN_>dX?Ta(Bgk65CrY+4W+V<~mzO^ESb9aMEkO
    zTKao>OUV2A0C-IDQwZ4n#GA*Y>@s>Y|i;>%*X
    z*T6;T>GKb++#=vR_h^hM`@hf3mhG*)P9nEKGPmhEBO*5GW$C{VbNJ@Ai=9mYx5~y^
    z1f2?adk;2#egh}oK4Xo+?i7xPFGwTK++6<=|M*W~Kc4JbVN%d5
    zN3%d1bLODoLPe<%DCawt#11gQQemjsz#ci0s%Oh%|d)?oRD$M#h_p3Bp`B9w9;)_o_J99^S7p%(i8v58GAG
    zz95>(GKOj7tOpyuMe?JF=CYec5B8>Nb?^5oI;!z_;+w=OdhM@4;wjMfHlc{$pO(3!
    z-aUXXqQ)o*AwwnaJClZ(KG7Zm*@=d}7&T0-iey2~aknc6l)jjFK@K6MVrvlVJDR4w
    zi){?k6k!j@a5q9-YV=&+CBp4+wCJ~pkblFH)E2(h1+&&kc5C4Usa9-l8+ryq(x;*p
    zb#2CVIx!C#Oh5_rbg+=Y9|o-OCsPGp!w{O1*ThBSan
    zYY$sXmZ&FJKvTIMNvUWbx@ngE?fh4151PYZW3W!BBgshrqZ?IitSa4xjgngT*3|0B>%c
    z$s+)cGapb`$G>&ZSOQGP;{Scj?r>*GvZWAJxogDdnE^T+dQz`YkaFXIfPh0bi)BaK
    z%}U^JiG=!gwiREhcr8s;RK)j_be!(Dn+%HGyFq|U6+zneWYBv6U;U#=KT(axw)CKX
    zWq{0owSQJoz}^!i`L$v5VNDS}=zml|&@K
    zn+?dBEsZ@ad!_KhE!>ha)AxSWRv8V%c0T~XG<5@&s-0|dHCbi;nC~78(A)#{R3PJdK^Eg)i$=&?T8EH(3e|Rvt(?flWBG(CwdfJl%8KY
    zm3S5`mlh}Jb;WJZvHt=kvwd$>nrmc)JhX&1%YR0xO>0(Oa{STytIv6*F>UliKz9KV
    z?`%-Hxk~bKtITk%;DK@<$o&b=i}#C_z-_=Feldwh*WmzDGbH)Y9nJopj^Cf67mjNaFc$V?+c#3J@kPbOK&!sJd)^JQQ
    zmJY|)?S3rn5QZCg#sL}^tfjs0t*{BZfR4!XKRN&uo?R;aK$$;+wL{G-LC4Ny*d{c>
    zA%9omzSiTOJ5$sSmowhqR4smFQ
    zTQv~n4fj9sbp`eUMamFG`fzTj0C+~dt^lmxJ9ba}j)Msekm)
    zjsQfF-l8XA;dqjr2yepddOVTAl9gy07bwG?N$t2Ftw`znIByXl7`yz2Lv?=;0;05T
    z@B<=@_IaL|9&cEA$KT@8e8*i$F2YQ3WSmQj
    zcW?>AirBo}V4W5k$fZntKd_f83j}OWm6|VFbPJsLcJ&i0mCkOv?WKIk7YP_uFm?Wd
    z4*^MQ`r|2*XSR&5*KzF(AetEa^AaRKBS}G3wt;AN%G!B29-tg6mGh%LhmCU6FZ237p*&!g&eKl>AxRLp
    OIr$K%sH9Y+5&j>Yu)ehb
    
    diff --git a/units/unit9-sim_files/figure-html/unnamed-chunk-11-1.png b/units/unit9-sim_files/figure-html/unnamed-chunk-11-1.png
    new file mode 100644
    index 0000000000000000000000000000000000000000..0fdcae23311954dc32f21b40f3da2722aac5ff2f
    GIT binary patch
    literal 30386
    zcmeHw2~?Bkx^)y4tqRBEsaBb4iPBaP1!W$zRB1$^tyPK$Q4u0C1PBo3;IT?u1vDrK
    zVNwLDNK$18GpH3rh9D?YARtqi1B59MlK**=*mLgr*ZuG9{qH&Ju64<3muXGD@69{z
    zXFvPd-%np#S*-tH>jxMNWc*@t}A
    z+X-{*tdFM~(Z|i@+{b=S-Xs^Iht6)@-P*f8zTo5INix*Za{v7UyNTWetxpF&ItLe7
    z34>7yNB=I1>YH%FVEoi@UmpC3pBL>Qb1S+l%fGJTBWR9a&C$w0-&Z%$xXgXA|8&S$X}*)dSn?zS;QW
    zjXNi4la|?e>R3%u$$X3vj6{=rFWR6KEV8|`(Up}j$S#NQC4QgG7|?=dw*1q
    ze0gWDNKH0DuivsS^6O(!L?mkO9mmM3d@mwC=EE?>gXq)g$7s53Z0RS%CYu=e!2ENQRD=RCNxkdVSpQbYy#4e=(ab9hS%8D-Y`c6|m
    zGdFj?ySsa8zLNAazYG)E?d&HUcHvgp1&LdheI$LGvU=&=|C&DVU+oYo=u-~pz3H-V
    zEBq6uN92UETG?!xE4^SxMpEF#9o9(+O-)S+q4R?U6BSDQHdn@|IDMI9u7sIaJlR`io9AZh;Q#K551Gs@9Hv{sD=DBq
    zt|rk)Z*g%^{mJ_a(J;k|dG72OMsVLw9F^@+x0_r)cKK8bOO=R9Z%c2G0BG%FXKx#~9oux;<@dY;1$=`{ihf;an4mM7zHK^b1Hy54ul2(ro5&)Oy1)A}h-_+xg2Tbhg85=+9xZ-4#m2nzme5{_CjF
    zed4(&vHm%}>B2}hj=I;e;(ltXW4%{n`Z0EoY?+5Wy{eDZx%gF%|3L>Iz+5ozfF&IP}Ywcf)?uaWDQq-
    z13fj)6N`8P0gIC47-ga>o@a;+yZL&wS}ZR0b<1$Ro{=IP>37o{CoZR!tW@yoXr_4Q
    zFyY?CYnJ)zQRW+ypImr-4_j5M*O>sbDEPPLJNsNSH*O?R6)~pon
    z{QSJbZ17oWyHi%d*KO>5PXq=9@#9#*J4oER5X&b0$(QQU!ajo%BfpW4rVPqyy6#)`
    zleC>)-DtYt>Z&AKB^%?*4R>I|UvgfYpT<#>g}oZEW{7k1nSIrGXr(9;ME&sLDYjkKrbdY_l6UoFsU)2k}Y-
    z;gq>>J>}WwYN9>3$)}2;dsI|Z7L3iLm-M{w=P)wAbhd<;T$#sYa?RD|>u)rFq~~#Z
    zcga-4Y0BcXZGpE_=)^UAuXwRcJX^T8I#!Ey+}h+ewm>*p9bPOOIM0EJ59^7SVCr5X
    z$gya8y!Dl*1Nv?o^xtu8aMR@vT2l;j`p^po?0hS^d;tKfOvWOyi
    zX@e9_j$u@{!>JW8{#3rPwKWc{`KNzf8FbxX@P4$ncyX4{u3siw;a#-_a{hlw3>Q)DTF5UjSzTN{7m}g2r*Cl!HR;SpKcLmeK%>+2O3%E%^+T^kJ
    z_K?|*IDM~fcCoqiD`9#&Fg$1B+(ev7kY63Zs06<@mjG4}Cz@Qg&@nhTSQTw;8b_oo
    zumh%ETae(C;#%V8zdiZr`^Wg<+W7sh4Dw&1l(;ssgM38E*GrGzUt3)qz@PD_yt?(7
    z!NhMHrJ$p~rs~!0out_gGMCmVbT%|JyjsF(dF-g;(s2JBQMM$z#M~|A=5H?S^RC(+
    zyG~dPC)_vf>!oci7jFm)znhlk)a4HiPR-va`w3!ojN)^X+4ksOt!;BTjaEuG4alQy
    zw*g#kh+8ZDjQ+kgnt^ao7SP;n`tFz;6kS}@lG=%t*sCP+~5+4M762?
    z?rk||c`|IR@3}v|)|r%}=an*_EEXh>v^-X?b=I-VbE|?8^XAv;aWl*oUC^|m)9DEm
    zkwCrFBfGWc*_E}VYE32<7qiQjn#m{){E9TM|RhFj9I0b(l_!~D)v4ORz;5@bWhmBVL
    zmQ5|CAwg&bVygh~>YRP{1M|F!^g<~L0J2%e{2B)quf0Gg79Sb_amG!o0k4>UwYb7k
    ztK?Rq5m_NHw3%*io9}s6G*@Z11Gl?$u6r9p+GLD!;EJw}1n*a$3EpIw-sVxn3${gm
    z?z--6w@OP(g=6Hh>dC%pNz^mT2_jN^YjjvOhc?^iR4R9Lbo?!rRps(vxhDYDF6fVU
    z1&|-rE8r%xVl?e0g?tvb(oE!KI`NZrZ=})KM=h-8(wSW5l#XnQ@0Y|%7yMxP$gBE)
    ztY|!5?ob&rMoPA{v9YPRzQw`3^VhTH^pcsj(?O%Tmm>}MH9BZT6S{x<>G>(yMf7u4
    zmlvV!g3nJD8%LSWUnndp%7&qLgE+}T+3U+lGtpdvSH+fsx$bMEiY7%FCGF1It(*<2x?Q6~o
    zB11L{ae;w>4NXnDuZ#fQp27R<3iZrkmT#jk%v%HUg-MpOC*Gt^CrsAqKmTdz-BKT5
    zFiU-~)CXuMUz!L1;|4W4_hR`y>KMd_EtqVwzb7KTAP{Zar_+}-~ifSCp!jyA+}#2b{*iC%PICg
    zsoov*oHX3bV9ietlUOFR=f~O$3KqtSbiBHMJIl!R=qU8@^K%9m!U&>F552W53G^*2
    zEPQhD?L$PSw75zSbWFGc`ESrTX19Gj@VR{tGLl{)%CHkKS^=+gVUkRlCF7_IyfQQV
    zd_(Nj601MN=uS^aZx;4UM%?J%*+;7dvP@&A4gDw;XtZrWYMCc-A7;{DO*i9{52xNt
    z7ENUs3_r?tAeUxl?!*hbH%DWuV>GM(x^nGI5RdYY#E+W?+=64Ma!pNnRMYdv;ZjjF
    zu!|Us{-@WSqe$}Of7-t_;xwRbIWvYsN#IM;)a)#w^@$UXT31(R(oLlP8|GJ~-$RD4
    z%zpx?r>UtaSJ_h;r8YSoWis8N;m{wXZ{Ld~7Mg6_>_
    zcMv@uaoF*D5`r;RG&3{*XFsfwtF&W!s5ERH9UV3Al*KTfUvC7K8e$0tkAl7!7*C+m
    zx5KZK5VypTVY`CG88%7~@L6V*mwdo4*nq|`a7
    ziVOg)_8WBy>TWR1{+=lIX@^YRs>looknZmXVsTYSZeE^30883q<(I5ska5vF`g)_S
    zt*FBJ<}%gE>BbCeYwAc_?oPm|mm}G~o_(&`Ed4xY-C>MsIz(z`@kXTnnoB(zN&e^R
    zy&&^9S3Yrli_X;&RoPHnfT1`GLmB{hJ{qf$;Q8zZBYD5;$u>4S8Vh)sm(mjod&v%*
    z>D*JrbjSh>D8VF0FNM#Rk9(jrN=Z^@0g14?OgMRoFq;Idt+|6^d%jlz;QnYVthX91
    zpo2{+cHv8ntxcxCqxQn&Q=62f5z6mcU^um(Wh79oLxzqEM6+edxvtI5kZBkk9gmOhv6TXW3yL>{{!xL@W97OdmRt=vy#OzAr8{^
    zr$l>5N!3
    zZv`ml0l6s@*`c_eBoc7B$)dqzZHR<6%zYr_SEZu{hXRcMxsu{S
    zr&-31ZJ>yo5VuDW39(fu*7$@aCrbr?v<4VK#Lvl9Y5{Ne>6nc4?h>yqe?E>F)vmcj
    zASr*WDR6D31CEY*d0Kk)^1C>IMG%5$RpFD?3NI-JpDF7<1({s{ZnJ4=Y1;uv$7wqq
    z0a}xPSxtE0PqI5+JugAQ2rB~E*nuk*_GzKRcVRys(z*b*9cKB6X%1Hl)z|CrIX+gz
    zD=nTa;i*g(mz0!90G?9aNg2MAOE3xx?;6ZVt^=Mo8e6GCK20y4e07#J4b-9$0?6u7
    z#?WMq7&mNGIv1;{akC;UacRmtOJLz*S2$(E1?P=M!OX@rva5M4YvmHRnOw#u`8d!;JMq$!Ulc)HRYEfu_3vQaD-rHNj**O@Reirg%n9i=CJ`8
    zHsxH$2v;F)``YfpPbDGWX#nW9#r_lQ!f?1F7sB8p)T7C!CL4Dg^|x!5g}Vw619$+%
    zk0NN{fQ8u+NhSj!s~_nTAW(7Q)+Js}>dbYu*|hug9W&8H6pngLtJMF?_ZPSb;)ra|
    zch3-J#Z7`D#ndupR~6tq`4Aw(dH)u+GAjirY(vnT4wSMngaZM9{IZ>@V~oW@z5teO
    zf^~5B#)6(mBPuo{!TShs%8ZhnJ}9OE`y)XD;`wGBU<`_f?RiH)jd{w>P6sHVT@J9r~8*8^{-KHQYW?pZ`g
    z0<;PsBt7O)D(G+bW4jBUZ7%&4@0tZ!e?S1O`CdK496S)gCm9ZjkP}A@I-X(G!SIzH
    z3t{G$jX+MwH4_^+R$Q;@6fbrnk`$3q*#Z3iN1#V%aspk`kH%3!hq<56Ww9VzAN~i~
    z;JGz2sP+&gf*~yrq!nQjFP_HR{(9!1m8qZ?((K&1o+t^btyB%#1;EcAj>Juxr}mBb
    zdgQ;miZqCj3KmpZhf-Ir-E9CXJ>l}x=43THy1Ze6I&kEG#*a~h1=aXEKxN^^m48(8`xN4U#4Bqz@4!LMJRebb&*KRID{)l79z5Un
    zAJ%MK70LAIFa$=F%?>Z!E1fi5DF95%fmGb6v$q7j!Ex0R&H+W$A%#
    z23zIID9LN~0^~ZR+B}j8(pZURxnxZ6Oxg$}aGom
    z`S{033IR9NGh1k7QL)YTWc$+jw~WY_?GRro>xY6$1O^-?wxTlY&R)mAmwR`X*CiSu
    zF4d{;)omP7r5Ik@tfl%`K&XwsjRdY$C9@o(fqXU5csP!#7&?A=aA4r0$J4Tv6c4i-
    z7HIw)5w@lVL)glqZnt~_fzClZtdF3Y
    z*Pxz<%nbYkn?cDtLmP(xr$~?p!<~acwq2J<=EQBr2k(Vs)M>~h8UE=<0_;wouvBDy
    zB-^pty{jRg0-=J<7^weCB6&kYTlYIqjQ|-n_3J4{Th98jSGteS56fjJVefF
    z-&hcLDE&5B`9;x-?ae36pI-m)O?K6%AAi332fcM4kQPPbmC84^oT=F2?9b})s!JfP
    zZ{`M;b;eH4tmjV^(!cmUJ$B`H!P*CZtMdKZuKc&y{cjTQKP}%Q-MtTNTX9DEF6<9n
    zu>I#FX;k1%`{sukEJ(&DzD~6ubd_Iv@bKZ06DLk=RZ{AlP>$Yl?BW)p9UfSsZX?PU
    zvimXW8AY4xH=00smcc;E<-#Bt@XLQLr5GySl8F%j8LQ*RkE3$Ln)v-YljnZZpkkwpC%-}h
    z$>Nzd*HKlc0!&{5!Z6Bm7?Ee%sW0M~&dycTiWUOg3Qboy>Nd4-z#yXMF7_+&&g
    z0|jP9J^$|6m4cy%nS}wl(r=d2;hUEpK75F@0dJa`cHn@0e?7KWwVhM`*DLN
    zv-y#2W5`-rB<8nfxu@a`=zVh9XMUEYv(E&r=u&<7;K65ToT#N5AbuoGk
    z46f+%7>BQ%2Z4?|r?b610Rcw|@
    z@Qwtx{dI}7OnTwx@Si*?R$eXGv(8K`r2GFYz3+juE4p-{?h;>xRPiTysN!!^r*)FC
    z!SVE?F3oc=sC8+v4)LJ)&8c-o0fTTdwyXwnAWihgGE~aJVa8S=aQ6A_BPf$$49Rrh
    z%MYRG^Wf!GPJk0@Zd$0uL7(pjd5tx!GP0J8U2cJ_koz4Sf>pShosCWGk+g>XMOa?e
    z(8yrosBFIbJT&bnk+v6uF%18m)BJDS6W)`gOBY1F{9oHwm--NcN%{Xgo0j@;sSnRc
    zH@ZIpY5v92rt9Q)pWohfD(6!;KtoWK)pb=9LpZXJ!p08AaYIW>iwh|W0`u6@-Z%w=
    z-WzY;yg5JK^;9q(ZkAB3Rb~u4=5S7c>~~I1509;!`jTb3;HF?Wkg_|O0A<7v7!1}E
    zr|P`9X`mpSpggPqI0Wy`kl8e-jj3*y#rh-j#moJTF1ossfVo7OcYcJkkr-#j?ZM`i>MAAXKdTOAa}0N`*=4w0W@0Kou?b8VkB1v%iCqCXMZS$GUJ22$S&_B8Drcl4#R_wuF05-ET%~jSt
    zej$JX{Ur)R;6ZV~lhT?I_OdgP(H|+Ai-keUN?rrG8;v!;x(+86EwYO}B*+I3FwU>`
    zF;uMS^6Y?WE~duCY&y22ttszF
    z`St-82a_T%pkrn%W8tlRFoCRf$>GsNkcQ9fpCvyeO!Y9}2WVFi^*q$evx~<-8r(Uw
    z*9aucC)_zqkX~q-;$>>oPcLKI*I!jsOX|&H4TeDN7h9;@73&cApl=3*m~HzjZ)`I+
    zEIlCWq1ZPW0N9mO{JSpUsQvx@Fb$v~pa5Y8${KmV4gi`bRH5gzht9nVhnqu=`eH!{
    zyeALFOD_{iu55yB-!jQ-~I64gM9%|Aats<699NN2tt{`oh>bQ
    zZmL`Cz#%y+qM7X?C_p6kZ7!QTgeu&wUcGvhrmG16iv<(eyC*^qRbJvIo9xVzp*ksX
    zaZ2`Umai#ypL=CfN(FBVW)?>e)Q*~^$;JWR!1p)MC*&1X6O4GpIxs~MRgUI*7JbY9
    zi{CldCB%g;3g6GJJeJ)7a;mw40e0=CT`%=YCnAI|@WMkNwjuX*@IuLv`rSW*0)0QI
    zTrD9qB5TPC99s_gySONwS~$&7IUzxOPt2PYwrFD|@)(LBmNGaVENv;~C!60bpv-?K
    z>i!8K;lFtQJR=*3KM=GowQva-EKM6&H~)IRXld#F!_w;-&4>E?hWh#&P&m62Fu*H$
    z)A38F(jJN1#E}(=;Nydi`~_bCVtx%eGWz6*28Q@NfSVX=4dy>6e*O1?fziHsO3NF|={Z`?A`BmGi!{%LDJ--f80uMf^Bw6h+vBS};Eqka=4TG;{@G
    z(E_REY`CN%3KbEfg5xcF&KwBuA7V3!etsIXx^(ok@QD$dp
    zW0MR$)Nfjr{~@H?t*OeSn!WB0I-XtSNPUQ-qD)+m0gXn>h7x+Rv4ZpnsSjFHPDI{T
    zF-!`F+!?|Q7)@VD23`pOxPnOR+aNlpilC*t->7sjd~H5`paRq4peVE6l?x~~vrN##
    z0owN=!cDokxi~6RU*!Z{2iTn}n1*_EpSDx)e^jajrfo3|_OBj=+@}|P8%)_TfI}BAUZkEc
    zV%)MX_V=Qm2O|m|m3d8d@3}9lNB6rn`OMErzi_}7rj9ZoQydk<5&5;QXlNkN(7+*h
    zwcEB5CV#y*duO1moueDV?7SLq8ruVgKP*SJ>Y2@G;vGA742J-s72X^eT9l5(Rzcy-
    zouQ#2t7Mrj^wVpY_Mf#v1}!y-av>8hHfIzR1YBF7#m8%dIt7cO2Pj|fq)*^Ml<3x;
    z|G0JmW+Q2=fa@%n?88w(R8V5yBfC~fG+gW5^KgP>wY6pwGVb2phMI#!xTq8#*wE^G
    zhpthvP;pcpwoES1KiJXH5r*R16DMv%!Ik@d5fZ{cWtkQVBGH`LZW+^y+!hhmr4dUF
    zP|(HiQ4{OqyupHs3uXYk2Cj@0zjaqKT!meKT?^6Z9i%oua!zkh3A=ewl1)RjeTUv0
    zy|d@8d5;z-av*+?*j(u{kh)~D8ZXiYUf3?YUp8uio0s`3No=b;pRGYQ1ju{@iE7jj
    zUw;WA3=$`R+xJUq8-RB*x)*5!4At%NtI
    zQ-fRyvJV~?tUDup`#+c=SZW~*(GpaV%+tU8Q?q$X=-QnBZ
    z2gnEUEa65Ic6&VxVhD!MB)^$(+ri=;4~6%K}n5I|8ZHnKiH08o&zZx
    zD)*uG!fEV}SR#~BpBsON4pZa=$qkcJ_=*}l>WZ(1RORb2gLJFJcnBeO1DGp-{Hr(U
    zLeul|bWz?9SqMpMK#*<|N$WC9^rXKi&}sBUMBwG2iYdBzPS@9I^E_R`5=d(lYkGkETrFlBmVlEe77_4@aqX
    z4DE-}+_LY&*VNt_Hj&sor)WBN=493>p!)i59S4rJF$BuF{Xr$b-Y+k0SN+23@~k^%
    zlu(Hev`^F&
    zN+zqcZ(H9PB!M+176%yRTi_KtCiEz>E$)DwWQxQ?5R=!=0n63AfYBc}s8g}X8v^8U
    z@$do$_Wa8mFj|zN9*ruX+KbH{iO5$|m&cx(X_Oe|`bAX7=<1fVby%isW;dCJ)BDRP
    z7)(iYS$KV-&}0RsFx*Veucm)J9q?a9ohGKN7Up)7cTgiin3o2)q|YyC`FOfPn^9=c
    zIu%2oa5|flnQ$q|b)d|57E0*CI2E1dF#FUftAZsqxe%|6<_L;cPW_JG4J&6Vcn<|K
    z$f_N%a-2N^vlp`MAYiU_)hna$D@O;E*-By{f_e3GRWU@q&<}CvFgqucW{WjYfqG*`
    zlvpuE198*UJ9CpkT)a-zI6r9fijq5vt4%JKtTqY%yFbx;b0X(jZEY>eic@=0sA`pH
    z2eAs^RAzf9AYO=`kg*kZi@xWXqUEFM1}K3I6@S^atYw^ip;W)JUBMs}U>A#FJrav5
    zQVAa!`;YsHJC+;${jJ{LT6)X&YeszcWIb+LGVwUrG)&r-B3mt8#_zzN*v#Iqh6rlT
    z=i-Qn2(}9Rdf5`rIfU_e0sd=A*Z>JPjZACfBoht1$S*X{(ePzi1}Nj6)g&w4lLU|w
    zXJe+rG9lCv77Zc%!A`cY5@;9zFPf>xO6Dhyqm?|YP8cZ7-*AE0k<9f}`j
    zl$lx^Pkc%39m!^S@Q%xVOW-yLZJ&oLvTl8LoQeevuoa|5RKX${>1Du~>i{g&SV+xm
    z$AKe541`l|6LSzqe|O>p%DNyuj&CF?D??cR_=E2d{Q&b)7n!Xm!6tm#uhmKO(%krS
    zMb;KV!fPkihtm6gYEd@kaKPFZjGOO2e!LfL%Or>bkX=|yx5zYP9%;&vD`9wrnz6qp
    zGVO&h5~<;l85pzB`OPoiNSK0Y{(E6+OXsu%U(s>4wGXa=`ez2ZC27f(2GfG2oPO@0Y%c4Hzq=4+?V?L!K4LQ>+4a{5t&LyU!(lhh?I9l#|(F66mu^LAm9L
    z8Eo!G?z;)9@$ut)`sDgko?@i1!e84^mdpejCSFX|Lue>$9vy|<2A
    zRcSKYpS(3jm5LJ6I5Q|<@P^*>^H*qji~Q%Rk=SO;VbzU9rdnduuOYLY6pAT`$$<>n
    zuUVD{$UtI{hqgIX_W@PRM*uy{YW~^Ji2MoKrnOjSX1@5R1LjGzvYhuCUM1cj}F|@eajg9IJE-M)RQqA?!eKs@tXz?rV
    z
    z3V+=R%^C>)!95570W#r0)g7?52}qeQ@i8%5^r==XQv>xlq8k&nle2Gf2Ef6{?Wbel
    zq4EXNIw3Q9m~9601F6}dCPO8Zc8jbdMi-%Lk;?~S%TPXgergc8Tf;c=AzP)lxG>8C
    zLc$xk6Q}1_*Cz_zoQZ9GyWkZKF6?gn;sqI>=obJyk%6E$+Aj2Pn{7fweW-n|nx2=g
    z-@Om&M`0d(BY>)Nd)O^KrbRFrz5~y1>l4)PsNX@;*mnb45tdacV~U&0=N)%>
    zQ6bzD#`=3`F~B=Rm$zRhK{5A9R22UT{E6&B-NE@kah`jKpgwX3wJ#!(wKMFly#8oT
    zGH0UIYf5VTmEe2ZO{``2bi@i82nsY?wjrzP0E?
    zc~Um&7?@4xM3qN^uHf%-`gMm}yzEQV2DdKXGxCXo>0DO~8y!mF6?cGOrk)2?0tyaJ
    zmGr~L;8G9*R-)t!s+gAB`f6v{(1Ve{rR}YX2|7!H;X49nBDEJrt+vp2*spCqWd*J7
    z9wdF8rv_eUIW)qV2m8!+7qRVB*JVX#NRQ*%IF+-giyp?W;h0%+rZfN1B{;F
    zA4b_zL8ZjbQUfTd0Zl-oDk|`Y%B(k^k&KiEHBs5rc{brSP_KwLM{UXxOta%1`3t&P
    zYXWJ|QCYAIW(4)*E$bzzmOuh$y$XVK_
    zcX%K!AtKQNbJe~agc1BvBE_i6j&WeCSttx@%XzQ{Yw6Dg=I^V?rp$X`bV=s}nT=}z
    zkcFzmz-5YzR#h)N1mc(;L*Z3F>dhQ$Kc6TpFF09WY4Dh#_ogf;vO^X6Mcj{Iib6Ze
    zIwm%9RZ1G+%%R!kUOWLAezUZKf`YyUkVY6GV>Hsp<4R{ZFjcm0ZZRdOPaTD`NgZRZ
    z$b?xc9PJdjSF7z2!JnFPtJYnZ)n}BA>7XTa0*p%j7QWsimzalcT
    zMQ-}Q31@}6BPlqjkH7$9FaZu9G?6zAwbrL2C?EZ|n$Asr>VE;P~$VT-?&zH_EKy_0WLXrW6vO7PbG_F-hgAtI$c
    zvhH!o1Rfs$pb(*$*0N>6D+uk|poeY`9L_@W4LFnDg}2wQOo;e(*i+ZIon`h!T!D+VuAUz3o_k?u?$oX96z;93ja*;M
    zPvwO613BT5XevMD44eG4z7o3U4W
    zTDUtvWWeeZaFZdE2i0mke}|MF^eCJdHcAIo+&)l{0I-i1$|AzrMf2{LI-m|Ur-S1UM-`uNtre?(eL_KPI~5SPo#JM!DesWXY|U}
    z7SrW>rap#5s3lmz+yb$aun#YC8iT+Z;Eqq*ob-7&s7t%eVrWG`E1mMt+qciBt^L44
    z2bAh*G8ebY6-ELm9|d>l3xZFnwX@IqQND>yEHkOHQ^P}svtzrf7filmgx5MO<7mjh
    z=Dc-9{0T&c*io}Ue#imyl!)62RwMdA8`!mX!Uhikjs^|}
    z2|XXjFQW7RA@pi^9I_7K?74qm{Xe=@tz|E4C2%`?bIeuMuBY;9HBf##3C{$&4}=)f
    z8bWn9vlt#}lobl(2>q>>DLhFCfwNjx5~@KtCHhsu{z1;l8){X4f`Z&nP$`1x{OhQG
    zz~VT50&`J(5@2%)?jzq@eYUNE25d6X)g#*2LsjLS2;n!TY-7q)xkPLD%ibKkywy{|3Q
    zYSFO^l@v&Pi6S&Adb$)GOXZuQh;D9ft@!^O8(_8qz{7Lo$nDcgx{^eP(4DVnr`
    z#SnAO2=$lU7?iwJwiRo6>3%^NeziE@HuA}bDePkj`j4sS!^=>C#K}s8j8D;Bsv%8j
    ziw;|wqvFbia32c#pfo3Pfz
    z&W2Zc4DAuXl+Y2eRU3lh;#csbD0Xmh_J%wjxENiem5CRo29F~;*)F6Vbd7d&8U$Rl
    z9x)0aK%$ywQ2EO@sh?ZM?jF5IkS~g%w?)CDlin(bN4T&#!JXEau=^>Y?VM&fyO{ou
    zvNsso*o;{a9GeF=r`QZOGw>3N0T^`wLes{OokaB64D4>gxA?^qyYsvLiuS9b*fobT
    zL2e5PT0YtUC1(G@f=NW~z}i75Tp|PpY6a{&(SJe(NZ?9zoPb%u6A&t(hZn?w6&6bG
    z!c0{5sXu4{)C37a!Mxy;oi&4n{O;iQYREa
    zag3hBh4y)mlIbQ8eWnADZ6%g1O1y!PNmEGDR-w4?q8Xa!5Ml)k+zmg)@pXm+4Dk!-
    z=bog%Gmw)
    kv2^blz<+VASlw{x$1Gl
    z$<@Qj{@PDoHg4|rt}gO23Nmtsf4c7Bal>6%R@V9N2gta(*~uOl*?A2vvhl{b%kCJA
    zXb}2Y6-J%5!C(SXHGe*7cq@K_jq@}2yv^W}ya@{BEvXk|%A0na(TLEtr|vPju=9-C
    zsYjcB4d&aFC@=M^{O3#LC1DMkh{h_>6K=!fBct!@u3mWZlYH7(|~
    z;q`d`O3wK%8T~`!>c;>4{?9!8XC3@++XroPi))kBObeZAHp`Yhln|1)J)e7#tO-R-b26C+FNH-(=}{
    zc72SzGO_MuWEn1PpY+$Q}NYw0$
    zHX{hT6kiSMu(&mFx7TmBfyr4UC1vJRC`X%nUH*vmu77;gnH()`8kK8Rb&(~H27B!i
    z(aX6f@_CQqx^rurR=Ixp@P|)NPk&TfrZMgI2Cx*AR
    z)s>f@yz-qNGrRdwK$8%zr>^AP?_+1%5F1&(*ssFyqp8e>C$S`gMBL61hMKiy8&Y-R
    ze2-z#1dV8~9>E;*of|rsb_%SD@aprSC1?`tTT*S>UtXq*$=fx?ZxL2FtU2>coPE8hVJkYYR0|_2
    zQHuLx`OC)#;SyEic)y^k*7o+{k!NAL1l5I3E43sMbllvwRRy1AN=Kdsi>WEUe*HQ<
    zE3083NIX$nrbI$JT82IqA9={Q(414@yy11I6OJ2Wm#A4{mfibd6`Y*BdTrCMxS1Ns
    z`~dc{gsJgunag<+ioG{Z@nKn0y|)HK=!y9-MTNALNuq1opx)ynyEU78dwUm@%-cFU
    zI|r=j-LLz0ZARnhHfW)2PoBxh$Z#7=PaBSRZ>)jYk+*Mt9xS3H2Me);z0l*{W@Jo#
    z-C
    z7SW~=4X#PhY&G^fQQwg#B{Q{$!X>tD-FmtB#sD89Y}fG&IurJRvZ&+AVi7jy(5_v(
    zl>H8y7QQsKLNla+TibMUELgtrIO1+haByJ%c8v|`>o)HwP^0Wn
    zDe_((O_S^@UG4Gd>1lfCOZ>qvFi!v)wXxl3;h4n3{Uu!o%rk+l5OQr0F)`C@cM&>Um2{Dl?TN
    zsj(7u_%flx0+&T6_nd9uElBN7NuAgT&G7xN*~}NuLTgp}r$f<}khiU`=BzN1U@^3I
    zbkIUoCadQ=ER#}GQzdGQNy}~UhHTTw6da@6{iA>}Z*#$0jfRmA&D6<~Thk5lt!oJN
    zaf-5fYV(GKAuw5WO;1i8;;5lX3+$JQtfRW&^?prA@MzX7;c`}p9uw4P%l^Wr@|u;Q
    zqH)HS{;yly&jjy$l;cl#WG+=<`$umITKe5|DVwhJqGV=d)E@6J3$K$QzSN$b2(7HF
    zY;9}XrwOmVyIk)e+(P5&-X^6(8k?_kSWKU(w;N`w_lEJzNWQI3Y+RhY*F>j?SLanF
    zYJ`E?z+FLGE_;zey@WPS|6;`=H8swqHYZ>{^VqRt*6)7X7^8;sk|`M?h~Or2i35kw
    zat=()T^+Hi#?=M=JG*(?NU+XB_$Mat-47b*)&1Asd7D+?^EW>k$wbdKodFHVB>wvUH
    zK`cpOvzTSf9m(L@b&|SuO8NUYKbgo~djZ|{q_w-Yqq=vGYyOq#T5~a?N87Srz}xLU
    z1r@tH>ecp~%IMR1q!xyEL}Oh4hg(T6b*ZF$-2-FWerLVsW-^PP>XR8?!R4I~(qR=%
    z^reyp=Lg0^QD-e*Oa&)1z5>oW8sN1Q#V@sI9))qaWomQ{emm*Ssk-)|F5_6=D>2qcSIM
    znrNly-?d#!4ElPPbMf+yZjJFrbI;Nm6OJCF)w?lOaAIGt`tU%cR*CK8w!YA0cf0eo
    zZSE9@c}d*!SWY604}&3+AC6_1dAC1`IsRIP;(NmoWxCYLW{g<#cb04&u{bs{uky!G_aQHmgqZ{?zhrIX-rd}Pp^+Xeh6Ij
    z!ALFmo1T-ym}5zs77iuiCTnb>&7-#KsUBUK$k)5KMTh_>YWT5LXib^V>`bpa*;Dz9
    zzrTMEjXsup{nOJ;0rO~V7nF{()mC{e!1vS!*8ul0a?(+&Gl{pp{E=XRpKmkHzcu~k
    zutnJk#r|7XE~TTXEUG2P#x)hKe9W3yz<9kM{@6YXI1Za{N1mKf$|+wk`O|+3Qkd?FqBZ)sN_?Vefi0J-9J$Sh*~&tgQMTLdtvg)|)fwQ+P~SAo5O
    zrOpQQ-STf3<#W=PUj0TlMY)G2Exqf;Mv_p8#CxI6FHdbY$jMv+r`L$LQo^
    zDRk1@W1WZVpD<@>G{DR{gdq~Uh#NIFHl`bf2YA(!=-ORyP&?`{j7)n*k2|MVve4}~
    zJF15-Jv)2d$!TJ${Iz4p!u9j$Qk{ty*&^H6KyluTY{n$
    zOZ-MO#3t#jRYjnV~$+?TaSC(4&XBN02Z!_cTIK`F;rZPt0ZZu@bqFK&@>Wn=?{U_K7by1$9q0
    zFz?Fx?VcTO$Q-ddeCbs(oe7XdN?8F7-0)Y7i3WI8L9-htUCQU%C2H){XJhHwsUzBG!*&z1)J_FsoAK)C_~q+g}0@`dU^Hzy_8hjD!f3b^4N>AK~Y9^ypp?|
    zRb}u5iOWgP$j|}{FEEFx7#4
    zqu+T{Zo1HY9M55?E`7e^GxX?e-nEbSUtZ4N-_Xz?q~hhor!w&>-mUgv$<({e`GD9K
    z7Y)%%Ndv!N;i{<
    zUcvn^@uH$ay(Lv=e2Pu2i?$nUO(%-b2--3=TNQhoCUI?TZ6#pP+`-z-Gk2g;?B
    zj!8oE)TvXIA;QMfWOT9nC(kC!Ukv2C0Wn?u3Wdw5PgEnC(#&DYIS6D9(gtkW*j|(4}^ipl;02gBpl$5p4cNYRKxBi
    zMl|qDm{|?atsER3
    z-RC}?mDCi2C`VoB>(s|#OpY8f&UPYt_%F|mCWsN~Y`a9C{XLXTg+K5v^G4$Wq7Cvc
    z=2-MC_)+{9>2sYMx9^wGgeVJf5xrPw+c
    z>EtlKp}o%0ugDN1197NQM!GpzX$E*klaOwZHVg!s>pzYbO-7-6ka$2bJr}HU;i4Fx
    zHO)uSE_Au(0iaAmbCSrYKyf=TgtcRBnX|J=Tvdi|2XO{=AFLSSWa$=BWsfv+&IcYp
    z*hJByLg+^czhWnu;GsVW?ng!x2h&vav6gP`H~q0>x>C4DEiEHMs>94(M}|0&r(2c%
    z;h*-@9nrymc#fvk#DROlGlSyX6X@{)%#h%Q$HMpP)Rxc^<19Hd(dL_?BX@pXR1XA_
    zx19T@rH(xK{|Zw7&-wp%%%5~JEdEqODNp824wJ^X5rBLI*}UNTFI>MzXpJDQ|xVf
    z&&=K8MdeK$z^io|MVP&IFhW~LqQ*WgAqp6*qW4xo{Njl~Y#Mn4p!>Zd7OxHXzLYr}
    zmv2)SB~fF^p;|7?I-#!+PjaWJh{UE#)Bu6)24;NmFW=N7LZ7wVLDD4vm8t}8kKXJP
    zr{FyH{xZ+HQNGvM5aXPQ4bL{`1pM3R#NJYOJGz=h3D@iY{hhSQIB6
    zDE!8oecIL;WBz%=BQO((&}9jY#`udyMpQ5!4NA25a4=oA&CfN3l-#Vx#>d@(FQ;W>
    zX4g9Alt(>~tvK=QOi1ln?D8kE2D={q{b;l!hrq+`5odkgFndRw^^=j2k@vyUSWd(5d42068jzOKrN1o7`-LD;SYGUYs
    zxpk{NzQ0uJVIRP(v)r%QsJ=3RXzmUxm!9(n|5$NA#~it`G5O9L?~2)kLRw@oOHFgf
    z;Y&>mewD#O0zzNkAQ=5(w~}}U1m&89JABFqU=qH_kT3shdbefV2e~l~%NL`JjZH-X
    zoY|x_KtGxt#&|K$)?a+8RQewQG_Zynbm6&d10$oz<>lqxn^mGEoaOPB*479K=9n=l
    z_NZ0mT-zL4ammKua(fDyzW~V;gk=rLB%meT8?Zu0#J-BE?cc)$@Js?RlIW@6$qU|C
    z05ADjKkss#Qdh-0!rJ9o&def;o)JJRfQl?uS>c>teQcd2@P%K
    z#sa!j6E4NA#B7qG`9}D-fG8&<2SnE&o?tR<;se3x7xRLrdj?rkN2A}q*Z3(AZ7rv?03F6fyvuBFY{wxvz9yA#hsMU5
    zN*!yA0+M+y^xLH$y>1iOm6p0^5L3SQ>$!-p7Z9Soncge)ms9_Vb{-M^N)W45>4_)Z
    zSs3k;YC??t&6aVgV|$Yo*EaE$L9T|$B;yk$ll<~XX=qRrXZ{5ZJy)5$C7F!n`XQ!_
    z=4iRnt;+x+z;C%FsSyZ3ci^XJzwZsL{&ZG6fx8sO?PPC9fN1$6WCm#ULfdg3lf6RW
    zZjFbdN$)j|9#}IXUDQh;<2kdysRZP;;LtgbbMWkOduL~z5;I-k_FV&J<5*ou?8^#H
    zolX4t=yF~~`sopl!(yN4-zj@nJ~vZa%({qwhmnKpLB2Ikg86v^w)p+Hw6ruzpz2FS
    zy1m;dY5n%sb?ATPAizO1R~&_9i$5<2mmAq|DkIaa@ABE<40%c{%*@zaCz>B4yNcr-
    ziJm{-Gvu%BUA^X_4qh)Ai=2d}Tj$jv`qc6oe$|K3mH|q8&jeQ|c=yMYmS!m)>VI|h
    zRCe)~`^W$?=hCp{Jv5ZmoC||7yHgbQ(duL4gQAZQ1%;JAq_vkNrVdLBT{aD0Qx#U-
    zB`-45z;kavwalwJJp-Z{ANk#qk{
    zyEp0FantcKFfbrw35z*1Ab+b1W>!-&q2T>EO4Jjc=jPrls^`TUFM>}BX%uBm`T@l0
    zvFHKbGGO|FrGzGA*@{I&DS0;91nZ&dM@0L3a3hSyzAd$w<@TbAIq2523S_5&w*>?oH}2fkCEGl
    zP>~q$JAHF)yvvif*5U#a{TI4Q7!@mSEjo%+e*Ds@1qz;9pM=H2YDADq-`u?X{WL$8
    zb%QXJe1~r=Y;++G<)>rZqio2^>T4f8*Y
    z5{|HWT}bSOUXK-|W({
    z4Au&Xiz{^MB0LLIS4O-ZpX$^b(a;qbZxT{SXcAy>s%OP~tnufIOabE{;etes9)7tc
    zLBX>-67&vs5dK~+axvFcXAbp3-ZMR8nS8MV7(EgHxlgy=#aMMvi=TflL$-haz_jn&(WQh*F=D>
    zps+CY^=r?t-;*Mz;Dt+$w`XY+%-kD}t9&3!pg_4|>7(rau3e^rGG@T7eurk+{OI*H
    z2SkZLoP^a5QY#sbBPm1M|xC&Ey1r8;o~UmNpFyb^f|x@=IUPm
    zjtS`3;X~&wSHpQgPCJK!%NGFz!b2C)7QT?HY_@?_QU
    zlfH*8qpPkjhOKZGSPpPC6wX2sPXXXFGsH7J0JS;$gcY1d7^6n5An%BoAMb#C`6dtG
    z;7cIfbXyt-qE!mJ_Kq&W3o25hcvJjUL~)R>)g%BGDI|}e#4w*YpkMmJEp)r7Gk4?y
    zvmxnU)a#~=61^!vdLx5HOT^6x;->|>;1ay#2%?*paj|LugCjwmbDRP@a0ZOme&ADT
    zfKB3|Sj0vH{tDNLQ=oo%Alt}d&U&9HN>5LxGYfHkUe-%f1I}dJ4;tfRW6FLL4)Z86
    zjq&&h6A3vaB}f#_=5UvDe5OANx&!}hJzO6He(*9Wz#m{1V+f(|p_=ena1iQ{co=0I
    z^sb9IXyCTk=S{N`U%3=|;>+)l9Y9(kBykyTh>M05EHPLWrbjUI>asyvr0&K(j{bsv
    zdO;73_ZB0MfrVQ@5(qPoPp1*fJA>djmrdd>iro5q8{NOe&4%Y^vRwTg5x2n_#B!|g
    zdPRHC;xzN2Mrso7-Mfd7H@zGJCD47e(J_U1{K{-;#-AyI(yzd|>{Bv-au2R0u0I05
    z6ojvaL@PBwqU%f3s4^nOsC)PITJ(t_QitgwR}v
    zwhR!qv@S>lhW?pGAbz|2BTV;glyjpjRVvP)NCR{abjGM*Z{gNp!o;hACh(3(ha%BM
    zm8u8IP_k=RsJRR>!f!Mr0t*Mhq}PD;p`gvp%Y#57ZhkRBYBD9-!U3la6p?6u9AxAO
    zLtxM+r&wrJMLw$%aX&9!|2=w{zIqt;Go*&XvvatSR$*#9^vkK@E{Q{SP!Aj_O3;;%
    z{blr2r+>!lwCm*=@|@?}#
    zw>Yt@^m4PXaXJq?J=@z9137LDd@_&*!2xw?N+!aa^
    zXtow-&*O&wXGA8VU+nTc_dW-oy9exy{hu5c_2r^fB?d%gqRyEP^S)J&
    zWKF`ZWx0KeD3983ykjV@PG-E#Oy!FrN_uG+L!#uX2?xOl7Uh~|9)1#%ib>{c`yaqWA1WT-|AMR!!HR6q7Lk5s7Y-$yja}%lT
    zkQdR0Kj756iK1h1{d1{E-v(@Wo!Vk=3%@*%mGgCgv49I0h8$z#cn&GCE8mU=+9GnD
    zEU9;38i5%DnP~SaQSXTgO|-Nyp0`*VCF37T
    zAk*d5@gpy;*|am_kOl)12@zT;2u=7qIC%91rs*XmO85Z8TJhxt=o;&`0*-tKXn7QPm5aY6(07z~GrlZxT+io@akwM1a4;
    zLM9}|P>KqcHvmzyd|{@rM-i*QfN~^oc`1R?%&+9p9~?Q?{xU!6AxjVt
    zy7cU{$MsYk$1i&0qhvNB|Ia03Mhfcy>qG
    zT(fqJy1y^l0J~4!n;7omk<;bBNb$ehM*vEm=Ma#9)N<*a;0&abpY#=%KsK6}(8aRe
    zV}~LPbZbyf)zGNKd2F)jpYu%}WO=lj*Gq}sZroBE>y40g#mZ!zBtQ$~MMbkGMC7c`
    zAuu<*J!$D@WZwMPL+*WScqFK<`hgNmH
    zJXu1M@5JXo8Br3w*qO$gO58mZW&r4w0F{G%bSTBk%+b*?sezaOe^CZs@1EB*M7CoB3CEJ0n)jv1ZB0e`OW*$MUQ=SdnhENt72u*0k5F#~B93OEoio=50fZ
    zgI|-=*SsB_og0w#NAVLJBm|F%R0T+ICj>sLJ;yRBEzwsB-7+606i`6fxH+R1j7B)v
    zY)XsxkGq704UCPWCG4+XuLinq+2;&A>l+QUYGB;q){y++%}HTGU(e<^Sj9*RIRhYu
    z+c-7w-g{hRgFLnvPT?u@>XWx7_aGU~ER9UQ!>|4rh_aWKBz3ebL8<5t>QkQV#J2wN
    zCrXQm2kSF>aiv<2CP)
    z(|t>=Kr&fv(G1q9iK!{k!w+`ZU%yk^7l^L>29$5~Ujai&)cn3}@3PN+_za`ry=4s3
    z9Vw6iE3X9n0YRcAO^-2tkoPJDjvbg>)%{Dc-)tPE*M6-tBqR^zlp&CdOdd0d>tm_2
    z4P9kq&7D1g?HwJAx$eS{kkYZ|;kvaT29Y{(2z{HJv-Lj47t%b8DH@zF2k{mtB#6zu@_;{(6euT+w@wT%eapQPO@YOQ2LlgpXZ37VMsDs=FjSg_E`K`?FexAr
    z-Tuf0o_BnS@qd=YnM>0oeD(V1)`1lWd4j~ck(0CDN9O++^?&?t
    z6AdK&M++5MW5i-FblD|s5t5TYGWMfYBH^%}sSr+#dj!KOgSR821~FbiT!kE(g^@%g
    z3yf5Rs9hEGyej3KvCJG>h`cVB-Qr1GQDDrhBeCiN9`8mcOo&|!+|h)Hc7P-C;NqE;
    ziWQI0{)Z|79e{yRh_(^J2KXV8)^F~ac*DE9o4u9#;lC*H%qI>R%g(Kq0_b8MNdn31
    zJYc}IcbjGFk-HYP$jQv)StQ;DgbX4gj#1v*hk>K?jNU>T_!qxCTXw5O7if}v35iQt
    zJhM*aBbkoG*vyf$*jS0ndDb;Dj19b(h?4?3-M7xPzj=WLU}~Z&2A!!v0qs&cAh!}a
    z(*j3tOL13l?rt@1;TaC~-5|BzyyNh>y!nb`6w0JkSwM6mq~_}ZIcWn^)42EiSk-7V
    zH_a#+^HP>UP=L*7iwOzbjT__pICx@h62>eMXO}$xz5PNpt;Ga^u|DF{7wR1+R
    zDC~*yWfSzsSB(G6u`XVMG!rs_su1!adLRJCG5p(h-H}h1NL)4p86YHoPltCgL#=~1
    z_Shxl7pcfn53{e*g^qB0v}f*UfOqcs+ab~0if;T(C%*uIZ9sV|5aTHz?E^3{I1z04
    zP}3>?2C&mH*5mw90PVbR!C-6n#OtD@$ESJT_lhv&Gp+-Jge2m;6cun)Sne#=4n$(*
    zVvAqJ@}NC*Ny5VB<`aj-O`m6>r_z8V2E+*{cp;;OAi>jKJX`TBw1c?Yne1$W_B4vF
    zc65Sgl?OFHl1OM0%sktSN=BYOkW%1i%a*q;^p+n-*W39!Bqbcco&~ATJt@Sw2Mlbs
    ztVR}VSqN6db(}hEmHVN*dWy)O0(YQdnRS9@4^3Umfewoy`|Y<}ZjLTJUU^^Okg`Ab
    z56Q%80b{dA5@;%M{j-T;4ZM}VtPd<=U@dX!%Y#|ED_%v@t>1@6MFb4^BFextxD#!i
    zhET2)A~yb~QY(FeJR7EFh^q8BiEAT`;JtaWfC+F%3CyW(5WUU9Cin8~2U|b$I>%pH
    zoV!#-RQYJ`@E;Y{|3xaXZ%LNU)c;OV;M4MwZ)#quTu@3n{jcIDsew&S0{Bm7KZme=0i$2b?)E
    zUl^0H9aSKVxQm*12VCj<{iREH#0HL)XHli;t6Mf&2dUlpp|5`UF;JH$I-iTlHtX&;
    zD3Z2#6k||D?Jd$7{kUb-0p$*4&6^|@iVB9C#X0+vET?`ZPUruCst^FLsS61QUi%*1
    z<^|+5d#NkF3xCS})!ADx3r@UTihy_BGub&GR_`#-?ypSMHw3DM;{w5yZCNeq{I_GC
    ztm^pV?runjpWE=;x4h2%;5GrM1n@8N-+}omu}^+enz!-OuT{pwpEo{pd;3R=TjR@`
    za=s`0(x%~dmiA+9JNrJhmf8H}gk%`C7gIf6+c=9%6I6Y4c3jx4p?9y`^HQ4kc_Ne`
    zh$Z!G-|NUulg5a;^F!H0;Oeo_{XcKskyBsRlQ`1WI5zL-MHwSYDBWQF-nZ<7nYhe~
    z1?_TjkaB6xFsSf{r`A*s)kNkDm)z@EnG1*$9c|eE^Ixa%6BDWz)Rc458kKElT-d6Z
    zg~Mn@j$D5&c;xJstlAgdyEO@^I;Pzr(VAX9mJL1P6eabAk8A~%c6$~p=ezvOmvmx3mr*23O~;l+g_xQUkM$Sc@@Z1v
    zD{s5>P?f{#79(d77JHoZ_9^T(DRuA{vuJ-f$iIt1cTWoykNH@>#PO(t>XGr;wgSvV
    z7g}A(7fSQSnRHS>Zq=Z%oA)eKg``I4XRD3>e*a9Fmh<1{pd4Z37YzGNz2@bn#!a1C
    zaVl2&jWN_2Z;Y}edaatu6(WAt&N>&f`yNJy2`#lZzOG}G+8R=A
    zx_>e7^r!rVeY96h~t|Ierh+i?cZD)PjYFfAf3NDPv
    z{+Vq_mQuc
    zl}=@731Qu?b7p=^XO8rwrDiyEL>sxT1n2a!JraF$$D{8Q6=82FY0gHqv#MY-g&Bdq
    zVen8sd8AvS=FoVow{k`E;`@owY}%TWu9HPwi7*eXtz{`^X^{UhJAw06Cw>0oe!UgW
    z-=AyvqBttbdUp8sN@rdQyTg^F7Ne9;BH=DQ4T!`mcar}qJ
    zTa0|^cgczwt?VPX{Ba-rcmrFZ#~@wj_cQGaEUXYY=C~51YUf6hYE(GZj0v?21@iHK
    zktd5ZJz9@3lJORymi{ogzWZ5ke@^fMCTG!U-H9zfuRdUo#5cV1DM+=%0`52u6^9_d
    zX++ox_|ZaA+6K(T6J^tZTT82O)a|an-xGCqn61=Kr<|Bv4l!+R?v90&-IKPI2~x)^
    z_?ZF>E$YP-yJ!%}z0?-bHXi3~GEnSqHj&Zpmgh1UrCRLaDP~rxyb_$9
    z(7p8pM9JL6M>O|_Fnp4yFs^XMC9Xl5+RIt0Rb4PU6~Kz|w*Jr*OYawc9R0`QaKPip
    z&iOLZJZxp&D%=n%{%O!%J4GKERf5Mj<+XEkp|P*ki+6{jSFm$NZm0LKU1x_miK`6H
    zrx<>UpqTm3x1nT))N)KnNJx5Sk1y#3%tkOU*iQ%7yty)8
    z8CV=J7*!Be=i@vW<-_E6~KVq-Bs
    zFP-R%Vhhpoq3SS)14(oyF|UBGJP`toU9EKT5J#Xf!l3N&)u;_TKgl0DN=L
    zVN;?%w%$^ms4TurJG*@S46*9yhR{0aq!Ul}SQ(A2Ov{R6vpatAGLF-CZjp^HT&zl|
    z@qg`*n<7`F7O24GFlr!!j3+;>1O8}lU~@Xqf9iv1lk0-y1+leYYJ-J8T0`E7rQpRm;j_rO_X*Lj&dI`O%Q|=SPQ%p9RZrztf_I_CWwhqR
    z;YrO#Ro`pbDl6560-64z`zZLi4}SA^&Qd}C>zC9UolD?LJR-HonpY_mc*_}HA@xfW
    zLl;v{Eea=y@2hfyF2nzN|g3$
    zQzzuhq~!T`CZt>4dtN}4wO>ejKf#xx51a@sEH>6$JfA2Ia@qIy<`=zlzRlBOM0km0XsW5`O)RD)oXMTpaQ~WZ6)c<9r?_w9P1GLM2tfRq0Ajp;1_hzI__*Z!X=<
    zv2KO^gE#A6#cmPTPk1d{G`E~b)#?h^*A~^T=56)(X-{GEP4Ay;cDJRlI1kdOdZY!7
    zr2g4_{Ziu0RDE2)FORevt-<@Tu-_o2nk2i(idHYV8nCj>Ejl1@F|6CzQYY=#V
    zi$R99uN0R^@1k5cbz;FJ)q!Pi
    zVO4c@3U5EtF)W|P(gc*HM6oP*T6bu78bfW5)jj@(aM_C32G`yJTmBovpSHKex=r3<
    zeS``-`ibjFk4k7clt%`n<=MU;_Hn-yA7u8wCoeK-hVOcxE!271iS$!Dy}P*P5!jcP
    zqj%$jamTYv3vvBs+WO>j;ZPOTS0ue#P3WLNWc=K%NlnWIlB}#~H07@SP8Fml^3oxC4@A4ukR@NG~
    zA5XJ4K%~4whE`W|bF99R(HXw1M)hZoDj|>6$|fiZ(EJ>D%2|ezYkhTgFa%Hg&S9LbuY-Y72`z+asP??%s_n
    z&*&g61j>vt+HOY%KO&Czrys$fLK2xNkAER7r#%T0t3@t=n7Ur06u;5iZqS3)|8T2L
    z&1S#~-%xMD9$Rm(XbZqA$g!%LP#)HCSq;otVe$G|e-X)#dR|Q*wZpmcO`ca@Q?g~G
    z1Ku;OCbcRmo|N>>I(93iJ-eh-;CZDt;P%Pn@1pm+nwDHLm)^B{%j(q!=q8ZZzh`ko20JBuQqof(PuaY5N%(P)!sK^PgSCCds90lh+Cp^oO=n#Ha_?YRXtnHD>73k{VPdNMyQk?LGf%-u>CU#c
    zKpCCX0}94QcG^Y_?cA*Fny8F1r|gWHsIjb^FmXjWtE%9Umb#F-9z1og&6ZL6$fv_}|FE>dqOSc&lX)1m*&4HO1U*c-uV>FJQmnRENuGcF(0ZIY
    zHQtd!hMWa)P6%9%Xho41y;~+^)hiWG6>D|ke#?@TQPue`j$OqzwCX(errNM_e+R-4
    zm-V!w??nl{^Ui`$%+dEjL(?u!bJG)tbewO`@MK$_Mm-{jN{%+dV;Yc3)2reh{-_+_
    zHPAd1dhz?xV=*VQRBi%y$Xe#tx!gf;a*;ezC@7|A?985x8jCH|C}Gbw;8ne~jXZlE
    z_mKLjM>CZzZB8Kh5Uzcjf5i{E5kn8L>X<7>wrSWKgM6lE79wW;Qx)^eOL4>`a+o4}H{Ppyi8Vtc)%NUaC&CoE
    z8pBde&*)bA-fIWO+hkqmO~YP&qlBYY_h{6uS!yO(OO;3KQW|A-`VG8pV{g5CA>!S%
    zyKCj!9yI}6km^4>}#Yfv5X8Hy|5;%WgQxpJW`NiKQMq9os^)EY5
    z_*b&4Z{FGaP{%OzZs0PPm$oV4p!!`UYj*1xCa@M(Lq|W3OQW9QHEO6rnhjo~hV8sY
    z4f!>z*g0mp3a*oVR&`Mp36}k3O0s1vheSNH?U?KJMj_F)?`YatLfzejF3@I}!yXi;
    zk2Mc181pTBX&%xSlr^6GQu~fhc0On{k2`kwK3e=}KX9nV
    zocEgy)^xs_D|ye*m
    zDWCDqi&M=*{Vy+0Os3Rq(S^G7L?9#8!w4JNL#jRPoXv$wh_c%mp;~?X`H=5}sJ#ZX
    zenmuOcdfpuGMg_JK5kS;pX>q8O!-yGpVwYcJ_bBmFP_Wx0nLnxYASe;2ZFM?1QGZ8
    zqeMrCEjy?)?M>~J+#LNwf~I}shkefuMei@V*kNsu?)%0(ws+=U$4j~A)d(~7XjN4N
    z^Tt8K+j?bW6rNe?^Nd+i006E-D9g((d2h8m7VtPlzh>+_(1j^hs2~JYm_SR8!>B$5
    zSY~hK;lJt~y7~;BtBJyG>gqyP=&Oh)_sQt=@0ye<3wIN@h*kAFXQbe?gZhM2#vS>i
    zWO&Wlf{zsyzY|VnXfx4!8gEp7DjmR~;mk?r%L-E?Y6j0de{I%yzjA9=uZb?lom5Xx
    zN{x8#bEO#nykmke#i2Z;PgV^+U*^
    zM+9xD>6Wf3l^*?+QWL%?8g9kkCEIo?Xg^+LHYChVN2kJyckBsZ+h{(5(Wv^2HVFj_8P
    zs3{)*t^j^+tK20oMAX^b{2@@SU#xH2i2LdT38kP7cOOlO-ml52!!u=hJ$gXufohde
    zCg>EuT7*Oiy|Bj(8uVP8a9^eJR+9@vUT#ovkEC^iseLiC^LNBCF
    zoqmAa5-22lz*dVC>#}QY@*9r8Y62C_WuEp=yE>KZe`ff1!0>PFIy`)9JoO^9YKpg=
    znn@4mt9`N(QsWFlWN3~+&gf9ErQd7tDfiJf>7P;@hq=ip+<#VgSDap=--XaP^$P8y
    zzeKDJoa}n0vf5u4V-qb?vGj%48>hk;WFn5)CHqkeq|;dUFwWcgxBUCDOD7>{Igyc3
    zDneh+dRFQiR-82Om3W){SpO_7xM7f-lT}T#gaFBXvZtHUJ>puy9Y1`_@3V7vu4F0m
    zoz*ONia%YVHnXOR!Sevma=;MN6Bw=(VxK>aP?;eW1&M55c
    zXgg9oUwph|)Q&ui!j|P>X{bCz$2Y}hh`fSmxTxcmVWgIQw@Wl*+bK|~>NJ?nET``H
    zv<U`7NCmp$!1BPFgHo*nA7_kD~ad(VSIKbmaFccu)29{h=(
    zSHh(I_u@(g$~tk&90jrP@q4tp_qO_Tl-OC%bZq)}e&6BR8{h7xl;hG#2#GVzyXo#|
    zV#1aGQCCFeuxm`QvHA(eS{VlQR{PQL=5CWl9qdZjVP6;L*rJK0j{0thZ%OuyW3Da@
    z;rL5M1`&RPc7KF`@`nCN^tQU)c8SuVf)?F6NPM(YS~}(9{oLgIZgI0s=EQ^Nno!4*
    z9MDlhlZ_y=c$0X#YQE6#x@~&zw3#9(``-xgYyldJJnw9_uPBWwzFw0
    zxLbxVs*cKHtVGib_S`(@d2sN|mYS$`r;vxHCf>JlT!tPR8XMVkJ{$vF5}c7$69I6$
    z?nC*^P;JMI*{KfMN7)~;JBi9%IzCS?tZF?JvgdzoEOTWMcDoXFd1b?geV^f2)i1CsYP39W+uH~cc4?Dfs-PgRMWkSX&2
    z@>_z1H-A%qm-+>loXp(3y`tt0<<1j{PD6SAPG+yOl^WtqXDdms42>h@o@YdiVb!}I
    zL;)Sl2cgc*jj9+;f~jEE2;*RZy+eQVfYJx;W6F^4U_*;bFdE~taB<4K#xUgSRX1Y-_K6mdtIckF0sSwsL(#HAdd*qd
    z#oj4DEZ&1$&(7;Wl@hkR+w`ICsd61Bevdy)I+xnN%EUU-?qKA)O^yce^O1JSRw^k(?B)jZ^D47GhYd=($z1&
    zik-)4LMIm?qCz0N@4h=*v>)t>ZV87I(0AoJAgPjtmwQX+%MV1?Jq6{%7B^m)EA)`{
    z`I=r~Ny_mgBE6`uRCNXF7Tmv*_8I(kg^t{rOd9U>+on&j7^cN6&S1ZlAh3BYr504-
    zD$|xR7y(c^5k;}#S&Mn`P**a_plBxiQ}3Q4cH4BxSbY&nCY1X1qe>Q3f8a;F@sOPRSaJMH;X#2!qmFrxJ5QW56`_rX+hkAb5R||4$Y|$S
    z_CT1=Howj!t3?g4D&yjVW#qy2_vRE4hNpYJo0R`;hDjr%cqr7@lLt_GfW=
    zhf9v<7+Z_L#~PS*FCIf}&!&+46sTB%KdTRDe1uh&-E?A77ifA+B(zvEtcJRuLvoFf
    z%`p7OY5eu^hL3JS(!%X@C9D|C+6#gAmHtK7B=j9z^)A>nM1tp37JvR=56mN!+`u7KP#XfTwp8>a&luAx+QyJ=(@r$zvcdSD
    zX)#eT94z+|xQUWoyY)0m+b;)VHcd>!?24Hb?k|!td9-|7$KJuErn-9!c8Hu^Lwx*T
    zY6pFx_!8Xh+uK6zs900Ef-_L^)6!bDZ?B`H+vh(+59dWP$}_JI*DrxSSc?*$LwG8r
    zB2USQZ_I6y-3JX=U+$3Ax?}_Lm?PC++h$w|EBLl>FTc7)l?gNd3w;3vGmmz+x!_@K
    zVeZ`BgqgSrLyjJ#EkKo2j8Bo;w#^in%qvshYur|A{I3nS;H<#gT9|bU2axW|@5<$z
    z%BA)@by=%HTN!xrZ|IAEjnY+=)I0TcH2sX5rW6aODc`bePReAqEDIZt@1AgYIWg751|F
    z1W@F^{Pk0?)#-Z%<2~z#pl}Kf-U2DWS7-m-+F26=FYgzS6CvAaY=z7%EG6g^h6YoO
    z2ICI?SINEdgIIzJ?DTl~Iw}LcJ96o}iSrdXjipjRhSmW6tA`jPyp(d_8U@Dj8!ASji5-PD)Gnfv#-r
    zLr~x4wg9zpUM2}sNDp-<0xdciH$XN&^n=$!BG;W)de`ujw+9Y?6D|Iq_q!k&FlC=mOpJ5fUbt(0aNaY1vBPcBRM;qfrZD__g3Q4c^+
    zZxr+`LXC_8lpLG7lXj`Eo3(b|dtph8kg7~sVSjf^O>qEKr
    zayXV5&Vp{B^}QwT&}zu*76Y|mngz+M3Ioy!?x-#TNor-v=bEAGQRe$;XfGk>(E6ex
    zQ2{-KVLT4BRM5CH8tI!s!Q`BgnK_H)(zs45vq&}7(TWbcQuG5zm5R!!(#~W{XoV0E
    z&lAD{p8~oj(9ZQFGC(nDu51SpJ=Dw5GkFp{jgL_N+v;4@2gx*~?3wEXIj4$2>iHT_
    z*8#5;jyh;D@Z1%5sJ{Y@z~cPZ?m1OjHtQUlE(4o~sw+^toeg5QuAn#NqUbr)5ky}UlC@@qH5Z&lqfP%qBYzC$A@f&{NT(1mp#
    z^)96jh3hpEK}?1!KTzch6pKHG+y8`0xKI-iUfEQm>TLbjf~T?mU%xnGAEQZ1&bhRz
    zn~Yj~Kv{uBNG|l*uw#@jPB!Nv;omHY)4|mupb`_*NCGz(gH*X7luLkmE$EJkss$hb
    z!GFp+s@$Ez31JPdhii@m3nY7-!O-ncFDcZCWYHlQG~%jIuk}$Tj0iQ#F&8l+@ndE^_n3Hs*8~J|23Z89T91O(T-ThD(3~>_sP%uVOA7Tz&l{8y>3>
    zwb6h#DFWThP_HF4pT&~b&3(SZKMc9F
    zdT4wrck{~!R2(UE^xDrz^fWmQQwY^$OME&0APBEP?K`1iup-LUp4=cB@o4YaRKRm+
    zha%}5k}E>}H)*Ah*%YMSO0z^IoEVR2=!rsM4J+mYYYRm>Zek$uO%UTkuNrx1m_zpR
    zWy2}voJj^Jy9Mk-LLBs01M5A1s{)-5bT=c)wiu2#K
    z==|UHc?Q(vKRpOA(k
    z*TcUdfai1;b9a@)|ELjYA3F4>oBS7cKKy4)@(3h7XXVjscILy4F$fL!g(mg%7T!AT
    z;xl)56R#@Z3RxoX9R_MQW*as{LDe<5+FmJ;kzfMhoX;r6Nc2xhH@1SNrRw~$jxU;c
    zJ8Z#h^?@x%0Ivah2v~Yt_Ze-YwNOOYjpE}RgVql1gb@2Hmx}ie9PVZ
    zG_XCWJ*R%U;S(V=!if?XOd!e
    z>$~Jv8=wiP5+`{Y{jk-L<}pz1K2K_e!QB0-I{*Lj?;Wiu2?}M5bycSY%?s*sFG86c
    zDt2i^D(2vwNB2P0BN^16kHPS+aGRT(^MosQ!7N}#P>~vDO^;r{OgL&I2(_TjIA>Ho
    z@KHA45o({sggQ`E0c8u_A=!$WypsY!C=HF9Xf}PNo}%ig`k0Sw4*f-tFMMfERDg6W
    zz5dSxAntbr+NVVh~I2yd_xTMN|NqHPp%jKpihJDdwo57%Ih(sHsw4$Ss7v
    zy{LvCmFWZ&DB?+P-`S=L12u8?TRW1@<)j0{o1HPG37pJp1g>G@jC+X1dKOR2rB&2xO|b_Qrb
    zpf7SYsPz(4GQBurhXVqfcc@m#{k6OA-_Q&6+PA0ez9dm|SzRL=rQX|yhmcMs|
    zk`&5lF&YXgvqz2a;!!C+nh|K=a}3Q0=Zh@oT98$dxtMILr_2AysNY6LJvnD%GJ66#
    zjn+a-h(73O3`!f+CEr{<0N^*1hUd_Fp*H}+O$fQrf7Cb&)lm|}-Y$RWp`OrU^!GHO
    z=%lRj_uzPBs>pQdOHeWr;YPoF_Pctx3C90Cv|vTG#DA~R11MFr5|B5a_$1*nXx@8f
    zp@*!uBOg@2kGD^+Y4}TcYdze06G^qCRC$#tAi-bAe-^5o(bCts(hSwR-90x(kd7U_
    z2Pikv!MGi*pw6zB~rp;7tG-nJ><_5U|Sn1u+iJ}Ts9Ni%UE^)uXJrSn214lX?qLIBZ72a4gyQ3K6h
    zfZ0F#Fxx
    z2fO!we0uk5b>ZWqyR?-gk05t5HjsiqQ7WV#TP>sxXkcbpd6X9Cq)m{K>HBffrH5~u
    z@f%ZFiq-dI9`=Q;$5jrNHM9gyE3YuMIH0M@FSeiF%iaBJ#L(O}6PIJ+%z~wqSz~P7
    z5&C2s5yua15`V4c+&k1z)gB@_?;R|q4RIoK!CIc(G5-yk_k4KA70L-TOA0YVoJwms^i5XZAK1IcUWB{yhEt+gS0$TiF<{k3+7v
    z__x4YmrRR{g^F*|K9VtQ+NzvX`#t$6lZi6)2?4HC{uGTekY_R|DdT!$Y5Cp5Kg~k9(TuTYaF#Sh&(7-uuRb
    zqq`HILVI@jQEZ-PM^Ur0(p=MF&Fbm?iuH{~=MKO1YcT4~@sD-hE1|Od{oDEZvgsZ()aA7Pjd%SFse`CL7)dB=2&atw
    zdT&IinGlGYAT@+Mi#A#8{*tH>7@Cp_&;_Y|at4YfAaaI++66dsL4;fY%9?sV{(1i0
    zn+7dVE}u!ssb0ZG7b4-H?Qh^_2N4x1>M0>Zb>8U;Dv{;o<-CC`u+LZcuXk&t@E8sT
    z_3V-j$%RAX0takMw2p(`PC4+dZL&od9N|PUdHUimcl5smg0Z!ck<1rj&0vess
    z9|u!LG`F$x-9Cn}~=2q6i{p@?YafA)p+IBV)chVn;qS0ih`Ih}@-0
    zu{sT1A$?#>z0G>NQ0X&zI0bgv*-6dKU%c9YvL&pnoGl~Y!=yu^O*wAu+qJE&x(7owOO;Eo#)MZx1
    z!IL&nC||sIQBUJ+2Eewfso?HaxvDf7JR269==nerSp6uwG=tv(#)t$|jLgc5u2E3G
    zy^!BJG&%m>sOyTQ2>P)8?$fn;I5S9et}^pv$F=5$bcU=vA?wFP(Z#I?$6_A&2Xxcv
    z%gdTP8`t=nq`l~>7(1b-I^T*ArP@NXZI+`&DPNglFC_x0!>~svfXRyevoX`}9(8o>xwK
    ztXWO`V_qNpyXaLjEck#5UShJaWplNJ8!yd!{T1u^ml(+J~_IPtc7_r}ouWTm$i)?*Xk2F9)pl}bsto&S}0ItWSeMX3B7a-3Hk)+&pz<9JnoC6
    z_x43613Eu|Pf^^K-&Wvn%?7pX5W>Ko*8JhpScOn>K*oq0#iZ2Wf{`z8?OFh=em|uB
    z9{UICQmnA@J-Y~VHs5B44;EwMJlYTt(GpLcSw<0~_0s80Qs~z7Y@Nyu^zsF${dgcg
    zLfu}2(LM}gBe2@9UN0DtxOBflYxJG^018Pc}GiMIn
    zomTNRy8%hqiRSwhWc?`8q~}C3OA0{9vD8vBs%Y_n2+dy+e4(4yu-;mSx*dE!;L*ge^8`Rc0TSHXcfUfQkP%{kiCMk8KO)}>Dnw9=D7;UQnwI&m
    zaJx$71DM8gh!BP;G;KY;bROMc0?nFz(45wRqJadU&m@$=!V|9-jZ7YlZdMbn%$r^q
    zII<0i&0K6f@X|sGygQ1u|HL-$@(KvOT!Ne|-D_gBqPGFX5)@T-#MrK@z^Yyba~Bdr
    z)T;n=WR>=8|9Z_SYizp
    zO%iBh&wO>QtIfXyV>h-Ey?hC)2tjT)CNb%HFK_aNzp=Cb3*|_>doqnIgS@{PkU2^}
    ze8iLvD_nks+RUUX6|lwFWQmEI!igicoFsBs>!#>5m;We0g3y37jOH;a-Vh8d#th9k?DW}5iX>{
    zt`jYGTfk9ZCp#7g13c4owC0QLJ7Eo}+Ce~gWfG;3^+qGdn*E11rMt^r?b`WYf
    z6%eJgkG>~<-XDb+&61(BE;BvdedyZj0~;v=LE`!%yrD3<_@fI=q{4Ce?(Iumnzcfi
    zai027jV(7`1y|&4rvF0vzkc*Rm`*BOM?@S{&8aPmVs6eDOEMz-99atqUxw7k
    z1)OO_MTOe;e$s1
    zJUyH0TY8$QZkrVn+8N5va9XRFtM;Fty=~2+%jQ#~rE|aVdr~~fFYX(BI*Xupz;+FG
    zDoj&9MzdV-P99#!XZcZ;hR`kp-8-|Lw;|Urp%WNz;c+3cCjMEbL~;DKQWVGFunhoF
    z!$YTkiO>FBIEXtUAhQr}>^pvA_7JOc_xH$@X45(c-{U7D{YjhyG~Pe-A4nkgfygw#
    zB-r+!XIN+rqX31rolbKR{^1IQJ%LJy4q_ej#+DFLuS!ly6b^&0N(2#`*X{ir8=U)w
    zeE<}ov(64-S~+T3YppKMBf1OphGP
    z2~R+axov*-XBE|A``Zd`{r1~p{9H|R(Yq8XaBZ`+e7bsq<`NKp1xvu}-6$PnC`#kK
    zXftN;$lV1_UO=;8$Z><0LeB9<)#K}_IttIsqjU;!p_}h!g{?vCX^hazPJ}=c#(CQ~
    zViw{ac(lV*JOccx0-i6ed5?r?VmK-i3MA?_{EHTYXp!yfKEw?#H3s;N_buE=*xD~?TVI*yU(^tPI*bjan
    zq4V2O^Q1jQq*C@m(-U1`Gj*N;(aC8I0pZdJRjFL3lCXEZ&H98NBhHzZpd8`7AaSZE
    zkv0_pijg91wVOyF3is{Xn-C6YSSPf)^Mr{ZW=L6pO>nG4Hasf}J?7*@w&N(8HE-oN
    z@D!~{mIn8frod2XZkRwq1^-yiU}SLSBrzCy&u}~uVJtZ(M~}!aWK%PEbKyu6p$SY0
    z7Td!vGyzyU5}i=JnT9^lu(@tiNre^FFJJaZc_rn(j=O{U8Gs>3ddFTp=COxs9DoVR
    z7hJc`sWF@ALSC7*VZ#PCz*h_DRJ_J4Mh2Q&rpU11AEM=I;?Yx=LJX#BOw!n0ll?Jk
    zf1wBzzqGQ+xhwWu!So*{i{|HPR6yB`!FR+oTEu#=w=9S7Dn@>?*mT0#t2x7>^#G~O?jmQH8*n*M{832zot>c@V$FUE!UuK|OAqi1L6dniS~;L*RR>G^fnA8
    zq05;xT#&o3A?D*9Wwn(~7DNoq;E@+W02~&jtY)dMS&7X3F^cFe%+_?px-uEeLWG(F(U+o?#($%;9U}TlhaC~Irz_>A12{$T!HP-V%6b6_qob2l6)zpRJ+))
    z&BqoNmD*fjN4kl-2!A+=u!AUmP@;fd(s6ibkT1R*{O)(j5GT0gk`-@eFe0t)0KIv$brBvkpC;`Rk8NvaYA-@(!nD&n1$2fR
    z+5o15$kg*YAyqo4F9lr(W0oC9aI8}%mZNHqygeSoyi=0km0IE-o(1moKkFBtTz?Za2*BOLv5>$bmKg^vIrj6wcrtP*Q>LC3LX_eTSq=
    z3f`S8Ib1qLv(ZkgT9YC8{7G5)}c70+Lijf`ViOBza8DJ2
    z2$2v(01**@uSAmUBc(+BSN^>@uX>FS_VC}I`u)Tc
    z`2WtMtPeFbXdTKJz%M;a`5+AXnwDl?e@r%$;#i26o%Z}01rFbO*5<0-;?&m)()
    z+oxO6HB;Z)cbyEG?kGS
    zU*|8ebA`dVU&YMQvcC7r1N#y4wCS(0Hq!Z;PA8{BLl$gQMkX7&~n5fSMg81SR5*>Bt*YL@W(Eb3~ZpQgz*lUGj`
    zi^vY47dkw(h73*)cO7dS!JF_88NoJwDteBv-vwvgFETzpUOM%f)DB&RDUbI)TuR#n
    z2Qqs!iIwm=8l9Y>&UM3wrh;zdpl%Q9;5FONhcYrUHm;RShK}0SFyant^JlKtlOw%^
    z!~Qh8j@yG~-UFkf!4ngDW8T=4{W{ueq-AaujW-tK&Ajg@cDg_7wxT^5w2(i6V3yF1N@b|drx50%AFRw0CJ%I0Xs
    z9NM_|TjV8PgAW&s+OC-Pqu)p(_avgEq;%{GdHdF&D_*?xXuV=HSpOJ4c{G7}tfAqb
    zmf%Ha`?5`YW3`~GT+e8GzGMG~M)hlE?0$3UY^bQ+YJM|;JGug0O;4-mc4cpMc-Gx(
    zwkc$&)Se}$=G5H-#G~n8Apw2(QX--GTC~n5-Ac;QmnmimRnLqVz
    zy%v2&L>?Ds;@`!i5cE(2!y!*R7{HI)UIwk9X1}8%cR9k+ro_
    zaftOYNBYjJ+-kaxjt*hWn4|reeRC2x&QQ^HlsIM#5859YtoBn=7N}T{!h+;?_U!HL-L)74`7!gYptEj$ykavV
    zy7G)^oISw@u4k`tUpJLJy5s_KtF=|}xMTA8oZjB|=;;8K(10DZ+0~q`2Y;}^xT#|q
    zw-?+C=GvlXxouyPl8Ns;K6~z*A;`0#JVQKas(dAJ}Ab9@alsC|uJx=}}Z<9OwQG*PPn8IFi4OxiwhlR<+u-p9?}PHb#!
    zI?dP{{kcwFuT9dq
    zs{03E%fAGKghZh@d{P==85x<8S)0mDxk1zP++0^&!p)U}lvs$$^_Uy8kJk4p+uL0S
    zd0J*PfvLt^>azU+Fu-AbpWe2Tf3J&v@A%L0q|)`QnPc~Cu>+SF8J!t5xM)6u+N!TU
    z>K{&Ec3(`YMl%}g?^R=}x}0qQb~!P5?x@U_GM{evJ2pExIpK~cG_q|j4pi2_b(-KD
    zBiGfT>2F#^00)^*7Uf4UJmc&O4tEHVj&o5OBEHhK&SNMPa{EI`rOEn`54pt9B?bm{
    zJ#^)EXRPS_e5~!>@u-gLxN{EL=e)Fnbv+Y~cocrbd7mEa&(PKcG&RW$Zm+Ga;r<|u
    z*~bB)wl+OTirlr3oj_#|IInw8Fw{OIFS*;V_JcoK$AP$06fFh<__A4a;+6MNi8mg!
    zI$A<5H14F9*tI#_|SpbxK?j}2IMmVL5pZ~^WpBw
    zCOAJT=7Sdb@U}Uf<{X`}-k*9LE!6||3_2d0jkv@L@2aTZAf@3#VNcQ6=N+GJLr6wZ
    zm#HRtTXnq@r!d{}+ySX0QvefR{vqDf_B6>8^Vu%F-As+7loX@&%`eeb>3FNWgEl>-N`<`W-YYT(}>JZQh&hn%w$O(tc8tpML>|hIowhzZ`#U
    zzx0^DmB4>mnUO)Pp<~}THjDOH5dFb#n&2r&Cb4e-p=`aj=~qXJ9vo?rOYC2ccbg%G
    zP@B!nVUfxb!x&sXstc|DmK3QifXnOvlwIfZtvbltgKH|
    zjcyNl9_>wo?{5aYz%L;1CNPkA&0bmZq;O#cLeWob5GY4+LxAaD;HpV@1F467`%#LA^r*y6TY=S$i%JRDeE
    zEiri_2*r79wK^`FQd$n?$Vw#0=W901hDnCLdv~EdR>*dJ!P(h)A4MpE>;kdipd#7t
    zbgfQNQjN~^$v6cR1^;t!KQffoeRZ?EEJH%{A1xm+j*z-M#w4MRXWMj$jdKr
    zzH|RZu8$Q`US3|@+vezBrQm?%
    z*n1_I4|G&_^T+-QoF>TgIB-~$qj62o~jlXem(5rpoZ5B*hMCs
    z6W5T1(@mcqp%ad<@)BDYx{MOsW&%r<<&z3As^Qg-M~khukIUBnb*?S1stl-(?#8+u
    zZ#H{|U1DNt0vQg2KyC@o1h5n~{~R2gJwba*LI%dh#yJb;vW5dfi4B;B;cf7;
    z5kF!BfD@^59BbG5hliJ5t}25-gBKbOe>ZW&de46}6$|?u7l-5I>6dT+^UeDD%z04X0k4$foyG|!HC8E*|6rym)mcf_1u@fI1*3b1Pw0N*IjE!
    z4cJ}+G^3Sw{AcgKyNhr-!=7YW@Y-MRC^f%Pb;ZPNU
    z(mz?&=;`a5J#a7nAMuO|WRDldU;!-;yN$_^i@(taI$tqPm6a-3R_EnhR8xZTqenB#
    zS-J6uV?cC++q-0v$98dEhfB~4hTNQrb&=JvGD^s9;F+o6+>fXWak%(cy!sO#w>3+B
    zQ4E&cb5UmG%rao30HLZSrHjnW%3J;OIGqdB+0vJ23vLt$vF-FMGqx0MX)?V{0iuiJ
    zrEIlw^5gzA310wmPw&jXB4k316j`cpP(_Fp1+~mM><^Sw`&#Cdy?8Vy+6Gz)p7O=3
    zRwISPP@cowk2}GpygW6>TV+`6O3mrY7?!u}S42Q3C_jaw$A7we)=LsQN#xVuuWR^j
    z-h6$IUT_L1EEGPi4icyHBO@bA(bY$G&5W)nKqNT)Xg8IJ+`n@=i9Pb(tU`N|r=+Bm
    zF2p13a5@Jk!vO5{0L49dx>o{{kR6xEm4Nn|&Iyefcw}>V`~}Di=Lw)rXMlpJ_d7@5
    zJw6@+fR4!)F*=I-C_^VFDx4a#sX1}MA^!|z0t)EC;X>s*=RpI*j}zS<%W4UAZw&KIjEkA&z1i
    z4xfQI85}$z{@pLKSlvqR8`3Wa+Pd&LLgXDfK
    z`9`Q~B_2@tZqvTB8vwY5o!brl8T}Hl#|x!%AGF2vZ;<}0`4Rk<*7TG6a#oYo
    zj7COAKPIw^il`@nytM0x``$b(ck@t<-j*7N5@+uBO`X8
    zIA-ve_aPiE;sg-Lo{Z14xUSR)96;&uR-66)w0}xj*(Fk1o(51A0bO2u1#X}T*#IoY
    z)%_o7S@5ER&T1$R26z}hRPxlVu&5{rY(<{YnM&e#ntT?=niUhfIfcr%1kygc&@p)*
    z+aK3pF<2bw8Uh(>W^LUF)Ims?*G>wqq2a0AsC%6Ia@ZugjH&PR<9X?T(70LQMOf4aELrV)0kj!Lb
    zE~D1ai~btyaThjjYZ!1+{-ytRirO@5yu>8{`d0vj;;;`An3wC>ch)ZjSKpkEz&C>_
    z{rk?|Xtsm0vj-S74T!lGr^X7E{lT*5elaxL_ev1|M;v~yE=mUfKevv1M4k8{gp136
    ztY0At^Y49t&PH`SKQ-=xSb}vJdMSOx~X_e~TD6
    zjWQ|tkA2~`se0`(Ovk4^LDG923-nbJ&|-a}m;OY67zO@`BPn!JmdRvZH~eFJKCAOP
    zxj~7#tzcf0>Pt$SoRY%QSnwxe;dXh1R@SxRT>t!;H7m8qdoBlL^M8f(f7Kdo^OY|IPQTmrVhJ)EK-)3{v~D9%r1qffj>(}ZfIp2PLK7F45Px%?nDBqh
    zMFl@|Tia&9w)q|hTbg>_?(TFzR)|AW0AB~#q9?JZnAsSkP#EL^I6inz>~8?i`26Pn
    ze^Tw@0ig5%AnhR(uG^orFOWn4(beiPHjdMmco!s3X;}|Sqch4=K$xX{eE?;+Q0kv4SGOw!64w30ZWpP
    zsl;iiw+xFOuMp}-00)EKTab_M{BqHsZ%rLAglqcD3J(>UQ!;sM)RuXBAGGQ7-MK>y
    z4ShEWHvQ>%5oc_SI}hTbD+PSi<>lp#WZEhs95`>{d@s<&zN;kvn+&RCpap@DK=F6Y
    z-pAyEJ|k%YI77FN1OF9*x-)rx+1e#DKrP#^q6-Y`3F18WhKI@=(^d}TnO>d=6Wb!f
    zxe>=e1lIqEx?vAm|MEfgLD{)8Yv*zwIF
    zp`n~Ll<&cb&en9q3i(T#|5-+cVLCLBXV!BSX9c=7-q6E&QFsu}R$H+&TD8-r=_
    zV$2N%Zr?Up#T)DEyM@8Z2GjVugBB}vOF8@xz66kRB5xO#lo+jz6gjp|rsJS>cM!N)
    z!l1d)ZJm5J@qf$N^OB(whi1^-e9e0r8qu4j+}gz)H8OwFf%Rn!^yro3>1NpC$_gG-
    z2@GW+CZVGOVE2wWM*fS38XAGWCV;^np=&1H#()zbguzlOi}4RQ0>=6sr00IbQ&esM
    zH-P8`jyPS|jgw)V#lCXqZyHe*-2RkqXQ_U{T%q?KGDHhat)c_$BBmfO%&ij}0H@
    zhg&;;Yl>*OPvVtUDljRa?w+1k0DWfdA4KzfKEh5^A!MrLy}fHTP4V#XlJtWAzI|K_
    zlqaz8;t-Wa&7oyjoC`8kg}{u>0*QZ-klABnY&Iu{hrwmY5JCdE=D80n|Ln@sn&Y;b
    zDMHD87>>A#EC**sEOQ(0EX4VMhWvQHS+XfjV)r_9YoH75jXg>^b^Duv(x@|bByZoo
    zg`iKdvEpHt=H?%dup1k8>6nodJvknqJYiffU*TEn;whbi56etb<7^;uu2@XD$fiVg*Af4jnDhkBhIN;@iz?PiWk^=
    z503HNE*mik2huN9@hDc0|t=w@2JIF+I6<2H98
    zSWS$@yBHHPBO~d4R=0{}Uowp`v<5LA+$(YI9}G}m9y2;IUOsx=VmG2tatAz!lB
    zb>dyVXfSJY^v~O`7~;aq;JMQSA%Rl@l4RCiyVy#_3#c`P-Cl9wb$1;O0=1dcR`2lx
    zF{Vigq^FVEewcPbQ|PG*R{zggtK(zDDhP8=jv0lI9$==n(Bc$BN|%)m<=Az@(n$9b
    zZs96HK@+#p$VVdP>sZiA)okJh5Vq@B_IS7F-xltH@aeLp1urso>Rp~Gi@!2BYg*y2
    zCh~_06?%2NF)!XKQ^TN|$)S)1%uo&Ri#|Gt$8%Esc_@6}04|LG|6i~Q_g62)KaOvc
    zzWm%ObIM&zjQ?}*Ur7}4qR+-v%B(BAJU2hEb?K^KL%azmo^Q$T#5XhqpTE4aVt+6P
    zkQBH=VWQ|twL9}b-jc<8Y$-x0P)fC|%3)xyB&!obIbzZHq>K5Au5hvFV^}9gx
    z)eFRvf#wy#nuJYkQ43##UH$~<4o<5VZ}qDw8`{~~t#5`hiq4;6K^7Zg*2f*gfQe|p
    z0Qk=_tfIUeXA;67oYg%In7nxF5WB^TeT^G9;YI-iIl95Pxe_p|$$)0r1gw7%r?;SX
    zP01SU2|Da>79?!`*Qz3O~2-JEOw$
    z_+VC3=+kwy-Jq_kxwi|>!eY_~rUcH-m99DkYISo(A4`XGKyjqj##8a|&O8t$wdKw$
    zkM+B91A#xps0()!!Fxqy4io33s{I|
    zt_*|S^sKDkZzV!se|uumMAxH1?cr$+RN7g*1C<&v+2d26qTn(g^`
    zIFHS7tx)S`4Y_Ue&l-12nEq5Ae}(QmrjSp
    z=BId{MT_q~W~lP;N}OC-?l6G6z5l)2%!CB7LPm`o1ci0S?*5kOBUFYw*ZrgS3M9Y!
    zBw)BghsJ#nEZR^VNr?2~PAXU7?Ete2Y7NcMMl6Y8e+1zdK*rX_90-|t-qC}d_V
    zdVy`Gce>5njRtGK(zid2H@eZef+ny0I*5Tflko*|MZw4ZtZ#6FX<(PHzg42nGc|)J
    z)Px`!wS0d^r2)Ppj$C^Rg;~B9t*3M$^Z?eI*3d11X$Re*O8Ikjl+pof3Pol#T8Zi`
    zt`azPaN2WSZ0INLBBt030gZls_*3idb~y{^cVu_M9vY?_=axF>is(m8s|iCh
    z{|;0v>UNcy9-;;lkqD|t>Fj&E6jxY-TZ|3)4Me#qxO+nuhu5!6PF{+SkwfP#KQ~6e
    zOO?~0;^R1+@4(mMb`=v;^v@t);m29!hel4SN?#v6%Ym5s;juSwOPvvoQprL#IDEHUk!i0fkY4VqkfUl2HnBVUe#ddax@%ROi%Yvi9Ou$nmS
    zbs=dtR}uNhm1&(*Q)=tSGR6Sueuc
    zWv^X%)6n&vO;h}dF%)|~f;&~$?$t_5!_uW*v~2wD5}wsM%nGd|pv&>xPKR6b^Zck`wRU5Ehrju_!1@h05BYK{K$gHt$aylY)?(<*>W_lXeR(3|jYR}i@))u-8V4@ss>
    z5$z;X*#x`11||4TlSH_KlFnovcMo#ZwXQqEmtoG(Tx&XL+YqRMB}1PK_zYR58(e!6
    zF1$2hstS$Li@_=>P!{^!(h9}l7z_ug=r<$Pf>#S)i~ZW-K6njY-gp
    zSr@T;;ti{yM4b@e^C56HN(LLqtoGlpVVcY*$2wwO?1Vg=SvTFAs=!z)9la;Ia$7EG
    z(I6~o|6YkfL14b3XCfdjiZ|>o>-F+pTc*d&*|BdM&#;OcUwv0qwI-v%YK%a;NfW)}
    z5Jz(BW1q5!%N$m<45`Dw$rehkEbl5n+LbRBnfW#ED*sdN;%3w+E|*Njp~SF$fv(+B
    z#E95IUG@9UH>r(#)hat-k7i1vNWs7mj-8knuisBg5DOhYYT8h?)$wty^my{tR59~o
    zRXt_XJ@orI7@wN7nQ;WM!AwW@J{3>(rqm8u*O_TKcG6W|%S&p{9a~tdJQpLX)YQS|
    zZ~8r?QH8~E_4y{}NAvO-;dv^UW^EmR&?~?%EZi*)JXP_*7x0ekZu
    z;5v
    zSFuEHe3}5bGayk@G{Tj5iHaSEj*IiRM|BAl#Xt#J(3O=L!%K_zJFB!8UZ>SeDgE8A7cZt!F7Wv>R9ZltQ>1=;K_tk-hqhLJSiw5Z@bPaC?ZM6l
    zoLUNEg0h}CwKm!ZQ;Qb`@C>&>f>&5ATa2$@p0xJf`ZW<;Ig|HjaCX7nFRHp-+xO__
    z4`1$_WMtHNX^)0C}cU2`X1dLTvpaG2rx>3;qXvHUGf41)c8gvy`YJpR;j(t<#w`ZHn*T`={X-
    z&JE14;cpOGKMy1g`|uLM!mwKYp;6Xp9UELIdV#~J0yb?|BVGsKLW+;A3}p9H_hqUhM!WA
    z7>q`JYS62sLKWXA1IscTo)8L3lRbYSPe?(t4HR%8vN-5^hie0HVOcFXNT$wPzf_pp
    z34rf^S?h3n_jpI5kA(tN2iJ;*iu1|s0HC0{w}PeYr4??^2$R`(J{QV4S?e1rMDVTk
    zT_FZl_7U)@YTx4PSaoQS>^}L_dCH^_$xrlhs!Y8u*;J!Zb8NRSQ=%Fg7?rb@^$p};
    zt9(L|3yT0Id~q-BEz|{vhcIP80LJ<9%WJ72Vck*6Q2+H_N5v
    zMzsk+0bN%;A%#NgB~XgV9LNPBqmG+~Fm7s8FWmi|9L&lP5hmeta?1c)4Z8Rr(}VPS
    zF-O=2rwbT~$=I3UaMqr+g))tAO27+%N%!6}-L5}lxAr&?uE`|Ycp3k?&-hV0PJ9sa
    z8<9^DA90>49=?6_Xtufo5h9ro$CI2CS`=8EILh$JYEn^T@%7KuG$o=|im$AOwf5oi
    zJktdk;$KN+cYTg5_1k#WxX4ntwyvq-4ewu5i#
    z9aQW&DNA%^F0a75eb=pjYDO-T+&Z6+8g-jE#VYk{4<~XMuk=Owy~eCl9=se@v0evj
    zLrb5P{Iwqrdmx4O`y3JCk!tFCL-^c@Yw{RIexwpnU?h%M)?&+9ZV2*UBevGZ(;%&?
    zkK}pqtMg@u*|#nudu>^FY2gTot+=JS$?N({-w|P2y3sc4VTJ;8{Q3s)hB#_U}_cg`ueqy#8bwL
    zmmJWtABu%w5SYbt5I4c|v==&+lsGm*9
    zJzSZ0T;Nh*dDGBH&NL*5!6FhSgRK$r;-O@I(f9&^3F}g>t471e^JUoFX9M$x!?(t7
    zc_K@KPHVDg*OVbs6ujKKp&tHeoBc}akML!EBw}4lx+KALwOsIbri(Z~m|SGdlC*@H
    z?KumOG*ZOlMfU=jZ!S96$!D$PgO%w$&0~q|v%6Tzat%=SW%v6e-?Zpg@}5f-tXyQv
    zS(PrKNyabKT~y!Jsi%;hc^*x~O8)j^q`4Ho)?G)Xa3VH*KIq#8Ab>FJ$S3$4?WeL@
    zcSkcILG8QBw2bTeL|2~pu3|}t?g&P@bvKqiTUE!YMhm)egGJ%H3GyZ!8DyLviJV)F
    zVkw=UE%aEQ?9a{e8hj&p@+Ao0$^S8=ZPhjN<%|)o2P
    z5c?z0w^EnS+@eXnAUy--#wKA5vbDt2sHK@Nkc30V1ibu4I((?E_<|mZ0kSdRP582O
    z8uS8Q>e7vY<}v!gxTSo$F4G$-*!?qXfP}>$QOb2VFNl-L>vWM!XG0$|gZAFJ$a9QF
    zWaV&HAoN|h6HnYO1gS)KN+v=Qs+xBtYfLZtCi`U99UkSCF`-mpR{A|L8srAZR{%+!
    zlhmMs#c~%8vw4A_XbVaX<@`A2G4c+I9-{0~m
    zDucNkE7@LX`q8Wz6^ek=jDUkONxnh+LoxGXh}XtadZDtZtQjP(*lykBy$|VT^ve#8
    z+~WOAsJ#w(FJc2+cl7g(9<2RDCp+;8N;b$oALxAJS$)p*&KJz$Z#^I
    zLRk$PP-d$(V8-dM)uE_{Cn?M*yz
    zAiMTPoe~G}X4=dSo=+lwCtP2XLl+3k&qvot494x*MD0~Csr9YbtSlPurr$r3Pt(~G
    zwN~2I?CzZsc~QQoX^B{iCQ63Kv^!KRViAWu4`6pypuW%_JWjF>fF?=D3F4QpNh>g{
    z{$xe6&#uDStDam1j2Xd%g7RK`tRUa8O>ijL5w`Ni>O*Fkv^3o`G$E}_sDrqp1wqNVN(-)k_r_MWeT;mpdu2M1s*JNu5Ta2Q3T}O
    zhUQK2wUt`kRV66<@W-}SnFbwgg
    z;f?)vSc$MGc78P<{Wc9A1EGAh5BXG8x$;u0hXWb6VZ}pXOB^Donlb}%4
    zEiRDs`9X>u`Z8z5UdZ$q@1&1WPAN5Z0THIjpJ11(@DI^{eHBJLwJeVrC(ITwW%m6Q
    z9{klpHOd7|Niwx}LOk_0{ptyY)X(?d*ypwh&9NT+s=qL8WAyjP4Q%89zWlRJaFI`u
    zMC+~wv$)K9RZH(4+uTKS827WPp?bLcNX~5;Z2A%1dd#5kV2vBi`Vy##*T5GrNE7$I
    zZPzC=x}bB@>quQ_-P6nVhiR&7W-7wuk2(`XL4Yja)Yp->1E24y1Q0gDN19!PA2^Yh
    zH-9v7Br$M>4-G&u-tbcQh}~2%&_J%m-h;cw!AX*S#Ys(?i-X?yx>+LGIJ`bI@rgn0lE}teYyBfRvm~W!v$3O
    z-pkRkOPRiFp6hQauEP>o%agxe&&jo*zyXg@=0+;K^f04=M8=hYy~izvBDk-IGxMoB
    zSK>k!C$iujRSLeq=Intu%g}(U!R)YRbaR$|;m~d=72p8h9EF@uhj|)!<;J|$RYX_p
    z6CXhG&v7ff31g6yqC!D-zf`M8O7nTbd}xZ>#fz&5?XUrvkj?qd2XQjEFkXkXio1#_
    zm}H;3KYIU-a%wr4%JvI<#XsF>aC-8rgMxbPp5`ljr&qC!n&GWo%c>A5>EgjHspxMU
    z$Y!>2Qmu^nRCvr-b;juKKDEE=b3L__F{3B2xKp-3hPxUb(m3ud3;SIvwG-1Z6ez#D
    z0@-e}>;dqLP^iwpSI^V%PMZFxRqfRhWW0m;x|#LkJ>B0ctLudM;1`-0ITl@(l@k1B
    zR7hgb>3ADlc~AYidH>rESw^gms`%o2C8-;QPJuJmNi!=l&o$dqIjh~KevwW*^)d0m
    z({C-Q6n2|pCE{ZW)SeotVoalgQ|=xBXLQ4&m{8y3ib=-B6mcuOwzQ-Gei{m|3N9=`
    zy7(Mbr=u!?YX+RErC?Tzcsdr?v%oUq)*if0VvzmY;!|+Ws|OGwQ2pLHm7|5=3L!Ph
    zm2IfUf&au+nEAVCm4(1?$`o
    z#VY_|J}gtCvS^Q3k=!@vR>-^YG&PRph^{cdENh?E|+;O@f^;|HJ-q1+;Hzn(FSK#tqJd|S#$wp{8F_Jr=t$Y+(3KQ6i
    z;J>^OSihlkb#?D7!
    zABYd3uWyd`oi12Af4ibY--oF`BYwpN*DREH%!LhAOo|-Pqs1?oR=Pd=Vfus#aeJ@p
    zZY52(ECGj+N%`Y@xf}|*$%D_hL15#==kNP
    zTjMe&C)Lq&pB_dGXYxdo82p5nivTS2_{E^j8{yYW@d?l!Ux($-=Zy)@4=2gdEMSiJ
    zs>3scf>fX6X{pBgv|_tSIWM+
    z8^WxgO|K&rAYLo$ZVn{Q%l+Qf>E#G7U9<}g!IOnq)lO=L|0-IMIL^PV582j1LGozv{^RLqeht8~sqv|@*tbP+UHxLfBL_d$n`YTxGy&Q~-U
    z8phKUC+u`AB1XHY_PGr|<%j8vQ#nwX%3!+kMKv|P7$Ion
    z0K8QRMMAaL@n9gfdRN2jc&PZ4sx#GLkE{<~KJs`hz=I0a%v1=`);QdT44#toAIoDW81>zSj%9>d6
    z>1XDTep>G7ogzDa+sj$}_1z#3HEOs{U{7IJQjxH)+^5zOth=Pk!)f&H7CQwDv=%Cw
    zcj+VEnCD`$J~OGTR=3N;&O*Qq=MvQQq*>oz7@St^CJEEy)MS=b5Or0
    zt!|Y<_EtZs&wgWqt3~j*;@-wG?Cj8!L^W>Qel(!bTSAe)aC8f)O
    zJ(CvHb3GQ$j=WuCDdVhBWJvNn?X$Ww-H
    zS9qyzgSi!5LKgld`Tq4=W09*5KGTM>aFtPSCc1kQh
    zW?Lg4q=fc(ib(suM$vS;7>rapswoFBE;e%Z6L{LQsHjq
    z6*cf7@?mAy@8Q*seOxD>PHXPEfb0*9WR@&FS5UIt`HKH|b9ll@IY7Okr9pr|jy&Ka
    z-Liw`aLUi$1%p*o*6rG>f`-$BL^}DtA{f(}ACl-cK^e$GBX}k6qpdO%ETYR4?-?
    zw@%`n&=gi=?8-K<)@e@xQzp_2CA&xgA;OQJQO0zofz0WMq!|XT3xprHsw?Qf6-Tn4
    z6MQeoVVZeqNsdY#wkpdLt)`ADro!xoEmEU?R0qqdrxGkJGdhb;^hjC=M;OC~v967;
    zX$l2KMpLS^T7uv)k7$c<+UdnR`T;+WM;RXDw?K%Cl4FT{o^n`Hqi7zk|MXLViAC9{
    zsu0}#P$)D@7+np|6}1RlIhdFxu3U&EH$WsH@V*R5@|wxr4;2eO}e35OQ+sM?gp?7UJZ@`LXh?Jm#pZ<=sm4
    zYkbm6X#dts*381e@_cp03^ht{N=2Q(Y<#I$Ne^D2)MO-6v&ZMR|CoYqWwM7Ul>)!>A
    zyirFYD-wf&J_Oo=E|f1N=wynVsWG=GA$_px#RTY(x
    z!?>wG9Owg1q$SR2>PXJq7a%d%gL1SADGg;w3?3z`7(Ig0#_-&<7)9^5g6rXRvdu=j
    zpdZ-AqCcDiR{!HsyrGp{tF!kMsQ13XHNm&ugK@2;Jp1yA``3v!Z+{K+{=P9Ma;?^K
    z%#hqbbmJj(3%Fd%G42o^Dn+}XkFd0>8%vw6BRf-6Y-A-OVT_&pa$y&XkNoOhLLbhZ
    zbLE8PJK=^Gso9s^P{xG^y>lzB68`$OL%ycv{y3S^&8C=z4=%gJT(9hmP{X$`Q1eJ5
    z<3U!m4@|7AKBxS8xR;rXxje@Nuz3R%%KLvm<>95hCt-sloqj0u
    zqU6GqKvmdS_&Jym4Gm^0zVa{TF_EkS{({>SuzDRFEy&w;6A|@YgjfX@`(uxRJoscP&%KhKjWTb
    ziRM`MoTy5APdp{E=`Q$mu{Ee?{-id(`0JuO6c&KKLmK-YyoKkWQ^u`L0~;{tJ^ctY
    z&7x~+_s@O2Yt>D4Ujd&l^6B1G2Gps*FIP0{Tr;%DAf=E!{RD+s9C-EXQL1z$HIBbE
    zc6u9pm@dqQBi`L_3Bup0I}u0DNpv>VX-9CW`Cg4*v1j+lD!XjrF9$?Q#okXBe#fUD
    z<9IlYj<@hTf<*B?mm~wky6h}%&aTDS%-@i6F#eA>21suAfI(#8c5fSFoi4KkWAMS>S$v}ojY`sZ
    zZ(_QpLenr%%$-T}gvPjdO66m&#BGbDHxEf1SuDuoiKpHN+qIn43#83lB35M1wRh@U
    zuMHYdQ2aiaa!nQVJq*n=$@0K}>
    zR``dt3&`yRhpTMIq5^bACfzoVSeReA{dMh3~JuYac3KrrBzSA2NP4pe2
    zKoZ=gU!+T-z|XV4n%x^Br4`e!WUhS|0KF##s^gsV{$Yi3-SNEpoGGa`{rurV=hdhP
    zH5xsQ@8sJ$JXYfwocxAtMvJiTAR`9%=a49s7(CA$V0inc96!!mt!7O2K#6}
    zlu|fsL8!AeF^zu+kpO*JjzsErUR8WMA9W~AUw|!lI&?tp*WPILb(-$V4GSjSnn??#
    zn4PmB^v#>&4
    zNx1u%0c@NT0cmW2OVyuwLh`fv0dek`=OadZ
    zAEnAw-h=7yl_i#iTFR%~PU2oxqjw$H)nF6EQ}=Nr-bX&`wX6sz4J%IewtOu?be_@9
    zLk&1cW3KOvhYPb@s8FLv?Vets+SzWh8G_3xe1aO3<-$)!X60_*Kzd`;g-)$cvqT9wT(p0QsL+M4n}1V=7H!dz{E06b3%%(YWBAu9nW;}nt6ZNOO{kJ
    zhWY0Z@+D>>&E-u9^}3+Fbcak0Pt0dJ(-3yUuFOrluC`61r$>5LON@#~?{C5=QLNi7A*PRg
    zV;IhckW3k}A+s)=oYDOVFQwzmuzLOjT>dB{+$dbNUe1N$=U7Y`d`l8Mh#-T;*DsFO
    z%>8dPx7_;}qIl07CTa>}G=#euOP?bd>6ZWYp7A7Q|5u0dMQ@u~*Jvt-dX}tEQ}c8U
    z>qNyvx9l%wFl)jIapVExg$O|hCXY?2?Vb_0%TR@@F~z9ZYjp9{7l=nBDjQVvqxgww
    zMUA!yU<)+e*1FTe>Z1`Md7Z!*9z$aA+`zy|HOnJ=VABjy^O7tXE*u*}1(G9_R(G_x
    zu=!l>iNS|i^%~D*Pgji-S0w*(W1N2a9N^;r=?
    z_x!JyT5AJ_L~SMWB-AhPcw9f7j17}g^VB1bfHI%tgb?$~!p71yAacuy(%MNGC8p8D
    zt1$rZH{OBEKv%+8UAe=E9qKQr!>m$fs%SP3B39%u5(@*z3uP~#UteLs;
    zJjQwg{6wPBD1qH#@=L`8Edx3BG+oPTzj6kLma?Xp4CxEb2L?966rke#eqI;syTy8l
    zEtXF^1)72Y=x@;1;gJ{sJM*C6YfpwZOcXP#GGzkmU_uSwQw07Gej<&L8jOMakjBSF
    z>u%TSL&~7`GvalhQCW}QF(63{B3>HjX==Pokzw0h21eS~?k7h0=LBLxNi#3A)dI^y
    z`vdfW_}k@nTbph+Z`f^>7e<$*wb1Neq2hGaUGH@-CUTo4M{(B!%kiLfE{&&8gDSn9
    z#gVYv8%-VQjBeC@t?=^^WD{W0!iXTs$NxOqAL!5HB#v1WtUyF79xZ^=4BA$@=Bur@
    zQm&;Z(-^pltq`a?twDaK={`v&v@>0S;pdIHDmk2fGj{^s;xc0xu!HI;`!3QY+&qmN
    z!fBU5r05-&?z)fsdiJJy239$2U!68h9)Q`V*9*~`@+_lb=fwdvwXqeQov|A0#w#tVt;)X`%lY2(
    zw44c5Y^5%^I^@6Nb@Xj!$CF)L+-SrUEsLjBpYoGx*sI`oiLl6?AgqWt3G
    z#m^1k${^Wfjko1hUs1hT(x@HKvUFN6zmXBq=^BoQK4b=UzEt`lQWS9#reR4=4{l>3
    zS69loEb#Lpj1{~)x_NVEeX~LJ$4Oe!MZ!z9FixWP=;3AhM$A)IJ)&aMBM-W4bL0`*
    zZ)abHG~W67rnu4I$L9Z`?9IcWZsWKA&y0o;ickn;%bx5zmF)Yz3uWKe3>kzdWG`gj
    z*Rk)J2qA0sJ(Qhf8Cjoe?(g$F$MO5``5ni7+<*A1na^jg>%E-k>wKn;a1FDX0H1u-
    zH#?kahxeOoEy3T&%zwR8T$5_Q9{EzdyO|VFL2fWX&Kb2SgdIq)nz9@f`
    z5ujAkfH|_6>G^EINn^)FEUI_jNPQA@?m&fo=V?#<+1&`F+LKb2%f>2#2r>P2C=;x3fNb
    zO(jk$go`!Emt9|0Ja05V3C2!_P%RG55d5a!ofeL!=5`|J!50V37Fe|vAUs>v#vj>r
    z2e)*Ul{V>%Mdz&8f+qGF2TJjGEMD?}-`#J!{p=_^m?_1YkdLgee||&@4ftLxjS}+9
    zprmmNun>GsKHR0``R`6m!u>#FrmCQEPSg+m8~&@Gc^v2Ym;81X7EZLdJ}sn0?DDYZ
    z2DeGMC_oePHY|$S9fR=cXD&~v!8&h{1xz@}Xq&A9)hnV-Z-$7y&W)bc2jUQPVHsfw
    z^_{EVs3J)(s;n4)@)Hl3fm7v8*HvQ5)^RU6)+I~XqXA>Fwi6^6_WyYlLys{DALr@J
    z4k>U@fSt8tZYd@2E6=35+V-4X`CjUx=sh{bdwN;-{oGs^DqzqrVg)4EYV!fjBwGCA2rvfv^waZ@McD3JwF^2w1ck|SIL
    zUQ3oK)=iBI+ncoIm94zRn*z$40X;7BMS3!Cepnj2Hxc5`mpcUHn1?SqTTN=#EWg+h
    zzix47GBmYXr!zWzI$0ud^}o}BdR>-!8|ViE7p0Q>*aJ|=a05*(C!KsqsD+gR6KmO&
    z{6jCJa}Lj-@?Q0W>wG&|Q2N9Q=~+S!1gssk5i+#F3RR42XB2t^=jN*OGuSzn&}lr8
    zOhirS1d@A6f5O}X=DNTQ;;I6h6;Qp#U(>oXsyTvZVA$+&
    zQEDi(O)qf4RnkMwTZe6o4-NOMu?R)KUH~`2$97}bR`f7>P0Ywq2SQSoPu{Y+7CeyU
    zW#JjOV+{j?QEPgZH7bM7qG|X3>?7Fk_JZmEeGhOn+KWz65P;);Id>*Ri{nnV9>gis
    zPUS2D_Ii1Qi(S3knW`Kvv~g;Gr#W?{s|o(VR%W>
    z{TR{UyGCzKwipfSXV2^`P$x|_D7#!<(^*YdtAL!{B;^sVb?=E56IsApvHh=vaBg07
    z+bxF%XvPT_*Rw;i9?lGmk^lFEg~Mez4Rwgu@apkSA${q}`zU^~i0R&3c@0};Ib-iy
    z0<=`6F3&^s3oA%P5fV<}&J6EUf;I!Y@x9g>j<@<*Qj1QmG4IAdVlAM#CW#uu7;j(n
    zJGNI8SWUN1I)PWdTUzgBD*^GyRYiu|$En=p;(jli9k#)cCg{$iS_(FI60AzS==yL0
    z=;EFa>Qq1il$c@u{~1#y4cH`V8Qmhx-JRzbGIXVM5gPsVpu&0~8)rZ$(FJ$jn87V{
    zt2#4ET!+~92pXUQ)4e*=NNUqrw(TqEdQkCrm=3;wrLw`04B|PcUp$`Vhzpuo%APmX00A(Rx4^x>kMT!
    zw=z6f-X7ksu!(aL+AbGFpx3Bprh+L!WwR0)ukc(XnnDX^(u{00KYPh$w&&8;eS2aq
    zl!Tt={(u&Pm0`)+}6LB^R#vQU9uj
    zlXGOr3or6o%8JFho_ivMOFN*=Rr1tDIH*Bnv~i7Qpk@?u)UhUSGQ5AM5S}h;R$tXU
    z&85i^q2PH;DUfH^;lfmPi!XwttpFhamUCL&qjK%`VLJw)G9se<%GD=b<1KT?lC)L&gs?ar(g8!6<9&
    zLAL0qmR$#=XaX!(;~9^xpszm3H5D!F=)-R;>uGJgUqFBcE{Gtp(ambSEj)?0ei>JE
    zz^LmvLR$)ikh
    zvJVKm;jY*_k`He;iCBVG(6%^PG0by<7%xY=yc_I>O=5r^sLFda{qvD;KJu#BW3{_E
    zJ>(Qj6hC;v!@-h+lS&{5RNxP95O34xl6X>SjJ6RQn>Z7IQtc_0XzrBNNP9vb?tweEz%Z4}sbT#`;J*R#va1(6N*AMnV$8
    zvvRGlRh4N6Rtjc&Mrq>FIR`PEx?k#qnkX?j&zVu(`M?u#a7nfN15~n2bY&~hr@p5T
    z5D$Px)S_2@n57$KBwx@KHPiaVq5sR_)w}CCHIMGI{}GqHkIL|W_MozzrZH)nUmyq;
    z<&G0t#>O3b2S|ng<35Zn*KSQS6^WLMNlUiAVRiEx7=~;%Nkt)Qp1AG)tkw7M1uN$%
    z)ua#ad)h;&gmPY8raRs~g>1-qPiV+eKtmi$H6evAIL~l>o)^9(J
    z&30NVpJ^DoG?UZP8KH=idiY0=Ei#I1UE+Eg=<*wO|Fu?T#4n3X@BN&j*F|uhUV48|
    zUH?>E=Uj+^rw}bZiJ0lUiXMa;Q3-o~*QUTc7X9;kSwng8T5`i
    z-{-PtuwX@1V+LMfeVW{}#9?WJt2ci>;@+Y5L`X^AD4bO#921yZY*~6^=vBne@qa&m
    zLJ6ej*r-H-aoUjs_0L#%QC0n|Coo5mt}e4O!>%82vAmU#;)|$PG>N@LGgLOq@$*Cm
    zpc$egwgtZwPhs}I=U}g-1VgEKju*oX2p|4F3?=VM`5pq@ZDMH*vle`>0yi*_
    zK^C^e%WT7DGwE^)ryHrtM1?@n@|G-R*U!XhIaVL;`_JEJ$yJy;lRszxZj>E8SSvif
    zKF&3VuWNVfJCTJlDDYfWz^QQG>CTHR_$0&se(8%Ip+7s=o9E-zcPx4PJYP%y8Ie-c
    zUHC#J{@MG8p|j@NdQ>FP!|iP)^CfAKS=_$n%9@E!yiIp&BQv;fa%ax1#c{Z?J!^GrddYAjPRjivc{7AR#&pN;
    zthu0R*@yH&X-@jO-nKe6x&bfylLz7A+Ha?d6^r-TCgi05&b#BFqyScJo?F^}&?Z-A
    z;*zQvvi9||NF@B2g%fx-^=Do=BiqCc4K@^H5w1deV3sC8N+M`5RAO9xa7``?5^8vo
    zZ;-HQ$d&eG@^Jo{N1Q$t^2UF6z%XL2cYWCi2(X+=eWpIBQPOSET50duw|hoiY4m;v
    z^q$W5I(R=Mq#!)uiThq&-q@_aV0r?6DqxiILscwF9U15?^FzM07mhEhxl7p!Z>Tie
    zmVYwRz1yT(tp8kZ$h6+grL^^CmnSoD1cU93@!dCvUy8zcA0hXH1b+ZmokUfw^!WZm
    z$`Q$#99Gox2TqzZDsP#mikW$c*NjZ1J0s4@X*P45tnemnaD#3<1DWbK#rc!9E_5Nw
    z1V`zh14H|1mTZ2oH`WK6s{yR^4oQ|?q+0sUwn~E6Z>Uw{3*6p
    zjo4PW$--@m$RZ|_BxK@s?%TA}{l$
    zKPzW6Bh?)hs5@psQX(k@ZopcbU6EGb8eWK9Qg&z{00qdRHoDQT#^GP^_}S2?e})k
    z+qY9NQd#Mf;@RqHJzo`AtUl@g`3PQg#e1-{zF(uYrjc@r4`D4D6L<#YbB0fJwH;@-
    z>jI_xI{ou9>r;+A0EH%e1n*d;%JrXrEcAlRV)J$eM`rz3`g)!W?a%Tupp&~}tcYit
    zT_+xF&=%zcFDWBH>#K!HqRxRP-kSbX9RIhcIy_r>8*0F-1TJ~i53DURKrG8Q%k@-V
    z6-=6CuQuZjH7(fI1+79}X3xMBaSmL>iFgenvMBmfR2R9IS=dPqRzoZ+)5W9tH__rS
    ze=I)|eN+JerM
    znoEgq;)RBg;a~u)OS}fkDL&SA2TJfeuZ)3hZSYS?FY!1Iv7^Rn?^m=MS^nU`PWpWR
    z>-{{fBtJL-`1Nmbu-Ad@akFJc-;*U^<_$jO?~A4C>YGlj3AIr@+?kZ)rZU^Ktc$&m
    zQYDSA1FyTUI`YtdhXELMotTMnQ|cIJzNIm0Cn`CZ3h`P>`z+q%<#bVQAb8&wEIl1L
    z{I&RqR$g}A5Q#0eoJlazf^l(YE^ouq>JT*=(8MY56)#SUk*|2s)(||vyXMVEtUjGO
    z7kS&50u%p3?yMt*p<`M6=;uHIc5z|cMdf#Z;FqNa&XQL6Xk9-onCgvaD(vvve4NmU
    znV@f+Je+k>_wM#KGuzy>c}-H&18BM7V;s=6PqO6K@y`{Pt=MF#yas$-1EC5U^(JO0jUO0D__=CZVZMgvE`#*roM+yG?2*B5j
    zjEr#bfd8n`&du9(3iC{T-z=>wg1-n8(ylFZO%hCU-T)ny29(N`$z)V;!V>)vicV)v
    zdj}!ot}SaLk{xtRL6w#A_02Ti2M^n_2YjZOedJS@odafSCm>pR;P85Y%Lg!v
    z@&HbR!!)OPjT`z-G|z_?6-}3Y#m?9y{q3?!$~x#y^(K3^Ie%k$FL5#P4YIsj9!!OR
    z=GM9DJ1@II$!39wA}=g$Amy3NK`l_lxw7A}iIpQsd+ZPnY>lQ?{lL-)WI7sS;B`YL
    zJQfomT>J+-j+zE5amVlZI7IPN)78c6;_#bVf38_)aScCW&o^*QakyQ99kARos{57j
    zqtWw>(fPmEyQJyrgAYZtj9^#Kq9Xf~k)p;957VEWdI69rm!B{1U;MJ69cl~2yO=wz
    z!{6vxOTMDi2N$!|Dk1;-kIp)6D4%h%;g&~gC|o4D@0nQ+I*9ZmT9^Z+U9P2D;sAW-
    z8zi@0%d(zFQ4KAyB9Jl4x!D?(-+g#%Hmfoaf7C(jZSQ?8
    zdi7oJ7^FO}3qhvGN^!bZA}EmQLcFl<>6aG5NI
    zaL?hWNlK@I`XFl5`KR6W|DJ_#VPXMbS9o%3Dl(XBd;Bx1^&8tOSoL!oPu}S&OM&#>
    zlf1r`*0WJwa}%SeJ21i7TDWDH8Kv%r1*_kesynOgN3a1EGK;r2G14z>#m&weXWWJZ
    z7s8#8=Wn@|o;q`^2udpJC}lI>qW3XR9-Fm!`YKR~%t9Ft-~FA!hTll5dfJyqZEl^v
    zPv3l%cX8qUpQr$U`tZX6pbOI6iKuwZ-u+M$C#hd}B_0h}-VjD!`b1MjEvzJg0e~xo
    zKkt!17(%u*6pmBo`y+M!H2|Pl0#eR@aD9Li!NE!)6yTt5-XC9^HrSDC#YzA3(LvEA
    zWN3Xj&c1(oMD3`iLj6Fe%Dbd*6`lF~1--^t+%FE)oCC*gw2&xI?BCdhM0Q0D_9{6g
    zqfMV8oFY0?QjPUgBz&AS6->fAL0b?#HKz0a9ufkQv3sx<co3ojdC|@iI*{zbbV1s^@h|V?tSpWAzUlp_T2D%fVIs>`l18AHT1%4Nv
    z@X1lUNg-Uhuz01Ili(@aUWKp=Ci;R8urk62=bG4$um_LXlO9}YtG0%54|12e-6veZ
    zPUHhL#(s6*KbE46-x06DK1PaqOu4qlr|s1A8cF|^YH4ia;9cH08UQ`R(NXcBwT+E-
    zK;(YhdXcZ1`anvGf)P+n=fk?5>Z(ov(zC~aC;>DyF%d3_JbHaWH-<8)?+M6%as+$j
    zX)Ns*maT8{k7X%+`(xv)+;b1GqWF8d>Q+R0QbDMLBjxNKSlB-5&oZU*Ffjd(YSD(>zyVb~|
    z8Hoi%ulK!HUTt}ryn%M{j7tsT$TIM>;NT!5M++X2mG7Z8JtQBCTsmM+jyeyj{qaUE
    zno1|@z&5~uA;v2u4yv==(gae7lq;^GR6|`?(9fxcsxJB7t92fO9m_-u7R*t#cWJp-
    zf8sx8smg|q)-OM?{=KO9eFc4XF%9?_pLED9oW$d9QDCGp`hGtmB4+@X10IB9ZQ;O&
    zHQlAA%~bvU{6TZ(XG_#sW^G05GH#BhpQIWLPO48%{5-wAX<|xVtV}ZLR>vh08@tZ-
    zB0OU55FF2G50{|MN2LZm2M=KLoLd>2>vKYsr2S^SlHILx4~q_+iXf
    z1;mZ0iTu0*VVGhh$TtorN)1Dfjy^6P_X{t)dn9{@AS^7*b+fD<4&jm%va@7e8jZGJ
    z*P`%Ipu_&{M;A^~WA64w5oSVT9q+K_XI
    z_a#gt80TBTr;XQ$!F`0HrBSa8zUuz@a9#fru{gjsn-rnLHZ
    zoMOSx_d@w!)g+L2uUx&E$~s~+$F)Ous{^NBsX|HE%JRSRoHGAeBdBaLsBRArBL@u^
    z((U_u#&@Z?r9VbCm}sxml9itb+fvB3Jg{0l))_7wQlH7c|IJ3ft@*ecF*muN{0XnJ
    zt%6$XhINyEf=pE>t>!-;@3f6Mkv=|yT2$S>$7a_X<^+kRT&1+#P3&}S6MfNhfO!(N
    zx%3C5UL?Kv>Ap?+OIDpAT;7Y+UW7UX?X+FKN(0d@IB4?&5fL1D^6UZ&Ce1c!L4Y+@
    zYETI`KN(5xJrkIccl&bny={B|Y8Uxg_^;W%)^>G6|J>iTsdFNWkC2(bF2*fV(7uss
    z$WoPsJJShw`6^OY2?>JHo>Cd|>*2h$7POiNtWB*(i*{KTlJ0ghCVVI^qHifi17At)
    zE=x5p;r8$bq${0VbPpxds75Bx)A36KHXQ;11D&&L7M2iUtK1#n@vYM*2ZW^1s4gU9#kj9J5BL0N>
    z1L3#aml+6ch?utV3LL))JBemWiCl7z^(f_orTwJG$Q(|~^u5%g@>1aRH6Akj$l^w}
    zQQEicZ~jkISI*3-T;BfU{QP|5C9iSa_hki(^^rUX1(F4>k&%V_WvBzt^)U!wiOSDs
    zEj4eW2U#1Ol@?#%ASvJ~E*b?8KVRbzufE^~sC?0%GL(_i*0yl$Vqw8}u$dQntrVdN
    zwb|OylA24n?Cj}$`En=K@17*DsCSF?FaL>UL&P65G6cHmDfbWfbOC{D*-pvQlUMpZ
    zhme{gjBB+QwXNW2-}-o|WAu_5^tUBe5^d17e$8
    zxca5pd%OMX*DE*%A};2`cdW!Ll-K_!@$Oh%;S)I-WwJ}6nAD&cHD9)*ku6=O5>KxqCA-3U
    zFWTs~+)ohcVij!Cbc=ELQ)*enkW%dCr&1#syq=qsYv74;(IjA#w>q8o1D-5B1EBn!
    z%WBD2sx>uqSC7L~Av}J?#fpG!2U{oI#K!=C3Q(qO-rLiZCtk43>gpfIgX&X`6iOvo
    z&}Juq^a>MF{=p4m4q9@RZ2POclf@d5t^jb8KP0?FTElI>uLLC*L3%jMN0X8fKJ_`1W!uH6^C{0GrxEUEdp+7Ck|N;%9KSw
    zMg3}iK4zY!aevW@;q=z4i~Wq`!xM(n$!^y^?s^C^+fSKT#P!7hqK5`SGiQM0+o5ds
    zmmIbjtQgXiM^R%0_J8
    zod%y#M%3ja?Sh&_*p9#n)VE?~(ptUZYyG2EkX08QWe*1jBzlDW|??t)z<3XGvwD#Drcsf|`pUtI0w;It`^~UcFAm^t3>n*p
    z$5E~d^}$^y5@t0j%qV2ousG;i$_c}1pvj#H{X~UxNnv%MT3iDq}&kw16b)#FpS~}3wuF@mvK6~`kp%I
    z7+jD3AUp>Kd3t%_Lb0;U0s%bt7DGl+y5E1TN!Qa~9|i{0(m-zly%1`DCOlXhJPa4Z
    zWSf!>HO&Kl;6MFM9V5c$O)d6bRhlP?5vNa!;AEDtGD3|W$cLs?`1#=b268a`u95q4
    zW&s>Zt6?1n(2*u_fngEH7)O6fzb?(uT)yZouQCcDZn#k=RgirjzKq!^t+@RoH
    zzaXHl1pZb!fl_~y7=sO$Q$XGkT#u9k^~}v<1_lN+b~QOa;7>8WJz}I~%xnLc8dQua8FXPG
    zS}aw~*1Sux@bCdMb&4zpiEf?~7^TPLzK6zH@|R_b?>QAeNhaj|M5K2Fk>-pa=|crz
    zwF*`+qnw2PKKY_9k0XK5f!0TxZ1H64KFUi@YRn#1rd=UOEEkhVXi-Kk45Rf{brl~F
    z3~HeQo5XL#o1;di@uM8L%t7R-%Dq8#_GWN+LYNdmheuO#3jll#q4JcIEkUqG7eIdx
    zf^;37#fb{ml?Dm{;5M*{l)?bK>>b0{q5;;w3R%#ravN~6^MI~LH%HS@MtE$34@SqY
    zD0;i7vyI(Qrmt~ih+3P9w|cGZzq!KySVvrv52$m-ZAu5;Pb21ecy%>-fiu_ayu2-9L%KlFjF;?^65X3>-r-&9f=xW_@bcm^RgjD
    zR#bA$;L2D6P@M+};gIC=3;{PGTzn%gS0d-(}i`H-I7i>(){hMEW2Wp#>SIy}tWv`B3tNdk6R@z#eFtnTB%Y#Atf9
    zC@~D?2XRUk|Fw1Ek}kSM`S2TqIk3%@L_Z&qufX+Pcm1GKc9gcxo8H0lT9n?2aJQcl
    z^xb3L+&e!RN=m(>{hm)bV8U}XDU3pv{QgF5Q<{5gjB(-kxNa5Ojc;Sd>fPAS_Caz|
    z4~_NWOmW#-5at}a0IY|WmX^*H$Pq=hP``7-OQEdJ3)Z>rX1mU7c*c05<4H1QrKrkPtCashUIKmQ5-qvjV~
    zV|XHHgy(IyM*9xV@gdV#_@q?q`+7Ja>$XGy;V@E}h{B^0Z&NFe>8Irq|eYtAG|il`)ltRHG4Z1PDrn1hCmSVXzL5e
    zxryqQJQ#Wmast5r0i(S3dhXEVlgFm?E+T$Kn~l^rLLQPG2gZy+
    zS<)FphUhvh;i*jwkTahXTD%ODP5@%&cWk(gj1G)Wibvnccbm95ytl1Ob?@z?$WSJ;
    zfn-^C);hG0K@E$YQ(9`hxN#9g3WPkt|F`wopUhLX5qg0e)6>%*N$CB>W$WQ4!owiB
    z;&5|f8H8T3P;nWc0Oc;J!?E=jIkd;Ln%m=7@l
    zeBb9(0VpF%2W|4Ogcuu^guyG@ht0Kq0pB{KTatU%@$!^wdatTADq{s9rIoy&}~6}H-LTrMXFH~oP02Yn9U``gP#
    zi|60FPf=l5J$Yzw3jYc^aYDX&kxR$2J
    zRQm$Py)cmmQSoKinfZ)Gw@?Sak*(Unzm*TrxTPO<)vg1Tv2!Ak(SS|hyp)Ewtd`c-
    zJSa)Y42%Rt@ux4ak$AiGm|w;+spQ3WVjq3GTKQ|#1%QO77~T*FX0jA25b@Keq*s(|
    zkY_2jcA^jN{(n0jws{a(hs(tRtsc>N6Erk6Wp}n(;5b0ul64q99$u1fo&3nVp1fILz
    zftxm0aT)2xzNp+@dOpp<>z%ahjZf>LJUw(acSBN4dO4|*>Zr8d9~-pz^8L@u9WtpgDnZrC}WWn4mVR}?J_nlM9h)6mYz#)Hwyin}QI4;QuAyJ-&MqeMU?01U!S
    z-`(b4uKvXJPasYJ#E*sGUhbM7YN?x){=;(AKS465oj=2*<=L#;+51pW%`!Yn;>Gjj
    zF5tt7YGXPrkrN!hzwy0E;;$%@n6V^ig&gBux;Yw508lrU*uH<&(}dY!`dsVWQ!&YX
    zN1yp7tfUct&XWA}v#^soUw)Cs3c2@7OX+^4;LV-5xPR5%G2%O1%%aW1%dyUdms;2?
    z^ot}w9g$ECvF~)Zq^(s63~t}{4&==vv}k;~odch|3VyV?epF6ieFMPp_jpiy|1HQStvrHmK__^^6-oFK#|*(=>bL
    z!L)6N3Hx8al*;vitVT27PQpsAgDrV0fok0^H+&*5kJz*r$JQ8y7XK9{hA$xYYXJ{4
    zo9^Bd$0D*_^TQQ{S>DYUT;9*PlmOjN@KA**sJ_fW?w)UtvKbsY27zA@6KxlVxOXLq
    z+c*r=I@I9pV1O3C^ACO%Zu{52CS}m2k(U3coLX0T+O~@#{+@gP$ReSIF1MjG+yFR2
    zapbfd7F+dMrAi=9VuzKMKYlr?X(@UV-)QyoKKhr?P$XO=
    zyCG~u>&t^WPNGGcSx#R{d>{QxNND@iQ5UkYlZfwiOokg6$!NnI!4Fl|*m!34vitv~
    z51RP>52V?(JD8suy^D(aDY^TS>}ZQ{V(2r7emr&o;lx>HeJ}3mLJnK(dC#9G;VW8p
    zm5}%Dj_hs7B|fQ8r;U|mmj=B(v1;CB?f_gY^sBNZkaTfpc8AWDsa)*`w>}famo`pw
    z;^ze9;$OyP
    z-=AM5hbTyCWsYPRaG^pwWs_qf)0ejc7f>7RE^VR`y4F<+JxZg
    zFYC-rP@l^4@CWL@O}OGJ(6i1ciE=W$)FX~`&<>CY4rkS5{HPaq*OhG1UuZmwSxkq3
    zKMFoMa_-v*;LN}f385W7i^EzRGJM`iD0+snfI6zcmGw(3GNkZGt0|b{&abQd#lX;|
    ziM9BLAk7ajPgxGre|KLw8Xs&u?M{Q-iag9hHf;i!v?J2Asfw`Dxpf9D$1-i7aH-o?
    zNIyRa0-qsEs6lwv6%UNkNTG(`DGoK=0s;by{2BkQ;X=)y+1g$OF|-u8h(E+DTv!aI
    z`^9amp?;g;9g*mK0(2W(gpgL2T-q%DlR5cBbtK2*c-w?;r;qTn`ikKC+?gJ>b}DU`
    zxjQXOGL;k6)*b96I;pg(gcjKSXMT|KbDzhi1GO6J9y#~$6q`VU=l#sg<3~L2wQLQ~
    zgnjsn-|fa3liPmfV3nvHYAM%AQYF$?fz7%mGd&%n0D3+mN=#%hGSEUN`irJ+nS8`J
    zTG;4Tyoj(?K=m2__X_K
    z(Df=#EE}Z(uz~=)d6%}f{K>Q*qv-?zDc7`(4k0<$W-I<2$b1P%YanXAWvYE9$i7f63;i1QZP!JNQ3MuNM0
    zphXaIDooE0)E@uSm;K-FrHyDa_vD7n?P`Rl2xUPLNAq;?#kXtek|<6vm9is`BEM~&
    zgjE%lg+6g>E9cDA+>5$`rt?_ZggF+Rx9EtckYhk7(>_gKbr4vMsjl)0J-BQ*!(&RMBVYkFXw}@i1
    z*b*-1SsSb+qEler{WQpuKSd8NviO<1`13fkc;U9aC$h=6>!-k^=%U_bxxoh;#z6r~;$F
    zu9Ux+vS5+VL3NnQFs8;U9>UD09Rb{Yti$(X11Ua2@rhxeY%3mfr!ONJfr~Yix#}=K
    z|HLYsRp%)3ysFO{Mp77n3deh8UPa&cCSHar~M#
    zEbU|XacEHQiFi+?#y}pq=+jBDq>bR&F`!Tu80ZNmqXomFy}LVmKMJAfy#Ox`q$tDG
    z5?~*2XWx_$GvV#6_0)-cmB(g37vy&S?fLff0ubEt_>^iQp%SX0y3)WBSZ#xlR8Wv(
    zY#cbkZOv7TiCq-^KSbV9j^%6TqCSn`AT=flW11@juoguqb$kIF-Yov$@b>t3?8&FG
    zNg_S^$P3h8OW*a8NQ5YLopAFq>o2jaj{tFd9#taOl6wH9137~zZWag9pLWwo4sf}1
    zH#{)sd$ILkMwW4=QH!<@)%hN4VXK|9h4uVDuy{9Z-Wvow^2-6Fxe7(5_x;qzIraIuw
    zvr?FxC6TEKrZOqZhf~jsOLcEzA`T$$w(dTh0ev0~w0bUc;=xhP__Y<-b{Vjazq?Bz
    zr)s_aLWdev&qrv)o`0|L#SM|#z3R>KPwry?F3j>`iOYHj@%Psx1$LG7^@rGfcY_jB
    zmL8g6`i?$w?6mbi|6n&NhW0ek6C9{?+t
    z09W{L(QY?zCl?_Of|KwkQao+_#rQM0?LL6~HQzl;Tq-Dpk_IWHxHxh8SI5r|j*gag
    zns>X_3o@cXamn-7@v?4_Tv-XzU0iYOge>PSOs&%
    z0_Z#UjaqR}s0z-fpkY)X<8Xs;QJ_@+`EZ1tn<;y7Q+qVYYQxp>p55F8A31%oUO`Uv$OL6dKFUZd>7ucypRxcR~%lH2Kmh>{@y%BTaOhQCMf(
    zB}s{1DM!k5-P!lpshKdA#A_(U8^#l$jPX?Ch_kmRU|YT3b|g_fIgQtlxo#)b2MARQ
    z%w~Y3w-8uOh%@v7C`7r4n&#V`wCDbLbiqb4!lRAE4f$`?`XdZhd;mWFmyt|r2`PKHn8+}JMOi0;lZZw85OpxUn3!v#mGOYB{F1u_YjlH1PmjwV5_I>o}-Z{fa^
    zYqc_s#cwkMK91TCIbMtVN;?=aR4_7ILf)zua`TmBEhQ})0utm-?~)O1p#b8wV?Su7
    z@h4rXXGS<+5oQ81-udUJT-&fxz;7!}96=Mud;Iuvr_5kvUO=B)l+W}lxK>48be=U@
    zsQ}{Q6(Wng-dTB=Nk){twG-JQhYFNRj)ON^U0Tf@U-x00s1Ztw+?&^rJD>p^>dDu+
    z;YIIR@00xVCD$?|tQ71!AhNKlAo$Kqrlfk5Pb6pcERE2VY_Y{tvuLZI~}J|nx$_@A4P2JXt9jsQR9o3N(YStFl?$(z7!pV*)>a=cB+u(8pR
    zYVP-bZ`J&wqxyKa86eVUPyKl>4)%Uf)_Tyb{jLFA{old_)hj|b1aR8!A%kR}JJO=2
    z2V5H!LMyc%uJS;;%lr7u0EDgr>l_tMXB23!U-s(Mh0mQ$}(
    z#>$E*B_#z75|97e49k97Q_-ytI%;+^((%_SNn?|PSFHdP#l=!074cB0$r_oF5Q0lDh
    z64HGAX%?sw`dYx{vZPP99&Vy6U)%lOp>X8&B*-|hGZy-$M~AX!Fd6X8sYdBd+gdZr
    zd_<6M!%Vv7c}aCTk?5s<5#ok7caCe(Z~>=)hyJEl)?bz~Weh&EYu11nTi|U|ll`d*
    z0(c{k3FFl-$s>k&OgQ;eJlf1jT*`3jSj6;xlbLYFe%-NNx6SF1hub|51SgXcl-d<8
    z0C*5BZ@A$U(!eL4^0oZtJ2V-Zo&yyivj3eQ7Z)9N@@cF@pTK@)9*oqKc^a>Px>>QY
    zK!5*3i1S@aOc=>~pr)FWV-Q769BP7#TyIS16)jQTghJLE%VEvpGv#3fJI%&OJxZ-O
    zFiGdtWYy1v!2K67_QNluej*`tHEPqIHhIeh?vi07k@Y$v9{;51d$qtkhsSUvP#8b=o{=1}0${+_2PKEx+
    zf0W#qAv_9OCRloCN4b_8`p&f6WOBpLBpOk(`~fLi0Dvm+y^-{19+wM-xfZz@gcjU7
    z|7nYA&VtR9+yH7{g}&UZjF~CE8?T2d_op$EDBjGmZtOO)kzK
    zwmF)dYe#x+v3Z*Y3)XuTb6?e;beQscfm!3pqN{I{fFTY>_>R0G*8z>Qvk
    z;T<{~qJIuI0P(!cKY;!dsctC?+BMBNm93mO=JMnem!`EVT9^8?e|L+NBROOthF>3e
    z6j+e)xx+|)TgEIcz<2dF(T{{rT7?X=l4CkxH!(TdJ61-i8>N-(Dep7pCgr(aBG5c<
    z?owcYV$Txrje*3>*P_pCVsJpx8IjcQQ!W?q4k5P)SwLv=iTKD
    z=m@zz-TYLgCs-?&vj52!WLA;3aFHacfI_{YB()QC-0oYID@}2f#XrILoDKCnWASCw
    zVdS=>J&}cGjfEb>b~#tnlS!8};L`OsE_Vl@E(Sfxeqw2xO?8WbQoDPjb3dW~oEYzy
    z%T6Nw{AZ#?N1c0{LO$`KmM?@fWIx7H4FTRd7HgdfpVTo(Fe6(8o>!%N@>Z}!bzmuEWlGOhu>%M%=q~(|
    za5Be<&t_@mlGPu!e7iYkv3kWIYbQ@V?He*sRvB+vc}Y(jL2!PTGIyRde*HY4L>5*G
    zT7U{oXwbn`a))-f`x`E+BKAb2I;kWzc2T+DMWuT1fC|wi
    z@hXK91D28FvsS$+Vy56tu>b%u*rkM@VWcL5BG*b-c~SkR8CC=gPVA
    zPnD7`Vcd0m)uXVYPhWRXbw-2Rf&x6}mKPet^xre|=UqI93Sa3Y;~$=AbTKJo*<{~)
    zD16AyrOdwm4IhDSX-lQqaP8e7o$%0F7njn-4AReK1+-e<#GGIYCjp1j?#epMcTuUDXTHA`sFJ{0t*~JfZi6i-}C~6|&H8kOwQP%t=h=|QI84IO*9Yti(
    zNm}U5+HK$EaPOc*0L!^p+$j?Z_Sc#aryl>SfTqJ%Z)2K`2f8Bfndwrwd|B`2@n-ql
    zPqgT}E4yzb@OjgqvToL#`IldyGR}#2W9$ywte>W4yBPnbMtrE#^_!h#tarzC%)`wu
    zJ4)OJt6N1Z6=}_CYE(t>hiVmxgy#!kn;$0t9!xY^F3frUP5gcC%)paxr2fl0{RXWd
    zZ(-qI_p?e?h>Os@b+DZtQyXqqW*k|@{jpb;o$i`u0sxKq7=xsXBAyuS*YIP&op~T?
    zn%0aB-2ck6Mf-yjFJ1x26{>qFFn75#x!KFr@l5H#F7VmZ*$as_8|$5a=~-?qxAuQk
    zAX}XOk;c1OZoUl7l4nJQ9N@IHQ~*qq&xbfXOK6g%@<-*<2r%hB{F-V2!ix-;ujgj#
    zRHBX)mex7;Ur<+-0n2gT@D+;dBc!Dua=?Tq{|dV1UbXdBtfYOrfsC?O1?kK0EnQ;$
    zwefLq1K?u3bff7(zH}NGu;aZ!w0KDC-B_N7s{vBZ&o4-|ikmstVWn+>s`tlhlcIBQ
    z^72+!PPqwlD#(atZXw>O*XBnHIrh+6tF*fq($l#V{a+-VWk6Kx8in@&1JX(ijfz8;
    zAgv%EAl=<5AR#G@Ly3R_5-QzY1JXH)gd)=2B@H4ibwAGie|~s2d*1!VTF1;*ZK{;i?eORl1I|=K7||?F4xk$g2pe}pAs}Fv|fkmJ6MVTdi1h^@08HaxJo+u
    zv*Mq?D__lMD+7G>XE2jZlnDOkIP
    z@|HMn#V}tJXPyC$Gbjcw&mk8!c_rM6iO_U){QlM0`Q#BfhRNVDJqF9yS~CYCDPoL?
    z$^`z)UkXoPkEX&;>JxWx(2o*SWP#*fptx?kdZx9!svuv+bo_Ru
    znn=Sblao$s`QN0{t@UujdP-ebshS)ofq6y%6@K*fZ{fb*V_u;hTDLMYz_7KIs1LnA
    z)@11IF!t}YObpk{VfIBG8}ENh1J`+nuYmnS@5L*j;Xs9TA9|+fEHR1PSFhH5{lV2G
    zmCG^HHi`~J1b%&Da?v@aqz5y$ATK4JZDJvu%K9T1O+yj&VB$dQm$Bap=lxCb82FIXvek->=hP+yW4ROB?OnFZ}I5+p%K^?`#N^y~lmeFe`OY(R7czAUN
    z*7$`xAW0Bxm%T~zK9Vj}hZ#qWz4;zkn{YoMlOn1rZY~i(9mp~GxT+Km;!0!~DIGL)
    zUkxC}fZLy7-bdwd)KPQUR#<1OSR55!!N$`fltjfDMFX8hCH&yBCLQbszBuVP7Lq9Ov7+d*D^VXdcQeB>^Kssj?52Ksr{rYo6(la`cL9=t}eGOBMnopQBI)ZKNdN8#O-iCK&$mv$07et37+{&6r+
    z#RWf4G6D(h*uXTELqRgqHG?gZeAvV#MEFnauSi9J5isHoD)qTOcULo0Ds+|=gQHEv
    zzpt1Iye&43c=h^?)%e=5bz@tM;cWC#9JS2V8pU;RgNffH-~);r*Hf0ysVzQ8`u!cy
    zrrLYAHQx0a0JtHY9~2JAHV2a`MJ_U{GC3lt|K8g3UnU*2wAqU2V~J$+W#!a-Ih@9@
    z>0qt{fbt`X!L?Y#$>FxDCXH#-u_AeM%{Z%=d|Tgrg-5(S<+|@Z=oHX+=tmXafW2+t
    z&qTfhP9Z&L6o9_1fbS^WL-O!*F{BD|*oK7$b0LY%@s;wf24htXSeyE@+w*8CT!RS5
    zC+1R%l+(2;_(tq3D~3R>Si{31_HrE48wcFK?0fX9XT_9T;Pbh0e5nZ|0>w;TyKvuJ
    z6mTaHPs4QZeeTBF0|j3R{zrb*XgdwV38WLu1f0FWip|(PtYtPam{94+6v+rEP-g%N
    z=T71-QEa6Bh7seI@l5KgI=kI-k^AAP-i08uOL%8u%lEuCoHlmU?>CaWU94i>iXstao4di>QBZ?Yzbt;gV(9sQnO)^Z>
    z6s_yu4b_!X(2!G&G#w?TKqPuxcXyfT$_ksYut%PyYzmG7J{vay?I&^!pOGY8`8#j!
    z%#mGr{Qkr|==ZavShm7%&h9&2Oseul;C3iw$@_KkhN3dyTmfa=p`@;Y+imPn_g84lX8hrvx#THe?I)Qayp1FsonL%ZBf
    zWq0omu<@UIs1EZ-y#}&mXB+#zewjj<0?<(8oyk@}B=)efBs@;1r)0QEkPtfxmT8r~
    zh5b{!?An!SZ>a}vdJW7k7b%thoJC)-{HgY9&kX`%j%hV;gMAA>2pWqAby2w)Ie{%g
    zOlf~Jff4iWcN-v&&K`=g5X#PdO9xgPhtj8D?;*rtkGyfMi9VW9aAfP1RRKIDSNQ^?
    z3BS&$j}F3@oCu%;5IM_f*vF@_)}{or{BE8%AiDw{fM7l+zIjyKoIRUsi|JznxQnSq
    z#R&&Wa12C$hRQsNJz@(iRD4I^#MB{{AE(Qr`2N77-e=0%z~G}&>zi_2`Bpp^0R>#Z
    z55tV%JNPixzKOJ^OZwJNS;7%*%x*^hH}(IIvEj;dv!~h!(|ua%(JX4fhEnobxfSIy
    zzUcGgI8DaPN{gs!Fd+AD-5(XG=W?rtd`XB>T2Oe{B5)tJCk-N5U%bJwWah?QlAgjA
    zS^ktiWyDGot_AXve&e_lXhSjlu>ao4WVf^Oq|)0+H`KBm@J1wOUbvkHCYskGH{x;u
    z!N8Odk4A=LCbK#ctbI68i5La=s81l#g1BlA{K
    zo@FO;ploiN`?Yco)!#fmu}|(>Y(Dl3XrtR`XO8kXo-k!W;YZ-ih|?kj*Gwlq@!!&9
    zUJi?cNXj5YIfwgETJ^#^!VU}ePF*2Me^9|k+_Jbc6EXGTn{;I2Qj1izhX702WAPiu
    zgrAM=)c@CE9Xji2ZMJXO!<+6Rm$6}T$3=Rzb8^Vb(|+q0nZG@BBleRdcz+EB@Vy_G%L;Z!KD36xiP|IXgh
    z40b(WNN=0mlL|Jr2!>3Bga+NbxxRpcGel`Sk1PdWc#-H3*FE7sRVdtH2~4|00Ll~O
    zq1!!|Z`0IiEnq(_)0bpbzq>sae%Ag%gpAj4@-=|l&U$TRd5=UP9L%1A(v*&+&%XFa
    zL&Ci_bIG=0Tb@G--*If^2^IGS*>8|Igrp5_RPni0L$0GQ
    zn@bhRY}f6XRIHy|gL-*ePuajmR|UCljkBixYyc#BQ>96sIW;&RYc)^VxhpN+lUEY)
    ze4Oq^YQA>~p5&=dyS^lL^4YzWMURSHTafQ!>ib7}#S7$NEvoREl1L6&-VN%r+JVvU
    zP47Gyolw`$K>Tgwo_Y8k$E8pGkH{N6f6?#h4S8#Bj2i)9&?rmyi#raahNCC3y*WO(
    z|KLBo48Li~%OV?HfhCeD#_kyJ5l8%pTg<=JvQ1KEk}gExhTCIyLY0`Xa^r6RH+BRz
    zn=+yc;pcSuE#H&H!v{OIe=-f8y#S}7K=5G2Q9&+aYQ}5?=1+Jlf#pM@))e^67QLJf`ChTC`1O{BCcs*Q^M#^u`sP8n^G?jNd
    zSE&n#)%P-3sVAkVJN5eP={%6)dcP9*Xght`%@m(JCE;uU?XLL*eEFPHxhMvL|DJX0
    zMb7!Z{JTK{kVtcf(_yR({B^6!?O2ln;d+j+vbUbIbA*1-?ab=?^2JI`cp*D1
    z%Z=jOiMk$@DXcUFFaGUNH?kSHuT%l19nnwl-)`9OK$(i6ua!dTj32C?JQu;Ihb?oL3xdqb-2U#flsN#GkJ8GmN+0Uc2y@JEdGvBQ38R`NVw4^(
    zXsUQqj(18K1Dsa+{wwYgrqM2}WY>&jrtaDALQ1|qq~ThSo)GKzGghauhi#`@2~Qp-
    z&iSnNkS3kIqTQ}lo*R=HM)XU#`&qpibsTv;NigULn;~Zc^Ny}U1s1&hh`ca$$
    zh0;%;#TF+6k*pv|4v2`6L*DSZW`x~_d7&93JL4BI9#nueZY!~07FS%^n|9ufjD+&a{UFD&tJ
    zkF-)mjL$3^f}z`@C{^-0)6r|2j*~MePyYsJ(u`bMlvoOSUximHoc*dJTmD`D1?XlG
    z{YU;s--81s-kkaVB}n~7$)z24aN11!^RLef6DWZ9jQJKB{`0%qIEa4AXdAK36E_V*
    zVeg;s#k9|UT_E~Ji&WW{mfw6>1}klASlPrT+@}mJj-oj(;s%l>vb?ewCqzFGT!l}k
    zC~$ZNyrC6bUuS$_>gvrZW=MK~6R^bBbIsg#j~QZ<5CSi~xAQU0=5s?>sR&!5zGT$0
    zrPmEYyNa_f`HYz2lAsoG)KivxK282B;n9`K%)Bq&&Dzkht6Ir5G+rPb+v$yzKJJ!n
    zgzu8d7N$wM=j`Fp<|>U>RhLk6skD+t38t6Mt48_)6HsU<$vTcLV$JNsT*xtaP{B!q
    z!q0`A6Tsh;U|t~vOXS-jITcGyByv4a!|mP>2>jvEGgYrBD`MPFA@#Qf3)7Z+3zR_?
    zBYz`_KZad@dobhLp6sX=#|s-&7=d}4kQ^cms36eqL3$!QUQN~GQ29qzrcB~z`wSo5i+iD#{33JeCT+gqlD9N+gi$LQbFxH
    z$@36RQNtThe>PXU=#ke|NBv?FWQ^2!WxN@qZ=j}1MU$C%Pbrx^Cf!UXhv!=xEhG9{
    z3Cz!oc)@yp+1M?H7l*r&5z%nrZ`{lLLz
    zNl?S@Fi)L3lbB%sjqt3(eSW@o6*GJEMk!e9Bn<^UFTclV6{mwU!;n1>QN>~k|dlCuv7EgRsHgXtMHi~gd^jM7@`AWYbm
    z)532&Gp$_Z1WTLyVqrznd`I7)yCczDH@KCquituwWRd7t4R#THALqf!F90`vzd`C8
    zKim6l9LOgrX`@0W(d$yOj`axv51U7!?A8V~K)(&Ouz29Ft!I<7v1IM^f?#Xav&;LG
    zw-W#UxQAFue9AR?8Au!XY^bq$utj^^l8VmIAumjCp{q*K7b-wg9XtU&XPw^g43z@q
    ztAiV7s~wC-eNaC;cjnW4rpE=P?K9%v1x)EI`ZD09s3_m1G@Oi!Ub*wJw7#YM@Vce6
    zvAGwri`_!amn?xV3oh~JC;$23ap&K#THvIj&7(U4W04a7UYzwx^YbSkF30Z)v#tP#
    zn&LQBRP^9bLyarAivSt)^|}}AaW-tUNH~EH$jT-)zPjL0qJRthw*($YgEw(Fie}Yu
    z4#fsI#E#$&OAy7fv;*6+fibiLU{20kPGZlE{8rGcL1?c}ltZD~k(Lki7PpFWpJTPxPti!4s1KcmSe3ACX?`jX*=SjQV0qm$H+vZw|5BW~Kx%S;YVxtJ6I
    zFE7TTWDT~J2Qg;wWNNPsXlG>5g$$Kzr}0pjjQR4|>i+6x9CzjY2S{o^%9SSYyjLav
    z^!81QvMa2+=xST|;j;vd`L^dH?I^t2>k}CmwEF1i7Mp(<~D?)Gsz{xJrc4Umu5y8DalJ1BwI)!Buk$&83c(nnHetUWcHArl*o(u
    zGE?NRI06RN@Fo4anVwbrjyqcy*5u|HZQAhh6?||PMn8N>1BN{Ga|=Wb+`}7=*7Ga_
    zD3H|4L!P}xJSW9qxu9Y)HukR?6qqCbmqYb;D0?-Q*5%0ClLvAQHR5F@@XbMtZij>;BHAf>5vU#P4<6Q=P4_KqN9;U6HxLIw@o?K0U*@^V;?@#
    zZMciPgL0gShpd~Nm#(mnJ(Lhv3a?XU2mr>7?Gp`G=w`z44R_P
    zCr$$5cow9)uOiNqdQ*ae_-Y1Dy@^%Ur1k(SU566q#Ah-wOMzT~oi=|^{j$^}NJ5QG
    zu>|;u#xLm=)T7}ZDe@4h+NU;NS(x!(lL99gcb7&*UpWPFM{0A0#}i(IN=z9PnDv$s
    zqxgy;15;-yYh!Wu-N)V+H4}tBhfm|4`!{iA95_UygO((Z*7O_KqWgwH4jElYl6WH_5=o)
    zN(A!)i?oK|t=)8==_6NyqJo$6U}TuIeW3RW{_oj2K5eF}#M4v$uF^!T_=x->2bjV>
    za5M%lQpO&BK6+31s^KMxA&nR~;z;x!Ckh60Tnyuw{sEMI!hfCrnm5G>WLYbTed7S;
    zzHHTI{{AO;HF_)kKoRm|Luu{k+>19h)#(Jw$Zc%;rR>LV!VIWCfB^CItcd;;|C+|p
    zv($~hJCWL53T;137Y~K0mC!et@tgl;8#uujEKMWGDn!A?5
    zG!FVX%qa#-zg(-myaRNL29sX4??AteH+PnFpK=^NfyX9ZvDtjgorn;O-MU#F5WV%0
    zraG^CwB?@2Qe%_LIP2(Av8UPxuQ*ywfaWQcl=ggo1AtH~rBvtly(w>MS7i`PVe9|8
    zuz&SYf$4EcbNR>XefbYR2>ZE1EK)t{)_#^z;(A;Q=xh@5AOxZxTj%DBnCorsvpqWN
    z7Q~akUM(E+jwLVmrEOi!wOmFK*Ryr-$=_ACAWEeDp1R&`)6?C$rIWsM#MN9y#$WN-
    zXUo}NawUkg;5)V&EIJs4qG_Nm&1g|&W`m&t=2+?dtJM#3SABD{TeTK@hM{4AdW42o
    zYZ`2}R4P`HqKj_ZHGnd0;W`xruU*2adtAJnI*48*2pKySQT;26@0yJpLFz9&DD*R5
    z@b#G2OPrYN+kr(q8u7uT(}aZ
    zZ`0sH{%5m1j?;qrod(5gofDaU${jPT%Yho5;)6V~h1Hqtl!SIMyTwd|Yq3@)ldg#Z
    zdrhRdSl(*t;Cr^sz9j7$(X%SxC%d~pNQsBMwa-Sua6r;@oRhc5P7W#|2sZ!D6*!8W
    z4&b2PzZ0R_hioQ4P(m=r-P8QAG3g0jd9GJ_9ONP6pD&Kn`ubGlgwI`-)|Mkxx}ne7
    zIA9bN5ieJ2BRJHx$4=A65;1REV(VUYk8FHYc6s-6H^(`ko7j&GE>*tK^@W5Jk01m`
    zaA=oB1K;zb34P{?2OPzc?(BoOzGJN}R@$Oca(QG^RiVXtq~>=T0BvqqFzzF~T6`|v
    z)**Zl~@IEbtNNA8p*_s8k@sbY
    z<800o)HKM${M0~*gwNZ#L!PO`ld?1Uml}lY0Q~;<$lb*6sY#6RE)Blx%V!=-j{W1g
    zk9bPq9^>CMra%4b#*+=ate@(DX5l!A^+L%6#mc!ehMX!CA2};O7)1D7XFg-gH$XL3&OQ^mkeJ>AP*%;TLecN#hMCsYMYl@JoS?%NNDl7hznge7
    zaZs~Xtx(&Q61cweUS{T}hXJ|RVu07Ey5W}fB$ik+);KR9M)bMb}fz995X?d6SEW<
    zM&-~G*uKrNO*~L7uP@_3Zl_DUIB5GaHCetO6*%;lPc(|)0uX&Qy5kVIE)>CKiW3zN
    z?B5}CA`HhO0stMjJJ&{8CsE0Hq|*3tDyVrkp7vG9Jh+Udwd6gA$?)oMY1;+dE
    zw$#Yqji0!_wH@z*EMV;M$JpTf3d}~=u`!4?{7Hv$zU;~>>jF%58zmxr*w
    zPz~mgn}YJMgOhj@pj4gaBzUbWwJ5^y@wi<8++~}v2;ynyF{zyW{4@GTAHn@!GKEVb
    z-d%^s3-NW2g4!-Fb51HAX2fi_Kq?k7pdiSCf&jp$ooaJ0k$jFPD#ceon8JqJGWsL`
    z&|+ZWbyBS2AFWR~vc&bEDwLbIsINC90extXPSo*?!M$xpKz&}&d8WaEx+wRWqKHOH
    zB)rrLqQpp@Z8uh=y?^Yoh^p4uFU
    zw5s1n$h10eEFK?tys=sR2`4aYpMRj<%bH2uLXsB#C9v@2SVMp9QDciISqB!l0qJAa
    zf7Ssu*YIJ(|94%uj1?JYht%B68ylM<0|vmOTDn;0eVlv)U=yVfH)QujE;`g3*n;fZ
    zTeNw7CsnB!mgW>ysvEC5x%pwK{etNp^T-X1Szfnw&^e>Bd=DZ%5v0{S1HFD~IhKCl
    z(X)(q&x#4=*|;*P6oEnvq(Z}W6%{65b8^`mO1{{n>mYxKx5`*Chvuw3#_~Y4gDPq8
    z)?$FR9V92@?+x*qlRaIWaNyC@_OW3@!OTI|jn7VJ(qBfO&4Z)OKHhT9J{|FWhvq<`FS~nqjGh6O~GN78#!uVvFdg?S2k(NnSL=1lqudzo?Z(h8BN%`
    zjSH8<$H!GJrU)1y(pMJB$Zm*8R(xqGr+Dml0-Cnu+K(2yOVwPNey^u13+4&tamTFE
    zB+EZs*WeF|{zp&PQIvvVEh#}T&_REFk-eXsnX?AtqIzzxs}4f$`jn5ag~xF#`M?6F
    z-A&w}4YArt8AcaZlErNShGEh?K`G?y#XJ6z?pjJT&l2Sesf7_TF&CL@B+4c7ZtHTz
    zBKj97Fe~GloIePAI%V5j#Z*~?vs{lIVwAsa`pzesCxqRuJ&<@w&zkUAM*F1pUc|!e
    zvD<9yaoc*MCwv=yA!o62o)}dr9(pf
    zfh`_G`aNslDj&uB9L)REXFyt=D{g-6*?!9Ekl*Hc^Ll9QF!NlEah0?h&e3^JB33iT3z&3sGC#4qBM)|4{+x=0LmTn5&)3?
    zJ7t>#PQW`&QHu83KLB@C&&K;|O`9#RC6zSPZF7dEE*R?pCs>DE@meRyKFcB0L)!JI@w$RK=Z{L#SGE)F#S2L5o`>
    zDR!bSY~rEb$W`34`uk`~p|-z}86aT}>wc~@GxTR?YvSDoJx<105iAhcuiAN$yVYIK;?6+)py
    zx3gnaaY~fq%}vKjG~LeFxADZW!v@kJmxO*!)P8!GyvJUQHP>}ER@sD0ZVLE@9KcBa
    zUW^d@x4e1*+XloUjU?_8Yk(4X7##w8hb6MD0J3&vSjl}ry9@-L?M_6IvQP25pG{zi
    zy-hadK&eFGG0QVwcJ9k8@)fHj-d3z}0ZA@8vdL9g-kKb!kbppY*o-)ZGR#qj)I2Zd
    zlh@ye3THd_kImIV$G}A%ahAz3FFhc&exF95yO4u-i#rKO(zoETJ6q0N
    z<49|*-YwM8yvw2Doqvqz$hz})W2x}qCBMJ(W6Dv}BW0#s$L#05cUJcmJIQO*%+X8|
    zI?XR1XNmc-b7z2)HwcRG_C1GT67n!NPXyRuBH-
    z^UNpUIC(>A$t{uTvEJmF98J#{Wl>L?MQq!vX=U;4L}Iiq~gXL^oJ!R_G0?KRSU7D9w8m>{oO@!2Dcx?0N`Vw5N+~UE%(4J
    zTyVX35!?wLlHLQ%T_No97y99c92qDH)zb+-0mLGz}J3oy6HzQFy
    zfyE5NaFs%X8diD?nR0FEeYwE$Q*Pq&F4rR_VDPspS3g$AUyJ06M6dyW$`4oO-iy{K
    z5dyE~U@EB*qQDLIhCP;0oTmkiY~es-oKy8PRhC@{-__ZTSWm11IU#Lzbl7+HBjZod
    z3jpfB+-IEL^H3|@{fO9@niS-itJ+=3je*>#ShFLo#{6`j&^UO552SZ
    z=+N-Ejpw{P%>x`FL`g?%nC}4{%k_$4nlD*`1GV~JLpS%FGA3j5{Ev#Y&4d0I>JD#>
    z5HfYJHe~tOBeqw1EWzh2$JND!i(h*pHOaI)ig1d)iH|48!K)w5)%)3kWNCI*)JkVA
    zSR7R1{mBi-63ODwA$QYSE(lbu#zWWRZ8A0Tkdg(U%x)AyB$Ikdmhm;8NPMn(U%aHs
    zMuPYTcP?RtcYd@=?l6*`UEI;b&Ai1{@~5*92^L
    zeNi~+J4)jrq)|6c8GE2cIdcnrNzMkWH0l9-f>$(pLz1fYzQXKn`%dQNi!?AeS8Ns-
    zv$A)PJT&*0LM9*b04rofK)?+<8ij+dI%jF@6TebTtesmQMB^&
    z5C{Gs-GAZ*X)bXoR1HL3Fxq2-Pz}yCK5~HT(XD*OT5#_Ro-8>?lrr)u(x%
    zl4WTeq;Z53aaW_pPJyhGfGy-!^ZrYdc^3JwgslF>5?#9*Tt8;UT
    zg@KTb)e-*d+N}Bg4kw-%FC4vHoNJ)m+dEWVRYmM$u^r!jsXH-}iltwaNo)ag
    zx`A~9%(y@xPm|W`4!s+Y2NJt1v4o&ath10Se@t*uMhKT
    zoMRSjEo8^6K`^{&IMEp;1lEu|t7+`lcjvw}{0jpoYFi6R(N>*T=mWSP;A)wyr@v%5s+*
    zmk8TvGQQM_?R%U*qvxRWPb;ir9q8l0)4M%%%AQXwNZjB$Cn_p_oYh87Api1nKr&Hq
    zo==)GCH+`{Mn+$lB8oR<*R+29-!)X24HdTH){v6x_I{~w?zdv)t}2jaj)_y_PGEy8
    zm^$IwUjpVIl}3tjz_Qi&HLE^&@i~^+#bqwMk$Cg1;%I1MtlX}JC&LoBE-J&Ct8zvT
    zy?6guJIdmQYCn|Pmc5Am9Zwzb$&n2xIGbfms|~U2%c@i>)Ok~
    ztLymJs-WfU7p)l4gEjeLMnmTWRp;Tac|&8Fa)
    zn+f;OU#vv&Jb;x_tbW^{SXotU!b4B6tsR=CxG=SnF$u0!zlv_IDkJ<;=s&i2j2-*IKruc#nTaVU^Qw0p$2)b>ZF7Ujh`lwOlXbd
    z2QEtddFS`g=;0Oj_Tk;D(fF`z%PvM%&AUvLv2ik#7#omA38XXFRzVa&8|SoJ5|u48
    z!Pn0=E|ok8m@G>E=@OF_&1KN{coG!n?tSKo+VuSk#sNS?9gJxOhefRb#wa^%6?tUg
    z1#=JB#@LUnhH=G3SQ3{QrNG6gXwW?Q{KkVOL0zL}2CIJIBxhj$ovG4FtR0(}_L2wV|
    zeT_VzC(8(Eh{)Bs0%58R`15X457~B}^C1sCxW5fXtCW;Qdx)djsQ^S-CjahoZ0*nk
    zlE4K^ZQtV`-j9{~%xwtfqjkOBgUAfL|C)$~aSf~;(E0Apad)sZk@Nl|!k_=XZcWZy
    zd(W?Kcw{A()N{L=5wj5IF#?Et!d<3xk#3ZjzsDefN)bp=)YZJR##QsY)4Ej@l&tFe
    zT9vhTJP94B@~$e-r@vO*et5=O?_SH6MGL-(;)XZiwf7H9&i_k^QAS{~W9b;xfT%((
    zU7s^xFC~P#qzZL6E172Ee*u7h&79nCJY^#eFuriUtZ>7n5h7RY)CMkT6G#FHG
    zL^f&u^!sGMp~C-ZRCkFF#GWWD$S*S6?Kz4Pm=87HkMs_6Cb_}Cvyp(Zc>gL#fMIKy
    zwstx=KbJeRrM0Aze^YJKx>GGFic
    z1Q;RwW-B>S9hdI*K)pk(T2uotbI8x~4?A^5yM2jXh^igoS}r0^L9ef;vWo>!17iuy0Z+{c_K1TF
    zQ>L=9$vLJLT>JTX)wi=IJ&w+JZNO3&hy%1fS@^j$N&*wf?AgzV!GH0!ovF_@2z+xY
    z7T6R9MVaK)l`B!T(y*;~DB7N3*((iK;~AK^7~_M8w9pC#*Esg}(oZb8^v#Zw?U{}0
    ziI39@#mU>tmOKiNl7MUB2qa>%D*#i7q&cTNR;cd2*3CEk
    z&~_hTt)0@V!N|8#q#ziv&`fypj|C1L(~~<)!FPgg-lgV|&SYk5Fxxcs+*#df89VMO
    z^897Ed-h9o&d11MyZH3&`}f6SXIm{tjXzIk(^m})Zwo!UVoH=LDAn_8n;~oU`RZ+q
    z-poSYzM;{+Hi*Kpftt2$ReD-Cj~*C#;*9Km7f0<)(o1|?!u4hefxQi9K$7+U6UU2>
    zO@?-1Tpz{tR?w9S)ZvItPiZsKc_geQWEfKYq(3i-+w*l}oY9zE9bP5>GC`#z4u$)s
    zM>$!*g%J}9b6kNrvch`dJu3V(Pq6Wb8zVF$h^RlP{CMy${d36gTsR6;vls0#10F{Dh{Gqi3xoD)*j50*#T(Eaz?$9el5UHbhY|24&mjf;9#Aal8S_losAyQ8_I$YqU
    zP1gf^M$9J|y?cML0xnwSE8VW22{r!%MaizXpF;ZB-@K1JiOehU?hg@(fif45;~^bP
    zzJ@XXv{>EOy0$@Hh`pzrs3_?$-rXyV+U)vUeA}03D}91S`caKg=<&42(_t$+tZg6Y
    ztmt?R_os&sN=Yl%{&9aO_q3ls{PKX%sBe6s!}BB~=;gob(T@Ge
    z0y3K3FRm%wBFD*2oZC4dx7wm*h
    zf&Bm#fg_L{`9TnsVl=jPJjR-=g1FDDDOX-UPmhO-wt8Q=No2kOdz*M;kvLoawT-lt
    z+vA!_+!XT!U=thua=bpBsdrXKol!F7m6x7XZ!Yj)B;?`^%wI%7k`*Zuw44;pXC?nI
    zLj7$KhcO*iXqrJp5$y|zlpXs-udd?>Egw1%R0$P!>ePv@hEFJD?W=~n*JZ4
    z!swuC{D=FU6+S17T#}HAZi*=zz6{L3ICoNc#73Lt>^};5gk8wYQHbcC!A)V=o;ow?oMJt6@nLS$~{Zn3AK_Gf3EMY@P=5
    zx3ju0c`0(B{tiM$KT9##XzrkS%{om%#u8#x=K8}V79|vzNKf$LQ!jPI1c%GtnGt*S
    zG&~ooWtt`|{=Fny-40y;xd~M-*C9{aIGk#ptXK+I_ctN(1eN3cNx><4%7GNwWj|bV
    z)lZTmc$1FM(&1=1toenr`k6$OL%U)7+xw^j8vHecjV-&;uWyj^&g205k;@mJLr4hx
    z`iL}1v|pDPg$etoN8otKC~$s&(X+q*_8iYxqqNsn1FoeZI{M(vu2-r0tsI|9zHebl
    zg4p_=8T0vgyRcMh3_d>3ygn7roxaZ>aXc=PNWVfaf4;9_#CWp_S2^AmCfav@d_X<^
    zuQ*WWU?UuMCT6qOaBbi}LTQ^*V5
    z%`7Zl;FzH5b}5L$@hG6i#IwVkETN|E;YKCNrCmmen3d#~^0@~T$Udy{eT5yubN`ni
    zu%B#ZhQm}k4tp-f7?Hta(bg)Bhk8x6|3<%&L-M_5a7}Cx3rnd;=TKOwA=*0qHnTL#
    zTqU|zbWp8Zs5@a(Oa%A-?2SXB`G0e+T0;~>?sP>my1DqJjzkMubAdOV{aiH8ACldE
    zM;6C~vgY9J;n%c|*=8Ml?Lo_^Y-i<$#@U0OQ8$V$`{3ev6$PO7v=1J2K}XiN>Mh*`
    z%FFP5o;^cZne7!4&3_{6&xJXr{}}r|v>|!%vayoq17`DJhXxe$7sQLBKlCeasJx{i
    zooAnhoF8{$Het`!pISjV#~lt%Wii-0SaRT_z!rkA{t(sV}a$=`Y6
    zR&xi&z9jc0y;(W*wW`MN;7nOCFw(&qnMc*(q0zPMrzRi8i1QX{W17e(>@v(`Q#iE?ut$~~vxq}klck5FXqg&`
    z8<5M*q8ltHcmJ#DA};N4h4q9~1)g~2?;b9+2laBmeQD0^=I$7W@yT4)1>fNaORcJI
    z_y?0%%G=SME+e-uu+PskUQw3&k5SFP-lZqIV0<=N^2h_SmWAxUMn`RNrnm;6Ome^Z
    z59BS71N=O+fo+@^)Jbgqs2{sO9Y=uml~XbYvUTwmt4T^Cs@4`#q8mI(+DOI`Q1$p-
    z;HTd5GbM+~Z~^rZ6}@lZT^QaJs1M7eBcA`q$90D?NMF-a8;h3sp4=N6&oKXU`l3a`
    z53pv!2mdemgg3Cq`JNHk!^uF6?JUy7MK_wy2w>~mxc>cd+uuWl(!$dn=W5@5uSLNM
    z_L+?J3-Y4bSjlIl*y*(a1T|arJ4=HfbUW#dO64u~UD_Y>}whdh{
    zrasI-W|EUh-$@X}uEBCO>yrQSC>W=@`i}hQ?RR)CTXzj;mz)@SwleXq#@si!<{!76
    z@_f}zrKnzoZSlaIo|BpXPT6snAvR~SpKSr}{Eca9r^V(W-^_oPcZzZHAIVpe{2S%a
    z-Ym(j4DBoWb*T#HFPvh>b)m*zJ0UW^sB!y#&qRU1TjedEcNy)VeGi`ZBOb^MYsVn-
    zI85?cnm6Qy0^c)UKO$`X{TXeyBWMrzkd)UF*X21BF7w}C;g4k)9a(Y5Pq``tWL_K-
    zALLA#2Lml7HHP8O>Lx+A+$UR^a-BM2?=kKY$raExbHWzO7gyYgW}J=C^>d*4p>dM|n;
    z|BXdd*t!I0JQ@&mIehd6qrzd`+AZjAN&in`Xr2$$S)qJAwMQgOVf6M4i?odyfXA!~2wphSLU8TS5EPnsL&XIlgVC;l#(cy;r!K
    zT!Q0KM}yHoW)3t&t~_8B{qPa+lUy>h-=Zbx+r4hbO)Y!id-HxBEU28ncl{Sxn~f59
    zgRFe$GrQO!B&&hxYV&z~K45zVXB4ro)f0`c49$I?LhYS@h|7RIzSF8+Hq=UNu0y~1
    zRn>zAu@i?QK=a21Tnp(e-QZo_rC)6_Kk%bw9-*w*N{Tkzodi*pkIuAtt#oFLcRY>B
    znTh5R=~U@3dO0ky(fwC;mE}SosWBGsX)*ZZgr9el9Tm%M<520)<4;SR|DKto@Y9)K
    zH{Dva43TptQhL9z&J(Yvqep`|VSJB^5&1qKDCc_TTNkvj3ZzPeFj<0Tv)g3`A{h-|B$V!K~cU`cMyNDSr7R8%O%`ZSQmumTeUve
    zyeoUM)p~0^J^N?PcXMbdC^jFNxz`e8H9);tt9oQ&SUpdTG5GK$a2ea*n(sSkfySk~
    z2J3=|wc2q^a)u}mPQEHhb}UDGkaH1SovuvD;aZDesSnqAybgNT6ZT+j^)76*w2NWqN0v0wa$ne*77#L
    zZ8e{n5u>6YX_cRP+O*_AHFV6Jk9Qx!oOa`#^k=8Txa{w^*-1zKpiDh~0O>GBu)etM5>Q}
    z`yXRp{T5aGeSL<3!J!4|?(RlFLb?Scm68rYKpKYbMnJj+q)S3z=#mg=L52{JmJWsQ
    z@%dcW`}6xBoa;XKz4zK{t-bf4L{cywz75q&QA=+?ZQh95ihWWL8J3gDWlp$Jcp5Ux
    z#L^5=zlevyj3AjTp$ebP!Rs9{Y5Gd%*C`3O$gt!7bm}5Mk*m7z$w~mjlC3AT9-2hp
    zAFVg6<9}se{lfL6)PKR6j@}f;?_9i47V>d32xls|Y$ORWxJ&14I9tlU>u@yn^v8-g
    ztS;&LXcOXO4(vTpte1(_5@%+&n-g4-dbnq#LOA92zx({za9jgUj!XyHnv2rEHq?qK
    zi1bjWusBjQya|u)#O$mxMJ@0$8@B8Ds*7-8M$g2hNhwqC){}#qWI3ek-cQBif|%|%
    zpa@x(|KNdxIut^9suL4uoz-FM+dYSvVH2qZYtD`(3^6Y{V0rLa?>%0VjZy?71uT*Y
    zijkX!P;`9s8%Xfz5=nYOcn;tNxp+)w&U}io0-!*S8X58*b}`4U+1U!Aafl!Zzm*}^
    zS%4D^t+Nyvq8ZoUrP&GGy^!C=IC|}opRaD@Z{SH{rW_HZ&sPy^M~=k9ku^~4z;^xI
    zDp`1`{{YnfuC94p4jq{bxVhi!x#b@%6}HnV+?7CPidt>#zsl)i)8%{!0L!DRA3eC{
    z8UCZqEVDF1$74m6~EyS%h`c!f}v
    z!KOgI(uiIJ(n#1+y{P^S+=T*jss7M)kkAiT4&LS7Xs6EJX1iBE7q$UokP8k;!7L9?o4Cas!h!Z&=Oik8`
    zx_Z+o6IuFP7wV0CWYJWgP^41{Jq!q=AXa^6`Q+>o~7R^+s&se7Atd{-YDvd2O|t4
    zq%-YbA=HAJi7>Solh(C>9<1;)6_nnlj3j1mS^r*#_;azq>S_~hP3$J23jyX|xFFoT
    znreraTw_%*9vX>e7`Q2zcCD@ZUi@nH-uu(x$zvK~1vvy?2
    zbn)EXyq>liZaoq}0N?yES{&94MJ+e1u2X_EvWywb6rt|KRl}?hny3t4z=}{U&f=v$
    zZaWy5n{4D93Df(IIzN9kPQuDLr`HjkRgrNFB-va1fP?yf(HW%>IeN%mw#oIO!G3DK
    z6QK43%<5Q$5&Gt(d{Jhs_Z_W(PQ#y)I7%RIb`B2X!xAgqe@vMk8jF4=^Hs8U^47r#
    zA7ybGPDB}bZ58
    zb~s>s$p}sG#-KZv>oJ-VsHTfNvZ}=T(^+!cG65wkXrpf*|Jq7oa(NQhe<`$T7u4+z
    z>AD*iMvV7$b6oKuUQj+2fszLK_jUJfcl(hYiZ$SLpC_BFO|560G?TUF&0sLUYeA|o
    z4o&#{zMijOUG%nb>GYz)H`gapZIF(jAi?{kb_8wDU&Jqi3e}#Gf!yKci(js!=>v&6
    z20GPi&g}Nelf*6w*_M=*fi|qoRM>=j@+&umj{ZaDK&=ywBUDzm6X)WK2F86(BRLoe
    z`Rb9BLN+DkBCaOgPnVYW@Hi{b(-G45Ig3lH9jM89Vpou^5-xiM2##-@NLALYD0jGO
    zta>`)C!#0PPF)Z16K8Yc)U
    z-(a%GgkZRA#W4q?%C4Y7N;O_Rz22Ai;=N(GPokbZ!T{sTa{|#-trus&;Oz)nN^+v-
    zlV*Bwf7%aZUQT5Zh6Rw{RfhR?mbE&OceN*sJn6@wu#OoF;mBLG)q@j=Ou3D@VGWe@XLTVN_;dGQPI
    z^5utlaS@Y=GWwY8zB0vx6bO|XnsMD>L<4q%wc?;0n;LXtl$#``i0*RYPg9Vz0&zfC
    z=sfP`O6AeFIl6ag4;0@Y-nfVU%)RxQv*T9ANVwLD$l;1sW46RmqcW=6&cP%pqaW)f
    zq?~wL0(0%{4-rBnzbbc-NEDFKAe>sU1@dKZP?5Q*WJr!W9P_m>5>^N#wU9`eX;eeT|CpCW}fW#$#$P)C+^ZzG)O}<3c5f%^WwZ$AC(J0Pq4{jJ;&}6n@}siKe;74ap+Cv-bx`X%mr-}uO6sT
    zrxrzEX&5?Iob4?Y?y>QF!sl<9Khi?f@?T~sTZr|({FpVIpzo1<)jo+s!1)=<
    z59{KsuQINJQ1k+dG)Vr<-y*_QK^3eYVj%v9HB6A)BowwuGds0uz*~MxFnC-#??Oz2
    zXsf;mC2V4I)AONBnYv>Pyp+V_}t&%r@E9V0qWRM~B;KswA;-v?!}Vz-#?HYboS?RR^o3BxfX;6CD6drT@xH*U%VMnU7@1w?wWj
    zgajH^VibK9!XV8}j$D!05(y@Xsa9bVg}Isy04>Y^MG#niQGseJ6>ECV_P{>|w3E#v+ds;JVc1VqVCI_&+^#8s
    z0KQ|0W9|`f{NoYh+J|{N+s+;g>>Bz;@yfB7%M+(tiKAS~nOBpV)wqv;4&^5C!BH{6
    zH4kQSJkOub2D8yclOuiMb~zcsTaYjPE$)WD7#sfG`s5)1cx5!W6-0>IYqWif)v4tT
    ztBYr4)0u2&)%xA|rbPsyKPnEE6(Vuolp{0Mm(TeSqKYriX=ZO#Cy3vb<8FW`P+;BF
    zbjBgy#A%EV_n;idybP{IiF|wg=sON+jpysJlpF!xkGiKPAxEAS-le2$j&2%ba>688rc52?$+
    z{}z8DN3u;#3nGZ4HOE3RTHP#8+Sz0mBX^xKv~r0tG>{%YBJ@$)QWsCVn)s;+Ti~|p
    z$}z7931G(+!h`o24M7m&Gk}F~YRfV4k~o*QmDloC7bI#9p|f+m^$K^yZ6Wq++i#j
    zcU;iZ{*&u4!1xZWA=`=3PRV9jv?fimB5Jr$ADX6uF6=XkdkumyLb8k;Bswo4nx@?g
    zf{_%+EJ>s%DtOfDBwyzdhP~wiPS)D|y6J`!g!LY0${F9h7`>@1Ve^}iP9-JQ52Ect
    zvF(&1m62D(a`i!w%xx(SXy#T*Ynl0Xpez(8ZJ|hhf6bR`!;B%qG96vyDIX=Vwm-EY
    z%oSS-U4K=+y{wrV62K>n2xa__(`A9%>GQoZ;Tn=v_^zNCN5o0l+ZItt{-i#O!Xs|L$Sh|NqV=J2RTWH&g*3tN=&3%2|hNzg8r!fYq1W80=t
    zAb}YY;1FNO?pj}s?jj+|0@09M-zP^_3NxcSZN`}*E{SwERuoCPVk9SrrB;ba_!2}_
    z7uEb-D6D~@6M*DVbORUT%V_6DP>MPea_I+4V3z#Y%pn~zw!N~N1C
    z7|RR}I@^#qraK3Ikj0M#*r$$dbY44I1H9~%$Q4j$5kLrs@<~f0D{s|$C5LL=tiQD1
    zbVZ`mq#u*pI81-bbR1msjdA9&e)f&8=k4%!lFFWRvg-zPkpSS$YrAzyi8EH)#XV$^^Q$yoXzPI@HEaktEP7&QAVY@q
    zfpw}sw=0NGue!L;ogZQCZ@GVAC=`uxNRy0ye$|mhQCt#jdrXt}Oud|FYsJP?MI?9I
    zlFNhM6@B66aMreHNNBgD02k#6g3Q5KW+uDfsT@<`UC0;p+=wveH&K^1*?y%eyeyRE
    zSRY#XsH_MSJoAT6lo&^f)^9x7C&&^8o6q^jcACE)j7b}cmVXVSStgI)I~g~6zRGFW
    z)0+UnL0*B?Da=(a1h>^f*FamC?2|ACCxLgv=(7O(eFELi3Es;A*A(iwLoHm@YC%(9h=?y>k
    zVHO{cle>Pdw^dK3R(I&PZvB2L&=%fheln>XY|;!*(eTKIaCIxC
    zfs|xO2uG`o`kUJCH;Gv#Hm;1OKdq+0`uW1erO0&}BcB?```dt2gPkDB_)cBd73hF_3Vc-kk@2+wy%6;VE
    z>NmzO;ZLc3t*?w?@@UT&6!2V|G|>!X?#Zp`*M@eQ)PT@vZYOf27`>LV@>J-}
    zmutSX-YRSQ*ezx!n7NuV)?Aw_-#qD+p;&mvV#H~@p`BGybC~0w1Z%kirh>h0&dybS
    zK~-W$x6q;K;97o)O*Ca3^SA|ZUVY9KL5PN1?xD1yj=vdPPu)x?H?diiET&LxOpXup)j)14
    z;^G~Vc&aQpvaNNhz%ub0T9j|7SMq`#8V})=f({>{>RZAov9%Sl5yH2oNzv5Gghy2{
    zELOcYmAQWCq~=UGhQwPRZtFfWB)5xmpt^`SH-)L#LJQd8cBG((10&J>;P_?KB7z>V
    z7_11l83PMx4E?AX+7(+z9jN_3)PfcnU-J1e5VMZ0Gp+x2$_9##`6ryiG
    zkp^=m@==f0CY%CsP@jrKEa!YM<`TiW73QII63JDY@$}*n0KS{doBD`TU`|DV*RP7U
    z(w$+ENG$OgY3sjWBzpD>YuV8XyAf_ex2JN9EausjArhvoalz6g5342iub)A63J^4LzckD9|jGFON8cnNeglPu)xFbAjPXbP&~FJCJuv&fnH{1P*r_2x?EQ+
    z%R$MBo};wuK)4maG5LFkvP8
    z^AWu)=Vu9|_uPX@URlVYjE7KfX%(40q%gPp+M~2~2-jt`o^v|C|DzIZky6~D&2=dU
    zhjkU^QPr`>d>RSiG)Qg|-rwR`hq^G=xPHo@Lp@&1KMxXOx!Z=c
    z`J~KJV0B*-OFa=
    z`1=I@z(;GTcM(daCNnvA$x^BSE1saI_D&ka`|O#r?^lnDYV`k=rVxt2z{=Sy`=oELS(vSGXOF(uB*!z
    zi^=s0*F1pe^$WH|;g#Ce4a#D-ricf?yJpAU$77{+B+#onq|*Zv6zrI~93m;o4Vp~U
    z(ApoAM=uM%;%4+82DV1_z=#N#dm~L-p+VlQl35*j)YSr
    z(??fb)3XZV-xKqGwm+n}%Bz|yE7{GEhzPP{Zs=p1mrUBsFaD2uC+siQ8~|)Ytlm8e
    z!E2f3)d)#NOf*#K4lMEntJ|I1g_x!0Fk_YMs1##5W*wjSU?@SoKA8vVm1JWa
    zo>#aHNBQQ2Ri!9OYPuiT!XEbpy?gD6TrA+eZgk496BBgQ8xzn2RZLq6*JI|4L-$Bv
    zE?^wcF0;rav<<-#(ig~*gzvc9B%?PKC96BrZ(crr?){9o&EUPC%=TYnjLV#yYQiZs
    zXCJ^s=&7^2#EaaiM3A+o%L+W409?!}qD7=u_6TcK6R*gank{Nz2VjqZ_RLf;+iQ$3
    z9iHNDCgO}AmP-(r`+7K0oS}q
    z_7a^?nb7`Miz^yG;_{302&3mneytixQd`-)_QASd%Fxq1jm91?9W>*2ahVBlOzadz
    z{9K`@G)R(@`o?k{GQrHVh$D^L^j)Y2tf_f5Gbe~<<^dGN*f*(@{FF5tx8Cz&;ReiZ_HWp_JSO>Va
    z8Z~H;+eNUt{_kA$^cI**^T$!-rQs6P{$$A#t^11%Fjx8Cyw#=<(eqUaY*}b7A@w5VT70YVTJ?!vNAYa&t?m?LzBNBQlQu%5b5nk=#Z@1&r
    z>w4d(f+lg?7-Z*GeR0_jB>ZP0RVMfwsPHu&J7TAo*Bp3!BQFH$1@cz25(eTD`hg#4
    zD>%Pk)t10U5HIUYY4C=(sofuwCAB|#mk}=0xH3G%#^LN!O&0857%R;4OGd*)D3H4S
    zChfU?Oii!JYyd0No-x|$V=$&@zt$jTqH!bB00rn)-U<)1E6+l7&6f6O1Vm~wuM-=3$nsIb=!5qNUMo4tM`VWf@}6c>nf%Mmy@VgH-ow`*pYe$o6S-eEi6GLF
    z_@a;N<&a3fay6?kl4z(8@R%xl7)mCHU?h(D^`p=nuo|Ebe{G*(@hmY4UWsCShp3@F
    zrRfDoL=fDEUKOGYybv%&E(Tn4QO}q?lT_o+X3lx0lm3eWnT}&VUVn=d<3OpME`M9&
    zNBG>vv6zLz_ldi`!-lY8(e&WYp;wu60vRXNhYi#xEy0H`X-N+@uSC8CRl*AW?!;ny
    zydAU&cgT=br@5g#uSI5~os-Fs|Jq
    zei7-gVk5+};OAnc*IgrW`8e>0o)3)lRM{$^W+AXeRp!$g#OZOl!`{z|#}_}9an1L#
    z;y3t26JCEWHw(XL8k+oE4vBloSb=*=gUlnGqIq)(xG@*h_rk7yuzeUu+?ole-k5M?
    zWxL^^Y#b@BGApahRQ2i%%5f(D(Wo6@c?$biqB6)b6}+p$7K9oyj47anCh*ax&qX%J
    zJK1oox@c;@)fCN|a}G`ID;TFRf_g_nxHvBypphh$)Rk~4T|k}H?f`ULi%zZ{Qn!RPO1+@04|Ju)4b7Rc06;H*Ca8u6C07SMs#J7Q#kdsbPV|DQfaJ4R!=y{gy!6T_|KZ+53}yIR%oRGB!-v(woEy^O?uc
    zVJhR!A@8>?6|G>$!59IZd_B1TX!qm-+~|fKn+v~h
    z+z}l8X{V<>aISmFPr7xCMAo*yUa`Y2E)TZJlPPpnyeki>D?q2Is~ycG8t0*sbU2(
    zag}tCCmE8b>Q*CXof@3r{unw&!O3o@w#eR<#PJyw
    z5ID_Lpfr3cH1-P1t`J4ov9Y%K>>O^!sHwR$sQDmtE@hK&YT8uV2RJe}`brQleik>c
    z{Y_hxW>>huS$ODB_hVgh!9bm}??(isP5qmP)N=Sj;l~YLzk$PTlj#ki{FGJ3F29cG
    z%r&U)!J-o03NgroW#5MAXEFtz;0hwXZqI_PKI8T)UU;3h)S&w!{(q4MMBzkx5Jd7%
    zLUfsF_V;COru6qXNMc)7PNR5?bclME@Sh09QWNj=LvJAR#6v*#
    znJZ^7Yfd#8@*$B9?M~kP!dvW0cLL)H@7sXzcxFwD~CX*E?wlZ~kda(a9Rx2ZDFD1V>V7p=je{gC-tXMz@I}XqVeY7Qo-FG)K^!VnnHjL++FdE8PvqXbWCJwT!$Y3TX4B^$^b`5vnquZMK)IT(SMiotlJO$G~eGf(xa>pd+
    zf|gC^`rU0v9dj;If&G`r(@H}uv1b)1meY7LyKncT$Z^nR_vq&ouQSUx$F+@E_IJXz
    z%mv|>6&xhultxGg_zoV3`nEjUZ=x&LxPuP^G;rOiDW_nDkm~rQu-bk8i1?Y;a1IE!
    z1#l{CXWH0IUViQzq4W+R;-C1O44L`2R^03r+e!>1Cl;<)k4(-}RgN_Fn3Nk{Iv*V*
    zNayhD(c=x3qG(VKdHi4OF63xmV|tIRT8#xi&U5
    z^4xU;LiNu_MsMYI=XbGHdaA<`O?{mP%Q%}rLri2f$fhZ8M~qrkfHYwWVk=g*KJrpe
    zgP1TD_Q!OEFJyQ1m_m~b`K3LAxd8veKjQI$#9TP`4qfbD-;jA+6R_i|gq%PXjUue%z^KlPS3D0dA!oMPAa!Io%9@aRb)iLMU=iY
    zPErWZ>vnvo;n9$NN-16=Ea`7KlyGVd-2XHGi4CjagQ|m23C!n)0tF$?ht(alMOA#Z
    zrocIbK@GnFElH1uCe0F)5iYq9Q)d7>lE$NXFX?hLdB9g|1KI0Q%)x9+G^0Nk7KSHJ
    z+^-G?&G#wqQwHBqigm>?7kIH)`-enNweIZ*{8^=KBL;vj&sBEsvpeytQrf$F44RS>ElFV6O@)v-UX17$3F-CUo=RbFWMZ!p%h4x
    zF3CpS^F1o}l%y*R;Wd!^r6N`M>Mipq)Y><1a9;ihls?gg(koI$5ZNF@KyLRYxCHe)
    z}(yn0OqRiY7)IHm%(H+e}($k2u{e@f(CAL!*oFwd!xiPfwxybps=*#m^N
    z;UIX0aobwTl;+GmGPU&T1)eL*#Dd>_`Pd%DegQ80O&cZoc(~n|3aLs$BrI^%|At_=
    zf&ugdn5(X3d23M);`z|_u7)A97Q!i$bFx3L4{^+$`Yc%?@tBqS6AEA2RY}Q_C1s_*
    zpbI|0w;y3MTGAxPQbzF~H$l|nFb^uD4na@qI7glg$^FMXibVLMSl%x+y$gUO38#Mt
    zk`Z%!i0@nzKYUk5mR7rcj#LIX4}J&Hh&BJ+o*jR<9O!Ly>#K<;{{>u~?dgg@HAS~%
    zo>jN0%9x<*FuNAmr%Zh%W?us7W*y0qQykZox?jI!5y#+MW*Dr_1`*1_k<481wVnN4
    zvZKQ;*!(O4I?0$lpTXE8YnHgdG+-p@L~M%IZWJW+OG+VglKKR235muoxF9bX=ylEK
    zT#Sv%p|$|BvuQFR{vP7|paRQO@X|H-FPwTV)ehI(GMLc^3dT1>gz?4FzTjh9}!{zGXoCTZ9p@MLVE_N2r9c`ZR57
    zU^@`k+zZAxr0pR~dLkroMxF(!dXlm1+OWB3pmH*SSLaZS0X5{CstAph?0z%|s!Tq`
    zxI{FZISXv!7bpJNYcCkmDRQtRV0A)Tv$G%M@qW<`-gS4(c=n<{`7als@je^1q-~s<
    zhKs?WhAfyfLJ5vc1s!@h13ebQ<_}-DmR*wYBZw)_9ta||1o=yyx}#GOBK!#I7*iH<
    zzHA0!yi5a;4UNrD`jYUqzgyT9tmU|PyP$if(
    zaW23)^2XS3XYgh0{Nb)t%wlR`Wc3W;)HolalyJ&>LmByuWqK~C)ccR
    z(ZBerfK8m;@?_pPpeI@GAI<9;%-!(?ode?BJ@;(RIg>1X>j@g%uBRSF#zEq)Sa~8x
    z9Lb^q&ZkOA#O+m;$tIrg&tjPF;n#$jW)-uOBe@54oMk|@Hp+ywzB!cfO$AF!_0~c_
    zgp!9a&$lkbhKv`?^|msqkQ@od6;s_WBhag}AD~S?`xP)mSta^2+!^QD;zHI#eDm>u
    z;Qa0yKR_*^nhncY{nJBNLxccA7b-^bzXW{JK<^|z2VIZ`7&I~yK>rVze}7!Zpf@S;
    zfqa=n*eg$+xAiONubxcW*Xu0+Jf%{{0Uwr7^We|o7C?wzZ3u{b1Fv|y;a`VY$&s7a
    ztK#7BZnjaFD@gFZ3?&{w7kD+t;I1&MT|hM9Mve^qG1rOM7)|KR&A
    zKh0U^u(qMrA&7ihe@6UG!K|a!1GByxWh|4Iz$OP%nc($-LqdSH)ee=~FW3`}TQt#v
    zy$~K{nQw*uPw>szDc{_lz8@^xlyv3NK~VeL5-w+zNs?5R(^u&_bxT|Rh@T(!$>IG(
    z4{_$lv39Thc4PfRB3fSbOm;){Fi%9e5Yzv{4SHqn04T}-v(zx@rBCR|cMJ5$ZpLbpaYO5p5!U8TVRispOf@P!6d=mzshF^ia{O7m#Xv3vZ^0L!xYe^=W
    zu^ninP*c`+jU)lYuR8}X$rb@q4OuYT;z7mI_p|gCgZc7JxS~E%VDan=QKB5R@nDNL
    z14aQXHq?T9_0fg&zkxjJGlzk!aUBR~L>bv5Y>(9n*wuBN)i>zDATDpU4MS^V&29L3@CUHJ^lg7c=47zC+O~)Pf7}x^rbhwt1#1r2_HqPumJ$I+YN4c%#b?k!G@O7_Gxg*c&
    z0O+1A3a_pbHgHVa*Wn-}q(4|tmht15S+l6iq(SNz5h7MDaWJy8NTfxVwi9(T_bH&Q
    z4GAg<@p$4xA600KO+WsX;zw-An%Qe8@34W}SVW+i!>__EfK*H4*f*W}RA$e?rb2k#
    zTjqjufo}R$zdmHiPZ4=Z`x>luy-U2VCwyauo>kNN0}&rF`7Cs?v!vQ(zCAC((IVU}
    zDe>n!ITJpY{TI13c;x&q-AQ3dwkFZ8#I3^-^v+Ks&}@X;1rYT?-fY@FhakTq+k=bj
    z28X7?_|l7NYBc&|Khxu*4EG<55+S!bi--i)Nh|d<-etzyx{Olwopt^`{m7A*
    z+u=x&o>&8pAs?QG?Z2n2tJT;cd3o7B-_!Bqyk@C;#eGuhm^A1&>2V=`z=Iz*bDw%^
    z?{8}T0tRiL+!}fR~#86tQ0CtEtN%vUAr#PHLVfw3hqIUHfCc*l8;$r&G|)EIgGg2sAKG_8N^
    zLX!sOq=mkr7NybF?L&xtfHE;rjx)$kXTe<6JA|pWv$6_XpzLKX3C;^LikGa}m2rJI7B_>HpKn8WwinpFQ;Crs5Ut%fGdsGq@H^UT4h!Z2(BD8ew!vQ*{=Vnu(ON$e
    zL^q&xkd0XvpQs4W#UZm;q(Jj!7%59mIe~~mY&RTtLwVGf)`bP1uGOuMMfbBI!XV50
    z?OPmRKY5zrJ~0EUHG3OZc+;Zo-pBB-;kUV=xS*`J5)$)}kTIDm9=m*Qu~A)$Nl6Ov
    zJq@uHKxKu0RB?#y!1<7G>)|8batB-o0GkX60JmS8?jEKUuzC<3f$y>&?+jl4`{lHz-fQ8Tj+YO6|-YVc?zy)!F9{UHgM48=+>PomL@(|#I~>%p7~E2JRF{~(w*Y}W>#HtFEV
    zGv&PYF<{RL3PQW#DV^rq0l!XXMWW@EUeY^KG`36yG?7+oy6@e6lB&=@M-lOT{<7VN}l#?O<9Oc{N
    zn3J{{ym%D|gpFI{nu86Mb+@1I}lAy5^aJRdH@qWp44)%IAg&=)@c7d3162^<>GA{9tiJ
    zLJJNwP;+A=@!(e{np(UsuWQ$NvIW}1WNA>}9u;=QpZ%Q|8A2vcm$rRQf%vDZ$ngG>
    zL@;i3h-lCOhk1O$CVNkwfL6ab5wCUDZCKs&khuh44eX|P<(579Z8&fXSyt0)jJEU+
    zQdJjO0Bt^T-{p+d_<7y;LTh>Whk;Hdx}B0rpACT}0#@3DW4?9mumUwK_T{Mm)W7Q0
    zp5N!eTf1_<*H@pKB}-iF22sk`D-lUxN+E##_wSLx`r`45)(u}Mo9uUA?+K54KU_rh
    ztG>ANncUZ_CQ$c>o8&w%Bb>6BFw=Q-fnz=g76bD!%)=+nu(~QS_c9-#qwcSVv^GmE
    zJns6J4VFO5m(8StJ~%g%aWVyJ`v4CKBrF5z%EWI_5vnl{2qQFnbk!)2p2p0d7|~B#
    zQd`LH7-qEceWoz%)$&S=q~WTwH$_=QiLkV(m2YZPq*>X)`0oFFE$l(yn)e7Q>N_rz
    zf%+7DpE0%qvg~Qz&Kn<6wOMg1J8$+GCRKatyP!a_Zn1hA^^C!Ft=roRAd&nu&8&vC+}?>u__1
    z$8Hr$_VIE-5CN{BQIC6yNh1jo9)GK1yh^a#Dh)TCtbjeRJ#tPVN;MQ5E_1hVd@g0%
    z^5g1Fg+Uzoh`P+Sf5
    z2L6F49npG=jyLf?B_|6cieEf(i&)gFvjW@+r_y#p9Si7pg@z179P)!E_S3v{#bx9F
    zbpk4vZ#FZ1<4S{YB_UXXWp5!jZVdOP_&fn+BXUqog+fBA8anf(s2!WkF>+)K>OIbb
    za;5@dh;_EQs4!G41pO>c!nMws<{|9ZSP%!r(%E?2IUAuTX6+k!eTV5anH1CrZq5N~
    zuPsQ9^|S*PSm}+h@^S
    zg!dC$a=$47pktqOK@M6-_Ha-Ec9mG9`(Vl%?6D{>lFp#Y{Eya$@rNgB~H5jYxCA>4+wp@qJ(>Hw8GG;CYq=lO1C2Pk`lPcJ2Y64m+rKpt&Uw
    znasfvgC>2Ub=h}`$J;Zj<%cf$=APfRR-!*%{_Z=BgR8lu*(3r-Wti{m=`)A
    z4t^(fwlF$9$n|Hi21GX?rZfh_>_(J|K6$o$J3pHP7{VTDtfpDj18xJz7cL`b++AesMt-`l?67f>-T3ko~|$G38yj+<)(Z~-9j7&A!wI_uXj
    z`_|1AHI$m!4Wa+}N3-U@drVq?0-%LrMBmB9rFUlL(WRp;@o+)KC-kO!-;m2>Py&9O
    z%>+)aztpaR%)AL^3S
    ztFZeL>-(A!rAdAf+psS&*aA0tuYse-;CCRF8J1rv0`u$>L>zPx8PK|e>Gn6_l!%65
    zd*Y%1Vnan{W3S1!%Z~=xe)VNAT~!$9*5~|jZImjFln|y+vC8@wq-kNa5BSYlyyEqs
    z8+oFn@E=j7C+(TS^xg`QMPPX&A*4wF<8uMmkwW5X<-wo(>0x#X>%>$*Mso*s19b#yCz;5e2ablI`mYR6mUT8l$F!MCx9?1*ZAoBfXZp1rn0A7WY2r%
    z!gc#rd0oVk`fFkzARV_Kjwf*56Xpt*9Z>1>3auV}gBM`o=L-1;`x$Akm9O6CUz1;*
    zWOP&Q3zCqMu2CO(W17BQw06C_-tId)6Z#K2R@q8n|2U2iMSki!uQt#3EJKWnzG}YD
    z0H@nrMEj_VL~{60*T-R2ovFJ}KpSpBtq8J4@?_0|L^T
    zv-w)iP#X?>E{;Oli!0@|(og4Y{C_!_x%_g&n1}Ah!4&{aVcueIHis|rCm$b49ysFD
    zNP-!8Z*Q-zsfp<>uH&f7?{-pj#e+z4gM1Q!A9U5{da|hk0ERDKg!B5ZVIS`rbW`MA
    ze%^K#Epc}JbLI9kh*dMEF0i{S*d184p^S!7Q!sMmpAppQA8LbB&t=KH9%jjUyAIDh
    z%6~;}SCjOeDQa?a%89)21Q@&^Vt~~R2(szm=?EiAC(9hzs>Y2qD5u!H6s$q5=UhaC
    zN>dn{6Qhj}5nt()C6>DRo2ISExlsrG$a&(`fcKNRoXIxos-(wF4V-O+v5rLF%(V}3ZRpxaLBVNuYqkfp30Z-^|R}B~HPa3e(j-n)aX8
    z2qt=>(9@{$s(=wa)md{wZSg0+#rMpN5J!L#Wa%INdMmA20Lh9>x>tsSTT>9ZkYqxB
    z$$e8UTvJ58iGH!xJ3vMDfMjXaF}A2+;M=IN66>eFgXXfoTw=wuZx)KdTT_V^)xR-}
    z$zh~XL_u`CnBhJDEA#Bx{c6}q-^t0zMZQ|188L|1KHWQg19&Jq0TZ7$-K&vaU8lLn
    zNGLvXX^F!Bz|kxeJc0oLaQ#<{Ilerl`0bam@ac8_$rGZ+Hf)g8>>`}XOt$OJyJ`sY
    zKyskl!0LQ(d-t2za8QLW{FU};jE{Le6u#4d6Ce1ov~T`qdIR|0Aa~w>(YDG=I8|w=
    zB$4(ZW@SPIb{ta%PH(MOHfn#1WB!6D5+`m;oSeq(Hz1-avu8LG29Bd(9WV!$Li+z`
    z_yLazw1F8e9a>MgfVQ;{&#f;)(P7vJGd_GaV8VdTOB<6p`=MNS!q2T7)_k0xbDAB+T
    zQB>{SZY|h39zp4X;H>>jc-V0y-Fn=My_R^llZa~h?hl+bhVX%%n&g%|bbO=jNU
    zU%$=>%ux(4$>UJb&t2paPZ7fO{b!zbRAw#j5H7qrKPL>La6{jjyB!0A&#EiOjfC4G
    zq3A9|`qZ!CsmpJ;c0zc+3BR$I{#AzjeXD}??X=vYwTYw)q7I@W=iIX&4t>ytmm+3p
    zASlWLr!7|aI^c>T<`?F5qLB&w2ujNjYD||c2_NpLonyZ&-%N?LdUhM^TqIhTqE`;m-b~Ap`RyOBVoWPzz=Ow~wJWAs|j0M{)Ezsk#F`3iKTY`Z&)F
    z^ovynQpz4uG}f2vp06QJUvhs>w*WSb!`1T4pIhtR=L}vPPIe#o(o*CICmhLO>yNu7
    zAuh~;TO7jMa%#Zo#pLeW#}uTpZ_@sS6>0
    zG8qvB_{U(|<|gDb=XsZ>lKF~xNy0#*WbmY5*%q0u5`
    zRjRJ>{o`L3I*xY+0UJp~xVqGzqfTcZd?U+(|D)|KqpJGe=-+)faFCYnl#=dllm-E5
    zknWO{hC>UAlyoanBHa>)5L6_jOE`!Ef)XNKck%uG@3?R7xbNt@I%?H#=Ey{DIK?tyL5CulY@gKy|giQHQhveu*(;o
    zz2(sCPeVB;prl<@X!fqQNaBPCG|plSC;~D8kMg-k%y-*hF-}l`qDTKrp1~zI?GLEU
    z!j`yZLdq|Tw)ei;?z-SO)4j)H;MDy?Ai;hssK9~>HSNzLh#a%v;o)(=4)XUmJYU!v
    zd)wo5KrO!qmxuU+zdrlu*1!`j-g7##CmwMi4&ITk$>C&t?LSENo#>>WXq`5aa_IOK
    zVpZElwM7xWlu%Q6{JoiL2_64OiaWKM6(4Q4Km6QtV($qiO)VwrFMW
    zHj?nuo$5BSNxb6^e=+sh#Ru4bnbnNt*GO8b9r?u5r}hODOTVP<-+nno8z4b5yYs=a
    z(A1nC=KgME`tV7Z#FOA6t~Qvd%&o5o$xnk-p9dMKtri7uqpoGs^EmHnaW4I)pk8uY~1t!t?`qC)`8!@c{)!=9wQ*6^lm4q+BzW_TV^HkWk}nWAJ;C+-Unei(sL)A2MO?M~ewqGsa9P$saYf
    z>KZu~*M-vXwk4VRdu>}+Dp@TqleXmOPdb`
    z#Vfdw-@0tm-9@p|F+;V8tj#syR%p?0e)6lY?i&NSA1=ac6G-AcE0v<{qEbfiT)R+$
    z$Z4%$3u|zJ3uA-ZcIT*wF?f@Cgd0XkF9p7-$;nA00yG?((Ea*!ek{D@uYZ<>56+jd
    zF*t;3?YZ>WKZm;$D89W~Y;;PV4rz`Y2ZTBFPTdYEqORQDkz7mzug$u^>Eg6E$n*}r
    zbw6_D0#j2Vy{jq%QCAQ}M2(_zzm!{UY;$~`ywPl5te7RO`5k{+IJ5;g22B3x0fLOO
    z6sSPbvVPjgAV+Wmq|y{6a5h`s1@D=~ZDa7I)Lr=xK}^65DZ1&S-S76={qg?c^|!08
    zyZ)nSTq%X3hqY@UinQ599#Nba;f86)=i|U7PzMpLKtYi>1vz?M
    z9?pW_%8ucHstpw9^RpKI(}s&z8jR<2L)FU>5_EkbE9aIF)b>4*8~yzm|B=K(BWcwK
    zb5wncfe!HUugj~Bx3=o-z3vd8*{M+lDM&(YClf5fzt8clZSLsDgFU?|HX(l%
    zYca9;=qu+10DEg(<-*dbGko|y@K2+|@Q|Z>2gGszL|XfBE99e>1tzye4qI{m^1~Q%
    z8%~l`#YT!Bc+a9fKmVg9IK*qP8WSQ
    z-6$O198;{jX|6}SfS6E!Y$Z%G{$&4K4l9g)0&a%8c=CuBmUJPMqau_clO>+O3MY^{
    zYGw5_l6!aqD<)QG&!fZDJg5*!4$~eCV7HWjN}=m+B>of=Mb-
    zXBL8i#kc4Hijrbu?QH;kbaKSO71PAs4xA^8zIQX)`x63#EH`61Vqg}JA$}ES=rI))
    zQt)eRGlwJmVny~z#LqC{z|Z~1&Q_1NnFm*=xVltihjEWP%gR-ekFC~T#~orv6SYBV;-)}Egme}$O9klH7Rwi&ExrtF2%x2Kt<{N
    zLKQ&AR<&_!C^G-rvG2$C$_HUw;8BMjjC`K0uC0B%(eWT;VX}Jp`qzVqGXRTXkgW>+
    z7AO<9Hk1s9v@qSoSvg4u_#Ii4%Rs$MoVd>B2j?hT(-nOLh7OpSw*lI;Hr8a&9}ru=
    z?E%VXfF{{}MVebb5A_I$XRio$eZ@$|qc*hjr122lAdu$|k*rvPgZ3Q}_W7uFE6>9ruf_`MzlMH|n
    z4%IzoeDvf>Dn6n3Cj9wWMB!JH*SNIpzMYnp)F^^*t0bBv`?{28w0q8-D!W=r305lF#r_5r5Y|{S(k(5E@&BiT<8%
    z#6|$l=g3H}L)Ih2ZqQhWwM2gYuL*WV2i%~USZ6^}*7f>lPx(+cWM2YEAo^#O*vRaVokMvlKMVrNDnFVhP~q1LPLt=x3)Km}fQC4xCfJP*7F?3w4ybl^L*(eVRDJtGJj|;)*=eAP?4bCTkViN3f@FwjLR0xGl5u9h
    zD>D
    z-)SlA_!}7H{3^>sQ;!8ALQnb`%-VmTy+nAI^p)DLEoLo+8pY(M>rqm@knr7n3&}%!s7OebDU$FfOz#G69BG8EC
    z>lk->)HCzvezen)Fd1!WiEfa}{=scC*XLw8hCs3-Z`?P*;GAd*G$>yNxvfq=dr)uQ
    ztn&p_LIyy;dNI0tK?p%IXJ3=!xNF+L^5JH+41-XhT=#F=>Dc}_ruv$=b3f@ZCWN%C
    z#ReZ^^P!WL(B8q$TI;RbgK?ZHUpp~hB=`?;@S2DVDwHouOQ6|DSp|`7-dgT+1H0Me
    zM&HU=n+X^{`C9zjINBxGhCf)xs;VN@8kPMCaypulbRQ{CQnno8(cb6af)x$^(uU+W9w|y5gk(v0@a~QKYpD)QbhLgmQpL}cO
    ztt4NQR0fY$^fdOd*Y$QF3|*^5kXXkuhK^q{$6Z-a9WfwFN1@MG^T!vE-t{Udz4Fk595^s&gR3ndCiEQ
    zHNWKECjhaC*u^RIo3>ehZ&G9H_yhiOc%jDwYW2`iLrdI~B{1!Jv#~Y+D!>=mF!%bZ
    zR7*iN+LnpT>T(Ut9CCS?UAEZdPSqeo=fWn#dnpvJFpy&%wmtd;1gegX2^qG>|Oz8
    z??8-|P)e!hZx=ZGn$&g&JIxQ=maxY`!{%sEt}%e_
    z1>dvd7kgkYo+VUt`o#%?&^N2<)mP8>5)T}taX&*xA%o7IZaswq&)3}NtP30=+V9d!
    z17APQJ|UQUUq_DPT#|;5%#}qjxf!K|jiPrgu&CR>ikU)Pfg2P}shIy3*0bqb_=$yA
    zn%`N?{qP;gjo$}qR3q_49~XmlnduyXTbkXy=b41Ur$mBjWtjkwj6{{2eoa}MBGz*M
    z{@33yCLKxm6&iFqnQ^+^zKy`HY+J=AEiIg_DoM*u)MZcU@c(JTs?1xBEui=D);DP8
    zc!lI5wIMu)>zzEsWv5h9S9?1SHRcv`5d`U3Fs?y!r#NBQX>jCn*Z)MH5ll#iddB9+I7~BUQ=TqThJv%
    z*B3SfeR;gYEQ9pcjk|R5D9P&-#`^a|l1!|qE>!Kbz=C~h^XAlLG)Y<
    z_kb6I@%m(FbwZOr7PKg+pIKIfR%a7gOk=($yVsXJ{mIuS$Kj~Z7ei>^cysk2}N
    z{y6*~ul4#0^!f%DR}eJxg8t!E!1Amv4b%g;h|O%&F#X%X^Oa+kU|3SZnV2G~I1uRigrrA3~Kpbo;Rq%37S~ynHTZ@dh6+`
    zJ{gf8zID!j{LA+?R5D&tz=e~(4tqZ40aR#t?Pi>rTYzcHxVgqMV
    zB6?1HJ%zSFfPSn;gO^hj4NKHBXx5u<_F&1>Q=EReeAXyg^W(3!
    zc}pjbIz~rOewb+v?EmTS!P4RXyS)eVW}(pY#TAMkd6bcDIJ>eT^jhoNneCt`#HQ|*
    z7fBdT3dA?gU=$e!oYY?o^pL3K8S}u>JttHI0xj;`2k~9f$9$mv_VT44{KM|5bWn3y
    zdc6=`y4!)eA!hMJ=KArXXf#_f?H=S=XRVWCKT<2}5zJ(V4so)Rh0VJn(p7J1${=H^=6^?Tj9jqwXvT*u{=258TpQww%2s(3d
    zQ%a9M_#7vU*Bv9RNO-nx;EMqv2lrAVm=0uDoo>TsI^M0|4R2xU5}BkTyk6s5pU>gX
    zvH%$dZjnc|
    zDf>+yummhL$stQ?)rn)E>nClwu6CPpG)06Q{f0rvhvhm6E7$H2uh0PyCo$>qj@LTD&R%FJCO91s!DXI
    z(vn^OIp@u7^6X6{-sdRugjc$E1x_fbm4@+7m4qN-&56$g2pm}hYwqJ0)2a(L&|5qv
    z@D{XiCPq2Tzy$D@Qt;8tynMFfyN_(3$Lzp~qojrI`pWK!PT8PFx7H;7syJNOim{HXwx}iZ
    z06H7`AoDakd^uhybFA5dZ4$lxuiAtTe!AV?C_X%soTFZoYvuHwt-`u@LDXI%`M
    z#am5Gu&$DiBhQ7tlkrIt>{h@4$t^JVJVUdjV9oLdeDZhAJf%?o|wP$NEvY$#-#4l>{}*
    z&mp%=RXSASD(9<$iB^(k{T^)Aq`N7V{Cy0F8T3W4C4d2j^XStLKF4;;10Inpq4Dw=
    zGLrFU{y5GgFU@pj#8%HIN@c`T%WX-}xACnP@5zX<#=bI1Gb&pT@OogbxxNt21N#<*
    zU02g?Gnrk7MH%logh@Q>p$>}<-^GQ`DNi#oGxy^2v{KtTr~IH6BS3%T5(~U#Gz_t0
    zxrJZ4yn?yA?Gq~vHuGxb1D!NdWy6xgl;(SQIsVYuWx1VKBH)E9+oY9^yAW9s3s+jOtH-n5QB5I$r)_)$7-rEYL@8
    z$Q?`m-1cn$Nnv7Y3PbWrZpP48M1`wE+tQhUl`g|rv45;_!e^vD+o$a5ZRKgA=!B!2aD6{
    zrF#J36bHu}fRJER8Vs%ozWxlcy)>lJfT1@2cxv4T&M08twdthXV3|MudJ
    z=9hooVRjpbX|fZT>3{PwVLTs)rx{Rd&egLuFmo|pt>aF_BX?^z8;93@#fb!AY(MtT
    zzjrDJ)c7qo^4?^Z1sSu%2_bJUIN@d2Z`5THQ
    zy1&R<)fcBLyoQSq)7Iw>HonNQ!Q)~c5t}Sxz^?)^GUhQD7#_n}6
    zwmxUUR_yUppRLX?C`XlKoST#t_KkqZI_P)J0fZPAM&9KfEdgm~L%Q%slxqnxNKrw>
    z^pPg9NYkl<=lt?zWTHPWwjR{}iz&=RMaUZCh1u(O_1A3r0Vfo}K$D
    zC##HCHYXbqSD<|i1XU4Xv|>RR=s*LMGCk@`!;hRIGB2NJb^Uhm0&7RGL*TaD#|<1d
    zU&dHDUp_|9)!TVZYKwyobPwa?i<2B2f_l#0_TW-J^3%)bmo7o!1-4V&*xdo$T)tV1
    zDFazXfZn-LqbS7N0j~hOQSmL!M?d`ndBkLC;jixRWuu{EukY?n2qC|Ym5z}vFAX1%
    zfK-W4rui~Iv&DXn7CCio)S&8fz+aX76}~xT=JB`i^TY|TBB&{d2ZTK>m+@Uz7D)T#
    z=Q50M?eikX-;<9-_@hWHy%a_7ef3X%|Kxb!-|C3!bO>MIW5xaZKK&a{@bW9`n9Q=c
    zm?%D)wKXHDzl2RN&R2JaB$;{e2oEGAQA
    z7WQ88)q8+R4Y$hL)p+SVeo!`Cz;@mhuJ--6GsIuA;S}DnE4pb5&x)%sm{y;*&poz>
    zNO7FW-v1gkyyaiEuzS2#JH~|`bfyDlT@14ikl01xPPZ>;n@jCF&Mo?k@;_{K$$v)$?clq&H1HRnf)MOO7u>}lVt-?3G&&4mncyf{$pO#?IJ1DPLKBtQ%=d`3-!@pOb4bmqa8rgF+yNfm0M~HmQK<)*R
    z^&Pkg5WG&1m4uv(UV=T$!mC!-Tb;O%qFus#@dfTke8j;v`kh;}b3bXm3N@pXwp6(?
    z3~~K>4RZHHw4oMPIm2o*u3OfbybNkzO_SOHnhbTWmmXwBMr5sN^twvSG5F|is{Mv<
    zJw|LgW|Q3k{1e>q}nj8FLNd;sSCMA;TK2goIQHwT*s`Nyl?!f
    z=~@4OqY{GupL)Wn@}XiOJafFJ|_JZ`1-L%d_hY+J_@xn+^RDmfLxWys#9^l4@c|_~~KIgcLJI~To
    zn@%Y8sf)yb@8eaVdsnt?bRbd~=ZJ?7KRyk4w0GeHtDn7X(Q}D+c(S(nx8r+;cX2vd
    zvy0Xo#t_XzQ5RkiDY5R`&Me|NmmrV=db_{{DYiu%9OhLT+HLe5rxm~`%aN&l>9GMf
    zue9#G<}=r5Yf0e4_gxs+VJEV_u$*QQ@_-h1z3<(B?eicHQGTBRl^eh6~B`O42xAKC2{9XvulG$i*k0hV!J@doJ+RKHK}Z6k|4faM8)N5
    zjbdzQ53=lMK$XkG<*`c;Wo*S<#_J)f-vdVO-gC@Jge@&Kz5*k;ftM%Y^#3Zc>v{lL
    z%t1MdUf<~}azp<5bnjwAeq}3w0Sv`v?OFXd4}0+7P;d||eOZrL>9Ya4PaIMYEI
    zFV{s7G*6~b3^j!ZWho9k(DEz9K50L~L9U-bM~Uu}LCrXp+akBvnb;z`Jns~IJ}}c2
    zpi3^v+Rcxk69BXnVME6hq+?Cd?vD&`LerCGsALEsgKCLc7TNH(D9g_4TG|h^cxs6f
    z#OsLKpo+WWEcb>PP>+Bh<`A+pr2IozU>5(3a0df=0do)d1|}ln^Tk6AGD|@L`_p}3
    zi}z=zHBE__ZvZ_oOMXH(_Tbp9`t^AIEj|JD8mmPBSr6;C$NLU{q$r!|@zf8k%n_#D
    zIr6ZfAmuh$%@L>6$;WxSu>nPN3DM_;^x?%S+!<|vyfbr_;BQk{v0q)FtT#oI1O`Dw
    z{|L~o#PgzA)t4{Wp9loCCW4FF`_DY!k709y-GLT8|7g3{CyzJEU0gWMRZP~O0MiPi
    zR<%Pk?PMZkfdv1|+tva{=@WyojdWnOC$uPpd`mK}re39pf1!t+TZeMIkTp+qlA%@x;}oT)$%sqQKAJcn_Xl+ZkfIZh*{eNXG*zD}Kikfj
    zJ;Qnf7HE1no9k34iS|V)hU>e&U+VsVJhwI2^DY>iJZ?W=DDAk!ju_yjSclHn%iSX6
    zmwn)E>oBdjPaR_wy>+
    zhu2Mq!(x#?N&!qnYN7TAJF7|5*^3#|Y(w`Ki8Qo}%{Jt3vp1qLlJq37jF
    zUBpU`bpT-fDS!pXqqvKLLj_Oxuz55N3xGTM)TmI-&>v@zSdfS*(&3~`IIi1%zy%Q2
    z?|xvo>??^kf$%fXibT$S1r2Ed>pPqQVZlA&$J5ZDx1lE3ggIN7XElCVq8e*>j&HQs
    zaI&C8X?vPNGTzp$mo)n(T3V*d3XZ@!QCX3DaVr0K$q?a(aAZm=i5)EhhEf{m$ndZS)f<&
    zKo&r|%IT%;&ta0yzhdO9$5NJKZkQz$9<2VK%k5?Dc7htRjvCSK-cJ`;0lP7*fX
    zk@JDmr4k?&N>JN>e^PYM&)Jnl4};}eM!b0hIk*Y3>bRcbu2Q~Q@H4osS-i#OE_?An
    z6vz3ysEItHCZRYz`#UNaI`{14gw4yE^RI8$ke=%3<80oQ3-rq1>?c5v
    z5YR3@O8YUl=L)%jX96&Knzx@kWqU}7eF+R++8TbEGEquo)Rj#p?+L6uF({(d%wlb!
    z*JFjMBtE?$KB@X?7@q%sp@L@Yo~(Br7sm+v3rgV>9Z|6cQKF#5CusXf%i_rDDnPN1
    zX@i5sU|XtxPC+uB2wP_`av(8tQ;$rxD_eQEK4FNKm3XfS~;7l(QH$FS(V;r#>a
    zjjpMYwA+PIQkm($a`V1F7i04Cu
    ztqEu*pPQa^980X>x`eoIjW+rq$Z%eAD}ne4J{n+;66rBUJaDnjf
    zGPp#D|HY?RWMb@ckPumCMMmxbmaI>d35FCMoJtf~?tPPo66hRpc?5Ly>(qct9ylug
    z&~U0g*3;_gPdkopcl?2%Unn)ol4P7!JkK8Cb#hsqgqKfN(5=1ZD53@tOzDUvx=ioP2-UEZqiRd*K~t2gaB#j
    z*Gi#>`X2>$WTHjr0zn34Z25CA^|to%(Z53i*r-0}D6PEo0}sRt$a6BSzJ&WC8;PfL
    zash@$pf0bkeWXXGrN2^)+4K}f7ZvpN9YP>WU@$FKL=>saje+hfwJ)}2bcMwKz_jL|
    zjJEQ41DiyaC8jt}0o0SW;X`3VxFql#ri?SNhQ#siEqkWNfA5m=gJ;(SbRYy6=JWst
    zPPtu<2_D|(pR#k9-V2p=myQhm#inosR2U2DwgJF%Vu8J*#!LdJy?#in0U{Wy8PVNl
    zjTS&+hw&!dVZ8-6sZ&{_iBE%7KYd+~y;tQ>h@1_TdH(8|N5Vg^zYu#0*D>H9wJ@Nj
    z!w5l`1qKD50~DZ+VFtF9&A$+{(pkZU_Nb&Pr(b|_D9dezsOuH;-Co%|P7@h;!_-gL
    zl|Qvc+V}cW!&51W&8vU-BUHn4#P>ID$tmXOTVt;`H$otn-L2PH#jy02;G?qtP9{*s
    zkJMPvI>edwO3;EYyKdp$ta2!MP5zhn|VmI6F-O=IjzE*N$Gs4_6L*g?lK
    z@v)Mhew5vID^93*AjwG;EC~K_(`kNO%6n`8wN@5Eo}QDhv}zOTRsNXwPpKf;yyX~U
    zJ;e#qX*#A79?TtLFw&IMk@%cM4@BQ(^uYYZ!=Xi|m6j!?9kKwul*N#3Qj4=so%=qg
    zA7)EIY4w%~G|e(dFhIFJZ~|ynHMAOT%SrAwdnE`g*L0Lz@$!~W&i{?jd1*)+qDQu%
    zV1%94RzwJ?S+j#tOfoe`Mt4dea!~2v-;d!`;&|YNx_={S1R9mlq
    zVz9r_!w<5x)$DUWaeR_PCnVHkMUKa2ygqPfv+;N=qm|Z4=
    zA7M~xEAQT|frlG(H=F})&`VP#>nG#Uk20tneGP7#za=qoq!uUm$mIMpI4IM)quZP@;zUP?MMIh?%khbLi+#}~j=OSux
    z0Z#oY+c4dSxlGFA=Nl0Ip3sVivm~Sqr~c+aAozvUPwlx
    z3<`G)0ASl6aX{}{2bS<6{l?P-%+;*)6?x~DNfc;=6z
    zijPL+({jEVtCNSf0XKRl-&!BG0?-uyNEJ(YaPb~>Uq5tw%xyV*4FerBxX3ReCIV*u
    z7J3{YF(`ON1247(HP*Rx`k7k3@h{aaxWp+h8!IMO`0}G8Mf`%URGgO|h+957t}Wi>
    zJ^A1WO{z2*csu_E#3b~!d6XT@S3q>rKT!mn%c5!9Wo|pekf)tVj|&hKQq}oJcW2GE
    z6R6!$yS=TE@yLs`&RuA=xqlg$zPj5|%ih;2;cpn(+&O2YnL1+o60G9p1Ni#6MS`Lq
    zUBBTIwX2n)l?wpE{>UoYKjC(KW11mL*=j@@w~#0<1%qp)j09
    zDf1}W2~(YHiA}#F*uRN@wn``A4DFw-V6Vh@pTL?PPK8aXtz(*@i_nEd1sgIOA1-$!
    zrF?FlSmA*AS@NDNXzPJIB~4ai4+TKUl^x775nSC`<%ot){atEf{{tk$cLm^WBfo;o
    zFx^_H%kW$mqq49CY$1@{gN048g;hC$osB4aa({BJL7tDnNG>J+`1TlX>Z)rF0W-B$
    z@9(|l5%1fBX<(8W%yjRAg#e7`6}`Dp$a!pB!XM>$jQ*tuC*2jxt@@P;=VLGBtwhxM
    zt$|{Gri90>l{zKGiycG09Ec8f_V2BM0vNMN_eBT5lkVkz(V0F*XW#&n^o%Jl;bs0a
    zS`;p$bvmXscqJ+7H)iaV
    zv#aqGx(hH+eztoiD!>bSq0M^y!k+VyxUSOn@E=j6H;*e#)14qs`S@R!Z6R2VOX)0z
    z!kc+k6Eb3AV(8tgon7>e_^>~osQ>It>c&bre|5UP&d$Tfr&K!|?K>)2{^5=`Z~(6Y
    zVw3A#F5@_6q|Dwkf-lf=d@iNoCP`^ov8zd}HQ(J8^LpylS>y7Czxa>G|H*p6UYQw-
    z35rTm>(wZ{Rdyz_*0I%Gc=^O8F+(>PlGdpNxpU6Id?6sl4Wp0mV#hkN@@Vuj=u^cw
    zBeP}S$<2v^zjs1krMoR&KXLY^XK0RC&b!O8PktOZ7QT*Q!Su5%U0LLf^4V|<@*YBV
    zOgKB|vzs#9t)KYr$s7iU(VQ~RKZU2!-P&eNCb@E~FXy+bVj(JBprVNRxa~J9Wc=Kr;MFo{QbR?y{JhGR~Gfp`QdA)_2VWd6|1G;^9{^L)35k*{Nu<>LVz15kR;y$^V79-lXkp>_Z9
    zqdTJ_IIp7z;Keu~X(`_VV?l*Wl&sGFc(f-TFn*(i0TtQjveN#}Wtc47#H!jFxDy(_
    zm(Zf}2QsJlzjEX-k#zU-MkpB{F*$yiXE$VT8ir~jj3$@Y?9UT(Q@nE~Gyaygz*N;JmZto%4SF#D+{A{y{4uAkMA?Uzbj3c@V)1rVuUMRNTL|M{
    z_OQPcApd58oVBGjjMU{}CU=i(p_tvf&CsJd$d@Z6j8rErKOgA;_c9!*?~UER0B_@h
    z$^LzuxOt)A?HfF+)nk+=_`#vfqBzM+dlL2KR493NzAYBRkmTjNu^)3U39RMB`%jVY
    zf+43*A{iUaRDr%Sali5h{Y%)27n`=2wZ}jPi#U!m0ceqjG%`v-Vhp{kf#HmMjqd4n
    zW_H0=udTx^dj~trjW2^MSs%UP(DPYS_XS-ljI4G5*LwJ&u~lu=Fa;;YL6tHG!i#&;
    zAn%g*aS+~rN-gDh!rrANIvL;f8E+!#+#^v{L
    zg8itDu&H+zRPZVwK~4>j$qxuvKLjP-g}np80^&+vp!FB#t*PG~`8@MiLmrQhfp~O0
    zsiTLzawEh#w$C(7KaIYjhi4;lfzfw5`_)h&RTYROe^xJxGzScT_d9--;5!FZr)B7p
    ztnhsr6j1>e*c*pH*jdOY5r{_$MdooZ_d4ek1*zC?WLko%I95U8gi~+{A$T_HP{Vfo
    z+HIK%3eu^E4dGwvsn`Ywg_kwqp)~~kh3Q8cS3TY-e`jO3I3LQ(z~|Ttoy{=Jj4`d0
    z_D`dlU~~Xca3aqCBjapH^=hznbkEn1YKlZ|FuHzt@huTR5`7#)3Q1-6JH?#
    z-lDrwS!?r_LFDx4%5*h%`gIhcwicTPk`{GNIYj_sMa{AzLBqxzb$=19H4IUFZ2pe(
    zZ>!AH_Pie7`g@5jKg{Edo}LwSiR|y6Md;wrRo6a#F`!LB2$-FV@V4D7iFpMnZyrTG
    zvc1}Picd7J;02lP^6pH>$@gDH&2zkWDo3Mhau#~%r5>fQjF4C%yxdzH-4S)DJ0Bf7
    zGQ)L-s8C8qzMEYwoX*5#MDH=3AUFGJzd3O${w)S`*BY36E|TNiovBjlqTL{gJ@kt+
    zklhsB36+22;ucYQAqtEBK?6H-q|I@7e!H+jN-a<9wYQ$#mm6Dy5*u7oA>I>O>s54F
    z1^9`fkxC-|8W3A=##&8F>*ta}DvWGL2zZQc+>?!N`1!wwtIk^qcqmElzX>0L%#m)J
    zM66K+j`PyR64<m~R_zsMqj$C&t6h~aO)P{nzN}Ktfo)!6~hUwcwL`}De
    zv`V@zq??Cq01So=+xDL%q5W)h@Ic3^A&Ml)6Ey{2ePKDd_uyl4XPR%CbtN~*UE=)+
    zupZ&7msYV4!p@oJM8Cyf`;SbnQm?eGoP_X!PMz2QBiJz!Brd5-=`%IqYuRC6mauhc
    zrHDvW#(3RYzYpeC2{#$byX!gQK9j9&DZixr-(F}9ZMQz|wuEF`CRpKfz}XZJ(=w2K
    zN~4X?%@N1Z``9%9tj*Q|cv4h9TagNI(hkUb{-x$LRUHd+MsSA98!ju%^+b2;Q^>0d
    zok2GC|1`ZQBuK!{A1FJ;!;b|nE{Y<^jl@;=`Z(nAuQnYz8<04xJ+59)$)f`y{;!BS
    z^(%1Wz}&422FJ{s!(;5;``(6A&E^Vd34S#F1QmL|fc?&u)cMk}28HEiO8(bM9~o))i6A&6LK|P<7Yl
    zlNHxPhfpigiUAjzi}81i5}p->@rv@bi+uP)tYT?M+9Qpp$)ou{X&-kI1LLUG*&8d8
    zt}7s#gdij~Lm<<-4V&NX{Fk2Ofr7OZk5w5)@GrWx`Os`SHJ(a`b^#;*lJaD8(mW*U
    zb%l}rjW!w;QNp2@=gkO#VBU^yx^wa2DN;+2?|awLXYz+O)gnCb$@rc7?!OKjUH)ku
    zw%|&$#jQnQ!o*@-mY~6eH$*CHeVURe$`-k=T%pG$Y$sE#j2IB|!y$9$?z4cz@Y<2}
    zR$NO0(43D1LRrr-Wnv%GW)BjcRerHQ$G3hpzN+h7x~%`F!s8{%P4b1LMoNO?SWIS~
    z56AsaC=isOAT5NNx9WWthAC(j)x6~4h#B(>fUSa|;lm9*Jr;HZJwom1PN>r%G
    zXYLahCuI88dY|MU_tYwye!`ZZ%v`6Ni(_VopIl$YOSh~&1-39)+riV|x_j8aZkv;$
    z2++#q*9qxhS=ajZ5$9{{y68l*W)#jE4CJzf
    zRO)2Yp!i01ZWk+$-XKSjwBb8dJ(GYd*}XB|U6$TFJ5NWAQ08{RM0nEmV;q^(
    z$tJ7RVax)uEwmPUJ@|%6rE$
    zbmuoPU@!paiCLZmUK{5ICB!cuMQ}&*b
    zsmaDf>?v1B20u;J%q)ILGLC@|Xcxn8d@7<@g^z~PHUb~ND^#Xi{FB?ufI=X=K;oB?
    z_M4~|HTw*>V%`Si`sR!ojLwBlF{jRr;cC?bQ@DXP)13>hQ1Z}ykq5r{RyT4oIE~TG
    zf8JYIS6hssPky%0mFciED>^1zkJ*CR79JJPxR8!)sm)XSPJ7WUMiD7!V+(Bu({{8m
    zPZi5iTdU!tRmO7~WdKeFf;2TXU(K!nNFxN*1?(<654{}M&Sp3D_la2gmroX0@751L
    z;ru(j$l(mF4U}^;pyVR&<4m(xz2uz<6rrL%7*OY&YIiroU9x;ga!_|97VUm)dGE|S=mabQo6>=xTi
    znpeZ!IVw7$;O~RfDnMhshGg_W4-8j&|6a}QX)PEssE-H-3Aui@4(S6@Z)wlgVaP(*
    zb6OEkd~5$mTC0O^&MCbjUp)Tq%F(hFQ|b`;h?lf%_?K-?G=+1b_w+e~f(|tepdL5b&!ETU|A59!8wE
    zMga81gV67!ujwBui7T2uXd^BsT=3Qv3&OWvf(nf~eXu1eJ%Sqyrk!!UuiOC-o|d`$
    z@t#}KLjzWqO2+7qH<#8(C_m#9y3~1^e>RC){@`W1EK>)K&FGh=pq_VwA4j*_m0G!-
    z0jL8$<7cHF*)>~wOi*CrA@A;$fgC)Z+nK-nu>t>xe+o|@z|~A!@UD@Tt+4M_3G0>FNq^50Hn@u{3=#VBgx^D29Tc9#ow<
    z6%Y_ue}VY0+kcJwp7^eUhY9{f4EJu+g2<9HDMPePv1BZoEinNM!1k)~aDRi8Z&5QY&vm=#_n
    zyR8|c5AFT0@Xfitk(zUYisS~XHyDR;3@4KL&&k9*EOcWgr;+N5sX+}4mi!e2n4-#E
    zY^2p+dR$dKJwsNQ2eH-wQizHm1zo|uR$4vnx)=Lq(5nO{{@BZV_6t;Y{O(;o2Z!_;
    z$5*9vcOZPI<>&KcUq$>x+2Vq*u|AB!Pj9#vj!{O_uzf>d~Ss4%5JazN$NX
    z*Dv{x+=$#Nu6}&hOcz_Ri|sN&kCE|sl=&r~5zP2FPnsu2zm!#G&jr2AgX2mie^J;0#ZXsCfNmD#*0^M-~N)fRFIIfMzUUtt=g
    zA{|S8JQQ))GJK!bo=R50+!F=8_>c$2wB~v0Se~Jo8%TUv)r#i!b)AX)7a9u2)ZnLb
    z*L#cdZ>*G|9Li}k{qvsV7wd#wRO28HmW#HIcyWo{b>3Vygzuu-NTGXb(G$o(^cfRm
    zjCW|ef#a6ilaRy~--X8wM`c8Yd(9hm@?^lun9}Z7+=?>6)})lGlUcfxh?H+;#82aJ
    zto4nJ)tiQ<=p>;C30l$m+VkH|@_7H*i9lad_jpAvaKz1Waxay=i6CU9xnOp3#Q5sg
    zW|?9hhU4^aauH~6exGWjy$En4C8SJSXT79Wt?Es0#)O#FX;)`yDUi_+~oqk
    zbr^>xua*}NsXLtvy6?8j2nJn*>d?|OqF>O7W+cE
    zLZt8|`Se@aEZY8h^2T2BCCR?(1Cxe%sURN6S8El6_Q4ZR{s;Kzz5kx9MK6J{=_B+c
    zRl*2_C_b0Xszo!)q+|Mhp~9t{dE>@BD*8$YXhr9|k$FD^d27$-u&F|5AD-$C!rN*Y
    zJcp8q7w6~|ib6nFxde%+DZJi}%-M5dg9qwrD8zpX^7k4k(9-Z`tI!`F-CcF;(O$!2
    zJT{Og9m7h+*u%x0tDWW)i{|wX>+zKDilbQ`eDx&5dpc4pV=YdlZn3VKl0|v-T
    zS?2BsY#}9JhFE^alHo#>q^Gzmp3k^VhFRgiI_kC`*yo)6#9H^dg4!-#!EAve
    zb^3AOo!_$!<&{cW4g{&NkeFcF5e+6WI+F8QU~?cw5dh|Mvp1hkzOY>ux-XhhEZQ`_z^}QT-q)1?^4u;ud*@M&@rK#*{I~nd
    zc``Y(f8BWcql?J#S3RGWom3a(a{$Q#863}jx;Y5wJf#kY>JQipo;?i7^3LSiM+*EM
    zjA}Kq5)?DPbp)~JK6o~u`L2zor3sWlJi@n!gvegfJ7VySZ!gotmZDnuchHtEMjCc)
    zIO1I+VcE&fR}E{y^*7-+KW2vfiKQaOVj`?a$X*k@z6vg|7DK!9@tPAVnaZro(_k!V
    zG0vnOOMZ01Xi0BwqPvKi3t8pkk(@wMCZ^7pnzgbCb*t}s{9U!K16uymRd3GqnE0@8OJ5IIic$ms+s^9bE5TYp{v3B9|o_j3J#E04Dzge$Nbn
    z&8^5j!t*D=-KBYE*nDWhamJVhRDa>_Rpo*k9%j59j9NIY!j*%hhbRiWXEgmC}SscdF*@vLpC&i*6I6l&s{7L{23&4)3t#-=P^{2b*D{
    zamdhIZZ_!scdZw=B6g6e*duowhL^Cp8hn9eh9w~h0IaR_x0?;E!#yZ_^lay9<}mU{
    zrc?LqEI-Y(&d&|b={34tm)C;MJw9FY{HM=%jZ29V$L89--Hx5dZSQ{#K&!=VrE{h;
    zo=Q`9W4(hH6lEACTidJrY$bH^Y#tHqJ;`>E`z}C#&%pXzZn{4uC)UK=#FZf6>PT_F
    z2zUP;l7-u9e*B%`!3;^gXdc4C;$FRWV_XeW1$nTNd5#L5Tha))Fgl)W6xdGVm51hD
    z^y85L4Myh3s)JeVt-ZKY#Vo}BPMtdNz(#;awZ#NL5)-N$*Pa@&yegU|HXA)u@~Is7
    zTyv?h^hi_nTz@crOH97Oc=yFFmJN){z3OzFQ25Tse6
    znfSm1l4vSmPE2_Z?Igwl2WmwjckH%8f_9v9|
    z*-qFF7(CFrv~7t=`0ud-+z|#)QH}lP_jB?8BNzMlG;S$rIuI0N;)Mjk^2{GAUu;7w
    z;?A!?iy`DA#@dk+=2YyimOx7Xsv|PZokk!rO$WCy0~eWJYzJ{}2L>Z$wp=We7S9Q`1SVdh;#
    z3AHnX*!dHnu#syG<~2h$&cE|NJbC`~PdoMf+B;k}$-S=>lbygPjcR>=1WKyk({N}}
    zJznn|WBd}^;d$t{a-v2~gz^;RG#t;K-)1+xb~*Ak%w}U#DdQICM3wJ>u>-XJY>}tI
    zBs4JhE%W<~haj{DU_nVy(q2*f0!+=HaK~{jkPnt`&v^4aq`>=TZHC`m``h+=SauPt
    zz25NbZUe`gGktzA{!DY0!@pnKAg6U4Xp`slgsk*;qFaxP%V-n(HLQJ_AB2w#INx-g
    zBfZ?;UkFqFlNBazad^*T_&Z`)bxjRXGa$tgQQn}ZjeNeDsWDNdW?0U=H_~xO(um%^
    z%u&pevarbM+bDBbE$mpR4E;`rhQ8GE?vI_Q&%oq^GrMmF7V9q^)vmFFTu5E#Oj%{H
    zz&oJb^jMohVnU7en|yyoW-BEv0WchcR~3v1y2=WeHuEGl^ZX*}%%rA>A3_)7XW6gI
    z=Z&Zdgo-i|&Sc6U@#^q>PHn+{&kQ{%JUq!3ABYGUnf*X}QVmHB7p9(!z
    z5PcLTJjXmIAFo2WBhwgYcn20dvO77uU{1?c6<`#(giy;X3taL`g4dn$9v
    zTXG;7BY$L}%A?gNY)7w4;icUWo#ZC%QGXL`I&d?+$`+`KZ`W<1({S}$bvhV5!zc#f
    z5`4PvLQOVg{5eK>)~9^0AZ?<>q_?b&Aq%cEnr;t@zKb@izcn>5!gmNA
    z#WH}HYgK&%3g2gneyJ`2kWAw
    z8xd?==RW1x>gIj(3S%;O__zK!PcYms?m{C$+uKK|l`MKC2wlS2rY=Y?n9)_F#izP&
    znOaT?Q&oZneohf#(H=FjJpwZx?2$wwX#ud1ci<%`*?Xad-`qG)
    zZn4<{Gq4jRwq3Rd;gB+wt)0P_gR_z+s~sF2^ZW>XMvP|3)~}}uQYzD^94R7OXlM`j
    zRTO{cvA)0Ey5%3B;Il3>QK37p0&`F2lr(%d6ryH?`TFqlS|j1tUtBeD9<9K59U$#8
    z{F;cTPaEu<)ls@3pVFHwtflD^hdsJ`yBgOGohdd%PBnLSEv|(OOex!nePiXWWPC(<*e<@w?nYRx^r&F;W9X2sC
    z{~Ef8hMw*uOq&};*EU`!q~FQ^nxDp#^`=?nSJtRUBkV0}5M@EKszoH`nT_gK4LDMK
    zkPVnL{%}aAqqj|^C{{09#KhSBjO|&j&gM6G=j^R{<=FF
    z_PJ!((m@7IwEaT561dQt>e4
    zlZW2FI=sF^%PnF1cL5drBnYkS%vl@t^-#q;^uuY~?>Ykg^>p>OY6j@UK&oz@z4WgE
    ze{bx9pr%f#nDMmD%|QKOt;4wZO|psQe!iU|ve9RZnC3(Z4V>A0iBVidftlw0T0vU6
    zclN>6p1c~`KXoc;oJDURdW60+6lKJyB$HV9Qd=a?5M4>IVC$ZycT`$Gm02dTw)ZVj
    z-k~TR5zKHnvV)chHpcPtpyVC;kG>5y*HEM=CzG3SG6E4(fS$3>io8jzWXV^<_1Mja
    zvpFR3|LfS>g-o1ww?9iP26USc^rDLtG(o!7p`dAQA5V62cVQp9l(@mdX7CoHgxlYI
    z9u`7zp;;hrAjS3_X2ugv+x*1T{*-V;(6ObEadTM#j;D_bwOpHL$?G~n+8&@xL$p(H
    z7?v*SO`|=>{3Gz)h#F+3=c2_temiP{P`7_Huerm&1`jsD<`Qs4O5fs{4Rx!?56Pfg
    zBI#d0m;zX)hGG6(U}a)#5r4H?{lRxV}fG0{==TxIQ6_kCL4HxcEeL5H;d;q|-(4KpJO-c1fN7-sZm9U@%Kg{i
    zb6lNsoE7^mErhdeaHbX1^iYD}7E@TqG-Sg#S3mfb5>eB7votgzIFwKqDc09TR!f7a
    zRFQW?VkXW;ff--r5ht!&djZetvL(;L8-d;0yBFW!6VN;$i-_!%RDAex*dsW&3#_G|
    zJ432XNIb@5|;>1+3JOku2v~!*rEd^%&YsicU4JYgZWEi0Wu__!29M)
    zqMBXcak94`8Z2$CB40;@l?NGI#^3Y><<=*IQm3P=B^t!SJ?)whIACN^Zz*_PY$u=g
    zbin<`YutMC@#OCV_!j|YP2l`A{!mTln3WDZ?#%5j0EH0?tmDu;OXs`immDvrmf17M
    zAS{@zIBn_lt?BlQ{0$xn5=_OdxT7CddyAQ2Wwf%VbylD@YUcpzm$U*BkUXJxUg0f
    z?#y8h+^``aUW06uPnOmID-UAu;^;8yA*|m%t^^TAkXw|#eZz^$c=AAXo`a%XSR|Vj
    zM9=v7L*o
    zD7S>We@6cW0W2iIfZ^wz^m1UYa5IaRq=CpFESw1)hEFQy6;5Cdvn;Y)5Y=Q!(6BJk
    zuxn!T)x|mc6-|fXbnlLJc3N+D3t~r@r}s%A1b*cI{6L8LdvOaPpvT+zPr%I~wdVL|QVdSO1Q?EN_S&Ly)MKs$m
    z=v8cT33kca;L0I}t)kJD;94Y0B}2GjWt7&0DG5)c^|uRjh_Lfd^1knFQua-qeh=>z
    zMVK^+4xnRG=K(D|?=j2a#f#^Sk{U!<
    z6B7|p`n;n=}tSxES4nh7q};qT+7a%XXdP+?~_rSgmgnur-Qt
    zd;Nf?pIjOun%TlwJKbKRsYEKmnI7zWOLs2?W9$Q*69)A
    zW!IH$SOgF_m@|x+g*5``L%z)CCfWO`lkG^%simH@z>id15w_5b?P=bj#?&Sq&w8VW
    z4x^Lz+Ik>6iuWP|#dY-E{tb@WfGYO=>Vx|-V4^Ut+W)sT6XAcj{w2z4^i_J!A@T9-hFhX?*-y}74F5s~a5dH1bO5}M>A71P1K
    zRGXiOOW*U%AP&u17&TXLY&RWCr@ZDCCypRmChsuV^NToIH6=R*J^b^mf>CWgB4|>vXRGb1S%t&}MJZ0Zccls!xoJ>QNh4o;X3#6nVi+;DRutMev(JQDV?6ENJv}}jI~8a>rniNJ
    z$wrf|#}qFzw;M+K{Tsx-D)v)of2q9O3%Ss6J(t-J@yykU<1|wB|J?foct@*W8vnF!
    z3b`P}lyKq##ER9vmt51{u1h@ZH#cmn^v_K=K%npj#=QEh=H@TY+25u{LM47?UmO5d
    zwJbDu(n4+7@@1^vRe=z@^PtW**!xd!+^iro8)OD<6Fqv(^PY$i5pfYr#|Pj)3l$IM
    zq%o%J1--}9JFVNLL;S2|=Dm&Dbxq;4J^*)EijUd`q&?D~D692yR9XDehiMv&%f_9~
    z#F?aZB#2%Uw3__sSDl`|1e7PwaMxc@cdXwBaM~`?XEi@rR2^t-cEwJr3-t<+zRIBb_1OmEG~{EnfS80>-lbs
    z*H#!{*(>UJ3*R#;%GsJ86>#a~Y<%xn==P#VxKXpO1=Go@JD#^BicGEtdgmOa4LT5?@Ocg6ofQ2k5({S9~lnsCWX@Z+>nj?zPlm7v8X0z2dI7c_(S
    z8toR?@M0~KkQfH}PZlFiiQXchIm!|#`=l-_q5(J^FD|Q1=Bd_93O0%*^4^={(^*0T
    zspvoQ^v_WR^N?(PbK`~m+(Dw6d^M@^n*sp>eRsIwAon(&6fU1D*`epB$m7zikap<)x)35AvSeOJk={OSj1!ar1!<_bf$$N>`PIiuvI1?2D9l@{#4t!Sby
    zuPqLuUfJ^Ag-t9gj&TMnr*E$*i#N^%!Ue~w+Q3K+z#)T6c%48WfI#JPqm1bvPvH1ZwU7-d*?$u$?OmTn{)z@KZSQ_z2kIxOfQGeF(>9v|Sh4
    zTTLEDXbRkf^J$nb92;oKKV@|Tr9!PG3!5sT3nWZ{zY9&Bo|jc;4f*$~4*bs7V7D1e
    z<3AhK_iH5Hum;9in2ugzz;5^Q>37$Ynv>!@RIEpEgb_RU8IquHLJib}}yqnRz$m%Z=y?rYml
    zf!VJV3M_oKSbkR9i|q*0s^QFf?{K8DRuL4>!ANcYB!g{~o;XR-8>>AI5tzFC+3$UB
    zH^Its+=Kqnb_-ErdRIrft7kD1_L~Z|^zS5E+B(gE$xoY(ydkE1-^=lS-Wpm&`fF|S
    z;+)z}vc0>b0=LR<6^YAJGO4V>8=oG{F+YCc&D9lm#3GMhQB~FL`ft@$5ik*pIAlt<
    zdQ=^d#W?AFs>Lbaz%Ot&7S|m`)Bq-TFOM**S$ZqkdNL0+DBj6&^Rw>Gj{>Q{Ga~ww
    z3A;BI)Gu1HmFcs;Xgagic1VY97|~J3a2aVI6+Fb5oq{(5(kF;i>^1%LCxD*|i2*KY
    zjcpfP`-wAki#dsii#5B9D>d%y+lRN42=`%(LQi_Zi|prnIJJmMPI0>4yzA6@>0U5q
    zk5v=>afe~7Hlm#W5#7t#J`|ppCCT->A2lyRHWzT
    zc8)C#O;OAjoTIvsi3Pe=J4atI*gkLGlX)>_U;s4i9f}B~G|g`UkMzRrNVlzqXICd
    za}@7*kG#;5s$2O17C9oMB)vQRPm;$31~t@@b$_&Y{Oy!V{MA0LVE@`yU~p%>|fa&1=>klK9#G6;d@|#3ZBdg!4KP&8W%e
    zDJDC#Wh@mM;>-@hjpZno2*+v&dqu_+#B=DAlj8C&8)J;bfI$clDwl~%W`(T;Szm}J
    zD7$g%UCbLl{yYbUzm8K{cScQalU-cj|CW&Lo#9qc3Qmzg?u%a;
    z#&{;Kqif*pi^g}m*arA1B_PucB5;_8&un?N3Ve*Ysfd$V?y91%z^XGDRDC$20ON*H~Zj{3O2)~Fef0SfXlGgwAa3SN*TEnBhemA15!rI*oed5cn
    zkuBfSW{QtW;m$Flp!BkuNhcfqgs55l#;29S-h6KKPSH4YWhS$O4Ww&NGdGEb)n#_Q
    zHib;XWF{>WC*yDawwm~+*cCM@Z!ma6Rb|`^7~A3n(RcueQ9zaBam_Lo-wNUo2^$tc
    zKL-dQLya>&4##J^)X#4|-iDGr>G=6;Aj6RQy$9E$P(V}clY@G&5K#=m$
    zt~&LM3V1PrZPjU%)1{m4U4jfze%>R7mxPtlD$yW(`ToNi#{w5xiE#WKC$=O;0b-PN
    zNc_7q=WkZUMP_TxAA$ZPo$0`_Yh|zh+Hl4p;fvji?;zWIZHM51hTtD@$3RHXBp
    z+g6YxTRun+%xE#j!d2avXv1OcCgvR7f_Xdp3si8VoMZ8s+{!ccch>%-SRK7%eXHR<
    zFqI?{<%eljk3hoyRsuk}TqDmnN+qSrxA&KoiZ2=84fns3{REM)!L~j^jD7nW72QS2
    zKl`Bsa;3i{-Kf>EnA`nFuFT8vgCgsCHTU
    zSa8zVbIsT{XakW6Ba(|!=`hJe%Ry<5<9x^ci2ch${|`uGcHRxpjc~zDIRUCw@?b_^
    z_X}I|qrRAYLxreX&!hC_uKd-YUmRJmxP7gv;kyqy-$L&k?OrX@CJhx+-pZVX1>jal3D{%TY*)8iQcDiii#Lvh&ZuGAe9gP%!sd_G0
    zN2_3&bDUmBoNbLeCO)>|OVay6B)pZ&o1ee|Zu@+0G=#(;!+ijc;b$$N+IQ;?
    z;5q)0c`8T93^R8&Qz@2GMOl9PEeKWy+arH}or)}uo!%I>qzx8Z?_LSwC2&yHiC=j*j7SED%8Ex8
    zjn#NRjDuzkB%xH^RdRTI!
    zzp}Onz4}>jxloLIAV@K~okRD6t{MEb0sf60i*bbBDT0W44fB_%o9ZFfZVL~md;6SZ
    zHo;{XsXUGRwI=Cg$bKNMY%TU;TyqKBz+(_&3Aq5@wf^{EQ`K83(_!k&Z2~`{qEN~Y
    zoFvhtZX7_Fp(d}KoUj*BX+Z_-E;AhXzM6H!C~DW_8NEE`kDki}U!FeXTB(tLv2q*N
    ziSQlBKzt3WNDLyRkZXH?yA1P*b>KC)c0K#@k-T`(@80lOkF0tg5@Syfdx+ETaCN&>
    ztfj;O*`B@vEfiwKE@i{rww(#g<6j`K+su?r1~O*v0xQmJ?Jvf;l=wPU>;=E_r`J9GftHE{;zT7jbovEw~;(_7_qTF8>W$EKmxfs
    z+EoJ!v$@eY5aCE~`0C>w^`-T{zhvd_@0l&#(t4UFIRdRBNP{T9>*vSB60687JJG7<
    zTDqKgI?7bd-YRukpWqH>bF0%&Q;M`xA~&%ZxJ5Bn4_t}>h7kHnky2I$N|Ba$w~>EJ
    zulM_CvMr%Y`*QG0jU+c};T-z+Z<>#~{MnS;z~}SnLou(vb}!ncz)(m@YMg)^Uk|2X
    zm>j}OL6($Smnu(@35DrP!JIR@ZYn7=!WHvTLKVnGYGfC!&aO)5%d&DMH4}b
    zPmu2vE0Un)Oh0qNwsi7KPOrg6xvZ&mNJqt_(GT-&O>j;1v+>Epia4)3l1>z(_+jxg
    z95GKEf2%j{`;!o1KN6!hHYcRvewFMnhCh#=C_Rb`t29E+I|jGI+>=^0|>@sACrLF<*0%cT~_t)JVST>CBj8<{c!*CEN3pxnr%B
    zb2@kA9^jX^F&cyGbx%8RY5!9YQ-Zg;2hsg7?c0Mxl~1^D@PvWs$1o9=m_|K|)9$_<
    zbXJnOIK6bHla{qOB+b`Y)N63E=_*~5GtFc=j~)p=PCSt)AN|JY!t}1+4!%t@>4~OW
    z;&YF7rGJf0aJh#jAfOWPFV&SJ`X&-fAb37kB+=Ub`0VT-UKZZxe(D$C?$(Prj8Gd`XJ|j@pl#
    zZ}EPKNSUF0I}FxBQqn;X>P6d{iLh)P*^Jgup5pzRugVba;);_2cd!0usATBrrxS^RcA^3>0HlmuR?
    zDpqwexqptK!>Im~ymd?3?K2WAs48dXQ$Obp%6mhTGr^;FR5*E)v+0PyF9_qYdMa@0Kc)TGe_
    zmdYG3h7+(Mt6?#@cKO%{q?u7kAs-L`M)Z73(8X{V99HS
    zgX+ybTQwhaNHy@>JqzgRudz
    z8m{I6oX#dEn&;f?^?zxe&RZK^m*#S?g9;Hfp|H50Yii9kKlh}qIpmZIh@ab;C&f@1
    zbW_f2BwyB|OgX#nJ2O$@2tAqS=~(SnC-mH2_PDqPZ?+^wxrEM$v@k4rwlsV-Y6*e!
    zt^B(FcLoQ(9d*U@xWOnfLNw`A7=Kfl6svMyOm8<^MPB=RAQVO0-oM#YjqmotQjcg-
    z_U9O~=wHE)7e7rvLT=4dI*bS@Gm*l}?N6%zfWz?4o`{sw07xzsRZ7qX+0+Q2R+wN)
    z-$#15QDMxxG~TE%CuQ`7y-Dt;eth=Hn|``V;^+x44~OoDr&z{9;u${>ukF&e^$v^=
    zJ1v@n$kt{~!Pm*t(y{Vw_r`;ZV|_=z?*KANKdqmVKi&8*fbPCTkh%G^i|R-i5jV@o
    z8FyLOu#L(>0ZB;UO)d^+l1e{l!EZTUVPX6I5Fg>`GlCxo5|^FZMJX?Llb&Pp#!JAS
    z|IO0va)NN!9H62k9FpnbWU43Eo~t!PJML}LKPhg5(wW=$Oz2n+Q*%Y!CawyX`(*hh
    z0np|-AFBXf86Ts8;1whw=jUc!;Q=HzSdwC=Bp&ks%05m5R8e+LU!E0w(my2A5Mm~P
    zuWZdDW|lzhcQ)~Yt-BD!qwiOOpNzFE4NwBXtABc+^nqkuGksHZ;%h?r_FR}XGJi{2WFf`
    zNh_yYbIZl6Jpcgbk9uf$IbXKVYzcJR
    zo*#wK9;=p~`kbDe(Rc~O9|Jawd5X_?J|LE#zx)IHEtqXzrkbuIcp)8D_WB(kY2RmK
    zBfE~_qndD+i_y|zvKF_fK)mN86Oao@r3gB2KYI;HPQ?X$UyB0gE4GTh3REWb=*@QI
    zIZ)6Bnx{&rfe0Q(4BY#xte6+9fV6xl9>#kN^(%<{YnJXgX2>P_@i*ugy4dwKYRG6?g;xtXckXZlpa~Nzcv>u871D6+H+87H|nEnv?s}%X;)bJKm
    zcYa<^RmQ)gW4`O?l=7@8!B`5A1MdJSgJ@V4`SPHyA|!DA+Eztd4hpXf2B?F*RwxZv
    zF4$QNzb6XtH#DNAr7XJk@|H=tB4*-+aj#&yJdV`s!({$_+_i}zI;Drg^9Vn;=5QcM
    z5|5Hph5J3pnI-31QG%^|n=&QbSF+S~>+#`PKUppQ1Weqqh9N>a!tC$dQ_+tNROEU7
    zyc0vqxC*i~E(|$tPOX-p6F(hRZ5WbK<@H+lQ4uGqNTZ{j{Dw)x-@l(A2CJBl-qyYJ
    z4sYU~E5Muc^dwPz{-cAqX~3LeciQnN-o@GT9>pI>0#UQj!aRjYAT}wD`w&0peNE?k
    zg&Wx@yN|fZT@g_y@H!vknDNxxLYoNe+XhDl#Xw_@k6&jq_+oC)JSVUxWyKT7k-klm
    zUjg`k^WC-XPT(sWXJT6WuA*#|O#$GVBIQ#YkuzY)Qq9$#lzgTod{Ms7+C5l$MTL+U
    z4S)z|e5=UR*fdgNZn5U4!ROXQpr@S#8*~m_bQr*1SB`xEwI5`W04s|PR1WOn<)p3H
    zS?cVU#!l<3Z_I)lBvgVvW}%(CUqtD+ef^H$M(pS>m=(Dxlx!;7=1)^rAK+)8?T?7L
    zHEHMJ>23V$I_V2*9z@ROJzWz>wZTR@u!PPJ3;(%%{gb?K^+m7!mlU=l99gmU=MPCy
    z!T>rj8%dkr^O^m7gBzfkv|yUAjnwG>4C15G$~lA{>>d^9ZCd@+fu1o77OhCm_oPiL
    z>zFsMo;nGc>=)xQ%CPpf`
    zmn>2UtyHSJ9z|r$%nqMfu_ndvoT@teI}j(!`AEA}B|l*A;mcjKdWKe|0L4RDI^2}2
    z7i^dWqrWiY4!7;@tAKPame-`};mPRNAhJ=At
    zHUSImGsu_G9ISu8z9nv6&z~;fQplvsrtriC-D6R4JZlkdBa!p?2zY=PLEk74LZpm{
    z%PJ1=ceY5ZxX=_t{~9&Cm~o?z*&>s`ii+q14cASxsqPj9CN0CIPm}QvM@0>`fJ>H@
    zL+t?r&1x?^Eh?=9w;ntl+%VP1?jlSJwINwqe~$=XiZJ?&9zIiR85ndQj0Cmd#%nMl
    zx7M3Rr4Pd!%{&d7$iT3sXE%K}F8Hk^HmIXr{0UsyO(Vpk&yY1=Cd$_50jYctPBEs6a`=}w{$DZ!)qG3ePuA>-u#4)bGUqSeB$z6#FYEBgY9
    z)ca0oc)%G#fO}RP*n_z10bE2=Q$Xv_#~7FfEhd1A7{pBcl<~q#7huGY5Ti8F>|E)3
    zTt02g?F&~p4YdbF>PayI=rB;tCGFQKXeVMz0c=GL)INXjEp#Zw2|nT_V1Wh8gLu(Q
    z8#U=dv#*i&ni9az9z!7Bkev8_=YG7%I&zRBP(J8=&Gtp)yQ{Xfe}mm8m^q0f1sdQO
    zc>Ooj=9&%xSEf9fX(Dt?#)sp678-uX3Lf1ARwxZ+&OZPujrs|4l
    zmK0DF6{QgkEH}8`$3TX}4rJvwLu`#W4PuZzuh_#6PWls79D^F)rP{4z9C~Kq{@nRcf#Nabg1lp1C(UyTjB)O`}HY|=7ldyLIV0|5H
    zJjB?7)_)3!fAmK@$sfQ%$Xdl-%JK~}mrj|O8z1z3ck`=1;Q)d!-zxe(xr4yhcti~~
    zd}b6q(Afr=p*rhukAyB(fd<`BUEYYWF7;RO9yBF@CU$Zl`NxQHA<(C6?VV$L@^TS2
    z7tu{%z?5j=np2-zT1r)RivkWN`OEH>sq{RxfyE>24-1f(TQX?#SfuQzKYvQWjhEfW
    zUdl9>;x39Hi_)26_I0wrB~fYz31u
    zeHHjjIatH-&6SnZ(CHC+QcA~tsr-;Ek?Hq>u*ioi_Ja+i&YHjcTh&G(vl{K+SxbN&ALxM}Jy9V=JtD9C
    zFyB9nv%L%cKXaYcddP`Mxxx9(Zm6GZG#WOC)P+lc^CI|d&?h_Yl|Z2TG78W_lbUe7
    ztbyqxWoiIEGhucFL|@fFjVAh`nsCewQ*7n=i`I-!K|IN!l$Djt!fIiSKwhqLeawOF
    zOBw_ONB9r)+$@7rM|0rdGfF)(!adsk@-&;biqsLWCgNA;`<7R>G(2#bo4~@$nm-xu
    zYa026#F>Ih;zUh;IkbR6YKm%Cnb6Bc1!wa!R3#@jP$-1{;>7em4-YwC7$}uujL&B^
    zJ0Dw|2c9=YML(?BtrgU3b?b0H_6IS`UjL%`{(XQ<yBs0#AP|O
    z^1JMkA7j`aj4EK{p9^++d*mAoKAB_TPBB<%Se;|Je_g!6OBI~ig7pvevihq?_E$y4
    zX#Bc0d>I~|rW=VT2ObzfLH;JE$CrZ3CcThh@AKp{z9|9mFXKy7_m!95_zc(a{sxR?
    z8k**TvkL*k__asy_8SgEeD}S
    zojpP}n5N^sRcY^%QuD0egoHyU8lNb%uf0sY;LxC8#d9^N9cgN&xyx)|=_wMa_i9CQ
    z2s--wS502r^LMf53LDHhz}2p2V$d?Zz1VFL=2|3m$&|y9q@Sxx4K?H9_rja|Au%Ss
    zYjzdG)xAG2fFqB&RHy(zd-B-dB;YyIV%EETjbPPJqzrKnFDKh9=r?fVFP@oir^?iQqyBPIsJqn8nfeRps
    z@~1bsE;o-&YXyOMj@7|##;?pOkKg*?&pY&V=`wY`E~wz&_v{J_RCUISe}Yq#xIC#F
    zP(knvZH)h9CvZ6T1-T-kiQbXyOTC60!3LRY(?nGY-a^_-tT6a$=%tS7!Atr1&QV
    zgm)mNUIjl}70wIEJ1xx%1wrt};~&X~NmN)>Tkbbpy`
    zONQyvVIA@3+$>7Rk--iccnY~DY$%xrM>JXfx7~cY`EAvYXuRdj
    zGybP!i|2hp&s-wZPzb#>Ttd?nUkNCU)v#+KMGIx>)~tpFNF^iO0b*3b;Xqfv_y|J(
    z&HGrg(es9e_=wiTo^|x{mQpMrk@9J=5@XTu?B=eGZiw+EA*RVaZ@0_)$;<{4V-9LO
    zNvx|Ix8ygCz71J9-M63~xyyt-Y
    zHE|A|T%`7%#sr&ez<6(M^;$#U7qXDOR$8B-1k@N1p61yY2Q-_gp1qD9oeR%oTkxgR3Zz8+akD
    zq{kcKtIMeF^A*=jn)oZ^0wn%A**jot5jRKlG>dXxK$Gs4Gxp5_|IXT^g|^t&PrlP^
    z*fGq*|30_(a9VXggCz+*Dyob%-~62(^^J1WH-%P&Qnx@Xan_(h9I)g606pn>C8*p%
    z5M&!4Y-=1U29_AG)}I&yYydUkmjie}I2ENRolE%(rQxJA!hGRNd(c#L{z=qdgbxLb
    zPceP$>p{P*zC8@LI$uVz8N$<5vY!5FdY0R>>R=y_Ot^mIh}k%uy%5$MzAz4-NsgZ%
    z5+&zO8FB{qp#Nwt|T&
    zK90VaJgC4C{_9>b@@-$`*j81E1TEIx+|OC%AO7A4b6h-xO7X(P0AS4_)HvuqSOagR
    z-t}Dj1o6d$0#mgWBz&Bumib6XqEQrGTbKKt@m+5>*DVsMap>9F-35Z!*w`Jo+zx!_
    z@5pn;J?SWm^F9j@APUl+UkUsN6WI=!Zo{TM|C_c59MVidS6>BEAa-_U^ik}|$FqC3
    z`{#HAVAK76e=&|@NW6lirJJ+B)>6iCYl2pRVkM?gmVkio-!SQEjj<~4x)+sJ>y8&#
    zN=?g=x=zu1m9$-y#9CxE`os?51^ynG0uXg#jRo{)I0Z{;Ue7V_MV8%
    z>mq?VaeSY8iFP0~Avz``R;}VFNcY!wB-NAD*dOPWR@B-*4spY-S`t|w4Ht1h$LwQvS
    z7NKSReqw0V-R_{^(+;zi;df6q1^$yg7c1tInz0^xJ=?pbfS)o!get`qX`CexmFIX>
    zG7<9r1jKAN+D2++;tND3z=&!LMCR$Y<}4)Uj~$`u=hUPrhBQvPwp0dljEXp?;lQ|D
    z2rxyGgqYf~ZZRrzPtzp)8mxcM-Y(gt+oXKsy__3?A(5Xbvxdh@KKK%Xd?Hrs^54Q`TUVim1D&7y!#nfY+R3iEe}w!QaC+f0Ur>3h*JeM6hs<*t2k&*2$t?2e
    zPb64zHvc^;bR*r%F2h++cCJAM7v1wF8-4Kd4+(XGn?S29Jx{f$Ed*c*N?%qO0&XQz*h`7#`$(
    zY8LUjS|3cOkCOO^YB$_a3JZ`}-$JuPTyOyJ*ARd$k^J2wQR9E-
    zw$dFIJb1dK(aE`qFk1YlWRRk<7yI@0{o%WSYEY^j%>DNpV59zRx;)Rj6sEWjyW$;K
    zGd)s_lBaU*TOGlB*-7rEUzUb=CU^s^K*Sejgy`6I5JFL*v_O17fm-C!J)ig5yE2WF
    z#EYRN`?W6?J}rC=gofBpy)^^B?@KW1MJGKW=E?Xz@r%{^j2H_HvmN#iOUTQ2o6&gz
    z$7$Gimn(KB$J-~;b0AG${+>j_?Fx%M^}Mxx#&_%*q(
    z^9$1N=4VyfelvUng{j)Xvu`+KP+;tPLp@~NQSW1<7uYND_33?f>Bdtfn5u1eZx}@m
    z!Hp$yT)OA-jZCA&A1pIJ`;8RCOC0@TEQPd5zz?cUiCt;?Lj!b)|9h7uWCr!I`Dj_A|m1Y3C0P0{v9I1p)ngm?r=N
    zw2bd7b%!5^khfA=X2O
    zIAg5-(DHuRJ4Vp*p9!HgQ{oPee+icUc^RJ+@bn?KchOy=#(!LRJM
    za?+b#F-g}r($4QCb9AvBf9d)V`1_gPkC)QJf05SyGmFQA!jE|4@hHK5AhTw_?Qna{
    za93?9EUXrNB$>AQ4veV43T{rXNM*%O&er4sV$$lP{CAan)k>2=ld2x#@E2e18NVLT?AH-Ca3UPN0WDvMpK?p0_k@+!j4NMeNwUe!E^K
    z@+QCG#I9@<3!=Is^wJOlM-~`jj4@_`SX<*vWOTPbQie+W!D&9dYq*~UJ++RtUo4ns
    z6XyyXmN}WWhj72^6eUItVdXh4iI}rCl2u~U^R1ZwQgFHnl|Jp9d*)YqblgTm<8V7?
    zT6ZyAXmDjZP?bf_G2mY-^YcphF3gakAJlgl!T(3oS%yW~Mq7AhC}{!dMnXcmL^`BJ
    z8kKG&1OcTPx+SF>6zPT`gaJW7IwYiq5Ky`s&ck=k`Rm0GuX&%^d#`n`gWL8aBIgx<
    zKkln;`)q9ToiumJXUgw(U{ly9_j6-T%a4NvzlulV1*=aa1DxG3KPYb5M$Qc1NWjrZ
    zPriy|T^`Rij>w}KY$ry`6lH$rhc_+bW}Tf}s_~mLPP@8SD*yi$H*n;zoLfx@N{~qd
    z=|ckQ^aS6yo`cO!C{`9CI(5&em7p8)qpY<;2tI5yG9?&|XZn5Ecjtpog>$0+^P+)#
    za|q{WAAzq6JKtT*XazD2b$ecd5Lt`lMc4|It2*ypNTpYT-6Mr%p4MN2bDlYanr05!
    zaTeUW)G~*B!RFLhB(H@9&F@l?IH3~r!`%(yW6hMGo(b2=%Jr=;fAgFP_@pWUKewm-R%E(;sNlERY2BSH#gh
    z7*{=~{`lDkE&f=;mqFC^bcKp3Rp@ob+!1oGiAR30mMh8zifGIk3IoICv54~J2)2q-
    zLgd?tzpQ&rW*7v`u2$4;iKu}Q$d4R}XJfq(CoZhd=z;=vtQg6aFOAoCkG*3nCvR?0
    zeL@FR|LlH0N1IT5W&m2YmXOAWd|Vh=sq2;X<>+oosjHoaGAEswpZi?K^vIHJ(fBga
    z-2j$}V5>O9ZVS2v^p_GirFbYMISi1ua(%Ha=b{pgS)~pQ=2yAOmSgke8n#&s>HBPf
    z4LV={ySe{ikpqFQBvy>>qaO@-y+6G|Rz{3uu{S98$88Nqpqycy50fk1*poJ$8$7_6
    z)Z6(ZE2~I#U^FN*R0Ju0rHAg^RLbf`{j7=hA)j5^Q8tCFWYTbchdTdbU-~$&FVUA2
    ztD1yU%VqHZ{b)0c<0kQpMfIDF^$xvz3ZU(zbi?kDXfOly|Dx7OH+3G4}=71
    z!#hLsHKpc
    z^ExbZ1g1mcOfi=vxS(SQV^Gdz=}2*s!pXTh2?FHSo_MRXoSfV2gL*~~ag}Od|A>fz
    ze-NSkl+KXtntW>@wH!zlzA(w#F(E}3!XP?Qb})8BO5ZfafU|RJ7kFV
    zwXG=#xW{@+`W$j{=`?t_D-N;uec3=QplvV5Pr+#P)?7}^=%)S3ENWmy>VRaIN?9njutK+A
    zS|ZmoXZvOQQ<^=V{9B#_$he?8?b%Z2L`vX~zzZQO@h{(pt1&6vFB3}91??bD*P?Wt
    z%SckLSiboDaTa8iYhck)5#c!NofpQ)BCLu+mPpwj=@7Q|MB~K8+%mYD2BMHTm(FIw@5e5`#I+&`KFQ565)k0j>A8sxtZE{4uM5KFu^asQ_1_MYIM
    zT93=$hWKB;y?rbWwtJ9B`}ts1Qxo>(>28=v+kVVrI(OOeI4`YtYV-H$D8|@%NZJlj
    z=G(g_O^1ksdIYp}ks+lEs$t^NA&-YLwejN6cM?9?b+qMv?FDhY6|PW`R%LB{&`rr7
    zU}6OUb!LInKW{f_T+C-@aN$9|#0AgArV;SL75iXB_K`2d`M90|5&2$;h6!O%k=svzF
    zH6hBhW*JI3SI!HsCmyMR#cbNRddqDvvD%3Ncaaz&D#-y$=FSj~+;jo`dn(+2QxD87
    zmZ!Ipq#JpbBu46?H0E%E-SUb}2Eae=gG`cYIZ}HX^;x)yezk{qD@hZ2`JJ}>H5{B|
    z&ee?0^AL8Qk`tRK>FoOj_I8E@+=2jpQNp&AXG{*h@DHr%UdY-C_`+so0ZU)Xz0zOQ
    zG7R%wsC|TT(ei%sx(g^BA{QnG^58ypzw(u8O7RsSR@K~z_k>@#{f2DcZ
    z3R-tS8z+uNAHVMYpZjY!2lWL^dNj(gX1=<2UvjQi*}gwr*eCGV>%aon_Hq22EezfdS8y
    zwU`-Wk*_^oFD+@+=N={UY{hX$;Wf5xriW4yh58G}6UoGA-0=BgbEG-=QGP?#B5fRz
    zift3Bu;!HksNtC==?~}q>X#s~3C1doV8kQ8zx!u=J?r_`FiVgEN%Dc-D1h4O>1wx6
    zoZtnov&*biU^+kim$7SJqy=Lb&R*{CeO4eX*mT_p%(r~|0zbGZ@ay!widd&%Fua!l
    zeNFA5Y`NluJg@naq
    zx$XWLgs>8ie8e*iny83`4)m>BSbY-!GIo4yTf-jaYX8`HxSqr%j0p!m&=70E6b50`
    zG$0U;`7ts4D#adkEb#`gi#|v5e&>AGqs|LAn&4;OkI-ym?kXx5jq-ww?*`{cQDZu5
    zJ^&ALrdJWLcCddt-d_E9i+o6bu7o)F$+>jr>@f57rOm!u5{cPIg}?1>Y;2bNuVQbZ
    zim`Mdm43&id)+*KM06e~C}Js;8nqwm??WAyGs3hNSp55H-@5e)9ah@UWT%5W1{nf!
    zc5+?Ry9uDnla{&f2amFdscJfeCx8-1x;=&Ofa7DLx;#t_hUQT=gL*7Z_cf(+s9Gge
    zdCpoqtfF}Ehbk2aEc#tlyz(r-o*%^6ME@y7tn~wb1Yvf5c01wYim96g(p@h4VGWQQ
    zk_Me~T!VwZ*Yx8>m+eO;-|L&K!+8E&m7ZM8Xwy%vZA?~V(eHuZDxoBy)amU4jK^n6
    zeds-Y0=q#p7*zj5r&84nIZ(rSq607IpaPI;&>gJVeZ&YJT&%`e(q5dkZmNm?>fkOo@RkFv}PPY4=>#sEc)
    z1Pys3P%XUF-W(NC-ab0
    zaePe{_v>^xu;dT(#;D2zj}8oSY0U?`Gz3eDIakbAM;f<+nf^EDd${d;xOb}{{yJ-B
    z<|Se^K^2uY?8BP*VZjrh-UYnyj-|hAX8>Kp=QJXYN8R|Q5ct~#?}RIA?)2ufTi6JC
    z3NEWT+vXT|WK&KJEeEO+3Y?Jk=5moOWfO@FV#>Q{%b0rCLqZ-VIw&vwD^Y?iL
    zCy;1i>`G$bWU9TxM`3AHKtoA2o3p1h{K_+PdMtLBU9Cq?@xzhdpa3C
    zj)XlgUtV5W6@c3j6&T&E{i9TIs&GjaB7oi7_1T)8Z9n7*OGeSReVF(wYMAb124u5~
    zAOMTDPhV^miKWL8!3b4nLjekGKt}ak_Gc}0ms_<<3cb_xE=a>28AJ_fFV~w-X6>9h
    z$tUJRb(&U`9rt>Gbor9q0ccf6koY>{Bdi#7VgxggNue8uw9LdUpb_=Q9ND#K^UnNh
    zsGGr}C8d+~DM-ag5jJpRBz>-FLxGfsy;_FS#6um@itfL!34oTBXq#95A|(NXa5vi$
    z_0THh3W^hIkGt$m9d!3F1@1I*0r;pG63FbezJA{fjY#{*czip5^F(hfd>WE`;p#e4
    zZPe!+3b!le@GUJHFnseTp;^e^A*p*%!d~+D$^h>Ko$^)R&2G~=-C0Zf`an&v3$g|?+v3>-K1Hg}4S1$b#2iyU
    z8e;qptBmT~UF8J(sSDC0JGXQyHQC>KZJSpMWgJ1CDU=NE+s#yG`S%MnXa7bjvM;+X
    zO5in0)EQhhVx=xYu$z~mhL7=7aAO+oy?g{l?t{GW$5hDd5RQs_!K3}P>AG^7XVq|{
    z3XKB@LDZet5^yG=cZfw~f;$HYo}VOr5HzSZdZiFT@n3t!Dt;8x8AMqawXQ5eb{0^v
    z8fXS=OWTjZ*iiqMH(VH#RGk4AM1$GKT>BI5ROF0T@WT6e!+m?>W8*d5NYrHl8OV
    z0sl_Om(HBq)G8-d#bEaUHewT~{er10P8wfOCBJI+ZB=JsJak}!#JqzgXVJ6#!no*Q
    z%h1=L;Y;xwT5fz(8_d10_m5xmvUgd($Wcix3RqBHO}HRdJc2>;db%mww2L$@nk&C`&%^C^l
    z^M{J*(X@x_9yo6x{+Bn}tEv0x%`GzLHX!PwAN>&WaYMiswysTf@sbZe@9W;6w+1rB
    zsp#EskNzCRS0~~TB5}BV$g|Tz)v&m-cj9I5eP+s`m1y(izpx97zgpljt$dxXy%#jN
    z;>yzYouZl5#T;_U2zVnjUuB=mV^%R{BQKS%03;{F@Y)~RS&_p6G%Ss+_#bZC_zK-I
    z^*5H^Wqe_PB=)#D?y;k0N5UtY;=^>Unw#+3gVyic>TlR+OR+KS6R
    z;_)a0ECBviKHzh8sjySe4;sI^oW+Za?g?oKRlthbTvzo{Yq+O7oXDgHFb8iK!d%2J`|B9s4?Pix`x$xXh*+ACAx=J>Ev`bFqCLEw1%l3L)>LUf9Dtn`~^uXq*Lo)
    z?zY9(_(SK!ZJ)ff$Iif2{SBi3dJ;iK(!z&fwO>QO#x6$En=Av>UBBy0RNpw5?Xv{G
    z893P$GvL{Xly$@x%!;@hx?y)7U{i>^J9#EHjn@|pp>5+vJ+OudL`#w06JUV^5eXz2
    zVwnMXdK0@;lNza8h|cU>mRHz&`#2J5HmKu5fjQ`6ub2EZP9(6|5I6T_&edS3FOYE6
    zu9K#(ivJg=KB4wg!1y9jDp(k)fJ4?;JSL(aY*1DyWHQeKr8%Ff_LF?JD^bv+ad~7GQ9%!64
    znkf8e**6_?Ph=D`mJ(S(JOWAj(?o`#=noK{FLtZ8zw7Rat4D-kFi*9o6r*{!>K_8k
    z&P>psw4yiNJ|J#7ldvM3WXwCj140cmEsl=C1{u$|;n;0my;EPAbLqwE_3ZiG|94qb
    z9P{J+Cbzy24F5GJBI(Dj&d|^A@$>s|@ildF0Jz|Vw-WCoH`VT8%RyL|Z1;Yn0tYXg
    zy95ibKe7!A*n3vDeB}PVxNrfvtR*49Go4yDVUl|r=p>d*mTRso-9jF5ARYuTxMsL1
    ztgQrvko-irH5G}ZbBuy-1u11K@q>1qDInthCf-ak0ni&W_+ku{8nARuL4BF1UxO;r
    zZ=-)$-X12v33#Q$PW9+nf?pJui=Xc$N
    z5Q7UzkG1U%bEM1Z0Ddd;XOQcxHK=iFA2UEa$DsV^MSVU3Fx)Bd2unA`;Z3j7cLYGz
    zB^<^S_9B>oeedwT#iRMv_5PPN%9v{VdsJKn3HtK
    zpfymS^VEe8BD}wqk*nxnfG}k4fB1FJyZlXt+Ta8qM{Nez&o$_f+xdq!EmNBvxF%<^
    zh@Iw5R0kG)@LjK9@2E(M7G!9FhCblZ+g;u#p$bkq?jlu^<4p=DFOuW;y2UnzQNe|I
    zoI54}77Tvq=pY9Zt9>I~{NjO|7&aDHFqCUgdh}dfnW~o~M+PKY<`8m^E~P%toJfD5
    z18erf)3zPM6{%{n(;@#_Xyy}m5%Sl|B_a{trZEaJ2HSt)-_T~n2R&6hg8_#4p97Ei
    zfOiOr_wUKXx6qD6?m6wtOOY1|6K&{j9@ug_zOu?*%5ajw`a!-KT*oL58i%lunI3AU}1gcUD2dz3L41e!q508BOd={%X-w(iq{)LK@7_QXTMpq+aXl
    zuno5>qJ39LFmX5RyA_ety777+xQYX%cG9ZsAU3QPW6f8U#eX+I&T5
    z`jkhXCd0iNnz>Hd4--CDgi!!Z_wEOyp`fs#&uCG4#3N+x=_|cx9>Wf&6i7sTSDzHx
    z#ao5yG_aI~v7GxynPzBE2)>Agh)w3yY|&WHBWtaN2jR~c$zAjYS{)4avIoHbdV
    zdJ_mNF&ZAAGZLtXM)9};flYxnQIxOdq0^^gyL1>9c8jpjA&IxY$C#P@Z=pm-34lQ6>O6THn@1bS<4Eo2<(xJ65n%F@<
    zZ#O&In}gERNB(~1-3vjKZt-DLsKyHc6?C!tzp!E^Oa%!|hgl_^AT1ZXrP6FD
    zcYG9hVUN}G6{ciGF}IhTrC#}GC;si_=oOyUH*VRL7yLa8X@?U=HD~KjJmFxv8(02*UeMv8yB*%3`G4
    zmQ+}dpAjAvOkKdInZFO{c6sk*aAN5ONanh*bz6mn-~j*0XYuTnh#fT+*&GGZ3jlaz
    z*yb4?CKm~Neor1DqEz}m`3&Y&PiK-P@N|O)_{vvG%xkhFPmB{U02r3&6)RvB3sVDSN|hHzh;
    zz`TjW4vBbp3Lt{Yvr!{^KGFu=LGADOFHWkKpOgTf#Bq!6(!QgxlT=6s$y-vNeA#cB82j_%G?R$u~b>q}8z)Rj0N8GzTA@YX6
    zG)*N
    z?=@EzzOoYK%!F-G@J@scdCT&{9l!AbIE6J;10zeW9MDIR(M81C2vrZW_
    zfJ{tz*hN#U)IERKO|)76_Tz=bz04kZpsam_HmwM24#DJIz`KY>7o6G?3|PPdPc&BP
    zL7_paK4#LpdNQx$a%gk!97Rm)^p6ki1FH}~MAHZLO|d-CR{Mmw6r|dh=OZxOm1xVq
    zzLxuiSv9iGC)|5{zv3z4ub|XC+i=8l^JvWec}oKEKSb*jnD%X+yAj)feM;|;2_ncy
    zT=h{*as|XA(Pf5s8s^1DNE5%~?DX52iZsdAtF+j_+x1ZY5bdH$e@zFW02gg7ET<7A
    zoK}LR%MtX1IsLFuXCHxu1N5W#MF+YN53aDl9wmxC4fi^yi?GLn=
    zl{#{&>@)wR=js44eRhzb?sNrPmkI;Wme1E}fId*`Q6cqDcLk-tGKQ@U`zjM1f+O~c
    z?HYlfaN+sTt-sK%$I$X~Umu4A*87km(}qJzd7~clD(cq`TWQpBG%facrUQYGJNU+A
    zp;H15+bA{r1Rd*SUbv7j-zw#^o&jaVKR>CX7h1=X^E5GeDLs@<-#s;>uhmjMytDks
    zRnSG|G@`8))a5Z2a*VlGewrP;ZsKz@SsnLHV(v-L=~_)_*2jQT)&eYEy;0jZsKe``
    z7etSk$r9xU3$Y1J2gLN*b@^6VKsx>RNM69MP}syr-3^rY*8OkRo#)-BI`&syM;WFG
    zns_XQ>>Z@E(^NFAQJsn0t}WJiref1VjSY38L_{Qdh439agpwVdeXMF5HU^cyGdS9(GeO@*4_hq{Rnw%L|1mul|{{CLRwMuqnZaZizKPQK2UyNif*W+b80
    zHKhRD@oof(6(b9L!DKQ)KticOIg^1l$LSHfmJqdMi+8(;`(ff{`_Hh?11_~wv)!v)
    zJs-)o8na)JllbRd{3)uN@9DbETJ%nLY1HS*i;8x9(|2?lCT=gx`f=Q$xTe)(B=rbP^b2t?ehz;t__^rol
    z(TflD!nrsU@P8uLD3B-awQP)0r8Sax$k*RPU<*^pLf^3|N~zz+d}Eb>s|P(`{ThnK
    zETn0ISv8n#BI`BmQi54{)8`4XkWK04Tb2nBOq%0*Wd~Vw>7o5^5$)Zaw$p=l<}MWZ
    zd`ukRj&7HvuCNem>Gmi^zg5Ln3VduEI%m_*I6K8Q)~NKr{SvhKKcBI5XZ}X4Ucg1vGLx1R@}DiSHakg_-F|B_UVH
    zTJ8QTkhlB>j{|h6VL<(0>epWsAeVGl71W*clmxdhWq3
    zCZ=;g0790b*NJ0Bfy-N6hzCGPcpN~K^f<+q3Y|q@DSF+`g(m=m50TRKPAT{3<<-9J
    zz<>NV*YCfhJadMfxU{7=JQUrUxP&G5JN}m0w93AD>U3Yy8G{96lgcFNC^rTE!}E*k
    zp=KmSvLHv7gIDqhdqsKd*_Th2XK1y2Ec;I@2>mzP5@+jkQi+=n=`Mf&zR!vGZ_3#)
    z3VxG#Q4MuMWyD@*b<8o@I%)**TXV2$hXo$dUD!vF|V`CcjSsD
    zm-4jjogWCn9aB|lz7ETx;hBb#vVJ5pZmvy72$C+m$)QpGMJf7L~S#8~TOPl6MGVO+JZ
    z;k2|pb?T+9HO(#$)F>aT%FMAxNJl~Zh0G^%O889aJ>2Uk`)_mW>L1_Rnb*xqV>;hl
    zZmhxdD_zNXh09lH9B0RssaKn@s-(~;u<;R6t2Gl#?i%s8V&t+#;zG)(DH~
    z?a6vhhesw1KG+*QsYN`C&3^r*;86RIiu8BaCR7*XJ*V13k~Tx^4f7`w&vU2ss0L&e
    zc$W`DsmG@j+f06+<(ab;929WyM4dL=M-$yq=2UPb9E$EL{N3W?MI#XCvYG@n(6+^>zF0rs5DuzAY5-le-kE
    zqov4=Cx7kIlzFJsn8=xMJU*EFftQT;k^a2DVVx_U=_5pmxri=#uH(Q#3_&|xNiNAS
    z7P&o%0hHp$v&O_12_wl#Mz;xF8>sXd;fGoq#3u^4<%sH^WU`vDi)PM)Y@ZnzZ_9^t
    zOE1zrvnw;GhqaG9`)H9|T8$w6Ep-m
    zCa&hUgE?{K&vBsrrRSFMIA2{M^pf5y^5$an>eYF23jN}R2>**=F){`gY6jX=bT=77
    z)7G_&sSlSz3M7_R%;CJ&>_)KPArY?&AzR4zDXbwZsO+s9z?+6{#UlJ;M)IPcrE^so
    zj0$2_sWjJzqq|Y!yY;@{7{d#SIOqnuuK`vi#~915$POlob#Mb5m8*-{YT)U+zDSVxmJ*
    zxE{E7XeALKxI=pXDhJ>bS(Vus!o(iF9ZY{NlEb5%xtkcY^PbpiXvydZ+2@$hX#xaV
    zABQwt0Fs0S7hR>c#bb6wqJ|HihUNST$UcSxJ|crp_1q2hEY*Lxrth2=Cqc^?c#02l
    z6w;-538K8m-e{^q&rEJN{ne!|Yo%200>w7Z&cey$IQn{n6rF5$9tPwekiCa;LUz-c
    z^VQR09iV5GbWao&*Nfg&DhS7L>eOUe$^CK+i3%SMmDh*UVDKayFT9Ue19oGfr>`wP
    z@xpZ_8Op*TQ8c?Wt1Aj>M5bo5jM=-V2ii*>--!$;^U~0hYKC;j7YqVRiae@2a
    zlm#6{qY1p08rC4?l&*3Akq1RzXGrR(3m~O*+zc(C!tP8;I>KPRFiN;bj9?BjOmk(e
    zvdsMJ8d77DKRjR6!*Flj7pQ5ax~}bmu_e{1xL*i~WSG8Ilnvs1T*!sV$pkG|O>c%|
    zl(OO&k_;K+0JH(fpGtu{fykjqs5oOBT$tr!)W;8r!B06MoCc{JrF6z{FqPnINbYvu
    zxNX_gDk$yutHg2Evbz$QHD%EV=?C?Sd!8(|#&$US{K12zKHT
    zc|WcE0W=Svq#w_!mU6H0Oquf6J83RbVjaBx06-J&B{RB5qetqXOV;VLZphlZCE#1$
    z{C1RZTYqtU8%zHr`;u~MpmqoYIt!w+nZQ&kdw{_BWx+EtMH;)&@NV{@&upUuMq!z>@m`gBskzG+F!<
    zceDN`eRCl^Q*zuQ)@=*5RGQ5(uB~}yfMHuk*aq}Y9EA`Le=9d0u{6D!dL
    zFK!9a>3u1?g?)A>V7`BMbKF>o8GsG|YV@U&YqnGrgg73;xz%Pzofe48bN=ndZc{c0
    zi5|GK&(Sb2@&yO}BzP?Ng3)Y{87h%KBvCSHWkRZQ1f_chq^YPQLlOa4Z4V1s49_nyGMxP!VDQv99q5h
    zj~#j|>5sL3FlK>9xxvcvT&J6tT4#Q8i;^!ogVoocI(v9+#I>#_JLk0Lnb@Zfl*#c(
    zW-a9ty$$Ff3L|;BO|4x+N)ytk3|F5fO5N_JNGE-wBGzyC!#-LX2@t7$8Ts_`=ksA^N%}{5lG3&z
    z8FSS*JND|X#MwoUKmNcZHp&}(!(zQp{wG=HkFs=1AH=$*$LaxG9V9#`3RXa@Ube{q(HtDdzIPhSyv35sM;6Xn4t5;cC|lgl|?usnu!I$*qN3syfJ@Q{IK8y?IDn`9Hf0@WPPOdE%o*`#F2-NwCoRym9_
    z=kd`9*)5|g*T^}XfY@MvFA^2So_d7%HVgJF3y7Sq~$-JPe<6!@Cf
    zDPuau!o|a$zb`d1#&5pOD%&~~%H6Gl+Z=y){7diT84Y>2_kHT2Y2+8HEA&>}MmWfX
    z&MzuXRrh$=Df8j6V+vhiIZ;G4Sq}
    zzIpH%rf;38uFNFp{QT9c%|AnVv&!-c-=HO?5b^_vwfge_V37wFfQU(c^V%Bc?ggLm
    z#f~T3xG}t({tqrIzSJAdh(2b(-t*P{0@>=DmcSw&nWaPeMVpiO-44?){Pw>ZEFgb(
    zVkL~qe(>!X*1dA)M1|3BB*TYieDH_|lF0Pd%*Q9%#lwFVO!9@G{f)`&(D~Zkn8;3M
    zXu0-8oPhvfC&g}O4Y5BoG?>ZV&3aY1we;+PdZ$9J_GluU9q-koex{EfYtbQ3^XMzn-$mq5(67-MWg
    zl-i^rtgw1DDSUCX+Wo6MCoy;fRWUfI-k3PzO0e!rcO!oIHh-CTME7oiJbg@xhk~?W
    z2Ek)U8~J;ZmEai(_^F7UN6fh|gQTTM`)r+emmS-UnCd4~>D^v@mT3&WekxkTb>N3w
    zTD5VXD!501W}pp`tm%dfu~B9tjxw8>FZx3@QXbDbXhiSHhNFQXco+gd(ZZ~z|S?={BO7tRmLF+UDkr&*TT
    zJzFmqLW-+MgT)98V=W#KAaR#fT}?|!XxnfjrlRkwbmXL#NIp8iNP#B{xI}sU-2cJg
    z8~9zZN#>#{!A4EQI;uDOnY{d0&Cmqd9gmL#2!1OjUF4x0OX*Am(Pnd&0wov?qt|1%
    zLJU8mA9dNX=_B5AKpG!?ygg5$Evl4s3r(K#J{r#UglD^${EW-U)VTyJ@jW|FWR3`guSZab|sG3ntPfRob75!K7Tu+
    z4Su!@skOuJdFw|^o9UYukvY0#Xo#=3Jn@o*VdsRT9ZD+8BZH$A2iIlB<{oS3naPo5
    zA0w-7Up_Rpgy=^aKt9@%7KX5qYU^`^n8IGK(q7r;AgU4)-RGnPvNsKVcSe1n6vH3h
    z%3#|j&3u)Gz51j>(9t#OF4xQlC#8KAit#%NI@jn8AsmuwK~x@JH04J0A!6JN_89P+
    zS#H`(4hQ()Xg`Nbh)45qkx``?i|3=RZ?HGy9zlJLA483Vy*ttEPEioMbbj#cK(8~K
    zJ5wS16|zn{Z^aD}fkWrUDW}Dl9a3qHE)#XHUH?
    zw4!shOTdO^CRRbp4zDJl)Q)mp8l2EHHZ9B?PCYY;>@14UpN_X>XG`P~TGjQqc`oyO
    zkN)_d7934rHTt{ACuL+4AlX46A}G)D8cBvAX`JT5Y|
    zOn^@o6DU(`Mh7oD6J~!9G=bb4Cfd0A6Mm~Wq+yAoe~#i1
    zFWJAPh4{jWyuWN~DY2SX=|rH{$;y&E)zZxVxsC`z^>@9xs~9j0nTBpyv{dcBt{Phz
    z=}a5M`Nx)Q-X}Ny%Zj(#k_~m(gAj};i@?XwMC;Ix(x1b;4k^WR!5F2-jT>h{!Zszm
    zf1#+_DeQ+L;zJ$yI6)+e^76w^o1dN@>eK)H&^M&6X9AfVWLMB!3tb3hR-Sf^!WiP9
    z>=PT%2m{0yP6&A%+_>>=>)Dkdn2==J&`zbin4iD)`?+Q_9-8e*N{TSva>d^ymAqBh
    zA+ZTK6Mz?BIm^p?OsBq52=K<6_0iHD^MPEoO}m9`<{x{IDoK+!vFh@{tMdk8z;Q_k
    zUZiNXBu4{9gHQTg|KTtwoF=ehN(uiWnMq46S^NWL_NT`bv&ny<(d(=EtZY`k#S;^<
    z9bL#|_@HvyirLN&;oAbNDsCQ|D0fe}NBr=pbt(6e^Zd|IOOpq%f`=F4l@(7?`kxgg
    z$<}HSZPu;J`kl$V2*>+Yp-N4n6IXMEr9|w6s~IY5HM*Xylw*+`XAY(?c>r2Lv0heP
    zmBarooWto+OEIt*qN!h(W9LgB18lA{hs)
    z0rg&t3vqx==1%?UA({b0z<7w$ip-M!PrCICG6nKa($CPOG(L4Lakah!)G@ldllv0}
    zsuK@&^?~Yvi5sYgp+3s6bW->mOZDmso&LFk1b^0dxYHEg-}W(3SFG5T9dq)>`D|Y-
    zWr&iR@=_o{?pybrVEvY-mVSfcjdg3QGEsdrB;zCjv=iV-9#|-0QAeg0_o@3tF(l$e
    zj1315FUf1A&p0c%DL){YjZt%KDS@YuW^_wSkT4yL#d<7DKI>djO)6iilKz&~gaNt@
    z`{cmHI%(ALGF*WI`3s;Rm*zsCa}A;kSYjtHM)wz*EJgA=lgxLy;<<%iJ~{I`$uO3_zzPW0}5OP+lVvTDw#~aM6@9&>t=o2
    z*UsCNUWZwr{CS=rKbrZeea%C|@Wf#FX$@GfeMJXv-serZJE)DssFGHB*jqqI_N~+T
    zEpt8NC$H0EV!rNH7Y4jysV$TJ3gcZ%S9`)F>E8=1AZ-7%NX*_OX#y$P^mXEyV(FFQ{xpZQkO|BFoKV)00KLr}k&ZBr%I#VlK<*w30rltgNgX2mL%nBCSjKNA;DyVuTk`BetJP
    z$$_sZCX-1Mc^DFT{R)$JjYpNAZ1?5K?~KS1JxV9NxWt@WD{of26qyb#H$FovM~3^#cvi=wX+2lZp!k(%kxy1T~~QIlnKw&a5f*^
    ztv+1;X}+#$R$@Aa4ykzPBkNSLTL+J$Ih7so|J-pS_Szw;$2+>3NbSW11ovvq=PAZy
    zYO>1-`Wibz1d=~zcQrrAfsb7Vm99=@MDwVU?gt<*8R;=!N6w$cfzKE5$8ZwilWxzc
    zBmZX37y+ypNVnvHNO`)je4t-1de)dAg7x9gpIjyRd7%z7_5=Zc09mP0g9qt>gfy(SaC0JdFO%
    z68^WxZ>wixr)BrcojWbI*YK6WirS0hRk$wA#y^PfE?#<5b#?aGUJg@^HmK(b&}%9A+9u!Af}TUn>4
    zZhb5FH#42^bpi7xQaYo3d_5mr!MqwlTAPR3V!+)FeDfHnxER-a895dBtc0cuG`qc!
    zNu}*~{q#WCEfQYDzSy6_4mC=~;{9=DRc-}n0pnh8>~9_=XecLi*^*E+dk71U_4zg$
    zn}9m)iRzfCE+Ai&2hpvBaq<8L6MvL+urg7t4xy<+Gz%k7wU1vVkJE#2Jd_uokEUtY
    ziy+mS5;j5I``n`&jWSHT1!y)9jhd+q%iO4+QY8Bdh-eJ5JoA6cOv?|UgMmw`0V&F4
    zGd*Y?dvpzPZzA^x6lIp6WxwOnnQg)+9M~3zI9qoS&>1ndmNM@^3s(RZzWh;)IePZ{Xo$iXiA}l&KKRUU
    zkZtdvB46#<5`I({7Hx6_Sx7q@36qouei3paqWO3IkB^XOgd@e;MH{Ln7sTJrx_Q1B
    zS1L9x`aSgp9_mXr=tbd--O=!8u8c;ON)@4H}q=v)uQX;8{!z>D-a3goJJgS+;X@=iD1Z|WTvf1R72kXc__O;OK
    zIB(C4K%+-{O)Z%elZvo(rF!8ED@1iXGIZVco~-IO3Mi+)y3wRUJnWN8TT7MIj!%|*?Ou$2o
    z)~{Q*ntYQngMbHM(mMT=Bp;F~b;<#9No;BVvOY_*W=GT=u>c|Z{ZGu+T^Oe2r-D2u
    zO3mtyv`M8nKZpqBuU23BfY3%V+@Ge90e?R07dvm#4
    z1TwuFc3cpzKfxP<
    z288Wys^dCknUmHV2|bKB4_{$cfjntrgdiz{VsqA9`n<+=wNd&PHJBFzc~rIXeQa9|
    zOy~4>WO*Nt|NP!qhUpc$JB)QayESxel|3Fpw~Pl%#V`+x0vVlI<4YHPb>BmM3_RU1
    zKW5dWr>PBMt--3Al-t9xc!@?2@i0*QkxLwuw$wUh&y{OsUh2ubMEtBR&FznR7Z`6c
    z-eBkV9;h^JMPF1WevTOuFM4Dd`Lx7CA-H=_A%I9OO7M?@p;3YsOLxDR8xRIw8;IEb
    zyjIl3GX-yA%?Ut+A4!q`+nKIHUk?QvAEMX45zu>TAqK%MkA6f
    z1g$?*#E08mYl%JUb_HanOO)X^uF@)ZU?K%7jx^9`WuIIo~uL`pG_`Us3HiULTztJ_T+M;K5jS?tnwqZAL_2#zgUq?U
    zvIi5xS@Ekm9G2V&Q8a^^|Do(HqoRD{ci$O?kP-oD1nHEPZjmkpq+#gp?g6ArN=i~d
    zN>Y?&kOpZfX=y>abKm~{`^8!3ti9IR?|doqJkNc9N23l$ege~2!GOY?lQY&W>-?pp#$ryPgul!4vVObGjiP99mMG8q2a$Ol6O?KFPUA*74uL4Lq7i82m$AqOb
    zkaM(RoKrQt!^>0TsoS&D;#sP-NdeQ3Dmu?CmH2+)PjRllKzaPA%S%MTWLA^|x}YI1
    zxrvR$2!`Htg%Q2PFbGfp_$%~*p<#Ok$-xb-{ssryoSGonLEEY@=&nvfT1eeNxyqlF
    z9!KcRXRX|Xhclw!^Ip0W#6VvLwMln6I{OD_QP!8
    zdw*A#K_0hxygw+LK!p^+^I8fJz{mf$=XCGH6<{`Y``v|^nLcChd;P;oTx08a@$Boj
    zvF;d>ZNq(E60!5fk!S7|Uq7<5-2Po{e4f&9ks5*c{Av`8KBm)mU>s8VqQ$vqaK%|f
    z6rBBol5X0|Th%!fV6aAPs-5EO6sjf`f5EjpvWkyA(pXz_UACYt&A%KlvHMM=nQIja$qLV-DSKZDAu
    z-3^U#ckm3dcd9W^DFN$=uJUpqHW>Ch*2^qpHpl+bq(f%`aSLydmVL1X{Sbr)jIF+d
    zj(Ok0jZpNKhDcur5|i9M!o*6+(EkFQ`1ntLrCLEGTuff>3K^kk9{py7f7^(aMbSII
    zDZOc)LnJyzm&|IBY;R61%XNF?z%V@?k
    zgQhS;1m>CgqADOT@+?Nrcs*4Q?)t*9>=xw``M53cXOj*CDEdg2z-b;Cxz$7A^s~*<
    z-~U5p>zN)c?bX$E``)eYKL)|}Kf=h?OpV#e;(6QjwO}X8`!(Daj=I(mhMeVN^Tv_Z
    zRH*14fP$B{6rt1-y|@^rA>@;Rk|uv7Rg%y6utU*36Ymhr;X_#*|KKPJ)kp9!4}pT^T(aeu8wZA
    z(R@1qB-G9Gz=7A;e&Fo~u_C~vS-0YL#vSXATvQ(15xeT;
    zjcHtZvH-(b7Se(?2=x%H&50Ysuw9T+;GdWp1L!SIC5*Js))dZi
    zJ0S}JIIrkm$8Gm>p9;Pg&EeNgXA6ZSXOym@m%V+EuO;PS>iIu)
    z4N~z(u5~D?PqhT^Yh_n>==0FTTlu$K>yvsUqR1)B=u5Q1FnKOq=dD
    zQteR1(uX$u3HkEyB_Lc{cwtB~pcv!9wFoA(uzrTuW*TY^3DV>pCG*m}J?ewRk~!+s
    zH6(>E9q-S%6OkcBus^BgE?%8?gkr>1i2-W?KK|#*%JJ{tLyoN$Jm(xvE>;sJc8t!2
    zo7Ug!e^_$*K3iv1f1fSl#j#}-hI*WV{lVU=dL3K9`a2+&ovUNd-
    zOY9m#NDweC#2CyQ1sf@i?yU6Lz#zXlfrcDk%Zz;g@@#Wf0fk@fP3(CE!RVt2OEV+U
    zH)LRLVTn5Z`;nDKM87tXL(X$wh{#@6Hx4`o1t)%Nw*gx!kUO6h%ta=1X-5+Sl%oT`
    zGHOKOX5)gUk<4{w#{CPk&NOL|typp}|&jiG-hK);NHwT?3fTZyuaQ{3oA@Ec?
    zDAlp~?^k&4`9E5<`)0Nmy2xU62SfGmJDLFg%PVi*41a!Qbr3w5XAZ$&h7dY1-qu1l
    zj`R|q9pxhlXkfWU?Bk9NOfnwg`htwLP&(otnn!FG%^kv17@lwAp%qCNIyjGCYXZy(}2Ae-!5Qy2;!4zF8GQMXr*m+bi@@1|hs
    z*ptieD}+8jyWxKe4h_d-O$=B%#%*vCX}pm}z6smYOZ&d`k#G;vdqAv>eGyB&&>
    zgL|tUCMVBbzY7aP%goLu3K|BZ0fwtBISEC8COFw?zuVbkvhBzb^9!M_!#XI`pJ}i`
    zstydF@7#*Vd0qJaa7ZBdyU6g^p1YO=M!L~NOY@i*|2r3=b5L84Aup_Aka2b<1#|Ms
    z$@E#Hq}J0EVuaQZ)*H;NR{>Nn!rTUZ(W}cpe8DXm+Zgc^*(X80CKVWgGqEcX%9GOjsEjOUTA
    z_`DEu|E(9)zgX`U|F4?oOs%hWpT56L(j*cLe<#K5F`IEP)0jFx65+K&$cwSbxP`QD
    zBdrhm+VHP^K}E&1aM#5}hN^lbPBH2HMEm0SUmDBo%qz4Nf^FHG>>)sDnd4UE%X*rI
    zc!MDGGs|2jm69+OvjbWB>IX3PrNLD6};cj*9&
    z|C?yKihW|r;hFaT$0f^^~Ds1)?S_ldxH~LxSa0>`Y#zfun
    zxKZ~ychvkX^1zqKgvHs(`WCR{Sp~M1n>YTne4vAQ3nr$zatvzNSy+Sxu8;%gOG>%kT?BeX-v=|qDkpI>tlAi
    zKZE?Mf_quI$B{i&M4PUZR*bHSvvvf7eyMBFST~~#+)LpQw5w{%T+YBJ_BinKV+lcl
    z${!eMuO!W=kRcwp*bz-Pv?#{fAZ!pa$!EA)pU|C`3AI0h0F3=0*`}ap+EfjM-3p>K
    z%*%t=k!$&l^?%<}W4?SMezSr%aeKa`Ca~yD*9u&G?EWn#r~~Z{3X0{AE1-SL}q~}pT5W2g@RhG=<
    zlD9VFThO-!ApVd!qAiXgKkex&c3xT+|r{7
    zx^2XswMS6-mZXf}1lQgYN3H!3Og6C4tu8gcD5-v{`RdSuIpdKYik<~GVw2@r)8~A&
    zS1*3oj;b&8s@r{tLH$zrYP#5R6X4>2>Ui!?dbC5
    zOJDeA>_|<&U(@dMQedDQhHz9xy*j6y{tNv;k13{>@BBtqR6>lyPPGpu*JWGmd*jo$
    z(da0fc)65e@kshtFV|Gav4n1U#1L^faC7+PCe6p+Wf-N{t7(0ZN~%sWOyVvec;;^M
    zJ_N_~Xv-X0}L#E2hab3Wu}9Cdry6meAhx(U7cGsXkK1moB0P&QE9St@0e5g|D(+;?JbJQq9lP-;#9>gfQpDQE8
    zcaKE_0X2A!dXnA-@i#wze7!K=C10tNpLAK_=IE&s!i~2K1tT57vSy^tW9HN~Ix$j|
    zBCs{)<0FEWDLl;_=X$3Sj*p*hs7FF_7wwC_lHU9@1p(F+sVyeUm44d$JS;#)nm6pI
    zE!5!1M1ymw`DBIwK9Si-_Nx7>NI>zXe?_!gZ&UVo5uEN%M_a)4*Sp&;LN!w3>$PMx
    z0{pRyi-rcs4H!`*h#v&wsehX*`ZtlV52D06K0f|+@43HdIA9en`O4=NX5nJGGGlu*
    zO0|V(dWw(x=<|&$$n+E=C_-B}d|q+!AnMc^-n}4d*1FbjMg1biQ3vD+wVor=Q13-`
    z2nPrBGaM2lhFF+6Zy94yZY$ebd{3xcOEBtqEYZ{3)Y$1c%s%|1rERy>)Q}2k!C#m6
    z`WImOiA4Op?#*<4MuluUUune3Lh@ubJI5#UaVb69xUgEE9(@fw8L`yaVVQ3io&lrq
    z#^$NfADO4jK<*jP#NxT8B1?d|1zs77!)P|=^93(L{IOJRBL#tZ=g_0)OsXGdjPNXO0c}b2CaX-fK@lW?UORhqRL^Yz?v{P2?Y|=JmeBO~ifm=@na5OHvYj{`X1EuMj8%q97xsY1SKm8Ib>8
    z>NlG9<8h%PD)j0=N|iW|R`ROG+SgEYXUvx_JrH8nfUUeHr?%H^4sVWJg8ztzlkd;i
    z%gOMS+Hk#DhdeTn!k)4MGv!2t(Ug*8IeB+Lsv7@D5g6vGu03FY8nRDyUowkqx-EGe
    zd&%Izv;A8>*F>(r!}^fS)(~PSL@+vkR@8-3T|8=~r_uNe4{W2kM=-o|3LplUU=a-c
    z%l3Cr6fp=S+RnEgLYA0=M$+ij>n?^5G&ojqS9$DZTmXlBx7={@DeZSUOXy?J(zi*;
    z$bgIGarDlY~OhiCmBj@R5E5XHV;5?<^+Lk(F@gotHct
    zDUynUM!`A3`?s`_qUX{NZ-?Y3fsTz-NZajFN2dN~{5F4#9*;4+Xmsj5b762LZ9-Bt
    zSGvEDTkrgyUKouURqFhi4Lu#}3nLdE91qjFXOE*gzt22fY
    zMAtmDyTsFxRKuVsnI%jt*Q~(7Y`AlpO>DMcs-FUl{lq0G_tgX#wuvZUR1I@QWwpW4
    zm8;cr3d{=p0`4+0w@P8|`rj4E^T?`-kZQWoFI9f|$F$x}BMY-b^
    zYtZS2>uV75jk~pD2-OI)QdtUGj4Anspk}HD)NNoKoz35n1qc2<=F|iNn`>nn!B+y~mqw$q
    z6Zy3bpsI;cv6n^XqBEM^e^S>NsnVZHM~r~}zsCsVzKyLWNT^o%
    zE$nOo5Q&R~E^IhQto(Ma(QPv%@Qv0x;+4ps-P|}`0U=MRr;5t&oia+2p&tw-Wt+1N
    z@~Ybs353=n+i
    zU;X@>PVzbjGG=d%J6*el&KhHOebha}cJ>RKmmr-!FTol>=55ZJu=83vwcE*5CsK$d
    zE3F>=*b!BJM6>4lrza+Uzlh|?lKzg+0g5^hyD#84=;N0<_MfnCd;;M!^q@&3sNq1O
    za9z3DNsz>3Ma$?~2k~+2P)9;z2}bk5QR?a}`f5E*-N54@{9zFUDVDi@%-zO+^$b^F
    z@uXN_3Ij~_UOz@XhTO)2bd
    zy|XmpJ5}cyd&7$#q#B?r&UhfJ0q{f5`!$geey*36INXhxf|IA%@-skYk0=o2K`=zQ
    z61w}zvafx%Ke6d~199}B`}DYslm(ChB;tX6#*aTE$&krp4c`dzRY4MHu*4UibfO%{
    z=F0DTn!o*HFy$NY>R4z{Ob4LOn7=-?@7APHF5@s*1PL)JaS4*phF&Zrzh#Zhd}^z9
    z*GQ!@N>f)sq9%&4$iA~>LmYHBSHov<%<$x5$$S&z$l-Ch(zDv095tkLR%2*0^Zo9Wg-1wM^
    zu%j6&9QY|p_Wf)sv;vE~5jD*a#aB!lGUat>Hk~U&tjr2Z8<$x()~4Wr-((n;+nYwd
    zv5}FW2P^*8a-+C!X2JdO&^j}lbnMEOI^CfXqgtuM%I34&EamK)wbu^S%EjE)r;3hX
    zX3kty^IRvopUBbzQ2I=w`IZAYUCYKsFusfDE5`2K-=}5Y+2ticrTJ=62SjHmP-#2s
    z-n)__^EKt(m^>j=kn}pS%fxOVkIRSb+DszF>mU?Qj42e!c?&XTzBy4Z0Dmo5*8$+u
    z_`jhh;Qka&d{UbTIA4(aN0HkG6ewbR{D6Z{YV+NBpx|-5yllH1a;m)8ox{y)v3pQb
    zVX1kL*w2ZjVfM8h)m@M=kMl+UTb2_=b^tWgH9jzK!PU%{Uscb_)0>cfE?Safl8wq2
    zGYGFZYUP#ScltTqiv#qanAY=4O~ffGquDWQ>6IfnLmHDLv69QAJDH-009QJKVurZ{
    zUQk`BJXjYi4~=4fM3DNQ7^>KH{SIz1$oW!{1biZ}N>?Fx*_gMV>!Ph~`*#3&*V-*P~l_>#ib`ZLZe>L{f5=Km9WCRAs7()|@aL&;Dg0BLjINK8*#WF5X0re;P+ovdTfdJy*{R
    zZy2&M{cFl$FS@pe&A;vK70ooHZmkT
    zM<~6-h}of8^J9CGY2QY%?k?+ybsv9Ga*xsM!2#dgVs=)?%A=ipEyu6%otd=%>1i9(
    z5~H}E5k)C|rS3)HvU5~Uy=TO#Ql!x$V#6Z5=I3(DaGW*i?v>3G5uWDvVc{`X5XB)=rcGwPUeK?x+aN>MX;dgvaVe)?u3^_bVCme0
    z=^qs}@8=*j@4{(jGM~^hLU!Xfb1+$0B
    zMtcE6x7j`EUM>?~9Kd|Fn*ZEyhETYv2vN~wecPvh^kJunvIg;QYU3fIuvgz2nCbUf
    zMWF*^@HokS8Nw`Xcf*5U3th!0M>(#_B>GAR75)xvtksw6j
    zd<#Z>0r6-yyy}w^Pdc{1in_eKd>yaPWRP4_+`pjh3u_IVNU2z4O32plzct{5HQWHd
    zZ^QZM+4J#!>&S%9nBm+{e{j4{LeopLe&_i*KjFKqgciNyWdk2tyexY4$8+iS+#}RT
    z%a!Lu+vV>X*{}?2wdld2_?tm~6oY_<@TY9v_x$-tB(Bgoa#*Yu6{YbvdXu5MET
    z&+h#ZcZa16W1B4p17cIt>6*_Bkfs!LFw*^GW7mql<9L@>^Y;6c^Zv&}?#Dv{BCVTX
    z^DFe6U3(r}GWrBRMR4;cT{Pb*eth5IS)Wgdxy8kbC9`fo$e2o&4i#mu@Vq
    z^5=cDCQs^vWW6gcHGNny1DolAIF5d(25Xy`KoD0Of1GiM?@Rk}f^&8zbK?f8{N!Hz
    zL5=eAv;$J|xe`wb$4hh~Wn|UJxQ4>&+6YFS`%KZ`0>lNV4b#j9kf~}<
    z?hPIybd2S=Cq5u0eKr=cBAsPM0)4Tek-Ch1lFg2qRC4HeCH
    z_sk(4Vc_p3Rz)=yUXoQd~fn3%WN$%*R!96sR-97tXhn?KLy?G
    z{#%9pFfEa5o!<-AW|<8k*9Bn|YRMlL<|*r17VE%dLHpQF3wX)!s56T;DHmUk2sOpz
    zxOvA1UW|S?71XW%9>qkF=9B1gy1DF|tD0j}*k8kO{Bcu3GzfF6VKzRJ8H2`u!ycRi
    z1?C?sS$OEdR4q}pV4h^hUf2FXKV&1S7lH=ID^(^08-XPBy=*}AW`ubw*G#F}{C=n(
    zOT&WFD9>-k-t3pOndV;coZ6ZU8B)N*&pt!?VV=BPuWp%5pEC
    zR;yULcTjdiW8lED1h2y7QP?EzREr@hjjJ9&J+AQftgRY?2@e(W$Msz9W2=FnEAJm^
    zW052);X05v%TxMJMgQ#lC!;U$#zOJL<)@8<#QcU9yp~kh*s{jkz%cd^^);9hd~ymr
    z&gz-InR2=%4gkR%G_(aHfoFAK-(^BhQC?*|^Y|d7pxrVcG0e-YW65YudBJi1-mCUV
    zCV-gKSt-QRIDL(mo7oe0*8^L~Q=p2>JVwh*a(Z^Q_uO`m=KOlio?}uvdNZ31Qdimg
    z*Kdp5+svv|A*UNm^mQKM;+R|YOxnW;Vd;@uqw{ZTAqE1lsSVC#G@eK5Mi|KqWqCu7Q8M&AF;Y5#bCved3-w(#(FygV>{@n)Ru+ua*;
    zc2j&-khw@Z5d4#)_yITc$ROZ{HiM$vR7|8bg5
    zPXsa@)VXb7ns$23;dx~py4sM9p?PMg!;*+wPvxC>E(~+)(k2Y`ON7^#hr(yoKVYB~
    zqbxSz#NOzb_i)Mq!jgdB)=@t>zG)6plMtoVu0Cvc+iJ~-yClaFC3&-G5Pt3uK&QpG
    zv9X`!pf(lWLT2gv<$k?)ro0rcj+=f!1vm_ueO7qyLm}Aa(~YU49U`@Mt_h^$u`0ED7mOaCjUFZW5g$v$DBHZahuQ0%BaHaEEI_?
    zT>@>g1JBu*qO6bamJa2D
    zwi#uj89?YeB)0#;2pVKNuNDT+c^c}Z@HEvXBn8S2s%Q4da40Az4WK&fU_lS*YipV1cD3zC@sAENH&
    zs3ixCK?=W$kuuAOKnbF{=EIy-D$Q=!5pgw+k@uR7uRV+&Xv`xYFod|V(>;uFesOG-
    zOJ41?B;K?dF9Otbi+=k>AP;-=-XMTF5Z6G$bDKQXAf&tqB{JXjZmE^
    z6=4||mbO>V^61nQ@(I8qZOjs&5=LeWu$gzGCWZ+aah2sK;Y!XuG22(naQ}aj^F4ijrl%Iqp71HVKE6Cbb2Au}h$1LDA)ZQC
    zZj2_$L&EpRV1qOhb}oyCgG)3HO8&IULhm>8zfV`imQ*L0__Rr4Z_r}45j)Sq;rI{I1n>Ff6tD69``
    zUUL5Tlv{q2x9k}17kIU-{u{WAh@Fq(3d~D_Xsu7$Ex(D}?{CjFK27BR3qC&iHQn}g
    zgLgs5d7+P#0UOJHz$fC9IP*I2mtZ5^mgT%nx`*E$q49i`!vU^kT+HNp0(W`P#bN9x
    z@*tuvX1=2ZXo8$7ULTiEAPb!<@~;CgDDb%KB3kIvY)m}`Fj>ELj`=jBiuiIqR2P7~
    zC*eFf|1}Kx%G*0@StOOz<^Xf#izY$ra1LiRRb@>*=HdP|G~_{Xn<7RRsUD?a-*y1X
    zsdCvqh82cog#u@@flr4x76g@IGDg
    z8Y5wX0}d}`QbJjOGuqH$UzN~JRIqPNDH>y1>yQ6Ldr9z#j+&e)^xxo
    zT*fgg-xLz@tFAP2#eF2{8OJJ5HJAbBk0#jiHxuCwi{m_d0u-#xjJM^9h%*z&4mOqe
    z%QhB%ueGY=Xp_j^?%ZiNkO5Z|p%T0Gu4Dq{{W15bhw{x^0fF_ez?2awz%(;e51d>-
    z)QoXHH2{q52G7kvEuvLI5a2<*yEPg1M=!~QA0n$%q6f}JWBLVDSw*;`sVH8`LF
    zg&QbbwbUb=Y1$Be?ynxZuMcaUcT_8U)gkcT#zM|0SQson!!8BaU)oI{dMjhU9TArx
    zugCJA;Zm%q=0~J|J!jB681Iz}IsdIcMo{DkVcy*a5I`q8y!_k@S+sm>34XA?$H6Rc
    z>c8nRrXGlXNzTk^@%i*iBeUts4L+5)+IyID|F;61<)fyFnlbqeC=Or%?AAYuC3y4U
    zTX+6O}B(js1+NH(|iJm66@P@wGpff21}!Gv66$$>drzKYX+3?tp7{mfuPun9K1U7)h_`Cf2ZwJhLlPv#_<+p?8QgID~GFe-LuU%r!@fBv>-l&`{49%9i=?6H$AduU>?~
    z#XnGSdKFzj1a?Uj#5%`vFWjtj;cg+HebTP|uXz!isNhepeSy4u9QC|-m8ZBZ3>w?m
    zTq}xjc#c2eWx5lSfZ+_h@bOzv`wkF$s4TGLUpL_w|3t(xm4t64USw}3DKCBoeW_1}
    zk=^W@5){3Rt^f=F3NdioY{6a~nS|*O
    zRTCY4i0z|uk8sE5hcnwV(nMWc-?=q
    zDNKa;N69F_O3Aio`t7H0C>k90woDY^@lw{jP9Fa~)F3{AUhnbKm=QFPq8PQir<(ZB
    zy||G^`e(6>3<*s8Eq5(7>fI1@OzRgelWHixKm!ekLjUOSfASU$A$17YDR&I=sp>U^M;i8T(Xjck>-3n(O66AF4ru(3XcUHll&+(^xOkSe%d`(^`|k5SECL7DLtG2n(7
    zBt1wjN;0Qm?2ztDQv~p;2A;Amf>ADxGgUW6HDZof_nXC4m%1nTudoJGzY><(N4l#p
    zoXSgLSV7EgH1j&B2utz_5@xn6>e9KO0e4^N^MD{Wi!>_?rAN^$7)!=`NVYlZO!%k)
    z2e_Hgbd*u9=4)Falu6JEHd!T7qxooOO-`^Z3rXb%5!^{{wcD+Df)vG4JS%4kM=!+?`5#kc1`SB=d!J2W{f?lufcD5--wHH=Hc-
    z-~prCd(aQ}&110i>||fwSK(d2u7^^6dnxi$2M^wO1R*Bms(w$8`p&{h!ys~$jt`bT
    zWc7T3BHLc=NzzaLJif)oXe=c7(06Qm;u1OfY2scKGEdAc0QE`Dw6r$_8$1x;a`CB9
    z%Ds(wr{i;aj5pDKJNAhpo~>O%j%KebmyVY9JA!GhvnotVMOPWmsC6I;Z9qf
    z&7*J=$ARn9WxD6MNj)f{(0Z{ic{*kuPz)P>qGO(1hQGbhUmLL9oc*?zdOMNKM{qo>{Ply;nrw7xJdWEL3=|KZbk>mUQG3AYDLHj2l
    z=y)D@@AO9}QkbY-bga$(z)m}YzgpiEX<$-FVqUi{%=Xfmc@ish7O3XgD4xLy8ZutQ
    z0;!5B6_VVTBY2%u0BSuf~^0l8VPw
    z3YL{R{7&WmbN35F+7mm*-i=46{M*~?fk)wkTNigWw6wmLcz-{p-HwVM`hjP+_77n~
    zj%`XK#iUO`UbZhP!u=-&XP7o6B-)lUtR{oAdiKjBn9c+sVdwAREc58PdBCdFxG=3r
    z>54Za%VXDi%evaoGHQf99Wsk)F}JEgqTsn=4=wmg`r?ImeTtTG49_=>yT4ABb}MWI
    znM2I{f4JIY-V%KaBh$%CUHW3okKLen#+e8*qIlANnFgeV7-%4NGD~avCs!#tL=kD7
    z8^tgFy0}!Vp7XCu04JHZ%DgCIeWcZaJu^pOKIY0BGHpmP8x=`~#Q)rJ6|N7r;4|fX
    zm$y;N_h-m@0@zAso(p9rYqwOl47UWoK>l>-j{fp6hCJ=C@A@)6&fOJzRIiG2uch;)
    z^8vrPLKYd)6%-Z99lScReC^J*6DLb
    z6ux58+r|6L&C#%RkE>ndd&qwu;^wEVgr!S5cNdH7jZZxxL}IkGv`27-p4aqw7RTwI
    z@vGTOGV5aezYb^i60vWSausJ|ZbK_r_^MH=B|W{E6eP{0xHS_WRAneAW)sCPUny01
    z6e{m`*Bf!nC`zm>)3M9F|MX0}
    zJXWh>*4V+Q>hCqzMI|GTEORUn3e0%7PUGqQfN1OdakD^gi2r)~GQ+Q;7DjE0wW{WW!2|x{7yMWEu@C14bfVnAXe?5e3~+hWJc&v*VVJ&D%jY62=Mu
    zm$cDs^SwjJ2vXVXw)hfw5N}Ck6nk*f-a70BI0~ols&&Z7YUD^g6RD5Ffgfm!=_jJ{
    zIod9eA^!-GPq!ABl_?wBFhYR{@gv58Hw1I*DHZZ^$9%5AQy7y0)1iO5C~fVKYYN$e
    zkm@L$Q`mZO|L4-(#3K7{xB1hzyF2h7$sYCt5HLThpLb{Y`1mHsGbRRlpIItR?(XEz!?){pD1Am6FmTXN3hZ>5o#GUfB<=zM`h^?0_
    zPC^j-ikT69RJ_YzuEwa|X#H9fc~nhE=jo%9jjM(Kmi2p_JHdd_Z4RRZtK}Hp%-=PoU2yp?QLe!=h+f)1mX!Hxl<5jOcMTFK8^?
    zEM}~VhvldWGeaFy5d042&5s_PaQLtsZ&uG@hSem-A6u?0&!ggJkK9%|?z5BhJZaiRmK*^oLkew34LKRQs5!QL1_C^e0EAAeCzFr_G1d)Z627fg8eCqa0~^SN@{6
    zHbiiH%42T1fYY3*H%sKb*<#@k@;i$!;T4J*7f2ARd>{bflst0N$sO*{l`*2xEt#(rP_?M_9>W1*V)
    z5PYN6=S8UQca-+54x|)6Ef&^L?8$jq^glE;WM2E+SLk=L@GC_OtIYMic)i>df;&{a
    znn{Wbco|5P^#Y3<@oeN){@X{fHUOiYffwq@{m<^#S;h`2%;W&LD0H4A@~|f<2=VcE
    zfQ2l!%1HZI8SR?3gBIPyn>{s`gFo{Rv9zISH&`8}4Ui(;#TqnwJV&?n9#QyCNSyQ=UojQe?RSa&jCqhj@CZBrfNU=-
    zFyT0G0Ez@-`rgWodMf)^zUi5l1V$)#EfLKT4a4$SfaF7oY<~?_wjsm0V>gYM7VYu2
    zdq#*Jh%%OHT-<2;{h^i62OAj~naPsxwA~xLiss*-EQGee7Mn#gB}ozq6~(WPMC$mX
    zq+&mD7eDSx!UsP^ZBgf2E9#t&PInVRw7+5-tHd7{EBu*_@EWrgnDrg5l=v!ltw$jx
    zDQ*6b$F{SP)$>N6p|G;+K8>03U(0PvX3CFgP2Hm%>5Yp&fT5}VVPin|Y!ml$!}}@U
    z#t?~bX5$KyL8^rLI%VTau>Jt{Zq7A)KyYMXOHqbG6`|wR6Y?Op)JGW
    z(Uxn;z&F_BV!s?XRU&SC0C}S;A>T(X+~6DZu}tdD;AG!o)4G3iRe1a_ecTT+T#^3{
    zR{&`VaU}k`gvH)sfYKB|h15*}J-9c-FTDp3o_w&CHX8HVRIj+VX
    z6itI82TxWFPiM>dimhdog`5Px#wI_A04PYqyjqSTwrbqiPSmX4Z!NX_-tJ(>^Hhl&
    z-@n<#JrM~ydt_MS6}=cZF;QRjQj3ET#ui)GBZrE%Y$1VZ4e0#a0cR*&e8vxQ#Ig!2
    z@!lo=y%vQ(wAM!N7WP}
    zC`aWJza~a^CQPk;WbJ_1Jz>vjd6W*N1fGlKc8=gl*o+j4u%BvKt3+0Fw(EL40I}{h
    zv3?;fiRe8@1#F228^bc7@O#}B>>x$*
    zkxer@#)CwXJFz6oi#`lhL9Su@GR`@)5V97co?Y=W{~Y4D_<_qf8N)e=o{v{N$?i*2
    zLv5vs>Th&BiyrsQ(5Po5CwX)N3g1~pmWAm|$x(*3z#vIDw(
    z&aFNgm9rq1{Etrq9ygd|evP<(LgBo@xUidKU{Z&df^zOA$ZyzZHM`NOrFl=jO>7Qn
    z#z#}$bZB~0`If_gt>@#t)3Zpj1x)}Q95H^i=`*n8QMTp!3(i{l;+_{XhdAswgjv61|`DwBtEGu>NQ&tVOV`%zIc
    zPhuY525J2enyUGf-g)5F>784SV$=-iWmx{`45NdD0!|gXQs@W2g^R*vT>H9M7~55L
    zpR&LjKp8iMZu3W?(%*c?fJ^ATM2adK-5isPb7)9}O73+Gb|C@kx2!j*K>{pfY
    z%mWkv_@|w0-59y>>Er&J1H#d6dlL
    zK;S#x&subyg)P4kE!G*=-q&y{lF2fIU8cH`;8rk@-MadxJP13f
    zmOuEPgYBfJen8HsujZ{@h|QXq5Mft&n2@;ZrUJo5p^WF)T1=*B^|wpwlIZvI-`*r^
    zM)-*wWPXo#n;7MFV0L#+EOQ+1usY70Sccj+#xs93n|oUta>dh1n}t#YhA=pA-BnC8
    z2pd6AQl<2Cc-8|k;fk8VnGK!HN6Zw$c&&`Xn8{|Bj=$=SmKri}GX%S>M%NtHXt@mv
    z6Io+w1K614j+d6+`~x9a)nj2E=v|3MfC*3go2D)U;+Y0QGVuNUh4rYcSARZH=z@-7
    z6)J*uQZMQ#o&#yOwF5j^6MzLvQk;ToA4kT8Sn&@aMOye-?YVFH>wF7(-@QjLRrwRv
    z@F0fW>tPrvk{S9;UA2abH2`smy=0IVe>kvT^$sdpdHyJk*!l20^!zY
    z|4|kefRyUF$n@dUbpQ;+q~A{q0#E49fu_Bzwt#P=Lgk!7h&*XHlJao@Cws)HE-6y8
    zka_%>b@ImAf8
    z-$$xII7yVO{95qnkSALGqdneoQ%`XvcJrRq%LuWjumHmo|lz?=Hgmjk(NDNAMNJ~hUgh+#YoA>(z
    za9!t|+53sL?uF<4s^kUM3jQJobJm%$iQEWuV%>58G=$iy;!8)u%)Kv%l|EQ-0p$Ao
    zZw3Z(-N>3Pb4*1w%d)`FeE`D6gm?-Rn1deiBsZ
    zlry7V;~!)UDO_$;z&|NmbgP($nqRNwL;QqvPLhgrT35-3rct?iHY0I^Xqf6s-7|Oj
    z!YTaB)>StvXgeyL>C?_flX2u?>hd_P(b4p+8jM=w>DSFk)Z^n$+@WfAM+~M%p%coV?n_M1A#GdghKff^4(I<~w$9}v;>o$<7=^bJSI1yKARR4aEJEv0&LZ{deYg~V5OO#@>~K4tOYTg
    zGWeBVk+QF-AgU%^0qwPt*_gatl_!xkP~
    z2Wok}?BQrdX(I{LgRHYp_dd+>fWlEbaj&|uO%Hk>%2V#RvKI1wB=XtVGoTsrFOA%n
    ztbtels?WC(NtiRsHWt0Vo!nZnxmR~t(C3**6(ZkFT;U~Wzu2oN$lMXGBt08fPVqF=
    z2m%mqED>#DFgKTP#H{*6wTHd^bNi?R&m7w~Ge2#|ENM_;frn#`1(53ps2UUD`;D9k
    z{kD8Z#e*UTXLfmzssY&(nq(OKp+9RJ&-(d_RIl4V589ykMainsb(Zot8?Uo#UAFRi
    z{YW$fI(Bv>x3!6f4*GBe+9#|Fx2u7AJw}zN)X7
    znkU`l7ebWx>-39=txwdO9*jrxt)Un~*De2qNoOUKyiqhjaXuz0B0lL)iK9@9RURQU
    zw&FPU?ScOr<{2uqzH1q&egq+VRZNiGGP^m$izu1^*bch=H#;2L5;isV#Gj&L9C&U9
    z?w;U)RFOlxz4*NFSS8h0f6v~&e0+QR-Itcj(}|-qiXkyYjRh&4?X$wEO^!?gE|x-)
    zSN){Q{Eo%Q>YY0nCy+b-PG5lf$ND0*d)~MJhkH;z0q`#XsoT6BOt3O-i&4
    z&PLZD?>aTLJD6+DZ~FK=k_JB%a?m2y?0x#WW6Ktl#wGJRLpSJdc|TL;1wTa`?@j$G
    z7L=yh|Ae$R5oP)qG!%g6RJ|yd=Go2#1Z|cMu$POLaeHQ&O`694QbN0MvTrFjWzSxp
    zFE_1pLK9t%#m%izTOnIhX_>!yVJ~Ob0oB@>JCt9EHuU8hsFFDE59m8^0>tD2`{Hkz(*b07c4d;K>_3wy*YY@-y9->a<^EiKGSt|h#|V!p(h
    z$ryj<)0|q}Ad+`hvh}>TciZqnv~nTv&mgo#=@9L#W2f*Bkuxo
    z7b&T?*R>)atI6W>_?jy+9d}hKokXCIt$}TJ9Jv>aKjxM4wgQ|p@V-Y2qQ0!ZvJo=$y4XMEQeZ6U#bVC%pQXz6a7$CSz$mG<}RsMfg
    zc+P$`VOOGHz-zwK=^?WE`sIaj()z*^0>Jf8P00%n)kN#{cFTt>V2W?{KKKMW@K}t$
    zr;4mex~6-14wiS}EET3a+*}L8he3Bi@+LmA3v&i|h`xjtuUP7KEbOoh$-5)I{>blC
    zD3w{TY2KcTmSK#2O$Wdh;QlDoLvZC>K4`3#3g(?~v_Sb)8I(
    z2WHzd*u98#_NAsmpUZHSE9n9LHxE)aCxY1r4uY6jLlqj&lJBbx^vLNJvqb?-$=!24
    zRD+aZJSsAjQl#Fi{7h!ax&?A6jH1uv1z+&7HFR+r-3U#xJ6OdA{J!AnFi5*-{k>2^
    z{kleRES{q2g?9L~5})vzl0gn>V$N^bfsvC1HI$c-h6X
    zKUc5v7iZenmhdD_?zU~O#SVA!2&Y}vY@Mb7Zk#N<@x5k`^nJ=S)fkUu2N&PFEp`zn
    z(-ehBOzZRyEy!sdpVmcj??^?ZPiSoAxN}Q@B_R1dSA^jq@(GfKKtHq`*K>
    zo#^+Di4j}w&|6V9Hk*u$cbtBgyNvvLgM)Yb^%
    zU%cA^K!A>~3mPurLd3izDp9|g)~9Q{BkNs54ssjcj_4_=x+KaPD-;Lho?*nPkA=0T
    z6<21;r{+IKtYt)M;wQq2qyNStjlq-eAK(^14W;0s34`=$hR4Qhi&g;vSzLmgZBx6$
    zLxx>dH@EwWV7*ehBm6dwJXi=&Zr3X;{q%}?y8r?QLZ8zV8z#LBgUdA)<@SnVu1}{N
    z#K;6a>8v~aJHegi8Svqe2N4sPHrA%`&b$^iyAq?We$OpK2=MBao^fO^K}eYrKq*LN
    zGIibEJ8Cel!pMj3t%;`Uo9*s+zh6q=tPj$Eh~-3_&ut0W-td2{GlZqdI=YIPvsmFp
    z@J$p(&SwH|SUuj5fz+JiG2o{bs4g}J%lD0l0O?KQIZX4NX{ewgD77O;BqT3f8JsRQr|1ic`N)XU#L_p=^I}xaS0zHras=*
    z&LHK*7pa*NjEEM>R9umJ$rWHKmFEyy7FSzh`PmMcF)fdWjDp?_5Z^s-A0B*AUs7WsS7q<$1P;UBv^-Gv;-D7`l->dCgE?-QuzQg#s2wVt9
    zS;b5hRzzVRPTZmQZTUv*NMF$W`(c>&=lCw5gVx)dL_zA|>GD9@rxraH7DiCrR61LbA
    z+r3~1XPZrFTdcJu|FbV68f9Bjpmg%)tAF$I80vIWYln_`pd`SR9YIiFCy
    zeBg2K{2NoqQzrddep0jGOCegG0Keu$c%==#n9Ko3F^af(k3
    z;+QwP+7MGl@Wz1rk4lP8=Ca!4#vKf*Ii_9FPO!KQatG4`7p+u#b76WvP>6`
    zj13A{1u>Ii)Mb+PQUpH6g}c@h9>Xi&AFTqO%sai_3O|r==)Y9mBW2u3VP5?sb;*+I
    zqGp6RqD&!-+pK6Qj-CB2iM)SbLf2wQk7TJ$#wy(szGQpGt3bk+!tO7Od-XFhoas_Ev
    zwe$+`1YV?dLY;?20l-<^9_IRK>01C^RG3MBSCn{B`CoT_{?KMBG%f%)K!d#TB)pTb
    zL`A3Zv>+R?>b1x1l|lC_XBqo}^&4sT-&l06#=Aq5bEjlXDT1yyJjE3t$C
    za?$h;3z^A0yY8fWT5^r*VM^FU$fr%WbZ_e!P}jqUUcUa}>nO5ku3Fd*2dNJi3<+CY
    zxbNfNX0$8;mMbGV*k}PujGoGnV)5kf0c3Mw!v%e(yNzd`Zz(LyLnD$*edrORxdkrD
    zOnR||B)=Amz4)W~NCV0a5~QryHzD%r8rJ76!(?M&D3!Kz0hDJ;RPJFIuEX2+_=}zQZKxOxVVS$MV^+_Y%z2a=1+ezzYH-mW
    zl6f8wjTm>hqVyq;w;-qcIQYfT;GN_45K!Q3a&^*L^-OxdHN^)`o4?I0_Vv9mhD-5
    z4Uftizk_bkD1!2YfNgsNS2gi#yM{f@+w<^{p%K69UH0?YE}oTtdW
    zZo-shne!QUQKE0RAonCtN!oT-FwSqjBFS%F(i8x;D@lgMMCTZtKF5yvmoz%L2|WPz
    zJ)b7-bzx{1@XeNsKW@CMmO#Dj)F5^hyD{iFmbEFU>A92-=7&Fc`Nft;YJN#V
    ztu@9~JqQnzkISKKMErfXM{rL)^pj0>00Z@-Ye&F~F|csLLpS>>Pk&~G5ozH^WH4np
    zb4gp6f`HJN4JU5E25bsu3@^)S5>csD%n%Ub8_TPr2sh$D|?_vhPQhbB_
    zg|NO~14JiBsP)?d_(*eOyI52weBU_=UP*zN;+qKN0&VhnEC7ob{JIU$f??SX8ta93
    zLXeUrDGl$~UvD0pBv)+OU)ri9W7jt;>*G8xpV#yqd0vZ$i4486CK>*|2rIQMASIUg
    zT(2^bx|`NX$@i)b;roy;CK1yKDDYMin5*Va#!r7eAj>nbL_!tu(-Df`D+1W3y7
    zTVHrSDI~7JCoV&2u$I`7@Ccl##FkSSK80E;@8?>t21MMxsqY;@GJCX#J
    zgk=~)Ux#K7Mo&KWt4QB8c1P)a$5mu5dVEy_@44`iL<>*ePd0GsAC?wW6Adag`?f|4
    zkcw;o;01q|FtmnUM}=NXH@Fqg12d_@28o|;hS|UO{KE)2h&(g6X%=c8yorja6c%y5
    zHj_HDWfVr}PjdZ+JDRNfiKk)IhM#W)XG7wrcg=7%KpPW-Q7aomJsGocGZ##}mqWH6
    zhY|spS0qQn^}3or|aIiCCxN3u`0y-*QniT<{J|rPG8Y~W#IVtZ$I1M-0$kNCV+xiA2wClxrx@z
    zTr^F?eFw5Doa99Le)k!`wBy2M3{*)Gr<>TtF(`$og@YI$!Y*B?v;OH4tpD}Pt6C@i
    zP%=Ak_ZkPcMq~Va$g$$k6*wAfW|F?Sg}L?a4|7(DEkPt84hVc!OJk&uSrRjziPOs?
    z)W)j+?B4+GY+tL?rsz=8Y&r~R&|K--kF5kmhnd!`&t$?%=P$nMUu{Ode8fAp!0i04
    z@37+HBk?%nf45dkRidgQ7M9aJFt@oLwdoo?9ui!>{~@l28|;T4k?VzXlcSzJIL}ef
    z67G|Ul&Uxa@-|Q8Q259E#j@?i!?f!(pwPRIBY;-?BAXFdw^DbpX)|0w{Yr8)Vavix
    zm@DKKVO%X`jOxFXZaPhc#5ekj={YJ$Wi9zO^&1fQyY;}c&C8iLHd;?1o>ZZfG|KET
    zyjhx6wdn<%W3e^Z&JUp#PMFER!$?&FAW||E{gcO$K=90onczf$gIv(!JCX~e57iQ8
    zUoyX}^t9xO7xJ6HoR2U0_(enVbK&LHziDT$Gr{8#mCULr(_`6-jm5iWZ)Bn7EwE#9
    zCqgzq;9~TGCK04A39Hq)_=`u6?-glWpZ?I^eWkI$$_|_ez1oZVv%;dv8a824-G2rG
    zR8D8QEMlGLCqDQZDw?y!b2Ta_Ae&J`oP#6NHjYIYwH$-0W3?p5cGH-PhLq^Us_dDx
    z%^w`aQsR~VTzS`^KEqB0RgF=k(UJfkgug}MSwp6019U0uCg@t9|KAVVXnsz
    zH0L&o8d>w}$z$^X>By*M!^u6DA@;FTeqjnqDs(AZlHn>kRD@z(JWh;R&!vnWkxW79
    z!D@2z1N%PX12>IrgjaH8Cng|?rn#xI>*wVqP&Cyka*5v{wU4CHFds*y=HoK#L=8EE
    z1w78J6~3ga2UA#!K8q=??2IO?9T);A0!-vC?5@qb=bY(^sD5iD#@;npWKaMbjFxyr
    zQnv$AAbJvkaF%StaGXWI^!#x^Iyk^9UKjH3%cOWnx!6M;90WCBSjH&9?O~L-w%xt`
    zw4C52!9imlOLK2ju$f-;Yx>uU3!6^6$#{vb6(hOsOxb@DUkm6pvH$eJ*cNuE?9y&=kus
    zB>BB+#jYHfVPFEblIM;VcW}|$L1}d|BI^bcD1t5`ZRzkg7=q;2Ny?nU{VkRVF-Nj
    zl{9#QsEqkn!0#OO=yDb$c|Q8aS0=*lo>HM7LeuAn4Ih*)`a}M!$>y>m4h;Q(K8C0s
    zTuP8khG$wKq-KYca*Krb(WSA*YH(<1=o+K0a~qcT-G|1pf9B-$q2PwI5W>OX@emu%
    zxYkb3N4F4^ON6w7xEAtY|5=*Z^{fylC_
    z=rN_ObH@1UC;%V7i0IAyqR#qDggvRe;IC+>i{YF8w`4H-!(Wu>VhLgq%mmo-d`TSJ
    z&|_w!k5|_d7e+!9_bh|`2MC87kv-Q;2VN4`4ucmb_|yb@k-Wz)+4@6#jV{$*@DoNh=&
    zC^MNFdr5*2FU>ZSvp6=j*1UFBYcNkO@k?qZDAzz+bY^>ylMgkmEk;=${z?2cp-Tqm
    z0pcc5vhrN8Yc2dTFy33W^p7EYm8%6y=`z4X0lqCxLY+#Tg4~YWKGQ4V9(S%V2{bI
    z-M`DxhkqM10~J(kj9eB%l%Tvmbx&sbCG&@N3;Zim7X)zIfj|
    zq2qW|d9fxRbO;^Cplm==9>3ctnnR9g(F-N|en(QR@r%s+S#E$(_RO^R!>aUlfw$OB
    zTD7ETcR^2;PE$+Osu#WIErL40S;JwJnbSb*pid=KtC*&xN3itMr7>MJOBDn~UK@=x
    zInF#&8Lz-PY$&0(k3%h!ByfLh(&YJnfi-{Y*Y({9w}IuMT0d~iL^Nk5!gfHag5Z25
    zFm&WP_vgVd#u-3TnMCL_2~cOTD}uI0@t?UI{ZG=*?_lDOBx6+=xXO74w?B{mlE&fjO2E?R0T7&mSvjCwihQ=~HNeOp
    zps-E#0KsnCZd!%ngSz?#dpb56s>}B?vXnR@u8hm4NlLS|gQ?QK2u{+hX3p+TQW00_
    zXyx^P>KVHG+jK^sgr&azmm8Q|1{Pe7gEOX7PDW-COC|IJoPaO_~Q;5sB3
    z#xAymEQyo&QNr@pcgH@>$)!I)v$$q-B;0A59@$ILy!cJd$x|aUJA;Bn3>2m=ead;9
    zUUCr+X!IN-XXqpe(>wWR+DUynRh92gKenpjBkRhuF@*R~L>T+fU`{;TiI^lgkLst;
    z7dMi(gi)*Byk$qpmNETvk`XR*o+?XNyD$gfaQpi%>Q?xFOek1UR!v5XKEMbsAnd^JCcpvR>Kn{RKvJ<=%t
    z3rp-9Q*&p{*rum$f`4oxjxnlVV$ZsMYX)oGPB@=(p6%Bk-`!Lc0&St1#TEsdkek;q
    zW)%gf5sVVQI47ENEkF#SP!akS5qy3$DQfYPGj9bp~*2`nCdjz3nYU^3WZ|iNXiy0gNiwL^~*dnc18Jdf9=v^+A{BxlIOr+(7#L
    z#6+^~J~Y8R*)Vyyrth`k`Vn@CuK`q`M94uPJO=Tpt;UBNvH<#uQ+0b|<+?0vcH
    zq3j1{`0}J-#24I?zFd
    z7T;gRt5Bs=i)~ty|JIVzC(LtSuPt7mVaY+@6E>f>dR^C(_((jwpLZHhBa6|XSL}cs
    zU*|Z};i7FpEZsgn()*cUd76kF6HC-O7_c)CA-PNg)t9!JW|+_}i7Yw$XZ@0`s&%c|
    z2NpnxK*c4(!|yoiNkHNptUjBjtlL3OmN@xmvK1(HY2OWD{yb*Ao{OO>w$EjZRq0X)
    z1PRC!-q7|R-H6TRkpjVR`-b3-gBU{Ec?otgR&i5GqQ%Rm9GY;|KV!cdevtj}X-jhz
    zr$UEerRVgjiWSWXXTx*CJn6PBX3&$LoJfWrmgi1z%KwsG8)nD-^6C&6D?-pBm2Z&^b^s5u!wy@HpqJicnZ*BPrqh)fVeT!5x$K3
    z#qtqY_YGl2BN~|Z5%toy+wt7adLjT~!sYP8T$;zcJOfT(-u&lVEzxPrS(75?xSi=f
    z{<|kSETT4si)tUR^PNi?=|n{N7#sd#9qQe|7_r3l+~l}q1c1DDcrPyZL%{W+#%6S3
    z;Nq*~=>;f}H8DK251rj>oo;atD03XZB0jEHQptWUG6|9kDKb}dQwP;epQ1WafNr`L
    z!C?PFtTLiNJx2|0#i$rQ#!_L!<#As}oRKkywh;L0s%@#Vc34u2{j1;4!1)yaKGS2P
    zWC;AvIHxELXgO6eHH66)_&lV*8|8azC9~q^1MVbkcg5jtl&Wd1&Issk`C96Kyd7Da
    z1}g`eNI3)<-^4gLorbDj{8Z{P(yG6MGWK8Y4SH;fVT1GhNOa?t43CJbnrtl7&&=~t
    z$=^y=*Mku{7MhCjP><4$mPA-)YKtgeGzgFvjswJwh@`Oi1gN#_GH*{}2UV`zqSPmIYD5+0xdMD49;_wWvZ#xSbk+=
    zanv+52Nz^%!J{nwL=K-e%uBiv-5xD!(UxNG?ryDe74u%vc{T7LUyr
    zwORkH0%Qyr4?v0JWqKci^@QG#Byho+pAsRaG~kA1y@3Pg$xj@`_<||hDUJU%BwzCq
    z+d?6gtX#c=Z#*lCrd}{2kb;w0(|rZdU64aJ5vtz4E;e^q5!;{;9u@lH_NTVcr&&rQ
    z!;P4}s0om>vH|OV>xD@&ECZN6D;{@^(;rZw|1eNqy}fulFK@^7CpAC;q=pAGdNRZq
    zjhj57$EsB3sVOFq#|TmL{mA?eUt2uC)4v-c3?HULZ#*(vztZAIrHPvYWX_pgw7=<+
    zz_7=M@Ltv>EW`2f2z4D7-#7L
    zG2S?kgGUj)axKU|J6h~rV;kr1Eifn6g^NiBRCmeR+D?G>fAA9we{r>meHXp|3|wSY
    zJC0PXdOYUN+mi%h1G@uJe(;^Z=e&6HN^j3Re_$T;Fv1L*?He>fHuknnJQ?8y)iCNX
    z01{VJA0>Jl7t6^&jX|3+fXJyrbG*s%MX5sPGE{tdpP2s5%RdkcW9sOsjwS3=IvVce
    zf#uY_o;9n~E%#c4>_p89;z#K3I~*La+#4vWcn^j?HvB-_YIQvk;;{-&6S~^z5XXYJ
    z=d8^@0S-{eQ_RzM;yGa-V&^U;vzjWCVGiulAlS7SWxp}3?eR4Wqqbw$U{ESn;~0*RkNxD?u67P#+qUNyj)~UvnrfACH*zd1{jJF=(}|0
    zQGS>2;ZY)SuV_6w4f;8xA+`7oN+h#)TO8Oharf=MJ%0{R&$LWF4Vka2n&XI4sAv+9z8oKKA3yL6{5kyPuY65~%l~`NIE>
    z3wX*`eCY#V5i7a3F_HawMm*k)>`dmFvfSbV5F{+K;xPIK=5F84^eblm`Yo$Z24+}S*c?(|>E&jq3Z
    zwdYYmX=x5m9&TGj_WR4AZOix&l;Ls0lJ7|L0`kr2n))i8;p>-7?2;WK9d^H3|$)#3s6;B1;dwm@J$3PlTBN*`B4{0xI2O}FCP1JgyJVEJc_vfx62QlbwEpX1g!}S
    z(k3=dPJ5N!Q9O6~Ca;C%M2RUl|D$hw#5nUy(a7b`s-_Nz9H>K61ETp&p1PEC?2+R>
    zEC;IiwBzwcc$Q;R-82XY-Fr18j#5EFmg;lZs;k9|zVo_}h=iq>09t`JvtXBLwxYvL
    z8qBrkPqV#0C_0mM)|4xn$N_;2*HS{%b4(B042O*^mU8ebShJE;W*-u+g(B9;EMjP-
    zM>n@<_?BcM@%aBWu^a7}q31NoPy|87CZk;}>qQ)O%1n>-s-h5ptzqs!JlMWtio7K`C+E0HRleSrp@;X
    zTsNJ3!b4%!Qip*rdzWH*kyUr##IgbapaltB8OWhL%HaJxW9rD`IDiOW7_)Xi7
    zwkL5~erv(S7;_Q|N9i!5GE8cM)!~NvP`MF2HI%j21W4tQm+_HYjhmAdnhssJ9xR&r
    z^6~ClqlALH
    z)ZYmJ(XpXcy%lXy^@^~7`##+pF`TbY>S4}?S?2dtNCr3J?#?FaKEsSmhjFn(hd~si
    zGOyQSW#Fl2>|NHm9EB@A??n*bqWpemzV^GAPP!{dUZzU>t0}I*BrPG|e~3N%|Fz}$
    z8o2~&h&u%jeJ__Z1)~;?nCgW(SqL?3L|F#=rs9j3vUGyt*Oy*2;4d=91|)})s7TYq
    z(~vJ-)Mb=W{MS%n^QqLYuf=EdK#svcRO}3X7i;lU729dI0bbs
    zPijp_6~gIftBmoAI6SqvRsaD)_1h${@My*@L&>eTHevnmiGYb`4b~+3rgAAdda8P4
    zyT;{30q8!ft5{YAO|(=ebIr!zvf)jo^HHW1kPjpDE5umksqm5W_R3&wgFj
    zQ$@p0=lw3c4vjNR^a44BXVw|#?3pH&4Rbp^m_c}sY||g&GRRUKheE@|xwv~Njj0L!
    z@Rs}2i9@;~%QFXYSdiKYG%xr8%yB;R)`DVT9$kQ_!%fez}q7L^1Z-V)F3%urM^z*=cv$7HhMiT=#Wt-7$$B-ho%kUj>7f%
    zUHH#?JM5u0a_LU^ZT(BWGZ@&*dZ7;X7A5QFiF9_m<^LJDmFlTgJ;m1Jfj9fOVIqAh
    z!m%rrj(V|uB7x5AP~lt7*7FFM5Bq$)Azq;(M!6bx3$j-XtQfan?C^geXvX;KByiIr
    zTl1k(3@5g*j__t|f_w>8_a1zpp`}9SUbek%0s%R0$3b-XNRxNO_G=*Qdgi}k2gPg^
    zB(GQYOrfpg*_X3U#;u<#&&@ZxtNB7qp7wQEi(pB!#O5k;{#Hyl&0btLhwL~EtKocH
    zb#-enGkVNCj9Pg@RX1ZjCzMhsBmScP-BKbkzm)5RXXEw;mE>5};$R5CxDQiZI^?78Dd3cAB4hn=
    z>A_N)kQ=x@`iEiGC3%=wKL2Vm8=$%0FKvO&dq;O)VU>gmO;`-)QEg)hsQP_}!1c=4
    zkVc0Kuch-V*$m$?-S%25$nWlgJB^{<|80Bg^kG&_nM-4VH_49$cM;Z~Yy<=k8GDwn
    ziKO-=h;H#
    z1C909t$xel6_s>-Ol^a2>y{;W;0eabyra4wZx_N}U^&%vh1OS@+kNw`=B=52xf9@Ww6c)Zqcm@sb)vkX
    zM|>zmM}6Afsywk-K=2FoZ~CKd|JLQ}2=c5x@1{2}<0qkb>?`zht44zxtt`P=UoI-!
    z_0QI{pF1sFZvY3Z@-_T1{mI`t_Q_KHM9d|Jh;TR05TTnz?2&+m-Jh
    zLK8qTUy<;mg!XU~*cj+fR-wuU_o^%mAx=p4;Ja@3_lF9MtCU)In{I#cq&E(~eGdSy1W*S47PTrF>fa(u
    zEOK3JT72$WL>L9a?rP^`3LhPLx-@OKn7_EYy6mlPsqmQ|siwrdYA+JLIV*KCX7ORr
    zfWaR@qtq7wuEKR#Dcd*FGk$?IjD+frl#b6o#hScd{3tYBgzIKE>hno)Yel;HABYM`
    zetIEHn*N|ml@1NBj*nWe6?wxAz|qz5;X(-c!2gdB@)vl2b{C56$L5|cl~^o+%qTOE
    z66u16)=wwwg~;zj56|9OjXdhAyD%gF0|i^)#(n<`jCtf*2^B)TxAr??m0q!lD1w{U
    z$W^aeaw=xVwQhkA*414UUh+4k)rafkYi9F7@`iqXAjE>5YC3E$ri#h%EHF`k0~dc$
    zjdn{7CV=vD@?}18F&KR>SXv_5y0*kJ{1i%bpG%F-XMYrBNQEx3_J+QDZSHfGS2DgU
    zh*emlioEfIeun`$7qA!3>E{1w`;Gtjvh$_TAA1uMd3fL;rxtD}6wDvU??3n^K{3Cb
    zjF`^&z)}tZ&IY3k*^2En$GAd60u}++hYS|b
    zBXT2t-kRD87q4JFaxJ}L?f4G34PbmEqB?z6kWI(Q;vQcoR*IbGoBA&OY{UAbh;FCC
    z?xxmTa4>wm9f7rOJ#(Rw>Tp|fbgywz@eZ^Kf`Rq=KyO>&1L}OEXic?0_sC66vc7}z
    z12{;4e9|q?CyIzEb}rqY?ZRQgy5cW1cuhz_y9}QL?E04jh{oKgaYQoN>x-~*zDIy~
    zb(f*!g?4)VQ?~|@Elj$nUdPo#NO%8)OarOcx6aHkpJ^13|F~!CkCeP{a6`794DIek!KhuCX_N&
    zjE_z5kzkt)ut5nZ@Uqn26{mO~UOUoNHLjfg)lVWEY#=j=7UeOIu?AE+(gQ|eUbE3!<5Dg9k}=vXD`B&Iw{_@_B_9{*{Lxz%FVXa{l6u4so(TrusxZZN_E
    zWX{e9orl%rC|PA`JWBK8Z~PXX1H47^zgO}mz25S$>%M`#V1=dO@g<+9P?+6&`al}z
    z*dgwNeGcQd2r$-rj8IKLw`-*3D8OWAySI{$NT_pFcJ;VP}n%%7UUiF5!&oh{u>
    zU~1y5v@27^;;@7&rmwcN$Sc@9bmCeuKtLXP+CJ@sT}`L+wrrvY5WGCn%o7VyrzSgb
    z%BFd~1?Ld_^?7YT^r-(+f7e(7%whWQ*H8(~+qIA;w^Gmy};U_$)Rluif?V=Cx9Inm@f?$Bp4Po0rf<$W;L;lkU{eKaJgPir)m0ccV9VLLO
    z_LweJkr_to67F`-^epr#`dQ0kvYkME56_d#8AN>#K9bQii;x+?jUzVAqSOr~cAqZp
    z@$otuXK|AZpS}sKgB%hLSNfUoaen|#>L;A9!zPzis^NaD+1b%a_IM7oMb}%weJfqm
    zZ3-a{u==}1KQeIBUwNelW?&f46wLDxuKslRz@yeylIi#GilgqaDLpJ^Ly$AqKu!mmHJ_|2T9N82&S5Dzq)#?^OWw8+6{
    zf{yaE?;T>!yuI^&Tkj-USpAVV
    z-+Ipim&y&bK^iWGgi&(i$^z*BKw7J$#+VuS2WbvO_Yci-5{
    z{x>9EP=7ieNS?!)hAoyB_d96y(&m+8^>o@RF3k`QNM$!E%2>4`Q@hT6bmUZYR-{pZ
    zXfe|Bp6vVQn^iBv3zb_!dIAck1_P`hmKhU$y?lFvHwShFZA{C5Qtcy$9rs0q+wN4}
    z`^u=?MTQupn#$OOt6CJU9F@p{F|M4P)bWmN!|5XMzib*w)ho)aDzOfNEq?Z|F6V*$
    z7!+tR|D;Z^bnaj-GJ|W^7>&-&>=TwvOaY&HTu|B&Uz}g$&O&JDL=GFG9}^ful0zWuKMCGMhSyE%S<-Vta7^NR22k)cW~2vVsrT!BsR
    zr&Gd7Hhv*>8n8D2i76f?6`I6Ih9ddJ?X}BlFW%?Xet*~@!5mZ}R7y8S(xGSbWLzU5
    zF}6`5Cv=PWzeAozzsQv0oA=|%U(+I3d?zr%`!MbXv-&gO4GSOT;rZ&CZ|6UF9f7Gw
    zgLtS!vT9*(>GDh1&|rHtxo=KA$3`13_dzrnm{f_pNw4D!efdcB@%qqDkD#$C)zM$_
    zk5cac0o$TqOj$1p*sT#DuYi$4Qc6OB_d5G+!@|{wnS_Fz&mhtGMgYk$9w#|grL{{g
    zCs!jBK0FSToZ)8#vF5w3ywR*LiVjk
    zGGG?#t>e;0Mq|V@m!k&jmA)P^lT@(leRTF+N&1o3jWrUt=fN}N*t5@IiQO2V&u%-7
    z1vW`43$u&cf<>HSM4U#v0LILi+m%ZIGBx6d>0?DPlVo^cb9b=#gghGnR=Rp1S*Sx|
    z3o4+Pcr6v8I{akAf9R?+k*LrJrp*g8K_Dot(|JEUo`|T-jz)ocgb?6B*IMypb~ALn
    z#l%}Q{HfNX4*Sk*Da+Dc`{}GO8;pDnA{p+pKU>z-gN|cS6Yb|sXP+c4tQdo^D+6FY
    zNO#7mb?48emH3M}L;mcU_F}d1F!86F@R<-;sa}Opzo>y?Qy564a`}Jd0zPVy#d~2-
    zn=ZUua=!C0?^wK`9=81@!-lg#aB(HTf7>nE-FX`KX@18rC?F%~t=nc}U=bqOVj{_M
    z$~18@+2SZm^Gd^w&iSF1mE^-J)j6-GUo7P3;S6kudUI{<7_N_9^L
    zdB)7deaeM!HmE|et9yPW9MNSM^s{<9nA&lGM-8~bp8TM4?~ATCo!eL!DF-u){a|?d
    zN$w<MUNunaA)XWXh7aG#!wuUal}U0qkJUvpNy+2DU7sgcK-m3r2ADBy
    zFo0(a24oRo+_yS|x%s4Add4JKB|Yw(mEt46CvUov$oPvn5VDWeg~t(pyMr<~d@S$s
    zwdmm4WO|9tg`E_FGJ)I6wNV%6`Co{~5F%80F--cJL>ZqW#tVy2WaBWl)!
    zUw7y|Dk=%3m6p@T?;2#V*zm_Oc+UhQPFlK$8`nSX=3xON!z)!o#G!cDYWN|nZ^aiK
    z0rK=X&ozeNtXOXIAJyoKp@^NCR^Pwk
    zS6?-*n6HjLQEYtGFt$n)NqdQk#9V{#!i?wZpa31lw4
    z-X6dF@!Vmm?3Ne{(;(4PoagdmC)S5Lrb>tYa<=Y1Wk=^iBQdmqF+7fWBacOFbLN)v
    zdGUq(g?8D+QQ!Gct0cqkT5b3Et1J2())J>;&a%`>X%$0`a?e?OQl&s~1z{QxtThNmh5dD5Lmu#vr)qoMXzOvn-X
    z>xZat-DT#i>^_cojM{eDA54!+Bpm6#)(gdwJ;kbgJV8nLKD^B8o?b>$F5647TJgnN
    z3Y3Y}FIAf0#&C^C1jPi~8m$|zc@xu>NJ_b|R@8&aIj6AIEN#u)oJ7_IJ$|o#-F`$x
    zl!gCS!taPuMIVzB`rdRwq`@=1Cv|%3?KU<=e5sSsrKavgRG}djU+?QBw@xe3bM|1U
    z*T7s`GlW@l#3_a{L0Osz@4w?HfYq}P9PJP89g12%q$CG-RT8S)mZwAXC;0MHZar0&
    zo~1&w4Nun#pO4N)9mQy{vG9!;=liZ?fE}AU=21=&nW_4J^
    zaMr?+-1)g&2lt)%xSLfEF$EDF=xM$NOw}O}wZK0lb@i>UIWlPL);8c?2rc^PFobB?
    z#To>FB#tY3DQ}799jc2Bw@KQ(+3Sw+&mW^W9UIT3%TZS=9eU^V*ZaD=!W_)R^!0b1
    z5e}hS^WP>U*DFCf|H3liU=QUAWG@ibHF0eL3Q=NWSw#7u
    z7UOJKPw%Ti_J;-P1WGzMroECLD?;*cubXrTbCbCZ!LyU4^!i^aW^(7Vk5Qp&~m#Oc+S@>o^&`$5irT{M|H$9&yO;0deXT
    zz+s!^cOGb~|J-lmIFU;V*ma$Aid}3~G$i@!I!6wUEl*4^@P0ow!d4|HzOe0lH=oqd
    z1Bj0fyWEPVBf=JnxMdvl$Y(ekcO>92iKv78BZyo@+EU$Y37ftz+RgkS;M&O&S=oiM
    zGIcP~%XrC-C)Cexk6Ezzr!WnixN`OZOpiM!`Ttn^s<5iMsNKEk5~M*&>F#ck4r%F<
    zmhOfvNSAhZ;$SW|p{I3|vFxZ6Jo
    zy7AuU%kOwjhc$
    zodEQsq}Lrk!Yq3^lNh+Pe@amoglBjNKq&c3Eqk_})&twuZ%F(~#0?BsTMsJIyj5qg#>GkSl~4a|
    znG{fx^b*Oe__s@%I?Vea=qmA4|6+6dn&?CF&R>GUZ!3zQ!5+k8*RSr2_=1~3f4f-ubC#H*m_!7_E3x|
    zNEdo41^hb^oO}8%YRo2A8SjFEu8~iJ<}RL6%nHan{F~VDri$k3bHf|iJ1@6=Cg>Mh
    zx~SSQ5k8_Ji!0q^>?R0&PAqgv?L>LRr5+9@#p3T*asyMZZ54;bMdd)l7M1{r-{%v=~#_FfuW$r&3?(Q^c8+Gb!2)m6I;c4^<52t9oQ>V
    zmz`dj$Fd5W9KA>Y&gza<-2mOcSr}iL_lJ6Wys|(I^%F3fdjZznNbal<9aMV6@*`-&
    zk7Qu(!uD9_N`4~U?CZP>p
    zr3Uwe`vY)ue-`>;2;0fz^$&}m_?-}P;2;h*uQ)GlUNJ$lCGK^bUj4e;v-YW+6j*Nw
    z0PC~rgduZ0ZPnYH{~*-HN5w}n>`ezdu*C#s;A^>=58lZPAh@O9wFON9s@oVb3wYCj
    zmJI;|ViMH;7(XE^bq+BNbzZ6be*TDlI9->dg7UH5+RByKG5xo4=s0ih1t^Nxk+cdn
    zpOaSk-*@m9Vfzrk5S=WjBaF{u?!MZ&YPH>mw)Zkr2}yx}c&oY~@JFF;t55pt)q}Bw
    zhvM;4Z>ts#_?>{tW$u!^1l|1!Vb?GPu7=yRnXy!er`Js85IG!RgMThHcKHlLn8|PG
    z14Wz=@+&oEXMP!k44aRPi|p`avC(coNJ5By&P>k$NG+g3!^t|YQ}5$_SiR6USCB!j
    zrF|315?&D^3doV^;f5t7hp`$HANH;TnXO}2;&a3)nn^+ACnEpl)wCPW8|Q2kAjnJB
    z*J{qUI}Y}3VZbu(kXia0oEOC%Z0IpNItOE7SI--j?8Ig^&pLATn0(|hjKpc^tGc4k;0Y?uCl&p^+HA5Chsgx}P4}AKBhQ{1k@LLiQi&;dhuu
    zQ%Zo=Y$!E@p;;4_Jq9iZa$!d6#>IRvLKYZB7t9bnVW`Tz5z=dP575&Sfe|L2fAo#-9+hgK^IqKaFB`l5&n1a
    zSD0$hyYz62BuP=O436%&AQaiuJV6b@1;;LV_1*2!#{+W76t=`tlOV8STk0rQEJ+dzOr#H
    zB|1A|0{cTj_7@JJwsSSIm@iMHK9I&hcUy$k@kWjoKER{n5B^@S^0h(?KfX{0mX@@8
    zTMNPIVM(8W{l5ycrAec1#=3^Yr
    zSJOK7V?OKW&x@f=XuE_jLPYZvSYPW|1>ftp4fI(p`JtczlLpssi!oeQ|KG}79&XT7vpPk)@!iM#ej7o#9{FMAiOI$o
    zC$%l-@=mw&_Yw!`L4Ijo%7?3As-Zjo<(VEpprW2aABC12``DO6y+J2;QfxT8IX>W9
    zefXKx45HL-hdU-S$7+y8BBI8{*>jTEZ&pfVpn&_EAZtgsXrz!h#F0IAoqsPgCzby?
    zP1-(~>mT<@7#+7kv!3lUU1;e+laQ78+oJ`fmmpd$UgE&^w9Q25=lz65O2e?Quw+KD
    zyGxNUx{bYf9XH^}RBzGF*u&jCtN{R=2WEWf?;|ii>*Qws@^yUb<+OL0a5W^LDL5&!9Yt&w{$Bxc%=z_~f|uXNN*5G1aqjnGPm8)FWaKs&
    zwtVEgh=n6$k
    zV4&CYeu}^T)lRxccj)bl6G&BcJ`MGMw?=IEFzaY;z=hVnmxM>p+VVv?{mRp9oibW#
    zPln$Oc@Pj&%A!^HsR5^%AkGLuACHyVK#tfnlZb5=7*AT&VRIyJfG|`0V{%-gwpxhV
    zAF^MpP>R%#jo#8jn%d0L6q0b{ekTsGAL?
    zLU)%Z*^iid4^^zM+81}hr4Bq42wL58Cs-X-jyMkMf^0O+nAI^;Zzsm
    z9!Yc*_)9(2RO2GZGa3)I503rE0zP|;1?EkEx`@+kjf#)hn5>?;ZFObJGF$v8wYZsZ
    zI6GMySSQn=#i+faBV7;S=I6o9Y(B5^5iY1nwQ()PR1y>ffn?L=y9l&Nn1=NL!aoF#
    ziD{tW5li?f0oGSN-zK6XrXou6wJDy*E1h7AdA3_|1R?d}w
    zV42Rv17ifz7hjO@(tg!iVwr+$z;g&uS@E}K-nAfHe)P*&V+k+g?9bEN?|R}$aAkRk
    zz`MV@5AnIOCI*2B!f?!5CLQD=;*^tUO)?UIx7YVW&Ufnuq?TV~&_=rH;2ZRNbT;$x
    zz_fE;=xzE7W9z-Zslwff@xsbe`O($v-9${k<=m(qchSP>{Mwo6&)On!AP;_R;;1%GKm%k8nW|Lkg!_J#j$5K$0Vvs3G177wYH
    zA@QmtpyLZ>G5p4NQ4>7AVs`kN$KhBj|rG|hq2_q*><8$U2uvxW2#SVK~`l&D@;
    z`9TIv@}tz!N7m=Cu4If}?Q>4O+v=yO$EPXcH8jJf*o*pXI(7gh=;)RGbD=5~rM?!d
    z_5(mWjRdd+)92YcHEO$KpFjL&!O1lmxQp?Gtk4*7B!7tDB#Fy@`CV+X)pclq17
    ztQGKi029a57KtK?GX$MLxz*|PCPsy45>Kd$n*^>~`WIQGqWq^mB<0E^BkgPB2l>gm
    z=hOGEKT~>vg;!4t@MvSUQqQn`SkE#2!EdVfiHU(go!y-x)Hb7TsWuky&!7Tf3D7xZ
    zaA%12+SDO^Rn*tT*nco^90ycjj?D172*zYR6vQbVgw|$M=X*DRJoYkE617Z3L}W)W
    zPg;QzZGE!A$%NsK7jXOMprnEO$5X&<`3=_M8&J_;u(Ui;7V+zk$9
    zYe;714-a|G4(Ttk0Q-ExN-_(tE|e(eJ-xNJfcO!mI=B=$3@$Am1%7Iy54`~T=r+Y+
    zwfP25X2=C9Fqhap5zeb$WwHw}1F$kMg?RZ6BBdlwsMYd(FknS=YkP5@MF=c{Q~9=i
    z%0J4;T?-tlI+_2Z_$%|B_
    zbw#U8NNon`0lN9=kN+ooswo3SN+Y@gf&@f{?Lgwc{ZCFkn;PnF
    z`I3p2KdI2cDEk?Nn)IJOzAp|Qz_#lz;>GdKoz@GekpQCZR2nSGaei+DGwIS>*#c>D
    z3mA1{Ok|NASwIY2vl_X~4crE_Vx+Wd8z4X&l2c~C+f5EUxU@y-q1hT4uIE8yd*kj;
    zS}ZRg@Q%VPOi&J68OSD(=Ic3r;keykuh(J=q3QtUzYsecq^E&v?K3V%yQ|awMwj=c
    z;p_JhQn<$q*vtc4msdw2C%;Y?@9%%k9RzNNIvx*8KRcLyo+OdFD7CfPC@)E)DTIVb
    zB7j9g@C1Hq#|EXrToP-ovBxxv01D*G)SCVnypgbr%qkjlWI^s!<&Dp6)wsXq5@`qW
    z>S7#!qE(SE{e}?ncS~QsP{6weHaYC7|Nb
    z5pcGX!jbPHxf}(XKBt%WP7!xT14t`VHG!5{0--gi@g
    zNbk`HSan3x^Wy-9^K?#>G#P!a0k}WgfT#8_f7>LUGkdG|V2nQHaXKe}%k<1$(E@Tq
    zr0;M=N=7rhex_&jZ*(2HjVNnc+HCW=b%CD9wIqo#v!gub$v|*bzWu|lG*hlX9|k;d
    z^)t6C{quP@uo93b&1P}
    z8j_rOaB2Lzc(?_F^(ap+zuYjY4-{Q1$^9uxjTC|}*l4`~jrlvdfZs>=Q>_`F@f3@(
    zJAk238P}Vy5PN-AIY4KWwR0%!O16=ElUyDi6+NdS-Nreww~Fwy!8jM~d~$R{2l=~7;RPUBtPgKxTBy<%vT!T^rCCOWq-NcIc-$*g}Gu$Kv#*(_Q}Y*z_(lxXbwe8}^gJY!nSi`O+*2@IUEND{L^F0|(z-;2VE@DzoA+K#u_Kg?6k
    zP3H-R#S2Iifndvj!bE4Jybg?I=bTY!(W432>hwBax2T>}M_ve`9ZsKDJk%kWu#SYw
    zl_ZVqh!Mzb!sfP191>s1{=P5)9>|A%2QOspj{<_T%(UEb4cTXs*y
    zMAxMD3i#(wFv=XEAGFZe9ohK>*Zxn5pkrQq)wE$=*Y*)h92sk=4(}A#*O_Tb+$pBI
    z*~g9iEfE*u6nwb>@YBW?M_Y
    zyv#ceID+cvM3~LJ20KvpZxHq_Q%>lPD!)&>oi-^lmJ|ABFn1wPYAjbwzxbagc!=cG
    z;(|}lB>+;odXh|cXq$)9Y4dinm1=(yX{WiX`BatxLsS5tB|bhp)2!NjQ;1GSlj~Um
    zIZQz^-U<_5g=N}wrzj6{QZ;Idg%+Cn|LoEB+F7uW&wMUXYM^5ri77)ghv>8ye>pa3
    zM5x)GK6nDwpw}}3^N49mUExjd=Y$(M5U0Lzd!0k9JP`va;o`_kteyAq)=lfXjL>=3
    z@8wjZg0~z0)iAbUvEARr+A3o&=z~Oiuh!18EY4PKCKW`wUVlh|c_z#{r^0$)v!8YF
    zvax!GIUQjuYCG#xf(?d;)Owcm(i#qa>I=>;kE;MAR6%m=Xk{|-p^(xL-!emo+
    zLPve=6Hq*uT{ME<6gkN3K<^>)b0bA)ec%vc{L&*l>ZGiUo*bmRIt?xFKAi8(diZ?;
    z^7=OVmk2ZCh-pcQ;LgQd7H_{`+3CpgRc=bE@fik9aw;@NI@t8?Go^y>d_v!LL
    zggtG^2id9PWfcW)*w&>C&F(J|Sw9JU|7GER0Ad;`{K{Y2ujyQA9}2BADrv?IHlYw;3MV50bDUwI1L(%86{0)$cV
    zMjn0J3p$nt$mw`s=}7LmM9(Lvii`2)yc|eGVdW|4GD{x7QvKIkr#Jnt9uZDJX~h0L(fpH=$3P}!N8wX9)z4Ij+`(%19VA_)=N{qH;JWz19`D7YZ8=_yM-QWx*m)l8!4VIgK|d;0zO89m4O+OlXv0U<
    z_1`Lu)+@0Jizhkan24qbGX$*Z7^&OlD+R^CaGwi?i89g&V)ZuY6NETlPFLUFoZdhE
    zwF5Wgwr8j98!Cu>X%7zfN*FQ8iWv5LHLNAZQHq36=21o&3Sc*eV@lZb#+pJMsWT?A
    z2psFnTdC4FJdzz)SY+64n~a2!?;5j3TUg`r~sK$me*Mb0TAdV>P}|E}t_B
    zaudOFk_1l2zjxwq4C1pgGX1o!I5gSgDj?nHAzSe>*FLsC%Bx<+b;UQ<8KOwXAJLbY#E`*LOOZ3rI|F#)fARuJ*j=G+ivE=#o
    zPI|i*CvGTLx1Ri%j=KM&rU5nUpJv132N|Y|knuw`T`&Tq2`fqV%rrDl{8D(c1A8(e
    z9EOzy8=`$P%v!a!O1n`ayCPnRvUJk2psJT7|F>3zh5C{kmJbQ5YxhB^F&5WHpsX(X
    zRNU!TvJH0ZRsoN+X&D}m3?G0JsiBiEXGprT?I#36MzBbjQ#4-plY2RJYX4{}l
    zkZZrjjqSd}5tDgvx3b;Y04;*C-(G@LpO%X6i(p4XNwsBL24YzHU_!#*ZF&=?vk|Vk
    zI0rl-WYTZa87>TH!u~*Ow7^1Q*vy?8gSEsjTZ?O;=Hq-~&APDi6plo%cq-qWCVOQp@w#`Z5nl2q5fU%`I&!YWMB)
    zAJ|aXvl@?kvkqJaBby+eu{BYb7ZWGw5Yn=cRy#e3w
    zc2nzL5i!Q5n3s!3k3r6-hJ8aPdgWTavqroG7L)cw|iMBRZnfd-fX>y@dO8F#EJs&Xnma=@&EJDo_$m3P
    zpO83E+FvC<)m@Ch8s|2BThk2t)&JwBKIuI=h5b{`_kNI6swU-c*F3tpWvA8O&!&mI
    zN^c|QCGah}OnHFwHVySmG`Ih{vf33^0p4Fu9#}p4YhV}maM2q^*)G7mB>edTGC>Se
    zH9|sh{)v8##@9^lI6w*qy5kcUnj}plnRTL*$Dqk`o1vw5sDXnTw)gTFnlGy5^Z$Zl
    zPUvcn;h_x@FZF9wVM2krU>HoQS$(|3bwvzU9rH3WH7IPCHf~gkMpz
    zZ`rgwAFL2fg@~?tAd}f@+*S$D_WJivAQ~vn0OSX2KvFmhp&tbQ%d=rttR)C#QX`3J
    zS`;5HuE>-2X3a(+M9BdlL{G$!bQYV#{n>!??h6~wna-~u_FiU>g=BpPDb0jS+Gog-)MP_s
    zQ_u6BnN$Hib+S?>w)4WVQDKgoH~VD2tFKvd=iTLxIXlqe8h`a7iXvgY-Dv7e3p{HW
    zF4{0@%#>|IzZ;&Uc;agKSIQt1$$b`B!ghi^S@WT(K-QL1BMevMbHHk3xhZ7DTAo;a
    zd`I~ugSD$83W03~^Xe$M7!24PfRL(kzq_&Ewd}Y$|BAQxy+dIvnH~`K2SXJo-ehj~
    zI&n1dxTT_G8vt`LyYKX;-@?-WGg{OC^Bc0TS@IQ)yQgQB#j$RLv9*s0(R=S`p%jKJ
    zr1^@I1!AIK_z?vo^D`+#k
    za}abqv81zJ%x;5b;{9InB9NhQa~cX!{?kcDp_I_vT_4QEs>>EWgGEVv@+=_
    z&*45j08SI&*_&6EB|x>VAL#uX`V3e{dDl5B^g-Cw5oWZ@utmO>h$f~rejeWSZp%Oc
    zxx|g?4cq{Tvab%_P2;vaEk_BQjLkyg9UN9+1muj<3`vA%(UxI%gjIzhZ-0IPqZ?W&
    z$+gF9Q(AkWis?eETAnHLl5)3QvJa#!$U1qk1?Q672*_q#6=b4*75mJl`0@To&z4T<
    zIu5l<7(k(aFhn$0OMa+X8#$!R|KOX63D=>>0tY#Z#;quj5;)KU3A@8BiKI4pGVdx6
    z&eC6Q+B{&};(gyq34nPH1S(Ve&u<6;QSjB^t}HLMoTn;^ND4C?^F^Dv0o;ol=z3sJ
    zqAMSdIOG6GF9x2z>5!d~S^EW<@L}SpB*Mu9s+YYd7Bqi`sZOTj08T-II5mst?cEwl
    z6BXj8^ed4)3u>q`7#uoJNP*ZaeWekjU=K(iX;%a!_Syd|0|AXb!d1PmH73{Pk&f3h
    z2yp~nqA)1y8HAo&L9%)+(1iaLi`0{3a`s-*{C38oLpBJ{fh-W+ff{0)V-9Las0=7T
    z_Hq0_+-07@im__**n+0{m>ZKt7NmgnjG3{HSYhJ^Q6Hf=B3_D(=20)P$!iJMzkxGi
    ze2Pv1SIMWrzc%)qm?3kiZ&&J&((KUbI;!7ceCr!&M_gE#DY*{a=6vt3lw8+H*BA4Rz)EP%am9!piw2Qlk!eOOImv=&D
    zyZeaQdo>3A3
    zH!-OFqfB2=QBnY%Ir6+r8{uhtgsF8A{*e!IALV@(2L#b?3Xs)@*OA_<2+K@40P(})
    zKn{L%MM&FDX~{1a1&-iMkyu-1XLSPfU#9fX`312XkobK))fH%FnH&BtgILscd`g$_
    z-GcU7FYWHzUWl=qHKbZtQBujRp}K9K`}$OXs_2m-&6m1YtYhdQc`8nr~P#7O(7iD+0
    zPJ$n0SY(hHo(hE;DSFl~)yf`NEWumX`O6y)DN0Bx7=f
    zLE=hb+mBzU0oQ}TOF5ToOo7>&Bn5pz(mD}V)s-FwzT#G($;X7pcubeX8s`qNu93q1
    zta~eR$#5wvxSR@y14voo=6%LtY6!j*m~bncrCk^Gd(Ur$o6iOe_`jmoGZAai4g(%`
    zW38$RKaGRxeeY3g
    z)}~!+-wrMxcW9o5KAg#I{%U$)3^-n#(=APR1k%q*!$yM}F9RAG>6TIn#kdGYT%i1g
    zfUqFvNn!NOFU~-e(CBK`00Kwbc&z9m(-U47>=}RRka(CLllrpnLC?YX_ofj8{#5tU
    zJ2;)6do<|lyxPDgp_*NqRk#PkCnR@J;-{DJMl>D+)FxM;qVfxF>1)SB;lqCGc46q`
    zpqOYLv(*LB48TPeuU@zARJFNw`Ca}C-&&!15WHJBxTCz=uD)lU_ndVY>bv_>A}%9w
    zl_a5HJCK_xN9pr{(q|R-deB<@mBcZPgjDYRX^HKDaXrBtXC6Ay8$ZNarGB0;C&u?|r6&XCLK-&;2B52xFLhCh<9~c9D^AHXDnH0pI
    z3e8T*@X$}I!CXIFX#08%pT^+;=Z2R2!BnX7SD^>PT7kiwQyHE8eALb$*{h~UoP>)z
    zhU8r-(ah7Flw#}xM}{NCrxTGM@kT0U2LO`w1|l12qxrmOdyXcN?E^(mSpTWOypKXr
    zn0t4~-kmU$gR(GiLVJ=sZ^e{})=(|47sw<+acO@8j1*E5qHT!Ww{iG6QJiJU=m5v!
    zQ-2maSD34%id&@m#qV0C7lJ&}#@V>P;o;(^`IXBDY)x}gIHHG;Y+;Y6&eI}}(P)#~
    zx7%mF?l*f)<0TSzwj1UgtgLE)vavbxqIIuvc2o6!Rdw@UI92g@iRyUn-E{6~BtAn#
    z#AfHsexXj+^EzC%vzqFbe^7=84aehwgj?$fb71F};{O_0j5
    zhJ^QGR?ijQ8L&boqtm8iN#22!8={q`(bY`Ob9_QGtwVP7P@~VFGVq{f)8$c2ISEfx
    ze73PY_pDu_p9Kvzn6C
    zz(S8#7`H%M>p_S_%`an?X{gO1K&>?Ssn4%_?PNxX3EUUa
    z-e)vnl+7gJhhM)E!e+x-emWzY_KtZlKA+*Iskk&_W{)p@pa-dj#^%?vkV#{4Rp1vq
    zeZ?!nCFSHjx;Am~G+hj4v>?#@IvQ%hr;-`Ai8Jd6%-(9@y4X@aka}8K){z67rRGF#
    z7za2Zd@}vO&}YM*3-%&19;@u5GdLwa+n{S6HT2&#rz(sl=D@=p&GuSQpwR7MU|`q^
    zIPe38TiK#MiQ*S6!$K4f>zPi)O`T_zyxav;O+0<&9|Oa2>>1wv5cXp;a5sHUSu5D-#aqo$9J8{xDSgM%RcNiQWe>8
    zq5cMO9XwZ({a;I8Jq)r+|gtmV+DaZ=a6H*
    z$6_5BiB3)Zcj4U
    z!PxI&UNlI98m^@ICCH*Y(^jSiTAXVjH^}<@igr*WtboLmwE^mubS2Xm0UST;J`vkG
    z^q#Z-z?t7rB$7&Miv9p<1!;r?ctH%L>_*lZ^N4~}oZ!RM(-0d(v4`pUzZ1}!Z_Vna
    zI&eXAMyYWk7UAud4m*!SN$NW}hg21#34u-R+mbV5MF5imuBOFd6hig%OqftUSdfX~
    zrPEshLnpZp-vv64fX7tpQ7>6TH7ONUiCuL^N<>8G{aH1tMEKd6hp^u{D@f^`DYAlK
    z;cuH7rX}844&{14Rz_Y1<3}Um7g8KiRw|p~qFR>W+Ge<-+u$RTB$lvXpw#^FWuzA6
    z?1j6$OG;qFj1lT&XL=-Fa|8Vo2dn=T{jCdo1K;k}(^iGPPYlc9o4vXK4?ETFcCJWi8Vn_3+@|n<?LuC7JF^lC2A$R>buvD{6VBZ
    zqya!hBeRmva=rngA-W9B&$il9xYqG
    zsJ;C2u9SbmgO=M1A(#txb4;HW@h{wt%MAv!3euTmhZPcc$NKS=+Lqi?h5-Y*)B`JX
    z%64nJ9|`TKw|ln6OPcNj4wqi8*rDS6E-x?tT2mugUtj-n$jrkddEq4I;wkRv?k;P`
    zQLK}@crQ$>;#-T`hQDehud}D#)lS_Ph`JS`3dPdV`1+Y_idp&ulpRk0f>n#g5X|-j
    zfa>*F?}p`fjQ^sb%ku;AQtPHWO(vRzgiWa;$GSL-0AHZHV1xNvFsK=*iZNR
    zIMxHHfUyC!6*k-dufc^rg64QCHR0tpt@O!z`Uf*$eBpv4X+mm~g2shZSpl(6Y5>St
    zv8kekDFidvRC6Dtqf|O8A-vr4L0lZW4U$hDXd)U3JqMi{nQ^MOTQ2V!!%S!3ztnBQLcYaGp
    zaJ+lkv{MSQXov);&4}ue!n$m(0Lmwb&;XPGf`
    z*XQXGEwlZ;gh9z8<~0exGIl}GfeG@8%fh2ju7n-1q%}=}iPPWapBMg~{MoV4LE1b4
    zUc8YU^q}?>aS$G-dFEm)Lh3@qy_9|6&u}*t
    ze@8MOa4~-xLUjv!qocF+*eb^d-L9ODrFMP_+?x#iz0;g~{ikH}_!mb>Q`g@X)u@HA^sbzYy+mw0
    zpFZep_3s2ijUvhn>T6K@Tef>an4C7)fn~-S(9>u%G1Y?GFNXd!8)FYjKA=QXK`tp*SeLCEtehs%Y0#g2l!Y
    zvu|O+?1A6JUTXq_#{93%@6Op*u}n`c)OcWk=B)|yTTd>Qm-M=7u7!5!wx9xEe_tA`
    z5d04QA}&WFOGbnIO4a_Aasp@R`vSf%lTVh)s>~4viYI#>ud?0-W)RyKGJraT+}t%Q
    zMrc)CKJaVc_}4x#zkm0OpHS^ZKKjbue>nH98m$w~QvyJUzUxM2EqWh|ceux{jS3-$-=N}-&naVF=(pMa(B%hf-
    zQ^AFEFje^sshJC~Y0B5Tgo^^Gw^7A=5?zvuZ9A|`9O>Qq)%R*m-^?d|sd4(A8c7gx((K^IMu0zk|mtW1eQt>`}36@*{l(2t3I%;g@U3k2%jd%mP;zAI=`=nA4#Z9}k

    1v)ml$uMB)@VR`db>(G2V%EQ4DvO>MX2HhJ8_%J#0hpit|bA4IP z7HbJlz_Hb>kq(h_Q+HfJ(ytD|{iO;%LNr=XDS~3}Qd3L^8tI|l))g(+)ZDX!->*@t zs?#<#%q%-{S#)D!-I@2Q`i7+rzSUtCoQy!@@XXMZ7p@6PRqQNJA#rG-6?4bOCU0;3^H07!n?q~{WdDyo zkixT}H&8*fBsbtV%A)n6u?TqRcydx!-}^NsvG8&YZHN!Q6h`uak5dF;ovX#$}H zYqv8lN{uZp2T#~Ub|%}>Cq>B{6mfm@&8eSL@0`1G#4o1fGLxU9bpxYc&ZDc@BU6ft z;9i8?JlXi7y7#PXpQ>21p`xL+ahXV`e@bu8xan5@y9~*NTt8S@TSrDzC{I5YEpQ|J z)z%@@NW9#&mQa$o8<$Y&Fpkdw*ZCN6=OOY;hhNjh(M*JJMm-=8P|!e)f)XTA?Je|q zFClK|HUcPzG}b5f4}h1>;k@`kdG?wF4u+V^i`oQi0~WO*H?>23Qzg2}GK<5pGcKJ#WzwD$Sz1>!ZUY0v)!+xaEmcS*OyqmfOMAN1sRa|T$>fnW~`LILZALEsB6^eC9P9oBif z#xV{cDeXPG$3jfDc!`3GYveWWv5${Gv3MgV8~*BQ{0*%&S9s#?v--dTa^~xpBrzu)}1Aer&=tp9|?~K;Fj)MJ)A>FU0~Gai7XK)?XnqV;=l%>!DJ*y)ZHI1-?7o4oUCOOmITJf(`PlK zj^Pu2jM=LbE~QDZ^W=#?$h1Hz;8>crvV$!?i|Yz<=5Z`5=oN3I|6tB3D^B48cysI_ zHG;u42NZ$FmjlFubBO@$XiP}`MY%;gvD%JNtP=EhQsQm1%T8s}SZdd;rSE2b_+~dM z)#sh+uAGkb{kDVR{SOJiIfu*QU)VNv4L5f5oe6W%2J=DIxNI|?Q>W-TEzT*TxDBpj6h4{_p zki7lHwAmiz zP}KdsdE#@--up9^x6hWt(vLiQV{m_e&s;byW+UCaXx|X~6W(A2p8Uei!Ue#>YR&sL zwX1!TPpthOCnseK-ChG8$S*2xh)U|Uu#-76Y zPngUzf|VJ4w02DSHgm^@ z*ue|xg6U~J-xadOP5+pB>*03~B3}E3HE#1iM?A*OUNo?of=&k6bk~9(xpIolM$s76 z99Y@d)FZjKM1QG+*eeL{w7*E10KP8T8w4Ry=dXT{dj$+pTbP1uDx+P)Ue<-oY-|uL z^M%v`*+^1O8lx|wx9`7JPh zb+TdW=ooi@i1Y9fn@U_66BBdbAtfbciYx4i;K3VQf%J(7vm{vWovaC{MW{;3kF^Nw zVrdYkmQ#ftul!EIzM@g|l~Jw8I^HfN91fXS)mD4vYuV99mlq%g`!^--SHfKp3*JL; zpAO%lyzYc#jg!jp7Xaw9>V;gA8JKshLhh{#IE^by!$;eqHUy1d$m z0optzD;Jd*Pg~coZQ7!nXPT|et@&C`dI;067y15tBFK2BHcI_g@UY6OBl>lYs6y8f zS=Y;pIX9enw_&lCE-zt<){`NTt%mA=%9C0N&~A@>(JTQo$hT0;H_2rzRn5WRr?zf| z9Hb~f*$^A4QIPb)Gu%J{Oz8CuZrFwCXecmIG?})4e-38n${ND=+ZqQhmXK;R%Pa=j zHX+jE92!qe-ya;zYl`c7HTT@4wesiE^lJIpi|u?cp--XpS%k! zW36kwHrWC|Jx#GnWQ0H1dO7=2wJjuIc|sM8kmX zk{pI1t}GFegvuFckGsa-uujw2NuZ>GO~u97}AC~9>aEl`f!$i>unSWryt84 zp!THSZ{1;UT}v_8?4i2n6gz|?x~~tC3Og@BIzW3+;?;SQeOf@=QM#{5{_M{`sJ-H@ zHdpu?^el-R&>OD%J23#`t3Or07B7Z-fgGr206qcYdNqC%;h2K0bNY`RFs%I3zxu)u znwzU~(QFgaE`T`IugD&x;(Kq|w)NOgC8@HFlw0aA8?AhTBs^rB=6ZZDM%BBpOxTbdRCVd~^H;10+?#e~Ds)6*-1 zI5_?fOR@KjBTzNXhBcWn6k>@q~-;CWBzVe6g;p>%R&hMZ^IN|1cmYtx7 zzNN4}sliiNp3P*|t3ak??{6_g;%YWWe#{A+_4{qii3|^ux3;!^6W|oRmsNHnD{4RY zK1;LAuljN+4E6NiR+N*?-qltkR6wq z^C{SdEQ0MXFr+^Rms;{)As&#PsrWlJ=fLBKn}%I7g(afZm210Spmx5f-})#m-r`x( z_se8l39t* zIESy}3!e!24%h_TA+(VYHQHgyN+4b3lTa=RE_NRg2gdZFB?X2$>$FrXX6m5{;*gK{e^FA+9zl``cZra=BnGM?j+C`5}WzZGukIsNk8ggc`1!{_23+8 z+PeGmWv4I{L?ieTu)9uM(-N1MFreV<{GdVc^=oxqXT;C0L8RD;=J8VExj7K1cYz)h zR=@d~{l3JU3DG=s{@@(KGrFUsb+fflfP|phhhF6&%!qV z1py*BJacMwlx%X}!ZhsJjnp+yde#{FrOU(2&D;5PZ*`-u2x;K;)tz^YjBpiBZ5cQ= zd?m6Qu)pR3JxXUWXu>KkH1NoLzK$Xw9IIMM@>IL5*~w?PTPbrb^3{Ze@4|eUlr_z) z^fs7e#b>IL620z%TK*e$|EKel&LMxK{AM3fzSE7t(ClPpYaMI z79|1W_5K{mg6$PfxQi7!`v3{7<)?>h37-2DL+t_E2Ho@SqC?) z^+Dc~oGL#}-TN}+rSCCXi*xT|8PWiR%X=fgkwVya*oAsHC{5-i*^_PmdJ0yFOwZ(;PvHe3lNPyxQM_kZT}&}e4@F#FN+@xguFsL&6bMEtC*B| z{hCx{_om1p?qmDVI3RsXi;I`&T>xGy=ss#!uYXxF>A{+poBOvXMQUl%b0h7~*EwBD zvb~pOM>ZuycA*T1!zocAuO1F|{x5=L=tk)b&sep9{WKA#Fs{iKAvSeTVGe)jCfOCvgR zrSIItW>e1n3`6(8m?#^&&&k^}Ozd#CD+{;d#E7=Y!KiGkYfEDpaEsedS&foQUQ8Xo z``)!Os$Kz8hXk|w z#_-RJPk8pKk2ll#>-_$sP!>ft+)4M|XO$EezXWWLC#Y$|MMk^X>UHf$JoQIiw!Jxe zdn{TyI;kItNPFuJCS_ZKXj#41!4kcLtB8TgHFi(UcK?J&y)1G1eU*ZD@|XEaA%ZAC zLgZfijbsG1uQWfu%?(O;CB!PuJBTif`S$omgm3B62AGQlaJr~kkfUWI7D7hxqhVI8 z$!ba%ai-r@CO`3aGpDMSrXw!%)p~ruY(wxKC*runZ~gW22z;eEX9^r&!YEpBFU0<+ zJ_c|%{8mNoy3+16>T0A0557~>y;tY=xO!orctS$C@`msSbSLU%XLq`ea?ec8)dWLg zv#yop(6`tRO%wMBH`}!+_uXgt+c4e9IL+c6lfR#v;uK5V&ACB*C+*sj`v{x1d z71J7X%V~n3uEYKRGL7h+5A&BqjVKpxgKOzFdLS}-q@OYSlT5WTkyUMJmMEiivNsqj zHbh!FFW!#`0HcLUx}v~^Q7&B#F!!viHw0hKzvDR~m5>U6Ho`$FcLVD-k}r?csG z>2TEGGd(_y(J4Yp8}7TaalgD!ypbZBxCEP_S+l>>LX%R!+irKE%aBxN5fcETY0<`S z*<8DY_fL1jgW}Q#;xohV^Krdufvy@zz7D<{PS2ed;$3?!^-r`GjC`lb6O&Xj6u{p5 ze2rp}k;C0@YfoLro&fyK-Xq6{?VqzDKK_w)spWQWp51}3SgSLy?JS7}qe_oFgP&>* znN?uOfe6T2lfIR#V8ocjPJqM9g%EFV)wF+st7R2a3KPO?gh6Pm#&mE!_I+Zh>b@x!Y@AyT%)oZ(sk{jzu2PhyF6FF-k?n`D&l(XvHQ3VcoQ-T zyyjS)#;hv_y#2{|U*3@eGoHM>rkvInzADsEpsH@ix^Mk!vjkJu^EwpH4j$5B5cq#n z{0(!|LE;BHh$&CmFn%rc4y3)MyHb8KfOZ%hYx&7Qr{k^6A>TI&1lhK@rK_Qb(chlu zyt|}f8yr`hZYh5z{nL6G*HhclM ze(^HS6=4h_P^R42dxV@j(2Z8ew5x;;=mHGK&IkM;El5|@-@5mU{=~4wOPGcQ&)2{x zYZhp%S&e-C)fkEeG2EDox4tx=@q2!Oi=T;o6}`;|l*~)^|J%PYM*QY3{wX=ILreNR z0nH+5o~^h(Y2QMteXEab?XyLQttae|@;0i$xBVlmldl+X#+KB}$ggdQT6cLT<~444 zEqyppXai{m%UD|#_2I!2+)2V$Z2QA{|7=fSb^HVC+ZiD)3<(^5{eyup3>9?1^r7H0 z)0ajNl3Vct$Uff&qka@JI{F_D$i62r(BUAuvcx6WyH@zx7yO3b%*4R_)3;Lvv{oD#su<;4wLGkV*uba(F-c84P5P;z;Rr!8}YaK=k zO|&IAnAZzc=qb43>-i$<_X{vq$G%n~^KSwb@5AJGZ=Gj;DtV9DG2TYiF(m1x#ir@X z-`ov07xt+TtAOAdDvNm?L$V-zdFntIe^4T|r=I{<=|uvUGh!|F{qvqnV|=B-pFy%# zjLa*L*FE*!!miGBf;VGYs*MGDS9gIcKExhOl_xN08l}6;OI;%>ewzKJdg)e|&0ffi|OS`QLMRnRyZkyLmmorBFV85IO7SzAt*!A>)+!9F<&IpcwIUnU8Ni6p-Vbn%``rJc2KtMz^l=-iUsFY`StvNjSmotV#JhtZO471#_ckdHZ#HrQZBkg4$uPUAV z<4r0{&Nz~%gTNA3Tld*Di`gucS0N!NUe)~9pr0|sOf(qQt|N04Po;YM6`gAqh0-ez zNbv;~kCwi(U+t1vQxxO9Z#;uHw+c^MIgoe{t!mI6bvo7P#B3m@DQ(qYnu! z4P5&IBr0-`T5~7BYHuj`k{tZgA&zhrB8v0yBoH+aY=%WXFP4e}b~tye<(hNAa@nj*5b zk5sX?ag}P?u-@5^&x#4qu2`rjz0hbrUhV!f^>cu2HEXEp-2U`|xbV$9M1sgiKu6i`*z!hFQ@D-^RmmzG!=~1RkvOmpyBY9M$6T>Qry^ z$8+Om%fbBgS;t`B7uazEO?6US?siXKQI1#|z7s9{`!XFS;@b5#AJrM5bP$e9v&v+$ z9ga$%8xTP;--2ws{{K64Tq&Z#M?EU0SYh4~nL*)I6H+ zS`84?rR^E$;WNGa_R*>W6(&vjNoZ7eOF3_TSC^`Eq3mBN{H_z!4c}7gbkG(%Ze+m*~?32smf5fsv}{f$9sA*`QLg7DOY{P_Uw;=A}6B>~8( zRcxiL!Ak4FN`+N$no<=kzZ&1kh+mm^<$&}VmVe6mw6wu=aN)vuv~E^YN_)=BM#Ye) z&p5wJC5;7&>mjOC0#1Rh)!daLxjHa`t*__>gM6K)agFL8gl&B*8?+FLUSbOS6;q2f zDI?Bd91+bRL!(Q0tf|9cal<5gjb%(4V=W*8;e`vc-iq}C ztOg%K;fGZOZL@Dop#m*!U#Rl=E9=$w*nrnhQagc~>6Mu#|7pctM~(FN(&2r11(xNu z^T0PtX>o_za!e&GNBsVrnUDQr!6lv0d!{TPB6C10z$@!1EdByao^3idZtKkI7Xqnu zR+civi?o5)xKjha8bfgM5MDwK!^`_Zv%an>$#D*JtaglWOqTY2@j+G^}qwHN0Xqe@(n-e6eKO?hzaWtdTRLc|@Fhg&tSP z!)eATfXT6RD;$*{+o*8gk=N1{$z4;KKmLz?i8!likr_C)R&P?IB()LykP5m@FEKy>8LN`y)p`>}8);f(+ z_r24CU6pB5^f3>LoH2>qD67p1OZng7Eguqi1%r&zvgv%;LmjNk%`NGD9JbIx_Wfkp zL&x-_B(9SpEX}QtPY*Oam4kcQ;x?!A-3dxto?`+6s?I?Sd1G;_u((9t<6OriHJ~8K zimM|bp{Y!@PE9{8? z`9bBB<6DhK<=zW+^JT_kLB^1qAr(e9j2L3}gcMV~vCn|W%z|?mo}R4SKkN1{3%34l z;@R1jE2n;TyyTlw3tmY~bQU!S`L8-*HP_6v`FYZan&!N`yxZ62JHLJ96ID8PUo#pr zw7M(RAm);gGjN6lpS)kJX8Vt^Qu0@coY5Lr(7Nt&W`tlm6z35<+_?9}c%Yk5M*H$9gQbDago4#m-zH7|70)C^I-`Sy*HCY$4C)an+R&8}a> z&*ddzX6)V_n_|@pW^^?PPI4c&9?$zIl4F@l{U)=eqmLfmi#@B_@#?y5+i@^owQR6u zd)w8F7h#fQxeI$5)q#kaYum|>DP~dPO9qKmW!NLht2s5CFBE;ZKWOY{SdGNQuWc=` zQt%&ryO&azcUyK|eLcH62h07p?Y+A$4ELP*5&6iknEm6mwC8vjs)LvW5**l*9&n7& zxGi@YYoALBH7NCY2bpXe%}*j^!k)r)5zTU_gv@iD>;br&Q8NDSxM>6m+YTi(hqkt5 zvr~g2KlbH|(Joye6|HrcR;{#;>D@9_R?fZRW*Z51)~N(G*u)3mON|6>ugR!Ebp$Ey zG*hI)Vscm{CQv(v)D8qPm0gI1vBD+*oHur^Cd}2Bb0*w;ZBpl>H|ny-h-cpx7)ADk zn$*F~x**Mk&`$LHMjq4ANcn!zswjq<=qt;hx@osrH=-=<60+EKRNHLG>8ym#FnxQA zcaJHzrbOg?CSYbJ%C#W)jG{AN(?5A&wZXvU^!%Uvqp7LA)^q_c`;F*e^qU`iM?WJY z>Yjzr#F9bSgZ0r)ngPze7Ja>|U3{+=lmTxRftt>;;ck;f^L@Q%b=%Ll)wDvLrR z2%5oh6tcqFG!%2FlN|v&)c^v1h9(t;>Q{gIJ~SCi0x{!}gt5I+2P0eYpK-o0*JR)k z8oO45iKE$I$kCiF!gu0+T*nC9QVu>s|L8CxM_P-M@dNh~`5l!+sTwEv!hICdCn6&x zqV7;mns9G)DQDdVhqOPY`LCV3&Q9NKdaLBwc^@98D}f&gmXIg~K;2p3)Xr09Qu&on zWfAFHkWMa=nKm)Ger9$Jw3Rn~AomNDRLYiO-1hIG z^bIL(@Z`lrcHSVwza3H_t_H=AzHENBi{uv5g4Ybl*N7PTI6S#Fbn+tFxxpM9@Omkr z1LgG@3e%;O_O$txVVF4}^qhmdIoV9H774z5-0E8IYW)1{EAE>=#@TqaoFM^y5!8a5 z(M5YU39d2u1^2agy_Xd<@*$BWDl6iy*JF!|sc*|GNM}I5olS)?Ch;Q)P+?RUa*$F4 zj#BsQcmtPr#w&Ha^-NC_o3Cyvqytm&Yvl9+BBxzG){lX?|B< z_C}0EXEWC!X1FH+A4;@WG9zGdZ0M0v(8YjjHagETmto^0!`%RPzW0;qg0S!C$0el| z8bV<>VN*XwRxJiGm4|fnL~46vc}Dp$1i3iWrZU6EyjVzEHf#40DZL9f)Mv`wcb zu=~mbCyu3UgqzJOb4*=uc*MvD?-J@AQRx~&eowtf=e5=jFVUl#0%myS03OBz;-7Q_ z@Ux&D!h#_uotM$n?M|f;hP!k8*5fxrdf~&DrQ@r3gosBqi^rDrWIfNIP5XJ_1f$@) zOy-ZAf_`0WFp>AG(_fGfQqJezQYNFfJt*nl+`L}Ql(A9_?Nw?CwJ^oQ;E z%i5PT$na%L_Qr9`6kLOlb0Bo1&$wVw7t~D~8F3{0q8yTNY`pQ}tmxNI{{xN7{`%wx zW)`LCY^frGjFE!#>z9?O zCecC@&!#o-FZx&umx7P&QWlYXL<-Q`A< zif`fyAG56Z0s#6e;AADVYn*Z1?-`m~d-Vr_endB}p}A(zGd{YqznY-6;(t{RowgI^ zjp7eJiuMaPSGW;bLb%_CbMT%^W8F}(#V}Z+2KlZ7-YnR+wLoW7-*uDUFdO{zf=;qP zC02Ol;0>2%+hi(?kjsr0p@P~c>(s-~w@Z|tOefVX@a-O&$+sPtgWrMsgg*u*6kpu# ztl&!AH;dhTuXNU0Ze?rb*E9P&?$%$->$I--{^RPxR2Vn59<0GUgY2&2=h-PEM;VmW z7`8oepzne|`S~N2iz%-xFgE~H_81uMHT_`G2lBEHPDXoI=w#~+U3zG}Ku z`+IM?AR7l12M=6;`F#thRy(GQC1A9IOJIII&x8f=MC5am@jM@8!K?Z+*YXi#P8bS0 zs?vWcO2ygIrh6Jwj8(+Uf-c{_x!Jukf!qr8UZnKf*WBVvl&OZNzAmr2A()%5=`BzZzc$!EVl1HFi{uiz(5`6e zKHoG0|2fTUD(V*C`}S4`0V;Lk&o+Xcg!TC~e=G}yzyy_5Tl~Ki*{{6SB)Z!N6Pawc z4GzNi=wl`fug%SbbV77;_?g?344GGo=}E9J<=F3w1`02xRK)QRTZmGdR`wb90fkPpb7E0ZsU?Qms=KNNg^^b?uZF_NbAQP6n~`luqw@8X1TYtFwh*gdlA^Gmz%gwf^%;p z!Lg_Cps2O+j%WWI&{a)7jqbbCtyg5H+)cDvFO#M`5NGNNN_abxc(L6`<{HE-RP8Pxsn_pCyN&$wY|$6HKna|EI%D=pdZv?y(&piRt8QK04eIoj-+^RtnF9;1#f(W| zJN>y_HnF^Zv!BoIQGqhznVY*}oN$XWn`I;5>M-NS2C#gx=dtcXnmNU>WhKP2m$MK@#0~4n5p9Qs!GM`o#(ggPvF!upXNLp$C*qG z;CTfpb*f6zsjc2kB`iE^+#|QnRDJ?V2l%9CTKk1O%zbIj}_6Ug}JV{h=CRUZg@qk2#T5$ zwZ3ZhosxI(77g1Mt?%-CR=6Y|@LSd%hjeYJ>sMz@ivqH_#l-fxRslZ?1BF+y%y;z& z)VjrFssO8Y4p|~%Myc#3FAMpDs+$AfNJ`*$_Znf}J($jfp?GU+E=tADKvQ`~g3n9O z)N~kw8IzDMrT#B*5 zMfpWWx;0^Ez6Ws`7aU7EpHZrHC8nyzEmrjRf=-Ca2i;!p<#$p!lYDAfHZPQY1i~dI zU@?2+?l+(z(Ep)wQM0MZ6PFElJiq%gl`4MQ#I#Vy(sxSg^7U^OI`*%uD9c!@Rs9dY z4$;sNt?|0b1cwZU)$$=hcLW+BEy*|PQe&n1j-hFbk_`*x48>(y_7AQ;&Yz7l#XeJi z8?9i}b+oO4R}E;kvMh61;2-ry&K>aS-F@Z4F*u+9h~Xm%7~&iDzOty;&C(Q7EH+fS zJf(A`jiHSpF~G^9K|<}^u@VcS;=VuP!zP6Pl9bJJ2_Ey5#(Bty+fLM% z81A$O?3ZV&k#zBa3ENzFn6~oKgg%jQAl6ga;U`Z^Q9alvvPy;yaAPC-M- zA>h{iL_}0eW!sT2r_>EVBXZfB877vzDZVBS4=i@ETX|!wQ;F`KoNyQYHcQ_SPbl~f zCfe$UdA4fAdxc$?!iu}}N=JL8l?&5+IJmBN<>t^bsZOv*&NzbzU*)60Ko+905j}r^ zOca(iBxSoH8yDimaHIB-FK1eSp0%N$YW^tP&H|fQUA<6K2N2qL-*zJxJAQ^O(SibE zV8W8W`#(NmOnM++FG_~e8S(uU=f9yYrX>W$vGoau&|x&s%*vVXmaQy>sRHiG77*kl zxVY;NC36tBYRGD)LVY`GCRV_5; z5b=*stC+(PbU-?dw3cMigFg`d6(i^j=1I8QGz=#lg-S3W#Tg$hG<)i<_By0s1e>p`h8={qlGt z6NulYq3 zItkgXGG0WwpXmK1{2 zfSTjl15o$FF6t34J_SIVxS;k##ccdth7y4l+8HK9gs+VOXu)=Q(@eY3eII(=wF|Pq z>7eA!To6HP(WVt_xnI?@d>1PZSjY}Y!K8ZVwWxq9nw+jg%JX)04JEDpi)yu0SWBH}ta6fA|yk59EWvhYRO zDKMhehoK{S!sVjbKaqWP$-ldPD>IVqeba6hzmTC30(fiLV>3*`I&EJ5*Gc9nuF@$I zqL420{H@Tc4|qG8pVwh=AhN>IcR$TfO{SsF_EpnCTf1iSMAHvmDJ%=AkS^#MkYK-4 z>H`>9;#S&*eHXfMHmju)2}(jJ;r>P78~v{qE-10JT1ywuz=QNMAuTZgUh5JUcpKH; z0xn~?{BacPkdhWCGzsnW9#G%oqvO|*1LFwXZ5%8;V~U`0X0EsG%Tg-5()l^HPIHb! zLWk{IX049Ii=$MSFnV#0^_(~X-s0dJT`DUgTHS;pf51p)hfYT#zN5ci71qn-_3d!{f&1${ zkMQ!9eF(*k7cxRRB$_)JSf&ea=5e?ikP+^I6@Q)>)=&aqL~pEfgr(X^S7N+*vKS-c% ztUe^LR;{ECxq;3y!t%|adE2xMzOhy;sFvN;>@i`VzX7|GGyV?%(KAR_zq&A$e9w3a z7s%%aPq-|*d8LBd{Y$b_q{?|kvFtgYY6Ox_3P`>)H*vR#YR^;SY__z#Lb z!=5vAR3c5Jgw-No!kA`e$Gg<_dQHbv}?aJ3fhY7?& z3fklr`^`(GJNYA;3(|-PR|{jvRleBQiO0C^VCU{PTL~b}eWlM~gNsxrI=yyoSQ38I z4bh$4(=2mG_B(GD{h}>0AA8&6y>sZxOyAD~=kAK_A@jtp(JirZ>iE=Vw1-h(rR#hh z-s&krSr$@#*nf|sI_}in1kI9>Q*Nqiw(Why1UM$%8}H-t^1t#+bOEt$HA&dY-u84g zccJkww)pcVURH#)L4}1;AXtu=FL`&Su0blnvmaD|^#?6P09rzum-Hp^C$O|6H9 zedRxgMW4}J!b1;SnIRM2TOAG&s|^9;ME4?}j(tFU+Ln+aWd=GS>{i)P)=B4go27;elNrpxPbmpxI0gdC$yn;!1@vZ%28wqzwhl|zZ2FqM3%kJHk+;OG06!b z8`?kdokn?+6S~i|2B>43(U4UC%U9ME8UiIDsW&!$&H6(FfePzb(g^uTvh=xV4iL4@*_*oN zf7&mvU2b46K!Wl}#`ZK6Uv4abIDQ`&VMG0Ij{PzV;>8hN+O1g; zmWAY$6Yy8q(X{tdxr)z&x)Rg}fV#Tm1X9cmM>tb0wl7FW%H+A?IserY*@~Pw2nMvf zeXHosbeG~(Lw<`1=yA60lnkc+`OT|o5rx8T5l?*xrG68d=F#V3_ge$7GP4VFPpd$; z-g8O(c=5*F6K80uZEc$l8mbQbAV4DWk61}>Kd8ZXDgg@c z6ziCaF#(D^&8-EH8Rv9yR$v~=an5*v%f>PLorszJxUq>JMp?r7r3RLc7xm_rFS;h3 zelTe!xZo7iJS%Jpgz2fz_|Z2;z>JTzFy3>JNCc`1b%)$zxZA=aRCv{)zdPbern%-r zs5V;?s?1ijjsJ+%GGPakPUY;rIspZ$M1MQ{a6(Ek;6h;Lu=>QvWbIkEBcWa1l=LKp zlv%=lx$^wcvd*Ipb;k2NCePlJm@e< zqP0ZUq3Gvj#Tlk9^PY|h#iX0;DR2xpHk6Ug9xqB35PcD^x@Z3Kd6!Gt?r-KftaTo{&4q_@I7%U02P}!)JXJr{9 z+izI0y!%E^T@q;_%2#QWim%o<13x>bQP62kT<-2(Uj3FQ2f*IeSGZLnLk@ScA~SAm~i<53Y*ao|K}lIK02ciWZq zo-d)@IpFQ;=vS%~2A8MuKhVs)3n%V>Rs{sb>n_=%{eapCdq4Mt&H#D6cU$1wBx7Eb zL*qw&)YJb9e?60*GU2E$gVpdRfXQ#w-$Tm=lU!>j+%o2{~<{z8~F6PG| zU^NKfClGN_0s=7a!z+VW_@!N$;TV-4u(-R6mitruB5(m;$F|aTAu@9qZm@=Key>g+ z|5}&`ik}I%y1_IQ&=1X>X#hTTf7Q~}nXnMz9MRgyzABegsu$*Fk0e9R+ZHx!r0yk9DBI?I&6v# z;o^@O#E^iyrl-ywzcbM{ajM(n$>8F^?CPQz<&LOpEmtQ%X%~^uiYxWU`gV1?M{2kt zJ!vYDicQ@%RU2E}e-YFf1ea$u+`5FPHpc_{AD3=~KV58N`eyD$(k8Jx{`X7))3MoH`pcy7frQCTqftepdGh#g*W4F68BX6g}_$ z>FB<_Lwh!T0p=k6`lKfqd{nbTq~jgJh*j7qPcxUs8!UN5Ems#U8J(PqlcdB-7V^ zzcpfWa9!2s2SQ}yHc>i2g#y;lyMM7o#@795K8ypAH($h+zfl&FktxYddH=qEtp~ogSzP$*X|#_IyF`E8{WrzlV#7 z?84qm<0|gJ@Cm^MM&OlPpa8+9zksPdNK)Lte-62t{MQ+ifbiuGs`)?OG*{E&z4m-i zVs`RD?l=-)nEEfnSC37d)lPl*t*P&2$q4>YoF}-AY!VW*Nqi!bPh4N*r@G63DX!=Q$jl|Skl*sN~+m(RL~^2 zCGeSqv(;DN@b&{vuDv}df1FB$p9th%(QT+Wjfc(iJBKi9`rG^&1OSnJ_$x`$V+(|V z#b>snR{&`EwnKyP${)oaZ1!vd5?P!NAO%8HivULnUY(K2jiE<^uJ>3$KzmC$g63j; zJt*5=?@?;RuJt})&@Lz>+BMmrgUVR#`n3i=KisQ#XXzm@_^4@YKc&Nc;PVe<%1ciSUEf0IHv|J#JB$RQ|?}{ zDbh#{32n&LO_hW5ifiyzdO7~?@MX0*>1pRpWmNfl!T%62-c>8F&C^mKq}>sLQ1+;n^wA0k0DcfyG&Egi)Apw6p>D`bE?hcRPNwD$oE$;BKDW@Jg&7 z*5@puyTMfV(oTS{39{--rw&&02dRa5>bL>De(Gh*3+R!=O1))R=WB!sl(7d*YE|)6 zGc!{GlNqoCtON~*uD%-l@ck_$p;QB&Hmcc(DpfH7RD3;$GQ^A87nO$-*YnQDD zSUak)lu7GU zPEKLu`Pjmu6bo2zFv84PwTg;;MAAz7q8}dxj#%{Rf&6*?!|f`U2)^~8Z^NJJU%JP)wBhjv;GCM~T7 z)}`6v*6~6Q!fJRnbQ{VZM#sg)$vOMQ19icCX0CQ)QkGi0GqS3Y^10BeTKo<01?d~~ z0BCqbH@5#8V=n2beYwNqb7mE{Yh?ib+J_=xr=LpojJ2~t&*&hfcLS;0838a4Z&;{lvbl-qdkwJ-b3pP7`c0yDN z7Mni!rBN+rfdP)G7_WE%rhBzt-JktQW2?r|))QK%(Q@@|X<3BwLTKRQf7*%>=$=r- zY^ZhfJ;t^F*6A|W9Oq7{tLa>G2T}gaLf!|d>a%=UuP8?_@Er^f~6$?(QrgGFA3*MWNqi_vkv*sVXnjwCFHQHWW z*d1+21e`?pn%=rp6bxsA8jrk&p?%yRD=hC~uz^%2;mo zD-P7oMr0c%lGc7`w7senvg0z2z{9?}YfwC8Ts*!5epQK>IpwVS8HJUDF_0zPwq-4Y z@hDvadIe~MUFLGZ&vw>X6BJkoeLgytjO;Hf1TCzx5}Azn3QV|cKROJI(U7>5Xz>!k z=(07Cq?-3Y39ZERHtOsklLRVI0B(Vk;s7MGG54RtbtMs#EyYTCz%6{JIMgw_-c@&fk}6|%0Z~l*5ooeqvc`{q!k?#XcsuEvMIbPB z9*+;I`Dd*2R7@jN#(ZD*SKO34pDQD>`wud2)I1+u&Xx(kyK4QKa8qMutcTBPf()L5 z_X2dl>A4IR9#wPm7v5p-o3U|(be>4{pqe4k^1NgZB~R_W`9ZZG@v!kz$CN9;G4cqY zK6QZkI|3FYMwbB70ZOhL5?J(N$i)O)uI0ZL*WFLlr-IIBM#0;C_*N+BvPb|q5=wA0 za8K@S?r`r0wJh4N6|_k^J9Pu9fXW`NGvCA>GEhi|B6xt)kzJEzwf=oOaTzH1T`EB0 z{hZ(Zb$v{GXWumHM;aIS;F5m0`R<(1xi|nq!%W;(x~Ao??ejf^3s9NK=ko@Z&-b*i zPBnS+q*(jU7n@tYbU!>{DtO?2U3edxOXJGC@O#49Mz|H7+a)uodhbyVkzFyqldp{@ zq~1W!tcB;n*;AYeTwkr<$AGkG^w=u*VY6Bs~84Kybj_pY0?Kpv*Lna|HEvgSSf?ZDK*3oUn* za&6r?qK#N~UQzy%nzU4=0V9!zXVvju-E7NC?PprNT|=jePm&mE3NtgNPXNV%Si2Q3 zmu^!mC2K;w*tnzc2VZav{%nIj+x+9HCh&ivOpjC91?Jw}(tg%%}$zoaILVK2z1dp zg6CPx0%RcxYv@4=x7yH#lG#I-(i(8Ux6OW#lnRk zQ~%6;6BDjApFMG?(iw;{5r+RWCdKgv+xR&gZ7s+~mc#}5FqxnE?9f39*Jd>zUf#LrD)(gEMW)f^0hT6h8Tg4igxTYZsear&5ccNQV6HTPxh{@z69p+3(7YiD7_O z3iRUHx-Nn+3D^<-a+tN>EvBGp~es`I3DuCDL&ErEWN^FYK+C|X z>&PP|LXUw_`8#JRNMPh!Lz;VlC{bB)+c7~JQk#ANTRIkm z;H_zQJh7ng?i29^dBxNGa1C(0Un`}k-ZcBK$okczIpJ)NYl?sHvw_C}+^ah6C2Vuo zNPQeTyJs|H%viNDazo+r!|@#RIIaZ6-yxp?yooc?Y4z!G^WvR3$JWm2X+X>ehQL-Y zA8?eL?{>v>+0kF`;u8pbAsX}d6{W?0`_?kk&Oi_4;LHsP7tvwN-%qa-TA5KQf)%p@ z8up9=B0D`iueO=iS5r&R+S-MHdtVy}yujF)@%~UPoCLa9|Je@N`34Os^$vz2OM|`h zD=H$gu|GYB)b%z>Z>G>4odTNtNes-ZB@@~V`ssIYF~LX$P4fOISZ5dB`1|F)}Pp&<+`H{;=DNzQog7OP<*v7xWk6VKr;~Ms38o#g;O2rhK z$~h5K1dl^=SF6=(`RJ={Js>xLZl21t_EN^1JLUv|<33ogSAFZuc75z^$X*RQ5RSox z5)}mS!OfR=Bf(0;1;uaSnwjTSL4!Mep2g}H%a#`xH7MO?X`;s+7Iti^j+F=ej>l+2>HP;_w%HmxNzmw~8dn0aw5AR4FF*z$!2yoMZ#xscA3G+J zwx*DcmnRs*&m8=y;7*+fQq888M)x{vGH=0JO=|o#ZWJ|h4D9sevZSIt&`;LRG;HUB zjekpWLu}xZ(dv1iP*2YeoA-)?`Ywsqw7}6~+aTy7X3UxNRpA(@U12 zo=1q?j9;6O)94K;v~Z#WIQ8ocLSH)&bM6Pryl%+T&_q8{5-a)Z=b{@)pSR8_=>GS# zSD}jSm$$`pI$~AIr|07&9BXp!7Czd&V2>5<&dY8WpQ^K&T>)h(3|OL01>zex7mfYlZ&YWm5&Vr2d$~VTtKd~{yOtXX06vNe(&PRN2$5}|F3dhdcbO9f@r0{`TnIR+~$hm06q50N8&K%xdyQ<9b|JZxasHn1SZMcwD+BVRb zkZgmZf{KXbXp17Eppr9+1Vun{5GcBBC4+!O2`VB=DhU#cU`v!xgxO4CzCv-cGg(v>M^%>9_84awQ%QGHIIIX8wS- zZmC|1mdH9%JQmr>ygI8QoH2N~FHqcIzUXy|jIn&gZ(s30yp9Bo-o>mJE&|Poma2Wq z$wWwdF_3bD6^ERXXE@?d)1f*xk+JiHjtm!ErjjOiz7@a>>AaPDV@I4_@^x=lMm4al zWDgm@TZyWw8{Vs<&X$c-IlAiF%m!O|rp{`KU%yi{vMnMJxc8MCTjICZed@-BezrFT ze{t;hbHY}?VD_Es+SadArjQ0_HZ;+9Ote8HZ*JGwI(R^3nrKMQs3|dPMKFRyw`^&3=r|xlqs1g}qF5S| z`+*;PndXm+VftU5`<>P!c5A5oQ!q%|gO1n=c$utKT~y|N|Egk`YC5>#&iC0CG@LFm zN3ZtdGE%aB>C}u^XoWolHpU`1<591>fCh+|o=xMQlD5@U5px#Ij`FaZ&=S5zOeZe; zFW$Qk+MNuYiawp6Fk#~|`oc_~*4UODk8G*2pe1B?n|JulE+>9;APPJ)w=NR3h+mn~ z_`#QNV2>>4Vn3-hp6T-^(lhz_ji<)U%Tz@vzrRToBt54t%+)E$ET3B#%r<(mz< zL5GU`q{_Y5I~A;oK<NZ$cwdsR*ueg%k^O4a|jVxA{Z`+nEC=?RY*oGGg-zc@V zbIYfci80yJ@=_fJ(p5+WRLQE0Py_Dqr1gCth8!lF^iJFx#V{+e?E_J&a@7lJbCCVe zd}ZBAB2NwWLd1a|KDCea3M@~B6x^uFmS4uYyKC;e*JA>QsD6Nivt67I0b==TEv$W* zbGlnUlp|$Oq;mtCRsb1@Ah|o8RCeo2K3=B`k~mWxrZ%{Sn_2jRDx~aG{R>Fe4 zu(c1lNO!0NToFN8O+OWw!=Q~TDvFW$7`;Ij`}(8`%`$4#Es%HW1ZIB$)8-)wJhAG)p4$%7t`Q~SX;t%xoX zB8t=5{`J71e^SBYh?T<;I(hf)AaBl1$0~dLPtadyuQsGr3xmjFaNSCcl#(X*ezsPK z>8e2f>>Cd2c;-l^&oJySzI@vLQMrNp2+NF3m+f1jQPCuy?R%p*G0l|OGSEA(F2~i@ z7Gf8aPMCJ@tranCnLht=>sO#7YIb9|@aOYxpHzPD_?Sb9?Y3=YlhbPr`DWSCw`4aF zg<~Z$y^dulD6sv5V0hs3b+xtOAmWhx{6>fy2<dR`0-TpIHBcLv0DNFDu4RIv<=R>orR!Q6=%-$^sRmcGmPI*D-W&VDbTWF`Br$ZQd)6|u zKACp1MGn$24c*v&dj4asM2q;DhIjfPhuVtWajXo;r!gHBD4~Y+CMezOmCz#RK>`pT zegz-6^ej#Bt;5vM^+Tgy9bqSQ!`Y~_nhB=i;7AE!`|>Nsy7l_Y*o+1sv$UJu{+OFx zy<^Pzg6>1lY)!9h<;o%#m;BnudQ{TtylZkJCIIF~K5t4kbVBo)0T5 z+2wwny%$Db2~`Xt%G*NGy+QyBl#iIN-`GSH)6OSqTPpM8^Mfn!+K^~)* zxYrG4{;eX5ndiRZAI?^H^j!CD9?L1Yk-O8m&3#Ly<`E9LKJZIe3g!3V)`#fE4t+iw zc~7eiS3>>?{;i)WRY#YSwPE3m+bF~$m!noWI2#ai^w=wRGAw_UdlsNN>Kax6W6 zYk=IT8~G*6oLrX&E3P!jiQxKw2&pF}wwCc={t2b>dA?Ral5w2}}tE+wSt@kOAJ@gE{5M z3M9^1r3J;--=g!0_}SRLM0v8hu(uYj+8pdvikz)ugyrXp*Hfc}Zfq$b_u8zhLqSxI z_;koPba|nPIrzLdS7QJ83%Qf8RPA2PSPPHlM(9~RFyt;Tl3ZBM=GHo=T27DrFa&~y zgpG)_s%o6PQD^jlGiezDPMXSYyU!4g3cY_H?NasFFt&}QsO8A>JWmWEQMyUxS7EWc z+XP<$P!U)*W;v)3cVYk~m`q2Cp>gxI&zhFEwy?ELO%G7*Iusd@Ax`OYO8Ovs*3dvP zH}1e*rg$OS%hGvy#-sZjg{8*`3pKq4-o6X(>R)!nxBM<`>@Ye}TV)tJq!`2Gni~cC z?$zTxvMDK9skoW9J#=XwdSUt_rwl`@)I^O7;VvhmnHt^C()aRM>$7woW0sFQWD9ZA z@=QbXmTYl0>!(1L?qF5Ec|u#Z^QsnSc9(e7XP$VGEbhcTDVV0HXr@dVW{N?vAf37LQ8R*A zx7|OoMKBX&bhnu=z5k>n2?y*+hd+c|I-$h55Z#PGU(D&Vrjx%uJ;bb}nVOY2{>$mD z`j+{H>f64BQ2pK!%n($dANNq>=dryqi$!kZ$Uy;FkW(U(g zgP(lXdFMn`(f4=ls>3AR+um~oH16%90hf1o^giAHvNGbCP{a=QD<)V(eXE0}V-)b43OYMC4!#=~@+#5+^ zaQalJ$&CRPDb~?sH;-1$l|ZqiQloE$W^0ydT7!4U4sN}cVRA9P_Sy(|h)v1NFX^??y?K`(e2?djJU=rh+i9rX>+&ikYv01S zb80_Vta{n;)Q<++X#H|?#%r66 zTQ3i+5Q4Wll?;bp?YrpH)j3?IS6|WL;1*;*tJ*jH;(b@g>OxcyiJ*JRZu#bwpv9eL zC#x7_;f>rgzDEszb#>KMIdQ2z@L4ub{u$q%L@g05siCxA4Yo3uO8&w(cEmHh$=W`T zW7{E?h#g-qD@9t>duR=+Rf4OXex@ZWDL|FB^m-9vw-gd{$7us!L9egKk67<|a!@3L zv+`ADw~gH-f$RFO5?MPJoln*XGMCRBVU`TcNdF=KdF|6neV1NhcR(iS-WQ#DKVt&< zrM)0nl~c3RvwpO*G9?DInPBWYVaT$422+oGxT3p5Hc*Y#jPu2NT((mC3{g?oCCx-& z%O&cPE)eZ&d{^KAw0FO8ZFL>-(YUfE+owkEC>s|iTL-FbJ3XT7_{(KqrbB!YT)xK1 z^?5p(nal>I|<(!}*Y-ge67xo7FhXi7ZLQ@Nen)>Ubo!Xd(BW_AP4?{QGN%<=W& zQyJAA7!BR+QMaF8oH!6Tok;Mxk>J_rLtZ>e2vZw$*U~<&#hD^MDw7*SJNaaX+87~_ z)}wLaB2R(6{le~TzHGJum#?56)abFXyU?=ot-7p9HdaR7CE0^Kt}@5P#wI_&L+-LV zVKdKm-Pt>Dhu#(gRW6THV;SeKvg@qDoJ0av8Ksjtk1@Ea!+!0;E@HZ>e8e#e2<@2j7It7J9omuZhsFk*{$rcj`dx_ zY*`h>N3)C*e=rlYxv8Oh!tk`Lj(n~(KY!ALmY0HjY*}?_GNQlD6#yUT`&bp4#< zl;P6_RONAw%J%&jb<1@@vEP|$gikVe|GEPc;@TumtU8{Sum{dGbaQrdtM#GIu;Z{B zQ>1Z4y2tq>?8e%X!*Y*R?Js`Zd*tQiTl@OBRQSA|joiOH%5j;}jk!oiF*~mq7(iM|${M-XTwPb2?@99n zY)Yq?4_arA9DWCs8u-WJ6V{kxFWoAr17tWc`dD_!WV-w3-&k(-71%luJ(TkR`DTk`5i$RjT3O zP~vm?u2&Wq;oB@Nlb%-E+j2`}4P}+XjWY5d|2T5N#?<&Xqp#281(p*G^=tAo%;#fb zfX>z4CXXGdl_^pFi50ail1LyX5!T<6uV%vA!dg_ zwLghaD9_Pd>}2KjnVHWY=RISy*OT_!Z@;i;Goo&TdZ;6k0G}n(QQ# z!?1EPa40%d3<`tlGt!SAKjx&EnoGV0f1eCId;Lh5PGqKzl(9_=#@s=C3+FHdm*VqF zCJQCzkaEO1kFP(|(jLv!|JrK@5a3gFCZ80UTMmWE&0k7@ng?ubWGIv(i#k)U*wNyI z-n&DnSV|4*GbuQ&TY!HUP69Q(2erWZaSk80vRB26+q?uv3C^z#ai20A;-pUJX`Il{ zJ=`oam02_rp~lZY3^lU?8$>GbH|dhD_A>z?#aV+#_xSpkjcGSOqFNE^imYrdS+0M= zY@7cTbI{s=QG-NNREp;a_;e#L%6FT!2Ts9rD7?#|;Xq?3Dd}uQ zy~PYkoi5zNlB%CLpwH{vt`=@jrQhm~7g&qN=Fl=!$}&CyC_Q_{!^hNKNTEnUn( z-qJz#dWR`CHkLp5}YLZ7Oj33p`~va_+_Jeq9I%oR=* zn9|pXIi5?C4rSY3QSz>z`wfw>fn|*i7=wXi`8(%~76z@VD)19uSHr|Y!?HuWvS**O zoPMHPpuVNpb#-_!a4u#>^nrni8+~gY$zH>V&9rECR_Z>Oz6R1tax`&kAZ(+U*PozT z(>^vfsN7^U=3P=!GVCu9>JKHkPWez8OrdEiLls>%!R0e(1XI;wLZ@5~O?Zn)ogQnR z4rYB#y(ZKdg4Yeyhw%kO=D?H1X2mgkS}=ooPU{?cGUC1 z1Jc)&u+SYM_|pY;1HDg(zB(&~08H2^(l3zT)5(k7Hc7$K&&9KhdKKqg;NSr6JS;Zk z0ke`fRKgAq$FZL@&{`(-`}>cp!)UIXEhs~-rnA=Jw4K63o6QTYhP1b6tR%0 zP0GYFGc%JsPhor~a_PVK8F}F3eMkf-{clhuOZHUx>TqB(Dq7$$8l_tUl~FxHS6VHn3Vt6`S}XFv2*LCZ!51N)dNk}huL#n zXL^g4du>SF=3ya;vTijjP#1Z%msl?GVw~^bLH`*1$Pv`!3mO_4UdwsOA+VoF>flUx z_~@&(DS3Hfb`=}TQk0QMb8G82;dF7#^X*9PHb5}sP4*aIb7;(^G%0XnetKziD$L8+ zUUfX6yhe|psEuRAJh5tEU#hx*la^C(ZqvNw8j~PuzfHpi(|)*Fb}o1Ys#=jzl~BvZSkY`^(dXZH+-iy!hx7M_pS@yQwbXE+;_f{;N_g_aef6vAVCjr-XQaR1# z>Cq<*1sB)Z*rvL&8t}GeW@e4F8Vm*_cW&GLKoPGmJIK&q7ph3V^%{Q|GPZ0^;DD2B zO_Y^0P0D1T?)0RCJiVpm8l%Jf{h{_)4*zOt!=1Fm11vnw*OZml+o7~o5qdgi-_Y+{ zcU&y@fdkv`4z57C!%%0XSw0~E}w_;-9KAr8@B&?GOEBDKnFY!+8#stMR zQMmq!BGc!6k^|0~%F5hOVwnR}T45rc=%Ff?bRyM+JEXA$# z6xjCF;pgFGOHSfKSUEO}h^$3==yn){2{;q)KFqU_r&!*M+Lr7-EY$730B15%#b=91 z&?nY5HbKawQPxxVn4u*piC**dY-IOs-=JDktY>mYc8Sn778ZeFa-a^$VVzv7g2iH$ zmEq`HQ+RJwTHW7YYWm&ByUi}puvmjFmD^zxt*avj9XOfpF$8A}U#ud@zLm7?Jv&SM zIyDv4+}tdhtjIqMRcf=S6p-sSJzcVgiNNT2C7+FI)Aj009AERAWrQF5`3c+cadQVd zHA^4m=Z`*R%qikS4ys*R=i%WY7ZjF3wg#FNXrX1yI#uu2t%7rU;aKrGuW{L0swuqA z^WWeI9SQqEx%s7R-U@w~QPT5E0ArP!n5QQx^1rO8P+S<&i)(^Q^Fz1pI8aqQo&;XB z;`$d>6561k7NICIYO50(SWbdRv(qs0IQ;!lg#}Ubn&7}%98681#GYPGb%*Lh32U6F z`i(hxxcWLA7l3ZjaH{kKl`tpZ6%q5;$Zl}gqNp$2<8rl^fIaTuYh=jY&VX%zB0_O%Y*n%ATQO<~(Ws{!lE4 zxUtR_w!#igGFzGdlfC@3$?Bi%p;Mx2F{iTLJl0k-H8oAZ(kKoz%DBeJH+V$sHprxy z&Lb$awzuzPW6RLVg1J~aTO!QQzgz0FuS1lLNzN^(RJ48EYMdf(K7_RgY3!J&7WX$f5fCAV-2fh=Vvr! zmK0ENh@HE4gRw<+BYRwtwn|EY*~ZTg8yN2P;2c!vybG({wfkm{^F$jsIGgu+M`gpP zli9Ikb^(j$1aKY?^73XEdEdW(x>`Y8C&}xfBy_zxlDH1 za&J1-$f0^aDqseWl^voqgR)`suD%0LF!uxhUz*>j3CBW1E%}s8HkM53F36ZGEW{D2 z&@5erl2O_Y4-5?rjqZQgN)FV;-};p>1X0rvZMl$CwO1pUI$J6-2nWUM42(a2xH}LT zBpI3YxbH8PQ;*61G8`llI_I-e>M&n_v^MF%EQgq7qd#(?P%*t~sJDr&r4G?)s`7Lc zmLZp6oj|83l0H9V4eiOZs_id!B7yk~+;OVt5VE&5V7=QR_}P5MHRn{2x7jV9J6+a= zGv(XZ*kEJJu^T7>YX)I9Oq&i%-aP7jAqxg$pvpT2)y2P0rJul$-@L{1L%=a_{`s%J ze3{b2Qn|>*%gamgG&6hd!LkLGe_E?kOsf-TAc6Y;tePVpu@ygI<{nGf^-Ik!!Geu< zKMc1d6*`b=VimDId8`lb+}gbZ3|YEmlPUP_o6kGuf1}vwlqB^~tjYQF=TS(gtXu^z zoRVehxy-b4=g#2iE|d3nwHKF{)f=X6+-nwo{rf`T+t2u0LQ=TcpFHGoV1`xBoMRU; z;pXA_$zbMG)Ov-D*xRtzYo{< z%TIg!cLwC&AG%*Lv3DH)Q&=F6HH4dklQR&xtYoMiqZkU-0303pm*D6jZyU`$wBgNy z?Jh|sc&MC2fiWysWynJWyu#jR;aRO$uv4Ic5BQ4J6KTo34KX5NXzJoU7H(As?k6MH z@>KCiM0}WwUHD=pGiz0kRWc8FF2RF?gI}kogK0FH6m0p!_=V=6_cp{(3U_nH05Hd) zefAad*+s*F?ocS61N{Kpq;Nt}XNE3u!ze_`KH-xgv$V?UlrY`l5rtdkxm44h|N zg?h@FD>-5Wxjb&RxAfk=z6N#^_nUdF;Rz^?I75gsrdu|}YttacP#klTo;%p^HZ94( z!2~~X)jr(=JHAgJ+=>Q!&-Q0EcpnP~X!O9GPobRsAU=Lv6MPKHQALXCn3ELFRRoG- z>NYKmW|z*UPkC@U3`(sW+cGY?bjHs@ChlX~v9Wh&e1uaLj`iLWt;#)AkkW62nN9e*lDo{6{ z5~g5)76c`kHj(^kVVoeMqFk&c6b1XMy_G`PLTCl%ay2PY^EM2wPz3-Q zywWV{@=G1_2qu`{SQ#=kDRs4_AY_Qr??gDMIH9U>*&FW^^s02Oyj1SN^0cG!)A7bQ zO&Ya=PMtql*nflZ0|+yPD!>Up>E`VGRN!Slk*Mpl`o%9J>7&PcDu?`UgI3AjpG2^f zhK8dF_x~&sZ28+Lkx%&<_QhwTd54S>^J zsI-Tx`t=4FsrqoIuelAUxQOP7hOa0sehOzD)xDQ8Os|#^jzTTxaO(O;#r5;mwlL?~ zYeKjhhE{7NKe5VvtE?lSeXZo^KfD7PS9pg@-^cZ@cSv!&27y(@r7{AF0mG?Y;BhAp zGQT=fHf21=DQyMOld(%FG!hC#&cX0d#Bm;1!SscpMuuAeFgL27XBh=uP(m!T>ko6Y z3m=1UO$Y{ZtTAr%#fujm)P5-0EHGXzJ4g-<>`Ty{K9jWFS)iF~oc978r6&YSKpz?v zD1Acn19$Yr)7V5#TPDpBy!*DC=WE;hKCTproA1Ns<1&j#A3i&)-SyNCtyo!oo1Gvio2DMD4qx{7;SJfW}2^Hg3X_W}wJq_PX(3AshY-sIT48 zD8Imu*VB4r%41|ir^R#Ypud^PB*NZNzVkQz!SQr-Y&|@Crt3}BXfLg zIWU31G2vZ64diy|xDGoa++?-ctvu2*8I^MqFinQ2e1D-dF4q5If@HINAS4WG!SsB& zX+&QCE`6fCY5l#=mdYyuXp3iHSN?hh_w<{<2yT4GI*uz!81xa#m)}E~n7O$yW>&d( z!xG~>?-8C#o-(D&gi4Ray4T7MUMHF-Xm&V$+$9Oy+ovdD6yG^~dBk1=6^R4uiGk@yJ{V46v^9AoZvGm{5CVzmuNeCzu9eeimqEBsQAXs$gP|lacS}Rc77# zGJa)Xi75vFkna3D?N52xvu5TSM8XiH1xOo|wkdn*b?J|V(0ka`k9-WX4DRx(B7K|= zgYC_Sh52v~-0_ih^x}KfrGBA}+0z8=N@ofV!d?RLBMK;tJSS2eAjYk!*yyX!mK!X- z5Dnu{jirraYhL*Aa!wI(VPefv0?{Y*_dqxvjFzecthlOh6Kdnh~@4EEm34;D?&lDqlbm~xTiObY*m!W+~nAeB^8T`*0KZV5?C~h-vh(0M0=42wu_BU&O z5iH>ucndTfN1XIt2T`V0f5eC#fzB73UxFU98FX8mN#SE*3!A$J;ECx$XgXg?S($t` ze*V*Ho+34`gDNw3301ATxE2f{WO4xb3}I7bm|KVGHRB0R>dvmNIs$DIVhxg5c$)|w8b=su{f)m$mI1A6v?N-zCixMij_hDE<_w?=%wiE z3o5)*_*?vc+owZr?QXaY3q%{gND-(ZDLPTsATWYe^Q zc}50P5BeO+*fG$s9!82z-RKt{8$;7hjg5T`Ky`@HQxLTjVJo>p^5zct(4;4Fsa;RBmm1<}D?;C8^~-R(?| zUnzrFkbjtL!gN8SIL!yF2O|tYQHP3hRW-XQHv-t3YNh5b^}6tSK`~#*sC&9*9|JX3 zZ;e=i^D!>)r}Tva@bfmIq0rFK_=lC{<>lcxNxS}oY+zysJg2g9d^XkqYXRsKhJEx? z6c`aOHC4UuZ3O=GsIL0k{1v20Xm1l zNBH}vgv#7C2n|~A%Q08MLueYzxA9d4?u{6h!KG}s&LzEZo{%wE1=;nmKeW`nl4=4M zNoL-~-hQr**nBeLV$1=~fk?rtIt0#tvLeTv^yaIiFB|}m5tgLoX>3+97!ga|xAEE; zI@N!Jm+robVZ_L`wHJM6`isTT_!O;8WNF{e?phRViN*)(i%3=hmSkr=FqGg_rWYDD!ZO($dlf6sJylmm@#RsMG{PW3B!% z{H+);FjgaS#W3ma4i^43SH(p`|J zst-JUw(APZ-R2DtQm-^1%FF{8rcvywANCj_PArX^Z4`oyc>qH*c<1{0C)7;W+UkbN zDk>_x$kpsI^lyHOPl-5F)AE^uzE23>Erz#+A3@aq6bnA1w!GiM-4$nmF`{vR*@R>5 zJsJ!z(z;Bky+k3qQYdPc&O7wiUd6%cq!$m~*k2a^#|7fVFEHC>Nx@OF%rxj zV?ZPXg3jpEJv}||s{fjxTbgVIz=ccL3*@tQjQ`CS=zt|>2}>?_)3>Q;e3H>Bd^NSvU#h$EynwG+*fZaBMoBS#~a@?`UL&bYCQ7)zd4D*0%pY5m{ zp2Rb1(GVSlCIZ-+QGqbITrWWHqxrlSTSfU$zXIZCO9>WhKE?+Iel#g5$??I12gQYj zg?Pu&PxEMB>EL;%3LdwbfMCn0qxp9~_BHS3R0%QLt(Fq^@eJ=6u02HyPW}H5P$)In z*erkl8`E;|-w(V0Z(e!r5VD&oT0$2fLPo*e@!EI56TW6(*c2;(K{U_*+u#d|^uqL&c`gy|; zatEUr9=2uO6|x*u-z2|a^*)9$L$q1K9T-lK(971=XRc?VH$+`LQ zamTnZ_;iaaD=P3-$mN4#$@(h@@qfZP1`>J)=;C)=YksiQ+-t;=*wBg?{*8s#f2Oan z77QI1n#se6yEC*e6JN+drjtSRLTD|Cs8z#4yAafJ24&5r!hM}MLZxq1(AhP7<|9ew zd{jN;t%#mJ{WJ1+fCxWYzL+XfEn3UHcRU&EQ*CZZNa8_mBS7CA4#3RUf`DK-eS848 zdptO2VSwGLfcxeJi#`zCSqHigpfGi?MllCr>%??7tuLl2h9L4J6k_vghne@p8rZ=| zP!bZfoRgZ!O%GF8v!1Lv%Rh|VWdz9qXe+S-SClv~#}KE82Rr3|3^+Q)?4p@C zK;`Rf(e6G;0>4@#w0|6E**8VD_wT>MgKr3S9EG^^k9TpU^8y!%tuTTTR$(`xNu$7= zp-{9fraT{y3@vFDxi#>M6)?W@Fjm5LfCipIsKP1OX_lvZMj;%@GlF%hxG|y8Ff%g) z4N3M{n|SYY6*bdWIujbI3E3f-X-mYMAspm}Jy+R+bkE1HiZqhqF!=ld{{CU!)8?XX zv;FC{igO->AYe>K<8T{T2tq@^89N35-f)E4sQ~aoI;e@>%RQoKI|UrW3j(2xqAXY; z0BBIUfsi#vei{VRr+C2txkEV4C+jvF4EtME8nm&|9GdmqCHCW%%GK={=1RLUc(V|2 zYdZSI2+4dPRZvk@M(bpM;f#=)ILs&{_lTgZqg5wnPX(Zi53;3rT=kIp132Ux^qKe{ zRFu&^u1(0Rg#kqV8}bj^uS!-UWEaZYYe2YMjov9%+(sv42%3P&`V;We8WHb?J(z2H zNVn6}=XO;Np&#T0Sj^z00hoFVCUjgF_LF8(ghyYZ+aQE9a1)V~w6wIOk0Ah<*o=MmOG$8FX*5PH8 z)hBtc`x?@lHnK697M(t(foLQ8{}B?Ae=!wp29% z{#!sm=D;EOplT=0xscb{+M|L-8WJ<0JnNDW-H?UVFyS%fFdSCDOa;UYv+;k zh!_dD8E9XIp~(HWOA|mC%LpTve!HS18Ww=HeFx$gFB=1921Qe_`bsOPrQJmi=AGU? z5Es`FD`-oN>VdT-mP8J=s_N7=BFhOfW2wGVu{kKjZP?w#_Uqw+E zOi8e@B&0C_UbO}Wn1obFi%Uz@D01!a3S}?*dV$9u|#B@}v(12nw&%K<&9f>8cXw84R}1Vy{}Iqx|JUA8=grHdvr8jIZhN9-mC z34mS$>b>j_wgoLuIW!s3T6Lg|#?+o?mEQ*%5jHm9uym`?*DxCpFe)|qCgnyl6(-Sr zIWT>|zM|QOn2fVuvAZ!Cr3_KOt!8doLNN`amV7@z@1aj5T8fO%|B-z8uVDTEb5G1a z_UCur3IHkOEb#M>H79b}+A`cniiDgTH5?u%9|tf&FJ9%k~*4GSKJPk8l#;i(;qoP_HtGyT8uN*baVx?>laSdf@O5 z=f#A_!(x7LF(Tn`K=Hq5-G~5oB?X7W1*pIiPAhm;&YHj+#`x-VW>PZr#TfU+D4l~8 z=FLC%{Q>5P@%jJomo6Tl?;HkpI~0U;2rL5(=OQpBNIBv#=rLXm4%IVne|U}3GGJZN z>pZ~{jhwFHL97z6i%UXl-3!T7k5*TEj?0g!q1>Qbi2zGnXRNZgGxE_>U4Q{_Ui!+?XIO zhT`8>gX1CgUZgrAylAa6no5}C3|`eY9*`e?ekgwkl~!*|dIKA!Qf^Xo|4TkMBEUPp zUxa*de}Cj5A_hklhy*|r!f_gMhdKB%WW*67)VLY}{i{IP>{FzX!$`25SvJOtXY}~VjRvd2T9Mm> zK~(Zx*bIY)(f*EP&qz^>Kr|UgL*RJ5pmm`L9W_~e$zJ^z4oHIDavva609D@Dw{6?D zTfmn^%0NKiJD_TqheTSKIvkIzYn6)PS}P!1OC6fRzgMJ`eS0JljBsJt31BPT6f3Q* zt(~DS0Y$xB0Y7_Z$i`^Xng1LYOIz=Ti{ET^=hY?m?Wdw zsjQ6uyj8@%Ktc_6B}|i<1nRdA=S|Js=g($xAbJR9D0vXQ06oGG6D<%F6-}0NV<0#| zXf&*AyTK2z2B*hl;=SPp7^U(=&&jtA5Qo+bZ43-mBM7-sj1Y{t*GV4$2gu-c@Oz5z|VDnth$Mf%_%Z&_Jc4qya~K`$I|HAFHZ>*W{d zp4URbpbQO^Cf$sHiry2YQKEA*I%;k9bc`NnWkqEIpE%8W3m@moHwf!kWrEVP7XslDFyJ%YLT;!fCJ6paFmy3}_klo~i_Cr(s0FlcEjn0wAtpNJqhGe2|ioB7iUokq12ux3_%Xkj3jU zb}y0Ni0DP|2m8kr*pW?Du`_&Uc8(kN5r$hkm6>-7ZD~j$TnYe(B@Z&^@Ok0{#cIEf zJ+lX?W1N^(jWFyZ9aKCy#Gae72Elr4=tr3K=wt8{bAPYPzvfyucL%uWx%hUBWb1Vm zp8afW;Jt?;q2b#V%A{_E%=}Wo(Ilypwcov!d_ktqf}XI=NJn&1|F`twY9)&T*qXm6 zM%%G3-&-S1Y@KHAPHaN_@PZ;;%mf0qS|kIJ0$g(#K)*Ld1czE4cB>>0OZ@VC91qb6 z$+#B^AZsFk`yXF^`fvLP^qu6=GI&^u$kPHb5Xo~8fZ9eZFi$pBFQCWYap{2&0C?sS zdKqv!kcS6fPIvv_`)nE@Qdyt1_T(A_QHS?|g-3DXeUKB1 zGBR0!P2#Jfj3@IUN%WNGEM(7E3`(3neP{>x;JzjO5YLLr$r&vu;5<7shU0Cr5oCc- z2*s-^uppJ78i8OBkQ!c#DSRRR{+b%{@A+3{dr`dN-j$cMAJGLyu00myoG|Ybms8tf z2}498v4sQ!Xtr_|cKrB4FsUEO(5Tb=yvZL=8RT2bAbpkZPI!%aq%=kBDljNv3MFIS zy4gm686Y$qXam4GLCkm|5Afb8Y&d@Fo;2-ANUa5x@HiP0${e(r1ER6Pxz z={w;FagTBpDe8?nAjX1x=muLVujc+AwkD-Am=!}+O_D>Pp+u=ODH$0V#VoM?d@=X) zPzp-4h$$}{8-wgckn!XOTE?sO8a{j2*ih*HGOH;Wyf$C2?x+o}7wg~^E1CEUO0;eU z)QAmlv{DdbDE81$90d6~(hM~NWQL;S18i)F9t5#vsRs*m4<=Qk3X-uk>cj<9l>Du0-;-fClKFq~lw5gFE z_1ZEy%DJ^#0F%JX!sm702xttHdj_sW13-HWg{{9oh)z5r2lMg&P;vSc zFqpcxK*`B3XbOkWzQp0|+X&!XRFZnXZ}c`j+BA>Z{$es@FyQM^%u;E+r4k;bGypuN zV%2v}F&SLTTHt`6c`3uc1#kK{9W!WV*AYXG(V=Nd=9{jp%kh{7a zuH_pV5db4|NR{EyylFt5oL-ryi*g;Bg>}M=J^lG9T_Xojogh^+IKkZ`DbUVCA)D{< z`lMk2!*c|W1w<^3$YwaP{s=Qj>n{fwNx$|(?01jhJ)DLUVW6St^WPKnG5X+te*WLl zLjGUtI>ylZ4s^EVl@B~68gB$d#?P0aF{|u|WI~GtKK~UF=YMQ?a07A!klezYGCzty z=~#fzGWBx!Q5;osB*v(1`Xj8E?kuFHhmUX9JK$#BseMDdXfk)&A}j*od`N&@g?I}= zZXURfjqxNU9^hIyaPHN@!${4+*e}G@6*m@<34;K;2Ewjk&{V5?k%LU{&4MsW<)4jJn=>Sq#i0y_@$5*lSdq{0&cBol?r61}n4osQo z?q7#RcS!MjfJy^-w0Ob29x%*&;2MBQg7$UrGse9mKVxxWxH^agxoHO}6yT!GpqGr2oYO|eHx+;t{}B6KR+nzNQ>Ed@1h$7d*x_Vc{v{OEKwJLW4-pe?SBA?J_{I4)?;03?cn2LWxN@+9VF?OTrLAq`jh_Z+)F1cL5;&?3W** zss2z@3VzZtk>9Q!#CT$;8(si%!VVyX9;9lZv?i2)%Mg`;%BpXg0d8c~V^D}S>DqFR zrCO1`peRX25pLZkOa*`aBP97hXzu^gAyO(}I{$OsxFsJi{0u%%oRB5w2VUg>l0%aM zg|pw3*pg~O902%Yq+cSFI`~Hl*jn^4c!aqhiF7*%p<~b|Y;1_?NC0XQO?{+2M+ATy zum#A1bdLyt$wHrbbxct`;#k5z&mn0Kl6eT(nL=DAH*-fL2y`xK9Da;B1|-zaPaP(YS*knNLJhDP$Rf99_}b`Sv*0lSu2DW9U3EYKqk!TNu+@25Y8X6N?$_5thx4=kJ<0Dj#2`UVf6u;-cAtIZ!O z2{Yp2Y-HY$=dY}c7RmOb2|vREAPCRqfg8kF8IZDb3vC?1x@D8vE{b%BoW<v_hg(gWz zjRHS74kcDfY^4iqw(EJ%ZH#37SKf+$R4rbxva(WEM&KRFA3%;QH+G$j^r=!{5Rq2d zGH?+N*-zIIxChN44Teubs76Aj1KM5WKFuiG+NZ%bB04}cOB_5YS7l^jnhw59a~g-R6q4GT}IjIoPbxM;`*& zum4&rwvP|R1Mq1{M;HCurWAgmH1#uJ6cBqBtqunv@g=|$rqw9LcJ%|R&(BEB&=0vt z3<)TT?4UeU>(CG%h)Y%Cz3Uhc#VCa$r^VHcfTb;v&?r}|WmE(rTqOx)F1iC&tuHSC zX5oc&ylsC7{IK~egGW0*b4$(Sw+{o5tqqEyIe&m?-0W1ts~xnGeJYNP5fZ@q_5^UEf}Yt%my@ z^Z(91v~Z#C(2;B!oYy0#^`11L-BhNQmiTUYP;Vw+Y zq&6-P5d)A-`ONzmT=ArS9py}tyKR{(8TuDQheD!)2ufi`ieNaE1TIS-aiEm|@omY6 z2A{*V5AzKXuTZxgq$cI7-L(n_PzudBBr}siOL!c1%xB*2aOux%l=zBrNGS*eMHMJg z13|eXg}_kI(73?_#m07#;Icz~c3g1mGLW`N%rcBNG@$vFA)f}FXF(elq47gjDsGjJ zPQq4!*0GzhlbK1zq)`Ac8RTRr18Vbp?Lp6tCJrMP4PnE3eFpy?D!0PSp5}$d-&d|m z-snkY@Vr%^g8!MJ^!a12y1!#FH#)pS?Nsq$q=iShxWgzP4&V}TDxxL2uZla-dLhbX zOfHnIJq$RV3(CP4m6AFLPLPTh$cDp!S*t;GIf9l!>EiV(4yI``#U4=mT^n?G<$cB~ zd8~(6QD}#dC=xhaPV2z*2NS$!ofttTisXyX`69rJ*n0tf6Rb(2FVD)Ez?q1AbQ^V@ z!{Jq&t^Thr1a=vezQLvb{)bSxcGVu+)^zH{YMh;oz1sWUmx26WbhK^m>OMFwR8=1B zZ+NdqV0*v`;vDNx!VT$qX@{{!X)(e$Nt89x$$QMH3^!sv=9x@v6hHrX1oYI+z?MIS zDLIh~ul`0f>Z>VraUv4&=E+nF3`pXakA7CbLDzu@JF!0U=s1W{noJ(dL_yr{{lfdt zU%mZ?bSCz^In%XI$u^^7sNH;r#B`mDnb^L>6zPgPmqq`6QB#pF(#hJFI>OQ8ARF5@wmq2hXjsF`d{V zGy@9iMt?Q|N65#wfY(q}D88Zt!r9y*AF4(7yPG}MDIp3gMGy~-gBT~69su8M@f>Ag<_}N{ZR4S27s@p4O2^KCGL{w20Kz} z`*Y7}$;z|AXfX_sJmL~kkcg0czB0=;gPOcL{$9|64S=s>9(!Iz17 z^5cX_nWuKz zo_1U9DL76C1b1F3HK-q)+A82bE&Ig#whzKq)dTV_C-C!6=KPYKkht61RY2ZmMeFM(gI_0PcopvPZEg_<|2caBrc_2|?&XLP0IcmhR>6DxMKu>z zP)wE}{QIigW}{z1qan-h3@m3vt#zyhohms72hKjTEztiyYOUWpgA0Q)TvQ{F4}e5~ z2V|E)5@P{5kR^xfVJEw`+zU50`(;BI{&Tgpylt!2CnSrRhWt*X4R=J2RmYp_95T+< zXgNF2^oj#aTi33km?(_Hp8*r77jYZZb#<^Gntb_$R?3mTxAHKhJ(>3O+{107FufWO zF^uMd-bm5QW4YA_3`f9{+J5;7ex$_cL{jH8ocRCh!OE-$ptDY(I!e{WnH&!sb}Ae! zdjRfbb3MfCv7<4j5^mv0J26Wt_i0K_UbSWXZ%M`+=*Vc>-|p(0Qus zB*LvhI5wqKq6}}^!swo3;(QovFU-UZD$>r1H&($;L6XYf1e*n zHxMy`(*9>*j55I^+MkW)vx3I=d8l3Ddl@wFS3tsmP7|?7LX-IY-){hhMm^$QZoT0C zY~%oPzW&?9oNZ6jh`0@E*Z<{T20?EXaTpz;Ww8Tp=Uctmf&0X{FmzA@T!qss903AB zhBfLGinR`?{%uwC!vnzt$%Az{FVSnAf^y9`6n(r7oY1>A(FWH(Pog1L?6ozr)2YCE zBvTd;`F~M9TT94d5brU1>_03tqkR`r)!>)DNB&y$T$r$fqA%Nyz?YSKCjPc={dV&h zOzeR#%>w@30hkt@MIIl+3(K;zna0Aorq=2-`W=p_)d$ONMn3PC-4M3W`7*ohdwhv$ zk@MOjRSy(H6q>*s@}ACu>It#nrj3B?1vf;4je8WlWYskF#N4Uu!L49il`I&xLoXtJ zx33i;DG`pQkVCS#h*!q~539b_T5Cl9%_$T;fgcnF?s~GmQoPJM)g~2a5=x*L{0J=x1shu&W^2p z9VJX2t_QG9YxW<<39enOKbm|cVj6bZ|EIn0j;iX~)<>h>8%w;gV8j;9QIr}BC@Li? z>H!f!iZrp&Ij9sBrQH~kEA~T^BA|j4DblMPG@=61n}7;YihwB6+u^)#Zcvl=a^L;E zF@A5ne=Z{n-41#XDfc#UY+b%lk~uka~Cg+F|nL0O|i?V@a|YTtg5 z2bBbug+>|{|HZD-B-NawOjXcl&efmqL6%B#p0U$$ECiAmChc*@WkZ9k5=*U>u?tUb88|krA_j#{-N<$xHU;YOmcs1_ z39Hgj=e(-u`4lxponzQQ8Zea#7-e-_E6n9BTl9Lx1#f(+^y}vrNC*{)PfdB9FYw}& zR6qNk{?>A{_!&KDF0khKtA|c#`@`nGdy#1do#xTVE?nl@b7Rvl64UQ_+Huu|YoREA zjxl6xIHoOwBU5TE?{v*0vm&@WK3(tFPl(aO>h7XuHWVA{?00OMK7S4C_2bKN_Ifz9 zbG~<*u_zYmTwTM}aE|zgV%3^{ES8*CU%D+gh$T{ht}5JBdv7;FKo(^$htuxvlzhRX ziU7aOjX$m9*;_U=y%|#|0`_AD_$x?O~ zs(!W7$nu!)Q|*a^r_V?JF^HAk`CwW>3Lu`4|8Ak@J7>!G?BRxj(*AhWS^Re7|7il6 zpv0#TS20ZC%9iIw9eUoB%Asa55hV#N6qf0c zb~2}ofxb;>s-ySn8b)cQG;hQ<2x5H9FT`jq1FtV1{d*fO-!MBS()BtLAzy8JdzbLI zS>W;QD1@neOF58+^Ynhk%!BGe-8stKP-LFd5pE{JiF;)ds~u78KU58_O|*D(_b%Bw z>%VaT^1#u-XWd&8t)LRS6?l+zQP#9YhwS6|TKtF1auZd44;>ULUePDB>+Z zir*_Eqj^{-GIDRYY#OIF$)b?P&P{(0TqeTTJ>#}rUsW&!q^<~0Qv{md6M#5GujL(z zR1=FZZT1x41_-l@w7{hnpu|2{!Fi~$Vy}7(y>soz_nmcOMiWgmBVFr`n|0`4dvegJxMfxaV+l?Ox?44WET)4$pz-)l-L%6=vB;;s6e0q?ZLClHfd5<)qTz2UnH zQHFl?{@K6ynk+#WwXz@!e@u=5Y}@*kT&P})xVK6=P_)0pZhRhLFW3mGkd+sqaSQAe z5E{7BCz!QYS@Qv)zuNdFBrJ9|o;r+B_LHmM`94dN)zO&4++qHf&sGw3aJRL@n&21; z5ENc#=|Ppy?H%W|+$h6754gJI$Li4Nv|5ExS6V2jf07I@sqJ-Il!V%ZpY)xhXLYY+en+6eMtm#>tO@TL>A z$mg2{UWEp)1t^4b<1^YU2V@^XmShE6_T{P9+b`B5#8K?tfJ>Tx`4PhMzBrZdwTjJ! z=*QH3uFk$!1NhCQ`8WOyg7^vUKTeRf0RHNfW(iw$6u zuY;E-G_vXQT~btjbEPf141rjCr}y?5RSAq)QOzkcL@KsWVCVMHGO8+33j_v8&r}%r z7O6+O6R-_VpwEji7?AJ#689y2uSe4+>%*hxwgr*^1{7marc=fDE10kupKWp+tZ6z3 zr!}WK8+Q4xW1N=_MoIaSq{lLxCOh;fb0^EA3)%t|&I zpC-b&hsVa=wzI{V`V+>^)aahjxdke!LDZ23k$TlziqMz{ zXo#Um)t1{YFTQ?mwkX4(z<0E#s$-2}7OKHfB;adw?`q;4OFw30mq)e=rLFV`JW*>Q z2i;c}=2o>JT1(8?^f}rZC06Pf5cG6}-A_P^L!Fz3bgCC~Duc|HAO|1^(sT_vp+zG* zRs-}QEN2tut&`ty|7;>sw--cYqLi%&V}Bfu37r5D*QpyCo{j8hHv*t0G!C#nHiDA? z@~AI^a5VgeoBrp3F&mOuSca}8f)7jNfLeSuxS&A=f>2m|lNhU456|hL*kP}jHx9Bz z%X~@6@_zbqcD~MDE+I;Cb$b1<)Jk=w z`ok0#>xU!go%eGzyxZwpk6^S}dFX?fuT%bemsh8NBh3x;4>KmH9bwF|>R2OB7VICV zVgShWJ9ff@sp4K?L1>uLgeXa!OZ_%Xn1mlk-Kw2|=`UcL_glpzb6MmJZ#}v!A|u95 zxIbk9&O>X))kql?0Qm*ro6{x88-;XOaF_v&F({mpgYh!(xcLN~Bm4*k#2+?>tY!o? zerlkDrTT?{N(IRhd=?|*!qjC)gn}^6@WPLA*%uv-A*iXe+NPUwJB9&=ukE( z&VGV-8Q2yoG#f;FGVjMrTsISN)zP;I8|qlVNzwpj@kXW0CJ?M@w5##oMi-yhsUSaUPP4TG?2 zz`_9VaZTtKbw;-4h^Ofb!Jm28mTMawEYGF_Z_3osBdp}9^b;GAv`7Z<)SaUK&Cpac z5ji5jmOlZRCjg8R-VTGCb3tY2b&_x|u)h04V$lnjdDH6%h}`m{jxf`tj?6R(o>^>g zY~`6J-%nq-;d^i@*71B}IIMKR=rF?2;6N}gk}1YJx!J!Swff+EX_Hm8aJ`wP-{BY; zglL9VkEfOBJ1X4v(aK=vIEN4l=7K7CFQS}!05g^Q`DnXII(bzt_4U((W%GAlviJ5b zi(_@5Aj=?iG_4oKqZY<-nHj}V$BaS%S_p5M2;Mz)AX^>zhF#Y;$`CU;W#(c-A7?EHaeat(QM>wb+tRVm0j!RDwe|J;zhnn@k&3F+73}w90 z+rvEbzYT-dO)otRJ&YgFQ^T|ldTA+lem5>wvFWCK{>_yP1xxu$-+?r=8-SqJJCQ+v zpAD9j$Dw#v04}$xFy-%7?@lB%X|E8ZOQW4@R{Q254$h&d^0)~UtFT7K(TEQ@W*C!B6q|qT#G;sle(v^Qk0a%0jcyz#|=`JD^1!i%mtJ(-+7_Kd8hzY?z&c zQkG?`xzvPM&U>uuZ$%}~*q^g-#B(cv|V@z{IohA}|*k_3_iq--@ zI2g@pr|MwFxiL}iz!p%6dt7!@; zSibxe;i$^MA}Php62kR`RAXXerG8mxyOY=l411VA_8bl>G=vdR>KHrZqAfsK%a7Bf zcESUIZ9oCCC)I*b&X!aW7rt4&jJn8&75pT72Z4^hQ}FAe+T`4Rc0t6$(<+u$YJ;1m z=x3_Pt80u0JsKMw+PrpO-?*HeaiYT?m8|k-|1z_P!dK#%QPOS<-GQ%~Y)vS;dpG5s zIOVt1RCzA^>^(1V&&i?V)ETD^?JYcEU}Vw#u9#FHD-^SLTwXlmWd@e+$YeaA*6Q)0 zMKieNAf;@$jcD%|bxek?qsGi8r-N{|cbLU6HocsFpqvjLF__g_&vSLZGXcw%!_4H6 zAAo|(4y6_&f+YetTRU>la7Gi4p}}^%-bRr@ffeK7HyweKxy3QyAP&PzQhjcofcTkS(hSlXZr331J z%AQu~Tg4QQWWm(Qq0&47vJ5j%HQ78QF>Qe~O-e$3m>V?*Cp;WGFBcgWmk;Tk{pH9Q zT_?E}n+xK%w0{Dbm}}!U9c%U8&@g4>CKJ$78nZ4-iQ(b%DW&wIH}2(I1_@+eq4vvp zqV?0V00VW?;Gf-UtH!Mdry6H|CHmMv#0T{7ib6Z8^3>(NXYRi0peD-)U(?3U( zLS{Q`gtDAnweRpnl1iz{s22yx2M`Q!^IySyoxe2_C-XO-I)>2I4-r5!w8MRTUdfc+ z-{&dv<20?Jd@5=MsSp*Ef5{Bl3Nra8P_0WGxdp;6lE_#im{66k_gb}AlIwoDmRN#_2ih3t&i7*@ai|bR znVbjFL##YvIg#+vO%ua)gq4L1mr_hc;W3=IEEzxwlkCBz5|cqD8b`OlQbybR;fxAo z5N%;ro1qg`XM$KcL_Ol<7=>n7AdzHQNzjt`6+1<+eT-W~P}fL-AesnVLma0QvIR%) zq;g(rc+KWg`50YhEs8d$JO&dj*HJg@fH|ZT2AaVg^Xe}IIH}%?bTX8CUZEZoNce{O zOzP}A=N5f_UWgjcJQkP9m|LR z_!{A<8JDF9by^t|11scRKCOUxtHuvD55sjzz;|3jdpJb(k3_5*iF58+D1WBU`dz}< zCEavDh3AuVB%FcJs~jV0=-gYGEtn^yuT=kAs&>XtQM#(GkCW}A>69d?i48hO5RKP> zSL((L2*=o_11cs`QmxYP4-sCNZss9u2q|kQMn~k73tLY%lMWM6@1Tk;{amtNyiFbTTU3;hMGB^Vl zQl8i_)gPoNj2-VefCQUMq{gqm%#34J5H6!+uv{qi%0HjZIHbfU8Eir_8dzM)XNcj! zm;Kqk$id5lv9&(t1OZNY_C43}U?Ak6!UV&m)K-SiCvvW}Ho5)KQ4-`#zB7JXwG%(1 zhhu8LOShpV7QR33WZTdW0tzF| z);0Lld{QuTGJgFhA#?Uvp-RN3z($fnKBc*1_cXT>%#eyNPTwQL_y-{g4Q8QFiuymgBw(R}=F8`|^7?k%>3QH3d_b3gB2{3#0QKtimLiAS#xRHa>n9u&@1!hoJ$AdW zMU2@3R8HaKXxha`JjcM)9Z}C5kD-!evno z0G;$+zLJ5lV-jN_9e)$jEg{tui-a?YP$o**J_VIZnip0cJ)Q6!8WiErUm*ff6!*w^ z$rSTryGX0Ukf%YQeuN__+)+>?g1Y3yU){tT@nx|*CmnDaNFn~#N0KjPjWKF<-%jSR z6tW}9mSv&r-hN7`EaGfip`vkb9?Y;EK zrO=;gSbF;x)G75NJ)z+s(fXneL&<3ytW(h~xl1WR@#wo-!U`hhH@!2SEiVBu6EisS zw~%Co9okW@4qv{RI@l9-7dc=i3>xQ~YlVXWLZpl`D(VGbu8@Q-uT$iwjnHCc{6KQrMm7(S5g>)S^vWZTc%CS41utWgy*uRzMl-K~rp$a6e>CgEcJ7gw4>V=f>dYaVQ%V2 z6E{CTfoPt+LGzqw-i4~p-+rZpVX<{EVl67}U{Pk21jxd1pO@$_LTNwhAW{jm%|}KW ztk!z62Y{1Tp)PI$zh;mEF>@A5=-6DUife@&oI!jcI%5m<5=fhnDt?RWs2OR4i6x}G zgU!9(s$%OnL^ibah^Lzy^-03WQI1>*aHxER>flMe(hUG_hMc$_raRHHjZ`F0S`HOo z0?%y5oWxxJ?1cxuI)Kmd^RH=k#_9h*9w}mvl&{H?D@*k_$(6L~p;Hj0`pzzNUcaT+ z9E_|MN-9RhNxVfwCJB#-MIcfVslo`oz8~7PV|!&F+NHE4(S(U09w;TE20-0SN`mlhc%G7BqCpY8%fYsM6M)V zA}s{PiisWPtHPeQw?pt>M5O|tR6;>9>zVqQ5FOYo7_Aos;LGmNogsO*A3{kAOsV5G z892J8iDRLyi@L$h`mn&BOfOFtU)!?xn!AUL`~zH0emQmDPl&Ukb|ySAfMUV}YMhp7 zg#rYb!r+>1jxB+vbR|j&>UZtfYPC1E*8vUl=Z0xU-cO(RG6j7d55brfI%y>dH@S=s zos`b%+)|1_toUsU0zvaC=aSP&4~H_4SD!a5;ulK$RPhGxm$Z4=ITxbYAL<|EeFj7Z zY4?=@Op$;O5fMq`D&oPBf*+)@$xoPquzs*@TpT1N ziPY~xczD|xa4J=CamW*&h`e;RDsp)cZ}bJEPsW02KIrVv-fEbuf5;iyreX`y=Ackp0VQ+Q)v%0huBGH#rMf7BZw0!GUzX-U{7Ek23=g1z)KN3hU?$A^TP%r2g%w>B(4mS^rbYx!iwUp53?N-S0_Fgu6p4mDRint-_nzi4v?}xgwz&O34 z_ZF&uSbTKjs^#xTRgD_<67*r^A>HMNTeZJ>m})Wr*F@PsB)oMNqtG{P06)RzzJ=&# z#w7!^|1}G;sFH5GosfSoTjVh5M<56ZWPxrti6R-X@UZG?v{=3hNp~aFER$Zw^+vim zABUqqoG-FYo7qHZM8k9#hv?V(UrogYBr;(NQ_8^7t>`#g`={0;$%Mln*)O!~UTVF|CY-jDOFU=% z;dwUH#TiIl`^1~s^3)I&dNR7=Us>yA|Yx za-dkaETT?)j84?7D8>t)H8vOeaf{`ogCIXq4tt6UoYj=cB+E(tfBe=TeW|%{a*Q@C z1%|%~MRhy5q+(+jB;=H6`unXhQK5E~Zy`#U56cCKr8kKoR2|nK}9e!e5UdHj)f zEIht#KX^e_9@QX5W3#bB+4bI)n(ggy7EeHu1flxQ@$rEfc%0_hQTV*iONCreGi8j_G4I_yr5}=zg_14zO0|xU)vBdYJcv3*c>01;v`E59UsC?$n z{*8bZTYm332pT0hj2j1a+HQ1k(h$h}1UCAlmZPaV&fdKsu`!ef;!L5ythY9!cYI>!JL z$@L`mQ!+uOrC?;Rdo9YFMzB-28U;+J74+laA>a_jDdU1aCnAo4vc~z{2^5u9T{cEM z4J{#=%&<-0>U-k>MZQ0v#OG6LXah=eDitp~!7%8NzO0LiwdJmw(o z@9ifQ9v+@!2e5kMKZDiUj1(6aX`$w|R1U<-19Gftpn94k*e9CJ1=>twce|b*dDZs; zp{PsjirQ@4usThjuIuleM1h&r5SdfOgH!XOU$lbZc`AZ%2uTt~N1+E=urL+j6v?*O zBR|G@02WAQX_bcaTrD)j_W%xzm)3A6Xt)(+aJxR69(OBWCy6ZUG z<*KV!1Xy#auSI++HLvS^O@-U!R?=Z@csXB}`g;uzTETzCxvp=6!o8#+4S9)b*SIG? zFR*zQ3J~s%hjE$_h{^vFMJrPtUtBDoF*-_FpZ~_c7Y5hQ>s<92Fk(ZTLB(9Wd&NXP zv`k`Crhl`CwQ8uZ!vNexc;4_g3Qni4hN35aF4m49y#w+w2`<1e(q2a_X1eP{;Jsg4 zeb|H2raLvMxFr|WfMteF<@vOV0@3RweKftD2^`WRvw*WG{lFI)q9!lt9hi9H*s=DP zWo#-oeTC0jG?&bbQ3Zaa)^J0~$xs1G+dhS2($E^qs`oXCA;T0F>+_mjm;mU%%qe|; zmww|PGhlVf59|XQ&VMK$Ph-Xl`NAWN1us>-aJi@yfUuBt}Cq7~9XL zLg$*C^L05lpFfhkLF84N?z`S(OvFRbk0NYA526H}{lm%|$kGpCUCU z+S`09EX*6x)b&lN`%n|q{`f6)uk-?pq0^2$1=D^WPt~z@_!%~C)NlBQ1)E|tS|e5U zw6n#K>w9FaKdh~z2&Z^+)u4Rk_Wx`b5i+J8oOnRE6&cCn{0D-50|+K*CpU?%;}_*` zI3d4q9zn?Zw96B|m@)^&G}7gz8Wv6Z#=$l_MeGb9am+sxqO~nOm?v^`Y3vMGMS$5= z6sP4XoLpr4WNFO1pSaB$MZGf6yBq?sVDQT7HC3@-Y2UiAgS}j0#KhpR4S_?m#qA?O zmgAA&=SVhRnOFCH{e-8>fC3%Up{H7l)f5e3J<%CD#?HM?*e<#d``B<{H~OX(LT2CI z0!URr8D*+VFOW<%L~mOkOuh{ms1wBS5zXUw>F283&)d(}nmTLj#*L#e;|bhaAjz@G zuMlpbeDj@{6n&}}YecYE1oe^+LWA|?v@-3SW8xo)IHH!5Dj-PqM*99T1j&S!;luJM ztfx($sT)EDG))(FD90T87_Wcl<=Lw|9szUo4CHyzbCor;GN|%}ezG_GtCY(BgdQ2Y z=Ko*HMJQq->E%Ej;vXt?-L7bZF5G&6WN;V>n0C%LM8GWR8vhE@)#=_0wl-uVxTla2 z{uu&BR{?D*D8f%8{^}?kR7*2%)y)GG=xm01>cHp)?C^V)3WC^wSZ(% z2}l>(LBvTUBYTsx1$^5|MdUEG{tkg9?diwvLLRtcg!1=Pug>O@4u#&Q-bWd} zS1^dFsE5Fd*KlI!ao^GmgsN)*_gGTF;Ix$rP${A0NH?N_a6+1d2;4@f^oc+h-sE!7 z^93Z`RA|QTf|P*7-%mjK_-gCuEe8X!6YOYLa-6dy@!Yfw!(!vK#KIt31!kJTKz@E2 z2=$QrOhifTluY2wcZ1g~L!yi*ENWd(rNv>fVE){?%6U+!uau@oz3Tq{ryzt}#Mi;@ zL!3!FfpU)oJNME6EeO6nA||Z29#DvtT7E}>M2)`Grx#;BoQ>KEtsk_h;&tm2NiA+ zam8g(S?k%?u~?uW$-d3w`M<-!0&Yssu0e|t^iVd9B9r|tgH4pxufdxkrk}(Qgj4ff zzdwHrILnL*-avDsGC6=5adh_;n1WiMid%kspnVyeM0U!SP-Gs+>xsh@$ln1Gk2V#f z)u6!GlSRu8g{VnSj!%;guH?B2h6-%Z%G9`k$SEXH6=hS|}J0>sXwi<)O z_qt3I?dgM)Ul(8Bv>qtAuq8zirM097@?>EO9JtlOL1;p{09ND^lO*b}mIb#*@d|ZQ zOFCNba`iRCV3I|nObqpJkThJ5cX2VywURe+4=I=<)_|bcO7&bvN_|W92i~=gt%wG@ zx}$(%G|D8CM4RMvK<}YC)J^%ZfYx;LPfa?Xa@PU{5xgLHVMbA&MIs$k!|diB0RbYp zpTzMOZ;gG|d6)LDg-T2-D=SUlii(LVrpgK`2cU*1ECj&&p|w?9&PMncKqWD~mxsYf zb$1+f-2{JDJ*$5;Y-HCQ-Z#?%`rrZ9ATs4t93~H4#Kjj4Ozeph+;p8BA~%S9sn!j5 z-%=HYVW~Eh-`&cN9O=2nLSvf4Z2>J{1xR^WNYxCWzD;P)9r$0Vtj-6lJb;=4=f4Ig zu2ai=zXt}${6$Y(Fsw^SAlV>yAe}2^1U(95h(7mszQ}2a^a%6zMt)(r<-BZj{oNZ6E1~%LL=4vG3KRGdngT731jcV zgtNB}xnlpuZlsA#!7hMnFyOK%-dD7nALVQ8lAbL$`tBsJt|5>_cAC}5>=dxr6yQvG zOyMWK1Fw$J#Ok+EHrd=6^m<7aJxLht3U?crg00}#(gR2l00hlqQ&W!O3q}Lop}!Rz zhS6G!`e6OG(?m2hdYax_N=xKy9Rjd$R`AG{2C6@6+elY7`g~=OMSfs4B-ysu+o1(* zNValAq1dHXAqs0xV2|=>o`5!1v9p`2e3g_vz)(hQrzr`*%A+C{;2>w4VMqk8VBf8f z`;Vyy1#)ECcz868gin7jal#b=%kWcasM3VYuQRLJ-%jvQ(<8%{ z9*%6MOvad6`j)98^8`zJHjFg`bEacec3%WU~f+KYm-*VvI#&hirrikf`z7h16l6 z-Y#1Gm|X3asKA;<)%M fKb3Gj6wHiYyljqzy6sKef)HDI_qNzA2Y&nyeP~T| literal 0 HcmV?d00001