From 20edee246cfdbed4f88a1639083d660adfe4b437 Mon Sep 17 00:00:00 2001 From: Kashif Rasul Date: Tue, 2 Dec 2025 19:54:57 +0100 Subject: [PATCH 01/16] add Transformer v5 changes --- src/chronos/chronos_bolt.py | 38 +++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/src/chronos/chronos_bolt.py b/src/chronos/chronos_bolt.py index 743ec06..6a4df36 100644 --- a/src/chronos/chronos_bolt.py +++ b/src/chronos/chronos_bolt.py @@ -13,7 +13,8 @@ from typing import List, Optional, Tuple, Union import torch import torch.nn as nn -from transformers import AutoConfig +from packaging import version +from transformers import AutoConfig, __version__ as transformers_version from transformers.models.t5.modeling_t5 import ( ACT2FN, T5Config, @@ -28,6 +29,27 @@ from .base import BaseChronosPipeline, ForecastType logger = logging.getLogger(__file__) +# Transformers v5 introduced breaking changes: +# - T5Stack.__init__ no longer accepts embed_tokens argument +# - _tied_weights_keys changed from list to dict format +_TRANSFORMERS_V5 = version.parse(transformers_version) >= version.parse("5.0.0.dev0") + + +def _create_t5_stack(config: T5Config, embed_tokens: nn.Embedding) -> T5Stack: + """ + Create a T5Stack with the given config and embed_tokens. + + This helper function provides backward compatibility between transformers v4 and v5. + In v4, T5Stack.__init__ accepts (config, embed_tokens). + In v5, T5Stack.__init__ only accepts (config), and embed_tokens must be set separately. + """ + if _TRANSFORMERS_V5: + stack = T5Stack(config) + stack.set_input_embeddings(embed_tokens) + return stack + else: + return T5Stack(config, embed_tokens) + @dataclass class ChronosBoltConfig: @@ -150,7 +172,15 @@ class ChronosBoltModelForForecasting(T5PreTrainedModel): r"output_patch_embedding\.", ] _keys_to_ignore_on_load_unexpected = [r"lm_head.weight"] # type: ignore - _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"] # type: ignore + # In transformers v5, _tied_weights_keys changed from list to dict {target: source} + _tied_weights_keys = ( # type: ignore + { + "encoder.embed_tokens.weight": "shared.weight", + "decoder.embed_tokens.weight": "shared.weight", + } + if _TRANSFORMERS_V5 + else ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"] + ) def __init__(self, config: T5Config): assert hasattr(config, "chronos_config"), "Not a Chronos config file" @@ -188,7 +218,7 @@ class ChronosBoltModelForForecasting(T5PreTrainedModel): encoder_config = copy.deepcopy(config) encoder_config.is_decoder = False encoder_config.use_cache = False - self.encoder = T5Stack(encoder_config, self.shared) + self.encoder = _create_t5_stack(encoder_config, self.shared) self._init_decoder(config) @@ -359,7 +389,7 @@ class ChronosBoltModelForForecasting(T5PreTrainedModel): decoder_config = copy.deepcopy(config) decoder_config.is_decoder = True decoder_config.num_layers = config.num_decoder_layers - self.decoder = T5Stack(decoder_config, self.shared) + self.decoder = _create_t5_stack(decoder_config, self.shared) def decode( self, From 784fcf1934b3896d978461b5858bc84bab723337 Mon Sep 17 00:00:00 2001 From: Kashif Rasul Date: Tue, 2 Dec 2025 19:55:24 +0100 Subject: [PATCH 02/16] add safe tensor --- test/dummy-chronos-model/model.safetensors | Bin 0 -> 111576 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 test/dummy-chronos-model/model.safetensors diff --git a/test/dummy-chronos-model/model.safetensors b/test/dummy-chronos-model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..40bef767d0a007f626e36c85c475399c66c0a70a GIT binary patch literal 111576 zcmd?yb+lIHx(57((ikWR2ug_(BGR$mrGkoBSlEStm?#G7wgpK+Iz$Y_KoAuJm9>^) zi>(;g*xg-QzTd^W&mU);+TT9s`|CSnT;sa#c%J#pC+<1lxp1Es>XfYezw=uT9C*sm z5rd8$G-A-efh|kVZ+YDCGfo*aqGf5z(?+zsuzky8hYlHj?9ekh3?4Ro$jKc#b{ID3 zTvo>p1BMPeuFr@OLr)!X;_y>DoZR7@p(h@Hf@bHpJa)vnrw!G9w=SJqwr_dn34>xC z(W!g;?p=@4UoJHKxZ}8w@$8`k2mRZm4jg>qpfmr+A#`!2|9H3S zE^Y{2O~pSww`c#47yB>ncinZ&dbdvh?d-1GIs;F&zy8My{byTcUH`7qqx*j|t(`jb zJ$1;gLx=nyY|V97YTvnIPlNd{=kPyX?_b`b>#o$kbJt$oJO019-oLy{*IlW7=UyGV z{Wp8$Ki=QJyi3+yseP9&ox1%uyQEXG!2IjGWL^KR(zWA%vr9U6*mdZsXAV7J=&-Za zi4NzS_&;t*2kk%GmFuq5-Z|^|->lkyyxu>xOS*Lak0*QGmD+dh)#JabWS#%Fv$(GR z_T4&m?E3F!ap)<7haNj{#PE~N-oH9w>pE@Ut!L-1|8ByLJMq*(!v_8@OT@Zf_3aGh z2n{{;|I=sJy6d#>-nmu{ZkA7pFP0VU8jBb?%j(|m4EJP|8j9ycb)bU8#M~E?tW?{-3?xKeb8!v+A_&O6_}f>RzlH|JC*Wm$SI8e^=?%@!xIE|BLFh zuG99NI(GcWTj{^u!T(8lDt4RRyA+>+UXITgdhEYiNsImHu4lKdCgQ^8{;#Cut3pZ1 zkcyI$mX#$XW9jN}GmL^UWhEt-;z43xRFsr_0B7*c#9x+|lx&E*u?OQlp(FGyl$3le zHl(zqds@Sb=H-_vrw*Y-TV3t%aG4XmW^!xM28d@r1*pY8Elm_v_+mK7x> zTkB^vy$`(&j4UlF>ET+F_1jDCEVzpQeYlnW1Mj`Oq~tkma*fvPkMTV1=HZMr^ZJJN zxyE7Q4fVGIpUeL(z6&=OYf3*)XB^`S?i)UVjQJyYxuT@x68(+CJ>d(vNAxTyxmK)9 zpI`B%?RU^n-nYh85AP$szTB?H@ONBWJac{~dpf=hMri*`c}dCP@E07%KB7=k@;G#& zAH@s!+sSQ9@6K0;u7;Ph_i?Sg@GsDnZ@Sz~@pE$e;EQqg-*7y-tfZtWOs1dWdj#6b zuP^V@QuEB8HEPd3nte3;SN%U`ywC7=*R}@SPLC1WSYAK4Kp$EA2lz85)#c36W-i}v z@TvG%{1eVz${1Q{H{UgO(f<#8w~9ScDBf4j(x!6W=DV4$!~X-M?^oFw!?w6N{>eJr zPX8hIFW3A)&e?RvI*GoAULf9?y&wB-`a5khx50_9Jv>xeQnGVd@mzF})0FQO{XJlO zx9MXR{7p`3zz;aJVGnt^UL)<=;OyUT@kqRz*m02avX(D%{SNyZoU`>PojtNCyC3WZ zng81OFSwq55T|}zp^ugH)36mx)ozM$onahp;9A&H&i7)Y=`y&aqNL=-^5Q)GK@S%@ zhMh6r1ADW7WuFVLz@_?oo4*c}L-u{f^ApaT)x&e}>7}kG|15f_alEMA5que6YR`xK zchfh(7}#9SbK=wKTrcbSHl6;nf1AKrFi&g)xd&9(pX}!N8W_#LKJKFLtm!JThg|bJ z{;bQFVh5Evlk}l!lv>WnNPQvgR{!8SFxzhHK%+_3>Dtq+~PWnQoswsohSM&H}q3{|@rb zV|SFdh5S)Cb!G^pU03+DqPYLBqqmZiy_`P(Xy(J}| z-{+k1{VHc?W4In)E;sX@G4vAql77qepJOk=d$2btv!?6^=nHXO{+!)*@^eo2qBdA2@26KC&T1Lr{>zTIJgek-lxt9WR+J&vbVSPQY9mG*`) zr_MeLo3S(3Ewt|@z8^hbo2*d_c}MX-Lod)@b-B5x%vbjLA7TgL-EjkLF2Ff^OXy?q za`+OSDip^zj-IQ1>gx`!RmPw7*@-{vlX!1!_ZPd)wU_eU*WKRduR%W#8_V6BPVN;ccYyBJRjZ^&)&#$^CJAByo@o=*8#XXf8HU^ zlY21xJ!n?229R@oIe*Skef+K5#~|k~^OpJg25QmM;B>kVgikJ2SLFUo=WN}H3;IZX z`+|NEHiZ$!-T^l#xFhH~hnod9A`l)H1#T&7or*q8) z>^^cXpjW}c`WfA;q~sj_vGiKLi&fBF@y24iXg7^cZOGhazm-Ai z#SxG_kUDxmWpU4TrOy=4yH9O64Ib34#QkJ!qr@(Q=h;8AJBX)NjD#K8SK$>fg>Hm5 zfo^h-l$SG@bC+{bv#i)|sxg1yd2qYD%f-5hwUnRxIhpT7s6(eF9V6aKyKNwS&*9$z zb}kiT_l8Zi{gAymy+CX`7^2++eD}Ck&S71sCGQsY%nHw1@tmIr>G$Dw@fm#cdX<#C z#(uS%_Y|DkbF;jh#{<}TPstoD&bUEwJ?d$BX-RmM<9&aHem!w7bMQ!Lds&#ybg zav$Z8zCVGZDvD!02UmqW>+aU(AUbCw&%3dFo5B9tUyFys{0g-QcHx^U|7`b={rCer z&;8V;f%0yVa|rz$oiRVo&fI1la)wuFH%)9qu{;w;(kI{}@x^>k%E^B0%$Mia4v_b% z`{C>5=7v2H_A3~dygTXsc%pdL)_)BwhEe>}jOj3VO>FP7;yc=GJV*QyzWi1kMQ;k% zvukU!2|WT*H~JaJ75qEmtaTIflNxaOKXSe{u6gu$*T{IXe@2L>#{R`Wggu6C6VRWg z{#H6KFq-cNn8E)6&b#FH+B^v_=r4P11#Acp@TG=LqZ?Pc9$tyBgcJGO;XKRtH=Zs9 z^UHqRz1&S_PIeNTBUa=7;yXZ9_T8?r3qBgMXD7++j%QaEf5%LhKaOudzTVJOe_Qiy zrQK?M=Gi$_?z{8};>XhM;T-x!?b7!=d>G$Q*d8)3kK>(j-UY9rhtrAQC^!9mrQbXF z`a#B)-)N&Mi|^GrqZhM3)b?S%65NyTIlfnLUA}>EnK9>(sNAZg*&b|PTfDQGz0s98= z8}KerMc&Wfq2II8DI9#02t0!N4y0+0cX$l zkP|)c=6)o!hWrkwD)ys(&tu-Z)7B7Z%65`4zj3{QfZWyRm@ zIV1bfyNKNb7qT0$ABu*D*qP50@lxCsXTHCI)cjt2*Bj3-74`}{V_gp)z`q!`f#>Ka z`Dfw1aT|O$PTjr+ZwbRIir*I=rC+8~my^E}z7Mx|jqLIC|EhVsj4yjUYj-t!nKqxx zUCWo+vH^c%7{qt5winUaN7-9F`P$Q$dyafAcOaedOoy)a%VhB%_)||h@{Pr*0au9K zCzhK3oA?60Wn$UKt=UJhhqJGxU#2gj+rsXy{|r7t-T<8Oa z3LeY1-9K;``*CfT;u5tu_q{+L8)}oilr@}*pXOUgSAn&39sD8AedQib$Fp(n`BhxO zev>|uJ_?o;JVSRd$8>7SX~uH4_{aQLRu*e(p4pw~-h2z>{GHBN)`J^h1=KRe?8Q9W zHWqKj&Y8VU{wT;9m;znkH)rv5eXgO$$)8tQto>EAIZvB@>^J_=&-L^oYw)w&v!IH$ zjd9jC_g~G}m&widzVwr4)0z6n{(s56e8nCH^Z9ZPa{s+xto-zOEPoI7EIHX9N5C`m zF!3kEUZu0AQq?k-P4AA*(a^|YrE&g{2$YEAm4Z1W?u?RU{AiM_20End~#5u1&X|uGV_}gbM{uS&q`G?WnD~oG#DZNMXan^n>`Pt)t;M~{k zmEO;wjLtdkBi0nZ&d#&5Cp{Klh`$uu+_;*vvo{CG*`B@(-yr@2%!k!toABj)eJWmq zJzKlpV%bYsiw|Igcx}Gl?2qix<@7P)FNrm#H#Cn~^XYsS^QE5+@Y(oVZEuFurHr|P zzkg-%89tCMp;wA$Pw&cJpFLjQFZe|EwftG*-Raz8bv&27A)T|FwMpF=M{fj=zrxk za3lT^^e*(F`fNsL|K%K3#ZB-+m;lw}F2ZM3vSA~r$9EcE+GTw4AE%$uc%axv`WR2| z2NT)ZC;cFM^b5XL(3YM3y%T-{u7`Or750P`mEMi;&X6;k@n?VE#h&6?FS1`^H?zL| z`Kr>HzdSdxj+wKJzdn8jZlp)?52G{pTj8VNSV&zj<6ENNtMKLJ&Oc7QnMc1%AUG^SwQb(@AyWrYlH{v|Qa=ss<9~654USMCt-ph4< zgf02z${Sx=JX2?gzsb&ezXRv7GvC|cXCSq`5#-r_3VsOxu5Hd?Z^*ga8n&w}zLP&s z=bhpa*UI;myZL9pEqpmoZ{fx8FyE8-0sKRmHNZ#XUx}I+Vq`o}D_mJ4p?7BGhCCo5X9t zr92%Li$J86n@Z07ryK0YYLtTc#Zdyo$1rG-2*0xzin*C({ItYh-EL2WzWZ>A@!{f ze?PvQuQmEfJindCvKy!+`QFl7KlxrVTl{nW_2rDi??5N*YssGv_wygbm;Kg_uEUr8 z(1b7hI5lCs{73Kucp}dD{-pDF$Yt8y%l-{=7IU3y#&8}!vCMhEqx6|?mwCTCU%SU3 z*P4WLmJfq@Ftn`rdvFiAH^FA?JNZ*ne#W)n2$;d{t?l*|#rNc#@n-a0u)FxP_f2JwBso{e9Jnw9nvUMZGmPp;bo_r`Bms8RS5 z$XWT5J(EsO=Ji9q)R(O7YwZ5wITx+61Mvuog;Z3v| zgg<6)W`E|}b#mXd&hO!~Am4a$ZU*W%@qFVr1lQotGjpKaCi=;{!?S$X!Ao!`++SK; zpS)X4q%VPeB6|SdG({^L=KiHd!r|ot6y$qiX+22{SYw?!2sobyV&$XQ(FW(5x z$GNwae0laA3-8kH<>vVx-M8SmE^ik+M(#y&-e%VqPo24tZyMC5tKdzcyLkQ<&>uI! z8An6yC-gENdO9SxlXx?ndwI50Tk+iAhp>#@#oVUfx7as`qV|4UwJMe#SlDfD#y_wX9urt;nT1~~_7^EG_{e{+4bqu-(LcHKGnCh?zf zU9s+BJLvC0oPGJ6Yi&foBK|w9fE(pMfb(wAP0ld+sUM%P^Nw;jUEsT1-Z)sze?7d+ zpZ$_K-G+S!-40gE$vkwV$IuPM(|^{lJAEV`C7wQdvrmW41!q+5BzaBY4YB-9yAQsd zo%t`7^BBGbE-NkG--g;wb59qFEzJbeMO;PyTgf{}AFtsv#cSg1Rt2^a=c(v}U+Qy)zNhdtVfXD>d>8+f{{+6&g!|>bBz7_6Ec9l7gr9_q z_g1F>0hvbXY% zm-D%ZeGTMy>L23cE9`giS0R1mx~b{8w=c!FqbuNc`a8T^Y4Q2gK>k+j)ZlS=E@V#f zzI27$-S~f^`|=;d-W=y$rYhvQnLV3Y*TFbZR z>0%w^K1shy?}_(;BlMfP_XOWGcAjTd@l8FQ8T=7sZl*%6zYWfMwt(8~A@aIta|-kj zI|Md{9_+@@h0Yj;LH1hSqYuQHgGLp__o|%1z1dsC0mj{pojslB#aD7m_-Ek<_|7hM zZt=D-Q#>_vbG&6Q`$%5aZ5d=Ps)@Z0w}@@T-btGa#OlCH?6<@oOia69*xR$`L(b>` zzIX9GICUrgcIy##)?y@{45J}6c?CWg`qE9|Cde4i<*U$t9eqsI{&m;AmG5`)E^x86 z%6;dVHwx!DoijH|-f;14`Enk6@tuU9=RZq-kF%e@r?|d5@#lP3%Dq4=Yk3v>2L6n9 zGG1S-fTzMYa$eETnbsj^DChG%dJ$g(_BpT@-*=F6w;mkmY}Vr6hp)1<_zd5cy&k*+ z8|eRBNPYa>JymFv`PmKf?%7K$bs^`kO=M?*dO zSb0muvafO;uQ!*Qv$H#$+U3+=)Opf92&c2`BQataP8GGu+)$py@ z`8dzMCU|6N@maKmex4Vr&$k&}i~j>{FU1*O-tBf4dy#G~o^!ZgMe#l}#<~0zaJ{@^ z*~h~j?BkPz>#~2>|2FKC>C5QJkebs7|ID|K*hGAlca&$A7vCFBDD@u3e;fbb;YN0K zycl1Dx0TlxCa^DKx0U;`{vI`Fd3GJ1x#RB%8Ba4jnD0^iA*_Z1TxuL^ ze`og5Flc%4Y%DUqhU|~Tf5N%fzr%C%V95TN&c2a8*!5CB`mmFix|f=q+A|tnq)&xk z==ELmF`U|(IjmpmI;?4Acr;u>XAf=RJ~Gzqy=wA$!8F(&4&}@Fy8^O)lZ|Z_eIuTY$HUz)L_gK=)pGh6 z%K$i?-i1H!dsn!|0(PDg4SE&d3;w2$v*}jwF8`k5J@HbU`AjXZs?WSvoWz%NJQW{~ z$HK1S`{-k<(&F4^PD=PQm)RH3vU3g|fQj0DBX&HUytmo)*gw%D<#%JBL#O7p+W_J`~}^x2-?mG4$|=6wpT!Z$(gJbgUOp5tEM<@*Be;jf`x>gd7j1K6M7-t2zz zI^Z|hsdd9}r9N`@Pb)2cGd}<~*FI+>_xr5)Wqb>?Z6mJ^y^1gMI$6$Z`WcQ7Vdt!k zhMd(=IQR9GHt*Bl)7k4;*X+$weQZxJ5$jBsz}@souCqcsHK-b2zKPwh-BU1%KhM)r z-w95ne}L=Rzu`mSNctkNns^8He4Otyc_!zZ;U2gzJKu9&6h92M;P2|1TjHtWC&PoT z{W082KQ1S8QGqv|NwcgB|77=F7dM9&L%2 zvirDpIi9B9oSEkODX_P}3$_1+PM=d?g9>L*-nOuoUJb`+pJ&dF@^glAKc_>!Ggk2D zJLskQnJqVG|9!std>7&K`7*!9;x}O-U ze;IDTj;{)I<=YTi@@HSZ4@VitjoNjwu9^Rwl~p+7OnsgQPeHEV6J~3lv3^o0K3{+2 zdrLpF*cZSH?Bnnw=@-vr-wK_y>0n;6e^a}ci+AJ;a-Sc-BDkVpzv7$qnQvLySC_Jn z!pE|^Ks7r34d&1HvWE0ybmn1u_NMyY5g*L%K&NhBC7$<D1pX<+Y>FbdBrfd?c24?!|PTB~9ei z6|X5T?{$;e>A#nFBWOqGZ0#)G2eQt8=idfjsh_L0xmnImmBo3UM&B=9!T+arhp?|^ zpD4Zoy%+royeZa$J(vCi&ZO7Z=XiZq!QY9WppTr(Ibw&x*YbA3_d-kh41KOokCAhY zb~Chjp8qTDHo{q#yCMDNT(rXnLS>s zwuL|Vy5R+UpU~NB&(b+FTi`qRTGN?}+l{La-_i62eDBfkKs(4>d{HPqU(Uv7iuWk9 zCiock#;}KbxuC+nrL%7a&~_g?B4;^5<{71!(qH~?M=o)nHf4cje#kUf2 zAE^oN8eih62V=E8UCu~$3Hx_#UxU=87a`BV*jcmetB3Fmc0+A8*X9@Po3VGGH^)_B zHOz#0ayn>}ef0}_3rG#fv$DDTd+5B&EHs`A=sfo`4|95{`M5Fru?qV}>{HiSk8aD~ zg1s-kLc0ojXa2lXRD(z9a>&`vTArtmtm7+Uujykk{(+quxe5W9C-l*0Bb;}_%*P#c!Fl?EUJhB`$Kia)`}4gx`>PTCEHtA>;H-cCF7+lm z^PgwP&$uc73fRXuJ}Xt9>4)*p;#uE%xJp?h;x3fq3h!93-$}nUh1u{o!FyrFWgSsagg6q!5g&z!q&w3}5?{5|>}&U;Q3@tnIAaxcX>XMggij?TwB z>E{hOYv2ub)}%3=cFWi~W7pxWwaPGj} z{xf`!+>7y0{@hFU;CMWco!t76K5r;^9zyoqZ{ic^W_0Ew>v^_*-iAl@o0_q?cDq7H zu|wsyphwEjnQOq#8Q+sVp00z3;vXUVKl6RF_?7I`k2U&Rsox{nt;C)wFV^ks-Ta-n zjhs60ki0wx^NjiuH^4v2T}odiz8if^MX}#iICDA(pMd9!mC$*w-HqJ~=eclEuj22z zuIwjZP@(vaKS!)DJNc=jd8f_yfb5%YW#$aOCpJ~T*_UnU%+q~z&VSygj)P%^;`?AD zZ8P@lvqQDXn)j?U5BOUCeR1ANGH-|Cw(P^`QteVF`tseW-3Ro|1#82f_lo>3sAL}q zx4=|5jW7GCcUkc_P5K_ipL_lg=Pd34Thb@!fL*~@L>L(D~t7{J)QO3 z3V$q?8j}4~gIyEH-XHd3r{6!V=X2~+w7DEFFy8cWFnhju&ca^wZhV)}9U*Hx-8fcM z6l+EH?W6pe^GWy~c6B@w^2|y->rCgpH*=V0T`$;<&O7Qj{3X5-XWp`AX@3`d;yShU z@f!a?_HXbwr2gfaFTgHz?r}5y-^s4f{tWnxzLLL%KGx7Dt4nAG`HDC6==lDv?|;+bey}@4mqK^g z5&o^eL+P}OFP#1}rsM~^K>WM@!`>ffOz{O7M{;(6^p$I5T-hIeDvR&14b4gRLQiva z15BlBi&dDnBgL|xy7Tp?^UTZ{ZN$zy(JAut?)4YWnK+W(xU5)T-XTPXg!g8Y4H zE`1MFmzVQ@K3|ne?}U(Vg4@X-tnD9ia;=H%cI=$1?5)~3^>{6;r_Eb9=V6uDRJatr zffD{ahu^IzzPIJ~!%=kh{0sPcXjf3GkvEI}L{83sO?j`#dyIY_-lsQ* z6?DG$-#}kM?~Zd$*3(D+#`Lm2^6mLGzTY6+qQbKhPr;wSa(I>O2M46uVZK2?c`^zO4!+-d7hT?H|PHqzsa95d<07>wCDfQ zbyA=5ttrpoAN+nkM_xa%G3+{UHr!NDUm)|H_n1xj_JCjbhTzNLK05EFP3VpA)%Y~1 zMrTeo7Qcku0;jg-9#*hZ6Zdod+~23rg z0i^9g74|=U6<&lN*R~BFP-g!5UZIbIryw<95#J;_YkZm9%)NKp*4z zMzU|?pFro`)$mppLv(*4QpX+g?kapb1`+RI=n1@JN^7Fp0iXR z*J6*9GdcaRr$Rm0i7#u|5h}ubQ(!!#roQKzZ{lSYYM8bc(o6JzD&1Dwdirh7m%X|z zU5DNfk7Pe9=Lq~RKHgm9oDX29HuaFxhaL>eVQcf1zuj!l-y2pHihchkHiG>$`x(gI z8VXDGF_wLqcsb5|d;vYgw&zP-&fLfD%kCl80B62tKuz{LaJ6@iP33jrJB9tNv3`S3 zVh`qj+I8}7x+i;2W84(yTV)^peveDp`R;KHU8MlptT1P}zqi@>?YaVI|NmZ5eD~c@ z{$u<*%Gr=V>vIs*|*afZ}xQ`dT%M#8MkS#NFU+m|n21+O?sVLf%=^?>6*@_)EM&X|eung=;QXE8>^87Pr>lMvz*V zdepJ|KzVIxGt^>Rpf0XFK6sA{uAI8{r!R8 za&89m*KuZ_)8{9APr|+0TmadF+Zan1dT)6Tz-RQA@CRf*f8fjB+m)^-_f~pq{5*aG zQZN207$0OW zt@u+b*OaQ~up8egVlVO4mis5)8F(o>`|bfKH;Lfgh zG43vBW88+F^S>9p9$x|ST$zCL49@wf#P8r2_5D0vz`hp00F7XT_Sv7GxK3(X8|^>G z4~b=s2Extk{9e2rZ^gbH>cBPZneZB9%%{o87&daPdHC`|@jU;4)93zjQlGzt%un`M zb-rnI+SM?I?A={)=5|}Y%jjBi+CtXzX?x}?IrYp>&f32G^Wpo?9_+s{de%o^6!VLVmTW>iFd>$d|#BR zqjFB9^InmfmS^L^kTd%#|1MBUrv@gzEB!U!cE)p)zEU&tn`wJ}tr0s=+l6$_clJ%@ zCvBdibEe9)Tc+J@bmpK59Iwrta=wJo<@OikyU==glz7g>{xF2^S2+!FeyeY$-OKuW zMeHEH=8!#<_p$TwUhKZ|?xwSc-oQE2_u&gGihsA;Q~V2d_F8Q@1H|8Cw_*>&4Pbvc z)o^2o|0H=g;knQV@_VQbUmv)L{d%Fe7jDxo&zwK_E`#cjbFxbQr}PYZC7uK$%Z6^u(PM$qN~cw z`_MJz#!s&Yd6wJ@JM-@;cZRx|x>ApC86F4eqrU#0!zXBy^L7Os3#ol;GY&dqxl(*5 z{5oI0C8kY&6P<_$vwzc1_I7Ihk@x_}T%J~NZF#9Hd5^dcuC~UhXZu2)y{T7a_(>Q5 zcfovp_TW#w&NvqEHTC_VjJ+8>qoVjbYdQN2cJ_9E`ULF;YF7?x_^*LCpd;j(i{Pq? z;yKEHdvbkQv7hSn5V2F;$33pMM9yW}JSm=iaZPOTJ8^1C#xqggIQOzd&bxd$>xaP# z`YC)C{s>MKp9FWpsnEUBHJ~p14xQl`dDY9DGrSvn4&4{B@3S7m=}loU{L1c*|Ae*T zm6hrlU*1DSsk+@jQJ&Pl4C@ALdIfyDu^P3f!ZQ^wUhvc$h}7XIyv7 zxzY8L{~-ON*h%zEdS6@*#?yJ%J(2FG-C|r9H^R@$uOZeNmhfeK&$zdor_aU8*tzH0 zcnJGK{)Kn|9t&;hYT|d$=fGBU=Km7;Ki~ls#lL$+*U)c6WBrrPvnKO1m~Rq$9!^d> zzHjlG3h}byZ{lHMo!Kw&Zy?@*|6=F?$3g0NP5dd&zHBUSHe}znt@NzW{uX);{Xj+W zd6+(4pf|v^_;bD8>4x;Za@W9}(U7xqH{|{%*@x4dkxp{2W@ipAr+?%7P)^44KzZ>y z#+BM0#5ag+4;s=z+?C_?j7I+b{Fm9UnMX99%F&~c|+_e z*PKMB?)(Dx%1J#)n?L1dpKgxN#mCA!3a4ggUMJv_;TG}9ko*6=P#nh@{8i*+{8#cF zK@WkGj4yk36aKmS+lX%)dMV%Qbk2W&`Ok~>;LBW`2_N&X!AJAmMCbQ!Iewcj>yWV| z?_4<3J$2Q`19aX;Z-xQ%E81qiUXBmNEAe0E|+gJVpboN}HRW3L$A`nt^qshKrFEbe(AjtEvs?SO zCYiU&3iSZS@V8;--M${K4ox8MLzA`L4_}0{E;V5)`&pd+wkR#$=ON;GSG$I9-Mp}S zvwy^W@Y(#iuaDVJ!ux#LW6xF;-w_6iodmgl_Q_WGVtqe>_hx?tFVpYyeXLDG_Fcwt z2aMp~4dz4o%lr=IznM;r$yxr1zKUKZ=LB{+WNq?Wd>MbP`!H^S_r{ZyIr+D%d(eaE^YCogi5`zn#|?3<%3@u}*?p@} z{Cma*eA&Ohx?axELpXi^4W`NIE0*7;o0k>O>K%M}4tEgGp1GIK9?v<=ef49nl9L+w zFnq?hAwCGQuQRsm#52|z_%1vTUZZo}zwiva9`49rLXUC%rS$Fead-o`6kdV6$F|2g z2kYTk+I=Fw8+4}gZaha$_VW1^_7=WO?i>8MMr}H^r8|8W{u_VZwU*#fID4S0c1__0 zzMbF~eP{ev;=$q%!Zz$jpMe?;10`hGC8CDw0OL#A_#T9C;>u*7c0?@_V&c(&M%xDkwpk1LDcMskjpLiTRfBK2W7G}g~_cG^7+cR~&0*wtL5Ze&f5 zmREsOBNpSFuWEcByZ(*rv2?~gl73K5_U?Xk)j(fYvcKm)NxNqBMlbgP+8m&v7JT>h!cv;>D zxDQN{(_BvWPWH+7_)?sjGNa7zF`RWizu6ZkebU*Oa(`d5vxc?g-;6VF1NpvX*Ji)Yeu16?**7zdCHFiY`rJ9}WX`1*1)zWndnd@MGO|3&s0_%N|o z;eE)r;)D3xL-ya=f3$DSz7@K_k&yF{`j9iYiT)-+8*Nihc7ybpwfYUl%E{il6JIXx z@AS$1*_(6WRXTg;81*5~;XJ#yf;ZgTReYb)i^|<6)P$zcTtB1WTe=3`r?mL*VQbQl z(>G~*6upV|weWO#&(P=5lf)aapT#?~@5Ff*nv5H>Khw5N!Mh#ZOx|erWbxM_=WC#E zCdqqR{#{VQ_X6IO?*_T4+s*l^6}&U?Z-O^~cG|VWkMZT5?ln1`D~k6Uy*_;eJR!C@ z-0WNS*WQC~7HjR=(;@F#IZNBpnfEVUFZZ8!h~4<}9XxfqSwTPA=YJDC-w(HwpLdTF z<>cO9rt^(rU))IkIcY=x3Mdq_Em9x_AUIK;X2nz zyT8*7_4y^9$Ih778e2!cci2C&x0Ksh+dt@|aQ@q?-^8|IkAhoWZz_9;SiX1e*Gt{N z`EQvf;P-IGw35C;-feKHc7Nwz!`GPI+Vu{BT_E?@R{R}!NY1D1TjW)xvySuVWaiU;7A`FDl>kbQ9~K2)1~@L}4W0-M1BrNz0bCMWm3 zDRiSBmbX3rOMIJN=7oO(eL8=8=r6W~|DE1v>Fn>8c>hxO2)9FrjFbNf{;X#yo&EO) zy;6U9=ALDq4rXt`pR+X>+VfY3^Z3TXVn_{ITWWu4_nP&}`hUbX1zOM-y4J-wHT3Gc zttZ_}EPFI?Fy5!xKXS4rx^<$;+NQ7dI5EPOrkLW6$aT z7I`h@9ih#5dJ>+%pM8-3F6K9!b<7@qi`~D}`wqJY^wlo+(x#_(NcQKDdCzzcSHsKj zX0F?SFYA+j8q!a~Zggk($o_u`=f0L1(@mx7KivqQu1$VxEu>$8{O>!=k#m>$A?#UV znbRrquBB(wo8uw)MtN648+vAW@wt(+lQr$jSAV(ZoBsC4LvU}t)a;yvtm_Pz09(S2 z`aBbF13Q%Z_N3qbu%+C|eDfjudN0_bwD^9KdbyW&d0)HE{2a)iy12%?d?hb6C^@NV zbIieEcm(@av58QFt_^kQ-2WzYz8{`PXRJT-XU=m^FTn}$xp>xcT&Z_v_jHmmW=}lq z`s1{3A$B%9&&t=uvxmlrWe?|?Tj89yE#Wc9p2&XMM!TW(wR}zK)Q8D9`>qCGSIB<@ zF@S%Xyo|3gPG7G<>ivCqEMLa+9R3WCk)PV|4bGf&)@PpKvy7=J{{eK)LH1%b7!4WI zFo-WT=tF!>sc-!_`YOKscjLK!-fv#Vol1*)`zX4%c+O1xt8lJ=io9GWwK99CJ^MYiqkZ`%+`5&Th{>k)8Lvf!YngIY)m$o_9l`hMZ3LP_c3R`<1E% zboNzK$T&X3=kV2(`!Rb1oSKp6^S8^*Gh88l7CwOgRQ`;+1>`!Z(;08MoDcZsz+7`) ziJ#JDJ^Zm)&i)zvt+h|zSBo9bmot|6KC4uX7ta}NB)*pY7d=~S61^)F=!N>qc^N=w zA7mZsz~=Op@Ex7GY)R+eFSo`E^ppO|>Fc15{s-YaS103*ao!p7ySgSj?`?UG)rSY+ z9OKG$dX{_M>+2ePu$=ATM|uTcGx3j%ZL>o0UF>rHWBJcBh7Np*=S*Z@+zWYLPS$=3 zJ-pl+%jwVV4h{JxLe{RYYY!3oQ~oA!6aPK(%is){0kh<#KDENPmGVO)*Lojr=g
@`{}M_%?=uHm}~N#?p29^GO_YiGPIG=i5g95P4tGS?BZE7s{K$ z&YAxb+VLL)kJCNGKGXgW?O$ZKg&W~|_&YSw{%t(bHT&VC%RLY1{cvjUl{n)+1h-{p zA2gtkg6#j@*^}s{`p-Gd8Jx@Cl+Js8YFG(e%D=69zlcA3GHbRi&Ursc+w8eTrOqQ9 zMjwW2i?v8Q<3FxvaerqVz4%h6X2|)2oxSmwoILM-=i5i_MDf(#QwqgrbM|)XbLMC< z)P>Zn?B%tPy)Xv0mXq_8dpz4Wfx5m0)Pob*p8T zv6`?MJr}+#m~))(;4|r}^kBKSP-{QMr0pI5A=Gw%rNhI0)HB4uxJ_IjA*64b^PTJLFx2M-g-Az3A-FgV{Z6NmxzAeNS;9h*I#g;gu`BprW z-V|?vpAgI3eqOLAa2I%zzS#I)p+|nJZw>pg&xO>9u{d?>HQbk-y^wYN6OPs9TlQnde>EJz z*NUCB{TrQc&952XDtdxG_BK}wPWAK@p+ zTFel84$p$&bk6YzE@4duwcAjw`E%@K0UxQCzWBDiH-0xBJBNgfj z{U@$x98xkoWi#G~__9~tsVKf9E|fo#9we3; zmix*1-kIJFDhi&r?2XI3$FiTKkESz!^XYT-mFGl5d2RUSK-M;Ex*c5;Zl+g@x28X& zpU~zN_9pDqihW~4-XHda?CT4}SINzOJ%ul|;8#3{-I70hHs>Vk^d@^Uoqdt|G88h$ zIoI>yEV>HL{8iO1Yq^QOmg3Z!f&7{G?1kkp1|DKR2A9J9aA2WWAM?CFy{veh&-r#@ zAHqHuUn%by$Uf|1?O!b~?)|;RcIL~yRNVzrqAc( zcBD^%i}^dSccuGPI+yZ))#p=q8UGmZ7W@ye&xgKr|59tnU(n`NdJ5iFEa$!o|0Mh= zPTMYY)+X`fkAZ2G#ed5<0e2IB8*&D7o$Q@ybR*c5-GhAsZc!$Oo~WN+a2MQ*ZyenO z=lNTW{@p&x+*|>T`E!o4M>4;8uI3En?C1W^l#_G4OkX(%&BT5a+k@Vht|2FLFkC!q zoNtZim#QQDIfolT*8NU7GoTk_ohvFl8}UH7*T9ebKk_Zc*>mTM?W$dB{S^8G{-@x0 zcE-04eU12`^a{QQ;Rrf)@fP@<{Rzx2EB2M=z&!C0eECL@_rW$WQ*Lco1Yh%K9M$*_ zD)XBcwuPhk@>?u>@(#$n?ZcmcKb{(v=Tpvf_S(Qgu_o_E=Qq!`{B8NR$M@h1@y58P zcYnR>jGZ++OLviQ3v??&grymE6Oem7l!?iIzed7QlL zuSfKKt(?@>9mUJ=q3|Bvxw3ff^7|s^{_pZnh0JHxu2N2FTLn(ddqF>0uUo})|IPUF zE}XMfgTJ%2-%Kp+YlwZu*G+65-G;74UkAsqkAu|}`Zwp9n^$2xWPGU&Cs!8Vea7Kl z{4ertCiXF%z5Q1~dwu3wt?0(`Hlcr|f8gJm-U}LO^DOR+EBL$6a|*@h)s1v}I?vs; z`k$$fK{##t!k%z0?RGZxNZSHdBc>M(!a z!*|E2Gb3^SHx2&(j3M^G4D*xUHaDBEtLfC1N%RHw)OY6TKE4xR2mY*C_T(1k?jc;2 zFVBbubnOaj2z#})ext8Pf6f0cu7l5qBjs#_FM{lsN&3E3AGa2Y??`L3 zJ4Wns_zZXqP?Q9I|K6 zgPen@P`yy}rCv8-ua9rW+vD%>S=ziWXS%)*bd59l28eyim$P#y-5bBj{~OLdZkV?8 zENxStufKK!olgZcaMU&wwK=NYzpWpNIF65AVQ@@H*I#D?Om zW7Z{SYkq~=M|b3V*gfZ*Ei~@bgTLsk#gX)=LUD{cSpG>K;Hrx&m#HzL(WC&Mo;`}p}2p3FvirQZQ1!vxr=!841Hv+_hx6^ z8qpi+;{-X+XHWI=+eGXhvG>F#;MXDNs+PP1#Jb^@xEf#X_tgsDsp+BmSc#8ujqK|@ zce4%y-UwL=UaUB zUgD1Kj9!cKnb@oDUGyAv|G z-SZCeK4h;QpzZaz2K#IAcinIHZT9S4@DyE7P9L%1m1-2;k8gokp4VxgHJC_O5x)?( z#lPUa;U4{E?y?Uu{%PJPZZ_7eL)K+8eNA0nvhH_aD}9ZZTMD_~oY{WjwIT0&wb}Kx zc}#47*oK~i>x+-0?}VDJm)dy}&RU(S&Cl%fU@LvRQBmBNmBz3yo$o}cH95D7+3j$B zyqk78qnERnh!2HT^pj#UAoqL_JMXhEZg3g9C0?Ze z2er#}r?}n{oHh6o4yurY+sfS>E~B%jc9NI#cn!N8@2#IFaIWC?8otq&`|E3u9vg(4SN>-hWHpdbs^^_XJVMV zSNXEe595LQe;a50zQuRquW-)#5ImXvcjyC6;rW7jhZE?xjO#``O72j;0g!tBsB72e zPd#YE_aMw>KMi>&ObyEW{%QQFQ>o>dmk-(fpf_I&+`!s=uCJfuuXU}|)E^-4N>B6W zx8BQmE57Wv?D2tx;+}Y%Jx1OpeBJOA_Kx&2eLjPC(RLut{?50w1@tKT06YYyz$|TU z(SHLv`*#m^p6Mm@Cb$oDr^~cUjXn%l$XNqJ*~@ThSa1F3_tYV3)S)>0{{y+JVNdpp zcqYt+>tL1mWB3Hfn&>)W{!W8kn`1j;*0UMST>*Gqkx$>&Ir;IVbk#-V$ zN1Nti=_~JGBl&957t-5l-$}d;evF;HmHjpbvVZfOu7TU)j&KCsS$@vbQ*!FS8}x_H zOrB4B(=XsVaDHe0f)9kV`7$SGz&CUk$XN1T^(fBsbu0P#{e2kj#ZLWwPpmV29N(^Z z1wMyA=l(4Au{h`G1-^$Mzd6?6JQohYS=+vRM?x1m^OC(h0cMD&eO3PXxC37^+?qdi zbv^tkjDwFLb9OJJ7N&O8rNdikzprs;&*a&43%wbgc%B&rIeBlpn}08Meg0hMM#!3n zpTW83HuO}wv%J)i^<3*m?W&0#$4(viv7-2HnP*jU&K6rH=L~J$XWv2RJlrMkFLu^A zXK*j}ET{tevNz;kr2m)gX~kYFKWFx37!B$BF|i}@Xt_C) zz4_;}bH2}G55~FwlVLi4&T8f)d#M%QBd{@Fc$j$VXKHXC$p4m1O?){23fH+wUiQ}E zcw0!#tS;W4&c4jt?ZN(@|2z6a`@c86wflR6oj%u-cRxLzJ_%ol^PBQedFRmCBRS96 zyJdW1*?Hb%?l!`CH)ullf-B*0Z7T}J-|ne**-LZz+T%}+xtCbp<7QVDpM%@*rv{%V z_XXoh&S3gyIkm;|e$tO`41SBh9^Vc6c!Mu%aW-G>;VALU{s8fc_~I(1f(e2sB5)oQTkP)c?#}xjpY8W`@BeUr7_Twsm~*YY_KLap^PIzh zpYbd-FW=bfWW>|FgE z!_GaP!Oj@-TmXX*DD zv5nao^Ef#t@k{c~#WVT&zIqJ*bpDz0C$sNj=iX9RE8t5xr$mRGy}I&#r_Esxs4PB9 zj_3cwt|EVX`Z;h7Z5-Yk=CO~gEcR&g)x~}EqunX?Ae>w(z6b2VzCm8*JkO#@?E36x z@F+X`EMt0t-G{ar?kevveU4`5xpt&@_I=i7FT4~|Lu%o?m#rvs9#`qx^;07{$j^6R zZRp$SCwr|uZ4`e5&Na`XZN{IE|Aen$1A6x1$uIy~!8rZgUs=3Q|IUAiy+_GDhRxtW z{copTQ?ZNqhp-QZoTt-hIlKGF*(X+s7r`~O((0lvZNSbov#*nvcg!m))dTz;?hH-kel91ux8mgFjO0D9H|-|e z6LzMZB|cu8m-xB2*~#Z$D7G)$V@)@~IctB4T?s=V&xacFZih|zc@Iea9ig8aY1!X3 z`DO5uSoXkC>`MLh;^#hhgthF2^7mn9Uo>RrTzrpP(6T4;Jwp>(-bL=fS)a?{26}y* zbN#uWC-hKyBZ$?9gOpMdew5D z=jdPb@*S~0vbVD~i}5~Usg2LEmx!&=X1-X~A!~9Sbmdw9|Pp4&XO)9xxSf9RH z+iS!+(tF@L@n^k@GnH{=etO`{OX|wU{K4?PcA1lTv?-9fb~J4*?7&WcvA2>lQ+yd5 z$NmVPgtv!FRUFZO)vka^E% zodx{2%ZmEDNbb4(%lKLQBjIKKX#M<-bKUHn)Z=6LXZH5K#(yIH;a2>Gc&vB_@zv}Z z|M{o7)}gf4uA9%D*!p2Cni++V(T%o%D9--t~XTTaV7u7I5Rf9NA% zu|9rtMn==~v#k$t>Tx%`H_q=6bZ4io=kvh+v@6*+;F<6iJ9}h%{p5M{MOpEg?`GVW zpSd_1XKtP-6}5A$SkBLUd;r`6+wvRX?2SC58dtiGTD&nk?+j()=h4oG%yI59bt!v0 zb@EO*$3V`{C$tYC`{)I+*2Y%iSJx(eJ;1&l7Km+(zs5V27tfulawpJo?xy1I^vrYq ztwH*4Y|S5{&lTTL>~nqgVUMA$4=1uOr#EMBjGx03<@D7ypXrXFt>Wh_?IAa7xd`9E z@4|kMUBxxBmiN%A(|Wsp8+yi@|6?=Hks6TS?Yx`zig@-xp2^v35919XbJNMS@{Bl< zzNt2OHvC3wSXR7SH^HCs52f!UKYJo$Y@kiclAp^#&SDFB`?GHq&vShV|8vOCZR?94 z2>bIt(ATN>k4onq|BX{)dW+v7cOPxL@pqNCfmltP{!gQ4j1}z1==t20nmQlml@;sx zGR~Sdk@Fk-Be8P!0&8<7yP?=!xIb=zr{MSSrFg6I;3y z*co^Fz1(>79C(7il>M=I2X>ix?}Fce?BT4%N%*Wv{lOq@a$fRpi|XRmWyQXzj<3Xy`Z}tG#o&JP=x)g{& zct(6rbA2EF8S+d^?T%hod=}2!_MjgQ^R&<0T=Jj4qO$l*bwB<@OPF5*)*#~^Yk%Z(Mb`TTZLh*TA=lm>&xC8(XUiMS-pm;KYWqAb_nEVh z^Pas_6`w~-J-t-kqp)eIs8?0#4aEDo*2e5Z=`--w@DyAimiPOPIJGeIy`h{v@iXFM zX*kZ?Sp& z;6(QB_SXIMk?e2fWFHQOZsJe!Z-l+{*@2exdl*0U_;2_1xVCxL=6&mHvFzJ@*<0c- z+>=87V`6a`vcN<707vO z3)R`v%Zf9-2fq${L~97k=_B0N8Rgz5U~7KP-v#((I1?_ZEcRay?K1CA(I&}B-Pnb_ zOuRAPUo4;7vM+1H!;tY;(mNUV5N$fk%Y3wFUo74nXTN_a?-Bi7&dy%_ojrxUM$XnH z^$b!2lK-=KJNdI=GGwjS7tb^DGO^UM_j?zgWv=CC9&*hc>5uB;O~^Y&_D1eG)+i*qi<%S&y_9{UZxhy9?5ob0o-J-*Vrk}=(fJJ61U z!|184sZ}q~uf`YP?dW-?rC!~{??nGx+o|lopbI_gJOj@ZI}1H*I2h z@tw2cA@M5`5>GPq#yq5GI@dW-PvDB*)?uE-}d&0JmHOqZHD^^yrC-^x#sYB!0 zS3&k*O`LayoY$Pchv7v2#yIZ`sY7`t498dFzl`ZmvEBLIOP;S`=*NB%r!KT(=b3dN zF5khf!HfCp@M?bEC*CYC-j!}<4-k7sY#_9)Ec_a@vtclNM$6gm2}i+Q z`~~G z7JsOAscE~i|HVh*`(crOo_D>}fj(u{ZgugU>_~d9a|peaylJ=@enW2N>PpDj%e=l! z&og1Mw$t$j^y}qiys1~4@t4D`{QsNhJkO?=6*cM}dam;j&VI^y{90d`|E2m~sCC=X10a8--MXlS0pFMRQG@<9L^(}Yb+WgM1TBe56HpR7E=Wa-UZRK>LXWn0gZP^>* z;bJ*^IdAplWG~!6AIv@sHlwFzbf&*YKb6)>o79P1_dD7ccJ^xSdnW%R+)`{;c6;`t zV!5x2`Dd^j!Byg^Az#q4X1R777zXQ!kHA}$)DJv~f1148*q`Bx@i>?PjretN`s)t& z8ShoJIgouhsM49lyNWG_Eg*C7Eo6_Sw&wnhft;(c^do2|Ku>5w%b7cbow?6-ZpY2D@GJ56kh+|Go_=$NujA)j-U0JU#ThOyE6&-LwCT`Y-b7mRABXg_lfFBNje%R> zI+#@|K96RP9mCK2Ymjr0{V|r7`&+`lji0*FSDO!U_WBC3x$Ki+GxpVZN0?t(?8BD$ zLv44qkFyqC#PXh)eYrDqQhm-aQj5q4moMt>O(&{x)TGMohCXgNne!fdg6`pNoEqCfBRR}bJ&CM`{wgl+=IWQvZ&$R@CwLvz7tFP(fA735^jOAa`R}6 zGu&5o@zkj6*dxVuFDuT*1METU*5cWlIY+bPEpW{a?9BHE{PS=HoI)E7*;fy1w*jsR zr?MxMiZh+s|EB)h(|acdV_kP6@nhNFiERnp*f~3^*cne<_5^$zbY*96*De+FR73mB zV>PiIaL!7rUEyuIyOJEo1;ImO`OgH-P{M=Jr&M>2EqulUpAsgX?}bvODQ^z6~^;U4xUxPQ4hfy~n! zc#n1)-l}Ae!BdrmmRf!!yE}{!pNIS5CUW*?-vF80U*H4wozPxvPy9FJY<*N#JnK^5 zs)=P!WiRADXB*Ey?y(-<@%O{IPAz%&^K%Z*VvnakZ9d=SKQI0mzM7VG&e_Yo zp8@UILn_q<_M4Den!TN8LY_Ohw{6%tv$^M-x2#pp`G)k4xD74!^%B=f4a*sQg_ipH z5&H~0#hAZjf99QJCfb#*0@}M^gH!E9IA?~N58_ob&+!! zEd4J(bNaE^P}*VIO~s?(Ay^1gXm8=fj=^~!{luDO?;ZgAmHbW-tt+Hen3CSb)d!FU1|T%d;|T-U%|kq5Q{T z7uc4+5*~%t{C3cko%g%l*|XU>O9!&oD;3YBD(tq9ai@kaa{Yz)dbzz}Pw2#Nk4s|b zvIoIIupaFd_}QH1TA7Q_D}7#&_X|!vtPOiWQ~nRo687Zp1Qq4Q^Xg^V2W3UgYtCL_ zyzl5E`z7^gAl`?bF`aHs4x`;J=W!Uw9!C2?{@Lsv<;5Ibfd|6@^t;7RWcMo-pTqNv z`-2{Tkyt-=Rq;IU7P&^|?qJuu0M~~mCGVN^Q`k8Vvv96+3FO@Tt^HQ)L$#TVkArAA z6WKc%=lqhn!VloQZ{%-Pr0)j&Tz3V&1g?Vu1_mY0+uVrVh zZ)ZOSec1by7oV{vvh%%B-ZvMpd%`sRWX|p>Q-|?-O}|s2zJ(B1%4X7DLLP;E2LHq&_^H7rYFQUq2)Sr%I!&f7S7ozGoC;BncE#Ji?tfg zUq#=d&uO!vu{N8D4>jhM z{3GSn#JR?y&{pggymhHKYuOKTl1tC`p;Pgf_$4_{!g7ArHhbzXc0IV9zY%=J-V-*f zEPig2d0E2Fy)}fR;bZ>x_#9X*H+y-U{_}jkA9jWbkTca0=N_}B^Vp+keQ;a;$#{48 zRUZ%Fez3Xw{)69!{Uy%X$oR@Ci#?e-GoN1*c4Xh}9@Z0E3vaP+r9CfJ1#;gR$CYAv z1{`S|uhZU!+;8UbBX*w0=Ruv4dLXAGZ6|gE+VR%%4*h1o=ULE-ejI->+{N#Tvj!W| zvwk`MPtqT+ES{Ig@td%pp=CUs__yF$I5q4=c525!cGffX`XsoNwk3aG`~p9FBK!3; z*qeVSv=YxA$=W~4-vdvF$Ju+~w8{E>fiJ};(MotBzJ`{y&zZ=1-W=ZGXTM#^&U{s6 zKTAuUYF#SUux0OJ@4qTmomNF&>SV5a5mbZ8@-yy3*g1nwv#&2XL-Hs zW_hob**EM8SP9eM0rB*;n4M=$>iHY?M_)XizPosyuT@|R+I)WIXehfO{)t|LooB)3 z_(_;&4DI;2{yFd_WUii)e;Ir0GV{xyiE{=vVdr`Dx^`Xp*W$nUd1ftv)b?5>;}F|U zY#6OGKkvBLiPc?g474xAj^Phy*MTGGIrG_56);TvI{L^;b&ubWKdxl|%FCWgy=g(K z;9tt#8V-XG<*tIU^hKrOKJ#x6+OxAi*THCZ_Q_1`uh(V~Ex(tL^O-TMp=Iro`vr_) zzl?vwS5+3D19Hytu68hNS}NXAwiCZz{57Z!jrh?Xky8u5k4HmlUHplVd(3|854mpq z?X`VR-aEJnKWjRvRP52z^n8DodmJaWM$Rbrk-Bo8*v4=uWPVfU=gMo2oAakao~x;I zx#oNPf3^Qpe_iFAOPkDY!EOy1Yu*bEke~ZHj`piId$PyTX7ksRQw?XXuAuG1&ib^6 zoB4U3{ULTf{We^KpL*O6GEZ&rAo{)Rr|>H1tw4Arp)byXvl-J2Wjh%g* zalBUPEYKd%<~H$bAZP4OXh!df+v7I)MSaCiJnQ>lW$_HpbLdI!Q>&J+b3b3=ywhZE z7V%SKj^&?x) zjm3QQqow}UGVW{Ht031bhly}Dq^}7u6moyJY5xvB39rz01NwZZ$KM_2dzWLyH->F# z75F>I`g|ci0J49kueN{j=8&_s3*_wNY>d{&4LJL#OuQQG#J{<+`23i&vmtF2{~!8K z@EAMak!O$g5zAgX98QIgT)PH6^M4=zl*-~dFS67B!D8>SuVgoa-0uWq$o`(I?PdJK z#J|&SIY0aIndHMs?3~$$@Jo26+?=7NIA>z5{QOPH{n$s-D{yMf0-UwW9=S_S-d~S` zg<`<~cIxk{RmJ!J&Dgc+!(iib&s=(I_F>wVrY^J*R~4e9K3`3 z!_Q^K{$9x5TFzid-yhH~hcWue8GVes6&@rnbKHY{2CWukjk2$6(p$oKc$lB-!ul`wh;sY!mk9 z>>l!t=8u(=vsp&(#QvAQlsyirK>oJxb@FqKDe$1&$^6;yFwCP3EGs^<4aGY`_Wmqf zO)ULouV%kJplvtW1oqyv?E546S@S-|(gCjr&DgEw&X@NMoJdQpxEY?Rv{u$IxjAdu z3t8i%efPE>41oP%5JVdSeg5-z!pHn)AA%2syh2*E-q5s(j;N%?q zpP$_1q`w0pdiqVD8N*)if4R~6LvmxMpCcfB{@*oXC!V%xmz@7^fB72^dqS?c`+s(F z&Vm+eiaK;8ei*8opCu>?OcXpQ|1-Jt4y>Lk50WKZ0Ot3c{w zzcs}(dWtp++3jE$%!00v|HtP!ZC1x;f6h*Qe8C#6W*A0<4=}1irrPtS@gg0ZT!@(Gx^)vtF8F=!*gQo zY1gy!TpG+iLEct4>)RWz6kEyehhU+0e=wA7;eG)=;tK%MBGPyYRm0%c9;Ja z{Zalv$bPIL{uIvr)S$g4K9IH?Zw_PGO<@PIHvBwC9#n@W&@$F{XggKNjSl1VlfLd} zZy}xw&kD{{ViFel%_k&(QL0`xoAomveolc>n0){rMmA=dttM*+`uEd5pG^ z*xUASH~upA@vx_BXYPlvo8xtmvz2)~Z%y(3JQqJt??}6u{TlwFyr`f1;he+VU-tG& z{v~)mzL8$5kIz8t&*04xTWl)(R-AY62JT@h`w_eVFBRK>J|{7L>f1Z8!o9p)Dn8f# zX}{(F?aKMe-1Ws-&vW(BO>XXe5L`^3N$-L4%=uVsJNm8qs%;(?)3e8Jq`kyGQO;Ai z7i|Qj-nE9s`ke`xueVFZ{N#Q!*C+7vZuGDGyZM9V)pXA@`Ac9jtseh!e9anX+SpSs zmS{JfHU)Np%W0!vtu`m%oQuXd^Obx0o&Pnv4dh)V>(h??mss{??lZOQ3I6H0HqJVp z1JxllHS6&H8gnamESyK11ZM~IW-yC>zP!xm2snwQY$+^YGvMyXR!z4)Ttxz z2!1ulK5r%`xwXZXvU4w+!wUA>^t$Z&o*7T@_h6@PT!}MJ=fN6wYTCjIYr?<9I5KzV z@c-bapS&~V?43*BSMCgc>d_KdXuoE@ZV`W$ze$C)XJ3X>6P^~!obHZW(4G-XUEfq6 znctpbE8t}DzOKC+`xN>kka@3-`_Xd#?-9@aZN%OXA1$77W)EbKeF@9hf8uYU6Qt(N zkh4D=$KHiDk9|8kIm00Lm^s-I_A;)E#Itq{aNeyya@{@ob@)e$52sxTpR-TEIZwaB z9P#v-y>Xm*=}LQ?wjAcM&%mv%#gFXi>^w7j;i_vqNAY{Evrz0vZ8J9w-EG)0zDnJ9~RPo`pxiCa!T`N!?~|P*J>_%w%Vsv+k*l zxz3Ql7{_RPD!V^C!%pqL2EGuhp`Y>W?1=;LIye@3$QvY9=~}t|IqZhGrrOgU4~ERo z$uJFm7th*!pxu7B>zd-{L0`(xxa-rVi%rD&JXxD|AUktcQ`;8&)T>6g9^{=UXY3WZ zOU1W_ui4MRY4qzWif3#)e2g)EL|@F#XQ}Vy-p1a0jo&jByNy3Z{1AAMowJoS%stP> z*;jY6kA&(K#e074`M%2H-7D`CPqPn~cMx7JXAeAD&UJ7!q~@PUuV+8qEN>08=I5F5 zB72ma2TH}yma8YI79*U2F0wzKzxzUkk}^USU5t6L*%1`}zz2 zO7BmuXPMhPd$KP-g6;SV`I|xy_nYrL|Is$JYc0E)H5x0n z1B`|(={Mq+#U9XhJftpfudiF!d6vv(9}XksZNg8z>CS%$+QaFz-2W5w#NU(GiryEt zglY8c@W_hdv($^d3%f6Wgjk*xGw><=73_Q--$8zGQ)O`$x{IZc1vt;}v)S+Cfv`Q~ zOsCyY`b}_{ywsP2*_rol_T<%gAyh3D@98zzsU*q4YFUa6{|`Ak-iKzfPcm6({qoD z**zh9bVuk+n}{C}Zw@v1li&tg*0sCb%w6`-qx|`hYtOMKQ}1qM=L}3`uY*N!q1g7g zn)`TC`&kui{#tfRWBeLVq@`YMOMe6BeYghwBRCvJh+hKhi=PF*&>rLGnfIi7dYFF# z{cYF!mUfI-9q2-v#2T&TbNjzRoujAayLOpilH+Qsk$OoJpDBnQFe;;%c&-yg1PboA9h@0zVKMQPM=x3BOvp<0+-Wj zLFVgC?d#D~V-Dr#tX3Dxyr=Eq^hV+n@yYl)cwL(+ct}O@?s7D~pLP!%Odrj^5_V>% zZZu(!#Os-l-Pz+HYu5>`Vo$<@U~yUTp5H`XYkD(U&ecA~kv*IC)nOm8ZE;ha_P2-E z6z?3P@Eyi+IPFaBvR}Hhzo%~onX~M%;?*;$B z8vX*PEAIh*U3nYeRRiiMPbV zOGSP7#xo!_bXz%@)7sEY&OFHaeFH6E;|l8yWstR>0Eet8YGU5YQVVK8-UCt_&Sp*ha6SKc+z_6Fu@yyZ zfA%`|f%q>ui`eJGT-Z;4>*&Yn^H#`uA4JPJK1=RD`k&0t_|AdD*(c+X@VEE};<@f5 zTx(77j?)Tfug}Mih^@3wdh;9c>*8^^9exDY7vHnI`0P}RmU+9HR)cm4JA3PN$liO0 zpLIF|R$7LDH;}iS-3xctHrLzGImvv_mG^SV z+S1pTTVk(<%;yAoE8rda_4rD;&#+sFy^UYRdH1>=e<}V>si=)vmm&PYcqDBWJGFOf z{!{WB@GqrjpLFAIMxV{jnaWxnif@HS#I|+Kj3s+Kea>fRy|Qk5;(u2c-%p>SzeC_r zZN9@(;WMZrFLRu+Wo_<-JhyUA4}#~!GB^Ujg}fP)GHy^_pVOo=eL;{f^to&H8=BAA&Ql!*J#&bN4WOOS=o_EZ$HmzLWe5 z?@e!vo8i3U4#Z82CD&~!zoYABekX`!jWWi)#D0Q2Q*url;oP*&SnA{!M8GCnmz47h*8u%I*$xdHOtozgWcCn4|jc~I3 zPx(t-?`nR|TK-1xdH6Z85wv`6Jp(@rd9PRk*@G8D_Rm~-?e+1SyzHs@>|yjh;R(pQ z&vx`GICGOZ$U2<`Wpexfr(Y)LNLn}7&l=@(@OO}(JDg9?8XRwJhVpYi!}0gx*$ayx zweAHNLAw^Rhi33|&Gog--=o@^{i|3x{mn|(!h6$N!qswS=`+u~)S!$zd+JU8iTuMW zd{*VB@7(vvP)1u1YFgilQgIgA^6%DWW86mWN72_5dt}pI#r)qVRzsWPaQbZoufpOr zMU7mf%}{=GNUgkGKkMUrDvNib3)pAEz4EfoneR8`w8i70Eqzplvw^RNdfFXJ&-Vtc z=vCk@c!<6ueX^X?wH;wo{^z){@l|2xOy}&>Dz{JAIrAT~ufctB&e)yoUGM_&yesF~ zlD#t*o^g#m#Z$j~!(MV0;eBZ{;s4ey_m_Q=y>T_pwLdi8i||!22>yVq%{djtJe=uT zH?#A6dfs&!^eN74D|X%kr-`N3oG+fWJ`KNS58cD=3m3xI{1t18`rJo+7ur)La|5T+ z^Ukpw{S0;oxwrB=^YdBc3R>P%v(`t^bGGKNAE7UWjB_X2>Fk5yD>#65jW%N;Yxp+4 z6nCVb$F5Z>&PE;nrEs^{x31Ndof@(;q^4!fu3n?2;+F6LE$`lk;LpqTMQaKfN9uMb zTvhBb_Lc0s9}R{6{C4!5+kNTt*qM*5-RnR2PC3RdTe5_ zC41vF#k)=gzC-(G@D^z+9@y@zM}5{JJF}&hX48Rie=7n{bNho zu|I?Ck<0n{4A)F-H~!c3BkT--ySQs2ia}Ja}U`w!};lN z@EUvC_*2W?H<8~2XaDuWefga$T#KFii&oBmQ=d7jFTwG0GS@ld z|9{3W=J18y_6^R@A=Wo9`CXiI@B#1xtpl_wSu3^km&)ROA#-1s_Pm;YAnhu9^+fG* z4!&U@1$(nEk(c+w#<(w3r{(uxrb52Q$+^zHtPtzLehlxZ&q3^bj;PS*aqO$;X+Mp& z8;ob?_k4E{`-purl=ElHe~F#v=c{lFltCkT>*0aoC$oRV%lSJ)ZSC@mTMD1>e}_NB z^L)y4{5$$r?7ZjCFbCVPQzO2o|L%80@;-4ttqRVbZd_hqh<>~3XJcAj+Uw%&*j;fS zyb{k8pNqRggGzNT&@OHAJ7F3760zLZXgm=<5^tx!C)mp&`+ZKOb0$CI&h^voCdhZZ zhvK|f>~E}1^*fXO9yDgp)mQ3fez)i{{3(Ajdy06Tbzf?eccGhT|5%sgY%V9C0rP*( zEaAVxo(Ch#-6!o+xp`N+otC}ueQ*0l+dAxwtBy9^@iy{r7h56sVAz;njh_AT3EKbrQY*dp4M?E3Vd z_|J-EZ`Fd#?|bkZdp$YpvybR42QPz}w8Qzm*+bZG6QIKKn<6y7g(W${jYG0t|}17%lZGb9j*0NdB{MJG%w$A@45sUF^n?^V&-8 zN$l-$uKz9VA9-2Jt=Ks~ui-<)HisT^C(^RMUEyhV&Uowc!fzut<6n&Xzie>VSG96$3h#JS0L+5P>Utr24FX>(u|Z7;lHwdX1QYI*zM9<*cd zRGjz1V-yH8&=^d7RGdp`F^(W`B3q8-EE@D;j7UtnZdhR>F@0>MFEg5A$w4~*1 zH4s0#%y$#e5>o%h;@cqSH+yXYG|*?Ra}|3JoSz@&-?p7aznwM|J}E2eWZLDqmgoBj z_IG0a%B=}Ib9EBVSkA+#DGyh=C;er;j%B}x|AIeZB`mHi>UK|UvL|!KuEb+q`)9Em zw0)7EeNjW~1>@?#zDz7@ndd_-bDi_CgWMkCZ`1#Um-TTt{TFr>_>kRS+uYMF{A=*% zaI$u(BiZXsAotszzY`oU)`_+o^ex#VVlU=;`cC_ruGfWLPwqQ%`p`x}KR6JwHrt6k zQ>IR^U#DdbuEsB}EY90A^dI;K%Ugh_i{FYLg4qs zJzh>en>4279d4oA?5An)kDTM+2KFuLe)`RxJXL-^Cw(n9&(BBbdG1f=uVU|H?2qW{ z747cgSLKh8x1ac0e$bG=H=Hde&-m8(0(^jWGuZ9fdEVqb={5cn{F?kn#h+x4#B1inE@NvzN%Z z4(D9uUhidR{YsEClXs%3`dY&84AUx$deBeoXt)S&qFrNL*XXOgoKf$|iR&pPAz47klJ^BziJ3{x$ z;$0^9^R>LXkh(oa&L8Z}*twsh*_-Mkdo2Hc;8pwzE$1!ov$_5{d1Glm7~dN4OW-;F z3;g@>3dq@9gg=9aU?4QFEY>}H=00uC=4X$7#$L7Bo{^Jh&X4*zgI!m=5{@sabM$t! z8t@{tmA^d6_yP|8|_S*Q`{$>z*PfbDO;}mEK3ZH?*MN!@or= z=dK@g=U*rGoUwN1XRl?gQ&-O5AIu-4-O23h*ppyS{`vY|CFfCkp6!pbZ-hMm@(hWc zdsqZH&oz3>)AnKZyh`6;v8Uh({JCPuPu_U`#1%Df)3)#VKd`y zS?NB-f8n2|-`{DuzrM5%a_7TFVySZ@%~v^l4RoX*Ca)%LFLoL2M)(>!(KGgi^kMwW zdpF3tQuh2poNJ~|9>=~zemQ&sx455H#xV_^6n`7v4|V08qs{(udcijQad0Co?;RQ6 zc>cYx75fxC3}^pLW~Xk?gt7cXAbTh8GathBWzLXmG+`eJse^k!_SQK12=UbZ^gT!) zJC&Ok`Vn|nc#2*H_7KZ_y@?mlYxzzp*G^r_{{4}D3qLh>n!anXuc1H0&wAu@N}dzf z;GE+-)b#E+_i!RD=Wqfnl%IXJguPTh^Nc0?d-6 z4sZthWqNAx*RTg|0)Kn@+x+#R75fPM17ttmR$1)xoRM|3oTYO7KF&RkppWMN4yku( zb6j~b{`Tx#=VqM!ur+-aWZ&F`Q_HhQ+Cm#RjlV#jm+7aK?{rV*|3W{9-;X{7kKkuN zr(X7FKWROaw;5#Ko|Ll-eS65yA-d37ifvBMT8_YDaB9d%sLIcL-ix;=D?b0cR9>uq zE4%}3sk|-N`I*PYcs*mw{#*uC_1_OKEU8g)o8qhac?Qnpw}qUK(`g&hE`m$>b@>DE zNc;nQNq-J+2AS*WIBS~sg&*~?Gc9{MwW5*Sx%hN?_QndxS`LNm!PlVf`Y?TO-xX*)XyI5>JWV$PX9k@*Sl0azp}Q?*ehup zIor9H+i7RxL!iId3HVTZ;wpCTYcxN#`$U|d0j3Uo#XgPJ31+S?&e`4ch3x&b=}12s zHlf{zvrc#8`Ouf0T9C6i*f_GDP1Wn1ohIeQd>=r|dZ#YmB$j?s%QoRZK%WmQ<#&J` z;SKQ}XnCd|g=a$*T1z}#Jm+{Iq=qhnzbcFO;`3>#g8pEonu9Z^&7q=Hd=A_NCqLK7 zURf${FuNwb89RL+1DhM)r#SCf+0!-gd3Ysld$ES>p0xBa3Kr9+;7%~9RP2SrX-lCI ze|P*=Wic-q)19<&ct6NK$U45D?L_vjrJ`nRO}m@_4&;7vkK@?~oBQ5!FUF}ApR%(T z*WzzIr?MV7S7T|PikH}(aptB1(r3=k`#7Jk9@g#$@r(GW6UT_Hgm&yN=-08&VlRP9 z?bp-g=YHzI2ka~K@r1m6aobYye7ljK{g(5vxi!mu<{8!vI@9il8`wX>=Wsp0afP|i z=4y7+)x~GlhGMzSn`J(`(RURe#b3bAyln#M`ylb`_wMW`?Bjf2ceUID_+LTJcxqby zhF{fE@g31fZRXIofy~ck89zJEp$+j=@h546`IYRv*OzeWRBB*;R@qu!_G}$F_1TZ% z9gTel{4>9c+%3c#K^6Y3{3&8<*=@|r?d}={SAn`;4aStkX=W52+o)@M3+H z*y~`3HaUY?v%O&qKmCpEU5x8{emSHLR6?H+7)DzBZwK%g&ngV&4co?U!NtORdhjXPmp07yB%GDra^q z?F8{Ea0UBB+OPV}oaa5QIW2W9KR+qQW5tKkACS|FeK4MgPsQ89H1^r}y)x_Qp7P%O z4|@oGMX9L&E!giu#+K+c!^w*2++-taAbbY<~gd=0L}-$(oxIInlHpYtw}wck#B z6zr(ox%~9AoPPuVI!Mjk2;WNk8TPdg7tnHVlliH+pYXfN*%WVqj}*H?Y=7LJ-wbBK z+q8kWHvDSb`CWuOlU`(3qcvym&3;(k5O@!At!kxW-Kygc`47705$x-=`HkO-{}6i_ zl(ab(HWFXWKi+xT&RVvm&F7DozoS?S$hu6TCGQgc>2mUYO1@WYMa%D}r(UEEj~359 zc$ju;@8VtZTKRP#=dJ>0AKw6dV5z)K%B&l{MJ#8&D*sxsWw;x^4_wZE7@rG0V4~dC zCHo!DfP2Jq-JKwHxI5Hx{Ym(=^5R|hV>vS*bJE=P@_lOu`zQ0?Qr<=K%Gueo$MN$F zX#-Os>oE`NY5y$zC7$2CxQRBqf&c|H{xG?*P7U??7484 z>($rx7JZ$?Prqx~%X@ooW;fD4&(+lB)Q#+cPPnUB<|>~P@>#y3xAzr&b-*WSpFWP) z<|f=zJl`?a;a|!Bk^c{WI=&C;vGd;dUN7ejrgf?{T~1f++CkQKnY>dV`|u~%J6pV# z*h;(!E$;)#%YMHX?=7||WIS6zc;)J1Pc&oSqTP3}IlHE|*%#GT72juWkvLRF(?@f8 zd6vG*pNHRvH`y)lcx`e{vUg|58%kS)JG!SY=pE^&>0^L1eE@$1%#brlPIY=dAD;`m z(~{o+{{|<(KG06ywvavc4KD9ptWiEcjAL&EsYe^}zh9*;>+?dn*?)OQTPl7MejUF9 z>+bdaq4)@x#lH(`i@&Gsa$`P*z6Kid9}#;lw!StLn?$S6Prchso2fX@yYx33KZBFo z4S&m@jwj&O+I5tl+BD`rEqkto+=^cAUCu#btzcL7En;W&um^Ed{_gZDkmuERw0SFw z+R_N;nXp>Ubp2$#UW2pb+;6;_h#kQ{5l$_0ZR5zvu#eH# z_4o&CmU=z0x3dl(i|nc^2ds8x>{Z4e}w1YM*6A; z55uMLOsz7hY+dy6wQkKW0(D#bG98rpOc zPwgLrv$t{{GAEheYqft?>=pXU+H6L@M!b($Pn>7une28rW9unrHalyP?>P^K_u+i8 z{7%G8+Gox7gcs#pOg|LojMdgp=505eH9TBSGk)eIW6X6w#9iojuJW#hN7FjP){t75 z@1FCHx=7x0?)g6YAX;kJ(d^vY-{P6$|LuK#IXml}dV2#qHR(#6_nL0{$a5`gc8J{6 zr+dxE5pt{R=W6~c$R6n`zb6i|PhNyCR~Mh54wru$zouCBTy6F$vBuC%KY34RLtkGX z3vdH|*7RfCNt?&<9IU?&Q-eLXcTwXy@$Yb-8;WIJ`fHQ=Sf8Hfz^iiihr7jJ zG5$O+^DZ_*ZpQmwVzhxcb2bp}mp6^S9NOvYb93>9oY%#EqviT}Kd;b!G;GdKy=g9< zwLJnKjJJn|?6G(se6pN8Hy`5XdAloqiM|d7L-tr1&K|u1AGx}CU(B;*1KLP_^7kk! z>iX6=&*GoxnX9Zvp4B;TBiS?PsWrKe^m4vpAZ&sEy~f={={Zoyykw-Q?p3t%s3xZ1NppO>)j zVsER@bvWnhbey`}i=B4Iz&o_m!EX2wW4acnhSt&d&Ui0KJ9}1td`--I|9>)HG{k)vna%bYy>lfth&3{q61OK-8^74Bo%kbsUUi*CR zea|&hzp}r(^)B8ohq2#=lKiUTIZN%~cvwQ;7w6q#Fgx!wi}@Ycd6ylFGat|6ayg4# zV@o*)!)>%N_j~+AgDC4E^{yca!mjW#&sx_FrB0lkDu>aq{oQsePG? zzvO2QcB5^G@2CGE=TH7Xeiip|3T+hs6?$q;_R7M_;=O(++7|q)#p}xJ&#tCTE&9JO z1#(teh#jrX3VG|{OZg4hsnSePp~(|RkV4ZopoKT&G$HKxe&L4tmFQ898{?+KD$0EuLpe*?GdP> zkGu;EVn5-$WH{WxRG_ds=Fooj7H%X4XVx%WnV9du@2 zj)%y3O8zDK*wpxMri~~o?)OPNU+gBahv^?c_Wi3+UF-$^8?Z#q3$!V;vHa|Toc&e! z0I?bwhwEJ^XD{|9(BHUrlCv3mCwa%R^W9wu@6P`M?v(#Bz8dDU$CMX8OS~O-28`K~M9E9SlOZ+0WRvAnd)y`IiKTAL%-`Hn69bcA}4@qG?e z=})gJYT8hK<}aT)Z^t9~J3ucvjrp0o)Utesm$SJmKi`v{VZ7b=nVWoH_ZxqJoa~jK zXdBX=WiNw_?G6|vUWwbdR!8xJ#D0dOA)G+v92Y8~TOpjG-F)3+Tkpo_t0fc!9njtmEhT|2p(yUvHc{ z_gq^y%G7Q<%$Cnq+ub%qM*iY(RoRwkp z*V*%F{FvW9I_+Tn_JhOpaU|ZDJ`diNmp%49dj(`| zUgu{oX5Z#MXJ~VnHZ%2;wOJoh2kX-2TKn8b-UG6~ex~n*pXYC+ZTfo;{^lP7dzr87 zy#vaNvE|y?-wndF`DMjBYxYYwIE?_9vL@_FQM zey+KJygPB`<2BmdVyTT?R~L0-I4!kzH)svlRQmkv9o;={Z9{hURO-ey8U+gR7W&ShglWF6iA3yuzICGOR zPJ{Ey%pu-H9m@N1`Wg*W*m)N1E7pm>zqVJf2e8}V*|0yP-et}<7H?DOUt~|DXV1Og+xcPl6uSv8G@gum3G|_z4bRB=9nPbj3a`*E!xQi)ct7oaX18^n zpSAyzKZd`l*tXhbJ-%Xp4tdYrp1&S6kh3M^ytJe5OM6e-jCX|ndxQL!#GVivguB3j zkbOGBb@C3E+Es%5tg|uhsgGarsr;OssZf*Fmi8t4UFgjI0`i<*!(Zf@x303*{M6lV z^t(xUF{fFNb+9vipf*3TGv}$5xA7OVAA%jU$#t_h^qh-1avpWvUFjRJ--xE) zn_aUne-dP^AEf8ZZ7Ftz{M4Kgko}Q*pK(>KEPmFJ^RgGDZPsCWrM(Zku-nVq81JHw ze)tH;{+U;4++sr^*R99Tett#XNLubG?-KKAo6}NLviH(&BisxQ5zm|sVPB$6N3n_g zoQ2)_efSGj7k%X%<$JWeOPtI83bL-LG0o+sR^=@1h%?6D;4N*3(@tcMVRzT|O!oS? zvAnIVDhNPV6`>nonU zl=r~e?2Pj?$XsXsvVU@o!R7WPJ^SZnv3$^XD&1M zE#x;7f1G{^KYOAbEzgV3*ssy@ea~9F6TK?^Np{Z1cDNimSNdM1xAz~~ef;;~4yZy; ze&#Xry}kGk_$hb^CVBVGzG(tmiS4hCO)86bs>#}&%)eGF>$w#@&zLvy8?@)(bp9T4 zayIiW^daONUJ8H7D;K{IA5PD{$vbyT+WWNQ@XxfX#17^Eio2=Z<8a=028wk|Kez(- z$M4WTlv^F<${Pj`urvPC#ip>U$gM#i!_HXtf}Dv*pblJZ->k-Ol@-sT=j7hW|AhY) zd#<_8yLk3Z?)OprHardALDqXaedfA#@x?gz*qS}a?_?Yb`I{X1dnEh)XCMBbeGnc9 zdqeWVyTe|v59|+#_k+X^gd-t;ry}^j{*#mZAbQ$I%imH-PB_}Za10hvG;)Jhe7)K+x&gg+r7a$bJ8DPEVHM~%}wkQ zeGoh6Z8_{Ac1*dwh4XH4D*i-Xdpr+PFS7sJ;tBNUAm6R!+4degwKi+sM7%yN&zQV3 z{kgjMjB%CN-B2!eBRosL({&ol|BYQ!EOlWpK1V#)Jei$6mb$bUPlE&Oht$ICg}isI z(kAce_u#xs{fM6u8?d_geD*H;d-`Fxtv%fgPlD66NexJ?8_XVy$Ca65IEB9>{6YJF z+PkkdtEw$p*kn;aM8J%oA_AfaifFAWR76AtvlvhW5yhNyBIlrF11f?65io2KsVZ{< z5p&q)obxts!}qp5_4&@l`2pvGi}tkMTkm7cF-KmDW%&yKQFYb8^B`?$0H=7)Wd46( zS!MB?^84@r{;qg^7^d8P(4E~GE-LAB{`%_PoxiDia<`ntPCu+t=ZWmh-|Rt+#P)~l zXgTllE`wkLu|HukbSo8iuzqF6gPnc%3_0C#zSDSt_N>^E{3~&DvDJ8qeyk(56@6Hh zv9$-^C^npSGu#V9U_9jRl09Z{Wih@xDU-E)A8ixb2zZs9b@?N-V3*-7<( ztD}x-;+dbDM#nwj2=**E3_1n!x4@@W7T-tYyOWIHOc)6{J3iHxzU%{FbV>hs=0wjt zhMzn849}WRZ^2Jre2*{HhDCToeh)lYJx8%KuRE}FF1M5O2rcLM`Rd5$kekI*XYQ8S zTZi#8Z)VGBqf8eV$UlJJfISLV;P z&hfAK^Yr}_;(M1D?=VANzH92tKT@zRoPDGgK2bdn%6k|_(_V(P={fh$r;X!p z$i9`GIgoX@8GoX2`&GSH^ZU?i(DLt`sWx<<*Td}j|YC-1Xp|p(MH_AQ9KZ1Q1dlH@vAF6+E&o4pt+niVVH#MxV%g5Gjr%~&?n-2_jwSziCFqMd*CDP5E=WX{7Rg69R(M| zzoDU=yyKb5jKQ0z^9%Jn%U+v36>p|p+sNCKy%8;YLRm?Da5X>c>|ENX>PW05y(i@E zwhezN?99$w{)C-#<}iFAz7xNL^Y1X3FMGlbrQ*HvZ(!MzvlqQ6XDVbJUJA?Dcgx!h zhVozF?~613??C491Eu2r_8tFQSQqksO(37yf92;+bvsVGK7`fUG1qg~k&|_hHJf>t zc8sPkQ0{Dc=I27O^XQB4570m?YxP@o4yUJ&vtRcRn-4SK9NM4ao!L9Gx51C$p`KfY zzxIrL4z8ixfBC=3ZH#j!9Hh>PrQ&XryWCUs)%>05^Vnaqx5Bv>y{lZ-T%KDO?=L5N zch0bPaZ6fvxjA>fllv2-ZPUcEezt=L_^0sGmgCqL(6aAj9$iDvesetgUVJ6I#I9q# zr_N3J$3S1&M0K~v&%l)B#Twk5_6l4fCwGE_@fXktPP7(Ytt`&9-u!%KuP5htWgGHG zhz+BS_nbHRdFM05Gge3A)8y^VUPONg_oFRgXTSOy_lKOkm zhPiLWw9ANy8X?)YoTJ)fWb z@g??`IO}4p*f4hH(S`in`QF48(1QLbRM76^FM#ir&Hj|Ncsl=W_=EiyJ!@rm^(Ne;Ko%XrHNfOS~;Kfom#@&ytPVyRrM? z?4#+=d-%)XD)?Ak9dPF6^pbrEj#ckbw5{-H{;TZ$U_85wmVd|Ikbg3N1AqIKBIlom}SG4K)1AZ0$4AQoJDP=coY1IoL2Z>{7p*5y)I{F=0JCNS1j*QUtV)|Ep~r-li6E~KZtws)2|!wC*msY z&Dgi2XN_mi>%o5$o`ISCZ^f3gZ@^vo*=sUSvW_n=E9O-Tv7Y?d_!4!#j)U}jze?jO zmOjt@XcO^0JS+3L5k2jnkMEV&pIwO`mfKW3`^*7i>ED`SX=58$CB8~L=h}ui`&icY zVt7dYewD@kbpcLa>;%V^iv8nu{3!oiWxv6be`a_E$jt3zuqr1kHpe1JK~pMf0$ZXtfND5 z_Jh};4Li?G&X4RDaQbMqXJmgIiZiy!-=CiE=&$yEiRZTjyU{Yo?+`y2vgU3UYf5j$ zpHU_s*W%AK&+`69^XtfKEcT7u{6;nFa4wu679NeS$6e&yfyeOML+0jTcxzl&ENl8W z7$E0)$Y-UO@LTFln+CBTk+(N~hd%*lkAFmb4*NxI8iTvQIG7BXcU!_6><=Knw;PM+ z!HM+j(PygjAb#Fs5qo|7OsN>JeCGe0oxOGlZU%Wq_OBYW6FmDX_Oq~em3>S7y=aqR z0c~G7SF#^2E7tgOT(?r&?Tzb-<^1?l-u-gw;oh{|9}i_Op}h>r>jcYbt#ID$RQ5E; zezdx**f&p9t}A4XZi#;p?*-YX{;brWa1y(RI{vq}@$crphqr|Jki3J%TjBIa_JH(T z3-+z-+(%DVe|_A#WFA2JB7L^Cc4Us$g0?cpN+vHC1) zVt4id@CW-m+!FW0=g1om8N-kGL)hu_8~J_9i+khG>_N25&$VF?J^OxN`b}^n?YZde zDkx+Bu6^n2lll4GaPB1et>rd2b1QpbPkAfM={`7fCUfyGcKRc0?rk`gy+SN^?%W}# z&}P8#>KO{#sAmlSHaQ33+`Y5s5Aw_(+4*Fiw85^i4ch=04>=F6C@?ZEJ^?ruWDi!PEG4}qna`DWIBVi!_Ze>5h zNAR;pX0GNPvd{G@H{SdSC1(ZuPF&8P$-arbllNJVeHXhz9h>63!v*Y~RmEACb1k1U zR*B{JsvXLTc~(>2!|(w84BG8@e|2RnpNDH`>DzVHvn_iaoHHS3X!el-a=PH0vAL_K ztxcg(W%2oTgm}iRS5-0355$@CH`BA{T!1G-FXb~IQto@`>{&TiGXBr;^Z6@f2GDk5 zXCCjtJ_yg0yC=I`yf%A#_7m#6(fA%uds+EWkTrLx{CRTrU~eXGcX(c2*5B!{l>RFm zOZ!RQi%>&dhlu^i?u|#_l-rnHODy-gQ`j|1#aW#`UfXkCm3Ink!QNB6Oqu*PH0!cG zJ##bjV+s3F@n;}&E&KZmw4ZQm$i4F4_!dZB*3cg?mc0nNvNJc|mzVwF40hJ+0XXyL zX7;n}I*EC&cJgv(X@EP?=ZJ5^zZ-9jKcS_be*D~lb2jC?sf+W@d5>SpiuPVpWsG1X zJ@1@y|5ESC@-wba!13k9I9|(M<@x(nnlm_Sa9uc!-W;xkCSr?edEa%F8zb*`cE;^P z@i*C9CkBs|6`!{j%0G#pcJ^gADHY#IZbzFV=Pekyt36U3m(jiue+YKx$Ij=FSLu27 zGyEsvpF2qf`!+aSZuW|!D~tQme{uS-Ic>7?+3PF#dB%rg4cTKoH~sgqcQ`@3wOD_# zXCd#A{(eon+S7-#Yrtvh`igdc$vE+wiPca?*5ApHu^J+$3!Vo#r!r@s#f$Nuo|U!G zQf}tyv2s?^vtDNLXTw}t8#z;y$r+vX(H%dj?8b5z!b$PvX5SdazYhtufIh;#PzVdvba zk2^!|WW#XgMo&3aa3XCxIhR8RcJ{89VJ@_iTVHN|gP!^FJse5TKK&j&YpRNWHa+Y0 zN}TVbYKqU`XOAC`e}jpzil6bCqg+$|Z|oz=yt_VogMSG>bA7J(3G7ecO#UY7+ecqL z$uEQ5X=CAi_AZcrFNyy&Ji-4IvOn#`zXrBv*HZp>b}yJ-S>u1dub;)v9V^ew8Xrf` z9eRfMeNh{-j`yaGRDb$*7<@+0Ivo+eRJ`lu>drp%CjS-XTgu7zEU`1c`l;tfydiB{ zIa!mbYr1&m>=Dp~eick%AE*35>`HdV@FI53&}*T@zg#`bPRILN{m(?Zh88CuUX_{hH4Lxhwt*D`~mk4yWZ_v1e7$*3HGP78{fp9-Tbe z-JX$q%_{z{{CmY`voEA=!oOCZ^=Ds2%N(krO^3m$A8$|Zf#>38%5H;mt{wCt1FbGD(i^1SP4*{@f?CNKmt7w%U!^XcNMqMxz`M$vNrcpKjYv&8b9;ADBZ z*VN&kgHOR-@o1c9X8g0}TY2U!^vsv6@ejnZhwVYnoLa^|iQg2mm)54eNLv8u)5Bl_ zOszC7>N}*oxR2)f+0!%6&xF)*E^RG&7qEB05BP6-bN~DgJKuTT!OocHKAXMvCGAOG zCwPoL2j~1~2;-reb{o!|Y$*37@oxOA-REe#v!~$Uco+P&_GjNY5qd+`?$KhmK+fhe z+<-O>U+!J5EERXxC-K*iets8Ugb%DN?zA7{6XY+W9RRQMABL=v7a{L@0sW;?u|FTf z&R+8c&UeCJ(Cff+o}aaP3hi_LmoTx)nu4sspS5QlJdmG#|15cvaPAX3<0kz3*f+CR zR9YYE`T?$F*AXknXX7gTR;gIekK^ZQAJLY=?a*Hxci`^kc>K)ejQfuA4$+3nD)*Z* z^Aley_5*Zbw}X}PGf%RHt8rWSUD>{J)`nsHp8T{i^-khvpKf32yePLGAkREc{XN;A zvHy~n{WkNnF3g1y{8M2Dd$F8z+0V<%9(1g-AJDRPbH8gy8w}(4*@H4SGDnB-bMJV= zJEdQiE4Prg4Lf7ls8swLVCsBRY$$F>+Z+B8?+ab^)j-@6nv188MwrtVl#2bJD}Sii z0>~Vl#%~MR-`c_%w0UBOu`^dsh3ps2@ke4kw66(!KD!3~g5OKL20!aQbM-#OHeelsG_y|k^^ zchaWgG58GpHJ*)khU^PhR~F~~Qg-(18T_21yYlZ-XU^@6?Nzk2p$tDk&scVspS?1B zZ2Bjksj@ep%3s9Kew(xV1)SW>m2t^|?f4J!uP^BzISu)lcgdZ|K3Yy0e-pS1TGFl+ zJ4o)M{Pg*@cmgecP5u+?u6UN5X0-It>$m|lqEF%9Ehlw7MC%S)v)8o_@4}h0o6_dW z$sF8N{>HS9_)gl+xSzZ|@fY;`HsDry^|ZAC?Njze@*czg!S~RvgTxlnrYV#0c~<`V z{GpJ3Itccs=kKaK0H?6W(B6=@IlE4|vqsrU*iD)3#nNBrz;M_^d=Bgb9pLd9_tyvy-bu-v~_ zjHX{9Kl5S%zD!?lfG1Vi?{EYBZ@e!441Wu6!c2bl#+1qD^&Pz5wX~eC@6kS~EbbuL z`)bO&180q8jUEdP+4+oI!S1Sj&W!=I-71UE?GxE;*mud<2%iXT*oVnG0B5cZgKni_ zjbsnnNGxkRaW|Vsse4kj_@VQubcGlk__8oYpx?0F9#~ag9=kfI1 z?Y?7Ab*DL>zOkIn^mA#qUD~$theP&| zN5u2}PxkBW*{kHHf9J4E;!WhA3Ogp3zBT)0_UG&km08JtnEeqRkB^4TgUpRT*wM0% z-@{pJKY7+QRmB|Ho&H;?xH~-~R)^jKRxz)TB)ZG&Tl(5qi4;1rrbU9+Cm>_C}$JM z9pgK(%)jxFzZsN%$i9%dnSJm<=naj^i|^aQqxi3uoLkDD1-sF5=1!rlP`(Aucx}nP zR%{l36`lw=Pji{h4VejG2;2&eSWD} zdzpum#kLh2hku6bq1AYayqD?S`2!&D^)!A$tTxQ#XMfEaJb~V>F=CT z?1AtA?4eAayGwa7_jYDagpI?rdhmpDJ=A-S-1Ygb#TM}QWiO%~2We0C|JCf<*)#EA z{z`ejLKkJ`(wD=Czd8HCSoSx|i@W^dN^^>xIg)+7xp>b0+<~va`-&e%dy=31XJ5!? z*6qYz5L*i;_eplW$So zlHI&ye)B(KFJNb1+7MDU&pb`J%#X+LPW(}Lxcs#1Mfy5;B7J^Yajs-f%RTH^`rYh3 zar){H+EwhearTkSftl9G3bC=;@@{3(H>2r!w|u_aN8WyN8uPQB|B%-fj}d!-KC-;n zi`EfeM$dV-l--0rSndRNU%9uz3dlR9-Umz86%1id#;c*7GPUu+c!qLQ#d3D$zHpwp zvd-RwUrWXaHW$lz+k{px`Eu67t!cR@eNWH%Gl<@apD}m>9$-H$_Dh+wL%f__hqfvF zDR&lsro0dMskb%G8074~jNMO8)=SR$-t;4&8~Z5yDy}bfjJ)COKJ4l8KlZG-`YZ3a zBkcyU%(-uH-m@kQgIh|)*kvAe=GVsQgWM&vXQ$sYkM6;r$jzO03%pNNQEs(oWlio+ zdyt)XJA-`%-db+z>PYX3>uFQ`lf*Nx8?kc^94_}6+AvtoeiApK=RTNc=N&eK8T^b_ z?q(Ap>v?nen``T5?7iiD$j>|GK9RYebus4C`Sem|bI*cTp_lfLQBZUrZ*b1VK| z@^9d;6kpBGdY#YjBDRwKJA48=@KY}9=po3R^g*08ur7Uaskpm8tKPTS1L)Q4^!2^` zjpS_3UkhiiNSl_49Sxt0*R3q}-0Tf$*Dc~5cV+m-(( zJ@@U)*t^kE-zZoL9puh|w)AcAq_W~%eO>t~_DEXxivFb<|NHM62Fc4_eF?vl^4qhQ z;rn11?E|@4*GEAge#S2IzN>Pz;Y`|k>OPtM8$Eqo#?BaxhxxEa$=wP+0vW%I<14gz z@LyOAZD^VA?^K$HIQ^Hk)d)6#m-)5Ii`@SFdUEo6?&SZ0+luX^t=SU~rp<$k;VikC zR}JWM`9H#rFb;CoeTTDG+*B%_|Bd{$=x;@X!B7`Uw6yO@cGiC@xCpj^LA330&Wh{h zJcrXSS+m1&`Z&LDyB8<#d+kgcbEa>j?E}SfrX3E`_;t#SEql7MOXa3**(0x}t-_l? zcetly9paq3`^(GSp<#LPxhQ-1cI=V#rMRuUbHs0oE%rVPf!%4@BQq{bV1m3!^!#nB zjoIzR(~rME#x3)zDNN@-RjTp7f1kgxT>mUD&ai)Z-W_7~;C1%pV)dMYp_%zNsY)0Q2HsfcX*;TBHK9S$Y zbI#@ODSkc7rf1GPh}YuJ5^K%winBkwCzf@Z{jRsNXV8CHR-CQfX<4gViRTPnx3aJ^ zA9Ig-a z5U5>Myl>h#wX*nkl3Dl-ZM`18RVMfN%!~B(2g(ef^`x~^_xAj|={vw?w43m+5YBk* zNL%cAE#&RN&N@o}uYyao~ewh9~zZukqb>LlkcV#mt@}8;pJnh?>eGq#(eQSOhdrZlk#Cw)$ z3wtZbohRemn4b0c7_B!u^JoKnDQwE_O&?vQUzXb^_?d?T#BN~czd8C**^6l7`5&?8 zvj2p7%G{*P6L2B_Yq5jbeU)u4wm!cJJjlKW7SVg*m(-K*MY1kN@N40$_fOYFMGi$a>mf^#%X`Wx1r5rPsUl_o$v&hEY_F5LM&sRXCDX0Ruy}9 z`n{jLFXg5DaQ6MB!p^+R`H^R&e5W$)q~*Umo5pX!uauj8a6fs!dDq?f$$b!Cil@mv zRox@S&*t}`=X2{wWx7B!_);HC!D-84@f+c4xsCaE(No`#%4E*n!){N{K7Fy+W7;r- z*4cBK^WUWJ&h9CHh}ft6)i}S~Xs6sWa1?DCTu+~eud6J6e=|fZ>nZQpiJpCQ0^|(I zd*=6(2k|rSc93%x3}oK`8TW}WfPIpBvtGYvp97gQyNR7%S^S$u2U<`5$Nu}C%V;&l z$KosDFBn-We!rD=HuA9b_i|53xqdXYCya&+^aVXZ>WKYRlhUZacXn_#5)Ovd^q?7d0nq zdftn)(`e_5-zE1`+UD%7X>UR1@$=@z4q^vGfAOr-TCkbet>#@H`WSZR+`Evr{Um;N zmAeytANE&Jl6#O?_RX9x`Q1~_h$&(tpugC`${fx<-E&&W$+`6)|KDQQLO=FzICX!9 z)9*LR%iMfPENf*J?M?M&JuKiiP{)0=tcBbG^Xx6@`FjERn*;e90ss6>fq(v{!T-sz z^LGXEHxlx<29h5;WfO}Z4)V7JQg7-DQZAg_{QZNJ3-b33^7jj3r`$iW{LO{fc}`;a z+Y0}*@t^$s?SdYVzmpIx&rg|@{ipuK(vHN!`5OO=m&#>SAplgCB#eu_p9O z&5u00HFSoL^jG|KVN*GWRr@>zouJkO&LDaRDCzSDa820Tc<)nPj8XQ#`Rvv(6lOz5 zSgXQ)LYd|9*`Kkmk^6#iTFxF!Kb1ZhE@b~@jK*QBxkJA%{As*I z&NbTIh5fwTOYqlF8+!2T;#u&Jazo`kNZW`#1+r%~7n@wI{p?1byFh*S(KdqYQ~6sJ z>%+;__C)wd-evsB@^T(@lyesS54?bXt7n|a?`%G9$bS%?6KhMmp52+SbUQygX9}yd;p464IcWd$Wl^c$K zpmo99$bT3=LLZ1*z-n4^_zT{V*R8@Bi1&^z-kbjc|3-EPV>|+yI*iF5Gt^!Bt{*stNw%e7J21MzYEykF+_GX5oaF20Gr zRyXGl`%`$MiY;~(`!@Wg@{PR1RqV&{JiJ6~V|w<+M*Qr9Z^KgW@^Y0q0RJ`L>Q@x& zD}C3Co$)+Z9i8On-TT7D^jY*SIOo?Ybv>>+Yi;hcpzLkF>2 zz#ZzH1?jK1s*3)}`=+l?*4fQ-sxBVSL{|WOdT1Y*6iQKGB5L<*~?b)Psh36WE{_dI*|P` zLRu$hH?xfA<@b8E0mF*yV>{IBEL;Afw-iMa;f3JAnZ&UUrc%XRNnK_U-_66L<{xALpwuS6@Gvyov zgW26_H?nVMCub<+9n&Yf!M@sck$A?g5$;}Id>^nke{KFT;=^d8Aa|k@an{q%aIJXi z%-k5PUpmsBqWueQWS@as7>oa~XRvc-?uu(wJ4f-mmHJcc7-iEpjpU~6Gvbr^TUe|0 z#M|*l(bL~KyYHs;;5UUi?Chz}tE-zbt!QPm+<`BH%%!}4FR_ey3w%+vz0;nPK3xsX z<$ocTv6{(Gd(Y?Z2MySnGacB!urs&EP36Un^}O%oE@1ChUHm?90e_145%35*Yb#@zcfJ~DUfsz) z2I^GUbL8ZmAJ9LS;@l^mVfT}F7+x-?CmtoXeq@PpfAhGNL$HNTB*w0tySo%B0@3V$g(pXp8lDfes)~DhU3T`8dQbz_rq_X{73PVui`i4*er?EJd<*+iu{m(Q z{0CvBc`_HD0c92TdH6S12Pl^C0KUnY7Nd=lEIE{{t=e(TnL9!8*`Z z-!G+SpUD`Ere(fw&TfaZC)T1J22ZMg8SOO4e7#1j9{oS`yI^DZOROP1?|27$7swpl z4LZ;!;)lhXL0$eNxPg{&-BE7(E_3J!{#?kjuQeyL-`&K{8n}wR0v5uBVmss7-s5TI zXIHTK53x7U#;@>1TK22X^w)9j!*%H&LO&QTehF+Oeir;hdy=0s?`iM!82?22Tb}g| z?KrWuVLRF+{uta4%H?i}=i>CsHuw`|yU=o{NFN?%&b`k+gk6D?cPy@g%*%hqHe;T3 zlKr4Jy&Zcsei|~?YvZ)zLiII)_|vszB&1DGL3`T8uo9-i6|@1`F{PsToO?CmTpg|-AfhyTJw z%k0(F#=vvGm;1gts^n!KO#PV$_t2J$jfKpqd`9^i(*6galX%8w-3oh;{c<4fP4;#0 zI(rg6L;Nmw&anmT^UI5UG5f&U%A~IL@^bf>kJFB`amM0hNFVmZ^XcPYKX&Ho7S;A3 z^Xv!l`^|+)e)=x^?*#aW9{X9GzRX;$1KA&TtJYt5EMB3`jNQ?YeqM^pX=_3H>kZ`_ z(6h%J${fSQ@#%L5$lZC$0>h%xVpG=jKqty zqaW=|>m6B?Ew6ccmK;6)Of&qh^4Nq z^^f3M_9AWE8_&Rx;nX<_Uxi0P_Ub{|#&|>62x>zGWL#7KLF)Zh&cf>d{~T$~zbcTEb)ENZ15MajYrEh!cvw}jKYZ;B z$R65RPWrSybdqx;Wc?gnx)y1Beds+5^x{!N7_J*_BQ?&0;b+5<1 zr^0;acbA*CWgT8bAI|<5>a#z@IVZAyGdK5<^D<65y2JJS6Y#q59E`0fVh6ETun)z* z%UQ@iALhV;>RUn2_xQI##``c@*6~?#|5X2Ee%f~q^kbilN5CKA?~CWTlki&A#T{ou zoVh+1KQ6Y+JSpQh=C6at;nw(Z+)#Y)^5V18TD0`rG+JHSCG5~F}P$=+Drzu3Ft4$9_vn^-65?>X{bt}?dtjpSCb zmqYq+u)jZ&ww2W5#6wzds&9o6XMNyFGuE{6_ps z>6s^;_*>AgW@k-htd7LD!Q*0`Ju_{|98aBd*%_~l+ur!E<;C~Yr>O4;ctV+P@m26C z)RdP#PTMjz_d(9Btkc8bIkEK38?>$PwvawNn}40ym9UE4Sbq(pFJK>z_lL}t*Tt8@ zckWCgSLEfseH5gwVQ{~^?7NS$r{IqAGOo4R zIS(GfZ^>_|uGTnnaecYB;N%}d%RZJhxHkS@dAaAXZ>6PgeuXalzm&TZ()ZbCG9E+a zc4epSJIX7=xAW`b=`ezwx|Yc8!M|I6efF;AQzQPnVx9TTm3;s%FL{6C+lxK{^6ZxQ zX*mzz(U5yf*6g-$ulNpPTd23D@oG|G-{yZRCwoTb+-x~_vmauo%p!jFk*wVg{I=|6 z%G{5;@uOwl55k$tEyYfU4V4>>Uxt0fGZu;GjC%!d0vj9mXYlP}o8g<_Wci=)mw4VZ ze%4z4cKmtxIkDlid~Q7hKMT26EQZX%iy`x8j=Z+&_(fjk)LiyZ`rfb-a`)MZUK6Kp z(gzu*v!Eom_uu+ua*m;O^8AcZJ_mmb`FDr&=^29)jLjf^-e(y8PCRpAA!M(60fy7A zfy|+q{5*3bW$#t*_Uxa<%IR-ZdKTV~wgF6&Gh3ZG^Rfq}-I-Hw@Q3jGRrsvRPu+R% zlc7Yb0riY;MOCpDw&dTV%x1Wi+z+Ey7jvZLu0{VpAXZnI!8r9chF9T^>SB*vsLUXK zGss?fyLvXl_f{5nq6^q(!hP~G>*huFz2cGh&(&RXT>2|H{41NL;>3ulep$?lHliRZ4Ivn6w94y^W! zy~VSCm%+Yr7UBbFv*4ex%ll=XWNu8udG-g|dl9}8`oeFJu{o!r=!Y{s>lSv-r{_JV zQMY2vZph9(aH?4Ln)Ago)~Dgu%%OYPz2HK~?`xJ;7yENJ@$R%$Rr&@_rRUDkgMJ3P zo!r~_9r*byas@5-)Qt79^sKFG*^kqgK-#%G?R56x@Fg5Vo36}Q$QZtbFU9TY=dstS zD%Qr@{7c~;v2Q%9BRhM@9*{jPV>YeYo{BeshiSQcAAvtBR~K!4NISA`Z;NY*J;@%$ z&i!Z*^yaswXWbq^zmc8(*xtMTiSLwC2iJrBD~iu+_3`)e*TP+(slFbY7_?z;R$bh6 zD)1uZSIcb*Z8NQ@KTrE}$D7D+f;0bi#l83)Dm;sw_ls7}e?y&Ft1rO`a?;mX{H z5BOU@4EKSI@3GJqvW5q8veZ8UM+SY z&bpjW9}Kx)UV`W1o9JtP|4@97c)7lqu3y&t{$b7UA8vKF=ex*r%!%Ks+>c;4Ybk5* zTzby(nf!c4TMTP{|KKjDU-DTi_u+5!Q|?M@e*aK>?^IKt%+epbz!{{X_BHe@A_> zqq*@Yw1v|viqEVo>6^3PHFq|o_kuPsmA(@mQBi!pTJ!sd;(PctzkgWs`-lHn8*6_5 zu;%v<`Oc~?Y+;{V^ZN(izqr~$toi-Jn%_UH`TfJ1-#`4n|NDpk508+cI{*Lx literal 0 HcmV?d00001 From ac46167d7467a989d3d1ae77375a42c5416999da Mon Sep 17 00:00:00 2001 From: Kashif Rasul Date: Fri, 5 Dec 2025 09:28:50 +0100 Subject: [PATCH 03/16] use from torch.nn import init --- src/chronos/chronos2/model.py | 55 +++++++++++++++++++++++------------ src/chronos/chronos_bolt.py | 32 ++++++++++++++------ 2 files changed, 59 insertions(+), 28 deletions(-) diff --git a/src/chronos/chronos2/model.py b/src/chronos/chronos2/model.py index 0397be2..fb813e0 100644 --- a/src/chronos/chronos2/model.py +++ b/src/chronos/chronos2/model.py @@ -10,9 +10,18 @@ from typing import cast import torch import torch.nn as nn from einops import rearrange, repeat +from packaging import version +from transformers import __version__ as transformers_version from transformers.modeling_utils import PreTrainedModel from transformers.utils import ModelOutput +# In transformers v5, use guarded init functions that check _is_hf_initialized +# to avoid re-initializing weights loaded from checkpoint +if version.parse(transformers_version) >= version.parse("5.0.0.dev0"): + from transformers import initialization as init +else: + from torch.nn import init + from chronos.chronos_bolt import InstanceNorm, Patch from .config import Chronos2CoreConfig, Chronos2ForecastingConfig @@ -264,53 +273,61 @@ class Chronos2Model(PreTrainedModel): self.post_init() def _init_weights(self, module): - super()._init_weights(module) - """Initialize the weights""" + """Initialize the weights. + + Uses transformers.initialization functions which are guarded against + re-initializing weights that have already been loaded from checkpoint + (they check the _is_hf_initialized flag on each parameter). + """ factor = self.config.initializer_factor if isinstance(module, Chronos2LayerNorm): - module.weight.data.fill_(factor * 1.0) + init.constant_(module.weight, factor * 1.0) elif isinstance(module, MLP): # Mesh TensorFlow FF initialization # See https://github.com/tensorflow/mesh/blob/master/mesh_tensorflow/transformer/transformer_layers.py#L56 # and https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/layers.py#L89 - module.wi.weight.data.normal_(mean=0.0, std=factor * ((self.config.d_model) ** -0.5)) + init.normal_(module.wi.weight, mean=0.0, std=factor * ((self.config.d_model) ** -0.5)) if hasattr(module.wi, "bias") and module.wi.bias is not None: - module.wi.bias.data.zero_() - module.wo.weight.data.normal_(mean=0.0, std=factor * ((self.config.d_ff) ** -0.5)) + init.zeros_(module.wi.bias) + init.normal_(module.wo.weight, mean=0.0, std=factor * ((self.config.d_ff) ** -0.5)) if hasattr(module.wo, "bias") and module.wo.bias is not None: - module.wo.bias.data.zero_() + init.zeros_(module.wo.bias) elif isinstance(module, MHA): # Mesh TensorFlow attention initialization to avoid scaling before softmax # See https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/transformer/attention.py#L136 d_model = self.config.d_model kv_proj_dim = self.config.d_kv n_heads = self.config.num_heads - module.q.weight.data.normal_(mean=0.0, std=factor * ((d_model * kv_proj_dim) ** -0.5)) - module.k.weight.data.normal_(mean=0.0, std=factor * (d_model**-0.5)) - module.v.weight.data.normal_(mean=0.0, std=factor * (d_model**-0.5)) - module.o.weight.data.normal_(mean=0.0, std=factor * ((n_heads * kv_proj_dim) ** -0.5)) + init.normal_(module.q.weight, mean=0.0, std=factor * ((d_model * kv_proj_dim) ** -0.5)) + init.normal_(module.k.weight, mean=0.0, std=factor * (d_model**-0.5)) + init.normal_(module.v.weight, mean=0.0, std=factor * (d_model**-0.5)) + init.normal_(module.o.weight, mean=0.0, std=factor * ((n_heads * kv_proj_dim) ** -0.5)) elif isinstance(module, (Chronos2Model)): - module.shared.weight.data.normal_(mean=0.0, std=factor * 1.0) + init.normal_(module.shared.weight, mean=0.0, std=factor * 1.0) elif isinstance(module, ResidualBlock): - module.hidden_layer.weight.data.normal_( + init.normal_( + module.hidden_layer.weight, mean=0.0, std=factor * (module.hidden_layer.weight.size(-1) ** -0.5), ) if hasattr(module.hidden_layer, "bias") and module.hidden_layer.bias is not None: - module.hidden_layer.bias.data.zero_() + init.zeros_(module.hidden_layer.bias) - module.residual_layer.weight.data.normal_( + init.normal_( + module.residual_layer.weight, mean=0.0, std=factor * (module.residual_layer.weight.size(-1) ** -0.5), ) if hasattr(module.residual_layer, "bias") and module.residual_layer.bias is not None: - module.residual_layer.bias.data.zero_() + init.zeros_(module.residual_layer.bias) - module.output_layer.weight.data.normal_( - mean=0.0, std=factor * (module.output_layer.weight.size(-1) ** -0.5) + init.normal_( + module.output_layer.weight, + mean=0.0, + std=factor * (module.output_layer.weight.size(-1) ** -0.5), ) if hasattr(module.output_layer, "bias") and module.output_layer.bias is not None: - module.output_layer.bias.data.zero_() + init.zeros_(module.output_layer.bias) def _validate_input( self, diff --git a/src/chronos/chronos_bolt.py b/src/chronos/chronos_bolt.py index 6a4df36..01610c2 100644 --- a/src/chronos/chronos_bolt.py +++ b/src/chronos/chronos_bolt.py @@ -32,8 +32,16 @@ logger = logging.getLogger(__file__) # Transformers v5 introduced breaking changes: # - T5Stack.__init__ no longer accepts embed_tokens argument # - _tied_weights_keys changed from list to dict format +# - _init_weights needs guarded init functions to avoid re-initializing loaded weights _TRANSFORMERS_V5 = version.parse(transformers_version) >= version.parse("5.0.0.dev0") +# In transformers v5, use guarded init functions that check _is_hf_initialized +# to avoid re-initializing weights loaded from checkpoint +if _TRANSFORMERS_V5: + from transformers import initialization as init +else: + from torch.nn import init + def _create_t5_stack(config: T5Config, embed_tokens: nn.Embedding) -> T5Stack: """ @@ -243,29 +251,35 @@ class ChronosBoltModelForForecasting(T5PreTrainedModel): self.device_map = None def _init_weights(self, module): - super()._init_weights(module) - """Initialize the weights""" + """Initialize the weights. + + Uses transformers.initialization functions which are guarded against + re-initializing weights that have already been loaded from checkpoint + (they check the _is_hf_initialized flag on each parameter). + """ factor = self.config.initializer_factor if isinstance(module, (self.__class__)): - module.shared.weight.data.normal_(mean=0.0, std=factor * 1.0) + init.normal_(module.shared.weight, mean=0.0, std=factor * 1.0) elif isinstance(module, ResidualBlock): - module.hidden_layer.weight.data.normal_( + init.normal_( + module.hidden_layer.weight, mean=0.0, std=factor * ((self.chronos_config.input_patch_size * 2) ** -0.5), ) if hasattr(module.hidden_layer, "bias") and module.hidden_layer.bias is not None: - module.hidden_layer.bias.data.zero_() + init.zeros_(module.hidden_layer.bias) - module.residual_layer.weight.data.normal_( + init.normal_( + module.residual_layer.weight, mean=0.0, std=factor * ((self.chronos_config.input_patch_size * 2) ** -0.5), ) if hasattr(module.residual_layer, "bias") and module.residual_layer.bias is not None: - module.residual_layer.bias.data.zero_() + init.zeros_(module.residual_layer.bias) - module.output_layer.weight.data.normal_(mean=0.0, std=factor * ((self.config.d_ff) ** -0.5)) + init.normal_(module.output_layer.weight, mean=0.0, std=factor * ((self.config.d_ff) ** -0.5)) if hasattr(module.output_layer, "bias") and module.output_layer.bias is not None: - module.output_layer.bias.data.zero_() + init.zeros_(module.output_layer.bias) def encode( self, context: torch.Tensor, mask: Optional[torch.Tensor] = None From c5ef52796dd1366b62b717b663a8ad4ad5b0c859 Mon Sep 17 00:00:00 2001 From: Kashif Rasul Date: Fri, 5 Dec 2025 09:31:17 +0100 Subject: [PATCH 04/16] Clean up comments related to Transformers v5 --- src/chronos/chronos_bolt.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/chronos/chronos_bolt.py b/src/chronos/chronos_bolt.py index 01610c2..5f218f1 100644 --- a/src/chronos/chronos_bolt.py +++ b/src/chronos/chronos_bolt.py @@ -29,10 +29,6 @@ from .base import BaseChronosPipeline, ForecastType logger = logging.getLogger(__file__) -# Transformers v5 introduced breaking changes: -# - T5Stack.__init__ no longer accepts embed_tokens argument -# - _tied_weights_keys changed from list to dict format -# - _init_weights needs guarded init functions to avoid re-initializing loaded weights _TRANSFORMERS_V5 = version.parse(transformers_version) >= version.parse("5.0.0.dev0") # In transformers v5, use guarded init functions that check _is_hf_initialized @@ -252,10 +248,6 @@ class ChronosBoltModelForForecasting(T5PreTrainedModel): def _init_weights(self, module): """Initialize the weights. - - Uses transformers.initialization functions which are guarded against - re-initializing weights that have already been loaded from checkpoint - (they check the _is_hf_initialized flag on each parameter). """ factor = self.config.initializer_factor if isinstance(module, (self.__class__)): From 1c270eb958c1538a86c7ba97b8ef3e2f87d4832f Mon Sep 17 00:00:00 2001 From: Kashif Rasul Date: Fri, 5 Dec 2025 09:32:08 +0100 Subject: [PATCH 05/16] Clean up comments related to weight initialization Remove comments about weight initialization in transformers. --- src/chronos/chronos2/model.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/chronos/chronos2/model.py b/src/chronos/chronos2/model.py index fb813e0..be1d436 100644 --- a/src/chronos/chronos2/model.py +++ b/src/chronos/chronos2/model.py @@ -15,8 +15,7 @@ from transformers import __version__ as transformers_version from transformers.modeling_utils import PreTrainedModel from transformers.utils import ModelOutput -# In transformers v5, use guarded init functions that check _is_hf_initialized -# to avoid re-initializing weights loaded from checkpoint + if version.parse(transformers_version) >= version.parse("5.0.0.dev0"): from transformers import initialization as init else: @@ -274,10 +273,6 @@ class Chronos2Model(PreTrainedModel): def _init_weights(self, module): """Initialize the weights. - - Uses transformers.initialization functions which are guarded against - re-initializing weights that have already been loaded from checkpoint - (they check the _is_hf_initialized flag on each parameter). """ factor = self.config.initializer_factor if isinstance(module, Chronos2LayerNorm): From b64d9db0b02e7412cc86b34f736955253c4f37b8 Mon Sep 17 00:00:00 2001 From: Kashif Rasul Date: Fri, 5 Dec 2025 09:35:35 +0100 Subject: [PATCH 06/16] Apply suggestion from @kashif --- src/chronos/chronos2/model.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/chronos/chronos2/model.py b/src/chronos/chronos2/model.py index be1d436..f3c5633 100644 --- a/src/chronos/chronos2/model.py +++ b/src/chronos/chronos2/model.py @@ -272,8 +272,7 @@ class Chronos2Model(PreTrainedModel): self.post_init() def _init_weights(self, module): - """Initialize the weights. - """ + """Initialize the weights""" factor = self.config.initializer_factor if isinstance(module, Chronos2LayerNorm): init.constant_(module.weight, factor * 1.0) From 0e85ae60969d08ec0e133c562b7e01008e283604 Mon Sep 17 00:00:00 2001 From: Kashif Rasul Date: Fri, 5 Dec 2025 09:37:03 +0100 Subject: [PATCH 07/16] Apply suggestion from @kashif --- src/chronos/chronos_bolt.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/chronos/chronos_bolt.py b/src/chronos/chronos_bolt.py index 5f218f1..2aa4f0f 100644 --- a/src/chronos/chronos_bolt.py +++ b/src/chronos/chronos_bolt.py @@ -247,8 +247,7 @@ class ChronosBoltModelForForecasting(T5PreTrainedModel): self.device_map = None def _init_weights(self, module): - """Initialize the weights. - """ + """Initialize the weights""" factor = self.config.initializer_factor if isinstance(module, (self.__class__)): init.normal_(module.shared.weight, mean=0.0, std=factor * 1.0) From f806213c2eb65a8672be7008b796b02bf3fbe38f Mon Sep 17 00:00:00 2001 From: Kashif Rasul Date: Fri, 5 Dec 2025 09:42:06 +0100 Subject: [PATCH 08/16] add back super --- src/chronos/chronos2/model.py | 1 + src/chronos/chronos_bolt.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/chronos/chronos2/model.py b/src/chronos/chronos2/model.py index f3c5633..cee0f3c 100644 --- a/src/chronos/chronos2/model.py +++ b/src/chronos/chronos2/model.py @@ -272,6 +272,7 @@ class Chronos2Model(PreTrainedModel): self.post_init() def _init_weights(self, module): + super()._init_weights(module) """Initialize the weights""" factor = self.config.initializer_factor if isinstance(module, Chronos2LayerNorm): diff --git a/src/chronos/chronos_bolt.py b/src/chronos/chronos_bolt.py index 2aa4f0f..d3d5166 100644 --- a/src/chronos/chronos_bolt.py +++ b/src/chronos/chronos_bolt.py @@ -247,6 +247,7 @@ class ChronosBoltModelForForecasting(T5PreTrainedModel): self.device_map = None def _init_weights(self, module): + super()._init_weights(module) """Initialize the weights""" factor = self.config.initializer_factor if isinstance(module, (self.__class__)): From fc785d9593322a94da571ed4f8889d0a9b11547e Mon Sep 17 00:00:00 2001 From: Kashif Rasul Date: Mon, 8 Dec 2025 14:04:38 +0100 Subject: [PATCH 09/16] stack.set_input_embeddings(embed_tokens) not needed as post_init will add it --- src/chronos/chronos_bolt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/chronos/chronos_bolt.py b/src/chronos/chronos_bolt.py index d3d5166..e8daba7 100644 --- a/src/chronos/chronos_bolt.py +++ b/src/chronos/chronos_bolt.py @@ -49,7 +49,6 @@ def _create_t5_stack(config: T5Config, embed_tokens: nn.Embedding) -> T5Stack: """ if _TRANSFORMERS_V5: stack = T5Stack(config) - stack.set_input_embeddings(embed_tokens) return stack else: return T5Stack(config, embed_tokens) From 378a0d19b28f9f17f46563ffe57aea744d771942 Mon Sep 17 00:00:00 2001 From: Kashif Rasul Date: Mon, 8 Dec 2025 14:05:29 +0100 Subject: [PATCH 10/16] return directly --- src/chronos/chronos_bolt.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/chronos/chronos_bolt.py b/src/chronos/chronos_bolt.py index e8daba7..84e9405 100644 --- a/src/chronos/chronos_bolt.py +++ b/src/chronos/chronos_bolt.py @@ -48,8 +48,7 @@ def _create_t5_stack(config: T5Config, embed_tokens: nn.Embedding) -> T5Stack: In v5, T5Stack.__init__ only accepts (config), and embed_tokens must be set separately. """ if _TRANSFORMERS_V5: - stack = T5Stack(config) - return stack + return T5Stack(config) else: return T5Stack(config, embed_tokens) From f29ff338e5e29d6e7bd90766a3d2b60971b9c7dd Mon Sep 17 00:00:00 2001 From: Kashif Rasul Date: Fri, 9 Jan 2026 19:14:50 +0100 Subject: [PATCH 11/16] fix buffer inits --- src/chronos/chronos2/layers.py | 64 +++++++++++++++++++++++++--------- src/chronos/chronos2/model.py | 6 +++- 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/src/chronos/chronos2/layers.py b/src/chronos/chronos2/layers.py index b00e8a8..15b3e0a 100644 --- a/src/chronos/chronos2/layers.py +++ b/src/chronos/chronos2/layers.py @@ -15,27 +15,59 @@ from transformers.utils import ModelOutput from .config import Chronos2CoreConfig -class RoPE(nn.Module): +class Chronos2RotaryEmbedding(nn.Module): """Applies rotary position embeddings (RoPE) to input tensors. + This implementation follows the transformers v5 pattern for RotaryEmbedding classes, + which enables automatic buffer reinitialization via the base `_init_weights` method. + Implementation adapted from: - https://github.com/huggingface/transformers/blob/965cf677695dd363285831afca8cf479cf0c600c/src/transformers/models/llama/modeling_llama.py#L95 + https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py """ - def __init__(self, dim: int, base: float = 10000): + inv_freq: torch.Tensor # type hint for register_buffer + + def __init__(self, config: Chronos2CoreConfig, device=None): super().__init__() - self.dim = dim - self.base = base - inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2, dtype=torch.int64).float() / self.dim)) - self.inv_freq: torch.Tensor # type hint for type checker - self.register_buffer("inv_freq", tensor=inv_freq, persistent=False) + self.config = config + self.rope_type = "default" + inv_freq, self.attention_scaling = self.compute_default_rope_parameters(config, device) + + self.register_buffer("inv_freq", inv_freq, persistent=False) + self.register_buffer("original_inv_freq", inv_freq.clone(), persistent=False) + + @staticmethod + def compute_default_rope_parameters( + config: Chronos2CoreConfig, + device: torch.device | None = None, + seq_len: int | None = None, + ) -> tuple[torch.Tensor, float]: + """ + Computes the inverse frequencies for RoPE embeddings. + + Args: + config: The model configuration containing rope_theta and d_kv. + device: The device to use for initialization. + seq_len: Unused, kept for API compatibility with transformers. + + Returns: + Tuple of (inv_freq tensor, attention_scaling factor). + """ + base = config.rope_theta + dim = config.d_kv + + attention_factor = 1.0 # Unused in default RoPE + + inv_freq = 1.0 / ( + base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim) + ) + return inv_freq, attention_factor @torch.no_grad() def forward(self, x: torch.Tensor, position_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: # x: [bs, num_attention_heads, seq_len, head_size] - self.inv_freq.to(x.device) - inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1) + inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device) position_ids_expanded = position_ids[:, None, :].float() # Force float32 since bfloat16 loses precision on long contexts # See https://github.com/huggingface/transformers/pull/29285 @@ -44,8 +76,8 @@ class RoPE(nn.Module): with torch.autocast(device_type=device_type, enabled=False): freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) emb = torch.cat((freqs, freqs), dim=-1) - cos = emb.cos() - sin = emb.sin() + cos = emb.cos() * self.attention_scaling + sin = emb.sin() * self.attention_scaling return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) @staticmethod @@ -78,8 +110,8 @@ class RoPE(nn.Module): """ cos = cos.unsqueeze(unsqueeze_dim) sin = sin.unsqueeze(unsqueeze_dim) - q_embed = (q * cos) + (RoPE.rotate_half(q) * sin) - k_embed = (k * cos) + (RoPE.rotate_half(k) * sin) + q_embed = (q * cos) + (Chronos2RotaryEmbedding.rotate_half(q) * sin) + k_embed = (k * cos) + (Chronos2RotaryEmbedding.rotate_half(k) * sin) return q_embed, k_embed @@ -164,7 +196,7 @@ class MHA(nn.Module): self.use_rope = use_rope if use_rope: - self.rope_embed = RoPE(dim=self.kv_proj_dim, base=config.rope_theta) + self.rope_embed = Chronos2RotaryEmbedding(config=config) def _eager_attention( self, @@ -277,7 +309,7 @@ class MHA(nn.Module): value_states = shape(self.v(hidden_states)) if self.use_rope: cos, sin = self.rope_embed(value_states, position_ids) - query_states, key_states = RoPE.apply_rotary_pos_emb(query_states, key_states, cos, sin) + query_states, key_states = Chronos2RotaryEmbedding.apply_rotary_pos_emb(query_states, key_states, cos, sin) if attn_implementation == "sdpa": attn_output, attn_weights = self._sdpa_attention(query_states, key_states, value_states, mask) diff --git a/src/chronos/chronos2/model.py b/src/chronos/chronos2/model.py index cee0f3c..4a9ff4c 100644 --- a/src/chronos/chronos2/model.py +++ b/src/chronos/chronos2/model.py @@ -297,8 +297,12 @@ class Chronos2Model(PreTrainedModel): init.normal_(module.k.weight, mean=0.0, std=factor * (d_model**-0.5)) init.normal_(module.v.weight, mean=0.0, std=factor * (d_model**-0.5)) init.normal_(module.o.weight, mean=0.0, std=factor * ((n_heads * kv_proj_dim) ** -0.5)) - elif isinstance(module, (Chronos2Model)): + elif isinstance(module, Chronos2Model): init.normal_(module.shared.weight, mean=0.0, std=factor * 1.0) + quantiles = torch.tensor( + module.chronos_config.quantiles, dtype=module.dtype, device=module.quantiles.device + ) + init.copy_(module.quantiles, quantiles) elif isinstance(module, ResidualBlock): init.normal_( module.hidden_layer.weight, From 16a91d5b584f870ec76315d9c42823a7fa8f4ecd Mon Sep 17 00:00:00 2001 From: Kashif Rasul Date: Fri, 9 Jan 2026 19:19:41 +0100 Subject: [PATCH 12/16] skip copy_ for v4 --- src/chronos/chronos2/model.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/chronos/chronos2/model.py b/src/chronos/chronos2/model.py index 4a9ff4c..c96d0bc 100644 --- a/src/chronos/chronos2/model.py +++ b/src/chronos/chronos2/model.py @@ -16,7 +16,9 @@ from transformers.modeling_utils import PreTrainedModel from transformers.utils import ModelOutput -if version.parse(transformers_version) >= version.parse("5.0.0.dev0"): +_TRANSFORMERS_V5 = version.parse(transformers_version) >= version.parse("5.0.0.dev0") + +if _TRANSFORMERS_V5: from transformers import initialization as init else: from torch.nn import init @@ -299,10 +301,11 @@ class Chronos2Model(PreTrainedModel): init.normal_(module.o.weight, mean=0.0, std=factor * ((n_heads * kv_proj_dim) ** -0.5)) elif isinstance(module, Chronos2Model): init.normal_(module.shared.weight, mean=0.0, std=factor * 1.0) - quantiles = torch.tensor( - module.chronos_config.quantiles, dtype=module.dtype, device=module.quantiles.device - ) - init.copy_(module.quantiles, quantiles) + if _TRANSFORMERS_V5: + quantiles = torch.tensor( + module.chronos_config.quantiles, dtype=module.dtype, device=module.quantiles.device + ) + init.copy_(module.quantiles, quantiles) elif isinstance(module, ResidualBlock): init.normal_( module.hidden_layer.weight, From f669e6c5ca58a85659a22471ab3ab47dce970cd3 Mon Sep 17 00:00:00 2001 From: Kashif Rasul Date: Fri, 9 Jan 2026 20:44:20 +0100 Subject: [PATCH 13/16] handle dtype warning --- src/chronos/base.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/chronos/base.py b/src/chronos/base.py index 7592c46..3f92972 100644 --- a/src/chronos/base.py +++ b/src/chronos/base.py @@ -347,9 +347,11 @@ class BaseChronosPipeline(metaclass=PipelineRegistry): from transformers import AutoConfig - torch_dtype = kwargs.get("torch_dtype", "auto") - if torch_dtype != "auto" and isinstance(torch_dtype, str): - kwargs["torch_dtype"] = cls.dtypes[torch_dtype] + # Handle both torch_dtype (deprecated) and dtype arguments + dtype_value = kwargs.pop("torch_dtype", None) or kwargs.pop("dtype", "auto") + if dtype_value != "auto" and isinstance(dtype_value, str): + dtype_value = cls.dtypes[dtype_value] + kwargs["dtype"] = dtype_value config = AutoConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) is_valid_config = hasattr(config, "chronos_pipeline_class") or hasattr(config, "chronos_config") From dc0c9a1b5b0e96f31123475cbe5ce58df98986a9 Mon Sep 17 00:00:00 2001 From: Kashif Rasul Date: Tue, 13 Jan 2026 09:55:35 +0100 Subject: [PATCH 14/16] fix bolt buffer for transformers v5 --- src/chronos/chronos_bolt.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/chronos/chronos_bolt.py b/src/chronos/chronos_bolt.py index 84e9405..5b68029 100644 --- a/src/chronos/chronos_bolt.py +++ b/src/chronos/chronos_bolt.py @@ -250,6 +250,12 @@ class ChronosBoltModelForForecasting(T5PreTrainedModel): factor = self.config.initializer_factor if isinstance(module, (self.__class__)): init.normal_(module.shared.weight, mean=0.0, std=factor * 1.0) + # Reinitialize quantiles buffer for transformers v5 meta device compatibility + if _TRANSFORMERS_V5: + quantiles = torch.tensor( + module.chronos_config.quantiles, dtype=module.dtype, device=module.quantiles.device + ) + init.copy_(module.quantiles, quantiles) elif isinstance(module, ResidualBlock): init.normal_( module.hidden_layer.weight, From 9ee10fc9b6a7bebb306f2f558de708e0b145a680 Mon Sep 17 00:00:00 2001 From: Kashif Rasul Date: Tue, 14 Apr 2026 11:41:05 +0200 Subject: [PATCH 15/16] update transformers version check --- pyproject.toml | 8 ++++---- scripts/training/train.py | 4 +++- src/chronos/chronos2/model.py | 2 +- src/chronos/chronos2/pipeline.py | 3 ++- src/chronos/chronos_bolt.py | 2 +- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d9e7117..dd4e4d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,8 +15,8 @@ license = { file = "LICENSE" } requires-python = ">=3.10" dependencies = [ "torch>=2.2,<3", - "transformers>=4.41,<5", - "accelerate>=0.34,<2", + "transformers>=4.41", + "accelerate>=1.1.0", "numpy>=1.21,<3", "einops>=0.7.0,<1", "scikit-learn>=1.6.0,<2", @@ -41,14 +41,14 @@ path = "src/chronos/__about__.py" [project.optional-dependencies] extras = [ "boto3>=1.10,<2", - "peft>=0.13.0,<0.18", + "peft>=0.18.1", "fev>=0.6.1", "pandas[pyarrow]>=2.0,<2.4", ] test = [ "pytest~=8.0", "boto3>=1.10,<2", - "peft>=0.13.0,<1", + "peft>=0.18.1", "fev>=0.6.1", "pandas[pyarrow]>=2.0,<2.4", ] diff --git a/scripts/training/train.py b/scripts/training/train.py index 09d5d8e..c586460 100644 --- a/scripts/training/train.py +++ b/scripts/training/train.py @@ -21,6 +21,7 @@ import torch import torch.distributed as dist from torch.utils.data import IterableDataset, get_worker_info import transformers +from packaging import version from transformers import ( AutoModelForSeq2SeqLM, AutoModelForCausalLM, @@ -46,6 +47,7 @@ from gluonts.transform import ( from chronos import ChronosConfig, ChronosTokenizer +_TRANSFORMERS_V5 = version.parse(transformers.__version__) >= version.parse("5.0.0") app = typer.Typer(pretty_exceptions_enable=False) @@ -661,7 +663,7 @@ def main( per_device_train_batch_size=per_device_train_batch_size, learning_rate=learning_rate, lr_scheduler_type=lr_scheduler_type, - warmup_ratio=warmup_ratio, + **({"warmup_steps": round(warmup_ratio * max_steps)} if _TRANSFORMERS_V5 else {"warmup_ratio": warmup_ratio}), optim=optim, logging_strategy="steps", logging_steps=log_steps, diff --git a/src/chronos/chronos2/model.py b/src/chronos/chronos2/model.py index c96d0bc..99eefb0 100644 --- a/src/chronos/chronos2/model.py +++ b/src/chronos/chronos2/model.py @@ -16,7 +16,7 @@ from transformers.modeling_utils import PreTrainedModel from transformers.utils import ModelOutput -_TRANSFORMERS_V5 = version.parse(transformers_version) >= version.parse("5.0.0.dev0") +_TRANSFORMERS_V5 = version.parse(transformers_version) >= version.parse("5.0.0") if _TRANSFORMERS_V5: from transformers import initialization as init diff --git a/src/chronos/chronos2/pipeline.py b/src/chronos/chronos2/pipeline.py index 223689d..2cefa5f 100644 --- a/src/chronos/chronos2/pipeline.py +++ b/src/chronos/chronos2/pipeline.py @@ -22,6 +22,7 @@ from transformers.utils.peft_utils import find_adapter_config_file import chronos.chronos2 from chronos.base import BaseChronosPipeline, ForecastType from chronos.chronos2 import Chronos2Model +from chronos.chronos2.model import _TRANSFORMERS_V5 from chronos.chronos2.dataset import Chronos2Dataset, DatasetMode, TensorOrArray from chronos.df_utils import convert_df_input_to_list_of_dicts_input from chronos.utils import interpolate_quantiles, weighted_quantile @@ -270,7 +271,7 @@ class Chronos2Pipeline(BaseChronosPipeline): per_device_eval_batch_size=batch_size, learning_rate=learning_rate, lr_scheduler_type="linear", - warmup_ratio=0.0, + **({"warmup_steps": 0} if _TRANSFORMERS_V5 else {"warmup_ratio": 0.0}), optim="adamw_torch_fused", logging_strategy="steps", logging_steps=100, diff --git a/src/chronos/chronos_bolt.py b/src/chronos/chronos_bolt.py index 5b68029..aa9c1b7 100644 --- a/src/chronos/chronos_bolt.py +++ b/src/chronos/chronos_bolt.py @@ -29,7 +29,7 @@ from .base import BaseChronosPipeline, ForecastType logger = logging.getLogger(__file__) -_TRANSFORMERS_V5 = version.parse(transformers_version) >= version.parse("5.0.0.dev0") +_TRANSFORMERS_V5 = version.parse(transformers_version) >= version.parse("5.0.0") # In transformers v5, use guarded init functions that check _is_hf_initialized # to avoid re-initializing weights loaded from checkpoint From 8f249a961eec90c0fa1865a0dd6b23367aadc730 Mon Sep 17 00:00:00 2001 From: Kashif Rasul Date: Tue, 14 Apr 2026 11:46:12 +0200 Subject: [PATCH 16/16] use maybe_autocast --- src/chronos/chronos2/layers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/chronos/chronos2/layers.py b/src/chronos/chronos2/layers.py index 15b3e0a..a632034 100644 --- a/src/chronos/chronos2/layers.py +++ b/src/chronos/chronos2/layers.py @@ -10,6 +10,7 @@ from einops import rearrange from torch import nn from transformers.activations import ACT2FN from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS +from transformers.utils.generic import maybe_autocast from transformers.utils import ModelOutput from .config import Chronos2CoreConfig @@ -73,7 +74,7 @@ class Chronos2RotaryEmbedding(nn.Module): # See https://github.com/huggingface/transformers/pull/29285 device_type = x.device.type device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu" - with torch.autocast(device_type=device_type, enabled=False): + with maybe_autocast(device_type=device_type, enabled=False): freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) emb = torch.cat((freqs, freqs), dim=-1) cos = emb.cos() * self.attention_scaling