From e2cfec586b2934f79743d6f3d4031b42e81f492c Mon Sep 17 00:00:00 2001 From: fchinembiri Date: Mon, 4 May 2026 18:46:36 +0200 Subject: [PATCH] refactor: remove sklearn inference.py, add async DW baseline loading - Deleted inference.py (sklearn path) in favor of hybrid_inference.py - Worker now uses ThreadPoolExecutor for async DW baseline loading - DW baseline URL sent to client as soon as ready, parallel to inference - Removed sklearn model fallback (only Hybrid_SpatioTemporal supported) - Updated docstring to reflect current module dependencies --- .../__pycache__/contracts.cpython-310.pyc | Bin 10736 -> 10773 bytes .../feature_computation.cpython-310.pyc | Bin 15955 -> 23700 bytes .../__pycache__/features.cpython-310.pyc | Bin 20838 -> 24413 bytes .../__pycache__/stac_client.cpython-310.pyc | Bin 9272 -> 9251 bytes .../__pycache__/storage.cpython-310.pyc | Bin 11806 -> 11405 bytes .../worker/__pycache__/worker.cpython-310.pyc | Bin 11192 -> 14763 bytes apps/worker/inference.py | 650 ------------------ apps/worker/worker.py | 437 +++++------- 8 files changed, 162 insertions(+), 925 deletions(-) delete mode 100644 apps/worker/inference.py diff --git a/apps/worker/__pycache__/contracts.cpython-310.pyc b/apps/worker/__pycache__/contracts.cpython-310.pyc index cb1fbabe4cad0facab80753f474913bc0be27595..99b47158514350512dff25022c337b0065e3681f 100644 GIT binary patch delta 2524 zcmZ`)TWl0n7~ZqpZnu{u1$u#{1sWIV2B8%MDwYDeEv=P`Qql|B?R0nAF3j$1&dh2{ z0*ew)=11se_iLjjS6-X;37@-9uu>u5cf?6R*D0p4| zM)i~kJq1smBPROyoeFyGgzkW94Q_8MN$qW`vg$HtslnXw0YK4LE(rGwxd!vxJy_ZRfcwH?N19F_u z@kosxIjIxM1Yim<377_~0VDxvCQ{9A>{RHRN#urLZ+2uKE+6GY%AK+)-FB*yRx)kb zS~{-F6ry6hau|b-R5@jF!;%KcPCzr0v4m!8C0U#rGiztEw&gA&zEer@;_f?lUf9W~ zx?MX%gxtBhZ~Gyn@W~E(xI^Zd^P~?$SL}6d_0Am1a7$$BkHOt5Sw@UVom`1j%qNVr zr0x}`!|dh(EX~QOjvJa~rSz~pZ6@Svh%^Hzl*bbDe@t0xc8D>eV`d~qMvmXw9m~q3 zQSV$u-V+_ z@$7lPr$TG92y_8(5wOy(oK^Q5wyZ@l?kVHj1!lt( zHCa?+RQ<9cT>TM(@PUufF>B7(Hf4oIEBQY+?Gt^Mu*Q9J6MWZjC47%@a8Aod=L#9q(fx!|mIfVf z+0pPRDV@rcnb3`xIdMw*VKkKgre&4a|2gC@)Q^v~Z~hG8m$3C|wk@5qS2bf0;}*GUUEL1v7nAC4cw_e=T2KB}K}L133=#Cm7a^JiixlKL zg-60Imfo}Z<6G8?y(KqNVR$U@@wrJczZG=KiIARv$0y`Xb#3c@(WHLgI-KL;mMF%b zGP*OFJr8NQj)$}^6;M5&4V)5+z=h-5EV@8_?v9KtX~#TGY|L}!J>U>S2+<+?)SYdk z^&i9fGT=kN2LN3iYz-Z`29Z&Fj(4)R;t;_MX z%X}G!l6&Hb^F4i(bdGN_YZy)-qstG7nK`*o zC7-3`KaDD)?+bOq{Ioi(_ipcn2vZrvvFsw)K_9!0pr%R-=mP&M0H${;{7*O*ip#L%numI@5Xj%1K_#sj*XnuD} zjbYOnIB{tckhrvCB*g;07K&k+AoQx_3HmxN(}XewI0-NTrvR$}rvWG?R?BYeRH~c# z;)h^wc4Qw;|BNys4ap2*3;@v4SdCYm%GmNO$)0ZHtwh>zjEaz6UB2K<6&%|)BAB<_ zc8rw2<}yq7A6H&vS_ApiZ;j1rYUSol`*4!Rg!CKBedg{yL#8tF#MQSepDZF9$%!)0 zC*dBDEF*5LUOtRO%%2l*U1Y-29bx8j8~omon^&947`B}?qt5AkTAo9s^MF!qEHSqy z)wNX}Vv@OiXVu8WaRdn2+Y$w7dDR7(8IA>)&hlXqXDGtw^jRdhF6xAwQCA$hs$zYw=vH6Xrw+ri5Rxy`m(P_~AQb_x0p0*056V+Q z1`j1~y5H`X&qZHQbE_|Jxwz=5ReEyxQgQ5bNo`xRx5&{^hJOT0e9o)TaWbz%;!B<* zDW>}wRrv4;tSjvdFWqR~PIujTk58r2rg7Q@z$HK_X_h$iIkll-V+6_gJmMFn+Z}J{ z7ZE0J$tkXGG)Bc0_vgk5(QuhOVnV)of84m*-y99S17+HsYVH%;G3=f`)IXe<=<9hw zvRi=X*Q-kvq%?C*V#YLNS#oi8cWrdV`v}6*jZR?9+|G5aV%uA^idB2W4QWkzbz?n) z{k@5yL%n^2BYt$@dzI&XrDbF21H{X^Zp*JC{w~S0O|e!QdD}{xtw*hF(ny{+Ta)=h zE^WuE%O?0$NX{q7BTMzTf0R671+30Tsgl+oIZ+|d@vWngEmh(8uSlgdLZb< zwvII)k;Pk{&8N*wBA+}XL)bay-fCMB2z?Cs6ZOl6Ev?reeu}-C;W*N(K4)a^OK@}a z8F5`@ql0ZpY)ui+7%)dG-wx$Pcc<|VkIw3jX*(U;J9o_L1>2MzT|0MKH*3j%ADOM+ zmyuMDa7_gJQwgq71g-$jaM^>b`FA$C zsGY1@$D8>k_?CEF#}HztJgEL|KU!4S{Wjn#;5`5mp^@kr=aKMg;)6XSBYh)@v8Uu0 zu=BZ#zh06L#KS4)08BRK8B>he1x`uuMqUS?zg}=kTDhV_52uy0vcwI3>zQ`U0WO^H z$AZrwe6*8=OvclvjG0W{NN@Fji;u9yoVFaEFTMz$c-0v*CqYS1=U)}m-de^$TxLMV J)uGro{{fk=5=;O9 diff --git a/apps/worker/__pycache__/feature_computation.cpython-310.pyc b/apps/worker/__pycache__/feature_computation.cpython-310.pyc index a40af852a25702fc1ba7eba7af6551c4fb34f869..6dfb2fc0133d10e9ec5f10d5466f751f3ceaf90c 100644 GIT binary patch literal 23700 zcmd6PdvqMvdEd=TP8K@tQhYDkF^Rs=v267_)j5JiF_Wf7!EO0;0f>*d}7u)tz> zac4jh@F12=DVA)@ksa5uWyvLF$1d}5)!IqyI&l)mNu9)L{AitIlD?9*4$}7YG&)Us z)L828`|ix_EC|Z!Ir*cDgFAP=d+&GO-|K#N>h0}04ZkmZ{eM?(`Jkr#1--<7dA!`O zX}U9z(KMk=XhIi8Ri85^44vYMYGNXRr&%@Uk`u|f)I@47J&_L5G85UQ98#0j++5p4 z9?;Z8`#G)FW@>iF;x1jIKarkje_X5S7d7!s(I)awn-iVlx5e*>9?g5QoJP#J7q+634`Gbf@n`T17>zYXP1zM>O^-oz>K$do~E;xx+M zG;xbKgJ<8w?J*p;1Jl`;&7e$(nV^p<==`_lzlnAX0QX~PQvuI&c;12MJMi3T7l3mc z+II)HZ_ply+BX=r?{Dm#NNt}Oilq*PsW0gB$@#yg6!AE>gJP$m*dr9~;NIRG^mY?^+iX$1x2@2iwqRZ?9(Gc{MR{H{$nPxmQeb+lRyy^4w?NeO2$( zHfo$UiW1Y5ccX?ir`zjA`#$UqJS%ukiJ3J^%myVkh^m-dGe<4Ru~F22430k05REl) z%m+9&i5Vf0<89ZqJj~H=ueUeY8|^-Olie?zmr@h^FfuNUjNNIkzq0?Tc3Gb|0N4Wc zaFZr3eA*Njsh2ALvdMWz#3lPcEd9WB>5H7N%RT^n2Z3+NK6vH+tD1NZ@Og+IQt=4i zGUwlHA4L8k)L;ZPeApfn3Gw_h2GuJ* z@^T`8|5WgN0p%YQ9~B?FYQ#$YG($hW3i^*2`iWK0pJC{!RnSi|^pRE2Pf@K{jW1S@ z%^jOKHg|mDIA-sd_%!)`){;n%K%|6G9Y z=TX!4mYROS-i~p;Ta9yoe>b&e;v`!Ai}uNxQ?c>=qJ8qpBf%#4mv+oC*KMFNfa?zKZdF6ea(KeKywaui1}AZ@p2z$B^$=?Z={&SUuubrcCk6 zQ4J=?nBvTwnKD7wf9(~0%0#{A>~;1z(ILK$dfx$jziyw4@QsMyu-_4T`^{h$z7Zb- ztfLX}Tk%n2NXy8J#2JnL3x8VrLtWEOElAs{EzC6*ht8KATUe8J$t9Rwo2t}oTUKgQ z)?{5;Wm#`nbsvkQG~o{dSOcye8~R@%)Z8Fn;8*@ghOS zkDNPRpjvG?)Tmstt06k;{6eKFY>Bi7OATwfR1>PD7*W@rYt&__Y5{0D4U8G$a}^9Q zx*|$$DVG~Rd3J2nI%AiddJTyeD>YHSXgLdWb0svI`(#^{nsvst8`fP94ChuzE-dJN zTd7v7t7&k?3klymQYpK>`EbQ?{p8t&M%6~9cBgbadr z1|*XwmQmVEZPMh6UP_pK+H%@Mj@hJq0`N58Pk3oz-g_O~IYs$5YReff<0WTx=^}py z`CTs~%zLhbJJSK)7lZsSdK%$h&f ztL5bQ!;f1RvE;GSY;4{VZR&b$h|pR!yD~L>zAl|%F8_AhZeT~s1v{v%&VJlKuRf3Bhl2h1@)ByS|j#_(;pc*Wb2*#1qykufI4^f`xUUe+mte`XJ7p=8A z%1Ey1ep0xL4O`OETX_?|6^a!yzENxVNwqxFlhu03z56cTEY};0exhpE`~)a~pTtN= z+fT|;ZOSg>Wg10&z2;}BOU1d8GwZiuboi|zF8P_siges!ZOKoAzHvma_?fCb=@uzH zD=~m7s(=?t)euNgQ(#+N)8sbP^ZM>x5(H}3lwBu^xvSJ@IJ+*^<*Y4t1*EkYqeR1K zcsGqmky_&v)sWl*M9w}0z4?ruF-$#^$R$j@%gpPD8yUSPLGUg;qj$ZTNvLO+2^jxv z(T%4}LlTx1+N&!D0zajwho8X0^0Qc2i**UgR8Al<{ACeI;h%T+ClGmy-JS3fo<6Io z-R&mjRyT#!t$T(rMB;f9J6;!NzqV{*kEcECaO~a;?QZPbo0d~Zp_8_lDup3)mtT;CF7snzFV*$BSnFAfQXe4X( zIU?Rw5Ng+Q<`=O4g5Aa#e*E51b2Dn43n+YL$+iaO7J%)%ZPkS+Fdo`@(Zr2fr!qf(@LjQ4+*? zoQ0|@Hxnf3xFzZO3A-kEi>Y8cC3OW=DxsjbM?3+j(~qFHBZEC+>L%?HGpFZ`4!uY3 z!@Jyz)Cd=Jn)sJX91PqD>zO1bbLlf`$xSYRWH$woP>302+7?jNovRjR zYzf|QIamX3WiF(_%_;x;Pv@;@JkMUiXpSr5L<+dyh%+_JEE3lT9bP@6?ZHqgKJY%2 zb4&!8T!$g=L1cx>06KxgNVf{0U^Nx0?aJV zQ5Fsok#Jx_IDq=f?DeQ;vS5(JlLUhl1OpI=NHA!FAdp}IU<~zVh>(DIi~=JVHb_xM zB$uWuvUYC5bdyEJ7HwTK(O`(t$hg<>ic*b!J2g>^{Amn?L8}or_*pUL-7D z+m#&1RIU$rVH!Gz`r16h&67B^+#c4`QpfF49UqI;@e=BIDXQZ+)bT3nc$Mp5{L39) z2S{!bu$0#h@gn_%_QyEfB0aNX%O9^18fE^r9PMK+hhGVs|i# zKR}sIq<4EgR};(YYH7T8q3sy4tH$y=9=UZM3?<$=Cc39F(Z@)HwVT2#RszI@npGlV z8(Y4y_0E!gs0>&TU1vdoL3>$|sJRxS7&x$>BF$9=0leneDHW zyG-;p5~nXtSJ0-I!~_Z2nS(3HL=03i8zFx0FDy%`9DX~20r&LFMlc;#1S23ac2pN%nxsifC z3N}$dyhO2~4A3j#5g3Z-nxCbmpqNt|MInGrxzi46aEjEH4jx0ws>-EkQ|TdBb=r1x%)%BXJWfv`gWIwDB_jQ)hNoPc}?t&vC-h*%PF=y%W#H+TlL z!=$IPy!hgBO6ZUhlY#vBJmf==eop(z<*dR0QktD)Sy6rwWE~nAvx<{psz^W^3vxIo zy&P$t$P zfSpkA9+adJmu}1kf|WdKp_gn7Q!=tOaBR>zS5SnwQ3q-$1+f-Kgs^o3fGLzNF;f7p z^BAf6q}71o3~e2lt3x$!?iD35G7$ZYahV@W!<)krYSgXsF=vZPsm5rIHf8y&mm!x?5DOFw`q?atHS@2F&7Ye~C%J>MSnKX93e$3a;$%i;| z)H^?22DzGokjZkqJ_AIdEdnBBwSK--Ee3eE5kd|E!Cu+JSV|>g`Wn)m0|+wh5ats( zD8+gH&qJ&?jjhH8eY?Iv-->^Fnnn*&ji-$NVXnjT%`VWfUO?rOK+!^=NCS%G)lcFD zQcujF1!Go%l>stb!K?*|3KHrJOjs^R5+ppxq032;UU1GhI)}}9m<_Bj@mR## zphqbV3%!x8vRLr}lV$U0T;K?K1~c{84*maAB3gR`JP`DmRamlkVK<#6Q&!M_Y_Za6 z+0r7>vMD~TEmpD+(y~!9tK|<;cqQ+Z{S>3c$nr-aw_1h(v(x-c$oJ(D$~i^>?F0EB z1;;5^z37!;$c`k2otSTjbh@q_m^hfViA5bS@HnGZ_MRd#{&4ieyZ`(ni(?1;M6DwI z1msRA&JD*;vazhi*h6||+FjENYNM&*o^H3|>)?@aR9T@+QlMy|(%E*9j?S~=#_-Sj%I&F)BwM)J2u-mCTD#0L{gmN+%BIBoJ0 zv!)297S?GKlbhtpO)VA2YM|R?Ozm6DCk{c-?UCWV)*Wk4>fLu>%rMpW4DY?;0W{g# zY3)%e+zK_81_8gV_p`hoyqWjMMhNy>WKc6+qu33HDPdozkVS`06@AHDb=g4vU@`rN zgTDUojmPUQ zRzJq(;X;8%at}w(pSKHPAPVk=*)ETao*D_=l~6}G-+v3MFqQCf6f@1j$>cRH>dw3u8! z*&?aHxl5xGhe?%s8klICFKEstWv+w*Vj!+xG4KzWg@k_fu8!WI0_?!?eDe=xP5hZJ zWua*4Gav;BlOhS3hGH=0BQ=FLqe46(4XhbbA1;G*2w2k?!U_dLG65tP<;eyRQZQ7h z+&*v-0h<~^9GVeN5pBq&d2j$&hILQmp+tQI(p1vT(;E&4$R^i=V?Ib+bkIkk$Zym( z(P0`$oM_*y;kYh2?~(Zp7Snw3{tqoYsjJs9L@Pcqg?CYNhI1I%s8A z&~DiJ0!TNtn~u$a6RG?RV1=I2-I$6{Dr;PK?97Tv%)k`eCO=N_QbNu^sS4digeTu1 zG_Zil(#58II9foX))6ct*oNGTw0i63$!w#Pa%+GLXNMj=Oa`0sf;v@fqTnTc-H8D$ zM^=7DA!J?}#Bn1GLu9)jv>X>2R|y9o6?Z|{h%S_&f!U?mBK4J8gq$!7IMsT?E-tjZ z;S9UR6^<*#gqY&BtQ^Ke?xJ8f0zZK&{A^SOq=1F8-`-M%pO06e>=3F#s)9wjYq}gI zl+dkE%Hz(jTj#g-}f2@2jz!BYqdU3|=;WD&MHvHTaEg!nnACPlYigrX+}m0GJF z)~aqtD%qf>&}2F>oSZCVl-ziUQqqgXGD-7tgs`wBirYgHqo3kVIjg0E?r60^i()5- z4-$zyiXtOzhGFQff2qIi%EQ)U^gv^xzg`?G(h*4)G^P&Ngm9>s$Z?99%>0O~L-GSa z)pArqrX2bQ&DEVFsO>s!Nt*%f;Ug0R2Ph^PaGukig?R#Y{_TLb{v}_^uoHm(0c}zj zDUp5}hMi}5*Ix$x3TRIXyeaLG=)x4J7qpG&h1-NP9SY#2If1@l&%#_JhzjL!lxT|F zIzTAgi1J7w%9FbSIs=0fWL=TR&Ia}BrPG8a=8TsjH$tv49fN0MaGVpMJlV;~d_+`- zYD{yDnOKdWK3pRXHMvI65~3@G?$Yfs8Zvz&*A!*+SeurdlJyz848H(auIXs9wyO+> z0(jl*!5Jbsn2g{*#Ow?zR|0~ER8Mfm74)2l{`dR=lmr4*g>Z3Ys_wqcNHi zZE+9~s*~M~)@(K-Rpw%%8S;G;VC1xG8NWU3+@2y($6rxN{1w&3Ur}ZAb)>cI2#)Gw zWk-rFWk-rA+p;@II3WoHltOp5A~FRY{bbv+E_k0n%*IN3R@K( zl;lT|#W$UKoYW>^5vq-Z=ey5Pg6=Y5^9^T@ZdxJpQYLj*Li(?Q?_clIJF8` zUC5ipKMPn#Q7Hr;7Jih?8iGg{EY>E36Me?;D1=u%1f(Pp8qdU0Tn65J#w+@A3MFCC zL5U&uFiKp|>EsSPMiogGea|B;~mpl(|Yo4msL5FO@=!#Iwy1)R-XBGVdtu z2ETZKJmKI3h?8@yPB=J)>y`I=iD=doxl_;)(d}?hYMzAOkO;XBBp#AcU|@v-2u^#o z63)+u4n&S0aWoNaBKq`pdl7}VleJ*`AU&l`PbiY~ApMD+taqsNiTCT|EcO-D`2voxrNd&r_830B}Mr5aqqim(tl z6&I2fm?9loL`XOiI9G;VF|h?lrv(<0QB}DL@m9bwW7=M1Fbn12T5zun7P3=N3zVkR zvX0XE^eTtVaHS9O$!JN4Vwj_-9DkgF0tz*7Lh0h{LZMUQYl6*JZ24iM#<;H+l|?Jb z1E*3*>h#w!pmiv?49n9${JU%a^Lr=GpE~Goyus)c(l|tgk=R*H{mJixMM0*fkcmG> z02W&_Vb}bO>nI7UIB-ZKA3WuE0&``OSFqovL|WMX z=iti6hw$*)qxGY5`bkxiQqs~_i!kB*^t9a-K3#c6LZjr zaqySK350nCI=V<*CC$~uHLXr{LF;jCz5@i`6ls*Z$1`Ve!hl1-4rt365bPr$$JtFJ zKdTE&1kV9}3UH9@7I+@;cM|z-frF64Wv>IWeP_RN*~^M90k0D1;0ls)b8Z`SW3=G1 zPH|#pQ5-%v6sI18CG;%)_XllPQRiT3FdMK-<>}Si=Z54zn1)enJUV7mo0uji+Uq!< zf+M+gvO#{d%9d14Ti3e^eJzppiaSWB3Pj{C@IqlOK>c5=KZV6#`NT#|Wk*4@3TS$atH{!7;(e zLEp{e;4cNS5PMoTjKDt@adL8it3#>>2Zwn*ucv+_e?ZNCl^U52@V%amws4Ep4U!An zN%7R3ci_d-C7e@f#|j-fZ98kUiI=cHScf%5^689+-3-wS#&_qYX1c3~hwj)P6gQP+ zOmOK!YU%r-8HsExtb~DA2jY<$#2fAfklUt9HZ5=EwiNAxpqHZ-^)ir`h#)g|+@~OK zBf4A=6?ltv&nJs5O&q|k**(0QoM$1G1a+xB)HLzH_mFf3Hj!MXHqns)t4+kpSZ!KI z_Xk|4OP9&Zn2egnvlw>r1$Kg%QlpsoV@L4n?w}gTNOR4QAE!7OP@qiFTq`a?N{0Ly z1kck7!oN1GEfoT%Y&%?K*beTRnp7dFDiVZ-Yl}LCHgGk6gd{(uCP3BcQ)a5zU%zHW z@1>@6A-yXH3k+;DbjFQ^`iA^709)20(Ezdpvy$?2h>()v!eR?0#kh*W!n#Y~%o>i? zu#kXXuxgPM#9#KqjtaZ(1zm2LU+*T##!EI;#L~imB9n%Lb{gg26ocG5a2lP2%Sd)g zgEwFfF4`F{2Q`JRIGD;33zY=Q3&PaPP=5d?+b%4$5j;ci`N8@1>|T;BeF+^sByinj zm4u|Ow7@4sT_cKnYN*>UAmC+p=%Ixgyij0sBq6Y_#NHumj3|i8aKtY2!lkP?rP)#I zp;B$=L`ep)he7Q|twRk-h#1BJiEPYx)D|GK1YKf+hu!LJ7~TRp7u*GkZO2wWmjDC& z9HZ+>EVhCH!8gUh9)VS+T(%pq*ua(>S5sm*TMfiF)^-R3yf0Zajich;kxC>H6|zHB7h4n)mfaMwS0>~*pmZs! zbVz9l%nM=})%*4Nh>ybjNj7OlsNPrQyI3i<*4zZNRYOgth}GJ9A1 z2{1a|^YF3x9A*D51;0qxv+9-#CYiab{2rxziBi&l6|n~VJVhcwx=qb3OVE;O5_!_Q z*O`%f*HTD6NfgNfc5`zV#9|VY!Bq8Lgc&f}oZe^j>&E{v)ABE=9ZM8^o&p+)g6a40 zWHePhvdOK6k#VgRrtEL4VPsyqM!0{Tf`3Z^kNm%**uO^rnmj}CISLvSI0*b?xr$>$ zj3iId8G`SkfLIKl0QiZDg47gV8N_DzuRp)qP?I$xxjgHU^vI;4^bMq)8k0414nOpK3q$YMN4GJ z625eIdw51@$_=mvrYInX{Z&S)%PL)*wUGzIAk%A2qQX2hoFo2~EXl=T?T z4XVG(?YJ1AN^Oqe$GAlJF?KY+unVeu{4Tb59po>9YXluvHlm)+S@?+1N?7i~8t7tS z*L*>j_qpr6E`noKbaVVBj`!5CZs6a#wNEpQ-E-oy_>b=^}IrQy!Ek_vz5R>wB4MJ?)sL_2hh&r*imNDN!nCA~gxWf0_#O+UO?j{nTSoq(7ubQ+% zHhLS-!@u%2QYZ=@6mW{6_M9l7&J8rKc&fLJ>Wz-Tp{(D*huY{3#l%i8g|dcBh`}(1 zl$|ukVvrBkQ(`D8ftVN$@@*Bn!Wi-nbBSHe^(J;lr4SQ)AO?;Edzk7w>h`N$jM?w= z`mStdD|H9V7Mqs)z5ZaVN4!nSO8tT{zZd_viaUWPzq~0Lvv`hUl*8Qu3X&FgK?vRK zrLNrMZKlgKUJ9myy-OFA>G-jcP&|bA=mu(0+-e+?oKBS7@-(y?x`l8U9xF$cTsan8 zYF8K8Aw+V9=r%fRdN#Z&XB8sWT%ff4HPPPJ5I|E;)1^OL#vWfv?Rb30+ztU)N@>WF zg#+SAemd}KB`Kg%V-|%d&a+vUM-*&lg{~J+bY$T+)$ebu!3|WyR zaomjLS z6%Sj_u+>bA{gV7nNE?u7BUaX;){*86MueNrn*Q2WMAT^*77J^}Gn&w#KO3GNorY2S zpsRh89*L&A4}5FWPhR`N0a^fhQtQ_jzoW= zBhjI6f%AF~tWerP#wPds&J94~B|)+AgXmX$jO9K_Czpj{&2l^!Iq%biU3=%-NEczhnZDfih~idK?f`N zNI;Byw<^A&a@joW3$caAeQaxqG)TtXv~vuwE{MlId+-?D8XS|d4zt{GTq(D3C_-OV2*23CZecB`##?9-%@h&Obu>GDh%oWY zkWK7aGT=mHGe(+z8reWVNLy6Uh*rHs%al;|!C?S2BB;1{0WKqTxfX$3>Nyd7TOw@a zsCDB85-s2crk2q>GSbCNbE+-WIC*bGOQWV;37gid@rhR<>31C+J9PHZ(__Vxr;m)C zEKVT;;ErxSF|s^#kvFmfzyhEeDwK5GGsPx$c>_zLYE z+%qz2EhUCWCYOvsYpD~t4o!l|FcPHPoEvE(UVZnT@OGobnczefPJ@-&eqg(Eaw)~$ zfcw$1JDUiiqiV22c3&e5X}Q9VrQ36Rn(J7A zcA*V$7sT?;^y>!PGiZFGt)V#++1fc<2#Dq$ z;$t!diCEU-a8xec=Lb zk$|A`VB#O9N$ZcYa{bo8-aUhb0L_lvz0Fa;g{$T#0mAPM5Df?@=U;@>^}mGFdF9~L zAAFmB|Lq`ZyN_Dmj=qJI5nbJZS49F_u)`lD`1r;CJ85|kJ$*2|?Y=4zSA7oCY9$sR z4^coLuR(o}P<^yV!}_8dwgV$aU_0k=R+WdhY>Aj7VOx&alKS}C1J)t zrF1@S2AgW4q_I^6n%Pq78Yv^Bu~?n1%Xb3`H|Y{3a<=nPtV}UMK@(v-XZ0;}=GMen zd&@G0oTv16Bfyovwb~>po!hi_>-d@o_oG>Qt(>|deDU!k&J!D-MQX%#Q^wGg(zt$> z3anL1A4fq}uz3D>^nQ)drIenOFH=`(vS%i_%s;V9+zB;|UNf+7WL93nKOs-_1NdjTmIf{1{5em$@1E1cs>3 z0h5he*`H!ijc;*o6N-(aPPP7=62CzK zbLR)C_v|lB>%#9;d^3{GB>BAKn4DD*d782$a15+IGc`)DIl4isK12j>6mC!OD&Ixv zZ2?s*LVff*!VK!-cuXwVhP>+VJ+n`JU-AHI)*R9eOfBCDH5y8^f%A5J;{yN2o4HK> zzTA%dx_mKzZ~nG?Hu%fr+jEDEyYp}4Pvq`2Hs{~SjG8Gr7u$d`9j2M0`xdD;G8uyO z>BbwlM?vRhq@n`?GXUXMiap?cP}DO?O6k|lH*#kFJ-N+fI)E!$Vmp`Fj;H!F63x3D zsn;{vyl&x-@T!nZ0xAv%7Z?fP0}TT~_y<=QoY|543mFz*nh4Ahxbs2nf+Jk7ZY3;z i3$Wal$ipN7V+6u3j7bMvYEnPSn*^i$bZ^9b=Kli0zG4^v delta 5668 zcmaJ_X>e256@K?=u_Q~zvJKcQMtH+AHpT{n?bvw1HcJ3w_6Ud&zSlya&G$WHOie`s zWQCCArfm`+wUcHcd+M~IGfDfSnNHKmbjoDf>AX&xADwoFGWpZ|=rq%&J?H9KmL15` z=+n7pyXP+F-23#*&&X%vwF!^MrNHx-+?OIX`xWJ{*vNhy(CAbYqOG$(Ra>1R3MJGM z%?UV8yMVGqoiSG+7r6F-d%qITwJOXrR!0>4(42t#J|#{@75XyGrS9|AKpuUCzDf&d z;dxtN9&n53eBkEO*XVV+m@WaX7wFe%DP0D10nl&I<+KdwLi#3si>4}R6*P*Z(h6D) z^n9S-rZsdW&>w27_*ehHnT@6#6A3iKlS6S{$R zfX-s41^7RuowN)1OTr56evt&0hLu1m-PofX_6C-*r9DdOush&mr4!59vMUPhVZJNM zHOnL+%J9>}PXNCR40~TB$iWuvqnpAiw&-SXvNW)aZh_z8KzWvra`3eEn$@(K_Ke(n z)7;_rhu?+ZKA`g!-g~m#`|j>m0h@9azB}6VXI60W-Y9FqE(K)6&#GQ zdsyo^0$r$P7pGY(wb1WeP_bh=e$8U?&zQg81^YIdqGu;nv`U%m2PFH!X|~BdC%Nb6 za4$%1-yH5klDlaR_hIw|`|FWtd#pXs9@`Mu01MboAB6!ucFqz=b|JMN2Ka75w^dgr1)YI8G4pm2QAm$kRm1#Fw64Qa06+SFW}>fp6r*S%@=D_Y=b!OHjrufXfMj8^G`8 z9;EXS<{{)Gc*V2%D~}feX;@;RQvi`T*t1aF@8ovWs6<$SFkfmVL)5TCM&NS7$l_x(xYi z(|nA;aEt|qBGD-OhI^0cS7SWFw6mbASpgibJe98#e&27m${`n`6HDl*=C|@@pm_^I zD{^g=#S^i)VO!($WJG}cOX<$$BfLv=Ew6I40ne|VJ|wy-7l;eXH+8h5rwzz8U2jAN zy5^k-z6{9u0LPqNMP4^TkH{_i{a#$1Fnr;$cJdjIwya=#L3nq!7Irf7GHA%r7uzZb z-z1vKKi-a>@8WRFw>q@Kx<%fe_UN`%6TVgSRb1+xGwXyW<+pWN@{B*yPS-AoiIUXN|r6($$2LI?H`4`Y&9X=G@~??~rYKT2}Of`d^=I^KZFm}p#v zBooZ8MH5M;&D~7is#Sbm)0`T&P}u0YT~{V@#tEQv94=x-$|N~QK<0$ojcZ&jQqEYj zbR6Sw4a?}{&auyyx!aU+EP0-g6=K_%h$TmK<~tVRu|zyFSl=hUT6v!!4JG1t^X%e_ z6<+aXZOz_kaYImx#A`F!4Z*>LHq&gJEp0@p%Lfm~?U2rd14!Yd_^qO{u3DU^D-zf0 zI!J~1bKT;_a^`)gBKNPWEea)g5Mta=DGFCLra~w>iXgYmAW{^-@0wd(528qJAi4f> zd*GfhEHDDY0+IP`=?!o{a)~aN-ca?%^g16$dl`gxdho%awJfBMK<)Lw0JJjD$j{9q z4%I-5Qjvo%7>>m)L$TEk@$G^Kc_xzs8#!AZdiC1_Rxc{31Ejo zf|6y}u=)kl65P;6p=4m`3MN_TL=bncw2_-UEHh#`9psUzOHG=(a|Z}-~ zM&dLv8jR3W=`)T{ei`Ik`f$QQ8yyRR8^a+dWX{PazHTm7FA!1O(q_l9%+E{(T8kWM!xmH(PhM_>g1Csxtv#Tvz!q_>mo7C;SZ3J_Oti`~<+ShzQ)XKIO%>+*oq< z(|&;m9;MMMKpXaxq3FnT@wko_a#PKR+>n_UG=~a@xUSv6eTDaee3y$9SgfjxxGipC z|C+x6%t5T-(~n!3%5OVqSt)nXcHpKrjGX_+u_?nD0m`sJeZn#<-s>pd^E%381X+X%8(TY|%VN_e`9r}l6AHL3EUL;{}M zutumgXN|g4(Ys@`UTS4j`Om+p*bB2NPsbvKJH_oC6`{_mez)_Z;?O`n=@%CV4!I16 ztTu2=iZ2E>lN(~)uGj+1J;SL-VvMq=9*X=8PBHF=6e<=sch$(Ef!AId8q@r$;gT29 zAf__^4F>lbf-eI&TQZL>!~F2*bnDh)p!|eSC{vqvPZ9N|BL2H)nJC=1H)Tg1=u_eN zr;el`k!9(rKqK=gml~C*QH8J~L(NzbzCLXr?>f>z-bN+WiUu;FXAB&0D@vJt@{Oo) zS5;T~0yW2J@3-2hd^5^rPr@q?Kx1sYo1$&Bm(ORV8p{vJbr_H;3n{b7yZuOejkH{*DQYfmQ}M#Mjq zC-_GhGj!$OBtAc|IK_Z8?D|*|^HH8ncqqXUIuc8c$t!{3)&?WVu~k|m91lgg4ZW;H zs6oZtEwN;R>%E*O_{@t>`uMziuA%u6gw!zNBM2`eVA13s02l>3H}~%74{q<>vun%7 zVE?Y3-d$YYozJ4wp1v^h7i5E-((weVga#A`5Icgwcqqn#LBp2hk+`0I^K(QaaV;4d zWPAuU696fA6AZyg88fUJgZHx-THE6Z9)qFAQA^G(o+yUhe204g*{cw+TJQ%E9zwth zXV^8E4n_^@2{xvs&x!X?h!r56%%379S7ZVc!c3YdHZ%j00c>p1VIwK!JFVsN|Y#xhj`kuM48rU>o5ZGEg+D%_!guj_HCh> zR-81WMDBLlNn+Kc+UdkSa+DJ{snw*Nb`&3N6DMs_uj83EZsV~N%XK`{HlF_I_dOh> zIi6%O`1X6>{l0hX_ufzMkTUKS%ejjD z;`|cor=Ose)bnKCxk?(K)wBjU9(sv3P%mIrA&EBLAwfbHXftgAscPC9O1Il+JG5&+ zqT>#sN>HYhcG51O)v8Xl4u3(VN;+3hRT`uNbPzZV`AU$6=w&)g4?xpP!*q;}gQG^e zNGItOU`_c-_D7d!gdU~GK(d)e=_B+6@LOn%o~CC2Yo&4eD18jDHkzQz^a8yIlI>JE zDf!seby8y~ZtY|7c(N~~#ur#T(YN(X-nfT6XZ*PCw*iLgUnOrD2O54vwv1BmGP$?4 z?tPaK(^zggZG90E$6hx6q3Jw%b*rQKA({NGp|wVwXNV^ z5mgcVyYc>fnK9NjMc&zZqU}7fzJ^rOc(eT)dC$1mQCagI5?%*j%Hg=_m{X&1c5OBs zQQ41;&vgt{{sj3y0N~3{oBSCP-ZftD@RI*9{;#8oylGVU_LDcaj{EMBavK4z9lx z_HhDYh5wW!5$$Z1Ao8ugC4(JmC5Qs?+KO8KzhBG0x|0Cm>e=T z``pG%(SWhgUqSxFSnKbdXADwp<}HN%Ol6yZ`z)q4pe3&*R5pSvTQCw2B=_xQ=aA`S zPMieA2ucu?B5)xnL*OSAMZrmQICk@8kfn_I* zu!Nh4f+PLItwV!fQ&N=Bp-5A-S?`Xv=siTD`}E$VjZ1X@x_pbU|KieOEAL(sI33n| zVZV2C z3v+mg4{dbw!A%H4V0&`dUc^F@-cLtU)6&H%eIK2McbUGQ@B5;}_kU5kE$hR4m=CQ= z(3dFh-{|4PsB}wSJ`ex1^l(OR=2z%_`XhQ*Ku@ESAm0T3X5n3{AE2oR0IARqqGS3X zKLG9zf_uvj_kZ>gGrtdJ4ETSJ9?67v7y0xEhDRUD(C!J^Lr8-x9fcw4Unkftk*~VN z1{@Hirq)SNN!BfT>EKNmon$WxLP(V`R)_QvK9Vu`-+YLVR7t##%2d7qe}w!{P#M9T z{2N&41Ku(5n}Y%9g8>+&?YtH;sIYPB%VC{ajMviks+6RU0phEY^l^B1!uv27D}{eQ zAA`wx{H84M@g4Z#G#*mu!^qbs_{2sdNDXWfK1%0dZ+Qf}pzjk|J#kwWolw|W=mh2& zDA1Kr#QW*V+cNY)S)SmQjloSxpX8Gp-L#*cqNksM?GGkTAD`s?d^Bi*HGgL7&hY=p z+3o+?L!;zJ#v6z3mxxs=)}~mc#apkB)RX3~!D3@2IZ%OEC4jgL$kzc`>;IkcHe+(^ zE99E-i?Kf^H;s+)LnpVF{I*3{niA_n&=0_8{Wbdn?lS4GMg{8JH+m1VlutHGW@jc& zojx(+fB3}o%nARQL9^!4N;pFOK{b$EVXEd|9rOp|%%2EfRU@-z`N9fgYAl)M`{RtN z?37s+^Q%`A>OxZ8R$c4$il@hWS_{WQ-ZO*V*|b&fd^n*-O2RSK+i?bz;w-$TQlGJL z_(oni#P^)hIq|*WKZTfmGF!vYuOfIJ!8Z~7HGs8>wE0u=VHpzK*B4;d5F-DK2=O%JvF`nd<+obOCGw{LgL z%nReRG40rSe}FO5A+T5=s%pkN)3e6NOua#8j#>|)%Ms)2GhNm)SOOX~eme7lt3t&A?pmP3YH(6W7x#{$!B9h#6t)~_TtEoA-IO%HxZ;)FV;v*EZ@^W95cp`)TI#n zu=dnSBBGv;C3<5NdLFpe?L8LOlIOJ~yLfTI0&3Z(8xTAEMVVC(1s7~$azu&}-4d1c zJlMRU?v5x?i*6OTC6X7l>b9s&w@2-IeiX_j*vuV^1)R`4YQ33~^#Tq*N~)l`S}a^F zf(ppV$vV-CVDojt?rx{H5UG^JhJHXV=EYoEl~|lhycqV`5?-=cLhXQ+@{)~Gn$Jr& zW!;r@ZIp2r)O}@1H+O*q*cTScbT{zIxr@82AlTHBWs|T)_>c&~I^kv9Jt8fig}0Lz z1GW#sDChQR=@z+oSGaes62_O4$OpIJYO#X5sDl>VlzBNXhOM~}cVnPe?xGiE=$;%s zl~!iWau?OFBPtpjBD2pPT5B|?j4-+EGH%6t=Pe;%(4W>gB)(z z<-ST#^LA0I1vMQw1PC9eTs9!pHI-6LmQpJut&sCnA-Jo%nT@_48IZWztm%z9Xw}0P zF>R`0hYQF}&IRNT=Z{{IkrQp|Cv$pSUx6Wp`m<<-a$-_t75%1^g$k>v}>hr%^dDr)O(M&%^6dA7Qj114_#JU1=JEEHy*@3)xvGfK{b2NNI!a;hmGj%IPmhWSec8DglL_X?w<->=8EhoicUP`v` z0$j_mM>NrHINtB1H1j41Wsx@{q&?gzir+=+VHVKufFu(~yQ(rR z98bl(@u2rt6mMQAfz-+`C-%Ybpz?8e0ZF`$H)O@Q=WpIK;Af#99!DFYy(` zu#hut{-{dBff&1nBvID=24XrgE&jwn|Jzj=ba3eHqZxGn(A#+YMS%nTP|=Ba^ZnU? z* zWf*{}S=8c>#5L39j|Q&#Lrhg;D+#|Ai6>OET+lQ%rfLgnDz>1e=}@z7hcJuTEB0&A zKqQh2VF8G_2TtD-76vEb6#hXyAR&oM7Ykr|5MAc-qC}-7naP|yEidEXlXObI0+P=u ztPEc4D%hK0e}U-7oG!ir9zKBbN~)LE_IzBOxueW9hQ}IHJ0oJxfNk*k$7mdVvfDuT zI5*z}HTDO{Tn4e@W@(JBhW(3ym_HU@_0w=L$o>-bzkxth@qdNb3ka~DX9)x|1V)l= z0P6FZ1=>P5ajjQ_!dYWaqUjq5)=}`2h@C@#6O=uL;8O_1e8lRCrOS63xTdAccdYzO zMT5HiQDi@cK-{r7MBVO>gqKuymf%eOHu64+;BmBh1Fkh4wvQ_`2CF@*hMHM352{=yrESPd~1ce*@B{ zHR%^UTtL-J2tI>ATvJ{~EDXRbPOn)wox#%g4^lB@mQq45ziJtilqr;joA4RnvxJk4WC- z0E=p-^|Bw20nii&1Lu`+jK;6P-Iul!Ndo%{uv5stf_zbYn-$-UuSa3VX`t0PVy<3W>wwA}&%wzHX5niUT+v#RJm#t5&KB{wkCT(y4e9 zuTm>lkd#Aq0nrIR7ZJEz4L4`bv@)oni)wJtfiiL}aY4I+>?1DOt+_Q0K+1KgB= zzU8`0ylVkE;3xxanPKqa^uopfTfnuMeTpP$YwgI(GWn7*b8H^Yp}%*m1+J7|I95l_ z8m}GmSy6+1+py0y7Hsb_tH$2B0rG|soEsvq7|+aonEDPFgw6X!c(Iodyo}%#q>|%4 zI}=ykc>laZL|l6R3(|@a{1Aoh>6zq3!p{)=9KpLeY^S8SZx)l?-ilG~GQTaFcN`yI ziE;MfhD)U=>P3UD9478E1vpDbyM~yj`;_xQo6gthuS0Ao^a5oX8 zVWSoT>xh%|anib@zarJ5KSB)S@pTy1`TeBJ7@MCWuNzOy?;plREE1c}^Q+htm-25S uR)kTBYl0MFct}hS|0~AN=iA5`qwJ9fTrw&ArDF~@zLcZKQ7xrPtp5*1*`?k9 delta 3070 zcmai0drVu`8NcUV`x+bbNJ4oVjCo$bfFVEt^9bhEq)EuKtgR`bgwJ*$JTFeNRL8Q` zq*K+FDb>-mKhkKxAFDE{n>tUMv`$r~Z0*!_U9(qGOKG!CYSuQJwn@#TXqEPT#}H`# z=xp8J`Of$IUgw;9&OJ9@XCJMz%!P~$hl0P4Z>>)cop~{{n2o+z$KyfBfXwrjV6HvO zo(*I01mr>fd228a#-R|3kjaN*Pzr983L*-Wt+OzL2`GmOv?_$kNUC22)z~jWi<)%? zT3CfAp%&^;R~*U=x#%C(3Y1_8Ou}*SK{GO?b}a}~0I&x-u_@wDn1)d}jBEwWz*pfYN|kUDX5le-9Qi8Hjwr@l;ssV@VR?z4SbOuC zA?~`KL;Oa`H5L&^O0Tn5MV)(zT}@na|DCaW;(5MFAcZ=;kaQdf%`KQ`ho)yk=6m9DO>5o<jm7uQ^ zw8$1^s~WcfJE?KTm3)QUiWFdL8rQ{QePcXL@@d3_EoG6T@U%@ENb@UTkJ*XIQ@A}( zi8-W)V_QTATI0>KcosW4v^X>JdcB$sPVt<#C_@M2xF(tR#gDvR)MTPY z$2T-<@;iw;-U(G)c9n~syi9SqDVJRqSDG4%lQuJiyCD)XO`OT3b!vLT44G>v8&<)5 zMU_ciuE(;GX{OV{IteleG6}K>vPF-t(Q58QSRnn7`vLMN#nlAA#-%@5Drl#XVqeVTQSZsb(i%0FydX6zA>ft3k|C}1DM@^g5 zl=FI?k7q$Uo&|eWw3ru}4#`w(Fr6Dr12SEbsoY@v>x_G$d&LrK#5bpjH?3B2?;7Tj zQp0^I-bi9ZiTR*?Kp8KHHA4@!yJ9W8`J%#GE-IJQ*lxa?H=R~4sAdcItyc5hR@jSeZmf+GLqBiD^lg~lvMs&&g)rk7 z=5c(ELSUo3r-_FEl?MYGx)-EwfOI%ZgV8MRJI82Pa=zlOnNn`&j+8pAMO`y)XkJlc z?a>-C#FFx!P-A;y9lT>B;Lmvz?ed+VgnhVc_tOw!&2m;BxunWLXr>DXLG#>&DjZ9v>>RkH;uy3g8@H_1 zuPL!!-n&`_O>htny@Z<_U!Hc}%YD2vY{8v3n%Lj*A9eF|9`SdxKZ)PAyKy)C5a0U;&(}QE&8etQh$}rwc1B$6eMNts6#1yA7fjXb0_$k59#KAxj ze(q)h{cMMLB~YM$hHAqr-VO}>wS=Tn0yn_`!Js%b=&9fKu_f0J32Kl|>Qkwj_mQnEvqfATtY=-~ zpM$Lx|9|eyo#Nr4av_F>97#twG#Qj&!;n&2_h?iMKbYzZ26CaM8V)0L@u$K`Gi@BFvP7$0YI71*mx^i0N8rb9=z&uQp zoL)&CrEU*JqHnTcGs7fD2;{s(h@T~xCs-z+D}|ZeMwm-*0F@1ROfTnPFHJ9GIoeJ} zfBnB{wru89rG+fj^e6m}-c;GkqA)nh-V!H+73_|<5OlFa;s-%P_fvc>ZU)QJn@DUV zm=X3d9~%)}W3B92u{3rs{!2`Rqm~Q#29@w8!LLcn4jO5uygtzlW}C@1>i(V}i{K9k zXelpIZc@lg&c`HuLLjB>mhyJuAZH3A=Q%0Asryh21~sW`nG||IhBRWe3ad&7XKb^$IEH$ zu>iXv=8v_v$ZbM1XvztFhosvCw+I|GVi!S_fL7dmR@^*R#YTl?V%X`t=h&~rmAIoc GyYPS5qVyX8 diff --git a/apps/worker/__pycache__/stac_client.cpython-310.pyc b/apps/worker/__pycache__/stac_client.cpython-310.pyc index a563253906f769f5669b048f9565a7bfc681dcf1..15bd0b3def1499393c69d2a99b6f03ad9b8f4c4c 100644 GIT binary patch delta 1246 zcmZ8gO^j1j6n>}e`)T`HiVh<)Ff%Wm0jVL3!NFukWlR7=j2e((GJ;y(w6~pyZC`ud zEz;Vi;y^$#5^@s~NK6K2qbp*%Vd27!iE)LA3-cDPkR=Oa(53g>my8K-@x61u^K-v* z&%O7LZywM1nM_KP-#3^4_Rf8u`Q6yIZx(%@&sLdVLccjXd}OQu{2~^u-LNd)x5iKK zcxM%~z6DzMFi-TXb=c7FYM*P|xT;m494{v>TV?&IwhrsiPxg|*+-F+P>Ki??7b_b( zwU0DhDVsg>S*@Sq=7uF6Skv+3Mr!!Px({$(+)KRyrg)N?*&gc1ICjGLp%-5wm=oja z8Ayx8^Z{`r{nIo1q;NZ(h}jT>5y*la6Ak;h_+N?K@S4p4E{VsPD=-i|d-&i98C!_q z9{I!f2=)@J5Iip-%+>_2VjkdbN(~{dQHYeVksZ$9ZPf?IM zM`|EUmuh~yEa!89Ar8d$$rlEfDB>$}#;fw0TqVnJn@!dzKb0NHjMrGBkczaCPW*y0 zbnewzfY*uml8CIwS-lvv>vb1B=}~HLkb6L|rCa=A8sf*v9Q-4Go!lIpB{@s=q_&4LaTD+J0?Q(2>>8jVAis&Zahg+wI6+G0BqtGb{FJgF026aP+~ z9jD=|D}m~Ad`wF^9K0>+&H_vizjfxpq2{|;$61o?JLop5Y{B{e6tE>AKLMx2RDS;m zHQ%)z`ME*Bcpw7@qL@Fl>uc!W5^0 zO0Y{ps5jATaNJJ>sSM5Fpn81K6H)Ktsf(-nP#xc*tlLqR8dE3KD;@?Mo1#1Ifi50S GFZ~C&P%viz delta 1250 zcmZ8gU5Fc16wXN|lgVVVY-w7%tJ$5+Zd)RyN>L#zR@${vOSS77HFYi7X>xaW>SPkn zOm^Ln{i&tOTI|a~s3K);=|dk&UJCjkKD55ohd#7<6?_sw5QO@J_uPRMI>UT(?stCf zcka*iB1{IoDRpIBLd z*s1}s&f!q+8W&(I^-XLu78-Qbcw)5laja{s>fKZ~Uf1`;K8WF&da9e+8Cx~O^p;8Y zjqyZg%j!KcZUgw#GhYRb9+?yLOlAVIRL&e6+cCZJR{6-(l;;XRJciiBem0_BJEewn zK6B@VXQgB-qs1+VLr#90eB)sKW9gEK8~x|G{bEHSsF@THo zFnbX$P<7z&p;NL}F^+rX4=W7&7~W;rFQGr^xj|U1H9b$%LbvG$tG#;zmjF)F?BK9< zltVPvpKsz6ogXa0X7Ag<7ZdP4{kij(=StFBOyWEjJIkOkEK2AnKWqzJDH=G+)XNN9 zOIF#j>)=Z#+4Lpc)T2aILNLqOk=i<2O05!_ss_Tr+T!=r-BoGKvGcTqzWI*hwS`(d zRL!QNbR|gDP<7P|s(6*N($13Odd^u-V8Er+Jjj-s%!=l!x}j(US9?cxza5|Kus6@i zsYNGn!VqzVsV+m4`Lyuc4S`N5uqDSuJyNnvKkb=>2b38)39Vjb=*I-KXe?hITINhO z^LCq9mYhwO^Cu=eE|J6;4pIxt9rZIcui3844OGchc_?(@I1rg#WZ0Sy` zCH(qh*^$hIC;XxrX=9zZ$r;JeZHNFrW8xAM1KfNyXg3-Ty3(W6US{_i!?td^F|r%} zqVGnouTQh%O$qYEDf>NTzkv?_t!S@n9Hcge7ZcU)C~KaF3k+(XY_?PtS6W3a5@2kz zA4RO{fFp1^$@nS#JUTnfCqbPLoMUT=ffoj^P{p2v{XMcvU~}W00oyLi)@^kB1u<#= zzXI5zU&rz=LuTPXjvMb-wtSo*5Mdw#*XTsydH9+(3PbP;eN`ywU-Aq`>0x2-%o#S< z8J?8TPqxtYLp;F5922SFkh*oz-B1_eu@kE_tM0zwtSdAuT diff --git a/apps/worker/__pycache__/storage.cpython-310.pyc b/apps/worker/__pycache__/storage.cpython-310.pyc index b685f9f9114e2608d20aa57eec15aa2de2b29198..f1e2e6764bb56ad0ea226d6994e98edb194910dd 100644 GIT binary patch delta 2998 zcma)8Yit|G5xzZ=$G6_p!=faLl0{Lpp@(hx5lMEW*g~Qtb*vV$*`V}^d!kMvdFKmrsfineHx9|ckaomo1T zMfgWa+;?*`v$H$j&d%MN&)>L|FoHp!1ix4Qb4UNxxFmgw4-fy)dPb6n`Ob@pUnbh9 zgLd4M=>gg)?qS*`?%lLU+#|GC+@m)YK0*4(bNn*NbpDKv_go8zC5x%$6oeOh#f&{L~U&^h>o+()|kWqGu3P7Bl1!aGVyhm?$IUAu!?s}{Q@ zC*xWiyk{yO$aiJ_BUglf*VPqzPqy6!rsq_)VSAR;Y<5viuX2Ynz&~~M^0P{W$CWVu zx#Hv3m0oTrUHsQdD}P(*lH&^hM2Usn7iXsp!?bMA(|Tsvnw@^e_ROf3&{ZzGza`iA z4>QBChBKOxW=3I{|JB<|lHBhbS0ca_>*ce)#B>-$+nrMj1wEf(A$;-b3^ZeVmTaQSb=SX*x!afWFmxl+Ms6=r}zN`Zk3gnwR2d`CI8!5ht?JA*li)()4mN>tpGCkCH8ZgDp!+tzQ$Izz zD|HP?(vAw-zpSY=wOCBAYF0IuQNG>S+l8@gqSXYkomc_8{HKjWp>yC7C$>MQuBR9b z)}a!T{9a>}e8eM7VRDI2Hbu*ptK^~%OWARhTr2J+=Y~e4Ef0}4$4j0qrQ|Nj)Y&er zO6)v!mE6tJme=y^dMQ|_vrU3S1|EQ(k{8fh^1y-f+=2sV`F8y!FG_hoj0jMl@be1~ zC+ZiZOA_V3Z)zoh%0HWKlc4Rt0J(ly&uK={;xDyKk|6)6Wj^SJdQ~lpv9CZzd^mKh zQ6%p*3K~-_-N-{)#n7>*E+Qq~H5GQrf*`fjGzbz}fq&T1>RAVGnBYAB zQ%B4%id;;y%^MFyx@yi*Agh~Js=8S7kCkT+yhapEk*)K;clJNImsOA&Yx%4J#g9#j zbyersi!PG5fg|_Nz`hT$IC6B)Hd$y_IzDaXFoWKyepyV9-5A{KY~IA4=_}zKJ8=!e#ihb&IJfm1XYt ze=$2&JG%`q$?x@@8cX90U;I#Z!8L=;z5#@iv@1%#1f_q=0d1FT{(-;WALD=S@8qoq z2l?*?>IWi&dOn>kQfNC7sLx{7GKVKtjK!2rr-ma1rY-5~aUcJaiI{jefa z&*W3kQwsb}Y_u#|HC9*k?IENi_$Z{KC>=u(FB72?ZxkV&c;JWmRBKdb-6%f?0Ldm% z{wm4}6Nf^K6UhpxyNnK3(19?N_xWw3vFeA?RkV8^8Aluh+`;PXC_yAqq+}t1Ju;ry6XB25#C8!xUKkA`@7W*91)YSQuy@ey(MuKEf|Q7W-s*^Uw!mMFe}e{Gt8cEBFyG=x~`M{OnMFW6;;) zYm-j~KKDiCP{V)O2rmu&R_VdfukeeBF7hhBlE}2ogTfBZ7rZR>jJ0g@e)(~gl*jRF6j)q>Z-OOTDa}R*GRRa x?H#EHd^Fge2?$lpYEy8eB=aTy^~3!{;Xgjy=l`28Tz60^OJ(0de)sV7{{TSCx}E?4 delta 3163 zcma(TTWlOxb?)r$?9=i36|ZAEo?XAP-o=UYuJfp4rw&ch1lf5sPMg(u@9cW)-5qmh zTtB+b3MWWGYD>#)fhr-`1zJ){CA1RoQzVMi4?d`nfR6!z1VVg(0P(>iw1{)=Y?{X8 zgI&#;bIv{I+ZcUP`@pDn>oD z?Uqcp(RT4}uhAW}Q_!6?dOPhBbXSd5XUa-$Wuu@3DT3*m*ro{{3TxpH+}JzJF?@SXGT$D%T6Gt6$;Z@c8>qSm*7A0 z_41_B!Gns*b4ouytHk-+N+VxW;&L*?-&OiAMkllqHH+gW(=9tQp)ET8Cr^x;re!jT zo34AO>S;4=xkczv8l!F3m8mFw6f(9QRvPFCJw_9>8+04#DD9*Dz-^*qbbzJ+YYwLA zarzkDMTbD5CAgc8(-U+*Jpk0!;6eIH`Zyh?heANM()6U1Jju_5e@^1OC-TEoADlQD zV*Bvm`w<*Ka1g;Tf#$vqh#2b%=PN0>&6Kg0P`;3CmC8F9SxMk)UtUThd>ISCR; z;)ILZQig%Dp@UmI z6O-m7_5}5o{Y}zp(DttfsITm&o>mF^ODW3${pBFQKsf-t7TADZvxDoQauBgn7&O8( zDC9!?YGW@6SHIr)HNwv|A0ZL`h33gfcmn)8XB2gF!RCK!-q#=;3!3vf(`>^m!I2NP z>>GLt-3=~kkQf_;H@h7b9xi7-QhSHrZix{u|6WTmcLj7;!ewbzf``n?Wdg2|=RFst zNy+Q-JhR^AL2%y#?l+e`v%YnO%7(Nd!`rh#sCQ0g$&1n@iTYMOS3T4>DXrD;MY1R* zm6c!f7q;%;KkMw_Bij!1=VIF{$BnG5E*kcXTGVZgYPO~tIaOaWEZa(}XUvk0bjh^U zoVie=`!?D!sI%w&#t%%r;G*}H4+nWUdR1IwaZYSm#*&+1v5QDhsa z8i1K{dfqHqI@<|!((9JqA_^=az;JGgA*%rIxA0go0CLDn6e7bv@L{trR-I_QLAu1T zlut1|XDm5^F|%Yt!CUNEkcTbK@qfko!y`a2vC$e&wW*z(x+_uu_cQK+t$$Q!+Fm6J znq|*&e|wMOdd7#^n|8%rtM~1RgMS`nAGFE72)r=icKd#!R=?Fg<%zzCQr!q@!>^_5 zvt*6m>^l17E65#0fRPcpZd=U)>3EG&&SaNxWe&j$$ZQxl3i=t-9>+9^e83nQ!DSV} zmk1DwxpIMM!BOlLOm{>Z*D`&Q61&q>p*f zo*M|jobbZD5Jy2Kf}(8B<~nL@pm0cEvYD3ASl-$krEkFc1>V{{T)7*@43=~{Xd5|4 znKh>~289a@HYjOD{cim^*iyf2*(|LBcslC^)q6h$5Fb2ac#ZD>gpx9W0U(z>c-gLc z%hI~&sj{~$Z+KQ=xRkv)8D_a><*(<_6@H^T!PQvz7I))e^`tqiK7O44Iliy$-Wi6~ z@!Dxr+f)}UomXNhemfej?8z)JBdyNq%gK*AW(DhgpMZ>S@4E*Kj@XP&=X{Ma_!c5rZ97 z)DrrlJswzldf=I34357RZg>SnG zB0FNS7K3e|CK81~`q&mR_)t`LoXs2qwiB3Vc&FIQV&1Z+QB?G%h#rmycZ7TuF;R7I zAy!3jzW@Y=Ul(jIfE9SG-TWszdYW27&7oHLk;uR0zQ_l1EcyZ4%RR|oDBURgW&WFF zCwY~Bn9Mg{1cp;r8@Z?TQr@1q%a;a5TUC_2?=xb_6#Fj!=|CGP@IMY5Q7)szEFVlA zsth8!2LYbL@xjMy$#(pfZ89yd3-5qh$YyoRay&&V?+!pkj4I&3fN)6BosaH~oHH2A zbLd8KeK=mc>fFit5bvW-;4qk4DCkEC>~Fov|C#C`e%_f*@K2;08)ctNy#G;t8A@;I Ol`2vtw2PlgkNy{?y#MR~ diff --git a/apps/worker/__pycache__/worker.cpython-310.pyc b/apps/worker/__pycache__/worker.cpython-310.pyc index 7954800e1774f3cc2f30af3e8db6f03042e64ad2..14f0aa17d9a93fcb28a2a040d4c1e1665a47883a 100644 GIT binary patch literal 14763 zcmb7rTWlOxnqJ-d-XvQTMN-tQbZ1*+la#C*bvH%HvL#9)Da&qK)7$J*q?+stb*fqv zYpXrV$lgpgBae1xk!)tAgbXGL2FS}k1e3f3&cnVeCX2iTK?(t~m;gzH1OXB(kXenL z$@l+N-J~=g2kAziI(07px&7zAcfF${soAK_;-ALVB(AH&nA#H;aqyqd@-xL>4_tfulQ&POZh zYDd1K+L`aHcICUOnS6%JW0ft{?tC}r=7nvc8hyrpV*Jw9&1nD z6bHmX^xtdk6Ni?xXR3V2qwLS`xArd{5QoL!Q#F6kI=Fa9WW`(Jh!}dRs|vpl<2~1U zUpu_6I(auFx2#2)Z~e#>E+NR-6;(#RYK@-%HOm@y_#j{)DyLI`KsP zM9ZJdpR!J2cQWWTVV!D0t{ubs<#3LZ)=81suC(eemM9Hn!N^Xa3r}=r>8h)Z)R_<-V+{W`~FxHGUE@t~^tUJ~j%)y{`&(sGu zQTIXE_ip(7ke_@?@{zUoiGp3wXWGB)cZxq(73Hc`pOE!s^KM-(S<)Ofr*E51(OxoT zqgF$~d|jI5+Po#LTFEjO>vMK4nVgxOyk(vqH>a&)#k6W-*r^ZWVJ?@KtxCCOnRQvZ zZ`qD4I^}u|9cJ&BZL?Yzjf!Qil%-{xPJO|0?qh&rrD7su+K#mxHnGQ(!)B>oL;sRv z=ayH^!H304SunIAbGf)$sTVO?QQGC&LKcl}r!I>N7B{_7u3fun7DaK{v1CwLEEP(X zvQ=}q!R5(I=FIG+3DdTUm?t%vw~9_fT7^=*y4-MrS<|q`$A-fy(|W|y!Fs5JSSidE zZC)(HTq)N?eZ>;y<-2C9g4(sWj$2^|tcppK$GpuW=CZUzxkP(JZI|n|vn=Z+%MR9D zEiTq&*;zH`%M~oew5#>Hb3a^hslLD>gi0b!|ag;SxsOxG|@f4YMv^CvR*43GyTod}b)>OHT zAtK0YZqzY6kbrjoXF}q6-W!Tje6QcMV=Nh5wAp>hG#6s*5aN9*by(Xm{>k3 z5>36PQTmZm!?kxbPxW+>e5%bT*_3;JLgIie;42nGm{qfQ*N4pe#hL)dE|_!JTGK99 zmw}-uGw)1aC)}?Vfq;Hudh+tMnF7lEWRQLL=FIG?J#>6_Zo#UTaD{UA!&0tPX#kOB zu2L@*E3L-2Zcfko#))GmPrLf~$cV(XQ>M!QtsjS_`Nn-5yKe+1=xcS`kK$%nwTF^e zlpmj$^=hFZE52T?IezkkVx?hCN?DhF8l{!;T#oL6_-`p_^BfI0oJSiBN7NW5-xY7xV&tStT5RKA3#XItAN=T z0^qdKhtbnEkt8)kP3mbqt!ntC)wmW%npO?Xc&z<(Iw=pJksp;7E@I8sF{jti3zdqF zAEQ?W8A2ULTh!OpHDg`DF{|$WtCnMy8WOb1G4ISyn1tEKLWQ6Jw6VP7h%@{CU^K{Q}lqVqW&m+ya7m?e%h!+>eS;dRW9bOdq$XUf5anwc4 zi#ht5;l>dt60Q+xU)<&iuel- ziZ&6HGq|p~q=^qFm*fIi1HIBG*z|GpJ@hFH9C>-ZY{_9+pStBJZ|0!&)QtJWC@eOL7Ri4!(*eP z8!$rz7&qT-RExD?&?Hfu1HB2$+fYONz}y*8i2hqdkZf~s<-S#WW2THW<0jJtQ&rYJ$L9bST@ZSJ{~d;SHFzmvxYxfHw5>o2(1ZZC-*d%|@nYjI(U zz3Uq7E%LhPUx$zrz8T>g_5^ilv5)J5w{EeY-na((+j>yiCk_Etkl!f|w_6OBm30j; zpudGJTP4uu-Y6OVjgs5~dW(_gh8TUGXr{cBr_Ph;H0J)~7D*Q@5`{8&K&3;x{jJnF ztcQ+JTw$`_lG+Kk+mwzo)6DWWQTAE71ZI7{E%sZY5OyCjE4USf`h0=Oo*fW3P&2yd zzFsLhNI)#&5;AtfE>wME6;i(2f1_-ZvS3QqGg9v>YYt-cg!=Vq$roK>l>m{o8;u#T#{ zLEZl&WC1tw5!PY4JKFQ$<1l5hMx5s8=-Bb$S8CReu#UlW8YDaY!KLfhE*EaxygYe* z#_hUR3p#O|ans$BGjBj%#h@mbAMLoGX=@n;=z}$@lGR_5!1OXg2?-_cKnu*9O#!iI z1LCnE^Y{=nLGxsAII%`;2@+K8<-Y-xm;V_F1SYFUq&;dks#Qz+@yVKPRlySFECym4 z?Z>1w50z5P`l(Gy*A+kM$Ob8F6-dG{XkTUMO<6;JLLKi>@*PSFNV1854YW_14~4f@ zengFKQ8G;l@w{xpk3zPEKI=#3p{qG8zOmFO4^tyL6F~~48sH|7 zFhn*0B5d2hHORS)3<0cki<1(n2H>K>=KG1(r4w&5q5&4p3(!>xtv0Po@;N49)AEf( zT#u`9HKS^eGc0xY>fLIu+O5X%7N=*JPohmfm!{Q$c5Tw=RD&<^uTHgF>qh%-)%Ys< zzvElv5-m0z-pm4B?AI6-?L{Yk%&7h`^{j36JL`-W8njo>YF~@LFA|gb7MXU!Y-lAw1=XMvsl&IYWl*U2Q!K5Jl zrZ9eziM;_vW?=jXG|eNBTH~t?NUc#wt$UmV^u`oOu?p!G4C#d?Ajx`BjMD{!QRXRH ziBwp!<+*lOiIQa7-Hd)3g_Z*~`+k-igY41ryB_^LJLo&w*sb_|}=yZsk#MZ?wu)N*Or1l_Za&B*6bi!Es zDlk1Z)0nd=_E$31EzJ%vyH0*TQ0cC2ZFXTVIGigg!B4#PB0|P1oX##fooDLmiST?f?bb*5bl3pKx&`k% zOn)0EKquKRhQzSQ(GIqIjs&YCCLOF0xp2qaFT5^jPJKa(pjM1NHLwE%%^htBW8T1G z9|;6BOQ+x2zBnL`iepbz2vIwo9g91?o#O3zt*m%EexiP%{;;FD%iFcK%Nrm80vc1d zp%}^&XkN9%7|eI}7b_^I???ExM>+28Y>gpK@L8X1&-_$-=0{(|n!CNo+HP;Bw|gDN z>vsRs-tJa^)bA1FRNqXs>(6*8(Jjt?u7x~Hzz>YGPh4y6cW2S>y~@7o{^o&A>wiCh>F)Fnc-upayZ+Q@?rTH4;q4RM4fUBO zZ+rU|`)aC~YV8=-dpKP0r{Q`B+Wl^BthZfuD_p_t=KgP9dkFEsGi~*s{07A7Z-6-S z|ANS~SDxweR(lq+K_3ri!0Y;wFcsK)uz9Fvh&w^ffuVoxaDYp&xy3+(55ZEWi4Ta9 z_9Dk(C6AbFbX3bZlbNYJD_-n zDAlxPw)vKGaBavt>}C5Ead)fY4SH_{xjxtin@6}d%QbIN%B4fju#*$R-rgq=hoRo= zZ4QeMy(8Yx4h4Nb>QQJPL^q92sW&_ib%x48eSV|nEcWZc{`P+5>D*gq9^(GTaHHHK z>=fU@9woLbYa{F6KF{>!`=YS9q?x0;zc#v#U2Bebxo3v-+K_)IJiQ`jyxbfG-HWV^ z!CHCL8}&xK(H+VYgtInd|X>S~m&H~cPQ&3Lkua9tEhnnZSGu}D0I`5rd8(&v&va4d1 zX|U^^qmyc0V48d0yFfIT>Fe)O-b;Gt*3Jk|G}krYn(<(^Hqt)3PeYtg#dj-btLK^* z(NhOTy}?(5n~4X+eUBx{A zxtCbGAU>n}+OGMWYyOODevfMsX!+;jZ-U}|zd0fPR`aq)CI7*@EPh}7ZAc}Fb>hQ6 zc%h$EnwOo6UJ@8S=|S?4{{_n7q_}XFB@}N$d|!OvUG^qX^S``F!0Qh9MSIOpy@?iP zdlOjgBu0wJl$Q`+JU8ybaEl&cj&U-wIvwKt3Fp$H&aRP3Ul8;UV8=( z@CRJ0ITIwUEnW^}H1CS|!{_?zavS7=A3cYc;hQD5>>s)18$~E!FaWi^V2v6(3^u8| z?9zfafqYjjpV@ra424B!lM{^mU1&DnfE}`BNen=-q9#v3N)B|C4@jg=gv#quHBcY_ zD-<-*BzoC+ylB_ztQcYIj zTpb;t?{>hqM?fan+lOUAzr%kU_QqNLKB~#FF*3t6?fn39mviq$ast6S!ROR{&mdp7x2!}+sass-2Y+B^aHuz<;= z(7U^FOl{jU&0+IS0Czj*8s&;8gq~jeWe;`ge}@VQD_b7mQe)1dQvp)`wmMwO%$3PY zvv;N^3pb}PPfiy;7;{ti%*jW~@Hk@W?#>oC*y!JEnZ1G&*!kLI_?j#g*a;}CqI=<< zIV)FbR^Qf)(V{i&4aYUz5yOx01(Tm*k@CBgJf`LDr<=x3aCp#Q4!MqyGq)>u48Erdo5kugFv)eY`AA~vZsHUso zOkzRD5x3VIBqTM%cM5F)9m%@g(=CH*Yl^1Z$wxcSloNdRC9xQHf1A(;yD{9_4tN6D zxW|X&Zv};)m*rBC$oySNhC1nQqY#!hrN0$Mp~xA)1qDyQDqLVYDYErim`##%ILt>l zOyebaWZa?YfKRoyy$#kJoS2$I(8fg3d8ZDaeU>Jq`iX!qi?Rf}>BD>~s|8*`u~KLo zNd07~UYCN*mA0P@;x+KH@G?(B_JBOV?@s3kcX6YFkOQ#4RA~4NDRV0D3c|)mQ5*6n z)Uh{g0#IOX@PU>rYQ3qvQf~kq5zoWrTU+fb)&iJVj{IsatduWPxtV+HT5h!(H?iyNNo1cQF&4P=1FwRc0Es8xJ!m8&bN8JZ4mlwCYz#(kT{`F! z_*o+qOJt#l0a6#h7GN=Sl{7bzipm4_%{ODsIR<(j;N zcXnsCVxHvhQc09iWI=k=obBUHDozI$cot`DenUi5fjI;mDHgo0OxYw^{}DX|+C$E0 zKU!U)Jh@pY+@)OijXEsmeq;p!OO_8ODbv9KiTo6Zfn8pxQD}(2HKhN**>z z+Eg9p3C=%MxB5;oaVGvlgCA{)Z9eYs66`hBZ;1N7oHu*P}D?kQ>ptM9) z`BMT+7$$#*l6R5#2}T)Wr;wmIFsT49662H~FR)%vC}gk73iV(Id8*}BAO}4oEu#)2 z(ImAZ1HEM7yemZHLKu4Bx2O$nzAb;B+7T8p_V}qllxF2aG8XBj`GpNxVnc?O#5xgL z6!_wVwGc=he@dX9QB@QY8(bcK5>LcZ@JXbRrxRjI=j#H|QPd1x`T$KjS_lM!iUXys z<%`yq-~4o|1xYeMM%ya1rGNmp)OUULuCK28>LWi+;To1KthRHH$Qw_AaB$DTt%00c zrU;>sOUi#hQ;)a!8R8<2ZbF4L2}OmKfHTP-(^CJ0lIPT^s{xv6)e6hyM^>dE<`E+l zGAqBUO1um#0<6rYs$@v;!CB-#;t^?kNdNPpNPu@2&?dNHyg7t4ehO#Zas&C9Hbx;J zi^w^4ri{40FnGc52;N$8N#Sc71(mrFO#6R8NJ>Ydi0+E36zgEX zf0T*zs~Ihb^&-!a22T^kdquwmmRXSmWWr&IRIvL662I9l;)eabGEkiA}xYn=l z)Z=;-5ft==*D0={RS;(p{d#vKqnaE`k%=A9_=7rXnf$6Zn$Zz^q3-^5k7mNx)uX2M z-H1FxOa|3|or(5ftPJ)3Dih5_(i+8U^lF9<_?S05W??)BjTZfCFq#3348G{`wedI; z*?3E$4v}qG^{Gg|J^&BfZt}6|7XdF5*{2}_ghtXdKo|go0gN!fxgZjx^Vfj+t4unB zHD{7cHfxk|3(qijV4e_P$4lbzS8zM~CI>Jadgipgp)w#MFcxy1L9-1Be ze?d*x>JLKORUCEO0(%s-_#RrcY^B*=_njHCCbUdPK@24+Nfg_F_%4ulGCtXG%5$4g zCM!@}wrz3%Tdw8dj|PF;Q?5Y)nSRIg?ZWLllXoTyQ%zb@!b+cMiHy5b+J8 zRnXVu*^u;!YPW~Njx4g&(C|DM$@$Y1>I(6O9J-R!@mK1&%+SjgByPv8Rp&n1lXQw! zBzc!efogpyuj7Xb)$T>c0aozI;#35OCJ74S)Ti9h8J`(+JF60Nolw z+yqQnnArk}Ppp+f*bKHp4cjm3tUu90nixpb#8S?kv+>`+V;8UeEv_+qVLW-Gj++PZ53w`q@AXIzkI+ z9XxMZ3})%6#Cl_J3p_4FsvB z3Oj3|;C8lfGT0XEX-ZgjNtQX9&y5a483Y%G5w4CPeUkRw9;j)o+ZP|P3J+whtkRJ~ zxrUJvVbxY;*^Lhm(?(d*?Fz0OkUlJU?Jfl4vT_f2vlxu-0)c~?d3ADzgEC~Dw&f|3 z2Hk>RJpYjyh^V%C^L-bgOhH@RmW*j`4!l80ziGZ^?J>d07wD=qNL2g9kJrqZI%1Wf z+Cbz*Tu>OTG}sbu+EG8iHAHyAx{(FcUCH6$Ve?k_e+Y`yqRYY4M}-S{-FAk@jvfm_ zlYTz-WoXK{a8cf+wR}j)$COaK%`9vnpU?}b2j8Zr-J(XM%(0Bx=tJ%o&;PEuVF5N7 zuR$&vySVO0?Ky6kkwGeK+Y+>IB(s!Zi*)0rdFkf0adR*bA^#~2@jenakwu^r1$3Qb zi@2{=-GoX17O^UzHZ&iC6U5A*fac~71e??POA5sSKP+1-Chbh7Wh0=l;04PbcQw7u^!?N9JH-D_v#fg=Ub1gQiCwA6N8G{bMbXa}+FU{|gb&NTP9m1XO)_RsJh#_AjU))lx)r<+@B$@c~MR7IIK=perxYE6FZ?$7|LJ zrbqJMqn5<_1&*xpI}vW)ATtBqD<~Slzaj9qi=se$bXeynxiP(i?F1>IhaU~d3=?WO zN8|3I&A3guCzLEx@-Hd*f|7qm$(NK6-SyK1q$SA9MJi`SEn#8hWZJak|D-}b?+qf& zIc9G0v$!4jyVZG&s@SBd8cMp0{&W$11JRNGjuHG3BOZuK zsKukn{SezBl#*E53F5Bl)RV+eBI~{QAzTdM%yG!98zLo%`_W&fN&E!-Zgr5u#wZvq RqeAGVSpZg2HU7*P|G)DEI2r%| literal 11192 zcma)CU2Ggja-P5a<&s>H6#qrFe$AE0m1J46P0O}uiIQbYqD1QFyguFBa(lR3aewHZ zSy7zK+Rn)b9D;M;APAB;6hL0`kjq1mJls=$aCysv1KeXUF9Gg?0L*icOLR6aU-iuX zkkSRW1bceAtE;Q4tE;N3rq zt!Oz-rMh0$bGm#QIYT}pxrls5b5Z$>$=qRr+#$vK}w@R|awes$v;i`+0&Vx3%1$H6-7&c?KgN%vcEFn8 zFRf}{sN#}!kWW5QRmD96x-(@~Ig~q8Ih;Ez={Qk7QaPGCD(ff9$12Bj$3fFUK1Cz& zoVE`Aner6W$hWL@xZ~}V^^$e`iP~(ruT)12#hTpsCsoO1Fkd&upSCjX{tpzc=Q1B= zs*&KiuCCJ@X9zognV;pa@N@h;f0e(6HFxvZH#I)98PA=t#;p@i)NgCKlesBtO8h&< zykSi>@m9~`{X$6Lo7PF*KcY1I(DN2r-li6Q5w%%adk3{kvUd4tEF3w9wkx|vUPa5h zyIQV=u~Zfv4^!KJDW@{ z-kH0_UY}tLR=&)vDxY?0(|EA8(wbE+RV`K%g%!(oMBXXYsu*x*rDU^8jn~T-TQ3RA zW=^eWIV+eTUoJCLnC)0=A&5PboMweu72^wzon6~tlaKP{5|`Mf*jj$0T+3s&ys%5v zVg|srQxo~3B|&eLs@LXOp6AybO9Y+yf>|h+tg0gcF3-)f#XGZ?m~G`jCxI+md8aNc zvrwz7)t!K9n)ck8>9C7gk0m)+4-Me!=2G64iZ0epEWxOuEjbD1rAtc zM#O_|c9N|LiyS6wF z5fNZrHWLwq#@DO0bz#U_-6{Bql3jzSSMtt}6)}WnSv=dHUt6G;1cA&qJr&T_b@bWdW`e3vdjeD@a(93_n@S ziXzTAz0xmW*$jIRV@e$RS6VJvqSYOsnPKyfEI|q*L_7Hr2_>JB$_l$X#g_A>GBt3L z@y_8zYldA4MWdyrCsxGam&GS09dQag+W6F^ScK4 zHRHx+0zN>d#~=k_oQeZf97N$K>vA8snRf)E*25@DpWaqUM`+e#ck+H-RKd0xc15xc zN|^;?g53()Ji~5gVu5HU<&lS)tkMy+aE>3^wW=TC^~##m5D&!uk3no-ZYyK8Z&vif!c~ zsgxM+Usdei_hMcgn7@?c2fP?C@Au*;2cJeH)k6}?Pr$cf@GSwFhDH=G0iOLtju{DB zFk)1y7ekI3lcTE~Lq zMU3XBHVuAyGtubsx;%ARgV{Rc{>3dgGO$x{ph7n%(4C>o{zjb{_QMU!uS;dybc{%@ zTlz;<7Nk|}(6@K!0I=F}%d@h$84jOfWt<4JwromeWCtn-I);w2ub1-<3LN^lfE!b{ z&5Cbqz>#u?Z8?QP<2+B&oFTo1H~&SiuhNQPtX6fcvk*S{nSldhqvX_ zxy9UJ23=`G`?FlBvXoz1x7bpxUgdTO{rbh?zkG$?4=-MHlh+h2iGRIn@#uQUIcojZN{bY+JF&P`X?l2xv)vkGhy6naIRr{Vtw`|3tMz&e;a z)}ljh2NU@!>Hn8cpE)=EOwITa>BeOuetZr-O9e{bPc2~gYLzQB*ms!rZ(V(-MqW<* zda1bLTz%(;AHS4$WSiT2ts0<}SZ0{JFUxMgYXVseMvL~uPq$n>6Ha;6DrfW^@>+Hx zRFEs+jyBQk1qH@|Hpa74?A#RGarSDkeX)9V6@E)qkgD7H8Bp!~2MYKqR-QC@)UH=5 z*!LN9i&-id3O^>SW%%{{j^EYc`Cjppj;NE@T!x<#g9}`O^O`ZlyEG7YVMV-0#rr5S ziNF-JWI=A5WWKmUK(ABrHWds-Cgw-sd*M>yN0#9*J7R*mhzVkV3feVa#~mV*5~ryi z4Wt!f8HyhUlKi!$56-0&k6>&nbQDbzUOy%G#FTpiFErq~pJ<*(zSka&(&fW4vFm0N zrg?t83u^4|pfD0~J+8*pw5olY)>CRq9n|~OLA6hf<1J3lu$~0WFk0h)8Ey3@jUJdW z`b*-kNA1)40NiHR;UnvR_l^?#92gU;m6fhL)3|hi$E)x?Rwc z1_dD+2ts(;6Q!Z!GJO(GpT2PcPM-m%@1SFNDv!V|)VZ`pPcx2=J~Bs6n`Pryb+_(55+5k=QEy^(5kH+sAterQZ- z^m^T1@37*fWQqEatPd$gqmlMvz`oBN^7cLK@zP$>>*9w$*BgB;ijO$GULOT9_|c~k zx{`zQ;w$jnncxbE{4<#c6V zqaW+;Zwz4W0r(*MTW}}6{h(wjpv3J1pQg)wmHx(H2d-?0E7ceRttrqt@P)c@T2<%| zv@!h`-UqLjk4lsjXFC@J1dE zw>ewQ692KLzYlldP`H9~{7wFr+yjlj{k-h+i(%g$>8G_lw@0)59e!!oDldb&IT`Me zVH>RRN_nI*+8C>hHO8^dgf}MF3798Z;{bRc(5Y=8yeH3Slg?RwmA@O(a_t3lzW3YE z=?-|~-T`mO8{qFlVkcVMy6#Q9fLq$e;&W8o;5WmW=3hX?t>1D^5K?g$5(#Ms$L^KKD+d}3(#^aH?;u+4@1p%%f$WAlM~=|AR8Y#rHF zcz%^={A<2cRe7Ot+&hF7AA>hJv31ltzO8zr-f>XFKPNgZsjHT8L4V29w~l$oJouHM zS>HO&i&XcH@RbYg(_0Ft#*U9k$<#LfJLGkbcXaC|{xIMqdCM=>3oZIrc^UNU;<|ep z^jARtq@)x9T5n|QC|@E<`O+Rrt1U_=UqETCgVMDUFp7ySKRzzOFBS!FSe$c@8>t+z1_&e zfu#^jGf-Qv%1C0zbwNg55qPEWYBQ|69D<=S?{vgzDI^Q9s-Ke7ECyj} zKN^Ioe4|t?IqpO|`a>oEdKe(G?XjN(O~RpQ5dy2YCCHi^2Mu&%WNudnf$rdg;4O&vw;9p1 zMAB)7(rNF|#ljtyB1^02lv>bCEBsA;*x~e*x!F5+7v{|Qh0Ajb=Djm+*8>tztKi@& z?2b3lA)L1tGK1BR@1jMreAOgbG`W>`-*~|8%DeUd2PJvkB)Xlh*rA}b+gzndXV8>r zEP)WVW0ooi1ym5_v~X9yLg)X5S&!zm-lnKvVU-G;Q^zU>NgludX3e?QMiofb(JLJ% zpM|qB%*r}ubAUqfJehjIw+cv}w;4V{2Ewdu?E-r-0(z6VnGQhOMJkauq=mZV&|Z1j7=+>|Nul+oFtSOAWHD^4sa49ELD zHz=`n9YhD)-Y4H>qN_<5Hj8ffekv4DDTjV{^Q|VhCES4r2)!duAPpI7GKazgZhuEd zTa&dL!-?As&53190dWnPJlmmAggYqB2{E7*JYkcoZ@NY%P))tf%$pfx-k==s67A7y zjl$DzKlDj67sn8#fCl4!GRVRq7*gE1MYHS+tP@Q`1}Z?ESVu;WA|^!{=8%z+J8oAP zUdbZuvGZ%1DpJSUdd-j0E`;?nH)iKiT4O4MQM#z4@iPx-) zGHJnpBB-OzQHk&cfjSD=Tyc9lLOeu~n?ALt#qR`>$e|ls$oBxd-5m{>L~g-w7%ZCE zs|;ks$Q^6Pm7as3V7?$LiC{hgW^P~0ZXjn&3c|6V{wG7A8TL#=e&{u$Q**=&L)c2DDP+QeK#-ve0-A0Di=%`Lh$j>4_T8;wvjfE*o;!#)g?xgC&dH7A_#g??@~i32gBhwdf@0l%Nz5dX2*~T%-IKSi8#U#n%y@MWoLr;ie>y z)Ifpw`dw?Nl!{doQ9f~<5F{jHmtcP2n#gRei7>C^ydO8sYQAEbCfNKfno@40WV2*= zmf9#%D9Fc=>zVd#p~`bX3nkzAI%6rWJTzNSDl z?QFnV@i7gN%#7j`#hqqz@2F-tc2IJ$j&sOGxpD2{kBU`PR%ycS*4}CA;^A zKFK>Yaq;*YptgU6n@Fl(Lp+lrlo}$M6vK>uGa5ni?O7^egl#=&OGeVVp&d0+8X&Y` zeO!<0Q7x*{A7YwuO(SUfi+(c^=~043bu!kcj%sPNHL07B)TO(IsTEOGL}>T)X^DUmqGdGle1EDd zj)-ka>(SCG=);@_QjmQ}MuJWxAftE&7^1&QB|x=0fVqN{WS>D$L|+F?d^b$mpqMat z8bM-HVr7`kC{qHxcz&Mo?>d$ZTCT{dsbU!J?ZC_W&GE~BXT zp!oUbAF@Ds*lhlN7q_2a5V8S_S@xjm1$$p7_)x zv3)Xp7 zOL`?ILHsEdn^chP^i#Cu%_Nt&N9{6Nkg%$CqzvgMu9kvtQ0O%5H5>5@xGmuOzqd%j zY_f}nlIo31eMLGe4Z8u2N_nzqTJ6!2F|A8eQ~#UHCT}U9D4!(Xf`QNpigIU^6&u7q pvSs+lBW?s5zo{c(5&kFDWdLU!+I>=H%_zH;rjJ4}BeeC;{trZQ7$X1x diff --git a/apps/worker/inference.py b/apps/worker/inference.py deleted file mode 100644 index 7a5367d..0000000 --- a/apps/worker/inference.py +++ /dev/null @@ -1,650 +0,0 @@ -"""GeoCrop inference pipeline (worker-side). - -This module is designed to be called by your RQ worker. -Given a job payload (AOI, year, model choice), it: - 1) Loads the correct model artifact from MinIO (or local cache). - 2) Loads/clips the DW baseline COG for the requested season/year. - 3) Queries Digital Earth Africa STAC for imagery and builds feature stack. - - IMPORTANT: Uses exact feature engineering from train.py: - - Savitzky-Golay smoothing (window=5, polyorder=2) - - Phenology metrics (amplitude, AUC, peak, slope) - - Harmonic features (1st/2nd order sin/cos) - - Seasonal window statistics (Early/Peak/Late) - 4) Runs per-pixel inference to produce refined classes at 10m. - 5) Applies neighborhood smoothing (majority filter). - 6) Writes output GeoTIFF (COG recommended) to MinIO. - -IMPORTANT: This implementation supports the current MinIO model format: - - Zimbabwe_Ensemble_Raw_Model.pkl (no scaler needed) - - Zimbabwe_Ensemble_Model.pkl (scaler needed) - - etc. -""" - -from __future__ import annotations - -import json -import os -import tempfile -from dataclasses import dataclass -from datetime import datetime -from pathlib import Path -from typing import Dict, Optional, Tuple, List - -# Try to import required dependencies -try: - import joblib -except ImportError: - joblib = None - -try: - import numpy as np -except ImportError: - np = None - -try: - import rasterio - from rasterio import windows - from rasterio.enums import Resampling -except ImportError: - rasterio = None - windows = None - Resampling = None - -try: - from config import InferenceConfig -except ImportError: - InferenceConfig = None - -try: - from features import ( - build_feature_stack_from_dea, - clip_raster_to_aoi, - load_dw_baseline_window, - majority_filter, - validate_aoi_zimbabwe, - ) -except ImportError: - pass - - -# ========================================== -# STEP 6: Model Loading and Raster Prediction -# ========================================== - -def load_model(storage, model_name: str): - """Load a trained model from MinIO storage. - - Args: - storage: MinIOStorage instance with download_model_file method - model_name: Name of model (e.g., "RandomForest", "XGBoost", "Ensemble") - - Returns: - Loaded sklearn-compatible model - - Raises: - FileNotFoundError: If model file not found - ValueError: If model has incompatible number of features - """ - # Create temp directory for download - import tempfile - with tempfile.TemporaryDirectory() as tmp_dir: - dest_dir = Path(tmp_dir) - - # Download model file from MinIO - # storage.download_model_file already handles mapping - model_path = storage.download_model_file(model_name, dest_dir) - - # Load model with joblib - model = joblib.load(model_path) - - # Validate model compatibility - if hasattr(model, 'n_features_in_'): - from feature_computation import FEATURE_ORDER_V1, FEATURE_ORDER_V2 - actual_features = model.n_features_in_ - - if actual_features == len(FEATURE_ORDER_V1): - print(f"Detected V1 model ({actual_features} features)") - elif actual_features == len(FEATURE_ORDER_V2): - print(f"Detected V2 model ({actual_features} features)") - else: - raise ValueError( - f"Model feature mismatch: model expects {actual_features} features. " - f"Available versions: V1 ({len(FEATURE_ORDER_V1)}), V2 ({len(FEATURE_ORDER_V2)})." - ) - - return model - - -def predict_raster( - model, - feature_cube: np.ndarray, - feature_order: List[str], -) -> np.ndarray: - """Run inference on a feature cube. - - Args: - model: Trained sklearn-compatible model - feature_cube: 3D array of shape (H, W, 51) containing features - feature_order: List of 51 feature names in order - - Returns: - 2D array of shape (H, W) with class predictions - - Raises: - ValueError: If feature_cube dimensions don't match feature_order - """ - # Validate dimensions - expected_features = len(feature_order) - actual_features = feature_cube.shape[-1] - - if actual_features != expected_features: - raise ValueError( - f"Feature dimension mismatch: feature_cube has {actual_features} features " - f"but feature_order has {expected_features}. " - f"feature_cube shape: {feature_cube.shape}, feature_order length: {len(feature_order)}. " - f"Expected 51 features matching FEATURE_ORDER_V1." - ) - - H, W, C = feature_cube.shape - - # Flatten spatial dimensions: (H, W, C) -> (H*W, C) - X = feature_cube.reshape(-1, C) - - # Identify nodata pixels (all zeros) - nodata_mask = np.all(X == 0, axis=1) - num_nodata = np.sum(nodata_mask) - - # Replace nodata with small non-zero values to avoid model issues - # The predictions will be overwritten for nodata pixels anyway - X_safe = X.copy() - if num_nodata > 0: - # Use epsilon to avoid division by zero in some models - X_safe[nodata_mask] = np.full(C, 1e-6) - - # Run prediction - y_pred = model.predict(X_safe) - - # Set nodata pixels to 0 (assuming class 0 reserved for nodata) - if num_nodata > 0: - y_pred[nodata_mask] = 0 - - # Reshape back to (H, W) - result = y_pred.reshape(H, W) - - return result - - -# ========================================== -# Legacy functions (kept for backward compatibility) -# ========================================== - - -# Model name to MinIO filename mapping -# Format: "Zimbabwe__Model.pkl" or "Zimbabwe__Raw_Model.pkl" -MODEL_NAME_MAPPING = { - # Ensemble models - "Ensemble": "Zimbabwe_Ensemble_Raw_Model.pkl", - "Ensemble_Raw": "Zimbabwe_Ensemble_Raw_Model.pkl", - "Ensemble_Scaled": "Zimbabwe_Ensemble_Model.pkl", - - # Individual models - "RandomForest": "Zimbabwe_RandomForest_Model.pkl", - "XGBoost": "Zimbabwe_XGBoost_Model.pkl", - "LightGBM": "Zimbabwe_LightGBM_Model.pkl", - "CatBoost": "Zimbabwe_CatBoost_Model.pkl", - - # Legacy/raw variants - "RandomForest_Raw": "Zimbabwe_RandomForest_Model.pkl", - "XGBoost_Raw": "Zimbabwe_XGBoost_Model.pkl", - "LightGBM_Raw": "Zimbabwe_LightGBM_Model.pkl", - "CatBoost_Raw": "Zimbabwe_CatBoost_Model.pkl", -} - -# Default class mapping if label encoder not available -# Based on typical Zimbabwe crop classification -DEFAULT_CLASSES = [ - "cropland_rainfed", - "cropland_irrigated", - "tree_crop", - "grassland", - "shrubland", - "urban", - "water", - "bare", -] - - -@dataclass -class InferenceResult: - job_id: str - status: str - outputs: Dict[str, str] - meta: Dict - - -def _local_artifact_cache_dir() -> Path: - d = Path(os.getenv("GEOCROP_CACHE_DIR", "/tmp/geocrop-cache")) - d.mkdir(parents=True, exist_ok=True) - return d - - -def get_model_filename(model_name: str) -> str: - """Get the MinIO filename for a given model name. - - Args: - model_name: Model name from job payload (e.g., "Ensemble", "Ensemble_Scaled") - - Returns: - MinIO filename (e.g., "Zimbabwe_Ensemble_Raw_Model.pkl") - """ - # Direct lookup - if model_name in MODEL_NAME_MAPPING: - return MODEL_NAME_MAPPING[model_name] - - # Try case-insensitive - model_lower = model_name.lower() - for key, value in MODEL_NAME_MAPPING.items(): - if key.lower() == model_lower: - return value - - # Default fallback - if "_raw" in model_lower: - return f"Zimbabwe_{model_name.replace('_Raw', '').title()}_Raw_Model.pkl" - else: - return f"Zimbabwe_{model_name.title()}_Model.pkl" - - -def needs_scaler(model_name: str) -> bool: - """Determine if a model needs feature scaling. - - Models with "_Raw" suffix do NOT need scaling. - All other models require StandardScaler. - - Args: - model_name: Model name from job payload - - Returns: - True if scaler should be applied - """ - # Check for _Raw suffix - if "_raw" in model_name.lower(): - return False - - # Ensemble without suffix defaults to raw - if model_name.lower() == "ensemble": - return False - - # Default: needs scaling - return True - - -def load_model_artifacts(cfg: InferenceConfig, model_name: str) -> Tuple[object, object, Optional[object], List[str]]: - """Load model, label encoder, optional scaler, and feature list. - - Supports current MinIO format: - - Zimbabwe_*_Raw_Model.pkl (no scaler) - - Zimbabwe_*_Model.pkl (needs scaler) - - Args: - cfg: Inference configuration - model_name: Name of the model to load - - Returns: - Tuple of (model, label_encoder, scaler, selected_features) - """ - cache = _local_artifact_cache_dir() / model_name.replace(" ", "_") - cache.mkdir(parents=True, exist_ok=True) - - # Get the MinIO filename - model_filename = get_model_filename(model_name) - model_key = f"models/{model_filename}" # Prefix in bucket - - model_p = cache / "model.pkl" - le_p = cache / "label_encoder.pkl" - scaler_p = cache / "scaler.pkl" - feats_p = cache / "selected_features.json" - - # Check if cached - if not model_p.exists(): - print(f"๐Ÿ“ฅ Downloading model from MinIO: {model_key}") - cfg.storage.download_model_bundle(model_key, cache) - - # Load model - model = joblib.load(model_p) - - # Load or create label encoder - if le_p.exists(): - label_encoder = joblib.load(le_p) - else: - # Try to get classes from model - print("โš ๏ธ Label encoder not found, creating default") - from sklearn.preprocessing import LabelEncoder - label_encoder = LabelEncoder() - # Fit on default classes - label_encoder.fit(DEFAULT_CLASSES) - - # Load scaler if needed - scaler = None - if needs_scaler(model_name): - if scaler_p.exists(): - scaler = joblib.load(scaler_p) - else: - print("โš ๏ธ Scaler not found but required for this model variant") - # Create a dummy scaler that does nothing - from sklearn.preprocessing import StandardScaler - scaler = StandardScaler() - # Note: In production, this should fail - scaler must be uploaded - - # Load selected features - if feats_p.exists(): - selected_features = json.loads(feats_p.read_text()) - else: - print("โš ๏ธ Selected features not found, will use all computed features") - selected_features = None - - return model, label_encoder, scaler, selected_features - - -def run_inference_job(cfg: InferenceConfig, job: Dict) -> InferenceResult: - """Main worker entry. - - job payload example: - { - "job_id": "...", - "user_id": "...", - "lat": -17.8, - "lon": 31.0, - "radius_m": 2000, - "year": 2022, - "season": "summer", - "model": "Ensemble" # or "Ensemble_Scaled", "RandomForest", etc. - } - """ - - job_id = str(job.get("job_id")) - - # 1) Validate AOI constraints - aoi = (float(job["lon"]), float(job["lat"]), float(job["radius_m"])) - validate_aoi_zimbabwe(aoi, max_radius_m=cfg.max_radius_m) - - year = int(job["year"]) - season = str(job.get("season", "summer")).lower() - - # Your training window (Sep -> May) - start_date, end_date = cfg.season_dates(year=year, season=season) - - model_name = str(job.get("model", "Ensemble")) - print(f"๐Ÿค– Loading model: {model_name}") - - model, le, scaler, selected_features = load_model_artifacts(cfg, model_name) - - # Determine if we need scaling - use_scaler = scaler is not None and needs_scaler(model_name) - print(f" Scaler required: {use_scaler}") - - # 2) Load DW baseline for this year/season (already converted to COGs) - # (This gives you the "DW baseline toggle" layer too.) - dw_arr, dw_profile = load_dw_baseline_window( - cfg=cfg, - year=year, - season=season, - aoi=aoi, - ) - - # 3) Build EO feature stack from DEA STAC - # IMPORTANT: This now uses full feature engineering matching train.py - print("๐Ÿ“ก Building feature stack from DEA STAC...") - feat_arr, feat_profile, feat_names, aux_layers = build_feature_stack_from_dea( - cfg=cfg, - aoi=aoi, - start_date=start_date, - end_date=end_date, - target_profile=dw_profile, - ) - - print(f" Computed {len(feat_names)} features") - print(f" Feature array shape: {feat_arr.shape}") - - # 4) Prepare model input: (H,W,C) -> (N,C) - H, W, C = feat_arr.shape - X = feat_arr.reshape(-1, C) - - # Ensure feature order matches training - if selected_features is not None: - name_to_idx = {n: i for i, n in enumerate(feat_names)} - keep_idx = [name_to_idx[n] for n in selected_features if n in name_to_idx] - - if len(keep_idx) == 0: - print("โš ๏ธ No matching features found, using all computed features") - else: - print(f" Using {len(keep_idx)} selected features") - X = X[:, keep_idx] - else: - print(" Using all computed features (no selection)") - - # Apply scaler if needed - if use_scaler and scaler is not None: - print(" Applying StandardScaler") - X = scaler.transform(X) - - # Handle NaNs (common with clouds/no-data) - X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0) - - # 5) Predict - print("๐Ÿ”ฎ Running prediction...") - y_pred = model.predict(X).astype(np.int32) - - # Back to string labels (your refined classes) - try: - refined_labels = le.inverse_transform(y_pred) - except Exception as e: - print(f"โš ๏ธ Label inverse_transform failed: {e}") - # Fallback: use default classes - refined_labels = np.array([DEFAULT_CLASSES[i % len(DEFAULT_CLASSES)] for i in y_pred]) - - refined_labels = refined_labels.reshape(H, W) - - # 6) Neighborhood smoothing (majority filter) - smoothing_kernel = job.get("smoothing_kernel", cfg.smoothing_kernel) - if cfg.smoothing_enabled and smoothing_kernel > 1: - print(f"๐Ÿงผ Applying majority filter (k={smoothing_kernel})") - refined_labels = majority_filter(refined_labels, k=smoothing_kernel) - - # 7) Write outputs (GeoTIFF only; COG recommended for tiling) - ts = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") - out_name = f"refined_{season}_{year}_{job_id}_{ts}.tif" - baseline_name = f"dw_{season}_{year}_{job_id}_{ts}.tif" - - with tempfile.TemporaryDirectory() as tmp: - refined_path = Path(tmp) / out_name - dw_path = Path(tmp) / baseline_name - - # DW baseline - with rasterio.open(dw_path, "w", **dw_profile) as dst: - dst.write(dw_arr, 1) - - # Refined - store as uint16 with a sidecar legend in meta (recommended) - # For now store an index raster; map index->class in meta.json - classes = le.classes_.tolist() if hasattr(le, 'classes_') else DEFAULT_CLASSES - class_to_idx = {c: i for i, c in enumerate(classes)} - - # Handle string labels - if refined_labels.dtype.kind in ['U', 'O', 'S']: - # String labels - create mapping - idx_raster = np.zeros((H, W), dtype=np.uint16) - for i, cls in enumerate(classes): - mask = refined_labels == cls - idx_raster[mask] = i - else: - # Numeric labels already - idx_raster = refined_labels.astype(np.uint16) - - refined_profile = dw_profile.copy() - refined_profile.update({"dtype": "uint16", "count": 1}) - - with rasterio.open(refined_path, "w", **refined_profile) as dst: - dst.write(idx_raster, 1) - - # Upload - refined_uri = cfg.storage.upload_result(local_path=refined_path, key=f"results/{out_name}") - dw_uri = cfg.storage.upload_result(local_path=dw_path, key=f"results/{baseline_name}") - - # Optionally upload aux layers (true color, NDVI/EVI/SAVI) - aux_uris = {} - for layer_name, layer in aux_layers.items(): - # layer: (H,W) or (H,W,3) - aux_path = Path(tmp) / f"{layer_name}_{season}_{year}_{job_id}_{ts}.tif" - - # Determine count and dtype - if layer.ndim == 3 and layer.shape[2] == 3: - count = 3 - dtype = layer.dtype - else: - count = 1 - dtype = layer.dtype - - aux_profile = dw_profile.copy() - aux_profile.update({"count": count, "dtype": str(dtype)}) - - with rasterio.open(aux_path, "w", **aux_profile) as dst: - if count == 1: - dst.write(layer, 1) - else: - dst.write(layer.transpose(2, 0, 1), [1, 2, 3]) - - aux_uris[layer_name] = cfg.storage.upload_result( - local_path=aux_path, key=f"results/{aux_path.name}" - ) - - meta = { - "job_id": job_id, - "year": year, - "season": season, - "start_date": start_date, - "end_date": end_date, - "model": model_name, - "scaler_used": use_scaler, - "classes": classes, - "class_index": class_to_idx, - "features_computed": feat_names, - "n_features": len(feat_names), - "smoothing": {"enabled": cfg.smoothing_enabled, "kernel": smoothing_kernel}, - } - - outputs = { - "refined_geotiff": refined_uri, - "dw_baseline_geotiff": dw_uri, - **aux_uris, - } - - return InferenceResult(job_id=job_id, status="done", outputs=outputs, meta=meta) - - -# ========================================== -# Self-Test -# ========================================== - -if __name__ == "__main__": - print("=== Inference Module Self-Test ===") - - # Check for required dependencies - missing_deps = [] - for mod in ['joblib', 'sklearn']: - try: - __import__(mod) - except ImportError: - missing_deps.append(mod) - - if missing_deps: - print(f"\nโš ๏ธ Missing dependencies: {missing_deps}") - print(" These will be available in the container environment.") - print(" Running syntax validation only...") - - # Test 1: predict_raster with dummy data (only if sklearn available) - print("\n1. Testing predict_raster with dummy feature cube...") - - # Create dummy feature cube (10, 10, 51) - H, W, C = 10, 10, 51 - dummy_cube = np.random.rand(H, W, C).astype(np.float32) - - # Create dummy feature order - from feature_computation import FEATURE_ORDER_V1 - feature_order = FEATURE_ORDER_V1 - - print(f" Feature cube shape: {dummy_cube.shape}") - print(f" Feature order length: {len(feature_order)}") - - if 'sklearn' not in missing_deps: - # Create a dummy model for testing - from sklearn.ensemble import RandomForestClassifier - - # Train a small model on random data - X_train = np.random.rand(100, C) - y_train = np.random.randint(0, 8, 100) - dummy_model = RandomForestClassifier(n_estimators=10, random_state=42) - dummy_model.fit(X_train, y_train) - - # Verify model compatibility check - print(f" Model n_features_in_: {dummy_model.n_features_in_}") - - # Run prediction - try: - result = predict_raster(dummy_model, dummy_cube, feature_order) - print(f" Prediction result shape: {result.shape}") - print(f" Expected shape: ({H}, {W})") - - if result.shape == (H, W): - print(" โœ“ predict_raster test PASSED") - else: - print(" โœ— predict_raster test FAILED - wrong shape") - except Exception as e: - print(f" โœ— predict_raster test FAILED: {e}") - - # Test 2: predict_raster with nodata handling - print("\n2. Testing nodata handling...") - - # Create cube with nodata (all zeros) - nodata_cube = np.zeros((5, 5, C), dtype=np.float32) - nodata_cube[2, 2, :] = 1.0 # One valid pixel - - result_nodata = predict_raster(dummy_model, nodata_cube, feature_order) - print(f" Nodata pixel value at [2,2]: {result_nodata[2, 2]}") - print(f" Nodata pixels (should be 0): {result_nodata[0, 0]}") - - if result_nodata[0, 0] == 0 and result_nodata[0, 1] == 0: - print(" โœ“ Nodata handling test PASSED") - else: - print(" โœ— Nodata handling test FAILED") - - # Test 3: Feature mismatch detection - print("\n3. Testing feature mismatch detection...") - - wrong_cube = np.random.rand(5, 5, 50).astype(np.float32) # 50 features, not 51 - - try: - predict_raster(dummy_model, wrong_cube, feature_order) - print(" โœ— Feature mismatch test FAILED - should have raised ValueError") - except ValueError as e: - if "Feature dimension mismatch" in str(e): - print(" โœ“ Feature mismatch test PASSED") - else: - print(f" โœ— Wrong error: {e}") - else: - print(" (sklearn not available - skipping)") - - # Test 4: Try loading model from MinIO (will fail without real storage) - print("\n4. Testing load_model from MinIO...") - try: - from storage import MinIOStorage - storage = MinIOStorage() - - # This will fail without real MinIO, but we can catch the error - model = load_model(storage, "RandomForest") - print(" Model loaded successfully") - print(" โœ“ load_model test PASSED") - except Exception as e: - print(f" (Expected) MinIO/storage not available: {e}") - print(" โœ“ load_model test handled gracefully") - - print("\n=== Inference Module Test Complete ===") - diff --git a/apps/worker/worker.py b/apps/worker/worker.py index 0e981b4..4772ce9 100644 --- a/apps/worker/worker.py +++ b/apps/worker/worker.py @@ -8,8 +8,7 @@ This module wires together all the step modules: - stac_client.py (DEA STAC search) - feature_computation.py (51-feature extraction) - dw_baseline.py (windowed DW baseline) -- inference.py (model loading + prediction) -- postprocess.py (majority filter smoothing) +- hybrid_inference.py (CNN + CatBoost ensemble inference) - cog.py (COG export) """ @@ -20,17 +19,17 @@ import os import sys import tempfile import traceback +from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, timezone from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple # Redis/RQ for job queue from redis import Redis from rq import Queue import numpy as np -import rasterio -from rasterio.io import MemoryFile + # ========================================== # Redis Configuration @@ -48,11 +47,9 @@ def _get_redis_conn(): redis_host = os.getenv("REDIS_HOST", "redis.geocrop.svc.cluster.local") redis_port_str = os.getenv("REDIS_PORT", "6379") - # Handle case where REDIS_PORT might be a full URL try: redis_port = int(redis_port_str) except ValueError: - # If it's a URL, extract the port if "://" in redis_port_str: import urllib.parse parsed = urllib.parse.urlparse(redis_port_str) @@ -60,7 +57,6 @@ def _get_redis_conn(): else: redis_port = 6379 - # MUST NOT use decode_responses=True because RQ uses pickle (binary) return Redis(host=redis_host, port=redis_port) @@ -85,17 +81,7 @@ def update_status( outputs: Optional[Dict] = None, error: Optional[Dict] = None, ) -> None: - """Update job status in Redis. - - Args: - job_id: Job identifier - status: Overall status (queued, running, failed, done) - stage: Current pipeline stage - progress: Progress percentage (0-100) - message: Human-readable message - outputs: Output file URLs (when done) - error: Error details (on failure) - """ + """Update job status in Redis.""" key = f"job:{job_id}:status" status_data = { @@ -113,8 +99,7 @@ def update_status( status_data["error"] = error try: - redis_conn.set(key, json.dumps(status_data), ex=86400) # 24h expiry - # Also update the job metadata in RQ if possible + redis_conn.set(key, json.dumps(status_data), ex=86400) from rq import get_current_job job = get_current_job() if job: @@ -126,39 +111,65 @@ def update_status( print(f"Warning: Failed to update Redis status: {e}") +def send_dw_baseline_if_ready(dw_future, storage, job_id, payload, update_func): + """Check if DW baseline is ready and send to client.""" + if dw_future is None: + return None + + if dw_future.done(): + try: + dw_result = dw_future.result() + if dw_result is not None: + dw_arr, dw_profile = dw_result + + # Save to temp file + import rasterio + dw_temp_path = Path(tempfile.mktemp(suffix=".tif")) + with rasterio.open(dw_temp_path, 'w', **dw_profile) as dst: + dst.write(dw_arr) + + # Upload to MinIO + dw_key = f"baselines/{job_id}/dw_baseline_{payload['year']}_{payload['season']}.tif" + storage.upload_result(dw_temp_path, dw_key) + + # Generate presigned URL + dw_url = storage.presign_get("geocrop-baselines", dw_key) + print(f"[{job_id}] DW baseline URL ready: {dw_url[:80]}...") + + # Notify client + update_func( + job_id, "running", "dw_ready", 30, + "Dynamic World baseline ready", + outputs={"dw_baseline_url": dw_url}, + ) + return dw_url + except Exception as e: + print(f"[{job_id}] DW baseline processing failed: {e}") + return None + + # ========================================== # Payload Validation # ========================================== def parse_and_validate_payload(payload: dict) -> tuple[dict, List[str]]: - """Parse and validate job payload. - - Args: - payload: Raw job payload dict - - Returns: - Tuple of (validated_payload, list_of_errors) - """ + """Parse and validate job payload.""" errors = [] - # Required fields required = ["job_id", "lat", "lon", "radius_m", "year"] for field in required: if field not in payload: errors.append(f"Missing required field: {field}") - # Validate AOI if "lat" in payload and "lon" in payload: lat = float(payload["lat"]) lon = float(payload["lon"]) - # Zimbabwe bounds check if not (-22.5 <= lat <= -15.6): errors.append(f"Latitude {lat} outside Zimbabwe bounds") if not (25.2 <= lon <= 33.1): errors.append(f"Longitude {lon} outside Zimbabwe bounds") - # Validate radius if "radius_m" in payload: radius = int(payload["radius_m"]) if radius > 5000: @@ -166,26 +177,22 @@ def parse_and_validate_payload(payload: dict) -> tuple[dict, List[str]]: if radius < 100: errors.append(f"Radius {radius}m below min 100m") - # Validate year if "year" in payload: year = int(payload["year"]) current_year = datetime.now().year if year < 2015 or year > current_year: errors.append(f"Year {year} outside valid range (2015-{current_year})") - # Validate model if "model" in payload: from contracts import VALID_MODELS if payload["model"] not in VALID_MODELS: errors.append(f"Invalid model: {payload['model']}. Must be one of {VALID_MODELS}") - # Validate kernel if "smoothing_kernel" in payload: kernel = int(payload["smoothing_kernel"]) if kernel not in [3, 5, 7]: errors.append(f"Invalid smoothing_kernel: {kernel}. Must be 3, 5, or 7") - # Set defaults validated = { "job_id": payload.get("job_id", "unknown"), "lat": float(payload.get("lat", 0)), @@ -207,30 +214,47 @@ def parse_and_validate_payload(payload: dict) -> tuple[dict, List[str]]: # ========================================== -# Main Job Runner +# Async DW Loading Helper +# ========================================== + +def _load_dw_async(storage, bbox, year, season) -> Optional[Tuple[np.ndarray, dict]]: + """Async wrapper for DW baseline loading.""" + from dw_baseline import load_dw_baseline_window + try: + dw_arr, dw_profile = load_dw_baseline_window( + storage=storage, + aoi_bbox_wgs84=bbox, + year=year, + season=season, + ) + print(f"[_dw_load] DW baseline loaded: shape={dw_arr.shape}") + return dw_arr, dw_profile + except Exception as e: + print(f"[_dw_load] DW baseline failed: {e}") + return None + + +# ========================================== +# Main Job Runner (Async) # ========================================== def run_job(payload_dict: dict) -> dict: - """Main job runner function. + """Main job runner with async DW baseline loading. - This is the RQ task function that orchestrates the full pipeline. + DW baseline loads in background while hybrid inference runs. + DW URL is sent to client as soon as it's ready, parallel to inference. """ from rq import get_current_job current_job = get_current_job() - # Extract job_id from payload or RQ job_id = payload_dict.get("job_id") if not job_id and current_job: job_id = current_job.id if not job_id: job_id = "unknown" - # Ensure job_id is in payload for validation payload_dict["job_id"] = job_id - - # Standardize payload from API format to worker format - # API sends: radius_km, model_name - # Worker expects: radius_m, model + if "radius_km" in payload_dict and "radius_m" not in payload_dict: payload_dict["radius_m"] = int(float(payload_dict["radius_km"]) * 1000) @@ -249,7 +273,6 @@ def run_job(payload_dict: dict) -> dict: ) return {"status": "failed", "error": str(e)} - # Parse and validate payload payload, errors = parse_and_validate_payload(payload_dict) if errors: update_status( @@ -259,220 +282,97 @@ def run_job(payload_dict: dict) -> dict: ) return {"status": "failed", "errors": errors} - # Update initial status - update_status(job_id, "running", "fetch_stac", 5, "Fetching STAC items...") + update_status(job_id, "running", "init", 5, "Starting inference pipeline...") - missing_outputs = [] - output_urls = {} dw_baseline_url = None + output_urls = {} + missing_outputs = [] try: - # ========================================== - # Stage 1: Fetch STAC - # ========================================== - print(f"[{job_id}] Fetching STAC items for {payload['year']} {payload['season']}...") - - from stac_client import DEASTACClient + # Get config and AOI bbox from config import InferenceConfig, MinIOStorage as ConfigMinIO - from dw_baseline import load_dw_baseline_window cfg = InferenceConfig() - # Initialize storage adapter for inference.py cfg.storage = ConfigMinIO() - # Get season dates start_date, end_date = cfg.season_dates(payload['year'], payload['season']) - # Calculate AOI bbox lat, lon, radius = payload['lat'], payload['lon'], payload['radius_m'] - - # Rough bbox from radius (in degrees) - radius_deg = radius / 111000 # ~111km per degree - bbox = [ - lon - radius_deg, # min_lon - lat - radius_deg, # min_lat - lon + radius_deg, # max_lon - lat + radius_deg, # max_lat - ] - - # Search STAC - stac_client = DEASTACClient() - - try: - items = stac_client.search_items( - bbox=bbox, - start_date=start_date, - end_date=end_date, - ) - print(f"[{job_id}] Found {len(items)} STAC items") - except Exception as e: - print(f"[{job_id}] STAC search failed: {e}") - # Continue but note that features may be limited + radius_deg = radius / 111000 + bbox = [lon - radius_deg, lat - radius_deg, lon + radius_deg, lat + radius_deg] # ========================================== - # Stage 2: Load DW Baseline + # Start DW baseline loading in background # ========================================== - update_status(job_id, "running", "load_dw", 10, "Loading Dynamic World baseline...") + update_status(job_id, "running", "load_dw", 10, "Loading Dynamic World baseline (async)...") + print(f"[{job_id}] Starting async DW baseline load...") - print(f"[{job_id}] Loading Dynamic World baseline for {payload['year']} {payload['season']}...") - - try: - # Load DW baseline for the AOI - dw_arr, dw_profile = load_dw_baseline_window( - storage=storage, - aoi_bbox_wgs84=bbox, - year=payload['year'], - season=payload['season'], - ) - print(f"[{job_id}] DW baseline loaded: shape={dw_arr.shape}") - - # Save to temporary TIF file - dw_temp_path = Path(tempfile.mktemp(suffix=".tif")) - with rasterio.open(dw_temp_path, 'w', **dw_profile) as dst: - dst.write(dw_arr) - print(f"[{job_id}] DW baseline saved to temp file: {dw_temp_path}") - - # Upload to MinIO - dw_key = f"baselines/{job_id}/dw_baseline_{payload['year']}_{payload['season']}.tif" - storage.upload_result(dw_temp_path, dw_key) - print(f"[{job_id}] DW baseline uploaded to: {dw_key}") - - # Generate presigned URL - dw_baseline_url = storage.presign_get("geocrop-baselines", dw_key) - print(f"[{job_id}] DW baseline URL: {dw_baseline_url[:80]}...") - - # Immediately update job status with DW baseline URL - update_status( - job_id, "running", "load_dw", 15, - "DW baseline loaded and uploaded", - outputs={"dw_baseline_url": dw_baseline_url}, + with ThreadPoolExecutor(max_workers=1) as dw_executor: + dw_future = dw_executor.submit( + _load_dw_async, + storage, bbox, payload['year'], payload['season'] ) - except Exception as e: - print(f"[{job_id}] Failed to load DW baseline: {e}") - # Continue without DW baseline - not critical for inference - dw_arr = None - dw_profile = None - - update_status(job_id, "running", "build_features", 20, "Building feature cube...") - - # ========================================== - # Stage 3: Build Feature Cube - # ========================================== - print(f"[{job_id}] Building feature cube...") - - from feature_computation import FEATURE_ORDER_V1 - - feature_order = FEATURE_ORDER_V1 - expected_features = len(feature_order) # Should be 51 - - print(f"[{job_id}] Expected {expected_features} features (FEATURE_ORDER_V1)") - - # Check if we have an existing feature builder in features.py - feature_cube = None - use_synthetic = False - - try: - from features import build_feature_stack_from_dea - print(f"[{job_id}] Trying build_feature_stack_from_dea for feature extraction...") + # ========================================== + # Start hybrid inference immediately (in parallel) + # ========================================== + update_status(job_id, "running", "load_model", 20, "Loading model artifacts...") - # Try to call it - this requires stackstac and DEA STAC access - try: - feature_cube = build_feature_stack_from_dea( - items=items, - bbox=bbox, - start_date=start_date, - end_date=end_date, - ) - print(f"[{job_id}] Feature cube built successfully: {feature_cube.shape if feature_cube is not None else 'None'}") - except Exception as e: - print(f"[{job_id}] Feature stack building failed: {e}") - print(f"[{job_id}] Falling back to synthetic features for testing") - use_synthetic = True - - except ImportError as e: - print(f"[{job_id}] Feature builder not available: {e}") - print(f"[{job_id}] Using synthetic features for testing") - use_synthetic = True - - # Generate synthetic features for testing when real data isn't available - if feature_cube is None: - print(f"[{job_id}] Generating synthetic features for pipeline test...") + model_dir = Path(tempfile.mkdtemp()) + print(f"[{job_id}] Downloading model artifacts...") - # Determine raster dimensions from DW baseline if loaded - if dw_arr is not None: - H, W = dw_arr.shape - else: - # Default size for testing - H, W = 100, 100 - - # Generate synthetic features: shape (H, W, 51) - - # Use year as seed for reproducible but varied features - np.random.seed(payload['year'] + int(payload.get('lon', 0) * 100) + int(payload.get('lat', 0) * 100)) - - # Generate realistic-looking features (normalized values) - feature_cube = np.random.rand(H, W, expected_features).astype(np.float32) - - # Add some structure - make center pixels different from edges - y, x = np.ogrid[:H, :W] - center_y, center_x = H // 2, W // 2 - dist = np.sqrt((y - center_y)**2 + (x - center_x)**2) - max_dist = np.sqrt(center_y**2 + center_x**2) - - # Add a gradient based on distance from center (simulating field pattern) - for i in range(min(10, expected_features)): - feature_cube[:, :, i] = (1 - dist / max_dist) * 0.5 + feature_cube[:, :, i] * 0.5 - - print(f"[{job_id}] Synthetic feature cube shape: {feature_cube.shape}") - - # ========================================== - # Stage 4: Load Model Artifacts - # ========================================== - update_status(job_id, "running", "load_model", 40, "Loading model artifacts...") - - is_hybrid = "hybrid" in payload['model'].lower() or "spatiotemporal" in payload['model'].lower() - - model_dir = Path(tempfile.mkdtemp()) - - if is_hybrid: - print(f"[{job_id}] Model type: Hybrid Spatio-Temporal. Downloading artifacts...") - # Expected files in MinIO: pipeline_meta.pkl, Temporal_FCN.pth, calibrated_hybrid_cb.pkl + # Download model artifacts for artifact in ["pipeline_meta.pkl", "Temporal_FCN.pth", "calibrated_hybrid_cb.pkl"]: try: storage.download_file(storage.bucket_models, artifact, model_dir / artifact) print(f"[{job_id}] Downloaded {artifact}") except Exception as e: - print(f"[{job_id}] Failed to download {artifact}: {e}") - # Try with 'hybrid/' prefix if direct fails try: storage.download_file(storage.bucket_models, f"hybrid/{artifact}", model_dir / artifact) print(f"[{job_id}] Downloaded {artifact} (from hybrid/ prefix)") except Exception as e2: - raise FileNotFoundError(f"Required artifact {artifact} not found in {storage.bucket_models}: {e2}") + raise FileNotFoundError( + f"Required artifact {artifact} not found in {storage.bucket_models}: {e2}" + ) + + update_status(job_id, "running", "fetch_stac", 30, "Fetching spatio-temporal data...") - # ========================================== - # Stage 5: Fetch Spatio-Temporal Data - # ========================================== - update_status(job_id, "running", "fetch_stac", 50, "Fetching spatio-temporal indices...") from hybrid_inference import DEAfricaSTACWrapper, CropInferencePipeline stac_wrapper = DEAfricaSTACWrapper() - # Calculate ranges for wrapper lat_range = (bbox[1], bbox[3]) lon_range = (bbox[0], bbox[2]) time_range = (start_date, end_date) + print(f"[{job_id}] Fetching STAC data from DEA...") unseen_pixel_df = stac_wrapper.fetch_and_format_data( lat_range=lat_range, lon_range=lon_range, time_range=time_range ) + print(f"[{job_id}] STAC data fetched: {len(unseen_pixel_df)} pixels") + + # Check if DW is ready while processing STAC + if dw_future.done(): + dw_result = dw_future.result() + if dw_result is not None: + dw_arr, dw_profile = dw_result + import rasterio + dw_temp_path = Path(tempfile.mktemp(suffix=".tif")) + with rasterio.open(dw_temp_path, 'w', **dw_profile) as dst: + dst.write(dw_arr) + dw_key = f"baselines/{job_id}/dw_baseline_{payload['year']}_{payload['season']}.tif" + storage.upload_result(dw_temp_path, dw_key) + dw_baseline_url = storage.presign_get("geocrop-baselines", dw_key) + update_status( + job_id, "running", "dw_ready", 35, + "Dynamic World baseline ready", + outputs={"dw_baseline_url": dw_baseline_url}, + ) + + update_status(job_id, "running", "infer", 50, "Running Hybrid Inference (CNN + CatBoost)...") + print(f"[{job_id}] Running hybrid inference...") - # ========================================== - # Stage 6: Hybrid Inference - # ========================================== - update_status(job_id, "running", "infer", 70, "Running Hybrid Inference (CNN + CatBoost)...") pipeline = CropInferencePipeline(model_dir=str(model_dir)) mapped_crops_df = pipeline.predict( @@ -480,17 +380,19 @@ def run_job(payload_dict: dict) -> dict: apply_spatial_smoothing=True, coord_cols=['lat', 'lon'] ) + print(f"[{job_id}] Inference complete, exporting results...") # ========================================== - # Stage 7: Export and Upload + # Export and Upload Results # ========================================== - update_status(job_id, "running", "export_cog", 90, "Exporting results...") + update_status(job_id, "running", "export_cog", 80, "Exporting results...") + output_dir = Path(tempfile.mkdtemp()) output_path = output_dir / "refined.tif" pipeline.export_to_geotiff(mapped_crops_df, output_path=str(output_path)) - output_urls = {} + # Upload results for filename in ["refined.tif", "refined_confidence.tif", "refined_cloud_mask.tif", "refined_legend.json"]: local_f = output_dir / filename if local_f.exists(): @@ -498,44 +400,47 @@ def run_job(payload_dict: dict) -> dict: storage.upload_result(local_f, result_key) output_urls[filename.replace(".","_url")] = storage.presign_get("geocrop-results", result_key) - else: - # Fallback to Legacy/Standard logic - print(f"[{job_id}] Using standard/ensemble inference logic...") - from inference import run_inference_job + # Check DW one more time (may have finished during inference) + if dw_baseline_url is None and dw_future.done(): + dw_result = dw_future.result() + if dw_result is not None: + dw_arr, dw_profile = dw_result + import rasterio + dw_temp_path = Path(tempfile.mktemp(suffix=".tif")) + with rasterio.open(dw_temp_path, 'w', **dw_profile) as dst: + dst.write(dw_arr) + dw_key = f"baselines/{job_id}/dw_baseline_{payload['year']}_{payload['season']}.tif" + storage.upload_result(dw_temp_path, dw_key) + dw_baseline_url = storage.presign_get("geocrop-baselines", dw_key) - # Create a mock job dict compatible with run_inference_job - job_payload = { - "job_id": job_id, - "lat": payload["lat"], - "lon": payload["lon"], - "radius_m": payload["radius_m"], - "year": payload["year"], - "season": payload["season"], - "model": payload["model"], - "smoothing_kernel": payload["smoothing_kernel"] - } - - inference_result = run_inference_job(cfg, job_payload) - output_urls = inference_result.outputs - - # Note: indices and true_color not yet implemented + # Wait for DW if still running + if dw_baseline_url is None: + print(f"[{job_id}] Waiting for DW baseline to finish...") + dw_result = dw_future.result(timeout=60) + if dw_result is not None: + dw_arr, dw_profile = dw_result + import rasterio + dw_temp_path = Path(tempfile.mktemp(suffix=".tif")) + with rasterio.open(dw_temp_path, 'w', **dw_profile) as dst: + dst.write(dw_arr) + dw_key = f"baselines/{job_id}/dw_baseline_{payload['year']}_{payload['season']}.tif" + storage.upload_result(dw_temp_path, dw_key) + dw_baseline_url = storage.presign_get("geocrop-baselines", dw_key) + + # ========================================== + # Final Status + # ========================================== + final_outputs = dict(output_urls) + if dw_baseline_url: + final_outputs["dw_baseline_url"] = dw_baseline_url + if payload['outputs'].get('indices'): missing_outputs.append("indices: not implemented") if payload['outputs'].get('true_color'): missing_outputs.append("true_color: not implemented") - # ========================================== - # Stage 8: Final Status - # ========================================== final_status = "partial" if missing_outputs else "done" - final_message = f"Inference complete" - if missing_outputs: - final_message += f" (partial: {', '.join(missing_outputs)})" - - # Include DW baseline URL in final outputs if available - final_outputs = dict(output_urls) - if dw_baseline_url: - final_outputs["dw_baseline_url"] = dw_baseline_url + final_message = f"Inference complete" + (f" ({', '.join(missing_outputs)})" if missing_outputs else "") update_status( job_id, @@ -556,7 +461,6 @@ def run_job(payload_dict: dict) -> dict: } except Exception as e: - # Catch-all for any unexpected errors error_trace = traceback.format_exc() print(f"[{job_id}] Error: {e}") print(error_trace) @@ -573,9 +477,10 @@ def run_job(payload_dict: dict) -> dict: "job_id": job_id, } -# Alias for API + run_inference = run_job + # ========================================== # RQ Worker Entry Point # ========================================== @@ -585,7 +490,6 @@ def start_rq_worker(): from rq import Worker import signal - # Ensure /app is in sys.path so we can import modules if '/app' not in sys.path: sys.path.insert(0, '/app') @@ -594,9 +498,7 @@ def start_rq_worker(): print(f"=== GeoCrop RQ Worker Starting ===") print(f"Listening on queue: {queue_name}") print(f"Redis: {os.getenv('REDIS_HOST', 'redis.geocrop.svc.cluster.local')}:{os.getenv('REDIS_PORT', '6379')}") - print(f"Python path: {sys.path[:3]}") - # Handle graceful shutdown def signal_handler(signum, frame): print("\nReceived shutdown signal, exiting gracefully...") sys.exit(0) @@ -624,22 +526,17 @@ if __name__ == "__main__": args = parser.parse_args() if args.test or not args.worker: - # Syntax-level self-test print("=== GeoCrop Worker Syntax Test ===") - # Test imports try: from contracts import STAGES, VALID_MODELS from storage import MinIOStorage - from feature_computation import FEATURE_ORDER_V1 print(f"โœ“ Imports OK") print(f" STAGES: {STAGES}") print(f" VALID_MODELS: {VALID_MODELS}") - print(f" FEATURE_ORDER length: {len(FEATURE_ORDER_V1)}") except ImportError as e: - print(f"โš  Some imports missing (expected outside container): {e}") + print(f"โš  Some imports missing: {e}") - # Test payload parsing print("\n--- Payload Parsing Test ---") test_payload = { "job_id": "test-123", @@ -647,7 +544,7 @@ if __name__ == "__main__": "lon": 31.0, "radius_m": 2000, "year": 2022, - "model": "Ensemble", + "model": "Hybrid_SpatioTemporal", "smoothing_kernel": 5, "outputs": {"refined": True, "dw_baseline": True}, } @@ -660,18 +557,8 @@ if __name__ == "__main__": print(f" job_id: {validated['job_id']}") print(f" AOI: ({validated['lat']}, {validated['lon']}) radius={validated['radius_m']}m") print(f" model: {validated['model']}") - print(f" kernel: {validated['smoothing_kernel']}") - - # Show what would run - print("\n--- Pipeline Overview ---") - print("Pipeline stages:") - for i, stage in enumerate(STAGES): - print(f" {i+1}. {stage}") - - print("\nNote: This is a syntax-level test.") - print("Full execution requires Redis, MinIO, and STAC access in the container.") print("\n=== Worker Syntax Test Complete ===") if args.worker: - start_rq_worker() + start_rq_worker() \ No newline at end of file