
    S9iB                     X   S SK r S SKrS SKrS SKrS SKJrJr  S SKJrJ	r	  Sr
SS0rSrSrS	rS
rS rS rS\S\\   4S jrS\S\4S jrS\\   4S jr\S:X  a  \R                  " 5       r\" 5       r\" S5        \ V s/ s H  n \" U 5      PM     sn r\" S5        \" \R6                  " \S    SS95        \" \SSS9 r\R<                  " \\SSS9  SSS5        \" S\" \5       S\ 35        \R                  " 5       r \" S\!" \ \-
  S5       S 35        ggs  sn f ! , (       d  f       NP= f)!    N)ListDict)ThreadPoolExecutoras_completedz7https://mygreenqube.com:4003/landRoute/plotListWithDocsinformationpaddy_carbon_pr_531582   
      zformatted_output.jsonc                    U (       d  / $ [        U [        5      (       aF  U R                  S5       Vs/ s H)  oR                  5       (       d  M  UR                  5       PM+     sn$ [        U 5      R                  5       /$ s  snf )N,)
isinstancestrsplitstrip)valuexs     -/var/www/html/banglarbhumi/fetch_land_data.py
safe_splitr      sa    	%#(;;s#3A#3awwy		#3AAJ Bs   A?A?c                    / nU R                  S5      n[        U[        5      (       aT  UR                  S5      nU(       a<  UR                  5       nU(       a%  UR	                  5       S:w  a  UR                  U5        U R                  S5      n[        U[        5      (       a  U H  n[        U[        5      (       d  M  UR                  S5      nU(       a  [        U[        5      (       d  MI  UR                  S5      nU(       d  Mc  UR                  5       nU(       a  UR	                  5       S:X  a  M  UR                  U5        M     [        [        R                  U5      5      $ )N	landUsersuserNamez	N.A. N.A.landCofarmerscoFInfo)getr   dictr   upperappendlistfromkeys)row
co_farmers	land_usernamecof_listcocof_infos          r   extract_cofarmersr(   (   s   J $I)T""}}Z(::<D

3!!$' ww'H(D!!Bb$''vvi(H :h#=#=<<
+D::<D 4::<;6d#- 2 j)**    page_noreturnc           
         U [         SS.n[        S[        S-   5       H  n [        R                  " [
        [        USS9nUR                  5         UR                  5       nUR	                  S0 5      R	                  S/ 5      n[        SU  S[        U5       S	35        Us  $    [        SU  35        / $ ! [         a5  n[        S
U SU  SU 35        [        R                  " S5         S nAM  S nAff = f)N   pageNo	pageLimitorderBy   headersparamstimeoutdatarowsu	   ✅ Page z
 fetched (z rows)z[Retry z] Page z error: u   ❌ Failed page )
PAGE_LIMITrangeMAX_RETRIESrequestsr   BASE_URLHEADERSraise_for_statusjsonprintlen	Exceptiontimesleep)r*   r5   attemptresponser7   r8   es          r   
fetch_pagerI   W   s    F K!O,	||HgfVXYH%%'==?D88FB'++FB7DIgYjT6BCK - 
WI
&'I  	GG9GG9HQC@AJJqMM	s   A=B44
C3>*C..C3r!   c                    [        U R                  S5      5      n[        U R                  S5      5      nU(       a  U(       d/  [        SU R                  S5       SU R                  S5       35        U R                  S/ 5       Vs/ s H,  nUR                  S5      (       d  M  UR                  S5      PM.     nn[        U 5      nU R                  S5      U R                  S5      UUUUS	.$ s  snf )
Njl_nodaag_nou   ⚠️ Missing JL/Daag → jl: z	 | daag: plot_img	file_pathrel_idland_convert_size)rO   rP   JL_NoDaag_Nopdf_urlsz
co-farmers)r   r   rA   r(   )r!   jl_list	daag_listimgrS   r"   s         r   transform_rowrW   t   s     )*G 3779-.I )/0@/A377S\K]J^_`
 77:r**C77; 	*   #3'J ''(# WW%89  s   C-$C-c            	         / n [        S5        [        R                  " [        [        S[
        SS.SS9nUR                  5         UR                  5       nUR                  S0 5      nUR                  SS5      nUS:X  a  [        S	5        / $ [        R                  " U[
        -  5      n[        S
U 35        [        SU 35        [        S[         S35        U R                  UR                  S/ 5      5        [        [        S9 n[        SUS-   5       Vs0 s H  nUR                  [        U5      U_M     nn[!        U5       H2  n	X   n U	R#                  5       n
U
(       a  U R                  U
5        M2  M4     S S S 5        U $ s  snf ! [$         a  n[        SU SU 35         S nAMf  S nAff = f! , (       d  f       U $ = f)NzFetching first page...r-   r.   r2   r3   r7   countr   zNo data found.zTotal records: zTotal pages: zUsing z threads...
r8   )max_workers   u   ❌ Error on page z: )rA   r<   r   r=   r>   r9   r?   r@   mathceilMAX_WORKERSextendr   r:   submitrI   r   resultrC   )all_rowsrG   
first_jsonr7   rY   total_pagesexecutorr*   futuresfuturer8   rH   s               r   fetch_all_data_parallelrh      s   H	
"#||' H J>>&"%DHHWa Ez	))EJ./K	OE7
#$	M+
'(	F;-}
-. OODHHVR() 
	4 !K!O4
4 OOJ0'94 	 

 #7+FoG;}}OOD) 	 , 
5 O
  ;*7)2aS9::; 
5	4 OsH   F4!F9F4(F6F4F4
F1F,&F4,F11F44
G__main__z
Transforming data...z
Sample Output:r[   )indentwzutf-8)encodingF)rj   ensure_asciiu   
💾 Saved z records to u   
⏱ Execution time: z seconds)"r<   r\   rD   r@   typingr   r   concurrent.futuresr   r   r=   r>   r9   r^   r;   OUTPUT_FILEr   r(   intrI   rW   rh   __name__
start_timeraw_datarA   transformed_datadumpsopenfdumprB   end_timeround)r!   s   0r   <module>r|      sw        ? E * 
% )+^ T
 :t  F1d 1n zJ '(H	
"# 7??hsc*h? 

	$**%a(
34 
k3	1Q		"AaeD 
2 
M#./0[M
JKyy{H	"5J)>#B"C8
LM-  @ 
2	1s   >D=D
D)