
    =8j                     x   d dl Z d dlZd dlZd dlZd dlmZmZ d dlmZm	Z	 dZ
ddiZdZdZd	Zd
Zd Zd Zdedee         fdZdedefdZdee         fdZedk    r ej                    Z e            Z ed           d eD             Z ed            e ej        ed          d                      eedd          5 Z ej        eedd           ddd           n# 1 swxY w Y    ed ee           de             ej                    Z  ed  e!e ez
  d           d!           dS dS )"    N)ListDict)ThreadPoolExecutoras_completedz7https://mygreenqube.com:4003/landRoute/plotListWithDocsinformationpaddy_carbon_pr_531582   
      zformatted_output.jsonc                     | sg S t          | t                    rd |                     d          D             S t          |                                           gS )Nc                 ^    g | ]*}|                                 |                                 +S  )strip).0xs     -/var/www/html/banglarbhumi/fetch_land_data.py
<listcomp>zsafe_split.<locals>.<listcomp>!   s-    AAAaqwwyyA		AAA    ,)
isinstancestrsplitr   )values    r   
safe_splitr      s]     	% BAA5;;s#3#3AAAAJJr   c                 X   g }|                      d          }t          |t                    r]|                     d          }|rF|                                }|r0|                                dk    r|                    |dd           |                      d          }t          |t                    r|D ]}t          |t                    s|                     d          }|rt          |t                    sE|                     d          }|s]|                                }|r|                                dk    r|                    |dd           t                      }g }|D ]<}	|	d	         |vr0|                    |	d	                    |                    |	           =|S )
N	landUsersuserNamez	N.A. N.A.T)nameis_main_userlandCofarmerscoFInfoFr   )	getr   dictr   upperappendlistsetadd)
row
co_farmers	land_userr   cof_listcocof_infoseenunique_co_farmersfarmers
             r   extract_cofarmersr2   (   s   J $$I)T"" }}Z(( 	::<<D 

33"" $($ $   
 ww''H(D!!  	 	Bb$'' vvi((H  :h#=#= <<
++D ::<<D  4::<<;66 $)      55D - -&>%%HHVF^$$$$$V,,,r   page_noreturnc           
      2   | t           dd}t          dt          dz             D ]}	 t          j        t
          t          |d          }|                                 |                                }|                    di                               dg           }t          d|  dt          |           d	           |c S # t          $ r6}t          d
| d|  d|            t          j        d           Y d }~d }~ww xY wt          d|             g S )N   pageNo	pageLimitorderBy   headersparamstimeoutdatarowsu	   ✅ Page z
 fetched (z rows)z[Retry z] Page z error: u   ❌ Failed page )
PAGE_LIMITrangeMAX_RETRIESrequestsr"   BASE_URLHEADERSraise_for_statusjsonprintlen	Exceptiontimesleep)r3   r>   attemptresponser@   rA   es          r   
fetch_pagerR   d   sB    F K!O,,  	|HgfVXYYYH%%'''==??D88FB''++FB77DBgBBTBBBCCCKKK 	 	 	@G@@G@@Q@@AAAJqMMMMMMMM	 

&W
&
&'''Is   BC
D,C==Dr)   c                    t          |                     d                    }t          |                     d                    }|r|s;t          d|                     d           d|                     d                      d |                     dg           D             }t          |           }|                     d          |                     d          ||||d	S )
Njl_nodaag_nou   ⚠️ Missing JL/Daag → jl: z	 | daag: c           	         g | ]~}|                     d           |                     d           |                     d          |                     d          |                     d          |                     d          dS )	file_pathfile_idterm_id	term_type
file_title)pdf_urlrX   rY   rZ   r[   )r"   )r   imgs     r   r   z!transform_row.<locals>.<listcomp>   s     
 
 
 	
ww{
77;''779%%779%%WW[))ggl++ 
 
 
r   plot_imgrel_idland_convert_size)r_   r`   JL_NoDaag_Nopdf_urlsz
co-farmers)r   r"   rJ   r2   )r)   jl_list	daag_listrc   r*   s        r   transform_rowrf      s     ))**G 3779--..I  a) a_0@0@__377S\K]K]__```
 
 wwz2&&
 
 
H #3''J ''(## WW%899   r   c            	         g } t          d           t          j        t          t          dt
          ddd          }|                                 |                                }|                    di           }|                    dd          }|dk    rt          d	           g S t          j	        |t
          z            }t          d
|            t          d|            t          dt           d           |                     |                    dg                      t          t                    5 fdt          d|dz             D             }t          |          D ]b}||         }	 |                                }	|	r|                     |	           7# t           $ r}
t          d| d|
            Y d }
~
[d }
~
ww xY w	 d d d            n# 1 swxY w Y   | S )NzFetching first page...r6   r7   r;   r<   r@   countr   zNo data found.zTotal records: zTotal pages: zUsing z threads...
rA   )max_workersc                 H    i | ]}                     t          |          |S r   )submitrR   )r   r3   executors     r   
<dictcomp>z+fetch_all_data_parallel.<locals>.<dictcomp>   s9     
 
 
 OOJ00'
 
 
r      u   ❌ Error on page z: )rJ   rE   r"   rF   rG   rB   rH   rI   mathceilMAX_WORKERSextendr   rC   r   resultrL   )all_rowsrP   
first_jsonr@   rh   total_pagesfuturesfuturer3   rA   rQ   rl   s              @r   fetch_all_data_parallelry      sp   H	
"###|'    H J>>&"%%DHHWa  Ezz	)EJ.//K	
#E
#
#$$$	
'+
'
'(((	
-;
-
-
-... OODHHVR(())) 
	4	4	4 ;
 
 
 
 K!O44
 
 

 #7++ 	; 	;FfoG;}} *OOD))) ; ; ;9799a99::::::::;	;; ; ; ; ; ; ; ; ; ; ; ; ; ; ; Os<   ?9G9+F%$G%
G/G	G	GGG"%G"__main__z
Transforming data...c                 ,    g | ]}t          |          S r   )rf   )r   r)   s     r   r   r      s     ???sc**???r   z
Sample Output:rn   )indentwzutf-8)encodingF)r|   ensure_asciiu   
💾 Saved z records to u   
⏱ Execution time: z seconds)"rE   ro   rM   rI   typingr   r   concurrent.futuresr   r   rF   rG   rB   rq   rD   OUTPUT_FILEr   r2   intrR   rf   ry   __name__
start_timeraw_datarJ   transformed_datadumpsopenfdumprK   end_timeroundr   r   r   <module>r      s               ? ? ? ? ? ? ? ? E * 
%     6 6 6x T
    :)t ) ) ) ) )^1d 1 1 1 1n zJ '&((H	E
"### @?h??? 
E
	E*$*%a(
3
3
3444 
k3	1	1	1 EQ	"AaeDDDDE E E E E E E E E E E E E E E 
E
J##.//
J
J[
J
JKKKty{{H	E
L55J)>#B#B
L
L
LMMMMM- s   C((C,/C,