
    2j1                        d Z ddlZddlZddlZddlZddlZddlZddlm	Z	 ddl
mZ ddlmZ ddlmZ ddlmZ ddlmZ d	Zd
ZdZ ed            G d de          Z e            Z	  eddd          Zn	#  dZY nxY wdZej                            e          r ee          Ze                     dd          Z!d Z"d Z#d Z$d Z%d Z&d Z'd)dZ(d Z)d Z*d Z+d  Z,d! Z-e.d"k    rnddl/Z/ e0e/j1                  d#k     r  ed$            ed%            e2             e/j1        d&         Z3 e-e3          Z4 e ej5        e4d'd#(                     dS dS )*u  
dl_pipeline.py

Hybrid Deep Learning Extraction Pipeline
CPU-Friendly Version

Architecture:
PDF
 ↓
DL Extraction Pipeline
 ↓
Confidence Scoring
 ↓
If score < 70:
    invoke existing DLM engine
 ↓
Merge Results
 ↓
Final JSON

Requirements:
pip install paddleocr paddlepaddle ultralytics transformers torch torchvision pdf2image pillow opencv-python

Optional:
sudo apt install poppler-utils
    N)Path)convert_from_path)	PaddleOCR)YOLO)pipeline)extract_land_recordF   cpuzmodels/layout_detector.ptzLoading PaddleOCR...c                        e Zd Z fdZ xZS )SafePaddleOCRc                 x    d| _         d| _        t                                          dd           d| _        d S )NzPP-OCRv3bnen)langocr_version)r   r   super__init__rec_lang)self	__class__s    )/var/www/html/banglarbhumi/dl_pipeline.pyr   zSafePaddleOCR.__init__B   s<    %	d
;;;     )__name__
__module____qualname__r   __classcell__)r   s   @r   r   r   A   s8                r   r   ztoken-classificationzai4bharat/IndicNER)modeldeviceu   ০১২৩৪৫৬৭৮৯
0123456789c                 >    | sdS |                      t                    S )N )	translateBN_TO_ENtexts    r   bn_to_enr'   k   s!     r>>(###r   c                 b    | sdS |                                  } t          j        dd|           } | S )Nr"   z\s+ )stripresubr%   s    r   normalize_textr-   q   s4     r::<<D6&#t$$DKr   c                     t          | d          }g }t          |          D ]P\  }}t          j        |          }t	          j        |t          j                  }|                    ||d           Q|S )N   )dpi)pageimage)r   	enumeratenparraycv2cvtColorCOLOR_RGB2BGRappend)pdf_pathpagesimagesir1   imgs         r   pdf_to_imagesr?      s    hC000EFU##  4htnnl3 122
 
 	 	 	 	
 Mr   c                    g }| D ]}|d         }|d         }t                               |          }|s/|d         D ]}|d         }t          |d         d                   }t          |d         d                   }	|sCd |D             }
d |D             }|                    ||	t          |
          t          |          t          |
          t          |          |d           |S )Nr1   r2   r      c                     g | ]
}|d          S )r    .0ps     r   
<listcomp>zrun_ocr.<locals>.<listcomp>       %%%1!A$%%%r   c                     g | ]
}|d          S )rA   rC   rD   s     r   rG   zrun_ocr.<locals>.<listcomp>   rH   r   )r&   confxminyminxmaxymaxr1   )
ocr_engineocrr-   floatr9   minmax)r<   	all_words	page_datapage_nor2   resultlinebboxr&   rJ   xsyss               r   run_ocrr\      s   I  	F#'"&& 	1I 	 	D7D!$q'!*--Da$$D %%%%%B%%%%%BBBBB     	, r   c           
      `   g }t           |S | D ]}|d         }|d         }t          |          }|D ]{}|j        }|D ]o}t          |j        d                   }	t	          |j        d                   }
|j        d                                         }|                    ||	|
|d           p||S )Nr2   r1   r   )r1   class
confidencerY   )	layout_modelboxesintclsrQ   rJ   xyxytolistr9   )r<   layout_resultsrU   r2   r1   resultsrra   boxrc   rJ   rd   s               r   detect_layoutrj      s    N  	'" u%% 	 	AGE  #'!*ooSXa[))x{))++%%  "& 	' '    		$ r   c                    d                     d | D                       }ddddddd}t          j        d|          }|r%t          |                    d                    |d<   t          j        d|          }|r%t          |                    d                    |d	<   t          j        d
|          }|r|                    d          |d<   t          j        d|          }|r|                    d          |d<   t          j        d|          }|r|                    d          |d<   t          j        d|          }|r%t          |                    d                    |d<   |S )Nr)   c                     g | ]
}|d          S r%   rC   rE   ws     r   rG   z%extract_header_dl.<locals>.<listcomp>   s    333!F)333r   r"   jl_nodaag_nomouzablockdistricttotal_land_acreu)   জে\.?এল\s*নং\s*([\d০-৯]+)rA   rp   u   \b([\d০-৯/]{2,6})\brq   u   মৌজা[:ঃ]?\s*(\S+)rr   u#   ব্লক[:ঃ]?\s*([A-Z0-9\-]+)rs   u   জেলা[:ঃ]?\s*([A-Z]+)rt   u   ([\d০-৯]+\.[\d০-৯]+)ru   )joinr+   searchr'   group)words	full_textrW   ms       r   extract_header_dlr|      s}   33U33344I  F 		>	JJA /"1771::..w 		,i88A 1$QWWQZZ00y 		0)<<A %''!**w 		8)DDA %''!**w 		3Y??A (WWQZZz 		19==A 9$,QWWQZZ$8$8 !Mr      c                 `   | sg S t          | d           } g }| d         g}| dd          D ]l}|d         }|d         |d         k    r8t          |d         |d         z
            |k     r|                    |           T|                    |           |g}m|                    |           |S )Nc                 "    | d         | d         fS )Nr1   rL   rC   )xs    r   <lambda>zgroup_rows.<locals>.<lambda>#  s    6AfI(> r   )keyr   rA   r1   rL   )sortedabsr9   )ry   y_thresholdrowscurrentrn   prevs         r   
group_rowsr     s     	5>>???EDQxjG122Y  qz fIf%%AfIV,--;;NN1 KK   cGGKKKr   c           
         t          |           }g }|D ]}d                    d |D                       }d}t          j        dt	          |                    }|r|                    d          }t          j        dt	          |                    }|s|sd}d}	t          r	 t          |          }
d |
D             }t          |          dk    rd                    |d d                   nd}t          |          d	k    rd                    |dd	                   nd}	n#  Y nxY w|sJ|	                                }t          |          dk    r|d
         }t          |          dk    r|d         }	|
                    |||	t          |          d
k    r|d         ndt          |          dk    r|d
         ndd           |S )Nr)   c                     g | ]
}|d          S r%   rC   rE   r   s     r   rG   z&extract_entries_dl.<locals>.<listcomp>G  s    4441QvY444r   r"   z\b\d[\d/]*\br   z\d+\.\d+c                     g | ]
}|d          S )wordrC   r   s     r   rG   z&extract_entries_dl.<locals>.<listcomp>`  s    888!F)888r         rA      
khatian_no
owner_namefather_husband_nameansha
area_acres)r   rv   r+   rw   r'   rx   findallner_pipelinelensplitr9   )ry   r   entriesrowrow_textkhatianr{   decimalsownerfather
ner_resulttokenstextss                r   extract_entries_dlr   ?  s   eDG 3 3884444455Iox'9'9:: 	!ggajjG:k8H+=+=>> 	 	 		)(33
88Z88803Fq0@0@,,,b25f++2B2B&1+... 	"NN$$E5zzQa5zzQq!#)$'MMQ$6$6Xa[[B),X!););(1++
 
 	 	 	 	 Ns   %A?D%%D)c                 d    d}t                               dg                     }|dk    r|dz  }g d}t           fd|D                       }||dz  z  }|r5t          j        d |D                       }|d	k     r|d
z  }n|dk     r|dz  }                     dg           }|r6t          d |D                       }|t          |          z  }	|	dk    r|d
z  }d}
|D ]4}|                    dd          }|rt          j        d|          s|
dz  }
5|
dk    r|dz  }t          |d          S )Nd   khatian_entriesr   (   )rp   rq   rr   rs   rt   c              3   F   K   | ]}                     |          d V  dS )rA   Nget)rE   hrW   s     r   	<genexpr>z(compute_dl_confidence.<locals>.<genexpr>  sG        zz!}}	     r      c                     g | ]
}|d          S )rJ   rC   rm   s     r   rG   z)compute_dl_confidence.<locals>.<listcomp>  s    4441V9444r   g333333?   g      ?
   c              3   D   K   | ]}|                     d           dV  dS )r   rA   Nr   )rE   es     r   r   z(compute_dl_confidence.<locals>.<genexpr>  sH       
 
55&&

 
 
 
 
 
r   g      ?r   r"   z
^\d+\.\d+$rA   )r   r   sumr4   meanr+   matchrS   )rW   ry   scoretotal_entriesimportant_headersmissing_headersavg_ocrr   missing_ownerratio	malformedr   areas   `            r   compute_dl_confidencer     s   E

#4b99::M        $    O
 
_q  E  '44e44455S==RKEEt^^RKE jj*B//G 
 
 

 
 
 
 

 G,3;;RKE I  uu\2&& 	55 	NI1}}ua==r   c                    |                                  }dD ]/}|                    |          s|                    |          ||<   0|                    dg           }|                    dg           }t          |          dk    r||d<   ng }t          t          |          t          |                    }t	          |          D ]}|t          |          k     r||         ni }	|t          |          k     r||         ni }
|	                                 }dD ]/}|                    |          s|
                    |          ||<   0|                    |           ||d<   d|d<   |S )Nro   r   r   r   Tfallback_used)copyr   r   rS   ranger9   )	dl_result
dlm_resultfinalr   
dl_entriesdlm_entriesmerged_entriesmax_lenr=   dl_edlm_emergedfields                r   merge_resultsr     s   NNE 
- 
- yy~~ 	-#,,E#J ,b11J..!2B77K
:!#.   c*oos;'7'788w 	* 	*A$%J$7$7:a==RD&'#k*:*:&:&:KNNEYY[[F 	5 	5 zz%(( 5$)IIe$4$4F5M!!&))))#1 !E/Lr   c                 (   t          |           }t          |          }t          |          }t          |          }t	          |          }t          |           j        |                    dd          |                    dd          |                    dd          |                    dd          |                    dd          |                    dd          t          |          |t          |          d
}t          ||          }||d	<   |S )
Nrp   r"   rq   rr   rs   rt   ru   )
source_filerp   rq   rr   rs   rt   ru   r   r   layout_objectsdl_confidence)
r?   r\   rj   r|   r   r   namer   r   r   )r:   r<   ry   layoutheaderr   rW   r   s           r   extract_land_record_dlr     s    8$$FFOOE6""Fu%%F ''G H~~*GR((::i,,GR((GR((JJz2..!::&7<<W"f++ F "&%00E#F?Mr   c                     t          d           t          |           }|d         }t          d|            |t          k    rd|d<   |S t          d           t          |           }t	          ||          }d|d<   |S )Nz
Running DL pipeline...r   zDL Confidence: DL_ONLYr   z*DL confidence low. Running DLM fallback...DL_PLUS_DLM)printr   DL_THRESHOLDextract_land_record_dlmr   )r:   r   r   r   r   s        r   process_land_recordr   *  s    	
$%%%&x00Io&E	
#E
#
#$$$  )	* 

6777(22J9j11F&F:Mr   __main__r   zUsage:zpython dl_pipeline.py file.pdfrA   F)ensure_asciiindent)r}   )6__doc__osr+   jsonr6   torchnumpyr4   pathlibr   	pdf2imager   	paddleocrr   ultralyticsr   transformersr   extract_landr   r   r   DEVICEYOLO_MODEL_PATHr   r   rO   r   r`   pathexistsstr	maketransr$   r'   r-   r?   r\   rj   r|   r   r   r   r   r   r   r   sysr   argvexitr:   rW   dumpsrC   r   r   <module>r      s   6 
			 				  



            ' ' ' ' ' '             ! ! ! ! ! ! H G G G G G 	-        I    ]__
8"  LL
LLL 7>>/"" )4((L ==9<HH$ $ $    *$ $ $V     N1 1 1p   B; ; ;DA A AP6 6 6z  H  B zJJJ
s38}}qh.///x{H  **F	E*$*V%
:
:
:;;;;; s   )A8 8A>