
    {-j70                         d dl Z d dlmZ d dlZddlmZ ddlmZm	Z	 ddl
mZmZ  e	d          rd dlZ ed           G d	 d
                      Z G d d          Z ed           G d d                      ZdS )    N)Path   )logging)class_requires_depsis_dep_available   )
YAMLReader
YAMLWriterz	faiss-cpuc                       e Zd ZdZdZdZdZd Zed             Z	ed             Z
ed             Zed	             Zed
             Zed             Zed             Zd Zd Zed             Zd Zed             ZdS )	IndexDatavectorz.indexid_mapz.yamlc                 p    || _         || _        |d         | _        |d         | _        |d         | _        d S )Nr   metric_type
index_type)_index_index_info_id_map_metric_type_index_type)selfindex
index_infos      n/var/www/html/banglarbhumi/venv/lib/python3.11/site-packages/paddlex/inference/pipelines/components/faisser.py__init__zIndexData.__init__#   s=    %!(+&}5%l3    c                     | j         S N)r   r   s    r   r   zIndexData.index*   s
    {r   c                 4    t          j        | j                  S r   )faissserialize_indexr   r   s    r   index_byteszIndexData.index_bytes.   s    $T[111r   c                     | j         S r   )r   r   s    r   r   zIndexData.id_map2   s
    |r   c                     | j         S r   )r   r   s    r   r   zIndexData.metric_type6   s      r   c                     | j         S r   )r   r   s    r   r   zIndexData.index_type:   s    r   c                 R    | j         | j        |                     | j                  dS )N)r   r   r   )r   r   _convert_intr   r   s    r   r   zIndexData.index_info>   s1     /+''44
 
 	
r   c                     t          j        |          }t          j        |d                   } | ||d                   S )Nr      )pickleloadsr!   deserialize_index)clsbytestupr   s       r   
from_byteszIndexData.from_bytesF   s;    l5!!'A//s5#a&!!!r   c                 j    t          j        | j                  | j        f}t	          j        |          S r   )r!   r"   r   r   r+   dumps)r   r0   s     r   to_byteszIndexData.to_bytesL   s+    $T[114?C|C   r   c                 >    d |                                 D             S )Nc                 N    i | ]"\  }}t          |          t          |          #S  )intstr.0kvs      r   
<dictcomp>z*IndexData._convert_int.<locals>.<dictcomp>Q   s*    :::41aAA:::r   items)r   r   s     r   r(   zIndexData._convert_intP   s    ::6<<>>::::r   c                 >    d |                                  D             S )Nc                 X    i | ]'\  }}t          j        |          t          |          (S r7   )npint64r9   r:   s      r   r>   z,IndexData._convert_int64.<locals>.<dictcomp>U   s,    ???1SVV???r   r?   )r   s    r   _convert_int64zIndexData._convert_int64S   s    ??????r   c                    t          |          }|                    dd           || j         | j         z                                  }|| j         | j         z                                  }| j        t          j	        v rt          j        | j        |           nt          j        | j        |           t                      }|                    || j        dd           d S )NT)parentsexist_okF)default_flow_styleallow_unicode)r   mkdir	VECTOR_FNVECTOR_SUFFIXas_posixIDMAP_FNIDMAP_SUFFIXr   FaissBuilderBINARY_METRIC_TYPEr!   write_index_binaryr   write_indexr
   writer   )r   save_dirvector_pathindex_info_pathyaml_writers        r   savezIndexData.saveW   s    >>td333T^"IT5G"I"IISSUU#&K8I&K&KKUUWW|>>>$TZ====dj+666 llO$	 	 	
 	
 	
 	
 	
r   c                    t          |t                    rt          |          }|| j         | j         z  }|| j         | j         z  }|                                sJ d| j         | j         d| d            |                                sJ d| j         | j         d| d            t                      }|	                    |          }d|v rd|v rd|v sJ d| d            t                              |d                   }|d         t          j        v r't          j        |                                          }n&t          j        |                                          }|j        t'          |          k    s
J d	            |||d         |d         fS t          |t                    sJ |j        |j        |j        |j        fS )
NzNot found the z	 file in !r   r   r   zThe index_info file(z]) may have been damaged, `id_map` or `metric_type` or `index_type` not found in `index_info`.z.data number in index is not equal in in id_map)
isinstancer9   r   rL   rM   rO   rP   existsr	   readr   rE   rQ   rR   r!   read_index_binaryrN   
read_indexntotallenr   r   r   r   )r.   r   
index_rootrW   rX   yaml_readerr   r   s           r   loadzIndexData.loadj   s;   eS!!  	ReJ$#-'L9J'L'LLK(cl+NC<L+N+NNO ""$$T TSSs/@SS5SSST T$  &&((R RQQc.>QQQQQR R( %,,K$))/::JJ&&!Z// J... Uo  U  U  U /./ --j.BCCF-(L,KKK/0D0D0F0FGG()=)=)?)?@@<3$ $   ?   &*]";Z=UUUeY/////;e.?AQQQr   N)__name__
__module____qualname__rL   rM   rO   rP   r   propertyr   r#   r   r   r   r   classmethodr1   r4   r(   staticmethodrE   rZ   rf   r7   r   r   r   r      s_       IMHL4 4 4   X 2 2 X2   X ! ! X!     X  
 
 X
 " " ["
! ! !; ; ; @ @ \@
 
 
& !R !R [!R !R !Rr   r   c                   $     e Zd Z fdZd Z xZS )FaissIndexerc                     t                                                       t                              |          \  | _        | _        | _        }d S r   )superr   r   rf   _indexerr   r   )r   r   r   	__class__s      r   r   zFaissIndexer.__init__   sA     	CL>>RWCXCX@t{D$4jjjr   c                 z   | j                             t          j        |          |          \  }}g }t	          ||          D ]\  }}	|                    g g d           t	          ||	          D ]X\  }
}|dk    rM|d         d                             |
           |d         d                             | j        |                    Y| j        t          j	        v r)t          j
        |d d df         |k              d         }n(t          j
        |d d df         |k               d         }|D ]
}d d d||<   |S )N)scorelabelr   rt   ru   )rq   searchrC   arrayzipappendr   r   rQ   rR   where)r   featurescore_threshamming_radiustopkscores_listids_listpredsscoresidsrt   ididxsidxs                 r   __call__zFaissIndexer.__call__   sX    $ 4 4RXg5F5F M MX{H55 	? 	?KFCLL233444 -- ? ?	r77"Ig&--e444"Ig&--dk"o>>>?
 |>>>8K1->??BDD8K1-;<<Q?D 	8 	8C#'$77E#JJr   )rg   rh   ri   r   r   __classcell__)rr   s   @r   rn   rn      sL        Y Y Y Y Y      r   rn   c                       e Zd ZdZdZdZdZedd            Zed             Z	e	 	 dd
            Z
ed             Zed             Zed             Zedd            ZdS )rQ   )hammingIPL2)FlatIVFHNSW32)r   )r   r   
BinaryHashNc           	      <   |dk    rD|t          t          t          |dz            d                    z   }|| j        v r|dz  }n|dz  }|| j        v r$|| j        v sJ d| d| j         d            d	|z   }|d
k    rt          j        d           d
}|dk    rd}|S )Nr      i   z,BFlatz,FlatzThe metric type(z) only support z index types!Br   z5The HNSW32 method dose not support 'remove' operationr   )r9   minr8   rR   BINARY_SUPPORT_INDEX_TYPEr   warning)r.   r   r   nums       r   _get_index_typezFaissBuilder._get_index_type   s     #c#c#(mmU*C*C&D&DDJc444h&

g%
 #000c;;;;j+jjc>[jjj <;;z)J!!OSTTT!JJr   c                     |dk    rt           j        S |dk    rt           j        S |dk    rt           j        S |dk    rt           j        S d S )Nr   jaccardr   r   )r!   METRIC_HammingMETRIC_JaccardMETRIC_INNER_PRODUCT	METRIC_L2)r.   r   s     r   _get_metric_typezFaissBuilder._get_metric_type   sY    )##''I%%''D  --D  ?" ! r   r   r   c                 B   || j         v sJ d| j          d            || j        v sJ d| j         d            t          |t                    r|                     ||          \  }}n||}}d  ||          D             }|| j        v rt          j        nt          j        }	t          j	        |          
                    |	          }|j        \  }
}|| j        v r?t          j        ||                     |||
          |                     |                    }nRt          j        ||                     |||
          |                     |                    }t          j        |          }i }|                     |||||d          \  }}t'          ||||d          S )NzSupported index types only: r\   Supported metric types only: c                     g | ]
}|d          S r|   r7   r;   ress     r   
<listcomp>z&FaissBuilder.build.<locals>.<listcomp>       IIIsC	NIIIr   newmoder   r   r   )SUPPORT_INDEX_TYPESUPPORT_METRIC_TYPEr]   r9   load_galleryrR   rC   uint8float32rx   astypeshaper!   index_binary_factoryr   r   index_factoryIndexIDMap2_add_galleryr   )r.   gallery_imgsgallery_labelpredict_funcr   r   gallery_docsgallery_listfeaturesdtype
vector_num
vector_dimr   r   s                 r   buildzFaissBuilder.build   s    #0000C#*@CCC 100 32222E3+BEEE 322 mS)) 	E),)9)9-)V)V&L,,)6,LIIll<.H.HIII'3+AAArz8H%%,,U33!)
J#000.##KZHH$$[11 EE '##KZHH$$[11 E
 %e,,E %%X|% & 
 

s c+ZXX
 
 	
r   c                    t                               |          \  }}}}|dk    rt          d          t          t                    rg }t                    5 }|                                }d d d            n# 1 swxY w Y   g |D ]Q}t          |                                	                    d          d                   }	
                    |	           Rt          j                  nt          j                  |                               fd|                                D             }t          ||||d          S )Nr   z:The index_type: HNSW32 dose not support 'remove' operation r   c                 $    i | ]\  }}|v	||S r7   r7   )r;   r<   r=   
remove_idss      r   r>   z'FaissBuilder.remove.<locals>.<dictcomp>#  s)    CCC1q
/B/Bq!/B/B/Br   r   )r   rf   RuntimeErrorr]   r9   open	readlinesr8   stripsplitrz   rC   asarrayr   r@   )
r.   r   r   r   r   r   linesflineid_s
    `        r   removezFaissBuilder.remove
  s    /8nnU.C.C+sK!!L   j#&& 
	0Ej!! &Q& & & & & & & & & & & & & & &J ' '$**,,,,S11!455!!#&&&&J//JJJ//J 	$$$CCCC		CCCc+ZXX
 
 	
s   A==BBc                    t                               |          \  }}}}|| j        v sJ d| j         d            t          |t                    r|                     ||          \  }}	n||}	}d  ||	          D             }
|| j        v rt          j        nt          j	        }t          j
        |
                              |          }
|                     ||||
|d          \  }}t          ||||d          S )Nr   r\   c                     g | ]
}|d          S r   r7   r   s     r   r   z'FaissBuilder.append.<locals>.<listcomp>4  r   r   rz   r   r   )r   rf   r   r]   r9   r   rR   rC   r   r   rx   r   r   )r.   r   r   r   r   r   r   r   r   r   r   r   s               r   rz   zFaissBuilder.append(  s'   .7nnU.C.C+sK32222E3+BEEE 322 mS)) 	E),)9)9-)V)V&L,,)6,LIIll<.H.HIII'3+AAArz8H%%,,U33 %%X|( & 
 

s c+ZXX
 
 	
r   c                    |r$t          |                                          dz   nd}t          j        dt	          |                    |z                       t          j                  }|dk    r4|| j        v r|                    |           n|	                    |           || j        vr|
                    ||           t          t          |          |          D ]
\  }	}
|
||	<   ||fS )Nr*   r   r   )maxkeysrC   arangerc   r   rD   rR   addtrainadd_with_idsry   list)r.   r   r   r   gallery_featuresr   r   start_idids_nowids              r   r   zFaissBuilder._add_gallery@  s     +.43sxxzz??Q&&19QL 1 122X=EEbhOO 5==c444		*++++,---c444/999
 W|44 	 	DAqCFFczr    r   c                    g }g }g }t          |          }t          |dd          5 }|                                }d d d            n# 1 swxY w Y   |D ]m}	|	                                                    |          \  }
}||
z  }|                    |                                           |                    |           n||fS )Nrzutf-8)encoding)r   r   r   r   r   rz   rN   )r.   gallery_label_pathgallery_imgs_root	delimiterr   fileslabelsrootr   r   pathru   	file_paths                r   r   zFaissBuilder.load_galleryX  s   %&&$cG<<< 	"KKMME	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	! 	!D**,,,,Y77KD%tILL++--...MM%    u}s   A		AAr   )r   r   )r   r   )rg   rh   ri   r   r   rR   r   rk   r   r   r   r   rz   r   r   r7   r   r   rQ   rQ      s        22% =   [2 # # [#  /
 /
 /
 [/
b 
 
 [
: 
 
 [
.   [.    [  r   rQ   )r+   pathlibr   numpyrC   utilsr   
utils.depsr   r   utils.ior	   r
   r!   r   rn   rQ   r7   r   r   <module>r      sY                    @ @ @ @ @ @ @ @ . . . . . . . .K   LLL [!!oR oR oR oR oR oR oR "!oRd       8 [!!y y y y y y y "!y y yr   