
    u-j                     T   d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZ d dlmZmZ d dlmZmZmZmZmZmZ d dlmZ d dlmZmZ g d	Zd
 ZddZddZd Zd Z d Z!d Z" G d d          Z#e j$        d             Z%d Z&d Z'd Z(d Z)d Z*e j$        d             Z+dS )    N)Future)partial)Path)
MethodTypeSimpleNamespace)BinaryIODictIterableListOptionalUnion) DEFAULT_MODELSCOPE_DATA_ENDPOINT)
CommitInfoCommitOperation)
z*.binz*.safetensorsz*.pthz*.ptz*.h5z*.ckptz*.zipz*.onnxz*.tarz*.gzc                    
 g } g d}ddgg d
t           j                            d          ddl}t          j        d         j        }|D ]}d|                                v r||         }|D ]ht          fd	|D                       rK	 t          j	        d
| |j
                  }t          |          |                                a#  Y exY wit           j                            d          J	 ddlt                    }d |D             }|                     fd|D                        n#  Y nxY wt           j                            d          	 ddl}	t          j        d         t#          d          rj        }|D ]}d|                                v r||         }|D ]ht          fd
D                       rK	 t          j	        d
| |	j
                  }t          |          |                                a#  Y exY winEt                    }d |D             }|                     
fd|D                        n#  Y nxY w| S )z/Find all modules in transformers/peft/diffusers)zAuto.*zT5.*BitsAndBytesConfigGenerationConfigzAwq.*zGPTQ.*BatchFeaturezQwen.*zLlama.*zIntern.*z
Deepseek.*PretrainedConfigPreTrainedTokenizerPreTrainedModelPreTrainedTokenizerFastz.*PeftModel.*z.*Config)z^(?!TF|Flax).*Pipeline$z^(?!TF|Flax).*Autoencoder.*z^(?!TF|Flax).*Model$z^(?!TF|Flax).*Adapter$ImageProjectionz^(?!TF|Flax).*UNet$z^(?!TF|Flax).*Scheduler$transformersNr   dummyc                 :    g | ]}t          j        |          S  re	fullmatch.0namevalues     `/var/www/html/banglarbhumi/venv/lib/python3.11/site-packages/modelscope/utils/hf_util/patcher.py
<listcomp>z,get_all_imported_modules.<locals>.<listcomp>C   s5         T511      .peftc                 <    g | ]}|                     d           |S __
startswithr"   attrs     r%   r&   z,get_all_imported_modules.<locals>.<listcomp>V   s9       4??43H3H  r'   c                 h    g | ]-t          fd D                       t                    .S )c                 :    g | ]}t          j        |          S r   r   r"   r#   _imports     r%   r&   z7get_all_imported_modules.<locals>.<listcomp>.<listcomp>Z   s8     F F F48BLw//F F Fr'   anygetattr)r"   r4   r)   peft_include_namess    @r%   r&   z,get_all_imported_modules.<locals>.<listcomp>Y   sr     ) ) )+2 F F F F<NF F F B B)g&&) ) )r'   	diffusers_import_structurec                 :    g | ]}t          j        |          S r   r   r!   s     r%   r&   z,get_all_imported_modules.<locals>.<listcomp>m   s5          $( !#T5 9 9     r'   c                 <    g | ]}|                     d           |S r+   r-   r/   s     r%   r&   z,get_all_imported_modules.<locals>.<listcomp>z   s9       !tt7L7L  r'   c                 h    g | ]-t          fd D                       t                    .S )c                 :    g | ]}t          j        |          S r   r   r3   s     r%   r&   z7get_all_imported_modules.<locals>.<listcomp>.<listcomp>   s5         T733  r'   r5   )r"   r4   diffusers_include_nameslazy_modules    @r%   r&   z,get_all_imported_modules.<locals>.<listcomp>}   sj     - - -6=    $;    -K11- - -r'   )	importlibutil	find_specr   sysmodulesr:   lowerr6   import_module__name__r7   appendr)   dirextendr9   hasattr)all_imported_modulestransformers_include_namesr   r:   keyvaluesmodule
attributesimportsr9   r?   r@   r)   r8   r$   s             @@@@@r%   get_all_imported_modulesrT      s   " " "" *:6  
 ~//;k.1'9$ 	 	C#))++%%&s+F      $>     
!*!8%II|'<"> "> ' 6 6,33E::::
 ~''3	KKK TJ !+  G !'' ) ) ) ) )6=) ) )    	D ~,,8#	 +k2K{$788 $/$A!, % %C#))++-- .s3F!' % %        ,C        
%%)2)@$-IIy/A*C *C(/(>(> 4 ; ;E B B B B% $
%%	%" !--
 %/   %++ - - - - -AH- - -    7	DB  s2   ACC?E E)I$ AHH $I(c                     d }d| v rddl m} |dg}n[d| v rddlm} |dg}nLd| v r@t	          |)t          |d          r|j                                        ng           g d	z   }nd
| v rg d	}|S )Nr   r   )GENERATION_CONFIG_NAME*.pyConfig)CONFIG_NAME	Tokenizervocab_files_names)zchat_template.jinjaz*.jsonrW   z*.txtz*.modelz
*.tiktoken	Processor)transformers.utilsrV   r   rY   listrL   r[   rP   )module_nameclsextra_allow_file_patternrV   rY   s        r%   _decide_allow_file_patternrb      s    #[((======$:G#D  	[	 	 ,,,,,,$/#9  		#	##'_.A!B!B_ !((***HJ$L $LO
 O
 O
$
   
	#	#$
 $
 $
  $#r'   Fc                    	 	 ddfd}fd}fd}fd}	 	 dddd	t           t          t          t          t                   f                  d
t           t          t          t          t                   f                  ffd}g }| D ]'t	          d          sj        }d|                                v pd|                                v }	|	si }
n	d	t          i}
	 t	          d          }t	          d          }t	          d          }t	          d          }n#  Y xY w|sd|                                v r[|rY	 |s|s|s|s|                               n|                     |fi |
           #  |                               Y xY w|rt	          d          sst          j
        j                  j        }d|v od|v }j        _        |s t          t          |fi |
          _        nt          t          |fi |
          _        |r;t	          d          s+j        _        t          t          |fi |
          _        |r;t	          d          s+j        _        t          t          |fi |
          _        |                               )d }ddlm} t	          |d          s |j        |_        ||_        ddlm} ||_        |S )zPatch all class to download from modelscope

    Args:
        wrap: Wrap the class or monkey patch the original class

    Returns:
        The classes after patched
    Nc                 l   ddl m} |                    dd           }d }|r| d}t          j                            |           sm|                    dd           }||dk    rd}||}|                    dd	          } || ||||
          }	|r t          j                            |	|          }	n| }	|	S )Nr   snapshot_download	subfolderz/*revisionmainmasterlocal_files_onlyF)rh   rk   ignore_file_patternallow_file_pattern)
modelscoperf   popospathexistsjoin)
pretrained_model_name_or_pathrl   rm   kwargsrf   rg   file_filterrh   rk   	model_dirs
             r%   get_model_dirz._patch_pretrained_class.<locals>.get_model_dir   s     	100000JJ{D11	 	+&***Kw~~;<< 	6zz*d33H8v#5#5#&%0"%zz*<eDD))-!!1$7#57 7 7I  ?GLLI>>	5Ir'   c                      ||                     dd          |                     dd          fi |} | j        j        | |g|R i |S )zPatch all from_pretrainedrl   Nrm   ro   _from_pretrained_origin__func__r`   rt   
model_argsru   rw   rx   s        r%   #patch_pretrained_model_name_or_pathzD_patch_pretrained_class.<locals>.patch_pretrained_model_name_or_path        "M"?"(**-BD"I"I"(**-A4"H"H, , %+, ,	 4s*3C K5?K K KCIK K 	Kr'   c                      ||                     dd          |                     dd          fi |} | j        j        | |g|R i |S )zPatch all get_config_dictrl   Nrm   )ro   _get_config_dict_originr|   r}   s        r%   patch_get_config_dictz6_patch_pretrained_class.<locals>.patch_get_config_dict   r   r'   c                      ||                     dd          |                     dd          fi |} | j        j        | ||g|R i |S )zPatch all peft.from_pretrainedrl   Nrm   rz   )r`   modelmodel_idr~   ru   rw   rx   s         r%   patch_peft_model_idz4_patch_pretrained_class.<locals>.patch_peft_model_id   s    !M("(**-BD"I"I"(**-A4"H"H, , %+, ,	 4s*3C	 K5?K K KCIK K 	Kr'   c                      ||                     dd          |                     dd          fi |} | j        j        | |fi |S )zPatch all _get_peft_typerl   Nrm   )ro   _get_peft_type_originr|   )r`   r   ru   rw   rx   s       r%   patch_get_peft_typez4_patch_pretrained_class.<locals>.patch_get_peft_type   sk    !M("(**-BD"I"I"(**-A4"H"H, , %+, ,	 2s(1#yKKFKKKr'   module_classr   rl   rm   c                     t           j        fd             fd} G  fdd           }t           d          s|`n-t	          j        	j                  j        }d|v rd|v r||_        t           d          s|`t           d	          s|`t           d
          s|`	 j
        |_
         j        |_        |S )a?  Get a custom wrapper class for  auto classes to download the models from the ModelScope hub
        Args:
            module_class (`PreTrainedModel`): The actual module class
            ignore_file_pattern (`str` or `List`, *optional*, default to `None`):
                Any file pattern to be ignored, like exact file names or file extensions.
            allow_file_pattern (`str` or `List`, *optional*, default to `None`):
                Any file pattern to be included, like exact file names or file extensions.
        Returns:
            The wrapped class
        c              3      K   d| vr| d<   d| vr| d<   |                      d          	 |t          |j        |          }|| d<   d V  |                     dd            |                     dd            d S )Nrm   rl   )getrb   rH   ro   )ru   r   r`   ra   rm   rl   s       r%   file_pattern_contextzP_patch_pretrained_class.<locals>.get_wrapped_class.<locals>.file_pattern_context   s      #611/A+,$F220C,-zz(* *-126B6N+E )3,0 ,0(/G+,EEEJJ,d333JJ+T22222r'   c                      |          5   |fd|}d d d            n# 1 swxY w Y    j         | |g|R i |}|S )N)r   r`   )from_pretrained)	r   r   r~   ru   rw   
module_objr   rx   r   s	         r%   r   zK_patch_pretrained_class.<locals>.get_wrapped_class.<locals>.from_pretrained	  s    %%f--  )M!-$  	 	               65eY M7AM M MEKM MJ s   &**c                        e Zd Zefd            Zefd            Zefd            Z	 ddeee	j
        f         def fdZ xZS )	H_patch_pretrained_class.<locals>.get_wrapped_class.<locals>.ClassWrapperc                      ||           5   |fi |}d d d            n# 1 swxY w Y    j         |g|R i |}j                            d          r||_        |S )N	AutoModel)r   rH   r.   rw   )	r`   rt   r~   ru   rw   r   r   rx   r   s	         r%   r   zX_patch_pretrained_class.<locals>.get_wrapped_class.<locals>.ClassWrapper.from_pretrained  s     *)&,DD 8 8 -.K !8 !806!8 !8I8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 :\96 *6 6 6.46 6
  (33K@@ 5+4J(!!s   
%))c                      |	|           5   |fd|}d d d            n# 1 swxY w Y    	j         |fi |}|S N)rl   rm   )_get_peft_type)
r`   r   ru   rw   r   rm   r   rx   rl   r   s
        r%   r   zW_patch_pretrained_class.<locals>.get_wrapped_class.<locals>.ClassWrapper._get_peft_type(  s    ))&,DD " " - !",?+=!" !" !	!" !"I" " " " " " " " " " " " " " " 9\8MMfMM
!!   (,,c                      |
|           5   |f	d|}d d d            n# 1 swxY w Y    
j         |g|R i |}|S r   )get_config_dict)r`   rt   r~   ru   rw   r   rm   r   rx   rl   r   s         r%   r   zX_patch_pretrained_class.<locals>.get_wrapped_class.<locals>.ClassWrapper.get_config_dict4  s     *)&,DD " " -5!",?+=!" !" !	!" !"I" " " " " " " " " " " " " " " :\96 *6 6 6.46 6
!!r   Tsave_directorysafe_serializationc                    |                     dd          }|rddlm} ddlm} ddlm} |                    d          }|                     dd           }|                     d	|                    t          j
        j                  d
                   }	 |            }
|
                    |           |
                    |	            |||	            t                      j        d||dd| |r ||	|||           d S d S )Npush_to_hubFr   )r   HubApi)
Repositorytokencommit_messagerepo_id)r   r   r   )	repo_name
output_dirr   r   r   )ro   modelscope.hub.push_to_hubr   modelscope.hub.apir   modelscope.hub.repositoryr   r   splitrp   rq   seplogincreate_reposupersave_pretrained)selfr   r   ru   r   r   r   r   r   r   api	__class__s              r%   r   zX_patch_pretrained_class.<locals>.get_wrapped_class.<locals>.ClassWrapper.save_pretrainedB  si    %jj>> :FFFFFF999999DDDDDD"JJw//E%+ZZ0@$%G%GN &

!&,,RW[99"=!? !?I !&((CIIe$$$OOI...J~y999'' #1'9 %  	    %K"+#1'5#	% % % % % %% %r'   )T)rH   
__module____qualname__classmethodr   r   r   r   strrp   PathLikeboolr   __classcell__)r   rm   r   rx   rl   r   s   @r%   ClassWrapperr     s       " " " " " " [" 	" 	" 	" 	" 	" 	" 	" 	" [	" " " " " " " " " ["  ,0$% $% %c2;&6 7$% %)$% $% $% $% $% $% $% $% $% $%r'   r   r   r   r   r   r   r   )
contextlibcontextmanagerrL   r   inspect	signature
parametersr   r   r   rH   r   )
r   rl   rm   ru   r   r   r   r   rx   vars
   ```    @r%   get_wrapped_classz2_patch_pretrained_class.<locals>.get_wrapped_class   sg     
	"	3 	3 	3 	3 	3 
#	"	3	 	 	 	 	 	 	N	% N	% N	% N	% N	% N	% N	% N	% N	% N	% N	%< N	% N	% N	%` |%677 	?,, *3+>??JJ*$$z)A)A/>,|%566 	,+|%677 	-,|%677 	-, , 5$0$=!r'   rH   	tokenizerconfigr   r   r   r   pipeliner{   r   r   r   r   c                    ddl m} dt          j        |          j        v r;|d         }t
          j                            |          sddlm	}  ||          |d<   d| v r|| 
                    d          \  }} t
          j                            |          s=i }t          |           }|||d<   d| v sd| v sd	| v r
t          |d
<   ddlm	}  ||fi |}|dz   | z   }  || g|R i |S )Nr   )$origin_get_class_from_dynamic_modulert   re   z--rm   rX   r\   rZ   rl   )!transformers.dynamic_module_utilsr   r   r   r   rp   rq   rr   rn   rf   r   rb   rl   )	class_referenceargsru   r   rt   rf   r   download_kwargsra   s	            r%   get_class_from_dynamic_modulez>_patch_pretrained_class.<locals>.get_class_from_dynamic_module  s   ZZZZZZ*g.?4/6 /66@A A,0G)7>>"?@@ K888888++,IJJQ?""'6'<'<T'B'B$G_7>>'** H"$+E#,% ,%(+70H $,...+2P2PT_crTrTr1D $-/888888++GGGGG%n>O33O >d > > >6<> > 	>r'   r   dynamic_module_utilsr   configuration_auto)NN)r   r   r   r   rL   rH   rF   rl   rI   r   r   r   r   r{   r   r   r   r   r   r   r   r   r   r   transformers.models.autor   )rM   wrapr   r   r   r   r   all_available_modulesr#   
skip_modelignore_file_pattern_kwargshas_from_pretrainedhas_get_peft_typehas_get_config_dicthas_save_pretrainedr   is_peftr   r   r   rx   r   s                       @@r%   _patch_pretrained_classr      s    +/)-   8K K K K KK K K K KK K K K KL L L L L DHBFR R+R!)%T#Y*?!@R !)sDI~)> ?R R R R R R Rh # <. <.;gc:66;| DJJLL0LH

4L
 	)+&& &':*&	")#/@"A"A '-= > >")#/@"A"A")#/@"A"A	H  %	.J$**,,..3F.2+ N4G N 1N:MN)005555)00))#LL1KLLN N N2%,,S11111" ?733L,N ,N ?$.s/BCCN
!Z/LJ*4L.1.A+ ?*5 C > >"<> >+? +?C'' +6 3 > >"<> >+? +?C' ! P6M)N)N P,/,>)%0/NN3MNN&P &P" # ;733L,N ,N ;.1.A+&11 : :8: :'; ';# "((----> > >4 211111'9; ; YDXDvA=Z:??????;X8  s   #A D$$D(:FFc                 ^   | D ]}|	 t          |d          }t          |d          }t          |d          }n#  Y ;xY w|r5t          |d          r%|j        |_        	 t          |d           n#  Y nxY w|r5t          |d          r%|j        |_        	 t          |d           n#  Y nxY w|r5t          |d          r%|j        |_        	 t          |d           #  Y xY wddlm	} t          |d	          r0|j
        |_        dd
lm} |j
        |_        t          |d	           d S d S )Nr   r   r   r{   r   r   r   r   r   r   )rL   r{   r   delattrr   r   r   r   r   r   r   r   r   r   )rM   r   r   r   r   r   r   s          r%   _unpatch_pretrained_classr     s   #  ;	")#/@"A"A '-= > >")#/@"A"A	H 	730I#J#J 	"%"=C67777 	.E!F!F 	!$!:C45555 	730I#J#J 	"%"=C67777111111#%KLL N=Q=v:??????;O;t8$&LMMMMM	N Ns,   0:> A11A5B((B,CC#c                      	 ddl m}  ddlm} n# t          $ r Y dS w xY wt          | d          rdS | j        | _        dd| _        dS )a   Monkey-patch the `kernels` library to route HF API calls to ModelScope.

    Only `kernels.utils._get_hf_api` is replaced; every download, file check
    and ref listing performed by `kernels` goes through `_MsKernelApi`, so
    the kernel loading/variant/lock logic stays untouched.
    r   utils_get_hf_apiN_get_hf_api_originc                     t                      S N)_MsKernelApi)
user_agents    r%   <lambda>z _patch_kernels.<locals>.<lambda>  s
     r'   r   )kernelsr   kernels.utilsr   ImportErrorrL   r   )kernels_utilsr   s     r%   _patch_kernelsr     s    222222-------   }233 '4'@M$ F F FM    
c                      	 ddl m}  ddlm} n# t          $ r Y d S w xY wt          | dd           }||| _        | `d S d S )Nr   r   r   r   )r   r   r   r   r   r7   r   )r   r   origins      r%   _unpatch_kernelsr     s    222222-------   ]$8$??F$*!,,, r   c                     | dv rdn| S )z<Translate an HF revision string into one ModelScope accepts.)Nri   rj   r   rh   s    r%   _ms_revisionr   $  s    >1188x?r'   c                   L    e Zd ZdZdddddddZddddZddd	Zd
 Zd ZdS )r   zMinimal `HfApi` look-alike that forwards to ModelScope. Only the
    handful of methods that `kernels` actually calls are implemented.
    NF)allow_patternsignore_patterns	cache_dirrh   rk   c          	      F    ddl m}  ||ft          |          ||||d|S )Nr   re   )rh   r   rk   r   r   )rn   rf   r   )	r   r   r   r   r   rh   rk   ru   ms_snapshot_downloads	            r%   rf   z_MsKernelApi.snapshot_download.  sZ     	IHHHHH##!(++-)+    	r'   )path_in_reporh   c                R   ddl m} ddlm}  |                                |t          |          |d          }g }|D ]g}	|	                    d          dk    r|	                    d          p|	                    d	          }
|                     ||
d
d                      h|S )Nr   )
RepoFolderr   F)rh   root	recursiveTypetreer   Name )rq   oidlast_commit)huggingface_hub.hf_apir   r   r   get_model_filesr   r   rI   )r   r   r   rh   ru   r   r   entriesfoldersentryrq   s              r%   list_repo_treez_MsKernelApi.list_repo_treeA  s     	655555------&((**!(++	 +  
  	L 	LEyy  F**99V$$9		&(9(9DNN::4RTJJJKKKKr'   r   c                h    ddl m}  |                                ||t          |                    S )Nr   r   r   )r   r   file_existsr   )r   r   filenamerh   ru   r   s         r%   r  z_MsKernelApi.file_existsV  sI    ------vxx##XX(>(> $ @ @ 	@r'   c                     ddl m ddlm}  |                                |          \  }}t          fd|pg D             fd|pg D             g           S )Nr   )
GitRefInfor   c                 2    g | ]} |d | |          S )zrefs/heads/r#   reftarget_commitr   r"   nr  s     r%   r&   z/_MsKernelApi.list_repo_refs.<locals>.<listcomp>b  sC        
'8Q'8'8JJJ  r'   c                 2    g | ]} |d | |          S )z
refs/tags/r  r   r  s     r%   r&   z/_MsKernelApi.list_repo_refs.<locals>.<listcomp>f  sC        
'7A'7'7qIII  r'   )branchestagsconverts)r	  r  r   r   get_model_branches_and_tagsr   )r   r   ru   r   r  r  r  s         @r%   list_repo_refsz_MsKernelApi.list_repo_refs[  s    555555------==gFF$    ".b     *"   	 	 	 		r'   c                 $    ddl m}  |d          )Nr   )EntryNotFoundErrorz2kernel-status.toml lookup is skipped on ModelScope)huggingface_hub.errorsr!  )r   r   ru   r!  s       r%   hf_hub_downloadz_MsKernelApi.hf_hub_downloadl  s1     	>=====  @B B 	Br'   )	rH   r   r   __doc__rf   r  r  r  r#  r   r'   r%   r   r   )  s          *.*.$(#'+0    , %) $	    * :> @ @ @ @ @
  "B B B B Br'   r   c               #      K   ddl m}  t          | d          rdV  dS t                       	 dV  t	                       dS # t	                       w xY w)zApply `_patch_kernels` for the duration of the `with` block, unless an
    outer patch (e.g. `patch_hub()`) is already in effect.
    r   r   r   N)r   r   rL   r   r   )r   s    r%   _kernels_patch_scoper&  u  sz      
 /.....}233 s   A Ac                 .      fd} x|_         |_        |S )a  Return a wrapper around `kernels.<attr_name>` that scopes the ModelScope
    patch on `kernels.utils._get_hf_api` to the call itself, so
    `from kernels import <attr_name>` stays on HuggingFace unless the user
    explicitly calls `patch_hub()` / `patch_context()`.
    c                      dd l }t                      5   t          |          | i |cd d d            S # 1 swxY w Y   d S )Nr   )r   r&  r7   )r   ru   r   	attr_names      r%   _wrappedz(_wrap_kernels_callable.<locals>._wrapped  s    !## 	@ 	@.77I..???	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@s   7;;)rH   r   )r)  r*  s   ` r%   _wrap_kernels_callabler+    s8    @ @ @ @ @
 1:9H-Or'   c                      dd l } ddl m} ddlm} ddlm} ddlm}  |            d d d ddt          dt          d	t          t                   d
t          t                   dt          t          t          d f         f
fd}d d d d d d dddt          dt          dt          t                   d	t          t                   d
t          t                   dt          t          t          d f         dt          t          t          d f         dt          t          t          d f         dt          ffd}d7dt          t          t          d f         dt          fd}d dddt          dt          t          t          d f         dt          ddfd}|d d d d dd ddt          dt          t          t          f         dt          t                   dt          t                   dt          t                   dt          t          t          d f         d
t          t                   d t          t          t          t                   t          f                  fd!            }	dd"lm}
 |d |
d d d#d$t          t          t          t           t"          f         dt          dt          dt          t          t          d f         d
t          t                   dt          t                   dt          t                   fd%            }|d d d |
d&dt          d't$          t&                   dt          dt          t                   dt          t          t          d f         d	t          t                   d
t          t                   dt          t(          t*          t(                   f         fd(            }	 	 	 d8d)t          t          t          f         d	t          t                   dt          t                   d*t          fd+}dd,l m} t/          |j        d-          sX|j        j        |j        _        d. |j        _        |j        j        |j        _        t;          ||j                  |j        _        t/          |d/          s$| j        j        |_         || _        || j        _        t/          |d0          s>|j!        |_"        t;          ||          |_!        |j!        | _!        |j!        | j        _!        t/          |d1          s>|j#        |_$        t;          ||          |_#        |j#        | _#        |j#        | j        _#        t/          |d2          sPdd3l%m&} |j'        |_(        t;          ||          |_'        |j'        | _'        |j'        | j        _'        |j'        |_'        t/          |d4          s>|j)        |_*        t;          |	|          |_)        |j)        | _)        |j)        | j        _)        t/          |d5          sJ|j+        |_,        t;          ||          |_+        |j+        | _+        |j+        | j        _+        |j+        |_+        t/          |d6          sR|j-        |_.        t;          ||          |_-        |j-        | _-        |j-        | j        _-        dd3l%m&} |j-        |_-        d S d S )9Nr   hf_api)r   )future_compatible)
get_logger)	repo_typerh   r   r   r  r1  rh   r   c                    |                     d           ddlm}  |            }|                    |           ||dk    rd}|                    |||          S )z!Patch huggingface_hub.file_existsNz[The passed in repo_type will not be used in modelscope. Now only model repo can be queried.r   r   ri   rj   r   )warningr   r   r   r  )	r   r   r  r1  rh   r   r   r   loggers	           r%   _file_existsz _patch_hub.<locals>._file_exists  s      NNm   	.-----fhh		%x611Hw8DDDr'   F)rg   r1  rh   r   	local_dirr   rk   rg   r   r6  rk   c                ^   t          |	          dk    r                    d           |dv sJ d| d            |dv rddlm}
 nddlm}
 dd	lm}  |            }|                    |           ||dk    rd} |
| |r t          j	        
                    ||          n|||||          S )z%Patch huggingface_hub.hf_hub_downloadr   zThe passed in library_name,library_version,user_agent,force_download,proxiesetag_timeout,headers,endpoint will not be used in modelscope.)Nr   datasetz
repo_type=z is not supported in ModelScope)Nr   model_file_downloaddataset_file_downloadr   Nri   rj   )	file_pathr   r6  rk   rh   )lenr3  modelscope.hub.file_downloadr:  r<  rn   r   r   rp   rq   rs   )r   r  rg   r1  rh   r   r6  r   rk   ru   file_downloadr   r   r4  s                r%   _file_downloadz"_patch_hub.<locals>._file_download  s%    v;;??NN23 3 3     OYOOO   ''YYYYYYY[[[[[[%%%%%%fhh		%x611H}'bgll9h777&-   	r'   returnc                     ddl m} ddl m}  |            }|                    |           d|                                d         pdiS )Nr   )ModelScopeConfigr   r#   unknown)r   rD  r   r   get_user_info)r   r   rD  r   r   s        r%   _whoamiz_patch_hub.<locals>._whoami  se    777777------fhh		%(6688;HyIIr'   )r   privaterH  RepoUrlc                x    ddl m}  |            }|rdnd} |j        |f||d|}ddlm}	  |	|d|          S )	af  
        Create a new repository on the hub.

        Args:
            repo_id: The ID of the repository to create.
            token: The authentication token to use.
            private: Whether the repository should be private.
            **kwargs: Additional arguments.

        Returns:
            RepoUrl: The URL of the created repository.
        r   r   rH  public)r   
visibility)rI  r   )urlr1  r   )r   r   r   modelscope.utils.repo_utilsrI  )
r   r   r   rH  ru   r   r   rL  repo_urlrI  s
             r%   r   z_patch_hub.<locals>.create_repo  s    $ 	.-----fhh")7YYx
"3?C ZC C;AC C777777w8wHHHHr'   rj   )r   r   commit_descriptionr   rh   r   folder_pathr   r   rP  r   c          	          ddl m}
 ||dk    rd} |
|||||||           ddlm}  |t           d| d||d 	          S )
Nr   _push_files_to_hubri   rj   )path_or_fileobjr   r   r   rP  rh   r   )r   z/models/z/files)
commit_urlr   rP  r  )r   rT  rN  r   r   )r   r   rQ  r   r   rP  r   rh   r   ru   rT  r   s               r%   upload_folderz!_patch_hub.<locals>.upload_folder  s     	BAAAAAx611H'%)1	 	 	 	 	;:::::z/HHHHH)1
 
 
 	
r'   )DEFAULT_REPOSITORY_REVISION)r   rh   r   rP  rU  c          	      H    ||dk    rd}ddl m}	  |	|||||||           d S )Nri   rj   r   rS  )r   rT  )
r   rU  r   r   r   rh   r   rP  ru   rT  s
             r%   upload_filez_patch_hub.<locals>.upload_file  s_     x611HAAAAAA?L'5#^5G	I 	I 	I 	I 	Ir'   )rP  r   r1  rh   
operationsc          	          ddl m}	  |	            }
t          d |D                       rt          d          ||dk    rd}d |D             }|
                    |||||||pd	           d S )
Nr   r   c                 *    g | ]}d |j         j        vS )Add)r   rH   r"   ops     r%   r&   z5_patch_hub.<locals>.create_commit.<locals>.<listcomp>?  s"    HHHrR\22HHHr'   z<ModelScope create_commit only support Add operation for now.ri   rj   c                     g | ]	}|j         
S r   )rU  r_  s     r%   r&   z5_patch_hub.<locals>.create_commit.<locals>.<listcomp>D  s    ===BR'===r'   r   )r   rQ  r   rP  r   rh   r1  )r   r   r6   
ValueErrorrW  )r   r   r[  r   rP  r   r1  rh   ru   r   r   	all_filess               r%   create_commitz!_patch_hub.<locals>.create_commit0  s     	.-----fhhHHZHHHII 	PNP P Px611H==*===	!)1*7 	 	, 	, 	, 	, 	,r'   repo_id_or_pathignore_metadata_errorsc                    ddl m}  |            }|                    |           t          j                            |          r|}nF|dk    s|ddlm}  ||d          }n+|dk    rddlm}	  |	|d          }nt          d|           t          |d	          5 }
 | |
                                |
          }t          |j        d          sg |j        _        |cd d d            S # 1 swxY w Y   d S )Nr   r   r   r9  z	README.mdr8  r;  z5repo_type should be `model` or `dataset`, but now is r)rf  r  )r   r   r   rp   rq   rr   rn   r:  r<  rb  openreadrL   datar  )r`   re  r1  r   rf  r   r   r=  r:  r<  f	repo_cards               r%   loadz_patch_hub.<locals>.loadN  s    	.-----fhh		%7>>/** 	'II'!!Y%6666666++O[IIII)##888888--o{KKIIS	SS   )S!! 	Q1GI I II9>622 )&(	#	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   AC--C14C1repocard_validate_originc                      d S r   r   )r   ru   s     r%   r   z_patch_hub.<locals>.<lambda>p  s    T r'   _hf_hub_download_origin_file_exists_origin_whoami_origin_create_repo_originhub_upload_folder_origin_upload_file_origin_create_commit_originr   )NNF)/huggingface_hubr.  r	  r   r/  rn   r0  r   r   r   r   r   r	   r   modelscope.utils.constantrX  bytesr   r
   r   r   r   rp  rL   RepoCardvalidaterq  rn  _load_originr   r@  r#  rs  r  rt  whoamiru  r]   rx  r   rv  rW  ry  rZ  rz  rd  r{  )r|  r.  r   r/  r0  r5  rA  rG  r   rW  rX  rZ  rd  rn  rp  rx  r4  s                   @r%   
_patch_hubr    s   &&&&&&******888888%%%%%%Z\\F $("&(,E E EE E
 C=E 3-E S$_%E E E E E E0 372615;?;?7;05$ $ $ $!$$ #+3-$ #+3-	$
 "*#$ #(T4"8$ #(T4"8$ $D#tO4$ *.$ $ $ $ $ $LJ JU4d?3 Jt J J J J 59$)	I I I I !dD1I "	I
 "+I I I I4  '+(,,0(,"*;?
 
 
 
 39%	

 sm
 !
 %SM
 S$_%
 3-
 "%S	3"78
 
 
 
B FEEEEE )-"=(,,0I I I sD%9:I 	I
 I S$_%I 3-I !I %SMI I I I$  -1(,#'"=, , ,, _-,
 , %SM, S$_%, C=, 3-, 
z6*--	., , , ,@ $(#', sDy) C= }	
 !%   > )(((((8$&899 E-5->-G*%A%A")1):)?&!+D(2C!D!D6455 G)8)F)V&*8'8F%56011 @%+%7"'c::&,&8#-3-?*6+,, 6 &"7C00!'(.%6011 -******%+%7"'S99&,&8#-3-?* ,6233 D'-';$)-==(.(<%/5/C,6011 2%+%7"'S99&,&8#-3-?*%16233 1'-';$)-==(.(<%/5/C,******"01 1r'   c                     dd l } ddl m} ddl m} t          |j        d          r+|j        j        |j        _        t          |j        d           t          |j        d          r+|j        j        |j        _	        t          |j        d           t          |d          r>|j
        | j        _        |j
        | _        |j
        | j        _        t          |d           t          |d          r9|j        |_        |j        | _        |j        | j        _        t          |d           t          |d          r9|j        |_        |j        | _        |j        | j        _        t          |d           t          |d	          rKdd
lm} |j        |_        |j        | _        |j        | j        _        |j        |_        t          |d	           t          |d          r9|j        |_        |j        | _        |j        | j        _        t          |d           t          |d          rE|j        |_        |j        | _        |j        | j        _        |j        |_        t          |d           t          |d          rM|j        |_        |j        | _        |j        | j        _        dd
lm} |j        |_        t          |d           d S d S )Nr   r-  ro  rq  r  rs  rt  ru  rv  rw  ry  rz  r{  )r|  r.  rp  rL   r  rq  r  r   r  rn  rs  r@  r#  rt  r  ru  r  r]   rx  rv  r   ry  rW  rz  rZ  r{  rd  )r|  r.  rp  rx  s       r%   _unpatch_hubr    s   &&&&&&((((((x "455 7%-%6%G"!#5666x .11 3!)!2!?!>222v011 38>8V%5*0*H'8>8V%51222v,-- /#7&,&8#-3-?*-...v'(( *-!'(.%()))v,-- /******#7&,&8#-3-?* ,-...v.// 1%;(.(<%/5/C,/000v,-- /#7&,&8#-3-?*%1-...v.// 1%;(.(<%/5/C,******"0/000001 1r'   c                  t    t                       t                       t          t                                 d S r   )r  r   r   rT   r   r'   r%   	patch_hubr    s2    LLL46677777r'   c                  t    t          t                                 t                       t                       d S r   )r   rT   r   r  r   r'   r%   unpatch_hubr    s0    688999NNNNNr'   c               #   J   K   t                       d V  t                       d S r   )r  r  r   r'   r%   patch_contextr    s%      KKK	EEEMMMMMr'   r   )F),r   rA   r   rp   r   rD   asyncior   	functoolsr   pathlibr   typesr   r   typingr   r	   r
   r   r   r   modelscope.hub.constantsr   rN  r   r   rl   rT   rb   r   r   r   r   r   r   r   r&  r+  r  r  r  r  r  r   r'   r%   <module>r     sR            				 				 



                   - - - - - - - - B B B B B B B B B B B B B B B B E E E E E E C C C C C C C C   e  e  e P$ $ $ $.! ! ! !D
#N #N #NLG G G$	- 	- 	-@ @ @
IB IB IB IB IB IB IB IBX      R1 R1 R1j91 91 91x8 8 8       r'   