
    Ƒi.                       S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKrS SK	r	S SK
r
S SKrS SKrS SKJr  S SKJr  S SKJr  S SKJr  S SKJr  S SKJrJrJrJrJrJrJrJrJ r   S SK!J"r"  S SK#r#S SK$r$S S	K$J%r%  S S
K&J'r'J(r(  S SK)J*r*  S SK+J,r,  S SK-J.r.J/r/J0r0J1r1J2r2J3r3J4r4J5r5J6r6J7r7J8r8J9r9J:r:J;r;J<r<J=r=J>r>J?r?J@r@JArAJBrBJCrCJDrDJErEJFrFJGrGJHrHJIrIJJrJJKrKJLrLJMrM  S SKNJOrOJPrPJQrQJRrRJSrSJTrTJUrUJVrVJWrWJXrX  S SKYJZrZ  S SK[J\r\J]r]  S SK^J_r_  S SK`Jara  S SKbJcrcJdrdJereJfrfJgrgJhrhJiri  S SKjJkrkJlrlJmrmJnrnJoroJprpJqrqJrrrJsrsJtrtJuruJvrvJwrwJxrxJyryJzrzJ{r{  S SK|J}r}J~r~Jr  S SKJr  S SKJrJrJrJrJrJrJrJr  S SKJr  \" 5       r " S S5      r " S S5      r " S S5      rg)    N)defaultdict)
HTTPStatus)	CookieJar)
expanduser)Path)	AnyBinaryIODictIterableListLiteralOptionalTupleUnion)	urlencode)Session)HTTPAdapterRetry)	HTTPErrortqdm) API_HTTP_CLIENT_MAX_RETRIESAPI_HTTP_CLIENT_TIMEOUTAPI_RESPONSE_FIELD_DATAAPI_RESPONSE_FIELD_EMAIL#API_RESPONSE_FIELD_GIT_ACCESS_TOKENAPI_RESPONSE_FIELD_MESSAGEAPI_RESPONSE_FIELD_USERNAMEDEFAULT_MAX_WORKERSDEFAULT_MODELSCOPE_INTL_DOMAINMODELSCOPE_CLOUD_ENVIRONMENTMODELSCOPE_CLOUD_USERNAMEMODELSCOPE_CREDENTIALS_PATHMODELSCOPE_DOMAINMODELSCOPE_PREFER_AI_SITEMODELSCOPE_REQUEST_IDMODELSCOPE_URL_SCHEMEONE_YEAR_SECONDSREQUESTS_API_HTTP_METHODTEMPORARY_FOLDER_NAME"UPLOAD_BLOB_TQDM_DISABLE_THRESHOLDUPLOAD_COMMIT_BATCH_SIZEUPLOAD_MAX_FILE_COUNTUPLOAD_MAX_FILE_COUNT_IN_DIRUPLOAD_MAX_FILE_SIZE#UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT$UPLOAD_SIZE_THRESHOLD_TO_ENFORCE_LFSVALID_SORT_KEYSDatasetVisibilityLicensesModelVisibility
VisibilityVisibilityMap)
InvalidParameterNotExistErrorNotLoginExceptionRequestErrordatahub_raise_on_errorhandle_http_post_errorhandle_http_responseis_okraise_for_http_statusraise_on_error)GitCommandWrapper)DatasetInfo	ModelInfo
Repository)	AigcModel)add_content_to_file
get_domainget_endpointget_readable_folder_sizeget_release_datetimeis_env_truemodel_id_to_group_owner_name)DEFAULT_DATASET_REVISIONDEFAULT_MODEL_REVISIONDEFAULT_REPOSITORY_REVISIONMASTER_MODEL_BRANCHMETA_FILES_FORMATREPO_TYPE_DATASETREPO_TYPE_MODELREPO_TYPE_SUPPORTConfigFieldsDatasetFormationsDatasetMetaFormatsDownloadChannelDownloadMode
Frameworks	ModelFileTasksVirgoDatasetConfig)get_file_hashget_file_sizeis_relative_path)
get_logger)DATASET_LFS_SUFFIXDEFAULT_IGNORE_PATTERNSMODEL_LFS_SUFFIXCommitHistoryResponse
CommitInfoCommitOperationCommitOperationAdd	RepoUtils)thread_executorc                      \ rS rSrSrS\\S4S\\   S\\   4S jjr	S\4S jr
SS\\   S
\\   4S jjr  SS\\   S\\   4S jjr\R                  \R"                  SSSSS4S\S\\   S\\   S\\   S\\   S\\   S\\   S\S   S\4S jjr   SS\S\S\\   S\\   S\S   S\4S jjrSS\S\\   S\\   4S jjrSS\S\\   4S jjr\SS4S\S\\   S\\   S\\   S\4
S jjrSSS.S\S\\   S\\   S\4S  jjr\SS!.S\S\\   S\\   S\4S" jjrSSS!.S\S\\   S\\   S\4S# jjr\\SS$.S\S\\   S\\   S\\   S\ \\4   4
S% jjr!SSS	SS&.S\S\\   S\\   S'\\   S\\   S\4S( jjr"  SS\S\S\\   S\\   4S) jjr#\$S* 5       r%\R                  \R"                  SS+S\&SSSS4
S\S,\S\\   S\\   S\\   S-\\   S.\\   S\\   S\\   S/\\ \'\   \4      S0\\ \\'\   4      S\\   4S1 jjr(    SS4\S5\\   S6\\   S\\   S\\   S\4S7 jjr)S2S3SSSSS8.S4\S5\\   S6\\   S9\\   S:\\   S\\   S\\   S\4S; jjr*SS<\ \\+4   S\+4S= jjr,  SS\S>\\   S<\ \\+4   S\'\   4S? jjr-   SS\S>\\   S<\ \\+4   S\\   S\'\   4
S@ jjr.SA r/   SS\SB\\+   S\\   4SC jjr0   SS\SB\\+   S\\   4SD jjr1  SS\S<\ \\+4   S\\   S\2\'\   \'\   4   4SE jjr3 SS\S<\ \\+4   S\2\'\   \'\   4   4SF jjr4\SS	S	0 S4S\S\\   SG\\   SH\\   S<\ \\+4   SI\\   S\\   S\'\   4SJ jjr5SSSK.S\SL\S\\   S\\   4SM jjr6S\R"                  \7R                  SSS4SN\SO\S\\   S\\   S\\   SP\\   S\\   S\\   S\4SQ jjr8  SSR\S\\   S\\   4SS jjr9  SSN\SO\S\\   S\\   4ST jjr:     SSN\SO\S\SW\SH\S5\S6\S\\   S\\   4SX jjr;\\&S2SYSSSZ.S\S\\   S\\   S5\S6\S\\   S\\   4S[ jjr<\&S\SUS2SVSSS].S\S\SW\SH\S5\S6\S\\   S\\   4S^ jjr=\&SS4SR\S\\   S\\   S\\   4S_ jjr>  SSN\SO\SR\S\S\\   S\\   4S` jjr?\$Sa\Sb\4Sc j5       r@  SSN\SO\S\Sb\Sa\Sd\AS\\   S\\   4Se jjrB\$Sf\CR                  S4S\\   4Sg jj5       rE\FS	SUS4Sh\SN\SO\S\\   Si\\   Sj\\   S\\   4Sk jjrG\FS4Sh\SN\SO\S\\   S\\   4
Sl jjrH\FSS4SN\SO\S\\   S\\   S\\   4
Sm jjrI\FSS4SN\SO\Sn\S\\   S\\   S\\   4So jjrJSSR\Sp\S\\   S\4Sq jjrK  SSN\SO\S\Sr\S\\   S\\   4Ss jjrL  SS\\   S\\   4St jjrM  SSu\SN\SO\S\S\\   S\\   S\4Sv jjrN  SSu\SN\SO\S\S\\   S\\   S\4Sw jjrOSS\\   4Sx jjrP  SSN\SO\Sy\S\\   S\\   SS4Sz jjrQS{ rRSS\S\\   S\4S| jjrSS\TR                  \S\R"                  SS	SUSS}.	S\S\ \\S4   S\\   S\\   S\\   S\\   S\\   S~\\   S\\   S\\U   S\4S jjrVSS\\&SSSS.S\S\W\X   S-\S\\   S\S\\   S\\   S\\   S\S\S\Y4S jjrZS\SSS2SS	\&S.S\ \\[\\\]4   S\S\S\ \S4   S\\   S-\\   S\\   S\\   S\\   S\\   S\\   S\Y4S jjr^SSSS\SS\_\&S.	S\S\ \\[\'\   \'\[   4   S\\   S-\\   S\\   S\ \S4   S\\   S\\ \'\   \4      S\\ \'\   \4      S\S\\   S\ \Y\'\Y   4   4S jjr`S	SS2SS.S\S\S\S\S\ \\[\\\]4   S\\   S\\   S\\   S\\   S\4S jjraSSS.S\S\S\'\b\\c4      S\\   S\\   S\'\b\\c4      4S jjrd  SS\ \\[\'\   \'\[   4   S\S\\ \'\   \4      S\\ \'\   \4      S\'\ \e\A4      4
S jjrf\$S\W\X   S-\S\b\\c4   4S j5       rgSS\h4S jjri\SSS.S\S\S\ \\'\   4   S\\   S\\   S\\   S\b\\c4   4S jjrj SS\S\kS   S\kS   S\ \S4   S\4
S jjrlSrmg)HubApic   zModel hub api interface.
    Nendpointtokenc                    Ub  UO	[        5       U l        X@l        S[        R	                  5       0U l        [        5       U l        [        USSSSSS9n[        US9nU R                  R                  S	U5        U R                  R                  S
U5        [         HA  n[        U R                  U[        R                  " [        U R                  U5      US95        MC     [!        5       U l        g)uz   The ModelScope HubApi。

Args:
    endpoint (str, optional): The modelscope server http|https address. Defaults to None.
N
user-agent      )  i  i  i  F)totalreadconnectbackoff_factorstatus_forcelistrespect_retry_after_header)max_retrieszhttp://zhttps://timeout)rJ   rp   rq   ModelScopeConfigget_user_agentheadersr   sessionr   r   mountr)   setattr	functoolspartialgetattrUploadingCheckupload_checker)selfrp   r   r}   rq   retryadaptermethods           R/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/modelscope/hub/api.py__init__HubApi.__init__g   s     %-$8ln
$&6&E&E&GHy1',
 %09g.:w/.Ff!!DLL&1#%& / -.    access_tokenc                     SSK Jn  SSKJn  U R                  (       a  U" U R                  5      R
                  O	[        5       nU" 5       nUR                  SUUSS9  U$ )z
Get jar cookies for authentication from access_token.

Args:
    access_token (str): user access token on ModelScope.

Returns:
    jar (CookieJar): cookies for authentication.
r   )RequestsCookieJar)urlparsem_session_id/)domainpath)requests.cookiesr   urllib.parser   rp   netlocrI   set)r   r   r   r   r   jars         r   _get_cookiesHubApi._get_cookies   sS     	7)8<ht}}-44:<! 	 	 
r   Fcookies_requiredc                     U=(       d2    U R                   =(       d    [        R                  R                  S5      nU(       a  U R	                  US9nO[
        R                  5       nUc  U(       a  [        S5      eU$ )a  
Get cookies for authentication from local cache or access_token.

Args:
    access_token (Optional[str]): user access token on ModelScope. If not provided, try to get from local cache.
    cookies_required (bool): whether to raise error if no cookies found, defaults to `False`.

Returns:
    cookies (CookieJar): cookies for authentication.

Raises:
    ValueError: If no credentials found and cookies_required is True.
MODELSCOPE_API_TOKENr   zNo credentials found.You can pass the `--token` argument, or use HubApi().login(access_token=`your_sdk_token`). Your token is available at https://modelscope.cn/my/myaccesstoken)rq   osenvirongetr   r   get_cookies
ValueError)r   r   r   rq   cookiess        r   r   HubApi.get_cookies   sk     T

Tbjjnn=S.T''U';G&224G?/T  r   c                    U=(       d2    U R                   =(       d    [        R                  R                  S5      nU(       d  gU(       d  U R                  nU S3nU R
                  R                  USU0U R                  U R                  5      S9n[        U5        UR                  5       n[        U5        U[           [           nUR                  n[        R!                  U5        [        R#                  U5        [        R%                  U[           [&           U[           [(           5        U[           [           U4$ )a$  Login with your SDK access token, which can be obtained from
   https://www.modelscope.cn user center.

Args:
    access_token (str): user access token on modelscope, set this argument or set `MODELSCOPE_API_TOKEN`.
    If neither of the tokens exist, login will directly return.
    endpoint: the endpoint to use, default to None to use endpoint specified in the class

Returns:
    cookies: to authenticate yourself to ModelScope open-api
    git_token: token to access your git repository.

Note:
    You only have to login once within 30 days.
r   NNz/api/v1/loginAccessToken)jsonr   )rq   r   r   r   rp   r   postbuilder_headersr   r@   r   rA   r   r   r   r   
save_tokensave_cookiessave_user_infor   r   )r   r   rp   r   rdrq   r   s           r   loginHubApi.login   s   ( $[tzz[RZZ^^DZ5[}}H=)LL.((6  8 	a FFHq)*+NO)) 	##E*%%g.''%&'BC%&'?@	B ()/129: 	:r    model_id
visibilitylicensechinese_nameoriginal_model_id
aigc_modelrG   returnc	                 R   Uc  [        S5      eU R                  USS9n	U(       d  U R                  n[        U5      u  pU
UUUUU[        R
                  R                  SS5      S.nUb  U S3nUR                  XR                  U R                  5      US9  UR                  UR                  UR                  UR                  UR                  UR                  UR                   =(       d    UUR"                  UR$                  UR&                  UR(                  UR*                  UR,                  UR.                  S	.5        UR0                  (       a  UR0                  US
'   OU S3nU R                  U R                  5      n[2        R4                  " S5      S   nUR7                  S5      R9                  SU 35      (       a  SUS'   U R:                  R=                  UUU	US9n[?        U5        URA                  5       n[C        U5        U SU 3nUb  URE                  XU5        U$ )a  Create model repo at ModelScope Hub.

Args:
    model_id (str): The model id in format {owner}/{name}
    visibility (int, optional): visibility of the model(1-private, 5-public), default 5.
    license (str, optional): license of the model, default apache-2.0.
    chinese_name (str, optional): chinese name of the model.
    original_model_id (str, optional): the base model id which this model is trained from
    endpoint: the endpoint to use, default to None to use endpoint specified in the class
    token (str, optional): access token for authentication
    aigc_model (AigcModel, optional): AigcModel instance for AIGC model creation.
        If provided, will create an AIGC model with automatic file upload.
        Refer to modelscope.hub.utils.aigc.AigcModel for details.

Returns:
    str: URL of the created model repository

Raises:
    InvalidParameter: If model_id is invalid or required AIGC parameters are missing.
    ValueError: If not login.

Note:
    model_id = {owner}/{name}
model_id is required!Tr   r   MODELSCOPE_TRAIN_IDr   )r   NameChineseNamer6   LicenseOriginalModelIdTrainIdz/api/v1/models/aigcr   r   rp   )TagShowNameCoverImagesAigcTypeTagDescriptionVisionFoundation	BaseModelWeightsNameWeightsSha256WeightsSize	ModelPathTriggerWordsModelSourceSubVisionFoundationOfficialTagsz/api/v1/models.r   en_USzX-Modelscope-Accept-Languager   r   r   /models/)#r8   r   rp   rN   r   r   r   preupload_weightsr   r   updatetagcover_images	aigc_typedescriptionbase_model_typebase_model_idweight_filenameweight_sha256weight_size
model_pathtrigger_wordsmodel_sourcebase_model_sub_typeofficial_tagsr    splitrstripendswithr   r   r@   r   rA   upload_to_repo)r   r   r   r   r   r   rp   rq   r   r   owner_or_groupnamebodyr   r   intl_endr   r   model_repo_urls                      r   create_modelHubApi.create_model   s(   B "#:;;"""M}}H;HE #'$0zz~~&;R@
 !Z23D((BVBVW[WcWcBdow(x KK)~~)66&00","8"8$.$>$>'55J9J)99!+!9!9)55'22 * 8 8)66'1'E'E   '''1'?'?^$ Z~.D&&t||4177<R@??3((1XJ886=G23LL	  
 	a FFHq$:XhZ8 !%%de<r   tag_namec                    Uc  [        S5      eUc  [        S5      eUR                  5       S;   a  [        SU S35      eU R                  USS9nU(       d  U R                  n[	        U5      u  pxUb  U S3n	UR                  X`R                  U R                  5      US	9  UR                  UUUUR                  UR                  UR                  UR                  UR                  UR                  S
.
n
OU SU S3n	SnUUS.n
U R                  R!                  U	U
UU R                  U R                  5      S9n[#        U5        UR%                  5       n['        U5        U SU SU 3nU$ )a  Create a tag for a model at ModelScope Hub.

Args:
    model_id (str): The model id in format {owner}/{name}
    tag_name (str): The tag name (e.g., "v1.0.0")
    endpoint: the endpoint to use, default to None to use endpoint specified in the class
    token (str, optional): access token for authentication
    aigc_model (AigcModel, optional): AigcModel instance for AIGC model tag creation.
        If provided, will create an AIGC model tag with automatic parameters.
        Refer to modelscope.hub.utils.aigc.AigcModel for details.

Returns:
    str: URL of the created tag

Raises:
    InvalidParameter: If model_id, tag_name, ref, or description is invalid.
    ValueError: If not login.

Note:
    model_id = {owner}/{name}
r   ztag_name is required!)mainmasterz
tag_name "zp" is not allowed. Please use a different tag name (e.g., "v1.0", "v1.1", "latest"). Reserved names: main, masterTr   z/api/v1/models/aigc/repo/tagr   )
r   r   r   r   r   r   r   r   r   r   /api/v1/models/z	/repo/tagr   )TagNameRefr   r   z/tags/)r8   lowerr   rp   rN   r   r   r   r   r   r   r   r   r   r   r   r   r@   r   rA   )r   r   r   rp   rq   r   r   r   r   r   r   revisionr   r   tag_urls                  r   create_model_tagHubApi.create_model_tagV  s   6 "#:;;"#:;;>>11"XJ '/ 0  """M}}H;HE !Z;<D((BVBVW[WcWcBdow(x  *66&')99!+!9!9)55 * 8 8&00$.$>$>D Zxj	BDH#D
 LL((6	  8 	a FFHqJhxjxjAr   c                 X   [         R                  " S[        SS9  U R                  USS9nU(       d  U R                  nUc  [        S5      eU SU 3nU R                  R                  UUU R                  U R                  5      S	9n[        U5        [        UR                  5       5        g)
a8  
@deprecated
Delete model_id from ModelScope.

Args:
    model_id (str): The model id.
    endpoint: the endpoint to use, default to None to use endpoint specified in the class
    token (str, optional): access token for authentication

Raises:
    ValueError: If not login.

Note:
    model_id = {owner}/{name}
This function is deprecated due to security reasons, and will be recovered in future versions with proper token authentication. rt   
stacklevelTr   N)Token does not exist, please login first.r   r   r   warningswarnDeprecationWarningr   rp   r   r   deleter   r   r@   rA   r   )r   r   rp   rq   r   r   r   s          r   delete_modelHubApi.delete_model  s      	Z		
 """M}}H?HII?8*5LL(/(,(<(<T\\(J   L 	a qvvx r   c                 8    U(       d  U R                   nU SU S3$ )Nr   z.git)rp   )r   r   rp   s      r   get_model_urlHubApi.get_model_url  s!    }}H?8*D99r   r  c                 &   U R                  USS9n[        U5      u  pgU(       d  U R                  nU(       a  U SU SU SU 3nO
U SU SU 3nU R                  R	                  XU R                  U R                  5      S9n	[        U	[        XQ5        U	R                  [        R                  :X  aU  [        U	R                  5       5      (       a  U	R                  5       [           $ [        U	R                  5       [            5      e[#        U	5        g)a  Get model information at ModelScope

Args:
    model_id (str): The model id.
    revision (str optional): revision of model.
    endpoint: the endpoint to use, default to None to use endpoint specified in the class
    token (str, optional): access token for authentication

Returns:
    The model detail information.

Raises:
    NotExistError: If the model is not exist, will throw NotExistError

Note:
    model_id = {owner}/{name}
Fr   r   r   
?Revision=r  N)r   rN   rp   r   r   r   r   r>   loggerstatus_coder   OKr?   r   r   r9   r   r@   )
r   r   r  rp   rq   r   r   r   r   r   s
             r   	get_modelHubApi.get_model  s    0 """N;HE}}HZ~.>avZPXzZDZ~.>avFDLLT%)%9%9$,,%G  IQ:==JMM)QVVXvvx 788#AFFH-G$HII!!$r   )	repo_typerq   repo_idr  c          	      N   [         R                  R                  [        5      nUb2  UR	                  5       S:w  a  [
        U-   n U R                  XUSUS9  U$ [        [        5      (       + n[        US9nU R                  XXsS9(       dB  [        U(       + S9n[        R                  SU S	U S
U S35         U R                  XUSUS9  U$ U$ ! [         a    [        R                  SU SU S35        e f = f! [         a     [        R                  SU SU SU 35        e f = f)a  Get proper endpoint for read operation (such as download, list etc.)
1. If user has set MODELSCOPE_DOMAIN, construct endpoint with user-specified domain.
   If the repo does not exist on that endpoint, throw 404 error, otherwise return the endpoint.
2. If domain is not set, check existence of repo in cn-site and ai-site (intl version) respectively.
   Checking order is determined by MODELSCOPE_PREFER_AI_SITE.
     a. if MODELSCOPE_PREFER_AI_SITE is not set ,check cn-site first before ai-site (intl version)
     b. otherwise check ai-site before cn-site
   return the endpoint with which the given repo_id exists.
   if neither exists, throw 404 error
r   T)r  r  rp   re_raiserq   Repo z does not exist on r   )cn_siter  rp   rq   z not exists on z#, will try on alternative endpoint r  rp   r   rq   z not exists on either z or )r   r   r   r$   stripr'   repo_exists	Exceptionr  errorrM   r%   rJ   warning)	r   r  r  rq   srp   check_cn_firstprefer_endpointalternative_endpoints	            r   get_endpoint_for_readHubApi.get_endpoint_for_read  s`    JJNN,-=QWWY"_,q0H  PXcgot u O()BCC&~>   U#/^9K#M NNU7)??:K L??S>TTUW X,  ;OZ^fk ! m ,+""+  uWI-@
!LM  uWI-COCTTXYmXnops   C 9C: (C7:*D$)r  rp   c                    [        U5      u  pEU R                  XUS9nU R                  U[        X#S9nU R	                  XSUS9n[        S0 UDXtUS.D6n	U	$ )a  Get model information including commit history.

Args:
    repo_id (str): The model id in the format of
        ``namespace/model_name``.
    revision (str, optional): Specific revision of the model.
        Defaults to ``DEFAULT_MODEL_REVISION``.
    endpoint (str, optional): Hub endpoint to use. When ``None``,
        use the endpoint specified when initializing :class:`HubApi`.

Returns:
    ModelInfo: The model detailed information returned by
    ModelScope Hub with commit history.
)r   r  rp   r  r  r  rp   T)r   r  	recursiverp   commitsauthorsiblings )rN   r  list_repo_commitsrU   get_model_filesrD   )
r   r  r  rp   r   _
model_datar4  r6  
model_infos
             r   r<  HubApi.model_info-  s    & 9A^^( $ D
(( ) ^''4( ( T ggW^fg
r   c                    [        U5      u  pEU R                  XUS9nU R                  U[        X#S9nU R	                  X=(       d    [
        SUS9n[        S0 UDXtUS.D6n	U	$ )a  Get dataset information including commit history.

Args:
    repo_id (str): The dataset id in the format of
        ``namespace/dataset_name``.
    revision (str, optional): Specific revision of the dataset.
        Defaults to ``None``.
    endpoint (str, optional): Hub endpoint to use. When ``None``,
        use the endpoint specified when initializing :class:`HubApi`.

Returns:
    DatasetInfo: The dataset detailed information returned by
    ModelScope Hub with commit history.
)
dataset_idr  rp   r1  T)r  r  r2  rp   r3  r7  )rN   get_datasetr8  rT   get_dataset_filesrO   rC   )
r   r  r  rp   r   r:  dataset_datar4  r6  dataset_infos
             r   rC  HubApi.dataset_infoM  s    & 9A''H ( F(('88 ) `))&J2JVZem * o #m\m7dlmr   )r  r  rp   c                    Ub
  U[         :X  a  U R                  XUS9$ U[        :X  a  U R                  XUS9$ [	        SU S[
         S35      e)a  Get repository information for models or datasets.

Args:
    repo_id (str): The repository id in the format of
        ``namespace/repo_name``.
    revision (str, optional): Specific revision of the repository.
        Currently only effective for model repositories. Defaults to
        ``DEFAULT_MODEL_REVISION``.
    repo_type (str, optional): Type of the repository. Supported
        values are ``"model"`` and ``"dataset"``. If not provided,
        ``"model"`` is assumed.
    endpoint (str, optional): Hub endpoint to use. When ``None``,
        use the endpoint specified when initializing :class:`HubApi`.

Returns:
    Union[ModelInfo, DatasetInfo]: The repository detailed information
    returned by ModelScope Hub.
)r  r  rp   Arg repo_type z# not supported. Please choose from r   )rU   r<  rT   rC  r8   rV   )r   r  r  r  rp   s        r   	repo_infoHubApi.repo_infom  sj    4 	_ <??7PX?YY))$$WRZ$[[YK'JK\J]]^_a 	ar   r$  r   c                p   Uc  U R                   nUb&  UR                  5       [        ;  a  [        SU-  5      eUb  UR	                  S5      S:w  a  [        SU-  5      eU R                  USS9n[        U5      u  pxUb#  UR                  5       [        :X  a  U SU SU 3n	O
U SU SU 3n	U R                  R                  XU R                  U R                  5      S	9n
[        U
[        XaS5      nUS
:X  a  gUS:X  a  U(       a  [        U
5      eg[        R                  SU S35        [        SU-  5      e)a  
Checks if a repository exists on ModelScope

Args:
    repo_id (`str`):
        A namespace (user or an organization) and a repo name separated
        by a `/`.
    repo_type (`str`, *optional*):
        `None` or `"model"` if getting repository info from a model. Default is `None`.
        TODO: support studio
    endpoint(`str`):
        None or specific endpoint to use, when None, use the default endpoint
        set in HubApi class (self.endpoint)
    re_raise(`bool`):
        raise exception when error
    token (`str`, *optional*): access token to use for checking existence.
Returns:
    True if the repository exists, False otherwise.
zNot support repo-type: %sr   ru   z5Invalid repo_id: %s, must be of format namespace/nameFr   /api/v1/datasets/r   r     Ti  z%Check repo_exists return status code r   zOFailed to check existence of repo: %s, make sure you have access authorization.)rp   r  rV   r'  countr   rN   rT   r   r   r   r   r>   r  r   r  )r   r  r  rp   r   rq   r   r   r   r   r   codes               r   r&  HubApi.repo_exists  sS   8 }}H!y'8@Q'Q7)CDDOc 2a 7SV__``"""N;GD!y'8<M'MZ00@$HDZ~.>avFDLLT%)%9%9$,,%G  I#AvwG3;S[l"KK?vQGHa r   c                    [         R                  " S[        SS9  U(       d  U R                  nU[        :X  a  U R                  UUUS9  O+U[        :X  a  U R                  UUUS9  O[        SU S35      e[        R                  SU S	35        g
)a  
@deprecated
Delete a repository from ModelScope.

Args:
    repo_id (`str`):
        A namespace (user or an organization) and a repo name separated
        by a `/`.
    repo_type (`str`):
        The type of the repository. Supported types are `model` and `dataset`.
    endpoint(`str`):
        The endpoint to use. If not provided, the default endpoint is `https://www.modelscope.cn`
        Could be set to `https://ai.modelscope.ai` for international version.
    token (str): Access token of the ModelScope.
r  rt   r  )r?  rp   rq   )r   rp   rq   rF  z not supported.r!  z deleted successfully.N)r  r  r  rp   rT   delete_datasetrU   r  r'  r  info)r   r  r  rp   rq   s        r   delete_repoHubApi.delete_repo  s    * 	Z		
 }}H))"!   
 /) !  
 nYKGHHeG9$:;<r   c                 T   [         R                  R                  U [        R                  5      n[
        R                  [        R                  [
        R                  [        R                  0n[        US5       n[        R                  " X#5        S S S 5        g ! , (       d  f       g = f)Nw)r   r   joinr]   CONFIGURATIONrW   	frameworkr\   torchtaskr^   otheropenr   dump)	model_dircfg_filecfgfiles       r   _create_default_configHubApi._create_default_config  si    77<<	9+B+BC""J$4$4u{{
 (C DIIc  !  s   9B
B'zupload modelr^  commit_messager   ignore_file_pattern
lfs_suffixc           
       
   [         R                  " S[        SS9   Uc  [        S5      eUc  [        S5      e[        R
                  R                  U5      (       a$  [        R
                  R                  U5      (       a  [        S5      e[        R
                  R                  U[        R                  5      n[        R
                  R                  U5      (       d?  [        R                  S[        R                   SU S	35        [        R                  U5        U R                  US
S9nUc  [!        S5      e[        R"                  " U5      n[%        U5      nU
c  / n
['        U
[(        5      (       a  U
/n
Ub  Uc  [        S5      eU R+                  XS9(       d,  [        R-                  SU-  5        U R/                  UUUUU	US9  [        R
                  R                  U[0        5      n[3        5       n[        R-                  SU SU S35        [        R-                  SU S35         [5        UXS9nUR7                  U5      nUU;  a*  [        R-                  SU-  5        UR9                  UU5        UR;                  UU5        [        R"                  " U5      nU H  nUS   S:w  d  M  [        R
                  R                  UU5      n[        R
                  R                  U5      (       a  [        R<                  " U5        Mj  [>        R@                  " US
S9  M     U H  nUS   S:w  d  M  [C        U
 Vs/ s H  n[D        RF                  " UU5      S LPM     sn5      (       a  MH  [        R
                  R                  UU5      n[        R
                  RI                  U5      (       a7  [>        RJ                  " U[        R
                  R                  UU5      5        M  [>        RL                  " UU5        M     U(       d7  [N        RN                  RQ                  5       RS                  S5      nSU< SU< 3nUb4  ['        U[(        5      (       a  U/OUnU H  nURU                  U5        M     URW                  UUUS9  Ub  URY                  Xw5        [        R-                  SU SU S 35         [>        R@                  " US
S9  g s  snf ! [Z         a    e f = f! [>        R@                  " US
S9  f = f)!NzThis function is deprecated and will be removed in future versions. Please use git command directly or use HubApi().upload_folder insteadrt   r  zmodel_id cannot be empty!zmodel_dir cannot be empty!z$model_dir must be a valid directory.zNo z file found in z, creating a default one.Tr   zMust login before upload!z5Visibility and License cannot be empty for new model.rq   zCreating new model [%s])r   r   r   r   r   rq   zPushing folder z
 as model r   zTotal folder size z;, this may take a while depending on actual pushing size...)r^  
clone_from
auth_tokenzCreating new branch %sr   )ignore_errorsz%Y_%m_%d_%H_%M_%Sz[automsg] push model z to hub at )rd  local_branchremote_branchzSuccessfully push folder z to remote repo [z].).r  r  r  r8   r   r   existsisfilerV  r]   rW  r  r)  rn   rb  r   r:   listdirrK   
isinstancestrr&  rQ  r   r*   rB   rF   get_remote_branches
new_branchcheckoutremoveshutilrmtreeanyresearchisdircopytreecopydatetimenowstrftimeadd_lfs_typepushtag_and_pushr'  )r   r   r^  r   r   r   rd  r   r  r   re  rf  rq   r_  r   files_to_savefolder_sizetmp_dirgit_wrapperrepobranchesfiles_in_repofsrcpatterndatelfs_suffix_listsuffixs                               r   
push_modelHubApi.push_model  s0    	T		
.	^ "#>??"#?@@ww~~i((BGGNN9,E,E"#IJJ77<<	9+B+BCww~~h''NNi--.oi[Habd)))4"""M?#$?@@

9-.y9&"$)3//#6"7"#Z[[6KK1H<=!%)"3   '',,y*?@')oi[
8*AFG(5pqr*	7HWD"66w?Hx'4x?@&&w9  (3JJw/M"Q43;'',,w2Cww~~c**		#c> # #Q43;M`aM`'BIIgq1=M`abb '',,y!4Cww}}S))RWW\\'1-EFC1 # "((,,.778KLd"$%2<Z2M2M:,S]-F%%f- .II-%&  ( !!#+KK3I;>OPXzY[\] MM'63 b,  		 MM'6s9   -A?S 0BS 5	S >#S
!ES S S##S& &S=ru   
   r   page_number	page_sizec           
         U R                  USS9nU(       d  U R                  nU S3nU R                  R                  USU< SU< SU< S3UU R	                  U R
                  5      S9n[        U[        Xa5        UR                  [        R                  :X  aW  [        UR                  5       5      (       a  UR                  5       [           n	U	$ [        UR                  5       [           5      e[!        U5        g	)
a  List models in owner or group.

Args:
    owner_or_group(str): owner or group.
    page_number(int, optional): The page number, default: 1
    page_size(int, optional): The page size, default: 10
    endpoint: the endpoint to use, default to None to use endpoint specified in the class
    token (str, optional): access token for authentication

Raises:
    RequestError: The request error.

Returns:
    dict: {"models": "list of models", "TotalCount": total_number_of_models_in_owner_or_group}
Fr   r   z	{"Path":"z", "PageNumber":z, "PageSize": })datar   r   N)r   rp   r   putr   r   r>   r  r  r   r  r?   r   r   r;   r   r@   )
r   r   r  r  rp   rq   r   r   r   r  s
             r   list_modelsHubApi.list_models  s    * """N}}H?+LL +y:((6  8 	Q@==JMM)QVVXvvx 78"1668,F#GHH!!$r   )r  r  sortr{  rp   rq   r  r{  c                   U(       d  U R                   nU S3nUUS.n	U(       a,  U[        ;  a  [        SU S[        [        5       35      eXIS'   U(       a  XYS'   U(       a  XS'   U R	                  USS	9n
U R                  U R                  5      nU R                  R                  UU	U
US
9n[        U5        UR                  5       nUR                  S5      SL a  SU;   a  US   $ UR                  S5      =(       d    Sn[        U5      e)a  List datasets via OpenAPI with pagination, filtering and sorting.

Args:
    owner_or_group (str): Search by dataset authors (including organizations and individuals).
    page_number (int, optional): The page number. Defaults to 1.
    page_size (int, optional): The page size. Defaults to 10.
    sort (str, optional): Sort key. If not provided, the server's default sorting is used.
        choose from ['default', 'downloads', 'likes', 'last_modified'].
    search (str, optional): Search by substring keywords in the dataset's Chinese name,
        English name, and authors (including organizations and individuals).
    endpoint (str, optional): Hub endpoint to use. When None, use the endpoint specified in the class.
    token (str, optional): Access token for authentication.

Returns:
    dict: The OpenAPI data payload, e.g.
          {
            "datasets": [...],
            "total_count": int,
            "page_number": int,
            "page_size": int
          }
z/openapi/v1/datasets)r  r  zInvalid sort key: z. Supported sort keys: r  r{  r5  Fr   paramsr   r   successTr  messagezFailed to list datasets)rp   r2   r8   listr   r   r   r   r   r@   r   r;   )r   r   r  r  r  r{  rp   rq   r   r  r   r   r   respmsgs                  r   list_datasetsHubApi.list_datasets  s#   > }}H/0 '""
 ?*&(.Ed?F[E\]_ _!6N%8-8"""N&&t||4LL	  
 	a vvx 88I$&6T>< ((9%B)BCs##r   use_cookiesc                     S n[        U[        5      (       a  UnU$ [        U[        5      (       a  U R                  US9nU$ )N)r   )rq  r   boolr   )r   r  r   s      r   _check_cookieHubApi._check_cookie  sG    k9--!G  T**&&&DGr   cutoff_timestampc                 p    U R                  UUUS9nU(       a  U Vs/ s H  oUS   PM	     nnU$ / nU$ s  snf )a  Get model branch and tags.

Args:
    model_id (str): The model id
    cutoff_timestamp (int): Tags created before the cutoff will be included.
                            The timestamp is represented by the seconds elapsed from the epoch time.
    use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                will load cookie from local. Defaults to False.

Returns:
    Tuple[List[str], List[str]]: Return list of branch name and tags
)r   r  r  Revision)list_model_revisions_detail)r   r   r  r  tags_detailsxtagss          r   list_model_revisionsHubApi.list_model_revisions  s\    " 77IYDO 8 Q " (4 |!*|  (* 	s   3c                 Z   U R                  U5      nUc
  [        5       nU(       d  U R                  nU SU S3U-  nU R                  R	                  XeU R                  U R                  5      S9n[        U[        XQ5        UR                  5       n[        U5        U[           n	U	S   S   $ )aE  Get model branch and tags.

Args:
    model_id (str): The model id
    cutoff_timestamp (int): Tags created before the cutoff will be included.
                            The timestamp is represented by the seconds elapsed from the epoch time.
    use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                will load cookie from local. Defaults to False.
    endpoint: the endpoint to use, default to None to use endpoint specified in the class

Returns:
    Tuple[List[str], List[str]]: Return list of branch name and tags
r   z/revisions?EndTime=%sr  RevisionMapTags)r  rL   rp   r   r   r   r   r>   r  r   rA   r   )
r   r   r  r  rp   r   r   r   r   rQ  s
             r   r  "HubApi.list_model_revisions_detail%  s    & $$[1#35}}H?8*4IJM]]LLT%)%9%9$,,%G  IQ:FFHq()M"6**r   c                 0    U H  nUS   U:X  d  M  Us  $    g )Nr  r7  )r   detailsr   items       r   get_branch_tag_detailHubApi.get_branch_tag_detailG  s#    DJ4'  r   r   c           	         U(       d  U R                   n[        5       n[        [        [        R                  R                  5       R                  5       5      5      nU R                  Xc  SOUUS9u  pxU(       a  U V	s/ s H  oS   PM	     sn	O/ n
U(       a  U V	s/ s H  oS   PM	     sn	O/ nXV[        -   :  a~  Uc  [        n[        R                  SU-  5        X*;  a  X+;  a  [        SU< SU< S35      eU R                  X5      nUc  U R                  Xr5      n[        R                  SU-  5        U$ Ub  X*;   a  U R                  Xr5      nU$ [        U5      S	:X  a;  Ub
  U[        :X  a  [        nO[        SU< S
U< S35      eU R                  Xr5      nU$ Uc  U(       a  U V	s/ s H  n	U	S   U::  d  M  U	PM     sn	O/ n[        U5      S	:  a  US	   S   nUS	   nOC[        nU R                  Xr5      nSSR!                  U5      -  n[        R#                  SU-  5        [        R#                  SU-  5        U$ X+;  a]  U[        :X  a'  [        R#                  S5        U R                  Xr5      nO=SSR!                  U5      -  n[        SU< S
U< SU< S35      eU R                  X5      n[        R                  SU-  5        U$ s  sn	f s  sn	f s  sn	f )NF)r  rp   r  z9Model revision not specified, using default [%s] version.zThe model: z has no revision : z .z!Development mode use revision: %sr   z has no revision:  !	CreatedAtz[%s],z5Model revision should be specified from revisions: %sz.Model revision not specified, use revision: %sz?Using the master branch is fragile, please use it with caution!z valid are: !z%Use user-specified model revision: %s)rp   rL   introundr  r  	timestamp#get_model_branches_and_tags_detailsr(   rR   r  rQ  r9   r  debuglenrV  r)  )r   r   r  r   rp   release_timestampcurrent_timestampall_branches_detailall_tags_detailr  all_branchesall_tagsrevision_detailrevisions_detailvls                  r   get_valid_revision_detail HubApi.get_valid_revision_detailM  s   
 }}H02h&7&7&;&;&=&G&G&I JK 04/W/W?%RZ 0X 0\,GZ/BC/B!*/BC`b?N?;?ajM?;TV3CCC.O  +0H#hX`$abb"88SO&"&"<"<=P"[LL<xGHN K #(@"&"<"<=P"[&&?#q(#x3F'F2H'QY[c(dee"&"<"<=P"[8 5 # Q` )8(M(7 /0()+:K(K )*(7(Meg % +,q0#3A#6z#B*:1*=#6*.*D*DEX*c#chhx&88'^bd'efNN#SV^#^_   /#'::"NN+lm.2.H.HI\.gO!'#((8*<!<B"/198R1I #J J +/*D*D_*_KK G( RSk D;6(Ms   >KK,K=Kc                 ,    U R                  UUUUS9S   $ )N)r   r  r   rp   r  )r  )r   r   r  r   rp   s        r   get_valid_revisionHubApi.get_valid_revision  s6    
 --x7?6=7? . A BLM 	Mr   c                 J   U R                  U5      nU(       d  U R                  nU SU S3nU R                  R                  XTU R	                  U R
                  5      S9n[        U[        XA5        UR                  5       n[        U5        U[           nUS   S   US   S   4$ )a  Get model branch and tags.

Args:
    model_id (str): The model id
    use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                will load cookie from local. Defaults to False.
    endpoint: the endpoint to use, default to None to use endpoint specified in the class

Returns:
    Tuple[List[str], List[str]]: Return list of branch name and tags
r   z
/revisionsr  r  Branchesr  )r  rp   r   r   r   r   r>   r  r   rA   r   )	r   r   r  rp   r   r   r   r   rQ  s	            r   r  *HubApi.get_model_branches_and_tags_details  s    " $$[1}}H?8*J?LLT%)%9%9$,,%G  IQ:FFHq()M":.]0CF0KKKr   c                     U R                  UUS9u  p4U(       a  U Vs/ s H  oUS   PM	     snO/ nU(       a  U Vs/ s H  oUS   PM	     snO/ nXg4$ s  snf s  snf )a9  Get model branch and tags.

Args:
    model_id (str): The model id
    use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                will load cookie from local. Defaults to False.

Returns:
    Tuple[List[str], List[str]]: Return list of branch name and tags
)r   r  r  )r  )r   r   r  branches_detailtags_detailr  r  r  s           r   get_model_branches_and_tags"HubApi.get_model_branches_and_tags  s     (,'O'OYa\g (P (i$ ) ,; ?ajM? .0 	 ! (3 {!*{ &( 	~	s
   AArootr2  r   c                    U(       d  U R                   nU(       a  U< SU< SU< SU< 3nOU< SU< SU< 3nU R                  U5      n	Ub  USU 3-   nUc  U R                  OUn[        [        R
                  " 5       R                  5      US'   U R                  R                  XUS9n
[        U
[        X5        U
R                  5       n[        U5        / nU[           S   (       d  [        R                  S	U S
U S35        U$ U[           S    H(  nUS   S:X  d	  US   S:X  a  M  UR                  U5        M*     U$ )ac  List the models files.

Args:
    model_id (str): The model id
    revision (Optional[str], optional): The branch or tag name.
    root (Optional[str], optional): The root path. Defaults to None.
    recursive (Optional[bool], optional): Is recursive list files. Defaults to False.
    use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                will load cookie from local. Defaults to False.
    headers: request headers
    endpoint: the endpoint to use, default to None to use endpoint specified in the class

Returns:
    List[dict]: Model file list.
r   z/repo/files?Revision=&Recursive=z/repo/files?Recursive=z&Root=zX-Request-IDr  FileszNo files found in model z at revision r   r   z
.gitignorez.gitattributes)rp   r  r   rr  uuiduuid4hexr   r   r>   r  r   rA   r   r)  append)r   r   r  r  r2  r  r   rp   r   r   r   r   filesra  s                 r   r9  HubApi.get_model_files  s:   . }}H(Hi9D (I/D$$[1F4&/)D")/$,,w"%djjl&6&6"7LL7  4 	Q:FFHq()'2NN5hZ}XJVWXYL-.w7DF||+tF|?O/OLL	 8
 r   )r  rq   filenamec                    U R                  US9nU R                  USUUc  SOUS9nU Vs/ s H  owS   PM	     nnX&;   $ s  snf )a^  Get if the specified file exists

Args:
    repo_id (`str`): The repo id to use
    filename (`str`): The queried filename, if the file exists in a sub folder,
        please pass <sub-folder-name>/<file-name>
    revision (`Optional[str]`): The repo revision
    token (`Optional[str]`): The access token
Returns:
    The query result in bool value
r   TF)r2  r  r  r   )r   r9  )r   r  r  r  rq   r   r  ra  s           r   file_existsHubApi.file_exists  sd    & """6$$!(g	 % 
 +00%$f%0   1s   ?dataset_name	namespacer   c	                    Ub  Uc  [        S5      eU R                  USS9n	U(       d  U R                  nU S3n
SU4SU4SU4SU4SU4SU4S.nU R                  R	                  U
UU	U R                  U R                  5      S9n[        XU5        [        UR                  5       5        U SU S	U 3n[        R                  S
U 35        U$ )a  
Create a dataset in ModelScope.

Args:
    dataset_name (str): The name of the dataset.
    namespace (str): The namespace (user or organization) for the dataset.
    chinese_name (str, optional): The Chinese name of the dataset. Defaults to ''.
    license (str, optional): The license of the dataset. Defaults to Licenses.APACHE_V2.
    visibility (int, optional): The visibility of the dataset. Defaults to DatasetVisibility.PUBLIC.
    description (str, optional): The description of the dataset. Defaults to ''.
    endpoint (str, optional): The endpoint to use. If not provided, the default endpoint is used.
    token (str, optional): The access token for authentication.

Returns:
    str: The URL of the created dataset repository.
Nz(dataset_name and namespace are required!Tr   z/api/v1/datasets)r   r   Ownerr   r6   Description)r  r   r   z
/datasets/r   zCreate dataset success: )r8   r   rp   r   r   r   r   r=   rA   r   r  rQ  )r   r  r  r   r   r   r   rp   rq   r   r   r  r   dataset_repo_urls                 r   create_datasetHubApi.create_dataset#  s   4 9#4"#MNN"""M}}H+,<( ,/I&g, +.
 LL((6	  
 	q.qvvx &Zz)Al^L./?.@ABr   r?  c                 X   [         R                  " S[        SS9  U R                  USS9nU(       d  U R                  nUc  [        S5      eU SU 3nU R                  R                  UUU R                  U R                  5      S	9n[        U5        [        UR                  5       5        g)
a  
@deprecated
Delete a dataset from ModelScope.

Args:
    dataset_id (str): The dataset id to delete.
    endpoint (str, optional): The endpoint to use. If not provided, the default endpoint is used.
    token (str, optional): The access token for authentication.

Returns:
    None
r  rt   r  Tr   Nr
  rJ  r  r  )r   r?  rp   rq   r   r   r   s          r   rP  HubApi.delete_dataset[  s      	Z		
 """M}}H?HII,ZL9LL(/(,(<(<T\\(J   L 	a qvvx r   c                     U(       d  U R                   nU SU SU 3nU R                  US9nU R                  R                  XVS9nUR	                  5       n[        XXU5        US   S   n	US   S   n
X4$ )zGet the dataset id and type. rJ  r   r   r   DataIdType)rp   r   r   r   r   r<   )r   r  r  rp   rq   datahub_urlr   r   r  r?  dataset_types              r   get_dataset_id_and_typeHubApi.get_dataset_id_and_type~  s     }}H!
"3I;a~N"""6LL[:vvx{!4&\$'
F|F+''r   Td   	root_pathc
                 j   [         R                  " S[        5        U R                  XXS9u  pU(       a  SOSnU(       d  U R                  nU SU
 S3nU(       a  UOSU(       a  UOSUXgS	.nU R                  U	S
9nU R                  R                  XUS9nUR                  5       n[        UUU5        U$ )z/
@deprecated: Use `get_dataset_files` instead.
zMThe function `list_repo_tree` is deprecated, use `get_dataset_files` instead.r  r  rp   rq   TrueFalserJ  
/repo/treer   r   r  Root	Recursive
PageNumberPageSizer   r  r   )
r  r  r  r  rp   r   r   r   r   r<   )r   r  r  r  r  r2  r  r  rp   rq   dataset_hub_idr  r  r  r   r   r  s                    r   list_repo_treeHubApi.list_repo_tree  s     	e(	* (,'C'C%X (D (\$ (FW	}}H!
"3N3C:N*2h'0)c	 +D """6LL[Ivvx{D!4r   2   )r  r  r  r  rp   rq   c          	         [        U5      (       a)  UR                  S5      S:X  a  UR                  S5      u  pO[        SU S35      eU(       d  U R                  nU(       a  U SU SU S3OU SU S3n
U=(       d    [
        UUS	.nU R                  US
9n U R                  R                  XXR                  U R                  5      S9n[        U5        UR                  5       n[        U5        UR                  S5      [        R                  :X  a  [         R"                  " U5      $ g! [$        R&                  R(                   a  n[+        SU S[-        U5       35      eSnAff = f)a  
Get the commit history for a repository.

Args:
    repo_id (str): The repository id, in the format of `namespace/repo_name`.
    repo_type (Optional[str]): The type of the repository. Supported types are `model` and `dataset`.
    revision (str): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`.
    page_number (int): The page number for pagination. Defaults to 1.
    page_size (int): The number of commits per page. Defaults to 50.
    endpoint (Optional[str]): The endpoint to use, defaults to None to use the endpoint specified in the class.
    token (Optional[str]): The access token.

Returns:
    CommitHistoryResponse: The commit history response.

Examples:
    >>> from modelscope.hub.api import HubApi
    >>> api = HubApi()
    >>> commit_history = api.list_repo_commits('meituan/Meeseeks')
    >>> print(f"Total commits: {commit_history.total_count}")
    >>> for commit in commit_history.commits:
    ...     print(f"{commit.short_id}: {commit.title}")
r   ru   Invalid repo_id: r  z/api/v1/s/z/commitsr   )r   r  r  r   r  Codez%Failed to get repository commits for : N)rb   rL  r   r   rp   rQ   r   r   r   r   r   r@   r   rA   r   r  rg   from_api_responserequests
exceptionsRequestExceptionr'  rr  )r   r  r  r  r  r  rp   rq   _owner_dataset_namecommits_urlr  r   r   r  es                   r   r8  HubApi.list_repo_commits  sf   B G$$s);q)@$+MM#$6!FM0	<==}}HMV
(9+RyIjy9 	 ::%!

 """6	Y  )0:N:Nt||:\ ! ^A!!$668D4 xx:==0,>>tDD 1 ""33 	YCG9BsSTvhWXX	Ys   BD, ,E)
E$$E)r   )r  r  r2  r  r  rp   rq   c                   [        U5      (       a)  UR                  S5      S:X  a  UR                  S5      u  pO[        SU S35      eU R	                  XXxS9u  pU(       d  U R
                  nU SU S3nUUU(       a  SOS	UUS
.nU R                  US9nU R                  R                  XUS9nUR                  5       n[        UUU5        US   S   $ )a  
Get the dataset files.

Args:
    repo_id (str): The repository id, in the format of `namespace/dataset_name`.
    revision (str): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`.
    root_path (str): The root path to list. Defaults to '/'.
    recursive (bool): Whether to list recursively. Defaults to True.
    page_number (int): The page number for pagination. Defaults to 1.
    page_size (int): The number of items per page. Defaults to 100.
    endpoint (Optional[str]): The endpoint to use, defaults to None to use the endpoint specified in the class.
    token (Optional[str]): The access token.

Returns:
    List: The response containing the dataset repository tree information.
        e.g. [{'CommitId': None, 'CommitMessage': '...', 'Size': 0, 'Type': 'tree'}, ...]
r   ru   r  r  r  rJ  r  r  r  r  r   r  r  r  )rb   rL  r   r   r  rp   r   r   r   r   r<   )r   r  r  r  r2  r  r  rp   rq   r  r  r  r  r  r  r   r   r  s                     r   rA  HubApi.get_dataset_files  s    8 G$$s);q)@$+MM#$6!FM0	<=='+'C'C&8 (D (Z$ }}H!
"3N3C:N #,'%!
 """6LL[Ivvx{D!4F|G$$r   c                 >   U R                  US9nU(       d  U R                  nU(       a  U SU SU 3nOU SU 3nU R                  R                  XeU R	                  U R
                  5      S9n[        U5        UR                  5       n[        XhU5        U[           $ )aP  
Get the dataset information.

Args:
    dataset_id (str): The dataset id.
    revision (Optional[str]): The revision of the dataset.
    endpoint (Optional[str]): The endpoint to use, defaults to None to use the endpoint specified in the class.
    token (Optional[str]): The access token.

Returns:
    dict: The dataset information.
r   rJ  r  r  )
r   rp   r   r   r   r   r@   r   r<   r   )	r   r?  r  rp   rq   r   r   r   r  s	            r   r@  HubApi.get_dataset&  s    & """6}}HZ0JxjQDZ0=DLL4+?+?+M  Oa vvxt1-+,,r   c           	      B   U(       d  U R                   nU SU SU 3nU R                  US9nU R                  R                  UUU R	                  U R
                  5      S9n	U	R                  5       n
[        XzU	5        U
S   nUc  [        SU SU SU S	35      eUS
   nU$ )z'Get the meta file-list of the dataset. rJ  z/repo/tree?Revision=r   r  r  z'The modelscope dataset [dataset_name = z, namespace = z, version = z] dose not existr  )	rp   r   r   r   r   r   r   r<   r9   )r   r  r  r?  r  rp   rq   r  r   r   r  	file_lists               r   get_dataset_meta_file_list!HubApi.get_dataset_meta_file_listI  s     }}H!
"3J<?ST\S]^"""6LL[%,%)%9%9$,,%G  I vvx{!4L	9,~V_U` a%J&689 9 g&	r   r  meta_cache_dirc                     [         R                  R                  U[        U 5       [        R
                  R                   35      n[        US5       nUR                  S5        SSS5        g! , (       d  f       g= f)z
Dump the data_type as a local file, in order to get the dataset
 formation without calling the datahub.
More details, please refer to the class
`modelscope.utils.constant.DatasetFormations`.
rU  z3*** Automatically-generated file, do not modify ***N)	r   r   rV  rr  rX   formation_mark_extvaluer\  write)r  r(  dataset_type_file_pathfps       r   dump_datatype_fileHubApi.dump_datatype_file_  s`     "$n14\1B0CDUDhDhDnDnCo.p"r(#."HHJK /..s   A--
A;r%  c	           
      $   [        [        5      n	[        U5      n
[        U
   nU R	                  US9n[
        R                  XTS9  U(       d  U R                  nU GH  nUS   n[        R                  R                  U5      S   nX;   d  M2  U SU SU SU SU 3	nU R                  R                  UUS	9n[        U5        [        R                  R                  XN5      n[        R                  R                  U5      (       a1  [         R#                  S
U SU S35        X   R%                  U5        M  ['        US5       nUR)                  UR*                  5        S S S 5        X   R%                  U5        GM!     X4$ ! , (       d  f       N)= f)Nr   )r  r(  r   r   rJ  r   /repo?Revision=
&FilePath=r  zReusing dataset z's python file ()wb)r   r  rX   rY   r   rn   r/  rp   r   r   splitextr   r   r@   rV  rn  r  r)  r  r\  r,  content)r   r  r  r  r(  r  r%  rp   rq   local_pathsdataset_formationdataset_meta_formatr   	file_info	file_path	extensionr  r   
local_pathr  s                       r   "get_dataset_meta_files_local_paths)HubApi.get_dataset_meta_files_local_pathsl  sw    "$'-l;01BC"""6 	!!|![}}H"I!&)I((3B7I/!)
*;I;a~ V**2:i[JLL$$['$B%a(WW\\.D
77>>*--NN*<.8HTUV  *11*=*d+qGGAII& ,&--j9# #& --	 ,+s   F
F	   c           	        ^ SSK nSSKJn  SSKn[        R
                  R                  XR                  U R                  SS95      R                  5       5      nU[        R                  :X  a:  [        R
                  R                  U5      (       a  [        R                  " U5        [        R
                  R                  U5      (       a  [        R                  SU 35        U$ [!        5       R#                  US9n[        R                  S5        [$        R&                  " XS	S
9n	[)        U	R*                  R'                  SS5      5      n
U" U
S	S9nU4S jnSn[-        US5       nU" U	5       H  nUR/                  [1        U5      5        U R3                  S5      (       a  U Vs/ s H0  nUR5                  5       (       d  M  [6        R8                  " U5      PM2     nn[1        U5      S:X  a  M  US:X  a  S	nOSnUR;                  U5      nUR=                  USUSS9  US-  nM  U H  nUR?                  US-   5        M     M     SSS5        URA                  5         U$ s  snf ! , (       d  f       N%= f)z?
Fetch the meta-data files from the url, e.g. csv/jsonl files.
r   Nr   zUTF-8encodingzReusing cached meta-data file: r   zLoading meta-data file ...T)r   streamzcontent-length)rw   dynamic_ncolsc              3      >#    / nU R                  5        H<  nUR                  S5      nUR                  U5        [        U5      T:  d  M6  Uv   / nM>     Uv   g 7f)Nutf-8)
iter_linesdecoder  r  )r  
chunk_datar  
chunk_sizes      r   	get_chunk3HubApi.fetch_meta_files_from_url.<locals>.get_chunk  sW     J){{7+!!$'z?j0$$!#J * s   AAAajsonlF\)indexheader
escapecharru   
)!hashlib	tqdm.autor   pandasr   r   rV  md5encode	hexdigestr[   FORCE_REDOWNLOADrn  rv  r  rQ  rn   r   r  r   r  r   r\  r   r  r   r%  r   loads	DataFrameto_csvr,  close)urlout_pathrL  moderq   rV  r   pdr   response
total_sizeprogressrM  iter_numr  chunklinewith_headerchunk_dfs     `                r   fetch_meta_files_from_url HubApi.fetch_meta_files_from_url  s    	"77<<++cjj'j6R*S*]*]*_`<000RWW^^H5M5MIIh77>>(##KK9(DEO(&&E&: 	01<<TB))--.>BC
j=	 (C A"8,E
+<<((:?P%$4::<-TZZ-%EP5zQ 1}&*&+!||E2HOOAU;SWOXMH !&t, !& - !$ 	! Q	 ! s&   &A I%&I I A)I% I%%
I3	file_nameviewextension_filterc                     U(       a  U(       a  U(       d  [        S5      eSXAUS.n[        U5      nU(       d  U R                  nU SU SU SU 3n	U	$ )Nz:Args (file_name, dataset_name, namespace) cannot be empty!SDK)Sourcer  FilePathViewrJ  r   /repo?)r   r   rp   )
r   ro  r  r  r  rp  rq  rp   r  file_urls
             r   get_dataset_file_urlHubApi.get_dataset_file_url  s`     IYZZ #(X^bc'}}HZ01\N&QWPXYr   c           	          U(       d  U R                   nU(       a:  [        R                  R                  U5      S   [        ;   a  U SU SU SU SU 3	nU$ )Nr   rJ  r   r2  r3  )rp   r   r   r6  rS   )r   ro  r  r  r  rp   s         r   get_dataset_file_url_origin"HubApi.get_dataset_file_url_origin  s`     }}H)))4R8<MM#*$5i[, P$$,:Z	{DIr   c                 `    U(       d  U R                   nU SU SU SU 3nU R                  XeS9$ )NrJ  r   /ststoken?Revision=rh  )rp   datahub_remote_call)r   r  r  r  rp   rq   r  s          r   get_dataset_access_config HubApi.get_dataset_access_config  sH     }}H!
"3I;a~ N++3*6'''AAr   check_cookiec                 6   U(       d  U R                   nU SU SU SU 3nU(       a  U R                  SS9nOU R                  US9nU R                  R	                  UUU R                  U R                  5      S9n	U	R                  5       n
[        U
5        U
S   $ )	NrJ  r   r  T)r  r   ra  r   r   r  )	rp   r  r   r   r   r   r   r   rA   )r   r  r  r  r  rp   rq   r  r   r   r  s              r   !get_dataset_access_config_session(HubApi.get_dataset_access_config_session  s     }}H!
"3I;a~ N++3*6((T(:G&&E&:GLL((6  8 vvxtF|r   versionc                    [         R                  R                  [        R                  S5      nU(       d  [        S[        R                   35      eU S3n[        R                  R                  U R                  US95      n[        UUS9n[        US9nU R                  R                  UUUU R                  U R                  5      SS9n	U	R                  5       n
U
S	   S
:w  a  [        SU
 35      eU
S   $ )z
Get virgo dataset meta info.
r   z"Virgo endpoint is not set in env: z/data/set/downloadr   )	dataSetIddataSetVersion)r  i  )ra  r   r   r   r   rM  r   zFailed to get virgo dataset: r  )r   r   r   r_   env_virgo_endpointRuntimeErrorr  utilsdict_from_cookiejarr   dictr   r   r   r   r   )r   r?  r  rq   virgo_endpointvirgo_dataset_urlr   rC  r  r   r  s              r   get_virgo_metaHubApi.get_virgo_meta  s     (:(M(MrR!CDVDiDiCjkll-..@A..44T5E5ESX5E5YZ "
 
 LL"3#'&-&*&:&:4<<&H&)	  +
 vvx<1!>tfEFFF|r   zip_file_namec                 "   U(       d  U R                   nU SU SU 3nU R                  US9nU R                  R                  XxU R	                  U R
                  5      S9n	U	R                  5       n
[        U
5        U
S   n[        R                  " US   5      nU SU 3nU R                  R                  XU R	                  U R
                  5      S9nUR                  5       n[        U5        US   nUS-   S-   U-   S	-   U-   S	-   U-   nUUS
'   U$ )NrJ  r   r   r  r  r6   r  z	-unzippedr:  Dir)	rp   r   r   r   r   r   r   rA   r7   )r   r  r  r  r  rp   rq   r  r   r   r  r  r   datahub_sts_urlr_stsresp_stsdata_stsfile_dirs                     r   &get_dataset_access_config_for_unzipped-HubApi.get_dataset_access_config_for_unzipped;  s-    }}H!
"3I;a~N"""6LL%)%9%9$,,%G  IvvxtF|"&&tL'9:
(M)<XJG  _)-)=)=dll)K ! M::<x F#+c1I=ClRUXX[hh"r   c	                     U(       d  U R                   nU SU SU SU SU SU SU 3n	U R                  US9n
U R                  R                  XSS	9nUR	                  5       n[        U5        US
   nU$ )NrJ  r   z/oss/tree/?MaxLimit=
&Revision=r  z&FilterDir=r   i  )ra  r   r   r  )rp   r   r   r   r   rA   )r   r  r  	max_limitis_recursiveis_filter_dirr  rp   rq   ra  r   r  s               r   list_oss_dataset_objectsHubApi.list_oss_dataset_objectsX  s     }}H
+I;a~ F#JxjL>Q\]j\km """6||C$Gyy{tF|r   object_namec           	      ,   U(       a  U(       a  U(       a  U(       d  [        S5      eU(       d  U R                  nU SU SU SU SU 3	nU R                  USS9nU R                  R	                  XxS9n	U	R                  5       n	[        U	5        U	S	   n	U	$ )
NArgs cannot be empty!rJ  r   z
/oss?Path=r  Tr   ra  r   Messager   rp   r   r   r  r   rA   
r   r  r  r  r  rp   rq   ra  r   r  s
             r   delete_oss_dataset_object HubApi.delete_oss_dataset_objectg  s     ,ix455}}H
+I;a~ZP[}\fgofpq"""M||""s"<yy{tIr   c           	      ,   U(       a  U(       a  U(       a  U(       d  [        S5      eU(       d  U R                  nU SU SU SU SU 3	nU R                  USS9nU R                  R	                  XxS9n	U	R                  5       n	[        U	5        U	S	   n	U	$ )
Nr  rJ  r   z/oss/prefix?Prefix=z/&Revision=Tr   r  r  r  r  s
             r   delete_oss_dataset_dirHubApi.delete_oss_dataset_dirw  s     ,ix455}}H
+I;a~EXYdXe f#*& """M||""s"<yy{tIr   c                     U R                  US9nU R                  R                  UUS[        R	                  5       0S9nUR                  5       n[        XU5        US   $ )Nr   rs   r  r  )r   r   r   r   r   r   r<   )r   ra  rq   r   r   r  s         r   r  HubApi.datahub_remote_call  sh    """6LL!#3#B#B#DE  G vvxs!,F|r   use_streamingc           	      ^   [         R                  " S5      S:H  nU(       d  U R                  nU(       GaM  U(       GaD  U(       Gd;  U(       Gd2   U R                  US9nU SU SU S3nU R                  R                  XU R                  U R                  5      S9n	[        U	5        [        R                  R                  n
Sn[        [         R                  ;   a  [         R                  [           n
[        [         R                  ;   a  [         R                  [           nU SU SU S	U
 S
U 3	nU R                  R                  XU R                  U R                  5      S9nUR                  5       n[!        U5        g g g g g ! ["         a  n[$        R'                  U5         S nAg S nAff = f)NCI_TESTr  r   rJ  r   z/download/increaser  r   z/download/uv/z?user=)r   getenvrp   r   r   r   r   r   r@   rZ   LOCALr+  r!   r   r"   r   rA   r'  r  r(  )r   r  r  r  rp   rq   
is_ci_testr   download_count_urldownload_count_respchannel	user_namedownload_uv_urldownload_uv_respr  s                  r   dataset_download_statistics"HubApi.dataset_download_statistics  s    YYy)V3
}}HIj ***> )1z1B9+Q|n\n%o"&*ll&7&78J@D@T@TUYUaUa@b '8 'd#%&9: *//55	/2::= jj)EFG,

: "

+D EI%-J.?	{!L>Yf%,IVI;#@ $(<<#4#4_=A=Q=QRVR^R^=_ $5 $a #3#8#8#: /0/ BOjI<2   Q s   D0F 
F,F''F,c                 b    [         [        [        R                  " 5       R                  5      0UE$ N)r&   rr  r  r  r  )r   r   s     r   r   HubApi.builder_headers  s*    %s4::<+;+;'<  	r   c                 d    UR                  S5      u  p4U(       d  U R                  nU SU SU S3$ )Nr   rJ  rw  )r   rp   )r   r  rp   
_namespacer  s        r   get_file_base_pathHubApi.get_file_base_path  s9    $+MM#$6!
}}H,ZL-OOr   )	rq   r   r  r   r   rp   exist_okcreate_default_configr   r  r  c       	   	      X   U(       d  [        S5      eU(       d  U R                  nU R                  XXrS9nU(       a=  U(       a'  U SU SU 3n[        R	                  SU SU 35        U$ [        SU S35      eUR                  S5      n[        U5      S:w  a  [        S	5      eUu  nnU[        :X  Ga  [        R                  R                  5        VVs0 s H!  u  nnUR                  S
5      (       a  M  UU_M#     nnnUR                  UR                  5       5      nUc  [        SU S35      eU R                  UUUUU
US9nU	(       ar  [        R                   " 5        nSSKJn  U" UXS9nSSSS.nUR                  S5      nU(       d  0 n0 UEUEn['        US[(        R*                  " U5      /SS9  SSS5        [-        SU S3SS9  U$ U[.        :X  a  [0        R                  R                  5        VVs0 s H!  u  nnUR                  S
5      (       a  M  UU_M#     nnnUR                  UR                  5       5      nUc  [        SU S35      eU R3                  UUUUUUS9n[-        SU S3SS9  U$ [        SU S[4         35      es  snnf ! , (       d  f       N= fs  snnf ) a  
Create a repository on the ModelScope Hub.

Args:
    repo_id (str): The repo id in the format of `owner_name/repo_name`.
    token (Union[str, bool, None]): The access token.
    visibility (Optional[str]): The visibility of the repo,
        could be `public`, `private`, `internal`, default to `public`.
    repo_type (Optional[str]): The repo type, default to `model`.
    chinese_name (Optional[str]): The Chinese name of the repo.
    license (Optional[str]): The license of the repo, default to `apache-2.0`.
    endpoint (Optional[str]): The endpoint to use.
        In the format of `https://www.modelscope.cn` or 'https://www.modelscope.ai'
    exist_ok (Optional[bool]): If the repo exists, whether to return the repo url directly.
    create_default_config (Optional[bool]): If True, create a default configuration file in the model repo.
    **kwargs: The additional arguments.

Returns:
    str: The repo url.
Repo id cannot be empty!r#  r   r  r!  z already exists, got repo url: z already exists!rt   zBInvalid repo id, should be in the format of `owner_name/repo_name`__NInvalid visibility: z9, supported visibilities: `public`, `private`, `internal`)r   r   r   r   r   rq   r   rE   )rj  pytorchztext-generationT)rX  rZ  allow_remoteconfig_jsonzconfiguration.json)ignore_push_errorz"New model created successfully at r   flush)r  r  r   r   r   rq   z$New dataset created successfully at Invalid repo type: , supported repos: )r   rp   r&  r  r)  r   r  rU   r5   __dict__items
startswithr   upperr   tempfileTemporaryDirectorymodelscope.hub.repositoryrF   rH   r   dumpsprintrT   r3   r  rV   )r   r  rq   r   r  r   r   rp   r  r  r   kwargsr&  repo_urlrepo_id_listr  	repo_namekvvisibilitiestemp_cache_dirrF   r  default_configr  configs                             r   create_repoHubApi.create_repo  s   H 788}}H ,,WT\,j#+*Ai[7) Dwi/NxjYZ 5	1A!BCC}}S)|!abb+	9'-<-E-E-K-K-Mh-MTQUVUaUabfUgDAqD-MLh*..z/?/?/ABJ! #7
| D[ "\ ] ] -- %)% . H %002nD%ngPD%. 1(,&N
 #)**]";K&&(>>+>F',tzz&/A.B*.0 3  6xjB$O* ' ++->-G-G-M-M-Oj-OTQWXWcWcdhWiDAqD-OLj*..z/?/?/ABJ! #7
| D[ "\ ] ] //&#)% 0 H 8
!DDQ
  29+=PQbPcdeec i 32& ks%   J7J"AJ:J&J&
J#      )commit_descriptionrq   r  r  rp   r}   r   
operationsr  r}   r   c          
         U(       d  [        S5      eU(       d  U R                  nU[        ;  a  [        SU S[         35      eU SU SU SU 3nU=(       d    SU 3nU=(       d    SnU R                  US	S
9nU R	                  UUS9nSn[        U	5       GHe  n US:  a  [        R                  SUS-    SU S35        [        R                  " UU R                  U R                  5      [        R                  " U5      UU
S9nUR                  S:w  am   UR                  5       nSUR                   SU SU 3nSUR                  s=::  a  S:  a"  O  O[        R#                  SUS-    SU 35        Of[        SU 35      eUR                  5       nUR%                  S0 5      R%                  SS5      n[        R                  SU 35        ['        UUUUS9s  $ XS-
  :  d  GMO  [2        R4                  " S5        GMh     [        R(                  R+                  S!U	 S"U 35      e! [        R                   a    UR                   n GN"f = f! [        R(                  R*                   a3  nUn[        R#                  SUS-    S[-        U5       35         SnANSnAf[.         a=  nUn[        R1                  S US-    S[-        U5       35        XS-
  :X  a  e  SnAGNSnAff = f)#a*  
Create a commit on the ModelScope Hub with retry mechanism.

Args:
    repo_id (str): The repo id in the format of `owner_name/repo_name`.
    operations (Iterable[CommitOperation]): The commit operations.
    commit_message (str): The commit message.
    commit_description (Optional[str]): The commit description.
    token (str): The access token. If None, will use the cookies from the local cache.
        See `https://modelscope.cn/my/myaccesstoken` to get your token.
    repo_type (Optional[str]): The repo type, should be `model` or `dataset`. Defaults to `model`.
    revision (Optional[str]): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`.
    endpoint (Optional[str]): The endpoint to use.
        In the format of `https://www.modelscope.cn` or 'https://www.modelscope.ai'
    max_retries (int): Number of max retry attempts (default: 3).
    timeout (int): Timeout for each request in seconds (default: 180).

Returns:
    CommitInfo: The commit info.

Raises:
    requests.exceptions.RequestException: If all retry attempts fail.
r  r  r  /api/v1/repos/r  z/commit/z
Commit to r   Tr   )r  rd  Nr   zAttempt ru   z to create commit for z...)r   r  r   r   rK  zHTTP z error from r  rv   iX  zServer error on attempt zClient request failed: r  oidzCommit succeeded: )
commit_urlrd  r  r  zRequest failed on attempt zUnexpected error on attempt zFailed to create commit after z attempts. Last error: )r   rp   rV   r   _prepare_commit_payloadranger  rQ  r  r   r   r   r   r  r  JSONDecodeErrortextr)  r   rh   r  r  rr  r'  r(  timesleep)r   r  r  rd  r  rq   r  r  rp   r}   r   ra  r   payloadlast_exceptionattemptre  error_detail	error_msgr  r  r  s                         r   create_commitHubApi.create_commit,  s   J 788}}H--29+=PQbPcdee
.2gYhxjQ'AZy+A/52"""M ..!) / 
 [)G1Q;KK(7Q;-7MgYVY Z[#== 00>G,## ''3.5'/}}
   4 45\#b'.*  h228S86w{m2i[Q )+B9+)NOO#==?D((62.225"=CKK"4SE :;%#&'5+=	 " q(

1k *p !!22,[M9PQ_P`a
 	
S  // 5'/}}56 &&77 U!"!;GaK=3q6(STT !";GaK=3q6(STAo- .sQ   A:IH'(AI6A#I' II
IIK$,)JK$'2KK$z[Uploading])rq   r  rd  r  buffer_size_mb	tqdm_descdisable_tqdmr  path_or_fileobjpath_in_repor  r  r  c                   U[         ;  a  [        SU S[          35      eU(       d  [        S5      eU R                  USS9  [        U[        [
        45      (       ae  [        R                  R                  [        R                  R                  U5      5      nU=(       d    [        R                  R                  U5      nOU(       d  [        S5      e[        U[        R                  5      (       a  UR                  5       nU R                  R                  U5        U R                  R!                  U/US9  Ub  UOSU S	3nUS
::  a  [        S5      e[#        UUS9nUS   nUS   nU R%                  UUUU R&                  SSS9  U R)                  UUUUUU
U	US9n[+        UUUS9nU R                  R-                  X5      (       a  SOSUl        US   Ul        U/n[3        SU S3SS9  U R5                  UUUUUUUS9nU$ )a  
Upload a file to the ModelScope Hub.

Args:
    path_or_fileobj (Union[str, Path, bytes, BinaryIO]):
        The local file path or file-like object (BinaryIO) or bytes to upload.
    path_in_repo (str): The path in the repo to upload to.
    repo_id (str): The repo id in the format of `owner_name/repo_name`.
    token (Union[str, None]): The access token. If None, will use the cookies from the local cache.
        See `https://modelscope.cn/my/myaccesstoken` to get your token.
    repo_type (Optional[str]): The repo type, default to `model`.
    commit_message (Optional[str]): The commit message.
    commit_description (Optional[str]): The commit description.
    buffer_size_mb (Optional[int]): The buffer size in MB for reading the file. Default to 1MB.
    tqdm_desc (Optional[str]): The description for the tqdm progress bar. Default to '[Uploading]'.
    disable_tqdm (Optional[bool]): Whether to disable the tqdm progress bar. Default to False.
    revision (Optional[str]): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`.

Returns:
    CommitInfo: The commit info.

Examples:
    >>> from modelscope.hub.api import HubApi
    >>> api = HubApi()
    >>> commit_info = api.upload_file(
    ...     path_or_fileobj='/path/to/your/file.txt',
    ...     path_in_repo='optional/path/in/repo/file.txt',
    ...     repo_id='your-namespace/your-repo-name',
    ...     commit_message='Upload file.txt to ModelScope hub'
    ... )
    >>> print(commit_info)
r  r  z$Path or file object cannot be empty!Tr   z#Arg `path_in_repo` cannot be empty!file_path_listr  zUpload z to ModelScope hubr   z4Buffer size: `buffer_size_mb` must be greater than 0)file_path_or_objr  	file_size	file_hashFr  rq   r  rp   r  r  r  r  sha256sizer  r  r  rq   r  r  file_hash_infolfsnormalis_uploadedzCommitting file to z ...r  r  r  rd  r  rq   r  r  )rV   r   r   rq  rr  r   r   r   abspathr   basenameioBufferedIOBaserx   r   
check_filecheck_normal_filesr`   r  rp   _upload_blobrj   is_lfs_upload_mode_is_uploadedr  r  )r   r  r  r  rq   r  rd  r  r  r  r  r  hash_info_dr
  r  
upload_resadd_operationr  commit_infos                      r   upload_fileHubApi.upload_file  sC   ^ --29+=PQbPcdeeCDD 	edCoT{33 ggoobgg.@.@.QRO'L277+;+;O+LL   !FGG or'8'899-224O&&7..+, 	/ 	
 -8N~Ug>h 	 QSTT),)
 %[1	$[1	$#,"&--"&/4 	 	6  ,, % - 	

 -?%+&-

 /3.A.A.H.H.d.dUjr"%/%>"#_
#G9D1>"&"4"4!)1 #5 #
 r   )	r  rd  r  rq   r  allow_patternsignore_patternsmax_workersr  folder_pathr&  r'  r(  c                  ^ ^^^ T(       d  [        S5      eUc  [        S5      eT[        ;  a  [        ST S[         35      eT R                  TSS9  U(       a  UOSnU	(       a  U	OSn	U	c  / n	O[        U	[        5      (       a  U	/n	U	[
        -  n	Ub  U	 Vs/ s H  oU;  d  M
  UPM     n	nUb  UOST S	3nU=(       d    S
n[        R                  S5        T R                  UUUU	S9n[        U5      S:X  a  [        SU S35      e[        R                  S[        U5       S35        T R                  R                  U VVs/ s H  u  pUPM	     snnTS9  T R                  TTTT R                  SSS9  [        U
SS9UUU U4S j5       nU" UTTTUUSSS9n/ nU Hd  nUS   nUS   nUS   nUS   n[        UUUS9nT R                  R!                  UT5      (       a  SOSUl        UUl        UR'                  U5        Mf     [        U5      S:X  a  [        SU S35      e[(        S:  a  [(        O
[        U5      n[        U5      S-
  U-  S-   n[+        S [        U5       S!U S"U S#3SS$9  / n[-        [/        U5      S%US&9 HE  nUUU-  US-   U-   nU S'US-    S(U S)3nT R1                  TUUUTTUS*9nUR'                  U5        MG     [        U5      S:X  a  US   $ U$ s  snf s  snnf )+a/  
Upload a folder to the ModelScope Hub.

Args:
    repo_id (str): The repo id in the format of `owner_name/repo_name`.
    folder_path (Union[str, Path, List[str], List[Path]]): The folder path or list of file paths to upload.
    path_in_repo (Optional[str]): The path in the repo to upload to.
    commit_message (Optional[str]): The commit message.
    commit_description (Optional[str]): The commit description.
    token (Union[str, None]): The access token. If None, will use the cookies from the local cache.
        See `https://modelscope.cn/my/myaccesstoken` to get your token.
    repo_type (Optional[str]): The repo type, default to `model`.
    allow_patterns (Optional[Union[List[str], str]]): The patterns to allow.
    ignore_patterns (Optional[Union[List[str], str]]): The patterns to ignore.
    max_workers (int): The maximum number of workers to use for uploading files concurrently.
        Defaults to `DEFAULT_MAX_WORKERS`.
    revision (Optional[str]): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`.

Returns:
    Union[CommitInfo, List[CommitInfo]]:
        The commit info or list of commit infos if multiple batches are committed.

Examples:
    >>> from modelscope.hub.api import HubApi
    >>> api = HubApi()
    >>> commit_info = api.upload_folder(
    ...     repo_id='your-namespace/your-repo-name',
    ...     folder_path='/path/to/your/folder',
    ...     path_in_repo='optional/path/in/repo',
    ...     commit_message='Upload my folder',
    ...     token='your-access-token'
    ... )
    >>> print(commit_info.commit_url)
"The arg `repo_id` cannot be empty!Nz%The arg `folder_path` cannot be None!r  r  Tr   z
Upload to z on ModelScope hubzUploading fileszPreparing files to upload ...)folder_path_or_filesr  r&  r'  r   z"No files to upload in the folder: r  z	Checking z files to upload ...r  Fr  )r(  r  c                    > U u  p#[        US9nUS   nUS   nT
R                  TT	UUUU[        :*  SU-   S-   TS9nUUUS   US.$ )	N)r	  r
  r  z[Uploading ]r  r  )file_path_in_repor<  r  r  )r`   r  r+   )	item_pairr  r/  r<  r   r
  r  r!  r  r  r   rq   s           r   _upload_items+HubApi.upload_folder.<locals>._upload_items	  s    +4( -!*!K )5I(5I#00# &*LL'*;;cA  1 	 J &7&)-8"-	 r   ru   )r  rq   r  rd  r  r  r  r/  r<  r  r  r  r  r  zCommitting z
 files in z batch(es) of size r   r  z[Committing batches] )descrw   z (batch r   r4  r  )r   rV   r   rq  rr  re   r  rQ  _prepare_upload_folderr  r   r  r  rp   rl   rj   r  r  r  r  r,   r  r   r  r  ) r   r  r)  r  rd  r  rq   r  r&  r'  r(  r  pprepared_repo_objectsr:  r  r1  uploaded_items_listr  item_dprepared_path_in_repoprepared_file_pathr  r  optcommit_batch_sizenum_batchescommit_infosibatch_operationsbatch_commit_messager#  s    ``    ``                        r   upload_folderHubApi.upload_folder&	  s   b ABBDEE--29+=PQbPcdee 	edC+9t-</$ " O--./O22 %**a~.E?  
 -8N
7)Se>f 	 0D3D 	34 $ ; ;!,%)+	 !< !
 $%*A+bQRRi$9 :;;OPQ..0EF0EWQD0EF 	/ 	

 	$#,"&--"&/4 	 	6 
[u	E	 
F	6 ,!)1	
 
)F)/0C)D!&,[&9 &} 5K#)*:#;N$2 2-C )-(;(;(B(BCUW`(a(augoC*Cc" *  z?aA+bQRR >VXY=Y!9_bcm_n:*/@@1DC
O,J{mCVWhViijk	)+eK(/FkZA)!.?*?!a%K\A\]&4%5Xa!eWAk]RS#T &*&8&8+3#5#! '9 'K , [ #&l"3q"8|AJlJg, Gs   	K%%K%K*
)r  r  r  rq   r  r  r  c       	   
      l   [        S SS S S9n
X4S./nU R                  UUUU	S9n[        U5      S:X  a  US   OS nUc#  [        R	                  SUS S  S	35        S
U
S'   U
$ U R                  U	S
S9nU(       a  [        U5      OS nUc  [        S5      eU R                  R                  SSUS    305        U R                  U R                  5      nUS-  S-  4S jn[        USS
UUS9 n[        U[        [        45      (       a6  [        US5       n[        R                   " US   UU" UU5      S9nS S S 5        O[        U["        5      (       a5  [        R                   " US   UU" [$        R&                  " U5      U5      S9nOK[        U[$        R(                  5      (       a!  [        R                   " US   UU" UU5      S9nO[        S5      eS S S 5        [+        WS9  UR-                  5       n[/        US9  US   U
S'   US   U
S'   US   U
S'   U
$ ! , (       d  f       NR= f! , (       d  f       N[= f)NF)ra  r  r  
status_msg)r  r  )r  r  objectsrq   ru   r   zBlob    z  has already uploaded, reuse it.Tr  r   r
  Cookiezm_session_id=r   rA  c              3   ~   #     U R                  U5      nU(       d  gUR                  [        U5      5        Uv   M9  7f)z8Lazy function (generator) to read a file piece by piece.N)rx   r   r  )file_objectpbarrL  cks       r   read_in_chunks+HubApi._upload_blob.<locals>.read_in_chunks
  s8      %%j1CG$ s   ;=B)rw   unit
unit_scaler3  disablerbra  )r   r  zInvalid data type to uploadrspr  r  r  rE  )r  _validate_blobr  r  r  r   r   r   r   r   r   rq  rr  r   r\  r  r  bytesr  BytesIOr  r@   r   rA   )r   r  r  r  r  r  r  r  r  rq   res_drF  upload_objectsupload_objectr   r   rM  rK  r  re  r  s                        r   r  HubApi._upload_blob	  s_    	
 "01,,	 - 
 .1-@A-Eq)4 LL5,LMN#'E- L"""M#*$w-?HIIXw~7N6O'PQR&&t||49G$9NQU9U 	 $
 $d,,$%'||%e, '+At4 H &% D%((#<<!%(#'

4(8$? D""3"344#<<!%(#'d3 !!>???
B 	(+}}4 $U+e#F|m"9olA &%
 
s%   )(H%!H2BH%
H"	H%%
H3)rp   rq   rF  c                   U(       d  U R                   nU SU SU S3nSUS.nU R                  USS9n[        R                  " UU R	                  U R
                  5      [        R                  " U5      US9n	[        U	S	9  U	R                  5       n
[        U
S	9  / nU
S
   S   nU H#  nUR                  US   S   S   US   S.5        M%     U$ )a  
Check the blob has already uploaded.
True -- uploaded; False -- not uploaded.

Args:
    repo_id (str): The repo id ModelScope.
    repo_type (str): The repo type. `dataset`, `model`, etc.
    objects (List[Dict[str, Any]]): The objects to check.
        oid (str): The sha256 hash value.
        size (int): The size of the blob.
    endpoint: the endpoint to use, default to None to use endpoint specified in the class
    token (str): The access token.

Returns:
    List[Dict[str, Any]]: The result of the check.
r  r  z/info/lfs/objects/batchupload)	operationrF  Tr   )r   r  r   rT  r  rF  actionshrefr  )ra  r  )rp   r   r  r   r   r   r   r  r@   rA   r  )r   r  r  rF  rp   rq   ra  r  r   re  r  rZ  resp_objectsobjs                 r   rV  HubApi._validate_blobE
  s    6 }}H
.2gY>UV "

 """M==((6G$	
 	(+}}4 F|I.C!!Ix08E
$   r   r,  c                    S nS n[        U[        5      (       a5  [        R                  R	                  US   5      (       a  UnO5[        S5      e[        R                  R	                  U5      (       a  U/nOUnUc  U R                  R                  U5        [        U5      R                  5       R                  5       nUR                  5       (       d  [        SU S35      e[        UR                  S5      5       Vs0 s H:  nUR                  5       (       d  M  UR                  U5      R!                  5       U_M<     nnOm0 nU He  n[        R                  R	                  U5      (       d  M)  U R                  R#                  U5        Xx[        R                  R%                  U5      '   Mg     [        [&        R(                  " UR+                  5       X4S95      n	U(       a  UR-                  S5       S3OSn
U	 Vs/ s H  nX-   [/        X   5      4PM     nn[0        R3                  S	[5        U5       S
35        U$ s  snf s  snf )Nr   z0Uploading multiple folders is not supported now.zProvided path: 'z' is not a directoryz**/*)r&  r'  r   r   z	Prepared z files for upload.)rq  r  r   r   ro  r   r   check_folderr   r   resolveis_dirsortedglobis_filerelative_toas_posixr  r  rk   filter_repo_objectskeysr%  rr  r  rQ  r  )r   r,  r  r&  r'  r)  
files_pathr   relpath_to_abspathfiltered_repo_objectsprefixrelpathr6  s                r   r4  HubApi._prepare_upload_folder
  s    
*D11ww~~215661
 !STTww~~23323
2,,[9{+668@@BK%%'' #3K=@T!UVV
 #;#3#3F#;<"<D<<> ?  -668$><  " "$"77>>$''''2248AErww'7'7'=> # !%))"'').!
 3?L&&s+,A.B 1!
0 s#5#>?@0 	 !

 	i$9 :;;MNO$$9"*!
s   >H=$H=<Ic                 t   U/ S.nSnU  GH  n[        U[        5      (       a;  UR                  (       a*  [        R	                  SUR
                   S35        US-  nMT  [        U[        5      (       a}  UR                  S:X  am  UR                  (       a  SOSUR
                  SUR                  R                  S	UR                  5       R                  5       S
S.nUS   R                  U5        M  [        U[        5      (       av  UR                  S:X  af  UR                  (       a  SOSUR
                  SUR                  R                  UR                  R                  S	S	S.nUS   R                  U5        GMq  [        SU S[        USS5       35      e   US:  a  [        R!                  SU S35        U$ )z>
Prepare the commit payload to be sent to the ModelScope hub.
)rd  r`  r   zSkipping file 'z(' in commit (ignored by gitignore file).ru   r  r   creater   base64)actionr   typer  r  r7  rD  r`  r  z(Unknown operation to commit. Operation: z. Upload mode: r  NzSkipped z/ file(s) in commit (ignored by gitignore file).)rq  rj   _should_ignorer  r  r  r  r  upload_infor  
b64contentrJ  r  r  r   r   rQ  )r  rd  r  nb_ignored_filesr_  commit_actions         r   r  HubApi._prepare_commit_payload
  s    -

  $I )%788Y=U=Uy/E/E.FFnop A%  )%788Y=S=SW_=_ +4*@*@hh%22$%1166 (335<<> (! 	"))-8 I'9::y?U?UY^?^ +4*@*@hh%22!%1166'33::! "! 	"))-8 !>yk J	>4@AC K $T aKK(#3"44cder   internal_timeoutc                     S[         S[        4S jnU R                   S3nU" X1S9nSnUbK  UR                  5       nSU;   a	  US   S   nOSnU(       a#  U" XaS	9nUb  UR                  R                  5       nU$ )
z
Get the internal acceleration domain.

Args:
    internal_timeout (float): The timeout for the request. Default to 0.2s

Returns:
    str: The internal acceleration domain. e.g. `cn-hangzhou`, `cn-zhangjiakou`
ra  r   c                      [         R                  " XS9nUR                  5         U$ ! [         R                  R                   a    S n U$ f = f)Nr~   )r  r   raise_for_statusr  r  )ra  r   re  s      r   send_request>HubApi._get_internal_acceleration_domain.<locals>.send_request  sO     #<<=))+ O &&77  O s   $(  AAz&/api/v1/repos/internalAccelerationInfo)ra  r   r   r  InternalRegionQueryAddressr~   )rr  floatrp   r   r  r%  )r   r  r  internal_urlinternal_info_response	region_id
query_addrdomain_responses           r   !_get_internal_acceleration_domain(HubApi._get_internal_acceleration_domain
  s    	c 	E 	 --(NO ".,!Y	!-%;%@%@%B"//3F;<XY
"$
".z"T". / 4 4 : : <Ir   )r  rp   rq   delete_patternsc          
      0   U[         ;  a  [        SU 35      eU(       d  [        S5      e[        U[        5      (       a  U/nU R	                  USS9nU(       d  U R
                  nUc  [        S5      eU R                  U R                  5      nU[        :X  a5  U R                  UU=(       d    [        SUUS9n	U	 V
s/ s H  oS   PM	     nn
OU[        :X  ai  / nS	nS
n  U R                  UU=(       d    [        SUUUUS9nU H"  nUS   S:w  d  M  UR#                  US   5        M$     [%        U5      U:  a  OUS	-  nMb  [        SU S[          35      e/ nU H<  nU H3  n[&        R&                  " UU5      (       d  M!  UR#                  U5          M:     M>     / / nnU H  n U[        :X  a2  UR)                  S5      u  nnU SU SU S3nU=(       d    [        US.nOCU[        :X  a$  UR)                  S5      u  nnU SU SU S3nSU0nO[        SU S[          35      eU R*                  R-                  UUXxS9n[/        U5        UR1                  5       n[3        U5        UR#                  U5        M     UU[%        U5      S.$ s  sn
f ! [         a0  n[        R!                  SU S[        U5       35         SnAGMl  SnAff = f! [         aA  nUR#                  U5        [        R!                  SU S[        U5       35         SnAGMp  SnAff = f)a  
Delete files in batch using glob (wildcard) patterns, e.g. '*.py', 'data/*.csv', 'foo*', etc.

Example:
    # Delete all Python and Markdown files in a model repo
    api.delete_files(
        repo_id='your_username/your_model',
        repo_type=REPO_TYPE_MODEL,
        delete_patterns=['*.py', '*.md']
    )

    # Delete all CSV files in the data/ directory of a dataset repo
    api.delete_files(
        repo_id='your_username/your_dataset',
        repo_type=REPO_TYPE_DATASET,
        delete_patterns='data/*.csv'
    )

Args:
    repo_id (str): 'owner/repo_name' or 'owner/dataset_name', e.g. 'Koko/my_model'
    repo_type (str): REPO_TYPE_MODEL or REPO_TYPE_DATASET
    delete_patterns (str or List[str]): List of glob patterns, e.g. '*.py', 'data/*.csv', 'foo*'
    revision (str, optional): Branch or tag name
    endpoint (str, optional): API endpoint
    token (str, optional): Access token
Returns:
    dict: Deletion result
zUnsupported repo_type: zdelete_patterns cannot be emptyTr   Nr
  )r  r2  rp   r  r   ru   r  )r  r  r2  r  r  rp   rq   zGet dataset: z file list failed, message: r  treer  r   r   z/file)r  ru  rJ  z/reporu  r  zFailed to delete r  )deleted_filesfailed_filestotal_files)rV   r   rq  rr  r   rp   r   r   rU   r9  rP   rT   rA  rO   r'  r  r(  r  r  fnmatchr   r   r  r@   r   rA   )r   r  r  r  r  rp   rq   r   r   r  r  
file_pathsr  r  dataset_filesr  file_info_d	to_deleter   delete_patternr  r  ownerr  ra  r  r  r   r  s                                r   delete_filesHubApi.delete_files   sq   H --6ykBCC>??os++./O"""M}}H?HII&&t||4 '((!;%;!# ) E .33UF)UJ3J++JKI:>:P:P '!)!E-E"&$/"+!)# ;Q ;M $1K"6*f4"))+f*=> $1 }%	1q / 2 6ykATUfTghii 	D"1??488$$T* #2  ')"|DC/'.}}S'9$E9%JoeWAi[NC$,$F0F$(F "33*1--*<'E<%J&7wa~USC"DF %'>ykI\]n\o%pqqLL''FG']%a(vvxt$$$T*- 8 +(y>
 	
I 4  ! LL=	9UVYZ[V\U]!^_`  C##D)0bQABBCs7   =J "J $CK

K$KK

L5LL)modeldatasetprivatepublicc                    U(       d  [        S5      eUS;  a  [        SU S35      e[        R                  " 5        VVs0 s H  u  pVXe_M	     nnnUR                  US5      nU R	                  USS9n	U[
        :X  Gac  U R                  XS9n
U R                   S	U 3nU
R                  S
5      nSn[        U[        5      (       a9  U(       a2  US   n[        U[        5      (       a  U(       a  UR                  S5      nU
R                  SS5      U
R                  SS5      USU
R                  SS5      U
R                  SS5      U
R                  SS5      U
R                  SS5      U
R                  SS5      U
R                  SS5      U
R                  S0 5      R                  SS5      U
R                  S0 5      R                  SS5      UU
R                  SS5      S.nOU[        :X  ap  UR                  S5      n[        U5      S:w  d  [        U5      (       d  [        SU S 35      eU R                  US!   US   US"9u  nnU R                   S#U 3nUSS$.nO[        S%U S&[          35      eU R"                  R%                  UUU	U R'                  U R(                  5      S'9n[+        U5        UR-                  5       n[/        U5        U$ s  snnf )(a  
Set the visibility of a repo.

Args:
    repo_id (str): The repo id in the format of `owner_name/repo_name`.
    repo_type (Literal['model', 'dataset']): The repo type, `model` or `dataset`.
    visibility (Literal['private', 'public']): The visibility to set, `private` or `public`.
    token (Union[str, None]): The access token. If None, will use the cookies from the local cache.
        See `https://modelscope.cn/my/myaccesstoken` to get your token.

Returns:
    dict: The response from the server.
r+  r  r  z-, supported visibilities: `private`, `public`   Tr   )r   rq   r   r^   r   r   r   r   ModelFrameworkPytorchrt   ApprovalModer  r   r   
ModelCoverSubScientificFieldNNEXAScientificFieldrt  r   )r   r  r6   ProtectedModer  r  r   r   r  r  r  rt  	ModelTaskr   r   zInvalid dataset repo_id: z-, should be in format of `owner/dataset_name`ru   )r  r  rq   rJ  )r6   r  r  r  r   )r   r7   r  r   r   rU   r  rp   rq  r  r  rT   r   r  allr  rV   r   r  r   r   r@   r   rA   )r   r  r  r   rq   r  r  visibility_mapvisibility_coder   r<  r   tasksmodel_tasksfirstr  repo_id_partsdataset_idxr:  r   r  s                        r   set_repo_visibilityHubApi.set_repo_visibility  s   & ABB223J<?lmnn;H;N;N;P)Q;P41!$;P)Q-11*a@"""M'FJmm_OG9=DNN7+EK%&&5aeT**u"'))F"3K)~~mR@",..1A9"M-!" *~q A)~~mR@&NN:r:$.NN3Er$J(nn\2>&0nn5I4&P#->>&"#=#A#ABSUW#X$..4882F(%>>)R8G  ++#MM#.M=!Q&c-.@.@ #<WIEr!stt!99*1-'* : NK mm_$5k]CD-!"G
 29+=PQbPcdeeLL((6	  8 	a vvxtw *Rs    K)rp   r   r   rq   r   )NFr   )NNNr  )ru   r  NN)F)NFN)FN)Tru   r  NN)ru   N)FNN)g?)n__name__
__module____qualname____firstlineno____doc__r   r   r   rr  r   r   r  r   r   r5   PUBLICr4   	APACHE_V2r  r   r  r  r  rP   r  r  r.  rD   r<  rC   rC  rU   r   rG  r&  rR  staticmethodrb  rQ   r   r  r  r  r   r  r  r  r  r  r  r   r  r  r9  r  r3   r  rP  r  r  r8  rA  r@  r&  r/  r  r?  r[   REUSE_DATASET_IF_EXISTSrm  rO   ry  r|  r  r  r  r  r  r  r  r  r  r   r  r6   rG   r  r   ri   rh   r  r   rW  r	   r$  r   rB  r  r
   r   rV  tupler4  r  r  r  r  r   r  __static_attributes__r7  r   r   rn   rn   c   s    ,008(,	!/#C=!/ !	!/F , PXY]P^ @ +/&*-:"3--: sm-:b 2A1G1G.6.@.@378:/3,09=g"g!)#g 'smg $,C=	g
 )1g  (}g %SMg "*+!6g CFgX 4804=AU#&U#&U $,C=U !)	U
 &.k%:U
 GJUn!!S !!HSM !!QYZ]Q^ !!F:c :Xc] : '=&*#'+%+% sm+% sm	+%
 C=+% 
+%` :>59	)#'*)# *2#)# &.c]	)# ?B	)#\ .D-1	 &c] &c]	 7@	F 04/3	!  (}  (}	 9D	H $3"8"&!a!a C=	!a
 3-!a 3-!a 
y+%	&!aN (,&*',#'88  }	8
 sm8 tn8 C=8 
8z /3+/	-= -="-= 'sm-= $C=	-=^ ! ! 0?/E/E,4,>,>153A(,-H6:JNAE*.P7 P7!P7  (}P7 %SM	P7
 "*#P7 $,C=P7 !P7 &c]P7 '/smP7 )1tCy#~1F(GP7  (c49n(=>P7 #3-P7h 23/1.2+/($'(!)#(  (}( 'sm	(
 $C=(
 59(Z 4513,0.204-1D$&)D$ $,C=D$ "*#	D$
 %SMD$ 'smD$ !)D$ &c]D$ 7;D$LtY)? I  /327	 'sm tY/	 =AI	6 /327&* + + 'sm + tY/	 +
 sm +
 04Cy +D ,0AE<@	C,/C ,4I+>C -5SM	CN %):>59	M%(M %-Y$7M &.c]	M 38&*	LL tY/L sm	L
 
tCy$s)#	$L@ 38 tY/ 
tCy$s)#	$	2 3I.249>C24264"%4"*3-4 'sm4 $,D>	4
 &+4?%;4 "*$4 #+3-4 <@:4v '+#'!! !
 sm! C=!@ 68080B0B3D3K3K4615.26 %(6 "%6  &.c]6  !)	6 
 $,C=6  %-SM6  "*#6  'sm6  8;6 t 26.2!!#&!!!)#!! 'sm!!L ;?7;	(.1(+.( +33-( (0}	(, *.*+(+15.2 %( "%  "%  #&	 
 #'  %(  #&  "*#  'sm J 6E4O-.+-4815=Y#&=Y &.c]=Y %-SM	=Y
 (+=Y &)=Y %-SM=Y "*#=YD +F+.,0-.+.48154%#&4% %(4% &)	4%
 &*4% (+4% &)4% %-SM4% "*#4%r #>"&#!-!- 3-!- 3-	!-
 }!-H ^b:>s s /2>AMUVY]*23-, 
L 
Lc 
L 
L  FJBF".s ".69".58". <?". OR". _c". 6>c]	".
 3;3-".H <@|GkGk9=6)1#6 6z '?#(/3&*  	
 sm 4. 'tn smJ '?&*  	
 sm sm$ '?&*#'BB B sm	B
 smB C=B$ '?&*#'  	
 sm sm C=4 s xPS} `d D JNFJ=@:= := ?B	
 :B# 7?sm< cg8<RZ[^R_(0  \`9=S  -0<?KSTW=)1#BE" Y]6:# S *-9<HPQT&.sm?B"hsm  \`;?        37  KSTW=  +3C=  DH  DP# P# PRU P -1(2(9(9'6*.%-%7%7&*',48.2kk dD)	k
 !k  }k #3-k c]k smk tnk $,D>k !+k 
kf 15'6&A&* v
v
 !1v

  v
 !)v
 v
  }v
 smv
 smv
 v
 v
 
v
| '+'6,004,-'4+0&A@ #3eX#=>@ 	@
 @ d#@  }@ %SM@ !)@ %SM@  }@ #4.@ sm@ 
@N +-,004&*'6>B?C2&A|K |K sD$s)T$Z?@	|K
 #3-|K %SM|K !)|K d#|K  }|K %U49c>%:;|K &eDIsN&;<|K |K sm|K 
z4
++	,|KL ,1'4,-#'_ _ 	_
 _ _ T5(23_ #4._  }_ %SM_ C=_ 
_N '+#'9 9 	9
 $sCx.)9 sm9 C=9 
d38n	9~ ?C?C7%"'T49d4j(H"I7% 7% %U49c>%:;	7%
 &eDIsN&;<7% 
eE4K 	!7%r = 1== 
c3h= =~%% %X 0F/3,0C
!C
 #C
 ',CcN&;C

  (}C
  (}C
 %SMC
 6:#s(^C
R 7;	T%(T'./A'BT )00C(DT $)d#3	T
 "&T Tr   rn   c                       \ rS rSr\" \5      rSrSrSr	Sr
Sr\S 5       r\S\4S j5       r\S	 5       r\S
 5       r\S\4S j5       r\S\S\4S j5       r\S\\\4   4S j5       r\S\\   4S j5       r\SS\\\S4   S\4S jj5       rSrg)r   i  r   	git_tokenuserr   Fc                  J    [         R                  " [        R                  SS9  g )NT)r  )r   makedirsr   path_credentialr7  r   r   make_sure_credential_path_exist0ModelScopeConfig.make_sure_credential_path_exist  s    
$44tDr   c                    [         R                  5         [        [        R                  R                  [         R                  [         R                  5      S5       n[        R                  " X5        S S S 5        g ! , (       d  f       g = f)Nzwb+)
r   r  r\  r   r   rV  r  COOKIES_FILE_NAMEpickler]  )r   r  s     r   r   ModelScopeConfig.save_cookies  s_    88:-==-??ABGILMKK#I I Is   A;;
B	c                  F   [         R                  R                  [        R                  [        R
                  5      n [         R                  R                  U 5      (       a  [        U S5       n[        R                  " U5      nU(       d
   S S S 5        g U Hk  nUR                  S:X  d  M  UR                  5       (       d  M,  [        R                  (       a  MC  S[        l        [        R                  S5          S S S 5        g    UsS S S 5        $ g ! , (       d  f       g = f)NrS  r   TzKNot logged-in, you can login for uploadingor accessing controlled entities.)r   r   rV  r   r  r  rn  r\  r  loadr   
is_expiredcookie_expired_warningr  rQ  )cookies_pathr  r   cookies       r   r   ModelScopeConfig.get_cookies  s    ww||$4$D$D$4$F$FH77>>,''lD)Q ++a. *) &F{{n49J9J9L9L 0 G G GBF(? %H I# *) &  *)  *) s*   -DD,DD!DD
D c                  ~   [         R                  R                  [        R                  [        R
                  5      n Sn[         R                  R                  U 5      (       a>  [        U S5       n[        UR                  5       R                  5       SS9nUsS S S 5        $ US:X  d  [        U5      S:w  ad  [        [        R                  " 5       R                  5      n[        R                  5         [        U S5       nUR!                  U5        S S S 5        U$ U$ ! , (       d  f       N= f! , (       d  f       U$ = f)Nr   rS  rH  rC      w+)r   r   rV  r   r  USER_SESSION_ID_FILE_NAMErn  r\  rr  readliner%  r  r  r  r  r  r,  )session_path
session_idr  wfs       r   get_user_session_id$ModelScopeConfig.get_user_session_id#  s    ww||$4$D$D$4$N$NP
77>>,''lD)Q !3!3!5H
! *) s:"4TZZ\--.J<<>lD)R$ * z *) *) s   /(D>D-
D*-
D<rq   c                    [         R                  5         [        [        R                  R                  [         R                  [         R                  5      S5       nUR                  U 5        S S S 5        g ! , (       d  f       g = f)Nr  )	r   r  r\  r   r   rV  r  GIT_TOKEN_FILE_NAMEr,  rq   r  s     r   r   ModelScopeConfig.save_token4  s^    88:-==-AACDHJMNGGENJ J Js   A66
Br  
user_emailc                    [         R                  5         [        [        R                  R                  [         R                  [         R                  5      S5       nUR                  U < SU< 35        S S S 5        g ! , (       d  f       g = f)Nr  :)	r   r  r\  r   r   rV  r  USER_INFO_FILE_NAMEr,  )r  r  r  s      r   r   ModelScopeConfig.save_user_info<  sd    88:-==-AACDHJMNGGy*56J J Js   A==
Br   c                  T    [        [        R                  R                  [        R
                  [        R                  5      SSS9 n U R                  5       nUR                  S5      S   UR                  S5      S   4sS S S 5        $ ! , (       d  f       g= f! [         a     gf = f)Nr   rH  rC  r  r   ru   r   )
r\  r   r   rV  r   r  r  rx   r   FileNotFoundError)r  rQ  s     r   get_user_infoModelScopeConfig.get_user_infoD  s    		GGLL!1!A!A!1!E!EG$	& *+vvxzz#q)4::c?1+==& & &  ! 		s0   AB 8B	?	B 	
BB B 
B'&B'c                     Sn  [        [        R                  R                  [        R
                  [        R                  5      SSS9 nUR                  5       n SSS5        U $ ! , (       d  f       U $ = f! [         a     U $ f = f)zk
Get token or None if not existent.

Returns:
    `str` or `None`: The token, `None` if it doesn't exist.

Nr   rH  rC  )	r\  r   r   rV  r   r  r  rx   r  r  s     r   	get_tokenModelScopeConfig.get_tokenR  s     	GGLL!1!A!A!1!E!EG$	& *+& & &  ! 		s0   AA6 	A$A6 $
A3.A6 3A6 6
BBN
user_agentc                 h   Sn[         [        R                  ;   a  [        R                  [            nSn[        [        R                  ;   a  [        R                  [           nSSKJn  SU< S[        R                  " 5       < S[        R                  5       < S[        R                  " 5       < S	[        R                  " 5       < S
U< SU< 3n[        U [        5      (       a.  USSR                  S U R                  5        5       5      -   -  nU$ [        U [        5      (       a  USU -   -  nU$ )zFormats a user-agent string with basic info about a request.

Args:
    user_agent (`str`, `dict`, *optional*):
        The user agent info in the form of a dictionary or a single string.

Returns:
    The formatted user-agent string.
customunknownr   )__version__zmodelscope/z	; python/z; session_id/z; platform/z; processor/z; env/z; user/z; c              3   4   #    U  H  u  pU S U 3v   M     g7f)r   Nr7  ).0r  r  s      r   	<genexpr>2ModelScopeConfig.get_user_agent.<locals>.<genexpr>  s     "M:L$!aS!::Ls   )r!   r   r   r"   
modelscoper  platformpython_versionr   r  	processorrq  r  rV  r  rr  )r  envr  r  uas        r   r   ModelScopeConfig.get_user_agentg  s     '2::5**9:C	$

2

#<=I*##%002 
 j$''$"M*:J:J:L"MMMMB 	 
C(($##B	r   r7  r  )r  r  r  r  r   r#   r  r  r  r  r  r  r  r  r   r   r   r  rr  r   r   r   r  r   r  r   r
   r   r  r7  r   r   r   r     s.    !<=O!%  )"E E $i $ $  $    #   7# 73 7 7 5c?   x}  ( "5sD#9 "s " "r   r   c                       \ rS rSrSr\\\\\	4S\
S\
S\
S\
S\
4
S jjrSS jrS\\\4   4S jrS\\\\\4   S\S	\4S jrS\\\\4      S\S	S
4S jrSrg
)r   i  aN  
Check the files and folders to be uploaded.

Args:
    max_file_count (int): The maximum number of files to be uploaded. Default to `UPLOAD_MAX_FILE_COUNT`.
    max_file_count_in_dir (int): The maximum number of files in a directory.
        Default to `UPLOAD_MAX_FILE_COUNT_IN_DIR`.
    max_file_size (int): The maximum size of a single file in bytes. Default to `UPLOAD_MAX_FILE_SIZE`.
    size_threshold_to_enforce_lfs (int): The size threshold to enforce LFS in bytes.
        Files larger than this size will be enforced to be uploaded via LFS.
        Default to `UPLOAD_SIZE_THRESHOLD_TO_ENFORCE_LFS`.
    normal_file_size_total_limit (int): The total size limit of normal files in bytes.
        Default to `UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT`.

Examples:
    >>> from modelscope.hub.api import UploadingCheck
    >>> upload_checker = UploadingCheck()
    >>> upload_checker.check_file('/path/to/your/file.txt')
    >>> upload_checker.check_folder('/path/to/your/folder')
    >>> is_lfs = upload_checker.is_lfs('/path/to/your/file.txt', repo_type='model')
    >>> print(f'Is LFS: {is_lfs}')
max_file_countmax_file_count_in_dirmax_file_sizesize_threshold_to_enforce_lfsnormal_file_size_total_limitc                 @    Xl         X l        X0l        X@l        XPl        g r  )r  r  r  r  r  )r   r  r  r  r  r  s         r   r   UploadingCheck.__init__  s"     -%:"*-J*,H)r   r   Nc           	      B   [        U[        [        45      (       a3  [        R                  R                  U5      (       d  [        SU S35      e[        U5      nX R                  :  a7  [        R                  SU R                  S-   S[        US-  S5       S35        gg)	z
Check a single file to be uploaded.

Args:
    file_path_or_obj (Union[str, Path, bytes, BinaryIO]): The file path or file-like object to be checked.

Raises:
    ValueError: If the file does not exist or exceeds the size limit.
File  does not existzFile exceeds size limit:    @z	 GB, got     GBN)rq  rr  r   r   r   rn  r   ra   r  r  r)  r  )r   r	  r
  s      r   r  UploadingCheck.check_file  s     &d4477>>"233 5)9(:/!JKK&'78	)))NN6t7I7IY7W6X Y""'	Y(?"C!DCI J *r   r)  c           	         SnSn[        U[        5      (       a  [        U5      nUR                  5        H  nUR	                  5       (       a^  US-  n[        U5      nXPR                  :  a=  [        R                  SU SU R                  S-   S3S[        US-  S5       S35        Mt  Mv  UR                  5       (       d  M  US-  nU R                  U5      u  pgXg-   U R                  :  a   [        S	U S
Xg-    SU R                   35      eX&-  nX7-  nM     X R                  :  a  [        SU SU R                   35      eX#4$ )z
Check a folder to be uploaded.

Args:
    folder_path (Union[str, Path]): The folder path to be checked.

Raises:
    ValueError: If the folder does not exist or exceeds the file count limit.
r   ru   r
  z exceeds size limit: r  r  zgot r  z
Directory z
 contains z items and exceeds limit: zTotal file count z and exceeds limit: )rq  rr  r   iterdirrk  ra   r  r  r)  r  rh  rf  r  r   r  )r   r)  
file_count	dir_countr  	item_sizesub_file_countsub_dir_counts           r   rf  UploadingCheck.check_folder  sg    
	k3''{+K'')D||~~a
!.t!4	111NNU4&0EdFXFX\eFfEggj#k%)%	Y0G*K)LC#PR 2 Q	040A0A$0G-"2d6P6PP$z$z.B`Aa b;;?;U;U:V&X Y Y,
*	 *" +++0<PQUQdQdPefgg$$r   r	  r  c                    Sn[        U[        [        45      (       a  [        U5      nUR                  5       (       d  [	        SU S35      eU[
        :X  a  UR                  [        ;  a  SnO6U[        :X  a  UR                  [        ;  a  SnO[	        SU S[         35      e[        U5      nX@R                  :  =(       d    U$ )a2  
Check if a file should be uploaded via LFS.

Args:
    file_path_or_obj (Union[str, Path, bytes, BinaryIO]): The file path or file-like object to be checked.
    repo_type (str): The repo type, either `model` or `dataset`.

Returns:
    bool: True if the file should be uploaded via LFS, False otherwise.
Tr
  r  Fr  r  )rq  rr  r   rn  r   rU   r  rf   rT   rd   rV   ra   r  )r   r	  r  hit_lfs_suffixr
  s        r   r  UploadingCheck.is_lfs  s     &d44#$45#**,, 5)9(:/!JKKO+#**2BB%*N//#**2DD%*N #6ykATUfTg!hii&'78	===OOr   r  c                    U Vs/ s H  o0R                  X25      (       a  M  UPM     nn[        U Vs/ s H  n[        U5      PM     sn5      nXPR                  :  a"  [	        SUS-   SU R                  S-   S35      egs  snf s  snf )a&  
Check a list of normal files to be uploaded.

Args:
    file_path_list (List[Union[str, Path]]): The list of file paths to be checked.
    repo_type (str): The repo type, either `model` or `dataset`.

Raises:
    ValueError: If the total size of normal files exceeds the limit.

Returns: None
zTotal size of non-lfs files i   zMB and exceeds limit: MBN)r  sumra   r  r   )r   r  r  r  normal_file_listrf  s         r   r  !UploadingCheck.check_normal_files
  s     .<`^T;;tC_D^`:JK:J$--:JKL
999;J+<V;W X3373T3TXc3d2eegi j j : aKs   BBB)r  r  r  r  r  )r   N)r  r  r  r  r  r-   r.   r/   r1   r0   r  r   r  r   rr  r   rf  rW  r	   r  r  r   r  r  r7  r   r   r   r     s    0 #8)E!51U0SII $'I 	I
 ,/I +.IJ&$%c4i(8 $%LPuS$x-G'H PUX P]a P>jeCI6F1G jTW j\` jr   r   )r  r  r   r  r   r  r  rz  rw  r  r  r  r  collectionsr   httpr   http.cookiejarr   os.pathr   pathlibr   typingr   r	   r
   r   r   r   r   r   r   r   r   r   r  r   requests.adaptersr   r   requests.exceptionsr   rW  r   modelscope.hub.constantsr   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   modelscope.hub.errorsr8   r9   r:   r;   r<   r=   r>   r?   r@   rA   modelscope.hub.gitrB   modelscope.hub.inforC   rD   r  rF   modelscope.hub.utils.aigcrG   modelscope.hub.utils.utilsrH   rI   rJ   rK   rL   rM   rN   modelscope.utils.constantrO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   modelscope.utils.file_utilsr`   ra   rb   modelscope.utils.loggerrc   modelscope.utils.repo_utilsrd   re   rf   rg   rh   ri   rj   rk   modelscope.utils.thread_utilsrl   r  rn   r   r   r7  r   r   <module>r3     s*      	 	   	      #  $  " " " "    0 ) 5 5 5 5 5 5 5 5 58J J J 1 6 0 /F F F	; 	; 	; 	; 	;; ; .4 4 4 :	V. V.r\N NbOj Ojr   