
    Ƒi"                         S SK Jr  S SKrS SKrS SKrS SKrS SKJr  S SKJr  S SK	J
r
  S SKJr  S SKJr  S SKJr  S S	KJrJrJr  S S
KJr  \" 5       rSrSrSrSrSrSr " S S\5      r " S S5      rg)    )print_functionN)hash_url_to_filename)CredentialsProvider)Credentials)HubApi)DataDownloadConfig)MS_CACHE_HOME)"DEFAULT_DATA_ACCELERATION_ENDPOINTMetaDataFields
UploadMode)
get_loggerAccessIdAccessSecretSecurityTokenBucket	BackupDirDirc                   8    \ rS rSrSrS\S\S\S\4S jrS rS	r	g
)CredentialProviderWrapper   zK
A custom credentials provider for oss2 that fetches temporary credentials
apidataset_name	namespacerevisionc                 h    Xl         X l        X0l        X@l        [        R
                  " 5       U l        g)z
Initializes the CredentialProviderWrapper with dataset information.

Args:
    dataset_name (str): The name of the dataset.
    namespace (str): The namespace of the dataset.
    revision (str): The revision of the dataset.
N)r   r   r   r   	threadingLock_lock)selfr   r   r   r   s        e/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/modelscope/msdatasets/utils/oss_utils.py__init__"CredentialProviderWrapper.__init__#   s'     (" ^^%
    c                    U R                      U R                  R                  U R                  U R                  SU R
                  S9n[        U[           U[           U[           S9sSSS5        $ ! , (       d  f       g= f)zj
oss2 SDK will call this method automatically when it finds the token is expired or needs authentication.
Fr   r   check_cookier   )access_key_idaccess_key_secretsecurity_tokenN)
r   r   !get_dataset_access_config_sessionr   r   r   r   	ACCESS_IDACCESS_SECRETSECURITY_TOKEN)r   
oss_configs     r    get_credentials)CredentialProviderWrapper.get_credentials3   sk     ZZCC!...."	 D (J (3",]";).9 ZZs   AA11
A?)r   r   r   r   r   N)
__name__
__module____qualname____firstlineno____doc__r   strr!   r/   __static_attributes__ r#   r    r   r      s.    &F &# &# && r#   r   c            
       b    \ rS rSrSrS r\S 5       rS\S\	S\4S jr
S	\S
\S\S\S\4
S jrSrg)OssUtilitiesE   zX
A utility class for handling Alibaba Cloud OSS operations such as upload and download.
c                    Xl         X l        X0l        [        5       U l        U R                  R                  U R                   U R                  SU R                  S9n[        R                  " S5      S:X  a  [        U l	        OSUS    S3U l	        [        R                  R                  [        S5      U l        [        R                  " 5       U l        S	U l        S
U l        SU l        [(        R*                  " U R                  S9U l        [(        R.                  " U R                  S9U l        [3        U R                  U R                   U R                  U R                  S9n[(        R4                  " U5      nU[6           U l        [(        R:                  " UU R                  U R8                  US   R=                  S5      S9U l        U[@           U l!        U[D           U l#        g)zX
Initializes the OssUtilities with the given OSS configuration and dataset information.
Fr%   ENABLE_DATASET_ACCELERATIONTruezhttps://Regionz.aliyuncs.comztmp/resumable_storei   i      )root)r   r   r   r   zoss-)authendpointbucket_nameregionN)$r   r   r   r   r   r*   osgetenvr
   rC   pathjoinr	   resumable_store_root_pathmultiprocessing	cpu_countnum_threads	part_sizemultipart_thresholdmax_retriesoss2ResumableDownloadStoreresumable_store_downloadResumableStoreresumable_store_uploadr   ProviderAuthV4BUCKETrD   r   lstripbucketDIRoss_dirBACK_DIRoss_backup_dir)r   r   r   r   r.   credential_providerrB   s          r    r!   OssUtilities.__init__J   s    )" 8XX??**nn]]	 @ $
 9923v=>DM&z(';&<MJDM)+m6K*M&*446(#3 (,(C(C//)1%&*&9&9//'1# 8**nn]]	$
 ""#67%f-kk]]((h'..v6	
 "#(2r#   c                     U(       a>  [        S[        U 5      [        U5      -  -  5      n[        SR                  U5      SSS9  g g )Nd   z{0}%  T)endflush)intfloatprintformat)consumed_bytestotal_bytesrates      r    _percentageOssUtilities._percentage{   s?    seN3eK6HHIJD)""4(b= r#   oss_file_namedownload_configreturnc                    UR                   n[        R                  R                  U R                  U5      n[        R                  R                  U R
                  U5      nUR                  nSnU(       aA  UR                  R                  U5      nU(       a  UR                  [        R                  5      nSn	  U(       a  Un
O$U R                  R                  U5      (       a  UOUn
[        U
SS9n[        R                  R                  X;5      nUR                  (       d$  [        R                  R                  U5      (       dW  [         R"                  " U R                  U
UU R$                  U R&                  U R(                  U R*                  U R,                  S9   U$ ! [.         aU  n[0        R3                  SU SU S35        U	S	-  n	XR4                  :  a  [0        R7                  S
U S35        Ue SnAOSnAff = fGML  )a  
Downloads a file from OSS to the local cache.

Args:
    oss_file_name (str): The name of the file in OSS to download.
    download_config (DataDownloadConfig): Configuration for the download process.

Returns:
    str: The local path to the downloaded file.
Fr   N)etag)rY   keyfilenamestoremultiget_thresholdrN   progress_callbackrM   zError downloading : , trying again...   zFailed to download  due to exceeded retries.)	cache_dirrF   rH   rI   r[   r]   splitmeta_args_mapgetr   ARGS_BIG_DATArY   object_existsr   force_downloadexistsrQ   resumable_downloadrS   rO   rN   rl   rM   	ExceptionloggerwarningrP   error)r   rn   ro   r|   candidate_keycandidate_key_backupr}   big_data	args_dictretry_countfile_oss_keyrt   
local_pathes                 r    downloadOssUtilities.download   s    $--	T\\=A!ww||D,?,?O%%'5599%@I$==)E)EF#0L48KK4M4M%5' 5'=,@ !/4HWW\\)>
"11":$ :$++#{{(!+";;+/+C+C"&..*.*:*:$($4$46    (r!<MNPq "2"22LL-m_<UV G	 3	/ s   ;C#F! !
H +AG;;H oss_object_namelocal_file_pathindicate_individual_progressupload_modec                    Sn[         R                  R                  U R                  U5      nU(       a  U R                  nOSn  U R
                  R                  U5      nU[        R                  :X  a0  U(       a)  [        R                  SU SUR                   S35         U$ [        R                  " U R
                  UUU R                  U R                  U R                   UU R"                  S9   U$ ! [$         aU  n	[        R'                  SU SU	 S	35        US
-  nXPR(                  :  a  [        R+                  SU S35        U	e Sn	A	OSn	A	ff = fGM  )ar  
Uploads a local file to OSS.

Args:
    oss_object_name (str): The name of the object in OSS.
    local_file_path (str): The local file path to upload.
    indicate_individual_progress (bool): Whether to show individual progress.
    upload_mode (UploadMode): The upload mode (e.g., OVERWRITE, APPEND).

Returns:
    str: The OSS object key where the file is uploaded.
r   NzSkip z in case of z mode.)rY   rs   rt   ru   rO   rN   rw   rM   zError uploading rx   ry   rz   zFailed to upload r{   )rF   rH   rI   r[   rl   rY   r   r   APPENDr   infovaluerQ   resumable_uploadrU   rO   rN   rM   r   r   rP   r   )
r   r   r   r   r   r   
object_keyrw   existr   s
             r    uploadOssUtilities.upload   sX    WW\\$,,@
' $ 0 0 $11*=*"3"33KK0[=N=N<OvV , ) %%;;",55(,(@(@"nn&7 $ 0 02    &&7r!<MNPq "2"22LL+O+<<UV G	 3	' s!   AC4 $AC4 4
E>AEE)r   rY   rD   r   rC   rP   rO   r   rM   r]   r[   rN   rS   rJ   rU   r   N)r1   r2   r3   r4   r5   r!   staticmethodrl   r6   r   r   boolr   r   r7   r8   r#   r    r:   r:   E   si    /3b > >
9c 9"499<9v4c 4C 4-14&4+.4r#   r:   ) 
__future__r   rK   rF   r   rQ   datasets.utils.file_utilsr   r   oss2.credentialsr   modelscope.hub.apir   .modelscope.msdatasets.download.download_configr   modelscope.utils.config_dsr	   modelscope.utils.constantr
   r   r   modelscope.utils.loggerr   r   r+   r,   r-   rW   r\   rZ   r   r:   r8   r#   r    <module>r      sv    &  	   : $ ( % M 4C C .		 	$ 3 $Nk kr#   