o 2j @sUddlZddlZddlZddlmZddlmZmZddlm Z ddl m Z ddl m Z mZddlmZddlmZdd lmZmZdd lmZmZmZdd lmZmZddlZddlZdd lmZm Z m!Z!dd l"m#Z#ddl$m%Z%ddl&m'Z'ddl(m)Z)m*Z*ddl+m,Z,m-Z-m.Z.m/Z/m0Z0ddl1m2Z2m3Z3ddl4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;ddlZ>m?Z?m@Z@mAZAddlBmCZCe GdddZDe GdddeDZEe GdddeDZFeGejHZIeeJd<GdddeIZKGd d!d!ejHeKd"ZLGd#d$d$ejMjNZOGd%d&d&ejMjNZPd'eQd(eQfd)d*ZRd+eQd(eQfd,d-ZSd.eQd/eTdBd(efd0d1ZUd2eLd.eQd3eQd4eVd5eQf d6d7ZWd8d9ZXeLZYdS):N)deque)IterableIterator) ExitStack)deepcopy) dataclassfield)datetime)chain)Path PurePosixPath)AnyNoReturnUnion)quoteunquote)_DEFAULT_CALLBACK NoOpCallback TqdmCallback) apply_config) isfilelike) constants)CommitOperationCopyCommitOperationDelete)BucketNotFoundErrorEntryNotFoundErrorHfHubHTTPErrorRepositoryNotFoundErrorRevisionNotFoundError) hf_hub_urlhttp_get)SPECIAL_REFS_REVISION_REGEX BucketFile BucketFolderHfApiLastCommitInfoRepoFile RepoFolder)HFValidationErrorhf_raise_for_status http_backoffhttp_stream_backoff parse_hf_uri)md5c@s0eZdZUdZeed<eed<defddZdS)HfFileSystemResolvedPathz_Top level Data structure containing information about a resolved Hugging Face file system path.rootpathreturncCs|jd|jdS)N/)r0r1rstripselfr7Y/var/www/html/whisper/venv/lib/python3.10/site-packages/huggingface_hub/hf_file_system.py unresolve+sz"HfFileSystemResolvedPath.unresolveN)__name__ __module__ __qualname____doc__str__annotations__r9r7r7r7r8r/$s r/c@sveZdZUdZeed<eed<eed<eed<eddZeed<eddZeed <ed dd Z ed Bed <d dZ d S)"HfFileSystemResolvedRepositoryPathzLData structure containing information about a resolved path in a repository. repo_typerepo_idrevision path_in_repoFinitr0r1N)defaultrepr _raw_revisioncCsftj|jd|j}|jr|d|j|_n|jtjkr*|dt |j|_n||_|j |_ dS)N@) rREPO_TYPES_URL_PREFIXESgetrArBrIr0rCDEFAULT_REVISION safe_revisionrDr1)r6 repo_pathr7r7r8 __post_init__=s  z0HfFileSystemResolvedRepositoryPath.__post_init__) r:r;r<r=r>r?rr0r1rIrQr7r7r7r8r@/s  r@c@s4eZdZUdZeed<eddZeed<ddZdS) HfFileSystemResolvedBucketPathzHData structure containing information about a resolved path in a bucket. bucket_idFrEr0cCsd|j|_dS)Nzbuckets/)rSr0r5r7r7r8rQOsz,HfFileSystemResolvedBucketPath.__post_init__N) r:r;r<r=r>r?rr0rQr7r7r7r8rRHs  rR _cached_basecs(eZdZdZfddZddZZS)_Cacheda Metaclass for caching HfFileSystem instances according to the args. This creates an additional reference to the filesystem, which prevents the filesystem from being garbage collected when all *user* references go away. A call to the :meth:`AbstractFileSystem.clear_instance_cache` must *also* be made for a filesystem instance to be garbage collected. This is a slightly modified version of `fsspec.spec._Cached` to improve it. In particular in `_tokenize` the pid isn't taken into account for the `fs_token` used to identify cached instances. The `fs_token` logic is also robust to defaults values and the order of the args. Finally new instances reuse the states from sister instances in the main thread. cstj|i|i|_dSN)super__init___cache)clsargskwargs __class__r7r8rXgs z_Cached.__init__c Ost||}|dd}|j|tg|Ri|}|j|tjg|Ri|}|s<|jr<||jvr<||_ |j|St j |g|Ri|}|sh|jrh||jvrh|j| }| D] \}} t||| q]||_||_||_|jr~|s~||_ ||j|<|S)Nskip_instance_cacheF)rpop _tokenize threading get_ident main_threadidentcachablerY_latesttype__call___get_instance_stateitemssetattr _fs_token_ storage_argsstorage_options) rZr[r\skipfs_tokenfs_token_main_threadobjinstance_stateattr state_valuer7r7r8rios&      z_Cached.__call__)r:r;r<r=rXri __classcell__r7r7r]r8rUWs rUcs^eZdZdZdZdZddddddedBdeeBdBdedBd edBffd d Z e d ed efddZ dedededBd e ee dBffddZded e ee dBffddZ dTdededBd eeBfddZdTdedBd dfddZ   dUdedededBdedBd edf d d!ZdTdededBd dfd"d#Z $  dVded%ed&edBdedBd df d'd(Z dWded*ed+ededBd eeeeefBf d,d-Z $ $   dXded%ed+ededBd edBd&edBf d.d/Zded0ed%ed eeeBfd1d2Z ded e!e eeeeefffd3d4 Z"dTded&edBd eeffd5d6 Z#  $ $ $ dYded&edBd7ed*ed+ededBd eeeeeeeffBfd8d9Z$dTd:ed;ededBd dfdd?Z'dZded+ededBd eeeffd@dAZ(dBdCZ)dDdEZ*dFdGZ+ded efdHdIZ,e-dfd[fdJdK Z.e/dLdMZ0dNdOZ1dPdQZ2dRdSZ3Z4S)\ HfFileSystemaz Access a remote Hugging Face Hub repository as if were a local file system. > [!WARNING] > [`HfFileSystem`] provides fsspec compatibility, which is useful for libraries that require it (e.g., reading > Hugging Face datasets directly with `pandas`). However, it introduces additional overhead due to this compatibility > layer. For better performance and reliability, it's recommended to use `HfApi` methods when possible. The file system supports paths for the `hf://` protocol, which follows those URL schemes: * Models, Datasets and Spaces repositories: ``` hf://[@]/ hf://datasets/[@]/ hf://spaces/[@]/ ``` * Buckets (generic storage): ``` hf://buckets// ``` Note: when using the [`HfFileSystem`] directly, passing the `hf://` protocol prefix is optional in paths. Args: endpoint (`str`, *optional*): Endpoint of the Hub. Defaults to . token (`bool` or `str`, *optional*): A valid user access token (string). Defaults to the locally saved token, which is the recommended method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). To disable authentication, pass `False`. block_size (`int`, *optional*): Block size for reading and writing files. expand_info (`bool`, *optional*): Whether to expand the information of the files. **storage_options (`dict`, *optional*): Additional options for the filesystem. See [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.__init__). Usage: ```python >>> from huggingface_hub import hffs >>> # List files >>> hffs.glob("my-username/my-model/*.bin") ['my-username/my-model/pytorch_model.bin'] >>> hffs.ls("datasets/my-username/my-dataset", detail=False) ['datasets/my-username/my-dataset/.gitattributes', 'datasets/my-username/my-dataset/README.md', 'datasets/my-username/my-dataset/data.json'] >>> # Read/write files >>> with hffs.open("my-username/my-model/pytorch_model.bin") as f: ... data = f.read() >>> with hffs.open("my-username/my-model/pytorch_model.bin", "wb") as f: ... f.write(data) ``` Specify a token for authentication: ```python >>> from huggingface_hub import HfFileSystem >>> hffs = HfFileSystem(token=token) ``` rJhfN)endpointtoken block_size expand_inforzr{r|r}csTtj|i||p tj|_||_t||d|_||_||_ i|_ i|_ i|_ dS)N)rzr{) rWrXrENDPOINTrzr{r%_apir|r}_repo_and_revision_exists_cache_bucket_exists_cachedircache)r6rzr{r|r}r[ror]r7r8rXs  zHfFileSystem.__init__threading_identr2cs\dptjd<dd<fddtD|||f}tt|}|S)zDeterministic token for cachingrzr{ci|]}||qSr7r7).0keyr\r7r8 z*HfFileSystem._tokenize..)rMrr~sortedr.r>encode hexdigest)rZrr[r\ tokenize_argshr7rr8ras  zHfFileSystem._tokenizerArBrCc Cs|||f|jvrqz |jj|||tjdWnKttfy=}zd|f|j|||f<d|f|j||df<WYd}~n8d}~wty`}zd|f|j|||f<d|j||df<WYd}~nd}~wwd|j|||f<d|j||df<|j|||fS)N)rCrAtimeoutFTN)rr repo_inforHF_HUB_ETAG_TIMEOUTrr)r)r6rArBrCer7r7r8_repo_and_revision_exists"   z%HfFileSystem._repo_and_revision_existrSc Csb||jvr,z|j|Wnty&}z d|f|j|<WYd}~n d}~wwd|j|<|j|S)NFr)rr bucket_infor)r6rSrr7r7r8_bucket_exists s   zHfFileSystem._bucket_existsr1cCs"||}|s td|ddkrtd|dttj|}|jr=||j \}}|s5t ||t |j |j dS|dur|j dur|j |kr|ddtjvr]|dddn|}|d d\}}d|vrq|ddn|d f\} } t| } | |krtd | d |d ||j||\} }| st ||t|j||| | dS|j dur|dur|j }||j|j |\} }| st ||d} d |vr|j dur|ddtjvr|dddn|}|d dd}|j r|dt|j d n|} |dur|ntj}t|j|j ||j | dS)a Resolve a Hugging Face file system path into its components. Args: path (`str`): Path to resolve. revision (`str`, *optional*): The revision of the repo to resolve. Defaults to the revision specified in the path. Returns: [`HfFileSystemResolvedPath`]: Resolved path information containing `repo_type`, `repo_id`, `revision` and `path_in_repo`. Raises: `ValueError`: If path contains conflicting revision information. `NotImplementedError`: If trying to list repositories. z [!WARNING] > Note: When possible, use `HfApi.delete_file()` for better performance. Args: path (`str`): Path to delete. recursive (`bool`, *optional*): If True, delete directory and all its contents. Defaults to False. maxdepth (`int`, *optional*): Maximum number of subdirectories to visit when deleting recursively. revision (`str`, *optional*): The git revision to delete from. r)rrrCc"g|] }|s|jqSr7isdirrr1rr1r5r7r8 "z#HfFileSystem.rm..rcrr7rrr5r7r8rrcSsg|]}t|dqS))rD)r)rrDr7r7r8rrzDelete  z recursively rJNz up to depth rr)rBrAr{ operationsrCrrr)r expand_pathrrRrrrS create_commitrBrAr{rCrMrr9) r6r1rrrCr\rpathsr paths_in_reporrr7r5r8rms(    zHfFileSystem.rmTdetailrefreshc s|j|d}|z|jf||d|}WnBty[|js(tdz|j|f||d|}Wn tyDg}Ynwfdd|D}t|dkrYtdYnw|r`|Sdd|DS)a List the contents of a directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.ls). > [!WARNING] > Note: When possible, use `HfApi.list_repo_tree()` for better performance. Args: path (`str`): Path to the directory. detail (`bool`, *optional*): If True, returns a list of dictionaries containing file information. If False, returns a list of file paths. Defaults to True. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to list from. Returns: `list[Union[str, dict[str, Any]]]`: List of file paths (if detail=False) or list of file information dictionaries (if detail=True). r)rrCNcg|] }|dkr|qSnamer7rorr7r8rz#HfFileSystem.ls..rcSsg|]}|dqSrr7rr7r7r8rs)rr9_ls_treerr1rrr)r6r1rrrCr\routr7rr8lss$  "    zHfFileSystem.lsc sT|dur|n jdurjnd}j||d}|}|j}|r"|nd}g} |jvr |s j|} | | g} |rdtfdd| D} | r| \} |dusZ|kr| djvri| | dnj| d} | | | fdd| D| sLg}|rt |t rfd d| D}|r| s|r|rt j | |}|d s||ks|t| |vr|d n||durԈt|dd }||8}fd d| D} tjD]}|d rj|dqjd| j|d |||d | St |trj|j|j |d}njj|j|j |||j|jd}|D]z}|d |j }t |t rN||j!d|j"|j#|j$|j%|j&d}n-t |t'rb||j!d|j$|j(|j)d}nt |t*rs|dd|j+|j%d}n|dd|j)d}|d}j,|g||t|dd |dus|kr| |q-| S)NFrrcs g|] }|ddkr|fqSrh directoryr7r path_infodepthr7r8rs z)HfFileSystem._ls_tree..rcs$g|]}|ddkrd|fqS)rhrrr7rrr7r8r)s   cs&g|]}|ddur|dqS) last_commitNr)rrr5r7r8r3s&r3cs"g|] }|dds|qS)rr3) startswithr) common_pathr7r8rDrT)rrrCr}r)prefixr)rexpandrCrAfilersizerhblob_idlfsxet_hashrsecurity)rrrhrmtime uploaded_atrrrrrhtree_idr)rrrhr)-r}rr9r0rextendrpopleftappendrr@osr1 commonprefixendswithr r4rrrlistrr`rrR_list_bucket_tree_with_foldersrSrlist_repo_treerBrCrAr'rrrrrrr#rrr(r setdefault)r6r1rrrCr}rr root_pathrcached_path_infosdirs_not_in_dircache dirs_to_visitdir_infodirs_not_expanded common_prefixcommon_path_depth cached_pathtreer cache_pathcache_path_info parent_pathr7)rrr6r8rs        E      zHfFileSystem._ls_treerc Cs|jj|||d}i}|rd|dnd}g}|D]O}|||s$qtt|jjd| dD]5} tdt | |j d} | j|vrO|| | || j<q3| j durh|| j} | j dusd| j | j krh| j | _ q3q|swt d|d |d |S) zrr) r6rSrr bucket_filesbucket_folders min_depthr bucket_entryparent_bucket_folder_strparent_bucket_folder bucket_folderr7r7r8rs4 "        z+HfFileSystem._list_bucket_tree_with_foldersc/s>|j||dd}tj|g|Ri|EdHdS)a Return all files below the given path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.walk). Args: path (`str`): Root path to list files from. Returns: `Iterator[tuple[str, list[str], list[str]]]`: An iterator of (path, list of directory names, list of file names) tuples. rCrN)rrMr9rWwalk)r6r1r[r\r]r7r8rs $zHfFileSystem.walkc s0|j||dd}tj|fd|i|S)a Find files by glob-matching. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.glob). Args: path (`str`): Path pattern to match. maxdepth (`int`, *optional*): Maximum depth to descend into directories. By default, no limit. Returns: `list[str]`: List of paths matching the pattern. rCrr)rrMr9rWglob)r6r1rr\r]r7r8rszHfFileSystem.globwithdirsc s|dur |dkr td|j||d}|}z|j|fd||d|Wn-tySz|j|fd|i|dd krB|iiniWn tyPiYnwYn(w|s^d d Dn|j|fi|} | dd krr| gnd dDt} |s| Sfdd| DS)a List all files below path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.find). Args: path (`str`): Root path to list files from. maxdepth (`int`, *optional*): Maximum depth to descend into subdirectories. withdirs (`bool`, *optional*): Include directory paths in the output. Defaults to False. detail (`bool`, *optional*): If True, returns a dict mapping paths to file information. Defaults to False. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to list from. Returns: `Union[list[str], dict[str, dict[str, Any]]]`: List of paths or dict of file information. Nrzmaxdepth must be at least 1rT)rrrrCrhrcSsg|] }|ddkr|qSrr7rr7r7r8r rz%HfFileSystem.find..rcSsi|]}|d|qSrr7rr7r7r8rrz%HfFileSystem.find..crr7r7)rrrr7r8rr)rrr9rrinfoFileNotFoundErrorr) r6r1rrrrrCr\rrnamesr7rr8finds4     zHfFileSystem.findpath1path2c KsH|j||d}|j||d}t|tst|trtd|j|jko'|j|jk}|rSd|d|}|jj|j|j|j| d|| ddt |j |j |jdgd n?|j |d |jd } | } Wd n1skwYd|d|}|jj| |j |j|j|j|j| d|| dd |j|d |j|d d S)a Copy a file within or between repositories. > [!WARNING] > Note: When possible, use `HfApi.upload_file()` for better performance. Args: path1 (`str`): Source path to copy from. path2 (`str`): Destination path to copy to. revision (`str`, *optional*): The git revision to copy from. rz)Copy from/to buckets is not available yetzCopy z to rrrJ)src_path_in_reporD src_revision)rBrArCrrrrNpath_or_fileobjrDrBr{rArCrrr)rrrRrrArBrrrCrMrrDopenread upload_filer{rr9) r6rrrCr\resolved_path1resolved_path2 same_reporfcontentr7r7r8cp_filesN      zHfFileSystem.cp_filecKs|j|fii|ddi}d|vr&|ddur!td|d|djSd|vr2|dr2|dSd|vr>|dr>|dStd |d |d ) ak Get the last modified time of a file. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.modified). Args: path (`str`): Path to the file. Returns: `datetime`: Last modified time of the file. r}TrNz9'modified' is not implemented for repository paths like 'r rrz'Cannot determined 'modified' for path 'z ' (info: ))rrdate)r6r1r\rr7r7r8modifiedSs   zHfFileSystem.modifiedc sN|j|d}||d|jdur|jnd}|jsRdddt|tr+dd<t|trQ|rQ|jj|j |j |j d d }idt |j |j|jd d nt|tr{|}|j||d fdd|j|D} | svtd| dnd|}|s||jvr||||jvrfdd|j|D} | std| d|sdus|rrddur|jj|j |j||j |j d} | std| d} t|j |j |j d|jd} t| tr| d| j| jd| j| j| j| j| jdn| d| jdd| j| jd|sfdddDdus%JS)aS Get information about a file or directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.info). > [!WARNING] > Note: When possible, use `HfApi.get_paths_info()` or `HfApi.repo_info()` for better performance > (or `HfApi.get_bucket_paths_info()` or `HfApi.bucket_info()` for buckets) Args: path (`str`): Path to get info for. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to get info from. Returns: `dict[str, Any]`: Dictionary containing file information (type, size, commit info, etc.). rr}NFrr)rrrhr)rArC)oidtitler+)rr)rcrrr7rrr7r8rrz%HfFileSystem.info..crrr7rrr7r8rr)rrCrArJ)rDrIr3rrrcrr7r7)rkrr7r8rrz%HfFileSystem.info..) rr9rMr}r1rr@rlist_repo_commitsrBrArCr& commit_idr/ created_atrRrrrrget_paths_inforDrIr'rrrrrrr) r6r1rrCr\rr}rrout1 paths_inforrr7)rr1r8rls           &    zHfFileSystem.infocKsDz|ddr |||j|fi|WdSty!YdSw)a Check if a file exists. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.exists). > [!WARNING] > Note: When possible, use `HfApi.file_exists()` for better performance. Args: path (`str`): Path to check. Returns: `bool`: True if file exists, False otherwise. rFT)rMrrOSError)r6r1r\r7r7r8existss   zHfFileSystem.existscC*z ||ddkWStyYdSw)ai Check if a path is a directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isdir). Args: path (`str`): Path to check. Returns: `bool`: True if path is a directory, False otherwise. rhrFrr7r6r1r7r7r8r   zHfFileSystem.isdircCr9)a` Check if a path is a file. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isfile). Args: path (`str`): Path to check. Returns: `bool`: True if path is a file, False otherwise. rhrFr:r;r7r7r8isfiler<zHfFileSystem.isfilecCsl||}t|tr|jd|jdt|j}nt|j|j |j |j |jd}| |r4| ddd}|S)z Get the HTTP URL of the given path. Args: path (`str`): Path to get URL for. Returns: `str`: HTTP URL to access the file or directory on the Hub. z /buckets/z /resolve/)rArCrzz/tree/r)rrrRrzrSrr1r rBrDrArCrreplace)r6r1rurlr7r7r8r?s   zHfFileSystem.urlc sT|d}t|dh}t|ttfrt|dkr)tj||f||d|St |r0|}n| |r>t j |dddSt|t tfrPt j t j|ddd}|dur]t|d}d}|} |j||d } |j||d d } || z+t|| ||| d|jt|tr|jndd || W|r|dSdS|r|ww) ad Copy single remote file to local. > [!WARNING] > Note: When possible, use `HfApi.hf_hub_download()` or `HfApi.download_bucket_files` for better performance. Args: rpath (`str`): Remote path to download from. lpath (`str`): Local path to download to. callback (`Callback`, *optional*): Optional callback to track download progress. Defaults to no callback. outfile (`IO`, *optional*): Optional file-like object to write to. If provided, `lpath` is ignored. rCr)callbackoutfileT)exist_okNFwbrr)r? temp_filedisplayed_filename expected_size resume_sizeheaders _tqdm_bar)rMsetkeysrrrrrWget_filerrrmakedirsr>r r1dirnamer!tellrrset_sizer!r?r9r_build_hf_headerstqdmseekclose) r6rpathlpathr@rAr\rCunhandled_kwargs close_file initial_posresolve_remote_pathrFr]r7r8rL1sF        zHfFileSystem.get_filecCtd)zA context within which files are committed together upon exit Requires the file class to implement `.commit()` and `.discard()` for the normal and exception cases. (Transactional commits are not supported.rr5r7r7r8 transactionos zHfFileSystem.transactioncCr[)z@Begin write transaction for deferring files, non-context versionr\r]r5r7r7r8start_transactionzszHfFileSystem.start_transactioncCstt||j|j|ffSrV) make_instancerhrnrorjr5r7r7r8 __reduce__s zHfFileSystem.__reduce__cCst|jt|jt|jdS)N)rrr)rrrrr5r7r7r8rjsz HfFileSystem._get_instance_staterV)rNN)FNN)TFN)FFNNN)NFFFN)FNr2N)5r:r;r<r= root_markerprotocolr>boolintrX classmethodratuple Exceptionrrr@rRrrrrrrrdictr rrrr#r$rrrrrr)r r,rr8rr=r?rrLpropertyr^r_rarjrwr7r7r]r8rxsB     O$  3 .   ,($ =<(n>   rx) metaclasscseZdZddedededBffdd ZfddZd ed ed efd d Z dddZ dde d dfddZ dfdd Z d efddZZS)rNfsr1rCc stz |j||d|_Wnty'}zd|ddvr"t|d|d}~wwtj||jfi||dS)NrwrrJB. Make sure the repository and revision exist before writing data.)rrrrMrWrXr9)r6rmr1rCr\rr]r7r8rXszHfFileSystemFile.__init__cst|dsdStSNr)hasattrrW__del__r5r]r7r8rrs  zHfFileSystemFile.__del__startendr2cCsLdd|d|di|jj}|}td||tjd}t||jS)Nrangebytes=-rGETrHr) rmrrQr?r+rHF_HUB_DOWNLOAD_TIMEOUTr*r()r6rsrtrHr?rr7r7r8 _fetch_ranges zHfFileSystemFile._fetch_rangecCstjddd|_dS)Nzhffs-F)rr)tempfileNamedTemporaryFilerDr5r7r7r8_initiate_uploadsz!HfFileSystemFile._initiate_uploadFfinalc Cs|jd|j}|j||rg|jt|jtr1|j j j |jj |jj |jjfgdn#|j j j|jj |jj|jj|j j|jj|jj|jd|jddt|jj |j j|jddSdS)Nr)addrrrr)bufferrSr"rDwriterTrrrRrmrrrSrr1r#rDrBr{rArCr\rMrremoverr9)r6rblockr7r7r8 _upload_chunks0         zHfFileSystemFile._upload_chunkr-cs|jdkr;|dus |dkr;|jdkr;|jj|jddd}|}|jt|7_|WdS1s6wYt|S)zRead remote file. If `length` is not provided or is -1, the entire file is downloaded and read. On POSIX systems the file is loaded in memory directly. Otherwise, the file is downloaded to a temporary file and read from there. rNr-r)r|)rlocrmr!r1r"rrW)r6lengthr'rr]r7r8r"s$ zHfFileSystemFile.readcC|j|jSrVrmr?r1r5r7r7r8r?zHfFileSystemFile.urlrVrb)Fr-)r:r;r<rxr>rXrrrfbytesr|rrerr"r?rwr7r7r]r8rs    rc seZdZ    d!dededededBd ed ef fd d Zd"d edefddZd#defddZd#de e dede fddZ defddZ fddZ ddZdd ZZS)$rrNrnonermr1rrCr| cache_typec s|dkr td||dkrtd|d|vr"td|dz |j||d|_Wn"tyN}zd|d d vrDt|d |WYd}~nd}~ww|jdd |_tj||jf|||d |d|_ |t |_ d|_ t |_dS)Nrz:HfFileSystemStreamFile only supports block_size=0 but got rz?HfFileSystemStreamFile only supports cache_type='none' but got rnz;HfFileSystemStreamFile only supports reading but got mode='r rrrJro)rrrr|r)rrrrrMr9detailsrWrXresponser _exit_stack_stream_iterator bytearray_stream_buffer) r6rmr1rrCr|rr\rr]r7r8rXs>    zHfFileSystemStreamFile.__init__rwhencecCs2|dkr |dkr dS||jkr|dkrdStd)NrrzCannot seek streaming HF file)rr)r6rrr7r7r8rSs zHfFileSystemStreamFile.seekr-rcCs|jdur |d} z |jdus|jdurWdS||j|}|jt|7_|WStyH|jdur=|j|r@|d}Ynwq )zRead the remote file. If the file is already open, we reuse the connection. Otherwise, open a new connection and read from it. If reading the stream fails, we retry with a new connection. NFT)r_open_connectionr_read_from_streamrrrirT)r6r retried_oncerr7r7r8r"s&    zHfFileSystemStreamFile.readiteratorr2cCs|dkrdS|dkr"t|j}|j|D]}||qt|S|t|jkr:t|jd|}|jd|=|St|j}|j|D]+}|t|}|t|krZ||qF||d||j||dt|St|S)zRead up to `length` bytes from stream buffer and stream. If length < 0, read until EOF. If EOF is reached before length, fewer bytes may be returned. rrN)rrrrrr)r6rrbufchunkresultneedr7r7r8r#s,         z(HfFileSystemStreamFile._read_from_streamcCrrVrr5r7r7r8r?ErzHfFileSystemStreamFile.urlcs"t|dsdS|jtSrp)rqrrTrWrrr5r]r7r8rrHs   zHfFileSystemStreamFile.__del__cCst|j|j|j|j|jjffSrV)reopenrmr1r blocksizecacherr5r7r7r8raOsz!HfFileSystemStreamFile.__reduce__c Cs|jd|_|}|jj}|jdkr d|jd|d<|j t d||t j d|_ zt|j WntyR}z|j jdkrMd|_ WYd}~dSd}~ww|j |_dS) z%Open a connection to the remote file.NrrvrwRangerxryi)rrrr?rmrrQrr enter_contextr,rrzrr*r status_code iter_bytes)r6r?rHrr7r7r8rRs0     z'HfFileSystemStreamFile._open_connection)rNrr)rr)r:r;r<rxr>rfrXrSr"rrrr?rrrarrwr7r7r]r8rs2#" rrCr2cCst|r|St|SrV)r"match safe_quoterr7r7r8rOqsrOscCs t|ddS)NrJ)safe)r)rr7r7r8rus rr1rcCsN|}t|tr |d}nt|tr|d}n t|tr"|d}t||)Nz (repository not found)z (revision not found)z (invalid repository id))rrrr)r)r1rmsgr7r7r8rys       rrmrr|rcCs|j||||dS)Nr)r!)rmr1rr|rr7r7r8rsrcCs0||i|}|D] \}}t|||q |SrV)rkrl)rZr[r\rtrmrurvr7r7r8r`sr`)Zrr}rb collectionsrcollections.abcrr contextlibrcopyr dataclassesrrr itertoolsr pathlibr r typingr rr urllib.parserrfsspechttpxfsspec.callbacksrrr fsspec.configr fsspec.utilsrrJr _commit_apirrerrorsrrrrr file_downloadr r!hf_apir"r#r$r%r&r'r(utilsr)r*r+r,r-utils.insecure_hashlibr.r/r@rRrhAbstractFileSystemrTr?rUrxspecAbstractBufferedFilerrr>rOrrirrfrr`hffsr7r7r7r8sf         $   5 K