o
    ï’2jP  ã                   @   s0  d Z ddlmZ ddlmZmZmZ eeef Ze	Z
	 eee	 ee	 f Z	 ee
ee
e
f ee
 f Z	 eeeeef ee f Z	 ee
ef Z	 eeef Z	 G dd„ deƒZG dd„ deƒZG dd	„ d	eƒZd
dlmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z! d
dl"m#Z#m$Z$m%Z%m&Z&m'Z' dS )u  Tokenizers â€” fast, batteries-included tokenization library.

Free-threaded Python (3.14t) note:
    Wheels built against free-threaded CPython declare ``Py_MOD_GIL_NOT_USED``
    and use ``RwLock``-guarded interior mutability so component setters are
    safe to call from multiple threads. Compound mutations
    (``tokenizer.post_processor.special_tokens = â€¦``) are still not atomic â€”
    use a Python lock if you need the read-then-write to be serialized.
    See ``docs/free-threading-audit.md`` for the full analysis.
é    )ÚEnum)ÚListÚTupleÚUnionc                   @   ó   e Zd ZdZdZdS )ÚOffsetReferentialÚoriginalÚ
normalizedN)Ú__name__Ú
__module__Ú__qualname__ÚORIGINALÚ
NORMALIZED© r   r   úN/var/www/html/whisper/venv/lib/python3.10/site-packages/tokenizers/__init__.pyr   G   ó    r   c                   @   r   )Ú
OffsetTypeÚbyteÚcharN)r
   r   r   ÚBYTEÚCHARr   r   r   r   r   L   r   r   c                   @   s    e Zd ZdZdZdZdZdZdS )ÚSplitDelimiterBehaviorÚremovedÚisolatedÚmerged_with_previousÚmerged_with_nextÚ
contiguousN)r
   r   r   ÚREMOVEDÚISOLATEDÚMERGED_WITH_PREVIOUSÚMERGED_WITH_NEXTÚ
CONTIGUOUSr   r   r   r   r   Q   s    r   é   )Ú
AddedTokenÚEncodingÚNormalizedStringÚPreTokenizedStringÚRegexÚTokenÚ	TokenizerÚdecodersÚmodelsÚnormalizersÚpre_tokenizersÚ
processorsÚtrainersÚ__version__)ÚBertWordPieceTokenizerÚByteLevelBPETokenizerÚCharBPETokenizerÚSentencePieceBPETokenizerÚSentencePieceUnigramTokenizerN)(Ú__doc__Úenumr   Útypingr   r   r   ÚintÚOffsetsÚstrÚTextInputSequenceÚPreTokenizedInputSequenceÚTextEncodeInputÚPreTokenizedEncodeInputÚInputSequenceÚEncodeInputr   r   r   Ú
tokenizersr#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   Úimplementationsr1   r2   r3   r4   r5   r   r   r   r   Ú<module>   s>    
þÿ	
þÿ	@ 