
    {bjP                     Z   S r SSKJr  SSKJrJrJr  \\\4   r\	r
 \\\	   \\	   4   r \\
\\
\
4   \\
   4   r \\\\\4   \\   4   r \\
\4   r \\\4   r  " S S\5      r " S S\5      r " S S	\5      rS
SKJrJrJrJrJrJrJrJrJrJrJrJrJ r J!r!  S
SK"J#r#J$r$J%r%J&r&J'r'  g)u  Tokenizers — fast, batteries-included tokenization library.

Free-threaded Python (3.14t) note:
    Wheels built against free-threaded CPython declare ``Py_MOD_GIL_NOT_USED``
    and use ``RwLock``-guarded interior mutability so component setters are
    safe to call from multiple threads. Compound mutations
    (``tokenizer.post_processor.special_tokens = …``) are still not atomic —
    use a Python lock if you need the read-then-write to be serialized.
    See ``docs/free-threading-audit.md`` for the full analysis.
    )Enum)ListTupleUnionc                       \ rS rSrSrSrSrg)OffsetReferentialG   original
normalized N)__name__
__module____qualname____firstlineno__ORIGINAL
NORMALIZED__static_attributes__r       ڏ/tmp/claude-0/-home-danvics-docker-quiz/c1e0577a-e42c-4a3d-b1ea-3edd61103a4e/scratchpad/stt/lib/python3.13/site-packages/tokenizers/__init__.pyr   r   G   s    HJr   r   c                       \ rS rSrSrSrSrg)
OffsetTypeL   bytecharr   N)r   r   r   r   BYTECHARr   r   r   r   r   r   L   s    DDr   r   c                   (    \ rS rSrSrSrSrSrSrSr	g)	SplitDelimiterBehaviorQ   removedisolatedmerged_with_previousmerged_with_next
contiguousr   N)
r   r   r   r   REMOVEDISOLATEDMERGED_WITH_PREVIOUSMERGED_WITH_NEXT
CONTIGUOUSr   r   r   r   r   r   Q   s    GH1)Jr   r      )
AddedTokenEncodingNormalizedStringPreTokenizedStringRegexToken	Tokenizerdecodersmodelsnormalizerspre_tokenizers
processorstrainers__version__)BertWordPieceTokenizerByteLevelBPETokenizerCharBPETokenizerSentencePieceBPETokenizerSentencePieceUnigramTokenizerN)(__doc__enumr   typingr   r   r   intOffsetsstrTextInputSequencePreTokenizedInputSequenceTextEncodeInputPreTokenizedEncodeInputInputSequenceEncodeInputr   r   r   
tokenizersr+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   implementationsr9   r:   r;   r<   r=   r   r   r   <module>rL      s  	  % % S/  5!$s)U3Z"78  	
.
./	
  	
#%>
>?	"#% 
 ')BBC O%<<= 
 
T      r   