§
    �ØjÔ  ã                   ó8   — d Z ddlmZmZ dZ G d„ d¦  «        ZdS )z3Token counting utilities for chunk size management.é    )ÚCallableÚOptionalg      @c                   óŽ   — e Zd ZdZddeeez           fd„Zededee         fd„¦   «         Z	edede
fd	„¦   «         Zdede
fd
„ZdS )ÚTokenCounterz�Counts tokens using tiktoken or a custom function.

    Falls back to character-based estimation if tiktoken is unavailable.
    NÚ	tokenizerc                 óÊ   — d | _         t          |¦  «        r|| _         n/t          |t          ¦  «        r|                      |¦  «        | _         | j         €| j        | _         d S d S ©N)Ú	_count_fnÚcallableÚ
isinstanceÚstrÚ_make_tiktoken_counterÚ_char_estimate)Úselfr   s     ú�/Users/tedpeters/Documents/Projects/Claude Projects/Misc Tools/notename-sheets/.venv/lib/python3.11/site-packages/pymupdf4llm/helpers/chunking/token_utils.pyÚ__init__zTokenCounter.__init__   si   € ØˆŒå�IÑÔð 	DØ&ˆDŒNˆNÝ˜	¥3Ñ'Ô'ð 	DØ!×8Ò8¸ÑCÔCˆDŒNàŒ>Ð!Ø!Ô0ˆDŒNˆNˆNð "Ð!ó    Úencoding_nameÚreturnc                 óf   ‡— 	 dd l }n# t          $ r Y d S w xY w|                     | ¦  «        Šˆfd„S )Nr   c                 óH   •— t          ‰                     | ¦  «        ¦  «        S r	   )ÚlenÚencode)ÚtextÚencs    €r   ú<lambda>z5TokenCounter._make_tiktoken_counter.<locals>.<lambda>$   s   ø€ �C §
¢
¨4Ñ 0Ô 0Ñ1Ô1€ r   )ÚtiktokenÚImportErrorÚget_encoding)r   r   r   s     @r   r   z#TokenCounter._make_tiktoken_counter   sY   ø€ ð	ØˆOˆOˆOˆOøÝð 	ð 	ð 	à�4�4ð	øøøð
 ×#Ò# MÑ2Ô2ˆØ1Ð1Ð1Ð1Ð1s   ƒ ˆ
•r   c                 óf   — t          dt          t          | ¦  «        t          z  ¦  «        ¦  «        S )Né   )ÚmaxÚintr   Ú_CHARS_PER_TOKEN_APPROX)r   s    r   r   zTokenCounter._char_estimate&   s&   € å�1•c�#˜d™)œ)Õ&=Ñ=Ñ>Ô>Ñ?Ô?Ð?r   c                 ó4   — |sdS |                       |¦  «        S )zCount tokens in text.r   )r
   )r   r   s     r   ÚcountzTokenCounter.count*   s!   € àð 	Ø�1Ø�~Š~˜dÑ#Ô#Ð#r   r	   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   Ústaticmethodr   r#   r   r&   © r   r   r   r   	   sÐ   € € € € € ðð ð
	1ð 	1 (¨3°©>Ô":ð 	1ð 	1ð 	1ð 	1ð ð	2¨cð 	2°h¸xÔ6Hð 	2ð 	2ð 	2ñ „\ð	2ð ð@˜Sð @ Sð @ð @ð @ñ „\ð@ð$˜#ð $ #ð $ð $ð $ð $ð $ð $r   r   N)r*   Útypingr   r   r$   r   r,   r   r   ú<module>r.      s]   ðØ 9Ð 9à %Ð %Ð %Ð %Ð %Ð %Ð %Ð %ð Ð ð%$ð %$ð %$ð %$ð %$ñ %$ô %$ð %$ð %$ð %$r   