§
    �ØjRM  ã            	       ó‚  — d Z ddlZddlmZmZ ddlmZ ddlmZm	Z	 ddl
mZmZmZmZ  ej        d¦  «        Z ej        d	¦  «        Zd
edee         fd„Z ej        d¦  «        Z ej        d¦  «        Z ej        dej        ¦  «         ej        dej        ¦  «         ej        dej        ¦  «         ej        dej        ¦  «        gZd
edeeee         f         fd„Zd
edefd„Zd
edefd„Zdee         deee f         fd„Z!dee         defd„Z"	 	 d defd„Z#dede d
edee          fd„Z$ G d„ d¦  «        Z%dS )!z7Step A: Convert LayoutBox textlines into SentenceUnits.é    N)ÚCounterÚdefaultdict)ÚOptionalé   )ÚSentenceUnitÚ
union_bbox)Úbox_to_markdownÚcreate_list_item_levelsÚfigure_placeholderÚtable_contentz'[.!?]\s*["\'\)\]]*(\s+)(?=[A-Z"\'\(\[])uZ   [.!?ã€‚ï¼�ï¼Ÿ]\s*["\'\)\]ã€�ã€�]*(\s*)(?=[A-Zê°€-íž£ã„±-ã…Žã…�-ã…£ä¸€-é¿¿"\'\(\[ã€Œã€Ž]|$)ÚtextÚreturnc                 óZ  — g }d}|                      | ¦  «        D ]]}| ||                     d¦  «        …                              ¦   «         }|r|                     |¦  «         |                     d¦  «        }Œ^| |d…                              ¦   «         }|r|                     |¦  «         |S )añ  Split *text* into sentences at *pattern*, losing only whitespace.

    ``pattern`` matches a sentence end and marks the boundary whitespace as
    group 1; everything else the match consumed (the punctuation and any
    closing quote/bracket) stays with the preceding sentence.  Re-joining
    the returned pieces reproduces *text* up to whitespace.

    ``re.split`` cannot be used for this: it discards the whole match, which
    is what dropped closing quotes and brackets from chunk text.
    r   r   N)ÚfinditerÚstartÚstripÚappendÚend)r   ÚpatternÚpiecesÚposÚmÚpieceÚtails          ú¢/Users/tedpeters/Documents/Projects/Claude Projects/Misc Tools/notename-sheets/.venv/lib/python3.11/site-packages/pymupdf4llm/helpers/chunking/sentence_builder.pyÚ_split_sentence_textr   &   s²   € ð €FØ
€CØ×Ò˜dÑ#Ô#ð ð ˆØ�S˜Ÿš ™œ�^Ô$×*Ò*Ñ,Ô,ˆØð 	!Ø�MŠM˜%Ñ Ô Ð Ø�eŠe�A‰hŒhˆˆØ���Œ:×ÒÑÔ€DØð Ø�Š�dÑÔÐØ€Mó    z[ \t]+z^[#>*_`\s]+z(^(?:figure|fig\.?\s*)\s*\d+(?:[.\-]\d+)*z'^(?:table|tbl\.?\s*)\s*\d+(?:[.\-]\d+)*z^\(\s*[a-z0-9]\s*\)z^(?:source|notes?)\s*:c                 ó  — |                       ¦   «         }|sdS t          D ]^}|                     |¦  «        rG|                     ¦   «         }|                     d¦  «        r dS |                     d¦  «        r dS  dS Œ_dS )z“Detect if text is a caption and determine target type.

    Returns (is_caption, target_type) where target_type is "figure", "table", or None.
    )FN)ÚfigureÚfig)Tr   )ÚtableÚtbl)Tr!   )TN)r   Ú_CAPTION_PATTERNSÚsearchÚlowerÚ
startswith)r   ÚstrippedÚpatÚlows       r   Ú_detect_captionr*   L   s›   € ð
 �zŠz‰|Œ|€HØð Øˆ{å ð ð ˆØ�:Š:�hÑÔð 	Ø—.’.Ñ"Ô"ˆCØ�~Š~Ð/Ñ0Ô0ð &Ø%�~�~Ø�~Š~Ð.Ñ/Ô/ð %Ø$�}�}Ø�:�:ð	ð ˆ;r   c                 óˆ   — |                       ¦   «                              ¦   «         }t                               d|¦  «        }|S )z>Normalize text for comparison: lowercase, collapse whitespace.ú )r   r%   Ú_MULTI_SPACEÚsub)r   Úts     r   Ú_normalize_textr0   a   s5   € à�
Š
‰Œ×ÒÑÔ€AÝ×Ò˜˜aÑ Ô €AØ€Hr   c                 ó8   — t                                d| ¦  «        S )zGStrip leading markdown decoration for pattern matching (captions, TOC).Ú )Ú_MD_DECORATIONr.   ©r   s    r   Ú_strip_md_decorationr5   h   s   € å×Ò˜b $Ñ'Ô'Ð'r   Ú	textlinesc                 ó¾  — t          ¦   «         }| D ]¤}|                     dg ¦  «        D ]‹}|                     dd¦  «        }t          |                     ¦   «         ¦  «        }|dk    rLt	          |                     dd¦  «        d¦  «        }|                     dd¦  «        }|||fxx         |z  cc<   ŒŒŒ¥|sdS |                     d¦  «        d         \  \  }}}||fS )	zJGet the most common (font_size, font_flags) from spans by character count.Úspansr   r2   r   Úsizer   Úflags)g        r   )r   ÚgetÚlenr   ÚroundÚmost_common)	r6   Úfont_counterÚtlÚspanr   Ú
char_countr9   r:   Ú_s	            r   Ú_get_dominant_fontrD   m   sø   € å‘9”9€LØð :ð :ˆØ—F’F˜7 BÑ'Ô'ð 	:ð 	:ˆDØ—8’8˜F BÑ'Ô'ˆDÝ˜TŸZšZ™\œ\Ñ*Ô*ˆJØ˜AŠ~ˆ~Ý˜TŸXšX f¨aÑ0Ô0°!Ñ4Ô4�ØŸš ¨!Ñ,Ô,�Ø˜d E˜]Ð+Ð+Ô+¨zÑ9Ð+Ð+Ñ+øð	:ð ð Øˆvà#×/Ò/°Ñ2Ô2°1Ô5Ñ�M€Tˆ5�1Ø�ˆ;Ðr   c                 ó<   ‡ — ˆ fd„}t           |¦   «         ¦  «        S )z0Compute the union bounding box of all textlines.c               3   ó  •K  — ‰D ]†} |                       d¦  «        }|€Œ	 t          |d         ¦  «        t          |d         ¦  «        t          |d         ¦  «        t          |d         ¦  «        fV — Œp# t          t          f$ r Y Œƒw xY wd S )NÚbboxr   r   é   é   )r;   ÚfloatÚ	TypeErrorÚ
IndexError)r@   rG   r6   s     €r   Ú_iter_bboxesz)_compute_bbox_union.<locals>._iter_bboxes‚   s¢   øè è € Øð 	ð 	ˆBØ—6’6˜&‘>”>ˆDØˆ|ØðÝ˜T !œW‘~”~¥u¨T°!¬W¡~¤~µu¸TÀ!¼W±~´~ÅuÈTÐRSÌWÁ~Ä~ÐVÐVÐVÐVÐVøÝ�zÐ*ð ð ð Ø�ðøøøð	ð 	s   ¡AA6Á6B
Â	B
)r   )r6   rM   s   ` r   Ú_compute_bbox_unionrN   €   s2   ø€ ðð ð ð ð õ �l�l‘n”nÑ%Ô%Ð%r   Úboxclassc                 ó  — | dk    rd|pddœS | dk    r |p|r|rt          |||¦  «        nd}d|pddœS | dk    rd	diS | d
v rddiS | dv rddiS | dk    rddiS | dv rddiS | dk    r!ddi}|rt          |¦  «        \  }}|�||d<   |S i S )a¶  Map LayoutBox.boxclass to structure hint flags.

    Returns only non-default values (SentenceUnit defaults handle the rest).

    Heading depth comes first from the engine's font-statistics assignment
    (LayoutBox.header_level, the same value the markdown renderers use for
    #/##/### depth), so the section hierarchy nests the way the rendered
    text does. The TOC match is a fallback for boxes the engine left
    unleveled.
    ÚtitleTr   )Úis_heading_hintÚheading_level_hintúsection-headerNrH   z	list-itemÚis_list_item©r!   útable-fallbackÚis_table_content©ÚpictureÚformulaÚis_figure_relatedÚfootnoteÚis_footnote©úpage-headerúpage-footerÚis_header_footerÚcaptionÚ
is_captionÚcaption_target_type)Ú_match_toc_levelr*   )	rO   ÚtocÚpage_nor   Úheader_levelÚlevelÚhintsrC   Útargets	            r   Ú_boxclass_to_hintsrm   Ž   s.  € ð �7ÒÐØ#'Ø&2Ð&7°að9ð 9ð 	9àÐ#Ò#Ð#Øð LØ47ÐJ¸DÐJÕ˜S '¨4Ñ0Ô0Ð0Àdð 	à#'¸u¸zÈÐJÐJÐJØ�;ÒÐØ Ð%Ð%ØÐ.Ð.Ð.Ø" DÐ)Ð)ØÐ)Ð)Ð)Ø# TÐ*Ð*Ø�:ÒÐØ˜tÐ$Ð$ØÐ1Ð1Ð1Ø" DÐ)Ð)Ø�9ÒÐØ˜tÐ$ˆØð 	6Ý'¨Ñ-Ô-‰IˆAˆvØÐ!Ø/5�Ð+Ñ,ØˆØ€Ir   rg   rh   c                 óB  — | r|sdS t          |¦  «        }|sdS | D ]‚}t          |¦  «        dk     rŒ|d         |d         |d         }}}||k    rŒ5t          t          |¦  «        ¦  «        }|sŒT|                     |¦  «        s|                     |¦  «        r|c S ŒƒdS )zTTry to match text against TOC entries for the given page to determine heading level.NrI   r   r   rH   )r0   r<   Ústrr&   )	rg   rh   r   ÚnormÚentryrj   rQ   Útoc_pageÚtoc_norms	            r   rf   rf   µ   sÔ   € àð �dð Øˆtå˜4Ñ Ô €DØð Øˆtàð 
ð 
ˆÝˆu‰:Œ:˜Š>ˆ>ØØ!& q¤¨5°¬8°U¸1´X�hˆuˆØ�wÒÐØÝ"¥3 u¡:¤:Ñ.Ô.ˆØð 	ØØ�?Š?˜8Ñ$Ô$ð 	¨×(;Ò(;¸DÑ(AÔ(Að 	ØˆLˆLˆLð	ð ˆ4r   c            	       ó¤  — e Zd ZdZddefd„Zdee         fd„Z	 ddee         fd„Z	dee         fd	„Z
dee         fd
„Z	 ddee         fd„Zdeeeef                  fd„Z	 ddee         dedee         fd„Zedee         dedeee                  fd„¦   «         Zedee         defd„¦   «         Zedee         dee         fd„¦   «         ZdS )ÚSentenceBuilderz9Converts ParsedDocument's LayoutBoxes into SentenceUnits.ÚdefaultÚsplitterc                 ób   — || _         d| _        |dk    rt          | _        d S t          | _        d S )Nr   Úmultilingual)rw   Ú_figure_countÚ_SENT_END_MULTIÚ	_split_reÚ_SENT_END_EN)Úselfrw   s     r   Ú__init__zSentenceBuilder.__init__Ð   s2   € Ø ˆŒØˆÔØ�~Ò%Ð%Ý,ˆDŒNˆNˆNå)ˆDŒNˆNˆNr   r   c                 ó@  — g }d}d}d| _         |j        D ]r}t          |j        ¦  «        }t	          |j        ¦  «        D ]G\  }}|                      |||||j        ||¬¦  «        }	|	D ]}
|                     |
¦  «         |dz  }|
}ŒŒHŒs|                      |¦  «        }|  	                    |¦  «        }t          t          |¦  «        dz
  ¦  «        D ]L}||dz            j        �:||         j        ||dz            j        k    r||dz            j        ||         _        ŒM|S )z=Build SentenceUnits from all pages/boxes in a ParsedDocument.r   N)ÚpageÚbox_idxÚboxÚsent_id_startrg   Ú	prev_unitÚlist_item_levelsr   )rz   Úpagesr
   ÚboxesÚ	enumerateÚ_process_boxrg   r   Ú_merge_same_page_hf_unitsÚ_renumber_sent_idsÚranger<   Úline_gap_beforerh   Úline_gap_after)r~   ÚdocÚunitsÚsent_idr…   r�   r†   r‚   rƒ   Ú	new_unitsÚuÚis               r   Úbuild_from_documentz#SentenceBuilder.build_from_documentØ   s\  € àˆØˆØˆ	ØˆÔà”Ið 	"ð 	"ˆDÝ6°t´zÑBÔBÐÝ )¨$¬*Ñ 5Ô 5ð "ð "‘�˜Ø ×-Ò-ØØ#ØØ")ØœØ'Ø%5ð .ñ ô �	ð #ð "ð "�AØ—L’L ‘O”O�OØ˜q‘L�GØ !�I�Ið"ð"ð  ×.Ò.¨uÑ5Ô5ˆØ×'Ò'¨Ñ.Ô.ˆå•s˜5‘z”z A‘~Ñ&Ô&ð 	Gð 	GˆAØ�Q˜‘UŒ|Ô+Ð7¸EÀ!¼HÔ<LÐPUÐVWÐZ[ÑV[ÔP\ÔPdÒ<dÐ<dØ*/°°A±¬,Ô*F��a”Ô'øàˆr   Nc           	      óV  — |j         }|dv r|                      |||||¦  «        S |dv r^| xj        dz  c_        |j        s|                      |||||¦  «        S |                      |||||||¦  «        }	|	D ]}
| j        |
_        Œ|	S |j        r|                      |||||||¦  «        S g S )z.Process a single LayoutBox into SentenceUnits.rV   rY   r   )rO   Ú_table_as_unitrz   r6   Ú_figure_as_unitÚ_split_sentencesÚfigure_number)r~   r�   r‚   rƒ   r„   rg   r…   r†   rO   r‘   r”   s              r   rŠ   zSentenceBuilder._process_boxú   s   € ð ”<ˆàÐ2Ð2Ð2Ø×&Ò& t¨W°c¸=È#ÑNÔNÐNàÐ-Ð-Ð-ð ÐÔ !Ñ#ÐÔØ”=ð TØ×+Ò+¨D°'¸3ÀÈsÑSÔSÐSØ×)Ò)¨$°¸¸mÈSØ*3Ð5EñGô GˆEàð 5ð 5�Ø"&Ô"4�”�ØˆLàŒ=ð 	FØ×(Ò(¨¨w¸¸]ÈCØ)2Ð4DñFô Fð Fð ˆ	r   c                 ó^  — t          |¦  «        \  }}t          |||¦  «                             ¦   «         }|sg S t          |j        ||j        |t          |dd¦  «        ¬¦  «        }	t          d||t          |¦  «        |j        ||j        |j	        |j
        |j        |j        f|pd|dœ	|	¤ŽgS )z-Create a single SentenceUnit for a table box.ri   N©ri   )	r’   r   Ú	norm_textrh   Ú	box_indexrO   rG   Útable_markdownÚ
table_html© )r   r	   r   rm   rO   Úpage_numberÚgetattrr   r0   Úx0Úy0Úx1Úy1)
r~   r�   r‚   rƒ   r„   rg   Útable_mdr¡   r   rk   s
             r   r˜   zSentenceBuilder._table_as_unit  sÚ   € å,¨SÑ1Ô1Ñˆ�*Ý˜t S¨'Ñ2Ô2×8Ò8Ñ:Ô:ˆØð 	ØˆIå" 3¤<°°dÔ6FÈÝ18¸¸nÈdÑ1SÔ1SðUñ Uô Uˆåð 
Ø!ØÝ% dÑ+Ô+ØÔ$ØØ”\Ø”&˜#œ& #¤&¨#¬&Ð1Ø#Ð+ tØ!ð
ð 
ð ð
ð 
ð ð 	r   c                 ób  — t          | j        |¦  «        }t          |j        ||j        |t          |dd¦  «        ¬¦  «        }t          |j        t          ¦  «        r|j        nd}t          d||t          |¦  «        |j        ||j        |j        |j        |j        |j        f| j        |dœ	|¤ŽgS )z?Create a single SentenceUnit for a figure/formula without text.ri   Nr�   )	r’   r   rž   rh   rŸ   rO   rG   r›   Ú
image_datar¢   )r   rz   rm   rO   r£   r¤   Ú
isinstanceÚimageÚbytesr   r0   r¥   r¦   r§   r¨   )	r~   r�   r‚   rƒ   r„   rg   r   rk   r«   s	            r   r™   zSentenceBuilder._figure_as_unit*  sÊ   € å! $Ô"4°cÑ:Ô:ˆÝ" 3¤<°°dÔ6FÈÝ18¸¸nÈdÑ1SÔ1SðUñ Uô Uˆå",¨S¬Y½Ñ">Ô">ÐH�S”Y�YÀDˆ
Ýð 
Ø!ØÝ% dÑ+Ô+ØÔ$ØØ”\Ø”&˜#œ& #¤&¨#¬&Ð1ØÔ,Ø!ð
ð 
ð ð
ð 
ð ð 	r   c                 óÖ  — |j         }|sg S t          ||||¦  «                             ¦   «         }	|	sg S t          |¦  «        \  }
}d}|rZ|j        |j        k    rJ|d                              d¦  «        �/t          |d         d         d         ¦  «        |j        d         z
  }t          |	¦  «        }t          |j        ||j        |t          |dd¦  «        ¬¦  «        }|                     d¦  «        s'|j        d	vrt          |¦  «        \  }}|r
d
|d<   ||d<   |                     d¦  «        pW|                     d¦  «        pB|                     d¦  «        p-|                     d¦  «        p|                     d¦  «        pd|	v }|s,t          |	| j        ¦  «        }|sg S t!          |¦  «        dk    }|r>t#          |¦  «        }t%          d||	t'          |	¦  «        |j        ||j        ||
||dœ
|¤ŽgS t#          |¦  «        }g }t)          |¦  «        D ]Q\  }}|                     t%          d||z   |t'          |¦  «        |j        ||j        ||
||dk    r|nddœ
|¤Ž¦  «         ŒR|S )zBSplit a box's rendered markdown into sentence-level SentenceUnits.Nr   rG   r   rI   ri   r�   rd   )	rQ   rT   r!   rW   rZ   r[   r]   r`   ra   Tre   rR   r^   rb   rU   ú
)
r’   r   rž   rh   rŸ   rO   rG   Úfont_size_dominantÚfont_flags_dominantrŽ   r¢   )r6   r	   r   rD   rh   r£   r;   rJ   rG   r5   rm   rO   r¤   r*   r   r|   r<   rN   r   r0   r‰   r   )r~   r�   r‚   rƒ   r„   rg   r…   r†   r6   ÚjoinedÚ	font_sizeÚ
font_flagsÚ	first_gapÚplainrk   Úis_capÚ
cap_targetÚkeep_singleÚ	sentencesrG   Úbox_bboxr‘   r•   Ú	sent_texts                           r   rš   z SentenceBuilder._split_sentences=  s  € ð ”Mˆ	Øð 	ØˆIõ
 !  s¨GÐ5EÑFÔF×LÒLÑNÔNˆØð 	ØˆIå 2°9Ñ =Ô =Ñˆ	�:Øˆ	Øð 	O˜Ô*¨dÔ.>Ò>Ð>Ø˜Œ|×Ò Ñ'Ô'Ð3Ý! )¨A¤,¨vÔ"6°qÔ"9Ñ:Ô:¸Y¼^ÈAÔ=NÑN�	õ % VÑ,Ô,ˆÝ" 3¤<°°dÔ6FÈÝ18¸¸nÈdÑ1SÔ1SðUñ Uô Uˆð �yŠy˜Ñ&Ô&ð 	:¨3¬<ð @
ð ,
ð ,
õ "1°Ñ!7Ô!7ÑˆF�JØð :Ø&*��lÑ#Ø/9�Ð+Ñ,ð �IŠIÐ'Ñ(Ô(ð ¨E¯IªI°mÑ,DÔ,Dð Ø�yŠyÐ+Ñ,Ô,ðØ05·	²	¸,Ñ0GÔ0Gðà�yŠy˜Ñ(Ô(ðð �vˆ~ð	 	ð ð 	.Ý,¨V°T´^ÑDÔDˆIØð Ø�	Ý˜i™.œ.¨AÒ-ˆKàð 	Ý& yÑ1Ô1ˆDÝ ð Ø%ØÝ)¨&Ñ1Ô1ØÔ(Ø!ØœØØ#,Ø$.Ø )ðð ð ðð ð ð õ ' yÑ1Ô1ˆØˆÝ% iÑ0Ô0ð 	ð 	‰LˆAˆyØ�LŠL�ð Ø%¨Ñ)ØÝ)¨)Ñ4Ô4ØÔ(Ø!ØœØØ#,Ø$.Ø-.°!ªV¨V 	 	¸ðð ð ðð ñ ô ð ð ð ˆr   c           	      ó¢  — |j         rt          |j         ¦  «        dk     rt          ¦   «         S t          t          ¦  «        }|j         D ]�}t          |j        ¦  «        D ]y\  }}|j        dv rk|j        rdt          t          |||¦  «        ¦  «        }|rDt          |j        dz  ¦  «        dz  }||j        ||f                              |j        |f¦  «         ŒzŒ‘t          dt          |j         ¦  «        dz  ¦  «        }t          ¦   «         }	|                     ¦   «         D ]-\  }
}t          |¦  «        |k    r|	                     |¦  «         Œ.|	S )z}Detect header/footer boxes that repeat across pages.

        Returns set of (page_no, box_index) tuples to exclude.
        rI   r_   é
   rH   g      à?)r‡   r<   Úsetr   Úlistr‰   rˆ   rO   r6   r0   r	   r=   r¦   r   r£   ÚmaxÚitemsÚupdate)r~   r�   Úhf_textsr�   r‚   rƒ   r   Úy_bucketÚ	thresholdÚrepeatedÚ_keyÚ	locationss               r   Údetect_repeated_headers_footersz/SentenceBuilder.detect_repeated_headers_footers“  sV  € ð
 Œyð 	�C ¤	™NœN¨QÒ.Ð.Ý‘5”5ˆLå�tÑ$Ô$ˆà”Ið 	eð 	eˆDÝ )¨$¬*Ñ 5Ô 5ð eð e‘�˜Ø”<Ð#AÐAÐAÀcÄmÐAÝ*­?¸4ÀÀgÑ+NÔ+NÑOÔO�DØð eÝ#(¨¬°"©Ñ#5Ô#5¸Ñ#:˜Ø  #¤,°¸$Ð!?Ô@×GÒGÈÔIYÐ[bÐHcÑdÔdÐdøðeõ ˜�3˜sœy™>œ>¨CÑ/Ñ0Ô0ˆ	Ý‘5”5ˆà'Ÿ~š~Ñ/Ô/ð 	+ð 	+‰OˆD�)Ý�9‰~Œ~ Ò*Ð*Ø—’ 	Ñ*Ô*Ð*øàˆr   ç      4@r‘   Úy_tolerancec                 óˆ  — |s|S g }d}|t          |¦  «        k     �r%||         }|j        s|                     |¦  «         |dz  }Œ>|g}|dz   }|t          |¦  «        k     r^||         }|j        r;|j        |j        k    r+|j        |j        k    r|                     |¦  «         |dz  }nn|t          |¦  «        k     °^t          |¦  «        dk    r|                     |¦  «         nC|                      ||¦  «        }	|	D ]*}
|                     |                      |
¦  «        ¦  «         Œ+|}|t          |¦  «        k     �°%|S )zEMerge consecutive same-page, same-boxclass HF units sharing a y-band.r   r   )r<   rb   r   rh   rO   Ú_group_by_y_bandÚ_merge_hf_group)r~   r‘   rÍ   Úresultr•   r”   ÚgroupÚjÚnxtÚbandsÚbands              r   r‹   z)SentenceBuilder._merge_same_page_hf_units°  sl  € ð ð 	ØˆLàˆØˆØ•#�e‘*”*Šn‰nØ�a”ˆAØÔ%ð Ø—’˜aÑ Ô Ð Ø�Q‘�Øð �CˆEØ�A‘ˆAØ•c˜%‘j”j’.�.Ø˜A”h�ØÔ(ð ØœK¨1¬9Ò4Ð4ØœL¨A¬JÒ6Ð6Ø—L’L Ñ%Ô%Ð%Ø˜‘F�A�Aàð •c˜%‘j”j’.�.õ �5‰zŒz˜QŠˆØ—’˜aÑ Ô Ð Ð ð ×-Ò-¨e°[ÑAÔA�Ø!ð >ð >�DØ—M’M $×"6Ò"6°tÑ"<Ô"<Ñ=Ô=Ð=Ð=ØˆAð7 •#�e‘*”*Šn‰nð: ˆr   c                 ót  — t          | d„ ¬¦  «        }g }|D ] }|j        d         |j        d         z   dz  }|rh|d         d         j        d         |d         d         j        d         z   dz  }t          ||z
  ¦  «        |k    r|d                              |¦  «         ŒŠ|                     |g¦  «         Œ¡|S )z6Group units by y-center proximity (greedy clustering).c                 ó>   — | j         d         | j         d         z   dz  S )Nr   rI   rH   ©rG   ©r”   s    r   ú<lambda>z2SentenceBuilder._group_by_y_band.<locals>.<lambda>Û  s   € °A´F¸1´IÀÄÀqÄ	Ñ4IÈQÑ3N€ r   ©Úkeyr   rI   rH   éÿÿÿÿ)ÚsortedrG   Úabsr   )r‘   rÍ   Úsorted_unitsrÕ   r”   ÚycÚlast_ycs          r   rÏ   z SentenceBuilder._group_by_y_band×  sÎ   € õ ˜eÐ)NÐ)NÐOÑOÔOˆØ*,ˆØð 	ð 	ˆAØ”&˜”)˜aœf QœiÑ'¨1Ñ,ˆBØð Ø  œ9 Rœ=Ô-¨aÔ0°5¸´9¸R´=Ô3EÀaÔ3HÑHÈAÑM�Ý�r˜G‘|Ñ$Ô$¨Ò3Ð3Ø˜"”I×$Ò$ QÑ'Ô'Ð'ØØ�LŠL˜!˜ÑÔÐÐØˆr   rÒ   c                 óH  — t          | ¦  «        dk    r| d         S t          | d„ ¬¦  «        }d                     d„ |D ¦   «         ¦  «        }t          d„ |D ¦   «         ¦  «        }t	          |d„ ¬¦  «        }g }|D ]6}|                     |j        ¦  «         |                     |j        ¦  «         Œ7t          |d         j
        |t          |¦  «        |d         j        |d         j        |d         j        ||j        |j        |d         j        d	¬
¦  «        }||_        |S )zCMerge a group of HF units into a single unit (left-to-right order).r   r   c                 ó   — | j         d         S )Nr   rÙ   rÚ   s    r   rÛ   z1SentenceBuilder._merge_hf_group.<locals>.<lambda>í  s   € °1´6¸!´9€ r   rÜ   z | c              3   ó$   K  — | ]}|j         V — Œd S ©Nr4   ©Ú.0r”   s     r   ú	<genexpr>z2SentenceBuilder._merge_hf_group.<locals>.<genexpr>î  ó$   è è € Ð7Ð7 Q˜!œ&Ð7Ð7Ð7Ð7Ð7Ð7r   c              3   ó$   K  — | ]}|j         V — Œd S rç   rÙ   rè   s     r   rê   z2SentenceBuilder._merge_hf_group.<locals>.<genexpr>ð  rë   r   c                 ó*   — t          | j        ¦  «        S rç   )r<   r   rÚ   s    r   rÛ   z1SentenceBuilder._merge_hf_group.<locals>.<lambda>ó  s   € µ#°a´f±+´+€ r   T)r’   r   rž   rh   rŸ   rO   rG   r±   r²   rŽ   rb   )r<   rß   Újoinr   rÂ   r   rŸ   ÚextendÚ_source_box_indicesr   r’   r0   rh   rO   r±   r²   rŽ   )rÒ   Úsorted_groupr   rG   ÚlongestÚsource_indicesr”   Úmergeds           r   rÐ   zSentenceBuilder._merge_hf_groupç  sA  € õ ˆu‰:Œ:˜Š?ˆ?Ø˜”8ˆOå˜eÐ)<Ð)<Ð=Ñ=Ô=ˆØ�zŠzÐ7Ð7¨,Ð7Ñ7Ô7Ñ7Ô7ˆåÐ7Ð7¨,Ð7Ñ7Ô7Ñ7Ô7ˆõ �lÐ(=Ð(=Ð>Ñ>Ô>ˆð ˆØð 	9ð 	9ˆAØ×!Ò! !¤+Ñ.Ô.Ð.Ø×!Ò! !Ô"7Ñ8Ô8Ð8Ð8åØ  ”OÔ+ØÝ% dÑ+Ô+Ø  ”OÔ+Ø" 1”oÔ/Ø! !”_Ô-ØØ&Ô9Ø 'Ô ;Ø(¨œOÔ;Ø!ð
ñ 
ô 
ˆð &4ˆÔ"Øˆr   c                 ó>   — t          | ¦  «        D ]\  }}||_        Œ| S )z&Renumber sent_ids sequentially from 0.)r‰   r’   )r‘   r•   r”   s      r   rŒ   z"SentenceBuilder._renumber_sent_ids  s-   € õ ˜eÑ$Ô$ð 	ð 	‰DˆAˆqØˆAŒIˆIØˆr   )rv   rç   )rÌ   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__ro   r   rÁ   r   r–   rŠ   r˜   r™   rš   rÀ   ÚtupleÚintrË   rJ   r‹   ÚstaticmethodrÏ   rÐ   rŒ   r¢   r   r   ru   ru   Í   s  € € € € € ØCÐCð*ð * ð *ð *ð *ð *ð ¨$¨|Ô*<ð  ð  ð  ð  ðF '+ðð Ø/3°LÔ/Aðð ð ð ð4ÈÈ\ÔHZð ð ð ð ð,ÈÈlÔI[ð ð ð ð ð( +/ðTð TØ37¸Ô3EðTð Tð Tð Tðl°c¸%ÀÀSÀ¼/Ô6Jð ð ð ð ð< 9=ð%ð %¨t°LÔ/Að %Ø05ð%ØAEÀlÔASð%ð %ð %ð %ðN ð  \Ô 2ð Ø&+ðØ04°T¸,Ô5GÔ0Hðð ð ñ „\ðð ð!˜t LÔ1ð !°lð !ð !ð !ñ „\ð!ðF ð $ |Ô"4ð ¸¸lÔ9Kð ð ð ñ „\ðð ð r   ru   )NNNN)&rù   ÚreÚcollectionsr   r   Útypingr   Úmodelsr   r   Útext_sourcer	   r
   r   r   Úcompiler}   r{   ro   rÁ   r   r-   r3   Ú
IGNORECASEr#   rú   Úboolr*   r0   r5   ÚdictrJ   rû   rD   rN   rm   rf   ru   r¢   r   r   ú<module>r     sÐ  ðØ =Ð =à 	€	€	€	Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø Ð Ð Ð Ð Ð à ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,ðð ð ð ð ð ð ð ð ð ð ð ð ˆrŒzðñô €ð �"”*ð<ñô €ð˜sð °°S´	ð ð ð ð ð0 ˆrŒz˜)Ñ$Ô$€ð �”˜NÑ+Ô+€ð €B„JÐ:¸B¼MÑJÔJØ€B„JÐ9¸2¼=ÑIÔIØ€B„JÐ% r¤}Ñ5Ô5Ø€B„JÐ(¨"¬-Ñ8Ô8ð	Ð ð˜#ð  %¨¨h°s¬mÐ(;Ô"<ð ð ð ð ð*˜#ð  #ð ð ð ð ð(˜sð ( sð (ð (ð (ð (ð
 $ t¤*ð °°u¸c°zÔ1Bð ð ð ð ð&& 4¨¤:ð &°%ð &ð &ð &ð &ð DHØ$(ð$ð $ ð $ð $ð $ð $ðN˜$ð ¨ð °Cð ¸HÀS¼Mð ð ð ð ð0Cð Cð Cð Cð Cñ Cô Cð Cð Cð Cr   