WjedZddlZddlZddlZddlmZd(dZ dZdZ dZ dZ d Z d Z d Zd Zd ZdZd)dZee fdZd*dZd+dZdZd,dZdZdZ d-d Zd!Zd"Zd#Zd$Zd%Zd.d'Z dS)/z PDF parsing utilities - Base module Maintained by: PDF Engineers Purpose: Stable PDF element extraction (text, images, vectors, checkboxes) N) defaultdictTcBtjd|}|\}}|ddz|ddzz|dz}tj|}d|D}g}g} |D]} |r1| d|kr | dd |fvs| d|kr | dd |fvr8| d jd kr-| d jd kr| | d v| d jd kr-| d jd kr| | d | d D]:} | ddkr| dd \} } t| j | j z dkr[t| j | j z d kr;| tj | | nzt| j | j z dkrZt| j | j z d kr:| tj | | | ddkr"| d}| tj |j|j| tj |j|j| tj |j|j| tj |j|j<|| fS)aReturn horizontal and vertical vector lines on a page. Optionally, omit vectors that have the color of the background. The background color is determined as the most frequent color on a pixel level as determined from a rasterized image (pixmap) of the page. rcVg|]&}|djdks|djdk$|'S)rect)heightwidth).0ps /Users/tedpeters/Documents/Projects/Claude Projects/TPC Houses/house-viewer/.venv/lib/python3.11/site-packages/pymupdf/layout/pymupdf_util_base.py z$get_vector_lines.. sA   !F)*:Q*>*>!F)/TUBUBUBUBUBUfillcolorNr itemslre)pymupdfTOOLS set_aa_level get_pixmapcolor_topusage sRGB_to_pdf get_drawingsr r appendabsyxRect normalizetltrblbr)pageomit_invisiblepixportionrsRGBbg_colorpathsh_linesv_linesritemp1p2rs rget_vector_linesr8 sk Mq!!! //  C''))NGU 8r>E!HM )E!H 4D"4((H  $$&&   EGG %A%A " & X%%gJ4"222W:))fI$!111  V9 a  AfIOb$8$8 NN1V9 % % %  vY_q QvY%5%:%: NN1V9 % % % gJ A ADAw#~~abbBrtbd{##q((S-=-=-B-BNN7<B#7#7#A#A#C#CDDDD%%**s24"$;/?/?"/D/DNN7<B#7#7#A#A#C#CDDDAw$QA NN7<ad33==?? @ @ @ NN7<ad33==?? @ @ @ NN7<ad33==?? @ @ @ NN7<ad33==?? @ @ @ @ A" G rg@@ c|j|jkr|j|jfn |j|jf\}}|j|jkr|j|jfn |j|jf\}}|j|jkr|j|jfn |j|jf\}} |j|jkr|j|jfn |j|jf\} } |||z ks|| |zks|| |z ks || |zkrdSdS)z?True if r1 and r2 are within tolerance (expanded-rect overlap).FTx1x0y1y0) r1r2delta_xdelta_yrr1_x0rr1_x1rr1_y0rr1_y1rr2_x0rr2_x1rr2_y0rr2_y1s r_are_neighborsrMgs')uru}}beRU^^25"%.NFF')uru}}beRU^^25"%.NFF')uru}}beRU^^25"%.NFF')uru}}beRU^^25"%.NFF'!!! FW$ $ $ FW$ $ $ FW$ $ $u 4rc|j|z }|j|z}|j|z }|j|z}||kr||}}||kr||}}||||fSN)r>r=r@r?)r rCrDr>r=r@r?s r_expanded_boundsrPwse 7 B 7 B 7 B 7 B BwwRB BwwRB B rc|j|jkr|j|jfn |j|jf\}}|j|jkr|j|jfn |j|jf\}}||||fS)z2Return normalized rect bounds as (x0, y0, x1, y1).r<)r r>r=r@r?s r_normalized_boundsrRsm#'7dg#5#5dgtw  DGTW;MFB#'7dg#5#5dgtw  DGTW;MFB B rctdz }tdz }t||jz |z}t||jz |z}|dkrd}n||kr|}|dkrd}n||kr|}||fS)z?Map a point to a containment-grid cell, clamped to page bounds.rr)CONTAINMENT_GRID_CELLSintr>r@) pxpy page_rectcell_wcell_hmax_xmax_ycxcys r_grid_cell_for_pointr_s "Q &E "Q &E b9<F * + +B b9<F * + +B Avv  e  Avv  e  8Orct|||||\}}tdz }tdz }g} dD]H} dD]C} || z} || z} d| cxkr|kr*nd| cxkr|krn,| | | fDI| S)a Return the point cell plus its immediate neighbors. This preserves correctness for degenerate rects (zero width / height) whose center may lie exactly on a grid boundary while the accepted container was indexed into the adjacent cell. r)rrr)r_rTr!)rVrWrXrYrZr]r^r[r\cellsdxdynxnys r_candidate_cells_for_pointrgs""b)VV D DFB "Q &E "Q &E E'' ' 'BbBbBB%A$4$4$4$4u$4$4$4$4$4 b"X&&&  ' Lrc#Kt|\}}}}tdz }tdz } t||jz |z} t||jz |z} t||jz |z} t||jz |z} | dkrd} n| |kr|} | dkrd} n| |kr|} | dkrd} n| | kr| } | dkrd} | | kr| } t | | dzD]}t | | dzD]}||fV dS)z-Yield containment-grid cells touched by rect.rrN)rRrTrUr>r@range)r rXrYrZr>r@r=r?r[r\cx0cx1cy0cy1r]r^s r_grid_keys_for_rectrnsa'--NBB "Q &E "Q &E rIL V+ , ,C rIL V+ , ,C rIL V+ , ,C rIL V+ , ,C Qww u Qww u Qww u Qww U{{Cq!!S1W%%  Br(NNNN rcN|sgSt|jtz d}t|jtz d}i}g}|d|D]}|d}t |\} } } } | | zdz } | | zdz }d}t | ||||D]3}||dD]}|||dvrd}n|rn4|rzt|}||t||||D]&}||vrg||<|||'|S)a Faster equivalent of: if not any(p["rect"] in f["rect"] for f in filtered_paths): filtered_paths.append(p) Because candidates are processed in descending area, any containing rectangle is already in filtered_paths. A rectangle that contains the candidate must also contain the candidate's center point, so we query only the accepted rectangles whose coverage includes that point's cell. g?Nr @FT) maxr rTr rRrggetlenr!rn) all_picturesrXlimitrYrZgridfiltered_pathsrr r>r@r=r?r]r^ containedcell accepted_idxkeys r_filter_contained_paths_fastr~s  #993 ? ?F !$::C @ @F DN &5& !++y+D11BB2g_2g_ .r2y&&QQ  D $r 2 2   >,7??? $IE@     >** a   &tYGG + +C$S I  \ * * * * + rc.|jfd|D}fd|D}td||zDdd}t |d}|sgS|||| S) zJIdentify and group connected vectors on a page (optimized implementation).c&g|] }|dzS)r rr)rrrXs rrz(get_picture_clusters..s" @ @ @qY6 " @ @ @rc&g|] }|dzSbboxrr)rimgrXs rrz(get_picture_clusters..s" G G G#i#f+% G G GrcBg|]}|jdks |jdkd|iS)rr r r )rr7s rrz(get_picture_clusters..s-NNN1 A &! rc8|dj|djzS)Nr r)r$s rz&get_picture_clusters..saio& (88rTr}reversero)rw)drawings x_tolerance y_tolerance)r r get_image_infosortedr~cluster_drawings)r+rrr1imagesrvryrXs @rget_picture_clustersrs I @ @ @ @D,=,=,?,? @ @ @E G G G G1D1D1F1F G G GFNNefnNNN 8 8L 2, QUVVVN    !  rhrc|dkr|dn|dg}|D]7}|j|j|j|jf\}}}}d} |D]} |dkrpt | j|z |krT|| j|z ksF|| j|zks8t | j|| _t| j|| _d} nqxt | j|z |krT|| j|z ksF|| j|zks8t | j|| _t| j|| _d} n| s'|tj |9|S)z Merge fragmented lines into longer continuous lines. lines: list of line objects with x0, y0, x1, y1 attributes orientation: 'h' for horizontal, 'v' for vertical tolerance: allowable gap or offset to consider lines as mergeable rc|j|jfSrO)r@r>r7s rrzmerge_lines..!!$r)r}c|j|jfSrO)r>r@rs rrzmerge_lines..#rrFT) sortr>r@r=r?r"minrsr!copydeepcopy) lines orientation tolerancemergedliner>r@r=r?matchedms r merge_linesrsc -- .... -- ... F//$'47DG;BB  Ac!!qtby>>Y..QTI=M8M8MQSVWVZ]fVfQfQfqtR==ADqtR==AD"GEqtby>>Y..QTI=M8M8MQSVWVZ]fVfQfQfqtR==ADqtR==AD"GE / MM$--- . . . Mr?c fd}tt}|D]a}t|j|jf\}}|j|jzdz }|||||f|btt} |D]j}t|j|jf\} } |j|jzdz } | || || f || kg} | D]\\}}}||kst|dkr"tt|}tt|D]}t|dzt|D]^}||||}}| ||||f}|r!||vr||vr| ||||f_| S)a> Find fully-closed border rectangles (e.g. full-border tables) using exact coordinate matching -- NOT proximity/tolerance-based clustering like page.cluster_drawings(). See design discussion: clustering's tolerance is itself an unvalidated heuristic, and real tables are built in inconsistent ways (lines-only, filled cells, text-wrapping vectors), so no single tolerance is guaranteed to generalize. Exact-match closure instead only asserts something narrower and more defensible: "these lines were clearly drawn to form a rectangle" -- true regardless of how close together two SEPARATE closed rectangles happen to sit. Core idea: a full-border rectangle's outer boundary is formed by two horizontal lines sharing the exact same x-span, and two vertical lines sharing the exact same y-span (the horizontal pair's y-values), positioned at exactly those two x-coordinates. Args: h_lines: list of objects with x0, y0, x1, y1 (horizontal lines, y0 == y1 up to float noise). As returned by get_vector_lines()/merge_lines(). v_lines: list of objects with x0, y0, x1, y1 (vertical lines, x0 == x1 up to float noise). tol: float. Coordinate-snapping tolerance to absorb float rounding noise ONLY (e.g. two lines both meant to be drawn at x=100.0 landing at 100.0 and 100.03) -- NOT a proximity threshold for deciding whether two nearby-but-distinct coordinates should be treated as "close enough" structurally. Keep well below the smallest real gap you'd expect between two genuinely different table/cell edges. Returns: list of (x0, y_top, x1, y_bottom) tuples: one closed rectangle per matching horizontal-line x-span/vertical-line y-span pair found. Checks every pair of horizontal lines sharing an x-span (not just the global min/max y for that span), so two separate rectangles that happen to share the same width don't get silently merged into one spanning both. c.t|z zSrO)round)vtols rsnapz&find_closed_border_rects..snaphsQW~~##rrqrr)rlistrr>r=r@r?r!setaddrrurirt)r2r3rr h_by_spanr7r>r=r# v_by_spanr@r?r$ closed_rectsx0kx1kys ys_sortedaby_topy_bottomv_xss ` rfind_closed_border_rectsrAs0N$$$$$D!!I 22qt %%B TAD[C 4488TT"XX&'..q1111C  I 55qt %%B TAD[C 4488TT"XX&'++DDGG4444L#//++ E E cB #::R1 3r77OO s9~~&& E EA1q5#i..11 E E"+A, ! x }}dd5kk44>>%BCCEC4KKC4KK ''eS((CDDD  E E rc|d|dzdz }|d|dzdz }t|D].\}\}}}}||cxkr|krn||cxkr|krn*|cS/dS)aG Which closed rect (from find_closed_border_rects) contains this bbox's center, if any. Returns the index into closed_rects, or -1 if the bbox isn't inside any of them. Used to build the 'same_rect_member' edge feature (see pymupdf_util_edge.get_edge_transform_yf): two nodes with the same (non -1) rect member id sit inside the same closed rectangle -- e.g. two cells of the same table -- while two nodes in different (or no) closed rectangles don't, even if they happen to be geometrically close (e.g. cells from two separate, adjacent tables). rrrqrrra) enumerate) rrr]r^idxr>r@r=r?s rget_rect_member_idrs q'DG s "B q'DG s "B!*>>>r>>>>>bBnnnn"nnnnnJJJ 2r?c ddlm}g}|r|d}d}t|D]\}}|||}||krwt |d|dt |d|dt |d|dt |d|dg} | ||<d}n|s||||S)z8Merge overlapping bounding boxes based on IoU threshold.r) compute_iourFrrT) common_utilrpoprrrsr!) boxes iou_thresholdrrbase has_mergediotheriounew_boxs r merge_boxesrs(((((( F  yy|| !%((  HAu+dE**C]""Qq**Qq**Qq**Qq**  #a! # MM$   !  " Mrc |D]T}|ddkrt|||||d)|ddkr6|dD]}|dd}|dd}|dd} |dd} g} |d D]} | | d d | } | d krd| kr~|| krx||krrd| krl|| krf||kr`||| | g} | |d vrP|d | |d | |tVdS)z"Extract text blocks from PDF page.typerblocksrrrrrspanstext bboxesN) text_extractr!joinstripBOX_TEXT) data_dictbox_type page_width page_heightrblockrr=r?x2y2txtspanrs rrrs.. =A   Hj+uX W W W  =A   'N . .DfaBfaBfaBfaBCW  ) ) 4<((((((3--%%''CbyyQ"WWrbJ6F6F1PR77WY]_W_W_dfjududuBB'y222h'..t444f%,,S111OOH---% . ..rc.||S)N)flags) get_textpage)r+rs rcreate_stext_pagers   5  ) ))rrrfc^ggddd}g}|jd|jd}}||d<||d<|} tj| jtj} | | j| j| j } | |d< dtj ztj ztj ztjztjztjztjztjztjztjz} tjd kr| tjz} | tjz} n| d z} tjd kr| tjz} |d krPdtj ztj ztjztjztjztjz} t5|| } | |d < d|vr.(sCCCcc&kCCCrr)rcH|d|dz |d|dz zS)Nrrrrrr)boxs rrz'extract_base_elements..*s#Q#a&SVc!f_(MrTrrrrrpicture_clustersc |j|jzSrOrrs rrz'extract_base_elements..>s!'AH,rvec_line)r,r)rrr vector_linesrc |j|jzSrOrrs rrz'extract_base_elements..d!(0Brc |j|jzSrOrrs rrz'extract_base_elements..srrtext_pmdictr)textpagecTg|]%}|jtjk|j|jf&Srr) field_typerPDF_WIDGET_TYPE_CHECKBOXr field_value)rwidgets rrz)extract_base_elements..s?III#w'GGG; 23GGGrr)3r rnp frombuffersamplesuint8reshaper r nrTEXT_PRESERVE_IMAGESTEXT_PRESERVE_WHITESPACETEXT_PRESERVE_LIGATURESTEXT_ACCURATE_BBOXESTEXT_COLLECT_VECTORSTEXT_COLLECT_STYLES TEXT_SEGMENTTEXT_PARAGRAPH_BREAKTEXT_COLLECT_STRUCTURETEXT_TABLE_HUNTmupdf_version_tupleTEXT_LAZY_VECTORS TEXT_CLIPTEXT_FUZZY_VECTORSTEXT_INHIBIT_SPACESrpprintpp extractDICTrrrrsr! BOX_IMAGErrr>r@r=r?r8rr BOX_HLINE BOX_VLINEget_textrwidgets BOX_CHECKBOX)r+ input_typefeature_set_name max_image_nummax_vec_line_numrrrrr- bytes_datapage_img stext_flagsrr img_bboxesr r=r?rrr pic_bboxesr2r3r checkboxesvalues rextract_base_elementsr-s  I H"ilDIaL J(Il*Im //  Cs{"(;;;J!!#*ci??H!Ig: * +. /- .*  + *  + )  *" #* +, -% &"j00w00 w((  r "j00w11 5  2312-.. /./0 1 #455J(Il, *CCT-@-@-B-BCCC  3??? MMW[\\\/  / /DQQBQQBQQBQQBB))))))))z)))))a2.J.J.J.J.J.J.J.J{.J.J.J.J.JBB'y222h'..t444f%,,R000OOI...Z'')$//  ,,    =.   / /DBBBBB))))))))z)))))a2.J.J.J.J.J.J.J.J{.J.J.J.J.J88aBBB'y222h'..t444f%,,R000OOI...Z+DFFFg3!DDDg3!DDD&-g$6 .!%=Wg$N$N .!&B&BDQQQRcScRcd / /DBBBBB))))))))z)))))a2.J.J.J.J.J.J.J.J{.J.J.J.J.J88aBBB'y222h'..t444f%,,R000OOI...&B&BDQQQRcScRcd / /DBBBBB****"**** *****qB/J/J/J/J/J/J/J/J{/J/J/J/J/J88aBBB'y222h'..t444f%,,R000OOI...yJ66  " "]]6**84FF]]6J]??IFY*k6JJJII$,,..IIIJ! . . e W W W W  % % % %R % % % %: % % % % %!r*E*E*E*EB*E*E*E*E+*E*E*E*E*EB#D9X...(#**4000&!((,,, ---$Ij rrrz h-line-vectorz v-line-vectorz check-boxrc |tkrdnd}|tkrdnd}|tkrdnd}|tkrdnd}t |dkr,t d|D}|t |z }nd}|||||||||zdS)u Create a custom_feature dict from a box_type string. This is the SINGLE SOURCE OF TRUTH for box_type → flag mapping. All modules (pymupdf_util.py, pymupdf_util_ext.py) must use this instead of duplicating the logic. rrc3BK|]}|dVdS)rN)isdigit)rcs r z&make_custom_feature..s/77a199;;7777777rg)r num_ratiois_textis_imageis_hline_vectoris_vline_vectoris_line is_vector)rrrrrusum) box_type_strrr4r5r6r7 num_countr3s rmake_custom_featurer=s 8++aaG I--qq1H'944aa!O'944aa!O 4yy1}}77477777 D )  !**$6   r)T)ro)rr)r)r)rrrr)r)!__doc__rnumpyrr collectionsrr8 X_TOLERANCE Y_TOLERANCEMAX_CELLS_FOR_GRIDrTrMrPrRr_rgrnr~rrrrrrrr-rrrrr!r=rrrrrDs  ######AAAAH         "*<,,,,^,7K4''''TCCCCL&2...:***HL>AFFFFT      r