|j~UdZddlZddlZddlZddlZddlZddlmZm Z m Z m Z m Z ddl mZddlmZeeeefeeeeeeeeffffZiddddd d d d d ddddddddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4 Zeeefed5<d6eDZeeefed7<d8ed9efd:Zejd9efd;Ziddddd dd dddddddddddddd dd"dd$dd&dd'dd)dd(dd d d d d d dddddddd< Zeeefed=<d Z dZ!d Z"d>Z#d Z$ejd?ed9efd@Z%dAedBedCedDed9ef dEZ&d8edFedGedHeedIeeeeeffd9eedzeeeff dJZ'efd8edKedLedMeed9edzf dNZ(d8edFedGedHeedIeeeeeffd9edzf dOZ)ejdHeed9eeeffdPZ*eZ+ejdQed9eefdRZ,dFedGedMeed9eedzfdSZ-dTe dUeee.eedzeeffdQedMeedzd9e/f dVZ0d8edHeed9e dzfdWZ1efd8edFedGedHeedMeed9edzf dXZ2ejdFedGed9eefdYZ3ejdFedGedHeed9eeeeefffdZZ4d8edFedGedHeed[eedzd\eedzd9edzfd]Z5d^edFedGed9eeefdzfd_Z6d`Z7daZ8dbZ9dcZ:d8edde.ed9e.efdeZ;dS)faConfusion group resolution for similar single-byte encodings. At runtime, loads pre-computed distinguishing byte maps from confusion.bin and uses them to resolve statistical scoring ties between similar encodings. Build-time computation (``compute_confusion_groups``, ``compute_distinguishing_maps``, ``serialize_confusion_data``) lives in ``scripts/confusion_training.py``. Note: ``from __future__ import annotations`` is intentionally omitted because this module is compiled with mypyc, which does not support PEP 563 string annotations. N) ART_LANGUAGE BigramProfile get_enc_indexget_idf_weightsscore_with_profile)DetectionResult)lookup_encodingLuLlLtLmLoMnMcMeNd Nl No Pc Pd PsPePiPfPoSmScSkSoZsZlZpCcCfCsCoCn) _INT_TO_CATEGORYci|]\}}|| SrG).0kvs /Users/tedpeters/Documents/Projects/Claude Projects/Misc Tools/notename-sheets/.venv/lib/python3.11/site-packages/chardet/pipeline/confusion.py rLKs"M"M"MDAq1a"M"M"MCATEGORY_TO_INTdatareturnci}d}tjd||\}|dz }t|D]=}tjd||\}|dz }||||zd}||z }tjd||\}|dz }||||zd}||z }tjd||\} |dz }g} i} t| D]o}tjd||\} } }|dz }| | t | d t |d f| | <pt| | f|||f<?|S) zLoad confusion group data from raw bytes. :param data: The raw binary content of a confusion.bin file. :returns: A :data:`DistinguishingMaps` dictionary keyed by encoding pairs. rz!Hr z!Br zutf-8z!BBBrr7)struct unpack_fromrangedecodeappendrEget frozenset)rOresultoffset num_pairs_ name_a_lenname_a name_b_lenname_b num_diffsdiff_bytes_list categoriesbv cat_a_int cat_b_ints rK%deserialize_confusion_data_from_bytesrgNs "$F F%dD&99LY aKF 9  LL*4v>> ! fv 223::7CC**4v>> ! fv 223::7CC*)$f== ! %'13 y!!  A'-'9&$'O'O $B 9 aKF  " "2 & & & $$Y55 $$Y55JrNN%.o$>$> #K  MrMctjdd}|}|st jdtdiS t|}n3#tj tf$r}d|}t||d}~wwxYwi}| D]1\\}}}t|p|} t|p|} ||| | f<2|S)zLoad confusion group data from the bundled confusion.bin file. :returns: A :data:`DistinguishingMaps` dictionary keyed by encoding pairs. zchardet.modelsz confusion.binuZchardet confusion.bin is empty — confusion resolution disabled; reinstall chardet to fixr ) stacklevelzcorrupt confusion.bin: N) importlib resourcesfilesjoinpath read_byteswarningswarnRuntimeWarningrgrRerrorUnicodeDecodeError ValueErroritemsr ) refrawraw_mapsemsg normalizedabvaluenorm_anorm_bs rKload_confusion_datarvs%   # #$4 5 5 > > O OC ..  C   '       %8== L, -%%%+++oo1$%&(J!))-- A ##(q ##(q', FF#$$ s(A88B(B##B() r.r/r0r1r2r4r3r6r5r7rrr_CATEGORY_PREFERENCEencodingc^td}tdD]} t|g|}n#t$rY3wxYwt |dkrKt j|}|dkrd||<k|ddks|dvrd||<t|S)u256-entry table: 0 = non-letter, 1 = uppercase letter, 2 = other letter. Combining marks count as letters: in decomposed text (Vietnamese under windows-1258) a base letter's neighbor is its diacritic, which is word-internal, not a word boundary. Whitespace deliberately counts as a plain non-letter: exempting space-adjacent letters from the isolated-letter demotion (to spare one-letter words like French ``à``) was tried and falsified by the accuracy suite — Irish/Finnish po files and the EBCDIC record set depend on space-adjacent demotions, so the cross-family decisive-override gate handles the ``à`` failure mode instead. rr r rL)rrr ) bytearrayrTbytesrUrslen unicodedatacategory)rtabler}chcats rK_letter_case_tablers cNNE 3ZZ s""8,,BB!    H  r77a<< "2&& $;;E!HH Vs]]c\11E!H <<s#A AArleftright case_tablect|d}|ddkr|S||}||}|dkr |dkrtS|dkr |dkrtS|S)aPreference for reading a byte as *cat*, adjusted for word shape. A letter reading only deserves its high preference when its neighbors make it look like part of a word under the same encoding: a letter with no letter neighbors is quoted/isolated punctuation in disguise, and a lowercase letter immediately followed by an uppercase one is not a word shape any of the supported languages produce. rrr r )rrW_IMPLAUSIBLE_LETTER_PREFERENCE)rrrrpref left_kind right_kinds rK_context_preferencersr # #C + +D 1v}} 4 IE"JA~~*//-- d{{zQ-- KrMenc_aenc_b diff_bytesrcct|\}}t|d|}|sdSt|}t|} d} d} d} d} d}d}t |dz }|D]Y}||\}}t |f}||}d}|dkr|tkr|dkr ||dz nd}||kr ||dznd}t||||}t|||| }||krF|ddkr |ddkrnx| ||z z } |ddkr|tkr | ||z z } |dz }nK||krE|ddkr |ddkrn,| ||z z } |ddkr|tkr | ||z z } |dz }|dz }|||dz}|dkr |tk[| | kr || | z | |fS| | kr || | z | |fSdS)uSContext-aware category voting. Returns ``(winner, margin, demotion_margin, demotion_events)``. For each occurrence of a distinguishing byte, compare the two encodings' readings: Unicode category preference, adjusted for word shape (see :func:`_context_preference`). The reading that makes more linguistic sense of the byte *in its context* collects the vote; occurrences vote independently, so repeated evidence counts. ``demotion_margin`` counts only the winner's votes earned where the *losing* side's letter reading was word-shape-implausible — evidence against an impossible reading, which is far stronger than the naive letters-beat-symbols preference. ``demotion_events`` counts how many distinct occurrences contributed to it, so callers can tell repeated evidence from one loud byte. N)Nrrrrr rP) _pair_byte_tablesrX translaterrrfind_MAX_VOTE_OCCURRENCESrr)rOrrrrcnon_diffr\relevanttable_atable_bvotes_avotes_b demotion_a demotion_bevents_aevents_bendrdcat_acat_bneedleposexaminedrrpref_apref_bs rK_vote_with_marginr sr8$J//KHah7788H } ''G ''GGGJJHH d))a-C"-"-!"~ uuiiQhh8&;;;$'!GG4a==D%(3YYDqMMAE(eWEEF(eWEEF8s??uQx3v.GQx365S+S+S"fvo5  A &8s??uQx3v.GQx365S+S+S"fvo5  A  MH))FC!G,,C;Qhh8&;;;<g'X==g'X== =rMenc_xenc_y languagescvt}t|||}|dS||\}}|\}} t||| ||\} } } } | 0| tkr%| tkrt |t kr| St||| ||}t |t kr ||| kr|SdS||n| S)uReturn the byte-evidence winner between two encodings, or ``None``. Mirrors the in-band pairwise rule of :func:`resolve_confusion_groups` (decisive demotion vote, else bigram rescore, else category vote) for callers outside the ranked-results scan — e.g. the classic-Mac line-ending promotion, whose platform prior must not override distinguishing-byte evidence. Returns ``None`` when the pair has no distinguishing map or the evidence is inconclusive. *languages* must carry what the two results being compared report, or the mirror breaks: the rescore would arbitrate the same pair under a different rule than the confusion stage just did, and a veto built on that answer can reverse a promotion the stage had settled. N)r_find_pair_keyr_DECISIVE_VOTE_MARGIN_DECISIVE_MIN_EVENTSr_CROSS_FAMILY_MIN_DIFFSresolve_by_bigram_rescore)rOrrrmapspair_keyrrcrr cat_winner_margindemotion_margindemotion_events bigram_winners rKconfusion_pair_winnerr`s(  DdE511Ht!(^J LE5z$_pair_byte_tables..s,<<1&).s6q!t?O?O?O?O?OrM)rXrrW)rs rK_modelled_languagesrsF ,223;;  rMc|sdSt|}t|}||z}||ksdSt|vs t|vr |thzS|S)uLanguages to score *enc_a* and *enc_b* under, or ``None`` for all. ``None`` means *unrestricted* — score every variant, the original max-over-models comparison. It is not an abstention. An encoding should not win on language coverage the other side lacks. On a Hungarian document the u-double-acute bytes score 0.014 against iso8859-2's *Czech* model (which reads them as a common r-hacek) and only 0.006 against iso8859-16's Hungarian one, so max-over-variants hands Hungarian text to the Czech reading. Dropping the languages only one side models removes that particular unfairness. This is deliberately a narrow rule, and two tempting generalisations were measured and rejected against the accuracy suite: * Restricting to *languages* themselves rather than the shared set costs 12 tests. As a consequence the restriction is a no-op for pairs whose language coverage already matches (67 of the 236 confusion pairs), which is accepted — those pairs have no coverage asymmetry to correct in the first place. * Restricting when *languages* is not wholly inside the shared set costs 6 tests: cp1125 models only Ukrainian, so a Belarusian cp866 document shares just ``uk`` with it, and scoring the *right* encoding under the *wrong* language loses to cp1125. Falling back to unrestricted is also what keeps a Vietnamese windows-1258 document, whose rival cp1252 models no Vietnamese, resolving correctly. The cost is that pairs modelling disjoint languages (koi8-r/koi8-u, mac-roman/mac-turkish) never restrict at all — about 20% of calls over the corpus. N)r _ART_LANGUAGE)rrrlangs_alangs_bshareds rK_comparable_languagesrsoF t!%((G!%((G w F   t=G#;#;'' MrMprofileindexct||}|sdStfd|DdS)aReturn the best bigram score for *enc*, restricted to *languages*. *languages* of ``None`` means every variant, the unrestricted max-over-models comparison. The ``default`` guards a caller-supplied set that names no variant of *enc*; note that scoring 0.0 hands the comparison to the rival rather than abstaining, so callers wanting an abstention must not rely on it. gc3NK|]\}}}|v t||V dSr)r)rHrmodel model_keyrrs rKrz&_best_variant_score..sQ  &eY DI$5$5 wy 9 9$5$5$5$5  rM)default)rWmax)rrrrvariantss` ` rK_best_variant_scorersiyy~~H s      *2       rMct|dkrdSt|\}}t|d|}|sdSt}i}t|dz }|tzt|krt }t |d|D]} t| f} || } | dkrT| r| | dz | |kr| | || | dz} | dkT|D];} || dz|| dzz} | | d|| z|| <$D*55Gt&ueY??J OOE % C CF % C CF    4rMc\t}tddD]}t|f} ||}n#t$rd}YnwxYw ||}n#t$rd}YnwxYw||kr||t |S)aByte values >= 0x80 that *enc_a* and *enc_b* decode to different text. The distinguishing set for a pair the confusion maps do not cover, computed from the codecs themselves. A byte only one side can decode counts as differing. Bytes below 0x80 are left out: the callers ask about high-byte evidence, and every single-byte Latin family agrees on ASCII anyway. rN)rrTrrUrsrrX)rroutr}rwtext_atext_bs rKdiffering_high_bytesrsEEC 4    QDkk !$E!2!2FF!   FFF  !$E!2!2FF!   FFF  V   GGAJJJ S>>s#A AAA11 B?BcXi}|D]}g}||fD]} t|g|}n%#t$r|dYHwxYw|t |dkrt j|nd|d|df||<|S)aUnicode general categories of each distinguishing byte under both encodings. The confusion maps carry this table for their own pairs; callers that arbitrate a pair the maps do not cover (the niche-Latin demotion's candidate against its swap target) build it here. A byte one side cannot decode reads as unassigned (``Cn``), which the vote treats as the least plausible reading of all. r7r r)rrUrsrVrrr)rrrrr}catsrrs rK_pair_categoriesrs)+E  & &5> L LC A3ZZ&&s++%    D!!!  KKCGGqLL ,R000d K K K KGT!W%a Ls#4AA languages_a languages_bc t||}|Bt}t||||}t||||} || kr|S| |kr|St||||t |||\} } } } | S)aDecide a pair on its distinguishing bytes: model rescore, then context. The in-band rule of :func:`resolve_confusion_groups` for a pair the confusion maps do not cover, with one difference: each side is scored under the languages the caller names for it rather than the shared set. The bigram rescore decides when the models have an opinion; when they score the distinguishing bigrams equally (usually both at zero, a byte neither model has seen in that context), the category vote reads word shape instead, so a letter between letters still beats a superscript between letters. Returns ``None`` when neither step can tell the two apart. :param data: The raw byte data to examine. :param enc_a: First encoding name. :param enc_b: Second encoding name. :param diff_bytes: Byte values where the two encodings differ. :param languages_a: Variants of *enc_a* to score; ``None`` for all. :param languages_b: Variants of *enc_b* to score; ``None`` for all. )rrrrr)rOrrrrrrrrrrrrrs rKarbitrate_distinguishing_bytesrs8$D*55G$WeUKHH$WeUKHH F??L F??L*; eUJ(8z(R(R++'FGY MrMrc.||f|vr||fS||f|vr||fSdS)zEFind the canonical key for a pair of encodings in the confusion maps.NrG)rrrs rKrrs; u~u~ u~u~ 4rM4g{Gzt?g?g?resultsct|dkr|S|d}|j|S|jtkr|St }|j}|t z}d|}|j}tdt|D]|}|jdkp||jz tk} | s|tks |j|krn;t|||j} | `|| \} } | \} }t|| || | \}}}}|1|tkr&|tkrt| tkr|}nftd|j|jfD}t!|| || |}| rt| tkr||n|}n | ||kr|}nd}| ||jkr0| rCt#|j||j|j}fdt'|D}|g|cS|}|j}dkr|St#|j||j|j}fdt'|D}|g|S)a Resolve confusion between similar encodings in the top results. Checks the top result against each candidate within a confidence band. Always checks position 1 (preserving original top-2 behavior); for positions 2+ only checks within the band. Uses bigram re-scoring with category voting as fallback. :param data: The raw byte data to examine. :param results: Detection results sorted by confidence descending. :returns: A reordered list of :class:`DetectionResult` with the winner first. r rNr c3K|]}||V dSrrG)rHrs rKrz+resolve_confusion_groups..qs4#####rMc&g|] \}}|k |SrGrG)rHjrrs rK z,resolve_confusion_groups..s"???$!QQArMc&g|] \}}|k |SrGrG)rHr r  champion_idxs rKrz,resolve_confusion_groups..s' B B B$!Q\0A0AA0A0A0ArM)rrlanguagerr confidenceCONFUSION_FLOOR_RATIOrTCONFUSION_BANDSTRICT_TIER_MAX_CONFrrrrrrXrr mime_type enumerate)rOrtoprtop_conffloorchampion champion_enc candidatein_bandrrrcrrr _vote_marginrrrlangsrpromotedrestrrs @@rKresolve_confusion_groupsr#s 7||a !*C | |}$$  D~H , ,ELH Vy'999   %'""# H @???)G"4"4???D$t$ $ $ $  ) q H C B B B)G,, B B BD  t rM)<__doc__ functoolsimportlib.resourcesrjrRrrochardet.modelsrrrrrchardet.pipelinerchardet.registryr dicttuplestrrXintDistinguishingMapsrE__annotations__rurNrrgcacherrrrrrrrrrrrrrrrlistfloatrrrrrrrrrrrr#rGrMrKr3s     -,,,,,,,,,,, #s(O )C.$sE#s(O34 457$t$t$t$t $ t $ t $t$t$t$t$$$$$$ !$"#$$             =$$$$sCx.H#N"M4D4J4J4L4L"M"M"Mc3hMMM%%:L%%%%P />("("( "( ! (  ! (  ! ( !( !( !( !( !( !( !( !( !(  !!(" !#($ =(((d38nH"# @SCUs*P P P P# P S%S/)* P  3:sC $% PPPPn!* +F+F +F +F +F~ +F  4Z +F+F+F+F\   #  S%S/)*   4Z "  #)C. #U5%<5H # # # #&  SYs^2 2 2~2s^d 2222j  T%d E3 6788 9 ~$   6A2 A2&s^A2TA2A2A2A2R!* -- - - -# - ~ -  4Z ----` CIcN2  (1# #uS#X 2( ( ( (# ( 3$& (3$&( 4Z((((V        38_t    (W W / "W /WWWWWWrM