jdZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z m Z ddl m Z ddlmZddlmZdZdZejed ejed gZejd ed Zejd edZdZdZGddejZdZejd edZdS)zCommon pickle round-trip tests for callbacks. These tests guard the contract that callbacks (and estimators they are attached to) must be picklable, and that an estimator pickled after a successful fit can be unpickled in a fresh Python interpreter. N) ProgressBarScoringMonitor)_message_consumers)MaxIterEstimator)make_regressioncFtjdtS)Nrich)pytest importorskipr/Users/tedpeters/Documents/Projects/Claude Projects/Misc Tools/notename-sheets/.venv/lib/python3.11/site-packages/sklearn/callback/tests/test_pickle.py_pbrs  ==r c"tdS)Nr2scoring)rr r r_smr s $ ' ' ''r ridrfactoryct|}tjtj|}t |t |usJt |jdksJdS)zJAn estimator with the callback registered but not yet fitted is picklable.N)r set_callbackspickleloadsdumpstypelen_skl_callbacks)r estimatorrestoreds r5test_estimator_with_callback_pickle_roundtrip_pre_fitr#*s{!""00;;I|FL3344H >>T)__ , , , , x& ' '1 , , , , , ,r cH|}td|}|tjtj|}t |t |usJt|jdksJdS)zBAn estimator with the callback registered and fitted is picklable.max_iterrN) rrfitrrrrrr )rcallbackr!r"s r6test_estimator_with_callback_pickle_roundtrip_post_fitr*3swyyH !,,,::8DDI MMOOO|FL3344H >>T)__ , , , , x& ' '1 , , , , , ,r ctjdtddd\}}td}t d t |}||| |}tj d |j sJtj d |j sJ| d }t|dksJtjtj|}||| |}tj d |j sJtj d |j sJ|jd d }t|dksJ|dj|djksJdS)zAn estimator with callbacks survives an in-process pickle round-trip. It also supports re-fitting after being unpickled and the callbacks accumulate new data from the re-fit. r r n_samples n_features random_staterrr%r&XyMaxIterEstimator - fit100%allselectrN)r r rrrrrr( readouterrresearchoutget_logsrrrrr data) capsysr3r4smr!captured original_logsr" restored_logss r1test_callbacks_refit_after_pickle_in_same_processrE>s   RAA F F FDAq  % % %B !,,,::;=="MMI MMAM  ""H 9. = === = 9Whl + +++ +KKuK--M }   " " " "|FL3344H LL1L  ""H 9. = === = 9Whl + +++ ++A.77u7EEM }   " " " "   M!$4$9 9 9 9 9 9 9r ctjdtddd\}}td}t dt |}||| |}tj d |j sJtj d |j sJ| d }t|dksJ|dz }t|d5} tj|| dddn #1swxYwYt#jdt'|d|djd} t+jt.jd| gdd} | j} tj d | sJtj d | sJdS)zAn estimator with callbacks survives unpickling in a fresh interpreter. It also supports re-fitting after being unpickled and the callbacks accumulate new data from the re-fit. r r%rr.rrr&r2r5r6r7r8rzest.pklwbNz import pickle from sklearn.callback import ScoringMonitor from sklearn.datasets import make_regression with open(a*, "rb") as f: est = pickle.load(f) X, y = make_regression(n_samples=20, n_features=3, random_state=1) est.fit(X=X, y=y) restored_logs = est._skl_callbacks[1].get_logs(select="all") assert len(restored_logs) == 2 assert restored_logs[0].data == z z-cTx)capture_outputtimeout)r r rrrrrr(r:r;r<r=r>ropenrdumptextwrapdedentstrr? subprocessrunsys executablestdoutdecode) tmp_pathr@r3r4rAr!rBrCpkl_pathf load_scriptresultrUs r0test_callbacks_refit_after_load_in_fresh_processr\_s$   RAA F F FDAq  % % %B !,,,::;=="MMI MMAM  ""H 9. = === = 9Whl + +++ +KKuK--M }   " " " ")#H h  " Iq!!!"""""""""""""""/ x==   *7q)9)>   K$^ {+D#F] ! ! # #F 9. 7 777 7 9Wf % %%% %%%s D//D36D3c(eZdZdZfdZdZxZS)_TraversalRecorderaaA pickler that records the id of every object it walks through. `persistent_id` is called for every object the pickler encounters, so an object is recorded whichever path leads to it, not only when it is a direct attribute of the object being pickled, see https://docs.python.org/3/library/pickle.html#pickle.Pickler.persistent_id. cttjtjt |_dS)N)protocol)super__init__ioBytesIOrHIGHEST_PROTOCOLsetwalked_through)self __class__s rrbz_TraversalRecorder.__init__s; 0GHHH!eer cT|jt|dS)N)rgaddr)rhobjs r persistent_idz _TraversalRecorder.persistent_ids$ 3(((tr )__name__ __module__ __qualname____doc__rbrm __classcell__)ris@rr^r^sQ$$$$$r r^cFtjfd}|S)aWrap a callback hook so that it first checks the callback it is called on. The check is that the pickler does not walk through the objects that listener threads mutate. They are found through the registered consumers: a consumer is normally a method bound to the container it fills, e.g. `self._log.append` for ScoringMonitor or `queue.put` for ProgressBar, so the container is what it is bound to. A consumer bound to nothing, e.g. a closure, is skipped, since there is then no way to tell what it mutates. cttj}d|D}t}|||jd|Dz}|rJd|jjd|g|Ri|S)Nc<g|]}t|d|jS)__self__)hasattrrv).0cs r z2_checked..checked_hook..s)KKK!GAz4J4JK1:KKKr c,h|]}t|Sr r)rx containers r z1_checked..checked_hook..s.V.V.Vr)}}.V.V.Vr z Pickling z walks through a container that a listener thread mutates concurrently. Keep it away from the pickler, either by handing over a copy of it in the callback's `__getstate__`, or by storing it outside of the callback instance.)listrvaluesr^rMrgrirn)rhargskwargs consumerswatchedrecorder offendershooks r checked_hookz_checked..checked_hooks+24455 KKyKKK%'' d+.V.Vg.V.V.VV    ?/ ? ? ?  }tD*4***6***r ) functoolswraps)rrs` r_checkedrs;_T+++++& r c |}dD]@}t|j|}||j|t|At d|dS)aJCheck that pickling a callback never traverses state its listener mutates. An estimator carrying a callback can be pickled by a background thread, e.g. loky's queue feeder dispatching a task to a worker, while the listener thread of that same callback mutates the callback's state as messages come in. Pickling a container that another thread mutates breaks the dump, which joblib reports as "Could not pickle the task to send it to the workers". The check runs from a hook, i.e. while the listeners are up, which is when such a dispatch would happen. )on_fit_task_beginon_fit_task_endr%r&N)getattrrisetattrrrrr()r monkeypatchr) hook_namers r0test_listener_state_is_not_walked_by_the_picklerrswyyH=KK x)955H. 8D>>JJJJa   ..x88<<>>>>>r ) rqrrcrr;rQrSrNr sklearn.callbackrrsklearn.callback._transportrsklearn.callback.tests._utilsrsklearn.datasetsrrrparamCALLBACK_FACTORIESmark parametrizer#r*rEr\Picklerr^rrr r rrs   88888888::::::::::::,,,,,, ((( FL'''FL)*** $677--87-$677--87-:::B1&1&1&h$   F$677??87???r