j2 dZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z ddl mZddlmZddlmZddlmZdd lmZddlZddlZdd lmZmZmZdd lmZdd lmZmZm Z m!Z!dd l"m#Z#m$Z$m%Z%m&Z&m'Z'eddl(m)Z)ddl*m+Z+ ddl,Z,n #e-$rdZ,YnwxYw ddl.m/Z/n #e-$rdZ/YnwxYw ddl0Z0n #e-$rdZ0YnwxYwddl1m2Z2m3Z3m4Z4m5Z5m6Z6ddl7m8Z8m9Z9m:Z:m;Z;mZ>m?Z?ej@jAZBe8jCZDeDEdddgeFe8jGzZHeHde8Dz ZHegZInddgZIeIdgzZJeKedr#eHLejMddZNdZOdZPdZQGddeRZSdd!ZTd"ZUdd#ZVd$ZWd%ZXd&ZYd'ZZe$d(e>e=gej@$d)d*d+ed?Zad@Zbe$dAe8e$dBe8dCZcdDZde dEZedFZfe$d2gdGdHZge$d1eJdIZhe dJZie!e e$dKgdLe$dMddNgdOZje$d1eJdPZke e$d1eJdQZle e$d1emeDdRhz e$dSdTdUgdVZne e$d1eIdWZoe e$d1e8dXZpdYZqe$d1e8e$dZdgd[fd\gd]fgd^Zre e$d1eJd_Zsd`Zte e$d1eIdaZudbZvdcZwddZxe e$d1eJej@ydefdgZzGdhdie5Z{djZ|e$d1eHdkZ}e e$d1eJe$d2gdldmZ~dnZej@jAdoZe&edudpfdqZdrZeJddZedsedtDe e$d1ee$d(e=e>gduZGdvdwe5Ze$d(e>e=gdxZe$d(e>e=gdyZdzZd{ZGd|d}ejj2Ze?d~ee e$d1gde$d(e>e=gdZe e$d2gde$d1eJe$d(e>e=gdZdZe e$deJe$deJe$d(e>e=gdZe e$d(e>e=ge$ddd gdZe$dgddZe$dgddZe dZe!e e$d1eIdZdZe e$d1eIdZdZdZdZdZe e$d1eIegndgze$deeege$dgddZdejejejZe dZdejejejZe dZdZdZe!e e$d1eIdZdZe!e dZdZdZej@$d2gddZdZdZej@$d2d7d\ge e$d1emeDdRhz dZe$d1eHe$d2gddZdZdZe!e$d1eDe$dSdTdUge$d2gddZe$d1eDe$dSdTdUge$d2gddZe$d1eDe$dSdTdUge$d2gddZe$d1eDe$ddTdUge$ddTdUge$d2gddZe$dgde$ddTdUge$ddTdUgdZej@jAe!e e$d1eIdZe$d1eddgngddZe e$d1eIdZe e$d(e>e=gdZe$d2dd7ge$dgddZe$d(e>e=gd„ZdÄZddĄZe e$d1ddge$d(e>e=gdńZddƄZej@je$d(e>e=ge$d1edgnddgdDŽZdȄZe$d1eddgndge&e/dudɬfdʄZe$d(e>e=ge&e0dudˬfd̄Ze$d(e>e=gd̈́Zd΄ZdτZdЄZdфZe!e e$d2gdҢdӄZe!e e$dgdբe$d2d7d+ge$d(e>e=gdքZe e$d2d7d+ge$dgdآe$d(e>e=gdلZe e$d2gdڢdۄZd܄Zdd݄Ze e$d2d7d\ge$d1eIe$d(e>e=gdބZe e$d2d7d\ge$d1eId߄Ze ej@$d2d7d\gdZe ej@$d2d7d\gdZdS)z Test the parallel module. N) nullcontextsqrt TimeoutError) PicklingError)sleep)format_exception)uuid4)dumploadparallel)mp)IS_GIL_DISABLEDnpwith_multiprocessing with_numpy)check_subprocess_call parametrizeraisesskipifwarns)get_reusable_executor)Queue) parallel_sum) LokyBackendMultiprocessingBackendParallelBackendBaseSequentialBackendThreadingBackend) BACKENDSParallel cpu_countdelayedeffective_n_jobsrparallel_backendparallel_configregister_parallel_backendmultiprocessingc6g|]}t|S)r").0 backend_strs /Users/tedpeters/Documents/Projects/Claude Projects/Misc Tools/notename-sheets/.venv/lib/python3.11/site-packages/joblib/test/test_parallel.py r0Xs#KKK;x ,..KKKloky threading get_contextspawnc0ttjSN)r"rDEFAULT_BACKENDr,r1r/get_default_backend_instancer9ds H, --r1c Bt|dt|ddS)N_pool_workers)getattrbackends r/ get_workersr@ls 7GWWj$%G%G H HHr1c ||z Sr7r,)xys r/divisionrDps q5Lr1c |dzSNr,rBs r/squarerIts a4Kr1ceZdZdZdZdS)MyExceptionWithFinickyInitz,An exception class with non trivial __init__cdSr7r,)selfabcds r/__init__z#MyExceptionWithFinickyInit.__init__{s r1N__name__ __module__ __qualname____doc__rRr,r1r/rKrKxs)66     r1rKFcH|dkr|rtddddnt|S)NrNrOrPrQ)rK ValueError)rBcustom_exceptions r/exception_raiserr\s8Avv   &sCc : : : Hr1c8tjdt)Ng?)timer KeyboardInterruptrHs r/interrupt_raiserr`sJt r1c|dz|z|zS)zNA module-level function so that it can be spawn with multiprocessing. rGr,)rBrCzs r/frcs a4!8a<r1cNttjdSNr)typerget_active_backendr,r1r/_active_backend_typerhs +--a0 1 11r1cdt||dtdDS)Nn_jobsr?c3PK|]!}tt|V"dSr7r%rIr-is r/ z parallel_func..sC:: ::::::r1r#range) inner_n_jobsr?s r/ parallel_funcrusE 98< 9 9 9::$)!HH:::  r1c.tdksJdSre)r$r,r1r/test_cpu_countrws ;;??????r1c.tdksJdSrer&r,r1r/test_effective_n_jobsrzs    ! ! ! ! ! !r1contextzbackend_n_jobs, expected_n_jobs)rqrqrkNr)z positive-intz negative-intNone)idsc|d|5td|ksJ dddn #1swxYwYtddksJdS)Nr3r}rry)r{backend_n_jobsexpected_n_jobss r/test_effective_n_jobs_Noners ^ 4 4 4@@ t,,,?????@@@@@@@@@@@@@@@ 4 ( ( (A - - - - - -s 266r?rk)rrGr|verbose)rG dcdtdDt|||dtdDksJdS)Nc,g|]}t|Sr,rIr-rBs r/r0z(test_simple_parallel..s ( ( (!F1II ( ( (r1rkr?rc3PK|]!}tt|V"dSr7rmrs r/rpz'test_simple_parallel..s2++Qogfooa++++++r1rsr#)r?rkrs r/test_simple_parallelrsw ) (uQxx ( ( (-Hw---++%((+++-,-, , , , , , ,r1rGcd}tjd}Gddt}|||d}|dt|Dt }|jD]H}||x}r/|\} } || | z It|dksJdS) Nrz(Done\s+\d+ out of \d+ \|)ceZdZgZdZdS)/test_parallel_pretty_print..ParallelLogc:|j|dSr7)messagesappend)rMmsgs r/_printz6test_parallel_pretty_print..ParallelLog._prints M  % % % % %r1N)rTrUrVrrr,r1r/ ParallelLogrs( & & & & &r1ri'rcHg|]}tt| Sr,r%rcrns r/r0z.test_parallel_pretty_print..s& 4 4 4jgajjmm 4 4 4r1r) recompiler#rssetrsearchspanaddlen) r?rkn_taskspatternrexecutorlensmessagesrNrOs r/test_parallel_pretty_printrsGj677G&&&&&h&&& {&'5IIIH H 4 4U7^^ 4 4 4555 55D$w'' '1 6688DAq HHQUOOO t99>>>>>>r1c|tjddtjd5}t d|dt d D}|gd ksJ dddn #1swxYwYd |D}|d vst|trd fd|D}t|dksJdS)Nname!some_new_name_for_the_main_thread)targetrvalueTrecordrGrjc3PK|]!}tt|V"dSr7rmrs r/rpz6test_main_thread_renamed_no_warning..sC6 6 #$OGFOOA  6 6 6 6 6 6 r1rq)rrcrg|]4}dt|jvt|jt2|5S)zworker timeout)strr isinstanceDeprecationWarningr-ws r/r0z7test_main_thread_renamed_no_warning..sJ 3qy>> 1 119&899 2 1 1 1r1)Nr*z3multi-threaded, use of fork() may lead to deadlocksc@g|]}t|jv|Sr,)rr)r-r message_parts r/r0z7test_main_thread_renamed_no_warning..s+NNN!. s,,,a ,,,r1c34K|]}d|jdvVdS)zbacked parallel loops cannotrN)args)r-eachs r/rpz)_assert_warning_nested..s?''CG.$)A,>''''''r1rF)rr simplefilterruallrr)r?rtexpectedrwarnings_are_correctwarnings_have_the_right_lengths r/_assert_warning_nestedrs;   - - -Bh'''gLAAAABBBBBBBBBBBBBBB-,8,,,H  K#&''KS'''$$ '6MH ""3x==A;M +(J,J Ju|ts&AA A z%parent_backend,child_backend,expected))r2r*T)r2r2F)r*r*T)r*r2T)r3r*T)r3r2Tc,td|fdtdDtd|fdtdD}|dkrt|sJdSt|sJdS)NrGrjc3XK|]$}ttddV%dS)rFr?rtrNr%rr-_ child_backends r/rpz0test_nested_parallel_warnings...sZ//  (&''!E   //////r1rc3XK|]$}ttdV%dS)rGrNr)r-rrrs r/rpz0test_nested_parallel_warnings..6sZ55  (&''!H   555555r1r3)r#rsanyr)parent_backendrrress `` r/test_nested_parallel_warningsr s/HA~...////q /// 5(!^ 4 4 455555q 555  C$$3xxx3xxxr1)r2r*r3cdg}d}tj||f}|||dsJdS)NFc$tjd5}tddtdDdddn #1swxYwYt t |t |dk|d<dS)NTrrGr}c3PK|]!}ttdV"dS)皙?Nr%r r-rs r/rpzPtest_background_thread_parallelism..background_thread..Ls2EEq~wu~~c22EEEEEEr1rr)rrr#rsprintr)is_run_parallelrs r/background_threadz=test_background_thread_parallelism..background_threadJs  $D 1 1 1 FX HA   EEE!HHEEE E E E F F F F F F F F F F F F F F F c(mm ]]a/s1AAA)rrr)r3Threadstartjoin)r?rrts r/"test_background_thread_parallelismrEsfgO000  18JKKKAGGIIIFFHHH 1  r1chtd|dtdDdS)NrGrjc3PK|]!}ttdV"dS{Gz?Nrmrs r/rpznested_loop..Ws2'P'P!(=(='P'P'P'P'P'Pr1rrr>s r/ nested_looprVs:'HAw''''P'PuQxx'P'P'PPPPPPr1rrcntd|fdtdDdS)NrGrjc3RK|]!}ttV"dSr7)r%rrs r/rpz#test_nested_loop..]sG//01 ]++//////r1rr)rrs `r/test_nested_looprZs\/HA~...////5:1XX///r1ctr7)rZr>s r/raise_exceptionrbs r1c 8tt5tdd5}|ttdtt dgdddn #1swxYwYddddS#1swxYwYdS)NrGr2rj)rrZr#r%rr)rs r/)test_nested_loop_with_exception_with_lokyrfs8   WW Q / / / W8 H*gk**6224LGO4L4LV4T4TU V V V W W W W W W W W W W W W W W WWWWWWWWWWWWWWWWWWWs5BAA7+ B7A; ;B>A; ?BBBctdtddfdtdDsJdS)z1Input is mutable when using the threading backendr)maxsizerGr3rjc3RK|]!}tjdV"dSrN)r%put)r-rqs r/rpz1test_mutate_input_with_threads..ps5+P+P!NGAENN1,=,=+P+P+P+P+P+Pr1N)rr#rsfull)rs@r/test_mutate_input_with_threadsrms` aA+HA{++++P+P+P+PuQxx+P+P+PPPP 6688OO8OOr1)rrGrqctd}d|Dt|d|DksJdS)z.Check the keyword argument processing of pmap. c0g|]}t|dSrrCrcrs r/r0z(test_parallel_kwargs..xs" # # #!Aa1III # # #r1r}c3TK|]#}tt|dV$dSrrNrrs r/rpz'test_parallel_kwargs..xsH??  1??????r1Nr)rklsts r/test_parallel_kwargsrtst ))C # #s # # #'>xv'>'>'>??$'???((      r1ctd}d|D}td|5}|j}||d|DksJ||d|DksJt%t |t |jusJdddn #1swxYwYtt |jJ||d|DksJtt |jJdSdS)Nrc0g|]}t|dSrrrs r/r0z4test_parallel_as_context_manager..s"'''a!q '''r1rrjc3TK|]#}tt|dV$dSrrrs r/rpz3test_parallel_as_context_manager..7==AZWQZZQ///======r1c3TK|]#}tt|dV$dSrrrs r/rpz3test_parallel_as_context_manager..r r1c3TK|]#}tt|dV$dSrrrs r/rpz3test_parallel_as_context_manager..s799A+++999999r1)rsr#_backendrr@)r?rrpmanaged_backends r/ test_parallel_as_context_managerr}s ))C''3'''H G , , , K*11===========11=========== >//;qz3J3JJJJJ K K K K K K K K K K K K K K K  ~1:&&... qq99S99999 9 9 9 9 ~1:&&...~..sA*B##B'*B'cGddtttd5tddfdt d Dd d d d S#1swxYwYd S) z`Check that pmap captures the errors when it is passed an object that cannot be pickled. ceZdZdZdS)1test_parallel_pickling..UnpicklableObjectc td)N123) RuntimeErrorrMs r/ __reduce__z.UnpicklableObject.__reduce__su%% %r1N)rTrUrVrr,r1r/UnpicklableObjectrs# & & & & &r1rzthe task to sendmatchrGr2rjc3bK|])}ttV*dSr7r%id)r-rrs r/rpz)test_parallel_pickling..sN+ + 12KGBKK))++ , ,+ + + + + + r1rN)objectrrr#rs)rs@r/test_parallel_picklingr!s &&&&&F&&&  %8 9 9 9  *6***+ + + + 6;Bii+ + +                      s4A,,A03A0 byteorder)<>= max_nbytes1Mcdtjdd|djj}t dd|fdtd D}|D]?\}}||ksJ||jjksJtj |@dS) Nc||jjfSr7)dtyper"rHs r/inspect_byteorderz=test_parallel_byteorder_corruption..inspect_byteorders!'###r1)rGrqi4rGr2)rkr?r&c3HK|]}tVdSr7r%)r-rr+rBs r/rpz5test_parallel_byteorder_corruption..sPGG*+"!""1%%GGGGGGr1rq) rarangereshapeviewr*r"r#rstestingassert_array_equal)r"r&initial_np_byteorderresult x_returnedbyteorder_in_workerr+rBs @@r/"test_parallel_byteorder_corruptionr9s $$$ ! V$$))Y*:*:*:;;A7, FXQ: F F FGGGGG/4QxxGGGF,255' '"&:::::"j&6&@@@@@ %%a444455r1cttd|ddtdDdksJdS)NrGrkr?timeoutc3PK|]!}ttdV"dS)gMbP?Nrrs r/rpz0test_parallel_timeout_success..sC<<*+u%%<<<<<<r1r)rr#rsr>s r/test_parallel_timeout_successr?st ;HAw ; ; ;<</4Ryy<<<           r1ctt5td|ddtdDddddS#1swxYwYdS)NrGrr<c3PK|]!}ttdV"dSrNrrs r/rpz-test_parallel_timeout_fail..sC: : #$NGENN2  : : : : : : r1r)rrr#rsr>s r/test_parallel_timeout_failrCs     97D999: : (-b : : :                      3AAA sequential return_as generatorgenerator_unorderedc ^tt5ttd||ddt dDdddn #1swxYwYttd||ddt dDdS)NrGr)rkr?rFr=c3PK|]!}ttdV"dSrBrrs r/rpz.sKRR'(r""RRRRRRr1rc3PK|]!}ttdV"dSrrrs r/rpz.sKM M %&NGENN4 M M M M M M r1)rrlistr#rsr?rFs r/)test_parallel_timeout_fail_with_generatorrNs%      QHAw)S Q Q QRR,1"IIRRR                     L7iLLLM M */))M M M   sAA""A&)A&c ttt5td|dt ddDdddn #1swxYwYtt 5td|ddDdddn #1swxYwYtd|5}t |jJt |j}tt5|dt ddDdddn #1swxYwYt |jJt |j|usJdtd D|d td DksJt |j}tt 5|d dDdddn #1swxYwYt |jJt |j|usJd td D|d td Dks"J|j |j |j |j f dddn #1swxYwYt |jJnOtt 5tdddDdddn #1swxYwYtt5tddt ddDdddn #1swxYwYtt5tdddtdDddddS#1swxYwYdS)NrGrjcPg|]#\}}tt||$Sr,r%rDr-rBrCs r/r0z&test_error_capture..s1IIITQ"""1a((IIIr1)rr)rrcHg|]}tt| Sr,r%r`rs r/r0z&test_error_capture..s,>>>!*)**1-->>>r1cPg|]#\}}tt||$Sr,rQrRs r/r0z&test_error_capture..s1RRRda+'(++Aq11RRRr1c0g|]}t|dSrrrs r/r0z&test_error_capture.. "111!Aa1III111r1rc3TK|]#}tt|dV$dSrrrs r/rpz%test_error_capture.. H>>'(  1""">>>>>>r1cHg|]}tt| Sr,rTrs r/r0z&test_error_capture..s,GGG13'"233A66GGGr1c0g|]}t|dSrrrs r/r0z&test_error_capture..rWr1c3TK|]#}tt|dV$dSrrrs r/rpz%test_error_capture..rYr1r}cHg|]}tt| Sr,rTrs r/r0z&test_error_capture..(s,MMM 9(8 9 9! < <MMMr1cPg|]#\}}tt||$Sr,rQrRs r/r0z&test_error_capture..-s1TTT1-GH--a33TTTr1r)rkrc3TK|]#}tt|dV$dS)T)r[Nr%r\rns r/rpz%test_error_capture..1s; T TQ &W% & &q4 @ @ @ T T T T T Tr1r;)rrZeroDivisionErrorr#zipr_r@rrs _iteratingn_completed_tasksn_dispatched_tasks _abortingrK)r?roriginal_workerss r/test_error_capturerhs~  ~ % & &   /HAw / / /IIS5H5HIII                   % & &   /HAw / / />>v>>>                   Q 0 0 0# Hx011===*8+<== )** T TRRc&&>Q>QRRRSSS T T T T T T T T T T T T T T T x011===x0119IIIII11uRyy111XX>>,1"II>>>66 +8+<== )** I IGGGGGHHH I I I I I I I I I I I I I I Ix011===x0119IIIII11uRyy111XX>>,1"II>>>66#*+" 9# # # # # # # # # # # # # # # N8,--5555 % & & O O HA   MMfMMM N N N O O O O O O O O O O O O O O O ! " "VVTTFF@S@STTTUUUVVVVVVVVVVVVVVV * + +  %1%%% T T%PR)) T T T                     s3AA #A ;%B,,B03B0?J$D7+ J7D; ;J>D; ?BJG9- J9G= =JG= BJJ"%J"$LL L#2M!!M%(M%2N??OOc0d d}td|5}ttd5|d|dDdddn #1swxYwYttd5|d |d Ddddn #1swxYwYttd5|d |d Ddddn #1swxYwYddddS#1swxYwYdS)Nrc3`KtdD]}||krtd|VdS)NIterator Raising Error)rsrZ)raise_atros r/ my_generatorz1test_error_in_task_iterator..my_generator8sHr  AH}} !9:::GGGG  r1rGrjrlrc3PK|]!}tt|V"dSr7rmrns r/rpz.test_error_in_task_iterator..A2 C CQogfooa  C C C C C Cr1)rmc3PK|]!}tt|V"dSr7rmrns r/rpz.test_error_in_task_iterator..Frpr1rc3PK|]!}tt|V"dSr7rmrns r/rpz.test_error_in_task_iterator..Js2 D DQogfooa  D D D D D Dr1r)r#rrZ)r?rnrs r/test_error_in_task_iteratorru5s G , , , E J&> ? ? ? D D A C C,,*B*B*B C C C C C C D D D D D D D D D D D D D D D J&> ? ? ? D D A C C,,*B*B*B C C C C C C D D D D D D D D D D D D D D DJ&> ? ? ? E E A D D,,*C*C*C D D D D D D E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E Es|D  A D A D  A !D : B& D &B* *D -B* .D  C3' D 3C7 7D :C7 ;D  DDc6|d|zdS)Nz Consumed %s)r)queueitems r/consumerryMs LL%&&&&&r1zbatch_size, expected_queue) Produced 0 Consumed 0 Produced 1 Consumed 1 Produced 2 Consumed 2 Produced 3 Consumed 3 Produced 4 Consumed 4 Produced 5 Consumed 5r) rzr|r~rr{r}rrrrrrctfd}td||fd|D|ksJtdksJdS)z=Test that with only one job, Parallel does act as a iterator.c3hKtdD]}d|z|VdSNr,z Produced %irsrrorws r/producerz'test_dispatch_one_job..producer~Hq  A LL* + + +GGGG  r1rrk batch_sizer?c3TK|]"}tt|V#dSr7r%ry)r-rBrws r/rpz(test_dispatch_one_job..sI??()%##??????r1 N)rLr#r)r?rexpected_queuerrws @r/test_dispatch_one_jobrQsV FFE ?HA*g>>>????-5XZZ??? N " " " " u::      r1ctj}|fd}tddd|fd|Dt}|ddksJ|d d d }|d ksJ|d }||ksJt dksJd S)zRCheck that using pre_dispatch Parallel does indeed dispatch items lazily. c3hKtdD]}d|z|VdSrrrs r/rz/test_dispatch_multiprocessing..producerrr1rGrrq)rkr pre_dispatchr?c3TK|]"}ttdV#dS)rNr)r-rrws r/rpz0test_dispatch_multiprocessing..sQFF,-%''FFFFFFr1rrzNrz Consumed anyr|rr)rManagerrLr#indexr)r?managerrqueue_contentsfirst_consumption_indexproduced_3_indexrws @r/test_dispatch_multiprocessingrs' jllG LLNNE FHA!!WEEEFFFF19FFF%[[N !  , , , ,-RaR066~FF "R ' ' ' '%++L99 5 5 5 5 5 u::      r1ctddd5}|dtdD|jdksJ ddddS#1swxYwYdS)NrGautor3rc3PK|]!}tt|V"dSr7rrns r/rpz/test_batching_auto_threading..0 . .Q+'"++a.. . . . . . .r1rr#rsrcompute_batch_size)rs r/test_batching_auto_threadingrs v{ C C C4q  . .%++ . . ....z,,..!33333444444444444444444AA##A'*A'ctdd|5}|dtdD|jdksJ ddddS#1swxYwYdS)NrGrrc3PK|]!}tt|V"dSr7rrns r/rpz2test_batching_auto_subprocesses..rr1rrr)r?rs r/test_batching_auto_subprocessesrs vw ? ? ?31  . .%++ . . .... z,,..22222 333333333333333333rctt5tddddtdDddddS#1swxYwYdS)zCMake sure that exception raised during dispatch are indeed capturedrGr)rkrrc3PK|]!}tt|V"dSr7r`rns r/rpz*test_exception_dispatch..sG7 7 -. %G$ % %a ( (7 7 7 7 7 7 r1r;N)rrZr#rsr,r1r/test_exception_dispatchrs     6A6667 7 27))7 7 7                      rDcftddtdDdS)NrGr}c3PK|]!}tt|V"dSr7r`r-js r/rpz(nested_function_inner..s6GG0w/0033GGGGGGr1r;rrros r/nested_function_innerrs8HAGGU2YYGGGGGGGGr1cftddtdDdS)NrGr}c3PK|]!}tt|V"dSr7)r%rrs r/rpz(nested_function_outer..s6LLQ5w455a88LLLLLLr1r;rrrs r/nested_function_outerrs8HALL%))LLLLLLLLr1z'https://github.com/joblib/loky/pull/255reasonctt5}td|dtdDdddn #1swxYwYt |j|j|j}d|}d|vsJd|vsJd |vsJt |jtusJdS) zEnsure errors for nested joblib cases gets propagated We rely on the Python 3 built-in __cause__ system that already report this kind of information to the user. rGrjc3PK|]!}tt|V"dSr7)r%rrns r/rpz1test_nested_exception_dispatch..sG, , 23 *G) * *1 - -, , , , , , r1r;Nrrr\) rrZr#rsr rfrtbr)r?excinfo report_linesreports r/test_nested_exception_dispatchrs%    w+7+++, , 7)rrZr#r(r,r1r/test_invalid_backendrs3  "4 5 5 5))(((()))))))))))))))  "4 5 5 5 ^ 4 4 4                    "4 5 5 5 ^ 4 4 4                  sx488B &A4( B 4A8 8B ;A8 <B  BB,C#=C ? C# C C#C C##C'*C'c>tt5}td|dddn #1swxYwYdt |jvsJtt5}td|dddn #1swxYwYdt |jvsJtt5}td|dddn #1swxYwYdt |jvsJtt5}td|dddn #1swxYwYdt |jvsJdS)Nrrjz&n_jobs == 0 in Parallel has no meaning?z2.3z$n_jobs could not be converted to int invalid_str)rrZr#_initialize_backendrr)r?rs r/test_invalid_njobsrs   Bw7+++??AAABBBBBBBBBBBBBBB 3s7=7I7I I I I I   DwW---AACCCDDDDDDDDDDDDDDD 3s7=7I7I I I I I   Fww///CCEEEFFFFFFFFFFFFFFF 1S5G5G G G G G   Nw w777KKMMMNNNNNNNNNNNNNNN 1S5G5G G G G G G GsG$AA  A <$B,,B03B0#$DDD $E::E>E>)2gffffff@rGct||}|dksJ|dtdD}tdt |DsJdS)NrjrGc3PK|]!}tt|V"dSr7rmrns r/rpz.test_njobs_converted_to_int..s2 2 21OGFOOA   2 2 2 2 2 2r1rc3BK|]\}}|t|kVdSr7r)r-rors r/rpz.test_njobs_converted_to_int..s099$!QqF1II~999999r1)r#_effective_n_jobsrsr enumerate)r?rkrrs r/test_njobs_converted_to_intrs 000A   A % % % % ! 2 2b 2 2 2 2 2C 99)C..999 9 999 999r1cdttz} t|t|tvsJt|tksJ t|=dS#t|=wxYw)Nz test_backend-)rr r)rr") test_backends r/test_register_parallel_backendr"sy"S\\1L#!,0CDDDx'''' %)<<<<<< \ " " "H\ """""s 8A## A-cJtj}ttksJ t dt ddtt ksJ |t_n#|t_wxYwttksJdS)Nr3T) make_default)rr8rhr9r)r"r!)default_backend_origs r/test_overwrite_default_backendr.s#3  ! !%A%C%C C C C C8!+x /DSWXXXX#%%)999999$8  #7 7777  ! !%A%C%C C C C C C Cs 6A22BzOnly without multiprocessingcpttd5tddtdDdddn #1swxYwYt d5tdtdDddddS#1swxYwYdS)Nz)joblib backend '.*' is not available on.*rr2r>c3PK|]!}tt|V"dSr7rmrns r/rpz2test_backend_no_multiprocessing..>s2 F F!3!3 F F F F F Fr1rqc3PK|]!}tt|V"dSr7rmrns r/rpz2test_backend_no_multiprocessing..Bs288!?76??1%%888888r1)r UserWarningr#rsr(r,r1r/test_backend_no_multiprocessingr;sL {"M N N NGG    F FU1XX F F FFFFGGGGGGGGGGGGGGG  ( ( (99 88uQxx888888999999999999999999s#1AAA//B++B/2B/c||d5tj\}}|dksJtddksJt}|jdksJ|dkr6t |t usJt |jt usJn|dkr6t |tusJt |jtusJn|dkr6t |tusJt |jtusJnJ| dr5t |tusJt |jtusJddddS#1swxYwYdS)Nrqr}r*r2r3test_) rrgr&r#rkrfrrrr! startswithr)r{ backend_nameactive_backend active_n_jobsrs r/check_backend_context_managerrEs a ( ( (;;(0(C(E(E% !!!!""a'''' JJx1}}}} , , ,''+AAAAA ##'====== V # #'';6666 ##{22222 [ ( (''+;;;;; ##'777777  $ $W - - ;''+>>>>> ##':::::#;;;;;;;;;;;;;;;;;;sE E((E,/E,cg|]}d|zS)ztest_backend_%dr,rns r/r0r0[s(Q(Q(Q1):Q)>(Q(Q(Qr1rqc|tvr!|t|ttt ksJt |tt ksJt ddfdtDtt ksJdS)NrGr3rjc3XK|]$}|tt|V%dSr7)r%r)r-rOr{s r/rpz/test_backend_context_manager..msR,, ,.-..w::,,,,,,r1)r"setitemrrhr9rr# all_backends_for_context_manager)rr?r{s `r/test_backend_context_managerr^shHg/BCCC  ! !%A%C%C C C C C!'7333 ! !%A%C%C C C C C,HA{+++,,,,1,,, ! !%A%C%C C C C C C Cr1ceZdZdZddZdS)ParameterizedParallelBackendz9Pretends to run conncurrently while running sequentially.Nc6|td||_dS)Nzparam should not be None)rZparam)rMrs r/rRz%ParameterizedParallelBackend.__init__zs! =788 8 r1r7rSr,r1r/rrws.CCr1rch|tdttt ksJ|ddd5t j\}}t|tusJ|jdksJ|dksJt}|j dksJ|j |usJ|dtdD}dddn #1swxYwY|dtdDksJtt ksJdS)N param_backend*rq)rrkc3PK|]!}tt|V"dSr7r%rrns r/rpz=test_parameterized_backend_context_manager..277MGDMM!$$777777r1rc,g|]}t|Sr,rrns r/r0z>test_parameterized_backend_context_manager..1111tAww111r1) rr"rrhr9rrgrfrr#rkrrs)rr{rrrrs r/*test_parameterized_backend_context_managerrs/3OPPP  ! !%A%C%C C C C C 1 5 5 588(0(C(E(E% N##'CCCCC#r))))!!!! JJx1}}}}z^++++!77eAhh77777888888888888888 11a111 1 1 1 1 ! !%A%C%C C C C C C CsB C((C,/C,c@ttksJ|tdd5tj\}}t |tusJ|jdksJ|dksJt}|jdksJ|j |usJ|dtdD}dddn #1swxYwY|dtdDksJttksJdS)N+)rrr}c3PK|]!}tt|V"dSr7rrns r/rpzFtest_directly_parameterized_backend_context_manager..rr1c,g|]}t|Sr,rrns r/r0zGtest_directly_parameterized_backend_context_manager..rr1) rhr9rrrgrfrr#rkrrs)r{rrrrs r/3test_directly_parameterized_backend_context_managerrs  ! !%A%C%C C C C C -B777 B B B88(0(C(E(E% N##'CCCCC#r))))!!!! JJx1}}}}z^++++!77eAhh77777888888888888888 11a111 1 1 1 1 ! !%A%C%C C C C C C CsB CCCcFtdtjS)Nr)r osgetpidr,r1r/sleep_and_return_pidrs #JJJ 9;;r1cttksJtdksJtddt dDS)NrrGr}c3NK|] }ttV!dSr7)r%rrs r/rpz"get_nested_pids..s4PP!;g&:;;==PPPPPPr1)rhr!r#rrsr,r1r/get_nested_pidsr so  ! !%5 5 5 5 5 :: ' ' ) )Q . . . . 81   PPuQxxPPP P PPr1c"eZdZdZfdZxZS) MyBackendz:Backend to test backward compatibility with older backendsc^tt|dSre)superr get_nested_backend)rM __class__s r/rzMyBackend.get_nested_backends&Y%%88::1==r1)rTrUrVrWr __classcell__)rs@r/r r s>DD>>>>>>>>>r1r back_compat_backend)r3r2r*rc||5tddtdD}|D]$}tt|dksJ% ddddS#1swxYwYdS)NrGr}c3NK|] }ttV!dSr7)r%r rs r/rpz6test_nested_backend_context_manager..s3'V'Vq(@(@(@(B(B'V'V'V'V'V'Vr1rr)r#rsrr)r{r? pid_groups pid_groups r/#test_nested_backend_context_managerrs   ,,'XQ''''V'VERTII'V'V'VVV # , ,Is9~~&&!+++++ ,,,,,,,,,,,,,,,,,,,sAA22A69A6)rGr|Nc*dtdfdtdD|5tdfdtdDddddS#1swxYwYdS)Nctt|ksJt|}t|ksJdSr7)rhr"r&r#r)expected_backend_typeexpected_n_jobs r/check_nested_backendz?test_nested_backend_in_sequential..check_nested_backendsW$%%2G)HHHHH*.99zz++--??????r1rr}c3^K|]'}ttjdV(dSr)r%rr8)r-rrs r/rpz4test_nested_backend_in_sequential..sMGH%$%%h&>BBr1rc3JK|]}tVdSr7r/)r-rr?rrks r/rpz4test_nested_backend_in_sequential..sJ  ?@ )G( ) )'6 : :      r1rr)r?rkr{rs`` @r/!test_nested_backend_in_sequentialrs5@@@HALQRTII  ( ( (        DI"II                        s5BB B c||5}|tur|d}|tur|d}|j|ksJ ddddS#1swxYwYdS)Nr?r)r(r' nesting_level)r{ inner_backendexpected_levelctxr?s r/check_nesting_levelr%s   73 o % %)nG & & &!fG$66666 777777777777777777s0A  AA outer_backendr"cBt|dtd|fdtdDd5tfdtdDddddS#1swxYwYdS)NrrGrjc3VK|]#}ttdV$dSrr%r%r-rr{r"s r/rpz-test_backend_nesting_level..sL..DE$#$$WmQ??......r1rr}c3VK|]#}ttdV$dSrr)r*s r/rpz-test_backend_nesting_level..sL  HI (G' ( (- C C      r1)r%r#rs)r{r&r"s` `r/test_backend_nesting_levelr,s/ 222-HA}---.....INr... q ) ) )        MRSUYY                        s2BBBwith_retrieve_callbackTcddlGfddt}td|d|d5}tdfdt d D|t ur|d jd ksJ|tur|djd ksJddddS#1swxYwYdS) Nrc8eZdZdZZjdZdS))test_retrieval_context..MyBackendrc32K|xjdz c_dVdSr~rrs r/retrieval_contextz;test_retrieval_context..MyBackend.retrieval_contexts" FFaKFF EEEEEr1N)rTrUrVrosupports_retrieve_callbackcontextmanagerr2) contextlibr-sr/r r0s= %;"  "   # "   r1r  retrievalcbtddt|DS)NrGr}c3PK|]!}tt|V"dSr7rrns r/rpz>test_retrieval_context..nested_call..s0!C!CQ+'"++a..!C!C!C!C!C!Cr1rr)ns r/ nested_callz+test_retrieval_context..nested_calls2!xq!!!!C!C%((!C!C!CCCCr1rGr}c3HK|]}t|VdSr7r/)r-ror:s r/rpz)test_retrieval_context..s6EEq/7;//22EEEEEEr1rr?r)r5r!r)r#rsr(ror')r{r-r r$r5r:s ` @@r/test_retrieval_contextr< s\$k9555DDD   !EEEEE!HHEEEEEE o % %y>#q(((( & & &q68q==== !!!!!!!!!!!!!!!!!!sA+B44B8;B8r)rr|gQ?ctt5t|ddddS#1swxYwYdS)Nr)rrZr#r>s r/test_invalid_batch_sizer?)s   ((J''''((((((((((((((((((s 377z)n_tasks, n_jobs, pre_dispatch, batch_size) )rGrGrr)rGrGrkrrrGrkr)irGrkrr@)rrrkr)rrkr)r 2 * n_jobsr)rrr)rDrrCrY)rArrCrc|||d}dt|D}tdi|dt|D}||ksJdS)N)rkrrc,g|]}t|Sr,rrns r/r0z0test_dispatch_race_condition..Ds222aq 222r1c3PK|]!}tt|V"dSr7rmrns r/rpz/test_dispatch_race_condition..Es2 L L!3!3 L L L L L Lr1r,r)rrkrrparamsrrs r/test_dispatch_race_conditionrI/su( J W WF225>>222H h    L LU7^^ L L LLLG h      r1ctj}tdd}|jd}|}||ksJdS)NrGr*rjr{)rget_start_methodr#_backend_kwargsget)mp_start_methodrr{ start_methods r/test_default_mp_contextrPIsa)++O#4555A##I..G++--L ? * * * * * *r1c@|dkrtjd}tjd}|ddtjjtd|fdtdDdS)Nr*r5rirGrjc3hK|],}ttjjV-dSr7)r%rdotT)r-rorNs r/rpzAtest_no_blas_crash_or_freeze_with_subprocesses..hs9'R'RA13(?(?'R'R'R'R'R'Rr1) rr4rrandom RandomStaterandnrSrTr#rs)r?rngrNs @r/.test_no_blas_crash_or_freeze_with_subprocessesrYRs###.)) )   # #C $AF1acNNN(HAw''''R'R'R'Rq'R'R'RRRRRRr1afrom joblib import Parallel, delayed def square(x): return x ** 2 backend = "{}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) c|dkr+tjdkrtjdt|}t tjd|gdddS)Nr*forkzVRequire fork start method to use interactively defined functions with multiprocessing.z-cr\[0, 1, 4, 9, 16\]r= stdout_regex) rrKpytestskip,UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_NO_MAINformatrsys executable)r?codes r/2test_parallel_with_interactively_defined_functionsrf{s ###(;(=(=(G(G  .    8 > >w G GD t$b?Tr1aimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed def run(f, x): return f(x) {define_func} if __name__ == "__main__": backend = "{backend}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) callable_position = "{callable_position}" if callable_position == "delayed": print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) elif callable_position == "args": print(Parallel(n_jobs=2, backend=backend)( delayed(run)(square, i) for i in range(5))) else: print(Parallel(n_jobs=2, backend=backend)( delayed(run)(f=square, x=i) for i in range(5))) z!def square(x): return x ** 2 zcdef gen_square(): def square(x): return x ** 2 return square square = gen_square() zsquare = lambda x: x ** 2 define_funccallable_position)r%rkwargsc |dvr/|tkstjdkrtjdt |||tj tj tj }| d}| |ttj|jgdddS) N)r*r5win32zNot picklable with pickle)rgr?rhjoblib_root_folderzunpicklable_func_script.pyrr\r]) SQUARE_MAINrcplatformr_r`)UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_MAINrbrpathdirnamejoblib__file__rwriterrdstrpath)r?rgrhtmpdirre code_files r/0test_parallel_with_unpicklable_functions_in_argsrxs...{""clg&=&= /000 4 ; ;+7??27??6?+K+KLL <  D  899I OOD *+*r1aimport sys import faulthandler # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed from functools import partial class MyClass: '''Class defined in the __main__ namespace''' def __init__(self, value): self.value = value def square(x, ignored=None, ignored2=None): '''Function defined in the __main__ namespace''' return x.value ** 2 square2 = partial(square, ignored2='something') # Here, we do not need the `if __name__ == "__main__":` safeguard when # using the default `loky` backend (even on Windows). # To make debugging easier faulthandler.dump_traceback_later(30, exit=True) # The following baroque function call is meant to check that joblib # introspection rightfully uses cloudpickle instead of the (faster) pickle # module of the standard library when necessary. In particular cloudpickle is # necessary for functions and instances of classes interactively defined in the # __main__ module. print(Parallel(backend="loky", n_jobs=2)( delayed(square2)(MyClass(i), ignored=[dict(a=MyClass(1))]) for i in range(5) )) )rlc|d}|tttj|jgdddS)Nz(joblib_interactively_defined_function.pyr\)r^r=)rrt5INTERACTIVE_DEFINED_FUNCTION_AND_CLASS_SCRIPT_CONTENTrrcrdrurvscripts r/7test_parallel_with_interactively_defined_functions_lokyr}s\ [[C D DF LLFGGG (*r1aimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed, hash import multiprocessing as mp mp.util.log_to_stderr(5) class MyList(list): '''MyList is interactively defined by MyList.append is a built-in''' def __hash__(self): # XXX: workaround limitation in cloudpickle return hash(self).__hash__() l = MyList() print(Parallel(backend="loky", n_jobs=2)( delayed(l.append)(i) for i in range(3) )) c|d}|tttj|jgddddS)Nz)joblib_interactive_bound_method_script.pyz\[None, None, None\] LokyProcess)r^ stderr_regexr=)rrt9INTERACTIVELY_DEFINED_SUBCLASS_WITH_METHOD_SCRIPT_CONTENTrrcrdrur{s r/:test_parallel_with_interactively_defined_bound_method_lokyr's] [[D E EF LLJKKK (,# r1cbtg}td|gksJdS)NrGr})iterr#)exhausted_iterators r/%test_parallel_with_exhausted_iteratorr3s;b 81   0 1 1R 7 7 7 7 7 7r1ct|tjstdt ||S)Nz#Expected np.memmap instance, got %r)rrmemmap TypeErrorrfcopy)rNs r/ check_memmapr8s; a # #H=tAwwGGG 6688Or1cd}tdd|d|dD}t||t|D]%\}}tj||&tdd|d|dD}t||t|D]%\}}tj||&dS) Nc3xKt|D]'}tjdtj|zV(dS)Nrr*)rsronesfloat32)r9ros r/generate_arrayszBtest_auto_memmap_on_arrays_from_generator..generate_arraysFsJq 4 4A'"BJ///!3 3 3 3 3 4 4r1rGrrkr&r?c3PK|]!}tt|V"dSr7r%rr-rNs r/rpz.LN@@%& a  @@@@@@r1rrc3PK|]!}tt|V"dSr7rrs r/rpz.Urr1)r#rbrrr3r4)r?rrr6rs r/)test_auto_memmap_on_arrays_from_generatorr>sM444 @haAw???@@*9/#*>*>@@@G W)F)FGG88 %%h7777 @haAw???@@*9/#*>*>@@@G W)F)FGG88 %%h777788r1c|Sr7r,)args r/identityr\s Jr1c|dj}tj}t j|dt j|dg}t||t|dtdfdd D\}td tj sJd j |ksJtj ||dS) Nz test.mmapuint8rr) mmap_moderGr}c3RK|]!}ttV"dSr7)r%r)r-rrs r/rpz.test_memmap_with_big_offset..hs6"J"J#478#4#4V#<#<"J"J"J"J"J"Jr1rtr)rrummapALLOCATIONGRANULARITYrzerosrr r r#rroffsetr3r4)rvfnamesizeobjr6rs @r/test_memmap_with_big_offsetr`s KK $ $ ,E  %D 8D ( ( ("'$g*F*F*F GCe %3 ' ' 'F"""""J"J"J"Jc"J"J"JJJIV fQi + +++ + !9 d " " " "J!!#v.....r1cdtj5}tdddtdDdddn #1swxYwYt |dksJ|d}t |jtsJt|jdksJdS)Nr)rkr=c3PK|]!}tt|V"dSr7rmrns r/rpzFtest_warning_about_timeout_not_supported_by_backend..ps2%L%LQogfooa&8&8%L%L%L%L%L%Lr12rzThe backend class 'SequentialBackend' does not support timeout. You have set 'timeout=1' in Parallel but the 'timeout' parameter will not be used.) r_rr#rsrrrrr)rrs r/3test_warning_about_timeout_not_supported_by_backendrns M8%1%%%%L%L%))%L%L%LLLLMMMMMMMMMMMMMMM x==A     A ai - --- - qy>>       s2AAAc|||<|Sr7r,) input_listrrs r/set_list_valuer{sJu Lr1)rrGrcdgdzt|ddfdtdD}t|}tdt |DsJdS)NrrrGr3rkrFr?c3VK|]#}tt||V$dSr7)r%r)r-rors r/rpzPtest_parallel_return_order_with_return_as_generator_parameter..sSQQ67 Aq11QQQQQQr1c3(K|] \}}||kVdSr7r,r-vrs r/rpzPtest_parallel_return_order_with_return_as_generator_parameter..s*::$!QqAv::::::r1)r#rsrLrrb)rkr6rs @r/=test_parallel_return_order_with_return_as_generator_parameterrs qJ PXV{K P P PQQQQ;@88QQQF &\\F ::#j&"9"9::: : ::: :::r1cB|rtdt|S)Nr;)r r)edelays r/_sqrt_with_delayrs   b 77Nr1cRt|d|dtdDtfdtdD}dgttddz}t d t ||DsJdS) NrHrc3`K|])}tt|dz|dkV*dS)rGrN)r%rrns r/rpzK_test_parallel_unordered_generator_returns_fastest_first..sYWW67! !!!Q$a11WWWWWWr1rc36K|]}tVdSr7)next)r-rr6s r/rpzK_test_parallel_unordered_generator_returns_fastest_first..s)==qd6ll======r1 rrGc3(K|] \}}||kVdSr7r,rs r/rpzK_test_parallel_unordered_generator_returns_fastest_first..s*SS$!QqAvSSSSSSr1)r#rssortedrLrrb)r?rkquickly_returnedexpected_quickly_returnedr6s @r/8_test_parallel_unordered_generator_returns_fastest_firstrsWXV/Dg V V VWW;@99WWWF====E!HH=====!"d5B<<&8&8 8 SS#&?AQ"R"RSSS S SSS Sr1c&t||dSr7)rr?rks r/7test_parallel_unordered_generator_returns_fastest_firstrs=WfMMMMMr1)rrGrr|cdgdgdzz}tt5tj}t||d|Ddddn #1swxYwYtj|z }|dksJdS)NrNrrrjc3ZK|]&}ttj|V'dSr7r%r^r rns r/rpz%test_abort_backend..s70X0XA1D1D1DQ1G1G0X0X0X0X0X0Xr1rk)rrr^r#)rkr?delayst_startdts r/test_abort_backendrsUbTCZ F   YY)++00000X0XQW0X0X0XXXXYYYYYYYYYYYYYYY w B 777777s8A##A'*A'c`tjtdt}d|d<|S)NgArFr)rrintbool)rr6s r/get_large_objectrs* WS\\ . . .FF1I Mr1ct|||5}|dtdD}t|t|~ddddS#1swxYwYdS)N)rkr?rFc3PK|]!}tt|V"dSr7)r%rrns r/rpz0_test_deadlock_with_generator..s6JJ13'"233A66JJJJJJr1r)r#rsr)r?rFrkrr6s r/_test_deadlock_with_generatorrs I F F F(JJb JJJJJ V  V  sAA""A&)A&c(t|||dSr7)r)r?rFrks r/test_deadlock_with_generatorrs "'9f=====r1c~ttd5t|||}|dtdD}t j}|dtdD}dddn #1swxYwYt j|z dks Jd ~dS) N)This Parallel instance is already runningrrMc3PK|]!}ttdV"dSrrrs r/rpz/test_multiple_generator_call..s2::1^WU^^A&&::::::r1rc3PK|]!}tt|V"dSr7rrns r/rpz/test_multiple_generator_call..s0;;1  A;;;;;;r1rrG[The error should be raised immediately when submitting a new task but it took more than 2s.)rrr#rsr^)r?rFrkrgrgen2s r/test_multiple_generator_callrs  $O P P P<<FGyIII H::b ::: : :)++x;;c ;;;;; <<<<<<<<<<<<<<< 9;; 1 $ $ $ $ % $ $ sA*B  BBct|||5}|dtdD}tj}ttd5|dtdD}dddn #1swxYwYtj|z dks Jd  dddn #1swxYwY~dS) NrMc3PK|]!}ttdV"dSrBrrs r/rpz7test_multiple_generator_call_managed..s2;;A^WU^^B'';;;;;;r1rrrc3PK|]!}tt|V"dSr7rrns r/rpz7test_multiple_generator_call_managed..s0==Q+'"++a..======r1rrGr)r#rsr^rr)r?rFrkrrrg2s r/$test_multiple_generator_call_managedrs^ &'Y ? ? ?  8 H;;r;;; ; ;)++ L(S T T T > >==%**=====B > > > > > > > > > > > > > > >y{{W$q((( ()(((                 s6A C #B C B C B 'C  CC return_as_1 return_as_2c t|||dtdD}t|||dtddD}|dkrt|}|dkrt|}tdt |tdDsJtdt |tddDsJdS) NrMc3VK|]$}tt|dzV%dSrGNrrns r/rpz9test_multiple_generator_call_separated..sOAA !  adAAAAAAr1rc3VK|]$}tt|dzV%dSrrrns r/rpz9test_multiple_generator_call_separated..sOBB !  adBBBBBBr1rkrHc3(K|] \}}||kVdSr7r,r-rros r/rpz9test_multiple_generator_call_separated..s*88FCsax888888r1c3(K|] \}}||kVdSr7r,rs r/rpz9test_multiple_generator_call_separated..s*==FCsax======r1)r#rsrrrb)r?rrrkrrs r/&test_multiple_generator_call_separatedrs6 AK@@@AA%*2YYAAA  A B&'[ A A ABB%*2r]]BBB  B+++ 1II+++ BZZ 88c!U2YY&7&7888 8 888 8 ==c"eBmm&<&<=== = === ===r1zbackend, error))r2T)r3F)rEFc |dkrttjdtd||}|dt dD}t j|d}|rttd n t}|5tj }td||d t dd D}|d krt|}td t|t dd DsJ dddn #1swxYwYtj |z dksJd} |jr,| dkr&| dz } tjd|jr| dk&tj |z dksJ|jdkr |jsJdSdS)Nr2zRequires multiprocessingrGrMc3PK|]!}ttdV"dSrBrrns r/rpz.0s277##777777r1rc tdS)NzGenerator collected)rr,r1r/z;test_multiple_generator_call_separated_gc..1su-B'C'Cr1z The executor underlying Parallelrc3VK|]$}tt|dzV%dSrrrns r/rpz.=sO@ @ $%MGDMM!Q$  @ @ @ @ @ @ r1rkrHc3(K|] \}}||kVdSr7r,rs r/rpz.Ds*@@Q3!8@@@@@@r1rrrqrr)rr_r`r#rsweakreffinalizerrrr^rrrbaliver rrf) r?rrerrorrrg_wrr$rretrys r/)test_multiple_generator_call_separated_gcrsh 6  .///7kBBBH77U2YY77777A  ACC D DD  |#EFFFF ]] A A )++ ?HQ; ? ? ?@ @ ).r2@ @ @    / / /q A@@#ar2*?*?@@@@@@@@@ A A A A A A A A A A A A A A A 9;; 1 $ $ $ $ E *   3 * 9;; 1 $ $ $ $!x!##q((!!!!)( "!sBD44D8;D8c|j}tdd||5}|dtjdgdzDt t j|dksJ dddn #1swxYwYtdD]'}t j|sntd(td tdd| }|d tjdgdzDtdD](}t j|sdStd)td ) NrGr)rkr&r? temp_folderc3PK|]!}tt|V"dSr7rrs r/rpz(test_memmapping_leaks..as5 G Gq ',   " " G G G G G Gr1rrrrz/temporary directory of Parallel was not removedrc3PK|]!}tt|V"dSr7rrs r/rpz(test_memmapping_leaks..qs5CC1glACCCCCCr1) rur#rrUrrlistdirrsr AssertionError)r?rvrrs r/test_memmapping_leaksrUs^F q'v N N N+RS  G GRY-=-=b-A-A,BQ,F G G GGGG2:f%%&&***** +++++++++++++++3ZZPPz&!!  E c NOOO a999AACC)9)9")=)=(>(BCCCCCC 3ZZPPz&!!  EE c NOOOsABB  B )Nr2r3ctd|dtdD}|dtdDksJdS)NrGrjc3HK|]}td|VdS)c |dzSrFr,rHs r/rz2test_lambda_expression...s !Q$r1Nr/rns r/rpz)test_lambda_expression..sH22'(""222222r1rcg|]}|dzSrGr,rns r/r0z*test_lambda_expression..s///q!t///r1rr)r?rs r/test_lambda_expressionr{sr 2ha11122,1"II222G //U2YY/// / / / / / /r1cd}d}d|z td||}|fdt|D|jj|jjksJ|jj|jjksJ|fdt|D|jj|jjksJ|jj|jjksJdS) zCTest that a parallel backend correctly resets its batch statistics.rGig@r)rrkr?c3\K|]&}ttjV'dSr7rr-ro task_times r/rpz6test_backend_batch_statistics_reset..8>>gdj)$$>>>>>>r1c3\K|]&}ttjV'dSr7rr s r/rpz6test_backend_batch_statistics_reset..r r1N)r#rsr_effective_batch_size_DEFAULT_EFFECTIVE_BATCH_SIZE_smoothed_batch_duration _DEFAULT_SMOOTHED_BATCH_DURATION)r?rkn_inputsrr s @r/#test_backend_batch_statistics_resetrs FHhIFG<<>>>eHoo>>>>>> : +qz/W W W W W + : 6 7 7 7 7A>>>>eHoo>>>>>> : +qz/W W W W W + : 6 7 7 7 7 7 7r1cdD]}tt|jtusJt|d}t|jtusJt|d}t|jt usJt|d}t|jtusJtdd d }t|jt usJ|d d5td }t|jt usJ|jdksJ dddn #1swxYwY|d d5td d}t|jt usJ|jd ksJ dddn #1swxYwY|d d5td }t|jtusJ|jdksJ dddn #1swxYwY|d d5td d}t|jtusJ|jd ksJ ddddS#1swxYwYdS)N)rrGr|r}threadsrkprefer processes sharedmemrkrequirerGr2)rkr?rrrqrr)rfr#rr9r!rrk)r{rkrs r/$test_backend_hinting_and_constraintsrs 4 4HF+++4559U9W9WWWWW F9 5 5 5AJ#33333 F; 7 7 7AJ;.... FK 8 8 8AJ#333333 6)<<;EE  E .s0001000r1rrc3PK|]!}tt|V"dSr7rmrns r/rpz6test_backend_hinting_always_running..sC55 555555r1)rrkc3PK|]!}tt|V"dSr7rmrns r/rpz6test_backend_hinting_always_running..s2CCA_WV__Q//CCCCCCr1)rsr#r()rkrexpected_resultsrs r/#test_backend_hinting_always_runningr%s+ 10eBii0004hfV44455$)"II555G & & & & & v 6 6 6DD(**CCrCCCCCDDDDDDDDDDDDDDD & & & & & & &s$/BB#&B#cGddt}||5tdd}t|j|usJtdd}t|j|usJ dddn #1swxYwYGdd t}||5tdd}t|j|usJ|\}}|d ksJ|d ksJtddd }t|jt usJ|\}}d }||ksJ|d ksJ dddn #1swxYwYtt5t|dddddS#1swxYwYdS)Nc"eZdZdZdZdZdZdS)[test_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackendTcdSr7r,rs r/ apply_asynczgtest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.apply_async Dr1c|Sr7r,rs r/r&zltest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.effective_n_jobsMr1NrTrUrVsupports_sharedmem use_threadsr*r&r,r1r/MyCustomThreadingBackendr(s=!         r1r1rGrrrrc"eZdZdZdZdZdZdS)\test_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackendFcdSr7r,rs r/r*zhtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.apply_asyncr+r1c|Sr7r,rs r/r&zmtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.effective_n_jobsr-r1Nr.r,r1r/MyCustomProcessingBackendr3s="         r1r6rr)rkrrzUsing ThreadingBackend as joblib backend instead of MyCustomProcessingBackend as the latter does not provide shared memory semantics.r?r) rr#rfr readouterrr!striprrZ)capsysr{r1rr6outerrrs r/9test_backend_hinting_and_constraints_with_custom_backendsr=s#6 ))++ , ,<< Ak 2 2 2AJ#;;;;; A{ 3 3 3AJ#;;;;;; <<<<<<<<<<<<<<<$7 **,, - - Ak 2 2 2AJ#<<<<<$$&&Sbyyyybyyyy A{B ? ? ?AJ#33333$$&&S 8  yy{{h&&&&byyyyy%(   KK2244kJJJJKKKKKKKKKKKKKKKKKKs7ABB B 8B;FFFGG  G cxtt5tddddn #1swxYwYtt5tddddn #1swxYwYtt5tdddddn #1swxYwYt}tt5tdddddn #1swxYwYtt5td dddddS#1swxYwYdSdS) Ninvalidrrrr)rrr2r7r*)rrZr#rr,r1r/,test_invalid_backend_hinting_and_constraintsr@s   ## """"###############   $$####$$$$$$$$$$$$$$$   ::  [9999:::::::::::::::  ~J   : : V[ 9 9 9 9 : : : : : : : : : : : : : : : J   E E . D D D D E E E E E E E E E E E E E E E E E E ~sT266A..A25A2 B++B/2B/C//C36C3D--D14D1c .td5}t|jj|jjfg}dkr|cdddS|fdt dD}||dzcdddS#1swxYwYdS)zCPerform nested parallel calls and introspect the backend on the wayrGr}rNc3VK|]#}ttdddz iV$dS)limitrNr,)r%_recursive_backend_info)r-rorirCs r/rpz*_recursive_backend_info..2sY   -G+ , , G G519 G G G      r1r)r#rfrrTr!rs)rCrir this_levelrs`` r/rDrD+s    'qAJ''0!*2JKL A::''''''''!     1XX     GAJ&''''''''''''''''''s.B /B  BBc||d5t}dddn #1swxYwY|dz}|dfdddg}||ksJdS)NrGr}Backendr)r!r)r rG)rDtitle)r{r?backend_types_and_levelstop_level_backend_typeexpected_types_and_levelss r/test_nested_parallelism_limitrL9s  # # #==#:#<#< ===============%]]__y8 #  ! $'@ @ @ @ @ @ @s )--c^tdtdDS)z6A horrible function that does recursive parallel callsc3NK|] }ttV!dSr7)r%_recursive_parallelrns r/rpz&_recursive_parallel..Ls4GG2g12244GGGGGGr1rGrr) nesting_limits r/rOrOJs+ 8::GGeAhhGGG G GGr1cn||d5tt5}tdddn #1swxYwYdddn #1swxYwY|j}|dkr1ddlm}t ||tfrtj dt |tsJdS)NrGr}r2r)TerminatedWorkerErrorz1Loky worker crash when serializing RecursionError) r BaseExceptionrOr&joblib.externals.loky.process_executorrRrrr_xfailRecursionError)r{r?rexcrRs r/test_thread_bomb_mitigationrXOsK  # # #"" M " " "g  ! ! ! " " " " " " " " " " " " " " """""""""""""""" -C& QPPPPP c1=A B B N LL M M M c> * *** ***s1A> AA AA AAAcvi}dD]$}tj|||<%|tdfS)N)OMP_NUM_THREADSOPENBLAS_NUM_THREADSMKL_NUM_THREADSVECLIB_MAXIMUM_THREADSNUMEXPR_NUM_THREADSNUMBA_NUM_THREADS ENABLE_IPCr)renvironrMr)env_varsvars r/_run_parallel_sumrdksFH , , s++ \#&& &&r1zNeed OpenMP helper compiledcptd|dtdD}ttdzd}|D]b\}}||ksJ|D]@\}}|dr|t |ksJ0|dksJ|dksJAcdS)NrGrjc3NK|] }ttV!dSr7)r%rdrs r/rpz-test_parallel_thread_limit..}sE22)*"!""$$222222r1r_THREADSr`1)r#rsrr$itemsendswithr)r?rexpected_num_threadsworker_env_varsomp_num_threadsrrs r/test_parallel_thread_limitrnzs2ha11122.3Ahh222Gy{{a/33,3$$("66666*0022 $ $KD%}}Z(( $$8 9 999999|++++|||||  $$$r1zThis test requires daskczttd5|dddddS#1swxYwYdS)NzPlease install daskrdask)rrZ)r{s r/)test_dask_backend_when_dask_not_installedrqs  "7 8 8 8s 044cNGddt}d}||5tjt|5t ddt dDdddn #1swxYwYddddS#1swxYwYdS)Nc"eZdZdZddZdZdS)3test_zero_worker_backend..ZeroWorkerBackendcdSrer,)rMrris r/rz=test_zero_worker_backend..ZeroWorkerBackend.configure1r1Nc td)NzNo worker availabler)rMfunccallbacks r/r*z?test_zero_worker_backend..ZeroWorkerBackend.apply_asyncs455 5r1cdSrer,rs r/r&zDtest_zero_worker_backend..ZeroWorkerBackend.effective_n_jobsrvr1r7)rTrUrVrr*r&r,r1r/ZeroWorkerBackendrtsF    6 6 6 6     r1r{z&ZeroWorkerBackend has no active workerrrGr}c3PK|]!}tt|V"dSr7rrns r/rpz+test_zero_worker_backend..s0@@!{wr{{1~~@@@@@@r1)r!r_rrr#rs)r{r{ expected_msgs r/test_zero_worker_backendr~sq,.check_globalss!!r1rGr}c3FK|]}tVdSr7r/r-rors r/rpz.E11%&   111111r1z changed valuec3FK|]}tVdSr7r/rs r/rpz.rr1)rr#rsr)workers_global_variablers @r/)test_globals_update_at_each_parallel_callrs* *""" =??. . . . .0ha0001111*/((111 & ' ',<+= = = = =) =??o - - - -0ha0001111*/((111 & ' 'O+< < < < < < >|LL  r1cv|D]5}|D]0}t||}t|||h}|d|vsJ16dS)Nr)rmin) workers_inforrchild_threadpool_inforparent_max_num_threadsrs r/check_child_num_threadsrsw".;;1 ; ;L%@k&& "K)?@@+NH .(:::::  ;;;r1)rGrrr|c|t}t|dkrtjdt d|dt dD}t |}|dkr|dd }ntt|zd}t|||dS) Nr&Need a version of numpy linked to BLASrr2rc3NK|] }ttV!dSr7r%rrns r/rpz;test_threadpool_limitation_in_child_loky..sMGG67/.//11GGGGGGr1rGrr rrr_r`r#rsr&rr$r)rkrworkers_threadpool_infosexpected_child_num_threadss r/(test_threadpool_limitation_in_child_lokyrs122K ;1 CDDDDFxvFFFGG;@88GGG  f % %F {{%0^M%B""%()>%B%B" +/Ir1inner_max_num_threads)rrGrNct}t|dkrtjd|d|5t |dt dD}dddn #1swxYwYt |}|d kr|dd }n$| tt|zd }n|}t|||dS) Nrrrr2rr}c3NK|] }ttV!dSr7rrns r/rpz>test_threadpool_limitation_in_child_context..sE; ; :; 3G2 3 3 5 5; ; ; ; ; ; r1rGrrr)r{rkrrrrs r/+test_threadpool_limitation_in_child_contextrsV122K ;1 CDDDD /D E E E  #:86#:#:#:; ; ?DQxx; ; ; $ $                 f % %F {{%0^M%B""  &%()>%B%B""%:" +/Is1BBBvar_name)r[r\rZct|dkrtjdtddt j} dt j<t|fdtd D}|ddgksJ|d d 5t|fd td D}dddn #1swxYwY|d d gksJ |t j=dS|t j<dS#|t j=n|t j<wxYw)NrzSkip test when n_jobs == 1Treusec@tj|Sr7)rrarM)rs r/_get_envz>test_threadpool_limitation_in_child_override.._get_env7sz~~h'''r14r}c3HK|]}tVdSr7r/r-rorrs r/rpz?test_threadpool_limitation_in_child_override..>s6)X)X!*;'(*;*;H*E*E)X)X)X)X)X)Xr1rGr2rc3HK|]}tVdSr7r/rs r/rpz?test_threadpool_limitation_in_child_override..BsG..01!!!(++......r1rh) r&r_r`rshutdownrrarMr#rs)r{rkroriginal_var_valuerrs ` @r/,test_threadpool_limitation_in_child_overrider's1$$ 0111%%%..000(((116" 8)(&))))X)X)X)X)XuUVxx)X)X)XXX3*$$$$ WV1 5 5 5  -hf---.....5:1XX...G               3*$$$$$  % 8$$$#5BJx   % 8$$#5BJx 5 5 5 5s1/AE4D< ED  ED  E!E!)rGrr|cd}||td}tdD]%}||td}||ksJ&dS)Nctdt|fdtdDdS)Nrr}c3RK|]!}ttV"dSr7)r%sum)r-rorBs r/rpzAtest_loky_reuse_workers..parallel_call..Vs1CCA  QCCCCCCr1r)rkrBs @r/ parallel_callz.test_loky_reuse_workers..parallel_callTsJ "IICCCCrCCCCCCCCr1Trr)rrs)rkrfirst_executorrrs r/test_loky_reuse_workersrNs DDD M&*666N2YY** f(t444>)))))**r1c2d|tj<dS)N initialized)rr)statuss r/_set_initializedrds'F29;;r1cLtj}||d}|dvs Jd||sdSd||<tjdz}t d|D}tj|krb||kr\tjdt d|D}tj|kr||k\tj|krtd|S) N)rstartedz2worker should have been in initialized state, got rr;c$g|] \}}|dk |Srr,r-pidrs r/r0z!_check_status..ts!HHHVS!iSr1rc$g|] \}}|dk |Srr,rs r/r0z!_check_status..ws!LLLaQ)^^^^^r1z-Waited more than 30s to start all the workers)rrrMr^rrir r)rrk wait_workersrstatedeadline n_starteds r/ _check_statusrhs! )++C JJsD ! !E . . . .DUDD / . . F3Ky{{RHHHv||~~HHHIII )++ Y%7%7 3LL6<<>>LLLMM  )++ Y%7%7 y{{hJKKK Jr1ctj}|||tf5t fdt dDddddS#1swxYwYdS)Nr?rk initializerinitargsc3TK|]"}ttV#dSr7r%rr-rorkrs r/rpz+test_initializer_context..s8NNa)7=))&&99NNNNNNr1r)rrdictrr#rs)rkr?r{rrs` @r/test_initializer_contextrs jllG \\^^F $    OO  NNNNN5::NNNNNNOOOOOOOOOOOOOOOOOOs2A>>BBctj}|t|tffdt dDdS)Nrc3TK|]"}ttV#dSr7rrs r/rpz,test_initializer_parallel..s8AAgmVV,,AAAAAAr1r)rrrr#rrs)rkr?rrs` @r/test_initializer_parallelrsjllG \\^^FH$  BAAAAeCjjAAA BBBBBr1cd}tj}|t}t |D]`}t dt ffdt D}|t|}at|ks JddS)Nrqr2rc3XK|]$}ttdV%dST)rNrrs r/rpz*test_initializer_reused..P   #GM " "66 E E E      r1z=The workers should be reused when the initializer is the same) rrrrrsr#runionrrk n_repetitionsrpidsrorrs` @r/test_initializer_reusedrs MjllG \\^^F 55D = ! ! ( ( ((Y         6]]     zz#g,,'' t99   G     r1cd}tj}t}t|D]t}|t dt ffdtD}|t|}ut||zks JddS)Nrqr2rc3XK|]$}ttdV%dSrrrs r/rpz.test_initializer_not_reused..rr1zFThe workers should not be reused when the initializer arguments change) rrrrsrr#rrrrs` @r/test_initializer_not_reusedrs MjllG 55D = ! ! ( ( ((Y         6]]     zz#g,,'' t99 . . . .P / . . . .r1)F)rr)rqr7)rWrrrrcr3r^rrr5rmathrr*rpicklerr tracebackr uuidr r_rrr r rjoblib._multiprocessing_helpersrjoblib.test.commonrrrrjoblib.testingrrrrrjoblib.externals.lokyrrwrposix ImportError _openmp_test_helper.parallel_sumr distributedjoblib._parallel_backendsrrrr r!joblib.parallelr"r#r$r%r&r'r(r)mark thread_unsafe pytestmarkrRETURN_GENERATOR_BACKENDSpoprkeysALL_VALID_BACKENDSPROCESS_BACKENDSPARALLEL_BACKENDShasattrrr4r9r@rDrI ExceptionrKr\r`rcrhrurwrzrrrrrrrrrrrrrrr!r9r?rCrrNrhruryrrrrrrrrUrrrrrrrrrrextendrsrrrrrr _parallel_backendsr rrr%r,r<r?rIrPrYrarfrorm SQUARE_LOCAL SQUARE_LAMBDArxrbrprqrsrzr}rrrrrrrrrrrrrrrrrrrrrrrrrr%r=r@rDrLrOno_coverrXrdrnrqr~rrrrrrrrrrrrrrr,r1r/rs  """"""(((((( &&&&&& ''''''''''...... UTTTTTTTTTTTTT>;;;;;;LLLL EEE>>>>>>>LLLKKK                      [ & *HMOO/666Vff]X]__555KK(KKKK:)62$ }4 72}7nbnW55666...III             222""" Y*:;<<% b"""--- . :000 .. =<. Y*++ X~~~&& Y %%,,&%'&,+,  Y*++ X1v,+( Y*++'',+'T0 +    2 Y@@@AA  BA QQQ _h''  x(()((' WWW  Xyyy!!"! Y)**//+*/:       [///** \At9%%55&%+* 5$ Y)**  +*  Y)**  +*  Y566,GHH [;(=>??@?IH$ Y())C C *)C L Y!!EE"!E,''' Y!!       $     %$''P  Q''"!R   Y)**+*<444 Y())33*)3   HHHMMM Y)**CDD--ED+*-,     +       Y*++HH,+H$ Y)** X}}}%%::&%+*:### D D D$=>>>99?>9;;;*$5QQQ#7 ''(Q(Qa(Q(Q(QRRR Y899 Y)?;<<DD=<:9D,#4 Y*:;<<DD=<D& Y*:;<<DD=<D( QQQ>>>>>)5>>>/;;; YWWWXX Y*:;<<,,=<YX, X}}}%% Y)** Y*:;<<  =<+*&% 0777 _/00 _/00 Y*:;<<   =<1010   Y*:;<<  %e}55!!65=<!: \===))((*)(  /    ! +++  Y())SS*) S, 0,  Y())  *) -)@    Y("*BB7)LMM ][, FGG  "?"?"?@@A@HGNM,(9P Fbgoobgoofo.N.NOOFPPQ6V   =, Fbgoobgoofo.N.NOOFPP-:2888    Y())88*) 86  / /  /    999-- ; ;.- ;"Aq6** Y566,GHHNNIH +* N Y*++ X~~~&&'&,+  Y122 [;(=>?? X~~~&&>>'&@?32 > Y122 [;(=>?? X~~~&&  '&@?32 & Y122 [;(=>?? X~~~&&  '&@?32 & Y122 ][*?@AA ][*?@AA X~~~&&>>'&BABA32>&  ][*?@AA ][*?@AA)")"BABA)"X  Y())PP*) PD  rz{##7R7R7R000 Y())*), Y*:;<<++=<+\ X1v X55566 ' '76 ' Y*:;<<3K3K=<3KlEEE* ' ' ' ' Y-.. Y*:;<< A A=</. AHHHH  Y*:;<< Y"*+6;:OQQ++RQ=<+2 ' ' ' Y2>$vGG %BCCC $ $DCHG $ Y*:;<< 4(ABBBCB=<  Y*:;<<AA=<A&===H + + + ; ; ;  X~~~&&'& 0   $ooo66 X2w Y*:;<<=< 76  6 X2w ZWWWXX Y*:;<< 6 6=<YX  6F Xzzz""**#"*((((. X1v Y()) Y*:;<< O O=<*) O X1v Y()) B B*) BAq6**+*0Aq6**+*s6BBB!B((B21B26B;;CC