o
    Df                    @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
mZ d dlmZ ddlmZmZ ddlmZ ddlmZmZmZmZmZ ddlmZmZmZ dd	lmZ dd
lmZ ddl m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z' d dlm(Z( d dl)m*Z*m+Z+ ddlm,Z,m-Z-m.Z. dZ/ej0d  dkZ1e2 dkrddl3m4Z5 eZ6n	d dlm7Z5 ej6Z6zej8Z8W n e9y   dZ8Y nw ej0dkrej:Z;nej;Z;d Z<dZ=dZ>d Z?dZ@dZAdZBdZCd ZDdZEdZFeGeddZHdZIeGedd ZDdZJdZKeL ZMejNZNdd ZOd d! ZPd"d# ZQd$d% ZRdHd&d'ZSG d(d) d)e;ZTG d*d+ d+eUZVG d,d- d-eUZWd.d/ ZXG d0d1 d1ZYG d2d3 d3eZZG d4d5 d5eZZ[G d6d7 d7eZZ\G d8d9 d9eZZ]G d:d; d;eZZ^G d<d= d=Z_G d>d? d?Z`G d@dA dAe`ZaG dBdC dCZbG dDdE dEebZcG dFdG dGe_ZddS )I    N)deque)partial   )	cpu_countget_context)util)TERM_SIGNALhuman_statuspickle_loadsreset_signalsrestart_state)	get_errnomem_rsssend_offset)ExceptionInfo)DummyProcess)CoroStopRestartFreqExceededSoftTimeLimitExceeded
TerminatedTimeLimitExceededTimeoutErrorWorkerLostError	monotonicQueueEmpty)FinalizedebugwarningzEchild process exiting after exceeding memory limit ({0}KiB / {1}KiB)
   Windows)kill_processtree)killg    _B)r!   r!            SIGUSR1g      $@EX_OKi,  皙?c                 C   s<   z| j }W n ty   d }Y nw |d u rtt |  S |S N)r   AttributeErrorr   fileno)
connectionnative r0   F/home/ubuntu/webapp/venv/lib/python3.10/site-packages/billiard/pool.py_get_send_offsetx   s   
r2   c                 C   s   t t|  S r+   )listmapargsr0   r0   r1   mapstar      r7   c                 C   s   t t| d | d S )Nr   r   )r3   	itertoolsstarmapr5   r0   r0   r1   starmapstar   s   r;   c                 O   s    t  j| g|R i | d S r+   )r   
get_loggererror)msgr6   kwargsr0   r0   r1   r=      s    r=   c                 C   s   | t  ur| | d S d S r+   )	threadingcurrent_threadstop)threadtimeoutr0   r0   r1   stop_if_not_current   s   rE   c                   @   sd   e Zd ZdZdd ZerdddZdd	 Zd
d Zdd Z	dS dddZdd	 Zdd Zdd Z	dS )LaxBoundedSemaphorez^Semaphore that checks that # release is <= # acquires,
    but ignores if # releases >= value.c                 C   s   |  j d8  _ |   d S Nr   )_initial_valueacquireselfr0   r0   r1   shrink   s   zLaxBoundedSemaphore.shrinkr   Nc                 C   s   t | | || _d S r+   
_Semaphore__init__rH   rK   valueverboser0   r0   r1   rO      s   
zLaxBoundedSemaphore.__init__c                 C   sR   | j  |  jd7  _|  jd7  _| j   W d    d S 1 s"w   Y  d S rG   )_condrH   _valuenotifyrJ   r0   r0   r1   grow   s
   "zLaxBoundedSemaphore.growc                 C   b   | j }|" | j| jk r|  jd7  _|  W d    d S W d    d S 1 s*w   Y  d S rG   )rS   rT   rH   
notify_allrK   condr0   r0   r1   release      
"zLaxBoundedSemaphore.releasec                 C   *   | j | jk rt|  | j | jk sd S d S r+   )rT   rH   rN   r[   rJ   r0   r0   r1   clear      
zLaxBoundedSemaphore.clearc                 C   s   t | || || _d S r+   rM   rP   r0   r0   r1   rO      s   
c                 C   sT   | j }| |  jd7  _|  jd7  _|  W d    d S 1 s#w   Y  d S rG   )_Semaphore__condrH   _Semaphore__valuerU   rY   r0   r0   r1   rV      s   
"c                 C   rW   rG   )r`   ra   rH   	notifyAllrY   r0   r0   r1   r[      r\   c                 C   r]   r+   )ra   rH   rN   r[   rJ   r0   r0   r1   r^      r_   r   N)
__name__
__module____qualname____doc__rL   PY3rO   rV   r[   r^   r0   r0   r0   r1   rF      s    

rF   c                       s0   e Zd ZdZ fddZdd Zdd Z  ZS )MaybeEncodingErrorzVWraps possible unpickleable errors, so they can be
    safely sent through the socket.c                    s*   t || _t || _t | j| j d S r+   )reprexcrQ   superrO   )rK   rk   rQ   	__class__r0   r1   rO      s   

zMaybeEncodingError.__init__c                 C   s   d| j jt| f S )Nz<%s: %s>)rn   rd   strrJ   r0   r0   r1   __repr__      zMaybeEncodingError.__repr__c                 C   s   d| j | jf S )Nz)Error sending result: '%r'. Reason: '%r'.)rQ   rk   rJ   r0   r0   r1   __str__   s   zMaybeEncodingError.__str__)rd   re   rf   rg   rO   rp   rr   __classcell__r0   r0   rm   r1   ri      s
    ri   c                   @   s   e Zd ZdZdS )WorkersJoinedzAll workers have terminated.N)rd   re   rf   rg   r0   r0   r0   r1   rt      s    rt   c                 C   s   t  r+   )r   )signumframer0   r0   r1   soft_timeout_sighandler      rw   c                   @   s   e Zd Z				dddZdd Zdd	 Zd
d ZdddZdd Zdd Z	e
edfddZdd Zdd Zdd ZefddZdd ZdS ) WorkerNr0   Tc                 C   sz   |d u st |tkr|dksJ || _|| _|| _|| _|| _|| _|	| _|||| _	| _
| _|
| _|| _| |  d S Nr   )typeintinitializerinitargsmaxtasksmax_memory_per_child	_shutdownon_exitsigprotectioninqoutqsynqwrap_exceptionon_ready_countercontribute_to_object)rK   r   r   r   r}   r~   r   sentinelr   r   r   r   r   r0   r0   r1   rO      s    zWorker.__init__c                 C   s   | j | j| j|_ |_|_| j j |_| jj |_| jr5| jj |_| jj |_	t
| jj|_n	d  |_ |_	|_| j jj|_| jjj|_t
| j j|_|S r+   )r   r   r   _writerr-   inqW_fd_readeroutqR_fdsynqR_fdsynqW_fdr2   send_syn_offset_send_syn_offsetsend
_quick_putrecv
_quick_getsend_job_offset)rK   objr0   r0   r1   r      s   zWorker.contribute_to_objectc                 C   s6   | j | j| j| j| j| j| j| j| j| j	| j
| jffS r+   )rn   r   r   r   r}   r~   r   r   r   r   r   r   rJ   r0   r0   r1   
__reduce__  s
   zWorker.__reduce__c                    s   t j d gd fdd	}|t _t }|   |   | j|d zGzt | j|d W n# tyR } zt	d| |dd | 
|d | W Y d }~nd }~ww W | 
|d d  d S W | 
|d d  d S | 
|d d  w )	Nc                    s   | d<  | S rz   r0   )status_exit	_exitcoder0   r1   exit  s   zWorker.__call__.<locals>.exitpidzPool process %r error: %rr   exc_infor   r+   )sysr   osgetpid_make_child_methods
after_forkon_loop_startworkloop	Exceptionr=   _do_exit)rK   r   r   rk   r0   r   r1   __call__  s&   *zWorker.__call__c              	   C   s~   |d u r
|rt nt}| jd ur| || tjdkr8z| jt||ff t	d W t
| d S t
| w t
| d S )Nwin32r   )
EX_FAILUREr)   r   r   platformr   putDEATHtimesleepr   r   )rK   r   exitcoderk   r0   r0   r1   r   *  s   

zWorker._do_exitc                 C      d S r+   r0   rK   r   r0   r0   r1   r   :     zWorker.on_loop_startc                 C   s   |S r+   r0   )rK   resultr0   r0   r1   prepare_result=  r   zWorker.prepare_resultc              
      s@  |pt  }jj}j}j}j}jpd}j}	j	}
j
  fdd}d}z|d u s5|r||k r|
 }|r|\}}|tksDJ |\}}}}}|t||| ||ff  r`||}|s`q+zd|	||i |f}W n ty{   dt f}Y nw z|t||||ff W n9 ty } z-t \}}}zt||d }tt||f}|t||d|f|ff W ~n~w W Y d }~nd }~ww |d7 }|dkrt }|dkrtd |dkr||krtt|| tW j|d S |d u s5|r||k s5|d	| |r||krtntW j|d S tW j|d S j|d w )
Nr   c                    s^   d}	 |dkrt d| jj dd   }|r*|\}}|tkr"dS |tks(J dS |d7 }q)Nr   r   <   z(!!!WAIT FOR ACK TIMEOUT: job:%r fd:%r!!!r   FT)r=   r   r   r-   NACKACK)jidireqtype_r6   _wait_for_synrK   r0   r1   wait_for_synL  s   z%Worker.workloop.<locals>.wait_for_synTFr   z'worker unable to determine memory usage)	completedzworker exiting after %d tasks)r   r   r   r   r   r   r   r   r   wait_for_jobr   TASKr   r   r   READYr   r   ri   r   r=   r    MAXMEM_USED_FMTformat
EX_RECYCLE_ensure_messages_consumedr   r)   )rK   r   nowr   r   r   r   r   r   r   r   r   r   r   r   args_jobr   funr6   r?   confirmr   rk   _tbwrappedeinfoused_kbr0   r   r1   r   @  sv   


%zWorker.workloopc                 C   sJ   | j sdS ttD ]}| j j|krtd|  dS tt q	td dS )zr Returns true if all messages sent out have been received and
        consumed within a reasonable amount of time Fz*ensured messages consumed after %d retriesTz<could not ensure all messages were consumed prior to exiting)	r   range)GUARANTEE_MESSAGE_CONSUMPTION_RETRY_LIMITrQ   r   r   r   ,GUARANTEE_MESSAGE_CONSUMPTION_RETRY_INTERVALr    )rK   r   retryr0   r0   r1   r     s   
z Worker._ensure_messages_consumedc                 C   s   t | jdr| jj  t | jdr| jj  | jd ur#| j| j  t| j	d t
d ur3tt
t zttjtj W d S  tyH   Y d S w )Nr   r   )full)hasattrr   r   closer   r   r}   r~   r   r   SIG_SOFT_TIMEOUTsignalrw   SIGINTSIG_IGNr,   rJ   r0   r0   r1   r     s   
zWorker.after_forkc                    sd   |j t|dr*|jj t|dr!|jr!|jtffdd	}|S  fdd}|S fdd}|S )Nr   get_payloadc                    s   d|  fS NTr0   )rD   loads)r   r0   r1   _recv     z'Worker._make_recv_method.<locals>._recvc                    s    | r	d fS dS NTFNr0   rD   )_pollgetr0   r1   r     s   
c                    s(   zd | dfW S  t jy   Y dS w NTr   r   r   r   )r   r0   r1   r     s
   )r   r   r   pollr   r
   )rK   connr   r0   )r   r   r   r1   _make_recv_method  s   

zWorker._make_recv_methodc                 C   s0   |  | j| _| jr|  | j| _d S d | _d S r+   )_make_protected_receiver   r   r   r   )rK   r   r0   r0   r1   r     s
   zWorker._make_child_methodsc                    s2   |  | | jr| jjnd tf fdd	}|S )Nc              
      s   r r| d t tz d\}}|sW d S W n( ttfyB } zt|tjkr2W Y d }~d S | dt|j t t	d }~ww |d u rO| d t t	|S )Nzworker got sentinel -- exiting      ?zworker got %s -- exiting)

SystemExitr)   EOFErrorIOErrorr   errnoEINTRr{   rd   r   )r   readyr   rk   _receiveshould_shutdownr0   r1   receive  s&   
z/Worker._make_protected_receive.<locals>.receive)r   r   is_setr   )rK   r   r   r0   r   r1   r     s   
zWorker._make_protected_receive)
NNr0   NNNTTNNr+   )rd   re   rf   rO   r   r   r   r   r   r   r   r   r   r   r   r   r
   r   r   r0   r0   r0   r1   ry      s$    

Mry   c                       sN   e Zd Zdd Zdd Z fddZdd Zdd
dZdd Zdd Z	  Z
S )
PoolThreadc                 O   s    t |  t| _d| _d| _d S NFT)r   rO   RUN_state_was_starteddaemonrK   r6   r?   r0   r0   r1   rO     s   

zPoolThread.__init__c              
   C   s   z|   W S  ty. } ztdt| j|dd tt t t	
  W Y d }~d S d }~w tyP } ztdt| j|dd td W Y d }~d S d }~ww )NzThread %r crashed: %rr   r   )bodyr   r=   r{   rd   _killr   r   r   r   r   r   r   rK   rk   r0   r0   r1   run  s    
zPoolThread.runc                    s    d| _ tt| j|i | d S r   )r   rl   r   startr  rm   r0   r1   r    s   zPoolThread.startc                 C   r   r+   r0   rJ   r0   r0   r1   on_stop_not_started  r   zPoolThread.on_stop_not_startedNc                 C   s    | j r
| | d S |   d S r+   )r   joinr  rK   rD   r0   r0   r1   rB   	  s   
zPoolThread.stopc                 C   
   t | _d S r+   )	TERMINATEr   rJ   r0   r0   r1   	terminate     
zPoolThread.terminatec                 C   r  r+   )CLOSEr   rJ   r0   r0   r1   r     r  zPoolThread.closer+   )rd   re   rf   rO   r  r  r  rB   r  r   rs   r0   r0   rm   r1   r     s    
r   c                       s$   e Zd Z fddZdd Z  ZS )
Supervisorc                    s   || _ t   d S r+   )poolrl   rO   )rK   r  rm   r0   r1   rO     s   zSupervisor.__init__c                 C   s   t d td | j}zH|j}td|j d|_tdD ]}| jtkr2|jtkr2|	  td q||_| jtkrS|jtkrS|	  td | jtkrS|jtks@W n t
yd   |  |   w t d d S )Nzworker handler startingg?
   r   r*   zworker handler exiting)r   r   r   r  r   
_processesr   r   r   _maintain_poolr   r   r	  )rK   r  
prev_stater   r0   r0   r1   r    s.   


zSupervisor.body)rd   re   rf   rO   r  rs   r0   r0   rm   r1   r    s    r  c                       s4   e Zd Z fddZdd Zdd Zdd Z  ZS )	TaskHandlerc                    s,   || _ || _|| _|| _|| _t   d S r+   )	taskqueuer   outqueuer  cacherl   rO   )rK   r  r   r  r  r  rm   r0   r1   rO   =     zTaskHandler.__init__c           
      C   sf  | j }| j}| j}t|jd D ]\}}d }d}z^t|D ]H\}}| jr)td  nJz|| W q ty=   td Y  n6 t	yd   |d d \}}	z|| 
|	dt f W n	 tya   Y nw Y qw |rqtd ||d  W qW  n7 t	y   |r|d d nd\}}	||v r|| 
|	d dt f |rtd ||d  Y qw td	 |   d S )
Nz'task handler found thread._state != RUNzcould not put task on queuer%   Fzdoing set_length()r   )r   r   ztask handler got sentinel)r  r  r   iterr   	enumerater   r   r   r   _setr   KeyErrorr   tell_others)
rK   r  r  r   taskseq
set_lengthtaskr   r   indr0   r0   r1   r  E  sR   
zTaskHandler.bodyc                 C   sj   | j }| j}| j}ztd |d  td |D ]}|d  qW n ty.   td Y nw td d S )Nz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got IOError when sending sentinelsztask handler exiting)r  r   r  r   r   )rK   r  r   r  pr0   r0   r1   r   o  s   

zTaskHandler.tell_othersc                 C   s   |    d S r+   )r   rJ   r0   r0   r1   r    r8   zTaskHandler.on_stop_not_started)rd   re   rf   rO   r  r   r  rs   r0   r0   rm   r1   r  ;  s
    *r  c                       sT   e Zd Z fddZdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
  ZS )TimeoutHandlerc                    s,   || _ || _|| _|| _d | _t   d S r+   )	processesr  t_softt_hard_itrl   rO   )rK   r'  r  r(  r)  rm   r0   r1   rO     r  zTimeoutHandler.__init__c                       t  fddt| jD dS )Nc                 3   &    | ]\}}|j  kr||fV  qd S r+   r   .0r   procr   r0   r1   	<genexpr>      
z1TimeoutHandler._process_by_pid.<locals>.<genexpr>NN)nextr  r'  r   r0   r   r1   _process_by_pid  
   zTimeoutHandler._process_by_pidc              
   C   sx   t d| | |j\}}|sd S |jdd z	t|jt W d S  ty; } zt|tj	kr0 W Y d }~d S d }~ww )Nzsoft time limit exceeded for %rTsoft)
r   r4  _worker_pidhandle_timeoutr  r   OSErrorr   r   ESRCH)rK   r   process_indexrk   r0   r0   r1   on_soft_timeout  s   
zTimeoutHandler.on_soft_timeoutc                 C   sz   |  rd S td| zt|j ty#   ||jdt f Y nw | |j\}}|j	dd |r;| 
| d S d S )Nzhard time limit exceeded for %rFr6  )r   r   r   _timeoutr  _jobr   r4  r8  r9  _trywaitkill)rK   r   r<  r=  r0   r0   r1   on_hard_timeout  s   

zTimeoutHandler.on_hard_timeoutc                 C   s   t d|j z!t|j|jkr"t d|j tt|jtj n|  W n	 t	y0   Y n
w |j
jddr:d S t d|j z&t|j|jkr^t d|j tt|jtj W d S t|jt W d S  t	yp   Y d S w )Nztimeout: sending TERM to %szIworker %s is a group leader. It is safe to kill (SIGTERM) the whole groupr*   r   z/timeout: TERM timed-out, now sending KILL to %szIworker %s is a group leader. It is safe to kill (SIGKILL) the whole group)r   _namer   getpgidr   killpgr   SIGTERMr  r:  _popenwaitSIGKILLr  rK   workerr0   r0   r1   rA    s*   zTimeoutHandler._trywaitkillc                 #   s    | j | j}}t }| j}| j}dd }| jtkrqt| j |r-t fdd|D } 	 D ]5\}}|j
}	|j}
|
d u rA|}
|j}|d u rJ|}||	|rT|| q1||vrf||	|
rf|| || q1d V  | jtksd S d S )Nc                 S   s"   | r|sdS t  | | krdS d S r   r   )r  rD   r0   r0   r1   
_timed_out  s
   z2TimeoutHandler.handle_timeouts.<locals>._timed_outc                 3   s    | ]	}| v r|V  qd S r+   r0   )r.  kr  r0   r1   r0        z1TimeoutHandler.handle_timeouts.<locals>.<genexpr>)r)  r(  setr>  rB  r   r   copyr  items_time_accepted_soft_timeoutr?  add)rK   r)  r(  dirtyr>  rB  rL  r   r   ack_timesoft_timeouthard_timeoutr0   rN  r1   handle_timeouts  s4   



zTimeoutHandler.handle_timeoutsc                 C   sP   | j tkr"z|  D ]}td q
W n	 ty   Y nw | j tkstd d S )Nr   ztimeout handler exiting)r   r   rZ  r   r   r   r   rK   r   r0   r0   r1   r    s   

zTimeoutHandler.bodyc                 G   s@   | j d u r
|  | _ zt| j  W d S  ty   d | _ Y d S w r+   )r*  rZ  r3  StopIterationrK   r6   r0   r0   r1   handle_event   s   

zTimeoutHandler.handle_event)rd   re   rf   rO   r4  r>  rB  rA  rZ  r  r^  rs   r0   r0   rm   r1   r&    s    &	r&  c                       sV   e Zd Z	d fdd	Zdd Zdd Zdd	d
ZdddZdd ZdddZ	  Z
S )ResultHandlerNc                    s^   || _ || _|| _|| _|| _|| _|| _d | _d| _|| _	|	| _
|
| _|   t   d S )NF)r  r   r  r   join_exited_workersputlockr   r*  _shutdown_completecheck_timeoutson_job_readyon_ready_counters_make_methodsrl   rO   )rK   r  r   r  r   r`  ra  r   rc  rd  re  rm   r0   r1   rO     s   zResultHandler.__init__c                 C   s   | j dd d S )NT)rZ  )finish_at_shutdownrJ   r0   r0   r1   r       z!ResultHandler.on_stop_not_startedc                    sl   j  jjj fdd} fdd}dd }t|t|t|i _fdd}|_d S )	Nc              	      s:   d_ z |  |||| W d S  ttfy   Y d S w rz   )R_ackr  r,   )r   r   time_acceptedr   r   )r  r   r0   r1   on_ack'  s   z+ResultHandler._make_methods.<locals>.on_ackc                    s   d ur| ||| z |  }W n
 t y   Y d S w jrOtt| d }|rO|jv rOj| }|  | jd7  _W d    n1 sJw   Y  | s[d ur[  z	|	|| W d S  t yn   Y d S w rG   )
r  re  r3  r  worker_pidsget_lockrQ   r   r[   r  )r   r   r   r   item
worker_pidr   )r  rd  ra  rK   r0   r1   on_ready/  s,   

z-ResultHandler._make_methods.<locals>.on_readyc              
   S   sJ   z	t | t W d S  ty$ } zt|tjkr W Y d }~d S d }~ww r+   )r   r$   r   r:  r   r   r;  )r   r   rk   r0   r0   r1   on_deathF  s   z-ResultHandler._make_methods.<locals>.on_deathc                    s<   | \}}z	 | |  W d S  t y   td|| Y d S w )NzUnknown job state: %s (args=%s))r  r   )r#  stater6   )state_handlersr0   r1   on_state_changeQ  s   z4ResultHandler._make_methods.<locals>.on_state_change)	r  ra  r   rd  r   r   r   rt  ru  )rK   rl  rq  rr  ru  r0   )r  rd  ra  r   rK   rt  r1   rf  !  s   

zResultHandler._make_methodsr   c              
   c   s    | j }| j}	 z||\}}W n ttfy& } ztd| t d }~ww | jr8| jtks1J td t |rP|d u rEtd t || |dkrOd S nd S d V  q)Nr    result handler got %r -- exitingz,result handler found thread._state=TERMINATEzresult handler got sentinelr   )r   ru  r   r   r   r   r   r  )rK   rD   r   ru  r   r#  rk   r0   r0   r1   _process_resultY  s4   
zResultHandler._process_resultc              	   C   sT   | j tkr(| jd u r| d| _zt| j W d S  ttfy'   d | _Y d S w d S rz   )r   r   r*  rw  r3  r\  r   )rK   r-   eventsr0   r0   r1   r^  t  s   

zResultHandler.handle_eventc                 C   sz   t d z3| jtkr*z
| dD ]}qW n	 ty   Y nw | jtks
W |   d S W |   d S W |   d S |   w )Nzresult handler startingr   )r   r   r   rw  r   rg  r[  r0   r0   r1   r  }  s    
zResultHandler.bodyFc              
   C   s  d| _ | j}| j}| j}| j}| j}| j}| j}d }	|r| jt	kr|d ur(|  z|d\}
}W n t
tfyJ } ztd| W Y d }~d S d }~ww |
rZ|d u rVtd q|| z|dd W n+ ty   t }|	sp|}	n||	 dkr|td Y ntdtt||	 d d	 Y nw |r| jt	ks!t|d
rtd ztdD ]}|j s n|  qW n t
tfy   Y nw tdt|| j d S )NTr   rv  z&result handler ignoring extra sentinel)shutdowng      @z!result handler exiting: timed outz6result handler: all workers terminated, timeout in %ssr   r   z"ensuring that outqueue is not fullr  z7result handler exiting: len(cache)=%s, thread._state=%s)rb  r   r  r  r   r`  rc  ru  r   r  r   r   r   rt   r   absminr   r   r   len)rK   rZ  r   r  r  r   r`  rc  ru  time_terminater   r#  rk   r   r   r0   r0   r1   rg    sj   



z ResultHandler.finish_at_shutdownr+   )r   r2  F)rd   re   rf   rO   r  rf  rw  r^  r  rg  rs   r0   r0   rm   r1   r_  	  s    
8
	r_  c                   @   sn  e Zd ZdZdZeZeZeZeZe	Z	e
Z
																	dwddZd	d
 Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd  Zdxd!d"Zd#d$ Zd%d& Zd'd( Zd)d* Zd+d, Zd-d. Zd/d0 Zd1d2 Z d3d4 Z!dyd5d6Z"dyd7d8Z#d9d: Z$d;d< Z%d=d> Z&d?d@ Z'dAdB Z(dCdD Z)dEdF Z*dGdH Z+dIdJ Z,di fdKdLZ-dzdMdNZ.		d{dOdPZ/dzdQdRZ0d|dSdTZ1		d|dUdVZ2di ddddddddddfdWdXZ3dYdZ Z4dzd[d\Z5		d{d]d^Z6		d{d_d`Z7e8dadb Z9dcdd Z:dedf Z;dgdh Z<e8didj Z=dkdl Z>dmdn Z?e8dodp Z@eAdqdr ZBeAdsdt ZCeDdudv ZEdS )}PoolzS
    Class which supports an async version of applying functions to arguments.
    TNr0   r   Fc                 K   s  |pt  | _|| _|   t | _i | _t| _|| _	|| _
|| _|| _|| _|| _|| _|p/t| _|
| _|| _|| _|| _|| _i | _|| _t|pR| j	d upR| j
d u| _|rdtd u rdttd d }|d u rl|  n|| _ |pwt!| j d | _"t#||	p~d| _#|d urt$|st%d|d urt$|st%d| jj&| _'g | _(i | _)i | _*|| _+|pt,| j | _-t.| j D ]}| /| q| 0| | _1|r| j12  | 3| j| j4| j5| j(| j| _6|r| j62  d | _7| jr
| 8| j(| j| j
| j	| _9t: | _;d| _<| =  |s	| j9j>| _7n	d | _9d| _<d | _;| ? | _@| j@j>| _A|r%| j@2  tB| | jC| j| jD| j5| j(| j1| j6| j@| j| j9| E f
dd| _Fd S )	NWSoft timeouts are not supported: on this platform: It does not have the SIGUSR1 signal.d   r   zinitializer must be a callablez on_process_exit must be callableF   )r6   exitpriority)Gr   _ctxsynack_setup_queuesr   
_taskqueue_cacher   r   rD   rX  _maxtasksperchild_max_memory_per_child_initializer	_initargs_on_process_exitLOST_WORKER_TIMEOUTlost_worker_timeouton_process_upon_process_downon_timeout_seton_timeout_cancelthreadsreadersallow_restartboolenable_timeoutsr   warningswarnUserWarningr   r  roundmax_restartsr   callable	TypeErrorProcess_Process_pool	_poolctrl_on_ready_countersputlocksrF   _putlockr   _create_worker_processr  _worker_handlerr  r  r   	_outqueue_task_handlerrc  r&  _timeout_handlerLock_timeout_handler_mutex_timeout_handler_started_start_timeout_handlerr^  create_result_handler_result_handlerhandle_result_eventr   _terminate_pool_inqueue_help_stuff_finish_args
_terminate)rK   r'  r}   r~   maxtasksperchildrD   rX  r  r  max_restart_freqr  r  r  r  r  	semaphorer  r  r  on_process_exitcontextr   r  r?   r   r0   r0   r1   rO     s   







zPool.__init__c                 O   s   | j |i |S r+   )r  )rK   r6   kwdsr0   r0   r1   r  H     zPool.Processc                 C   s   | | j|dS )N)target)r   r  rJ  r0   r0   r1   WorkerProcessK     zPool.WorkerProcessc              
   K   s:   | j | j| j| j| j| j| j| j| j| j	f	d| j
i|S )Nre  )r_  r  r   r  _poll_result_join_exited_workersr  r   rc  rd  r  )rK   extra_kwargsr0   r0   r1   r  N  s   zPool.create_result_handlerc                 C   r   r+   r0   )rK   r   r   r   r   r0   r0   r1   rd  W  r   zPool.on_job_readyc                 C   s   | j | j| jfS r+   )r  r  r  rJ   r0   r0   r1   r  Z  r  zPool._help_stuff_finish_argsc                 C   s   zt  W S  ty   Y dS w rG   )r   NotImplementedErrorrJ   r0   r0   r1   r   ]  s
   zPool.cpu_countc                 G   s   | j j| S r+   )r  r^  r]  r0   r0   r1   r  c  r8   zPool.handle_result_eventc                 C   r   r+   r0   )rK   rK  queuesr0   r0   r1   _process_register_queuesf  r   zPool._process_register_queuesc                    r+  )Nc                 3   r,  r+   r   r-  r   r0   r1   r0  j  r1  z'Pool._process_by_pid.<locals>.<genexpr>r2  )r3  r  r  r   r0   r   r1   r4  i  r5  zPool._process_by_pidc                 C   s   | j | jd fS r+   )r  r  rJ   r0   r0   r1   get_process_queueso  r   zPool.get_process_queuesc                 C   s   | j r| j nd }|  \}}}| jd}| | j|||| j| j| j	|| j
| j| j| j|d}| j| | ||||f |jdd|_d|_||_|  || j|j< || j|j< | jrg| | |S )Nr   )r   r   r   r   r  
PoolWorkerT)r  r  Eventr  Valuer  ry   r  r  r  r  r  _wrap_exceptionr  r  appendr  namereplacer  indexr  r  r   r  r  )rK   r   r   r   r   r   r   wr0   r0   r1   r  r  s,   


zPool._create_worker_processc                 C   r   r+   r0   rJ  r0   r0   r1   process_flush_queues  r   zPool.process_flush_queuesc                    sl  d}dd t | j D D ]}|pt }|j\}}|| |jkr'| || q|r2t| js2t	 i i }t
tt| jD ]]}| j| }|j}	|j}
|
du sU|	durtd| |
durb|  td| ||j< |	||j< |	ttfvrt|ddstd|j|jt|	d	d
 | | | j|= | j|j= | j|j= q@r4dd | jD  t | j D ]d}t fdd| D d}|r| || | s||pd	}	|}|rt|ddr||	 q|  |||	 q|j!}|j"}|r|# s| ||j q|r|# s| ||j q D ]}| j$r,|s'| %| | $| qt | S g S )zCleanup after any worker processes which have exited due to
        reaching their specified lifetime. Returns True if any workers were
        cleaned up.
        Nc                 S   s   g | ]}|  s|jr|qS r0   )r   _worker_lost)r.  r   r0   r0   r1   
<listcomp>  s
    
z-Pool._join_exited_workers.<locals>.<listcomp>z!Supervisor: cleaning up worker %dzSupervisor: worked %d joined_controlled_terminationFz Process %r pid:%r exited with %rr   r   c                 S   s   g | ]}|j qS r0   r   r.  r  r0   r0   r1   r    s    c                 3   s$    | ]}|v s| vr|V  qd S r+   r0   r.  r   all_pidscleanedr0   r1   r0    s    z,Pool._join_exited_workers.<locals>.<genexpr>_job_terminated)&r3   r  valuesr   r  _lost_worker_timeoutmark_as_worker_lostr|  r  rt   reversedr   r   rG  r   r	  r   r)   r   getattrr=   r  r	   r  r  r  r3  rm  on_job_process_downr   r   _set_terminatedon_job_process_lost	_write_to_scheduled_for	_is_aliver  _process_cleanup_queues)rK   ry  r   r   	lost_timelost_ret	exitcodesr   rK  r   popenacked_by_goner/  write_to	sched_forr0   r  r1   r    s   














zPool._join_exited_workersc                 C   r   r+   r0   )rK   r   rK  r0   r0   r1   on_partial_read  r   zPool.on_partial_readc                 C   r   r+   r0   rJ  r0   r0   r1   r    r   zPool._process_cleanup_queuesc                 C   r   r+   r0   )rK   r   pid_goner0   r0   r1   r    r   zPool.on_job_process_downc                 C   s   t  |f|_d S r+   )r   r  )rK   r   r   r   r0   r0   r1   r    r  zPool.on_job_process_lostc                 C   s>   zt dt||j t y   |d dt f Y d S w )Nz(Worker exited prematurely: {0} Job: {1}.F)r   r   r	   r@  r  r   )rK   r   r   r0   r0   r1   r    s   
zPool.mark_as_worker_lostc                 C      | S r+   r0   rJ   r0   r0   r1   	__enter__  r   zPool.__enter__c                 G   s   |   S r+   )r  )rK   r   r0   r0   r1   __exit__  s   zPool.__exit__c                 C   r   r+   r0   rK   nr0   r0   r1   on_grow  r   zPool.on_growc                 C   r   r+   r0   r  r0   r0   r1   	on_shrink  r   zPool.on_shrinkc                 C   s`   t |  D ]%\}}|  jd8  _| jr| j  |  | d ||d kr+ d S qtd)Nr   z&Can't shrink pool. All processes busy!)r  _iterinactiver  r  rL   terminate_controlledr  
ValueError)rK   r  r   rK  r0   r0   r1   rL     s   

zPool.shrinkc                 C   s:   t |D ]}|  jd7  _| jr| j  q| | d S rG   )r   r  r  rV   r  )rK   r  r   r0   r0   r1   rV     s   
z	Pool.growc                 c   s"    | j D ]
}| |s|V  qd S r+   )r  _worker_activerJ  r0   r0   r1   r    s   

zPool._iterinactivec                 C   s(   | j  D ]}|j| v r dS qdS )NTF)r  r  r   rm  )rK   rK  r   r0   r0   r1   r    s
   zPool._worker_activec              	   C   s   t | jt| j D ]5}| jtkr dS z|r$|| ttfvr$| j	  W n t
y3   | j	  Y nw | |   td q
dS )zBring the number of pool processes up to the specified number,
        for use after reaping workers which have exited.
        Nzadded worker)r   r  r|  r  r   r   r)   r   r   step
IndexErrorr  _avail_indexr   )rK   r  r   r0   r0   r1   _repopulate_pool#  s   


zPool._repopulate_poolc                    sD   t | j| jk s
J tdd | jD  t fddt| jD S )Nc                 s   s    | ]}|j V  qd S r+   )r  )r.  r%  r0   r0   r1   r0  4  s    z$Pool._avail_index.<locals>.<genexpr>c                 3   s    | ]	}| vr|V  qd S r+   r0   )r.  r   indicesr0   r1   r0  5  rO  )r|  r  r  rP  r3  r   rJ   r0   r  r1   r  2  s   zPool._avail_indexc                 C   s
   |    S r+   )r  rJ   r0   r0   r1   did_start_ok7  r  zPool.did_start_okc                 C   s<   |   }| | tt|D ]}| jdur| j  qdS )zF"Clean up any exited workers and start replacements for them.
        N)r  r  r   r|  r  r[   )rK   joinedr   r0   r0   r1   r  :  s   


zPool._maintain_poolc              
   C   sz   | j jtkr9| jtkr;z|   W d S  ty"   |   |     ty8 } zt|t	j
kr3t| d }~ww d S d S r+   )r  r   r   r  r   r   r	  r:  r   r   ENOMEMMemoryErrorr  r0   r0   r1   maintain_poolC  s   zPool.maintain_poolc                    sF    j   _ j   _ jjj _ jjj _	 fdd}| _
d S )Nc                    s    j j| rd  fS dS r   )r  r   r   r   r   rJ   r0   r1   r  V  s   z(Pool._setup_queues.<locals>._poll_result)r  SimpleQueuer  r  r   r   r   r   r   r   r  rK   r  r0   rJ   r1   r  P  s   
zPool._setup_queuesc                 C   sj   | j r1| jd ur3| j | jsd| _| j  W d    d S W d    d S 1 s*w   Y  d S d S d S r   )r  r  r  r  r  rJ   r0   r0   r1   r  \  s   "zPool._start_timeout_handlerc                 C       | j tkr| ||| S dS )z8
        Equivalent of `func(*args, **kwargs)`.
        N)r   r   apply_asyncr   )rK   funcr6   r  r0   r0   r1   applye  s   
z
Pool.applyc                 C   s"   | j tkr| ||t| S dS )z
        Like `map()` method but the elements of the `iterable` are expected to
        be iterables as well and will be unpacked as arguments. Hence
        `func` and (a, b) becomes func(a, b).
        N)r   r   
_map_asyncr;   r   rK   r
  iterable	chunksizer0   r0   r1   r:   l  s   
zPool.starmapc                 C   s"   | j tkr| ||t|||S dS )z=
        Asynchronous version of `starmap()` method.
        N)r   r   r  r;   rK   r
  r  r  callbackerror_callbackr0   r0   r1   starmap_asyncv  s
   
zPool.starmap_asyncc                 C   r  )zx
        Apply `func` to each element in `iterable`, collecting the results
        in a list that is returned.
        N)r   r   	map_asyncr   r  r0   r0   r1   r4     s   
zPool.mapc                       | j tkrdS |p| j}|dkr,t| j|d| j fddt|D jf S |dks2J t	
 ||}t| j|d| jfddt|D jf dd D S )zP
        Equivalent of `map()` -- can be MUCH slower than `Pool.map()`.
        Nr   r  c                 3   *    | ]\}}t j| |fi ffV  qd S r+   r   r@  r.  r   xr
  r   r0   r1   r0        zPool.imap.<locals>.<genexpr>c                 3   *    | ]\}}t  j|t|fi ffV  qd S r+   r   r@  r7   r  r   r0   r1   r0    r  c                 s       | ]
}|D ]}|V  qqd S r+   r0   r.  chunkro  r0   r0   r1   r0        )r   r   r  IMapIteratorr  r  r   r  _set_lengthr  
_get_tasksrK   r
  r  r  r  task_batchesr0   r  r1   imap  s4   


z	Pool.imapc                    r  )zL
        Like `imap()` method but ordering of results is arbitrary.
        Nr   r  c                 3   r  r+   r  r  r  r0   r1   r0    r  z&Pool.imap_unordered.<locals>.<genexpr>c                 3   r  r+   r  r  r  r0   r1   r0    r  c                 s   r   r+   r0   r!  r0   r0   r1   r0    r#  )r   r   r  IMapUnorderedIteratorr  r  r   r  r%  r  r&  r'  r0   r  r1   imap_unordered  s4   


zPool.imap_unorderedc                 C   s  | j tkrdS |	p| j}	|
p| j}
|p| j}|	r%tdu r%ttd d}	| j tkr|du r1| j	n|}|r?| j
dur?| j
  t| j|||||	|
|| j| j|| jrT| jnd|d}|
s]|	ra|   | jrw| jt|jd|||ffgdf |S | t|jd|||ff |S dS )a  
        Asynchronous equivalent of `apply()` method.

        Callback is called when the functions return value is ready.
        The accept callback is called when the job is accepted to be executed.

        Simplified the flow is like this:

            >>> def apply_async(func, args, kwds, callback, accept_callback):
            ...     if accept_callback:
            ...         accept_callback()
            ...     retval = func(*args, **kwds)
            ...     if callback:
            ...         callback(retval)

        Nr  )r  r  callbacks_propagatesend_ackcorrelation_id)r   r   rX  rD   r  r   r  r  r  r  r  rI   ApplyResultr  r  r  r  r-  r  r  r  r   r   r@  r   )rK   r
  r6   r  r  r  accept_callbacktimeout_callbackwaitforslotrX  rD   r  r,  r.  r   r0   r0   r1   r	    sF   






	zPool.apply_asyncc                 C   r   r+   r0   )rK   responser   r   fdr0   r0   r1   r-    r   zPool.send_ackc              
   C   st   |  |\}}|d ur8z	t||pt W n ty/ } zt|tjkr$ W Y d }~d S d }~ww d|_d|_d S d S r   )	r4  r  r   r:  r   r   r;  r  r  )rK   r   sigr/  r   rk   r0   r0   r1   terminate_job  s   
zPool.terminate_jobc                 C   s   |  ||t|||S )z<
        Asynchronous equivalent of `map()` method.
        )r  r7   r  r0   r0   r1   r  	  s   zPool.map_asyncc           	         s   | j tkrdS t|dst|}|du r(tt|t| jd \}}|r(|d7 }t|dkr0d}t|||}t	| j
|t|||d| j fddt|D df S )	zY
        Helper function to implement map, starmap and their async counterparts.
        N__len__r&   r   r   r  c                 3   r  r+   r  r  mapperr   r0   r1   r0  &  r  z"Pool._map_async.<locals>.<genexpr>)r   r   r   r3   divmodr|  r  r  r&  	MapResultr  r  r   r  )	rK   r
  r  r:  r  r  r  extrar(  r0   r9  r1   r    s(   

zPool._map_asyncc                 c   s0    t |}	 tt||}|sd S | |fV  qr+   )r  tupler9   islice)r
  itsizer  r0   r0   r1   r&  *  s   
zPool._get_tasksc                 C   s   t d)Nz:pool objects cannot be passed between processes or pickled)r  rJ   r0   r0   r1   r   3  s   zPool.__reduce__c                 C   sP   t d | jtkr&t| _| jr| j  | j  | j	d  t
| j d S d S )Nzclosing pool)r   r   r   r  r  r^   r  r   r  r   rE   rJ   r0   r0   r1   r   8  s   


z
Pool.closec                 C   s$   t d t| _| j  |   d S )Nzterminating pool)r   r  r   r  r  r  rJ   r0   r0   r1   r  B  s   
zPool.terminatec                 C   s   t |  d S r+   )rE   )task_handlerr0   r0   r1   _stop_task_handlerH  s   zPool._stop_task_handlerc                 C   s   | j ttfv s	J td t| j td | | j td t| j td t	| j
D ]\}}td|d t| j
| |jd urG|  q.td d S )Nzjoining worker handlerjoining task handlerjoining result handlerzresult handler joinedzjoining worker %s/%s (%r)r   zpool join complete)r   r  r  r   rE   r  rC  r  r  r  r  r|  rG  r	  )rK   r   r%  r0   r0   r1   r	  L  s   


z	Pool.joinc                 C   s   | j  D ]}|  qd S r+   )r  r  rP  )rK   er0   r0   r1   restart[  s   
zPool.restartc                 C   sZ   t d | j  | r'| j r+| j  td | r)| j sd S d S d S d S )Nz7removing tasks from inqueue until task handler finishedr   )	r   _rlockrI   is_aliver   r   r   r   r   )inqueuerB  r  r0   r0   r1   _help_stuff_finish_  s   


"zPool._help_stuff_finishc                 C   s   | d  d S r+   )r   )clsr  r  r0   r0   r1   _set_result_sentinelh  s   zPool._set_result_sentinelc                 C   s:  t d |  |  |d  t d | j|
  |  | || |	d ur,|	  |rFt|d drFt d |D ]
}| rE|  q;t d | | t d |  |	d urdt d |	t	 |rt|d drt d	 |D ]}|
 rt d
|j |jd ur|  qst d |r|  |r|  d S d S )Nzfinalizing poolz&helping task handler/workers to finishr   r  zterminating workersrD  rE  zjoining timeout handlerzjoining pool workerszcleaning up worker %dzpool workers joined)r   r  r   rK  rM  r   r  rC  rB   TIMEOUT_MAXrI  r   rG  r	  r   )rL  r  rJ  r  r  worker_handlerrB  result_handlerr  timeout_handlerhelp_stuff_finish_argsr%  r0   r0   r1   r  l  sJ   




zPool._terminate_poolc                 C      dd | j D S )Nc                 S   s   g | ]}|j jqS r0   )rG  r   r  r0   r0   r1   r        z*Pool.process_sentinels.<locals>.<listcomp>)r  rJ   r0   r0   r1   process_sentinels  rh  zPool.process_sentinels)NNr0   NNNNNr   NNNNTNFFFNNNFr~  )r   r+   )NNNrc   )Frd   re   rf   rg   r  ry   r  r  r&  r_  r   rO   r  r  r  rd  r  r   r  r  r4  r  r  r  r  r  r  r  r  r  r  r  r  r  rL   rV   r  r  r  r  r  r  r  r  r  r  r:   r  r4   r)  r+  r	  r-  r6  r  r  staticmethodr&  r   r   r  rC  r	  rG  rK  classmethodrM  r  propertyrU  r0   r0   r0   r1   r    s    
z	
S

		



	


8

	






6r  c                   @   s   e Zd ZdZdZdZdddddedddddfddZdd Zdd Z	d	d
 Z
dd Zdd Zdd Zdd Zd$ddZdd Zd$ddZd$ddZdd Zd%ddZd d! Zd"d# ZdS )&r/  Nr0   c                 C   s   || _ t | _t | _tt| _|| _	|| _
|| _|| _|| _|| _|| _|| _|	| _|
| _|p2d| _|| _d| _d| _d | _d | _d | _| || j< d S )Nr0   F)r.  r  _mutexr@   r  _eventr3  job_counterr@  r  	_callback_accept_callback_error_callback_timeout_callbackr?  rT  r  _on_timeout_set_on_timeout_cancel_callbacks_propagate	_send_ack	_accepted
_cancelledr8  rS  _terminated)rK   r  r  r0  r1  r  rX  rD   r  r  r  r,  r-  r.  r0   r0   r1   rO     s,   


zApplyResult.__init__c                 C   s   dj | jj| j| j|  dS )Nz&<{name}: {id} ack:{ack} ready:{ready}>)r  idackr   )r   rn   rd   r@  rd  r   rJ   r0   r0   r1   rp     s   zApplyResult.__repr__c                 C   s
   | j  S r+   )rZ  r   rJ   r0   r0   r1   r     r  zApplyResult.readyc                 C      | j S r+   )rd  rJ   r0   r0   r1   accepted  rx   zApplyResult.acceptedc                 C   s   |   sJ | jS r+   )r   _successrJ   r0   r0   r1   
successful  s   zApplyResult.successfulc                 C   s
   d| _ dS )zOnly works if synack is used.TN)re  rJ   r0   r0   r1   _cancel  s   
zApplyResult._cancelc                 C   s   | j | jd  d S r+   )r  popr@  rJ   r0   r0   r1   discard  rq   zApplyResult.discardc                 C   s
   || _ d S r+   )rf  rK   ru   r0   r0   r1   r    r  zApplyResult.terminatec                 C   s6   zt |pd  t y   | d dt f Y d S w Nr   F)r   r  r   rp  r0   r0   r1   r    s
   zApplyResult._set_terminatedc                 C   s   | j r| j gS g S r+   r8  rJ   r0   r0   r1   rm    r  zApplyResult.worker_pidsc                 C   s   | j | d S r+   )rZ  rH  r
  r0   r0   r1   rH    r  zApplyResult.waitc                 C   s*   |  | |  st| jr| jS | jjr+   )rH  r   r   rk  rT   	exceptionr
  r0   r0   r1   r     s   
zApplyResult.getc              
   O   sb   |r/z
||i | W d S  | j y     ty. } ztd|dd W Y d }~d S d }~ww d S )Nz"Pool callback raised exception: %rr   r   )rb  r   r=   )rK   r   r6   r?   rk   r0   r0   r1   safe_apply_callback  s   zApplyResult.safe_apply_callbackFc                 C   s0   | j d ur| j| j ||r| jn| jd d S d S )N)r7  rD   )r_  rt  rT  r?  )rK   r7  r0   r0   r1   r9    s   

zApplyResult.handle_timeoutc                 C   s   | j ` | jr| |  |\| _| _| j  | jr"| j| j	d  | j
r0| jr0| | j
| j | jd urK| jrS| js[| | j| j W d    d S W d    d S W d    d S W d    d S 1 sfw   Y  d S r+   )rY  ra  rk  rT   rZ  rP  rd  r  rn  r@  r\  rt  r^  rK   r   r   r0   r0   r1   r    s4   


"zApplyResult._setc                 C   s  | j  | jr(| jr(d| _|r| t|| j|W  d    S 	 W d    d S d| _|| _|| _|  r=| j	
| jd  | jrI| | | j| j t}| jrz5z| || W n | jyb   t}  tyl   t}Y nw W | jr|r| ||| j|W  d    S n| jr|r| ||| j|     Y W  d    S w w | jr|r| ||| j| W d    d S W d    d S W d    d S 1 sw   Y  d S r   )rY  re  rc  rd  r   r@  rS  r8  r   r  rn  r`  rT  r?  r   r]  _propagate_errorsr   )rK   r   rk  r   r   r3  r0   r0   r1   rj  $  sZ   




"zApplyResult._ackr+   r~  )rd   re   rf   r  r  r  r  rO   rp   r   rj  rl  rm  ro  r  r  rm  rH  r   rt  r9  r  rj  r0   r0   r0   r1   r/    s4    



	

r/  c                   @   s4   e Zd Zdd Zdd Zdd Zdd Zd	d
 ZdS )r<  c                 C   s   t j| |||d d| _|| _d g| | _dg| | _d g| | _d g| | _|| _|dkr<d| _	| j
  || j= d S || t||  | _	d S )Nr8  TFr   )r/  rO   rk  _lengthrT   rd  r8  rS  
_chunksize_number_leftrZ  rP  r@  r  )rK   r  r  lengthr  r  r0   r0   r1   rO   L  s   
zMapResult.__init__c                 C   s   |\}}|r>|| j || j |d | j < |  jd8  _| jdkr<| jr*| | j  | jr5| j| jd  | j	  d S d S d| _
|| _ | jrM| | j  | jrX| j| jd  | j	  d S )Nr   r   F)rT   rx  ry  r\  rd  r  rn  r@  rZ  rP  rk  r^  )rK   r   success_resultsuccessr   r0   r0   r1   r  ^  s$   
zMapResult._setc                 G   sn   || j  }t|d | j  | j}t||D ]}d| j|< || j|< || j|< q|  r5| j	| j
d  d S d S Nr   T)rx  r{  rw  r   rd  r8  rS  r   r  rn  r@  )rK   r   rk  r   r6   r  rB   jr0   r0   r1   rj  r  s   


zMapResult._ackc                 C   s
   t | jS r+   )allrd  rJ   r0   r0   r1   rj  |  r  zMapResult.acceptedc                 C   rS  )Nc                 S   s   g | ]}|r|qS r0   r0   r  r0   r0   r1   r    rT  z)MapResult.worker_pids.<locals>.<listcomp>rr  rJ   r0   r0   r1   rm    r  zMapResult.worker_pidsN)rd   re   rf   rO   r  rj  rj  rm  r0   r0   r0   r1   r<  J  s    
r<  c                   @   sZ   e Zd ZdZefddZdd ZdddZeZdd	 Z	d
d Z
dd Zdd Zdd ZdS )r$  Nc                 C   sZ   t t  | _tt| _|| _t | _	d| _
d | _d| _i | _g | _|| _| || j< d S rq  )r@   	Conditionr  rS   r3  r[  r@  r  r   _itemsr=  rw  _ready	_unsorted_worker_pidsr  )rK   r  r  r0   r0   r1   rO     s   
zIMapIterator.__init__c                 C   r  r+   r0   rJ   r0   r0   r1   __iter__  r   zIMapIterator.__iter__c                 C   s   | j F z| j }W n6 tyA   | j| jkrd| _t| j | z| j }W n ty>   | j| jkr<d| _tt	w Y nw W d    n1 sLw   Y  |\}}|rY|S t
|r   )rS   r  popleftr  r=  rw  r  r\  rH  r   r   )rK   rD   ro  r|  rQ   r0   r0   r1   r3    s0   zIMapIterator.nextc                 C   s   | j \ | j|kr<| j| |  jd7  _| j| jv r6| j| j}| j| |  jd7  _| j| jv s| j   n|| j|< | j| jkrWd| _| j	| j
= W d    d S W d    d S 1 sbw   Y  d S r}  )rS   r=  r  r  r  rn  rU   rw  r  r  r@  ru  r0   r0   r1   r    s"   

"zIMapIterator._setc                 C   sh   | j ' || _| j| jkr"d| _| j   | j| j= W d    d S W d    d S 1 s-w   Y  d S r   )rS   rw  r=  r  rU   r  r@  )rK   rz  r0   r0   r1   r%    s   
"zIMapIterator._set_lengthc                 G   s   | j | d S r+   )r  r  )rK   r   rk  r   r6   r0   r0   r1   rj    r  zIMapIterator._ackc                 C   ri  r+   )r  rJ   r0   r0   r1   r     rx   zIMapIterator.readyc                 C   ri  r+   )r  rJ   r0   r0   r1   rm    rx   zIMapIterator.worker_pidsr+   )rd   re   rf   r  r  rO   r  r3  __next__r  r%  rj  r   rm  r0   r0   r0   r1   r$    s    
r$  c                   @   s   e Zd Zdd ZdS )r*  c                 C   s|   | j 1 | j| |  jd7  _| j   | j| jkr,d| _| j| j= W d    d S W d    d S 1 s7w   Y  d S r}  )	rS   r  r  r=  rU   rw  r  r  r@  ru  r0   r0   r1   r    s   
"zIMapUnorderedIterator._setN)rd   re   rf   r  r0   r0   r0   r1   r*    s    r*  c                   @   s:   e Zd ZddlmZ eZdddZdd Zed	d
 Z	dS )
ThreadPoolr   )r  Nr0   c                 C   s   t | ||| d S r+   )r  rO   )rK   r'  r}   r~   r0   r0   r1   rO     rq   zThreadPool.__init__c                    s:   t   _t   _ jj _ jj _ fdd}| _d S )Nc                    s(   z	d j | dfW S  ty   Y dS w r   )r   r   r   rJ   r0   r1   r    s
   z.ThreadPool._setup_queues.<locals>._poll_result)r   r  r  r   r   r   r   r  r  r0   rJ   r1   r    s   


zThreadPool._setup_queuesc                 C   sV   | j  | j  | jd gt|  | j   W d    d S 1 s$w   Y  d S r+   )	not_emptyqueuer^   extendr|  rX   )rJ  rB  r  r0   r0   r1   rK    s
   
"zThreadPool._help_stuff_finish)NNr0   )
rd   re   rf   dummyr  r   rO   r  rV  rK  r0   r0   r0   r1   r    s    
r  r+   )erQ  r   r9   r   r   r   r   r@   r   r  collectionsr   	functoolsr    r   r   r   commonr   r	   r
   r   r   compatr   r   r   r   r   r  r   
exceptionsr   r   r   r   r   r   r   r   r  r   r   r   r   r    r   version_inforh   system_winr#   r  rI  r$   rN  r,   	SemaphorerN   r   r  r  r   r   r   r   r   r)   r   r   r  r   r  r   r   countr[  r  r2   r7   r;   r=   rE   rF   r   ri   rt   rw   ry   r   r  r  r&  r_  r  r/  r<  r$  r*  r  r0   r0   r0   r1   <module>   s   $	



;  )%K  :     o =R