a
    wDfh/                     @  s   d dl mZ d dlmZ d dlmZ d dlmZmZm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ ed	d
ZG dd deZeddZG dd deZG dd deZeddZG dd deZdS )    )annotations)
namedtuple)starmap)PipeProcesscurrent_process)sleep)default_timer)Callback)import_requiredTaskData)keytask
start_timeend_timeZ	worker_idc                      sl   e Zd ZdZdd Z fddZ fddZdd	 Zd
d Zdd Z	dd Z
dd Zdd Zdd Z  ZS )Profilera  A profiler for dask execution at the task level.

    Records the following information for each task:
        1. Key
        2. Task
        3. Start time in seconds since the epoch
        4. Finish time in seconds since the epoch
        5. Worker id

    Examples
    --------

    >>> from operator import add, mul
    >>> from dask.threaded import get
    >>> from dask.diagnostics import Profiler
    >>> dsk = {'x': 1, 'y': (add, 'x', 10), 'z': (mul, 'y', 2)}
    >>> with Profiler() as prof:
    ...     get(dsk, 'z')
    22

    >>> prof.results        # doctest: +SKIP
    [TaskData(key='y', task=(add, 'x', 10), start_time=..., end_time=..., worker_id=...),
     TaskData(key='z', task=(mul, 'y', 2), start_time=..., end_time=..., worker_id=...)]

    These results can be visualized in a bokeh plot using the ``visualize``
    method. Note that this requires bokeh to be installed.

    >>> prof.visualize()    # doctest: +SKIP

    You can activate the profiler globally

    >>> prof.register()

    If you use the profiler globally you will need to clear out old results
    manually.

    >>> prof.clear()
    >>> prof.unregister()

    c                 C  s"   i | _ g | _i | _d | _d | _d S N)_resultsresults_dskr   r   self r   e/nfs/NAS7/SABIOD/METHODE/ermites/ermites_venv/lib/python3.9/site-packages/dask/diagnostics/profile.py__init__<   s
    zProfiler.__init__c                   s   |    t | _t  S r   clearr	   r   super	__enter__r   	__class__r   r   r   C   s    zProfiler.__enter__c                   s   t  | _t j| S r   r	   r   r   __exit__r   argsr   r   r   r"   H   s    zProfiler.__exit__c                 C  s   | j | d S r   r   updater   dskr   r   r   _startL   s    zProfiler._startc                 C  s   t  }||| |f| j|< d S r   r	   r   )r   r   r(   statestartr   r   r   _pretaskO   s    zProfiler._pretaskc                 C  s    t  }| j|  ||f7  < d S r   r*   )r   r   valuer(   r+   idendr   r   r   	_posttaskS   s    zProfiler._posttaskc                 C  s>   dd | j  D }|  jttt| 7  _| j   d S )Nc                 S  s"   i | ]\}}t |d kr||qS )   )len).0kvr   r   r   
<dictcomp>X       z$Profiler._finish.<locals>.<dictcomp>)r   itemsr   listr   r   valuesr   )r   r(   r+   failedr   r   r   r   _finishW   s    zProfiler._finishc                 K  s*   ddl m} || j| j| j| jfi |S )Nr   )
plot_tasks)"dask.diagnostics.profile_visualizer>   r   r   r   r   )r   kwargsr>   r   r   r   _plot\   s    zProfiler._plotc                 K  s   ddl m} || fi |S zVisualize the profiling run in a bokeh plot.

        See also
        --------
        dask.diagnostics.profile_visualize.visualize
        r   )	visualizer?   rC   r   r@   rC   r   r   r   rC   c   s    zProfiler.visualizec                 C  s,   | j   | jdd= i | _d| _d| _dS z#Clear out old results from profilerN)r   r   r   r   r   r   r   r   r   r   r   n   s
    
zProfiler.clear)__name__
__module____qualname____doc__r   r   r"   r)   r-   r1   r=   rA   rC   r   __classcell__r   r   r   r   r      s   )r   ResourceData)timememcpuc                      s   e Zd ZdZdddZdd Zdd Zd	d
 Z fddZ fddZ	dd Z
dd Zdd ZeZdd Zdd Zdd Z  ZS )ResourceProfilera   A profiler for resource use.

    Records the following each timestep
        1. Time in seconds since the epoch
        2. Memory usage in MB
        3. % CPU usage

    Examples
    --------

    >>> from operator import add, mul
    >>> from dask.threaded import get
    >>> dsk = {'x': 1, 'y': (add, 'x', 10), 'z': (mul, 'y', 2)}
    >>> with ResourceProfiler() as prof:
    ...     get(dsk, 'z')
    22

    These results can be visualized in a bokeh plot using the ``visualize``
    method. Note that this requires bokeh to be installed.

    >>> prof.visualize() # doctest: +SKIP

    You can activate the profiler globally

    >>> prof.register()

    If you use the profiler globally you will need to clear out old results
    manually.

    >>> prof.clear()

    Note that when used as a context manager data will be collected throughout
    the duration of the enclosed block. In contrast, when registered globally
    data will only be collected while a dask scheduler is active.

    >>> prof.unregister()
       c                 C  s(   || _ d| _d | _g | _d | _d | _d S NF)_dt_entered_trackerr   r   r   r   dtr   r   r   r      s    zResourceProfiler.__init__c                 C  s   | j d uo| j  S r   )rU   is_aliver   r   r   r   _is_running   s    zResourceProfiler._is_runningc                 C  s0   |   st| j| _| j  | jjd d S )Ncollect)rY   _TrackerrS   rU   r,   parent_connsendr   r   r   r   _start_collect   s    
zResourceProfiler._start_collectc                 C  s4   |   r0| jjd | jtt| jj  d S )N	send_data)	rY   rU   r\   r]   r   extendr   rL   recvr   r   r   r   _stop_collect   s    zResourceProfiler._stop_collectc                   s(   d| _ |   t | _|   t  S NT)rT   r   r	   r   r^   r   r   r   r   r   r   r      s
    zResourceProfiler.__enter__c                   s.   d| _ |   |   t | _t j|  d S rR   )rT   rb   closer	   r   r   r"   r#   r   r   r   r"      s
    zResourceProfiler.__exit__c                 C  s   |    d S r   )r^   r'   r   r   r   r)      s    zResourceProfiler._startc                 C  s   | j s|   d S r   )rT   rb   )r   r(   r+   r<   r   r   r   r=      s    zResourceProfiler._finishc                 C  s   |   r| j  d| _dS )z%Shutdown the resource tracker processN)rY   rU   shutdownr   r   r   r   rd      s    
zResourceProfiler.closec                 C  s   g | _ d | _d | _d S r   )r   r   r   r   r   r   r   r      s    zResourceProfiler.clearc                 K  s&   ddl m} || j| j| jfi |S )Nr   )plot_resources)r?   rf   r   r   r   )r   r@   rf   r   r   r   rA      s    zResourceProfiler._plotc                 K  s   ddl m} || fi |S rB   rD   rE   r   r   r   rC      s    zResourceProfiler.visualize)rQ   )rG   rH   rI   rJ   r   rY   r^   rb   r   r"   r)   r=   rd   __del__r   rA   rC   rK   r   r   r   r   rP   z   s   &
rP   c                      s:   e Zd ZdZd fdd	Zdd Zdd Zd	d
 Z  ZS )r[   z.Background process for tracking resource usagerQ   c                   s2   t    d| _|| _t j| _t \| _| _	d S rc   )
r   r   daemonrW   r   pid
parent_pidr   r\   
child_connrV   r   r   r   r      s
    

z_Tracker.__init__c                 C  s*   | j js| j d | j   |   d S )Nre   )r\   closedr]   rd   joinr   r   r   r   re      s    
z_Tracker.shutdownc                   s    | j g fdd| j  D  S )Nc                   s&   g | ]}|j  kr| d kr|qS )Zzombie)ri   status)r4   pri   r   r   
<listcomp>   s   z)_Tracker._update_pids.<locals>.<listcomp>)parentchildren)r   ri   r   rp   r   _update_pids   s    z_Tracker._update_pidsc              	   C  s  t dd}|| j| _t }g }z| j }W n tyF   Y q"Y n0 |dkrVqq"|dkr| |}|rv| j	 st
 }d }}|D ]>}	z|	 j}
|	 }W n ty   Y q0 ||
7 }||7 }q|||d |f t| j qhq"|dkr"| j| g }q"| j  d S )Npsutilz9Tracking resource usage requires `psutil` to be installedre   rZ   r   g    .Ar_   )r   r   rj   rr   r   rk   ra   KeyboardInterruptrt   pollr	   Zmemory_infoZrssZcpu_percent	Exceptionappendr   rW   r]   rd   )r   ru   ri   datamsgZpsZticrN   rO   ro   Zmem2Zcpu2r   r   r   run   s>    



z_Tracker.run)rQ   )	rG   rH   rI   rJ   r   re   rt   r|   rK   r   r   r   r   r[      s
   r[   	CacheData)r   r   metricZ
cache_timeZ	free_timec                      sf   e Zd ZdZdddZ fddZ fddZd	d
 Zdd Zdd Z	dd Z
dd Zdd Z  ZS )CacheProfilera]  A profiler for dask execution at the scheduler cache level.

    Records the following information for each task:
        1. Key
        2. Task
        3. Size metric
        4. Cache entry time in seconds since the epoch
        5. Cache exit time in seconds since the epoch

    Examples
    --------

    >>> from operator import add, mul
    >>> from dask.threaded import get
    >>> from dask.diagnostics import CacheProfiler
    >>> dsk = {'x': 1, 'y': (add, 'x', 10), 'z': (mul, 'y', 2)}
    >>> with CacheProfiler() as prof:
    ...     get(dsk, 'z')
    22

    >>> prof.results    # doctest: +SKIP
    [CacheData(key='y', task=(add, 'x', 10), metric=1, cache_time=..., free_time=...),
     CacheData(key='z', task=(mul, 'y', 2), metric=1, cache_time=..., free_time=...)]

    The default is to count each task (``metric`` is 1 for all tasks). Other
    functions may used as a metric instead through the ``metric`` keyword. For
    example, the ``nbytes`` function found in ``cachey`` can be used to measure
    the number of bytes in the cache.

    >>> from cachey import nbytes                   # doctest: +SKIP
    >>> with CacheProfiler(metric=nbytes) as prof:  # doctest: +SKIP
    ...     get(dsk, 'z')
    22

    The profiling results can be visualized in a bokeh plot using the
    ``visualize`` method. Note that this requires bokeh to be installed.

    >>> prof.visualize() # doctest: +SKIP

    You can activate the profiler globally

    >>> prof.register()

    If you use the profiler globally you will need to clear out old results
    manually.

    >>> prof.clear()
    >>> prof.unregister()

    Nc                 C  s>   |    |r|ndd | _|r&|| _n|r4|j| _nd| _d S )Nc                 S  s   dS )NrQ   r   )r.   r   r   r   <lambda>`  r8   z(CacheProfiler.__init__.<locals>.<lambda>count)r   _metric_metric_namerG   )r   r~   Zmetric_namer   r   r   r   ^  s    
zCacheProfiler.__init__c                   s   |    t | _t  S r   r   r   r   r   r   r   h  s    zCacheProfiler.__enter__c                   s   t  | _t j| S r   r!   r#   r   r   r   r"   m  s    zCacheProfiler.__exit__c                 C  s   | j | d S r   r%   r'   r   r   r   r)   q  s    zCacheProfiler._startc           
   	   C  sb   t  }| ||f| j|< |d | j @ D ]0}| j|\}}	| jt||| ||	| q,d S )NZreleased)r	   r   _cachekeyspopr   ry   r}   )
r   r   r.   r(   r+   r/   tr5   r~   r,   r   r   r   r1   t  s
    zCacheProfiler._posttaskc              	   C  sH   t  }| j D ](\}\}}| jt||| ||| q| j  d S r   )r	   r   r9   r   ry   r}   r   )r   r(   r+   r<   r   r5   r~   r,   r   r   r   r=   {  s    zCacheProfiler._finishc                 K  s.   ddl m} || j| j| j| j| jfi |S )Nr   )
plot_cache)r?   r   r   r   r   r   r   )r   r@   r   r   r   r   rA     s    zCacheProfiler._plotc                 K  s   ddl m} || fi |S rB   rD   rE   r   r   r   rC     s    zCacheProfiler.visualizec                 C  s"   g | _ i | _i | _d| _d| _dS rF   )r   r   r   r   r   r   r   r   r   r     s
    zCacheProfiler.clear)NN)rG   rH   rI   rJ   r   r   r"   r)   r1   r=   rA   rC   r   rK   r   r   r   r   r   *  s   3

r   N)
__future__r   collectionsr   	itertoolsr   multiprocessingr   r   r   rM   r   Ztimeitr	   Zdask.callbacksr
   Z
dask.utilsr   r   r   rL   rP   r[   r}   r   r   r   r   r   <module>   s$   e
p;