a %Dfp @sUddlmZddlZddlZddlZddlZddlZddlZddlZddl m Z m Z ddl m Z mZmZmZmZmZmZmZe rddlZddlmZedZnedZedZedZeeegefZGd d d ZGd d d eZ Gd ddeZ!GdddeZ"GdddeZ#GdddeZ$GdddeZ%GdddeZ&GdddeeefZ'GdddeZ(deiZ)de*d<d&d d!d"d#d$d%Z+ee e$e!e#e"e%e&e(f D]Z,e+e,qdS)') annotationsN)FutureThreadPoolExecutor) TYPE_CHECKINGAnyCallableClassVarGeneric NamedTuple OrderedDictTypeVar) ParamSpecPTZfsspecc@sDeZdZUdZdZded<dddddd d Zd d d d ddZdS) BaseCacheagPass-though cache: doesn't keep anything, calls every time Acts as base class for other cachers Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is none ClassVar[str]nameintFetcherNone blocksizefetchersizereturncCs||_||_||_dSN)rrrselfrrrr[/nfs/NAS7/SABIOD/METHODE/ermites/ermites_venv/lib/python3.9/site-packages/fsspec/caching.py__init__9szBaseCache.__init__ int | NonebytesstartstoprcCs<|dur d}|dur|j}||jks,||kr0dS|||SNr)rrrr%r&rrr _fetch>szBaseCache._fetchN)__name__ __module__ __qualname____doc__r__annotations__r!r*rrrr r's  rcsteZdZdZdZdddddddd fd d Zd d ddZddddddZdd ddZdddddZ Z S) MMapCachezmemory-mapped sparse file cache Opens temporary file, which is filled blocks-wise when data is requested. Ensure there is enough disc space in the temporary location. This cache method might only work on posix mmapNrrz str | Nonezset[int] | Noner)rrrlocationblocksrcs8t||||durtn||_||_||_dSr)superr!setr3r2 _makefilecache)rrrrr2r3 __class__rr r!SszMMapCache.__init__zmmap.mmap | bytearrayrcCsddl}ddl}|jdkr tS|jdus8tj|js|jdurT|}t |_ n t |jd}| |jd| d|n t |jd}|||jS)Nrzwb+1zr+b)r1tempfiler bytearrayr2ospathexists TemporaryFiler5r3openseekwriteflushfileno)rr1r=fdrrr r6`s       zMMapCache._makefiler"r#r%endrc std|d||dur"d}|dur0j}|jksB||krFdS|j}|j}fddt||dD}|r|d}|j}t|jj}td|d |d|||j||<j |qvj||S) NzMMap cache fetching -rr(csg|]}|jvr|qSr)r3).0irrr r(z$MMapCache._fetch..r;zMMap get block #z () loggerdebugrrrangepopminrr7r3add) rr%rJZ start_blockZ end_blockZneedrMZsstartsendrrNr r*vs$    zMMapCache._fetchdict[str, Any]cCs|j}|d=|S)Nr7)__dict__copyrstaterrr __getstate__s zMMapCache.__getstate__r[rcCs|j|||_dSr)rXupdater6r7rZrrr __setstate__s zMMapCache.__setstate__)NN) r+r,r-r.rr!r6r*r\r_ __classcell__rrr8r r0Hs r0csBeZdZdZdZdddddfdd Zd d d d d d ZZS)ReadAheadCachea!Cache which reads only when we get beyond a block of data This is a much simpler version of BytesCache, and does not attempt to fill holes in the cache or keep fragments alive. It is best suited to many small reads in a sequential order (e.g., reading lines from a file). Z readaheadrrrrcs&t|||d|_d|_d|_dS)Nr(r)r4r!r7r%rJrr8rr r!szReadAheadCache.__init__r"r#rIcCs|dur d}|dus||jkr$|j}||jks6||kr:dS||}||jkrp||jkrp|j||j||jS|j|kr|jkrnn(|j||jd}|t|8}|j}nd}t|j||j}||||_||_|jt|j|_||jd|Sr')rr%rJr7lenrTrr)rr%rJlpartrrr r*s& zReadAheadCache._fetchr+r,r-r.rr!r*r`rrr8r rasracsBeZdZdZdZdddddfdd Zd d d d d d ZZS)FirstChunkCachezCaches the first block of a file only This may be useful for file types where the metadata is stored in the header, but is randomly accessed. firstrrrrcst|||d|_dSr)r4r!r7rr8rr r!szFirstChunkCache.__init__r"r#rIcCs|pd}|p|j}||jkr|jdurh||jkrX|d|}|d|j|_||dS|d|j|_|j||}||jkr|||j|7}|S|||SdSNr)rrr7r)rr%rJdatardrrr r*s       zFirstChunkCache._fetchrerrr8r rfsrfcseZdZdZdZd!ddddddfdd Zd d d d ZddZdd ddZdddddZ ddddddZ dddfdd Z dddddddd Z Z S)" BlockCachea Cache holding memory as a set of blocks. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. Z blockcache rrrrrr maxblocksrcs<t|||t|||_||_t||j|_ dSr) r4r!mathceilnblocksrm functools lru_cache _fetch_block_fetch_block_cachedrrrrrmr8rr r!szBlockCache.__init__strr:cCsd|jd|jd|jdS)NzrrrprNrrr __repr__s  zBlockCache.__repr__cCs |jSz The statistics on the block cache. Returns ------- NamedTuple Returned directly from the LRU Cache used internally. rt cache_inforNrrr r~ s zBlockCache.cache_inforWcCs|j}|d=|S)NrtrXrZrrr r\szBlockCache.__getstate__r]cCs&|j|t|d|j|_dS)Nrm)rXr^rqrrrsrtrZrrr r_s  zBlockCache.__setstate__r"r#rIcCst|dur d}|dur|j}||jks,||kr0dS||j}||j}t||dD]}||qR|j||||dS)Nrr(r;start_block_numberend_block_number)rrrRrt _read_cache)rr%rJrr block_numberrrr r*s    zBlockCache._fetch)rrcsT||jkr"td|d|jd||j}||j}td|t||}|S)= Fetch the block of data for `block_number`. 'block_number=(' is greater than the number of blocks ()zBlockCache fetching block %drp ValueErrorrrPinfor4r*)rrr%rJblock_contentsr8rr rs6s    zBlockCache._fetch_blockr%rJrrrc Cs||j}||j}||kr2||}|||S|||dg}|t|jt|d||||d|d|SdSz Read from our block cache. Parameters ---------- start, end : int The start and end byte positions. start_block_number, end_block_number : int The start and end block numbers. Nr;r(rrtextendmaprRappendjoin rr%rJrrZ start_posZend_posblockoutrrr rFs     zBlockCache._read_cache)rk)r+r,r-r.rr!r{r~r\r_r*rsrr`rrr8r rjs rjcs^eZdZUdZdZded<dddddd d fd d Zd d ddddZddddZZ S) BytesCacheaKCache which holds data in a in-memory bytes object Implements read-ahead by the block size, for semi-random reads progressing through the file. Parameters ---------- trim: bool As we read more data, whether to discard the start of the buffer when we are more than a blocksize ahead of it. r#rrTrrboolr)rrrtrimrcs,t|||d|_d|_d|_||_dS)Nr()r4r!r7r%rJr)rrrrrr8rr r!}s zBytesCache.__init__r"rIcCs\|dur d}|dur|j}||jks,||kr0dS|jdurx||jkrx|jdurx||jkrx||j}|j||||S|jrt|j||j}n|}||ks||jkrdS|jdus||jkr|jdus||jkr||||_||_n|jdusJ|jdusJ||jkrj|jdus2|j||jkrH||||_||_n |||j}||_||j|_nj|jdur||jkr|j|jkrnB||j|jkr||||_||_n||j|}|j||_|jt|j|_||j}|j||||}|jrX|j|j|jd}|dkrX|j|j|7_|j|j|d|_|S)Nrr(r;) rr%rJr7rrTrrbr)rr%rJoffsetZbendnewrnumrrr r*sl     zBytesCache._fetchr:cCs t|jSr)rbr7rNrrr __len__szBytesCache.__len__)T) r+r,r-r.rr/r!r*rr`rrr8r rns   ArcsPeZdZUdZdZded<dddddd d fd d Zddd dddZZS)AllBytesz!Cache entire contents of the fileallrrNr"zFetcher | Nonez bytes | Noner)rrrrircs0t||||dur&|d|j}||_dSrh)r4r!rrri)rrrrrir8rr r!szAllBytes.__init__r#r$cCs|j||Sr)rir)rrr r*szAllBytes._fetch)NNNN r+r,r-r.rr/r!r*r`rrr8r rs   rcsZeZdZUdZdZded<idfddddd d d fd d Zddddfdd ZZS)KnownPartsOfAFilea Cache holding known file parts. Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is data: dict A dictionary mapping explicit `(start, stop)` file-offset tuples with known bytes. strict: bool, default True Whether to fetch reads that go beyond a known byte-range boundary. If `False`, any read that ends outside a known part will be zero padded. Note that zero padding will not be used for reads that begin outside a known byte-range. partsrrTrrzdict[tuple[int, int], bytes]rr)rrrristrict_c st|||||_|rt|}|dg}||dg} |ddD]f\} } |d\} } | | kr| | f|d<| d|| | f7<qL|| | f| || | fqLtt|| |_ n||_ dS)Nrr;) r4r!rsortedkeysrSrdictzipri)rrrrrirrZ old_offsetsoffsetsr3r%r&Zstart0Zstop0r8rr r!s     zKnownPartsOfAFile.__init__r"r#r$cs |dur d}|dur|j}d}|jD]\\}}}||krH|kr(nq(||}|||||}|jr||kr|krnn |d||t|7}|S|}qq(|jdurtd||fdtd||fdt d|d||t ||S) Nrr(z&Read is outside the known file parts: z. z%. IO/caching performance may be poor!z!KnownPartsOfAFile cache fetching rK) rriitemsrrbrrwarningswarnrPrQr4r*)rr%r&rZloc0loc1rioffr8rr r*s* zKnownPartsOfAFile._fetchrrrr8r rs  rc@sveZdZdZGdddeZdddddd d Zd d d dddZdddddZd dddddZ ddddZ dS) UpdatableLRUzg Custom implementation of LRU cache that allows updating keys Used by BackgroudBlockCache c@s.eZdZUded<ded<ded<ded<dS)UpdatableLRU.CacheInforhitsmissesmaxsizecurrsizeN)r+r,r-r/rrrr CacheInfoIs rzCallable[P, T]rr)funcmax_sizercCs0t|_||_||_d|_d|_t|_ dSrh) collectionsr _cache_func _max_size_hits_misses threadingLock_lock)rrrrrr r!Os  zUpdatableLRU.__init__zP.argszP.kwargsr)argskwargsrcOs|rtd||jL||jvrZ|j||jd7_|j|WdSWdn1sn0Y|j|i|}|jF||j|<|jd7_t|j|j kr|jj ddWdn1s0Y|S)Nz Got unexpected keyword argument r;Flast) TypeErrorrrr move_to_endrrrrbrpopitem)rrrresultrrr __call__Ws  6 ,zUpdatableLRU.__call__rr)rrcGs4|j||jvWdS1s&0YdSr)rr)rrrrr is_key_cachedjszUpdatableLRU.is_key_cached)rrrcGsR|j8||j|<t|j|jkr0|jjddWdn1sD0YdS)NFr)rrrbrr)rrrrrr add_keyns zUpdatableLRU.add_keyrr:cCsH|j.|j|jt|j|j|jdWdS1s:0YdS)N)rrrr)rrrrbrrrrNrrr r~tszUpdatableLRU.cache_infoN)r) r+r,r-r.r rr!rrrr~rrrr rBsrcseZdZUdZdZded<d$dddddd fd d Zd d ddZdd ddZdd ddZ dd ddZ ddddddZ d%dd ddfdd Z dddddd!d"d#Z ZS)&BackgroundBlockCachea Cache holding memory as a set of blocks with pre-loading of the next block in the background. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. If the next block is not in cache, it is loaded in a separate thread in non-blocking way. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. Z backgroundrrrkrrrrlcsZt|||t|||_||_t|j||_t dd|_ d|_ d|_ t |_dS)Nr; max_workers)r4r!rnrorprmrrsrtr_thread_executor_fetch_future_block_number _fetch_futurerr_fetch_future_lockrur8rr r!s zBackgroundBlockCache.__init__rvr:cCsd|jd|jd|jdS)Nz sV(    !Q* ]b<R