a sDfG@sddlmZmZmZddlmZddlmZddlZddlZddl Z ddl Z ddl Z ddl Z ddZd9d d Zd:d d Zdd lmZddlZddlZddlZddlZddlZddlZz ddlZWneydZYn0ejZdZdZdZdZ dZ!gdZ"dZ#dZ$dZ%dZ&Gddde'Z(ddde!e ddfddZ)ej*efddZ+Gdd d e'Z,d!d"Z-d#d$Z.d;d%d&Z/dd5d6Z6d7d8Z7dS)?)print_functionabsolute_importdivision) __version__)reportNcCshtjtt|}tj|dtj|ddg}|D]}tj|r:|Sq:t d||fdS)Nexamplesz..z+Could not find examples for %s at any of %s) ospathdirnameinspectgetfile importlib import_modulejoinexists ValueError)name module_path candidates candidaterU/nfs/NAS7/SABIOD/METHODE/ermites/ermites_venv/lib/python3.9/site-packages/pyct/cmd.py_find_exampless   rFcCs$t||||t||d||ddS)z Copy examples and fetch data (if any) to the supplied path. See copy-examples and fetch-data for more flexibility. NOTE: force operates both on example and data over-writing pre-existing files. F)require_datasets use_test_dataforceN) copy_examples fetch_data)rr verboserrrrrrsrcCsbt|}tj|}tj|r0|s0td||r@td|tjj |||dtd|dS)z#Copy examples to the supplied path.zSPath %s already exists; please move it away, choose a different path, or use force.zCopying examples from %s)rzCopied examples to %sN) rr r abspathrrprint distutilsdir_util copy_tree)rr rrsourcerrrr*s   r) OrderedDictz%s[%s%s] %i/%i - %s z %s %s %i/%i .# )|/-\ dataz .data_stubsc@sNeZdZddZddZdddeeddfd d Zdd d Zd dZ ddZ dS)BarcCs|SNrselfrrr __enter__rsz Bar.__enter__cCs |dS)NF)done)r3exc_typeexc_valexc_tbrrr__exit__usz Bar.__exit__ NrcCs||_||_||_|durDzt |_WntyBd|_Yn0||_||_||_||_ t |_ g|_ d|_ t |_||j |_d|_|jr|ddS)z3Bar is a class for printing the status of downloadsNTr)labelwidthhideSTREAMisattyAttributeError empty_char filled_char expected_sizeeverytimestartittimesetaetadelta format_timeetadisp last_progressshow)r3r<r=r>rBrCrDrErrr__init__ys(    z Bar.__init__c Cs|dur||_|jdur td||_t|jtkrt|_|jt d|jt |dg|_t |jt t |j|j||_ | |j |_t|j||j}|js||jdks||jkrtt|j|j||j|j|||j|jftdS)Nzexpected_size not initializedrr)rD ExceptionrMrFrJ ETA_INTERVALrHETA_SMA_WINDOWrGsumfloatlenrIrKrLintr=r>rEr?write BAR_TEMPLATEr<rCrBflush)r3progresscountxrrrrNs6   zBar.showc Csft|j|_||j}|jsbtt|j|j |j |j d|j |j |ftdtdS)Nr )rFrGelapsedrKr>r?rWrXr<rCr=rBrMrDrY)r3Z elapsed_disprrrr5s   zBar.donecCstdt|S)Nz%H:%M:%S)rFstrftimegmtime)r3secondsrrrrKszBar.format_time)N) __name__ __module__ __qualname__r4r9BAR_EMPTY_CHARBAR_FILLED_CHARrOrNr5rKrrrrr0qs   r0r:r;c cst|durt|n|}t|||tt||d6} t|D]\} } | V| | dq4Wdn1sf0YdS)z/Progress iterator. Wrap your iterables with it.N)r<r=r>rBrCrDrEr)rUr0rerf enumeraterN) itr<r=r>rBrCrDrEr[bariitemrrrris rics:Gddd|}fdd}|tjjj|t||S)Nc@s eZdZdS)z#ordered_load..OrderedLoaderN)rbrcrdrrrr OrderedLoadersrlcs||||Sr1)Zflatten_mappingZconstruct_pairs)loadernodeobject_pairs_hookrrconstruct_mappings z'ordered_load..construct_mapping)Zadd_constructoryamlresolver BaseResolverZDEFAULT_MAPPING_TAGload)streamLoaderrprlrqrror ordered_loads rxc@s(eZdZdZddZddZddZdS) DirectoryContextz2 Context Manager for changing directories cCst|_||_dSr1)r getcwdold_dirnew_dir)r3r rrrrOs zDirectoryContext.__init__cCst|jdSr1)r chdirr|r2rrrr4szDirectoryContext.__enter__cGst|jdSr1)r r}r{)r3argsrrrr9szDirectoryContext.__exit__N)rbrcrd__doc__rOr4r9rrrrrysrycCstd|tj|dd}zxt|dZ}t|jd}t|jdd|ddd d D]}|rX| || qXWd n1s0YWn$t j |rt |Yn0d S) zGiven a url, output_path and title, write the contents of a requests get operation to the url in binary mode and print the title of operationzDownloading {0}T)rvwbzcontent-lengthi) chunk_sizeri)rDrEN)r!formatrequestsgetopenrVheadersri iter_contentrWrYr r rremove)url output_pathtitlerespf total_lengthchunkrrr_url_to_binary_writes " ,  rcCs.|drJt|d}|Wdn1s40Yt|n|drt|d}|Wdn1s~0Yt|n|drt|d}|Wdn1s0Yt|nL|dr*t|d }|Wdn1s0Yt|dS) zEExtract a local archive, e.g. zip or tar, then delete the archiveztar.gzzr:gzNtarzr:ztar.bz2zr:bz2zipr)endswithtarfiler extractallr rzipfileZipFile)rrZzipfrrr_extract_downloaded_archives  &  &  &  (rcsLtj|st|t|d}|dgD]}tj|s2d}qLq2|durz|sztd|dWddS|d|dd}d rԇfd d |dD}fd d |dD} d d | D} n6g}tj dg} |d} t | t t fs | g} t || | } t| D]\} \} }|| dt|}|dur~t|sltj|ddr~td|qtj|t|}|rtj|rtj||}td||t||qn"|rtj|std|t| |t| qWdn1s"0YtdurHtdtddS)ayProcess each download spec in datasets.yml Typically each dataset list entry in the yml has "files" and "url" and "title" keys/values to show local files that must be present / extracted from a decompression of contents downloaded from the url. If a url endswith '/', then all files given are assumed to be added to the url pattern at the end FfilesTz Skipping {0}rNrz {} of {}r+csg|] }|qSrr).0r)rrr )z$_process_dataset..csg|]}tjt|qSr)r r rDATA_DIR)rfname)hererrr*sc Ss6g|].}d|dd|dr$dnddqS)r'Ngz*)rsplitr)rrrrrr-srrr:z%Copying test data file '{0}' to '{1}'z6No test file found for: {}. Using regular file insteadzIthis download script requires the requests module: conda install requests)r r rmakedirsryrr!rrr isinstancetuplelistrrgrUglobreplacerDATA_STUBS_DIRshutilcopyfilerrrsysexit)Zdataset output_dirrrrZrequires_downloadrZ title_fmturlsZ output_pathsunpackedZzippedidxrunpackZ running_titletesttargetr)rrr_process_dataset sZ         ,  , r datasets.ymlTc Cstj|}tj||}tj|s8tjt||}tj|sX|durXtddStd|tj|tft|X}t | }| D]2\} } tj|| } | D]} t | | |||dqqWdn1s0YdS)zFetch sample datasets as defined by path/datasets if it exists or else module's own examples/datasets otherwise. Datasets are placed in path/data FzNo datasets to downloadNz-Fetching data defined in %s and placing in %s)rr) r r r rrrr!rrrxreaditemsr) rr ZdatasetsrrrZ info_filerinfoZtopicZ downloadsrdrrrrMs    rc Cstj|}tj|s t|}tj|t}tj|t}tj|rZtt |dkrft ddSt |D]}tj||}tj |st d|qptj||}tj |rptj |}tj |}||krt d|t |qpt d||qpdS)z=Remove up any data files that are copied from test files rzNo test files foundNz!Test file was not copied to data:zRemoving copied test file:z?Size of test file {:.2e} did not match size of data file {:.2e})r r r rrrrrrUlistdirr!isfilegetsizerr) rr data_dirZtest_dirrZ data_fileZ test_fileZdata_sZtest_srrr clean_datads.         rcGs>d|vr|jdddddd|vr:|jdtdd|d dS) N-vz --verboser[r)actiondefault--pathzwhere to place outputz {}-examples)typehelpr) add_argumentstrr)parserrr~rrr_add_common_argssrcs|jfddddS)Ncs"fifddtDS)Ncs i|]}|dkr|t|qSfunc)getattr)rkr~rr rz3_set_defaults....)varsrfnrrrrz_set_defaults..r) set_defaults)rrrrrr _set_defaultssrcCsN|durgd}d|vrV|jdttd}|jddddt||d d t||td |vr|jd ttd}|jd td dd|jdddd|jddddt||d t||td|vr|jdtt d}|jdddd|jddddt||d d t||t d|vrJ|jdtt d}t||d t||t dS)zF Add all commands in pyct.cmd unless specific cmds are listed N)r copy-examples fetch-data clean-datarrz--force store_truezQif PATH already exists, force overwrite existing files if older than source files)rrrrrz --datasetsrz^*name* of datasets file; must exist either in path specified by --path or in package/examples/)rrrz,Force any existing data files to be replacedz--use-test-datazUse data's test files, if any, instead of fetching full data. If test file not in '.data_stubs', fall back to fetching full data.rzif PATH already exists, force overwrite existing examples if older than source examples. ALSO force any existing data files to be replacedr) add_parserr getdocrrrrrrrr)rrcmdsr~Z eg_parserZd_parserZ egd_parserZ cd_parserrrr add_commandssD        rcCs$t|}|jddd|jddS)N --versionversion %(prog)s rr)rrrr)rrmodrrr add_versions rcCs^tjd|d}|jdd}t||||t|||}t|drP||n|ddS)z_ If module has no other commands, use this function to add all of the ones in pyct.cmd z %s commands descriptionavailable commandsrrmust supply command to runN) argparseArgumentParseradd_subparsersrr parse_argshasattrrerror)rrr~r subparsersrrrsubstitute_mains   rcCstjdd}|jdddtd|jdd}|jd ttd }|j td |jd d t ddd| }t |dr|j |jn |ddS)NzCommands relating to versioningrrrrrrrrrrpackagespackage+zname of package)metavarrnargsrrr)rrrrrrr rrrrrrrrr)rrZ report_parserr~rrrmains     r)FFF)FF)FF)rTFF)NN)NN)8 __future__rrrr:rrr rr rdistutils.dir_utilr"rrrr collectionsr&rrrrFrrrr ImportErrorstderrr?rXZ MILL_TEMPLATEZ DOTS_CHARrfreZ MILL_CHARSrQrRrrobjectr0rirwrxryrrrrrrrrrrrrrrrsd      J   @  ,