a ]Df@sdZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlmZddlZddlmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#ddl$m%Z%ddl&m'Z'ddl(m)Z)ddl*m+Z+dd l,m-Z-e.d Z/e.d ej0Z1e.d Z2e.d ej0j3Z4d5Z6gdZ7dZ8dZ9e9j:dj:ej;edZddZ?dBddZ@dCddZAdDddZBdedfd d!ZCd"d#ZDe.d$ej0ZEeDd%d&ZFGd'd(d(ZGGd)d*d*eGZHGd+d,d,eZIe.d-jJZKd.d/ZLd0d1ZMdEd2d3ZNd4d5ZOGd6d7d7ZPGd8d9d9e jQZRejSjTfd:d;ZUdd?ZWd@dAZXdS)Fz$PyPI and direct package downloading.Nwraps) CHECKOUT_DIST Distribution BINARY_DISTnormalize_path SOURCE_DIST Environmentfind_distributions safe_name safe_version to_filename Requirement DEVELOP_DISTEGG_DIST parse_version)logDistutilsError) translate)Wheelunique_everseenz^egg=([-A-Za-z0-9_.+!]+)$zhref\s*=\s*['"]?([^'"> ]+)z([^<]+)\n\s+\(md5\)z([-+.a-z0-9]{2,}):z.tar.gz .tar.bz2 .tar .zip .tgz) PackageIndexdistros_for_urlparse_bdist_wininstinterpret_distro_namezrR)filenamerHr#r#r$distros_for_filenamesrXc cs|d}|s,tdd|ddDr,dStt|D]}||ddr8q^q8t|}t||d|d|d||d|||dVdS)zGenerate the interpretation of a source distro name Note: if `location` is a filesystem filename, you should call ``pkg_resources.normalize_path()`` on it before passing it to this routine! rJcss|]}td|VqdS)z py\d\.\d$N)rerF).0pr#r#r$ z(interpret_distro_name..Nr5) py_versionrCrT)r9anyrangerQisdigitrjoin)rKrRrHr_rCrTr;r[r#r#r$rs rcstfdd}|S)zs Wrap a function returning an iterable such that the resulting iterable only ever yields unique items. cst|i|SNr)argskwargsfuncr#r$wrapperszunique_values..wrapperr)rhrir#rgr$ unique_valuessrjz3<([^>]*\srel\s{0,10}=\s{0,10}['"]?([^'" >]+)[^>]*)>c cst|D]d}|\}}tttj|d}d|vsDd|vr t |D]}t j |t |dVqNq dD]@}||}|dkrtt ||}|rtt j |t |dVqtdS)zEFind rel="homepage" and rel="download" links in `page`, yielding URLs,Zhomepager2r5)z Home PagezDownload URLr1N)RELfinditergroupssetmapstrstripr)r9HREFr6rurljoin htmldecoderGfindsearch)r:pagerFtagrelZrelsposr#r#r$find_external_linkss   r|c@s(eZdZdZddZddZddZdS) ContentCheckerzP A null content checker that defines the interface for checking content cCsdS)z3 Feed a block of data to the hash. Nr#selfblockr#r#r$feedszContentChecker.feedcCsdS)zC Check the hash. Return False if validation fails. Tr#rr#r#r$is_validszContentChecker.is_validcCsdS)zu Call reporter with information about the checker (hash name) substituted into the template. Nr#)rreportertemplater#r#r$reportszContentChecker.reportN)__name__ __module__ __qualname____doc__rrrr#r#r#r$r}sr}c@sBeZdZedZddZeddZddZ dd Z d d Z d S) HashCheckerzK(?Psha1|sha224|sha384|sha256|sha512|md5)=(?P[a-f0-9]+)cCs||_t||_||_dSrd) hash_namehashlibnewhashexpected)rrrr#r#r$__init__ s zHashChecker.__init__cCsBtj|d}|stS|j|}|s0tS|fi|S)z5Construct a (possibly null) ContentChecker from a URLr1)r6rr7r}patternrw groupdict)clsr:rArFr#r#r$from_urls zHashChecker.from_urlcCs|j|dSrd)rupdater~r#r#r$rszHashChecker.feedcCs|j|jkSrd)r hexdigestrrr#r#r$r szHashChecker.is_validcCs||j}||Srd)r)rrrmsgr#r#r$r#s zHashChecker.reportN) rrrrYcompilerr classmethodrrrrr#r#r#r$rs rcsTeZdZdZdNfdd Zfdd ZdOd d ZdPd dZdQddZddZ ddZ ddZ ddZ ddZ dRddZddZdSfdd Zd!d"Zd#d$Zd%d&Zd'd(Zd)d*ZdTd+d,ZdUd-d.Zd/d0Zd1Zd2d3Zd4d5ZdVd6d7Zd8d9Zd:d;Zdd?Z d@dAZ!e"dWdBdCZ#dDdEZ$dFdGZ%dHdIZ&dJdKZ'dLdMZ(Z)S)Xrz;A distribution index that scans web pages for download URLshttps://pypi.org/simple/*NTcsltj|i||dd|d |_i|_i|_i|_td t t |j |_ g|_tjj|_dS)Nr0|)superrr* index_url scanned_urls fetched_urls package_pagesrYrrcrprrFallowsto_scanr6requesturlopenopener)rrhostsZ ca_bundleZ verify_sslrekw __class__r#r$r+s zPackageIndex.__init__cs0zt|jWnty"YdS0t|Srd)rrM Exceptionradd)rdistrr#r$r=s  zPackageIndex.addFc Cs||jvr|sdSd|j|<t|s2||dStt|}|r\||sPdS|d||sn|rn||jvrtt|j |dS||sd|j|<dS| d|d|j|<d}| |||}|durdSt |t jjr|jdkr| d|jd|j|j<d|jd d vr(|dS|j}|}t |tsvt |t jjrXd }n|jd phd }||d }|t|D](} t j|t| d} |!| q|"|j#rt$|dddkr|%||}dS)zexistswarnisdirrealpathlistdirrrcrXrrrpr)rfnnestedr>itemrr#r#r$r{s    zPackageIndex.process_filenamecCsft|}|o|ddk}|s8|tj|drentryr#r#r$r\s z.PackageIndex.scan_egg_links..)filterrVr>rr itertoolsstarmap scan_egg_link)r search_pathdirsZ egg_linksr#r#r$scan_egg_linkss zPackageIndex.scan_egg_linkscCsttj||&}ttdttj|}Wdn1s>0Yt |dkrXdS|\}}t tj||D]*}tjj|g|R|_ t |_ ||qrdS)Nr^)openrVr>rcrrrprqrrrQr rKrrCr)rr>rZ raw_lineslinesZegg_pathZ setup_pathrr#r#r$rs4 zPackageIndex.scan_egg_linkcCsd}||js|Stttjj|t|jdd}t|dksRd|dvrV|St |d}t |d}d|j | i|<t|t|fS)N)NNr0r^r4r5rT)r+rrrpr6rr8rQr9r r r setdefaultr)r )rrZNO_MATCH_SENTINELr;pkgverr#r#r$_scans &  zPackageIndex._scanc Cst|D]:}z"|tj|t|dWq tyBYq 0q ||\}}|s\dSt ||D]H}t |\}}| dr|s|r|d||f7}n | || |qftdd|S)z#Process the contents of a PyPI pager5r.pyz #egg=%s-%scSsd|dddS)Nz%sr5r^)rG)mr#r#r$r]z,PackageIndex.process_index..)rsrmrr6rrtrurGr r|rBr*need_version_infoscan_urlPYPI_MD5sub) rr:rxrFrrnew_urlr-fragr#r#r$rs$"    zPackageIndex.process_indexcCs|d|dS)NzPPage at %s links to .py file(s) without version info; an index scan is required.)scan_allrr:r#r#r$rszPackageIndex.need_version_infocGs<|j|jvr,|r"|j|g|R|d||jdS)Nz6Scanning index of all packages (this may take a while))rrrrrrrrer#r#r$rs   zPackageIndex.scan_allcCsz||j|jd|j|js:||j|jd|j|jsR||t|j|jdD]}||qfdS)Nr0r#) rr unsafe_namerrkeyrLnot_found_in_indexr)r requirementr:r#r#r$ find_packagess zPackageIndex.find_packagescsN|||||jD]"}||vr0|S|d||qt||S)Nz%s does not match %s)prescanrrrrobtain)rr installerrrr#r$rs zPackageIndex.obtaincCsL||jd||sH|t|td|jjtj |fdS)z- checker is a ContentChecker zValidating %%s checksum for %sz7%s validation failed for %s; possible download problem?N) rrrrrVunlinkrrr,r>rR)rcheckerrWtfpr#r#r$ check_hashs zPackageIndex.check_hashcCsN|D]D}|jdus0t|r0|ds0tt|r<||q|j|qdS)z;Add `urls` to the list that will be prescanned for searchesNfile:)rrr+rrrappend)rurlsr:r#r#r$add_find_linkss  zPackageIndex.add_find_linkscCs"|jrtt|j|jd|_dS)z7Scan urls scheduled for prescanning (e.g. --find-links)N)rrrprrr#r#r$r%szPackageIndex.prescancCs<||jr|jd}}n |jd}}|||j|dS)Nz#Couldn't retrieve index page for %rz3Couldn't find index page for %r (maybe misspelled?))rrrrr)rrmethrr#r#r$r+s  zPackageIndex.not_found_in_indexcCs~t|tsjt|}|rR||d||}t|\}}|drN||||}|Stj |rb|St |}t | ||ddS)aLocate and/or download `spec` to `tmpdir`, returning a local path `spec` may be a ``Requirement`` object, or a string containing a URL, an existing local filename, or a project/version requirement spec (i.e. the string form of a ``Requirement`` object). If it is the URL of a .py file with an unambiguous ``#egg=name-version`` tag (i.e., one that escapes ``-`` as ``_`` throughout), a trivial ``setup.py`` is automatically created alongside the downloaded file. If `spec` is a ``Requirement`` object or a string containing a project/version requirement spec, this method returns the location of a matching distribution (possibly after downloading it to `tmpdir`). If `spec` is a locally existing file or directory name, it is simply returned unchanged. If `spec` is a URL, it is downloaded to a subpath of `tmpdir`, and the local filename is returned. Various errors may be raised if a problem occurs during downloading. r5rrKN)rrr _download_urlrGrBr* gen_setuprVr>rr%rfetch_distribution)rr!tmpdirr<foundr-rAr#r#r$r26s    zPackageIndex.downloadc sd|id}d fdd }|rH|||}|s^|dur^|||}|durjdurx||}|dur|s|||}|durdrdpd|dSd||j|jd SdS) a|Obtain a distribution suitable for fulfilling `requirement` `requirement` must be a ``pkg_resources.Requirement`` instance. If necessary, or if the `force_scan` flag is set, the requirement is searched for in the (online) package index as well as the locally installed packages. If a distribution matching `requirement` is found, the returned distribution's ``location`` is the value you would have gotten from calling the ``download()`` method with the matching distribution's URL or filename. If no matching distribution is found, ``None`` is returned. If the `source` flag is set, only source distributions and source checkout links will be considered. Unless the `develop_ok` flag is set, development and system eggs (i.e., those using the ``.egg-info`` format) will be ignored. zSearching for %sNcs|dur }||jD]v}|jtkrFsF|vrd|d|<q||vo\|jtkp\ }|r|j}||_tj |jr|SqdS)Nz&Skipping development or system egg: %sr5) rrCrrrr2rKdownload_locationrVr>r)reqenvrtestloc develop_okrskippedsourcer r#r$rvvs$ z-PackageIndex.fetch_distribution..findz:No local packages or working download links found for %s%sza source distribution of rzBest match: %s)rK)N)rrrrrcloner ) rrr  force_scanrrZ local_indexrrvr#rr$r Xs4         zPackageIndex.fetch_distributioncCs"|||||}|dur|jSdS)a3Obtain a file suitable for fulfilling `requirement` DEPRECATED; use the ``fetch_distribution()`` method now instead. For backward compatibility, this routine is identical but returns the ``location`` of the downloaded distribution instead of a distribution object. N)r rK)rrr rrrr#r#r$fetchszPackageIndex.fetchc Cst|}|r*ddt||ddDp,g}t|dkrtj|}tj||krtj ||}tj |r~tj ||st |||}ttj |dd<}|d|dj|djtj|dfWdn1s0Y|S|r td||fntd dS) NcSsg|]}|jr|qSr#)rM)rZdr#r#r$ sz*PackageIndex.gen_setup..r5zsetup.pywzIfrom setuptools import setup setup(name=%r, version=%r, py_modules=[%r]) rzCan't unambiguously interpret project/version identifier %r; any dashes in the name or version should be escaped using underscores. %rzpCan't process plain .py files without an '#egg=name-version' suffix to enable automatic setup script generation.)rErFrrGrQrVr>rRdirnamercrsamefileshutilcopy2rwriterLrMsplitextr) rrWrAr rFrrRdstrr#r#r$rsD    " zPackageIndex.gen_setupi c Cs:|d|d}zt|}||}t|tjjrLtd||j |j f|}d}|j }d}d|vr| d} t tt| }||||||t|d`} ||} | r|| | | |d7}||||||qqq|||| Wdn1s0Y|W|r"|Sn|r4|0dS) NzDownloading %szCan't download %s: %s %srr1zcontent-lengthzContent-Lengthwbr5)rrrrrr6rrrrr dl_blocksizeget_allmaxrpint reporthookrrrrrr) rr:rWfprrblocknumbssizesizesrrr#r#r$ _download_tosD        .zPackageIndex._download_tocCsdSrdr#)rr:rWr)Zblksizer+r#r#r$r' szPackageIndex.reporthookc Cs|drt|Szt||jWSttjjfy}zHddd|j D}|r`| ||nt d||f|WYd}~n(d}~0t j jy}z|WYd}~Sd}~0t j jy}z4|r| ||jnt d||jf|WYd}~nd}~0tjjyT}z6|r,| ||jnt d||jf|WYd}~n\d}~0tjjtfy}z2|r| ||nt d||f|WYd}~n d}~00dS)Nr cSsg|] }t|qSr#)rq)rZargr#r#r$rr]z)PackageIndex.open_url..z%s %szDownload error for %s: %sz;%s returned a bad status line. The server might be down, %s)r+ local_openopen_with_authrr httpclient InvalidURLrcrerrr6rrURLErrorreason BadStatusLineline HTTPExceptionOSError)rr:warningvrr#r#r$rsB ( zPackageIndex.open_urlcCst|\}}|r0d|vr4|dddd}qnd}|drJ|dd}tj||}|dksj|d rv|||S|d ks|d r|||S|d r| ||S|d krt j t j |dS||d|||SdS)Nz...\_Z__downloaded__rIr(svnzsvn+gitzgit+zhg+rr^T)rBreplacer*rVr>rcr+ _download_svn _download_git _download_hgr6r url2pathnamerr7r_attempt_download)rr<r:r r,rArWr#r#r$r1s$        zPackageIndex._download_urlcCs||ddS)NT)rrr#r#r$rNszPackageIndex.scan_urlcCs6|||}d|ddvr.||||S|SdS)Nrrr)r-rr)_invalid_download_html)rr:rWrr#r#r$rGQs zPackageIndex._attempt_downloadcCst|td|dS)NzUnexpected HTML page found at )rVrr)rr:rrWr#r#r$rHXs z#PackageIndex._invalid_download_htmlcCstd|dS)Nz/Invalid config, SVN download is not supported: r)rr: _filenamer#r#r$rC\szPackageIndex._download_svncCsptj|\}}}}}|ddd}|ddd}d}d|vrR|dd\}}tj||||df}||fS)N+r5r1r4r@r)r6rurlsplitr9rsplit urlunsplit)r: pop_prefixr<netlocr>r@rrevr#r#r$_vcs_split_rev_from_url_sz$PackageIndex._vcs_split_rev_from_urlcCsl|ddd}|j|dd\}}|d||td||f|durh|d|td ||f|S) Nr4r5rTrOzDoing git clone from %s to %szgit clone --quiet %s %szChecking out %szgit -C %s checkout --quiet %sr9rRrrVsystemrr:rWrQr#r#r$rDqs zPackageIndex._download_gitcCsl|ddd}|j|dd\}}|d||td||f|durh|d|td ||f|S) Nr4r5rTrSzDoing hg clone from %s to %szhg clone --quiet %s %szUpdating to %szhg --cwd %s up -C -r %s -qrTrVr#r#r$rEs zPackageIndex._download_hgcGstj|g|RdSrd)rrrr#r#r$rszPackageIndex.debugcGstj|g|RdSrd)rrrr#r#r$rszPackageIndex.infocGstj|g|RdSrd)rrrr#r#r$rszPackageIndex.warn)rrNT)F)F)F)N)N)FFFN)FF)N)F)*rrrrrrrrrrrrrrrrrrrrrr2r rrr#r-r'rrrrGrHrC staticmethodrRrDrErrr __classcell__r#r#rr$r(s\ 5      & R .$ " rz!&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?cCs|d}t|S)Nr)rGrunescape)rFwhatr#r#r$ decode_entitys r[cCs tt|S)a Decode HTML entities in the given text. >>> htmldecode( ... 'https://../package_name-0.1.2.tar.gz' ... '?tokena=A&tokenb=B">package_name-0.1.2.tar.gz') 'https://../package_name-0.1.2.tar.gz?tokena=A&tokenb=B">package_name-0.1.2.tar.gz' ) entity_subr[)textr#r#r$rus rucsfdd}|S)Ncsfdd}|S)Nc s>t}tz|i|Wt|St|0dSrd)socketgetdefaulttimeoutsetdefaulttimeout)rerfZ old_timeout)rhtimeoutr#r$_socket_timeouts  z@socket_timeout.._socket_timeout.._socket_timeoutr#)rhrbrargr$rbsz'socket_timeout.._socket_timeoutr#)rarbr#rcr$socket_timeouts rdcCs2tj|}|}t|}|}|ddS)a9 Encode auth from a URL suitable for an HTTP header. >>> str(_encode_auth('username%3Apassword')) 'dXNlcm5hbWU6cGFzc3dvcmQ=' Long auth strings should not cause a newline to be inserted. >>> long_auth = 'username:' + 'password'*10 >>> chr(10) in str(_encode_auth(long_auth)) False  r)r6rr8encodebase64 b64encoderrB)authZauth_sZ auth_bytesZ encoded_bytesencodedr#r#r$ _encode_auths  rkc@s(eZdZdZddZddZddZdS) Credentialz: A username/password pair. Use like a namedtuple. cCs||_||_dSrdusernamepassword)rrnror#r#r$rszCredential.__init__ccs|jV|jVdSrdrmrr#r#r$__iter__szCredential.__iter__cCs dt|S)Nz%(username)s:%(password)s)varsrr#r#r$__str__szCredential.__str__N)rrrrrrprrr#r#r#r$rlsrlcs8eZdZfddZeddZddZddZZS) PyPIConfigcsLtgdd}t|tjtjdd}tj|rH| |dS)z% Load from ~/.pypirc )rnro repositoryr~z.pypircN) dictfromkeysrrrVr>rc expanduserrr)rdefaultsrcrr#r$rs   zPyPIConfig.__init__cs&fddD}ttj|S)Ncs g|]}|dr|qS)rt)rrr)rZsectionrr#r$rsz2PyPIConfig.creds_by_repository..)sectionsrvrp_get_repo_cred)rZsections_with_repositoriesr#rr$creds_by_repositorys zPyPIConfig.creds_by_repositorycCs6||d}|t||d||dfS)Nrtrnro)rrrrl)rr{repor#r#r$r}s zPyPIConfig._get_repo_credcCs*|jD]\}}||r |Sq dS)z If the URL indicated appears to be a repository defined in this config, return the credential for that repository. N)r~itemsr+)rr:rtcredr#r#r$find_credential s  zPyPIConfig.find_credential) rrrrpropertyr~r}rrXr#r#rr$rss   rscCs@tj|}|\}}}}}}|dr2tjd|dvrHt|\} } nd} | st |} | rt | } | j |f} t j dg| R| rdt| } || ||||f} tj| }tj|}|d| n tj|}|dt||}| rparamsr@rriaddressrrr;rrr(s2h2path2Zparam2Zquery2Zfrag2r#r#r$r1s8          r1cCs |d\}}}|r|nd|fS)zNsplituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'.rKN) rpartition)hostuserdelimr#r#r$rCsrcCs|Srdr#)r:r#r#r$ fix_sf_urlNsrc Cs,tj|\}}}}}}tj|}tj|r{name})r,zB{url}{files}re)r:files)OK)rzPath not foundz Not foundrz text/html)r6rr7rrFrVr>isfilerr*rrrcrrrformatioStringIOrr)r:r<r=r>paramr@rrWrrfilepathr(bodyrstatusmessagerZ body_streamr#r#r$r0Rs.    &    r0)N)N)N)r)YrsysrVrYrrr^rgrr configparserr http.clientr2 urllib.parser6urllib.request urllib.error functoolsrr pkg_resourcesrrrrrr r r r r rrrr distutilsrdistutils.errorsrfnmatchrZsetuptools.wheelrZ setuptools.extern.more_itertoolsrrrEIrsrrFrr9rP__all__Z_SOCKET_TIMEOUTZ_tmplr version_inforr%rrBrrDrXrrjrlr|r}rrrr\r[rurdrkrlRawConfigParserrsrrr1rrr0r#r#r#r$s @         #  "  !  (/