mxi9?dZddlmZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddlmZmZmZddlmZddlmZddlmZddlmZmZmZdd lmZdd lmZdd l m!Z!m"Z"dd l#m$Z$dd l%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,ddl-m.Z.ddl/m0Z0ddl1m2Z2ddl3m4Z4m5Z5m6Z6e jne8Z9ee:e:fZ;d2dZ<Gdde=Z>d3dZ?Gdde=Z@d4dZAd5dZBd6dZCGdd ZDGd!d"eZEd7d#ZFeFd8d%ZGed&'Gd(d$ZHGd)d*eZI d9 d:d+ZJ d; dzO The main purpose of this module is to expose LinkCollector.collect_sources(). ) annotationsN)IterableMutableMappingSequence) dataclass) HTMLParser)Values)Callable NamedTupleProtocol)requests)Response) RetryErrorSSLError)NetworkConnectionError)Link) SearchScope) PipSession)raise_for_status)is_archive_fileredact_auth_from_url)vcs)CandidatesFromPage LinkSource build_sourcectjD]6}|jj|s#|t |dvs4|cSy)zgLook for VCS schemes in the URL. Returns the matched VCS scheme, or None if there's no match. z+:N)rschemeslower startswithlen)urlschemes T/home/htdocs/ttos/venv/lib/python3.12/site-packages/pip/_internal/index/collector.py_match_vcs_schemer&.sE ++ 99; ! !& )c#f+.>$.FM c eZdZdfd ZxZS)_NotAPIContentcBt|||||_||_yN)super__init__ content_type request_desc)selfr.r/ __class__s r%r-z_NotAPIContent.__init__:s" |4((r')r.strr/r2returnNone)__name__ __module__ __qualname__r- __classcell__r1s@r%r)r)9s ))r'r)c|jjdd}|j}|jdryt ||j j )z Check the Content-Type header to ensure the response contains a Simple API Response. Raises `_NotAPIContent` if the content type is not a valid content-type. Content-TypeUnknown)z text/htmlz#application/vnd.pypi.simple.v1+html#application/vnd.pypi.simple.v1+jsonN)headersgetr r!r)requestmethod)responser.content_type_ls r%_ensure_api_headerrD@s[##'' BL!'')N     x'7'7'>'> ??r'c eZdZy)_NotHTTPN)r5r6r7r'r%rFrFVsr'rFctjj|\}}}}}|dvr t|j |d}t |t |y)z Send a HEAD request to the URL, and ensure the response contains a simple API Response. Raises `_NotHTTP` if the URL is not available for a HEAD request, or `_NotAPIContent` if the content type is not a valid content type. >httphttpsT)allow_redirectsN)urllibparseurlsplitrFheadrrD)r#sessionr$netlocpathqueryfragmentresps r%_ensure_api_responserVZsV-3LL,A,A#,F)FFD% &&j <<T< 2DTtr'ctt|jr t||tj dt ||j|djgddd}t|t|tj dt ||jjd d |S) aYAccess an Simple API response with GET, and return the response. This consists of three parts: 1. If the URL looks suspiciously like an archive, send a HEAD first to check the Content-Type is HTML or Simple API, to avoid downloading a large file. Raise `_NotHTTP` if the content type cannot be determined, or `_NotAPIContent` if it is not HTML or a Simple API. 2. Actually perform the request. Raise HTTP exceptions on network failures. 3. Check the Content-Type header to make sure we got a Simple API response, and raise `_NotAPIContent` otherwise. rPzGetting page %sz, )r=z*application/vnd.pypi.simple.v1+html; q=0.1ztext/html; q=0.01z max-age=0)Acceptz Cache-Control)r>zFetched page %s as %sr;r<) rrfilenamerVloggerdebugrr?joinrrDr>)r#rPrUs r%_get_simple_responser^lstCy))*S'2 LL"$8$=> ;; ii()+   D4Tt LLS! 3 Kr'c|rHd|vrDtjj}|d|d<|jd}|r t |Sy)z=Determine if we have any encoding information in our headers.r;z content-typecharsetN)emailmessageMessage get_paramr2)r>mr`s r%_get_encoding_from_headersrfsK>W, MM ! ! ##N3.++i( w<  r'c$eZdZddZddZddZy)CacheablePageContentc.|jsJ||_yr+)cache_link_parsingpager0rks r%r-zCacheablePageContent.__init__s&&&& r'ct|t|xr-|jj|jjk(Sr+) isinstancetyperkr#)r0others r%__eq__zCacheablePageContent.__eq__s-%d,P%**..1PPr'c@t|jjSr+)hashrkr#r0s r%__hash__zCacheablePageContent.__hash__sDIIMM""r'N)rk IndexContentr3r4)rpobjectr3bool)r3int)r5r6r7r-rqrurGr'r%rhrhsQ#r'rhceZdZddZy) ParseLinkscyr+rGrls r%__call__zParseLinks.__call__sr'Nrkrvr3zIterable[Link])r5r6r7r}rGr'r%r{r{sAr'r{cztjdfd tjdfd }|S)z Given a function that parses an Iterable[Link] from an IndexContent, cache the function's result (keyed by CacheablePageContent), unless the IndexContent `page` has `page.cache_link_parsing == False`. c:t|jSr+)listrk)cacheable_pagefns r%wrapperz*with_cached_index_content..wrappersB~**+,,r'c`|jrt|St|Sr+)rjrhr)rkrrs r%wrapper_wrapperz2with_cached_index_content..wrapper_wrappers+  " "/56 6BtH~r')rrhr3 list[Link])rkrvr3r) functoolscachewraps)rrrs` @r%with_cached_index_contentrs@__--__R r'rvc#TK|jj}|jdr^tj|j }|j dgD])}tj||j}|&|+yt|j}|jxsd}|j|j j||j}|jxs|}|jD]!} tj | ||}||#yw)z\ Parse a Simple API's Index Content, and yield its anchor elements as Link objects. r=filesNzutf-8)page_urlbase_url)r.r r!jsonloadscontentr?r from_jsonr#HTMLLinkParserencodingfeeddecoderanchors from_element) rkrCdatafilelinkparserrr#ranchors r% parse_linksrs &&,,.N  !FGzz$,,'HHWb) D>>$1D|J    DHH %F}}'H KK ##H-. ((C%#H..  #I <  sD&D(T)frozencPeZdZUdZded<ded<ded<ded<d Zd ed <dd Zy )rvaRepresents one response (or page), along with its URL. :param encoding: the encoding to decode the given content. :param url: the URL from which the HTML was downloaded. :param cache_link_parsing: whether links parsed from this page's url should be cached. PyPI index urls should have this set to False, for example. bytesrr2r. str | Nonerr#Trxrjc,t|jSr+)rr#rts r%__str__zIndexContent.__str__s#DHH--r'N)r3r2)r5r6r7__doc____annotations__rjrrGr'r%rvrvs/N H##.r'c4eZdZdZdfd ZddZddZxZS)rzf HTMLParser that keeps the first base HREF and a list of all anchor elements' attributes. cPt|d||_d|_g|_y)NT)convert_charrefs)r,r-r#rr)r0r#r1s r%r-zHTMLLinkParser.__init__s( $/$( 46 r'c|dk(r(|j|j|}|||_yy|dk(r%|jjt |yy)Nbasea)rget_hrefrappenddict)r0tagattrshrefs r%handle_starttagzHTMLLinkParser.handle_starttagsT &=T]]2=='D $  CZ LL  U ,r'c*|D]\}}|dk(s |cSy)NrrG)r0rnamevalues r%rzHTMLLinkParser.get_href!s&  KD%v~  r')r#r2r3r4)rr2rlist[tuple[str, str | None]]r3r4)rrr3r)r5r6r7rr-rrr8r9s@r%rr s 7-r'rc<|tj}|d||y)Nz%Could not fetch URL %s: %s - skipping)r[r\)rreasonmeths r%_handle_get_simple_failr(s  ||| 0$?r'ct|j}t|j|jd||j|S)Nr;)rr#rj)rfr>rvrr#)rBrjrs r%_make_index_contentr2sE*(*:*:;H ( LL-  r'c|jjddd}t|}|rtj d||yt j j|\}}}}}}|dk(rtjjt jj|rL|jds|dz }t j j|d}tjd| t!|| }t#||j$ S#t&$rtj d |Yyt($r6}tj d ||j*|j,Yd}~yd}~wt.$r}t1||Yd}~yd}~wt2$r}t1||Yd}~yd}~wt4$r6}d } | t7|z } t1|| tj8Yd}~yd}~wt:j<$r}t1|d|Yd}~yd}~wt:j>$rt1|dYywxYw)N#rrzICannot look at %s URL %s because it does not support lookup as web pages.r/z index.htmlz# file: URL is directory, getting %srX)rjz`Skipping page %s because it looks like an archive, and cannot be checked by a HTTP HEAD request.zSkipping page %s because the %s request got Content-Type: %s. The only supported Content-Types are application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html, and text/htmlz4There was a problem confirming the ssl certificate: )rzconnection error: z timed out) r#splitr&r[warningrLrMurlparseosrRisdirr@ url2pathnameendswithurljoinr\r^rrjrFr)r/r.rrrrr2infor ConnectionErrorTimeout) rrPr# vcs_schemer$_rRrUexcrs r%_get_index_contentr?s ((..a  #C#3'J W    &||44S9FAtQ1 BGGMM&..*E*Ed*KL||C 3JC ll""3 5 :C@U#C9:#4DH=,E77 H= F H= F11 H==,G..H=H"H=<H=c"eZdZUded<ded<y)CollectedSourceszSequence[LinkSource | None] find_links index_urlsN)r5r6r7rrGr'r%rr|s++++r'rcveZdZdZ ddZe d d dZed dZd dZ d dZ y) LinkCollectorz Responsible for collecting Link objects from all configured locations, making network requests as needed. The class's main method is its collect_sources() method. c ||_||_yr+) search_scoperP)r0rPrs r%r-zLinkCollector.__init__s ) r'c0|jg|jz}|jr0|s.tj ddj d|Dg}|j xsg}tj|||j}t||}|S)z :param session: The Session to use to make requests. :param suppress_no_index: Whether to ignore the --no-index option when constructing the SearchScope object. zIgnoring indexes: %s,c32K|]}t|ywr+r).0r#s r% z'LinkCollector.create..sIs-c2Is)rrno_index)rPr) index_urlextra_index_urlsrr[r\r]rrcreater)clsrPoptionssuppress_no_indexrrrlink_collectors r%rzLinkCollector.creates''(7+C+CC   $5 LL&IjII J''-2 "))!!%% '% r'c.|jjSr+)rrrts r%rzLinkCollector.find_linkss  +++r'c0t||jS)z> Fetch an HTML page containing package links. rX)rrP)r0locations r%fetch_responsezLinkCollector.fetch_responses"(DLLAAr'ctjfdjjDj }tjfdj Dj }t jtjrwtj||Dcgc]}||jd|j!}}t|ddg|z}t jdj|t!t#|t#|Scc}w)Nc 3hK|])}t|jjdd+yw)Fcandidates_from_pagepage_validator expand_dirrj project_nameNrrPis_secure_originrlocrrr0s r%rz0LinkCollector.collect_sources..sA 4  %9#||<< #()    4 /2c 3hK|])}t|jjdd+yw)TrNrrs r%rz0LinkCollector.collect_sources..sA 5  %9#||<<#')    5 rz* z' location(s) to search for versions of : )rr) collections OrderedDictrget_index_urls_locationsvaluesrr[ isEnabledForloggingDEBUG itertoolschainrr"r\r]rr)r0rrindex_url_sourcesfind_links_sourcessliness``` r%collect_sourceszLinkCollector.collect_sourcess2 (33 4 ((AA,O 4   &( )44 5  5   &(    w}} -#);=NO=QVV%7QVVH E u:,##/.3E LL5) *./-.  s>$D<N)rPrrrr3r4)F)rPrrr rrxr3r)r3z list[str])rrr3IndexContent | None)rr2rrr3r) r5r6r7rr- classmethodrpropertyrrrrGr'r%rrs"   #(    B,,B , , 1,   , r'r)r#r2r3r)rBrr3r4)r#r2rPrr3r4)r#r2rPrr3r)r>ResponseHeadersr3r)rr{r3r{r~r+)rrrzstr | ExceptionrzCallable[..., None] | Noner3r4)T)rBrrjrxr3rv)rrrPrr3r)Or __future__rr email.messagerarrrrr urllib.parserLurllib.requestcollections.abcrrr dataclassesr html.parserroptparser typingr r r pip._vendorr pip._vendor.requestsrpip._vendor.requests.exceptionsrrpip._internal.exceptionsrpip._internal.models.linkr!pip._internal.models.search_scoperpip._internal.network.sessionrpip._internal.network.utilsrpip._internal.utils.filetypesrpip._internal.utils.miscrpip._internal.vcsrsourcesrrr getLoggerr5r[r2r r& Exceptionr)rDrFrVr^rfrhr{rrrvrrrrrrrGr'r%r"s#  >>!" !)@;*94899!AA   8 $ c*)Y)@, y $<~ # #BB(8 $...(Z>(,@ @ @ %@ @48  ,0  :z,z, h h r'