Mercurial > repos > shellac > sam_consensus_v3
comparison env/lib/python3.9/site-packages/pip/_internal/req/req_file.py @ 0:4f3585e2f14b draft default tip
"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"
author | shellac |
---|---|
date | Mon, 22 Mar 2021 18:12:50 +0000 |
parents | |
children |
comparison
equal
deleted
inserted
replaced
-1:000000000000 | 0:4f3585e2f14b |
---|---|
1 """ | |
2 Requirements file parsing | |
3 """ | |
4 | |
5 import optparse | |
6 import os | |
7 import re | |
8 import shlex | |
9 import urllib.parse | |
10 | |
11 from pip._internal.cli import cmdoptions | |
12 from pip._internal.exceptions import InstallationError, RequirementsFileParseError | |
13 from pip._internal.models.search_scope import SearchScope | |
14 from pip._internal.network.utils import raise_for_status | |
15 from pip._internal.utils.encoding import auto_decode | |
16 from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
17 from pip._internal.utils.urls import get_url_scheme, url_to_path | |
18 | |
19 if MYPY_CHECK_RUNNING: | |
20 from optparse import Values | |
21 from typing import ( | |
22 Any, | |
23 Callable, | |
24 Dict, | |
25 Iterator, | |
26 List, | |
27 NoReturn, | |
28 Optional, | |
29 Text, | |
30 Tuple, | |
31 ) | |
32 | |
33 from pip._internal.index.package_finder import PackageFinder | |
34 from pip._internal.network.session import PipSession | |
35 | |
36 ReqFileLines = Iterator[Tuple[int, Text]] | |
37 | |
38 LineParser = Callable[[Text], Tuple[str, Values]] | |
39 | |
40 | |
41 __all__ = ['parse_requirements'] | |
42 | |
43 SCHEME_RE = re.compile(r'^(http|https|file):', re.I) | |
44 COMMENT_RE = re.compile(r'(^|\s+)#.*$') | |
45 | |
46 # Matches environment variable-style values in '${MY_VARIABLE_1}' with the | |
47 # variable name consisting of only uppercase letters, digits or the '_' | |
48 # (underscore). This follows the POSIX standard defined in IEEE Std 1003.1, | |
49 # 2013 Edition. | |
50 ENV_VAR_RE = re.compile(r'(?P<var>\$\{(?P<name>[A-Z0-9_]+)\})') | |
51 | |
52 SUPPORTED_OPTIONS = [ | |
53 cmdoptions.index_url, | |
54 cmdoptions.extra_index_url, | |
55 cmdoptions.no_index, | |
56 cmdoptions.constraints, | |
57 cmdoptions.requirements, | |
58 cmdoptions.editable, | |
59 cmdoptions.find_links, | |
60 cmdoptions.no_binary, | |
61 cmdoptions.only_binary, | |
62 cmdoptions.prefer_binary, | |
63 cmdoptions.require_hashes, | |
64 cmdoptions.pre, | |
65 cmdoptions.trusted_host, | |
66 cmdoptions.use_new_feature, | |
67 ] # type: List[Callable[..., optparse.Option]] | |
68 | |
69 # options to be passed to requirements | |
70 SUPPORTED_OPTIONS_REQ = [ | |
71 cmdoptions.install_options, | |
72 cmdoptions.global_options, | |
73 cmdoptions.hash, | |
74 ] # type: List[Callable[..., optparse.Option]] | |
75 | |
76 # the 'dest' string values | |
77 SUPPORTED_OPTIONS_REQ_DEST = [str(o().dest) for o in SUPPORTED_OPTIONS_REQ] | |
78 | |
79 | |
80 class ParsedRequirement: | |
81 def __init__( | |
82 self, | |
83 requirement, # type:str | |
84 is_editable, # type: bool | |
85 comes_from, # type: str | |
86 constraint, # type: bool | |
87 options=None, # type: Optional[Dict[str, Any]] | |
88 line_source=None, # type: Optional[str] | |
89 ): | |
90 # type: (...) -> None | |
91 self.requirement = requirement | |
92 self.is_editable = is_editable | |
93 self.comes_from = comes_from | |
94 self.options = options | |
95 self.constraint = constraint | |
96 self.line_source = line_source | |
97 | |
98 | |
99 class ParsedLine: | |
100 def __init__( | |
101 self, | |
102 filename, # type: str | |
103 lineno, # type: int | |
104 args, # type: str | |
105 opts, # type: Values | |
106 constraint, # type: bool | |
107 ): | |
108 # type: (...) -> None | |
109 self.filename = filename | |
110 self.lineno = lineno | |
111 self.opts = opts | |
112 self.constraint = constraint | |
113 | |
114 if args: | |
115 self.is_requirement = True | |
116 self.is_editable = False | |
117 self.requirement = args | |
118 elif opts.editables: | |
119 self.is_requirement = True | |
120 self.is_editable = True | |
121 # We don't support multiple -e on one line | |
122 self.requirement = opts.editables[0] | |
123 else: | |
124 self.is_requirement = False | |
125 | |
126 | |
127 def parse_requirements( | |
128 filename, # type: str | |
129 session, # type: PipSession | |
130 finder=None, # type: Optional[PackageFinder] | |
131 options=None, # type: Optional[optparse.Values] | |
132 constraint=False, # type: bool | |
133 ): | |
134 # type: (...) -> Iterator[ParsedRequirement] | |
135 """Parse a requirements file and yield ParsedRequirement instances. | |
136 | |
137 :param filename: Path or url of requirements file. | |
138 :param session: PipSession instance. | |
139 :param finder: Instance of pip.index.PackageFinder. | |
140 :param options: cli options. | |
141 :param constraint: If true, parsing a constraint file rather than | |
142 requirements file. | |
143 """ | |
144 line_parser = get_line_parser(finder) | |
145 parser = RequirementsFileParser(session, line_parser) | |
146 | |
147 for parsed_line in parser.parse(filename, constraint): | |
148 parsed_req = handle_line( | |
149 parsed_line, | |
150 options=options, | |
151 finder=finder, | |
152 session=session | |
153 ) | |
154 if parsed_req is not None: | |
155 yield parsed_req | |
156 | |
157 | |
158 def preprocess(content): | |
159 # type: (str) -> ReqFileLines | |
160 """Split, filter, and join lines, and return a line iterator | |
161 | |
162 :param content: the content of the requirements file | |
163 """ | |
164 lines_enum = enumerate(content.splitlines(), start=1) # type: ReqFileLines | |
165 lines_enum = join_lines(lines_enum) | |
166 lines_enum = ignore_comments(lines_enum) | |
167 lines_enum = expand_env_variables(lines_enum) | |
168 return lines_enum | |
169 | |
170 | |
171 def handle_requirement_line( | |
172 line, # type: ParsedLine | |
173 options=None, # type: Optional[optparse.Values] | |
174 ): | |
175 # type: (...) -> ParsedRequirement | |
176 | |
177 # preserve for the nested code path | |
178 line_comes_from = '{} {} (line {})'.format( | |
179 '-c' if line.constraint else '-r', line.filename, line.lineno, | |
180 ) | |
181 | |
182 assert line.is_requirement | |
183 | |
184 if line.is_editable: | |
185 # For editable requirements, we don't support per-requirement | |
186 # options, so just return the parsed requirement. | |
187 return ParsedRequirement( | |
188 requirement=line.requirement, | |
189 is_editable=line.is_editable, | |
190 comes_from=line_comes_from, | |
191 constraint=line.constraint, | |
192 ) | |
193 else: | |
194 if options: | |
195 # Disable wheels if the user has specified build options | |
196 cmdoptions.check_install_build_global(options, line.opts) | |
197 | |
198 # get the options that apply to requirements | |
199 req_options = {} | |
200 for dest in SUPPORTED_OPTIONS_REQ_DEST: | |
201 if dest in line.opts.__dict__ and line.opts.__dict__[dest]: | |
202 req_options[dest] = line.opts.__dict__[dest] | |
203 | |
204 line_source = f'line {line.lineno} of {line.filename}' | |
205 return ParsedRequirement( | |
206 requirement=line.requirement, | |
207 is_editable=line.is_editable, | |
208 comes_from=line_comes_from, | |
209 constraint=line.constraint, | |
210 options=req_options, | |
211 line_source=line_source, | |
212 ) | |
213 | |
214 | |
215 def handle_option_line( | |
216 opts, # type: Values | |
217 filename, # type: str | |
218 lineno, # type: int | |
219 finder=None, # type: Optional[PackageFinder] | |
220 options=None, # type: Optional[optparse.Values] | |
221 session=None, # type: Optional[PipSession] | |
222 ): | |
223 # type: (...) -> None | |
224 | |
225 if options: | |
226 # percolate options upward | |
227 if opts.require_hashes: | |
228 options.require_hashes = opts.require_hashes | |
229 if opts.features_enabled: | |
230 options.features_enabled.extend( | |
231 f for f in opts.features_enabled | |
232 if f not in options.features_enabled | |
233 ) | |
234 | |
235 # set finder options | |
236 if finder: | |
237 find_links = finder.find_links | |
238 index_urls = finder.index_urls | |
239 if opts.index_url: | |
240 index_urls = [opts.index_url] | |
241 if opts.no_index is True: | |
242 index_urls = [] | |
243 if opts.extra_index_urls: | |
244 index_urls.extend(opts.extra_index_urls) | |
245 if opts.find_links: | |
246 # FIXME: it would be nice to keep track of the source | |
247 # of the find_links: support a find-links local path | |
248 # relative to a requirements file. | |
249 value = opts.find_links[0] | |
250 req_dir = os.path.dirname(os.path.abspath(filename)) | |
251 relative_to_reqs_file = os.path.join(req_dir, value) | |
252 if os.path.exists(relative_to_reqs_file): | |
253 value = relative_to_reqs_file | |
254 find_links.append(value) | |
255 | |
256 if session: | |
257 # We need to update the auth urls in session | |
258 session.update_index_urls(index_urls) | |
259 | |
260 search_scope = SearchScope( | |
261 find_links=find_links, | |
262 index_urls=index_urls, | |
263 ) | |
264 finder.search_scope = search_scope | |
265 | |
266 if opts.pre: | |
267 finder.set_allow_all_prereleases() | |
268 | |
269 if opts.prefer_binary: | |
270 finder.set_prefer_binary() | |
271 | |
272 if session: | |
273 for host in opts.trusted_hosts or []: | |
274 source = f'line {lineno} of {filename}' | |
275 session.add_trusted_host(host, source=source) | |
276 | |
277 | |
278 def handle_line( | |
279 line, # type: ParsedLine | |
280 options=None, # type: Optional[optparse.Values] | |
281 finder=None, # type: Optional[PackageFinder] | |
282 session=None, # type: Optional[PipSession] | |
283 ): | |
284 # type: (...) -> Optional[ParsedRequirement] | |
285 """Handle a single parsed requirements line; This can result in | |
286 creating/yielding requirements, or updating the finder. | |
287 | |
288 :param line: The parsed line to be processed. | |
289 :param options: CLI options. | |
290 :param finder: The finder - updated by non-requirement lines. | |
291 :param session: The session - updated by non-requirement lines. | |
292 | |
293 Returns a ParsedRequirement object if the line is a requirement line, | |
294 otherwise returns None. | |
295 | |
296 For lines that contain requirements, the only options that have an effect | |
297 are from SUPPORTED_OPTIONS_REQ, and they are scoped to the | |
298 requirement. Other options from SUPPORTED_OPTIONS may be present, but are | |
299 ignored. | |
300 | |
301 For lines that do not contain requirements, the only options that have an | |
302 effect are from SUPPORTED_OPTIONS. Options from SUPPORTED_OPTIONS_REQ may | |
303 be present, but are ignored. These lines may contain multiple options | |
304 (although our docs imply only one is supported), and all our parsed and | |
305 affect the finder. | |
306 """ | |
307 | |
308 if line.is_requirement: | |
309 parsed_req = handle_requirement_line(line, options) | |
310 return parsed_req | |
311 else: | |
312 handle_option_line( | |
313 line.opts, | |
314 line.filename, | |
315 line.lineno, | |
316 finder, | |
317 options, | |
318 session, | |
319 ) | |
320 return None | |
321 | |
322 | |
323 class RequirementsFileParser: | |
324 def __init__( | |
325 self, | |
326 session, # type: PipSession | |
327 line_parser, # type: LineParser | |
328 ): | |
329 # type: (...) -> None | |
330 self._session = session | |
331 self._line_parser = line_parser | |
332 | |
333 def parse(self, filename, constraint): | |
334 # type: (str, bool) -> Iterator[ParsedLine] | |
335 """Parse a given file, yielding parsed lines. | |
336 """ | |
337 yield from self._parse_and_recurse(filename, constraint) | |
338 | |
339 def _parse_and_recurse(self, filename, constraint): | |
340 # type: (str, bool) -> Iterator[ParsedLine] | |
341 for line in self._parse_file(filename, constraint): | |
342 if ( | |
343 not line.is_requirement and | |
344 (line.opts.requirements or line.opts.constraints) | |
345 ): | |
346 # parse a nested requirements file | |
347 if line.opts.requirements: | |
348 req_path = line.opts.requirements[0] | |
349 nested_constraint = False | |
350 else: | |
351 req_path = line.opts.constraints[0] | |
352 nested_constraint = True | |
353 | |
354 # original file is over http | |
355 if SCHEME_RE.search(filename): | |
356 # do a url join so relative paths work | |
357 req_path = urllib.parse.urljoin(filename, req_path) | |
358 # original file and nested file are paths | |
359 elif not SCHEME_RE.search(req_path): | |
360 # do a join so relative paths work | |
361 req_path = os.path.join( | |
362 os.path.dirname(filename), req_path, | |
363 ) | |
364 | |
365 yield from self._parse_and_recurse(req_path, nested_constraint) | |
366 else: | |
367 yield line | |
368 | |
369 def _parse_file(self, filename, constraint): | |
370 # type: (str, bool) -> Iterator[ParsedLine] | |
371 _, content = get_file_content(filename, self._session) | |
372 | |
373 lines_enum = preprocess(content) | |
374 | |
375 for line_number, line in lines_enum: | |
376 try: | |
377 args_str, opts = self._line_parser(line) | |
378 except OptionParsingError as e: | |
379 # add offending line | |
380 msg = f'Invalid requirement: {line}\n{e.msg}' | |
381 raise RequirementsFileParseError(msg) | |
382 | |
383 yield ParsedLine( | |
384 filename, | |
385 line_number, | |
386 args_str, | |
387 opts, | |
388 constraint, | |
389 ) | |
390 | |
391 | |
392 def get_line_parser(finder): | |
393 # type: (Optional[PackageFinder]) -> LineParser | |
394 def parse_line(line): | |
395 # type: (str) -> Tuple[str, Values] | |
396 # Build new parser for each line since it accumulates appendable | |
397 # options. | |
398 parser = build_parser() | |
399 defaults = parser.get_default_values() | |
400 defaults.index_url = None | |
401 if finder: | |
402 defaults.format_control = finder.format_control | |
403 | |
404 args_str, options_str = break_args_options(line) | |
405 | |
406 opts, _ = parser.parse_args(shlex.split(options_str), defaults) | |
407 | |
408 return args_str, opts | |
409 | |
410 return parse_line | |
411 | |
412 | |
413 def break_args_options(line): | |
414 # type: (str) -> Tuple[str, str] | |
415 """Break up the line into an args and options string. We only want to shlex | |
416 (and then optparse) the options, not the args. args can contain markers | |
417 which are corrupted by shlex. | |
418 """ | |
419 tokens = line.split(' ') | |
420 args = [] | |
421 options = tokens[:] | |
422 for token in tokens: | |
423 if token.startswith('-') or token.startswith('--'): | |
424 break | |
425 else: | |
426 args.append(token) | |
427 options.pop(0) | |
428 return ' '.join(args), ' '.join(options) | |
429 | |
430 | |
431 class OptionParsingError(Exception): | |
432 def __init__(self, msg): | |
433 # type: (str) -> None | |
434 self.msg = msg | |
435 | |
436 | |
437 def build_parser(): | |
438 # type: () -> optparse.OptionParser | |
439 """ | |
440 Return a parser for parsing requirement lines | |
441 """ | |
442 parser = optparse.OptionParser(add_help_option=False) | |
443 | |
444 option_factories = SUPPORTED_OPTIONS + SUPPORTED_OPTIONS_REQ | |
445 for option_factory in option_factories: | |
446 option = option_factory() | |
447 parser.add_option(option) | |
448 | |
449 # By default optparse sys.exits on parsing errors. We want to wrap | |
450 # that in our own exception. | |
451 def parser_exit(self, msg): | |
452 # type: (Any, str) -> NoReturn | |
453 raise OptionParsingError(msg) | |
454 # NOTE: mypy disallows assigning to a method | |
455 # https://github.com/python/mypy/issues/2427 | |
456 parser.exit = parser_exit # type: ignore | |
457 | |
458 return parser | |
459 | |
460 | |
461 def join_lines(lines_enum): | |
462 # type: (ReqFileLines) -> ReqFileLines | |
463 """Joins a line ending in '\' with the previous line (except when following | |
464 comments). The joined line takes on the index of the first line. | |
465 """ | |
466 primary_line_number = None | |
467 new_line = [] # type: List[str] | |
468 for line_number, line in lines_enum: | |
469 if not line.endswith('\\') or COMMENT_RE.match(line): | |
470 if COMMENT_RE.match(line): | |
471 # this ensures comments are always matched later | |
472 line = ' ' + line | |
473 if new_line: | |
474 new_line.append(line) | |
475 assert primary_line_number is not None | |
476 yield primary_line_number, ''.join(new_line) | |
477 new_line = [] | |
478 else: | |
479 yield line_number, line | |
480 else: | |
481 if not new_line: | |
482 primary_line_number = line_number | |
483 new_line.append(line.strip('\\')) | |
484 | |
485 # last line contains \ | |
486 if new_line: | |
487 assert primary_line_number is not None | |
488 yield primary_line_number, ''.join(new_line) | |
489 | |
490 # TODO: handle space after '\'. | |
491 | |
492 | |
493 def ignore_comments(lines_enum): | |
494 # type: (ReqFileLines) -> ReqFileLines | |
495 """ | |
496 Strips comments and filter empty lines. | |
497 """ | |
498 for line_number, line in lines_enum: | |
499 line = COMMENT_RE.sub('', line) | |
500 line = line.strip() | |
501 if line: | |
502 yield line_number, line | |
503 | |
504 | |
505 def expand_env_variables(lines_enum): | |
506 # type: (ReqFileLines) -> ReqFileLines | |
507 """Replace all environment variables that can be retrieved via `os.getenv`. | |
508 | |
509 The only allowed format for environment variables defined in the | |
510 requirement file is `${MY_VARIABLE_1}` to ensure two things: | |
511 | |
512 1. Strings that contain a `$` aren't accidentally (partially) expanded. | |
513 2. Ensure consistency across platforms for requirement files. | |
514 | |
515 These points are the result of a discussion on the `github pull | |
516 request #3514 <https://github.com/pypa/pip/pull/3514>`_. | |
517 | |
518 Valid characters in variable names follow the `POSIX standard | |
519 <http://pubs.opengroup.org/onlinepubs/9699919799/>`_ and are limited | |
520 to uppercase letter, digits and the `_` (underscore). | |
521 """ | |
522 for line_number, line in lines_enum: | |
523 for env_var, var_name in ENV_VAR_RE.findall(line): | |
524 value = os.getenv(var_name) | |
525 if not value: | |
526 continue | |
527 | |
528 line = line.replace(env_var, value) | |
529 | |
530 yield line_number, line | |
531 | |
532 | |
533 def get_file_content(url, session): | |
534 # type: (str, PipSession) -> Tuple[str, str] | |
535 """Gets the content of a file; it may be a filename, file: URL, or | |
536 http: URL. Returns (location, content). Content is unicode. | |
537 Respects # -*- coding: declarations on the retrieved files. | |
538 | |
539 :param url: File path or url. | |
540 :param session: PipSession instance. | |
541 """ | |
542 scheme = get_url_scheme(url) | |
543 | |
544 if scheme in ['http', 'https']: | |
545 # FIXME: catch some errors | |
546 resp = session.get(url) | |
547 raise_for_status(resp) | |
548 return resp.url, resp.text | |
549 | |
550 elif scheme == 'file': | |
551 url = url_to_path(url) | |
552 | |
553 try: | |
554 with open(url, 'rb') as f: | |
555 content = auto_decode(f.read()) | |
556 except OSError as exc: | |
557 raise InstallationError( | |
558 f'Could not open requirements file: {exc}' | |
559 ) | |
560 return url, content |