comparison env/lib/python3.9/site-packages/pip/_internal/req/req_file.py @ 0:4f3585e2f14b draft default tip

"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"
author shellac
date Mon, 22 Mar 2021 18:12:50 +0000
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:4f3585e2f14b
1 """
2 Requirements file parsing
3 """
4
5 import optparse
6 import os
7 import re
8 import shlex
9 import urllib.parse
10
11 from pip._internal.cli import cmdoptions
12 from pip._internal.exceptions import InstallationError, RequirementsFileParseError
13 from pip._internal.models.search_scope import SearchScope
14 from pip._internal.network.utils import raise_for_status
15 from pip._internal.utils.encoding import auto_decode
16 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
17 from pip._internal.utils.urls import get_url_scheme, url_to_path
18
19 if MYPY_CHECK_RUNNING:
20 from optparse import Values
21 from typing import (
22 Any,
23 Callable,
24 Dict,
25 Iterator,
26 List,
27 NoReturn,
28 Optional,
29 Text,
30 Tuple,
31 )
32
33 from pip._internal.index.package_finder import PackageFinder
34 from pip._internal.network.session import PipSession
35
36 ReqFileLines = Iterator[Tuple[int, Text]]
37
38 LineParser = Callable[[Text], Tuple[str, Values]]
39
40
41 __all__ = ['parse_requirements']
42
43 SCHEME_RE = re.compile(r'^(http|https|file):', re.I)
44 COMMENT_RE = re.compile(r'(^|\s+)#.*$')
45
46 # Matches environment variable-style values in '${MY_VARIABLE_1}' with the
47 # variable name consisting of only uppercase letters, digits or the '_'
48 # (underscore). This follows the POSIX standard defined in IEEE Std 1003.1,
49 # 2013 Edition.
50 ENV_VAR_RE = re.compile(r'(?P<var>\$\{(?P<name>[A-Z0-9_]+)\})')
51
52 SUPPORTED_OPTIONS = [
53 cmdoptions.index_url,
54 cmdoptions.extra_index_url,
55 cmdoptions.no_index,
56 cmdoptions.constraints,
57 cmdoptions.requirements,
58 cmdoptions.editable,
59 cmdoptions.find_links,
60 cmdoptions.no_binary,
61 cmdoptions.only_binary,
62 cmdoptions.prefer_binary,
63 cmdoptions.require_hashes,
64 cmdoptions.pre,
65 cmdoptions.trusted_host,
66 cmdoptions.use_new_feature,
67 ] # type: List[Callable[..., optparse.Option]]
68
69 # options to be passed to requirements
70 SUPPORTED_OPTIONS_REQ = [
71 cmdoptions.install_options,
72 cmdoptions.global_options,
73 cmdoptions.hash,
74 ] # type: List[Callable[..., optparse.Option]]
75
76 # the 'dest' string values
77 SUPPORTED_OPTIONS_REQ_DEST = [str(o().dest) for o in SUPPORTED_OPTIONS_REQ]
78
79
80 class ParsedRequirement:
81 def __init__(
82 self,
83 requirement, # type:str
84 is_editable, # type: bool
85 comes_from, # type: str
86 constraint, # type: bool
87 options=None, # type: Optional[Dict[str, Any]]
88 line_source=None, # type: Optional[str]
89 ):
90 # type: (...) -> None
91 self.requirement = requirement
92 self.is_editable = is_editable
93 self.comes_from = comes_from
94 self.options = options
95 self.constraint = constraint
96 self.line_source = line_source
97
98
99 class ParsedLine:
100 def __init__(
101 self,
102 filename, # type: str
103 lineno, # type: int
104 args, # type: str
105 opts, # type: Values
106 constraint, # type: bool
107 ):
108 # type: (...) -> None
109 self.filename = filename
110 self.lineno = lineno
111 self.opts = opts
112 self.constraint = constraint
113
114 if args:
115 self.is_requirement = True
116 self.is_editable = False
117 self.requirement = args
118 elif opts.editables:
119 self.is_requirement = True
120 self.is_editable = True
121 # We don't support multiple -e on one line
122 self.requirement = opts.editables[0]
123 else:
124 self.is_requirement = False
125
126
127 def parse_requirements(
128 filename, # type: str
129 session, # type: PipSession
130 finder=None, # type: Optional[PackageFinder]
131 options=None, # type: Optional[optparse.Values]
132 constraint=False, # type: bool
133 ):
134 # type: (...) -> Iterator[ParsedRequirement]
135 """Parse a requirements file and yield ParsedRequirement instances.
136
137 :param filename: Path or url of requirements file.
138 :param session: PipSession instance.
139 :param finder: Instance of pip.index.PackageFinder.
140 :param options: cli options.
141 :param constraint: If true, parsing a constraint file rather than
142 requirements file.
143 """
144 line_parser = get_line_parser(finder)
145 parser = RequirementsFileParser(session, line_parser)
146
147 for parsed_line in parser.parse(filename, constraint):
148 parsed_req = handle_line(
149 parsed_line,
150 options=options,
151 finder=finder,
152 session=session
153 )
154 if parsed_req is not None:
155 yield parsed_req
156
157
158 def preprocess(content):
159 # type: (str) -> ReqFileLines
160 """Split, filter, and join lines, and return a line iterator
161
162 :param content: the content of the requirements file
163 """
164 lines_enum = enumerate(content.splitlines(), start=1) # type: ReqFileLines
165 lines_enum = join_lines(lines_enum)
166 lines_enum = ignore_comments(lines_enum)
167 lines_enum = expand_env_variables(lines_enum)
168 return lines_enum
169
170
171 def handle_requirement_line(
172 line, # type: ParsedLine
173 options=None, # type: Optional[optparse.Values]
174 ):
175 # type: (...) -> ParsedRequirement
176
177 # preserve for the nested code path
178 line_comes_from = '{} {} (line {})'.format(
179 '-c' if line.constraint else '-r', line.filename, line.lineno,
180 )
181
182 assert line.is_requirement
183
184 if line.is_editable:
185 # For editable requirements, we don't support per-requirement
186 # options, so just return the parsed requirement.
187 return ParsedRequirement(
188 requirement=line.requirement,
189 is_editable=line.is_editable,
190 comes_from=line_comes_from,
191 constraint=line.constraint,
192 )
193 else:
194 if options:
195 # Disable wheels if the user has specified build options
196 cmdoptions.check_install_build_global(options, line.opts)
197
198 # get the options that apply to requirements
199 req_options = {}
200 for dest in SUPPORTED_OPTIONS_REQ_DEST:
201 if dest in line.opts.__dict__ and line.opts.__dict__[dest]:
202 req_options[dest] = line.opts.__dict__[dest]
203
204 line_source = f'line {line.lineno} of {line.filename}'
205 return ParsedRequirement(
206 requirement=line.requirement,
207 is_editable=line.is_editable,
208 comes_from=line_comes_from,
209 constraint=line.constraint,
210 options=req_options,
211 line_source=line_source,
212 )
213
214
215 def handle_option_line(
216 opts, # type: Values
217 filename, # type: str
218 lineno, # type: int
219 finder=None, # type: Optional[PackageFinder]
220 options=None, # type: Optional[optparse.Values]
221 session=None, # type: Optional[PipSession]
222 ):
223 # type: (...) -> None
224
225 if options:
226 # percolate options upward
227 if opts.require_hashes:
228 options.require_hashes = opts.require_hashes
229 if opts.features_enabled:
230 options.features_enabled.extend(
231 f for f in opts.features_enabled
232 if f not in options.features_enabled
233 )
234
235 # set finder options
236 if finder:
237 find_links = finder.find_links
238 index_urls = finder.index_urls
239 if opts.index_url:
240 index_urls = [opts.index_url]
241 if opts.no_index is True:
242 index_urls = []
243 if opts.extra_index_urls:
244 index_urls.extend(opts.extra_index_urls)
245 if opts.find_links:
246 # FIXME: it would be nice to keep track of the source
247 # of the find_links: support a find-links local path
248 # relative to a requirements file.
249 value = opts.find_links[0]
250 req_dir = os.path.dirname(os.path.abspath(filename))
251 relative_to_reqs_file = os.path.join(req_dir, value)
252 if os.path.exists(relative_to_reqs_file):
253 value = relative_to_reqs_file
254 find_links.append(value)
255
256 if session:
257 # We need to update the auth urls in session
258 session.update_index_urls(index_urls)
259
260 search_scope = SearchScope(
261 find_links=find_links,
262 index_urls=index_urls,
263 )
264 finder.search_scope = search_scope
265
266 if opts.pre:
267 finder.set_allow_all_prereleases()
268
269 if opts.prefer_binary:
270 finder.set_prefer_binary()
271
272 if session:
273 for host in opts.trusted_hosts or []:
274 source = f'line {lineno} of {filename}'
275 session.add_trusted_host(host, source=source)
276
277
278 def handle_line(
279 line, # type: ParsedLine
280 options=None, # type: Optional[optparse.Values]
281 finder=None, # type: Optional[PackageFinder]
282 session=None, # type: Optional[PipSession]
283 ):
284 # type: (...) -> Optional[ParsedRequirement]
285 """Handle a single parsed requirements line; This can result in
286 creating/yielding requirements, or updating the finder.
287
288 :param line: The parsed line to be processed.
289 :param options: CLI options.
290 :param finder: The finder - updated by non-requirement lines.
291 :param session: The session - updated by non-requirement lines.
292
293 Returns a ParsedRequirement object if the line is a requirement line,
294 otherwise returns None.
295
296 For lines that contain requirements, the only options that have an effect
297 are from SUPPORTED_OPTIONS_REQ, and they are scoped to the
298 requirement. Other options from SUPPORTED_OPTIONS may be present, but are
299 ignored.
300
301 For lines that do not contain requirements, the only options that have an
302 effect are from SUPPORTED_OPTIONS. Options from SUPPORTED_OPTIONS_REQ may
303 be present, but are ignored. These lines may contain multiple options
304 (although our docs imply only one is supported), and all our parsed and
305 affect the finder.
306 """
307
308 if line.is_requirement:
309 parsed_req = handle_requirement_line(line, options)
310 return parsed_req
311 else:
312 handle_option_line(
313 line.opts,
314 line.filename,
315 line.lineno,
316 finder,
317 options,
318 session,
319 )
320 return None
321
322
323 class RequirementsFileParser:
324 def __init__(
325 self,
326 session, # type: PipSession
327 line_parser, # type: LineParser
328 ):
329 # type: (...) -> None
330 self._session = session
331 self._line_parser = line_parser
332
333 def parse(self, filename, constraint):
334 # type: (str, bool) -> Iterator[ParsedLine]
335 """Parse a given file, yielding parsed lines.
336 """
337 yield from self._parse_and_recurse(filename, constraint)
338
339 def _parse_and_recurse(self, filename, constraint):
340 # type: (str, bool) -> Iterator[ParsedLine]
341 for line in self._parse_file(filename, constraint):
342 if (
343 not line.is_requirement and
344 (line.opts.requirements or line.opts.constraints)
345 ):
346 # parse a nested requirements file
347 if line.opts.requirements:
348 req_path = line.opts.requirements[0]
349 nested_constraint = False
350 else:
351 req_path = line.opts.constraints[0]
352 nested_constraint = True
353
354 # original file is over http
355 if SCHEME_RE.search(filename):
356 # do a url join so relative paths work
357 req_path = urllib.parse.urljoin(filename, req_path)
358 # original file and nested file are paths
359 elif not SCHEME_RE.search(req_path):
360 # do a join so relative paths work
361 req_path = os.path.join(
362 os.path.dirname(filename), req_path,
363 )
364
365 yield from self._parse_and_recurse(req_path, nested_constraint)
366 else:
367 yield line
368
369 def _parse_file(self, filename, constraint):
370 # type: (str, bool) -> Iterator[ParsedLine]
371 _, content = get_file_content(filename, self._session)
372
373 lines_enum = preprocess(content)
374
375 for line_number, line in lines_enum:
376 try:
377 args_str, opts = self._line_parser(line)
378 except OptionParsingError as e:
379 # add offending line
380 msg = f'Invalid requirement: {line}\n{e.msg}'
381 raise RequirementsFileParseError(msg)
382
383 yield ParsedLine(
384 filename,
385 line_number,
386 args_str,
387 opts,
388 constraint,
389 )
390
391
392 def get_line_parser(finder):
393 # type: (Optional[PackageFinder]) -> LineParser
394 def parse_line(line):
395 # type: (str) -> Tuple[str, Values]
396 # Build new parser for each line since it accumulates appendable
397 # options.
398 parser = build_parser()
399 defaults = parser.get_default_values()
400 defaults.index_url = None
401 if finder:
402 defaults.format_control = finder.format_control
403
404 args_str, options_str = break_args_options(line)
405
406 opts, _ = parser.parse_args(shlex.split(options_str), defaults)
407
408 return args_str, opts
409
410 return parse_line
411
412
413 def break_args_options(line):
414 # type: (str) -> Tuple[str, str]
415 """Break up the line into an args and options string. We only want to shlex
416 (and then optparse) the options, not the args. args can contain markers
417 which are corrupted by shlex.
418 """
419 tokens = line.split(' ')
420 args = []
421 options = tokens[:]
422 for token in tokens:
423 if token.startswith('-') or token.startswith('--'):
424 break
425 else:
426 args.append(token)
427 options.pop(0)
428 return ' '.join(args), ' '.join(options)
429
430
431 class OptionParsingError(Exception):
432 def __init__(self, msg):
433 # type: (str) -> None
434 self.msg = msg
435
436
437 def build_parser():
438 # type: () -> optparse.OptionParser
439 """
440 Return a parser for parsing requirement lines
441 """
442 parser = optparse.OptionParser(add_help_option=False)
443
444 option_factories = SUPPORTED_OPTIONS + SUPPORTED_OPTIONS_REQ
445 for option_factory in option_factories:
446 option = option_factory()
447 parser.add_option(option)
448
449 # By default optparse sys.exits on parsing errors. We want to wrap
450 # that in our own exception.
451 def parser_exit(self, msg):
452 # type: (Any, str) -> NoReturn
453 raise OptionParsingError(msg)
454 # NOTE: mypy disallows assigning to a method
455 # https://github.com/python/mypy/issues/2427
456 parser.exit = parser_exit # type: ignore
457
458 return parser
459
460
461 def join_lines(lines_enum):
462 # type: (ReqFileLines) -> ReqFileLines
463 """Joins a line ending in '\' with the previous line (except when following
464 comments). The joined line takes on the index of the first line.
465 """
466 primary_line_number = None
467 new_line = [] # type: List[str]
468 for line_number, line in lines_enum:
469 if not line.endswith('\\') or COMMENT_RE.match(line):
470 if COMMENT_RE.match(line):
471 # this ensures comments are always matched later
472 line = ' ' + line
473 if new_line:
474 new_line.append(line)
475 assert primary_line_number is not None
476 yield primary_line_number, ''.join(new_line)
477 new_line = []
478 else:
479 yield line_number, line
480 else:
481 if not new_line:
482 primary_line_number = line_number
483 new_line.append(line.strip('\\'))
484
485 # last line contains \
486 if new_line:
487 assert primary_line_number is not None
488 yield primary_line_number, ''.join(new_line)
489
490 # TODO: handle space after '\'.
491
492
493 def ignore_comments(lines_enum):
494 # type: (ReqFileLines) -> ReqFileLines
495 """
496 Strips comments and filter empty lines.
497 """
498 for line_number, line in lines_enum:
499 line = COMMENT_RE.sub('', line)
500 line = line.strip()
501 if line:
502 yield line_number, line
503
504
505 def expand_env_variables(lines_enum):
506 # type: (ReqFileLines) -> ReqFileLines
507 """Replace all environment variables that can be retrieved via `os.getenv`.
508
509 The only allowed format for environment variables defined in the
510 requirement file is `${MY_VARIABLE_1}` to ensure two things:
511
512 1. Strings that contain a `$` aren't accidentally (partially) expanded.
513 2. Ensure consistency across platforms for requirement files.
514
515 These points are the result of a discussion on the `github pull
516 request #3514 <https://github.com/pypa/pip/pull/3514>`_.
517
518 Valid characters in variable names follow the `POSIX standard
519 <http://pubs.opengroup.org/onlinepubs/9699919799/>`_ and are limited
520 to uppercase letter, digits and the `_` (underscore).
521 """
522 for line_number, line in lines_enum:
523 for env_var, var_name in ENV_VAR_RE.findall(line):
524 value = os.getenv(var_name)
525 if not value:
526 continue
527
528 line = line.replace(env_var, value)
529
530 yield line_number, line
531
532
533 def get_file_content(url, session):
534 # type: (str, PipSession) -> Tuple[str, str]
535 """Gets the content of a file; it may be a filename, file: URL, or
536 http: URL. Returns (location, content). Content is unicode.
537 Respects # -*- coding: declarations on the retrieved files.
538
539 :param url: File path or url.
540 :param session: PipSession instance.
541 """
542 scheme = get_url_scheme(url)
543
544 if scheme in ['http', 'https']:
545 # FIXME: catch some errors
546 resp = session.get(url)
547 raise_for_status(resp)
548 return resp.url, resp.text
549
550 elif scheme == 'file':
551 url = url_to_path(url)
552
553 try:
554 with open(url, 'rb') as f:
555 content = auto_decode(f.read())
556 except OSError as exc:
557 raise InstallationError(
558 f'Could not open requirements file: {exc}'
559 )
560 return url, content