Mercurial > repos > shellac > sam_consensus_v3
comparison env/lib/python3.9/site-packages/virtualenv/seed/wheels/periodic_update.py @ 0:4f3585e2f14b draft default tip
"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"
author | shellac |
---|---|
date | Mon, 22 Mar 2021 18:12:50 +0000 |
parents | |
children |
comparison
equal
deleted
inserted
replaced
-1:000000000000 | 0:4f3585e2f14b |
---|---|
1 """ | |
2 Periodically update bundled versions. | |
3 """ | |
4 | |
5 from __future__ import absolute_import, unicode_literals | |
6 | |
7 import json | |
8 import logging | |
9 import os | |
10 import ssl | |
11 import subprocess | |
12 import sys | |
13 from datetime import datetime, timedelta | |
14 from itertools import groupby | |
15 from shutil import copy2 | |
16 from textwrap import dedent | |
17 from threading import Thread | |
18 | |
19 from six.moves.urllib.error import URLError | |
20 from six.moves.urllib.request import urlopen | |
21 | |
22 from virtualenv.app_data import AppDataDiskFolder | |
23 from virtualenv.info import PY2 | |
24 from virtualenv.util.path import Path | |
25 from virtualenv.util.subprocess import CREATE_NO_WINDOW, Popen | |
26 | |
27 from ..wheels.embed import BUNDLE_SUPPORT | |
28 from ..wheels.util import Wheel | |
29 | |
30 if PY2: | |
31 # on Python 2 datetime.strptime throws the error below if the import did not trigger on main thread | |
32 # Failed to import _strptime because the import lock is held by | |
33 try: | |
34 import _strptime # noqa | |
35 except ImportError: # pragma: no cov | |
36 pass # pragma: no cov | |
37 | |
38 | |
39 def periodic_update(distribution, for_py_version, wheel, search_dirs, app_data, do_periodic_update, env): | |
40 if do_periodic_update: | |
41 handle_auto_update(distribution, for_py_version, wheel, search_dirs, app_data, env) | |
42 | |
43 now = datetime.now() | |
44 | |
45 u_log = UpdateLog.from_app_data(app_data, distribution, for_py_version) | |
46 u_log_older_than_hour = now - u_log.completed > timedelta(hours=1) if u_log.completed is not None else False | |
47 for _, group in groupby(u_log.versions, key=lambda v: v.wheel.version_tuple[0:2]): | |
48 version = next(group) # use only latest patch version per minor, earlier assumed to be buggy | |
49 if wheel is not None and Path(version.filename).name == wheel.name: | |
50 break | |
51 if u_log.periodic is False or (u_log_older_than_hour and version.use(now)): | |
52 updated_wheel = Wheel(app_data.house / version.filename) | |
53 logging.debug("using %supdated wheel %s", "periodically " if updated_wheel else "", updated_wheel) | |
54 wheel = updated_wheel | |
55 break | |
56 | |
57 return wheel | |
58 | |
59 | |
60 def handle_auto_update(distribution, for_py_version, wheel, search_dirs, app_data, env): | |
61 embed_update_log = app_data.embed_update_log(distribution, for_py_version) | |
62 u_log = UpdateLog.from_dict(embed_update_log.read()) | |
63 if u_log.needs_update: | |
64 u_log.periodic = True | |
65 u_log.started = datetime.now() | |
66 embed_update_log.write(u_log.to_dict()) | |
67 trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, periodic=True, env=env) | |
68 | |
69 | |
70 DATETIME_FMT = "%Y-%m-%dT%H:%M:%S.%fZ" | |
71 | |
72 | |
73 def dump_datetime(value): | |
74 return None if value is None else value.strftime(DATETIME_FMT) | |
75 | |
76 | |
77 def load_datetime(value): | |
78 return None if value is None else datetime.strptime(value, DATETIME_FMT) | |
79 | |
80 | |
81 class NewVersion(object): | |
82 def __init__(self, filename, found_date, release_date): | |
83 self.filename = filename | |
84 self.found_date = found_date | |
85 self.release_date = release_date | |
86 | |
87 @classmethod | |
88 def from_dict(cls, dictionary): | |
89 return cls( | |
90 filename=dictionary["filename"], | |
91 found_date=load_datetime(dictionary["found_date"]), | |
92 release_date=load_datetime(dictionary["release_date"]), | |
93 ) | |
94 | |
95 def to_dict(self): | |
96 return { | |
97 "filename": self.filename, | |
98 "release_date": dump_datetime(self.release_date), | |
99 "found_date": dump_datetime(self.found_date), | |
100 } | |
101 | |
102 def use(self, now): | |
103 compare_from = self.release_date or self.found_date | |
104 return now - compare_from >= timedelta(days=28) | |
105 | |
106 def __repr__(self): | |
107 return "{}(filename={}), found_date={}, release_date={})".format( | |
108 self.__class__.__name__, | |
109 self.filename, | |
110 self.found_date, | |
111 self.release_date, | |
112 ) | |
113 | |
114 def __eq__(self, other): | |
115 return type(self) == type(other) and all( | |
116 getattr(self, k) == getattr(other, k) for k in ["filename", "release_date", "found_date"] | |
117 ) | |
118 | |
119 def __ne__(self, other): | |
120 return not (self == other) | |
121 | |
122 @property | |
123 def wheel(self): | |
124 return Wheel(Path(self.filename)) | |
125 | |
126 | |
127 class UpdateLog(object): | |
128 def __init__(self, started, completed, versions, periodic): | |
129 self.started = started | |
130 self.completed = completed | |
131 self.versions = versions | |
132 self.periodic = periodic | |
133 | |
134 @classmethod | |
135 def from_dict(cls, dictionary): | |
136 if dictionary is None: | |
137 dictionary = {} | |
138 return cls( | |
139 load_datetime(dictionary.get("started")), | |
140 load_datetime(dictionary.get("completed")), | |
141 [NewVersion.from_dict(v) for v in dictionary.get("versions", [])], | |
142 dictionary.get("periodic"), | |
143 ) | |
144 | |
145 @classmethod | |
146 def from_app_data(cls, app_data, distribution, for_py_version): | |
147 raw_json = app_data.embed_update_log(distribution, for_py_version).read() | |
148 return cls.from_dict(raw_json) | |
149 | |
150 def to_dict(self): | |
151 return { | |
152 "started": dump_datetime(self.started), | |
153 "completed": dump_datetime(self.completed), | |
154 "periodic": self.periodic, | |
155 "versions": [r.to_dict() for r in self.versions], | |
156 } | |
157 | |
158 @property | |
159 def needs_update(self): | |
160 now = datetime.now() | |
161 if self.completed is None: # never completed | |
162 return self._check_start(now) | |
163 else: | |
164 if now - self.completed <= timedelta(days=14): | |
165 return False | |
166 return self._check_start(now) | |
167 | |
168 def _check_start(self, now): | |
169 return self.started is None or now - self.started > timedelta(hours=1) | |
170 | |
171 | |
172 def trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, env, periodic): | |
173 wheel_path = None if wheel is None else str(wheel.path) | |
174 cmd = [ | |
175 sys.executable, | |
176 "-c", | |
177 dedent( | |
178 """ | |
179 from virtualenv.report import setup_report, MAX_LEVEL | |
180 from virtualenv.seed.wheels.periodic_update import do_update | |
181 setup_report(MAX_LEVEL, show_pid=True) | |
182 do_update({!r}, {!r}, {!r}, {!r}, {!r}, {!r}) | |
183 """, | |
184 ) | |
185 .strip() | |
186 .format(distribution, for_py_version, wheel_path, str(app_data), [str(p) for p in search_dirs], periodic), | |
187 ] | |
188 debug = env.get(str("_VIRTUALENV_PERIODIC_UPDATE_INLINE")) == str("1") | |
189 pipe = None if debug else subprocess.PIPE | |
190 kwargs = {"stdout": pipe, "stderr": pipe} | |
191 if not debug and sys.platform == "win32": | |
192 kwargs["creationflags"] = CREATE_NO_WINDOW | |
193 process = Popen(cmd, **kwargs) | |
194 logging.info( | |
195 "triggered periodic upgrade of %s%s (for python %s) via background process having PID %d", | |
196 distribution, | |
197 "" if wheel is None else "=={}".format(wheel.version), | |
198 for_py_version, | |
199 process.pid, | |
200 ) | |
201 if debug: | |
202 process.communicate() # on purpose not called to make it a background process | |
203 | |
204 | |
205 def do_update(distribution, for_py_version, embed_filename, app_data, search_dirs, periodic): | |
206 versions = None | |
207 try: | |
208 versions = _run_do_update(app_data, distribution, embed_filename, for_py_version, periodic, search_dirs) | |
209 finally: | |
210 logging.debug("done %s %s with %s", distribution, for_py_version, versions) | |
211 return versions | |
212 | |
213 | |
214 def _run_do_update(app_data, distribution, embed_filename, for_py_version, periodic, search_dirs): | |
215 from virtualenv.seed.wheels import acquire | |
216 | |
217 wheel_filename = None if embed_filename is None else Path(embed_filename) | |
218 embed_version = None if wheel_filename is None else Wheel(wheel_filename).version_tuple | |
219 app_data = AppDataDiskFolder(app_data) if isinstance(app_data, str) else app_data | |
220 search_dirs = [Path(p) if isinstance(p, str) else p for p in search_dirs] | |
221 wheelhouse = app_data.house | |
222 embed_update_log = app_data.embed_update_log(distribution, for_py_version) | |
223 u_log = UpdateLog.from_dict(embed_update_log.read()) | |
224 now = datetime.now() | |
225 if wheel_filename is not None: | |
226 dest = wheelhouse / wheel_filename.name | |
227 if not dest.exists(): | |
228 copy2(str(wheel_filename), str(wheelhouse)) | |
229 last, last_version, versions = None, None, [] | |
230 while last is None or not last.use(now): | |
231 download_time = datetime.now() | |
232 dest = acquire.download_wheel( | |
233 distribution=distribution, | |
234 version_spec=None if last_version is None else "<{}".format(last_version), | |
235 for_py_version=for_py_version, | |
236 search_dirs=search_dirs, | |
237 app_data=app_data, | |
238 to_folder=wheelhouse, | |
239 env=os.environ, | |
240 ) | |
241 if dest is None or (u_log.versions and u_log.versions[0].filename == dest.name): | |
242 break | |
243 release_date = release_date_for_wheel_path(dest.path) | |
244 last = NewVersion(filename=dest.path.name, release_date=release_date, found_date=download_time) | |
245 logging.info("detected %s in %s", last, datetime.now() - download_time) | |
246 versions.append(last) | |
247 last_wheel = Wheel(Path(last.filename)) | |
248 last_version = last_wheel.version | |
249 if embed_version is not None: | |
250 if embed_version >= last_wheel.version_tuple: # stop download if we reach the embed version | |
251 break | |
252 u_log.periodic = periodic | |
253 if not u_log.periodic: | |
254 u_log.started = now | |
255 u_log.versions = versions + u_log.versions | |
256 u_log.completed = datetime.now() | |
257 embed_update_log.write(u_log.to_dict()) | |
258 return versions | |
259 | |
260 | |
261 def release_date_for_wheel_path(dest): | |
262 wheel = Wheel(dest) | |
263 # the most accurate is to ask PyPi - e.g. https://pypi.org/pypi/pip/json, | |
264 # see https://warehouse.pypa.io/api-reference/json/ for more details | |
265 content = _pypi_get_distribution_info_cached(wheel.distribution) | |
266 if content is not None: | |
267 try: | |
268 upload_time = content["releases"][wheel.version][0]["upload_time"] | |
269 return datetime.strptime(upload_time, "%Y-%m-%dT%H:%M:%S") | |
270 except Exception as exception: | |
271 logging.error("could not load release date %s because %r", content, exception) | |
272 return None | |
273 | |
274 | |
275 def _request_context(): | |
276 yield None | |
277 # fallback to non verified HTTPS (the information we request is not sensitive, so fallback) | |
278 yield ssl._create_unverified_context() # noqa | |
279 | |
280 | |
281 _PYPI_CACHE = {} | |
282 | |
283 | |
284 def _pypi_get_distribution_info_cached(distribution): | |
285 if distribution not in _PYPI_CACHE: | |
286 _PYPI_CACHE[distribution] = _pypi_get_distribution_info(distribution) | |
287 return _PYPI_CACHE[distribution] | |
288 | |
289 | |
290 def _pypi_get_distribution_info(distribution): | |
291 content, url = None, "https://pypi.org/pypi/{}/json".format(distribution) | |
292 try: | |
293 for context in _request_context(): | |
294 try: | |
295 with urlopen(url, context=context) as file_handler: | |
296 content = json.load(file_handler) | |
297 break | |
298 except URLError as exception: | |
299 logging.error("failed to access %s because %r", url, exception) | |
300 except Exception as exception: | |
301 logging.error("failed to access %s because %r", url, exception) | |
302 return content | |
303 | |
304 | |
305 def manual_upgrade(app_data, env): | |
306 threads = [] | |
307 | |
308 for for_py_version, distribution_to_package in BUNDLE_SUPPORT.items(): | |
309 # load extra search dir for the given for_py | |
310 for distribution in distribution_to_package.keys(): | |
311 thread = Thread(target=_run_manual_upgrade, args=(app_data, distribution, for_py_version, env)) | |
312 thread.start() | |
313 threads.append(thread) | |
314 | |
315 for thread in threads: | |
316 thread.join() | |
317 | |
318 | |
319 def _run_manual_upgrade(app_data, distribution, for_py_version, env): | |
320 start = datetime.now() | |
321 from .bundle import from_bundle | |
322 | |
323 current = from_bundle( | |
324 distribution=distribution, | |
325 version=None, | |
326 for_py_version=for_py_version, | |
327 search_dirs=[], | |
328 app_data=app_data, | |
329 do_periodic_update=False, | |
330 env=env, | |
331 ) | |
332 logging.warning( | |
333 "upgrade %s for python %s with current %s", | |
334 distribution, | |
335 for_py_version, | |
336 "" if current is None else current.name, | |
337 ) | |
338 versions = do_update( | |
339 distribution=distribution, | |
340 for_py_version=for_py_version, | |
341 embed_filename=current.path, | |
342 app_data=app_data, | |
343 search_dirs=[], | |
344 periodic=False, | |
345 ) | |
346 msg = "upgraded %s for python %s in %s {}".format( | |
347 "new entries found:\n%s" if versions else "no new versions found", | |
348 ) | |
349 args = [ | |
350 distribution, | |
351 for_py_version, | |
352 datetime.now() - start, | |
353 ] | |
354 if versions: | |
355 args.append("\n".join("\t{}".format(v) for v in versions)) | |
356 logging.warning(msg, *args) | |
357 | |
358 | |
359 __all__ = ( | |
360 "periodic_update", | |
361 "do_update", | |
362 "manual_upgrade", | |
363 "NewVersion", | |
364 "UpdateLog", | |
365 "load_datetime", | |
366 "dump_datetime", | |
367 "trigger_update", | |
368 "release_date_for_wheel_path", | |
369 ) |