Mercurial > repos > shellac > sam_consensus_v3
comparison env/lib/python3.9/site-packages/psutil/tests/__init__.py @ 0:4f3585e2f14b draft default tip
"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"
author | shellac |
---|---|
date | Mon, 22 Mar 2021 18:12:50 +0000 |
parents | |
children |
comparison
equal
deleted
inserted
replaced
-1:000000000000 | 0:4f3585e2f14b |
---|---|
1 # -*- coding: utf-8 -*- | |
2 | |
3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. | |
4 # Use of this source code is governed by a BSD-style license that can be | |
5 # found in the LICENSE file. | |
6 | |
7 """ | |
8 Test utilities. | |
9 """ | |
10 | |
11 from __future__ import print_function | |
12 import atexit | |
13 import contextlib | |
14 import ctypes | |
15 import errno | |
16 import functools | |
17 import gc | |
18 import inspect | |
19 import os | |
20 import random | |
21 import re | |
22 import select | |
23 import shutil | |
24 import signal | |
25 import socket | |
26 import stat | |
27 import subprocess | |
28 import sys | |
29 import tempfile | |
30 import textwrap | |
31 import threading | |
32 import time | |
33 import warnings | |
34 from socket import AF_INET | |
35 from socket import AF_INET6 | |
36 from socket import SOCK_STREAM | |
37 | |
38 import psutil | |
39 from psutil import AIX | |
40 from psutil import FREEBSD | |
41 from psutil import LINUX | |
42 from psutil import MACOS | |
43 from psutil import POSIX | |
44 from psutil import SUNOS | |
45 from psutil import WINDOWS | |
46 from psutil._common import bytes2human | |
47 from psutil._common import print_color | |
48 from psutil._common import supports_ipv6 | |
49 from psutil._compat import FileExistsError | |
50 from psutil._compat import FileNotFoundError | |
51 from psutil._compat import PY3 | |
52 from psutil._compat import range | |
53 from psutil._compat import super | |
54 from psutil._compat import u | |
55 from psutil._compat import unicode | |
56 from psutil._compat import which | |
57 | |
58 if PY3: | |
59 import unittest | |
60 else: | |
61 import unittest2 as unittest # requires "pip install unittest2" | |
62 | |
63 try: | |
64 from unittest import mock # py3 | |
65 except ImportError: | |
66 with warnings.catch_warnings(): | |
67 warnings.simplefilter("ignore") | |
68 import mock # NOQA - requires "pip install mock" | |
69 | |
70 if sys.version_info >= (3, 4): | |
71 import enum | |
72 else: | |
73 enum = None | |
74 | |
75 | |
76 __all__ = [ | |
77 # constants | |
78 'APPVEYOR', 'DEVNULL', 'GLOBAL_TIMEOUT', 'TOLERANCE_SYS_MEM', 'NO_RETRIES', | |
79 'PYPY', 'PYTHON_EXE', 'ROOT_DIR', 'SCRIPTS_DIR', 'TESTFN_PREFIX', | |
80 'UNICODE_SUFFIX', 'INVALID_UNICODE_SUFFIX', | |
81 'CI_TESTING', 'VALID_PROC_STATUSES', 'TOLERANCE_DISK_USAGE', 'IS_64BIT', | |
82 "HAS_CPU_AFFINITY", "HAS_CPU_FREQ", "HAS_ENVIRON", "HAS_PROC_IO_COUNTERS", | |
83 "HAS_IONICE", "HAS_MEMORY_MAPS", "HAS_PROC_CPU_NUM", "HAS_RLIMIT", | |
84 "HAS_SENSORS_BATTERY", "HAS_BATTERY", "HAS_SENSORS_FANS", | |
85 "HAS_SENSORS_TEMPERATURES", "HAS_MEMORY_FULL_INFO", | |
86 # subprocesses | |
87 'pyrun', 'terminate', 'reap_children', 'spawn_testproc', 'spawn_zombie', | |
88 'spawn_children_pair', | |
89 # threads | |
90 'ThreadTask' | |
91 # test utils | |
92 'unittest', 'skip_on_access_denied', 'skip_on_not_implemented', | |
93 'retry_on_failure', 'TestMemoryLeak', 'PsutilTestCase', | |
94 'process_namespace', 'system_namespace', 'print_sysinfo', | |
95 # install utils | |
96 'install_pip', 'install_test_deps', | |
97 # fs utils | |
98 'chdir', 'safe_rmpath', 'create_exe', 'decode_path', 'encode_path', | |
99 'get_testfn', | |
100 # os | |
101 'get_winver', 'kernel_version', | |
102 # sync primitives | |
103 'call_until', 'wait_for_pid', 'wait_for_file', | |
104 # network | |
105 'check_net_address', | |
106 'get_free_port', 'bind_socket', 'bind_unix_socket', 'tcp_socketpair', | |
107 'unix_socketpair', 'create_sockets', | |
108 # compat | |
109 'reload_module', 'import_module_by_path', | |
110 # others | |
111 'warn', 'copyload_shared_lib', 'is_namedtuple', | |
112 ] | |
113 | |
114 | |
115 # =================================================================== | |
116 # --- constants | |
117 # =================================================================== | |
118 | |
119 # --- platforms | |
120 | |
121 PYPY = '__pypy__' in sys.builtin_module_names | |
122 # whether we're running this test suite on a Continuous Integration service | |
123 APPVEYOR = 'APPVEYOR' in os.environ | |
124 GITHUB_ACTIONS = 'GITHUB_ACTIONS' in os.environ or 'CIBUILDWHEEL' in os.environ | |
125 CI_TESTING = APPVEYOR or GITHUB_ACTIONS | |
126 # are we a 64 bit process? | |
127 IS_64BIT = sys.maxsize > 2 ** 32 | |
128 | |
129 | |
130 # --- configurable defaults | |
131 | |
132 # how many times retry_on_failure() decorator will retry | |
133 NO_RETRIES = 10 | |
134 # bytes tolerance for system-wide related tests | |
135 TOLERANCE_SYS_MEM = 5 * 1024 * 1024 # 5MB | |
136 TOLERANCE_DISK_USAGE = 10 * 1024 * 1024 # 10MB | |
137 # the timeout used in functions which have to wait | |
138 GLOBAL_TIMEOUT = 5 | |
139 # be more tolerant if we're on CI in order to avoid false positives | |
140 if CI_TESTING: | |
141 NO_RETRIES *= 3 | |
142 GLOBAL_TIMEOUT *= 3 | |
143 TOLERANCE_SYS_MEM *= 3 | |
144 TOLERANCE_DISK_USAGE *= 3 | |
145 | |
146 # --- file names | |
147 | |
148 # Disambiguate TESTFN for parallel testing. | |
149 if os.name == 'java': | |
150 # Jython disallows @ in module names | |
151 TESTFN_PREFIX = '$psutil-%s-' % os.getpid() | |
152 else: | |
153 TESTFN_PREFIX = '@psutil-%s-' % os.getpid() | |
154 UNICODE_SUFFIX = u("-ƒőő") | |
155 # An invalid unicode string. | |
156 if PY3: | |
157 INVALID_UNICODE_SUFFIX = b"f\xc0\x80".decode('utf8', 'surrogateescape') | |
158 else: | |
159 INVALID_UNICODE_SUFFIX = "f\xc0\x80" | |
160 ASCII_FS = sys.getfilesystemencoding().lower() in ('ascii', 'us-ascii') | |
161 | |
162 # --- paths | |
163 | |
164 ROOT_DIR = os.path.realpath( | |
165 os.path.join(os.path.dirname(__file__), '..', '..')) | |
166 SCRIPTS_DIR = os.path.join(ROOT_DIR, 'scripts') | |
167 HERE = os.path.realpath(os.path.dirname(__file__)) | |
168 | |
169 # --- support | |
170 | |
171 HAS_CONNECTIONS_UNIX = POSIX and not SUNOS | |
172 HAS_CPU_AFFINITY = hasattr(psutil.Process, "cpu_affinity") | |
173 HAS_CPU_FREQ = hasattr(psutil, "cpu_freq") | |
174 HAS_GETLOADAVG = hasattr(psutil, "getloadavg") | |
175 HAS_ENVIRON = hasattr(psutil.Process, "environ") | |
176 HAS_IONICE = hasattr(psutil.Process, "ionice") | |
177 HAS_MEMORY_MAPS = hasattr(psutil.Process, "memory_maps") | |
178 HAS_NET_IO_COUNTERS = hasattr(psutil, "net_io_counters") | |
179 HAS_PROC_CPU_NUM = hasattr(psutil.Process, "cpu_num") | |
180 HAS_PROC_IO_COUNTERS = hasattr(psutil.Process, "io_counters") | |
181 HAS_RLIMIT = hasattr(psutil.Process, "rlimit") | |
182 HAS_SENSORS_BATTERY = hasattr(psutil, "sensors_battery") | |
183 try: | |
184 HAS_BATTERY = HAS_SENSORS_BATTERY and bool(psutil.sensors_battery()) | |
185 except Exception: | |
186 HAS_BATTERY = False | |
187 HAS_SENSORS_FANS = hasattr(psutil, "sensors_fans") | |
188 HAS_SENSORS_TEMPERATURES = hasattr(psutil, "sensors_temperatures") | |
189 HAS_THREADS = hasattr(psutil.Process, "threads") | |
190 SKIP_SYSCONS = (MACOS or AIX) and os.getuid() != 0 | |
191 | |
192 # --- misc | |
193 | |
194 | |
195 def _get_py_exe(): | |
196 def attempt(exe): | |
197 try: | |
198 subprocess.check_call( | |
199 [exe, "-V"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) | |
200 except Exception: | |
201 return None | |
202 else: | |
203 return exe | |
204 | |
205 if GITHUB_ACTIONS: | |
206 if PYPY: | |
207 return which("pypy3") if PY3 else which("pypy") | |
208 elif FREEBSD: | |
209 return os.path.realpath(sys.executable) | |
210 else: | |
211 return which('python') | |
212 elif MACOS: | |
213 exe = \ | |
214 attempt(sys.executable) or \ | |
215 attempt(os.path.realpath(sys.executable)) or \ | |
216 attempt(which("python%s.%s" % sys.version_info[:2])) or \ | |
217 attempt(psutil.Process().exe()) | |
218 if not exe: | |
219 raise ValueError("can't find python exe real abspath") | |
220 return exe | |
221 else: | |
222 exe = os.path.realpath(sys.executable) | |
223 assert os.path.exists(exe), exe | |
224 return exe | |
225 | |
226 | |
227 PYTHON_EXE = _get_py_exe() | |
228 DEVNULL = open(os.devnull, 'r+') | |
229 atexit.register(DEVNULL.close) | |
230 | |
231 VALID_PROC_STATUSES = [getattr(psutil, x) for x in dir(psutil) | |
232 if x.startswith('STATUS_')] | |
233 AF_UNIX = getattr(socket, "AF_UNIX", object()) | |
234 | |
235 _subprocesses_started = set() | |
236 _pids_started = set() | |
237 | |
238 | |
239 # =================================================================== | |
240 # --- threads | |
241 # =================================================================== | |
242 | |
243 | |
244 class ThreadTask(threading.Thread): | |
245 """A thread task which does nothing expect staying alive.""" | |
246 | |
247 def __init__(self): | |
248 super().__init__() | |
249 self._running = False | |
250 self._interval = 0.001 | |
251 self._flag = threading.Event() | |
252 | |
253 def __repr__(self): | |
254 name = self.__class__.__name__ | |
255 return '<%s running=%s at %#x>' % (name, self._running, id(self)) | |
256 | |
257 def __enter__(self): | |
258 self.start() | |
259 return self | |
260 | |
261 def __exit__(self, *args, **kwargs): | |
262 self.stop() | |
263 | |
264 def start(self): | |
265 """Start thread and keep it running until an explicit | |
266 stop() request. Polls for shutdown every 'timeout' seconds. | |
267 """ | |
268 if self._running: | |
269 raise ValueError("already started") | |
270 threading.Thread.start(self) | |
271 self._flag.wait() | |
272 | |
273 def run(self): | |
274 self._running = True | |
275 self._flag.set() | |
276 while self._running: | |
277 time.sleep(self._interval) | |
278 | |
279 def stop(self): | |
280 """Stop thread execution and and waits until it is stopped.""" | |
281 if not self._running: | |
282 raise ValueError("already stopped") | |
283 self._running = False | |
284 self.join() | |
285 | |
286 | |
287 # =================================================================== | |
288 # --- subprocesses | |
289 # =================================================================== | |
290 | |
291 | |
292 def _reap_children_on_err(fun): | |
293 @functools.wraps(fun) | |
294 def wrapper(*args, **kwargs): | |
295 try: | |
296 return fun(*args, **kwargs) | |
297 except Exception: | |
298 reap_children() | |
299 raise | |
300 return wrapper | |
301 | |
302 | |
303 @_reap_children_on_err | |
304 def spawn_testproc(cmd=None, **kwds): | |
305 """Creates a python subprocess which does nothing for 60 secs and | |
306 return it as a subprocess.Popen instance. | |
307 If "cmd" is specified that is used instead of python. | |
308 By default stdin and stdout are redirected to /dev/null. | |
309 It also attemps to make sure the process is in a reasonably | |
310 initialized state. | |
311 The process is registered for cleanup on reap_children(). | |
312 """ | |
313 kwds.setdefault("stdin", DEVNULL) | |
314 kwds.setdefault("stdout", DEVNULL) | |
315 kwds.setdefault("cwd", os.getcwd()) | |
316 kwds.setdefault("env", os.environ) | |
317 if WINDOWS: | |
318 # Prevents the subprocess to open error dialogs. This will also | |
319 # cause stderr to be suppressed, which is suboptimal in order | |
320 # to debug broken tests. | |
321 CREATE_NO_WINDOW = 0x8000000 | |
322 kwds.setdefault("creationflags", CREATE_NO_WINDOW) | |
323 if cmd is None: | |
324 testfn = get_testfn() | |
325 try: | |
326 safe_rmpath(testfn) | |
327 pyline = "from time import sleep;" \ | |
328 "open(r'%s', 'w').close();" \ | |
329 "sleep(60);" % testfn | |
330 cmd = [PYTHON_EXE, "-c", pyline] | |
331 sproc = subprocess.Popen(cmd, **kwds) | |
332 _subprocesses_started.add(sproc) | |
333 wait_for_file(testfn, delete=True, empty=True) | |
334 finally: | |
335 safe_rmpath(testfn) | |
336 else: | |
337 sproc = subprocess.Popen(cmd, **kwds) | |
338 _subprocesses_started.add(sproc) | |
339 wait_for_pid(sproc.pid) | |
340 return sproc | |
341 | |
342 | |
343 @_reap_children_on_err | |
344 def spawn_children_pair(): | |
345 """Create a subprocess which creates another one as in: | |
346 A (us) -> B (child) -> C (grandchild). | |
347 Return a (child, grandchild) tuple. | |
348 The 2 processes are fully initialized and will live for 60 secs | |
349 and are registered for cleanup on reap_children(). | |
350 """ | |
351 tfile = None | |
352 testfn = get_testfn(dir=os.getcwd()) | |
353 try: | |
354 s = textwrap.dedent("""\ | |
355 import subprocess, os, sys, time | |
356 s = "import os, time;" | |
357 s += "f = open('%s', 'w');" | |
358 s += "f.write(str(os.getpid()));" | |
359 s += "f.close();" | |
360 s += "time.sleep(60);" | |
361 p = subprocess.Popen([r'%s', '-c', s]) | |
362 p.wait() | |
363 """ % (os.path.basename(testfn), PYTHON_EXE)) | |
364 # On Windows if we create a subprocess with CREATE_NO_WINDOW flag | |
365 # set (which is the default) a "conhost.exe" extra process will be | |
366 # spawned as a child. We don't want that. | |
367 if WINDOWS: | |
368 subp, tfile = pyrun(s, creationflags=0) | |
369 else: | |
370 subp, tfile = pyrun(s) | |
371 child = psutil.Process(subp.pid) | |
372 grandchild_pid = int(wait_for_file(testfn, delete=True, empty=False)) | |
373 _pids_started.add(grandchild_pid) | |
374 grandchild = psutil.Process(grandchild_pid) | |
375 return (child, grandchild) | |
376 finally: | |
377 safe_rmpath(testfn) | |
378 if tfile is not None: | |
379 safe_rmpath(tfile) | |
380 | |
381 | |
382 def spawn_zombie(): | |
383 """Create a zombie process and return a (parent, zombie) process tuple. | |
384 In order to kill the zombie parent must be terminate()d first, then | |
385 zombie must be wait()ed on. | |
386 """ | |
387 assert psutil.POSIX | |
388 unix_file = get_testfn() | |
389 src = textwrap.dedent("""\ | |
390 import os, sys, time, socket, contextlib | |
391 child_pid = os.fork() | |
392 if child_pid > 0: | |
393 time.sleep(3000) | |
394 else: | |
395 # this is the zombie process | |
396 s = socket.socket(socket.AF_UNIX) | |
397 with contextlib.closing(s): | |
398 s.connect('%s') | |
399 if sys.version_info < (3, ): | |
400 pid = str(os.getpid()) | |
401 else: | |
402 pid = bytes(str(os.getpid()), 'ascii') | |
403 s.sendall(pid) | |
404 """ % unix_file) | |
405 tfile = None | |
406 sock = bind_unix_socket(unix_file) | |
407 try: | |
408 sock.settimeout(GLOBAL_TIMEOUT) | |
409 parent, tfile = pyrun(src) | |
410 conn, _ = sock.accept() | |
411 try: | |
412 select.select([conn.fileno()], [], [], GLOBAL_TIMEOUT) | |
413 zpid = int(conn.recv(1024)) | |
414 _pids_started.add(zpid) | |
415 zombie = psutil.Process(zpid) | |
416 call_until(lambda: zombie.status(), "ret == psutil.STATUS_ZOMBIE") | |
417 return (parent, zombie) | |
418 finally: | |
419 conn.close() | |
420 finally: | |
421 sock.close() | |
422 safe_rmpath(unix_file) | |
423 if tfile is not None: | |
424 safe_rmpath(tfile) | |
425 | |
426 | |
427 @_reap_children_on_err | |
428 def pyrun(src, **kwds): | |
429 """Run python 'src' code string in a separate interpreter. | |
430 Returns a subprocess.Popen instance and the test file where the source | |
431 code was written. | |
432 """ | |
433 kwds.setdefault("stdout", None) | |
434 kwds.setdefault("stderr", None) | |
435 srcfile = get_testfn() | |
436 try: | |
437 with open(srcfile, 'wt') as f: | |
438 f.write(src) | |
439 subp = spawn_testproc([PYTHON_EXE, f.name], **kwds) | |
440 wait_for_pid(subp.pid) | |
441 return (subp, srcfile) | |
442 except Exception: | |
443 safe_rmpath(srcfile) | |
444 raise | |
445 | |
446 | |
447 @_reap_children_on_err | |
448 def sh(cmd, **kwds): | |
449 """run cmd in a subprocess and return its output. | |
450 raises RuntimeError on error. | |
451 """ | |
452 shell = True if isinstance(cmd, (str, unicode)) else False | |
453 # Prevents subprocess to open error dialogs in case of error. | |
454 flags = 0x8000000 if WINDOWS and shell else 0 | |
455 kwds.setdefault("shell", shell) | |
456 kwds.setdefault("stdout", subprocess.PIPE) | |
457 kwds.setdefault("stderr", subprocess.PIPE) | |
458 kwds.setdefault("universal_newlines", True) | |
459 kwds.setdefault("creationflags", flags) | |
460 p = subprocess.Popen(cmd, **kwds) | |
461 _subprocesses_started.add(p) | |
462 if PY3: | |
463 stdout, stderr = p.communicate(timeout=GLOBAL_TIMEOUT) | |
464 else: | |
465 stdout, stderr = p.communicate() | |
466 if p.returncode != 0: | |
467 raise RuntimeError(stderr) | |
468 if stderr: | |
469 warn(stderr) | |
470 if stdout.endswith('\n'): | |
471 stdout = stdout[:-1] | |
472 return stdout | |
473 | |
474 | |
475 def terminate(proc_or_pid, sig=signal.SIGTERM, wait_timeout=GLOBAL_TIMEOUT): | |
476 """Terminate a process and wait() for it. | |
477 Process can be a PID or an instance of psutil.Process(), | |
478 subprocess.Popen() or psutil.Popen(). | |
479 If it's a subprocess.Popen() or psutil.Popen() instance also closes | |
480 its stdin / stdout / stderr fds. | |
481 PID is wait()ed even if the process is already gone (kills zombies). | |
482 Does nothing if the process does not exist. | |
483 Return process exit status. | |
484 """ | |
485 if POSIX: | |
486 from psutil._psposix import wait_pid | |
487 | |
488 def wait(proc, timeout): | |
489 if isinstance(proc, subprocess.Popen) and not PY3: | |
490 proc.wait() | |
491 else: | |
492 proc.wait(timeout) | |
493 if WINDOWS and isinstance(proc, subprocess.Popen): | |
494 # Otherwise PID may still hang around. | |
495 try: | |
496 return psutil.Process(proc.pid).wait(timeout) | |
497 except psutil.NoSuchProcess: | |
498 pass | |
499 | |
500 def sendsig(proc, sig): | |
501 # XXX: otherwise the build hangs for some reason. | |
502 if MACOS and GITHUB_ACTIONS: | |
503 sig = signal.SIGKILL | |
504 # If the process received SIGSTOP, SIGCONT is necessary first, | |
505 # otherwise SIGTERM won't work. | |
506 if POSIX and sig != signal.SIGKILL: | |
507 proc.send_signal(signal.SIGCONT) | |
508 proc.send_signal(sig) | |
509 | |
510 def term_subproc(proc, timeout): | |
511 try: | |
512 sendsig(proc, sig) | |
513 except OSError as err: | |
514 if WINDOWS and err.winerror == 6: # "invalid handle" | |
515 pass | |
516 elif err.errno != errno.ESRCH: | |
517 raise | |
518 return wait(proc, timeout) | |
519 | |
520 def term_psproc(proc, timeout): | |
521 try: | |
522 sendsig(proc, sig) | |
523 except psutil.NoSuchProcess: | |
524 pass | |
525 return wait(proc, timeout) | |
526 | |
527 def term_pid(pid, timeout): | |
528 try: | |
529 proc = psutil.Process(pid) | |
530 except psutil.NoSuchProcess: | |
531 # Needed to kill zombies. | |
532 if POSIX: | |
533 return wait_pid(pid, timeout) | |
534 else: | |
535 return term_psproc(proc, timeout) | |
536 | |
537 def flush_popen(proc): | |
538 if proc.stdout: | |
539 proc.stdout.close() | |
540 if proc.stderr: | |
541 proc.stderr.close() | |
542 # Flushing a BufferedWriter may raise an error. | |
543 if proc.stdin: | |
544 proc.stdin.close() | |
545 | |
546 p = proc_or_pid | |
547 try: | |
548 if isinstance(p, int): | |
549 return term_pid(p, wait_timeout) | |
550 elif isinstance(p, (psutil.Process, psutil.Popen)): | |
551 return term_psproc(p, wait_timeout) | |
552 elif isinstance(p, subprocess.Popen): | |
553 return term_subproc(p, wait_timeout) | |
554 else: | |
555 raise TypeError("wrong type %r" % p) | |
556 finally: | |
557 if isinstance(p, (subprocess.Popen, psutil.Popen)): | |
558 flush_popen(p) | |
559 pid = p if isinstance(p, int) else p.pid | |
560 assert not psutil.pid_exists(pid), pid | |
561 | |
562 | |
563 def reap_children(recursive=False): | |
564 """Terminate and wait() any subprocess started by this test suite | |
565 and any children currently running, ensuring that no processes stick | |
566 around to hog resources. | |
567 If resursive is True it also tries to terminate and wait() | |
568 all grandchildren started by this process. | |
569 """ | |
570 # Get the children here before terminating them, as in case of | |
571 # recursive=True we don't want to lose the intermediate reference | |
572 # pointing to the grandchildren. | |
573 children = psutil.Process().children(recursive=recursive) | |
574 | |
575 # Terminate subprocess.Popen. | |
576 while _subprocesses_started: | |
577 subp = _subprocesses_started.pop() | |
578 terminate(subp) | |
579 | |
580 # Collect started pids. | |
581 while _pids_started: | |
582 pid = _pids_started.pop() | |
583 terminate(pid) | |
584 | |
585 # Terminate children. | |
586 if children: | |
587 for p in children: | |
588 terminate(p, wait_timeout=None) | |
589 gone, alive = psutil.wait_procs(children, timeout=GLOBAL_TIMEOUT) | |
590 for p in alive: | |
591 warn("couldn't terminate process %r; attempting kill()" % p) | |
592 terminate(p, sig=signal.SIGKILL) | |
593 | |
594 | |
595 # =================================================================== | |
596 # --- OS | |
597 # =================================================================== | |
598 | |
599 | |
600 def kernel_version(): | |
601 """Return a tuple such as (2, 6, 36).""" | |
602 if not POSIX: | |
603 raise NotImplementedError("not POSIX") | |
604 s = "" | |
605 uname = os.uname()[2] | |
606 for c in uname: | |
607 if c.isdigit() or c == '.': | |
608 s += c | |
609 else: | |
610 break | |
611 if not s: | |
612 raise ValueError("can't parse %r" % uname) | |
613 minor = 0 | |
614 micro = 0 | |
615 nums = s.split('.') | |
616 major = int(nums[0]) | |
617 if len(nums) >= 2: | |
618 minor = int(nums[1]) | |
619 if len(nums) >= 3: | |
620 micro = int(nums[2]) | |
621 return (major, minor, micro) | |
622 | |
623 | |
624 def get_winver(): | |
625 if not WINDOWS: | |
626 raise NotImplementedError("not WINDOWS") | |
627 wv = sys.getwindowsversion() | |
628 if hasattr(wv, 'service_pack_major'): # python >= 2.7 | |
629 sp = wv.service_pack_major or 0 | |
630 else: | |
631 r = re.search(r"\s\d$", wv[4]) | |
632 if r: | |
633 sp = int(r.group(0)) | |
634 else: | |
635 sp = 0 | |
636 return (wv[0], wv[1], sp) | |
637 | |
638 | |
639 # =================================================================== | |
640 # --- sync primitives | |
641 # =================================================================== | |
642 | |
643 | |
644 class retry(object): | |
645 """A retry decorator.""" | |
646 | |
647 def __init__(self, | |
648 exception=Exception, | |
649 timeout=None, | |
650 retries=None, | |
651 interval=0.001, | |
652 logfun=None, | |
653 ): | |
654 if timeout and retries: | |
655 raise ValueError("timeout and retries args are mutually exclusive") | |
656 self.exception = exception | |
657 self.timeout = timeout | |
658 self.retries = retries | |
659 self.interval = interval | |
660 self.logfun = logfun | |
661 | |
662 def __iter__(self): | |
663 if self.timeout: | |
664 stop_at = time.time() + self.timeout | |
665 while time.time() < stop_at: | |
666 yield | |
667 elif self.retries: | |
668 for _ in range(self.retries): | |
669 yield | |
670 else: | |
671 while True: | |
672 yield | |
673 | |
674 def sleep(self): | |
675 if self.interval is not None: | |
676 time.sleep(self.interval) | |
677 | |
678 def __call__(self, fun): | |
679 @functools.wraps(fun) | |
680 def wrapper(*args, **kwargs): | |
681 exc = None | |
682 for _ in self: | |
683 try: | |
684 return fun(*args, **kwargs) | |
685 except self.exception as _: # NOQA | |
686 exc = _ | |
687 if self.logfun is not None: | |
688 self.logfun(exc) | |
689 self.sleep() | |
690 continue | |
691 if PY3: | |
692 raise exc | |
693 else: | |
694 raise | |
695 | |
696 # This way the user of the decorated function can change config | |
697 # parameters. | |
698 wrapper.decorator = self | |
699 return wrapper | |
700 | |
701 | |
702 @retry(exception=psutil.NoSuchProcess, logfun=None, timeout=GLOBAL_TIMEOUT, | |
703 interval=0.001) | |
704 def wait_for_pid(pid): | |
705 """Wait for pid to show up in the process list then return. | |
706 Used in the test suite to give time the sub process to initialize. | |
707 """ | |
708 psutil.Process(pid) | |
709 if WINDOWS: | |
710 # give it some more time to allow better initialization | |
711 time.sleep(0.01) | |
712 | |
713 | |
714 @retry(exception=(FileNotFoundError, AssertionError), logfun=None, | |
715 timeout=GLOBAL_TIMEOUT, interval=0.001) | |
716 def wait_for_file(fname, delete=True, empty=False): | |
717 """Wait for a file to be written on disk with some content.""" | |
718 with open(fname, "rb") as f: | |
719 data = f.read() | |
720 if not empty: | |
721 assert data | |
722 if delete: | |
723 safe_rmpath(fname) | |
724 return data | |
725 | |
726 | |
727 @retry(exception=AssertionError, logfun=None, timeout=GLOBAL_TIMEOUT, | |
728 interval=0.001) | |
729 def call_until(fun, expr): | |
730 """Keep calling function for timeout secs and exit if eval() | |
731 expression is True. | |
732 """ | |
733 ret = fun() | |
734 assert eval(expr) | |
735 return ret | |
736 | |
737 | |
738 # =================================================================== | |
739 # --- fs | |
740 # =================================================================== | |
741 | |
742 | |
743 def safe_rmpath(path): | |
744 "Convenience function for removing temporary test files or dirs" | |
745 def retry_fun(fun): | |
746 # On Windows it could happen that the file or directory has | |
747 # open handles or references preventing the delete operation | |
748 # to succeed immediately, so we retry for a while. See: | |
749 # https://bugs.python.org/issue33240 | |
750 stop_at = time.time() + GLOBAL_TIMEOUT | |
751 while time.time() < stop_at: | |
752 try: | |
753 return fun() | |
754 except FileNotFoundError: | |
755 pass | |
756 except WindowsError as _: | |
757 err = _ | |
758 warn("ignoring %s" % (str(err))) | |
759 time.sleep(0.01) | |
760 raise err | |
761 | |
762 try: | |
763 st = os.stat(path) | |
764 if stat.S_ISDIR(st.st_mode): | |
765 fun = functools.partial(shutil.rmtree, path) | |
766 else: | |
767 fun = functools.partial(os.remove, path) | |
768 if POSIX: | |
769 fun() | |
770 else: | |
771 retry_fun(fun) | |
772 except FileNotFoundError: | |
773 pass | |
774 | |
775 | |
776 def safe_mkdir(dir): | |
777 "Convenience function for creating a directory" | |
778 try: | |
779 os.mkdir(dir) | |
780 except FileExistsError: | |
781 pass | |
782 | |
783 | |
784 @contextlib.contextmanager | |
785 def chdir(dirname): | |
786 "Context manager which temporarily changes the current directory." | |
787 curdir = os.getcwd() | |
788 try: | |
789 os.chdir(dirname) | |
790 yield | |
791 finally: | |
792 os.chdir(curdir) | |
793 | |
794 | |
795 def create_exe(outpath, c_code=None): | |
796 """Creates an executable file in the given location.""" | |
797 assert not os.path.exists(outpath), outpath | |
798 if c_code: | |
799 if not which("gcc"): | |
800 raise ValueError("gcc is not installed") | |
801 if isinstance(c_code, bool): # c_code is True | |
802 c_code = textwrap.dedent( | |
803 """ | |
804 #include <unistd.h> | |
805 int main() { | |
806 pause(); | |
807 return 1; | |
808 } | |
809 """) | |
810 assert isinstance(c_code, str), c_code | |
811 with open(get_testfn(suffix='.c'), 'wt') as f: | |
812 f.write(c_code) | |
813 try: | |
814 subprocess.check_call(["gcc", f.name, "-o", outpath]) | |
815 finally: | |
816 safe_rmpath(f.name) | |
817 else: | |
818 # copy python executable | |
819 shutil.copyfile(PYTHON_EXE, outpath) | |
820 if POSIX: | |
821 st = os.stat(outpath) | |
822 os.chmod(outpath, st.st_mode | stat.S_IEXEC) | |
823 | |
824 | |
825 def get_testfn(suffix="", dir=None): | |
826 """Return an absolute pathname of a file or dir that did not | |
827 exist at the time this call is made. Also schedule it for safe | |
828 deletion at interpreter exit. It's technically racy but probably | |
829 not really due to the time variant. | |
830 """ | |
831 while True: | |
832 name = tempfile.mktemp(prefix=TESTFN_PREFIX, suffix=suffix, dir=dir) | |
833 if not os.path.exists(name): # also include dirs | |
834 return os.path.realpath(name) # needed for OSX | |
835 | |
836 | |
837 # =================================================================== | |
838 # --- testing | |
839 # =================================================================== | |
840 | |
841 | |
842 class TestCase(unittest.TestCase): | |
843 | |
844 # Print a full path representation of the single unit tests | |
845 # being run. | |
846 def __str__(self): | |
847 fqmod = self.__class__.__module__ | |
848 if not fqmod.startswith('psutil.'): | |
849 fqmod = 'psutil.tests.' + fqmod | |
850 return "%s.%s.%s" % ( | |
851 fqmod, self.__class__.__name__, self._testMethodName) | |
852 | |
853 # assertRaisesRegexp renamed to assertRaisesRegex in 3.3; | |
854 # add support for the new name. | |
855 if not hasattr(unittest.TestCase, 'assertRaisesRegex'): | |
856 assertRaisesRegex = unittest.TestCase.assertRaisesRegexp | |
857 | |
858 # ...otherwise multiprocessing.Pool complains | |
859 if not PY3: | |
860 def runTest(self): | |
861 pass | |
862 | |
863 | |
864 # monkey patch default unittest.TestCase | |
865 unittest.TestCase = TestCase | |
866 | |
867 | |
868 class PsutilTestCase(TestCase): | |
869 """Test class providing auto-cleanup wrappers on top of process | |
870 test utilities. | |
871 """ | |
872 | |
873 def get_testfn(self, suffix="", dir=None): | |
874 fname = get_testfn(suffix=suffix, dir=dir) | |
875 self.addCleanup(safe_rmpath, fname) | |
876 return fname | |
877 | |
878 def spawn_testproc(self, *args, **kwds): | |
879 sproc = spawn_testproc(*args, **kwds) | |
880 self.addCleanup(terminate, sproc) | |
881 return sproc | |
882 | |
883 def spawn_children_pair(self): | |
884 child1, child2 = spawn_children_pair() | |
885 self.addCleanup(terminate, child2) | |
886 self.addCleanup(terminate, child1) # executed first | |
887 return (child1, child2) | |
888 | |
889 def spawn_zombie(self): | |
890 parent, zombie = spawn_zombie() | |
891 self.addCleanup(terminate, zombie) | |
892 self.addCleanup(terminate, parent) # executed first | |
893 return (parent, zombie) | |
894 | |
895 def pyrun(self, *args, **kwds): | |
896 sproc, srcfile = pyrun(*args, **kwds) | |
897 self.addCleanup(safe_rmpath, srcfile) | |
898 self.addCleanup(terminate, sproc) # executed first | |
899 return sproc | |
900 | |
901 def assertProcessGone(self, proc): | |
902 self.assertRaises(psutil.NoSuchProcess, psutil.Process, proc.pid) | |
903 if isinstance(proc, (psutil.Process, psutil.Popen)): | |
904 assert not proc.is_running() | |
905 try: | |
906 status = proc.status() | |
907 except psutil.NoSuchProcess: | |
908 pass | |
909 else: | |
910 raise AssertionError("Process.status() didn't raise exception " | |
911 "(status=%s)" % status) | |
912 proc.wait(timeout=0) # assert not raise TimeoutExpired | |
913 assert not psutil.pid_exists(proc.pid), proc.pid | |
914 self.assertNotIn(proc.pid, psutil.pids()) | |
915 | |
916 | |
917 @unittest.skipIf(PYPY, "unreliable on PYPY") | |
918 class TestMemoryLeak(PsutilTestCase): | |
919 """Test framework class for detecting function memory leaks, | |
920 typically functions implemented in C which forgot to free() memory | |
921 from the heap. It does so by checking whether the process memory | |
922 usage increased before and after calling the function many times. | |
923 | |
924 Note that this is hard (probably impossible) to do reliably, due | |
925 to how the OS handles memory, the GC and so on (memory can even | |
926 decrease!). In order to avoid false positives, in case of failure | |
927 (mem > 0) we retry the test for up to 5 times, increasing call | |
928 repetitions each time. If the memory keeps increasing then it's a | |
929 failure. | |
930 | |
931 If available (Linux, OSX, Windows), USS memory is used for comparison, | |
932 since it's supposed to be more precise, see: | |
933 https://gmpy.dev/blog/2016/real-process-memory-and-environ-in-python | |
934 If not, RSS memory is used. mallinfo() on Linux and _heapwalk() on | |
935 Windows may give even more precision, but at the moment are not | |
936 implemented. | |
937 | |
938 PyPy appears to be completely unstable for this framework, probably | |
939 because of its JIT, so tests on PYPY are skipped. | |
940 | |
941 Usage: | |
942 | |
943 class TestLeaks(psutil.tests.TestMemoryLeak): | |
944 | |
945 def test_fun(self): | |
946 self.execute(some_function) | |
947 """ | |
948 # Configurable class attrs. | |
949 times = 200 | |
950 warmup_times = 10 | |
951 tolerance = 0 # memory | |
952 retries = 10 if CI_TESTING else 5 | |
953 verbose = True | |
954 _thisproc = psutil.Process() | |
955 | |
956 def _get_mem(self): | |
957 # USS is the closest thing we have to "real" memory usage and it | |
958 # should be less likely to produce false positives. | |
959 mem = self._thisproc.memory_full_info() | |
960 return getattr(mem, "uss", mem.rss) | |
961 | |
962 def _get_num_fds(self): | |
963 if POSIX: | |
964 return self._thisproc.num_fds() | |
965 else: | |
966 return self._thisproc.num_handles() | |
967 | |
968 def _log(self, msg): | |
969 if self.verbose: | |
970 print_color(msg, color="yellow", file=sys.stderr) | |
971 | |
972 def _check_fds(self, fun): | |
973 """Makes sure num_fds() (POSIX) or num_handles() (Windows) does | |
974 not increase after calling a function. Used to discover forgotten | |
975 close(2) and CloseHandle syscalls. | |
976 """ | |
977 before = self._get_num_fds() | |
978 self.call(fun) | |
979 after = self._get_num_fds() | |
980 diff = after - before | |
981 if diff < 0: | |
982 raise self.fail("negative diff %r (gc probably collected a " | |
983 "resource from a previous test)" % diff) | |
984 if diff > 0: | |
985 type_ = "fd" if POSIX else "handle" | |
986 if diff > 1: | |
987 type_ += "s" | |
988 msg = "%s unclosed %s after calling %r" % (diff, type_, fun) | |
989 raise self.fail(msg) | |
990 | |
991 def _call_ntimes(self, fun, times): | |
992 """Get 2 distinct memory samples, before and after having | |
993 called fun repeadetly, and return the memory difference. | |
994 """ | |
995 gc.collect(generation=1) | |
996 mem1 = self._get_mem() | |
997 for x in range(times): | |
998 ret = self.call(fun) | |
999 del x, ret | |
1000 gc.collect(generation=1) | |
1001 mem2 = self._get_mem() | |
1002 self.assertEqual(gc.garbage, []) | |
1003 diff = mem2 - mem1 # can also be negative | |
1004 return diff | |
1005 | |
1006 def _check_mem(self, fun, times, warmup_times, retries, tolerance): | |
1007 messages = [] | |
1008 prev_mem = 0 | |
1009 increase = times | |
1010 for idx in range(1, retries + 1): | |
1011 mem = self._call_ntimes(fun, times) | |
1012 msg = "Run #%s: extra-mem=%s, per-call=%s, calls=%s" % ( | |
1013 idx, bytes2human(mem), bytes2human(mem / times), times) | |
1014 messages.append(msg) | |
1015 success = mem <= tolerance or mem <= prev_mem | |
1016 if success: | |
1017 if idx > 1: | |
1018 self._log(msg) | |
1019 return | |
1020 else: | |
1021 if idx == 1: | |
1022 print() # NOQA | |
1023 self._log(msg) | |
1024 times += increase | |
1025 prev_mem = mem | |
1026 raise self.fail(". ".join(messages)) | |
1027 | |
1028 # --- | |
1029 | |
1030 def call(self, fun): | |
1031 return fun() | |
1032 | |
1033 def execute(self, fun, times=None, warmup_times=None, retries=None, | |
1034 tolerance=None): | |
1035 """Test a callable.""" | |
1036 times = times if times is not None else self.times | |
1037 warmup_times = warmup_times if warmup_times is not None \ | |
1038 else self.warmup_times | |
1039 retries = retries if retries is not None else self.retries | |
1040 tolerance = tolerance if tolerance is not None else self.tolerance | |
1041 try: | |
1042 assert times >= 1, "times must be >= 1" | |
1043 assert warmup_times >= 0, "warmup_times must be >= 0" | |
1044 assert retries >= 0, "retries must be >= 0" | |
1045 assert tolerance >= 0, "tolerance must be >= 0" | |
1046 except AssertionError as err: | |
1047 raise ValueError(str(err)) | |
1048 | |
1049 self._call_ntimes(fun, warmup_times) # warm up | |
1050 self._check_fds(fun) | |
1051 self._check_mem(fun, times=times, warmup_times=warmup_times, | |
1052 retries=retries, tolerance=tolerance) | |
1053 | |
1054 def execute_w_exc(self, exc, fun, **kwargs): | |
1055 """Convenience method to test a callable while making sure it | |
1056 raises an exception on every call. | |
1057 """ | |
1058 def call(): | |
1059 self.assertRaises(exc, fun) | |
1060 | |
1061 self.execute(call, **kwargs) | |
1062 | |
1063 | |
1064 def print_sysinfo(): | |
1065 import collections | |
1066 import datetime | |
1067 import getpass | |
1068 import locale | |
1069 import platform | |
1070 import pprint | |
1071 try: | |
1072 import pip | |
1073 except ImportError: | |
1074 pip = None | |
1075 try: | |
1076 import wheel | |
1077 except ImportError: | |
1078 wheel = None | |
1079 | |
1080 info = collections.OrderedDict() | |
1081 | |
1082 # OS | |
1083 if psutil.LINUX and which('lsb_release'): | |
1084 info['OS'] = sh('lsb_release -d -s') | |
1085 elif psutil.OSX: | |
1086 info['OS'] = 'Darwin %s' % platform.mac_ver()[0] | |
1087 elif psutil.WINDOWS: | |
1088 info['OS'] = "Windows " + ' '.join( | |
1089 map(str, platform.win32_ver())) | |
1090 if hasattr(platform, 'win32_edition'): | |
1091 info['OS'] += ", " + platform.win32_edition() | |
1092 else: | |
1093 info['OS'] = "%s %s" % (platform.system(), platform.version()) | |
1094 info['arch'] = ', '.join( | |
1095 list(platform.architecture()) + [platform.machine()]) | |
1096 if psutil.POSIX: | |
1097 info['kernel'] = platform.uname()[2] | |
1098 | |
1099 # python | |
1100 info['python'] = ', '.join([ | |
1101 platform.python_implementation(), | |
1102 platform.python_version(), | |
1103 platform.python_compiler()]) | |
1104 info['pip'] = getattr(pip, '__version__', 'not installed') | |
1105 if wheel is not None: | |
1106 info['pip'] += " (wheel=%s)" % wheel.__version__ | |
1107 | |
1108 # UNIX | |
1109 if psutil.POSIX: | |
1110 if which('gcc'): | |
1111 out = sh(['gcc', '--version']) | |
1112 info['gcc'] = str(out).split('\n')[0] | |
1113 else: | |
1114 info['gcc'] = 'not installed' | |
1115 s = platform.libc_ver()[1] | |
1116 if s: | |
1117 info['glibc'] = s | |
1118 | |
1119 # system | |
1120 info['fs-encoding'] = sys.getfilesystemencoding() | |
1121 lang = locale.getlocale() | |
1122 info['lang'] = '%s, %s' % (lang[0], lang[1]) | |
1123 info['boot-time'] = datetime.datetime.fromtimestamp( | |
1124 psutil.boot_time()).strftime("%Y-%m-%d %H:%M:%S") | |
1125 info['time'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") | |
1126 info['user'] = getpass.getuser() | |
1127 info['home'] = os.path.expanduser("~") | |
1128 info['cwd'] = os.getcwd() | |
1129 info['pyexe'] = PYTHON_EXE | |
1130 info['hostname'] = platform.node() | |
1131 info['PID'] = os.getpid() | |
1132 | |
1133 # metrics | |
1134 info['cpus'] = psutil.cpu_count() | |
1135 info['loadavg'] = "%.1f%%, %.1f%%, %.1f%%" % ( | |
1136 tuple([x / psutil.cpu_count() * 100 for x in psutil.getloadavg()])) | |
1137 mem = psutil.virtual_memory() | |
1138 info['memory'] = "%s%%, used=%s, total=%s" % ( | |
1139 int(mem.percent), bytes2human(mem.used), bytes2human(mem.total)) | |
1140 swap = psutil.swap_memory() | |
1141 info['swap'] = "%s%%, used=%s, total=%s" % ( | |
1142 int(swap.percent), bytes2human(swap.used), bytes2human(swap.total)) | |
1143 info['pids'] = len(psutil.pids()) | |
1144 pinfo = psutil.Process().as_dict() | |
1145 pinfo.pop('memory_maps', None) | |
1146 info['proc'] = pprint.pformat(pinfo) | |
1147 | |
1148 print("=" * 70, file=sys.stderr) # NOQA | |
1149 for k, v in info.items(): | |
1150 print("%-17s %s" % (k + ':', v), file=sys.stderr) # NOQA | |
1151 print("=" * 70, file=sys.stderr) # NOQA | |
1152 sys.stdout.flush() | |
1153 | |
1154 | |
1155 def _get_eligible_cpu(): | |
1156 p = psutil.Process() | |
1157 if hasattr(p, "cpu_num"): | |
1158 return p.cpu_num() | |
1159 elif hasattr(p, "cpu_affinity"): | |
1160 return random.choice(p.cpu_affinity()) | |
1161 return 0 | |
1162 | |
1163 | |
1164 class process_namespace: | |
1165 """A container that lists all Process class method names + some | |
1166 reasonable parameters to be called with. Utility methods (parent(), | |
1167 children(), ...) are excluded. | |
1168 | |
1169 >>> ns = process_namespace(psutil.Process()) | |
1170 >>> for fun, name in ns.iter(ns.getters): | |
1171 ... fun() | |
1172 """ | |
1173 utils = [ | |
1174 ('cpu_percent', (), {}), | |
1175 ('memory_percent', (), {}), | |
1176 ] | |
1177 | |
1178 ignored = [ | |
1179 ('as_dict', (), {}), | |
1180 ('children', (), {'recursive': True}), | |
1181 ('is_running', (), {}), | |
1182 ('memory_info_ex', (), {}), | |
1183 ('oneshot', (), {}), | |
1184 ('parent', (), {}), | |
1185 ('parents', (), {}), | |
1186 ('pid', (), {}), | |
1187 ('wait', (0, ), {}), | |
1188 ] | |
1189 | |
1190 getters = [ | |
1191 ('cmdline', (), {}), | |
1192 ('connections', (), {'kind': 'all'}), | |
1193 ('cpu_times', (), {}), | |
1194 ('create_time', (), {}), | |
1195 ('cwd', (), {}), | |
1196 ('exe', (), {}), | |
1197 ('memory_full_info', (), {}), | |
1198 ('memory_info', (), {}), | |
1199 ('name', (), {}), | |
1200 ('nice', (), {}), | |
1201 ('num_ctx_switches', (), {}), | |
1202 ('num_threads', (), {}), | |
1203 ('open_files', (), {}), | |
1204 ('ppid', (), {}), | |
1205 ('status', (), {}), | |
1206 ('threads', (), {}), | |
1207 ('username', (), {}), | |
1208 ] | |
1209 if POSIX: | |
1210 getters += [('uids', (), {})] | |
1211 getters += [('gids', (), {})] | |
1212 getters += [('terminal', (), {})] | |
1213 getters += [('num_fds', (), {})] | |
1214 if HAS_PROC_IO_COUNTERS: | |
1215 getters += [('io_counters', (), {})] | |
1216 if HAS_IONICE: | |
1217 getters += [('ionice', (), {})] | |
1218 if HAS_RLIMIT: | |
1219 getters += [('rlimit', (psutil.RLIMIT_NOFILE, ), {})] | |
1220 if HAS_CPU_AFFINITY: | |
1221 getters += [('cpu_affinity', (), {})] | |
1222 if HAS_PROC_CPU_NUM: | |
1223 getters += [('cpu_num', (), {})] | |
1224 if HAS_ENVIRON: | |
1225 getters += [('environ', (), {})] | |
1226 if WINDOWS: | |
1227 getters += [('num_handles', (), {})] | |
1228 if HAS_MEMORY_MAPS: | |
1229 getters += [('memory_maps', (), {'grouped': False})] | |
1230 | |
1231 setters = [] | |
1232 if POSIX: | |
1233 setters += [('nice', (0, ), {})] | |
1234 else: | |
1235 setters += [('nice', (psutil.NORMAL_PRIORITY_CLASS, ), {})] | |
1236 if HAS_RLIMIT: | |
1237 setters += [('rlimit', (psutil.RLIMIT_NOFILE, (1024, 4096)), {})] | |
1238 if HAS_IONICE: | |
1239 if LINUX: | |
1240 setters += [('ionice', (psutil.IOPRIO_CLASS_NONE, 0), {})] | |
1241 else: | |
1242 setters += [('ionice', (psutil.IOPRIO_NORMAL, ), {})] | |
1243 if HAS_CPU_AFFINITY: | |
1244 setters += [('cpu_affinity', ([_get_eligible_cpu()], ), {})] | |
1245 | |
1246 killers = [ | |
1247 ('send_signal', (signal.SIGTERM, ), {}), | |
1248 ('suspend', (), {}), | |
1249 ('resume', (), {}), | |
1250 ('terminate', (), {}), | |
1251 ('kill', (), {}), | |
1252 ] | |
1253 if WINDOWS: | |
1254 killers += [('send_signal', (signal.CTRL_C_EVENT, ), {})] | |
1255 killers += [('send_signal', (signal.CTRL_BREAK_EVENT, ), {})] | |
1256 | |
1257 all = utils + getters + setters + killers | |
1258 | |
1259 def __init__(self, proc): | |
1260 self._proc = proc | |
1261 | |
1262 def iter(self, ls, clear_cache=True): | |
1263 """Given a list of tuples yields a set of (fun, fun_name) tuples | |
1264 in random order. | |
1265 """ | |
1266 ls = list(ls) | |
1267 random.shuffle(ls) | |
1268 for fun_name, args, kwds in ls: | |
1269 if clear_cache: | |
1270 self.clear_cache() | |
1271 fun = getattr(self._proc, fun_name) | |
1272 fun = functools.partial(fun, *args, **kwds) | |
1273 yield (fun, fun_name) | |
1274 | |
1275 def clear_cache(self): | |
1276 """Clear the cache of a Process instance.""" | |
1277 self._proc._init(self._proc.pid, _ignore_nsp=True) | |
1278 | |
1279 @classmethod | |
1280 def test_class_coverage(cls, test_class, ls): | |
1281 """Given a TestCase instance and a list of tuples checks that | |
1282 the class defines the required test method names. | |
1283 """ | |
1284 for fun_name, _, _ in ls: | |
1285 meth_name = 'test_' + fun_name | |
1286 if not hasattr(test_class, meth_name): | |
1287 msg = "%r class should define a '%s' method" % ( | |
1288 test_class.__class__.__name__, meth_name) | |
1289 raise AttributeError(msg) | |
1290 | |
1291 @classmethod | |
1292 def test(cls): | |
1293 this = set([x[0] for x in cls.all]) | |
1294 ignored = set([x[0] for x in cls.ignored]) | |
1295 klass = set([x for x in dir(psutil.Process) if x[0] != '_']) | |
1296 leftout = (this | ignored) ^ klass | |
1297 if leftout: | |
1298 raise ValueError("uncovered Process class names: %r" % leftout) | |
1299 | |
1300 | |
1301 class system_namespace: | |
1302 """A container that lists all the module-level, system-related APIs. | |
1303 Utilities such as cpu_percent() are excluded. Usage: | |
1304 | |
1305 >>> ns = system_namespace | |
1306 >>> for fun, name in ns.iter(ns.getters): | |
1307 ... fun() | |
1308 """ | |
1309 getters = [ | |
1310 ('boot_time', (), {}), | |
1311 ('cpu_count', (), {'logical': False}), | |
1312 ('cpu_count', (), {'logical': True}), | |
1313 ('cpu_stats', (), {}), | |
1314 ('cpu_times', (), {'percpu': False}), | |
1315 ('cpu_times', (), {'percpu': True}), | |
1316 ('disk_io_counters', (), {'perdisk': True}), | |
1317 ('disk_partitions', (), {'all': True}), | |
1318 ('disk_usage', (os.getcwd(), ), {}), | |
1319 ('net_connections', (), {'kind': 'all'}), | |
1320 ('net_if_addrs', (), {}), | |
1321 ('net_if_stats', (), {}), | |
1322 ('net_io_counters', (), {'pernic': True}), | |
1323 ('pid_exists', (os.getpid(), ), {}), | |
1324 ('pids', (), {}), | |
1325 ('swap_memory', (), {}), | |
1326 ('users', (), {}), | |
1327 ('virtual_memory', (), {}), | |
1328 ] | |
1329 if HAS_CPU_FREQ: | |
1330 getters += [('cpu_freq', (), {'percpu': True})] | |
1331 if HAS_GETLOADAVG: | |
1332 getters += [('getloadavg', (), {})] | |
1333 if HAS_SENSORS_TEMPERATURES: | |
1334 getters += [('sensors_temperatures', (), {})] | |
1335 if HAS_SENSORS_FANS: | |
1336 getters += [('sensors_fans', (), {})] | |
1337 if HAS_SENSORS_BATTERY: | |
1338 getters += [('sensors_battery', (), {})] | |
1339 if WINDOWS: | |
1340 getters += [('win_service_iter', (), {})] | |
1341 getters += [('win_service_get', ('alg', ), {})] | |
1342 | |
1343 ignored = [ | |
1344 ('process_iter', (), {}), | |
1345 ('wait_procs', ([psutil.Process()], ), {}), | |
1346 ('cpu_percent', (), {}), | |
1347 ('cpu_times_percent', (), {}), | |
1348 ] | |
1349 | |
1350 all = getters | |
1351 | |
1352 @staticmethod | |
1353 def iter(ls): | |
1354 """Given a list of tuples yields a set of (fun, fun_name) tuples | |
1355 in random order. | |
1356 """ | |
1357 ls = list(ls) | |
1358 random.shuffle(ls) | |
1359 for fun_name, args, kwds in ls: | |
1360 fun = getattr(psutil, fun_name) | |
1361 fun = functools.partial(fun, *args, **kwds) | |
1362 yield (fun, fun_name) | |
1363 | |
1364 test_class_coverage = process_namespace.test_class_coverage | |
1365 | |
1366 | |
1367 def serialrun(klass): | |
1368 """A decorator to mark a TestCase class. When running parallel tests, | |
1369 class' unit tests will be run serially (1 process). | |
1370 """ | |
1371 # assert issubclass(klass, unittest.TestCase), klass | |
1372 assert inspect.isclass(klass), klass | |
1373 klass._serialrun = True | |
1374 return klass | |
1375 | |
1376 | |
1377 def retry_on_failure(retries=NO_RETRIES): | |
1378 """Decorator which runs a test function and retries N times before | |
1379 actually failing. | |
1380 """ | |
1381 def logfun(exc): | |
1382 print("%r, retrying" % exc, file=sys.stderr) # NOQA | |
1383 | |
1384 return retry(exception=AssertionError, timeout=None, retries=retries, | |
1385 logfun=logfun) | |
1386 | |
1387 | |
1388 def skip_on_access_denied(only_if=None): | |
1389 """Decorator to Ignore AccessDenied exceptions.""" | |
1390 def decorator(fun): | |
1391 @functools.wraps(fun) | |
1392 def wrapper(*args, **kwargs): | |
1393 try: | |
1394 return fun(*args, **kwargs) | |
1395 except psutil.AccessDenied: | |
1396 if only_if is not None: | |
1397 if not only_if: | |
1398 raise | |
1399 raise unittest.SkipTest("raises AccessDenied") | |
1400 return wrapper | |
1401 return decorator | |
1402 | |
1403 | |
1404 def skip_on_not_implemented(only_if=None): | |
1405 """Decorator to Ignore NotImplementedError exceptions.""" | |
1406 def decorator(fun): | |
1407 @functools.wraps(fun) | |
1408 def wrapper(*args, **kwargs): | |
1409 try: | |
1410 return fun(*args, **kwargs) | |
1411 except NotImplementedError: | |
1412 if only_if is not None: | |
1413 if not only_if: | |
1414 raise | |
1415 msg = "%r was skipped because it raised NotImplementedError" \ | |
1416 % fun.__name__ | |
1417 raise unittest.SkipTest(msg) | |
1418 return wrapper | |
1419 return decorator | |
1420 | |
1421 | |
1422 # =================================================================== | |
1423 # --- network | |
1424 # =================================================================== | |
1425 | |
1426 | |
1427 # XXX: no longer used | |
1428 def get_free_port(host='127.0.0.1'): | |
1429 """Return an unused TCP port. Subject to race conditions.""" | |
1430 with contextlib.closing(socket.socket()) as sock: | |
1431 sock.bind((host, 0)) | |
1432 return sock.getsockname()[1] | |
1433 | |
1434 | |
1435 def bind_socket(family=AF_INET, type=SOCK_STREAM, addr=None): | |
1436 """Binds a generic socket.""" | |
1437 if addr is None and family in (AF_INET, AF_INET6): | |
1438 addr = ("", 0) | |
1439 sock = socket.socket(family, type) | |
1440 try: | |
1441 if os.name not in ('nt', 'cygwin'): | |
1442 sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) | |
1443 sock.bind(addr) | |
1444 if type == socket.SOCK_STREAM: | |
1445 sock.listen(5) | |
1446 return sock | |
1447 except Exception: | |
1448 sock.close() | |
1449 raise | |
1450 | |
1451 | |
1452 def bind_unix_socket(name, type=socket.SOCK_STREAM): | |
1453 """Bind a UNIX socket.""" | |
1454 assert psutil.POSIX | |
1455 assert not os.path.exists(name), name | |
1456 sock = socket.socket(socket.AF_UNIX, type) | |
1457 try: | |
1458 sock.bind(name) | |
1459 if type == socket.SOCK_STREAM: | |
1460 sock.listen(5) | |
1461 except Exception: | |
1462 sock.close() | |
1463 raise | |
1464 return sock | |
1465 | |
1466 | |
1467 def tcp_socketpair(family, addr=("", 0)): | |
1468 """Build a pair of TCP sockets connected to each other. | |
1469 Return a (server, client) tuple. | |
1470 """ | |
1471 with contextlib.closing(socket.socket(family, SOCK_STREAM)) as ll: | |
1472 ll.bind(addr) | |
1473 ll.listen(5) | |
1474 addr = ll.getsockname() | |
1475 c = socket.socket(family, SOCK_STREAM) | |
1476 try: | |
1477 c.connect(addr) | |
1478 caddr = c.getsockname() | |
1479 while True: | |
1480 a, addr = ll.accept() | |
1481 # check that we've got the correct client | |
1482 if addr == caddr: | |
1483 return (a, c) | |
1484 a.close() | |
1485 except OSError: | |
1486 c.close() | |
1487 raise | |
1488 | |
1489 | |
1490 def unix_socketpair(name): | |
1491 """Build a pair of UNIX sockets connected to each other through | |
1492 the same UNIX file name. | |
1493 Return a (server, client) tuple. | |
1494 """ | |
1495 assert psutil.POSIX | |
1496 server = client = None | |
1497 try: | |
1498 server = bind_unix_socket(name, type=socket.SOCK_STREAM) | |
1499 server.setblocking(0) | |
1500 client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) | |
1501 client.setblocking(0) | |
1502 client.connect(name) | |
1503 # new = server.accept() | |
1504 except Exception: | |
1505 if server is not None: | |
1506 server.close() | |
1507 if client is not None: | |
1508 client.close() | |
1509 raise | |
1510 return (server, client) | |
1511 | |
1512 | |
1513 @contextlib.contextmanager | |
1514 def create_sockets(): | |
1515 """Open as many socket families / types as possible.""" | |
1516 socks = [] | |
1517 fname1 = fname2 = None | |
1518 try: | |
1519 socks.append(bind_socket(socket.AF_INET, socket.SOCK_STREAM)) | |
1520 socks.append(bind_socket(socket.AF_INET, socket.SOCK_DGRAM)) | |
1521 if supports_ipv6(): | |
1522 socks.append(bind_socket(socket.AF_INET6, socket.SOCK_STREAM)) | |
1523 socks.append(bind_socket(socket.AF_INET6, socket.SOCK_DGRAM)) | |
1524 if POSIX and HAS_CONNECTIONS_UNIX: | |
1525 fname1 = get_testfn() | |
1526 fname2 = get_testfn() | |
1527 s1, s2 = unix_socketpair(fname1) | |
1528 s3 = bind_unix_socket(fname2, type=socket.SOCK_DGRAM) | |
1529 for s in (s1, s2, s3): | |
1530 socks.append(s) | |
1531 yield socks | |
1532 finally: | |
1533 for s in socks: | |
1534 s.close() | |
1535 for fname in (fname1, fname2): | |
1536 if fname is not None: | |
1537 safe_rmpath(fname) | |
1538 | |
1539 | |
1540 def check_net_address(addr, family): | |
1541 """Check a net address validity. Supported families are IPv4, | |
1542 IPv6 and MAC addresses. | |
1543 """ | |
1544 import ipaddress # python >= 3.3 / requires "pip install ipaddress" | |
1545 if enum and PY3 and not PYPY: | |
1546 assert isinstance(family, enum.IntEnum), family | |
1547 if family == socket.AF_INET: | |
1548 octs = [int(x) for x in addr.split('.')] | |
1549 assert len(octs) == 4, addr | |
1550 for num in octs: | |
1551 assert 0 <= num <= 255, addr | |
1552 if not PY3: | |
1553 addr = unicode(addr) | |
1554 ipaddress.IPv4Address(addr) | |
1555 elif family == socket.AF_INET6: | |
1556 assert isinstance(addr, str), addr | |
1557 if not PY3: | |
1558 addr = unicode(addr) | |
1559 ipaddress.IPv6Address(addr) | |
1560 elif family == psutil.AF_LINK: | |
1561 assert re.match(r'([a-fA-F0-9]{2}[:|\-]?){6}', addr) is not None, addr | |
1562 else: | |
1563 raise ValueError("unknown family %r", family) | |
1564 | |
1565 | |
1566 def check_connection_ntuple(conn): | |
1567 """Check validity of a connection namedtuple.""" | |
1568 def check_ntuple(conn): | |
1569 has_pid = len(conn) == 7 | |
1570 assert len(conn) in (6, 7), len(conn) | |
1571 assert conn[0] == conn.fd, conn.fd | |
1572 assert conn[1] == conn.family, conn.family | |
1573 assert conn[2] == conn.type, conn.type | |
1574 assert conn[3] == conn.laddr, conn.laddr | |
1575 assert conn[4] == conn.raddr, conn.raddr | |
1576 assert conn[5] == conn.status, conn.status | |
1577 if has_pid: | |
1578 assert conn[6] == conn.pid, conn.pid | |
1579 | |
1580 def check_family(conn): | |
1581 assert conn.family in (AF_INET, AF_INET6, AF_UNIX), conn.family | |
1582 if enum is not None: | |
1583 assert isinstance(conn.family, enum.IntEnum), conn | |
1584 else: | |
1585 assert isinstance(conn.family, int), conn | |
1586 if conn.family == AF_INET: | |
1587 # actually try to bind the local socket; ignore IPv6 | |
1588 # sockets as their address might be represented as | |
1589 # an IPv4-mapped-address (e.g. "::127.0.0.1") | |
1590 # and that's rejected by bind() | |
1591 s = socket.socket(conn.family, conn.type) | |
1592 with contextlib.closing(s): | |
1593 try: | |
1594 s.bind((conn.laddr[0], 0)) | |
1595 except socket.error as err: | |
1596 if err.errno != errno.EADDRNOTAVAIL: | |
1597 raise | |
1598 elif conn.family == AF_UNIX: | |
1599 assert conn.status == psutil.CONN_NONE, conn.status | |
1600 | |
1601 def check_type(conn): | |
1602 # SOCK_SEQPACKET may happen in case of AF_UNIX socks | |
1603 SOCK_SEQPACKET = getattr(socket, "SOCK_SEQPACKET", object()) | |
1604 assert conn.type in (socket.SOCK_STREAM, socket.SOCK_DGRAM, | |
1605 SOCK_SEQPACKET), conn.type | |
1606 if enum is not None: | |
1607 assert isinstance(conn.type, enum.IntEnum), conn | |
1608 else: | |
1609 assert isinstance(conn.type, int), conn | |
1610 if conn.type == socket.SOCK_DGRAM: | |
1611 assert conn.status == psutil.CONN_NONE, conn.status | |
1612 | |
1613 def check_addrs(conn): | |
1614 # check IP address and port sanity | |
1615 for addr in (conn.laddr, conn.raddr): | |
1616 if conn.family in (AF_INET, AF_INET6): | |
1617 assert isinstance(addr, tuple), type(addr) | |
1618 if not addr: | |
1619 continue | |
1620 assert isinstance(addr.port, int), type(addr.port) | |
1621 assert 0 <= addr.port <= 65535, addr.port | |
1622 check_net_address(addr.ip, conn.family) | |
1623 elif conn.family == AF_UNIX: | |
1624 assert isinstance(addr, str), type(addr) | |
1625 | |
1626 def check_status(conn): | |
1627 assert isinstance(conn.status, str), conn.status | |
1628 valids = [getattr(psutil, x) for x in dir(psutil) | |
1629 if x.startswith('CONN_')] | |
1630 assert conn.status in valids, conn.status | |
1631 if conn.family in (AF_INET, AF_INET6) and conn.type == SOCK_STREAM: | |
1632 assert conn.status != psutil.CONN_NONE, conn.status | |
1633 else: | |
1634 assert conn.status == psutil.CONN_NONE, conn.status | |
1635 | |
1636 check_ntuple(conn) | |
1637 check_family(conn) | |
1638 check_type(conn) | |
1639 check_addrs(conn) | |
1640 check_status(conn) | |
1641 | |
1642 | |
1643 # =================================================================== | |
1644 # --- compatibility | |
1645 # =================================================================== | |
1646 | |
1647 | |
1648 def reload_module(module): | |
1649 """Backport of importlib.reload of Python 3.3+.""" | |
1650 try: | |
1651 import importlib | |
1652 if not hasattr(importlib, 'reload'): # python <=3.3 | |
1653 raise ImportError | |
1654 except ImportError: | |
1655 import imp | |
1656 return imp.reload(module) | |
1657 else: | |
1658 return importlib.reload(module) | |
1659 | |
1660 | |
1661 def import_module_by_path(path): | |
1662 name = os.path.splitext(os.path.basename(path))[0] | |
1663 if sys.version_info[0] == 2: | |
1664 import imp | |
1665 return imp.load_source(name, path) | |
1666 elif sys.version_info[:2] <= (3, 4): | |
1667 from importlib.machinery import SourceFileLoader | |
1668 return SourceFileLoader(name, path).load_module() | |
1669 else: | |
1670 import importlib.util | |
1671 spec = importlib.util.spec_from_file_location(name, path) | |
1672 mod = importlib.util.module_from_spec(spec) | |
1673 spec.loader.exec_module(mod) | |
1674 return mod | |
1675 | |
1676 | |
1677 # =================================================================== | |
1678 # --- others | |
1679 # =================================================================== | |
1680 | |
1681 | |
1682 def warn(msg): | |
1683 """Raise a warning msg.""" | |
1684 warnings.warn(msg, UserWarning) | |
1685 | |
1686 | |
1687 def is_namedtuple(x): | |
1688 """Check if object is an instance of namedtuple.""" | |
1689 t = type(x) | |
1690 b = t.__bases__ | |
1691 if len(b) != 1 or b[0] != tuple: | |
1692 return False | |
1693 f = getattr(t, '_fields', None) | |
1694 if not isinstance(f, tuple): | |
1695 return False | |
1696 return all(type(n) == str for n in f) | |
1697 | |
1698 | |
1699 if POSIX: | |
1700 @contextlib.contextmanager | |
1701 def copyload_shared_lib(suffix=""): | |
1702 """Ctx manager which picks up a random shared CO lib used | |
1703 by this process, copies it in another location and loads it | |
1704 in memory via ctypes. Return the new absolutized path. | |
1705 """ | |
1706 exe = 'pypy' if PYPY else 'python' | |
1707 ext = ".so" | |
1708 dst = get_testfn(suffix=suffix + ext) | |
1709 libs = [x.path for x in psutil.Process().memory_maps() if | |
1710 os.path.splitext(x.path)[1] == ext and | |
1711 exe in x.path.lower()] | |
1712 src = random.choice(libs) | |
1713 shutil.copyfile(src, dst) | |
1714 try: | |
1715 ctypes.CDLL(dst) | |
1716 yield dst | |
1717 finally: | |
1718 safe_rmpath(dst) | |
1719 else: | |
1720 @contextlib.contextmanager | |
1721 def copyload_shared_lib(suffix=""): | |
1722 """Ctx manager which picks up a random shared DLL lib used | |
1723 by this process, copies it in another location and loads it | |
1724 in memory via ctypes. | |
1725 Return the new absolutized, normcased path. | |
1726 """ | |
1727 from ctypes import wintypes | |
1728 from ctypes import WinError | |
1729 ext = ".dll" | |
1730 dst = get_testfn(suffix=suffix + ext) | |
1731 libs = [x.path for x in psutil.Process().memory_maps() if | |
1732 x.path.lower().endswith(ext) and | |
1733 'python' in os.path.basename(x.path).lower() and | |
1734 'wow64' not in x.path.lower()] | |
1735 if PYPY and not libs: | |
1736 libs = [x.path for x in psutil.Process().memory_maps() if | |
1737 'pypy' in os.path.basename(x.path).lower()] | |
1738 src = random.choice(libs) | |
1739 shutil.copyfile(src, dst) | |
1740 cfile = None | |
1741 try: | |
1742 cfile = ctypes.WinDLL(dst) | |
1743 yield dst | |
1744 finally: | |
1745 # Work around OverflowError: | |
1746 # - https://ci.appveyor.com/project/giampaolo/psutil/build/1207/ | |
1747 # job/o53330pbnri9bcw7 | |
1748 # - http://bugs.python.org/issue30286 | |
1749 # - http://stackoverflow.com/questions/23522055 | |
1750 if cfile is not None: | |
1751 FreeLibrary = ctypes.windll.kernel32.FreeLibrary | |
1752 FreeLibrary.argtypes = [wintypes.HMODULE] | |
1753 ret = FreeLibrary(cfile._handle) | |
1754 if ret == 0: | |
1755 WinError() | |
1756 safe_rmpath(dst) | |
1757 | |
1758 | |
1759 # =================================================================== | |
1760 # --- Exit funs (first is executed last) | |
1761 # =================================================================== | |
1762 | |
1763 | |
1764 # this is executed first | |
1765 @atexit.register | |
1766 def cleanup_test_procs(): | |
1767 reap_children(recursive=True) | |
1768 | |
1769 | |
1770 # atexit module does not execute exit functions in case of SIGTERM, which | |
1771 # gets sent to test subprocesses, which is a problem if they import this | |
1772 # module. With this it will. See: | |
1773 # https://gmpy.dev/blog/2016/how-to-always-execute-exit-functions-in-python | |
1774 if POSIX: | |
1775 signal.signal(signal.SIGTERM, lambda sig, frame: sys.exit(sig)) |