Mercurial > repos > guerler > springsuite
comparison planemo/lib/python3.7/site-packages/psutil/tests/__init__.py @ 1:56ad4e20f292 draft
"planemo upload commit 6eee67778febed82ddd413c3ca40b3183a3898f1"
| author | guerler |
|---|---|
| date | Fri, 31 Jul 2020 00:32:28 -0400 |
| parents | |
| children |
comparison
equal
deleted
inserted
replaced
| 0:d30785e31577 | 1:56ad4e20f292 |
|---|---|
| 1 # -*- coding: utf-8 -*- | |
| 2 | |
| 3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. | |
| 4 # Use of this source code is governed by a BSD-style license that can be | |
| 5 # found in the LICENSE file. | |
| 6 | |
| 7 """ | |
| 8 Test utilities. | |
| 9 """ | |
| 10 | |
| 11 from __future__ import print_function | |
| 12 import atexit | |
| 13 import contextlib | |
| 14 import ctypes | |
| 15 import errno | |
| 16 import functools | |
| 17 import gc | |
| 18 import inspect | |
| 19 import os | |
| 20 import random | |
| 21 import re | |
| 22 import select | |
| 23 import shutil | |
| 24 import signal | |
| 25 import socket | |
| 26 import stat | |
| 27 import subprocess | |
| 28 import sys | |
| 29 import tempfile | |
| 30 import textwrap | |
| 31 import threading | |
| 32 import time | |
| 33 import warnings | |
| 34 from socket import AF_INET | |
| 35 from socket import AF_INET6 | |
| 36 from socket import SOCK_STREAM | |
| 37 | |
| 38 import psutil | |
| 39 from psutil import AIX | |
| 40 from psutil import LINUX | |
| 41 from psutil import MACOS | |
| 42 from psutil import POSIX | |
| 43 from psutil import SUNOS | |
| 44 from psutil import WINDOWS | |
| 45 from psutil._common import bytes2human | |
| 46 from psutil._common import print_color | |
| 47 from psutil._common import supports_ipv6 | |
| 48 from psutil._compat import FileExistsError | |
| 49 from psutil._compat import FileNotFoundError | |
| 50 from psutil._compat import PY3 | |
| 51 from psutil._compat import range | |
| 52 from psutil._compat import super | |
| 53 from psutil._compat import u | |
| 54 from psutil._compat import unicode | |
| 55 from psutil._compat import which | |
| 56 | |
| 57 if PY3: | |
| 58 import unittest | |
| 59 else: | |
| 60 import unittest2 as unittest # requires "pip install unittest2" | |
| 61 | |
| 62 try: | |
| 63 from unittest import mock # py3 | |
| 64 except ImportError: | |
| 65 with warnings.catch_warnings(): | |
| 66 warnings.simplefilter("ignore") | |
| 67 import mock # NOQA - requires "pip install mock" | |
| 68 | |
| 69 if sys.version_info >= (3, 4): | |
| 70 import enum | |
| 71 else: | |
| 72 enum = None | |
| 73 | |
| 74 | |
| 75 __all__ = [ | |
| 76 # constants | |
| 77 'APPVEYOR', 'DEVNULL', 'GLOBAL_TIMEOUT', 'TOLERANCE_SYS_MEM', 'NO_RETRIES', | |
| 78 'PYPY', 'PYTHON_EXE', 'ROOT_DIR', 'SCRIPTS_DIR', 'TESTFN_PREFIX', | |
| 79 'UNICODE_SUFFIX', 'INVALID_UNICODE_SUFFIX', 'TRAVIS', 'CIRRUS', | |
| 80 'CI_TESTING', 'VALID_PROC_STATUSES', 'TOLERANCE_DISK_USAGE', 'IS_64BIT', | |
| 81 "HAS_CPU_AFFINITY", "HAS_CPU_FREQ", "HAS_ENVIRON", "HAS_PROC_IO_COUNTERS", | |
| 82 "HAS_IONICE", "HAS_MEMORY_MAPS", "HAS_PROC_CPU_NUM", "HAS_RLIMIT", | |
| 83 "HAS_SENSORS_BATTERY", "HAS_BATTERY", "HAS_SENSORS_FANS", | |
| 84 "HAS_SENSORS_TEMPERATURES", "HAS_MEMORY_FULL_INFO", | |
| 85 # subprocesses | |
| 86 'pyrun', 'terminate', 'reap_children', 'spawn_testproc', 'spawn_zombie', | |
| 87 'spawn_children_pair', | |
| 88 # threads | |
| 89 'ThreadTask' | |
| 90 # test utils | |
| 91 'unittest', 'skip_on_access_denied', 'skip_on_not_implemented', | |
| 92 'retry_on_failure', 'TestMemoryLeak', 'PsutilTestCase', | |
| 93 'process_namespace', 'system_namespace', 'print_sysinfo', | |
| 94 # install utils | |
| 95 'install_pip', 'install_test_deps', | |
| 96 # fs utils | |
| 97 'chdir', 'safe_rmpath', 'create_exe', 'decode_path', 'encode_path', | |
| 98 'get_testfn', | |
| 99 # os | |
| 100 'get_winver', 'get_kernel_version', | |
| 101 # sync primitives | |
| 102 'call_until', 'wait_for_pid', 'wait_for_file', | |
| 103 # network | |
| 104 'check_net_address', | |
| 105 'get_free_port', 'bind_socket', 'bind_unix_socket', 'tcp_socketpair', | |
| 106 'unix_socketpair', 'create_sockets', | |
| 107 # compat | |
| 108 'reload_module', 'import_module_by_path', | |
| 109 # others | |
| 110 'warn', 'copyload_shared_lib', 'is_namedtuple', | |
| 111 ] | |
| 112 | |
| 113 | |
| 114 # =================================================================== | |
| 115 # --- constants | |
| 116 # =================================================================== | |
| 117 | |
| 118 # --- platforms | |
| 119 | |
| 120 PYPY = '__pypy__' in sys.builtin_module_names | |
| 121 # whether we're running this test suite on a Continuous Integration service | |
| 122 TRAVIS = 'TRAVIS' in os.environ | |
| 123 APPVEYOR = 'APPVEYOR' in os.environ | |
| 124 CIRRUS = 'CIRRUS' in os.environ | |
| 125 GITHUB_WHEELS = 'CIBUILDWHEEL' in os.environ | |
| 126 CI_TESTING = TRAVIS or APPVEYOR or CIRRUS or GITHUB_WHEELS | |
| 127 # are we a 64 bit process? | |
| 128 IS_64BIT = sys.maxsize > 2 ** 32 | |
| 129 | |
| 130 | |
| 131 # --- configurable defaults | |
| 132 | |
| 133 # how many times retry_on_failure() decorator will retry | |
| 134 NO_RETRIES = 10 | |
| 135 # bytes tolerance for system-wide related tests | |
| 136 TOLERANCE_SYS_MEM = 5 * 1024 * 1024 # 5MB | |
| 137 TOLERANCE_DISK_USAGE = 10 * 1024 * 1024 # 10MB | |
| 138 # the timeout used in functions which have to wait | |
| 139 GLOBAL_TIMEOUT = 5 | |
| 140 # be more tolerant if we're on travis / appveyor in order to avoid | |
| 141 # false positives | |
| 142 if CI_TESTING: | |
| 143 NO_RETRIES *= 3 | |
| 144 GLOBAL_TIMEOUT *= 3 | |
| 145 TOLERANCE_SYS_MEM *= 3 | |
| 146 TOLERANCE_DISK_USAGE *= 3 | |
| 147 | |
| 148 # --- file names | |
| 149 | |
| 150 # Disambiguate TESTFN for parallel testing. | |
| 151 if os.name == 'java': | |
| 152 # Jython disallows @ in module names | |
| 153 TESTFN_PREFIX = '$psutil-%s-' % os.getpid() | |
| 154 else: | |
| 155 TESTFN_PREFIX = '@psutil-%s-' % os.getpid() | |
| 156 UNICODE_SUFFIX = u("-ƒőő") | |
| 157 # An invalid unicode string. | |
| 158 if PY3: | |
| 159 INVALID_UNICODE_SUFFIX = b"f\xc0\x80".decode('utf8', 'surrogateescape') | |
| 160 else: | |
| 161 INVALID_UNICODE_SUFFIX = "f\xc0\x80" | |
| 162 ASCII_FS = sys.getfilesystemencoding().lower() in ('ascii', 'us-ascii') | |
| 163 | |
| 164 # --- paths | |
| 165 | |
| 166 ROOT_DIR = os.path.realpath( | |
| 167 os.path.join(os.path.dirname(__file__), '..', '..')) | |
| 168 SCRIPTS_DIR = os.path.join(ROOT_DIR, 'scripts') | |
| 169 HERE = os.path.realpath(os.path.dirname(__file__)) | |
| 170 | |
| 171 # --- support | |
| 172 | |
| 173 HAS_CONNECTIONS_UNIX = POSIX and not SUNOS | |
| 174 HAS_CPU_AFFINITY = hasattr(psutil.Process, "cpu_affinity") | |
| 175 HAS_CPU_FREQ = hasattr(psutil, "cpu_freq") | |
| 176 HAS_GETLOADAVG = hasattr(psutil, "getloadavg") | |
| 177 HAS_ENVIRON = hasattr(psutil.Process, "environ") | |
| 178 HAS_IONICE = hasattr(psutil.Process, "ionice") | |
| 179 HAS_MEMORY_MAPS = hasattr(psutil.Process, "memory_maps") | |
| 180 HAS_NET_IO_COUNTERS = hasattr(psutil, "net_io_counters") | |
| 181 HAS_PROC_CPU_NUM = hasattr(psutil.Process, "cpu_num") | |
| 182 HAS_PROC_IO_COUNTERS = hasattr(psutil.Process, "io_counters") | |
| 183 HAS_RLIMIT = hasattr(psutil.Process, "rlimit") | |
| 184 HAS_SENSORS_BATTERY = hasattr(psutil, "sensors_battery") | |
| 185 try: | |
| 186 HAS_BATTERY = HAS_SENSORS_BATTERY and bool(psutil.sensors_battery()) | |
| 187 except Exception: | |
| 188 HAS_BATTERY = True | |
| 189 HAS_SENSORS_FANS = hasattr(psutil, "sensors_fans") | |
| 190 HAS_SENSORS_TEMPERATURES = hasattr(psutil, "sensors_temperatures") | |
| 191 HAS_THREADS = hasattr(psutil.Process, "threads") | |
| 192 SKIP_SYSCONS = (MACOS or AIX) and os.getuid() != 0 | |
| 193 | |
| 194 # --- misc | |
| 195 | |
| 196 | |
| 197 def _get_py_exe(): | |
| 198 def attempt(exe): | |
| 199 try: | |
| 200 subprocess.check_call( | |
| 201 [exe, "-V"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) | |
| 202 except Exception: | |
| 203 return None | |
| 204 else: | |
| 205 return exe | |
| 206 | |
| 207 if GITHUB_WHEELS: | |
| 208 if PYPY: | |
| 209 return which("pypy3") if PY3 else which("pypy") | |
| 210 else: | |
| 211 return which('python') | |
| 212 elif MACOS: | |
| 213 exe = \ | |
| 214 attempt(sys.executable) or \ | |
| 215 attempt(os.path.realpath(sys.executable)) or \ | |
| 216 attempt(which("python%s.%s" % sys.version_info[:2])) or \ | |
| 217 attempt(psutil.Process().exe()) | |
| 218 if not exe: | |
| 219 raise ValueError("can't find python exe real abspath") | |
| 220 return exe | |
| 221 else: | |
| 222 exe = os.path.realpath(sys.executable) | |
| 223 assert os.path.exists(exe), exe | |
| 224 return exe | |
| 225 | |
| 226 | |
| 227 PYTHON_EXE = _get_py_exe() | |
| 228 DEVNULL = open(os.devnull, 'r+') | |
| 229 atexit.register(DEVNULL.close) | |
| 230 | |
| 231 VALID_PROC_STATUSES = [getattr(psutil, x) for x in dir(psutil) | |
| 232 if x.startswith('STATUS_')] | |
| 233 AF_UNIX = getattr(socket, "AF_UNIX", object()) | |
| 234 | |
| 235 _subprocesses_started = set() | |
| 236 _pids_started = set() | |
| 237 | |
| 238 | |
| 239 # =================================================================== | |
| 240 # --- threads | |
| 241 # =================================================================== | |
| 242 | |
| 243 | |
| 244 class ThreadTask(threading.Thread): | |
| 245 """A thread task which does nothing expect staying alive.""" | |
| 246 | |
| 247 def __init__(self): | |
| 248 super().__init__() | |
| 249 self._running = False | |
| 250 self._interval = 0.001 | |
| 251 self._flag = threading.Event() | |
| 252 | |
| 253 def __repr__(self): | |
| 254 name = self.__class__.__name__ | |
| 255 return '<%s running=%s at %#x>' % (name, self._running, id(self)) | |
| 256 | |
| 257 def __enter__(self): | |
| 258 self.start() | |
| 259 return self | |
| 260 | |
| 261 def __exit__(self, *args, **kwargs): | |
| 262 self.stop() | |
| 263 | |
| 264 def start(self): | |
| 265 """Start thread and keep it running until an explicit | |
| 266 stop() request. Polls for shutdown every 'timeout' seconds. | |
| 267 """ | |
| 268 if self._running: | |
| 269 raise ValueError("already started") | |
| 270 threading.Thread.start(self) | |
| 271 self._flag.wait() | |
| 272 | |
| 273 def run(self): | |
| 274 self._running = True | |
| 275 self._flag.set() | |
| 276 while self._running: | |
| 277 time.sleep(self._interval) | |
| 278 | |
| 279 def stop(self): | |
| 280 """Stop thread execution and and waits until it is stopped.""" | |
| 281 if not self._running: | |
| 282 raise ValueError("already stopped") | |
| 283 self._running = False | |
| 284 self.join() | |
| 285 | |
| 286 | |
| 287 # =================================================================== | |
| 288 # --- subprocesses | |
| 289 # =================================================================== | |
| 290 | |
| 291 | |
| 292 def _reap_children_on_err(fun): | |
| 293 @functools.wraps(fun) | |
| 294 def wrapper(*args, **kwargs): | |
| 295 try: | |
| 296 return fun(*args, **kwargs) | |
| 297 except Exception: | |
| 298 reap_children() | |
| 299 raise | |
| 300 return wrapper | |
| 301 | |
| 302 | |
| 303 @_reap_children_on_err | |
| 304 def spawn_testproc(cmd=None, **kwds): | |
| 305 """Creates a python subprocess which does nothing for 60 secs and | |
| 306 return it as a subprocess.Popen instance. | |
| 307 If "cmd" is specified that is used instead of python. | |
| 308 By default stdin and stdout are redirected to /dev/null. | |
| 309 It also attemps to make sure the process is in a reasonably | |
| 310 initialized state. | |
| 311 The process is registered for cleanup on reap_children(). | |
| 312 """ | |
| 313 kwds.setdefault("stdin", DEVNULL) | |
| 314 kwds.setdefault("stdout", DEVNULL) | |
| 315 kwds.setdefault("cwd", os.getcwd()) | |
| 316 kwds.setdefault("env", os.environ) | |
| 317 if WINDOWS: | |
| 318 # Prevents the subprocess to open error dialogs. This will also | |
| 319 # cause stderr to be suppressed, which is suboptimal in order | |
| 320 # to debug broken tests. | |
| 321 CREATE_NO_WINDOW = 0x8000000 | |
| 322 kwds.setdefault("creationflags", CREATE_NO_WINDOW) | |
| 323 if cmd is None: | |
| 324 testfn = get_testfn() | |
| 325 try: | |
| 326 safe_rmpath(testfn) | |
| 327 pyline = "from time import sleep;" \ | |
| 328 "open(r'%s', 'w').close();" \ | |
| 329 "sleep(60);" % testfn | |
| 330 cmd = [PYTHON_EXE, "-c", pyline] | |
| 331 sproc = subprocess.Popen(cmd, **kwds) | |
| 332 _subprocesses_started.add(sproc) | |
| 333 wait_for_file(testfn, delete=True, empty=True) | |
| 334 finally: | |
| 335 safe_rmpath(testfn) | |
| 336 else: | |
| 337 sproc = subprocess.Popen(cmd, **kwds) | |
| 338 _subprocesses_started.add(sproc) | |
| 339 wait_for_pid(sproc.pid) | |
| 340 return sproc | |
| 341 | |
| 342 | |
| 343 @_reap_children_on_err | |
| 344 def spawn_children_pair(): | |
| 345 """Create a subprocess which creates another one as in: | |
| 346 A (us) -> B (child) -> C (grandchild). | |
| 347 Return a (child, grandchild) tuple. | |
| 348 The 2 processes are fully initialized and will live for 60 secs | |
| 349 and are registered for cleanup on reap_children(). | |
| 350 """ | |
| 351 tfile = None | |
| 352 testfn = get_testfn(dir=os.getcwd()) | |
| 353 try: | |
| 354 s = textwrap.dedent("""\ | |
| 355 import subprocess, os, sys, time | |
| 356 s = "import os, time;" | |
| 357 s += "f = open('%s', 'w');" | |
| 358 s += "f.write(str(os.getpid()));" | |
| 359 s += "f.close();" | |
| 360 s += "time.sleep(60);" | |
| 361 p = subprocess.Popen([r'%s', '-c', s]) | |
| 362 p.wait() | |
| 363 """ % (os.path.basename(testfn), PYTHON_EXE)) | |
| 364 # On Windows if we create a subprocess with CREATE_NO_WINDOW flag | |
| 365 # set (which is the default) a "conhost.exe" extra process will be | |
| 366 # spawned as a child. We don't want that. | |
| 367 if WINDOWS: | |
| 368 subp, tfile = pyrun(s, creationflags=0) | |
| 369 else: | |
| 370 subp, tfile = pyrun(s) | |
| 371 child = psutil.Process(subp.pid) | |
| 372 grandchild_pid = int(wait_for_file(testfn, delete=True, empty=False)) | |
| 373 _pids_started.add(grandchild_pid) | |
| 374 grandchild = psutil.Process(grandchild_pid) | |
| 375 return (child, grandchild) | |
| 376 finally: | |
| 377 safe_rmpath(testfn) | |
| 378 if tfile is not None: | |
| 379 safe_rmpath(tfile) | |
| 380 | |
| 381 | |
| 382 def spawn_zombie(): | |
| 383 """Create a zombie process and return a (parent, zombie) process tuple. | |
| 384 In order to kill the zombie parent must be terminate()d first, then | |
| 385 zombie must be wait()ed on. | |
| 386 """ | |
| 387 assert psutil.POSIX | |
| 388 unix_file = get_testfn() | |
| 389 src = textwrap.dedent("""\ | |
| 390 import os, sys, time, socket, contextlib | |
| 391 child_pid = os.fork() | |
| 392 if child_pid > 0: | |
| 393 time.sleep(3000) | |
| 394 else: | |
| 395 # this is the zombie process | |
| 396 s = socket.socket(socket.AF_UNIX) | |
| 397 with contextlib.closing(s): | |
| 398 s.connect('%s') | |
| 399 if sys.version_info < (3, ): | |
| 400 pid = str(os.getpid()) | |
| 401 else: | |
| 402 pid = bytes(str(os.getpid()), 'ascii') | |
| 403 s.sendall(pid) | |
| 404 """ % unix_file) | |
| 405 tfile = None | |
| 406 sock = bind_unix_socket(unix_file) | |
| 407 try: | |
| 408 sock.settimeout(GLOBAL_TIMEOUT) | |
| 409 parent, tfile = pyrun(src) | |
| 410 conn, _ = sock.accept() | |
| 411 try: | |
| 412 select.select([conn.fileno()], [], [], GLOBAL_TIMEOUT) | |
| 413 zpid = int(conn.recv(1024)) | |
| 414 _pids_started.add(zpid) | |
| 415 zombie = psutil.Process(zpid) | |
| 416 call_until(lambda: zombie.status(), "ret == psutil.STATUS_ZOMBIE") | |
| 417 return (parent, zombie) | |
| 418 finally: | |
| 419 conn.close() | |
| 420 finally: | |
| 421 sock.close() | |
| 422 safe_rmpath(unix_file) | |
| 423 if tfile is not None: | |
| 424 safe_rmpath(tfile) | |
| 425 | |
| 426 | |
| 427 @_reap_children_on_err | |
| 428 def pyrun(src, **kwds): | |
| 429 """Run python 'src' code string in a separate interpreter. | |
| 430 Returns a subprocess.Popen instance and the test file where the source | |
| 431 code was written. | |
| 432 """ | |
| 433 kwds.setdefault("stdout", None) | |
| 434 kwds.setdefault("stderr", None) | |
| 435 srcfile = get_testfn() | |
| 436 try: | |
| 437 with open(srcfile, 'wt') as f: | |
| 438 f.write(src) | |
| 439 subp = spawn_testproc([PYTHON_EXE, f.name], **kwds) | |
| 440 wait_for_pid(subp.pid) | |
| 441 return (subp, srcfile) | |
| 442 except Exception: | |
| 443 safe_rmpath(srcfile) | |
| 444 raise | |
| 445 | |
| 446 | |
| 447 @_reap_children_on_err | |
| 448 def sh(cmd, **kwds): | |
| 449 """run cmd in a subprocess and return its output. | |
| 450 raises RuntimeError on error. | |
| 451 """ | |
| 452 shell = True if isinstance(cmd, (str, unicode)) else False | |
| 453 # Prevents subprocess to open error dialogs in case of error. | |
| 454 flags = 0x8000000 if WINDOWS and shell else 0 | |
| 455 kwds.setdefault("shell", shell) | |
| 456 kwds.setdefault("stdout", subprocess.PIPE) | |
| 457 kwds.setdefault("stderr", subprocess.PIPE) | |
| 458 kwds.setdefault("universal_newlines", True) | |
| 459 kwds.setdefault("creationflags", flags) | |
| 460 p = subprocess.Popen(cmd, **kwds) | |
| 461 _subprocesses_started.add(p) | |
| 462 if PY3: | |
| 463 stdout, stderr = p.communicate(timeout=GLOBAL_TIMEOUT) | |
| 464 else: | |
| 465 stdout, stderr = p.communicate() | |
| 466 if p.returncode != 0: | |
| 467 raise RuntimeError(stderr) | |
| 468 if stderr: | |
| 469 warn(stderr) | |
| 470 if stdout.endswith('\n'): | |
| 471 stdout = stdout[:-1] | |
| 472 return stdout | |
| 473 | |
| 474 | |
| 475 def terminate(proc_or_pid, sig=signal.SIGTERM, wait_timeout=GLOBAL_TIMEOUT): | |
| 476 """Terminate a process and wait() for it. | |
| 477 Process can be a PID or an instance of psutil.Process(), | |
| 478 subprocess.Popen() or psutil.Popen(). | |
| 479 If it's a subprocess.Popen() or psutil.Popen() instance also closes | |
| 480 its stdin / stdout / stderr fds. | |
| 481 PID is wait()ed even if the process is already gone (kills zombies). | |
| 482 Does nothing if the process does not exist. | |
| 483 Return process exit status. | |
| 484 """ | |
| 485 if POSIX: | |
| 486 from psutil._psposix import wait_pid | |
| 487 | |
| 488 def wait(proc, timeout): | |
| 489 if isinstance(proc, subprocess.Popen) and not PY3: | |
| 490 proc.wait() | |
| 491 else: | |
| 492 proc.wait(timeout) | |
| 493 if WINDOWS and isinstance(proc, subprocess.Popen): | |
| 494 # Otherwise PID may still hang around. | |
| 495 try: | |
| 496 return psutil.Process(proc.pid).wait(timeout) | |
| 497 except psutil.NoSuchProcess: | |
| 498 pass | |
| 499 | |
| 500 def sendsig(proc, sig): | |
| 501 # XXX: otherwise the build hangs for some reason. | |
| 502 if MACOS and GITHUB_WHEELS: | |
| 503 sig = signal.SIGKILL | |
| 504 # If the process received SIGSTOP, SIGCONT is necessary first, | |
| 505 # otherwise SIGTERM won't work. | |
| 506 if POSIX and sig != signal.SIGKILL: | |
| 507 proc.send_signal(signal.SIGCONT) | |
| 508 proc.send_signal(sig) | |
| 509 | |
| 510 def term_subproc(proc, timeout): | |
| 511 try: | |
| 512 sendsig(proc, sig) | |
| 513 except OSError as err: | |
| 514 if WINDOWS and err.winerror == 6: # "invalid handle" | |
| 515 pass | |
| 516 elif err.errno != errno.ESRCH: | |
| 517 raise | |
| 518 return wait(proc, timeout) | |
| 519 | |
| 520 def term_psproc(proc, timeout): | |
| 521 try: | |
| 522 sendsig(proc, sig) | |
| 523 except psutil.NoSuchProcess: | |
| 524 pass | |
| 525 return wait(proc, timeout) | |
| 526 | |
| 527 def term_pid(pid, timeout): | |
| 528 try: | |
| 529 proc = psutil.Process(pid) | |
| 530 except psutil.NoSuchProcess: | |
| 531 # Needed to kill zombies. | |
| 532 if POSIX: | |
| 533 return wait_pid(pid, timeout) | |
| 534 else: | |
| 535 return term_psproc(proc, timeout) | |
| 536 | |
| 537 def flush_popen(proc): | |
| 538 if proc.stdout: | |
| 539 proc.stdout.close() | |
| 540 if proc.stderr: | |
| 541 proc.stderr.close() | |
| 542 # Flushing a BufferedWriter may raise an error. | |
| 543 if proc.stdin: | |
| 544 proc.stdin.close() | |
| 545 | |
| 546 p = proc_or_pid | |
| 547 try: | |
| 548 if isinstance(p, int): | |
| 549 return term_pid(p, wait_timeout) | |
| 550 elif isinstance(p, (psutil.Process, psutil.Popen)): | |
| 551 return term_psproc(p, wait_timeout) | |
| 552 elif isinstance(p, subprocess.Popen): | |
| 553 return term_subproc(p, wait_timeout) | |
| 554 else: | |
| 555 raise TypeError("wrong type %r" % p) | |
| 556 finally: | |
| 557 if isinstance(p, (subprocess.Popen, psutil.Popen)): | |
| 558 flush_popen(p) | |
| 559 pid = p if isinstance(p, int) else p.pid | |
| 560 assert not psutil.pid_exists(pid), pid | |
| 561 | |
| 562 | |
| 563 def reap_children(recursive=False): | |
| 564 """Terminate and wait() any subprocess started by this test suite | |
| 565 and any children currently running, ensuring that no processes stick | |
| 566 around to hog resources. | |
| 567 If resursive is True it also tries to terminate and wait() | |
| 568 all grandchildren started by this process. | |
| 569 """ | |
| 570 # Get the children here before terminating them, as in case of | |
| 571 # recursive=True we don't want to lose the intermediate reference | |
| 572 # pointing to the grandchildren. | |
| 573 children = psutil.Process().children(recursive=recursive) | |
| 574 | |
| 575 # Terminate subprocess.Popen. | |
| 576 while _subprocesses_started: | |
| 577 subp = _subprocesses_started.pop() | |
| 578 terminate(subp) | |
| 579 | |
| 580 # Collect started pids. | |
| 581 while _pids_started: | |
| 582 pid = _pids_started.pop() | |
| 583 terminate(pid) | |
| 584 | |
| 585 # Terminate children. | |
| 586 if children: | |
| 587 for p in children: | |
| 588 terminate(p, wait_timeout=None) | |
| 589 gone, alive = psutil.wait_procs(children, timeout=GLOBAL_TIMEOUT) | |
| 590 for p in alive: | |
| 591 warn("couldn't terminate process %r; attempting kill()" % p) | |
| 592 terminate(p, sig=signal.SIGKILL) | |
| 593 | |
| 594 | |
| 595 # =================================================================== | |
| 596 # --- OS | |
| 597 # =================================================================== | |
| 598 | |
| 599 | |
| 600 def get_kernel_version(): | |
| 601 """Return a tuple such as (2, 6, 36).""" | |
| 602 if not POSIX: | |
| 603 raise NotImplementedError("not POSIX") | |
| 604 s = "" | |
| 605 uname = os.uname()[2] | |
| 606 for c in uname: | |
| 607 if c.isdigit() or c == '.': | |
| 608 s += c | |
| 609 else: | |
| 610 break | |
| 611 if not s: | |
| 612 raise ValueError("can't parse %r" % uname) | |
| 613 minor = 0 | |
| 614 micro = 0 | |
| 615 nums = s.split('.') | |
| 616 major = int(nums[0]) | |
| 617 if len(nums) >= 2: | |
| 618 minor = int(nums[1]) | |
| 619 if len(nums) >= 3: | |
| 620 micro = int(nums[2]) | |
| 621 return (major, minor, micro) | |
| 622 | |
| 623 | |
| 624 def get_winver(): | |
| 625 if not WINDOWS: | |
| 626 raise NotImplementedError("not WINDOWS") | |
| 627 wv = sys.getwindowsversion() | |
| 628 if hasattr(wv, 'service_pack_major'): # python >= 2.7 | |
| 629 sp = wv.service_pack_major or 0 | |
| 630 else: | |
| 631 r = re.search(r"\s\d$", wv[4]) | |
| 632 if r: | |
| 633 sp = int(r.group(0)) | |
| 634 else: | |
| 635 sp = 0 | |
| 636 return (wv[0], wv[1], sp) | |
| 637 | |
| 638 | |
| 639 # =================================================================== | |
| 640 # --- sync primitives | |
| 641 # =================================================================== | |
| 642 | |
| 643 | |
| 644 class retry(object): | |
| 645 """A retry decorator.""" | |
| 646 | |
| 647 def __init__(self, | |
| 648 exception=Exception, | |
| 649 timeout=None, | |
| 650 retries=None, | |
| 651 interval=0.001, | |
| 652 logfun=None, | |
| 653 ): | |
| 654 if timeout and retries: | |
| 655 raise ValueError("timeout and retries args are mutually exclusive") | |
| 656 self.exception = exception | |
| 657 self.timeout = timeout | |
| 658 self.retries = retries | |
| 659 self.interval = interval | |
| 660 self.logfun = logfun | |
| 661 | |
| 662 def __iter__(self): | |
| 663 if self.timeout: | |
| 664 stop_at = time.time() + self.timeout | |
| 665 while time.time() < stop_at: | |
| 666 yield | |
| 667 elif self.retries: | |
| 668 for _ in range(self.retries): | |
| 669 yield | |
| 670 else: | |
| 671 while True: | |
| 672 yield | |
| 673 | |
| 674 def sleep(self): | |
| 675 if self.interval is not None: | |
| 676 time.sleep(self.interval) | |
| 677 | |
| 678 def __call__(self, fun): | |
| 679 @functools.wraps(fun) | |
| 680 def wrapper(*args, **kwargs): | |
| 681 exc = None | |
| 682 for _ in self: | |
| 683 try: | |
| 684 return fun(*args, **kwargs) | |
| 685 except self.exception as _: # NOQA | |
| 686 exc = _ | |
| 687 if self.logfun is not None: | |
| 688 self.logfun(exc) | |
| 689 self.sleep() | |
| 690 continue | |
| 691 if PY3: | |
| 692 raise exc | |
| 693 else: | |
| 694 raise | |
| 695 | |
| 696 # This way the user of the decorated function can change config | |
| 697 # parameters. | |
| 698 wrapper.decorator = self | |
| 699 return wrapper | |
| 700 | |
| 701 | |
| 702 @retry(exception=psutil.NoSuchProcess, logfun=None, timeout=GLOBAL_TIMEOUT, | |
| 703 interval=0.001) | |
| 704 def wait_for_pid(pid): | |
| 705 """Wait for pid to show up in the process list then return. | |
| 706 Used in the test suite to give time the sub process to initialize. | |
| 707 """ | |
| 708 psutil.Process(pid) | |
| 709 if WINDOWS: | |
| 710 # give it some more time to allow better initialization | |
| 711 time.sleep(0.01) | |
| 712 | |
| 713 | |
| 714 @retry(exception=(FileNotFoundError, AssertionError), logfun=None, | |
| 715 timeout=GLOBAL_TIMEOUT, interval=0.001) | |
| 716 def wait_for_file(fname, delete=True, empty=False): | |
| 717 """Wait for a file to be written on disk with some content.""" | |
| 718 with open(fname, "rb") as f: | |
| 719 data = f.read() | |
| 720 if not empty: | |
| 721 assert data | |
| 722 if delete: | |
| 723 safe_rmpath(fname) | |
| 724 return data | |
| 725 | |
| 726 | |
| 727 @retry(exception=AssertionError, logfun=None, timeout=GLOBAL_TIMEOUT, | |
| 728 interval=0.001) | |
| 729 def call_until(fun, expr): | |
| 730 """Keep calling function for timeout secs and exit if eval() | |
| 731 expression is True. | |
| 732 """ | |
| 733 ret = fun() | |
| 734 assert eval(expr) | |
| 735 return ret | |
| 736 | |
| 737 | |
| 738 # =================================================================== | |
| 739 # --- fs | |
| 740 # =================================================================== | |
| 741 | |
| 742 | |
| 743 def safe_rmpath(path): | |
| 744 "Convenience function for removing temporary test files or dirs" | |
| 745 def retry_fun(fun): | |
| 746 # On Windows it could happen that the file or directory has | |
| 747 # open handles or references preventing the delete operation | |
| 748 # to succeed immediately, so we retry for a while. See: | |
| 749 # https://bugs.python.org/issue33240 | |
| 750 stop_at = time.time() + GLOBAL_TIMEOUT | |
| 751 while time.time() < stop_at: | |
| 752 try: | |
| 753 return fun() | |
| 754 except FileNotFoundError: | |
| 755 pass | |
| 756 except WindowsError as _: | |
| 757 err = _ | |
| 758 warn("ignoring %s" % (str(err))) | |
| 759 time.sleep(0.01) | |
| 760 raise err | |
| 761 | |
| 762 try: | |
| 763 st = os.stat(path) | |
| 764 if stat.S_ISDIR(st.st_mode): | |
| 765 fun = functools.partial(shutil.rmtree, path) | |
| 766 else: | |
| 767 fun = functools.partial(os.remove, path) | |
| 768 if POSIX: | |
| 769 fun() | |
| 770 else: | |
| 771 retry_fun(fun) | |
| 772 except FileNotFoundError: | |
| 773 pass | |
| 774 | |
| 775 | |
| 776 def safe_mkdir(dir): | |
| 777 "Convenience function for creating a directory" | |
| 778 try: | |
| 779 os.mkdir(dir) | |
| 780 except FileExistsError: | |
| 781 pass | |
| 782 | |
| 783 | |
| 784 @contextlib.contextmanager | |
| 785 def chdir(dirname): | |
| 786 "Context manager which temporarily changes the current directory." | |
| 787 curdir = os.getcwd() | |
| 788 try: | |
| 789 os.chdir(dirname) | |
| 790 yield | |
| 791 finally: | |
| 792 os.chdir(curdir) | |
| 793 | |
| 794 | |
| 795 def create_exe(outpath, c_code=None): | |
| 796 """Creates an executable file in the given location.""" | |
| 797 assert not os.path.exists(outpath), outpath | |
| 798 if c_code: | |
| 799 if not which("gcc"): | |
| 800 raise ValueError("gcc is not installed") | |
| 801 if isinstance(c_code, bool): # c_code is True | |
| 802 c_code = textwrap.dedent( | |
| 803 """ | |
| 804 #include <unistd.h> | |
| 805 int main() { | |
| 806 pause(); | |
| 807 return 1; | |
| 808 } | |
| 809 """) | |
| 810 assert isinstance(c_code, str), c_code | |
| 811 with open(get_testfn(suffix='.c'), 'wt') as f: | |
| 812 f.write(c_code) | |
| 813 try: | |
| 814 subprocess.check_call(["gcc", f.name, "-o", outpath]) | |
| 815 finally: | |
| 816 safe_rmpath(f.name) | |
| 817 else: | |
| 818 # copy python executable | |
| 819 shutil.copyfile(PYTHON_EXE, outpath) | |
| 820 if POSIX: | |
| 821 st = os.stat(outpath) | |
| 822 os.chmod(outpath, st.st_mode | stat.S_IEXEC) | |
| 823 | |
| 824 | |
| 825 def get_testfn(suffix="", dir=None): | |
| 826 """Return an absolute pathname of a file or dir that did not | |
| 827 exist at the time this call is made. Also schedule it for safe | |
| 828 deletion at interpreter exit. It's technically racy but probably | |
| 829 not really due to the time variant. | |
| 830 """ | |
| 831 while True: | |
| 832 name = tempfile.mktemp(prefix=TESTFN_PREFIX, suffix=suffix, dir=dir) | |
| 833 if not os.path.exists(name): # also include dirs | |
| 834 return os.path.realpath(name) # needed for OSX | |
| 835 | |
| 836 | |
| 837 # =================================================================== | |
| 838 # --- testing | |
| 839 # =================================================================== | |
| 840 | |
| 841 | |
| 842 class TestCase(unittest.TestCase): | |
| 843 | |
| 844 # Print a full path representation of the single unit tests | |
| 845 # being run. | |
| 846 def __str__(self): | |
| 847 fqmod = self.__class__.__module__ | |
| 848 if not fqmod.startswith('psutil.'): | |
| 849 fqmod = 'psutil.tests.' + fqmod | |
| 850 return "%s.%s.%s" % ( | |
| 851 fqmod, self.__class__.__name__, self._testMethodName) | |
| 852 | |
| 853 # assertRaisesRegexp renamed to assertRaisesRegex in 3.3; | |
| 854 # add support for the new name. | |
| 855 if not hasattr(unittest.TestCase, 'assertRaisesRegex'): | |
| 856 assertRaisesRegex = unittest.TestCase.assertRaisesRegexp | |
| 857 | |
| 858 # ...otherwise multiprocessing.Pool complains | |
| 859 if not PY3: | |
| 860 def runTest(self): | |
| 861 pass | |
| 862 | |
| 863 | |
| 864 # monkey patch default unittest.TestCase | |
| 865 unittest.TestCase = TestCase | |
| 866 | |
| 867 | |
| 868 class PsutilTestCase(TestCase): | |
| 869 """Test class providing auto-cleanup wrappers on top of process | |
| 870 test utilities. | |
| 871 """ | |
| 872 | |
| 873 def get_testfn(self, suffix="", dir=None): | |
| 874 fname = get_testfn(suffix=suffix, dir=dir) | |
| 875 self.addCleanup(safe_rmpath, fname) | |
| 876 return fname | |
| 877 | |
| 878 def spawn_testproc(self, *args, **kwds): | |
| 879 sproc = spawn_testproc(*args, **kwds) | |
| 880 self.addCleanup(terminate, sproc) | |
| 881 return sproc | |
| 882 | |
| 883 def spawn_children_pair(self): | |
| 884 child1, child2 = spawn_children_pair() | |
| 885 self.addCleanup(terminate, child2) | |
| 886 self.addCleanup(terminate, child1) # executed first | |
| 887 return (child1, child2) | |
| 888 | |
| 889 def spawn_zombie(self): | |
| 890 parent, zombie = spawn_zombie() | |
| 891 self.addCleanup(terminate, zombie) | |
| 892 self.addCleanup(terminate, parent) # executed first | |
| 893 return (parent, zombie) | |
| 894 | |
| 895 def pyrun(self, *args, **kwds): | |
| 896 sproc, srcfile = pyrun(*args, **kwds) | |
| 897 self.addCleanup(safe_rmpath, srcfile) | |
| 898 self.addCleanup(terminate, sproc) # executed first | |
| 899 return sproc | |
| 900 | |
| 901 def assertProcessGone(self, proc): | |
| 902 self.assertRaises(psutil.NoSuchProcess, psutil.Process, proc.pid) | |
| 903 if isinstance(proc, (psutil.Process, psutil.Popen)): | |
| 904 assert not proc.is_running() | |
| 905 self.assertRaises(psutil.NoSuchProcess, proc.status) | |
| 906 proc.wait(timeout=0) # assert not raise TimeoutExpired | |
| 907 assert not psutil.pid_exists(proc.pid), proc.pid | |
| 908 self.assertNotIn(proc.pid, psutil.pids()) | |
| 909 | |
| 910 | |
| 911 @unittest.skipIf(PYPY, "unreliable on PYPY") | |
| 912 class TestMemoryLeak(PsutilTestCase): | |
| 913 """Test framework class for detecting function memory leaks, | |
| 914 typically functions implemented in C which forgot to free() memory | |
| 915 from the heap. It does so by checking whether the process memory | |
| 916 usage increased before and after calling the function many times. | |
| 917 | |
| 918 Note that this is hard (probably impossible) to do reliably, due | |
| 919 to how the OS handles memory, the GC and so on (memory can even | |
| 920 decrease!). In order to avoid false positives, in case of failure | |
| 921 (mem > 0) we retry the test for up to 5 times, increasing call | |
| 922 repetitions each time. If the memory keeps increasing then it's a | |
| 923 failure. | |
| 924 | |
| 925 If available (Linux, OSX, Windows), USS memory is used for comparison, | |
| 926 since it's supposed to be more precise, see: | |
| 927 https://gmpy.dev/blog/2016/real-process-memory-and-environ-in-python | |
| 928 If not, RSS memory is used. mallinfo() on Linux and _heapwalk() on | |
| 929 Windows may give even more precision, but at the moment are not | |
| 930 implemented. | |
| 931 | |
| 932 PyPy appears to be completely unstable for this framework, probably | |
| 933 because of its JIT, so tests on PYPY are skipped. | |
| 934 | |
| 935 Usage: | |
| 936 | |
| 937 class TestLeaks(psutil.tests.TestMemoryLeak): | |
| 938 | |
| 939 def test_fun(self): | |
| 940 self.execute(some_function) | |
| 941 """ | |
| 942 # Configurable class attrs. | |
| 943 times = 200 | |
| 944 warmup_times = 10 | |
| 945 tolerance = 0 # memory | |
| 946 retries = 10 if CI_TESTING else 5 | |
| 947 verbose = True | |
| 948 _thisproc = psutil.Process() | |
| 949 | |
| 950 def _get_mem(self): | |
| 951 # USS is the closest thing we have to "real" memory usage and it | |
| 952 # should be less likely to produce false positives. | |
| 953 mem = self._thisproc.memory_full_info() | |
| 954 return getattr(mem, "uss", mem.rss) | |
| 955 | |
| 956 def _get_num_fds(self): | |
| 957 if POSIX: | |
| 958 return self._thisproc.num_fds() | |
| 959 else: | |
| 960 return self._thisproc.num_handles() | |
| 961 | |
| 962 def _log(self, msg): | |
| 963 if self.verbose: | |
| 964 print_color(msg, color="yellow", file=sys.stderr) | |
| 965 | |
| 966 def _check_fds(self, fun): | |
| 967 """Makes sure num_fds() (POSIX) or num_handles() (Windows) does | |
| 968 not increase after calling a function. Used to discover forgotten | |
| 969 close(2) and CloseHandle syscalls. | |
| 970 """ | |
| 971 before = self._get_num_fds() | |
| 972 self.call(fun) | |
| 973 after = self._get_num_fds() | |
| 974 diff = after - before | |
| 975 if diff < 0: | |
| 976 raise self.fail("negative diff %r (gc probably collected a " | |
| 977 "resource from a previous test)" % diff) | |
| 978 if diff > 0: | |
| 979 type_ = "fd" if POSIX else "handle" | |
| 980 if diff > 1: | |
| 981 type_ += "s" | |
| 982 msg = "%s unclosed %s after calling %r" % (diff, type_, fun) | |
| 983 raise self.fail(msg) | |
| 984 | |
| 985 def _call_ntimes(self, fun, times): | |
| 986 """Get 2 distinct memory samples, before and after having | |
| 987 called fun repeadetly, and return the memory difference. | |
| 988 """ | |
| 989 gc.collect(generation=1) | |
| 990 mem1 = self._get_mem() | |
| 991 for x in range(times): | |
| 992 ret = self.call(fun) | |
| 993 del x, ret | |
| 994 gc.collect(generation=1) | |
| 995 mem2 = self._get_mem() | |
| 996 self.assertEqual(gc.garbage, []) | |
| 997 diff = mem2 - mem1 # can also be negative | |
| 998 return diff | |
| 999 | |
| 1000 def _check_mem(self, fun, times, warmup_times, retries, tolerance): | |
| 1001 messages = [] | |
| 1002 prev_mem = 0 | |
| 1003 increase = times | |
| 1004 for idx in range(1, retries + 1): | |
| 1005 mem = self._call_ntimes(fun, times) | |
| 1006 msg = "Run #%s: extra-mem=%s, per-call=%s, calls=%s" % ( | |
| 1007 idx, bytes2human(mem), bytes2human(mem / times), times) | |
| 1008 messages.append(msg) | |
| 1009 success = mem <= tolerance or mem <= prev_mem | |
| 1010 if success: | |
| 1011 if idx > 1: | |
| 1012 self._log(msg) | |
| 1013 return | |
| 1014 else: | |
| 1015 if idx == 1: | |
| 1016 print() # NOQA | |
| 1017 self._log(msg) | |
| 1018 times += increase | |
| 1019 prev_mem = mem | |
| 1020 raise self.fail(". ".join(messages)) | |
| 1021 | |
| 1022 # --- | |
| 1023 | |
| 1024 def call(self, fun): | |
| 1025 return fun() | |
| 1026 | |
| 1027 def execute(self, fun, times=None, warmup_times=None, retries=None, | |
| 1028 tolerance=None): | |
| 1029 """Test a callable.""" | |
| 1030 times = times if times is not None else self.times | |
| 1031 warmup_times = warmup_times if warmup_times is not None \ | |
| 1032 else self.warmup_times | |
| 1033 retries = retries if retries is not None else self.retries | |
| 1034 tolerance = tolerance if tolerance is not None else self.tolerance | |
| 1035 try: | |
| 1036 assert times >= 1, "times must be >= 1" | |
| 1037 assert warmup_times >= 0, "warmup_times must be >= 0" | |
| 1038 assert retries >= 0, "retries must be >= 0" | |
| 1039 assert tolerance >= 0, "tolerance must be >= 0" | |
| 1040 except AssertionError as err: | |
| 1041 raise ValueError(str(err)) | |
| 1042 | |
| 1043 self._call_ntimes(fun, warmup_times) # warm up | |
| 1044 self._check_fds(fun) | |
| 1045 self._check_mem(fun, times=times, warmup_times=warmup_times, | |
| 1046 retries=retries, tolerance=tolerance) | |
| 1047 | |
| 1048 def execute_w_exc(self, exc, fun, **kwargs): | |
| 1049 """Convenience method to test a callable while making sure it | |
| 1050 raises an exception on every call. | |
| 1051 """ | |
| 1052 def call(): | |
| 1053 self.assertRaises(exc, fun) | |
| 1054 | |
| 1055 self.execute(call, **kwargs) | |
| 1056 | |
| 1057 | |
| 1058 def print_sysinfo(): | |
| 1059 import collections | |
| 1060 import datetime | |
| 1061 import getpass | |
| 1062 import platform | |
| 1063 | |
| 1064 info = collections.OrderedDict() | |
| 1065 info['OS'] = platform.system() | |
| 1066 if psutil.OSX: | |
| 1067 info['version'] = str(platform.mac_ver()) | |
| 1068 elif psutil.WINDOWS: | |
| 1069 info['version'] = ' '.join(map(str, platform.win32_ver())) | |
| 1070 if hasattr(platform, 'win32_edition'): | |
| 1071 info['edition'] = platform.win32_edition() | |
| 1072 else: | |
| 1073 info['version'] = platform.version() | |
| 1074 if psutil.POSIX: | |
| 1075 info['kernel'] = '.'.join(map(str, get_kernel_version())) | |
| 1076 info['arch'] = ', '.join( | |
| 1077 list(platform.architecture()) + [platform.machine()]) | |
| 1078 info['hostname'] = platform.node() | |
| 1079 info['python'] = ', '.join([ | |
| 1080 platform.python_implementation(), | |
| 1081 platform.python_version(), | |
| 1082 platform.python_compiler()]) | |
| 1083 if psutil.POSIX: | |
| 1084 s = platform.libc_ver()[1] | |
| 1085 if s: | |
| 1086 info['glibc'] = s | |
| 1087 info['fs-encoding'] = sys.getfilesystemencoding() | |
| 1088 info['time'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") | |
| 1089 info['user'] = getpass.getuser() | |
| 1090 info['pid'] = os.getpid() | |
| 1091 print("=" * 70) # NOQA | |
| 1092 for k, v in info.items(): | |
| 1093 print("%-14s %s" % (k + ':', v)) # NOQA | |
| 1094 print("=" * 70) # NOQA | |
| 1095 | |
| 1096 | |
| 1097 def _get_eligible_cpu(): | |
| 1098 p = psutil.Process() | |
| 1099 if hasattr(p, "cpu_num"): | |
| 1100 return p.cpu_num() | |
| 1101 elif hasattr(p, "cpu_affinity"): | |
| 1102 return p.cpu_affinity()[0] | |
| 1103 return 0 | |
| 1104 | |
| 1105 | |
| 1106 class process_namespace: | |
| 1107 """A container that lists all Process class method names + some | |
| 1108 reasonable parameters to be called with. Utility methods (parent(), | |
| 1109 children(), ...) are excluded. | |
| 1110 | |
| 1111 >>> ns = process_namespace(psutil.Process()) | |
| 1112 >>> for fun, name in ns.iter(ns.getters): | |
| 1113 ... fun() | |
| 1114 """ | |
| 1115 utils = [ | |
| 1116 ('cpu_percent', (), {}), | |
| 1117 ('memory_percent', (), {}), | |
| 1118 ] | |
| 1119 | |
| 1120 ignored = [ | |
| 1121 ('as_dict', (), {}), | |
| 1122 ('children', (), {'recursive': True}), | |
| 1123 ('is_running', (), {}), | |
| 1124 ('memory_info_ex', (), {}), | |
| 1125 ('oneshot', (), {}), | |
| 1126 ('parent', (), {}), | |
| 1127 ('parents', (), {}), | |
| 1128 ('pid', (), {}), | |
| 1129 ('wait', (0, ), {}), | |
| 1130 ] | |
| 1131 | |
| 1132 getters = [ | |
| 1133 ('cmdline', (), {}), | |
| 1134 ('connections', (), {'kind': 'all'}), | |
| 1135 ('cpu_times', (), {}), | |
| 1136 ('create_time', (), {}), | |
| 1137 ('cwd', (), {}), | |
| 1138 ('exe', (), {}), | |
| 1139 ('memory_full_info', (), {}), | |
| 1140 ('memory_info', (), {}), | |
| 1141 ('name', (), {}), | |
| 1142 ('nice', (), {}), | |
| 1143 ('num_ctx_switches', (), {}), | |
| 1144 ('num_threads', (), {}), | |
| 1145 ('open_files', (), {}), | |
| 1146 ('ppid', (), {}), | |
| 1147 ('status', (), {}), | |
| 1148 ('threads', (), {}), | |
| 1149 ('username', (), {}), | |
| 1150 ] | |
| 1151 if POSIX: | |
| 1152 getters += [('uids', (), {})] | |
| 1153 getters += [('gids', (), {})] | |
| 1154 getters += [('terminal', (), {})] | |
| 1155 getters += [('num_fds', (), {})] | |
| 1156 if HAS_PROC_IO_COUNTERS: | |
| 1157 getters += [('io_counters', (), {})] | |
| 1158 if HAS_IONICE: | |
| 1159 getters += [('ionice', (), {})] | |
| 1160 if HAS_RLIMIT: | |
| 1161 getters += [('rlimit', (psutil.RLIMIT_NOFILE, ), {})] | |
| 1162 if HAS_CPU_AFFINITY: | |
| 1163 getters += [('cpu_affinity', (), {})] | |
| 1164 if HAS_PROC_CPU_NUM: | |
| 1165 getters += [('cpu_num', (), {})] | |
| 1166 if HAS_ENVIRON: | |
| 1167 getters += [('environ', (), {})] | |
| 1168 if WINDOWS: | |
| 1169 getters += [('num_handles', (), {})] | |
| 1170 if HAS_MEMORY_MAPS: | |
| 1171 getters += [('memory_maps', (), {'grouped': False})] | |
| 1172 | |
| 1173 setters = [] | |
| 1174 if POSIX: | |
| 1175 setters += [('nice', (0, ), {})] | |
| 1176 else: | |
| 1177 setters += [('nice', (psutil.NORMAL_PRIORITY_CLASS, ), {})] | |
| 1178 if HAS_RLIMIT: | |
| 1179 setters += [('rlimit', (psutil.RLIMIT_NOFILE, (1024, 4096)), {})] | |
| 1180 if HAS_IONICE: | |
| 1181 if LINUX: | |
| 1182 setters += [('ionice', (psutil.IOPRIO_CLASS_NONE, 0), {})] | |
| 1183 else: | |
| 1184 setters += [('ionice', (psutil.IOPRIO_NORMAL, ), {})] | |
| 1185 if HAS_CPU_AFFINITY: | |
| 1186 setters += [('cpu_affinity', ([_get_eligible_cpu()], ), {})] | |
| 1187 | |
| 1188 killers = [ | |
| 1189 ('send_signal', (signal.SIGTERM, ), {}), | |
| 1190 ('suspend', (), {}), | |
| 1191 ('resume', (), {}), | |
| 1192 ('terminate', (), {}), | |
| 1193 ('kill', (), {}), | |
| 1194 ] | |
| 1195 if WINDOWS: | |
| 1196 killers += [('send_signal', (signal.CTRL_C_EVENT, ), {})] | |
| 1197 killers += [('send_signal', (signal.CTRL_BREAK_EVENT, ), {})] | |
| 1198 | |
| 1199 all = utils + getters + setters + killers | |
| 1200 | |
| 1201 def __init__(self, proc): | |
| 1202 self._proc = proc | |
| 1203 | |
| 1204 def iter(self, ls, clear_cache=True): | |
| 1205 """Given a list of tuples yields a set of (fun, fun_name) tuples | |
| 1206 in random order. | |
| 1207 """ | |
| 1208 ls = list(ls) | |
| 1209 random.shuffle(ls) | |
| 1210 for fun_name, args, kwds in ls: | |
| 1211 if clear_cache: | |
| 1212 self.clear_cache() | |
| 1213 fun = getattr(self._proc, fun_name) | |
| 1214 fun = functools.partial(fun, *args, **kwds) | |
| 1215 yield (fun, fun_name) | |
| 1216 | |
| 1217 def clear_cache(self): | |
| 1218 """Clear the cache of a Process instance.""" | |
| 1219 self._proc._init(self._proc.pid, _ignore_nsp=True) | |
| 1220 | |
| 1221 @classmethod | |
| 1222 def test_class_coverage(cls, test_class, ls): | |
| 1223 """Given a TestCase instance and a list of tuples checks that | |
| 1224 the class defines the required test method names. | |
| 1225 """ | |
| 1226 for fun_name, _, _ in ls: | |
| 1227 meth_name = 'test_' + fun_name | |
| 1228 if not hasattr(test_class, meth_name): | |
| 1229 msg = "%r class should define a '%s' method" % ( | |
| 1230 test_class.__class__.__name__, meth_name) | |
| 1231 raise AttributeError(msg) | |
| 1232 | |
| 1233 @classmethod | |
| 1234 def test(cls): | |
| 1235 this = set([x[0] for x in cls.all]) | |
| 1236 ignored = set([x[0] for x in cls.ignored]) | |
| 1237 klass = set([x for x in dir(psutil.Process) if x[0] != '_']) | |
| 1238 leftout = (this | ignored) ^ klass | |
| 1239 if leftout: | |
| 1240 raise ValueError("uncovered Process class names: %r" % leftout) | |
| 1241 | |
| 1242 | |
| 1243 class system_namespace: | |
| 1244 """A container that lists all the module-level, system-related APIs. | |
| 1245 Utilities such as cpu_percent() are excluded. Usage: | |
| 1246 | |
| 1247 >>> ns = system_namespace | |
| 1248 >>> for fun, name in ns.iter(ns.getters): | |
| 1249 ... fun() | |
| 1250 """ | |
| 1251 getters = [ | |
| 1252 ('boot_time', (), {}), | |
| 1253 ('cpu_count', (), {'logical': False}), | |
| 1254 ('cpu_count', (), {'logical': True}), | |
| 1255 ('cpu_stats', (), {}), | |
| 1256 ('cpu_times', (), {'percpu': False}), | |
| 1257 ('cpu_times', (), {'percpu': True}), | |
| 1258 ('disk_io_counters', (), {'perdisk': True}), | |
| 1259 ('disk_partitions', (), {'all': True}), | |
| 1260 ('disk_usage', (os.getcwd(), ), {}), | |
| 1261 ('net_connections', (), {'kind': 'all'}), | |
| 1262 ('net_if_addrs', (), {}), | |
| 1263 ('net_if_stats', (), {}), | |
| 1264 ('net_io_counters', (), {'pernic': True}), | |
| 1265 ('pid_exists', (os.getpid(), ), {}), | |
| 1266 ('pids', (), {}), | |
| 1267 ('swap_memory', (), {}), | |
| 1268 ('users', (), {}), | |
| 1269 ('virtual_memory', (), {}), | |
| 1270 ] | |
| 1271 if HAS_CPU_FREQ: | |
| 1272 getters += [('cpu_freq', (), {'percpu': True})] | |
| 1273 if HAS_GETLOADAVG: | |
| 1274 getters += [('getloadavg', (), {})] | |
| 1275 if HAS_SENSORS_TEMPERATURES: | |
| 1276 getters += [('sensors_temperatures', (), {})] | |
| 1277 if HAS_SENSORS_FANS: | |
| 1278 getters += [('sensors_fans', (), {})] | |
| 1279 if HAS_SENSORS_BATTERY: | |
| 1280 getters += [('sensors_battery', (), {})] | |
| 1281 if WINDOWS: | |
| 1282 getters += [('win_service_iter', (), {})] | |
| 1283 getters += [('win_service_get', ('alg', ), {})] | |
| 1284 | |
| 1285 ignored = [ | |
| 1286 ('process_iter', (), {}), | |
| 1287 ('wait_procs', ([psutil.Process()], ), {}), | |
| 1288 ('cpu_percent', (), {}), | |
| 1289 ('cpu_times_percent', (), {}), | |
| 1290 ] | |
| 1291 | |
| 1292 all = getters | |
| 1293 | |
| 1294 @staticmethod | |
| 1295 def iter(ls): | |
| 1296 """Given a list of tuples yields a set of (fun, fun_name) tuples | |
| 1297 in random order. | |
| 1298 """ | |
| 1299 ls = list(ls) | |
| 1300 random.shuffle(ls) | |
| 1301 for fun_name, args, kwds in ls: | |
| 1302 fun = getattr(psutil, fun_name) | |
| 1303 fun = functools.partial(fun, *args, **kwds) | |
| 1304 yield (fun, fun_name) | |
| 1305 | |
| 1306 test_class_coverage = process_namespace.test_class_coverage | |
| 1307 | |
| 1308 | |
| 1309 def serialrun(klass): | |
| 1310 """A decorator to mark a TestCase class. When running parallel tests, | |
| 1311 class' unit tests will be run serially (1 process). | |
| 1312 """ | |
| 1313 # assert issubclass(klass, unittest.TestCase), klass | |
| 1314 assert inspect.isclass(klass), klass | |
| 1315 klass._serialrun = True | |
| 1316 return klass | |
| 1317 | |
| 1318 | |
| 1319 def retry_on_failure(retries=NO_RETRIES): | |
| 1320 """Decorator which runs a test function and retries N times before | |
| 1321 actually failing. | |
| 1322 """ | |
| 1323 def logfun(exc): | |
| 1324 print("%r, retrying" % exc, file=sys.stderr) # NOQA | |
| 1325 | |
| 1326 return retry(exception=AssertionError, timeout=None, retries=retries, | |
| 1327 logfun=logfun) | |
| 1328 | |
| 1329 | |
| 1330 def skip_on_access_denied(only_if=None): | |
| 1331 """Decorator to Ignore AccessDenied exceptions.""" | |
| 1332 def decorator(fun): | |
| 1333 @functools.wraps(fun) | |
| 1334 def wrapper(*args, **kwargs): | |
| 1335 try: | |
| 1336 return fun(*args, **kwargs) | |
| 1337 except psutil.AccessDenied: | |
| 1338 if only_if is not None: | |
| 1339 if not only_if: | |
| 1340 raise | |
| 1341 raise unittest.SkipTest("raises AccessDenied") | |
| 1342 return wrapper | |
| 1343 return decorator | |
| 1344 | |
| 1345 | |
| 1346 def skip_on_not_implemented(only_if=None): | |
| 1347 """Decorator to Ignore NotImplementedError exceptions.""" | |
| 1348 def decorator(fun): | |
| 1349 @functools.wraps(fun) | |
| 1350 def wrapper(*args, **kwargs): | |
| 1351 try: | |
| 1352 return fun(*args, **kwargs) | |
| 1353 except NotImplementedError: | |
| 1354 if only_if is not None: | |
| 1355 if not only_if: | |
| 1356 raise | |
| 1357 msg = "%r was skipped because it raised NotImplementedError" \ | |
| 1358 % fun.__name__ | |
| 1359 raise unittest.SkipTest(msg) | |
| 1360 return wrapper | |
| 1361 return decorator | |
| 1362 | |
| 1363 | |
| 1364 # =================================================================== | |
| 1365 # --- network | |
| 1366 # =================================================================== | |
| 1367 | |
| 1368 | |
| 1369 def get_free_port(host='127.0.0.1'): | |
| 1370 """Return an unused TCP port.""" | |
| 1371 with contextlib.closing(socket.socket()) as sock: | |
| 1372 sock.bind((host, 0)) | |
| 1373 return sock.getsockname()[1] | |
| 1374 | |
| 1375 | |
| 1376 def bind_socket(family=AF_INET, type=SOCK_STREAM, addr=None): | |
| 1377 """Binds a generic socket.""" | |
| 1378 if addr is None and family in (AF_INET, AF_INET6): | |
| 1379 addr = ("", 0) | |
| 1380 sock = socket.socket(family, type) | |
| 1381 try: | |
| 1382 if os.name not in ('nt', 'cygwin'): | |
| 1383 sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) | |
| 1384 sock.bind(addr) | |
| 1385 if type == socket.SOCK_STREAM: | |
| 1386 sock.listen(5) | |
| 1387 return sock | |
| 1388 except Exception: | |
| 1389 sock.close() | |
| 1390 raise | |
| 1391 | |
| 1392 | |
| 1393 def bind_unix_socket(name, type=socket.SOCK_STREAM): | |
| 1394 """Bind a UNIX socket.""" | |
| 1395 assert psutil.POSIX | |
| 1396 assert not os.path.exists(name), name | |
| 1397 sock = socket.socket(socket.AF_UNIX, type) | |
| 1398 try: | |
| 1399 sock.bind(name) | |
| 1400 if type == socket.SOCK_STREAM: | |
| 1401 sock.listen(5) | |
| 1402 except Exception: | |
| 1403 sock.close() | |
| 1404 raise | |
| 1405 return sock | |
| 1406 | |
| 1407 | |
| 1408 def tcp_socketpair(family, addr=("", 0)): | |
| 1409 """Build a pair of TCP sockets connected to each other. | |
| 1410 Return a (server, client) tuple. | |
| 1411 """ | |
| 1412 with contextlib.closing(socket.socket(family, SOCK_STREAM)) as ll: | |
| 1413 ll.bind(addr) | |
| 1414 ll.listen(5) | |
| 1415 addr = ll.getsockname() | |
| 1416 c = socket.socket(family, SOCK_STREAM) | |
| 1417 try: | |
| 1418 c.connect(addr) | |
| 1419 caddr = c.getsockname() | |
| 1420 while True: | |
| 1421 a, addr = ll.accept() | |
| 1422 # check that we've got the correct client | |
| 1423 if addr == caddr: | |
| 1424 return (a, c) | |
| 1425 a.close() | |
| 1426 except OSError: | |
| 1427 c.close() | |
| 1428 raise | |
| 1429 | |
| 1430 | |
| 1431 def unix_socketpair(name): | |
| 1432 """Build a pair of UNIX sockets connected to each other through | |
| 1433 the same UNIX file name. | |
| 1434 Return a (server, client) tuple. | |
| 1435 """ | |
| 1436 assert psutil.POSIX | |
| 1437 server = client = None | |
| 1438 try: | |
| 1439 server = bind_unix_socket(name, type=socket.SOCK_STREAM) | |
| 1440 server.setblocking(0) | |
| 1441 client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) | |
| 1442 client.setblocking(0) | |
| 1443 client.connect(name) | |
| 1444 # new = server.accept() | |
| 1445 except Exception: | |
| 1446 if server is not None: | |
| 1447 server.close() | |
| 1448 if client is not None: | |
| 1449 client.close() | |
| 1450 raise | |
| 1451 return (server, client) | |
| 1452 | |
| 1453 | |
| 1454 @contextlib.contextmanager | |
| 1455 def create_sockets(): | |
| 1456 """Open as many socket families / types as possible.""" | |
| 1457 socks = [] | |
| 1458 fname1 = fname2 = None | |
| 1459 try: | |
| 1460 socks.append(bind_socket(socket.AF_INET, socket.SOCK_STREAM)) | |
| 1461 socks.append(bind_socket(socket.AF_INET, socket.SOCK_DGRAM)) | |
| 1462 if supports_ipv6(): | |
| 1463 socks.append(bind_socket(socket.AF_INET6, socket.SOCK_STREAM)) | |
| 1464 socks.append(bind_socket(socket.AF_INET6, socket.SOCK_DGRAM)) | |
| 1465 if POSIX and HAS_CONNECTIONS_UNIX: | |
| 1466 fname1 = get_testfn() | |
| 1467 fname2 = get_testfn() | |
| 1468 s1, s2 = unix_socketpair(fname1) | |
| 1469 s3 = bind_unix_socket(fname2, type=socket.SOCK_DGRAM) | |
| 1470 for s in (s1, s2, s3): | |
| 1471 socks.append(s) | |
| 1472 yield socks | |
| 1473 finally: | |
| 1474 for s in socks: | |
| 1475 s.close() | |
| 1476 for fname in (fname1, fname2): | |
| 1477 if fname is not None: | |
| 1478 safe_rmpath(fname) | |
| 1479 | |
| 1480 | |
| 1481 def check_net_address(addr, family): | |
| 1482 """Check a net address validity. Supported families are IPv4, | |
| 1483 IPv6 and MAC addresses. | |
| 1484 """ | |
| 1485 import ipaddress # python >= 3.3 / requires "pip install ipaddress" | |
| 1486 if enum and PY3 and not PYPY: | |
| 1487 assert isinstance(family, enum.IntEnum), family | |
| 1488 if family == socket.AF_INET: | |
| 1489 octs = [int(x) for x in addr.split('.')] | |
| 1490 assert len(octs) == 4, addr | |
| 1491 for num in octs: | |
| 1492 assert 0 <= num <= 255, addr | |
| 1493 if not PY3: | |
| 1494 addr = unicode(addr) | |
| 1495 ipaddress.IPv4Address(addr) | |
| 1496 elif family == socket.AF_INET6: | |
| 1497 assert isinstance(addr, str), addr | |
| 1498 if not PY3: | |
| 1499 addr = unicode(addr) | |
| 1500 ipaddress.IPv6Address(addr) | |
| 1501 elif family == psutil.AF_LINK: | |
| 1502 assert re.match(r'([a-fA-F0-9]{2}[:|\-]?){6}', addr) is not None, addr | |
| 1503 else: | |
| 1504 raise ValueError("unknown family %r", family) | |
| 1505 | |
| 1506 | |
| 1507 # =================================================================== | |
| 1508 # --- compatibility | |
| 1509 # =================================================================== | |
| 1510 | |
| 1511 | |
| 1512 def reload_module(module): | |
| 1513 """Backport of importlib.reload of Python 3.3+.""" | |
| 1514 try: | |
| 1515 import importlib | |
| 1516 if not hasattr(importlib, 'reload'): # python <=3.3 | |
| 1517 raise ImportError | |
| 1518 except ImportError: | |
| 1519 import imp | |
| 1520 return imp.reload(module) | |
| 1521 else: | |
| 1522 return importlib.reload(module) | |
| 1523 | |
| 1524 | |
| 1525 def import_module_by_path(path): | |
| 1526 name = os.path.splitext(os.path.basename(path))[0] | |
| 1527 if sys.version_info[0] == 2: | |
| 1528 import imp | |
| 1529 return imp.load_source(name, path) | |
| 1530 elif sys.version_info[:2] <= (3, 4): | |
| 1531 from importlib.machinery import SourceFileLoader | |
| 1532 return SourceFileLoader(name, path).load_module() | |
| 1533 else: | |
| 1534 import importlib.util | |
| 1535 spec = importlib.util.spec_from_file_location(name, path) | |
| 1536 mod = importlib.util.module_from_spec(spec) | |
| 1537 spec.loader.exec_module(mod) | |
| 1538 return mod | |
| 1539 | |
| 1540 | |
| 1541 # =================================================================== | |
| 1542 # --- others | |
| 1543 # =================================================================== | |
| 1544 | |
| 1545 | |
| 1546 def warn(msg): | |
| 1547 """Raise a warning msg.""" | |
| 1548 warnings.warn(msg, UserWarning) | |
| 1549 | |
| 1550 | |
| 1551 def is_namedtuple(x): | |
| 1552 """Check if object is an instance of namedtuple.""" | |
| 1553 t = type(x) | |
| 1554 b = t.__bases__ | |
| 1555 if len(b) != 1 or b[0] != tuple: | |
| 1556 return False | |
| 1557 f = getattr(t, '_fields', None) | |
| 1558 if not isinstance(f, tuple): | |
| 1559 return False | |
| 1560 return all(type(n) == str for n in f) | |
| 1561 | |
| 1562 | |
| 1563 if POSIX: | |
| 1564 @contextlib.contextmanager | |
| 1565 def copyload_shared_lib(suffix=""): | |
| 1566 """Ctx manager which picks up a random shared CO lib used | |
| 1567 by this process, copies it in another location and loads it | |
| 1568 in memory via ctypes. Return the new absolutized path. | |
| 1569 """ | |
| 1570 exe = 'pypy' if PYPY else 'python' | |
| 1571 ext = ".so" | |
| 1572 dst = get_testfn(suffix=suffix + ext) | |
| 1573 libs = [x.path for x in psutil.Process().memory_maps() if | |
| 1574 os.path.splitext(x.path)[1] == ext and | |
| 1575 exe in x.path.lower()] | |
| 1576 src = random.choice(libs) | |
| 1577 shutil.copyfile(src, dst) | |
| 1578 try: | |
| 1579 ctypes.CDLL(dst) | |
| 1580 yield dst | |
| 1581 finally: | |
| 1582 safe_rmpath(dst) | |
| 1583 else: | |
| 1584 @contextlib.contextmanager | |
| 1585 def copyload_shared_lib(suffix=""): | |
| 1586 """Ctx manager which picks up a random shared DLL lib used | |
| 1587 by this process, copies it in another location and loads it | |
| 1588 in memory via ctypes. | |
| 1589 Return the new absolutized, normcased path. | |
| 1590 """ | |
| 1591 from ctypes import wintypes | |
| 1592 from ctypes import WinError | |
| 1593 ext = ".dll" | |
| 1594 dst = get_testfn(suffix=suffix + ext) | |
| 1595 libs = [x.path for x in psutil.Process().memory_maps() if | |
| 1596 x.path.lower().endswith(ext) and | |
| 1597 'python' in os.path.basename(x.path).lower() and | |
| 1598 'wow64' not in x.path.lower()] | |
| 1599 if PYPY and not libs: | |
| 1600 libs = [x.path for x in psutil.Process().memory_maps() if | |
| 1601 'pypy' in os.path.basename(x.path).lower()] | |
| 1602 src = random.choice(libs) | |
| 1603 shutil.copyfile(src, dst) | |
| 1604 cfile = None | |
| 1605 try: | |
| 1606 cfile = ctypes.WinDLL(dst) | |
| 1607 yield dst | |
| 1608 finally: | |
| 1609 # Work around OverflowError: | |
| 1610 # - https://ci.appveyor.com/project/giampaolo/psutil/build/1207/ | |
| 1611 # job/o53330pbnri9bcw7 | |
| 1612 # - http://bugs.python.org/issue30286 | |
| 1613 # - http://stackoverflow.com/questions/23522055 | |
| 1614 if cfile is not None: | |
| 1615 FreeLibrary = ctypes.windll.kernel32.FreeLibrary | |
| 1616 FreeLibrary.argtypes = [wintypes.HMODULE] | |
| 1617 ret = FreeLibrary(cfile._handle) | |
| 1618 if ret == 0: | |
| 1619 WinError() | |
| 1620 safe_rmpath(dst) | |
| 1621 | |
| 1622 | |
| 1623 # =================================================================== | |
| 1624 # --- Exit funs (first is executed last) | |
| 1625 # =================================================================== | |
| 1626 | |
| 1627 | |
| 1628 # this is executed first | |
| 1629 @atexit.register | |
| 1630 def cleanup_test_procs(): | |
| 1631 reap_children(recursive=True) | |
| 1632 | |
| 1633 | |
| 1634 # atexit module does not execute exit functions in case of SIGTERM, which | |
| 1635 # gets sent to test subprocesses, which is a problem if they import this | |
| 1636 # module. With this it will. See: | |
| 1637 # https://gmpy.dev/blog/2016/how-to-always-execute-exit-functions-in-python | |
| 1638 if POSIX: | |
| 1639 signal.signal(signal.SIGTERM, lambda sig, frame: sys.exit(sig)) |
