"""Server for mypy daemon mode. This implements a daemon process which keeps useful state in memory to enable fine-grained incremental reprocessing of changes. """ import argparse import base64 import io import json import os import pickle import subprocess import sys import time import traceback from contextlib import redirect_stderr, redirect_stdout from typing import AbstractSet, Any, Callable, Dict, List, Optional, Sequence, Tuple, Set from typing_extensions import Final import mypy.build import mypy.errors import mypy.main from mypy.find_sources import create_source_list, InvalidSourceList from mypy.server.update import FineGrainedBuildManager, refresh_suppressed_submodules from mypy.dmypy_util import receive from mypy.ipc import IPCServer from mypy.fscache import FileSystemCache from mypy.fswatcher import FileSystemWatcher, FileData from mypy.modulefinder import BuildSource, compute_search_paths, FindModuleCache, SearchPaths from mypy.options import Options from mypy.suggestions import SuggestionFailure, SuggestionEngine from mypy.typestate import reset_global_state from mypy.version import __version__ from mypy.util import FancyFormatter, count_stats MEM_PROFILE: Final = False # If True, dump memory profile after initialization if sys.platform == 'win32': from subprocess import STARTUPINFO def daemonize(options: Options, status_file: str, timeout: Optional[int] = None, log_file: Optional[str] = None) -> int: """Create the daemon process via "dmypy daemon" and pass options via command line When creating the daemon grandchild, we create it in a new console, which is started hidden. We cannot use DETACHED_PROCESS since it will cause console windows to pop up when starting. See https://github.com/python/cpython/pull/4150#issuecomment-340215696 for more on why we can't have nice things. It also pickles the options to be unpickled by mypy. """ command = [sys.executable, '-m', 'mypy.dmypy', '--status-file', status_file, 'daemon'] pickled_options = pickle.dumps((options.snapshot(), timeout, log_file)) command.append(f'--options-data="{base64.b64encode(pickled_options).decode()}"') info = STARTUPINFO() info.dwFlags = 0x1 # STARTF_USESHOWWINDOW aka use wShowWindow's value info.wShowWindow = 0 # SW_HIDE aka make the window invisible try: subprocess.Popen(command, creationflags=0x10, # CREATE_NEW_CONSOLE startupinfo=info) return 0 except subprocess.CalledProcessError as e: return e.returncode else: def _daemonize_cb(func: Callable[[], None], log_file: Optional[str] = None) -> int: """Arrange to call func() in a grandchild of the current process. Return 0 for success, exit status for failure, negative if subprocess killed by signal. """ # See https://stackoverflow.com/questions/473620/how-do-you-create-a-daemon-in-python sys.stdout.flush() sys.stderr.flush() pid = os.fork() if pid: # Parent process: wait for child in case things go bad there. npid, sts = os.waitpid(pid, 0) sig = sts & 0xff if sig: print("Child killed by signal", sig) return -sig sts = sts >> 8 if sts: print("Child exit status", sts) return sts # Child process: do a bunch of UNIX stuff and then fork a grandchild. try: os.setsid() # Detach controlling terminal os.umask(0o27) devnull = os.open('/dev/null', os.O_RDWR) os.dup2(devnull, 0) os.dup2(devnull, 1) os.dup2(devnull, 2) os.close(devnull) pid = os.fork() if pid: # Child is done, exit to parent. os._exit(0) # Grandchild: run the server. if log_file: sys.stdout = sys.stderr = open(log_file, 'a', buffering=1) fd = sys.stdout.fileno() os.dup2(fd, 2) os.dup2(fd, 1) func() finally: # Make sure we never get back into the caller. os._exit(1) def daemonize(options: Options, status_file: str, timeout: Optional[int] = None, log_file: Optional[str] = None) -> int: """Run the mypy daemon in a grandchild of the current process Return 0 for success, exit status for failure, negative if subprocess killed by signal. """ return _daemonize_cb(Server(options, status_file, timeout).serve, log_file) # Server code. CONNECTION_NAME: Final = "dmypy" def process_start_options(flags: List[str], allow_sources: bool) -> Options: _, options = mypy.main.process_options( ['-i'] + flags, require_targets=False, server_options=True ) if options.report_dirs: print("dmypy: Ignoring report generation settings. Start/restart cannot generate reports.") if options.junit_xml: print("dmypy: Ignoring report generation settings. " "Start/restart does not support --junit-xml. Pass it to check/recheck instead") options.junit_xml = None if not options.incremental: sys.exit("dmypy: start/restart should not disable incremental mode") if options.follow_imports not in ('skip', 'error', 'normal'): sys.exit("dmypy: follow-imports=silent not supported") return options def ignore_suppressed_imports(module: str) -> bool: """Can we skip looking for newly unsuppressed imports to module?""" # Various submodules of 'encodings' can be suppressed, since it # uses module-level '__getattr__'. Skip them since there are many # of them, and following imports to them is kind of pointless. return module.startswith('encodings.') ModulePathPair = Tuple[str, str] ModulePathPairs = List[ModulePathPair] ChangesAndRemovals = Tuple[ModulePathPairs, ModulePathPairs] class Server: # NOTE: the instance is constructed in the parent process but # serve() is called in the grandchild (by daemonize()). def __init__(self, options: Options, status_file: str, timeout: Optional[int] = None) -> None: """Initialize the server with the desired mypy flags.""" self.options = options # Snapshot the options info before we muck with it, to detect changes self.options_snapshot = options.snapshot() self.timeout = timeout self.fine_grained_manager: Optional[FineGrainedBuildManager] = None if os.path.isfile(status_file): os.unlink(status_file) self.fscache = FileSystemCache() options.raise_exceptions = True options.incremental = True options.fine_grained_incremental = True options.show_traceback = True if options.use_fine_grained_cache: # Using fine_grained_cache implies generating and caring # about the fine grained cache options.cache_fine_grained = True else: options.cache_dir = os.devnull # Fine-grained incremental doesn't support general partial types # (details in https://github.com/python/mypy/issues/4492) options.local_partial_types = True self.status_file = status_file # Since the object is created in the parent process we can check # the output terminal options here. self.formatter = FancyFormatter(sys.stdout, sys.stderr, options.show_error_codes) def _response_metadata(self) -> Dict[str, str]: py_version = f'{self.options.python_version[0]}_{self.options.python_version[1]}' return { 'platform': self.options.platform, 'python_version': py_version, } def serve(self) -> None: """Serve requests, synchronously (no thread or fork).""" command = None server = IPCServer(CONNECTION_NAME, self.timeout) try: with open(self.status_file, 'w') as f: json.dump({'pid': os.getpid(), 'connection_name': server.connection_name}, f) f.write('\n') # I like my JSON with a trailing newline while True: with server: data = receive(server) resp: Dict[str, Any] = {} if 'command' not in data: resp = {'error': "No command found in request"} else: command = data['command'] if not isinstance(command, str): resp = {'error': "Command is not a string"} else: command = data.pop('command') try: resp = self.run_command(command, data) except Exception: # If we are crashing, report the crash to the client tb = traceback.format_exception(*sys.exc_info()) resp = {'error': "Daemon crashed!\n" + "".join(tb)} resp.update(self._response_metadata()) server.write(json.dumps(resp).encode('utf8')) raise try: resp.update(self._response_metadata()) server.write(json.dumps(resp).encode('utf8')) except OSError: pass # Maybe the client hung up if command == 'stop': reset_global_state() sys.exit(0) finally: # If the final command is something other than a clean # stop, remove the status file. (We can't just # simplify the logic and always remove the file, since # that could cause us to remove a future server's # status file.) if command != 'stop': os.unlink(self.status_file) try: server.cleanup() # try to remove the socket dir on Linux except OSError: pass exc_info = sys.exc_info() if exc_info[0] and exc_info[0] is not SystemExit: traceback.print_exception(*exc_info) def run_command(self, command: str, data: Dict[str, object]) -> Dict[str, object]: """Run a specific command from the registry.""" key = 'cmd_' + command method = getattr(self.__class__, key, None) if method is None: return {'error': f"Unrecognized command '{command}'"} else: if command not in {'check', 'recheck', 'run'}: # Only the above commands use some error formatting. del data['is_tty'] del data['terminal_width'] return method(self, **data) # Command functions (run in the server via RPC). def cmd_status(self, fswatcher_dump_file: Optional[str] = None) -> Dict[str, object]: """Return daemon status.""" res: Dict[str, object] = {} res.update(get_meminfo()) if fswatcher_dump_file: data = self.fswatcher.dump_file_data() if hasattr(self, 'fswatcher') else {} # Using .dumps and then writing was noticeably faster than using dump s = json.dumps(data) with open(fswatcher_dump_file, 'w') as f: f.write(s) return res def cmd_stop(self) -> Dict[str, object]: """Stop daemon.""" # We need to remove the status file *before* we complete the # RPC. Otherwise a race condition exists where a subsequent # command can see a status file from a dying server and think # it is a live one. os.unlink(self.status_file) return {} def cmd_run(self, version: str, args: Sequence[str], is_tty: bool, terminal_width: int) -> Dict[str, object]: """Check a list of files, triggering a restart if needed.""" stderr = io.StringIO() stdout = io.StringIO() try: # Process options can exit on improper arguments, so we need to catch that and # capture stderr so the client can report it with redirect_stderr(stderr): with redirect_stdout(stdout): sources, options = mypy.main.process_options( ['-i'] + list(args), require_targets=True, server_options=True, fscache=self.fscache, program='mypy-daemon', header=argparse.SUPPRESS) # Signal that we need to restart if the options have changed if self.options_snapshot != options.snapshot(): return {'restart': 'configuration changed'} if __version__ != version: return {'restart': 'mypy version changed'} if self.fine_grained_manager: manager = self.fine_grained_manager.manager start_plugins_snapshot = manager.plugins_snapshot _, current_plugins_snapshot = mypy.build.load_plugins( options, manager.errors, sys.stdout, extra_plugins=() ) if current_plugins_snapshot != start_plugins_snapshot: return {'restart': 'plugins changed'} except InvalidSourceList as err: return {'out': '', 'err': str(err), 'status': 2} except SystemExit as e: return {'out': stdout.getvalue(), 'err': stderr.getvalue(), 'status': e.code} return self.check(sources, is_tty, terminal_width) def cmd_check(self, files: Sequence[str], is_tty: bool, terminal_width: int) -> Dict[str, object]: """Check a list of files.""" try: sources = create_source_list(files, self.options, self.fscache) except InvalidSourceList as err: return {'out': '', 'err': str(err), 'status': 2} return self.check(sources, is_tty, terminal_width) def cmd_recheck(self, is_tty: bool, terminal_width: int, remove: Optional[List[str]] = None, update: Optional[List[str]] = None) -> Dict[str, object]: """Check the same list of files we checked most recently. If remove/update is given, they modify the previous list; if all are None, stat() is called for each file in the previous list. """ t0 = time.time() if not self.fine_grained_manager: return {'error': "Command 'recheck' is only valid after a 'check' command"} sources = self.previous_sources if remove: removals = set(remove) sources = [s for s in sources if s.path and s.path not in removals] if update: known = {s.path for s in sources if s.path} added = [p for p in update if p not in known] try: added_sources = create_source_list(added, self.options, self.fscache) except InvalidSourceList as err: return {'out': '', 'err': str(err), 'status': 2} sources = sources + added_sources # Make a copy! t1 = time.time() manager = self.fine_grained_manager.manager manager.log(f"fine-grained increment: cmd_recheck: {t1 - t0:.3f}s") if not self.following_imports(): messages = self.fine_grained_increment(sources, remove, update) else: assert remove is None and update is None messages = self.fine_grained_increment_follow_imports(sources) res = self.increment_output(messages, sources, is_tty, terminal_width) self.flush_caches() self.update_stats(res) return res def check(self, sources: List[BuildSource], is_tty: bool, terminal_width: int) -> Dict[str, Any]: """Check using fine-grained incremental mode. If is_tty is True format the output nicely with colors and summary line (unless disabled in self.options). Also pass the terminal_width to formatter. """ if not self.fine_grained_manager: res = self.initialize_fine_grained(sources, is_tty, terminal_width) else: if not self.following_imports(): messages = self.fine_grained_increment(sources) else: messages = self.fine_grained_increment_follow_imports(sources) res = self.increment_output(messages, sources, is_tty, terminal_width) self.flush_caches() self.update_stats(res) return res def flush_caches(self) -> None: self.fscache.flush() if self.fine_grained_manager: self.fine_grained_manager.flush_cache() def update_stats(self, res: Dict[str, Any]) -> None: if self.fine_grained_manager: manager = self.fine_grained_manager.manager manager.dump_stats() res['stats'] = manager.stats manager.stats = {} def following_imports(self) -> bool: """Are we following imports?""" # TODO: What about silent? return self.options.follow_imports == 'normal' def initialize_fine_grained(self, sources: List[BuildSource], is_tty: bool, terminal_width: int) -> Dict[str, Any]: self.fswatcher = FileSystemWatcher(self.fscache) t0 = time.time() self.update_sources(sources) t1 = time.time() try: result = mypy.build.build(sources=sources, options=self.options, fscache=self.fscache) except mypy.errors.CompileError as e: output = ''.join(s + '\n' for s in e.messages) if e.use_stdout: out, err = output, '' else: out, err = '', output return {'out': out, 'err': err, 'status': 2} messages = result.errors self.fine_grained_manager = FineGrainedBuildManager(result) if self.following_imports(): sources = find_all_sources_in_build(self.fine_grained_manager.graph, sources) self.update_sources(sources) self.previous_sources = sources # If we are using the fine-grained cache, build hasn't actually done # the typechecking on the updated files yet. # Run a fine-grained update starting from the cached data if result.used_cache: t2 = time.time() # Pull times and hashes out of the saved_cache and stick them into # the fswatcher, so we pick up the changes. for state in self.fine_grained_manager.graph.values(): meta = state.meta if meta is None: continue assert state.path is not None self.fswatcher.set_file_data( state.path, FileData(st_mtime=float(meta.mtime), st_size=meta.size, hash=meta.hash)) changed, removed = self.find_changed(sources) changed += self.find_added_suppressed(self.fine_grained_manager.graph, set(), self.fine_grained_manager.manager.search_paths) # Find anything that has had its dependency list change for state in self.fine_grained_manager.graph.values(): if not state.is_fresh(): assert state.path is not None changed.append((state.id, state.path)) t3 = time.time() # Run an update messages = self.fine_grained_manager.update(changed, removed) if self.following_imports(): # We need to do another update to any new files found by following imports. messages = self.fine_grained_increment_follow_imports(sources) t4 = time.time() self.fine_grained_manager.manager.add_stats( update_sources_time=t1 - t0, build_time=t2 - t1, find_changes_time=t3 - t2, fg_update_time=t4 - t3, files_changed=len(removed) + len(changed)) else: # Stores the initial state of sources as a side effect. self.fswatcher.find_changed() if MEM_PROFILE: from mypy.memprofile import print_memory_profile print_memory_profile(run_gc=False) status = 1 if messages else 0 messages = self.pretty_messages(messages, len(sources), is_tty, terminal_width) return {'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status} def fine_grained_increment(self, sources: List[BuildSource], remove: Optional[List[str]] = None, update: Optional[List[str]] = None, ) -> List[str]: """Perform a fine-grained type checking increment. If remove and update are None, determine changed paths by using fswatcher. Otherwise, assume that only these files have changes. Args: sources: sources passed on the command line remove: paths of files that have been removed update: paths of files that have been changed or created """ assert self.fine_grained_manager is not None manager = self.fine_grained_manager.manager t0 = time.time() if remove is None and update is None: # Use the fswatcher to determine which files were changed # (updated or added) or removed. self.update_sources(sources) changed, removed = self.find_changed(sources) else: # Use the remove/update lists to update fswatcher. # This avoids calling stat() for unchanged files. changed, removed = self.update_changed(sources, remove or [], update or []) changed += self.find_added_suppressed(self.fine_grained_manager.graph, set(), manager.search_paths) manager.search_paths = compute_search_paths(sources, manager.options, manager.data_dir) t1 = time.time() manager.log(f"fine-grained increment: find_changed: {t1 - t0:.3f}s") messages = self.fine_grained_manager.update(changed, removed) t2 = time.time() manager.log(f"fine-grained increment: update: {t2 - t1:.3f}s") manager.add_stats( find_changes_time=t1 - t0, fg_update_time=t2 - t1, files_changed=len(removed) + len(changed)) self.previous_sources = sources return messages def fine_grained_increment_follow_imports(self, sources: List[BuildSource]) -> List[str]: """Like fine_grained_increment, but follow imports.""" t0 = time.time() # TODO: Support file events assert self.fine_grained_manager is not None fine_grained_manager = self.fine_grained_manager graph = fine_grained_manager.graph manager = fine_grained_manager.manager orig_modules = list(graph.keys()) self.update_sources(sources) changed_paths = self.fswatcher.find_changed() manager.search_paths = compute_search_paths(sources, manager.options, manager.data_dir) t1 = time.time() manager.log(f"fine-grained increment: find_changed: {t1 - t0:.3f}s") seen = {source.module for source in sources} # Find changed modules reachable from roots (or in roots) already in graph. changed, new_files = self.find_reachable_changed_modules( sources, graph, seen, changed_paths ) sources.extend(new_files) # Process changes directly reachable from roots. messages = fine_grained_manager.update(changed, []) # Follow deps from changed modules (still within graph). worklist = changed[:] while worklist: module = worklist.pop() if module[0] not in graph: continue sources2 = self.direct_imports(module, graph) # Filter anything already seen before. This prevents # infinite looping if there are any self edges. (Self # edges are maybe a bug, but...) sources2 = [source for source in sources2 if source.module not in seen] changed, new_files = self.find_reachable_changed_modules( sources2, graph, seen, changed_paths ) self.update_sources(new_files) messages = fine_grained_manager.update(changed, []) worklist.extend(changed) t2 = time.time() def refresh_file(module: str, path: str) -> List[str]: return fine_grained_manager.update([(module, path)], []) for module_id, state in list(graph.items()): new_messages = refresh_suppressed_submodules( module_id, state.path, fine_grained_manager.deps, graph, self.fscache, refresh_file ) if new_messages is not None: messages = new_messages t3 = time.time() # There may be new files that became available, currently treated as # suppressed imports. Process them. while True: new_unsuppressed = self.find_added_suppressed(graph, seen, manager.search_paths) if not new_unsuppressed: break new_files = [BuildSource(mod[1], mod[0]) for mod in new_unsuppressed] sources.extend(new_files) self.update_sources(new_files) messages = fine_grained_manager.update(new_unsuppressed, []) for module_id, path in new_unsuppressed: new_messages = refresh_suppressed_submodules( module_id, path, fine_grained_manager.deps, graph, self.fscache, refresh_file ) if new_messages is not None: messages = new_messages t4 = time.time() # Find all original modules in graph that were not reached -- they are deleted. to_delete = [] for module_id in orig_modules: if module_id not in graph: continue if module_id not in seen: module_path = graph[module_id].path assert module_path is not None to_delete.append((module_id, module_path)) if to_delete: messages = fine_grained_manager.update([], to_delete) fix_module_deps(graph) self.previous_sources = find_all_sources_in_build(graph) self.update_sources(self.previous_sources) # Store current file state as side effect self.fswatcher.find_changed() t5 = time.time() manager.log(f"fine-grained increment: update: {t5 - t1:.3f}s") manager.add_stats( find_changes_time=t1 - t0, fg_update_time=t2 - t1, refresh_suppressed_time=t3 - t2, find_added_supressed_time=t4 - t3, cleanup_time=t5 - t4) return messages def find_reachable_changed_modules( self, roots: List[BuildSource], graph: mypy.build.Graph, seen: Set[str], changed_paths: AbstractSet[str]) -> Tuple[List[Tuple[str, str]], List[BuildSource]]: """Follow imports within graph from given sources until hitting changed modules. If we find a changed module, we can't continue following imports as the imports may have changed. Args: roots: modules where to start search from graph: module graph to use for the search seen: modules we've seen before that won't be visited (mutated here!!) changed_paths: which paths have changed (stop search here and return any found) Return (encountered reachable changed modules, unchanged files not in sources_set traversed). """ changed = [] new_files = [] worklist = roots[:] seen.update(source.module for source in worklist) while worklist: nxt = worklist.pop() if nxt.module not in seen: seen.add(nxt.module) new_files.append(nxt) if nxt.path in changed_paths: assert nxt.path is not None # TODO changed.append((nxt.module, nxt.path)) elif nxt.module in graph: state = graph[nxt.module] for dep in state.dependencies: if dep not in seen: seen.add(dep) worklist.append(BuildSource(graph[dep].path, graph[dep].id)) return changed, new_files def direct_imports(self, module: Tuple[str, str], graph: mypy.build.Graph) -> List[BuildSource]: """Return the direct imports of module not included in seen.""" state = graph[module[0]] return [BuildSource(graph[dep].path, dep) for dep in state.dependencies] def find_added_suppressed(self, graph: mypy.build.Graph, seen: Set[str], search_paths: SearchPaths) -> List[Tuple[str, str]]: """Find suppressed modules that have been added (and not included in seen). Args: seen: reachable modules we've seen before (mutated here!!) Return suppressed, added modules. """ all_suppressed = set() for state in graph.values(): all_suppressed |= state.suppressed_set # Filter out things that shouldn't actually be considered suppressed. # # TODO: Figure out why these are treated as suppressed all_suppressed = {module for module in all_suppressed if module not in graph and not ignore_suppressed_imports(module)} # Optimization: skip top-level packages that are obviously not # there, to avoid calling the relatively slow find_module() # below too many times. packages = {module.split('.', 1)[0] for module in all_suppressed} packages = filter_out_missing_top_level_packages(packages, search_paths, self.fscache) # TODO: Namespace packages finder = FindModuleCache(search_paths, self.fscache, self.options) found = [] for module in all_suppressed: top_level_pkg = module.split('.', 1)[0] if top_level_pkg not in packages: # Fast path: non-existent top-level package continue result = finder.find_module(module, fast_path=True) if isinstance(result, str) and module not in seen: # When not following imports, we only follow imports to .pyi files. if not self.following_imports() and not result.endswith('.pyi'): continue found.append((module, result)) seen.add(module) return found def increment_output(self, messages: List[str], sources: List[BuildSource], is_tty: bool, terminal_width: int) -> Dict[str, Any]: status = 1 if messages else 0 messages = self.pretty_messages(messages, len(sources), is_tty, terminal_width) return {'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status} def pretty_messages(self, messages: List[str], n_sources: int, is_tty: bool = False, terminal_width: Optional[int] = None) -> List[str]: use_color = self.options.color_output and is_tty fit_width = self.options.pretty and is_tty if fit_width: messages = self.formatter.fit_in_terminal(messages, fixed_terminal_width=terminal_width) if self.options.error_summary: summary: Optional[str] = None n_errors, n_notes, n_files = count_stats(messages) if n_errors: summary = self.formatter.format_error(n_errors, n_files, n_sources, use_color=use_color) elif not messages or n_notes == len(messages): summary = self.formatter.format_success(n_sources, use_color) if summary: # Create new list to avoid appending multiple summaries on successive runs. messages = messages + [summary] if use_color: messages = [self.formatter.colorize(m) for m in messages] return messages def update_sources(self, sources: List[BuildSource]) -> None: paths = [source.path for source in sources if source.path is not None] if self.following_imports(): # Filter out directories (used for namespace packages). paths = [path for path in paths if self.fscache.isfile(path)] self.fswatcher.add_watched_paths(paths) def update_changed(self, sources: List[BuildSource], remove: List[str], update: List[str], ) -> ChangesAndRemovals: changed_paths = self.fswatcher.update_changed(remove, update) return self._find_changed(sources, changed_paths) def find_changed(self, sources: List[BuildSource]) -> ChangesAndRemovals: changed_paths = self.fswatcher.find_changed() return self._find_changed(sources, changed_paths) def _find_changed(self, sources: List[BuildSource], changed_paths: AbstractSet[str]) -> ChangesAndRemovals: # Find anything that has been added or modified changed = [(source.module, source.path) for source in sources if source.path and source.path in changed_paths] # Now find anything that has been removed from the build modules = {source.module for source in sources} omitted = [source for source in self.previous_sources if source.module not in modules] removed = [] for source in omitted: path = source.path assert path removed.append((source.module, path)) # Find anything that has had its module path change because of added or removed __init__s last = {s.path: s.module for s in self.previous_sources} for s in sources: assert s.path if s.path in last and last[s.path] != s.module: # Mark it as removed from its old name and changed at its new name removed.append((last[s.path], s.path)) changed.append((s.module, s.path)) return changed, removed def cmd_suggest(self, function: str, callsites: bool, **kwargs: Any) -> Dict[str, object]: """Suggest a signature for a function.""" if not self.fine_grained_manager: return { 'error': "Command 'suggest' is only valid after a 'check' command" " (that produces no parse errors)"} engine = SuggestionEngine(self.fine_grained_manager, **kwargs) try: if callsites: out = engine.suggest_callsites(function) else: out = engine.suggest(function) except SuggestionFailure as err: return {'error': str(err)} else: if not out: out = "No suggestions\n" elif not out.endswith("\n"): out += "\n" return {'out': out, 'err': "", 'status': 0} finally: self.flush_caches() def cmd_hang(self) -> Dict[str, object]: """Hang for 100 seconds, as a debug hack.""" time.sleep(100) return {} # Misc utilities. MiB: Final = 2 ** 20 def get_meminfo() -> Dict[str, Any]: res: Dict[str, Any] = {} try: import psutil # type: ignore # It's not in typeshed yet except ImportError: res['memory_psutil_missing'] = ( 'psutil not found, run pip install mypy[dmypy] ' 'to install the needed components for dmypy' ) else: process = psutil.Process() meminfo = process.memory_info() res['memory_rss_mib'] = meminfo.rss / MiB res['memory_vms_mib'] = meminfo.vms / MiB if sys.platform == 'win32': res['memory_maxrss_mib'] = meminfo.peak_wset / MiB else: # See https://stackoverflow.com/questions/938733/total-memory-used-by-python-process import resource # Since it doesn't exist on Windows. rusage = resource.getrusage(resource.RUSAGE_SELF) if sys.platform == 'darwin': factor = 1 else: factor = 1024 # Linux res['memory_maxrss_mib'] = rusage.ru_maxrss * factor / MiB return res def find_all_sources_in_build(graph: mypy.build.Graph, extra: Sequence[BuildSource] = ()) -> List[BuildSource]: result = list(extra) seen = {source.module for source in result} for module, state in graph.items(): if module not in seen: result.append(BuildSource(state.path, module)) return result def fix_module_deps(graph: mypy.build.Graph) -> None: """After an incremental update, update module dependencies to reflect the new state. This can make some suppressed dependencies non-suppressed, and vice versa (if modules have been added to or removed from the build). """ for module, state in graph.items(): new_suppressed = [] new_dependencies = [] for dep in state.dependencies + state.suppressed: if dep in graph: new_dependencies.append(dep) else: new_suppressed.append(dep) state.dependencies = new_dependencies state.dependencies_set = set(new_dependencies) state.suppressed = new_suppressed state.suppressed_set = set(new_suppressed) def filter_out_missing_top_level_packages(packages: Set[str], search_paths: SearchPaths, fscache: FileSystemCache) -> Set[str]: """Quickly filter out obviously missing top-level packages. Return packages with entries that can't be found removed. This is approximate: some packages that aren't actually valid may be included. However, all potentially valid packages must be returned. """ # Start with a empty set and add all potential top-level packages. found = set() paths = ( search_paths.python_path + search_paths.mypy_path + search_paths.package_path + search_paths.typeshed_path ) paths += tuple(os.path.join(p, '@python2') for p in search_paths.typeshed_path) for p in paths: try: entries = fscache.listdir(p) except Exception: entries = [] for entry in entries: # The code is hand-optimized for mypyc since this may be somewhat # performance-critical. if entry.endswith('.py'): entry = entry[:-3] elif entry.endswith('.pyi'): entry = entry[:-4] elif entry.endswith('-stubs'): # Possible PEP 561 stub package entry = entry[:-6] if entry.endswith('-python2'): entry = entry[:-8] if entry in packages: found.add(entry) return found