Be more careful with caching local partial types

Author: ilevkivskyi

mypy/build.py

@@ -898,7 +898,6 @@ def __init__(
             self.errors,
             self.plugin,
             self.import_map,
-            parallel_worker,
         )
         self.all_types: dict[Expression, Type] = {}  # Enabled by export_types
         self.indirection_detector = TypeIndirectionVisitor()
@@ -2056,14 +2055,26 @@ def find_cache_meta(
 
     # Ignore cache if (relevant) options aren't the same.
     # Note that it's fine to mutilate cached_options since it's only used here.
-    cached_options = m.options
+    cached_options = m.options.copy()
     current_options = options_snapshot(id, manager)
     if manager.options.skip_version_check:
         # When we're lax about version we're also lax about platform.
         cached_options["platform"] = current_options["platform"]
     if "debug_cache" in cached_options:
         # Older versions included debug_cache, but it's silly to compare it.
         del cached_options["debug_cache"]
+    if "local_partial_types" in cached_options:
+        local_partial_types = cached_options["local_partial_types"]
+        del cached_options["local_partial_types"]
+        is_parallel = manager.options.num_workers > 0
+        if not local_partial_types and cached_options["is_parallel"] != is_parallel:
+            # If local partial types are disabled, behavior is too different for
+            # parallel and sequential runs, see write_cache for details.
+            return None
+        del cached_options["is_parallel"]
+    else:
+        # Cache from an old mypy version.
+        return None
     if cached_options != current_options:
         manager.log(f"Metadata abandoned for {id}: options differ")
         if manager.options.verbosity >= 2:
@@ -2227,7 +2238,6 @@ def validate_meta(
             meta.mtime = mtime
             meta.path = path
             meta.size = size
-            meta.options = options_snapshot(id, manager)
             meta_file, _, _ = get_cache_names(id, path, manager.options)
             if manager.logging_enabled:
                 manager.log(
@@ -2272,6 +2282,8 @@ def write_cache(
     trans_dep_hash: bytes,
     source_hash: str,
     ignore_all: bool,
+    local_partial_types: bool,
+    is_parallel: bool,
     manager: BuildManager,
 ) -> tuple[bytes, tuple[CacheMeta, str] | None]:
     """Write cache files for a module.
@@ -2371,6 +2383,16 @@ def write_cache(
     # important, or otherwise the options would never match when
     # verifying the cache.
     assert source_hash is not None
+    # Local partial types need special handling as they behave differently
+    # in sequential and parallel runs:
+    #   * In sequential run we respected the option, but use different SCC
+    #   processing logic depending on whether they are enabled or disabled.
+    #   * In parallel run they are always on, and we give an error if a user
+    #   tries to disable them.
+    extra_options = {
+        "local_partial_types": local_partial_types,
+        "is_parallel": is_parallel,
+    }
     meta = CacheMeta(
         id=id,
         path=path,
@@ -2382,7 +2404,7 @@ def write_cache(
         data_file=data_file,
         suppressed=suppressed,
         imports_ignored=imports_ignored,
-        options=options_snapshot(id, manager),
+        options=options_snapshot(id, manager) | extra_options,
         suppressed_deps_opts=suppressed_deps_opts,
         dep_prios=dep_prios,
         dep_lines=dep_lines,
@@ -2727,6 +2749,11 @@ def new_state(
                 meta, meta_ex = meta_pair
                 interface_hash = meta.interface_hash
                 meta_source_hash = meta.hash
+                # Update the local partial types in case they were set by an inline config.
+                # So we can select the correct SCC processing logic without reading the file.
+                local_partial_types = meta.options["local_partial_types"]
+                if options.local_partial_types != local_partial_types:
+                    options = options.apply_changes({"local_partial_types": local_partial_types})
         if path and source is None and manager.fscache.isdir(path):
             source = ""
 
@@ -3637,6 +3664,8 @@ def write_cache(self) -> tuple[CacheMeta, str] | None:
             self.trans_dep_hash,
             self.source_hash,
             self.ignore_all,
+            self.options.local_partial_types,
+            self.options.num_workers > 0,
             self.manager,
         )
         if new_interface_hash == self.interface_hash:

mypy/semanal.py

@@ -461,7 +461,6 @@ def __init__(
         errors: Errors,
         plugin: Plugin,
         import_map: dict[str, set[str]],
-        parallel_worker: bool,
     ) -> None:
         """Construct semantic analyzer.
 
@@ -495,8 +494,6 @@ def __init__(
         self.errors = errors
         self.modules = modules
         self.import_map = import_map
-        # True if this analysis is run in a parallel worker process.
-        self.parallel_worker = parallel_worker
         self.msg = MessageBuilder(errors, modules)
         self.missing_modules = missing_modules
         self.missing_names = [set()]
@@ -740,7 +737,7 @@ def refresh_top_level(self, file_node: MypyFile) -> None:
             self.ad_hoc_error(
                 "--allow-redefinition-old and --allow-redefinition should not be used together"
             )
-        if not self.options.local_partial_types and self.parallel_worker:
+        if not self.options.local_partial_types and self.options.num_workers > 0:
             self.ad_hoc_error("--local-partial-types must be enabled in parallel mode")
         self.recurse_into_functions = False
         self.add_implicit_module_attrs(file_node)

mypy/test/testcmdline.py

@@ -58,7 +58,7 @@ def test_python_cmdline(testcase: DataDrivenTestCase, step: int) -> None:
     with open(program_path, "w", encoding="utf8") as file:
         for s in testcase.input:
             file.write(f"{s}\n")
-    args = parse_args(normalize_devnull(testcase.input[0]))
+    args = parse_args(normalize_devnull(testcase.input[step - 1]), step)
     custom_cwd = parse_cwd(testcase.input[1]) if len(testcase.input) > 1 else None
     args.append("--show-traceback")
     if "--error-summary" not in args:
@@ -122,7 +122,7 @@ def test_python_cmdline(testcase: DataDrivenTestCase, step: int) -> None:
         )
 
 
-def parse_args(line: str) -> list[str]:
+def parse_args(line: str, step: int) -> list[str]:
     """Parse the first line of the program for the command line.
 
     This should have the form
@@ -133,7 +133,8 @@ def parse_args(line: str) -> list[str]:
 
       # cmd: mypy pkg/
     """
-    m = re.match("# cmd: mypy (.*)$", line)
+    step_str = "" if step == 1 else str(step)
+    m = re.match(f"# cmd{step_str}: mypy (.*)$", line)
     if not m:
         return []  # No args; mypy will spit out an error.
     return shlex.split(m.group(1))

test-data/unit/check-incremental.test

@@ -8251,3 +8251,27 @@ y = 2
 [stale]
 [out2]
 [out3]
+
+[case testNoLocalPartialTypesInlineRespectedIncremental_no_parallel]
+import a
+[file a.py]
+# mypy: no-local-partial-types
+import dep
+
+items = []
+
+def add() -> None:
+    items.append(1)
+
+reveal_type(items)
+[file dep.py]
+value: int = 1
+[file dep.py.2]
+value: str = "x"
+[builtins fixtures/list.pyi]
+[rechecked a, dep]
+[stale dep]
+[out]
+tmp/a.py:9: note: Revealed type is "builtins.list[builtins.int]"
+[out2]
+tmp/a.py:9: note: Revealed type is "builtins.list[builtins.int]"

test-data/unit/cmdline.test

@@ -1330,3 +1330,23 @@ error: Cache must be enabled in parallel mode
 from foo.api import bar as bar
 [file foo-stubs/api/bar.pyi]
 [out]
+
+[case testCacheInvalidatedForNoLocalPartialTypesParallelVsSequential]
+# cmd: mypy a.py
+# cmd2: mypy a.py --num-workers=2
+[file a.py]
+# mypy: no-local-partial-types
+
+items = []
+
+def add() -> None:
+    items.append(1)
+
+reveal_type(items)
+[out]
+a.py:8: note: Revealed type is "list[int]"
+== Return code: 0
+[out2]
+a.py:1: error: --local-partial-types must be enabled in parallel mode
+a.py:3: error: Need type annotation for "items" (hint: "items: list[<type>] = ...")
+a.py:8: note: Revealed type is "list[Any]"

test-data/unit/exportjson.test

@@ -298,6 +298,8 @@ from typing_extensions import Final
     ],
     "suppressed": [],
     "options": {
+        "is_parallel": false,
+        "local_partial_types": true,
         "other_options": "<hash>",
         "platform": ...
     },