Properly track module_hidden and module_public for incomplete symbols (#8450)

This fixes some crash bugs involving import * from an import cycle.

Author: msullivan

mypy/fixup.py

@@ -22,7 +22,7 @@
 def fixup_module(tree: MypyFile, modules: Dict[str, MypyFile],
                  allow_missing: bool) -> None:
     node_fixer = NodeFixer(modules, allow_missing)
-    node_fixer.visit_symbol_table(tree.names)
+    node_fixer.visit_symbol_table(tree.names, tree.fullname)
 
 
 # TODO: Fix up .info when deserializing, i.e. much earlier.
@@ -42,7 +42,7 @@ def visit_type_info(self, info: TypeInfo) -> None:
             if info.defn:
                 info.defn.accept(self)
             if info.names:
-                self.visit_symbol_table(info.names)
+                self.visit_symbol_table(info.names, info.fullname)
             if info.bases:
                 for base in info.bases:
                     base.accept(self.type_fixer)
@@ -64,7 +64,7 @@ def visit_type_info(self, info: TypeInfo) -> None:
             self.current_info = save_info
 
     # NOTE: This method *definitely* isn't part of the NodeVisitor API.
-    def visit_symbol_table(self, symtab: SymbolTable) -> None:
+    def visit_symbol_table(self, symtab: SymbolTable, table_fullname: str) -> None:
         # Copy the items because we may mutate symtab.
         for key, value in list(symtab.items()):
             cross_ref = value.cross_ref
@@ -76,7 +76,7 @@ def visit_symbol_table(self, symtab: SymbolTable) -> None:
                     stnode = lookup_qualified_stnode(self.modules, cross_ref,
                                                      self.allow_missing)
                     if stnode is not None:
-                        assert stnode.node is not None
+                        assert stnode.node is not None, (table_fullname + "." + key, cross_ref)
                         value.node = stnode.node
                     elif not self.allow_missing:
                         assert False, "Could not find cross-ref %s" % (cross_ref,)

mypy/nodes.py

@@ -3030,6 +3030,7 @@ def serialize(self, prefix: str, name: str) -> JsonDict:
                         and fullname != prefix + '.' + name
                         and not (isinstance(self.node, Var)
                                  and self.node.from_module_getattr)):
+                    assert not isinstance(self.node, PlaceholderNode)
                     data['cross_ref'] = fullname
                     return data
             data['node'] = self.node.serialize()

mypy/semanal.py

@@ -1742,13 +1742,25 @@ def process_imported_symbol(self,
                                 fullname: str,
                                 context: ImportBase) -> None:
         imported_id = as_id or id
+        # 'from m import x as x' exports x in a stub file or when implicit
+        # re-exports are disabled.
+        module_public = (
+            not self.is_stub_file
+            and self.options.implicit_reexport
+            or as_id is not None
+        )
+        module_hidden = not module_public and fullname not in self.modules
+
         if isinstance(node.node, PlaceholderNode):
             if self.final_iteration:
                 self.report_missing_module_attribute(module_id, id, imported_id, context)
                 return
             else:
                 # This might become a type.
-                self.mark_incomplete(imported_id, node.node, becomes_typeinfo=True)
+                self.mark_incomplete(imported_id, node.node,
+                                     module_public=module_public,
+                                     module_hidden=module_hidden,
+                                     becomes_typeinfo=True)
         existing_symbol = self.globals.get(imported_id)
         if (existing_symbol and not isinstance(existing_symbol.node, PlaceholderNode) and
                 not isinstance(node.node, PlaceholderNode)):
@@ -1760,14 +1772,6 @@ def process_imported_symbol(self,
             # Imports are special, some redefinitions are allowed, so wait until
             # we know what is the new symbol node.
             return
-        # 'from m import x as x' exports x in a stub file or when implicit
-        # re-exports are disabled.
-        module_public = (
-            not self.is_stub_file
-            and self.options.implicit_reexport
-            or as_id is not None
-        )
-        module_hidden = not module_public and fullname not in self.modules
         # NOTE: we take the original node even for final `Var`s. This is to support
         # a common pattern when constants are re-exported (same applies to import *).
         self.add_imported_symbol(imported_id, node, context,
@@ -1866,6 +1870,7 @@ def visit_import_all(self, i: ImportAll) -> None:
                     self.add_imported_symbol(name, node, i,
                                              module_public=module_public,
                                              module_hidden=not module_public)
+
         else:
             # Don't add any dummy symbols for 'from x import *' if 'x' is unknown.
             pass
@@ -4338,6 +4343,7 @@ def add_imported_symbol(self,
                             module_public: bool = True,
                             module_hidden: bool = False) -> None:
         """Add an alias to an existing symbol through import."""
+        assert not module_hidden or not module_public
         symbol = SymbolTableNode(node.kind, node.node,
                                  module_public=module_public,
                                  module_hidden=module_hidden)
@@ -4421,7 +4427,9 @@ def record_incomplete_ref(self) -> None:
         self.num_incomplete_refs += 1
 
     def mark_incomplete(self, name: str, node: Node,
-                        becomes_typeinfo: bool = False) -> None:
+                        becomes_typeinfo: bool = False,
+                        module_public: bool = True,
+                        module_hidden: bool = False) -> None:
         """Mark a definition as incomplete (and defer current analysis target).
 
         Also potentially mark the current namespace as incomplete.
@@ -4440,7 +4448,9 @@ def mark_incomplete(self, name: str, node: Node,
             assert self.statement
             placeholder = PlaceholderNode(fullname, node, self.statement.line,
                                           becomes_typeinfo=becomes_typeinfo)
-            self.add_symbol(name, placeholder, context=dummy_context())
+            self.add_symbol(name, placeholder,
+                            module_public=module_public, module_hidden=module_hidden,
+                            context=dummy_context())
         self.missing_names.add(name)
 
     def is_incomplete_namespace(self, fullname: str) -> bool:

test-data/unit/check-incremental.test

@@ -5361,3 +5361,26 @@ reveal_type(z)
 tmp/c.py:2: note: Revealed type is 'a.A'
 [out2]
 tmp/c.py:2: note: Revealed type is 'a.<subclass of "A" and "B">'
+
+[case testStubFixupIssues]
+import a
+[file a.py]
+import p
+[file a.py.2]
+import p
+p.N
+
+[file p/__init__.pyi]
+from p.util import *
+
+[file p/util.pyi]
+from p.params import N
+class Test: ...
+x: N
+
+[file p/params.pyi]
+import p.util
+class N(p.util.Test):
+    ...
+[out2]
+tmp/a.py:2: error: "object" has no attribute "N"

test-data/unit/fine-grained.test

@@ -9572,3 +9572,25 @@ c.py:2: note: Revealed type is 'a.A'
 ==
 c.py:2: note: Revealed type is 'a.<subclass of "A" and "B">'
 
+[case testStubFixupIssues]
+[file a.py]
+import p
+[file a.py.2]
+import p
+# a change
+
+[file p/__init__.pyi]
+from p.util import *
+
+[file p/util.pyi]
+from p.params import N
+class Test: ...
+
+[file p/params.pyi]
+import p.util
+class N(p.util.Test):
+    ...
+
+[builtins fixtures/list.pyi]
+[out]
+==