Repository: edgedb/edgedb

Issue:
Unable to create migration altering a property from required to optional
<!-- Please search existing issues to avoid creating duplicates. -->
- EdgeDB Version: EdgeDB CLI 1.1.0+8bc2c20
- OS Version: MacOS Catalina 10.15.7

Steps to Reproduce:

1. Create a new EdgeDB project (version 1.0) with `default` module as below:
```
module default {
    abstract type Person {
        required property name -> str;
    }

    type PC extending Person {
    }
}
```
2. Create and apply migration
```
user@local> edgedb migration create
did you create object type 'default::Person'? [y,n,l,c,b,s,q,?]
> y
did you create object type 'default::PC'? [y,n,l,c,b,s,q,?]
> y
Created ./dbschema/migrations/00001.edgeql, id: m1clwlsejlohyzkgrc234l4etgml2mnq6ttpfvvpaykmrfujj5lzaq

user@local> edgedb migrate
Applied m1clwlsejlohyzkgrc234l4etgml2mnq6ttpfvvpaykmrfujj5lzaq (00001.edgeql)
Note: adding first migration disables DDL. More info: https://edgedb.com/p/bare_ddl

```
3. Remove `required` from `default::Person::name` property and create new migration.
```
user@local> edgedb migration create
did you make property 'name' of object type 'default::Person' optional? [y,n,l,c,b,s,q,?]
> y
edgedb error: EdgeDB could not resolve migration with the provided answers. Please retry with different answers.

```

As you see from the terminal output above, the migration to make property `default::Person::name` optional could not be created. The error message is quite confusing `EdgeDB could not resolve migration with the provided answers. Please retry with different answers`.


How can I fix it?



Discussion:
It's a bug, we are looking into it.  A workaround for now is to declare the `name` property as `optional` explicitly:


```
    abstract type Person {
        optional property name -> str;
    }
```

Resolving patch:
diff --git a/edb/schema/inheriting.py b/edb/schema/inheriting.py
--- a/edb/schema/inheriting.py
+++ b/edb/schema/inheriting.py
@@ -78,9 +78,16 @@ def inherit_fields(
         fields: Optional[Iterable[str]] = None,
         ignore_local: bool = False,
     ) -> s_schema.Schema:
+        from . import referencing as s_referencing
+
         mcls = self.get_schema_metaclass()
         scls = self.scls
 
+        is_owned = (
+            isinstance(scls, s_referencing.ReferencedObject)
+            and scls.get_owned(schema)
+        )
+
         field_names: Iterable[str]
         if fields is not None:
             field_names = set(scls.inheritable_fields()) & set(fields)
@@ -92,7 +99,8 @@ def inherit_fields(
         deferred_complex_ops = []
 
         for field_name in field_names:
-            ignore_local_field = ignore_local or field_name in inherited_fields
+            was_inherited = field_name in inherited_fields
+            ignore_local_field = ignore_local or was_inherited
             field = mcls.get_field(field_name)
 
             try:
@@ -118,8 +126,9 @@ def inherit_fields(
 
             if (
                 (
-                    (result is not None or ours is not None)
-                    and (result != ours or inherited)
+                    result != ours
+                    or inherited
+                    or (was_inherited and not is_owned)
                 ) or (
                     result is None and ours is None and ignore_local
                 )
diff --git a/edb/schema/ordering.py b/edb/schema/ordering.py
--- a/edb/schema/ordering.py
+++ b/edb/schema/ordering.py
@@ -843,7 +843,12 @@ def write_ref_deps(
                     for ref in refs:
                         write_ref_deps(ref, referrer, this_name_str)
 
-                if isinstance(obj, referencing.ReferencedInheritingObject):
+                if (
+                    isinstance(obj, referencing.ReferencedInheritingObject)
+                    # Changes to owned objects can't necessarily be merged
+                    # in with parents, so we make sure not to.
+                    and not obj.get_owned(new_schema)
+                ):
                     implicit_ancestors = [
                         b.get_name(new_schema)
                         for b in obj.get_implicit_ancestors(new_schema)
diff --git a/edb/server/compiler/compiler.py b/edb/server/compiler/compiler.py
--- a/edb/server/compiler/compiler.py
+++ b/edb/server/compiler/compiler.py
@@ -42,6 +42,7 @@
 from edb.common import debug
 from edb.common import verutils
 from edb.common import uuidgen
+from edb.common import ast
 
 from edb.edgeql import ast as qlast
 from edb.edgeql import codegen as qlcodegen
@@ -1097,7 +1098,7 @@ def _compile_ql_migration(
                     proposed_steps = []
 
                     if proposed_ddl:
-                        for ddl_text, ast, top_op in proposed_ddl:
+                        for ddl_text, ddl_ast, top_op in proposed_ddl:
                             # get_ast has a lot of logic for figuring
                             # out when an op is implicit in a parent
                             # op. get_user_prompt does not have any of
@@ -1109,13 +1110,26 @@ def _compile_ql_migration(
                             # *that*.
                             # This is stupid, and it is slow.
                             top_op2 = s_ddl.cmd_from_ddl(
-                                ast,
+                                ddl_ast,
                                 schema=schema,
                                 modaliases=current_tx.get_modaliases(),
                             )
                             prompt_key2, prompt_text = (
                                 top_op2.get_user_prompt())
 
+                            # Similarly, some placeholders may not have made
+                            # it into the actual query, so filter them out.
+                            used_placeholders = {
+                                p.name for p in ast.find_children(
+                                    ddl_ast,
+                                    lambda n: isinstance(n, qlast.Placeholder))
+                            }
+                            required_user_input = tuple(
+                                (k, v) for k, v in (
+                                    top_op.get_required_user_input().items())
+                                if k in used_placeholders
+                            )
+
                             # The prompt_id still needs to come from
                             # the original op, though, since
                             # orig_cmd_class is lost in ddl.
@@ -1131,9 +1145,7 @@ def _compile_ql_migration(
                                 prompt=prompt_text,
                                 prompt_id=prompt_id,
                                 data_safe=top_op.is_data_safe(),
-                                required_user_input=tuple(
-                                    top_op.get_required_user_input().items(),
-                                ),
+                                required_user_input=required_user_input,
                                 operation_key=prompt_key2,
                             )
                             proposed_steps.append(step)


Test patch:
diff --git a/tests/test_edgeql_data_migration.py b/tests/test_edgeql_data_migration.py
--- a/tests/test_edgeql_data_migration.py
+++ b/tests/test_edgeql_data_migration.py
@@ -8785,6 +8785,108 @@ async def test_edgeql_migration_annotation_05(self):
 
         await self.migrate("")
 
+    async def test_edgeql_migration_reset_optional_01(self):
+        await self.migrate(r'''
+            abstract type Person {
+                required property name -> str;
+            }
+
+            type PC extending Person;
+        ''')
+
+        await self.migrate(r'''
+            abstract type Person {
+                property name -> str;
+            }
+
+            type PC extending Person;
+        ''')
+
+        await self.migrate(r'''
+            abstract type Person {
+                required property name -> str;
+            }
+
+            type PC extending Person;
+        ''', user_input=['""'])
+
+    async def test_edgeql_migration_reset_optional_02(self):
+        await self.migrate(r'''
+            abstract type Person {
+                required property name -> str;
+            }
+
+            type PC extending Person {
+                overloaded required property name -> str;
+            }
+        ''')
+
+        await self.migrate(r'''
+            abstract type Person {
+                property name -> str;
+            }
+
+            type PC extending Person {
+                overloaded property name -> str;
+            }
+        ''')
+
+    async def test_edgeql_migration_reset_optional_03(self):
+        await self.migrate(r'''
+            abstract type Person {
+                required property name -> str;
+            }
+
+            type PC extending Person {
+                overloaded required property name -> str;
+            }
+        ''')
+
+        await self.migrate(r'''
+            abstract type Person {
+                optional property name -> str;
+            }
+
+            type PC extending Person {
+                overloaded optional property name -> str;
+            }
+        ''')
+
+    @test.xfail('''
+        This fails because we try to set the parent as required while the
+        child is still optional.
+
+        For this to work, we need to process the *child* first,
+        but in order for the reverse case above to work we need to
+        process the *parent* first.
+
+        I don't know if there is any way to fix this sort of thing without
+        exposing this kind of semantic understanding to ordering.
+
+        Maybe suppressing some kinds of intermediate-state errors during
+        migrations would be OK?
+    ''')
+    async def test_edgeql_migration_reset_optional_04(self):
+        await self.migrate(r'''
+            abstract type Person {
+                optional property name -> str;
+            }
+
+            type PC extending Person {
+                overloaded optional property name -> str;
+            }
+        ''')
+
+        await self.migrate(r'''
+            abstract type Person {
+                required property name -> str;
+            }
+
+            type PC extending Person {
+                overloaded required property name -> str;
+            }
+        ''', user_input=["''", "''"])
+
     async def test_edgeql_migration_invalid_scalar_01(self):
         with self.assertRaisesRegex(
                 edgedb.SchemaError,