Repository: zulip/zulip

Issue:
Improve behavior of deleting an option for a "List of options" custom profile field
As [discussed on CZO](https://chat.zulip.org/#narrow/stream/378-api-design/topic/custom.20profile.20fields.20option.20deletion/near/1384923), when an administrator deletes an option for a "List of options" custom profile field, users have the invalid value in the database while in UI the empty option is selected. Also, we do not show any warning in the UI to the admin user deleting the option. We should:

- [ ] When an option gets deleted, we should delete the field from all the users that had that option chosen.
- [ ] When an admin goes to delete an option, we should first give them a confirm/cancel modal saying that this will delete the field for N users. We should show the modal even when zero users have the option selected.


Discussion:
Hello @zulip/server-settings members, this issue was labeled with the "area: settings (admin/org)" label, so you may want to check it out!

<!-- areaLabelAddition -->

Hi,
Can I take up this issue if nobody is working on it?

Yeah, go for it. The New Application Feature tutorial should give you the background needed to find the code for this.
@ritikBhandari are you working on this?
Actually, no. I am working on another issue and have opened a Draft PR for the same. But I can work on this one if you want.
I was asking as if you are not working then I can take this one, should be quick to do.
Sure.

Resolving patch:
diff --git a/zerver/actions/custom_profile_fields.py b/zerver/actions/custom_profile_fields.py
--- a/zerver/actions/custom_profile_fields.py
+++ b/zerver/actions/custom_profile_fields.py
@@ -78,6 +78,17 @@ def do_remove_realm_custom_profile_fields(realm: Realm) -> None:
     CustomProfileField.objects.filter(realm=realm).delete()
 
 
+def remove_custom_profile_field_value_if_required(
+    field: CustomProfileField, field_data: ProfileFieldData
+) -> None:
+    old_values = set(orjson.loads(field.field_data).keys())
+    new_values = set(field_data.keys())
+    removed_values = old_values - new_values
+
+    if removed_values:
+        CustomProfileFieldValue.objects.filter(field=field, value__in=removed_values).delete()
+
+
 def try_update_realm_custom_profile_field(
     realm: Realm,
     field: CustomProfileField,
@@ -91,6 +102,9 @@ def try_update_realm_custom_profile_field(
         field.field_type == CustomProfileField.SELECT
         or field.field_type == CustomProfileField.EXTERNAL_ACCOUNT
     ):
+        if field.field_type == CustomProfileField.SELECT:
+            assert field_data is not None
+            remove_custom_profile_field_value_if_required(field, field_data)
         field.field_data = orjson.dumps(field_data or {}).decode()
     field.save()
     notify_realm_custom_profile_fields(realm)
diff --git a/zerver/migrations/0397_remove_custom_field_values_for_deleted_options.py b/zerver/migrations/0397_remove_custom_field_values_for_deleted_options.py
new file mode 100644
--- /dev/null
+++ b/zerver/migrations/0397_remove_custom_field_values_for_deleted_options.py
@@ -0,0 +1,35 @@
+# Generated by Django 3.2.13 on 2022-06-17 17:39
+import orjson
+from django.db import migrations
+from django.db.backends.postgresql.schema import BaseDatabaseSchemaEditor
+from django.db.migrations.state import StateApps
+
+
+def remove_custom_field_values_for_deleted_options(
+    apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
+) -> None:
+    SELECT_TYPE = 3
+    CustomProfileField = apps.get_model("zerver", "CustomProfileField")
+    CustomProfileFieldValue = apps.get_model("zerver", "CustomProfileFieldValue")
+
+    select_type_fields = CustomProfileField.objects.filter(field_type=SELECT_TYPE)
+    for field in select_type_fields:
+        field_data = orjson.loads(field.field_data)
+        current_options = list(field_data.keys())
+        CustomProfileFieldValue.objects.filter(field=field).exclude(
+            value__in=current_options
+        ).delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("zerver", "0396_remove_subscription_role"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            remove_custom_field_values_for_deleted_options,
+            elidable=True,
+        ),
+    ]
diff --git a/zilencer/management/commands/populate_db.py b/zilencer/management/commands/populate_db.py
--- a/zilencer/management/commands/populate_db.py
+++ b/zilencer/management/commands/populate_db.py
@@ -726,8 +726,8 @@ def assign_time_zone_by_delivery_email(delivery_email: str, new_time_zone: str)
                 hint="Or drink, if you'd prefer",
             )
             field_data: ProfileFieldData = {
-                "vim": {"text": "Vim", "order": "1"},
-                "emacs": {"text": "Emacs", "order": "2"},
+                "0": {"text": "Vim", "order": "1"},
+                "1": {"text": "Emacs", "order": "2"},
             }
             favorite_editor = try_add_realm_custom_profile_field(
                 zulip_realm, "Favorite editor", CustomProfileField.SELECT, field_data=field_data
@@ -754,7 +754,7 @@ def assign_time_zone_by_delivery_email(delivery_email: str, new_time_zone: str)
                     {"id": phone_number.id, "value": "+1-234-567-8901"},
                     {"id": biography.id, "value": "Betrayer of Othello."},
                     {"id": favorite_food.id, "value": "Apples"},
-                    {"id": favorite_editor.id, "value": "emacs"},
+                    {"id": favorite_editor.id, "value": "1"},
                     {"id": birthday.id, "value": "2000-01-01"},
                     {"id": favorite_website.id, "value": "https://zulip.readthedocs.io/en/latest/"},
                     {"id": mentor.id, "value": [hamlet.id]},
@@ -770,7 +770,7 @@ def assign_time_zone_by_delivery_email(delivery_email: str, new_time_zone: str)
                         "value": "I am:\n* The prince of Denmark\n* Nephew to the usurping Claudius",
                     },
                     {"id": favorite_food.id, "value": "Dark chocolate"},
-                    {"id": favorite_editor.id, "value": "vim"},
+                    {"id": favorite_editor.id, "value": "0"},
                     {"id": birthday.id, "value": "1900-01-01"},
                     {"id": favorite_website.id, "value": "https://blog.zulig.org"},
                     {"id": mentor.id, "value": [iago.id]},


Test patch:
diff --git a/zerver/tests/test_custom_profile_data.py b/zerver/tests/test_custom_profile_data.py
--- a/zerver/tests/test_custom_profile_data.py
+++ b/zerver/tests/test_custom_profile_data.py
@@ -106,8 +106,8 @@ def test_create_select_field(self) -> None:
 
         data["field_data"] = orjson.dumps(
             {
-                "python": {"text": "Python"},
-                "java": {"text": "Java"},
+                "0": {"text": "Python"},
+                "1": {"text": "Java"},
             }
         ).decode()
         result = self.client_post("/json/realm/profile_fields", info=data)
@@ -115,8 +115,8 @@ def test_create_select_field(self) -> None:
 
         data["field_data"] = orjson.dumps(
             {
-                "python": {"text": "Python", "order": ""},
-                "java": {"text": "Java", "order": "2"},
+                "0": {"text": "Python", "order": ""},
+                "1": {"text": "Java", "order": "2"},
             }
         ).decode()
         result = self.client_post("/json/realm/profile_fields", info=data)
@@ -125,7 +125,7 @@ def test_create_select_field(self) -> None:
         data["field_data"] = orjson.dumps(
             {
                 "": {"text": "Python", "order": "1"},
-                "java": {"text": "Java", "order": "2"},
+                "1": {"text": "Java", "order": "2"},
             }
         ).decode()
         result = self.client_post("/json/realm/profile_fields", info=data)
@@ -133,8 +133,8 @@ def test_create_select_field(self) -> None:
 
         data["field_data"] = orjson.dumps(
             {
-                "python": {"text": "Python", "order": 1},
-                "java": {"text": "Java", "order": "2"},
+                "0": {"text": "Python", "order": 1},
+                "1": {"text": "Java", "order": "2"},
             }
         ).decode()
         result = self.client_post("/json/realm/profile_fields", info=data)
@@ -146,8 +146,8 @@ def test_create_select_field(self) -> None:
 
         data["field_data"] = orjson.dumps(
             {
-                "python": {"text": "Duplicate", "order": "1"},
-                "java": {"text": "Duplicate", "order": "2"},
+                "0": {"text": "Duplicate", "order": "1"},
+                "1": {"text": "Duplicate", "order": "2"},
             }
         ).decode()
         result = self.client_post("/json/realm/profile_fields", info=data)
@@ -155,8 +155,8 @@ def test_create_select_field(self) -> None:
 
         data["field_data"] = orjson.dumps(
             {
-                "python": {"text": "Python", "order": "1"},
-                "java": {"text": "Java", "order": "2"},
+                "0": {"text": "Python", "order": "1"},
+                "1": {"text": "Java", "order": "2"},
             }
         ).decode()
         result = self.client_post("/json/realm/profile_fields", info=data)
@@ -497,8 +497,8 @@ def test_update(self) -> None:
 
         field_data = orjson.dumps(
             {
-                "vim": "Vim",
-                "emacs": {"order": "2", "text": "Emacs"},
+                "0": "Vim",
+                "1": {"order": "2", "text": "Emacs"},
             }
         ).decode()
         result = self.client_patch(
@@ -509,9 +509,9 @@ def test_update(self) -> None:
 
         field_data = orjson.dumps(
             {
-                "vim": {"order": "1", "text": "Vim"},
-                "emacs": {"order": "2", "text": "Emacs"},
-                "notepad": {"order": "3", "text": "Notepad"},
+                "0": {"order": "1", "text": "Vim"},
+                "1": {"order": "2", "text": "Emacs"},
+                "2": {"order": "3", "text": "Notepad"},
             }
         ).decode()
         result = self.client_patch(
@@ -594,7 +594,7 @@ def test_update_profile_data_successfully(self) -> None:
             ("Phone number", "*short* text data"),
             ("Biography", "~~short~~ **long** text data"),
             ("Favorite food", "long short text data"),
-            ("Favorite editor", "vim"),
+            ("Favorite editor", "0"),
             ("Birthday", "1909-03-05"),
             ("Favorite website", "https://zulip.com"),
             ("Mentor", [self.example_user("cordelia").id]),
@@ -667,7 +667,7 @@ def test_update_select_field_successfully(self) -> None:
         data = [
             {
                 "id": field.id,
-                "value": "emacs",
+                "value": "1",
             }
         ]
 
@@ -727,6 +727,30 @@ def test_do_update_value_not_changed(self) -> None:
             do_update_user_custom_profile_data_if_changed(iago, data)
             mock_notify.assert_not_called()
 
+    def test_removing_option_from_select_field(self) -> None:
+        self.login("iago")
+        realm = get_realm("zulip")
+        field = CustomProfileField.objects.get(name="Favorite editor", realm=realm)
+        self.assertTrue(
+            CustomProfileFieldValue.objects.filter(field_id=field.id, value="0").exists()
+        )
+        self.assertTrue(
+            CustomProfileFieldValue.objects.filter(field_id=field.id, value="1").exists()
+        )
+
+        new_options = {"1": {"text": "Emacs", "order": "1"}}
+        result = self.client_patch(
+            f"/json/realm/profile_fields/{field.id}",
+            info={"name": "Favorite editor", "field_data": orjson.dumps(new_options).decode()},
+        )
+        self.assert_json_success(result)
+        self.assertFalse(
+            CustomProfileFieldValue.objects.filter(field_id=field.id, value="0").exists()
+        )
+        self.assertTrue(
+            CustomProfileFieldValue.objects.filter(field_id=field.id, value="1").exists()
+        )
+
 
 class ListCustomProfileFieldTest(CustomProfileFieldTestCase):
     def test_list(self) -> None:
diff --git a/zerver/tests/test_users.py b/zerver/tests/test_users.py
--- a/zerver/tests/test_users.py
+++ b/zerver/tests/test_users.py
@@ -608,7 +608,7 @@ def test_admin_user_can_change_profile_data(self) -> None:
             "Phone number": "short text data",
             "Biography": "long text data",
             "Favorite food": "short text data",
-            "Favorite editor": "vim",
+            "Favorite editor": "0",
             "Birthday": "1909-03-05",
             "Favorite website": "https://zulip.com",
             "Mentor": [cordelia.id],