Repository: zulip/zulip

Issue:
slack import: Support importing custom profile fields
We don't currently support custom profile fields.  I think @rheaparekh is working on this, so pre-assigning it to her.


Discussion:

Resolving patch:
diff --git a/zerver/lib/slack_data_to_zulip_data.py b/zerver/lib/slack_data_to_zulip_data.py
--- a/zerver/lib/slack_data_to_zulip_data.py
+++ b/zerver/lib/slack_data_to_zulip_data.py
@@ -71,8 +71,9 @@ def slack_workspace_to_realm(domain_name: str, realm_id: int, user_list: List[Ze
                  zerver_useractivityinterval=[],
                  zerver_realmfilter=[])
 
-    zerver_userprofile, avatars, added_users = users_to_zerver_userprofile(
-        slack_data_dir, user_list, realm_id, int(NOW), domain_name)
+    zerver_userprofile, avatars, added_users, zerver_customprofilefield, \
+        zerver_customprofilefield_value = users_to_zerver_userprofile(slack_data_dir, user_list,
+                                                                      realm_id, int(NOW), domain_name)
     channels_to_zerver_stream_fields = channels_to_zerver_stream(slack_data_dir,
                                                                  realm_id,
                                                                  added_users,
@@ -84,6 +85,10 @@ def slack_workspace_to_realm(domain_name: str, realm_id: int, user_list: List[Ze
     # for documentation on zerver_defaultstream
     realm['zerver_userprofile'] = zerver_userprofile
 
+    # Custom profile fields
+    realm['zerver_customprofilefield'] = zerver_customprofilefield
+    realm['zerver_customprofilefield_value'] = zerver_customprofilefield_value
+
     realm['zerver_defaultstream'] = channels_to_zerver_stream_fields[0]
     realm['zerver_stream'] = channels_to_zerver_stream_fields[1]
     realm['zerver_subscription'] = channels_to_zerver_stream_fields[3]
@@ -130,23 +135,44 @@ def build_realmemoji(custom_emoji_list: ZerverFieldsT,
 def users_to_zerver_userprofile(slack_data_dir: str, users: List[ZerverFieldsT], realm_id: int,
                                 timestamp: Any, domain_name: str) -> Tuple[List[ZerverFieldsT],
                                                                            List[ZerverFieldsT],
-                                                                           AddedUsersT]:
+                                                                           AddedUsersT,
+                                                                           List[ZerverFieldsT],
+                                                                           List[ZerverFieldsT]]:
     """
     Returns:
     1. zerver_userprofile, which is a list of user profile
     2. avatar_list, which is list to map avatars to zulip avatard records.json
     3. added_users, which is a dictionary to map from slack user id to zulip
        user id
+    4. zerver_customprofilefield, which is a list of all custom profile fields
+    5. zerver_customprofilefield_values, which is a list of user profile fields
     """
     logging.info('######### IMPORTING USERS STARTED #########\n')
     zerver_userprofile = []
+    zerver_customprofilefield = []  # type: List[ZerverFieldsT]
+    zerver_customprofilefield_values = []  # type: List[ZerverFieldsT]
     avatar_list = []  # type: List[ZerverFieldsT]
     added_users = {}
 
+    # The user data we get from the slack api does not contain custom profile data
+    # Hence we get it from the slack zip file
+    slack_data_file_user_list = get_data_file(slack_data_dir + '/users.json')
+
+    # To map user id with the custom profile fields of the corresponding user
+    slack_user_custom_field_map = {}
+    # To store custom fields corresponding to their ids
+    custom_field_map = {}  # type: ZerverFieldsT
+
+    for user in slack_data_file_user_list:
+        if 'fields' in user['profile']:
+            # Make sure the content of fields is not 'None'
+            if user['profile']['fields']:
+                slack_user_custom_field_map[user['id']] = user['profile']['fields']
+
     # We have only one primary owner in slack, see link
     # https://get.slack.help/hc/en-us/articles/201912948-Owners-and-Administrators
     # This is to import the primary owner first from all the users
-    user_id_count = 0
+    user_id_count = custom_field_id_count = customprofilefield_id = 0
     primary_owner_id = user_id_count
     user_id_count += 1
 
@@ -174,6 +200,17 @@ def users_to_zerver_userprofile(slack_data_dir: str, users: List[ZerverFieldsT],
         # timezone
         timezone = get_user_timezone(user)
 
+        # Check for custom profile fields
+        if slack_user_id in slack_user_custom_field_map:
+            # For processing the fields
+            custom_field_map, customprofilefield_id = build_customprofile_field(
+                zerver_customprofilefield, slack_user_custom_field_map[slack_user_id],
+                customprofilefield_id, realm_id, custom_field_map)
+            # Store the custom field values for the corresponding user
+            custom_field_id_count = build_customprofilefields_values(
+                custom_field_map, slack_user_custom_field_map[slack_user_id], user_id,
+                custom_field_id_count, zerver_customprofilefield_values)
+
         userprofile = dict(
             enable_desktop_notifications=DESKTOP_NOTIFICATION,
             is_staff=False,  # 'staff' is for server administrators, which don't exist in Slack.
@@ -238,8 +275,54 @@ def users_to_zerver_userprofile(slack_data_dir: str, users: List[ZerverFieldsT],
             user_id_count += 1
 
         logging.info(u"{} -> {}".format(user['name'], userprofile['email']))
+
+    process_customprofilefields(zerver_customprofilefield, zerver_customprofilefield_values)
     logging.info('######### IMPORTING USERS FINISHED #########\n')
-    return zerver_userprofile, avatar_list, added_users
+    return zerver_userprofile, avatar_list, added_users, zerver_customprofilefield, \
+        zerver_customprofilefield_values
+
+def build_customprofile_field(customprofile_field: List[ZerverFieldsT], fields: ZerverFieldsT,
+                              customprofilefield_id: int, realm_id: int,
+                              custom_field_map: ZerverFieldsT) -> Tuple[ZerverFieldsT, int]:
+    # The name of the custom profile field is not provided in the slack data
+    # Hash keys of the fields are provided
+    # Reference: https://api.slack.com/methods/users.profile.set
+    for field, value in fields.items():
+        if field not in custom_field_map:
+            field_name = ("slack custom field %s" % str(customprofilefield_id + 1))
+            customprofilefield = dict(
+                id=customprofilefield_id,
+                realm=realm_id,
+                name=field_name,
+                field_type=1  # For now this is defaulted to 'SHORT_TEXT'
+                              # Processing is done in the function 'process_customprofilefields'
+            )
+            custom_field_map[field] = customprofilefield_id
+            customprofilefield_id += 1
+            customprofile_field.append(customprofilefield)
+    return custom_field_map, customprofilefield_id
+
+def build_customprofilefields_values(custom_field_map: ZerverFieldsT, fields: ZerverFieldsT,
+                                     user_id: int, custom_field_id: int,
+                                     custom_field_values: List[ZerverFieldsT]) -> int:
+    for field, value in fields.items():
+        custom_field_value = dict(
+            id=custom_field_id,
+            user_profile=user_id,
+            field=custom_field_map[field],
+            value=value['value'])
+        custom_field_values.append(custom_field_value)
+        custom_field_id += 1
+    return custom_field_id
+
+def process_customprofilefields(customprofilefield: List[ZerverFieldsT],
+                                customprofilefield_value: List[ZerverFieldsT]) -> None:
+    # Process the field types by checking all field values
+    for field in customprofilefield:
+        for field_value in customprofilefield_value:
+            if field_value['field'] == field['id'] and len(field_value['value']) > 50:
+                field['field_type'] = 2  # corresponding to Long Text
+                break
 
 def get_user_email(user: ZerverFieldsT, domain_name: str) -> str:
     if 'email' in user['profile']:


Test patch:
diff --git a/zerver/tests/test_slack_importer.py b/zerver/tests/test_slack_importer.py
--- a/zerver/tests/test_slack_importer.py
+++ b/zerver/tests/test_slack_importer.py
@@ -145,13 +145,19 @@ def test_get_timezone(self) -> None:
         self.assertEqual(get_user_timezone(user_timezone_none), "America/New_York")
         self.assertEqual(get_user_timezone(user_no_timezone), "America/New_York")
 
-    def test_users_to_zerver_userprofile(self) -> None:
+    @mock.patch("zerver.lib.slack_data_to_zulip_data.get_data_file")
+    def test_users_to_zerver_userprofile(self, mock_get_data_file: mock.Mock) -> None:
+        custom_profile_field_user1 = {"Xf06054BBB": {"value": "random1"},
+                                      "Xf023DSCdd": {"value": "employee"}}
+        custom_profile_field_user2 = {"Xf06054BBB": {"value": "random2"},
+                                      "Xf023DSCdd": {"value": "employer"}}
         user_data = [{"id": "U08RGD1RD",
                       "team_id": "T5YFFM2QY",
                       "name": "john",
                       "deleted": False,
                       "real_name": "John Doe",
-                      "profile": {"image_32": "", "email": "jon@gmail.com", "avatar_hash": "hash"}},
+                      "profile": {"image_32": "", "email": "jon@gmail.com", "avatar_hash": "hash",
+                                  "fields": custom_profile_field_user1}},
                      {"id": "U0CBK5KAT",
                       "team_id": "T5YFFM2QY",
                       "is_admin": True,
@@ -162,6 +168,7 @@ def test_users_to_zerver_userprofile(self) -> None:
                       "real_name": "Jane Doe",
                       "deleted": False,
                       "profile": {"image_32": "https:\/\/secure.gravatar.com\/avatar\/random.png",
+                                  "fields": custom_profile_field_user2,
                                   "email": "jane@foo.com", "avatar_hash": "hash"}},
                      {"id": "U09TYF5Sk",
                       "team_id": "T5YFFM2QY",
@@ -172,6 +179,7 @@ def test_users_to_zerver_userprofile(self) -> None:
                       "profile": {"image_32": "https:\/\/secure.gravatar.com\/avatar\/random1.png",
                                   "email": "bot1@zulipchat.com", "avatar_hash": "hash"}}]
 
+        mock_get_data_file.return_value = user_data
         # As user with slack_id 'U0CBK5KAT' is the primary owner, that user should be imported first
         # and hence has zulip_id = 1
         test_added_users = {'U08RGD1RD': 1,
@@ -179,8 +187,19 @@ def test_users_to_zerver_userprofile(self) -> None:
                             'U09TYF5Sk': 2}
         slack_data_dir = './random_path'
         timestamp = int(timezone_now().timestamp())
-        zerver_userprofile, avatar_list, added_users = users_to_zerver_userprofile(
-            slack_data_dir, user_data, 1, timestamp, 'test_domain')
+        mock_get_data_file.return_value = user_data
+        zerver_userprofile, avatar_list, added_users, customprofilefield, \
+            customprofilefield_value = users_to_zerver_userprofile(slack_data_dir, user_data, 1,
+                                                                   timestamp, 'test_domain')
+
+        # Test custom profile fields
+        self.assertEqual(customprofilefield[0]['field_type'], 1)
+        self.assertEqual(customprofilefield[1]['name'], 'slack custom field 2')
+
+        self.assertEqual(len(customprofilefield_value), 4)
+        self.assertEqual(customprofilefield_value[0]['field'], 0)
+        self.assertEqual(customprofilefield_value[0]['user_profile'], 1)
+        self.assertEqual(customprofilefield_value[2]['user_profile'], 0)
 
         # test that the primary owner should always be imported first
         self.assertDictEqual(added_users, test_added_users)
@@ -316,7 +335,7 @@ def test_channels_to_zerver_stream(self, mock_get_data_file: mock.Mock) -> None:
 
     @mock.patch("zerver.lib.slack_data_to_zulip_data.build_zerver_realm", return_value=[{}])
     @mock.patch("zerver.lib.slack_data_to_zulip_data.users_to_zerver_userprofile",
-                return_value=[[], [], {}])
+                return_value=[[], [], {}, [], []])
     @mock.patch("zerver.lib.slack_data_to_zulip_data.channels_to_zerver_stream",
                 return_value=[[], [], {}, [], [], {}])
     def test_slack_workspace_to_realm(self, mock_channels_to_zerver_stream: mock.Mock,