add trust remote code param for dataset with scripts

app_leaderboard.py

@@ -21,7 +21,7 @@ def get_records_from_dataset_repo(dataset_id):
     logger.info(f"Dataset {dataset_id} has splits {dataset_split}")
 
     try:
-        ds = datasets.load_dataset(dataset_id, dataset_config[0])[dataset_split[0]]
+        ds = datasets.load_dataset(dataset_id, dataset_config[0], split=dataset_split[0])
         df = ds.to_pandas()
         return df
     except Exception as e:

fetch_utils.py

@@ -14,7 +14,7 @@ def check_dataset_and_get_config(dataset_id):
 
 def check_dataset_and_get_split(dataset_id, dataset_config):
     try:
-        ds = datasets.load_dataset(dataset_id, dataset_config)
+        ds = datasets.load_dataset(dataset_id, dataset_config, trust_remote_code=True)
     except Exception as e:
         # Dataset may not exist
         logging.warning(

text_classification.py

@@ -254,7 +254,7 @@ def infer_output_label_column(
 
 def check_dataset_features_validity(d_id, config, split):
     # We assume dataset is ok here
-    ds = datasets.load_dataset(d_id, config)[split]
+    ds = datasets.load_dataset(d_id, config, split=split, trust_remote_code=True)
     try:
         dataset_features = ds.features
     except AttributeError:
@@ -278,7 +278,7 @@ def get_example_prediction(model_id, dataset_id, dataset_config, dataset_split):
     prediction_result = None
     try:
         # Use the first item to test prediction
-        ds = datasets.load_dataset(dataset_id, dataset_config)[dataset_split]
+        ds = datasets.load_dataset(dataset_id, dataset_config, split=dataset_split, trust_remote_code=True)
         if "text" not in ds.features.keys():
             # Dataset does not have text column
             prediction_input = ds[0][select_the_first_string_column(ds)]

text_classification_ui_helpers.py

@@ -59,11 +59,9 @@ def check_dataset(dataset_id):
                 gr.update(),
                 ""
             )
-        splits = list(
-                    datasets.load_dataset(
-                        dataset_id, configs[0]
-                    ).keys()
-                )
+        splits = datasets.get_dataset_split_names(
+                        dataset_id, configs[0], trust_remote_code=True
+                    )
         return (
             gr.update(choices=configs, value=configs[0], visible=True),
             gr.update(choices=splits, value=splits[0], visible=True),
@@ -176,7 +174,7 @@ def precheck_model_ds_enable_example_btn(
         return (gr.update(), gr.update(), "")
     
     try:
-        ds = datasets.load_dataset(dataset_id, dataset_config)
+        ds = datasets.load_dataset(dataset_id, dataset_config, trust_remote_code=True)
         df: pd.DataFrame = ds[dataset_split].to_pandas().head(5)
         ds_labels, ds_features = get_labels_and_features_from_dataset(ds[dataset_split])
 
@@ -243,7 +241,7 @@ def align_columns_and_show_prediction(
 
     model_labels = list(prediction_response.keys())
     
-    ds = datasets.load_dataset(dataset_id, dataset_config)[dataset_split]
+    ds = datasets.load_dataset(dataset_id, dataset_config, split=dataset_split, trust_remote_code=True)
     ds_labels, ds_features = get_labels_and_features_from_dataset(ds)
 
     # when dataset does not have labels or features
@@ -334,7 +332,7 @@ def try_submit(m_id, d_id, config, split, inference, inference_token, uid):
     check_column_mapping_keys_validity(all_mappings)
 
     # get ds labels and features again for alignment
-    ds = datasets.load_dataset(d_id, config)[split]
+    ds = datasets.load_dataset(d_id, config, split=split, trust_remote_code=True)
     ds_labels, ds_features = get_labels_and_features_from_dataset(ds)
     label_mapping, feature_mapping = construct_label_and_feature_mapping(all_mappings, ds_labels, ds_features)