Repository: huggingface/transformers
Pull request: Add trust_remote_code in LightGlueConfig

# What does this PR do?

Add trust_remote_code parameter to LightGlue config so that LightGlue can be used with other not implemented keypoint detectors than SuperPoint.

With `from_pretrained`, `trust_remote_code` is not provided to the config, so I had to include it into the config itself as an attribute.

Discussed on Slack.

## Who can review?

@qubvel 


Changed files:
filename: src/transformers/models/lightglue/configuration_lightglue.py
patch: @@ -65,6 +65,8 @@ class LightGlueConfig(PretrainedConfig):
             The dropout ratio for the attention probabilities.
         attention_bias (`bool`, *optional*, defaults to `True`):
             Whether to use a bias in the query, key, value and output projection layers during self-attention.
+        trust_remote_code (`bool`, *optional*, defaults to `False`):
+            Whether to trust remote code when using other models than SuperPoint as keypoint detector.
 
     Examples:
         ```python
@@ -98,8 +100,14 @@ def __init__(
         hidden_act: str = "gelu",
         attention_dropout=0.0,
         attention_bias=True,
+        trust_remote_code: bool = False,
         **kwargs,
     ):
+        # LightGlue can be used with other models than SuperPoint as keypoint detector
+        # We provide the trust_remote_code argument to allow the use of other models
+        # that are not registered in the CONFIG_MAPPING dictionary (for example DISK)
+        self.trust_remote_code = trust_remote_code
+
         if descriptor_dim % num_attention_heads != 0:
             raise ValueError("descriptor_dim % num_heads is different from zero")
 
@@ -124,9 +132,15 @@ def __init__(
             keypoint_detector_config["model_type"] = (
                 keypoint_detector_config["model_type"] if "model_type" in keypoint_detector_config else "superpoint"
             )
-            keypoint_detector_config = CONFIG_MAPPING[keypoint_detector_config["model_type"]](
-                **keypoint_detector_config, attn_implementation="eager"
-            )
+            if keypoint_detector_config["model_type"] not in CONFIG_MAPPING:
+                keypoint_detector_config = AutoConfig.from_pretrained(
+                    keypoint_detector_config["_name_or_path"], trust_remote_code=self.trust_remote_code
+                )
+            else:
+                keypoint_detector_config = CONFIG_MAPPING[keypoint_detector_config["model_type"]](
+                    **keypoint_detector_config, attn_implementation="eager"
+                )
+
         if keypoint_detector_config is None:
             keypoint_detector_config = CONFIG_MAPPING["superpoint"](attn_implementation="eager")
 

filename: src/transformers/models/lightglue/modeling_lightglue.py
patch: @@ -513,8 +513,9 @@ class LightGlueForKeypointMatching(LightGluePreTrainedModel):
 
     def __init__(self, config: LightGlueConfig):
         super().__init__(config)
-
-        self.keypoint_detector = AutoModelForKeypointDetection.from_config(config.keypoint_detector_config)
+        self.keypoint_detector = AutoModelForKeypointDetection.from_config(
+            config.keypoint_detector_config, trust_remote_code=config.trust_remote_code
+        )
 
         self.keypoint_detector_descriptor_dim = config.keypoint_detector_config.descriptor_decoder_dim
         self.descriptor_dim = config.descriptor_dim

filename: src/transformers/models/lightglue/modular_lightglue.py
patch: @@ -79,6 +79,8 @@ class LightGlueConfig(PretrainedConfig):
             The dropout ratio for the attention probabilities.
         attention_bias (`bool`, *optional*, defaults to `True`):
             Whether to use a bias in the query, key, value and output projection layers during self-attention.
+        trust_remote_code (`bool`, *optional*, defaults to `False`):
+            Whether to trust remote code when using other models than SuperPoint as keypoint detector.
 
     Examples:
         ```python
@@ -112,8 +114,14 @@ def __init__(
         hidden_act: str = "gelu",
         attention_dropout=0.0,
         attention_bias=True,
+        trust_remote_code: bool = False,
         **kwargs,
     ):
+        # LightGlue can be used with other models than SuperPoint as keypoint detector
+        # We provide the trust_remote_code argument to allow the use of other models
+        # that are not registered in the CONFIG_MAPPING dictionary (for example DISK)
+        self.trust_remote_code = trust_remote_code
+
         if descriptor_dim % num_attention_heads != 0:
             raise ValueError("descriptor_dim % num_heads is different from zero")
 
@@ -138,9 +146,15 @@ def __init__(
             keypoint_detector_config["model_type"] = (
                 keypoint_detector_config["model_type"] if "model_type" in keypoint_detector_config else "superpoint"
             )
-            keypoint_detector_config = CONFIG_MAPPING[keypoint_detector_config["model_type"]](
-                **keypoint_detector_config, attn_implementation="eager"
-            )
+            if keypoint_detector_config["model_type"] not in CONFIG_MAPPING:
+                keypoint_detector_config = AutoConfig.from_pretrained(
+                    keypoint_detector_config["_name_or_path"], trust_remote_code=self.trust_remote_code
+                )
+            else:
+                keypoint_detector_config = CONFIG_MAPPING[keypoint_detector_config["model_type"]](
+                    **keypoint_detector_config, attn_implementation="eager"
+                )
+
         if keypoint_detector_config is None:
             keypoint_detector_config = CONFIG_MAPPING["superpoint"](attn_implementation="eager")
 
@@ -584,8 +598,9 @@ class LightGlueForKeypointMatching(LightGluePreTrainedModel):
 
     def __init__(self, config: LightGlueConfig):
         super().__init__(config)
-
-        self.keypoint_detector = AutoModelForKeypointDetection.from_config(config.keypoint_detector_config)
+        self.keypoint_detector = AutoModelForKeypointDetection.from_config(
+            config.keypoint_detector_config, trust_remote_code=config.trust_remote_code
+        )
 
         self.keypoint_detector_descriptor_dim = config.keypoint_detector_config.descriptor_decoder_dim
         self.descriptor_dim = config.descriptor_dim