Repository: activeloopai/Hub
File: deeplake/auto/structured/dataframe.py (Python)

Code before:
                }  # If column parameters are not specified for a column, use the sanitized column name

        self.column_params = column_params

    def _get_most_frequent_image_extension(self, fn_iterator):
        # TODO: Make this generic and work for any htype that requires compression

        supported_image_extensions = tuple(
            HTYPE_SUPPORTED_COMPRESSIONS["image"] + ["jpg"]
        )
        image_extensions: DefaultDict[str, int] = defaultdict(int)

        for file in fn_iterator:
            if file.endswith(supported_image_extensions):
                ext = pathlib.Path(file).suffix[
                    1:
                ]  # Get extension without the . symbol
                image_extensions[ext] += 1
            else:
                raise IngestionError(f"The following file is not supported: {file}")

        most_frequent_image_extension = max(
            image_extensions, key=lambda k: image_extensions[k], default=None
        )

        return most_frequent_image_extension

    def _parse_tensor_params(self, key, inspect_limit=1000):
        """Parse the tensor parameters for a column. Required parameters that are not specified will be inferred by inspecting up to 'inspect_limit' rows in the data."""

        tensor_params = {}

        tensor_params = self.column_params[key]

        dtype = self.source[key].dtype
        if (
            "htype" not in tensor_params.keys()
        ):  # Auto-set some typing parameters if htype is not specified
            if dtype == np.dtype("object"):
                types = [type(v) for v in self.source[key][0:inspect_limit].values]

                if len(set(types)) != 1:
                    raise IngestionError(
                        f"Dataframe has different data types inside '{key}' column. Please make sure all data is given column is compatible with a single Deep Lake htype, or try specifying the htype manually."
                    )

                if types[0] == str:
                    tensor_params.update(
                        htype="text"
                    )  # Use "text" htype for text data when the htype is not specified tensor_params
            else:

Review comment (refactor):
Couple of issues: `endswith` check should include the period (`.`) otherwise a file named `abcdjpg` for example would be counted as a jpeg. Second, creating `pathlib.Path` object in a for loop for basic string operations is not good. Consider refactoring:

```suggestion

    def _get_most_frequent_image_extension(self, fn_iterator):
        # TODO: Make this generic and work for any htype that requires compression
        supported_image_extensions = tuple("." + fmt for fmt in HTYPE_SUPPORTED_COMPRESSIONS["image"] + ["jpg"])
        image_extensions: DefaultDict[str, int] = defaultdict(int)
        for file in fn_iterator:
            if file.lower().endswith(supported_image_extensions):
                ext = file.split(".")[1]
                image_extensions[ext] += 1
            else:
                raise IngestionError(f"The following file is not supported: {file}")

        most_frequent_image_extension = max(
            image_extensions, key=lambda k: image_extensions[k], default=None
        )
        return most_frequent_image_extension
```

Code after:
        """Parse the tensor parameters for a column. Required parameters that are not specified will be inferred by inspecting up to 'inspect_limit' rows in the data."""

        tensor_params: Dict = self.column_params[key]

        dtype = self.source[key].dtype
        if (
            "htype" not in tensor_params
        ):  # Auto-set some typing parameters if htype is not specified
            if dtype == np.dtype("object"):
                types = [type(v) for v in self.source[key][0:inspect_limit].values]

                if len(set(types)) != 1:
                    raise IngestionError(
                        f"Dataframe has different data types inside '{key}' column. Please make sure all data is given column is compatible with a single Deep Lake htype, or try specifying the htype manually."
                    )

                if types[0] == str:
                    tensor_params.update(
                        htype="text"
                    )  # Use "text" htype for text data when the htype is not specified tensor_params
            else:
                tensor_params.update(
                    dtype=dtype,
                    create_shape_tensor=tensor_params.get("create_shape_tensor", False),
                )  # htype will be auto-inferred for numeric data unless the htype is specified in tensor_params

        # TODO: Make this more robust so it works for all htypes where sample_compression is required and should be inferred from the data itself
        if (
            "image" in tensor_params.get("htype", "")
            and "sample_compression" not in tensor_params
            and "chunk_compression" not in tensor_params
        ):
            tensor_params.update(
                sample_compression=self._get_most_frequent_image_extension(
                    self.source[key].values
                )
            )

        return tensor_params

    def _get_extend_values(self, tensor_params: dict, key: str):  # type: ignore
        """Method creates a list of values to be extended to the tensor, based on the tensor parameters and the data in the dataframe column"""

        extend_values: List[Optional[Union[Sample, LinkedSample, np.ndarray]]]

        if "htype" in tensor_params and "link[" in tensor_params["htype"]:
            extend_values = [
                link(value, creds_key=self.creds_key) if value is not None else None
                for value in self.source[key].values
            ]
        elif "htype" in tensor_params and "image" in tensor_params["htype"]: