Repository: activeloopai/Hub
File: deeplake/auto/structured/dataframe.py (Python)

Code before:

        for file in fn_iterator:
            if file.endswith(supported_image_extensions):
                ext = pathlib.Path(file).suffix[
                    1:
                ]  # Get extension without the . symbol
                image_extensions[ext] += 1
            else:
                raise IngestionError(f"The following file is not supported: {file}")

        most_frequent_image_extension = max(
            image_extensions, key=lambda k: image_extensions[k], default=None
        )

        return most_frequent_image_extension

    def _parse_tensor_params(self, key, inspect_limit=1000):
        """Parse the tensor parameters for a column. Required parameters that are not specified will be inferred by inspecting up to 'inspect_limit' rows in the data."""

        tensor_params = {}

        tensor_params = self.column_params[key]

        dtype = self.source[key].dtype
        if (
            "htype" not in tensor_params.keys()
        ):  # Auto-set some typing parameters if htype is not specified
            if dtype == np.dtype("object"):
                types = [type(v) for v in self.source[key][0:inspect_limit].values]

                if len(set(types)) != 1:
                    raise IngestionError(
                        f"Dataframe has different data types inside '{key}' column. Please make sure all data is given column is compatible with a single Deep Lake htype, or try specifying the htype manually."
                    )

                if types[0] == str:
                    tensor_params.update(
                        htype="text"
                    )  # Use "text" htype for text data when the htype is not specified tensor_params
            else:
                tensor_params.update(
                    dtype=dtype,
                    create_shape_tensor=tensor_params.get("create_shape_tensor", False),
                )  # htype will be auto-inferred for numeric data unless the htype is specified in tensor_params

        # TODO: Make this more robust so it works for all htypes where sample_compression is required and should be inferred from the data itself
        if (
            "image" in tensor_params.get("htype", "")
            and "sample_compression" not in tensor_params.keys()
            and "chunk_compression" not in tensor_params.keys()
        ):

Review comment (suggestion):
```suggestion
            "htype" not in tensor_params
```

Code after:
                if len(set(types)) != 1:
                    raise IngestionError(
                        f"Dataframe has different data types inside '{key}' column. Please make sure all data is given column is compatible with a single Deep Lake htype, or try specifying the htype manually."
                    )

                if types[0] == str:
                    tensor_params.update(
                        htype="text"
                    )  # Use "text" htype for text data when the htype is not specified tensor_params
            else:
                tensor_params.update(
                    dtype=dtype,
                    create_shape_tensor=tensor_params.get("create_shape_tensor", False),
                )  # htype will be auto-inferred for numeric data unless the htype is specified in tensor_params

        # TODO: Make this more robust so it works for all htypes where sample_compression is required and should be inferred from the data itself
        if (
            "image" in tensor_params.get("htype", "")
            and "sample_compression" not in tensor_params
            and "chunk_compression" not in tensor_params
        ):
            tensor_params.update(
                sample_compression=self._get_most_frequent_image_extension(
                    self.source[key].values
                )
            )

        return tensor_params

    def _get_extend_values(self, tensor_params: dict, key: str):  # type: ignore
        """Method creates a list of values to be extended to the tensor, based on the tensor parameters and the data in the dataframe column"""

        extend_values: List[Optional[Union[Sample, LinkedSample, np.ndarray]]]

        if "htype" in tensor_params and "link[" in tensor_params["htype"]:
            extend_values = [
                link(value, creds_key=self.creds_key) if value is not None else None
                for value in self.source[key].values
            ]
        elif "htype" in tensor_params and "image" in tensor_params["htype"]:
            extend_values = [
                read(value, creds=self.creds) if value is not None else None
                for value in self.source[key].values
            ]
        else:
            extend_values = self.source[key].values

        return extend_values

    def fill_dataset(self, ds: Dataset, progressbar: bool = True) -> Dataset:
        """Fill dataset with data from the dataframe - one tensor per column