Repository: great-expectations/great_expectations

Issue:
Progress Bar for profiler
Some processes can take a long time, like the BasicSuiteBuilderProfiler and a progress bar can make it much more bearable.

There is a great progress bar package `tqdm` (https://github.com/tqdm/tqdm) that would be great addition to these longer process. Would this addition interest others?





Discussion:
@peterdhansen This sounds like a great contribution - yes, please go ahead. We looked at adding a progress bar in the past, but ran out of time.

Resolving patch:
diff --git a/great_expectations/profile/basic_suite_builder_profiler.py b/great_expectations/profile/basic_suite_builder_profiler.py
--- a/great_expectations/profile/basic_suite_builder_profiler.py
+++ b/great_expectations/profile/basic_suite_builder_profiler.py
@@ -3,6 +3,7 @@
 
 import numpy as np
 from dateutil.parser import parse
+from tqdm.auto import tqdm
 
 from great_expectations.core.expectation_configuration import ExpectationConfiguration
 from great_expectations.dataset.util import build_categorical_partition_object
@@ -615,53 +616,61 @@ def _profile(cls, dataset, configuration=None):
 
         column_cache = {}
         if selected_columns:
-            for column in selected_columns:
-                cardinality = cls._get_column_cardinality_with_caching(
-                    dataset, column, column_cache
-                )
-                column_type = cls._get_column_type_with_caching(
-                    dataset, column, column_cache
-                )
-
-                if cardinality in [
-                    ProfilerCardinality.TWO,
-                    ProfilerCardinality.VERY_FEW,
-                    ProfilerCardinality.FEW,
-                ]:
-                    cls._create_expectations_for_low_card_column(
+            with tqdm(
+                total=len(selected_columns), desc="Profiling Columns", delay=5
+            ) as pbar:
+                for column in selected_columns:
+                    pbar.set_postfix_str(column)
+                    cardinality = cls._get_column_cardinality_with_caching(
                         dataset, column, column_cache
                     )
-                elif cardinality in [
-                    ProfilerCardinality.MANY,
-                    ProfilerCardinality.VERY_MANY,
-                    ProfilerCardinality.UNIQUE,
-                ]:
-                    # TODO we will want to finesse the number and types of
-                    #  expectations created here. The simple version is deny/allow list
-                    #  and the more complex version is desired per column type and
-                    #  cardinality. This deserves more thought on configuration.
-                    dataset.expect_column_values_to_be_unique(column)
-
-                    if column_type in [ProfilerDataType.INT, ProfilerDataType.FLOAT]:
-                        cls._create_expectations_for_numeric_column(dataset, column)
-                    elif column_type in [ProfilerDataType.DATETIME]:
-                        cls._create_expectations_for_datetime_column(
-                            dataset,
-                            column,
-                            excluded_expectations=excluded_expectations,
-                            included_expectations=included_expectations,
-                        )
-                    elif column_type in [ProfilerDataType.STRING]:
-                        cls._create_expectations_for_string_column(
-                            dataset,
-                            column,
-                            excluded_expectations=excluded_expectations,
-                            included_expectations=included_expectations,
-                        )
-                    elif column_type in [ProfilerDataType.UNKNOWN]:
-                        logger.debug(
-                            f"Skipping expectation creation for column {column} of unknown type: {column_type}"
+                    column_type = cls._get_column_type_with_caching(
+                        dataset, column, column_cache
+                    )
+
+                    if cardinality in [
+                        ProfilerCardinality.TWO,
+                        ProfilerCardinality.VERY_FEW,
+                        ProfilerCardinality.FEW,
+                    ]:
+                        cls._create_expectations_for_low_card_column(
+                            dataset, column, column_cache
                         )
+                    elif cardinality in [
+                        ProfilerCardinality.MANY,
+                        ProfilerCardinality.VERY_MANY,
+                        ProfilerCardinality.UNIQUE,
+                    ]:
+                        # TODO we will want to finesse the number and types of
+                        #  expectations created here. The simple version is deny/allow list
+                        #  and the more complex version is desired per column type and
+                        #  cardinality. This deserves more thought on configuration.
+                        dataset.expect_column_values_to_be_unique(column)
+
+                        if column_type in [
+                            ProfilerDataType.INT,
+                            ProfilerDataType.FLOAT,
+                        ]:
+                            cls._create_expectations_for_numeric_column(dataset, column)
+                        elif column_type in [ProfilerDataType.DATETIME]:
+                            cls._create_expectations_for_datetime_column(
+                                dataset,
+                                column,
+                                excluded_expectations=excluded_expectations,
+                                included_expectations=included_expectations,
+                            )
+                        elif column_type in [ProfilerDataType.STRING]:
+                            cls._create_expectations_for_string_column(
+                                dataset,
+                                column,
+                                excluded_expectations=excluded_expectations,
+                                included_expectations=included_expectations,
+                            )
+                        elif column_type in [ProfilerDataType.UNKNOWN]:
+                            logger.debug(
+                                f"Skipping expectation creation for column {column} of unknown type: {column_type}"
+                            )
+                    pbar.update()
 
         if excluded_expectations:
             # NOTE: we reach into a private member here because of an expected future
diff --git a/great_expectations/profile/user_configurable_profiler.py b/great_expectations/profile/user_configurable_profiler.py
--- a/great_expectations/profile/user_configurable_profiler.py
+++ b/great_expectations/profile/user_configurable_profiler.py
@@ -26,6 +26,8 @@
 
 logger = logging.getLogger(__name__)
 
+from tqdm.auto import tqdm
+
 
 class UserConfigurableProfiler:
 
@@ -234,18 +236,21 @@ def _build_expectation_suite_from_semantic_types_dict(self):
                 self.profile_dataset, self.primary_or_compound_key
             )
 
-        for column_name, column_info in self.column_info.items():
-            semantic_types = column_info.get("semantic_types")
-            for semantic_type in semantic_types:
-                semantic_type_fn = self.semantic_type_functions.get(semantic_type)
-                semantic_type_fn(
-                    profile_dataset=self.profile_dataset, column=column_name
+        with tqdm(
+            desc="Profiling Columns", total=len(self.column_info), delay=5
+        ) as pbar:
+            for column_name, column_info in self.column_info.items():
+                pbar.set_postfix_str(f"Column={column_name}")
+                semantic_types = column_info.get("semantic_types")
+                for semantic_type in semantic_types:
+                    semantic_type_fn = self.semantic_type_functions.get(semantic_type)
+                    semantic_type_fn(
+                        profile_dataset=self.profile_dataset, column=column_name
+                    )
+                self._build_expectations_for_all_column_types(
+                    self.profile_dataset, column_name
                 )
-
-        for column_name in self.column_info.keys():
-            self._build_expectations_for_all_column_types(
-                self.profile_dataset, column_name
-            )
+                pbar.update()
 
         expectation_suite = self._build_column_description_metadata(
             self.profile_dataset
@@ -267,33 +272,36 @@ def _profile_and_build_expectation_suite(self):
                 column_list=self.primary_or_compound_key,
             )
         self._build_expectations_table(profile_dataset=self.profile_dataset)
-        for column_name, column_info in self.column_info.items():
-            data_type = column_info.get("type")
-            cardinality = column_info.get("cardinality")
-
-            if data_type in ("FLOAT", "INT", "NUMERIC"):
-                self._build_expectations_numeric(
-                    profile_dataset=self.profile_dataset,
-                    column=column_name,
-                )
+        with tqdm(desc="Profiling", total=len(self.column_info), delay=5) as pbar:
+            for column_name, column_info in self.column_info.items():
+                pbar.set_postfix_str(f"Column={column_name}")
+                data_type = column_info.get("type")
+                cardinality = column_info.get("cardinality")
+
+                if data_type in ("FLOAT", "INT", "NUMERIC"):
+                    self._build_expectations_numeric(
+                        profile_dataset=self.profile_dataset,
+                        column=column_name,
+                    )
 
-            if data_type == "DATETIME":
-                self._build_expectations_datetime(
-                    profile_dataset=self.profile_dataset,
-                    column=column_name,
-                )
+                if data_type == "DATETIME":
+                    self._build_expectations_datetime(
+                        profile_dataset=self.profile_dataset,
+                        column=column_name,
+                    )
 
-            if (
-                OrderedProfilerCardinality[self.value_set_threshold]
-                >= OrderedProfilerCardinality[cardinality]
-            ):
-                self._build_expectations_value_set(
+                if (
+                    OrderedProfilerCardinality[self.value_set_threshold]
+                    >= OrderedProfilerCardinality[cardinality]
+                ):
+                    self._build_expectations_value_set(
+                        profile_dataset=self.profile_dataset, column=column_name
+                    )
+
+                self._build_expectations_for_all_column_types(
                     profile_dataset=self.profile_dataset, column=column_name
                 )
-
-            self._build_expectations_for_all_column_types(
-                profile_dataset=self.profile_dataset, column=column_name
-            )
+                pbar.update()
 
         expectation_suite = self._build_column_description_metadata(
             self.profile_dataset


Test patch: