Repository: sdv-dev/RDT (language: python)

Issue:
Move profiling functions into RDT library
### Problem Description

Currently the `profiling.py` file lives in the tests package, and isn't shipped with `RDT` when being installed from `pypi`. We want to reuse these functions in other libraries, so it would be useful for them to ship with `RDT`. To do this we can move the `profiling.py` file and possibly other functions from performance evaluation into a `performance` package inside the `RDT`  package.

### Expected behavior

When installing `RDT`, one should be able to do
```
from rdt.performance import x, y, z
```
to import any performance or profiling related functions.

### Additional context

Look into any functions inside `profiling.py` and `test_performance.py` that may be recycled in other libraries. All of those should be pulled out.



Discussion / hints:

Fix patch:
diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 000000000..44e3460b5
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,2 @@
+[run]
+omit = rdt/performance/*
\ No newline at end of file
diff --git a/rdt/performance/__init__.py b/rdt/performance/__init__.py
new file mode 100644
index 000000000..ac9f82097
--- /dev/null
+++ b/rdt/performance/__init__.py
@@ -0,0 +1,9 @@
+"""Functions to evaluate and test the performance of the RDT Transformers."""
+
+from rdt.performance import profiling
+from rdt.performance.performance import evaluate_transformer_performance
+
+__all__ = [
+    'evaluate_transformer_performance',
+    'profiling',
+]
diff --git a/tests/datasets/__init__.py b/rdt/performance/datasets/__init__.py
similarity index 82%
rename from tests/datasets/__init__.py
rename to rdt/performance/datasets/__init__.py
index 46e3aa173..e11d70056 100644
--- a/tests/datasets/__init__.py
+++ b/rdt/performance/datasets/__init__.py
@@ -2,8 +2,8 @@
 
 from collections import defaultdict
 
-from tests.datasets import boolean, categorical, datetime, numerical
-from tests.datasets.base import BaseDatasetGenerator
+from rdt.performance.datasets import boolean, categorical, datetime, numerical
+from rdt.performance.datasets.base import BaseDatasetGenerator
 
 __all__ = [
     'boolean',
diff --git a/tests/datasets/base.py b/rdt/performance/datasets/base.py
similarity index 92%
rename from tests/datasets/base.py
rename to rdt/performance/datasets/base.py
index e3a0c5a21..6937c8442 100644
--- a/tests/datasets/base.py
+++ b/rdt/performance/datasets/base.py
@@ -11,7 +11,7 @@ class BaseDatasetGenerator(ABC):
     @staticmethod
     @abstractmethod
     def generate(num_rows):
-        """This method serves as a template for dataset generators.
+        """Return array of data. This method serves as a template for dataset generators.
 
         Args:
             num_rows (int):
diff --git a/tests/datasets/boolean.py b/rdt/performance/datasets/boolean.py
similarity index 98%
rename from tests/datasets/boolean.py
rename to rdt/performance/datasets/boolean.py
index 8783cb8f6..98d7668a3 100644
--- a/tests/datasets/boolean.py
+++ b/rdt/performance/datasets/boolean.py
@@ -4,7 +4,7 @@
 
 import numpy as np
 
-from tests.datasets.base import BaseDatasetGenerator
+from rdt.performance.datasets.base import BaseDatasetGenerator
 
 MAX_PERCENT_NULL = 50  # cap the percentage of null values at 50%
 MIN_PERCENT = 20  # the minimum percentage of true or false is 20%
diff --git a/tests/datasets/categorical.py b/rdt/performance/datasets/categorical.py
similarity index 98%
rename from tests/datasets/categorical.py
rename to rdt/performance/datasets/categorical.py
index 5a8f8d1af..4834f835e 100644
--- a/tests/datasets/categorical.py
+++ b/rdt/performance/datasets/categorical.py
@@ -4,9 +4,9 @@
 
 import numpy as np
 
-from tests.datasets.base import BaseDatasetGenerator
-from tests.datasets.datetime import RandomGapDatetimeGenerator
-from tests.datasets.utils import add_nans
+from rdt.performance.datasets.base import BaseDatasetGenerator
+from rdt.performance.datasets.datetime import RandomGapDatetimeGenerator
+from rdt.performance.datasets.utils import add_nans
 
 
 class CategoricalGenerator(BaseDatasetGenerator, ABC):
diff --git a/tests/datasets/datetime.py b/rdt/performance/datasets/datetime.py
similarity index 97%
rename from tests/datasets/datetime.py
rename to rdt/performance/datasets/datetime.py
index 4b7a05235..7391d7f92 100644
--- a/tests/datasets/datetime.py
+++ b/rdt/performance/datasets/datetime.py
@@ -6,8 +6,8 @@
 import numpy as np
 import pandas as pd
 
-from tests.datasets.base import BaseDatasetGenerator
-from tests.datasets.utils import add_nans
+from rdt.performance.datasets.base import BaseDatasetGenerator
+from rdt.performance.datasets.utils import add_nans
 
 
 class DatetimeGenerator(BaseDatasetGenerator, ABC):
diff --git a/tests/datasets/numerical.py b/rdt/performance/datasets/numerical.py
similarity index 98%
rename from tests/datasets/numerical.py
rename to rdt/performance/datasets/numerical.py
index cffd775c1..d4caa9ea3 100644
--- a/tests/datasets/numerical.py
+++ b/rdt/performance/datasets/numerical.py
@@ -4,8 +4,8 @@
 
 import numpy as np
 
-from tests.datasets.base import BaseDatasetGenerator
-from tests.datasets.utils import add_nans
+from rdt.performance.datasets.base import BaseDatasetGenerator
+from rdt.performance.datasets.utils import add_nans
 
 
 class NumericalGenerator(BaseDatasetGenerator, ABC):
diff --git a/tests/datasets/utils.py b/rdt/performance/datasets/utils.py
similarity index 100%
rename from tests/datasets/utils.py
rename to rdt/performance/datasets/utils.py
diff --git a/rdt/performance/performance.py b/rdt/performance/performance.py
new file mode 100644
index 000000000..ec3279423
--- /dev/null
+++ b/rdt/performance/performance.py
@@ -0,0 +1,62 @@
+"""Functions for evaluating transformer performance."""
+
+import numpy as np
+import pandas as pd
+
+from rdt.performance.profiling import profile_transformer
+
+DATASET_SIZES = [1000, 10000, 100000]
+
+
+def _get_dataset_sizes(data_type):
+    """Get a list of (fit_size, transform_size) for each dataset generator.
+
+    Based on the data type of the dataset generator, return the list of
+    sizes to run performance tests on. Each element in this list is a tuple
+    of (fit_size, transform_size).
+
+    Args:
+        input_type (str):
+            The type of data that the generator returns.
+
+    Returns:
+        sizes (list[tuple]):
+            A list of (fit_size, transform_size) configs to run tests on.
+    """
+    sizes = [(s, s) for s in DATASET_SIZES]
+
+    if data_type == 'categorical':
+        sizes = [(s, max(s, 1000)) for s in DATASET_SIZES if s <= 10000]
+
+    return sizes
+
+
+def evaluate_transformer_performance(transformer, dataset_generator):
+    """Evaluate the given transformer's performance against the given dataset generator.
+
+    Args:
+        transformer (rdt.transformers.BaseTransformer):
+            The transformer to evaluate.
+        dataset_generator (rdt.tests.datasets.BaseDatasetGenerator):
+            The dataset generator to performance test against.
+
+    Returns:
+        pandas.DataFrame:
+            The performance test results.
+    """
+    transformer_instance = transformer()
+
+    sizes = _get_dataset_sizes(dataset_generator.DATA_TYPE)
+
+    out = []
+    for fit_size, transform_size in sizes:
+        performance = profile_transformer(
+            transformer=transformer_instance,
+            dataset_generator=dataset_generator,
+            fit_size=fit_size,
+            transform_size=transform_size,
+        )
+        size = np.array([fit_size, transform_size, transform_size] * 2)
+        out.append(performance / size)
+
+    return pd.DataFrame(out).max(axis=0)
diff --git a/tests/performance/profiling.py b/rdt/performance/profiling.py
similarity index 100%
rename from tests/performance/profiling.py
rename to rdt/performance/profiling.py


Test patch:
diff --git a/tests/contributing.py b/tests/contributing.py
index c8350211c..881aaea7a 100644
--- a/tests/contributing.py
+++ b/tests/contributing.py
@@ -11,14 +11,15 @@
 import pytest
 from tabulate import tabulate
 
+from rdt.performance import evaluate_transformer_performance
+from rdt.performance.datasets import get_dataset_generators_by_type
 from rdt.transformers import get_transformer_class, get_transformers_by_type
 from tests.code_style import (
     get_test_location, validate_test_location, validate_test_names, validate_transformer_addon,
     validate_transformer_importable_from_parent_module, validate_transformer_module,
     validate_transformer_name, validate_transformer_subclass)
-from tests.datasets import get_dataset_generators_by_type
 from tests.integration.test_transformers import validate_transformer
-from tests.performance import evaluate_transformer_performance, validate_performance
+from tests.performance import validate_performance
 from tests.quality.test_quality import (
     TEST_THRESHOLD, get_regression_scores, get_results_table, get_test_cases)
 
diff --git a/tests/datasets/tests/test_boolean.py b/tests/datasets/tests/test_boolean.py
index ddb201d12..6362f2f22 100644
--- a/tests/datasets/tests/test_boolean.py
+++ b/tests/datasets/tests/test_boolean.py
@@ -1,6 +1,6 @@
 import pandas as pd
 
-from tests.datasets import boolean
+from rdt.performance.datasets import boolean
 
 NUM_ROWS = 50
 
diff --git a/tests/datasets/tests/test_categorical.py b/tests/datasets/tests/test_categorical.py
index e602971ba..649c1f9fd 100644
--- a/tests/datasets/tests/test_categorical.py
+++ b/tests/datasets/tests/test_categorical.py
@@ -1,7 +1,7 @@
 import numpy as np
 import pandas as pd
 
-from tests.datasets import categorical
+from rdt.performance.datasets import categorical
 
 
 class TestRandomIntegerGenerator:
diff --git a/tests/datasets/tests/test_datetime.py b/tests/datasets/tests/test_datetime.py
index d4e7f673b..465511986 100644
--- a/tests/datasets/tests/test_datetime.py
+++ b/tests/datasets/tests/test_datetime.py
@@ -3,7 +3,7 @@
 import numpy as np
 import pandas as pd
 
-from tests.datasets import datetime
+from rdt.performance.datasets import datetime
 
 
 class TestRandomGapDatetimeGenerator:
diff --git a/tests/datasets/tests/test_numerical.py b/tests/datasets/tests/test_numerical.py
index 98fc6a00f..6b687d8f0 100644
--- a/tests/datasets/tests/test_numerical.py
+++ b/tests/datasets/tests/test_numerical.py
@@ -1,7 +1,7 @@
 import numpy as np
 import pandas as pd
 
-from tests.datasets import numerical
+from rdt.performance.datasets import numerical
 
 
 class TestRandomIntegerGenerator:
diff --git a/tests/datasets/tests/test_utils.py b/tests/datasets/tests/test_utils.py
index 591b32daa..61b599d82 100644
--- a/tests/datasets/tests/test_utils.py
+++ b/tests/datasets/tests/test_utils.py
@@ -2,7 +2,7 @@
 
 import numpy as np
 
-from tests.datasets import utils
+from rdt.performance.datasets import utils
 
 
 def test_add_nulls_int():
diff --git a/tests/integration/test_transformers.py b/tests/integration/test_transformers.py
index 327d422d7..556cc257f 100644
--- a/tests/integration/test_transformers.py
+++ b/tests/integration/test_transformers.py
@@ -5,8 +5,8 @@
 import pytest
 
 from rdt import HyperTransformer
+from rdt.performance.datasets import BaseDatasetGenerator
 from rdt.transformers import BaseTransformer
-from tests.datasets import BaseDatasetGenerator
 
 DATA_SIZE = 1000
 TEST_COL = 'test_col'
diff --git a/tests/performance/__init__.py b/tests/performance/__init__.py
index d16d5c5af..acd600b4d 100644
--- a/tests/performance/__init__.py
+++ b/tests/performance/__init__.py
@@ -1,12 +1,7 @@
 """Functions to evaluate and test the performance of the RDT Transformers."""
 
-from tests.performance import profiling
-from tests.performance.test_performance import (
-    TEST_NAMES, evaluate_transformer_performance, validate_performance)
+from tests.performance.test_performance import validate_performance
 
 __all__ = [
-    'evaluate_transformer_performance',
-    'profiling',
-    'TEST_NAMES',
     'validate_performance',
 ]
diff --git a/tests/performance/test_performance.py b/tests/performance/test_performance.py
index d2a3299a8..a7cf7e08e 100644
--- a/tests/performance/test_performance.py
+++ b/tests/performance/test_performance.py
@@ -1,75 +1,21 @@
 """Test whether the performance of the Transformers is the expected one."""
 
-import importlib
-
 import numpy as np
 import pandas as pd
 import pytest
 
+from rdt.performance.datasets import get_dataset_generators_by_type
+from rdt.performance.performance import evaluate_transformer_performance
+from rdt.performance.profiling import profile_transformer
 from rdt.transformers import get_transformers_by_type
 from rdt.transformers.numerical import BayesGMMTransformer
-from tests.datasets import get_dataset_generators_by_type
-from tests.performance.profiling import profile_transformer
-
-TEST_NAMES = [
-    'Fit Time',
-    'Transform Time',
-    'Reverse Transform Time',
-    'Fit Memory',
-    'Transform Memory',
-    'Reverse Transform Memory',
-]
-
-DATASET_SIZES = [1000, 10000, 100000]
-
-
-def get_instance(obj, **kwargs):
-    """Create new instance of the ``obj`` argument.
-
-    Args:
-        obj (str):
-            Full name of class to import.
-    """
-    instance = None
-    if isinstance(obj, str):
-        package, name = obj.rsplit('.', 1)
-        instance = getattr(importlib.import_module(package), name)(**kwargs)
-
-    return instance
 
-
-def get_fqn(obj):
-    """Get the fully qualified name of the given object."""
-    return f'{obj.__module__}.{obj.__name__}'
-
-
-def _get_dataset_sizes(data_type):
-    """Get a list of (fit_size, transform_size) for each dataset generator.
-
-    Based on the data type of the dataset generator, return the list of
-    sizes to run performance tests on. Each element in this list is a tuple
-    of (fit_size, transform_size).
-
-    Args:
-        input_type (str):
-            The type of data that the generator returns.
-
-    Returns:
-        sizes (list[tuple]):
-            A list of (fit_size, transform_size) configs to run tests on.
-    """
-    sizes = [(s, s) for s in DATASET_SIZES]
-
-    if data_type == 'categorical':
-        sizes = [(s, max(s, 1000)) for s in DATASET_SIZES if s <= 10000]
-
-    return sizes
+SANDBOX_TRANSFORMERS = [BayesGMMTransformer]
 
 
 def _get_performance_test_cases():
     """Get all the (transformer, dataset_generator) combinations for testing."""
     all_test_cases = []
-    sandbox = [BayesGMMTransformer]
 
     dataset_generators = get_dataset_generators_by_type()
     transformers = get_transformers_by_type()
@@ -78,7 +24,7 @@ def _get_performance_test_cases():
         dataset_generators_for_type = dataset_generators.get(data_type, [])
 
         for transformer in transformers_for_type:
-            if transformer in sandbox:
+            if transformer in SANDBOX_TRANSFORMERS:
                 continue
 
             for dataset_generator in dataset_generators_for_type:
@@ -90,37 +36,6 @@ def _get_performance_test_cases():
 test_cases = _get_performance_test_cases()
 
 
-def evaluate_transformer_performance(transformer, dataset_generator):
-    """Evaluate the given transformer's performance against the given dataset generator.
-
-    Args:
-        transformer (rdt.transformers.BaseTransformer):
-            The transformer to evaluate.
-        dataset_generator (rdt.tests.datasets.BaseDatasetGenerator):
-            The dataset generator to performance test against.
-
-    Returns:
-        pandas.DataFrame:
-            The performance test results.
-    """
-    transformer_instance = transformer()
-
-    sizes = _get_dataset_sizes(dataset_generator.DATA_TYPE)
-
-    out = []
-    for fit_size, transform_size in sizes:
-        performance = profile_transformer(
-            transformer=transformer_instance,
-            dataset_generator=dataset_generator,
-            fit_size=fit_size,
-            transform_size=transform_size,
-        )
-        size = np.array([fit_size, transform_size, transform_size] * 2)
-        out.append(performance / size)
-
-    return pd.DataFrame(out).max(axis=0)
-
-
 def validate_performance(performance, dataset_generator, should_assert=False):
     """Validate the performance of all transformers for a dataset_generator.
 
diff --git a/tests/performance/tests/test_profiling.py b/tests/performance/tests/test_profiling.py
index 53925b22b..de93d4cdf 100644
--- a/tests/performance/tests/test_profiling.py
+++ b/tests/performance/tests/test_profiling.py
@@ -5,13 +5,13 @@
 
 import numpy as np
 
+from rdt.performance.datasets import BaseDatasetGenerator
+from rdt.performance.profiling import profile_transformer
 from rdt.transformers import NumericalTransformer
-from tests.datasets import BaseDatasetGenerator
-from tests.performance.profiling import profile_transformer
 
 
-@patch('tests.performance.profiling.mp')
-@patch('tests.performance.profiling.deepcopy', spec_set=deepcopy)
+@patch('rdt.performance.profiling.mp')
+@patch('rdt.performance.profiling.deepcopy', spec_set=deepcopy)
 def test_profile_transformer(deepcopy_mock, multiprocessor_mock):
     """Test the ``profile_transformer`` function.
 
diff --git a/tests/unit/transformers/test_base.py b/tests/unit/transformers/test_base.py
index e2a4df6b4..455eb5043 100644
--- a/tests/unit/transformers/test_base.py
+++ b/tests/unit/transformers/test_base.py
@@ -738,6 +738,21 @@ class Dummy(BaseTransformer):
         assert dummy_transformer.column_prefix == 'a#b#'
         assert dummy_transformer.output_columns == ['a#b#.value', 'a#b#.is_null']
 
+    def test__fit_raises_error(self):
+        """Test ``_fit`` raises ``NotImplementedError``."""
+
+        # Setup
+        data = pd.DataFrame({
+            'a': [1, 2, 3],
+            'b': [4, 5, 6],
+            'c': [7, 8, 9]
+        })
+        transformer = BaseTransformer()
+
+        # Run / Assert
+        with pytest.raises(NotImplementedError):
+            transformer._fit(data)
+
     def test_fit(self):
         """Test the ``fit`` method.
 
@@ -794,6 +809,21 @@ def _fit(self, data):
         assert dummy_transformer.column_prefix == 'a#b'
         assert dummy_transformer.output_columns == ['a#b.value', 'a#b.is_null']
 
+    def test__transform_raises_error(self):
+        """Test ``_transform`` raises ``NotImplementedError``."""
+
+        # Setup
+        data = pd.DataFrame({
+            'a': [1, 2, 3],
+            'b': [4, 5, 6],
+            'c': [7, 8, 9]
+        })
+        transformer = BaseTransformer()
+
+        # Run / Assert
+        with pytest.raises(NotImplementedError):
+            transformer._transform(data)
+
     def test_transform_incorrect_columns(self):
         """Test the ``transform`` method when the columns are not in the data.
 
@@ -979,6 +1009,21 @@ def test_fit_transform(self):
         self.transform.assert_called_once_with(data)
         assert output == self.transform.return_value
 
+    def test__reverse_transform_raises_error(self):
+        """Test ``_reverse_transform`` raises ``NotImplementedError``."""
+
+        # Setup
+        data = pd.DataFrame({
+            'a.value': [1, 2, 3],
+            'b.value': [4, 5, 6],
+            'c.value': [7, 8, 9]
+        })
+        transformer = BaseTransformer()
+
+        # Run / Assert
+        with pytest.raises(NotImplementedError):
+            transformer._reverse_transform(data)
+
     def test_reverse_transform_incorrect_columns(self):
         """Test the ``reverse_transform`` method when the columns are not in the data.