Repository: ViCCo-Group/thingsvision (language: python)

Issue:
Update tests
Unittests should be more splitted in a test directory


Discussion / hints:

Fix patch:
diff --git a/.coveragerc b/.coveragerc
index db1a950f..e2da53f0 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -4,6 +4,7 @@ source = thingsvision
 omit = 
 	*/__init__.py
 	*/usr/local/lib*
+	./thingsvision/tests/*
 	./thingsvision/clip.py
 	./thingsvision/dataset.py
 
@@ -11,5 +12,6 @@ omit =
 omit =
 	*/__init__.py
 	*/usr/local/lib*
+	./thingsvision/tests/*
 	./thingsvision/clip.py
 	./thingsvision/dataset.py
diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
index 7bed032a..12bc2aa0 100644
--- a/.github/workflows/coverage.yml
+++ b/.github/workflows/coverage.yml
@@ -1,5 +1,9 @@
 name: Code Coverage
-on: [push, pull_request]
+on: 
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
 jobs:
   run:
     runs-on: ${{ matrix.os }}
@@ -24,7 +28,7 @@ jobs:
     - name: Generate coverage report
       run: |
         pip install coverage
-        coverage run unittests.py
+        coverage run -m unittest discover thingsvision/tests
         coverage xml
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v2


Test patch:
diff --git a/.github/workflows/python-package.yml b/.github/workflows/tests.yml
similarity index 94%
rename from .github/workflows/python-package.yml
rename to .github/workflows/tests.yml
index d50f81ee..1ad88d66 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/tests.yml
@@ -30,4 +30,4 @@ jobs:
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
     - name: Test with unittest
       run: |
-        python unittests.py -v
+        python -m unittest discover thingsvision/tests
diff --git a/thingsvision/tests/__init__.py b/thingsvision/tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/thingsvision/tests/helper.py b/thingsvision/tests/helper.py
new file mode 100644
index 00000000..1df2b76c
--- /dev/null
+++ b/thingsvision/tests/helper.py
@@ -0,0 +1,187 @@
+import os
+import re
+from typing import Tuple, List, Any
+
+import thingsvision.vision 
+from thingsvision.dataloader import DataLoader
+from thingsvision.dataset import ImageDataset
+from thingsvision.model_class import Model
+
+import numpy as np
+
+import skimage
+import imageio
+
+import torch 
+import torch.nn as nn
+
+import tensorflow as tf
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.layers import Dense
+
+DATA_PATH = './data'
+TEST_PATH = './test_images'
+OUT_PATH = './test'
+
+PT_MODEL_AND_MODULE_NAMES = {
+    'vgg16_bn': ['features.23', 'classifier.3'],
+    'vgg19_bn': ['features.23', 'classifier.3'],
+    'cornet_r': ['decoder.flatten'],
+    'cornet_rt': ['decoder.flatten'],
+    'cornet_s': ['decoder.flatten'],
+    'cornet_z': ['decoder.flatten'],
+    'clip-ViT': ['visual'],
+    'clip-RN': ['visual']
+}
+
+TF_MODEL_AND_MODULES_NAMES = {
+    'VGG16': ['block1_conv1', 'flatten'],
+    'VGG19': ['block1_conv1', 'flatten']
+}
+
+BACKENDS = ['tf', 'pt']
+
+CLIP = [True if re.search(r'^clip', model_name)
+        else False for model_name in PT_MODEL_AND_MODULE_NAMES]
+
+FILE_FORMATS = ['hdf5', 'npy', 'mat', 'txt']
+DISTANCES = ['correlation', 'cosine', 'euclidean', 'gaussian']
+
+BATCH_SIZE = 16
+NUM_OBJECTS = 1854
+# we want to iterate over two batches to exhaustively test mini-batching
+NUM_SAMPLES = int(BATCH_SIZE * 2)
+DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
+
+
+tf_model = Sequential()
+tf_model.add(Dense(1, input_dim=1, activation='relu',
+             use_bias=False, name='relu'))
+weights = np.array([[[1]]])
+tf_model.get_layer('relu').set_weights(weights)
+
+
+class NN(nn.Module):
+
+    def __init__(self, in_size: int, out_size: int):
+        super(NN, self).__init__()
+        self.linear = nn.Linear(in_size, out_size, bias=False)
+        self.relu = nn.ReLU()
+        # exchange weight value with 1.
+        self.linear.weight = nn.Parameter(torch.tensor([1.]))
+
+    def forward(self, x):
+        print('input %s' % x)
+        print('weight %s' % self.linear.weight)
+        x = self.linear(x)
+        act = self.relu(x)
+        print('act %s' % act)
+        return act
+
+
+pt_model = NN(1, 1)
+
+
+class SimpleDataset(object):
+
+    def __init__(self, values: List[int], backend: str):
+        self.values = values
+        self.backend = backend
+
+    def __getitem__(self, idx: int) -> Tuple[Any, Any]:
+        target = 0
+        value = self.values[idx]
+
+        if self.backend == 'pt':
+            value = torch.tensor([float(value)])
+            target = torch.tensor([target])
+        elif self.backend == 'tf':
+            value = tf.convert_to_tensor([float(value)])
+            target = tf.convert_to_tensor([target])
+
+        return value, target
+
+    def __len__(self) -> int:
+        return len(self.values)
+
+def iterate_through_all_model_combinations():
+    for backend in BACKENDS:
+        MODEL_AND_MODULE_NAMES = None
+        if backend == 'pt':
+            MODEL_AND_MODULE_NAMES = PT_MODEL_AND_MODULE_NAMES
+        elif backend == 'tf':
+            MODEL_AND_MODULE_NAMES = TF_MODEL_AND_MODULES_NAMES
+
+        for model_name in MODEL_AND_MODULE_NAMES:
+            model, dataset, dl = create_model_and_dl(model_name, backend)
+            yield model, dataset, dl, MODEL_AND_MODULE_NAMES[model_name], model_name
+
+
+def create_model_and_dl(model_name, backend):
+    """Iterate through all backends and models and create model, dataset and data loader."""
+    model = Model(
+        model_name=model_name,
+        pretrained=True,
+        device=DEVICE,
+        backend=backend
+    )
+
+    dataset = ImageDataset(
+        root=TEST_PATH,
+        out_path=OUT_PATH,
+        backend=backend,
+        imagenet_train=None,
+        imagenet_val=None,
+        things=None,
+        things_behavior=None,
+        add_ref_imgs=None,
+        transforms=model.get_transformations()
+    )
+    dl = DataLoader(
+        dataset,
+        batch_size=BATCH_SIZE,
+        backend=backend,
+    )
+    return model, dataset, dl
+
+def create_test_images(n_samples: int = NUM_SAMPLES) -> None:
+    """Create an artificial image dataset to be used for performing tests."""
+    if not os.path.exists(OUT_PATH):
+        os.makedirs(OUT_PATH)
+    if not os.path.exists(TEST_PATH):
+        test_img_1 = skimage.data.hubble_deep_field()
+        test_img_2 = skimage.data.coffee()
+        test_imgs = list(map(lambda x: x / x.max(), [test_img_1, test_img_2]))
+
+        classes = ['hubble', 'coffee']
+        for cls in classes:
+            PATH = os.path.join(TEST_PATH, cls)
+            if not os.path.exists(PATH):
+                os.makedirs(PATH)
+
+        for i in range(n_samples):
+            if i > n_samples // 2:
+                test_img = np.copy(test_imgs[0])
+                cls = classes[0]
+            else:
+                test_img = np.copy(test_imgs[1])
+                cls = classes[1]
+            H, W, C = test_img.shape
+            # add random Gaussian noise to test image
+            noisy_img = test_img + np.random.randn(H, W, C)
+            noisy_img = noisy_img.astype(np.uint8)
+            imageio.imsave(os.path.join(
+                TEST_PATH, cls, f'test_img_{i+1:03d}.png'), noisy_img)
+        print('\n...Successfully created image dataset for testing.\n')
+
+def download_item_names():
+    if not os.path.isfile(os.path.join(DATA_PATH, 'item_names.tsv')):
+        try:
+            os.system(
+                "curl -O https://raw.githubusercontent.com/ViCCo-Group/THINGSvision/master/get_files.sh")
+            os.system("bash get_files.sh")
+        except:
+            raise RuntimeError(
+                "Download the THINGS item names <tsv> file to run test;\n"
+                "See README.md for details."
+            )
diff --git a/thingsvision/tests/test_model/__init__.py b/thingsvision/tests/test_model/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/thingsvision/tests/test_model/test_extraction/__init__.py b/thingsvision/tests/test_model/test_extraction/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/thingsvision/tests/test_model/test_extraction/test_custom_model.py b/thingsvision/tests/test_model/test_extraction/test_custom_model.py
new file mode 100644
index 00000000..5f41a5c2
--- /dev/null
+++ b/thingsvision/tests/test_model/test_extraction/test_custom_model.py
@@ -0,0 +1,39 @@
+import re
+import unittest
+
+import helper
+from thingsvision.dataloader import DataLoader
+from thingsvision.model_class import Model
+
+import numpy as np 
+
+class ExtractionTestCase(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        helper.create_test_images()
+        
+    def test_custom_model(self):
+        layer_name = 'relu'
+        values = [2, -10]
+        backends = [['pt', helper.pt_model, 'vgg16'], ['tf', helper.tf_model, 'VGG16']]
+        for backend, custom_model, vgg_model in backends:
+            dataset = helper.SimpleDataset(values, backend)
+            dl = DataLoader(
+                dataset,
+                batch_size=1,
+                backend=backend,
+            )
+            model = Model(vgg_model, pretrained=False, device=helper.DEVICE, backend=backend)
+
+            model.model = custom_model
+            features, _ = model.extract_features(
+                dl, layer_name, batch_size=helper.BATCH_SIZE, flatten_acts=False, device=helper.DEVICE)
+
+            expected_features = np.array([[2], [0]])
+            np.testing.assert_allclose(features, expected_features)
+
+
+    
+
+    
\ No newline at end of file
diff --git a/thingsvision/tests/test_model/test_extraction/test_post_extraction.py b/thingsvision/tests/test_model/test_extraction/test_post_extraction.py
new file mode 100644
index 00000000..364d6a16
--- /dev/null
+++ b/thingsvision/tests/test_model/test_extraction/test_post_extraction.py
@@ -0,0 +1,54 @@
+import unittest
+
+import helper
+import thingsvision.vision as vision
+import numpy as np 
+
+class PostExtractionTestCase(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        helper.create_test_images()
+
+    def test_postprocessing(self):
+        """Test different postprocessing methods (e.g., centering, normalization, compression)."""
+        model_name = 'vgg16_bn'
+        model, dataset, dl = helper.create_model_and_dl(model_name, 'pt')
+        module_name = helper.PT_MODEL_AND_MODULE_NAMES[model_name][0]
+        features, _ = model.extract_features(
+            data_loader=dl,
+            module_name=module_name,
+            batch_size=helper.BATCH_SIZE,
+            flatten_acts=False,
+            device=helper.DEVICE,
+        )
+        flattened_features = features.reshape(helper.NUM_SAMPLES, -1)
+        centred_features = vision.center_features(flattened_features)
+        normalized_features = vision.normalize_features(flattened_features)
+        transformed_features = vision.compress_features(
+            flattened_features, rnd_seed=42, retained_var=.9)
+
+        self.assertTrue(centred_features.mean(axis=0).sum() < 1e-3)
+        self.assertEqual(np.linalg.norm(normalized_features, axis=1).sum(),
+                         np.ones(features.shape[0]).sum())
+        self.assertTrue(
+            transformed_features.shape[1] < flattened_features.shape[1])
+
+    def test_storing(self):
+        """Test storing possibilities."""
+        model_name = 'vgg16_bn'
+        model, dataset, dl = helper.create_model_and_dl(model_name, 'pt')
+        module_name = helper.PT_MODEL_AND_MODULE_NAMES[model_name][0]
+        features, _ = model.extract_features(
+            data_loader=dl,
+            module_name=module_name,
+            batch_size=helper.BATCH_SIZE,
+            flatten_acts=False,
+            device=helper.DEVICE,
+        )
+        for format in helper.FILE_FORMATS:
+            # tests whether features can be saved in any of the formats as four-dimensional tensor
+            vision.save_features(
+                features=features,
+                out_path=helper.OUT_PATH,
+                file_format=format,
+            )
\ No newline at end of file
diff --git a/thingsvision/tests/test_model/test_extraction/test_pretrained_model.py b/thingsvision/tests/test_model/test_extraction/test_pretrained_model.py
new file mode 100644
index 00000000..06031b11
--- /dev/null
+++ b/thingsvision/tests/test_model/test_extraction/test_pretrained_model.py
@@ -0,0 +1,40 @@
+import re
+import unittest
+
+import helper
+from thingsvision.dataloader import DataLoader
+from thingsvision.model_class import Model
+
+import numpy as np 
+
+class ExtractionPretrainedTestCase(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        helper.create_test_images()
+        
+    def test_extraction_pretrained_modells(self):
+        """Tests basic feature extraction pipeline."""
+        for model, dataset, dl, module_names, model_name in helper.iterate_through_all_model_combinations():
+            clip = re.search(r'^clip', model_name)
+
+            for module_name in module_names:
+                self.assertEqual(len(dataset), len(dl) * helper.BATCH_SIZE)
+
+                features, targets = model.extract_features(
+                    data_loader=dl,
+                    module_name=module_name,
+                    batch_size=helper.BATCH_SIZE,
+                    flatten_acts=False,
+                    device=helper.DEVICE,
+                    clip=clip
+                )
+
+                self.assertTrue(isinstance(features, np.ndarray))
+                self.assertTrue(isinstance(targets, np.ndarray))
+                self.assertEqual(features.shape[0], len(dataset))
+                self.assertEqual(len(targets), features.shape[0])
+
+                if module_name.startswith('classifier'):
+                    self.assertEqual(features.shape[1],
+                                     model.model.classifier[int(module_name[-1])].out_features)
diff --git a/thingsvision/tests/test_model/test_extraction/test_torch_vs_tensorflow.py b/thingsvision/tests/test_model/test_extraction/test_torch_vs_tensorflow.py
new file mode 100644
index 00000000..f7734c17
--- /dev/null
+++ b/thingsvision/tests/test_model/test_extraction/test_torch_vs_tensorflow.py
@@ -0,0 +1,46 @@
+import unittest
+
+import helper
+from thingsvision.model_class import Model
+from thingsvision.dataloader import DataLoader
+
+import numpy as np 
+
+class ExtractionPTvsTFTestCase(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        helper.create_test_images()
+        
+    def test_custom_torch_vs_tf_extraction(self):
+        layer_name = 'relu'
+        values = [2, -10]
+        backend = 'tf'
+        tf_dataset = helper.SimpleDataset(values, backend)
+        tf_dl = DataLoader(
+            tf_dataset,
+            batch_size=1,
+            backend=backend,
+        )
+
+        model = Model('VGG16', pretrained=False,
+                      device=helper.DEVICE, backend=backend)
+        model.model = helper.tf_model
+        tf_features, _ = model.extract_features(
+            tf_dl, layer_name, batch_size=helper.BATCH_SIZE, flatten_acts=False, device=helper.DEVICE)
+
+        backend = 'pt'
+        pt_dataset = helper.SimpleDataset(values, backend)
+        pt_dl = DataLoader(
+            pt_dataset,
+            batch_size=1,
+            backend=backend,
+        )
+        model = Model('vgg16', pretrained=False,
+                      device=helper.DEVICE, backend=backend)
+        model.model = helper.pt_model
+        pt_features, _ = model.extract_features(
+            pt_dl, layer_name, batch_size=helper.BATCH_SIZE, flatten_acts=False, device=helper.DEVICE)
+        np.testing.assert_allclose(tf_features, pt_features)
+
+        expected_features = np.array([[2], [0]])
+        np.testing.assert_allclose(pt_features, expected_features)
diff --git a/thingsvision/tests/test_model/test_model.py b/thingsvision/tests/test_model/test_model.py
new file mode 100644
index 00000000..c9e9b9e5
--- /dev/null
+++ b/thingsvision/tests/test_model/test_model.py
@@ -0,0 +1,17 @@
+import unittest
+
+import helper
+
+import numpy as np 
+
+class ModelLoadingTestCase(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        helper.create_test_images()
+        
+    def test_mode_and_device(self):
+        model_name = 'vgg16_bn'
+        model, dataset, dl = helper.create_model_and_dl(model_name, 'pt')
+        self.assertTrue(hasattr(model.model, helper.DEVICE))
+        self.assertFalse(model.model.training)
diff --git a/thingsvision/tests/test_rest.py b/thingsvision/tests/test_rest.py
new file mode 100644
index 00000000..ce2fa8f6
--- /dev/null
+++ b/thingsvision/tests/test_rest.py
@@ -0,0 +1,111 @@
+import os 
+import unittest
+
+import thingsvision.vision as vision 
+import helper 
+
+import numpy as np
+import pandas as pd 
+
+class RDMTestCase(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        helper.create_test_images()
+        model_name = 'vgg16_bn'
+        model, dataset, dl = helper.create_model_and_dl(model_name, 'pt')
+        module_name = helper.PT_MODEL_AND_MODULE_NAMES[model_name][0]
+        features, _ = model.extract_features(
+            data_loader=dl,
+            module_name=module_name,
+            batch_size=helper.BATCH_SIZE,
+            flatten_acts=False,
+            device=helper.DEVICE,
+        )
+        for format in helper.FILE_FORMATS:
+            # tests whether features can be saved in any of the formats as four-dimensional tensor
+            vision.save_features(
+                features=features,
+                out_path=helper.OUT_PATH,
+                file_format=format,
+            )
+
+
+    def test_rdms(self):
+        """Test different distance metrics on which RDMs are based."""
+
+        features = np.load(os.path.join(helper.OUT_PATH, 'features.npy'))
+        features = features.reshape(helper.NUM_SAMPLES, -1)
+
+        rdms = []
+        for distance in helper.DISTANCES:
+            rdm = vision.compute_rdm(features, distance)
+            self.assertEqual(len(rdm.shape), 2)
+            self.assertEqual(features.shape[0], rdm.shape[0], rdm.shape[1])
+            vision.plot_rdm(helper.OUT_PATH, features, distance)
+            rdms.append(rdm)
+
+        for rdm_i, rdm_j in zip(rdms[:-1], rdms[1:]):
+            corr = vision.correlate_rdms(rdm_i, rdm_j)
+            self.assertTrue(isinstance(corr, float))
+            self.assertTrue(corr > float(-1))
+            self.assertTrue(corr < float(1))
+
+
+class ComparisonTestCase(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        helper.create_test_images()
+
+    def test_comparison(self):
+        backend = 'pt'
+        compare_model_names = ['vgg16_bn', 'vgg19_bn']
+        compare_module_names = ['features.23', 'classifier.3']
+        corr_mat = vision.compare_models(
+            root=helper.TEST_PATH,
+            out_path=helper.OUT_PATH,
+            model_names=compare_model_names,
+            module_names=compare_module_names,
+            pretrained=True,
+            batch_size=helper.BATCH_SIZE,
+            backend=backend,
+            flatten_acts=True,
+            clip=helper.CLIP,
+            save_features=False,
+        )
+        self.assertTrue(isinstance(corr_mat, pd.DataFrame))
+        self.assertEqual(corr_mat.shape, (len(
+            compare_model_names), len(compare_module_names)))
+
+
+class LoadItemsTestCase(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        helper.download_item_names()
+
+    def test_item_name_loading(self):
+        item_names = vision.load_item_names(helper.DATA_PATH)
+        self.assertTrue(isinstance(item_names, np.ndarray))
+        self.assertEqual(len(item_names), helper.NUM_OBJECTS)
+
+
+class FileNamesTestCase(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        helper.create_test_images()
+
+    def test_filenames(self):
+        import os
+        print(os.getcwd())
+        file_names = open(os.path.join(helper.OUT_PATH, 'file_names.txt'), 'r').read().split()
+        img_files = []
+        for root, _, files in os.walk(helper.TEST_PATH):
+            for f in files:
+                if f.endswith('png'):
+                    img_files.append(os.path.join(root, f))
+        self.assertEqual(sorted(file_names), sorted(img_files))
+
+
diff --git a/unittests.py b/unittests.py
deleted file mode 100644
index f0fd9e07..00000000
--- a/unittests.py
+++ /dev/null
@@ -1,378 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-import imageio
-import os
-import re
-import skimage
-import shutil
-import unittest
-
-import numpy as np
-import pandas as pd
-import torch
-import tensorflow as tf
-import torch.nn as nn
-import thingsvision.vision as vision
-
-from thingsvision.model_class import Model
-from thingsvision.dataset import ImageDataset
-from thingsvision.dataloader import DataLoader
-
-from tensorflow.keras.models import Sequential
-from tensorflow.keras.layers import Dense
-from typing import Tuple, List, Any
-
-DATA_PATH = './data'
-TEST_PATH = './test_images'
-OUT_PATH = './test'
-
-PT_MODEL_AND_MODULE_NAMES = {
-    'vgg16_bn': ['features.23', 'classifier.3'],
-    'vgg19_bn': ['features.23', 'classifier.3'],
-    'cornet_r': ['decoder.flatten'],
-    'cornet_rt': ['decoder.flatten'],
-    'cornet_s': ['decoder.flatten'],
-    'cornet_z': ['decoder.flatten'],
-    'clip-ViT': ['visual'],
-    'clip-RN': ['visual']
-}
-
-TF_MODEL_AND_MODULES_NAMES = {
-    'VGG16': ['block1_conv1', 'flatten'],
-    'VGG19': ['block1_conv1', 'flatten']
-}
-
-CLIP = [True if re.search(r'^clip', model_name)
-        else False for model_name in PT_MODEL_AND_MODULE_NAMES]
-
-FILE_FORMATS = ['hdf5', 'npy', 'mat', 'txt']
-DISTANCES = ['correlation', 'cosine', 'euclidean', 'gaussian']
-
-BATCH_SIZE = 16
-NUM_OBJECTS = 1854
-# we want to iterate over two batches to exhaustively test mini-batching
-NUM_SAMPLES = int(BATCH_SIZE * 2)
-DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
-BACKENDS = ['pt', 'tf']
-
-if not os.path.isfile(os.path.join(DATA_PATH, 'item_names.tsv')):
-    try:
-        os.system(
-            "curl -O https://raw.githubusercontent.com/ViCCo-Group/THINGSvision/master/get_files.sh")
-        os.system("bash get_files.sh")
-    except:
-        raise RuntimeError(
-            "Download the THINGS item names <tsv> file to run test;\n"
-            "See README.md for details."
-        )
-
-tf_model = Sequential()
-tf_model.add(Dense(1, input_dim=1, activation='relu',
-             use_bias=False, name='relu'))
-weights = np.array([[[1]]])
-tf_model.get_layer('relu').set_weights(weights)
-
-
-class NN(nn.Module):
-
-    def __init__(self, in_size: int, out_size: int):
-        super(NN, self).__init__()
-        self.linear = nn.Linear(in_size, out_size, bias=False)
-        self.relu = nn.ReLU()
-        # exchange weight value with 1.
-        self.linear.weight = nn.Parameter(torch.tensor([1.]))
-
-    def forward(self, x):
-        print('input %s' % x)
-        print('weight %s' % self.linear.weight)
-        x = self.linear(x)
-        act = self.relu(x)
-        print('act %s' % act)
-        return act
-
-
-pt_model = NN(1, 1)
-
-
-class SimpleDataset(object):
-
-    def __init__(self, values: List[int], backend: str):
-        self.values = values
-        self.backend = backend
-
-    def __getitem__(self, idx: int) -> Tuple[Any, Any]:
-        target = 0
-        value = self.values[idx]
-
-        if self.backend == 'pt':
-            value = torch.tensor([float(value)])
-            target = torch.tensor([target])
-        elif self.backend == 'tf':
-            value = tf.convert_to_tensor([float(value)])
-            target = tf.convert_to_tensor([target])
-
-        return value, target
-
-    def __len__(self) -> int:
-        return len(self.values)
-
-
-def iterate_through_all_model_combinations():
-    for backend in BACKENDS:
-        MODEL_AND_MODULE_NAMES = None
-        if backend == 'pt':
-            MODEL_AND_MODULE_NAMES = PT_MODEL_AND_MODULE_NAMES
-        elif backend == 'tf':
-            MODEL_AND_MODULE_NAMES = TF_MODEL_AND_MODULES_NAMES
-
-        for model_name in MODEL_AND_MODULE_NAMES:
-            model, dataset, dl = create_model_and_dl(model_name, backend)
-            yield model, dataset, dl, MODEL_AND_MODULE_NAMES[model_name], model_name
-
-
-def create_model_and_dl(model_name, backend):
-    """Iterate through all backends and models and create model, dataset and data loader."""
-    model = Model(
-        model_name=model_name,
-        pretrained=True,
-        device=DEVICE,
-        backend=backend
-    )
-
-    dataset = ImageDataset(
-        root=TEST_PATH,
-        out_path=OUT_PATH,
-        backend=backend,
-        imagenet_train=None,
-        imagenet_val=None,
-        things=None,
-        things_behavior=None,
-        add_ref_imgs=None,
-        transforms=model.get_transformations()
-    )
-    dl = DataLoader(
-        dataset,
-        batch_size=BATCH_SIZE,
-        backend=backend,
-    )
-    return model, dataset, dl
-
-
-class ModelLoadingTestCase(unittest.TestCase):
-
-    def test_mode_and_device(self):
-        model_name = 'vgg16_bn'
-        model, dataset, dl = create_model_and_dl(model_name, 'pt')
-        self.assertTrue(hasattr(model.model, DEVICE))
-        self.assertFalse(model.model.training)
-
-
-class ExtractionTestCase(unittest.TestCase):
-
-    def test_extraction_pretrained_modells(self):
-        """Tests basic feature extraction pipeline."""
-        for model, dataset, dl, module_names, model_name in iterate_through_all_model_combinations():
-            clip = re.search(r'^clip', model_name)
-
-            for module_name in module_names:
-                self.assertEqual(len(dataset), len(dl) * BATCH_SIZE)
-
-                features, targets = model.extract_features(
-                    data_loader=dl,
-                    module_name=module_name,
-                    batch_size=BATCH_SIZE,
-                    flatten_acts=False,
-                    device=DEVICE,
-                    clip=clip
-                )
-
-                self.assertTrue(isinstance(features, np.ndarray))
-                self.assertTrue(isinstance(targets, np.ndarray))
-                self.assertEqual(features.shape[0], len(dataset))
-                self.assertEqual(len(targets), features.shape[0])
-
-                if module_name.startswith('classifier'):
-                    self.assertEqual(features.shape[1],
-                                     model.model.classifier[int(module_name[-1])].out_features)
-
-    def test_postprocessing(self):
-        """Test different postprocessing methods (e.g., centering, normalization, compression)."""
-        model_name = 'vgg16_bn'
-        model, dataset, dl = create_model_and_dl(model_name, 'pt')
-        module_name = PT_MODEL_AND_MODULE_NAMES[model_name][0]
-        features, _ = model.extract_features(
-            data_loader=dl,
-            module_name=module_name,
-            batch_size=BATCH_SIZE,
-            flatten_acts=False,
-            device=DEVICE,
-        )
-        flattened_features = features.reshape(NUM_SAMPLES, -1)
-        centred_features = vision.center_features(flattened_features)
-        normalized_features = vision.normalize_features(flattened_features)
-        transformed_features = vision.compress_features(
-            flattened_features, rnd_seed=42, retained_var=.9)
-
-        self.assertTrue(centred_features.mean(axis=0).sum() < 1e-3)
-        self.assertEqual(np.linalg.norm(normalized_features, axis=1).sum(),
-                         np.ones(features.shape[0]).sum())
-        self.assertTrue(
-            transformed_features.shape[1] < flattened_features.shape[1])
-
-    def test_storing(self):
-        """Test storing possibilities."""
-        model_name = 'vgg16_bn'
-        model, dataset, dl = create_model_and_dl(model_name, 'pt')
-        module_name = PT_MODEL_AND_MODULE_NAMES[model_name][0]
-        features, _ = model.extract_features(
-            data_loader=dl,
-            module_name=module_name,
-            batch_size=BATCH_SIZE,
-            flatten_acts=False,
-            device=DEVICE,
-        )
-        for format in FILE_FORMATS:
-            # tests whether features can be saved in any of the formats as four-dimensional tensor
-            vision.save_features(
-                features=features,
-                out_path=OUT_PATH,
-                file_format=format,
-            )
-
-    def test_custom_torch_vs_tf_extraction(self):
-        layer_name = 'relu'
-        values = [2, -10]
-        backend = 'tf'
-        tf_dataset = SimpleDataset(values, backend)
-        tf_dl = DataLoader(
-            tf_dataset,
-            batch_size=1,
-            backend=backend,
-        )
-
-        model = Model('VGG16', pretrained=False,
-                      device=DEVICE, backend=backend)
-        model.model = tf_model
-        tf_features, _ = model.extract_features(
-            tf_dl, layer_name, batch_size=BATCH_SIZE, flatten_acts=False, device=DEVICE)
-
-        backend = 'pt'
-        pt_dataset = SimpleDataset(values, backend)
-        pt_dl = DataLoader(
-            pt_dataset,
-            batch_size=1,
-            backend=backend,
-        )
-        model = Model('vgg16', pretrained=False,
-                      device=DEVICE, backend=backend)
-        model.model = pt_model
-        pt_features, _ = model.extract_features(
-            pt_dl, layer_name, batch_size=BATCH_SIZE, flatten_acts=False, device=DEVICE)
-        np.testing.assert_allclose(tf_features, pt_features)
-
-        expected_features = np.array([[2], [0]])
-        np.testing.assert_allclose(pt_features, expected_features)
-
-
-class RDMTestCase(unittest.TestCase):
-
-    def test_rdms(self):
-        """Test different distance metrics on which RDMs are based."""
-
-        features = np.load(os.path.join(OUT_PATH, 'features.npy'))
-        features = features.reshape(NUM_SAMPLES, -1)
-
-        rdms = []
-        for distance in DISTANCES:
-            rdm = vision.compute_rdm(features, distance)
-            self.assertEqual(len(rdm.shape), 2)
-            self.assertEqual(features.shape[0], rdm.shape[0], rdm.shape[1])
-            vision.plot_rdm(OUT_PATH, features, distance)
-            rdms.append(rdm)
-
-        for rdm_i, rdm_j in zip(rdms[:-1], rdms[1:]):
-            corr = vision.correlate_rdms(rdm_i, rdm_j)
-            self.assertTrue(isinstance(corr, float))
-            self.assertTrue(corr > float(-1))
-            self.assertTrue(corr < float(1))
-
-
-class ComparisonTestCase(unittest.TestCase):
-
-    def test_comparison(self):
-        backend = 'pt'
-        compare_model_names = ['vgg16_bn', 'vgg19_bn']
-        compare_module_names = ['features.23', 'classifier.3']
-        corr_mat = vision.compare_models(
-            root=TEST_PATH,
-            out_path=OUT_PATH,
-            model_names=compare_model_names,
-            module_names=compare_module_names,
-            pretrained=True,
-            batch_size=BATCH_SIZE,
-            backend=backend,
-            flatten_acts=True,
-            clip=CLIP,
-            save_features=False,
-        )
-        self.assertTrue(isinstance(corr_mat, pd.DataFrame))
-        self.assertEqual(corr_mat.shape, (len(
-            compare_model_names), len(compare_module_names)))
-
-
-class LoadItemsTestCase(unittest.TestCase):
-
-    def test_item_name_loading(self):
-        item_names = vision.load_item_names(DATA_PATH)
-        self.assertTrue(isinstance(item_names, np.ndarray))
-        self.assertEqual(len(item_names), NUM_OBJECTS)
-
-
-class FileNamesTestCase(unittest.TestCase):
-
-    def test_filenames(self):
-        file_names = open(os.path.join(OUT_PATH, 'file_names.txt'), 'r').read().split()
-        img_files = []
-        for root, _, files in os.walk(TEST_PATH):
-            for f in files:
-                if f.endswith('png'):
-                    img_files.append(os.path.join(root, f))
-        self.assertEqual(sorted(file_names), sorted(img_files))
-
-
-def create_test_images(n_samples: int) -> None:
-    """Create an artificial image dataset to be used for performing tests."""
-    if not os.path.exists(OUT_PATH):
-        os.makedirs(OUT_PATH)
-    test_img_1 = skimage.data.hubble_deep_field()
-    test_img_2 = skimage.data.coffee()
-    test_imgs = list(map(lambda x: x / x.max(), [test_img_1, test_img_2]))
-
-    classes = ['hubble', 'coffee']
-    for cls in classes:
-        PATH = os.path.join(TEST_PATH, cls)
-        if not os.path.exists(PATH):
-            os.makedirs(PATH)
-
-    for i in range(n_samples):
-        if i > n_samples // 2:
-            test_img = np.copy(test_imgs[0])
-            cls = classes[0]
-        else:
-            test_img = np.copy(test_imgs[1])
-            cls = classes[1]
-        H, W, C = test_img.shape
-        # add random Gaussian noise to test image
-        noisy_img = test_img + np.random.randn(H, W, C)
-        noisy_img = noisy_img.astype(np.uint8)
-        imageio.imsave(os.path.join(
-            TEST_PATH, cls, f'test_img_{i+1:03d}.png'), noisy_img)
-    print('\n...Successfully created image dataset for testing.\n')
-
-
-if __name__ == '__main__':
-    create_test_images(NUM_SAMPLES)
-    unittest.main()
-    shutil.rmtree(TEST_PATH)
-    shutil.rmtree(OUT_PATH)