Repository: h1alexbel/samples-filter (language: python)

Issue:
Example of repository as encoded vector



Discussion / hints:

Fix patch:
diff --git a/models/Makefile b/models/Makefile
index a0aa0ef..2e62d41 100644
--- a/models/Makefile
+++ b/models/Makefile
@@ -22,7 +22,7 @@
 
 .SHELLFLAGS: -e -o pipefail -c
 .ONESHELL:
-.PHONY: install test $(DATA_FOLDER)/clean $(DATA_FOLDER)/dataset trainrf transformer
+.PHONY: install test if transformer
 .SILENT:
 
 ## The shell to use.
@@ -44,13 +44,9 @@ test:
 #  We need to find out how to properly train isolation forest model with
 #  gathered data in order to detect anomalies (SRs). Don't forget to remove
 #  this puzzle.
-isolation-forest:
+if:
 	echo "training model based on isolation forest algorithm..."
 
-# Train model with Random-Forest algorithm.
-random-forest:
-	python3 random-forest.py
-
 # Train text transformer.
 transformer:
 	python3 transformer.py
diff --git a/models/model/embeddings.py b/models/model/embeddings.py
new file mode 100644
index 0000000..8b0d289
--- /dev/null
+++ b/models/model/embeddings.py
@@ -0,0 +1,60 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2024 Aliaksei Bialiauski
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+from transformers import BertTokenizer, BertModel
+import torch
+
+"""
+Generate embeddings for a set of tokens.
+"""
+
+
+class Embeddings:
+    def __init__(self, tokens, length, encoder="bert-base-uncased"):
+        self.tokens = tokens
+        self.length = length
+        self.tokenizer = BertTokenizer.from_pretrained(encoder)
+        self.model = BertModel.from_pretrained(encoder)
+
+    def embed(self):
+        print(f"Generating embeddings for {self.tokens}")
+        print(f"Encoder: {self.tokenizer}, {self.model}, output length: {self.length}")
+        inputs = []
+        masks = []
+        for tokens in self.tokens:
+            ids = self.tokenizer.encode_plus(
+                tokens,
+                add_special_tokens=True,
+                return_tensors='pt',
+                padding='max_length',
+                truncation=True,
+                max_length=self.length
+            )
+            inputs.append(ids["input_ids"])
+            masks.append(ids["attention_mask"])
+        inputs = torch.cat(inputs, dim=0)
+        masks = torch.cat(masks, dim=0)
+        with torch.no_grad():
+            outputs = self.model(inputs, attention_mask=masks)
+            states = outputs.last_hidden_state
+        embeddings = states[0].numpy()
+        print(f"Generated embeddings {embeddings}")
+        return embeddings
diff --git a/models/model/vector.py b/models/model/vector.py
new file mode 100644
index 0000000..61d3072
--- /dev/null
+++ b/models/model/vector.py
@@ -0,0 +1,43 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2024 Aliaksei Bialiauski
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import numpy as np
+
+"""
+Vectorizes the embedding matrixes with a scalar values.
+"""
+
+
+class Vector:
+    def __init__(self, *matrixes, **scalars):
+        self.matrixes = [np.array(mat) for mat in matrixes]
+        self.scalars = {key: val for key, val in scalars.items()}
+
+    def plain(self):
+        flattened = []
+        for matrix in self.matrixes:
+            flattened.append(matrix.flatten())
+        vector = np.concatenate(flattened)
+        scals = []
+        for scalar in self.scalars.values():
+            scals.append(np.reshape(scalar, (-1,)).astype(float))
+        values = np.concatenate(scals)
+        return np.concatenate((vector, values))
diff --git a/models/requirements.txt b/models/requirements.txt
index 62fb64d..60094fe 100644
--- a/models/requirements.txt
+++ b/models/requirements.txt
@@ -1,2 +1,5 @@
 nltk==3.8.1
 pytest==8.2.0
+transformers==4.40.0
+torch==2.2.2
+numpy>=1.22.2


Test patch:
diff --git a/models/model_tests/test_embeddings.py b/models/model_tests/test_embeddings.py
new file mode 100644
index 0000000..547c216
--- /dev/null
+++ b/models/model_tests/test_embeddings.py
@@ -0,0 +1,43 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2024 Aliaksei Bialiauski
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import unittest
+
+from model.embeddings import Embeddings
+
+"""
+Test cases for Embeddings. 
+"""
+
+
+class TestEmbeddings(unittest.TestCase):
+
+    def test_generates_embeddings_for_tokens(self):
+        shape = Embeddings(
+            ["apache", "kafka", "examples", "learning"],
+            4
+        ).embed().shape
+        expected = (4, 768)
+        self.assertEqual(
+            shape,
+            expected,
+            f"received matrix's shape {shape} does not match with expected {expected}"
+        )
diff --git a/models/model_tests/test_pre_name.py b/models/model_tests/test_pre_name.py
index cb1ba25..d30a49a 100644
--- a/models/model_tests/test_pre_name.py
+++ b/models/model_tests/test_pre_name.py
@@ -33,12 +33,7 @@ class TestPreName(unittest.TestCase):
     def test_preprocesses_name(self):
         input = "streaming-with-flink/examples-java"
         tokens = PreName(input).tokens()
-        expected = [
-            "streaming",
-            "flink",
-            "example",
-            "java"
-        ]
+        expected = ["streaming", "flink", "example", "java"]
         self.assertEqual(
             tokens,
             expected,
diff --git a/models/model_tests/test_vector.py b/models/model_tests/test_vector.py
new file mode 100644
index 0000000..119894b
--- /dev/null
+++ b/models/model_tests/test_vector.py
@@ -0,0 +1,48 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2024 Aliaksei Bialiauski
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import unittest
+
+from model.vector import Vector
+
+"""
+Test cases for Vector.
+"""
+
+
+class TestVector(unittest.TestCase):
+
+    def test_vectorizes_embeddings_and_scalars(self):
+        plain = Vector(
+            [[1.1, 2.1], [3.1, 4.1]],
+            [[1.2, 2.2], [3.2, 4.2]],
+            cpd=1,
+            rc=0.01,
+            ic=0.5
+        ).plain().tolist()
+        expected = [
+            1.1, 2.1, 3.1, 4.1, 1.2, 2.2, 3.2, 4.2, 1, 0.01, 0.5
+        ]
+        self.assertEqual(
+            plain,
+            expected,
+            f"received vector {plain} does not match with expected {expected}"
+        )