Repository: hCaptcha/hmt-basemodels

Issue:
add preprocessing schema
add basic preprocessing schema for labeling request image preprocessing configuration

Discussion:

Patch:
diff --git a/basemodels/manifest/data/preprocess.py b/basemodels/manifest/data/preprocess.py
new file mode 100644
index 0000000..8ba5827
--- /dev/null
+++ b/basemodels/manifest/data/preprocess.py
@@ -0,0 +1,12 @@
+from schematics.models import Model
+from schematics.types import StringType, DictType, UnionType, IntType, FloatType
+
+class Preprocess(Model):
+    pipeline = StringType(required=True,choices=["FaceBlurPipeline"])
+    config = DictType(UnionType([FloatType, IntType, StringType]))
+
+    def to_dict(self):
+        p = { "pipeline": self.pipeline }
+        if self.config is not None:
+            p["config"] = self.config
+        return p
\ No newline at end of file
diff --git a/basemodels/manifest/manifest.py b/basemodels/manifest/manifest.py
index ba082d7..0da9ca2 100644
--- a/basemodels/manifest/manifest.py
+++ b/basemodels/manifest/manifest.py
@@ -9,6 +9,7 @@ from schematics.types import StringType, DecimalType, BooleanType, IntType, Dict
 
 from .data.groundtruth import validate_groundtruth_entry
 from .data.taskdata import validate_taskdata_entry
+from .data.preprocess import Preprocess
 
 BASE_JOB_TYPES = [
     "image_label_binary",
diff --git a/basemodels/pydantic/__init__.py b/basemodels/pydantic/__init__.py
index 66c6f10..fb3f851 100644
--- a/basemodels/pydantic/__init__.py
+++ b/basemodels/pydantic/__init__.py
@@ -1,3 +1,4 @@
 from .manifest import validate_manifest_uris, Manifest, NestedManifest, RequestConfig, TaskData, Webhook
 from .manifest.data import validate_taskdata_entry, validate_groundtruth_entry
 from .via import ViaDataManifest
+from .manifest.data.preprocess import Pipeline, Preprocess
\ No newline at end of file
diff --git a/basemodels/pydantic/manifest/data/preprocess.py b/basemodels/pydantic/manifest/data/preprocess.py
new file mode 100644
index 0000000..32f8667
--- /dev/null
+++ b/basemodels/pydantic/manifest/data/preprocess.py
@@ -0,0 +1,16 @@
+import enum
+import typing
+import pydantic
+
+class Pipeline(str, enum.Enum):
+    FaceBlurPipeline = 'FaceBlurPipeline'
+
+class Preprocess(pydantic.BaseModel):
+    pipeline: Pipeline
+    config: typing.Optional[dict]
+
+    def to_dict(self):
+        p = { "pipeline": self.pipeline.value }
+        if self.config is not None:
+            p["config"] = self.config
+        return p
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
index 1fbdcc4..8156772 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "hmt-basemodels"
-version = "0.1.1"
+version = "0.1.2"
 description = ""
 authors = ["Intuition Machines, Inc <support@hcaptcha.com>"]
 packages = [
diff --git a/setup.py b/setup.py
index 4ff40fd..264080f 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ import setuptools
 
 setuptools.setup(
     name="hmt-basemodels",
-    version="0.1.1",
+    version="0.1.2",
     author="HUMAN Protocol",
     description="Common data models shared by various components of the Human Protocol stack",
     url="https://github.com/hCaptcha/hmt-basemodels",


Test patch:
diff --git a/tests/test_preprocess.py b/tests/test_preprocess.py
new file mode 100644
index 0000000..3f3776b
--- /dev/null
+++ b/tests/test_preprocess.py
@@ -0,0 +1,41 @@
+import unittest
+
+from schematics.exceptions import DataError
+from basemodels.manifest import Preprocess
+
+class PipelineTest(unittest.TestCase):
+    def test_preprocess(self):
+        config = {}
+        p = Preprocess({"pipeline": "FaceBlurPipeline", "config": config})
+
+        self.assertEqual(p.pipeline, "FaceBlurPipeline")
+        self.assertEqual(p.config, config)
+
+        p = Preprocess({"pipeline": "FaceBlurPipeline"})
+
+        self.assertIsNone(p.config)
+
+
+    def test_preprocess_raise(self):
+        with self.assertRaises(DataError):
+            Preprocess().validate()
+
+        with self.assertRaises(DataError):
+            Preprocess({"pipeline": ""}).validate()
+
+        with self.assertRaises(DataError):
+            Preprocess({"pipeline": "FaceBlurPipeline", "config": 1}).validate()
+
+
+    def test_preprocess_to_dict(self):
+        config = { "radius": 3 }
+        p = Preprocess({"pipeline": "FaceBlurPipeline", "config": config})
+
+        self.assertEqual(p.to_dict(), { "pipeline": "FaceBlurPipeline", "config": config })
+
+        p = Preprocess({"pipeline": "FaceBlurPipeline"})
+
+        self.assertEqual(p.to_dict(), { "pipeline": "FaceBlurPipeline" })
+
+
+
diff --git a/tests/test_pydantic_preprocess.py b/tests/test_pydantic_preprocess.py
new file mode 100644
index 0000000..4731b35
--- /dev/null
+++ b/tests/test_pydantic_preprocess.py
@@ -0,0 +1,41 @@
+import unittest
+
+from pydantic.error_wrappers import ValidationError
+from basemodels.pydantic import Preprocess, Pipeline
+
+class PipelineTest(unittest.TestCase):
+    def test_preprocess(self):
+        config = {}
+        p = Preprocess(pipeline=Pipeline.FaceBlurPipeline, config=config)
+
+        self.assertEqual(p.pipeline, Pipeline.FaceBlurPipeline)
+        self.assertEqual(p.config, config)
+
+        p = Preprocess(pipeline=Pipeline.FaceBlurPipeline)
+
+        self.assertIsNone(p.config)
+
+
+    def test_preprocess_raise(self):
+        with self.assertRaises(ValidationError):
+            Preprocess()
+
+        with self.assertRaises(ValidationError):
+            Preprocess(pipeline="")
+
+        with self.assertRaises(ValidationError):
+            Preprocess(pipeline=Pipeline.FaceBlurPipeline, config=1)
+
+
+    def test_preprocess_to_dict(self):
+        config = { "radius": 3 }
+        p = Preprocess(pipeline=Pipeline.FaceBlurPipeline, config=config)
+
+        self.assertEqual(p.to_dict(), { "pipeline": Pipeline.FaceBlurPipeline.value, "config": config })
+
+        p = Preprocess(pipeline=Pipeline.FaceBlurPipeline)
+
+        self.assertEqual(p.to_dict(), { "pipeline": Pipeline.FaceBlurPipeline.value })
+
+
+