Repository: hCaptcha/hmt-basemodels (language: python)

Issue:
Migrate Manifest model from schematics to pydantic
Goal here is to duplicate the Manifest model into [pydantic](https://pydantic-docs.helpmanual.io/) so that we can use this second generation of the model to expose via JSONSchema, Swagger, etc and use in various other places

- Create models in pydantic (Anything related to manifest)
- Bonus for converting over the VIA model as well (optional)
- Ensure that the existing tests work with both pydantic model as well as existing schematics
- Maintain support for old schematics library for now (as it's still in use)


Discussion / hints:
Issue Status: 1. **Open** 2. Started 3. Submitted 4. Done 

<hr>

__This issue now has a funding of 0.35 ETH (151.87 USD @ $433.91/ETH)  attached to it as part of the hCaptcha fund.__

 * If you would like to work on this issue you can 'start work' [on the Gitcoin Issue Details page](https://gitcoin.co/issue/hCaptcha/hmt-basemodels/41/100023620).
* Want to chip in? Add your own contribution [here](https://gitcoin.co/issue/hCaptcha/hmt-basemodels/41/100023620).
* Questions? Checkout <a href='https://gitcoin.co/help'>Gitcoin Help</a> or the <a href='https://chat.gitcoin.co/'>Gitcoin Chat</a>
* $464,282.60 more funded OSS Work available on the [Gitcoin Issue Explorer](https://gitcoin.co/explorer)


Fix patch:
diff --git a/Pipfile b/Pipfile
index 7585321..ab22f6e 100644
--- a/Pipfile
+++ b/Pipfile
@@ -12,6 +12,8 @@ httpretty = "*"
 schematics = "*"
 marshmallow = "*"
 requests = "*"
+pydantic = "*"
+typing-extensions = "*"
 
 [requires]
 python_version = "3.6"
diff --git a/Pipfile.lock b/Pipfile.lock
index 0a89fa7..af8ff38 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "1c4e4c8b0128cc0b1975a5e00e50d6248d8e7f33af19a10e634377bdfffbfbd5"
+            "sha256": "aec8b048c51e79027bc581380cc257bc95856b76ac8cdcb8a573c5a73acbc33d"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -18,10 +18,10 @@
     "default": {
         "certifi": {
             "hashes": [
-                "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304",
-                "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519"
+                "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3",
+                "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"
             ],
-            "version": "==2020.4.5.1"
+            "version": "==2020.6.20"
         },
         "chardet": {
             "hashes": [
@@ -32,26 +32,50 @@
         },
         "idna": {
             "hashes": [
-                "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb",
-                "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa"
+                "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
+                "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
             ],
-            "version": "==2.9"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.10"
         },
         "marshmallow": {
             "hashes": [
-                "sha256:c2673233aa21dde264b84349dc2fd1dce5f30ed724a0a00e75426734de5b84ab",
-                "sha256:f88fe96434b1f0f476d54224d59333eba8ca1a203a2695683c1855675c4049a7"
+                "sha256:67bf4cae9d3275b3fc74bd7ff88a7c98ee8c57c94b251a67b031dc293ecc4b76",
+                "sha256:a2a5eefb4b75a3b43f05be1cca0b6686adf56af7465c3ca629e5ad8d1e1fe13d"
             ],
             "index": "pypi",
-            "version": "==3.6.0"
+            "version": "==3.7.1"
+        },
+        "pydantic": {
+            "hashes": [
+                "sha256:1783c1d927f9e1366e0e0609ae324039b2479a1a282a98ed6a6836c9ed02002c",
+                "sha256:2dc946b07cf24bee4737ced0ae77e2ea6bc97489ba5a035b603bd1b40ad81f7e",
+                "sha256:2de562a456c4ecdc80cf1a8c3e70c666625f7d02d89a6174ecf63754c734592e",
+                "sha256:36dbf6f1be212ab37b5fda07667461a9219c956181aa5570a00edfb0acdfe4a1",
+                "sha256:3fa799f3cfff3e5f536cbd389368fc96a44bb30308f258c94ee76b73bd60531d",
+                "sha256:40d765fa2d31d5be8e29c1794657ad46f5ee583a565c83cea56630d3ae5878b9",
+                "sha256:418b84654b60e44c0cdd5384294b0e4bc1ebf42d6e873819424f3b78b8690614",
+                "sha256:4900b8820b687c9a3ed753684337979574df20e6ebe4227381d04b3c3c628f99",
+                "sha256:530d7222a2786a97bc59ee0e0ebbe23728f82974b1f1ad9a11cd966143410633",
+                "sha256:54122a8ed6b75fe1dd80797f8251ad2063ea348a03b77218d73ea9fe19bd4e73",
+                "sha256:6c3f162ba175678218629f446a947e3356415b6b09122dcb364e58c442c645a7",
+                "sha256:b49c86aecde15cde33835d5d6360e55f5e0067bb7143a8303bf03b872935c75b",
+                "sha256:b5b3489cb303d0f41ad4a7390cf606a5f2c7a94dcba20c051cd1c653694cb14d",
+                "sha256:cf3933c98cb5e808b62fae509f74f209730b180b1e3c3954ee3f7949e083a7df",
+                "sha256:eb75dc1809875d5738df14b6566ccf9fd9c0bcde4f36b72870f318f16b9f5c20",
+                "sha256:f769141ab0abfadf3305d4fcf36660e5cf568a666dd3efab7c3d4782f70946b1",
+                "sha256:f8af9b840a9074e08c0e6dc93101de84ba95df89b267bf7151d74c553d66833b"
+            ],
+            "index": "pypi",
+            "version": "==1.6.1"
         },
         "requests": {
             "hashes": [
-                "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee",
-                "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6"
+                "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b",
+                "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"
             ],
             "index": "pypi",
-            "version": "==2.23.0"
+            "version": "==2.24.0"
         },
         "schematics": {
             "hashes": [
@@ -61,12 +85,22 @@
             "index": "pypi",
             "version": "==2.1.0"
         },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918",
+                "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c",
+                "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"
+            ],
+            "index": "pypi",
+            "version": "==3.7.4.3"
+        },
         "urllib3": {
             "hashes": [
-                "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527",
-                "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115"
+                "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a",
+                "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461"
             ],
-            "version": "==1.25.9"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
+            "version": "==1.25.10"
         }
     },
     "develop": {
diff --git a/README.md b/README.md
index 4c4b96b..6b16370 100644
--- a/README.md
+++ b/README.md
@@ -41,7 +41,47 @@ Lint the python files:
 yapf --diff ./basemodels/__init__.py ./test.py
 mypy ./basemodels/__init__.py ./test.py --ignore-missing-imports
 ```
-
+## How to use model
+Using the model (based on schematics library)
+```python
+import basemodels
+from schematics.exceptions import ValidationError
+
+model = {
+    'job_mode': 'batch',
+    'request_type': 'image_label_area_select',
+    'unsafe_content': False,
+    'task_bid_price': 1,
+    ...
+}
+manifest = basemodels.Manifest(model)
+try:
+    manifest.validate()
+except ValidationError, e:
+    print(e.messages)
+```
+  
+Using the new model (based on pydantic library)
+```python
+import basemodels.pydantic as pydantic_models
+from pydantic import ValidationError
+
+model = {
+    'job_mode': 'batch',
+    'request_type': 'image_label_area_select',
+    'unsafe_content': False,
+    'task_bid_price': 1,
+    ...
+}
+# Validate model on creation
+try:
+   manifest = pydantic_models.Manifest(**model)
+except ValidationError as e:
+   print(e.json())
+# Or creating model without validation
+manifest = pydantic_models.Manifest.construct(**model)
+# See https://pydantic-docs.helpmanual.io/usage/models/#creating-models-without-validation
+```
 ## Note for maintainers: Deploying to PyPi
 
 A build will automatically be deployed to PyPi from master if tagged with a version number.  This version number should  match the version in the `setup.py` file.
@@ -67,3 +107,5 @@ Open an issue!
 ## License
 
 MIT © HUMAN Protocol 2020
+
+
diff --git a/basemodels/pydantic/__init__.py b/basemodels/pydantic/__init__.py
new file mode 100644
index 0000000..66c6f10
--- /dev/null
+++ b/basemodels/pydantic/__init__.py
@@ -0,0 +1,3 @@
+from .manifest import validate_manifest_uris, Manifest, NestedManifest, RequestConfig, TaskData, Webhook
+from .manifest.data import validate_taskdata_entry, validate_groundtruth_entry
+from .via import ViaDataManifest
diff --git a/basemodels/pydantic/manifest/__init__.py b/basemodels/pydantic/manifest/__init__.py
new file mode 100644
index 0000000..fb2e078
--- /dev/null
+++ b/basemodels/pydantic/manifest/__init__.py
@@ -0,0 +1,1 @@
+from .manifest import Manifest, NestedManifest, RequestConfig, TaskData, Webhook, validate_manifest_uris
diff --git a/basemodels/pydantic/manifest/data/__init__.py b/basemodels/pydantic/manifest/data/__init__.py
new file mode 100644
index 0000000..8f6f1be
--- /dev/null
+++ b/basemodels/pydantic/manifest/data/__init__.py
@@ -0,0 +1,2 @@
+from .groundtruth import validate_groundtruth_entry
+from .taskdata import validate_taskdata_entry
diff --git a/basemodels/pydantic/manifest/data/groundtruth.py b/basemodels/pydantic/manifest/data/groundtruth.py
new file mode 100644
index 0000000..f80f67e
--- /dev/null
+++ b/basemodels/pydantic/manifest/data/groundtruth.py
@@ -0,0 +1,89 @@
+from typing_extensions import Literal
+from typing import Dict, Union
+from pydantic import BaseModel, HttpUrl, validate_model
+from typing import List, Optional
+
+
+def create_wrapper_model(type):
+    class WrapperModel(BaseModel):
+            data: Optional[type]
+            class Config:
+                    arbitrary_types_allowed = True
+    
+    return WrapperModel
+
+def validate_wrapper_model(Model, data):
+    Model.validate({"data": data})
+
+groundtruth_entry_key_type = HttpUrl
+GroundtruthEntryKeyModel = create_wrapper_model(groundtruth_entry_key_type)
+"""
+Groundtruth file format for `image_label_binary` job type:
+
+{
+  "https://domain.com/file1.jpeg": ["false", "false", "false"],
+  "https://domain.com/file2.jpeg": ["true", "true", "true"]
+}
+"""
+ilb_groundtruth_entry_type = List[Literal["true", "false"]]
+ILBGroundtruthEntryModel = create_wrapper_model(ilb_groundtruth_entry_type)
+"""
+Groundtruth file format for `image_label_multiple_choice` job type:
+
+{
+  "https://domain.com/file1.jpeg": [
+    ["cat"],
+    ["cat"],
+    ["cat"]
+  ],
+  "https://domain.com/file2.jpeg": [
+    ["dog"],
+    ["dog"],
+    ["dog"]
+  ]
+}
+"""
+ilmc_groundtruth_entry_type = List[List[str]]
+ILMCGroundtruthEntryModel = create_wrapper_model(ilmc_groundtruth_entry_type)
+
+
+class ILASGroundtruthEntry(BaseModel):
+    entity_name: float
+    entity_type: str
+    entity_coords: List[Union[int, float]]
+
+
+"""
+Groundtruth file format for `image_label_area_select` job type
+
+{
+  "https://domain.com/file1.jpeg": [
+    [
+      {
+        "entity_name": 0,
+        "entity_type": "gate",
+        "entity_coords": [275, 184, 454, 183, 453, 366, 266, 367]
+      }
+    ]
+  ]
+}
+"""
+ilas_groundtruth_entry_type = List[List[ILASGroundtruthEntry]]
+ILASGroundtruthEntryModel = create_wrapper_model(ilas_groundtruth_entry_type)
+
+groundtruth_entry_models_map = {
+    "image_label_binary": ILBGroundtruthEntryModel,
+    "image_label_multiple_choice": ILMCGroundtruthEntryModel,
+    "image_label_area_select": ILASGroundtruthEntryModel
+}
+
+
+def validate_groundtruth_entry(key: str, value: Union[dict, list], request_type: str):
+    """ Validate key & value of groundtruth entry based on request_type """
+    GroundtruthEntryValueModel = groundtruth_entry_models_map.get(request_type)
+
+    if GroundtruthEntryValueModel is None:
+        return
+
+    validate_wrapper_model(GroundtruthEntryKeyModel, key)
+    validate_wrapper_model(GroundtruthEntryValueModel, value)
diff --git a/basemodels/pydantic/manifest/data/taskdata.py b/basemodels/pydantic/manifest/data/taskdata.py
new file mode 100644
index 0000000..92068ce
--- /dev/null
+++ b/basemodels/pydantic/manifest/data/taskdata.py
@@ -0,0 +1,45 @@
+from typing import Dict
+from pydantic import BaseModel, HttpUrl, validate_model, ValidationError, validator
+from uuid import UUID
+from typing import Optional
+
+# New type
+class AtLeastTenCharUrl(HttpUrl):
+    min_length = 10
+
+class TaskDataEntry(BaseModel):
+    """
+    Taskdata file format:
+
+    [
+      {
+        "task_key": "407fdd93-687a-46bb-b578-89eb96b4109d",
+        "datapoint_uri": "https://domain.com/file1.jpg",
+        "datapoint_hash": "f4acbe8562907183a484498ba901bfe5c5503aaa"
+      },
+      {
+        "task_key": "20bd4f3e-4518-4602-b67a-1d8dfabcce0c",
+        "datapoint_uri": "https://domain.com/file2.jpg",
+        "datapoint_hash": "f4acbe8562907183a484498ba901bfe5c5503aaa"
+      }
+    ]
+    """
+    task_key: Optional[UUID]
+    datapoint_uri: HttpUrl
+
+    @validator("datapoint_uri", always=True)
+    def validate_datapoint_uri(cls, value):
+        if len(value) < 10:
+            raise ValidationError("datapoint_uri need to be at least 10 char length.")
+
+    datapoint_hash: Optional[str]
+
+
+def validate_taskdata_entry(value: dict):
+    """ Validate taskdata entry """
+    if not isinstance(value, dict):
+        raise ValidationError("taskdata entry should be dict", TaskDataEntry())
+
+    *_, validation_error = validate_model(TaskDataEntry, value)
+    if validation_error:
+          raise validation_error
diff --git a/basemodels/pydantic/manifest/manifest.py b/basemodels/pydantic/manifest/manifest.py
new file mode 100644
index 0000000..5c63e3a
--- /dev/null
+++ b/basemodels/pydantic/manifest/manifest.py
@@ -0,0 +1,368 @@
+import uuid
+import requests
+from requests.exceptions import RequestException
+from typing_extensions import Literal
+from typing import Dict, Callable, Any, Union, Type, List, Optional
+from enum import Enum
+from uuid import UUID
+from .data.groundtruth import validate_groundtruth_entry
+from .data.taskdata import validate_taskdata_entry
+from pydantic import BaseModel, validator, ValidationError, validate_model, HttpUrl
+from pydantic.fields import Field
+from decimal import Decimal
+
+# Validator function for taskdata and taskdata_uri fields
+def validator_taskdata_uri(cls, value, values, **kwargs):
+    taskdata = values.get('taskdata')
+    taskdata_uri = values.get('taskdata_uri')
+    if taskdata is not None and len(taskdata) > 0 and taskdata_uri is not None:
+        raise ValidationError(u'Specify only one of taskdata {} or taskdata_uri {}'.format(taskdata, taskdata_uri))
+    return value
+
+# A validator function for UUID fields
+def validate_uuid(cls, value):
+    return value or uuid.uuid4()
+
+# Base job types
+class BaseJobTypesEnum(str, Enum):
+    image_label_binary = "image_label_binary"
+    image_label_multiple_choice = "image_label_multiple_choice"
+    text_free_entry = "text_free_entry"
+    text_multiple_choice_one_option = "text_multiple_choice_one_option"
+    text_multiple_choice_multiple_options = "text_multiple_choice_multiple_options"
+    image_label_area_adjust = "image_label_area_adjust"
+    image_label_area_select = "image_label_area_select"
+    image_label_single_polygon = "image_label_single_polygon"
+    image_label_multiple_polygons = "image_label_multiple_polygons"
+    image_label_semantic_segmentation_one_option = "image_label_semantic_segmentation_one_option"
+    image_label_semantic_segmentation_multiple_options = "image_label_semantic_segmentation_multiple_options"
+    image_label_text = "image_label_text"
+    multi_challenge = "multi_challenge"
+
+# Return a request type validator function
+class RequestTypeValidator(object):
+    def __init__(self, multi_challenge: bool = True):
+        self.multi_challenge = multi_challenge
+    def validate(self, cls, value, values):
+        """
+        validate request types for all types of challenges
+        multi_challenge should always have multi_challenge_manifests
+        """
+        if value == BaseJobTypesEnum.multi_challenge:
+            if not self.multi_challenge:
+                raise ValidationError("multi_challenge request is not allowed here.")
+            if "multi_challenge_manifests" not in values:
+                raise ValidationError("multi_challenge requires multi_challenge_manifests.")
+        elif value in [BaseJobTypesEnum.image_label_multiple_choice, BaseJobTypesEnum.image_label_area_select]:
+            if values.get('multiple_choice_min_choices', 1) > values.get('multiple_choice_max_choices', 1):
+                raise ValidationError(
+                    "multiple_choice_min_choices cannot be greater than multiple_choice_max_choices")
+        return value
+
+# Shape types enum
+class ShapeTypes(str, Enum):
+    point = "point"
+    bounding_box = "bounding_box"
+    polygon = "polygon"
+
+class Model(BaseModel):
+    def to_primitive(self):
+        return self.dict()
+
+    # Helper function for using in the unittest
+    def check(self, return_new=False):
+        self.__class__.validate(self)
+        out_dict, _, validation_error = validate_model(self.__class__, self.__dict__)
+        if validation_error:
+            raise validation_error
+
+        # For compatibility with tests
+        if return_new:
+            return self.__class__(**out_dict)
+
+class Webhook(Model):
+    """ Model for webhook configuration """
+    webhook_id: UUID
+    chunk_completed: Optional[List[str]]
+    job_completed: Optional[List[str]]
+
+    # States that might be interesting later
+    # job_skipped: List[str] = None
+    # job_inserted : List[str] = None
+    # job_activated : List[str] = None
+
+
+class TaskData(Model):
+    """ objects within taskdata list in Manifest """
+    task_key: UUID
+    datapoint_uri: HttpUrl
+
+    @validator("datapoint_uri", always=True)
+    def validate_datapoint_uri(cls, value):
+        if len(value) < 10:
+            raise ValidationError("datapoint_uri need to be at least 10 char length.")
+
+    datapoint_hash: str = Field(..., min_length=10, strip_whitespace=True)
+
+
+class RequestConfig(Model):
+    """ definition of the request_config object in manifest """
+    version: int = 0
+    shape_type: Optional[ShapeTypes]
+    min_points: Optional[int]
+    max_points: Optional[int]
+    min_shapes_per_image: Optional[int]
+    max_shapes_per_image: Optional[int]
+    restrict_to_coords: Optional[bool]
+    minimum_selection_area_per_shape: Optional[int]
+    multiple_choice_max_choices: Optional[int] = 1
+    multiple_choice_min_choices: Optional[int] = 1
+
+
+class InternalConfig(Model):
+    """ discarded from incoming manifests """
+    exchange: Optional[Dict[str, Union[str, int, float]]]
+    reco: Optional[Dict[str, Union[str, int, float]]]
+    repo: Optional[Dict[str, Union[str, int, float]]]
+    other: Optional[Dict[str, Union[str, int, float]]]
+    # Accept one layer of nested
+    mitl: Optional[Dict[str, Union[str, int, float, Dict[str, Union[str, int, float]]]]]
+
+    class Config:
+        arbitrary_types_allowed = True
+
+
+class NestedManifest(Model):
+    """ The nested manifest description for multi_challenge jobs """
+    # We will set a default dynamic value for job_id
+    job_id: Optional[UUID]
+    validate_job_id = validator('job_id', always=True, allow_reuse=True)(validate_uuid)
+
+    request_type: BaseJobTypesEnum
+    validate_request_type = validator('request_type', allow_reuse=True, \
+            always=True)(RequestTypeValidator(multi_challenge=False).validate)
+    requester_restricted_answer_set: Optional[Dict[str, Dict[str, str]]]
+
+    @validator('requester_restricted_answer_set', always=True)
+    def validate_requester_restricted_answer_set(cls, value, values, **kwargs):
+        """image_label_area_select should always have a single RAS set"""
+
+        # validation runs before other params, so need to handle missing case
+        if "request_type" not in values:
+            raise ValidationError("request_type missing")
+        if values['request_type'] == BaseJobTypesEnum.image_label_area_select:
+            if not value or len(value.keys()) == 0:
+                value = {'label': {}}
+                values['requester_restricted_answer_set'] = value
+        return value
+
+    requester_description: Optional[str]
+    requester_max_repeats: int = 100
+    requester_min_repeats: int = 1
+    requester_question: Optional[Dict[str, str]]
+    requester_question_example: Optional[Union[HttpUrl, List[HttpUrl]]]
+
+    @validator('requester_question_example')
+    def validate_requester_question_example(cls, value, values, **kwargs):
+        # validation runs before other params, so need to handle missing case
+        if not ("request_type" in values):
+            raise ValidationError("request_type missing")
+
+        # based on https://github.com/hCaptcha/hmt-basemodels/issues/27#issuecomment-590706643
+        supports_lists = [
+            BaseJobTypesEnum.image_label_area_select, BaseJobTypesEnum.image_label_binary
+        ]
+
+        if isinstance(value, list) and not values['request_type'] in supports_lists:
+            raise ValidationError("Lists are not allowed in this challenge type")
+        return value
+
+    unsafe_content: bool = False
+    requester_accuracy_target: float = 0.1
+
+    request_config: Optional[RequestConfig]
+
+    # Groundtruth data is stored as a URL or optionally as an inlined json-serialized stringtype
+    groundtruth_uri: Optional[HttpUrl]
+    groundtruth: Optional[str]
+
+    @validator('groundtruth', always=True)
+    def validate_groundtruth(cls, v, values, **kwargs):
+        if "groundtruth_uri" in values and "groundtruth" in values:
+            raise ValidationError("Specify only groundtruth_uri or groundtruth, not both.")
+        return v
+
+    # Configuration id -- XXX LEGACY
+    confcalc_configuration_id: Optional[str]
+    webhook: Optional[Webhook]
+
+    class Config:
+        arbitrary_types_allowed = True
+
+
+class Manifest(Model):
+    """ The manifest description. """
+    job_mode: Literal["batch", "online", "instant_delivery"]
+    
+    # We will set a default dynamic value for job_api_key
+    job_api_key: Optional[UUID]
+    validate_job_api_key = validator('job_api_key', always=True, allow_reuse=True)(validate_uuid)
+
+    # We will set a default dynamic value for job_id
+    job_id: Optional[UUID]
+    validate_job_id = validator('job_id', always=True, allow_reuse=True)(validate_uuid)
+
+    job_total_tasks: Optional[int]
+    multi_challenge_manifests: Optional[List[NestedManifest]]
+    request_type: BaseJobTypesEnum
+    validate_request_type = validator('request_type', allow_reuse=True, \
+            always=True)(RequestTypeValidator().validate)
+
+    requester_restricted_answer_set: Optional[Dict[str, Dict[str, str]]]
+
+    @validator('requester_restricted_answer_set', always=True)
+    def validate_requester_restricted_answer_set(cls, value, values, **kwargs):
+        """image_label_area_select should always have a single RAS set"""
+        # validation runs before other params, so need to handle missing case
+        if not ("request_type" in values):
+            raise ValidationError("request_type missing")
+
+        if values['request_type'] == BaseJobTypesEnum.image_label_area_select:
+            if not value or len(value.keys()) == 0:
+                value = {'label': {}}
+                values['requester_restricted_answer_set'] = value
+        return value
+
+    requester_description: Optional[str]
+    requester_max_repeats: int = 100
+    requester_min_repeats: int = 1
+    requester_question: Optional[Dict[str, str]]
+
+    requester_question_example: Optional[Union[HttpUrl, List[HttpUrl]]]
+
+    @validator('requester_question_example')
+    def validate_requester_question_example(cls, value, values, **kwargs):
+        # validation runs before other params, so need to handle missing case
+        if not ("request_type" in values):
+            raise ValidationError("request_type missing")
+
+        # based on https://github.com/hCaptcha/hmt-basemodels/issues/27#issuecomment-590706643
+        supports_lists = [
+            BaseJobTypesEnum.image_label_area_select, BaseJobTypesEnum.image_label_binary
+        ]
+
+        if isinstance(value, list) and not (values['request_type'] in supports_lists):
+            raise ValidationError("Lists are not allowed in this challenge type")
+        return value
+
+    unsafe_content: bool = False
+    task_bid_price: Decimal
+    oracle_stake: Decimal
+    expiration_date: Optional[int]
+    requester_accuracy_target: float = 0.1
+    manifest_smart_bounty_addr: Optional[str]
+    hmtoken_addr: Optional[str]
+    minimum_trust_server: float = 0.1
+    minimum_trust_client: float = 0.1
+    recording_oracle_addr: str
+    reputation_oracle_addr: str
+    reputation_agent_addr: str
+    requester_pgp_public_key: Optional[str]
+    ro_uri: Optional[str]
+    repo_uri: Optional[str]
+    batch_result_delivery_webhook: Optional[HttpUrl]
+    online_result_delivery_webhook: Optional[HttpUrl]
+    instant_result_delivery_webhook: Optional[HttpUrl]
+
+    request_config: Optional[RequestConfig]
+
+    # If taskdata is directly provided
+    taskdata: Optional[List[TaskData]]
+
+    # If taskdata is separately stored
+    taskdata_uri: Optional[HttpUrl]
+
+    # Groundtruth data is stored as a URL or optionally as an inlined json-serialized stringtype
+    groundtruth_uri: Optional[HttpUrl]
+    groundtruth: Optional[str]
+
+    @validator('groundtruth', always=True)
+    def validate_groundtruth(cls, value, values):
+        if "groundtruth_uri" in values and "groundtruth" in values:
+            raise ValidationError("Specify only groundtruth_uri or groundtruth, not both.")
+        return value
+
+    # internal config options for param tests etc.
+    internal_config: Optional[InternalConfig]
+
+    # Configuration id -- XXX LEGACY
+    confcalc_configuration_id: Optional[str]
+    restricted_audience: Optional[Dict[str, Union[float, List[Dict[str, Dict[str, float]]]]]]
+
+    validate_taskdata_uri = validator('taskdata_uri', allow_reuse=True, always=True)(validator_taskdata_uri)
+    validate_taskdata = validator('taskdata', allow_reuse=True, always=True)(validator_taskdata_uri)
+    webhook: Optional[Webhook]
+
+    class Config:
+        arbitrary_types_allowed = True
+
+
+def validate_groundtruth_uri(manifest: dict):
+    """
+    Validate groundtruth_uri 
+    Returns entries count if succeeded
+    """
+    request_type = manifest.get('request_type', '')
+    uri_key = "groundtruth_uri"
+    uri = manifest.get(uri_key)
+    if uri is None:
+        return
+    try:
+        response = requests.get(uri, timeout=1)
+        response.raise_for_status()
+
+        entries_count = 0
+        data = response.json()
+        if isinstance(data, dict):
+            for k, v in data.items():
+                entries_count += 1
+                validate_groundtruth_entry(k, v, request_type)
+        else:
+            for v in data:
+                entries_count += 1
+                validate_groundtruth_entry("", v, request_type)
+
+    except (ValidationError, RequestException) as e:
+        raise ValidationError(f"{uri_key} validation failed: {e}", Manifest) from e
+
+    if entries_count == 0:
+        raise ValidationError(f"fetched {uri_key} is empty", Manifest)
+
+def validate_taskdata_uri(manifest: dict):
+    """
+    Validate taskdata_uri 
+    Returns entries count if succeeded
+    """
+    uri_key = "taskdata_uri"
+    uri = manifest.get(uri_key)
+    if uri is None:
+        return
+    try:
+        response = requests.get(uri, timeout=1)
+        response.raise_for_status()
+
+        entries_count = 0
+        data = response.json()
+        for v in data:
+            entries_count += 1
+            validate_taskdata_entry(v)
+    except (ValidationError, RequestException) as e:
+        raise ValidationError(f"{uri_key} validation failed: {e}", Manifest) from e
+
+    if entries_count == 0:
+        raise ValidationError(f"fetched {uri_key} is empty", Manifest)
+
+def validate_manifest_uris(manifest: dict):
+    """ Fetch & validate manifest's remote objects """
+    validate_taskdata_uri(manifest)
+    validate_groundtruth_uri(manifest)
diff --git a/basemodels/pydantic/via.py b/basemodels/pydantic/via.py
new file mode 100644
index 0000000..d7d52c0
--- /dev/null
+++ b/basemodels/pydantic/via.py
@@ -0,0 +1,58 @@
+"""
+The VIA Format intends to define a standard HMT format for data files that contain results
+and the annotations associated with that image data.  Each file takes the general form of:
+
+{
+    "datapoints": [{
+        "task_uri": "https://mydomain.com/image.jpg",
+        "metadata": {
+            "filename": "image.jpg"
+        },
+        "class_attributes": {
+            "0": {
+                # This nested class_attributes field will soon be deprecated
+                "class_attributes": {
+                    "dog": False,
+                    "cat": False
+                }
+            }
+        },
+        "regions": [{
+            "region_attributes": {
+                "region_key": "region_value"
+            },
+            "shape_attributes": {
+                "coords": [x1, y1, x2, y2, x3, y3, x4, y4],
+                "name": "shape_type"
+            }
+        }],
+    }]
+}
+
+
+"""
+from typing import Dict, List, Any, Optional, Union
+from pydantic import BaseModel, HttpUrl
+
+
+class RegionAttributesBaseModel(BaseModel):
+    region_attributes: Dict[str, Any]
+    shape_attributes: Dict[str, Any]
+
+
+class ClassAttributeBaseModel(BaseModel):
+    # inner class attributes field not required, here for legacy purposes
+    class_attributes: Optional[Dict[str, Any]]
+    regions: Optional[List[RegionAttributesBaseModel]]
+
+
+class DatapointBaseModel(BaseModel):
+    task_uri: HttpUrl
+    metadata: Optional[Dict[str, Any]]
+    class_attributes: Optional[Union[Any, Dict[str, ClassAttributeBaseModel]]]
+
+
+class ViaDataManifest(BaseModel):
+    """ Main entrypoint to define the VIA Data Format """
+    datapoints: List[DatapointBaseModel]
+    version: int = 1
diff --git a/setup.py b/setup.py
index 20a837a..4e286e7 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@
 
 setuptools.setup(
     name="hmt-basemodels",
-    version="0.0.23",
+    version="0.1.0",
     author="HUMAN Protocol",
     description="Common data models shared by various components of the Human Protocol stack",
     url="https://github.com/hCaptcha/hmt-basemodels",
@@ -13,4 +13,7 @@
         "Programming Language :: Python"
     ],
     packages=setuptools.find_packages(),
-    install_requires=["schematics>=2.1.0", "marshmallow>=3.2.1", "requests>=2"])
+    install_requires=[
+        "schematics>=2.1.0", "marshmallow>=3.2.1", "requests>=2", "typing-extensions>=3.7.4.3",
+        "pydantic>=1.6.1"
+    ])


Test patch:
diff --git a/test.py b/test.py
index 5d1e7fd..5a7c27e 100755
--- a/test.py
+++ b/test.py
@@ -1,10 +1,13 @@
 #!/usr/bin/env python3
-
+from pydantic import BaseModel, ValidationError, validator
+from typing import Any
 import logging
 import os
 import sys
 import schematics
 import basemodels
+# New pydantic model
+import basemodels.pydantic as pydantic_basemodels
 import unittest
 import httpretty
 import json
@@ -17,6 +20,59 @@
 REC_ORACLE = "0xD979105297fB0eee83F7433fC09279cb5B94fFC6"
 FAKE_ORACLE = "0x1413862c2b7054cdbfdc181b83962cb0fc11fd92"
 
+# Test both version of models
+SCHEMATICS = "schematics"
+PYDANTIC = "pydantic"
+test_modes = {SCHEMATICS: basemodels, PYDANTIC: pydantic_basemodels}
+
+# Library related errors
+validation_base_errors = {SCHEMATICS: schematics.exceptions.BaseError, PYDANTIC: ValidationError}
+
+# Library related errors
+validation_data_errors = {SCHEMATICS: schematics.exceptions.DataError, PYDANTIC: ValidationError}
+
+
+# A helper function for create manifest models based on model library
+def create_manifest(data: dict):
+    if test_mode == SCHEMATICS:
+        return basemodels.Manifest(data)
+    return pydantic_basemodels.Manifest.construct(**data)
+
+
+# A helper function for create nested manifest models based on model library
+def create_nested_manifest(data: dict):
+    if test_mode == SCHEMATICS:
+        return basemodels.NestedManifest(data)
+    return pydantic_basemodels.NestedManifest.construct(**data)
+
+
+# A helper function for create nested manifest models based on model library
+def create_webhook(data: dict):
+    if test_mode == SCHEMATICS:
+        return basemodels.Webhook(data)
+    return pydantic_basemodels.Webhook.construct(**data)
+
+
+# Json serializer for models based on libraries
+def to_json(model):
+    if test_mode == SCHEMATICS:
+        return json.dumps(model.to_primitive())
+
+    # Pydantic json serializer
+    return model.json()
+
+
+# A helper function for providing validatation function based on libraries
+def validate_func(model):
+    if test_mode == SCHEMATICS:
+        return model.validate
+    return model.check
+
+
+# To be changed in runtime
+test_mode = SCHEMATICS
+test_models = basemodels
+
 
 def a_manifest(number_of_tasks=100,
                bid_amount=1.0,
@@ -26,7 +82,7 @@ def a_manifest(number_of_tasks=100,
                request_type=IMAGE_LABEL_BINARY,
                request_config=None,
                job_mode='batch',
-               multi_challenge_manifests=None) -> basemodels.Manifest:
+               multi_challenge_manifests=None) -> Any:
     internal_config = {'exchange': {'a': 1, 'b': 'c'}}
     model = {
         'requester_restricted_answer_set': {
@@ -64,14 +120,13 @@ def a_manifest(number_of_tasks=100,
     if request_config:
         model.update({'request_config': request_config})
 
-    manifest = basemodels.Manifest(model)
-    manifest.validate()
-
+    manifest = create_manifest(model)
+    validate_func(manifest)()
     return manifest
 
 
 def a_nested_manifest(request_type=IMAGE_LABEL_BINARY, minimum_trust=.1,
-                      request_config=None) -> basemodels.Manifest:
+                      request_config=None) -> Any:
     model = {
         'requester_restricted_answer_set': {
             '0': {
@@ -93,8 +148,8 @@ def a_nested_manifest(request_type=IMAGE_LABEL_BINARY, minimum_trust=.1,
     if request_config:
         model.update({'request_config': request_config})
 
-    manifest = basemodels.NestedManifest(model)
-    manifest.validate()
+    manifest = create_nested_manifest(model)
+    validate_func(manifest)()
 
     return manifest
 
@@ -108,18 +163,18 @@ def test_basic_construction(self):
 
     def test_can_serialize(self):
         """ validate that we can dump this to json in downstream services """
-        j = json.dumps(a_manifest().to_primitive())
+        j = to_json(a_manifest())
 
     def test_can_fail_toconstruct(self):
         """Tests that the manifest raises an Error when called with falsy parameters."""
         a_manifest(-1)
-        self.assertRaises(schematics.exceptions.DataError, a_manifest, "invalid amount")
+        self.assertRaises(validation_data_errors[test_mode], a_manifest, "invalid amount")
 
     def test_can_fail_toconstruct2(self):
         """Tests that validated fields can't be broken without an exception."""
         mani = a_manifest()
         mani.taskdata_uri = 'test'
-        self.assertRaises(schematics.exceptions.DataError, mani.validate)
+        self.assertRaises(validation_data_errors[test_mode], validate_func(mani))
 
     def test_can_make_request_config_job(self):
         """Test that jobs with valid request_config parameter work"""
@@ -151,7 +206,7 @@ def test_can_bad_request_config(self):
         manifest = a_manifest()
         manifest.request_type = 'image_label_area_select'
         manifest.request_config = {'shape_type': 'not-a-real-option'}
-        self.assertRaises(schematics.exceptions.DataError, manifest.validate)
+        self.assertRaises(validation_data_errors[test_mode], validate_func(manifest))
 
     def test_gets_default_restrictedanswerset(self):
         """Make sure that the image_label_area_select jobs get a default RAS"""
@@ -176,16 +231,24 @@ def test_gets_default_restrictedanswerset(self):
             'job_total_tasks': 5,
             'taskdata_uri': FAKE_URL
         }
-        manifest = basemodels.Manifest(model)
 
-        manifest.validate()
-        self.assertGreater(len(manifest['requester_restricted_answer_set'].keys()), 0)
+        manifest = create_manifest(model)
+
+        func = validate_func(manifest)
+        # Return new object for pydantic library
+        if test_mode == PYDANTIC:
+            manifest = func(True)
+        else:
+            func()
+
+        self.assertGreater(
+            len(manifest.to_primitive()['requester_restricted_answer_set'].keys()), 0)
 
     def test_confcalc_configuration_id(self):
         """ Test that key is in manifest """
         manifest = a_manifest()
         manifest.confcalc_configuration_id = 'test_conf_id'
-        manifest.validate()
+        validate_func(manifest)()
 
         self.assertTrue("confcalc_configuration_id" in manifest.to_primitive())
 
@@ -194,21 +257,21 @@ def test_url_or_list_for_example(self):
         model = a_manifest()
 
         model.requester_question_example = "https://test.com"
-        self.assertTrue(model.validate() is None)
+        self.assertTrue(validate_func(model)() is None)
         self.assertIsInstance(model.to_primitive()['requester_question_example'], str)
 
         model.requester_question_example = ["https://test.com"]
-        self.assertTrue(model.validate() is None)
+        self.assertTrue(validate_func(model)() is None)
         self.assertIsInstance(model.to_primitive()['requester_question_example'], list)
 
         model.requester_question_example = "non-url"
-        self.assertRaises(schematics.exceptions.DataError, model.validate)
+        self.assertRaises(validation_data_errors[test_mode], validate_func(model))
         model.requester_question_example = ["non-url"]
-        self.assertRaises(schematics.exceptions.DataError, model.validate)
+        self.assertRaises(validation_data_errors[test_mode], validate_func(model))
 
         # we now allow lists in non-ilb types
         model.request_type = "image_label_area_select"
-        self.assertTrue(model.validate)
+        self.assertTrue(validate_func(model))
 
     def test_restricted_audience(self):
         """ Test that restricted audience is in the Manifest """
@@ -226,7 +289,7 @@ def test_restricted_audience(self):
             }],
             "min_difficulty": 2,
         }
-        manifest.validate()
+        validate_func(manifest)()
         self.assertTrue("restricted_audience" in manifest.to_primitive())
         self.assertTrue("minimum_client_confidence" in manifest.to_primitive()
                         ["restricted_audience"]["confidence"][0])
@@ -331,9 +394,9 @@ def test_realistic_multi_challenge_example(self):
             }]
         }
 
-        model = basemodels.Manifest(obj)
+        model = create_manifest(obj)
         # print(model.to_primitive())
-        self.assertTrue(model.validate() is None)
+        self.assertTrue(validate_func(model)() is None)
 
     def test_webhook(self):
         """ Test that webhook is correct """
@@ -342,13 +405,13 @@ def test_webhook(self):
             "job_completed": ["http://servicename:4000/api/webhook"]
         }
 
-        webhook_model = basemodels.Webhook(webhook)
-        webhook_model.validate()
+        webhook_model = create_webhook(webhook)
+        validate_func(webhook_model)()
         self.assertTrue("webhook_id" in webhook_model.to_primitive())
 
         model = a_manifest()
         model.webhook = webhook
-        model.validate()
+        validate_func(model)()
         self.assertTrue("webhook" in model.to_primitive())
 
 
@@ -381,7 +444,12 @@ def test_via_legacy_case(self):
             }]
         }
 
-        parsed = basemodels.ViaDataManifest().dump(content)
+        # Also test the marshmallow model from the old package
+        parsed: Dict
+        if test_mode == SCHEMATICS:
+            parsed = test_models.ViaDataManifest().dump(content)
+        else:
+            parsed = test_models.ViaDataManifest(**content).dict()
         self.assertEqual(len(parsed['datapoints']), 1)
         self.assertEqual(parsed['version'], 1)
 
@@ -409,7 +477,12 @@ def test_via_v1_case(self):
             }]
         }
 
-        parsed = basemodels.ViaDataManifest().dump(content)
+        # Also test the marshmallow model from the old package
+        parsed: Dict
+        if test_mode == SCHEMATICS:
+            parsed = test_models.ViaDataManifest().dump(content)
+        else:
+            parsed = test_models.ViaDataManifest(**content).dict()
         self.assertEqual(len(parsed['datapoints']), 1)
         self.assertEqual(parsed['version'], 1)
         self.assertIn('dog', parsed['datapoints'][0]['class_attributes'])
@@ -426,12 +499,12 @@ def validate_groundtruth_response(self, request_type, body):
 
         self.register_http_response(uri, manifest, body)
 
-        basemodels.validate_manifest_uris(manifest)
+        test_models.validate_manifest_uris(manifest)
 
     def test_no_uris(self):
         """ should not raise if there are no uris to validate """
         manifest = {}
-        basemodels.validate_manifest_uris(manifest)
+        test_models.validate_manifest_uris(manifest)
 
     def test_groundtruth_uri_ilb_valid(self):
         body = {
@@ -444,14 +517,14 @@ def test_groundtruth_uri_ilb_valid(self):
     def test_groundtruth_uri_ilb_invalid(self):
         body = {"not_uri": ["false", "false", True]}
 
-        with self.assertRaises(schematics.exceptions.BaseError):
+        with self.assertRaises(validation_base_errors[test_mode]):
             self.validate_groundtruth_response("image_label_binary", body)
 
     def test_groundtruth_uri_ilb_invalid_format(self):
         """ should raise if groundtruth_uri contains array instead of object """
         body = [{"key": "value"}]
 
-        with self.assertRaises(schematics.exceptions.BaseError):
+        with self.assertRaises(validation_base_errors[test_mode]):
             self.validate_groundtruth_response("image_label_binary", body)
 
     def test_groundtruth_uri_ilmc_valid(self):
@@ -465,7 +538,7 @@ def test_groundtruth_uri_ilmc_valid(self):
     def test_groundtruth_uri_ilmc_invalid_key(self):
         body = {"not_uri": [["cat"], ["cat"], ["cat"]]}
 
-        with self.assertRaises(schematics.exceptions.BaseError):
+        with self.assertRaises(validation_base_errors[test_mode]):
             self.validate_groundtruth_response("image_label_multiple_choice", body)
 
     def test_groundtruth_uri_ilmc_invalid_value(self):
@@ -473,7 +546,7 @@ def test_groundtruth_uri_ilmc_invalid_value(self):
             "https://domain.com/file1.jpeg": [True, False],
         }
 
-        with self.assertRaises(schematics.exceptions.BaseError):
+        with self.assertRaises(validation_base_errors[test_mode]):
             self.validate_groundtruth_response("image_label_multiple_choice", body)
 
     def test_groundtruth_uri_ilas_valid(self):
@@ -496,13 +569,13 @@ def test_groundtruth_uri_ilas_invalid_key(self):
             }]]
         }
 
-        with self.assertRaises(schematics.exceptions.BaseError):
+        with self.assertRaises(validation_base_errors[test_mode]):
             self.validate_groundtruth_response("image_label_area_select", body)
 
     def test_groundtruth_uri_ilas_invalid_value(self):
         body = {"https://domain.com/file1.jpeg": [[True]]}
 
-        with self.assertRaises(schematics.exceptions.BaseError):
+        with self.assertRaises(validation_base_errors[test_mode]):
             self.validate_groundtruth_response("image_label_area_select", body)
 
     def test_taskdata_empty(self):
@@ -513,8 +586,8 @@ def test_taskdata_empty(self):
 
         self.register_http_response(uri, manifest, body)
 
-        with self.assertRaises(schematics.exceptions.BaseError):
-            basemodels.validate_manifest_uris(manifest)
+        with self.assertRaises(validation_base_errors[test_mode]):
+            test_models.validate_manifest_uris(manifest)
 
     def test_taskdata_invalid_format(self):
         """ should raise if taskdata_uri contains object instead of array """
@@ -524,8 +597,8 @@ def test_taskdata_invalid_format(self):
 
         self.register_http_response(uri, manifest, body)
 
-        with self.assertRaises(schematics.exceptions.BaseError):
-            basemodels.validate_manifest_uris(manifest)
+        with self.assertRaises(validation_base_errors[test_mode]):
+            test_models.validate_manifest_uris(manifest)
 
     def test_taskdata_uri_valid(self):
         uri = "https://uri.com"
@@ -543,7 +616,7 @@ def test_taskdata_uri_valid(self):
 
         self.register_http_response(uri, manifest, body)
 
-        basemodels.validate_manifest_uris(manifest)
+        test_models.validate_manifest_uris(manifest)
 
     def test_taskdata_uri_invalid(self):
         uri = "https://uri.com"
@@ -552,8 +625,8 @@ def test_taskdata_uri_invalid(self):
 
         self.register_http_response(uri, manifest, body)
 
-        with self.assertRaises(schematics.exceptions.BaseError):
-            basemodels.validate_manifest_uris(manifest)
+        with self.assertRaises(validation_base_errors[test_mode]):
+            test_models.validate_manifest_uris(manifest)
 
     def test_groundtruth_and_taskdata_valid(self):
         taskdata_uri = "https://td.com"
@@ -583,7 +656,7 @@ def test_groundtruth_and_taskdata_valid(self):
         self.register_http_response(taskdata_uri, manifest, taskdata)
         self.register_http_response(groundtruth_uri, manifest, groundtruth)
 
-        basemodels.validate_manifest_uris(manifest)
+        test_models.validate_manifest_uris(manifest)
 
     def test_mitl_in_internal_config(self):
         """ Test that mitl config can be part of the internal configuration """
@@ -608,12 +681,15 @@ def test_mitl_in_internal_config(self):
         }
 
         model["internal_config"]["mitl"] = mitl_config
-        manifest = basemodels.Manifest(model)
-        manifest.validate()
+        manifest = create_manifest(model)
+        validate_func(manifest)()
         self.assertTrue(True)
 
 
 if __name__ == "__main__":
     logging.basicConfig()
     logging.getLogger("urllib3").setLevel(logging.INFO)
-    unittest.main()
+    for mode in test_modes:
+        test_mode = mode
+        test_models = test_modes[mode]
+        unittest.main(exit=False)