Repository: frictionlessdata/frictionless-py

Issue:
Multiprocessing introduces high overhead for small validation jobs
# Overview

Validation can now be done on just metadata and table headers, so migrating `goodtables-pandas` should now be very straightforward. Thank you. (see https://github.com/frictionlessdata/frictionless-py/issues/503, closed by #514 and #515)

```python
from frictionless import validate, Query

validate(..., query=Query(limit_rows=1), nolookup=True, noinfer=True)
```
*NOTE: `limit_rows=1` because `Query(limit_rows=0)` queries all rows, rather than no rows.*

However, I wanted to point out that with the validation process so reduced, the use of `multiprocessing.Pool` introduces a significant overhead. My test package with three resources takes 1.2 s, instead of just 0.2 s if I replace:

https://github.com/frictionlessdata/frictionless-py/blob/09cc98e1966d6f97f4eecb47757f45f8a946c5e7/frictionless/validate/inquiry.py#L54-L55

with:

```python
for task in tasks:
    reports.append(validate(**task))
```

So it may be worth considering a global or local configuration to set the max number of threads (threads > 1) or disable parallel processing altogether (threads == 1).

---

Please preserve this line to notify @roll (lead of this repository)


Comments:

Patch:
diff --git a/frictionless/validate/inquiry.py b/frictionless/validate/inquiry.py
index 4136259b..8e570d92 100644
--- a/frictionless/validate/inquiry.py
+++ b/frictionless/validate/inquiry.py
@@ -9,7 +9,7 @@ from .. import exceptions
 
 
 @Report.from_validate
-def validate_inquiry(source):
+def validate_inquiry(source, *, nopool=False):
     """Validate inquiry
 
     API      | Usage
@@ -18,6 +18,7 @@ def validate_inquiry(source):
 
     Parameters:
         source (dict|str): an inquiry descriptor
+        nopool? (bool): disable multiprocessing
 
     Returns:
         Report: validation report
@@ -44,13 +45,14 @@ def validate_inquiry(source):
             continue
         tasks.append(task)
 
-    # Validate task
-    if len(tasks) == 1:
-        report = validate(**helpers.create_options(tasks[0]))
-        reports.append(report)
+    # Validate sequentially
+    if len(tasks) == 1 or nopool:
+        for task in tasks:
+            report = validate(**helpers.create_options(task))
+            reports.append(report)
 
-    # Validate tasks
-    if len(tasks) > 1:
+    # Validate in-parallel
+    else:
         with Pool() as pool:
             reports.extend(pool.map(partial(helpers.apply_function, validate), tasks))
 
diff --git a/frictionless/validate/package.py b/frictionless/validate/package.py
index ebea2872..0fd8c9a9 100644
--- a/frictionless/validate/package.py
+++ b/frictionless/validate/package.py
@@ -8,7 +8,13 @@ from .. import exceptions
 
 @Report.from_validate
 def validate_package(
-    source, basepath=None, trusted=False, noinfer=False, nolookup=False, **options
+    source,
+    basepath=None,
+    trusted=False,
+    noinfer=False,
+    nolookup=False,
+    nopool=False,
+    **options
 ):
     """Validate package
 
@@ -22,6 +28,7 @@ def validate_package(
         trusted? (bool): don't raise an exception on unsafe paths
         noinfer? (bool): don't call `package.infer`
         nolookup? (bool): don't read lookup tables skipping integrity checks
+        nopool? (bool): disable multiprocessing
         **options (dict): options for every extracted table
 
     Returns:
@@ -62,7 +69,7 @@ def validate_package(
 
     # Validate inquiry
     inquiry = Inquiry(descriptor)
-    report = validate_inquiry(inquiry)
+    report = validate_inquiry(inquiry, nopool=nopool)
 
     # Return report
     return Report(time=timer.time, errors=report["errors"], tables=report["tables"])


Test patch:
diff --git a/tests/validate/test_inquiry.py b/tests/validate/test_inquiry.py
index 38a4a976..2340deb3 100644
--- a/tests/validate/test_inquiry.py
+++ b/tests/validate/test_inquiry.py
@@ -101,3 +101,20 @@ def test_validate_with_multiple_packages():
         [3, 3, None, "primary-key-error"],
         [4, 4, None, "blank-row"],
     ]
+
+
+def test_validate_with_multiple_packages_with_nopool():
+    report = validate(
+        {
+            "tasks": [
+                {"source": "data/package/datapackage.json"},
+                {"source": "data/invalid/datapackage.json"},
+            ]
+        },
+        nopool=True,
+    )
+    assert report.flatten(["tablePosition", "rowPosition", "fieldPosition", "code"]) == [
+        [3, 3, None, "blank-row"],
+        [3, 3, None, "primary-key-error"],
+        [4, 4, None, "blank-row"],
+    ]
diff --git a/tests/validate/test_package.py b/tests/validate/test_package.py
index 5dd51144..299497a8 100644
--- a/tests/validate/test_package.py
+++ b/tests/validate/test_package.py
@@ -146,6 +146,15 @@ def test_validate_package_dialect_header_false():
     assert report.valid
 
 
+def test_validate_with_nopool():
+    report = validate("data/invalid/datapackage.json", nopool=True)
+    assert report.flatten(["tablePosition", "rowPosition", "fieldPosition", "code"]) == [
+        [1, 3, None, "blank-row"],
+        [1, 3, None, "primary-key-error"],
+        [2, 4, None, "blank-row"],
+    ]
+
+
 # Checksum
 
 DESCRIPTOR_SH = {