Repository: frictionlessdata/frictionless-py

Issue:
Implement `Resource.analyze` function
# Overview

> Is there any tooling around that does more than describe to analyse the data.  For example tools that would give you distributions for number fields, most common word statistics for text fields, distinct counts, counts for fields that are not blank, most common categories used, and many others?   Essentially statistics that are currently not in the tablular data package resource specification?  Also could help detect more kinds of data formats.
As example of some of this could be done by pandas describe function https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.describe.html, however there is a lot more potential than that.
Also is it ok to add suche extra statistics to resouces without causing validation errors?
I am thinking such tooling would be interesting as it could give greater insight into the data before the need to analyse it.

Discussion:

Patch:
diff --git a/data/analysis-data.csv b/data/analysis-data.csv
new file mode 100644
index 00000000..ab09fc51
--- /dev/null
+++ b/data/analysis-data.csv
@@ -0,0 +1,10 @@
+type_school,school_accreditation,gender,interest,residence,parent_age,parent_salary,house_area,average_grades,parent_was_in_college,in_college
+Academic,A,Male,Less Interested,Urban,56,,83.0,,False,True
+Academic,A,Male,Less Interested,Urban,57,4410000,76.8,86.91,False,True
+Academic,B,Female,Very Interested,Urban,50,6500000,80.6,87.43,False,True
+Vocational,B,Male,Very Interested,Rural,49,6600000,78.2,,True,True
+Academic,A,Female,Very Interested,Urban,57,5250000,75.1,86.79,False,False
+Vocational,B,Female,Less Interested,Rural,48,3770000,65.3,86.79,True,False
+Academic,A,Male,Very Interested,Rural,52,6680000,85.5,90.39,True,True
+Academic,B,Male,Very Interested,Rural,53,5890000,83.3,84.65,True,False
+Academic,B,Female,Uncertain,Rural,52,6730000,80.3,10000,True,True
\ No newline at end of file
diff --git a/data/package-1067.json b/data/package-1067.json
new file mode 100644
index 00000000..4b3c3a98
--- /dev/null
+++ b/data/package-1067.json
@@ -0,0 +1,87 @@
+{
+  "name": "capitals and schools",
+  "resources": [
+    { "name": "capital-valid",
+      "path": "capital-valid.csv",
+      "schema": {
+        "fields": [
+          {
+            "name": "id",
+            "type": "number"
+          },
+          {
+            "name": "name",
+            "type": "string"
+          }
+        ]
+      }
+    },
+    {"name": "capital-invalid", 
+     "path": "capital-invalid.csv",
+      "schema": {
+        "fields": [
+          {
+            "name": "id",
+            "type": "integer"
+          },
+          {
+            "name": "name",
+            "type": "string"
+          }
+        ]
+      }
+    },
+    { "name": "analysis-data",
+      "path": "analysis-data.csv",
+      "schema": {
+        "fields": [
+          {
+            "name": "type_school",
+            "type": "string"
+          },
+          {
+            "name": "school_accreditation",
+            "type": "string"
+          },
+          {
+            "name": "gender",
+            "type": "string"
+          },
+          {
+            "name": "interest",
+            "type": "string"
+          },
+          {
+            "name": "residence",
+            "type": "string"
+          },
+          {
+            "name": "parent_age",
+            "type": "integer"
+          },
+          {
+            "name": "parent_salary",
+            "type": "integer"
+          },
+          {
+            "name": "house_area",
+            "type": "number"
+          },
+          {
+            "name": "average_grades",
+            "type": "number"
+          },
+          {
+            "name": "parent_was_in_college",
+            "type": "boolean"
+          },
+          {
+            "name": "in_college",
+            "type": "boolean"
+          }
+        ]
+      }
+    }
+  ]
+}
+
diff --git a/frictionless/package/analyze.py b/frictionless/package/analyze.py
new file mode 100644
index 00000000..fc1f16f8
--- /dev/null
+++ b/frictionless/package/analyze.py
@@ -0,0 +1,24 @@
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .package import Package
+
+
+def analyze(package: "Package", *, detailed=False):
+    """Analyze the resources of the package
+    This feature is currently experimental, and its API may change
+    without warning.
+
+    Parameters:
+        package (Package): package object
+        detailed? (bool): detailed analysis
+
+    Returns:
+        dict: dict of resource analysis
+
+    """
+    result = {}
+    for number, resource in enumerate(package.resources, start=1):
+        key = resource.fullpath if not resource.memory else f"memory{number}"
+        result[key] = resource.analyze(detailed=detailed)
+    return result
diff --git a/frictionless/package/package.py b/frictionless/package/package.py
index 624faab3..5cc1ca9e 100644
--- a/frictionless/package/package.py
+++ b/frictionless/package/package.py
@@ -11,6 +11,7 @@ from ..detector import Detector
 from ..resource import Resource
 from ..field import Field
 from ..system import system
+from .analyze import analyze
 from .describe import describe
 from .extract import extract
 from .transform import transform
@@ -130,6 +131,7 @@ class Package(Metadata):
     extract = extract
     transform = transform
     validate = validate
+    analyze = analyze
 
     def __init__(
         self,
diff --git a/frictionless/resource/analyze.py b/frictionless/resource/analyze.py
new file mode 100644
index 00000000..f0c2d238
--- /dev/null
+++ b/frictionless/resource/analyze.py
@@ -0,0 +1,263 @@
+import statistics
+from math import nan
+from typing import TYPE_CHECKING, Union, List
+from collections import Counter
+from decimal import Decimal
+from math import fsum, sqrt
+from .. import helpers
+
+if TYPE_CHECKING:
+    from .resource import Resource
+
+
+def analyze(resource: "Resource", *, detailed=False) -> dict:
+    """Analyze the resource
+    This feature is currently experimental, and its API may change
+    without warning.
+
+    Parameters:
+        resource (Resource): resource object
+        detailed? (bool): detailed analysis
+
+    Returns:
+        dict: resource analysis
+
+    """
+    timer = helpers.Timer()
+    resource.infer()
+
+    # Row stats
+    analysis_report = {}
+    analysis_report["variableTypes"] = {}
+    analysis_report["notNullRows"] = 0
+    analysis_report["rowsWithNullValues"] = 0
+    analysis_report["fieldStats"] = {}
+
+    columns_data = {}
+    numeric = ["integer", "numeric", "number"]
+    for row in resource:
+        null_columns = 0
+        for field_name in row:
+            field = resource.schema.get_field(field_name)
+            cell = field.read_cell(row.get(field_name))[0]
+            if field.name not in columns_data:
+                columns_data[field.name] = []
+            if cell is None:
+                if field.type in numeric:
+                    cell = nan
+                null_columns += 1
+            if isinstance(cell, Decimal):
+                cell = float(cell)
+            columns_data[field.name].append(cell)
+
+        if null_columns > 0:
+            analysis_report["rowsWithNullValues"] += 1
+
+    # Field/Column Stats
+    if columns_data and detailed:
+        analysis_report["correlations"] = {}
+        for field in resource.schema.fields:
+            analysis_report["fieldStats"][field.name] = {}
+
+            if field.type not in analysis_report["variableTypes"]:
+                analysis_report["variableTypes"][field.type] = 0
+            analysis_report["variableTypes"][field.type] += 1
+
+            # summary - categorical data
+            if field.type not in [*numeric, "boolean"]:
+                analysis_report["fieldStats"][field.name]["type"] = "categorical"
+                analysis_report["fieldStats"][field.name]["values"] = set(
+                    columns_data[field.name]
+                )
+
+            # descriptive statistics - numeric data
+            if field.type in numeric:
+                analysis_report["fieldStats"][field.name]["type"] = "numeric"
+                rows_without_nan_values = [
+                    cell for cell in columns_data[field.name] if cell is not nan
+                ]
+
+                # skip rows with nan values
+                if len(rows_without_nan_values) < 2:
+                    continue
+
+                analysis_report["fieldStats"][field.name].update(
+                    _statistics(rows_without_nan_values)
+                )
+                analysis_report["fieldStats"][field.name]["outliers"] = []
+                analysis_report["fieldStats"][field.name][
+                    "missingValues"
+                ] = resource.stats["rows"] - len(rows_without_nan_values)
+
+                # calculate correlation between variables(columns/fields)
+                for field_y in resource.schema.fields:
+                    if field_y.type in numeric:
+                        # filter rows with nan values, correlation return nan if any of the
+                        # row has nan value.
+                        var_x, var_y = [], []
+                        for cell_x, cell_y in zip(
+                            columns_data[field.name], columns_data[field_y.name]
+                        ):
+                            if nan not in [cell_x, cell_y]:
+                                var_x.append(cell_x)
+                                var_y.append(cell_y)
+
+                        # check for atleast 2 data points for correlation calculation
+                        if len(var_x) > 2:
+                            if field.name not in analysis_report["correlations"]:
+                                analysis_report["correlations"][field.name] = []
+                            correlation_result = {
+                                "fieldName": field_y.name,
+                                "corr": _correlation(var_x, var_y),
+                            }
+                            analysis_report["correlations"][field.name].append(
+                                correlation_result
+                            )
+
+                # calculate outliers
+                lower_bound, upper_bound = analysis_report["fieldStats"][field.name][
+                    "bounds"
+                ]
+                for cell in columns_data[field.name]:
+                    if cell is not nan:
+                        if not lower_bound < cell < upper_bound:
+                            analysis_report["fieldStats"][field.name]["outliers"].append(
+                                cell
+                            )
+
+    analysis_report["notNullRows"] = (
+        resource.stats["rows"] - analysis_report["rowsWithNullValues"]
+    )
+    analysis_report["averageRecordSizeInBytes"] = 0
+    if resource.stats["rows"]:
+        analysis_report["averageRecordSizeInBytes"] = (
+            resource.stats["bytes"] / resource.stats["rows"]
+        )
+    analysis_report["timeTaken"] = timer.time
+    return {**analysis_report, **resource.stats}
+
+
+# TODO:This is a temporary function to use with statistics library as
+# python 3.7 does not support quantiles
+# code: https://github.com/python/cpython/blob/3.8/Lib/statistics.py#L620
+def _quantiles(data, *, n=4, method="exclusive") -> List:
+    """Divide *data* into *n* continuous intervals with equal probability.
+    Returns a list of (n - 1) cut points separating the intervals.
+    Set *n* to 4 for quartiles (the default).  Set *n* to 10 for deciles.
+    Set *n* to 100 for percentiles which gives the 99 cuts points that
+    separate *data* in to 100 equal sized groups.
+    The *data* can be any iterable containing sample.
+    The cut points are linearly interpolated between data points.
+    If *method* is set to *inclusive*, *data* is treated as population
+    data.  The minimum value is treated as the 0th percentile and the
+    maximum value is treated as the 100th percentile.
+    """
+    if n < 1:
+        raise statistics.StatisticsError("n must be at least 1")
+    data = sorted(data)
+    ld = len(data)
+    if ld < 2:
+        raise statistics.StatisticsError("must have at least two data points")
+    if method == "inclusive":
+        m = ld - 1
+        result = []
+        for i in range(1, n):
+            j, delta = divmod(i * m, n)
+            interpolated = (data[j] * (n - delta) + data[j + 1] * delta) / n
+            result.append(interpolated)
+        return result
+    if method == "exclusive":
+        m = ld + 1
+        result = []
+        for i in range(1, n):
+            j = i * m // n  # rescale i to m/n
+            j = 1 if j < 1 else ld - 1 if j > ld - 1 else j  # clamp to 1 .. ld-1
+            delta = i * m - j * n  # exact integer math
+            interpolated = (data[j - 1] * (n - delta) + data[j] * delta) / n
+            result.append(interpolated)
+        return result
+    raise ValueError(f"Unknown method: {method!r}")
+
+
+def _common_values(data: Union[float, int]) -> Union[float, int]:
+    """Finds highly common data with frequency
+
+    Args:
+        data (float|int): data
+
+    Returns:
+        (float|int): highly common element and its count
+    """
+    column = Counter(data)
+    common_value = column.most_common(1)
+    if common_value and common_value[0][1] > 1:
+        return common_value[0][0]
+    return None
+
+
+def _statistics(data: Union[float, int]) -> dict:
+    """Calculate the descriptive statistics of the data
+
+    Args:
+        data (float|int): data
+
+    Returns:
+        dict : statistics of the data
+    """
+    resource_stats = {}
+    resource_stats["mean"] = statistics.mean(data)
+    resource_stats["median"] = statistics.median(data)
+    resource_stats["mode"] = _common_values(data)
+    resource_stats["variance"] = statistics.variance(data)
+    resource_stats["quantiles"] = _quantiles(data)
+    resource_stats["stdev"] = statistics.stdev(data)
+    resource_stats["max"] = max(data)
+    resource_stats["min"] = min(data)
+    resource_stats["bounds"] = _find_bounds(resource_stats["quantiles"])
+    resource_stats["uniqueValues"] = len(set(data))
+    return resource_stats
+
+
+def _find_bounds(quartiles: List):
+    """Calculate the higher and lower bound of distribution
+
+    Args:
+        quantiles (List): list of quartiles of distribution
+
+    Returns:
+        List: upper and lower bound
+    """
+    q1, _, q3 = quartiles
+    inter_quartile_range = q3 - q1
+    upper_bound = round(q3 + (1.5 * inter_quartile_range))
+    lower_bound = round(q1 - (1.5 * inter_quartile_range))
+    return [lower_bound, upper_bound]
+
+
+# TODO:This is a temporary function to use with statistics library as
+# python 3.7 does not support correlation
+# code: https://github.com/python/cpython/blob/3.10/Lib/statistics.py#L889
+def _correlation(x, y) -> float:
+    """Pearson's correlation coefficient
+    Return the Pearson's correlation coefficient for two inputs. Pearson's
+    correlation coefficient *r* takes values between -1 and +1. It measures the
+    strength and direction of the linear relationship, where +1 means very
+    strong, positive linear relationship, -1 very strong, negative linear
+    relationship, and 0 no linear relationship
+    """
+    n = len(x)
+    if len(y) != n:
+        raise statistics.StatisticsError(
+            "correlation requires that both inputs have same number of data points"
+        )
+    if n < 2:
+        raise statistics.StatisticsError("correlation requires at least two data points")
+    xbar = fsum(x) / n
+    ybar = fsum(y) / n
+    sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y))
+    sxx = fsum((xi - xbar) ** 2.0 for xi in x)
+    syy = fsum((yi - ybar) ** 2.0 for yi in y)
+    try:
+        return sxy / sqrt(sxx * syy)
+    except ZeroDivisionError:
+        raise statistics.StatisticsError("at least one of the inputs is constant")
diff --git a/frictionless/resource/resource.py b/frictionless/resource/resource.py
index 24be82c5..b8a6f059 100644
--- a/frictionless/resource/resource.py
+++ b/frictionless/resource/resource.py
@@ -14,6 +14,7 @@ from ..header import Header
 from ..system import system
 from ..field import Field
 from ..row import Row
+from .analyze import analyze
 from .describe import describe
 from .extract import extract
 from .transform import transform
@@ -141,6 +142,7 @@ class Resource(Metadata):
     extract = extract
     transform = transform
     validate = validate
+    analyze = analyze
 
     def __init__(
         self,


Test patch:
diff --git a/tests/package/analyze/__init__.py b/tests/package/analyze/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/package/analyze/test_general.py b/tests/package/analyze/test_general.py
new file mode 100644
index 00000000..9d27ce06
--- /dev/null
+++ b/tests/package/analyze/test_general.py
@@ -0,0 +1,253 @@
+from frictionless import Package, helpers
+
+
+IS_UNIX = not helpers.is_platform("windows")
+
+
+def test_analyze_package():
+    package = Package("data/package-1067.json")
+    analysis = package.analyze()
+    assert len(analysis) == 3
+    path_1 = "data/capital-valid.csv" if IS_UNIX else "data\\capital-valid.csv"
+    path_2 = "data/capital-invalid.csv" if IS_UNIX else "data\\capital-invalid.csv"
+    path_3 = "data/analysis-data.csv" if IS_UNIX else "data\\analysis-data.csv"
+    assert analysis[path_1]["rows"] == 5
+    assert list(analysis[path_1].keys()) == [
+        "variableTypes",
+        "notNullRows",
+        "rowsWithNullValues",
+        "fieldStats",
+        "averageRecordSizeInBytes",
+        "timeTaken",
+        "hash",
+        "bytes",
+        "fields",
+        "rows",
+    ]
+    assert analysis[path_2]["rows"] == 11
+    assert list(analysis[path_2].keys()) == [
+        "variableTypes",
+        "notNullRows",
+        "rowsWithNullValues",
+        "fieldStats",
+        "averageRecordSizeInBytes",
+        "timeTaken",
+        "hash",
+        "bytes",
+        "fields",
+        "rows",
+    ]
+    assert analysis[path_3]["rows"] == 9
+    assert list(analysis[path_3].keys()) == [
+        "variableTypes",
+        "notNullRows",
+        "rowsWithNullValues",
+        "fieldStats",
+        "averageRecordSizeInBytes",
+        "timeTaken",
+        "hash",
+        "bytes",
+        "fields",
+        "rows",
+    ]
+
+
+def test_analyze_package_detailed():
+    package = Package("data/package-1067.json")
+    analysis = package.analyze(detailed=True)
+    path_1 = "data/capital-valid.csv" if IS_UNIX else "data\\capital-valid.csv"
+    path_2 = "data/capital-invalid.csv" if IS_UNIX else "data\\capital-invalid.csv"
+    path_3 = "data/analysis-data.csv" if IS_UNIX else "data\\analysis-data.csv"
+    assert analysis[path_1]["rows"] == 5
+    assert list(analysis[path_1].keys()) == [
+        "variableTypes",
+        "notNullRows",
+        "rowsWithNullValues",
+        "fieldStats",
+        "correlations",
+        "averageRecordSizeInBytes",
+        "timeTaken",
+        "hash",
+        "bytes",
+        "fields",
+        "rows",
+    ]
+    assert analysis[path_2]["rows"] == 11
+    assert list(analysis[path_2].keys()) == [
+        "variableTypes",
+        "notNullRows",
+        "rowsWithNullValues",
+        "fieldStats",
+        "correlations",
+        "averageRecordSizeInBytes",
+        "timeTaken",
+        "hash",
+        "bytes",
+        "fields",
+        "rows",
+    ]
+    assert analysis[path_3]["rows"] == 9
+    assert list(analysis[path_3].keys()) == [
+        "variableTypes",
+        "notNullRows",
+        "rowsWithNullValues",
+        "fieldStats",
+        "correlations",
+        "averageRecordSizeInBytes",
+        "timeTaken",
+        "hash",
+        "bytes",
+        "fields",
+        "rows",
+    ]
+
+
+def test_analyze_package_invalid_data():
+    descriptor = {
+        "name": "capitals and schools",
+        "resources": [
+            {"name": "capital-invalid", "path": "data/invalid.csv"},
+        ],
+    }
+    package = Package(descriptor)
+    analysis = package.analyze()
+    assert (
+        round(analysis["data/invalid.csv"]["averageRecordSizeInBytes"]) == 12
+        if IS_UNIX
+        else 14
+    )
+    assert analysis["data/invalid.csv"]["fields"] == 4
+    assert analysis["data/invalid.csv"]["fieldStats"] == {}
+    assert analysis["data/invalid.csv"]["rows"] == 4
+    assert analysis["data/invalid.csv"]["rowsWithNullValues"] == 3
+    assert analysis["data/invalid.csv"]["notNullRows"] == 1
+    assert analysis["data/invalid.csv"]["variableTypes"] == {}
+
+
+def test_analyze_package_detailed_variable_types():
+    package = Package("data/package-1067.json")
+    analysis = package.analyze(detailed=True)
+    path_1 = "data/capital-valid.csv" if IS_UNIX else "data\\capital-valid.csv"
+    path_2 = "data/capital-invalid.csv" if IS_UNIX else "data\\capital-invalid.csv"
+    path_3 = "data/analysis-data.csv" if IS_UNIX else "data\\analysis-data.csv"
+    assert len(analysis) == 3
+    assert analysis[path_1]["variableTypes"] == {
+        "number": 1,
+        "string": 1,
+    }
+    assert analysis[path_2]["variableTypes"] == {
+        "integer": 1,
+        "string": 1,
+    }
+    assert analysis[path_3]["variableTypes"] == {
+        "boolean": 2,
+        "integer": 2,
+        "number": 2,
+        "string": 5,
+    }
+
+
+def test_analyze_package_detailed_non_numeric_values_summary():
+    package = Package("data/package-1067.json")
+    analysis = package.analyze(detailed=True)
+    path_1 = "data/capital-valid.csv" if IS_UNIX else "data\\capital-valid.csv"
+    path_2 = "data/capital-invalid.csv" if IS_UNIX else "data\\capital-invalid.csv"
+    path_3 = "data/analysis-data.csv" if IS_UNIX else "data\\analysis-data.csv"
+    assert list(analysis[path_1]["fieldStats"]["name"].keys()) == [
+        "type",
+        "values",
+    ]
+    assert list(analysis[path_2]["fieldStats"]["name"].keys()) == [
+        "type",
+        "values",
+    ]
+    assert list(analysis[path_3]["fieldStats"]["gender"].keys()) == [
+        "type",
+        "values",
+    ]
+
+
+def test_analyze_package_detailed_numeric_values_descriptive_summary():
+    package = Package("data/package-1067.json")
+    analysis = package.analyze(detailed=True)
+    path = "data/analysis-data.csv" if IS_UNIX else "data\\analysis-data.csv"
+    assert list(analysis[path]["fieldStats"]["parent_age"].keys()) == [
+        "type",
+        "mean",
+        "median",
+        "mode",
+        "variance",
+        "quantiles",
+        "stdev",
+        "max",
+        "min",
+        "bounds",
+        "uniqueValues",
+        "outliers",
+        "missingValues",
+    ]
+
+
+def test_analyze_package_detailed_numeric_descriptive_statistics():
+    package = Package("data/package-1067.json")
+    analysis = package.analyze(detailed=True)
+    path = "data/analysis-data.csv" if IS_UNIX else "data\\analysis-data.csv"
+    assert analysis[path]["fieldStats"]["parent_age"]["bounds"] == [
+        39,
+        67,
+    ]
+    assert analysis[path]["fieldStats"]["parent_age"]["max"] == 57
+    assert analysis[path]["fieldStats"]["parent_age"]["mean"] == 52.666666666666664
+    assert analysis[path]["fieldStats"]["parent_age"]["median"] == 52
+    assert analysis[path]["fieldStats"]["parent_age"]["min"] == 48
+    assert analysis[path]["fieldStats"]["parent_age"]["missingValues"] == 0
+    assert analysis[path]["fieldStats"]["parent_age"]["mode"] == 57
+    assert analysis[path]["fieldStats"]["parent_age"]["quantiles"] == [49.5, 52.0, 56.5]
+    assert analysis[path]["fieldStats"]["parent_age"]["stdev"] == 3.391164991562634
+    assert analysis[path]["fieldStats"]["parent_age"]["uniqueValues"] == 7
+    assert analysis[path]["fieldStats"]["parent_age"]["variance"] == 11.5
+    assert analysis[path]["fieldStats"]["parent_age"]["outliers"] == []
+
+
+def test_analyze_package_detailed_non_numeric_summary():
+    package = Package("data/package-1067.json")
+    analysis = package.analyze(detailed=True)
+    path_1 = "data/capital-valid.csv" if IS_UNIX else "data\\capital-valid.csv"
+    path_2 = "data/analysis-data.csv" if IS_UNIX else "data\\analysis-data.csv"
+    assert analysis[path_1]["fieldStats"]["name"]["type"] == "categorical"
+    assert analysis[path_1]["fieldStats"]["name"]["values"] == {
+        "Berlin",
+        "London",
+        "Madrid",
+        "Paris",
+        "Rome",
+    }
+    assert analysis[path_2]["fieldStats"]["school_accreditation"]["type"] == "categorical"
+    assert analysis[path_2]["fieldStats"]["school_accreditation"]["values"] == {
+        "A",
+        "B",
+    }
+
+
+def test_analyze_package_detailed_invalid_data():
+    descriptor = {
+        "name": "capitals and schools",
+        "resources": [
+            {"name": "capital-invalid", "path": "data/invalid.csv"},
+        ],
+    }
+    package = Package(descriptor)
+    analysis = package.analyze(detailed=True)
+    path = "data/invalid.csv"
+    assert round(analysis[path]["averageRecordSizeInBytes"]) == 12 if IS_UNIX else 14
+    assert analysis[path]["fields"] == 4
+    assert list(analysis[path]["fieldStats"].keys()) == [
+        "id",
+        "name",
+        "field3",
+        "name2",
+    ]
+    assert analysis[path]["rows"] == 4
+    assert analysis[path]["rowsWithNullValues"] == 3
+    assert analysis[path]["notNullRows"] == 1
+    assert analysis[path]["variableTypes"] == {"integer": 3, "string": 1}
diff --git a/tests/resource/analyze/test_general.py b/tests/resource/analyze/test_general.py
new file mode 100644
index 00000000..d54447b3
--- /dev/null
+++ b/tests/resource/analyze/test_general.py
@@ -0,0 +1,236 @@
+from frictionless import Resource, helpers
+
+
+IS_UNIX = not helpers.is_platform("windows")
+
+
+def test_analyze_resource():
+    resource = Resource({"path": "data/analysis-data.csv"})
+    analysis = resource.analyze()
+    assert list(analysis.keys()) == [
+        "variableTypes",
+        "notNullRows",
+        "rowsWithNullValues",
+        "fieldStats",
+        "averageRecordSizeInBytes",
+        "timeTaken",
+        "hash",
+        "bytes",
+        "fields",
+        "rows",
+    ]
+    assert round(analysis["averageRecordSizeInBytes"]) == 85 if IS_UNIX else 86
+    assert analysis["fields"] == 11
+    assert analysis["rows"] == 9
+    assert analysis["rowsWithNullValues"] == 2
+    assert analysis["notNullRows"] == 7
+    assert analysis["variableTypes"] == {}
+
+
+def test_analyze_resource_detailed():
+    resource = Resource({"path": "data/analysis-data.csv"})
+    analysis = resource.analyze(detailed=True)
+    assert list(analysis.keys()) == [
+        "variableTypes",
+        "notNullRows",
+        "rowsWithNullValues",
+        "fieldStats",
+        "correlations",
+        "averageRecordSizeInBytes",
+        "timeTaken",
+        "hash",
+        "bytes",
+        "fields",
+        "rows",
+    ]
+    assert round(analysis["averageRecordSizeInBytes"]) == 85 if IS_UNIX else 86
+    assert analysis["fields"] == 11
+    assert analysis["rows"] == 9
+    assert analysis["rowsWithNullValues"] == 2
+    assert analysis["notNullRows"] == 7
+    assert analysis["variableTypes"] == {
+        "boolean": 2,
+        "integer": 2,
+        "number": 2,
+        "string": 5,
+    }
+
+
+def test_analyze_resource_detailed_non_numeric_values_summary():
+    resource = Resource({"path": "data/analysis-data.csv"})
+    analysis = resource.analyze(detailed=True)
+    assert list(analysis["fieldStats"]["gender"].keys()) == ["type", "values"]
+
+
+def test_analyze_resource_detailed_numeric_values_descriptive_summray():
+    resource = Resource({"path": "data/analysis-data.csv"})
+    analysis = resource.analyze(detailed=True)
+    assert list(analysis["fieldStats"]["parent_age"].keys()) == [
+        "type",
+        "mean",
+        "median",
+        "mode",
+        "variance",
+        "quantiles",
+        "stdev",
+        "max",
+        "min",
+        "bounds",
+        "uniqueValues",
+        "outliers",
+        "missingValues",
+    ]
+
+
+def test_analyze_resource_detailed_numeric_descriptive_statistics():
+    resource = Resource({"path": "data/analysis-data.csv"})
+    analysis = resource.analyze(detailed=True)
+    assert analysis["fieldStats"]["parent_age"]["bounds"] == [39, 67]
+    assert analysis["fieldStats"]["parent_age"]["max"] == 57
+    assert analysis["fieldStats"]["parent_age"]["mean"] == 52.666666666666664
+    assert analysis["fieldStats"]["parent_age"]["median"] == 52
+    assert analysis["fieldStats"]["parent_age"]["min"] == 48
+    assert analysis["fieldStats"]["parent_age"]["missingValues"] == 0
+    assert analysis["fieldStats"]["parent_age"]["mode"] == 57
+    assert analysis["fieldStats"]["parent_age"]["quantiles"] == [49.5, 52.0, 56.5]
+    assert analysis["fieldStats"]["parent_age"]["stdev"] == 3.391164991562634
+    assert analysis["fieldStats"]["parent_age"]["uniqueValues"] == 7
+    assert analysis["fieldStats"]["parent_age"]["variance"] == 11.5
+    assert analysis["fieldStats"]["parent_age"]["outliers"] == []
+
+
+def test_analyze_resource_detailed_numeric_descriptive_statistics_with_missingValues():
+    resource = Resource({"path": "data/analysis-data.csv"})
+    analysis = resource.analyze(detailed=True)
+    assert analysis["fieldStats"]["average_grades"]["bounds"] == [81, 96]
+    assert analysis["fieldStats"]["average_grades"]["max"] == 10000.0
+    assert analysis["fieldStats"]["average_grades"]["mean"] == 1503.28
+    assert analysis["fieldStats"]["average_grades"]["median"] == 86.91
+    assert analysis["fieldStats"]["average_grades"]["min"] == 84.65
+    assert analysis["fieldStats"]["average_grades"]["missingValues"] == 2
+    assert analysis["fieldStats"]["average_grades"]["mode"] == 86.79
+    assert analysis["fieldStats"]["average_grades"]["quantiles"] == [86.79, 86.91, 90.39]
+    assert round(analysis["fieldStats"]["average_grades"]["stdev"]) == 3747
+    assert analysis["fieldStats"]["average_grades"]["uniqueValues"] == 6
+    assert round(analysis["fieldStats"]["average_grades"]["variance"]) == 14037774
+    assert analysis["fieldStats"]["average_grades"]["outliers"] == [10000.0]
+
+
+def test_analyze_resource_detailed_descriptive_statistics_with_outliers():
+    resource = Resource({"path": "data/analysis-data.csv"})
+    analysis = resource.analyze(detailed=True)
+    assert analysis["fieldStats"]["average_grades"]["bounds"] == [81, 96]
+    assert analysis["fieldStats"]["average_grades"]["outliers"] == [10000.0]
+
+
+def test_analyze_resource_detailed_descriptive_statistics_variables_correlation():
+    resource = Resource({"path": "data/analysis-data.csv"})
+    analysis = resource.analyze(detailed=True)
+    assert list(analysis["correlations"].keys()) == [
+        "parent_age",
+        "parent_salary",
+        "house_area",
+        "average_grades",
+    ]
+    assert (
+        analysis["correlations"]["average_grades"][0]["fieldName"] == "parent_age"
+        and analysis["correlations"]["average_grades"][0]["corr"] == -0.09401771232099933
+    )
+    assert (
+        analysis["correlations"]["average_grades"][1]["fieldName"] == "parent_salary"
+        and analysis["correlations"]["average_grades"][1]["corr"] == 0.4241304392492213
+    )
+    assert (
+        analysis["correlations"]["average_grades"][2]["fieldName"] == "house_area"
+        and analysis["correlations"]["average_grades"][2]["corr"] == 0.14354348594097088
+    )
+    assert (
+        analysis["correlations"]["average_grades"][3]["fieldName"] == "average_grades"
+        and analysis["correlations"]["average_grades"][3]["corr"] == 1.0
+    )
+
+
+def test_analyze_resource_detailed_non_numeric_summary():
+    resource = Resource({"path": "data/analysis-data.csv"})
+    analysis = resource.analyze(detailed=True)
+    assert list(analysis["fieldStats"]["gender"].keys()) == ["type", "values"]
+    assert analysis["fieldStats"]["gender"]["values"] == {"Male", "Female"}
+
+
+def test_analyze_resource_detailed_non_numeric_data_identification():
+    data = [
+        ["gender", "country"],
+        ["male", "usa"],
+        ["female", "usa"],
+        ["male", "italy"],
+        ["female", "italy"],
+        ["female", "italy"],
+    ]
+    resource = Resource(data)
+    analysis = resource.analyze(detailed=True)
+    assert analysis["fieldStats"]["gender"]["type"] == "categorical"
+    assert analysis["fieldStats"]["gender"]["values"] == {"male", "female"}
+    assert analysis["fieldStats"]["country"]["type"] == "categorical"
+    assert analysis["fieldStats"]["country"]["values"] == {"usa", "italy"}
+
+
+def test_analyze_resource_with_empty_rows():
+    data = [["a", "b"]]
+    resource = Resource(data)
+    analysis = resource.analyze()
+    assert list(analysis.keys()) == [
+        "variableTypes",
+        "notNullRows",
+        "rowsWithNullValues",
+        "fieldStats",
+        "averageRecordSizeInBytes",
+        "timeTaken",
+        "hash",
+        "bytes",
+        "fields",
+        "rows",
+    ]
+    assert analysis["rows"] == 0
+
+
+def test_analyze_resource_detailed_with_empty_rows():
+    data = [["a", "b"]]
+    resource = Resource(data)
+    analysis = resource.analyze(detailed=True)
+    assert list(analysis.keys()) == [
+        "variableTypes",
+        "notNullRows",
+        "rowsWithNullValues",
+        "fieldStats",
+        "averageRecordSizeInBytes",
+        "timeTaken",
+        "hash",
+        "bytes",
+        "fields",
+        "rows",
+    ]
+    assert analysis["rows"] == 0
+
+
+def test_analyze_resource_with_invalid_data():
+    resource = Resource({"path": "data/invalid.csv"})
+    analysis = resource.analyze()
+    assert round(analysis["averageRecordSizeInBytes"]) == 12 if IS_UNIX else 14
+    assert analysis["fields"] == 4
+    assert analysis["fieldStats"] == {}
+    assert analysis["rows"] == 4
+    assert analysis["rowsWithNullValues"] == 3
+    assert analysis["notNullRows"] == 1
+    assert analysis["variableTypes"] == {}
+
+
+def test_analyze_resource_detailed_with_invalid_data():
+    resource = Resource({"path": "data/invalid.csv"})
+    analysis = resource.analyze(detailed=True)
+    assert round(analysis["averageRecordSizeInBytes"]) == 12 if IS_UNIX else 14
+    assert analysis["fields"] == 4
+    assert list(analysis["fieldStats"].keys()) == ["id", "name", "field3", "name2"]
+    assert analysis["rows"] == 4
+    assert analysis["rowsWithNullValues"] == 3
+    assert analysis["notNullRows"] == 1
+    assert analysis["variableTypes"] == {"integer": 3, "string": 1}