Repository: frictionlessdata/frictionless-py (language: python)

Issue:
Normalize bytes/hash calculation between UNIX and Windows?
# Overview

At the moment, it doesn't match. Shall we normalize line endings etc? It's complicated because `python.csv` requires opening files without a universal newline.

On the other hand, the normalization might just be the wrong way to go because it's checksums. This issue requires proper analysis.

PS.
Search for `IS_UNIX` flag in the tests


Discussion / hints:

Fix patch:
diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000000..c7b9c26e5d
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,3 @@
+# Don't apply EOL normalization for csv files
+*.csv  -text
+*.txt  -text
\ No newline at end of file


Test patch:
diff --git a/tests/analyzer/test_package.py b/tests/analyzer/test_package.py
index 6c23d5cf0c..4e13436259 100644
--- a/tests/analyzer/test_package.py
+++ b/tests/analyzer/test_package.py
@@ -2,9 +2,7 @@
 
 import pytest
 
-from frictionless import Package, platform
-
-IS_UNIX = platform.type != "windows"
+from frictionless import Package
 
 pytestmark = pytest.mark.skipif(
     sys.version_info < (3, 10),
@@ -122,11 +120,7 @@ def test_analyze_package_invalid_data():
     }
     package = Package(descriptor)
     analysis = package.analyze()
-    assert (
-        round(analysis["capital-invalid"]["averageRecordSizeInBytes"]) == 12
-        if IS_UNIX
-        else 14
-    )
+    assert round(analysis["capital-invalid"]["averageRecordSizeInBytes"]) == 12
     assert analysis["capital-invalid"]["fields"] == 4
     assert analysis["capital-invalid"]["fieldStats"] == {}
     assert analysis["capital-invalid"]["rows"] == 4
@@ -241,7 +235,7 @@ def test_analyze_package_detailed_invalid_data():
     package = Package(descriptor)
     analysis = package.analyze(detailed=True)
     name = "capital-invalid"
-    assert round(analysis[name]["averageRecordSizeInBytes"]) == 12 if IS_UNIX else 14
+    assert round(analysis[name]["averageRecordSizeInBytes"]) == 12
     assert analysis[name]["fields"] == 4
     assert list(analysis[name]["fieldStats"].keys()) == [
         "id",
diff --git a/tests/analyzer/test_resource.py b/tests/analyzer/test_resource.py
index 02799e1b7d..53da72cc12 100644
--- a/tests/analyzer/test_resource.py
+++ b/tests/analyzer/test_resource.py
@@ -2,11 +2,8 @@
 
 import pytest
 
-from frictionless import platform
 from frictionless.resources import TableResource
 
-IS_UNIX = platform.type != "windows"
-
 pytestmark = pytest.mark.skipif(
     sys.version_info < (3, 10),
     reason="Supported on Python3.10+",
@@ -32,7 +29,7 @@ def test_analyze_resource():
         "fields",
         "rows",
     ]
-    assert round(analysis["averageRecordSizeInBytes"]) == 85 if IS_UNIX else 86
+    assert round(analysis["averageRecordSizeInBytes"]) == 85
     assert analysis["fields"] == 11
     assert analysis["rows"] == 9
     assert analysis["rowsWithNullValues"] == 2
@@ -57,7 +54,7 @@ def test_analyze_resource_detailed():
         "fields",
         "rows",
     ]
-    assert round(analysis["averageRecordSizeInBytes"]) == 85 if IS_UNIX else 86
+    assert round(analysis["averageRecordSizeInBytes"]) == 85
     assert analysis["fields"] == 11
     assert analysis["rows"] == 9
     assert analysis["rowsWithNullValues"] == 2
@@ -225,7 +222,7 @@ def test_analyze_resource_detailed_with_empty_rows():
 def test_analyze_resource_with_invalid_data():
     resource = TableResource(path="data/invalid.csv")
     analysis = resource.analyze()
-    assert round(analysis["averageRecordSizeInBytes"]) == 12 if IS_UNIX else 14
+    assert round(analysis["averageRecordSizeInBytes"]) == 12
     assert analysis["fields"] == 4
     assert analysis["fieldStats"] == {}
     assert analysis["rows"] == 4
@@ -237,7 +234,7 @@ def test_analyze_resource_with_invalid_data():
 def test_analyze_resource_detailed_with_invalid_data():
     resource = TableResource(path="data/invalid.csv")
     analysis = resource.analyze(detailed=True)
-    assert round(analysis["averageRecordSizeInBytes"]) == 12 if IS_UNIX else 14
+    assert round(analysis["averageRecordSizeInBytes"]) == 12
     assert analysis["fields"] == 4
     assert list(analysis["fieldStats"].keys()) == ["id", "name", "field3", "name2"]
     assert analysis["rows"] == 4
diff --git a/tests/checks/test_baseline.py b/tests/checks/test_baseline.py
index f247422517..1f32c1e3b6 100644
--- a/tests/checks/test_baseline.py
+++ b/tests/checks/test_baseline.py
@@ -1,6 +1,4 @@
-import pytest
-
-from frictionless import Resource, platform
+from frictionless import Resource
 
 # General
 
@@ -29,7 +27,6 @@ def test_validate_invalid():
 # Stats
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_validate_baseline_stats_hash():
     hash = "sha256:a1fd6c5ff3494f697874deeb07f69f8667e903dd94a7bc062dd57550cea26da8"
     resource = Resource("data/table.csv", hash=hash)
@@ -37,7 +34,6 @@ def test_validate_baseline_stats_hash():
     assert report.task.valid
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_validate_baseline_stats_hash_invalid():
     hash = "6c2c61dd9b0e9c6876139a449ed87933"
     resource = Resource("data/table.csv", hash="bad")
@@ -50,14 +46,12 @@ def test_validate_baseline_stats_hash_invalid():
     ]
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_validate_baseline_stats_bytes():
     resource = Resource("data/table.csv", bytes=30)
     report = resource.validate()
     assert report.task.valid
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_validate_baseline_stats_bytes_invalid():
     resource = Resource("data/table.csv", bytes=40)
     report = resource.validate()
@@ -68,14 +62,12 @@ def test_validate_baseline_stats_bytes_invalid():
     ]
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_validate_baseline_stats_rows():
     resource = Resource("data/table.csv", rows=2)
     report = resource.validate()
     assert report.task.valid
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_validate_baseline_stats_rows_invalid():
     resource = Resource("data/table.csv", rows=3)
     report = resource.validate()
diff --git a/tests/console/commands/test_describe.py b/tests/console/commands/test_describe.py
index 6fe6ebfc62..75ee93ac53 100644
--- a/tests/console/commands/test_describe.py
+++ b/tests/console/commands/test_describe.py
@@ -13,7 +13,6 @@
 # General
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_console_describe():
     actual = runner.invoke(console, "describe data/table.csv --stats --yaml")
     assert actual.exit_code == 0
diff --git a/tests/console/commands/test_validate.py b/tests/console/commands/test_validate.py
index 423e527102..89f7e54ef3 100644
--- a/tests/console/commands/test_validate.py
+++ b/tests/console/commands/test_validate.py
@@ -4,7 +4,7 @@
 import yaml
 from typer.testing import CliRunner
 
-from frictionless import Detector, Dialect, platform, validate
+from frictionless import Detector, Dialect, validate
 from frictionless.console import console
 
 runner = CliRunner()
@@ -81,7 +81,6 @@ def test_console_validate_field_missing_values():
     assert no_time(json.loads(actual.stdout)) == no_time(expect.to_descriptor())
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_console_validate_chucksum_hash():
     actual = runner.invoke(
         console,
@@ -95,7 +94,6 @@ def test_console_validate_chucksum_hash():
     assert no_time(json.loads(actual.stdout)) == no_time(expect.to_descriptor())
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_console_validate_chucksum_bytes():
     actual = runner.invoke(console, "validate data/table.csv --json --bytes 30")
     expect = validate("data/table.csv", bytes=30)
diff --git a/tests/package/test_describe.py b/tests/package/test_describe.py
index 4e3f9dfba3..8292f0403f 100644
--- a/tests/package/test_describe.py
+++ b/tests/package/test_describe.py
@@ -99,7 +99,6 @@ def test_describe_package_basepath():
     assert package.get_resource("chunk2").basepath == "data"
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_describe_package_hashing():
     package = Package.describe("data/chunk*.csv", stats=True)
     assert (
diff --git a/tests/resource/test_dereference.py b/tests/resource/test_dereference.py
index b3dabb247e..bb06add167 100644
--- a/tests/resource/test_dereference.py
+++ b/tests/resource/test_dereference.py
@@ -8,7 +8,6 @@
 # General
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_resource_dereference():
     resource = Resource(path="data/table.csv", schema="data/schema.json")
     resource.infer(stats=True)
diff --git a/tests/resource/test_describe.py b/tests/resource/test_describe.py
index 5ce0323722..e18b20dc79 100644
--- a/tests/resource/test_describe.py
+++ b/tests/resource/test_describe.py
@@ -1,6 +1,4 @@
-import pytest
-
-from frictionless import Detector, Dialect, Resource, formats, platform
+from frictionless import Detector, Dialect, Resource, formats
 
 # General
 
@@ -25,7 +23,6 @@ def test_resource_describe():
     }
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_resource_describe_with_stats():
     resource = Resource.describe("data/table.csv", stats=True)
     assert isinstance(resource, Resource)
diff --git a/tests/resource/test_infer.py b/tests/resource/test_infer.py
index 666763caf9..e72f48582d 100644
--- a/tests/resource/test_infer.py
+++ b/tests/resource/test_infer.py
@@ -1,11 +1,8 @@
-import pytest
-
-from frictionless import Resource, platform
+from frictionless import Resource
 
 # General
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_resource_infer():
     resource = Resource(path="data/table.csv")
     resource.infer(stats=True)
@@ -30,7 +27,6 @@ def test_resource_infer():
     }
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_resource_infer_source_non_tabular():
     resource = Resource(path="data/text.txt")
     resource.infer(stats=True)
diff --git a/tests/resource/test_read.py b/tests/resource/test_read.py
index 759410011b..643110243a 100644
--- a/tests/resource/test_read.py
+++ b/tests/resource/test_read.py
@@ -2,12 +2,11 @@
 
 import pytest
 
-from frictionless import Resource, platform, resources
+from frictionless import Resource, resources
 
 # General
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 @pytest.mark.skipif(sys.version_info < (3, 7), reason="Requires Python3.7+")
 def test_resource_read_bytes():
     resource = Resource(path="data/text.txt")
@@ -15,13 +14,11 @@ def test_resource_read_bytes():
     assert bytes == b"text\n"
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_resource_read_bytes_bug_1341():
     assert len(Resource("data/issue-1341.csv").read_bytes()) == 16792
     assert len(Resource("data/issue-1066.csv").read_bytes()) == 3289808
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_resource_read_text():
     resource = Resource(path="data/text.txt")
     text = resource.read_text()
diff --git a/tests/resources/table/test_stats.py b/tests/resources/table/test_stats.py
index 2c2a6c0f7a..716f41fe31 100644
--- a/tests/resources/table/test_stats.py
+++ b/tests/resources/table/test_stats.py
@@ -1,6 +1,6 @@
 import pytest
 
-from frictionless import Dialect, platform
+from frictionless import Dialect
 from frictionless.resources import TableResource
 
 BASEURL = "https://raw.githubusercontent.com/frictionlessdata/frictionless-py/master/%s"
@@ -9,7 +9,6 @@
 # General
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_resource_stats_hash():
     with TableResource(path="data/doublequote.csv") as resource:
         resource.read_rows()
@@ -19,7 +18,6 @@ def test_resource_stats_hash():
         )
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_resource_stats_hash_compressed():
     with TableResource(path="data/doublequote.csv.zip") as resource:
         resource.read_rows()
@@ -30,7 +28,6 @@ def test_resource_stats_hash_compressed():
 
 
 @pytest.mark.vcr
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_resource_stats_hash_remote():
     with TableResource(path=BASEURL % "data/doublequote.csv") as resource:
         resource.read_rows()
@@ -40,14 +37,12 @@ def test_resource_stats_hash_remote():
         )
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_resource_stats_bytes():
     with TableResource(path="data/doublequote.csv") as resource:
         resource.read_rows()
         assert resource.stats.bytes == 7346
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_resource_stats_bytes_compressed():
     with TableResource(path="data/doublequote.csv.zip") as resource:
         resource.read_rows()
@@ -55,7 +50,6 @@ def test_resource_stats_bytes_compressed():
 
 
 @pytest.mark.vcr
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_resource_stats_bytes_remote():
     with TableResource(path=BASEURL % "data/doublequote.csv") as resource:
         resource.read_rows()
diff --git a/tests/validator/resource/test_stats.py b/tests/validator/resource/test_stats.py
index 51881087a8..ab19b9239c 100644
--- a/tests/validator/resource/test_stats.py
+++ b/tests/validator/resource/test_stats.py
@@ -1,12 +1,8 @@
-import pytest
-
-from frictionless import platform
 from frictionless.resources import TableResource
 
 # General
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_resource_validate_stats_hash():
     hash = "sha256:a1fd6c5ff3494f697874deeb07f69f8667e903dd94a7bc062dd57550cea26da8"
     resource = TableResource(path="data/table.csv", hash=hash)
@@ -14,7 +10,6 @@ def test_resource_validate_stats_hash():
     assert report.task.valid
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_resource_validate_stats_hash_invalid():
     hash = "6c2c61dd9b0e9c6876139a449ed87933"
     resource = TableResource(path="data/table.csv", hash="bad")
@@ -27,14 +22,12 @@ def test_resource_validate_stats_hash_invalid():
     ]
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_resource_validate_stats_bytes():
     resource = TableResource(path="data/table.csv", bytes=30)
     report = resource.validate()
     assert report.task.valid
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_resource_validate_stats_bytes_invalid():
     resource = TableResource(path="data/table.csv", bytes=40)
     report = resource.validate()
@@ -45,14 +38,12 @@ def test_resource_validate_stats_bytes_invalid():
     ]
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_resource_validate_stats_rows():
     resource = TableResource(path="data/table.csv", rows=2)
     report = resource.validate()
     assert report.task.valid
 
 
-@pytest.mark.skipif(platform.type == "windows", reason="Fix on Windows")
 def test_resource_validate_stats_rows_invalid():
     resource = TableResource(path="data/table.csv", rows=3)
     report = resource.validate()