Repository: Wenzel/checksec.py (language: python)

Issue:
Testing: add E2E tests
Add E2E tests with real binaries, and test JSON output


Discussion / hints:

Fix patch:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ead3d5c..5cf4704 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -99,10 +99,15 @@ jobs:
 
   test:
     needs: build
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
 
       - name: Set up Python 3.7 🐍
         uses: actions/setup-python@v1
@@ -113,13 +118,7 @@ jobs:
         run: python -m pip install nox==2020.8.22
 
       - name: Run tests
-        run: nox -s test
-
-      - name: Install package 📦
-        run: python -m pip install .
-
-      - name: Run smoke test
-        run: checksec /usr/lib
+        run: nox -s test_e2e -- -s
 
   release:
     needs: test
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..cd4ed58
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "tests/binaries"]
+	path = tests/binaries
+	url = git@github.com:Wenzel/checksec.py-test-binaries.git
diff --git a/noxfile.py b/noxfile.py
index abf7e05..052e839 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -1,6 +1,6 @@
 import nox
 
-nox.options.sessions = ["fmt", "lint", "test"]
+nox.options.sessions = ["fmt", "lint", "test_e2e"]
 
 
 @nox.session
@@ -31,7 +31,28 @@ def run(session):
 
 
 @nox.session
-def test(session):
+def test_unit(session):
+    # run unit tests
+    args = session.posargs
     session.install("-r", "requirements.txt")
     session.install("pytest==6.0.2", "coverage==5.3")
-    session.run("coverage", "run", "-m", "pytest", "-v")
+    session.run("coverage", "run", "--concurrency=multiprocessing", "-m", "pytest", "-v", "-k", "unit", *args)
+    session.run("coverage", "combine")
+    session.run("coverage", "report")
+
+
+@nox.session
+def test_e2e(session):
+    args = session.posargs
+    session.install(".")
+    session.install("pytest==6.0.2", "coverage==5.3")
+    session.run("coverage", "run", "--concurrency=multiprocessing", "-m", "pytest", "-v", "-k", "e2e", *args)
+    session.run("coverage", "combine")
+    session.run("coverage", "report")
+
+
+@nox.session
+def coverage(session):
+    args = session.posargs if session.posargs else ["report"]
+    session.install("coverage==5.3")
+    session.run("coverage", *args)
diff --git a/setup.cfg b/setup.cfg
index ca4960b..7ed044c 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -3,3 +3,10 @@ max-line-length=120
 exclude = .git, .nox, venv, build, dist
 # black compatibility
 extend-ignore = E203
+
+[coverage:run]
+# measure branch coverage in addition to statement coverage
+branch = True
+# restrict measurement to avoid .nox/*
+source =
+    checksec
diff --git a/setup.py b/setup.py
index d0899c9..f3dc43e 100644
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 import setuptools
 
-with open("README.md", "r") as fh:
+with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 
 with open("requirements.txt") as f:
@@ -8,7 +8,7 @@
 
 setuptools.setup(
     name="checksec.py",
-    version="0.5.0",
+    version="0.5.1",
     author="Mathieu Tarral",
     author_email="mathieu.tarral@protonmail.com",
     description="Checksec tool implemented in Python",


Test patch:
diff --git a/tests/binaries b/tests/binaries
new file mode 160000
index 0000000..1b842c5
--- /dev/null
+++ b/tests/binaries
@@ -0,0 +1,1 @@
+Subproject commit 1b842c58b2164694c7dc040f3a7331d5c58f9021
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 0000000..e8d62dc
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,13 @@
+import json
+import subprocess
+from pathlib import Path
+from typing import Dict
+
+
+def run_checksec(bin_path: Path, libc_path: Path = None) -> Dict:
+    """Runs checksec from command line, returns json output as dict"""
+    cmd = ["checksec", str(bin_path), "-j"]
+    if libc_path:
+        cmd.extend(["-s", str(libc_path)])
+    output = subprocess.check_output(cmd)
+    return json.loads(output.decode())
diff --git a/tests/e2e/test_e2e_elf.py b/tests/e2e/test_e2e_elf.py
new file mode 100644
index 0000000..0e7a22e
--- /dev/null
+++ b/tests/e2e/test_e2e_elf.py
@@ -0,0 +1,66 @@
+"""E2E tests for ELF"""
+
+from pathlib import Path
+
+import pytest
+
+from checksec.elf import PIEType, RelroType
+from tests.conftest import run_checksec
+
+ELF_BINARIES = Path(__file__).parent.parent / "binaries" / "elf"
+
+
+@pytest.mark.parametrize("is_enabled", [False, True])
+@pytest.mark.parametrize("prop", ["nx", "canary", "rpath", "runpath", "symbols", "fortify_source"])
+def test_bool_prop(prop: str, is_enabled: bool):
+    """Test that boolean prop is disabled/enabled"""
+    libc_path = ELF_BINARIES / "libc-2.27.so"
+    bin_path = ELF_BINARIES / f"{prop}_{'enabled' if is_enabled else 'disabled'}"
+    chk_data = run_checksec(bin_path, libc_path)
+    assert chk_data[str(bin_path)][prop] == is_enabled
+
+
+@pytest.mark.parametrize("relro_type", list(RelroType))
+def test_relro(relro_type: RelroType):
+    """Test that relro type is No/Partial/Full"""
+    bin_path = ELF_BINARIES / f"relro_{relro_type.name.lower()}"
+    chk_data = run_checksec(bin_path)
+    assert chk_data[str(bin_path)]["relro"] == relro_type.name
+
+
+@pytest.mark.parametrize("pie_type", list(PIEType))
+def test_pie(pie_type):
+    """Test that PIE is No/Partial/Full"""
+    bin_path = ELF_BINARIES / f"pie_{pie_type.name.lower()}"
+    chk_data = run_checksec(bin_path)
+    assert chk_data[str(bin_path)]["pie"] == pie_type.name
+
+
+def test_fortified():
+    """Test the fortified functions"""
+    libc_path = ELF_BINARIES / "libc-2.27.so"
+    bin_path = ELF_BINARIES / "fortify_funcs"
+    chk_data = run_checksec(bin_path, libc_path)
+    fortified_funcs = ["__fprintf_chk@@GLIBC_2.3.4", "__printf_chk@@GLIBC_2.3.4"]
+    assert chk_data[str(bin_path)]["fortified"] == len(fortified_funcs)
+
+
+def test_fortifiable():
+    """Test the fortifiable functions"""
+    libc_path = ELF_BINARIES / "libc-2.27.so"
+    bin_path = ELF_BINARIES / "fortify_funcs"
+    chk_data = run_checksec(bin_path, libc_path)
+    fortified_funcs = ["__fprintf_chk@@GLIBC_2.3.4", "__printf_chk@@GLIBC_2.3.4"]
+    non_fortified_funcs = ["fgets"]
+    assert chk_data[str(bin_path)]["fortify-able"] == len(fortified_funcs) + len(non_fortified_funcs)
+
+
+def test_fortify_score():
+    """Test the fortify score"""
+    libc_path = ELF_BINARIES / "libc-2.27.so"
+    bin_path = ELF_BINARIES / "fortify_funcs"
+    chk_data = run_checksec(bin_path, libc_path)
+    fortified_funcs = ["__fprintf_chk@@GLIBC_2.3.4", "__printf_chk@@GLIBC_2.3.4"]
+    non_fortified_funcs = ["fgets"]
+    total = len(fortified_funcs) + len(non_fortified_funcs)
+    assert chk_data[str(bin_path)]["fortify_score"] == round((2 * 100) / total, 0)
diff --git a/tests/e2e/test_e2e_pe.py b/tests/e2e/test_e2e_pe.py
new file mode 100644
index 0000000..7091afd
--- /dev/null
+++ b/tests/e2e/test_e2e_pe.py
@@ -0,0 +1,31 @@
+"""E2E tests for PE"""
+
+from pathlib import Path
+
+import pytest
+
+from tests.conftest import run_checksec
+
+PE_BINARIES = Path(__file__).parent.parent / "binaries" / "pe"
+
+
+@pytest.mark.parametrize("is_enabled", [False, True])
+@pytest.mark.parametrize(
+    "prop",
+    [
+        "nx",
+        "canary",
+        "dynamic_base",
+        "aslr",
+        "high_entropy_va",
+        "safe_seh",
+        "force_integrity",
+        "guard_cf",
+        "isolation",
+    ],
+)
+def test_bool_prop(prop: str, is_enabled: bool):
+    """Test that boolean prop is disabled/enabled"""
+    bin_path = PE_BINARIES / f"{prop}_{'enabled' if is_enabled else 'disabled'}.exe"
+    chk_data = run_checksec(bin_path)
+    assert chk_data[str(bin_path)][prop] == is_enabled
diff --git a/tests/main.c b/tests/main.c
deleted file mode 100644
index 056f966..0000000
--- a/tests/main.c
+++ /dev/null
@@ -1,26 +0,0 @@
-// https://www.programiz.com/c-programming/examples/concatenate-string
-
-#include <stdio.h>
-int main() {
-  char s1[100] = "programming ", s2[] = "is awesome";
-  int length, j;
-
-  // store length of s1 in the length variable
-  length = 0;
-  while (s1[length] != '\0') {
-    ++length;
-  }
-
-  // concatenate s2 to s1
-  for (j = 0; s2[j] != '\0'; ++j, ++length) {
-    s1[length] = s2[j];
-  }
-
-  // terminating the s1 string
-  s1[length] = '\0';
-
-  printf("After concatenation: ");
-  puts(s1);
-
-  return 0;
-}
\ No newline at end of file
diff --git a/tests/test_elf.py b/tests/test_elf.py
deleted file mode 100644
index fa1b228..0000000
--- a/tests/test_elf.py
+++ /dev/null
@@ -1,98 +0,0 @@
-from pathlib import Path
-from subprocess import check_call
-from tempfile import NamedTemporaryFile
-
-from checksec.elf import ELFSecurity, PIEType, RelroType
-
-MAIN_PATH = Path(__file__).parent / "main.c"
-
-
-def test_nx_enabled():
-    with NamedTemporaryFile() as tmp_f:
-        cmdline = ["gcc", str(MAIN_PATH), "-o", tmp_f.name]
-        check_call(cmdline)
-        elf = ELFSecurity(Path(tmp_f.name))
-        assert elf.has_nx
-
-
-def test_nx_disabled():
-    with NamedTemporaryFile() as tmp_f:
-        cmdline = ["gcc", "-z", "execstack", str(MAIN_PATH), "-o", tmp_f.name]
-        check_call(cmdline)
-        elf = ELFSecurity(Path(tmp_f.name))
-        assert not elf.has_nx
-
-
-def test_canary_enabled():
-    with NamedTemporaryFile() as tmp_f:
-        cmdline = ["gcc", "-fstack-protector", str(MAIN_PATH), "-o", tmp_f.name]
-        check_call(cmdline)
-        elf = ELFSecurity(Path(tmp_f.name))
-        assert elf.has_canary
-
-
-def test_canary_disabled():
-    with NamedTemporaryFile() as tmp_f:
-        cmdline = ["gcc", "-fno-stack-protector", str(MAIN_PATH), "-o", tmp_f.name]
-        check_call(cmdline)
-        elf = ELFSecurity(Path(tmp_f.name))
-        assert not elf.has_canary
-
-
-def test_relro_disabled():
-    with NamedTemporaryFile() as tmp_f:
-        cmdline = ["gcc", "-Wl,-z,norelro", str(MAIN_PATH), "-o", tmp_f.name]
-        check_call(cmdline)
-        elf = ELFSecurity(Path(tmp_f.name))
-        assert elf.relro == RelroType.No
-
-
-def test_relro_full():
-    with NamedTemporaryFile() as tmp_f:
-        cmdline = ["gcc", "-Wl,-z,relro,-z,now", str(MAIN_PATH), "-o", tmp_f.name]
-        check_call(cmdline)
-        elf = ELFSecurity(Path(tmp_f.name))
-        assert elf.relro == RelroType.Full
-
-
-# TODO: DSO and PIE
-def test_pie_disabled():
-    with NamedTemporaryFile() as tmp_f:
-        cmdline = ["gcc", "-no-pie", str(MAIN_PATH), "-o", tmp_f.name]
-        check_call(cmdline)
-        elf = ELFSecurity(Path(tmp_f.name))
-        assert elf.pie == PIEType.No
-
-
-def test_rpath_disabled():
-    with NamedTemporaryFile() as tmp_f:
-        cmdline = ["gcc", str(MAIN_PATH), "-o", tmp_f.name]
-        check_call(cmdline)
-        elf = ELFSecurity(Path(tmp_f.name))
-        assert not elf.has_rpath
-
-
-def test_rpath_enabled():
-    with NamedTemporaryFile() as tmp_f:
-        rpath = "/opt/lib"
-        cmdline = ["gcc", "-Wl,--disable-new-dtags", f"-Wl,-rpath,{rpath}", str(MAIN_PATH), "-o", tmp_f.name]
-        check_call(cmdline)
-        elf = ELFSecurity(Path(tmp_f.name))
-        assert elf.has_rpath
-
-
-def test_runpath_disabled():
-    with NamedTemporaryFile() as tmp_f:
-        cmdline = ["gcc", str(MAIN_PATH), "-o", tmp_f.name]
-        check_call(cmdline)
-        elf = ELFSecurity(Path(tmp_f.name))
-        assert not elf.has_runpath
-
-
-def test_runpath_enabled():
-    with NamedTemporaryFile() as tmp_f:
-        rpath = "/opt/lib"
-        cmdline = ["gcc", "-Wl,--enable-new-dtags", f"-Wl,-rpath,{rpath}", str(MAIN_PATH), "-o", tmp_f.name]
-        check_call(cmdline)
-        elf = ELFSecurity(Path(tmp_f.name))
-        assert elf.has_runpath