Repository: unascribed/FlexVer (language: python)

Issue:
Test vectors
It would be useful for testing new implementations and verifying consistency between implementations to have a large series of test vectors, that any implementation can read and verify. For this purpose, I suggest a trivial plain text format, like the following, called something like `test_vectors.txt` in the root of the repo:

```sql
# Basic numeric ordering (lexical string sort fails these)
10 > 2
100 > 10

# Trivial common numerics
1.0 < 1.1
1.0 < 1.0.1
1.1 > 1.0.1

# SemVer compatibility
1.5 > 1.5-pre1
1.5 = 1.5+foobar

# SemVer incompatibility
1.5 < 1.5-2
1.5-pre10 > 1.5-pre2

# Optional features
## Too large for a 64-bit integer or double, checks if codepoint-wise or integer-parse is being used
36893488147419103232 <? 36893488147419103233

# ...and so on
```

We would also want weird Unicode edge cases in here, and tests for things like the leading-zero pitfall with integer-parse. (Open question: Does there exist a Unicode codepoint that sorts differently as UTF-16 and as UTF-32, when compared as outlined in the spec?)

It would likely be good to start this by moving the current test vectors out of being hardcoded in each language implementation.

A series of test vectors for decompositions may also be useful, such as:

```sql
1.0.1-pre2+foobar5 n1 t. n0 t. n1 p-pre n2 a+foobar n5
```
Test vectors
It would be useful for testing new implementations and verifying consistency between implementations to have a large series of test vectors, that any implementation can read and verify. For this purpose, I suggest a trivial plain text format, like the following, called something like `test_vectors.txt` in the root of the repo:

```sql
# Basic numeric ordering (lexical string sort fails these)
10 > 2
100 > 10

# Trivial common numerics
1.0 < 1.1
1.0 < 1.0.1
1.1 > 1.0.1

# SemVer compatibility
1.5 > 1.5-pre1
1.5 = 1.5+foobar

# SemVer incompatibility
1.5 < 1.5-2
1.5-pre10 > 1.5-pre2

# Optional features
## Too large for a 64-bit integer or double, checks if codepoint-wise or integer-parse is being used
36893488147419103232 <? 36893488147419103233

# ...and so on
```

We would also want weird Unicode edge cases in here, and tests for things like the leading-zero pitfall with integer-parse. (Open question: Does there exist a Unicode codepoint that sorts differently as UTF-16 and as UTF-32, when compared as outlined in the spec?)

It would likely be good to start this by moving the current test vectors out of being hardcoded in each language implementation.

A series of test vectors for decompositions may also be useful, such as:

```sql
1.0.1-pre2+foobar5 n1 t. n0 t. n1 p-pre n2 a+foobar n5
```


Discussion / hints:

Fix patch:
diff --git a/java/build.gradle b/java/build.gradle
index 4bb98f8..78c112b 100644
--- a/java/build.gradle
+++ b/java/build.gradle
@@ -8,6 +8,20 @@ sourceCompatibility = targetCompatibility = 8
 archivesBaseName = 'FlexVer'
 version = '1.0.2'
 
+repositories {
+	mavenCentral()
+}
+
+dependencies {
+    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.9.2'
+    testImplementation 'org.junit.jupiter:junit-jupiter-engine:5.9.2'
+	testImplementation 'org.junit.jupiter:junit-jupiter-params:5.9.2'
+}
+
+test {
+    useJUnitPlatform()
+}
+
 compileJava {
 	options.release = 8
 }
diff --git a/rust/src/lib.rs b/rust/src/lib.rs
index 8b093a9..22a0887 100644
--- a/rust/src/lib.rs
+++ b/rust/src/lib.rs
@@ -161,67 +161,78 @@ impl Ord for FlexVer<'_> {
 
 #[cfg(test)]
 mod tests {
+    use std::path::PathBuf;
+    use std::fs;
+
     use super::*;
 
-    fn test(left: &str, right: &str, result: Ordering) {
-        assert_eq!(compare(&left, &right), result);
-        assert_eq!(
-            compare(&right, &left),
-            match result {
-                Less => Greater,
-                Greater => Less,
-                Equal => Equal,
-            }
-        );
+    const ENABLED_TESTS: &'static [&str] = &[ "test_vectors.txt" ];
+
+    fn test(left: &str, right: &str, expected: Ordering) -> Result<(), String> {
+        if compare(left, right) != expected {
+            return Err(format!("Expected {:?} but found {:?}", expected, compare(left, right)));
+        }
+
+        // Assert commutativity, if right > left than left < right and vice versa  
+        let inverse = match expected {
+            Less => Greater,
+            Greater => Less,
+            Equal => Equal,
+        };
+        if compare(right, left) != inverse {
+            return Err(format!("Comparison method violates its general contract!"));
+        }
+
+        Ok(())
     }
 
     #[test]
-    fn test_compare() {
-        test("b1.7.3", "a1.2.6", Greater);
-        test("b1.2.6", "a1.7.3", Greater);
-        test("a1.1.2", "a1.1.2_01", Less);
-        test("1.16.5-0.00.5", "1.14.2-1.3.7", Greater);
-        test("1.0.0", "1.0.0-2", Less);
-        test("1.0.0", "1.0.0_01", Less);
-        test("1.0.1", "1.0.0_01", Greater);
-        test("1.0.0_01", "1.0.1", Less);
-        test("0.17.1-beta.1", "0.17.1", Less);
-        test("0.17.1-beta.1", "0.17.1-beta.2", Less);
-        test("1.4.5_01", "1.4.5_01+fabric-1.17", Equal);
-        test("1.4.5_01", "1.4.5_01+fabric-1.17+ohno", Equal);
-        test("14w16a", "18w40b", Less);
-        test("18w40a", "18w40b", Less);
-        test("1.4.5_01+fabric-1.17", "18w40b", Less);
-        test("13w02a", "c0.3.0_01", Less);
-        test("0.6.0-1.18.x", "0.9.beta-1.18.x", Less);
+    fn standardized_tests() {
+        let test_folder = PathBuf::from("../test");
+        let errors = ENABLED_TESTS.iter().flat_map(|test_file_name| {
+            let test_file = test_folder.join(test_file_name);
+            fs::read_to_string(test_file).unwrap()
+                .lines()
+                .enumerate()
+                .filter(|(_, line)| !line.starts_with("#"))
+                .filter(|(_, line)| !line.is_empty())
+                .map(|(num, line)| {
+                    let split: Vec<&str> = line.split(" ").collect();
+                    if split.len() != 3 { panic!("{}:{} Line formatted incorrectly, expected 2 spaces: {}", test_file_name, num, line) }
+                    let ord = match split[1] {
+                        "<" => Less,
+                        "=" => Equal,
+                        ">" => Greater,
+                        _ => panic!("{} is not a valid ordering", split[1])
+                    };
+                    test(split[0], split[2], ord).map_err(|message| (line.to_owned(), message))
+                }).collect::<Vec<_>>()
+            })
+            .filter_map(|res| res.err())
+            .collect::<Vec<_>>();
+        
+        if !errors.is_empty() {
+            errors.iter().for_each(|(line, message)| println!("{}: {}", line, message));
+            panic!()
+        }
     }
 
     #[test]
-    fn test_ord() {
-        assert!(FlexVer("b1.7.3") > FlexVer("a1.2.6"));
-        assert!(FlexVer("b1.2.6") > FlexVer("a1.7.3"));
-        assert!(FlexVer("a1.1.2") < FlexVer("a1.1.2_01"));
-        assert!(FlexVer("1.16.5-0.00.5") > FlexVer("1.14.2-1.3.7"));
-        assert!(FlexVer("1.0.0") < FlexVer("1.0.0-2"));
-        assert!(FlexVer("1.0.0") < FlexVer("1.0.0_01"));
-        assert!(FlexVer("1.0.1") > FlexVer("1.0.0_01"));
-        assert!(FlexVer("1.0.0_01") < FlexVer("1.0.1"));
-        assert!(FlexVer("0.17.1-beta.1") < FlexVer("0.17.1"));
-        assert!(FlexVer("0.17.1-beta.1") < FlexVer("0.17.1-beta.2"));
-        assert!(FlexVer("1.4.5_01") == FlexVer("1.4.5_01+fabric-1.17"));
-        assert!(FlexVer("1.4.5_01") == FlexVer("1.4.5_01+fabric-1.17+ohno"));
-        assert!(FlexVer("14w16a") < FlexVer("18w40b"));
-        assert!(FlexVer("18w40a") < FlexVer("18w40b"));
-        assert!(FlexVer("1.4.5_01+fabric-1.17") < FlexVer("18w40b"));
-        assert!(FlexVer("13w02a") < FlexVer("c0.3.0_01"));
-        assert!(FlexVer("0.6.0-1.18.x") < FlexVer("0.9.beta-1.18.x"));
+    fn test_min() {
+        assert_eq!(FlexVer("1.0.0"), FlexVer("1.0.0").min(FlexVer("1.0.0")));
+        assert_eq!(FlexVer("a1.2.6"), FlexVer("b1.7.3").min(FlexVer("a1.2.6")));
+        assert_eq!(FlexVer("a1.7.3"), FlexVer("b1.2.6").min(FlexVer("a1.7.3")));
+    }
 
+    #[test]
+    fn test_max() {
         assert_eq!(FlexVer("b1.7.3"), FlexVer("b1.7.3").max(FlexVer("a1.2.6")));
         assert_eq!(FlexVer("b1.2.6"), FlexVer("b1.2.6").max(FlexVer("a1.7.3")));
-        assert_eq!(FlexVer("a1.2.6"), FlexVer("b1.7.3").min(FlexVer("a1.2.6")));
-        assert_eq!(FlexVer("a1.7.3"), FlexVer("b1.2.6").min(FlexVer("a1.7.3")));
         assert_eq!(FlexVer("1.0.0"), FlexVer("1.0.0").max(FlexVer("1.0.0")));
-        assert_eq!(FlexVer("1.0.0"), FlexVer("1.0.0").min(FlexVer("1.0.0")));
+    }
+
+    #[test]
+    fn test_clamp() {
         assert_eq!(
             FlexVer("1.1.0"),
             FlexVer("1.1.0").clamp(FlexVer("1.0.0"), FlexVer("1.2.0"))


Test patch:
diff --git a/go/flexver/main_test.go b/go/flexver/main_test.go
index c16ed29..49978d0 100644
--- a/go/flexver/main_test.go
+++ b/go/flexver/main_test.go
@@ -9,219 +9,233 @@
 package flexver
 
 import (
+	"bufio"
 	"fmt"
+	"os"
 	"reflect"
+	"strings"
 	"testing"
-	"unicode/utf8"
 )
 
-func TestBasicSort(t *testing.T) {
-	var input = []string{
-		"0.17.2", "0.1.0", "1.0.0", "1000", "0.17.2-pre.1", "1.16.5+pre",
-	}
-	var expect = []string{
-		"0.1.0", "0.17.2-pre.1", "0.17.2", "1.0.0", "1.16.5+pre", "1000",
-	}
-	VersionSlice(input).Sort()
-	if !reflect.DeepEqual(input, expect) {
-		t.Fatalf("Failed to sort strings: got %v (expected %v)", input, expect)
-	}
+var ENABLED_TESTS = []string{
+	"test_vectors.txt",
+	"large.txt",
 }
 
 const (
-	opLT        = -1
-	opEQ        = 0
-	opGT        = 1
-	shouldError = 10
+	opLT = -1
+	opEQ = 0
+	opGT = 1
 )
 
-func opChar(op int) string {
-	if op < 0 {
-		return "<"
-	} else if op == opEQ {
-		return "="
-	} else if op == opGT {
-		return ">"
-	} else {
-		return "!"
+func RunCompare(t *testing.T, lefthand string, righthand string, ordering int) {
+	res := Compare(lefthand, righthand)
+	if (ordering == opLT && !(res < 0)) ||
+		(ordering == opEQ && !(res == 0)) ||
+		(ordering == opGT && !(res > 0)) {
+		t.Errorf("Compare returned %v", res)
 	}
-}
 
-type testInstance struct {
-	a  string
-	op int
-	b  string
-}
+	res2, err := CompareError(lefthand, righthand)
+	if err != nil {
+		t.Fatalf("CompareError returned an unexpected error: %v", err)
+	}
 
-func t(a string, op int, b string) testInstance {
-	return testInstance{a: a, op: op, b: b}
+	if res != res2 {
+		t.Error("CompareError did not give the same result as Compare")
+	}
 }
 
-func (i testInstance) name() string {
-	a := i.a
-	b := i.b
-	if !utf8.ValidString(a) {
-		a = "badutf8"
+func RunLess(t *testing.T, lefthand string, righthand string, ordering int) {
+	res := Less(lefthand, righthand)
+	if ordering == opLT {
+		if !res {
+			t.Error("Less incorrectly returned false")
+		}
+	} else {
+		if res {
+			t.Error("Less incorrectly returned true")
+		}
 	}
-	if !utf8.ValidString(b) {
-		b = "badutf8"
+
+	res2, err := LessError(lefthand, righthand)
+	if err != nil {
+		t.Fatalf("LessError returned an unexpected error: %v", err)
 	}
-	return a + " " + opChar(i.op) + " " + b
-}
 
-var specTests = []testInstance{
-	t("b1.7.3", opGT, "a1.2.6"),
-	t("a1.1.2", opLT, "a1.1.2_01"),
-	t("1.16.5-0.00.5", opGT, "1.14.2-1.3.7"),
-	t("1.0.0", opLT, "1.0.0_01"),
-	t("1.0.1", opGT, "1.0.0_01"),
-	t("0.17.1-beta.1", opLT, "0.17.1"),
-	t("0.17.1-beta.1", opLT, "0.17.1-beta.2"),
-	t("1.4.5_01", opEQ, "1.4.5_01+exp-1.17"),
-	t("1.4.5_01", opEQ, "1.4.5_01+exp-1.17-moretext"),
-	t("14w16a", opLT, "18w40b"),
-	t("18w40a", opLT, "18w40b"),
-	t("1.4.5_01+exp-1.17", opLT, "18w40b"),
-	t("13w02a", opLT, "c0.3.0_01"),
-	t("0.6.0-1.18.x", opLT, "0.9.beta-1.18.x"),
-	t("36893488147419103232", opLT, "36893488147419103233"),
-	t("1.0", opLT, "1.1"),
-	t("1.0", opLT, "1.0.1"),
-	t("10", opGT, "2"),
+	if res != res2 {
+		t.Error("LessError did not give the same result as Less")
+	}
 }
 
-var miscTests = []testInstance{
-	// Empty strings
-	t("", opEQ, ""),
-	t("1", opGT, ""),
-	t("", opLT, "1"),
-	// Invalid UTF8
-	t("\xc3\x28", shouldError, ""),
-	t("", shouldError, "\xc3\x28"),
-	// Check boundary between textual and prerelease
-	t("a-a", opLT, "a"),
-	// Check boundary between textual and appendix
-	t("a+a", opEQ, "a"),
-	// Dash is included in prerelease comparison (if stripped it will be a smaller component)
-	// Note that a-a < a=a regardless since the prerelease splits the component creating a smaller first component; 0 is added to force splitting regardless
-	t("a0-a", opLT, "a0=a"),
-	// Pre-releases must contain only non-digit
-	t("1.16.5-10", opGT, "1.16.5"),
-	// Pre-releases can have multiple dashes (should not be split)
-	// Reasoning for test data: "p-a!" > "p-a-" (correct); "p-a!" < "p-a t-" (what happens if every dash creates a new component)
-	t("-a-", opGT, "-a!"),
-}
+func RunEqual(t *testing.T, lefthand string, righthand string, ordering int) {
+	res := Equal(lefthand, righthand)
+	if ordering == opEQ {
+		if !res {
+			t.Error("Equal incorrectly returned false")
+		}
+	} else {
+		if res {
+			t.Error("Equal incorrectly returned true")
+		}
+	}
 
-var allTests = append(append([]testInstance{}, specTests...), miscTests...)
-
-func TestCompare(t *testing.T) {
-	for _, v := range allTests {
-		t.Run(v.name(), func(t *testing.T) {
-			if v.op == shouldError {
-				// Catch panic
-				defer func() { _ = recover() }()
-			}
-
-			res := Compare(v.a, v.b)
-			if (v.op == opLT && res >= 0) ||
-				(v.op == opEQ && res != 0) ||
-				(v.op == opGT && res <= 0) ||
-				(v.op == shouldError) {
-				t.Fatalf("Unexpected result %v", res)
-			}
-		})
+	res2, err := EqualError(lefthand, righthand)
+	if err != nil {
+		t.Fatalf("EqualError returned an unexpected error: %v", err)
+	}
+
+	if res != res2 {
+		t.Error("EqualError did not give the same result as Equal")
 	}
 }
 
-func TestLess(t *testing.T) {
-	for _, v := range allTests {
-		t.Run(v.name(), func(t *testing.T) {
-			if v.op == shouldError {
-				// Catch panic
-				defer func() { _ = recover() }()
-			}
-
-			res := Less(v.a, v.b)
-			if (v.op == opLT && res == false) ||
-				(v.op == opEQ && res == true) ||
-				(v.op == opGT && res == true) ||
-				(v.op == shouldError) {
-				t.Fatalf("Unexpected result %v", res)
-			}
-		})
+func TestStandardized(t *testing.T) {
+	for _, test := range ENABLED_TESTS {
+		ProcessTestfile(t, test)
 	}
 }
 
-func TestEqual(t *testing.T) {
-	for _, v := range allTests {
-		t.Run(v.name(), func(t *testing.T) {
-			if v.op == shouldError {
-				// Catch panic
-				defer func() { _ = recover() }()
-			}
-
-			res := Equal(v.a, v.b)
-			if (v.op == opLT && res == true) ||
-				(v.op == opEQ && res == false) ||
-				(v.op == opGT && res == true) ||
-				(v.op == shouldError) {
-				t.Fatalf("Unexpected result %v", res)
-			}
+func ProcessTestfile(t *testing.T, test string) {
+	file, err := os.Open("../../test/" + test)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		if strings.HasPrefix(line, "#") {
+			continue
+		}
+		if len(line) == 0 {
+			continue
+		}
+
+		split := strings.Split(line, " ")
+		if len(split) != 3 {
+			t.Fatal("Line formatted incorrectly, expected 2 spaces: " + line)
+		}
+
+		ord := 0
+		switch split[1] {
+		case "<":
+			ord = opLT
+		case "=":
+			ord = opEQ
+		case ">":
+			ord = opGT
+		}
+
+		lefthand := split[0]
+		righthand := split[2]
+
+		t.Run(line, func(t *testing.T) {
+			t.Run("Compare", func(t *testing.T) {
+				RunCompare(t, lefthand, righthand, ord)
+			})
+
+			t.Run("Less", func(t *testing.T) {
+				RunLess(t, lefthand, righthand, ord)
+			})
+
+			t.Run("Equal", func(t *testing.T) {
+				RunEqual(t, lefthand, righthand, ord)
+			})
 		})
 	}
+
+	if err := scanner.Err(); err != nil {
+		t.Fatal(err)
+	}
 }
 
-func TestCompareError(t *testing.T) {
-	for _, v := range allTests {
-		t.Run(v.name(), func(t *testing.T) {
-			res, err := CompareError(v.a, v.b)
-			if err == nil {
-				if (v.op == opLT && res >= 0) ||
-					(v.op == opEQ && res != 0) ||
-					(v.op == opGT && res <= 0) ||
-					(v.op == shouldError) {
-					t.Fatalf("Unexpected result %v", res)
-				}
-			} else if v.op != shouldError {
-				t.Fatalf("Unexpected error %v", err)
-			}
-		})
+func TestInvalid(t *testing.T) {
+	// Run through some invalid inputs and fail the test if it didn't return an error
+	_, err := CompareError("\xc3\x28", "")
+	if err == nil {
+		t.Fatal()
+	}
+	_, err = LessError("\xc3\x28", "")
+	if err == nil {
+		t.Fatal()
+	}
+	_, err = EqualError("\xc3\x28", "")
+	if err == nil {
+		t.Fatal()
+	}
+	_, err = CompareError("", "\xc3\x28")
+	if err == nil {
+		t.Fatal()
+	}
+	_, err = LessError("", "\xc3\x28")
+	if err == nil {
+		t.Fatal()
+	}
+	_, err = EqualError("", "\xc3\x28")
+	if err == nil {
+		t.Fatal()
 	}
 }
 
-func TestLessError(t *testing.T) {
-	for _, v := range allTests {
-		t.Run(v.name(), func(t *testing.T) {
-			res, err := LessError(v.a, v.b)
-			if err == nil {
-				if (v.op == opLT && res == false) ||
-					(v.op == opEQ && res == true) ||
-					(v.op == opGT && res == true) ||
-					(v.op == shouldError) {
-					t.Fatalf("Unexpected result %v", res)
-				}
-			} else if v.op != shouldError {
-				t.Fatalf("Unexpected error %v", err)
-			}
-		})
+func TestInvalidPanic(t *testing.T) {
+	// Ensures that the AssertPanic function is working, as Go doesn't have any built-in way to assert a function panics
+	if !DetectPanic(func() { panic("Test") }) || DetectPanic(func() {}) {
+		t.Fatal()
+	}
+
+	// Run through some invalid inputs and fail the test if it doesn't panic
+	if !DetectPanic(func() { Compare("\xc3\x28", "") }) {
+		t.Fatal()
+	}
+
+	if !DetectPanic(func() { Less("\xc3\x28", "") }) {
+		t.Fatal()
+	}
+
+	if !DetectPanic(func() { Equal("\xc3\x28", "") }) {
+		t.Fatal()
+	}
+
+	if !DetectPanic(func() { Compare("", "\xc3\x28") }) {
+		t.Fatal()
+	}
+
+	if !DetectPanic(func() { Less("", "\xc3\x28") }) {
+		t.Fatal()
 	}
+
+	if !DetectPanic(func() { Equal("", "\xc3\x28") }) {
+		t.Fatal()
+	}
+
 }
 
-func TestEqualError(t *testing.T) {
-	for _, v := range allTests {
-		t.Run(v.name(), func(t *testing.T) {
-			res, err := EqualError(v.a, v.b)
-			if err == nil {
-				if (v.op == opLT && res == true) ||
-					(v.op == opEQ && res == false) ||
-					(v.op == opGT && res == true) ||
-					(v.op == shouldError) {
-					t.Fatalf("Unexpected result %v", res)
-				}
-			} else if v.op != shouldError {
-				t.Fatalf("Unexpected error %v", err)
-			}
-		})
+// Will return true if the given function panics and false if it returned correctly
+func DetectPanic(f func()) (ret bool) {
+	defer func() {
+		if r := recover(); r != nil {
+			ret = true
+		}
+	}()
+
+	f()
+
+	return false // If we reached this statement, we didn't panic
+}
+
+func TestBasicSort(t *testing.T) {
+	var input = []string{
+		"0.17.2", "0.1.0", "1.0.0", "1000", "0.17.2-pre.1", "1.16.5+pre",
+	}
+	var expect = []string{
+		"0.1.0", "0.17.2-pre.1", "0.17.2", "1.0.0", "1.16.5+pre", "1000",
+	}
+	VersionSlice(input).Sort()
+	if !reflect.DeepEqual(input, expect) {
+		t.Fatalf("Failed to sort strings: got %v (expected %v)", input, expect)
 	}
 }
 
diff --git a/java/src/test/java/com/unascribed/flexver/FlexVerBasicTest.java b/java/src/test/java/com/unascribed/flexver/FlexVerBasicTest.java
index 7c1e03e..428e50f 100644
--- a/java/src/test/java/com/unascribed/flexver/FlexVerBasicTest.java
+++ b/java/src/test/java/com/unascribed/flexver/FlexVerBasicTest.java
@@ -1,92 +1,49 @@
 package com.unascribed.flexver;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
 
-import com.unascribed.flexver.FlexVerComparator.NumericVersionComponent;
-import com.unascribed.flexver.FlexVerComparator.SemVerPrereleaseVersionComponent;
-import com.unascribed.flexver.FlexVerComparator.VersionComponent;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
 
 public class FlexVerBasicTest {
+	public static final String[] ENABLED_TESTS = { "test_vectors.txt", "large.txt" };
 
-	private static final boolean ANSI = false;
-	
-	private static void test(String a, String b, int expect) {
-		int c = signum(FlexVerComparator.compare(a, b));
-		int c2 = signum(FlexVerComparator.compare(b, a));
-		if (-c2 != c) {
-			throw new IllegalArgumentException("Comparison method violates its general contract! ("+a+" <=> "+b+" is not commutative)");
-		}
-		if (c != expect) throw new IllegalArgumentException("Expected "+expect+", got "+c);
-		String res = "";
-		if (c < 0) res = "<";
-		if (c == 0) res = "=";
-		if (c > 0) res = ">";
-		System.out.println(represent(a)+(ANSI?"\u001B[0m ":" ")+res+" "+represent(b));
-	}
-	
-	private static int signum(int i) {
-		return i < 0 ? -1 : i > 0 ? 1 : 0;
-	}
+	@ParameterizedTest(name = "{0} {2} {1}")
+	@MethodSource("getEqualityTests")
+	public void testEquality(String a, String b, Ordering expected) {
+		Ordering c = Ordering.fromComparison(FlexVerComparator.compare(a, b));
+		Ordering c2 = Ordering.fromComparison(FlexVerComparator.compare(b, a));
 
-	private static String represent(String str) {
-		List<VersionComponent> d = FlexVerComparator.decompose(str);
-		StringBuilder out = new StringBuilder(ANSI ? "" : "`");
-		for (VersionComponent vc : d) {
-			if (ANSI) {
-				int color = 90;
-				if (vc instanceof NumericVersionComponent) {
-					color = 96;
-				} else if (vc instanceof SemVerPrereleaseVersionComponent) {
-					color = 91;
-				} else {
-					color = 95;
-				}
-				out.append("\u001B[").append(color).append("m");
-			}
-			out.append(vc);
-			if (!ANSI) {
-				out.append(" ");
-			}
-		}
-		if (str.contains("+")) {
-			if (ANSI) {
-				out.append("\u001B[90m");
-			}
-			out.append(str.substring(str.indexOf('+')));
-			out.append(" ");
-		}
-		if (!ANSI) {
-			out.setLength(out.length()-1);
-			out.append("`");
-		}
-		return out.toString();
+		// When inverting the operands we're comparing, the ordering should be inverted too
+		assertEquals(c2.invert(), c, "Comparison method violates its general contract! ("+a+" <=> "+b+" is not commutative)");
+
+		assertEquals(expected, c, "Ordering.fromComparison produced "+a+" "+c+" "+b);
 	}
 
-	public static void main(String[] args) {
-		test("b1.7.3", "a1.2.6", 1);
-		test("b1.2.6", "a1.7.3", 1);
-		test("a1.1.2", "a1.1.2_01", -1);
-		test("1.16.5-0.00.5", "1.14.2-1.3.7", 1);
-		test("1.0.0", "1.0.0_01", -1);
-		test("1.0.1", "1.0.0_01", 1);
-		test("1.0.0_01", "1.0.1", -1);
-		test("0.17.1-beta.1", "0.17.1", -1);
-		test("0.17.1-beta.1", "0.17.1-beta.2", -1);
-		test("1.4.5_01", "1.4.5_01+fabric-1.17", 0);
-		test("1.4.5_01", "1.4.5_01+fabric-1.17+ohgod", 0);
-		test("14w16a", "18w40b", -1);
-		test("18w40a", "18w40b", -1);
-		test("1.4.5_01+fabric-1.17", "18w40b", -1);
-		test("13w02a", "c0.3.0_01", -1);
-		test("0.6.0-1.18.x", "0.9.beta-1.18.x", -1);
-		// 2^65. Too large for a 64-bit integer or a double
-		test("36893488147419103232", "36893488147419103233", -1);
-		test("37", "12", 1);
-		test("12", "13", -1);
-		test("12", "21", -1);
-		test("43", "103", -1);
+	public static Stream<Arguments> getEqualityTests() throws IOException {
+		Path testsFolder = Path.of("../test").toAbsolutePath();
+		List<String> lines = new ArrayList<>();
 
-		test("1.0", "1.1", -1);
-	}
+		for (String test : ENABLED_TESTS) {
+			lines.addAll(Files.readAllLines(testsFolder.resolve(test)));
+		}
 
+		return lines.stream()
+			.filter(line -> !line.startsWith("#"))
+			.filter(line -> !line.isEmpty())
+			.map(line -> {
+				String[] split = line.split(" ", -1);
+				if (split.length != 3) throw new IllegalArgumentException("Line formatted incorrectly, expected 2 spaces: "+line);
+				return Arguments.of(split[0], split[2], Ordering.fromString(split[1]));
+			});
+	}
 }
diff --git a/java/src/test/java/com/unascribed/flexver/Ordering.java b/java/src/test/java/com/unascribed/flexver/Ordering.java
new file mode 100644
index 0000000..3e72779
--- /dev/null
+++ b/java/src/test/java/com/unascribed/flexver/Ordering.java
@@ -0,0 +1,52 @@
+package com.unascribed.flexver;
+
+import java.util.Arrays;
+import java.util.NoSuchElementException;
+
+public enum Ordering {
+    LESS("<"),
+    EQUAL("="),
+    GREATER(">");
+
+    public final String charRepresentation;
+
+    Ordering(String charRepresentation) {
+        this.charRepresentation = charRepresentation;
+    }
+
+    @Override
+    public String toString() {
+        return charRepresentation;
+    }
+
+    public Ordering invert() {
+        switch (this) {
+            case LESS:
+                return Ordering.GREATER;
+            case EQUAL:
+                return this;
+            case GREATER:
+                return Ordering.LESS;
+            default:
+                throw new IllegalArgumentException();
+        }
+    }
+
+    public static Ordering fromString(String str) {
+        return Arrays.stream(Ordering.values())
+            .filter(ord -> ord.charRepresentation.equals(str))
+            .findFirst()
+            .orElseThrow(() -> new NoSuchElementException("'"+str+"' is not a valid ordering"));
+    }
+
+    /**
+     * Converts an integer returned by a method like {@link FlexVerComparator#compare(String, String)} to an {@link Ordering}
+     */
+    public static Ordering fromComparison(int i) {
+        if (i < 0) return Ordering.LESS;
+        if (i == 0) return Ordering.EQUAL;
+        if (i > 0) return Ordering.GREATER;
+
+        throw new AssertionError();
+    }
+}
diff --git a/js/test.js b/js/test.js
index f30f308..99ecc0f 100644
--- a/js/test.js
+++ b/js/test.js
@@ -1,43 +1,43 @@
 const flexVerCompare = require("./dist/node.js");
+const fs = require('fs');
+
+const ENABLED_TESTS = [ "test_vectors.txt", "large.txt" ];
 
 function signum(i) {
 	return i < 0 ? -1 : i > 0 ? 1 : 0;
 }
 
-function test(a, b, expect) {
+function runTest(a, b, expect) {
 	var c = signum(flexVerCompare(a, b));
 	var c2 = signum(flexVerCompare(b, a));
 	if (-c2 != c) {
-		throw new Error("Comparison method violates its general contract! ("+a+" <=> "+b+" is not commutative)");
+		console.log(`FAIL: Comparison method violates its general contract! (${a} <=> ${b} is not commutative)`);
 	}
 	var res = "";
 	if (c < 0) res = "<";
 	if (c == 0) res = "=";
 	if (c > 0) res = ">";
-	console.log(a+" "+res+" "+b+(c != expect ? " DOES NOT MATCH" : ""));
+
+	if (res != expect) {
+		console.log(`FAIL: expected ${a} ${expect} ${b} but got '${res}'`);
+	}
 }
 
-test("b1.7.3", "a1.2.6", 1);
-test("b1.2.6", "a1.7.3", 1);
-test("a1.1.2", "a1.1.2_01", -1);
-test("1.16.5-0.00.5", "1.14.2-1.3.7", 1);
-test("1.0.0", "1.0.0_01", -1);
-test("1.0.1", "1.0.0_01", 1);
-test("1.0.0_01", "1.0.1", -1);
-test("0.17.1-beta.1", "0.17.1", -1);
-test("0.17.1-beta.1", "0.17.1-beta.2", -1);
-test("1.4.5_01", "1.4.5_01+fabric-1.17", 0);
-test("1.4.5_01", "1.4.5_01+fabric-1.17+ohgod", 0);
-test("14w16a", "18w40b", -1);
-test("18w40a", "18w40b", -1);
-test("1.4.5_01+fabric-1.17", "18w40b", -1);
-test("13w02a", "c0.3.0_01", -1);
-test("0.6.0-1.18.x", "0.9.beta-1.18.x", -1);
-// 2^65. Too large for a 64-bit integer or a double
-test("36893488147419103232", "36893488147419103233", -1);
-test("37", "12", 1);
-test("12", "13", -1);
-test("12", "21", -1);
-test("43", "103", -1);
+ENABLED_TESTS.forEach(test => {
+	fs.readFile(`../test/${test}`, "utf-8", (err, data) => {
+		if (err) throw err;
+		data.split("\n")
+			.filter(line => !line.startsWith("#"))
+			.filter(line => line.length != 0)
+			.map(line => {
+				var split = line.split(" ")
+				if (split.length != 3) throw test+" Line formatted incorrectly, expected 2 spaces: "+line;
+				
+				var lefthand = split[0];
+				var righthand = split[2];
+				var ordering = split[1];
 
-test("1.0", "1.1", -1);
+				return runTest(lefthand, righthand, ordering)
+			})
+	})
+})
diff --git a/test/large.txt b/test/large.txt
new file mode 100644
index 0000000..972eb5d
--- /dev/null
+++ b/test/large.txt
@@ -0,0 +1,4 @@
+# The tests in this file may be disabled by some implementations
+
+# Too large for a 64-bit integer or double, checks if codepoint-wise or integer-parse is being used
+36893488147419103232 < 36893488147419103233
\ No newline at end of file
diff --git a/test/test_vectors.txt b/test/test_vectors.txt
new file mode 100644
index 0000000..11b7b06
--- /dev/null
+++ b/test/test_vectors.txt
@@ -0,0 +1,59 @@
+# This test file is formatted as "<lefthand> <operator> <righthand>", seperated by the space character
+# Implementations should ignore lines starting with "#" and lines that have a length of 0
+
+# Basic numeric ordering (lexical string sort fails these)
+10 > 2
+100 > 10
+
+# Trivial common numerics
+1.0 < 1.1
+1.0 < 1.0.1
+1.1 > 1.0.1
+
+# SemVer compatibility
+1.5 > 1.5-pre1
+1.5 = 1.5+foobar
+
+# SemVer incompatibility
+1.5 < 1.5-2
+1.5-pre10 > 1.5-pre2
+
+# Empty strings
+ = 
+1 > 
+ < 1
+
+# Check boundary between textual and prerelease
+a-a < a
+
+# Check boundary between textual and appendix
+a+a = a
+
+# Dash is included in prerelease comparison (if stripped it will be a smaller component)
+# Note that a-a < a=a regardless since the prerelease splits the component creating a smaller first component; 0 is added to force splitting regardless
+a0-a < a0=a
+
+# Pre-releases must contain only non-digit
+1.16.5-10 > 1.16.5
+
+# Pre-releases can have multiple dashes (should not be split)
+# Reasoning for test data: "p-a!" > "p-a-" (correct); "p-a!" < "p-a t-" (what happens if every dash creates a new component)
+-a- > -a!
+
+# Misc
+b1.7.3 > a1.2.6
+b1.2.6 > a1.7.3
+a1.1.2 < a1.1.2_01
+1.16.5-0.00.5 > 1.14.2-1.3.7
+1.0.0 < 1.0.0_01
+1.0.1 > 1.0.0_01
+1.0.0_01 < 1.0.1
+0.17.1-beta.1 < 0.17.1
+0.17.1-beta.1 < 0.17.1-beta.2
+1.4.5_01 = 1.4.5_01+fabric-1.17
+1.4.5_01 = 1.4.5_01+fabric-1.17+ohgod
+14w16a < 18w40b
+18w40a < 18w40b
+1.4.5_01+fabric-1.17 < 18w40b
+13w02a < c0.3.0_01
+0.6.0-1.18.x < 0.9.beta-1.18.x
\ No newline at end of file