Repository: memononen/libtess2
Pull request #71: Fail if coords are outside of a safe range
Author: login: sfreilich
Created: 2025-04-16T20:30:18Z | Merged: 2025-04-24T21:25:58Z

This avoids an assert-fail crash in debug mode, and reads of uninitialized memory and possibly crashes in optimized builds.


Diff:
diff --git a/Include/tesselator.h b/Include/tesselator.h
index 3d43155..8993daa 100755
--- a/Include/tesselator.h
+++ b/Include/tesselator.h
@@ -138,6 +138,14 @@ typedef struct TESSalloc TESSalloc;
 
 #define TESS_NOTUSED(v) do { (void)(1 ? (void)0 : ( (void)(v) ) ); } while(0)
 
+// These two constants define the valid input coordinate range the library is
+// able to operate on. Tesselation will fail if any of the coordinates are not
+// within this range. Clients are responsible for dealing with inputs outside of
+// this range (e.g. clamping or filtering invalid points, scaling down the
+// coordinate space).
+#define TESS_MAX_VALID_INPUT_VALUE ((TESSreal) (1<<23))
+#define TESS_MIN_VALID_INPUT_VALUE (-TESS_MAX_VALID_INPUT_VALUE)
+
 // Custom memory allocator interface.
 // The internal memory allocator allocates mesh edges, vertices and faces
 // as well as dictionary nodes and active regions in buckets and uses simple
diff --git a/Source/tess.c b/Source/tess.c
index 2a3b96f..7525fd6 100755
--- a/Source/tess.c
+++ b/Source/tess.c
@@ -918,6 +918,11 @@ void OutputContours( TESStesselator *tess, TESSmesh *mesh, int vertexSize )
 	}
 }
 
+int IsValidCoord(TESSreal coord) {
+  return coord <= TESS_MAX_VALID_INPUT_VALUE &&
+      coord >= TESS_MIN_VALID_INPUT_VALUE;
+}
+
 void tessAddContour( TESStesselator *tess, int size, const void* vertices,
 					int stride, int numVertices )
 {
@@ -942,7 +947,9 @@ void tessAddContour( TESStesselator *tess, int size, const void* vertices,
 	{
 		const TESSreal* coords = (const TESSreal*)src;
 		src += stride;
-		if (isnan(coords[0]) || isnan(coords[1]) || (size > 2 && isnan(coords[2]))) {
+		if (!IsValidCoord(coords[0]) ||
+		    !IsValidCoord(coords[1]) ||
+		    (size > 2 && !IsValidCoord(coords[2]))) {
 			// "Out of memory" isn't quite right, but give up and bail out
 			tess->outOfMemory = 1;
 			return;
diff --git a/Tests/libtess2_test.cc b/Tests/libtess2_test.cc
index 5039128..6d40934 100644
--- a/Tests/libtess2_test.cc
+++ b/Tests/libtess2_test.cc
@@ -167,10 +167,9 @@ TEST_F(Libtess2Test, FloatOverflowQuad) {
                      {kFloatMin, kFloatMax},
                      {kFloatMax, kFloatMax},
                      {kFloatMax, kFloatMin}});
-  EXPECT_NE(tessTesselate(tess, TESS_WINDING_POSITIVE, TESS_POLYGONS,
+  EXPECT_EQ(tessTesselate(tess, TESS_WINDING_POSITIVE, TESS_POLYGONS,
                           kNumTriangleVertices, kComponentCount, nullptr),
             0);
-  EXPECT_EQ(tessGetElementCount(tess), 2);
 }
 
 TEST_F(Libtess2Test, SingularityQuad) {
@@ -188,34 +187,30 @@ TEST_F(Libtess2Test, DegenerateQuad) {
                      {0.64113313f, -1.f},
                      {-0.f, -0.f},
                      {-3.40282347e+38f, 1.f}});
-  EXPECT_NE(tessTesselate(tess, TESS_WINDING_POSITIVE, TESS_POLYGONS,
+  EXPECT_EQ(tessTesselate(tess, TESS_WINDING_POSITIVE, TESS_POLYGONS,
                           kNumTriangleVertices, kComponentCount, nullptr),
             0);
-  EXPECT_EQ(tessGetElementCount(tess), 2);
 }
 
 TEST_F(Libtess2Test, WidthOverflowsTri) {
   AddPolyline(tess, {{-2e+38f, 0}, {0, 0}, {2e+38f, -1}});
-  EXPECT_NE(tessTesselate(tess, TESS_WINDING_POSITIVE, TESS_POLYGONS,
+  EXPECT_EQ(tessTesselate(tess, TESS_WINDING_POSITIVE, TESS_POLYGONS,
                           kNumTriangleVertices, kComponentCount, nullptr),
             0);
-  EXPECT_EQ(tessGetElementCount(tess), 1);
 }
 
 TEST_F(Libtess2Test, HeightOverflowsTri) {
   AddPolyline(tess, {{0, 0}, {0, 2e+38f}, {-1, -2e+38f}});
-  EXPECT_NE(tessTesselate(tess, TESS_WINDING_POSITIVE, TESS_POLYGONS,
+  EXPECT_EQ(tessTesselate(tess, TESS_WINDING_POSITIVE, TESS_POLYGONS,
                           kNumTriangleVertices, kComponentCount, nullptr),
             0);
-  EXPECT_EQ(tessGetElementCount(tess), 1);
 }
 
 TEST_F(Libtess2Test, AreaOverflowsTri) {
   AddPolyline(tess, {{-2e+37f, 0.f}, {0, 5}, {1e37f, -5}});
-  EXPECT_NE(tessTesselate(tess, TESS_WINDING_POSITIVE, TESS_POLYGONS,
+  EXPECT_EQ(tessTesselate(tess, TESS_WINDING_POSITIVE, TESS_POLYGONS,
                           kNumTriangleVertices, kComponentCount, nullptr),
             0);
-  EXPECT_EQ(tessGetElementCount(tess), 1);
 }
 
 TEST_F(Libtess2Test, NanQuad) {
@@ -229,7 +224,7 @@ TEST_F(Libtess2Test, NanQuad) {
   EXPECT_EQ(tessGetElementCount(tess), 0);
 }
 
-TEST_F(Libtess2Test, DegenerateSquiggle) {
+TEST_F(Libtess2Test, AvoidsCrashWhileFindingIntersection) {
   // Previously, this failed an assert while finding an intersection because
   // that fell back to taking a midpoint between two coordinates in a way that
   // could get the wrong answer because of the sum overflowing max float.
@@ -261,10 +256,41 @@ TEST_F(Libtess2Test, DegenerateSquiggle) {
                      {-1.f, 3.40282347e+38f},
                      {-0.f, 0.34419331f},
                      {1.f, 1.f}});
-  EXPECT_NE(tessTesselate(tess, TESS_WINDING_POSITIVE, TESS_POLYGONS,
+  EXPECT_EQ(tessTesselate(tess, TESS_WINDING_POSITIVE, TESS_POLYGONS,
+                          kNumTriangleVertices, kComponentCount, nullptr),
+            0);
+}
+
+TEST_F(Libtess2Test, AvoidsCrashInAddRightEdges) {
+  AddPolyline(tess, {{{-0.5f, 1.f},
+                      {3.40282347e+38f, 0.f},
+                      {0.349171013f, 1.f},
+                      {1.f, 0.f},
+                      {1.f, -0.f},
+                      {0.594775498f, -0.f},
+                      {0.f, -0.f},
+                      {-0.f, 1.f},
+                      {0.f, 1.f},
+                      {2.20929384f, 1.f},
+                      {1.f, 1.f},
+                      {-0.f, -0.f},
+                      {3.40282347e+38f, -0.f},
+                      {-1.f, 0.f},
+                      {1.70141173e+38f, 0.391036272f},
+                      {3.40282347e+38f, 0.371295959f},
+                      {3.40282347e+38f, -0.f},
+                      {0.f, 0.234747186f},
+                      {-1.f, 1.f},
+                      {-1.f, -0.f},
+                      {3.40282347e+38f, 1.f},
+                      {-0.f, -0.f},
+                      {3.40282347e+38f, 1.f},
+                      {0.434241712f, 0.f},
+                      {1.f, 0.211511821f},
+                      {3.40282347e+38f, 1.f}}});
+  EXPECT_EQ(tessTesselate(tess, TESS_WINDING_POSITIVE, TESS_POLYGONS,
                           kNumTriangleVertices, kComponentCount, nullptr),
             0);
-  EXPECT_EQ(tessGetElementCount(tess), 13);
 }
 
 }  // namespace


Reviews:
user: login: memononen
state: COMMENTED
author_association: OWNER
submitted_at: 2025-04-21T22:21:00Z
user: login: sfreilich
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-24T13:54:49Z
user: login: memononen
state: COMMENTED
author_association: OWNER
submitted_at: 2025-04-24T14:52:02Z
user: login: sfreilich
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-24T15:01:28Z
user: login: memononen
body: Few nags, should be trivial to fix. Other than that the implementation looks good.
state: APPROVED
author_association: OWNER
submitted_at: 2025-04-24T15:32:47Z
user: login: sfreilich
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-24T18:20:35Z
user: login: sfreilich
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-24T18:30:37Z
user: login: sfreilich
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-24T18:31:28Z

Review comments:
diff_hunk: @@ -942,7 +949,15 @@ void tessAddContour( TESStesselator *tess, int size, const void* vertices,
 	{
 		const TESSreal* coords = (const TESSreal*)src;
 		src += stride;
-		if (isnan(coords[0]) || isnan(coords[1]) || (size > 2 && isnan(coords[2]))) {
+		if (isnan(coords[0]) ||
path: Source/tess.c
user: login: memononen
body: make the per component check a helper function.
created_at: 2025-04-17T15:57:28Z
author_association: OWNER
diff_hunk: @@ -61,6 +62,12 @@ static void Normalize( TESSreal v[3] )
 
 #define ABS(x)	((x) < 0 ? -(x) : (x))
 
+// Coords need to be limited to this range to avoid overflow when computing
+// the projection vectors of the sentinel edges. This is based on logic which
+// computes the bounding box of the projection vectors in InitEdgeDict.
+static float kMaxSafeCoord = (FLT_MAX - 0.01f) / 2.0f;
path: Source/tess.c
user: login: memononen
body: That 0.01 will do nothing, the next smaller number from:
FLT_MAX = 340282346638528859811704183484516925440.000000
towards zero:
next = 340282326356119256160033759537265639424.000000
diff = 20282409603651670423947251286016.000000

At 2<<23 each float is 1.0 apart. You can use e.g. `float next_v = nextafterf(v, 0.f)` to poke around how the float works.

A common gamedev wisdom is that the max workable float range is +-4000 (assuming 1m world units). At that you loose the fractions when big numbers are multiplied.

I would suggest the max coord to be 2<<23. My rationale is that after that we start to loose precision, but it's not very scientific.  Half of float max sounds like it might still overflow in the intersection calculations (there's multiplies there).

I expect that some people will run into this issue with existing data with this new behavior. I think the minimum we should do is to let the user know the failure reason. I think it is time to change outOfMemory to enum, which desrcibes the error case better. I think we can do that in separate change.
created_at: 2025-04-17T16:26:14Z
author_association: OWNER
diff_hunk: @@ -61,6 +62,12 @@ static void Normalize( TESSreal v[3] )
 
 #define ABS(x)	((x) < 0 ? -(x) : (x))
 
+// Coords need to be limited to this range to avoid overflow when computing
+// the projection vectors of the sentinel edges. This is based on logic which
+// computes the bounding box of the projection vectors in InitEdgeDict.
+static float kMaxSafeCoord = (FLT_MAX - 0.01f) / 2.0f;
path: Source/tess.c
user: login: memononen
body: One more thing... if we dont clamp, then I think we should expose the max coord in the public header so that users can clamp themselves.
created_at: 2025-04-17T16:28:31Z
author_association: OWNER
diff_hunk: @@ -61,6 +62,12 @@ static void Normalize( TESSreal v[3] )
 
 #define ABS(x)	((x) < 0 ? -(x) : (x))
 
+// Coords need to be limited to this range to avoid overflow when computing
+// the projection vectors of the sentinel edges. This is based on logic which
+// computes the bounding box of the projection vectors in InitEdgeDict.
+static float kMaxSafeCoord = (FLT_MAX - 0.01f) / 2.0f;
path: Source/tess.c
user: login: sfreilich
body: Yeah, makes sense. (Why doesn't `InitEdgeDict` use `nextafterf` instead of a flat `0.01`?)

How do you want to expose the constant? I'm more used to C++ conventions than C. Should that be done with `#define` or is `const TESSreal` preferable?
created_at: 2025-04-24T13:54:49Z
author_association: COLLABORATOR
diff_hunk: @@ -61,6 +62,12 @@ static void Normalize( TESSreal v[3] )
 
 #define ABS(x)	((x) < 0 ? -(x) : (x))
 
+// Coords need to be limited to this range to avoid overflow when computing
+// the projection vectors of the sentinel edges. This is based on logic which
+// computes the bounding box of the projection vectors in InitEdgeDict.
+static float kMaxSafeCoord = (FLT_MAX - 0.01f) / 2.0f;
path: Source/tess.c
user: login: memononen
body: The epsilon in `InitEdgeDict()` is to handle zero dimension. The usual case is that the sentinel is placed bounding box width away from each end.

A #define is good here. I have had some issues with const values defined in headers in the past, as you may end up with the same symbol in different compilation units (or something like that). I think `static const ...`  is fine for per function or compilation unit constants.
created_at: 2025-04-24T14:52:02Z
author_association: OWNER
diff_hunk: @@ -61,6 +62,12 @@ static void Normalize( TESSreal v[3] )
 
 #define ABS(x)	((x) < 0 ? -(x) : (x))
 
+// Coords need to be limited to this range to avoid overflow when computing
+// the projection vectors of the sentinel edges. This is based on logic which
+// computes the bounding box of the projection vectors in InitEdgeDict.
+static float kMaxSafeCoord = (FLT_MAX - 0.01f) / 2.0f;
path: Source/tess.c
user: login: sfreilich
body: Makes sense.
created_at: 2025-04-24T15:01:28Z
author_association: COLLABORATOR
diff_hunk: @@ -918,6 +919,10 @@ void OutputContours( TESStesselator *tess, TESSmesh *mesh, int vertexSize )
 	}
 }
 
+bool IsValidCoord(TESSreal coord) {
path: Source/tess.c
user: login: memononen
body: The library is not using bool (it's later C addition), use int instead.
created_at: 2025-04-24T15:21:36Z
author_association: OWNER
diff_hunk: @@ -138,6 +138,11 @@ typedef struct TESSalloc TESSalloc;
 
 #define TESS_NOTUSED(v) do { (void)(1 ? (void)0 : ( (void)(v) ) ); } while(0)
 
+// Value that avoids loss-of-precision or overflow issues when creating distinct
+// sentinel edges in InitEdgeDict.
+#define TESS_MAX_VALUE ((TESSreal) (2<<23))
path: Include/tesselator.h
user: login: memononen
body: I think that should be 1<<23 (I may have used 2 in earlier comment).

I think this needs more user facing comment, and const names. I suggest `TESS_MAX_VALID_INPUT_VALUE`.
```
These two constants define the valid input coordinate range the library is able to operate on. If any input coordinate is out of range, `tessTesselate()` will fail. The user is expected to handle out of range coordinates (e.g. clamp or discard points).
``` 
created_at: 2025-04-24T15:30:26Z
author_association: OWNER
diff_hunk: @@ -918,6 +919,10 @@ void OutputContours( TESStesselator *tess, TESSmesh *mesh, int vertexSize )
 	}
 }
 
+bool IsValidCoord(TESSreal coord) {
path: Source/tess.c
user: login: sfreilich
body: Are you sure? It's part of C99. I do think it's reasonable to expect C99 support?
created_at: 2025-04-24T18:20:35Z
author_association: COLLABORATOR
diff_hunk: @@ -138,6 +138,11 @@ typedef struct TESSalloc TESSalloc;
 
 #define TESS_NOTUSED(v) do { (void)(1 ? (void)0 : ( (void)(v) ) ); } while(0)
 
+// Value that avoids loss-of-precision or overflow issues when creating distinct
+// sentinel edges in InitEdgeDict.
+#define TESS_MAX_VALUE ((TESSreal) (2<<23))
path: Include/tesselator.h
user: login: sfreilich
body: Done.

I think I must've had the same mental hiccup you did with 2^23 vs. 1<<23. 
created_at: 2025-04-24T18:30:37Z
author_association: COLLABORATOR
diff_hunk: @@ -918,6 +919,10 @@ void OutputContours( TESStesselator *tess, TESSmesh *mesh, int vertexSize )
 	}
 }
 
+bool IsValidCoord(TESSreal coord) {
path: Source/tess.c
user: login: sfreilich
body: Tentatively made the suggested change, it's not a big deal either way.
created_at: 2025-04-24T18:31:28Z
author_association: COLLABORATOR

PR comments:
user: login: sfreilich
created_at: 2025-04-16T20:31:19Z
body: I still don't understand exactly why this constraint is getting violated, though. But fuzztester seems to find a lot of polylines that violate it, even without getting stuck in NaN-land.
author_association: COLLABORATOR
user: login: memononen
created_at: 2025-04-16T21:53:09Z
body: I think it is unrealistic that the algorithm is able to deal with max float. The issue with the large floats are that they quickly turn into infs, and then down the line either 0 or inf depending how they are used. The brittleness is in `tesedgeIntersect()`.

To get to the bottom of the issue, I think you should check what the s/t values are after projection, as well as what the sentinel edges are `InitEdgeDict()`.

One solution could be to clamp the input points to a safe range. We should be able to figure out the sane range from `tesedgeIntersect()`.
author_association: OWNER
user: login: sfreilich
created_at: 2025-04-17T15:08:09Z
body: For this particular example, the edge that violated that assert had:

```
e->Org->s,t: 1.000000,0.000000, e->Dst->s,t: 0.594775,0.000000
```

That is, `Org->s > Dst->s` and it's not looking at `t`.

You're right that those sentinel edges are overflowing. The projection vectors for those are:

```
e->Org->s,t: inf,-2.010000, e->Dst->s,t: -340282346638528859811704183484516925440.000000,-2.010000
e->Org->s,t: inf,1.010000, e->Dst->s,t: -340282346638528859811704183484516925440.000000,1.010000
```

The code in `InitEdgeDict` which handles this is:  

```
w = (tess->bmax[0] - tess->bmin[0]) + (TESSreal)0.01;
h = (tess->bmax[1] - tess->bmin[1]) + (TESSreal)0.01;

smin = tess->bmin[0] - w;
smax = tess->bmax[0] + w;
tmin = tess->bmin[1] - h;
tmax = tess->bmax[1] + h;
```

That seems equivalent to:

```
smin = 2 * tess->bmin[0] - tess->bmax[0] - (TESSreal)0.01 ;
smax = 2 * tess->bmax[0] - tess->bmin[0] + (TESSreal)0.01;
```

Are you suggesting clamping the values generated for the projection vectors in tessProjectPolygon? (Clamping to `[(FLOAT_MIN + 0.01) / 2.0, (FLOAT_MAX - 0.01) / 2 .0]`???) Or are you suggesting clamping the input coords somewhere in `tesedgeIntersect`? I looked at that, but I'm a bit lost?
author_association: COLLABORATOR
user: login: sfreilich
created_at: 2025-04-17T15:12:39Z
body: `s` is the dot-product of `coords` and `(0, 1, 0)`. `t` is the dot-product of `coords` and `(0, 0, 1)` or `(0, 0, -1)`. So that implies the same safe bounds for `coords` as for `s` and `t`. I think.
author_association: COLLABORATOR
user: login: sfreilich
created_at: 2025-04-17T15:20:15Z
body: And I'd also guess that it might be better to fail in `tessAddContour` rather than modify values that fall out of that range.
author_association: COLLABORATOR
user: login: sfreilich
created_at: 2025-04-17T15:48:49Z
body: Made an attempt at this. It changes some of the previous test-cases to exit with error due to invalid input.
author_association: COLLABORATOR
user: login: memononen
created_at: 2025-04-21T22:21:37Z
body: I had bunch of pending comments from the other day, and did not realize that, sorry :)
author_association: OWNER
user: login: sfreilich
created_at: 2025-04-24T15:01:44Z
body: Updated, please take a look.
author_association: COLLABORATOR
user: login: sfreilich
created_at: 2025-04-25T14:08:01Z
body: I messed up the range in the change description, the bounds are +/- 2^23, not 2^32.
author_association: COLLABORATOR