Fix incorrect rasterization at tile borders (#476)

* Fix incorrect rasterization at tile borders

Clip the polygon at the tile border and disregard coordinates outside of
the tile before clamping.

* Added unit test with minimal repro case for the rasterization issue fixed in PR #476

Co-authored-by: Graham Pentheny <graham.pentheny@gmail.com>

Recast/Source/RecastRasterization.cpp

@@ -264,7 +264,8 @@ static bool rasterizeTri(const float* v0, const float* v1, const float* v2,
 	// Calculate the footprint of the triangle on the grid's y-axis
 	int y0 = (int)((tmin[2] - bmin[2])*ics);
 	int y1 = (int)((tmax[2] - bmin[2])*ics);
-	y0 = rcClamp(y0, 0, h-1);
+	// use -1 rather than 0 to cut the polygon properly at the start of the tile
+	y0 = rcClamp(y0, -1, h-1);
 	y1 = rcClamp(y1, 0, h-1);
 	
 	// Clip the triangle into all grid cells it touches.
@@ -283,7 +284,7 @@ static bool rasterizeTri(const float* v0, const float* v1, const float* v2,
 		dividePoly(in, nvIn, inrow, &nvrow, p1, &nvIn, cz+cs, 2);
 		rcSwap(in, p1);
 		if (nvrow < 3) continue;
-		
+		if (y < 0) continue;
 		// find the horizontal bounds in the row
 		float minX = inrow[0], maxX = inrow[0];
 		for (int i=1; i<nvrow; ++i)
@@ -293,7 +294,10 @@ static bool rasterizeTri(const float* v0, const float* v1, const float* v2,
 		}
 		int x0 = (int)((minX - bmin[0])*ics);
 		int x1 = (int)((maxX - bmin[0])*ics);
-		x0 = rcClamp(x0, 0, w-1);
+		if (x1 < 0 || x0 >= w) {
+			continue;
+		}
+		x0 = rcClamp(x0, -1, w-1);
 		x1 = rcClamp(x1, 0, w-1);
 
 		int nv, nv2 = nvrow;
@@ -305,7 +309,7 @@ static bool rasterizeTri(const float* v0, const float* v1, const float* v2,
 			dividePoly(inrow, nv2, p1, &nv, p2, &nv2, cx+cs, 0);
 			rcSwap(inrow, p2);
 			if (nv < 3) continue;
-			
+			if (x < 0) continue;
 			// Calculate min and max of the span.
 			float smin = p1[1], smax = p1[1];
 			for (int i = 1; i < nv; ++i)

Tests/Recast/Tests_Recast.cpp

@@ -657,6 +657,43 @@ TEST_CASE("rcRasterizeTriangle")
 	}
 }
 
+TEST_CASE("rcRasterizeTriangle overlapping bb but non-overlapping triangle")
+{
+	// This is a minimal repro case for the issue fixed in PR #476 (https://github.com/recastnavigation/recastnavigation/pull/476)
+    rcContext ctx;
+
+	// create a heightfield
+    float cellSize = 1;
+    float cellHeight = 1;
+    int width = 10;
+    int height = 10;
+    float bmin[] = { 0, 0, 0 };
+    float bmax[] = { 10, 10, 10 };
+    rcHeightfield heightfield;
+    REQUIRE(rcCreateHeightfield(&ctx, heightfield, width, height, bmin, bmax, cellSize, cellHeight));
+
+	// rasterize a triangle outside of the heightfield.
+    unsigned char area = 42;
+    int flagMergeThr = 1;
+    float verts[] =
+	{
+        -10.0, 5.5, -10.0,
+        -10.0, 5.5, 3,
+        3.0, 5.5, -10.0
+    };
+    REQUIRE(rcRasterizeTriangle(&ctx, &verts[0], &verts[3], &verts[6], area, heightfield, flagMergeThr));
+    
+	// ensure that no spans were created
+    for (int x = 0; x < width; ++x)
+	{
+        for (int z = 0; z < height; ++z)
+		{
+            rcSpan* span = heightfield.spans[x + z * heightfield.width];
+			REQUIRE(span == NULL);
+        }
+    }
+}
+
 TEST_CASE("rcRasterizeTriangles")
 {
 	rcContext ctx;