Add consistent input validation to dtNavMeshQuery (#374)

Validate input values, including that points are finite.

Detour/Include/DetourCommon.h

@@ -283,6 +283,28 @@ inline bool dtVequal(const float* p0, const float* p1)
 	return d < thr;
 }
 
+/// Checks that the specified vector's components are all finite.
+///  @param[in]		v	A point. [(x, y, z)]
+/// @return True if all of the point's components are finite, i.e. not NaN
+/// or any of the infinities.
+inline bool dtVisfinite(const float* v)
+{
+	bool result =
+		dtMathIsfinite(v[0]) &&
+		dtMathIsfinite(v[1]) &&
+		dtMathIsfinite(v[2]);
+
+	return result;
+}
+
+/// Checks that the specified vector's 2D components are finite.
+///  @param[in]		v	A point. [(x, y, z)]
+inline bool dtVisfinite2D(const float* v)
+{
+	bool result = dtMathIsfinite(v[0]) && dtMathIsfinite(v[2]);
+	return result;
+}
+
 /// Derives the dot product of two vectors on the xz-plane. (@p u . @p v)
 ///  @param[in]		u		A vector [(x, y, z)]
 ///  @param[in]		v		A vector [(x, y, z)]

Detour/Include/DetourMath.h

@@ -8,6 +8,9 @@ Members in this module are wrappers around the standard math library
 #define DETOURMATH_H
 
 #include <math.h>
+// This include is required because libstdc++ has problems with isfinite
+// if cmath is included before math.h.
+#include <cmath>
 
 inline float dtMathFabsf(float x) { return fabsf(x); }
 inline float dtMathSqrtf(float x) { return sqrtf(x); }
@@ -16,5 +19,6 @@ inline float dtMathCeilf(float x) { return ceilf(x); }
 inline float dtMathCosf(float x) { return cosf(x); }
 inline float dtMathSinf(float x) { return sinf(x); }
 inline float dtMathAtan2f(float y, float x) { return atan2f(y, x); }
+inline bool dtMathIsfinite(float x) { return std::isfinite(x); }
 
 #endif

Detour/Source/DetourNavMeshQuery.cpp

@@ -223,6 +223,9 @@ dtStatus dtNavMeshQuery::findRandomPoint(const dtQueryFilter* filter, float (*fr
 {
 	dtAssert(m_nav);
 
+	if (!filter || !frand || !randomRef || !randomPt)
+		return DT_FAILURE | DT_INVALID_PARAM;
+
 	// Randomly pick one tile. Assume that all tiles cover roughly the same area.
 	const dtMeshTile* tile = 0;
 	float tsum = 0.0f;
@@ -319,8 +322,13 @@ dtStatus dtNavMeshQuery::findRandomPointAroundCircle(dtPolyRef startRef, const f
 	dtAssert(m_openList);
 	
 	// Validate input
-	if (!startRef || !m_nav->isValidPolyRef(startRef))
+	if (!m_nav->isValidPolyRef(startRef) ||
+		!centerPos || !dtVisfinite(centerPos) ||
+		maxRadius < 0 || !dtMathIsfinite(maxRadius) ||
+		!filter || !frand || !randomRef || !randomPt)
+	{
 		return DT_FAILURE | DT_INVALID_PARAM;
+	}
 	
 	const dtMeshTile* startTile = 0;
 	const dtPoly* startPoly = 0;
@@ -513,6 +521,9 @@ dtStatus dtNavMeshQuery::closestPointOnPoly(dtPolyRef ref, const float* pos, flo
 	if (!tile)
 		return DT_FAILURE | DT_INVALID_PARAM;
 
+	if (!pos || !dtVisfinite(pos) || !closest)
+		return DT_FAILURE | DT_INVALID_PARAM;
+	
 	// Off-mesh connections don't have detail polygons.
 	if (poly->getType() == DT_POLYTYPE_OFFMESH_CONNECTION)
 	{
@@ -608,6 +619,9 @@ dtStatus dtNavMeshQuery::closestPointOnPolyBoundary(dtPolyRef ref, const float*
 	if (dtStatusFailed(m_nav->getTileAndPolyByRef(ref, &tile, &poly)))
 		return DT_FAILURE | DT_INVALID_PARAM;
 
+	if (!pos || !dtVisfinite(pos) || !closest)
+		return DT_FAILURE | DT_INVALID_PARAM;
+	
 	// Collect vertices.
 	float verts[DT_VERTS_PER_POLYGON*3];	
 	float edged[DT_VERTS_PER_POLYGON];
@@ -660,6 +674,9 @@ dtStatus dtNavMeshQuery::getPolyHeight(dtPolyRef ref, const float* pos, float* h
 	if (dtStatusFailed(m_nav->getTileAndPolyByRef(ref, &tile, &poly)))
 		return DT_FAILURE | DT_INVALID_PARAM;
 
+	if (!pos || !dtVisfinite2D(pos))
+		return DT_FAILURE | DT_INVALID_PARAM;
+	
 	if (poly->getType() == DT_POLYTYPE_OFFMESH_CONNECTION)
 	{
 		const float* v0 = &tile->verts[poly->verts[0]*3];
@@ -768,6 +785,8 @@ dtStatus dtNavMeshQuery::findNearestPoly(const float* center, const float* halfE
 	if (!nearestRef)
 		return DT_FAILURE | DT_INVALID_PARAM;
 
+	// queryPolygons below will check rest of params
+	
 	dtFindNearestPolyQuery query(this, center);
 
 	dtStatus status = queryPolygons(center, halfExtents, filter, &query);
@@ -972,8 +991,12 @@ dtStatus dtNavMeshQuery::queryPolygons(const float* center, const float* halfExt
 {
 	dtAssert(m_nav);
 
-	if (!center || !halfExtents || !filter || !query)
+	if (!center || !dtVisfinite(center) ||
+		!halfExtents || !dtVisfinite(halfExtents) ||
+		!filter || !query)
+	{
 		return DT_FAILURE | DT_INVALID_PARAM;
+	}
 
 	float bmin[3], bmax[3];
 	dtVsub(bmin, center, halfExtents);
@@ -1022,13 +1045,19 @@ dtStatus dtNavMeshQuery::findPath(dtPolyRef startRef, dtPolyRef endRef,
 	dtAssert(m_nodePool);
 	dtAssert(m_openList);
 
-	if (pathCount)
+	if (!pathCount)
-		*pathCount = 0;
+		return DT_FAILURE | DT_INVALID_PARAM;
+
+	*pathCount = 0;
 	
 	// Validate input
 	if (!m_nav->isValidPolyRef(startRef) || !m_nav->isValidPolyRef(endRef) ||
-		!startPos || !endPos || !filter || maxPath <= 0 || !path || !pathCount)
+		!startPos || !dtVisfinite(startPos) ||
+		!endPos || !dtVisfinite(endPos) ||
+		!filter || !path || maxPath <= 0)
+	{
 		return DT_FAILURE | DT_INVALID_PARAM;
+	}
 
 	if (startRef == endRef)
 	{
@@ -1263,18 +1292,21 @@ dtStatus dtNavMeshQuery::initSlicedFindPath(dtPolyRef startRef, dtPolyRef endRef
 	m_query.status = DT_FAILURE;
 	m_query.startRef = startRef;
 	m_query.endRef = endRef;
-	dtVcopy(m_query.startPos, startPos);
+	if (startPos)
-	dtVcopy(m_query.endPos, endPos);
+		dtVcopy(m_query.startPos, startPos);
+	if (endPos)
+		dtVcopy(m_query.endPos, endPos);
 	m_query.filter = filter;
 	m_query.options = options;
 	m_query.raycastLimitSqr = FLT_MAX;
 	
-	if (!startRef || !endRef)
-		return DT_FAILURE | DT_INVALID_PARAM;
-	
 	// Validate input
-	if (!m_nav->isValidPolyRef(startRef) || !m_nav->isValidPolyRef(endRef))
+	if (!m_nav->isValidPolyRef(startRef) || !m_nav->isValidPolyRef(endRef) ||
+		!startPos || !dtVisfinite(startPos) ||
+		!endPos || !dtVisfinite(endPos) || !filter)
+	{
 		return DT_FAILURE | DT_INVALID_PARAM;
+	}
 
 	// trade quality with performance?
 	if (options & DT_FINDPATH_ANY_ANGLE)
@@ -1530,8 +1562,14 @@ dtStatus dtNavMeshQuery::updateSlicedFindPath(const int maxIter, int* doneIters)
 
 dtStatus dtNavMeshQuery::finalizeSlicedFindPath(dtPolyRef* path, int* pathCount, const int maxPath)
 {
+	if (!pathCount)
+		return DT_FAILURE | DT_INVALID_PARAM;
+
 	*pathCount = 0;
 
+	if (!path || maxPath <= 0)
+		return DT_FAILURE | DT_INVALID_PARAM;
+	
 	if (dtStatusFailed(m_query.status))
 	{
 		// Reset query.
@@ -1615,12 +1653,13 @@ dtStatus dtNavMeshQuery::finalizeSlicedFindPath(dtPolyRef* path, int* pathCount,
 dtStatus dtNavMeshQuery::finalizeSlicedFindPathPartial(const dtPolyRef* existing, const int existingSize,
 													   dtPolyRef* path, int* pathCount, const int maxPath)
 {
+	if (!pathCount)
+		return DT_FAILURE | DT_INVALID_PARAM;
+
 	*pathCount = 0;
 
-	if (existingSize == 0)
+	if (!existing || existingSize <= 0 || !path || !pathCount || maxPath <= 0)
-	{
+		return DT_FAILURE | DT_INVALID_PARAM;
-		return DT_FAILURE;
-	}
 	
 	if (dtStatusFailed(m_query.status))
 	{
@@ -1824,13 +1863,18 @@ dtStatus dtNavMeshQuery::findStraightPath(const float* startPos, const float* en
 {
 	dtAssert(m_nav);
 
+	if (!straightPathCount)
+		return DT_FAILURE | DT_INVALID_PARAM;
+
 	*straightPathCount = 0;
 
-	if (!maxStraightPath)
+	if (!startPos || !dtVisfinite(startPos) ||
-		return DT_FAILURE | DT_INVALID_PARAM;
+		!endPos || !dtVisfinite(endPos) ||
-	
+		!path || pathSize <= 0 || !path[0] ||
-	if (!path[0])
+		maxStraightPath <= 0)
+	{
 		return DT_FAILURE | DT_INVALID_PARAM;
+	}
 	
 	dtStatus stat = 0;
 	
@@ -2070,13 +2114,19 @@ dtStatus dtNavMeshQuery::moveAlongSurface(dtPolyRef startRef, const float* start
 	dtAssert(m_nav);
 	dtAssert(m_tinyNodePool);
 
+	if (!visitedCount)
+		return DT_FAILURE | DT_INVALID_PARAM;
+
 	*visitedCount = 0;
 
-	// Validate input
+	if (!m_nav->isValidPolyRef(startRef) ||
-	if (!startRef)
+		!startPos || !dtVisfinite(startPos) ||
-		return DT_FAILURE | DT_INVALID_PARAM;
+		!endPos || !dtVisfinite(endPos) ||
-	if (!m_nav->isValidPolyRef(startRef))
+		!filter || !resultPos || !visited ||
+		maxVisitedSize <= 0)
+	{
 		return DT_FAILURE | DT_INVALID_PARAM;
+	}
 	
 	dtStatus status = DT_SUCCESS;
 	
@@ -2485,15 +2535,22 @@ dtStatus dtNavMeshQuery::raycast(dtPolyRef startRef, const float* startPos, cons
 {
 	dtAssert(m_nav);
 
+	if (!hit)
+		return DT_FAILURE | DT_INVALID_PARAM;
+
 	hit->t = 0;
 	hit->pathCount = 0;
 	hit->pathCost = 0;
 
 	// Validate input
-	if (!startRef || !m_nav->isValidPolyRef(startRef))
+	if (!m_nav->isValidPolyRef(startRef) ||
-		return DT_FAILURE | DT_INVALID_PARAM;
+		!startPos || !dtVisfinite(startPos) ||
-	if (prevRef && !m_nav->isValidPolyRef(prevRef))
+		!endPos || !dtVisfinite(endPos) ||
+		!filter ||
+		(prevRef && !m_nav->isValidPolyRef(prevRef)))
+	{
 		return DT_FAILURE | DT_INVALID_PARAM;
+	}
 	
 	float dir[3], curPos[3], lastPos[3];
 	float verts[DT_VERTS_PER_POLYGON*3+3];	
@@ -2735,11 +2792,18 @@ dtStatus dtNavMeshQuery::findPolysAroundCircle(dtPolyRef startRef, const float*
 	dtAssert(m_nodePool);
 	dtAssert(m_openList);
 
+	if (!resultCount)
+		return DT_FAILURE | DT_INVALID_PARAM;
+
 	*resultCount = 0;
 
-	// Validate input
+	if (!m_nav->isValidPolyRef(startRef) ||
-	if (!startRef || !m_nav->isValidPolyRef(startRef))
+		!centerPos || !dtVisfinite(centerPos) ||
+		radius < 0 || !dtMathIsfinite(radius) ||
+		!filter || maxResult < 0)
+	{
 		return DT_FAILURE | DT_INVALID_PARAM;
+	}
 	
 	m_nodePool->clear();
 	m_openList->clear();
@@ -2902,8 +2966,18 @@ dtStatus dtNavMeshQuery::findPolysAroundShape(dtPolyRef startRef, const float* v
 	dtAssert(m_nodePool);
 	dtAssert(m_openList);
 
+	if (!resultCount)
+		return DT_FAILURE | DT_INVALID_PARAM;
+
 	*resultCount = 0;
 
+	if (!m_nav->isValidPolyRef(startRef) ||
+		!verts || nverts < 3 ||
+		!filter || maxResult < 0)
+	{
+		return DT_FAILURE | DT_INVALID_PARAM;
+	}
+	
 	// Validate input
 	if (!startRef || !m_nav->isValidPolyRef(startRef))
 		return DT_FAILURE | DT_INVALID_PARAM;
@@ -3089,11 +3163,18 @@ dtStatus dtNavMeshQuery::findLocalNeighbourhood(dtPolyRef startRef, const float*
 	dtAssert(m_nav);
 	dtAssert(m_tinyNodePool);
 
+	if (!resultCount)
+		return DT_FAILURE | DT_INVALID_PARAM;
+
 	*resultCount = 0;
 
-	// Validate input
+	if (!m_nav->isValidPolyRef(startRef) ||
-	if (!startRef || !m_nav->isValidPolyRef(startRef))
+		!centerPos || !dtVisfinite(centerPos) ||
+		radius < 0 || !dtMathIsfinite(radius) ||
+		!filter || maxResult < 0)
+	{
 		return DT_FAILURE | DT_INVALID_PARAM;
+	}
 
 	static const int MAX_STACK = 48;
 	dtNode* stack[MAX_STACK];
@@ -3302,6 +3383,9 @@ dtStatus dtNavMeshQuery::getPolyWallSegments(dtPolyRef ref, const dtQueryFilter*
 {
 	dtAssert(m_nav);
 
+	if (!segmentCount)
+		return DT_FAILURE | DT_INVALID_PARAM;
+	
 	*segmentCount = 0;
 
 	const dtMeshTile* tile = 0;
@@ -3309,6 +3393,9 @@ dtStatus dtNavMeshQuery::getPolyWallSegments(dtPolyRef ref, const dtQueryFilter*
 	if (dtStatusFailed(m_nav->getTileAndPolyByRef(ref, &tile, &poly)))
 		return DT_FAILURE | DT_INVALID_PARAM;
 
+	if (!filter || !segmentVerts || maxSegments < 0)
+		return DT_FAILURE | DT_INVALID_PARAM;
+	
 	int n = 0;
 	static const int MAX_INTERVAL = 16;
 	dtSegInterval ints[MAX_INTERVAL];
@@ -3455,8 +3542,13 @@ dtStatus dtNavMeshQuery::findDistanceToWall(dtPolyRef startRef, const float* cen
 	dtAssert(m_openList);
 	
 	// Validate input
-	if (!startRef || !m_nav->isValidPolyRef(startRef))
+	if (!m_nav->isValidPolyRef(startRef) ||
+		!centerPos || !dtVisfinite(centerPos) ||
+		maxRadius < 0 || !dtMathIsfinite(maxRadius) ||
+		!filter || !hitDist || !hitPos || !hitNormal)
+	{
 		return DT_FAILURE | DT_INVALID_PARAM;
+	}
 	
 	m_nodePool->clear();
 	m_openList->clear();