From 15025e8519798de296c180446c725d8a98cde957 Mon Sep 17 00:00:00 2001 From: Rafael Stahl Date: Tue, 23 Jun 2015 17:51:49 +0200 Subject: [PATCH] handle the case when allocSpan fails to allocate memory. adds bool return types to rcAddSpan, rcRasterizeTriangle and rcRasterizeTriangles --- Recast/Include/Recast.h | 16 +++-- Recast/Source/RecastRasterization.cpp | 70 +++++++++++++++++----- RecastDemo/Source/Sample_SoloMesh.cpp | 6 +- RecastDemo/Source/Sample_TempObstacles.cpp | 3 +- RecastDemo/Source/Sample_TileMesh.cpp | 3 +- 5 files changed, 74 insertions(+), 24 deletions(-) diff --git a/Recast/Include/Recast.h b/Recast/Include/Recast.h index 0705033..d92d659 100644 --- a/Recast/Include/Recast.h +++ b/Recast/Include/Recast.h @@ -763,6 +763,7 @@ void rcCalcGridSize(const float* bmin, const float* bmax, float cs, int* w, int* /// @param[in] bmax The maximum bounds of the field's AABB. [(x, y, z)] [Units: wu] /// @param[in] cs The xz-plane cell size to use for the field. [Limit: > 0] [Units: wu] /// @param[in] ch The y-axis cell size to use for field. [Limit: > 0] [Units: wu] +/// @returns True if the operation completed successfully. bool rcCreateHeightfield(rcContext* ctx, rcHeightfield& hf, int width, int height, const float* bmin, const float* bmax, float cs, float ch); @@ -806,7 +807,8 @@ void rcClearUnwalkableTriangles(rcContext* ctx, const float walkableSlopeAngle, /// @param[in] smax The maximum height of the span. [Limit: <= #RC_SPAN_MAX_HEIGHT] [Units: vx] /// @param[in] area The area id of the span. [Limit: <= #RC_WALKABLE_AREA) /// @param[in] flagMergeThr The merge theshold. [Limit: >= 0] [Units: vx] -void rcAddSpan(rcContext* ctx, rcHeightfield& hf, const int x, const int y, +/// @returns True if the operation completed successfully. +bool rcAddSpan(rcContext* ctx, rcHeightfield& hf, const int x, const int y, const unsigned short smin, const unsigned short smax, const unsigned char area, const int flagMergeThr); @@ -820,7 +822,8 @@ void rcAddSpan(rcContext* ctx, rcHeightfield& hf, const int x, const int y, /// @param[in,out] solid An initialized heightfield. /// @param[in] flagMergeThr The distance where the walkable flag is favored over the non-walkable flag. /// [Limit: >= 0] [Units: vx] -void rcRasterizeTriangle(rcContext* ctx, const float* v0, const float* v1, const float* v2, +/// @returns True if the operation completed successfully. +bool rcRasterizeTriangle(rcContext* ctx, const float* v0, const float* v1, const float* v2, const unsigned char area, rcHeightfield& solid, const int flagMergeThr = 1); @@ -835,7 +838,8 @@ void rcRasterizeTriangle(rcContext* ctx, const float* v0, const float* v1, const /// @param[in,out] solid An initialized heightfield. /// @param[in] flagMergeThr The distance where the walkable flag is favored over the non-walkable flag. /// [Limit: >= 0] [Units: vx] -void rcRasterizeTriangles(rcContext* ctx, const float* verts, const int nv, +/// @returns True if the operation completed successfully. +bool rcRasterizeTriangles(rcContext* ctx, const float* verts, const int nv, const int* tris, const unsigned char* areas, const int nt, rcHeightfield& solid, const int flagMergeThr = 1); @@ -850,7 +854,8 @@ void rcRasterizeTriangles(rcContext* ctx, const float* verts, const int nv, /// @param[in,out] solid An initialized heightfield. /// @param[in] flagMergeThr The distance where the walkable flag is favored over the non-walkable flag. /// [Limit: >= 0] [Units: vx] -void rcRasterizeTriangles(rcContext* ctx, const float* verts, const int nv, +/// @returns True if the operation completed successfully. +bool rcRasterizeTriangles(rcContext* ctx, const float* verts, const int nv, const unsigned short* tris, const unsigned char* areas, const int nt, rcHeightfield& solid, const int flagMergeThr = 1); @@ -863,7 +868,8 @@ void rcRasterizeTriangles(rcContext* ctx, const float* verts, const int nv, /// @param[in,out] solid An initialized heightfield. /// @param[in] flagMergeThr The distance where the walkable flag is favored over the non-walkable flag. /// [Limit: >= 0] [Units: vx] -void rcRasterizeTriangles(rcContext* ctx, const float* verts, const unsigned char* areas, const int nt, +/// @returns True if the operation completed successfully. +bool rcRasterizeTriangles(rcContext* ctx, const float* verts, const unsigned char* areas, const int nt, rcHeightfield& solid, const int flagMergeThr = 1); /// Marks non-walkable spans as walkable if their maximum is within @p walkableClimp of a walkable neihbor. diff --git a/Recast/Source/RecastRasterization.cpp b/Recast/Source/RecastRasterization.cpp index a6138ca..a4cef74 100644 --- a/Recast/Source/RecastRasterization.cpp +++ b/Recast/Source/RecastRasterization.cpp @@ -82,7 +82,7 @@ static void freeSpan(rcHeightfield& hf, rcSpan* ptr) hf.freelist = ptr; } -static void addSpan(rcHeightfield& hf, const int x, const int y, +static bool addSpan(rcHeightfield& hf, const int x, const int y, const unsigned short smin, const unsigned short smax, const unsigned char area, const int flagMergeThr) { @@ -90,6 +90,8 @@ static void addSpan(rcHeightfield& hf, const int x, const int y, int idx = x + y*hf.width; rcSpan* s = allocSpan(hf); + if (!s) + return false; s->smin = smin; s->smax = smax; s->area = area; @@ -99,7 +101,7 @@ static void addSpan(rcHeightfield& hf, const int x, const int y, if (!hf.spans[idx]) { hf.spans[idx] = s; - return; + return true; } rcSpan* prev = 0; rcSpan* cur = hf.spans[idx]; @@ -152,6 +154,8 @@ static void addSpan(rcHeightfield& hf, const int x, const int y, s->next = hf.spans[idx]; hf.spans[idx] = s; } + + return true; } /// @par @@ -161,12 +165,19 @@ static void addSpan(rcHeightfield& hf, const int x, const int y, /// from the existing span, the span flags are merged. /// /// @see rcHeightfield, rcSpan. -void rcAddSpan(rcContext* /*ctx*/, rcHeightfield& hf, const int x, const int y, +bool rcAddSpan(rcContext* ctx, rcHeightfield& hf, const int x, const int y, const unsigned short smin, const unsigned short smax, const unsigned char area, const int flagMergeThr) { -// rcAssert(ctx); - addSpan(hf, x,y, smin, smax, area, flagMergeThr); + rcAssert(ctx); + + if (!addSpan(hf, x, y, smin, smax, area, flagMergeThr)) + { + ctx->log(RC_LOG_ERROR, "rcAddSpan: Out of memory."); + return false; + } + + return true; } // divides a convex polygons into two convex polygons on both sides of a line @@ -227,7 +238,7 @@ static void dividePoly(const float* in, int nin, -static void rasterizeTri(const float* v0, const float* v1, const float* v2, +static bool rasterizeTri(const float* v0, const float* v1, const float* v2, const unsigned char area, rcHeightfield& hf, const float* bmin, const float* bmax, const float cs, const float ics, const float ich, @@ -248,7 +259,7 @@ static void rasterizeTri(const float* v0, const float* v1, const float* v2, // If the triangle does not touch the bbox of the heightfield, skip the triagle. if (!overlapBounds(bmin, bmax, tmin, tmax)) - return; + return true; // Calculate the footprint of the triangle on the grid's y-axis int y0 = (int)((tmin[2] - bmin[2])*ics); @@ -315,9 +326,12 @@ static void rasterizeTri(const float* v0, const float* v1, const float* v2, unsigned short ismin = (unsigned short)rcClamp((int)floorf(smin * ich), 0, RC_SPAN_MAX_HEIGHT); unsigned short ismax = (unsigned short)rcClamp((int)ceilf(smax * ich), (int)ismin+1, RC_SPAN_MAX_HEIGHT); - addSpan(hf, x, y, ismin, ismax, area, flagMergeThr); + if (!addSpan(hf, x, y, ismin, ismax, area, flagMergeThr)) + return false; } } + + return true; } /// @par @@ -325,7 +339,7 @@ static void rasterizeTri(const float* v0, const float* v1, const float* v2, /// No spans will be added if the triangle does not overlap the heightfield grid. /// /// @see rcHeightfield -void rcRasterizeTriangle(rcContext* ctx, const float* v0, const float* v1, const float* v2, +bool rcRasterizeTriangle(rcContext* ctx, const float* v0, const float* v1, const float* v2, const unsigned char area, rcHeightfield& solid, const int flagMergeThr) { @@ -335,7 +349,13 @@ void rcRasterizeTriangle(rcContext* ctx, const float* v0, const float* v1, const const float ics = 1.0f/solid.cs; const float ich = 1.0f/solid.ch; - rasterizeTri(v0, v1, v2, area, solid, solid.bmin, solid.bmax, solid.cs, ics, ich, flagMergeThr); + if (!rasterizeTri(v0, v1, v2, area, solid, solid.bmin, solid.bmax, solid.cs, ics, ich, flagMergeThr)) + { + ctx->log(RC_LOG_ERROR, "rcRasterizeTriangle: Out of memory."); + return false; + } + + return true; } /// @par @@ -343,7 +363,7 @@ void rcRasterizeTriangle(rcContext* ctx, const float* v0, const float* v1, const /// Spans will only be added for triangles that overlap the heightfield grid. /// /// @see rcHeightfield -void rcRasterizeTriangles(rcContext* ctx, const float* verts, const int /*nv*/, +bool rcRasterizeTriangles(rcContext* ctx, const float* verts, const int /*nv*/, const int* tris, const unsigned char* areas, const int nt, rcHeightfield& solid, const int flagMergeThr) { @@ -360,8 +380,14 @@ void rcRasterizeTriangles(rcContext* ctx, const float* verts, const int /*nv*/, const float* v1 = &verts[tris[i*3+1]*3]; const float* v2 = &verts[tris[i*3+2]*3]; // Rasterize. - rasterizeTri(v0, v1, v2, areas[i], solid, solid.bmin, solid.bmax, solid.cs, ics, ich, flagMergeThr); + if (!rasterizeTri(v0, v1, v2, areas[i], solid, solid.bmin, solid.bmax, solid.cs, ics, ich, flagMergeThr)) + { + ctx->log(RC_LOG_ERROR, "rcRasterizeTriangles: Out of memory."); + return false; + } } + + return true; } /// @par @@ -369,7 +395,7 @@ void rcRasterizeTriangles(rcContext* ctx, const float* verts, const int /*nv*/, /// Spans will only be added for triangles that overlap the heightfield grid. /// /// @see rcHeightfield -void rcRasterizeTriangles(rcContext* ctx, const float* verts, const int /*nv*/, +bool rcRasterizeTriangles(rcContext* ctx, const float* verts, const int /*nv*/, const unsigned short* tris, const unsigned char* areas, const int nt, rcHeightfield& solid, const int flagMergeThr) { @@ -386,8 +412,14 @@ void rcRasterizeTriangles(rcContext* ctx, const float* verts, const int /*nv*/, const float* v1 = &verts[tris[i*3+1]*3]; const float* v2 = &verts[tris[i*3+2]*3]; // Rasterize. - rasterizeTri(v0, v1, v2, areas[i], solid, solid.bmin, solid.bmax, solid.cs, ics, ich, flagMergeThr); + if (!rasterizeTri(v0, v1, v2, areas[i], solid, solid.bmin, solid.bmax, solid.cs, ics, ich, flagMergeThr)) + { + ctx->log(RC_LOG_ERROR, "rcRasterizeTriangles: Out of memory."); + return false; + } } + + return true; } /// @par @@ -395,7 +427,7 @@ void rcRasterizeTriangles(rcContext* ctx, const float* verts, const int /*nv*/, /// Spans will only be added for triangles that overlap the heightfield grid. /// /// @see rcHeightfield -void rcRasterizeTriangles(rcContext* ctx, const float* verts, const unsigned char* areas, const int nt, +bool rcRasterizeTriangles(rcContext* ctx, const float* verts, const unsigned char* areas, const int nt, rcHeightfield& solid, const int flagMergeThr) { rcAssert(ctx); @@ -411,6 +443,12 @@ void rcRasterizeTriangles(rcContext* ctx, const float* verts, const unsigned cha const float* v1 = &verts[(i*3+1)*3]; const float* v2 = &verts[(i*3+2)*3]; // Rasterize. - rasterizeTri(v0, v1, v2, areas[i], solid, solid.bmin, solid.bmax, solid.cs, ics, ich, flagMergeThr); + if (!rasterizeTri(v0, v1, v2, areas[i], solid, solid.bmin, solid.bmax, solid.cs, ics, ich, flagMergeThr)) + { + ctx->log(RC_LOG_ERROR, "rcRasterizeTriangles: Out of memory."); + return false; + } } + + return true; } diff --git a/RecastDemo/Source/Sample_SoloMesh.cpp b/RecastDemo/Source/Sample_SoloMesh.cpp index 7056857..1fd2cc9 100644 --- a/RecastDemo/Source/Sample_SoloMesh.cpp +++ b/RecastDemo/Source/Sample_SoloMesh.cpp @@ -438,7 +438,11 @@ bool Sample_SoloMesh::handleBuild() // the are type for each of the meshes and rasterize them. memset(m_triareas, 0, ntris*sizeof(unsigned char)); rcMarkWalkableTriangles(m_ctx, m_cfg.walkableSlopeAngle, verts, nverts, tris, ntris, m_triareas); - rcRasterizeTriangles(m_ctx, verts, nverts, tris, m_triareas, ntris, *m_solid, m_cfg.walkableClimb); + if (!rcRasterizeTriangles(m_ctx, verts, nverts, tris, m_triareas, ntris, *m_solid, m_cfg.walkableClimb)) + { + m_ctx->log(RC_LOG_ERROR, "buildNavigation: Could not rasterize triangles."); + return false; + } if (!m_keepInterResults) { diff --git a/RecastDemo/Source/Sample_TempObstacles.cpp b/RecastDemo/Source/Sample_TempObstacles.cpp index 967f50a..5601648 100644 --- a/RecastDemo/Source/Sample_TempObstacles.cpp +++ b/RecastDemo/Source/Sample_TempObstacles.cpp @@ -348,7 +348,8 @@ static int rasterizeTileLayers(BuildContext* ctx, InputGeom* geom, rcMarkWalkableTriangles(ctx, tcfg.walkableSlopeAngle, verts, nverts, tris, ntris, rc.triareas); - rcRasterizeTriangles(ctx, verts, nverts, tris, rc.triareas, ntris, *rc.solid, tcfg.walkableClimb); + if (!rcRasterizeTriangles(ctx, verts, nverts, tris, rc.triareas, ntris, *rc.solid, tcfg.walkableClimb)) + return 0; } // Once all geometry is rasterized, we do initial pass of filtering to diff --git a/RecastDemo/Source/Sample_TileMesh.cpp b/RecastDemo/Source/Sample_TileMesh.cpp index ebe670a..6fbc1c3 100644 --- a/RecastDemo/Source/Sample_TileMesh.cpp +++ b/RecastDemo/Source/Sample_TileMesh.cpp @@ -1018,7 +1018,8 @@ unsigned char* Sample_TileMesh::buildTileMesh(const int tx, const int ty, const rcMarkWalkableTriangles(m_ctx, m_cfg.walkableSlopeAngle, verts, nverts, ctris, nctris, m_triareas); - rcRasterizeTriangles(m_ctx, verts, nverts, ctris, m_triareas, nctris, *m_solid, m_cfg.walkableClimb); + if (!rcRasterizeTriangles(m_ctx, verts, nverts, ctris, m_triareas, nctris, *m_solid, m_cfg.walkableClimb)) + return 0; } if (!m_keepInterResults)