Fix for Issue 55: Negative array access in dtNavMesh::raycast()

This commit is contained in:
Mikko Mononen 2010-03-18 08:06:12 +00:00
parent 21f76bc0a5
commit 70c17e8212
7 changed files with 1811 additions and 2133 deletions

File diff suppressed because it is too large Load Diff

View File

@ -281,14 +281,14 @@
<key>PBXSmartGroupTreeModuleOutlineStateSelectionKey</key>
<array>
<array>
<integer>45</integer>
<integer>35</integer>
<integer>52</integer>
<integer>46</integer>
<integer>1</integer>
<integer>0</integer>
</array>
</array>
<key>PBXSmartGroupTreeModuleOutlineStateVisibleRectKey</key>
<string>{{0, 458}, {358, 643}}</string>
<string>{{0, 429}, {358, 643}}</string>
</dict>
<key>PBXTopSmartGroupGIDs</key>
<array/>
@ -323,7 +323,7 @@
<key>PBXProjectModuleGUID</key>
<string>6B8632A30F78115100E2684A</string>
<key>PBXProjectModuleLabel</key>
<string>Sample_Debug.cpp</string>
<string>NavMeshTesterTool.cpp</string>
<key>PBXSplitModuleInNavigatorKey</key>
<dict>
<key>Split0</key>
@ -331,11 +331,11 @@
<key>PBXProjectModuleGUID</key>
<string>6B8632A40F78115100E2684A</string>
<key>PBXProjectModuleLabel</key>
<string>Sample_Debug.cpp</string>
<string>NavMeshTesterTool.cpp</string>
<key>_historyCapacity</key>
<integer>0</integer>
<key>bookmark</key>
<string>6B92D9C8114101C100E82EC3</string>
<string>6B904C6D1152164400A12E63</string>
<key>history</key>
<array>
<string>6B8DE70D10B01BBF00DF20FB</string>
@ -352,8 +352,6 @@
<string>6BF7C69B11172159002B3F46</string>
<string>6B324AE6111C07AB00EBD2FD</string>
<string>6B324AEA111C0D9700EBD2FD</string>
<string>6B324AFB111C0F2700EBD2FD</string>
<string>6B324B1F111C10C700EBD2FD</string>
<string>6B324B4F111C1AC800EBD2FD</string>
<string>6B324C45111C5C5A00EBD2FD</string>
<string>6B324CC3111C6F6300EBD2FD</string>
@ -368,39 +366,42 @@
<string>6BE90FDF112A770600F5C17A</string>
<string>6BE91085112A898E00F5C17A</string>
<string>6BE910BA112A91D900F5C17A</string>
<string>6BE910BB112A91D900F5C17A</string>
<string>6BE910C2112A92A300F5C17A</string>
<string>6BE910D4112A933500F5C17A</string>
<string>6BE910D9112B1AF000F5C17A</string>
<string>6BE910F21130617300F5C17A</string>
<string>6BE9113A11379F5000F5C17A</string>
<string>6BE9113C11379F5000F5C17A</string>
<string>6B8036BB113BAF47005ED67B</string>
<string>6B8036D0113BAF79005ED67B</string>
<string>6B8036F7113BB746005ED67B</string>
<string>6B8036F8113BB746005ED67B</string>
<string>6B803729113D1079005ED67B</string>
<string>6B80372B113D1079005ED67B</string>
<string>6B803743113D1190005ED67B</string>
<string>6B803796113D1629005ED67B</string>
<string>6BC8BE261140ED5F00555B22</string>
<string>6BC8BE4A1140F01D00555B22</string>
<string>6BC8BE4B1140F01D00555B22</string>
<string>6B92D9661140FBDA00E82EC3</string>
<string>6B92D96A1140FBDA00E82EC3</string>
<string>6B92D96B1140FBDA00E82EC3</string>
<string>6B92D97A1140FE6000E82EC3</string>
<string>6B92D97B1140FE6000E82EC3</string>
<string>6B92D97D1140FE6000E82EC3</string>
<string>6B92D9881140FE9700E82EC3</string>
<string>6B92D99D1141004F00E82EC3</string>
<string>6B92D9AF114100A300E82EC3</string>
<string>6B92D9B81141017500E82EC3</string>
<string>6B92D9B91141017500E82EC3</string>
<string>6B92D9BA1141017500E82EC3</string>
<string>6B92D9BB1141017500E82EC3</string>
<string>6B92D9C5114101C100E82EC3</string>
<string>6B92D9C6114101C100E82EC3</string>
<string>6B904B231144E89000A12E63</string>
<string>6B904B601144F03800A12E63</string>
<string>6B904B6D1144F11000A12E63</string>
<string>6B904B7E1144F2C000A12E63</string>
<string>6B904B7F1144F2C000A12E63</string>
<string>6B904B9D1144F39D00A12E63</string>
<string>6B904BA61144F3FA00A12E63</string>
<string>6B904BAC1144F43300A12E63</string>
<string>6B904BB61144F49D00A12E63</string>
<string>6B904BBE1144F50500A12E63</string>
<string>6B904BD01144FCF000A12E63</string>
<string>6B904BE91144FDC900A12E63</string>
<string>6B904C091145340E00A12E63</string>
<string>6B904C40115211FB00A12E63</string>
<string>6B904C41115211FB00A12E63</string>
<string>6B904C521152132800A12E63</string>
<string>6B904C591152139200A12E63</string>
<string>6B904C631152164400A12E63</string>
<string>6B904C641152164400A12E63</string>
<string>6B904C651152164400A12E63</string>
</array>
<key>prevStack</key>
<array>
@ -444,14 +445,11 @@
<string>6BE91035112A7D9600F5C17A</string>
<string>6BE91089112A898E00F5C17A</string>
<string>6BE9108A112A898E00F5C17A</string>
<string>6BE910A5112A8CF900F5C17A</string>
<string>6BE910C7112A92A300F5C17A</string>
<string>6BE9113F11379F5000F5C17A</string>
<string>6B8036A5113BAA82005ED67B</string>
<string>6B8036A6113BAA82005ED67B</string>
<string>6B8036C2113BAF47005ED67B</string>
<string>6B8036C5113BAF47005ED67B</string>
<string>6B8036C6113BAF47005ED67B</string>
<string>6B8036D3113BAF79005ED67B</string>
<string>6B8036D4113BAF79005ED67B</string>
<string>6B8036D5113BAF79005ED67B</string>
@ -459,50 +457,36 @@
<string>6B80379B113D1629005ED67B</string>
<string>6BC8BE281140ED5F00555B22</string>
<string>6B92D9121140F58200E82EC3</string>
<string>6B92D91C1140F5F800E82EC3</string>
<string>6B92D9261140F6BF00E82EC3</string>
<string>6B92D9321140F7C500E82EC3</string>
<string>6B92D9411140F84100E82EC3</string>
<string>6BE910F91130617300F5C17A</string>
<string>6B92D94E1140F99700E82EC3</string>
<string>6B92D94F1140F99700E82EC3</string>
<string>6B92D9501140F99700E82EC3</string>
<string>6B92D9511140F99700E82EC3</string>
<string>6B92D95F1140FB3D00E82EC3</string>
<string>6B92D96D1140FBDA00E82EC3</string>
<string>6B92D96E1140FBDA00E82EC3</string>
<string>6B92D96F1140FBDA00E82EC3</string>
<string>6B92D9701140FBDA00E82EC3</string>
<string>6B92D9711140FBDA00E82EC3</string>
<string>6B92D9721140FBDA00E82EC3</string>
<string>6B92D9731140FBDA00E82EC3</string>
<string>6B92D9741140FBDA00E82EC3</string>
<string>6B92D9751140FBDA00E82EC3</string>
<string>6B92D97F1140FE6000E82EC3</string>
<string>6B92D9801140FE6000E82EC3</string>
<string>6B92D9811140FE6000E82EC3</string>
<string>6B92D9821140FE6000E82EC3</string>
<string>6B92D9831140FE6000E82EC3</string>
<string>6B92D9841140FE6000E82EC3</string>
<string>6B92D9851140FE6000E82EC3</string>
<string>6B92D98A1140FE9700E82EC3</string>
<string>6B92D9A21141004F00E82EC3</string>
<string>6B92D9A31141004F00E82EC3</string>
<string>6B92D9A41141004F00E82EC3</string>
<string>6B92D9A51141004F00E82EC3</string>
<string>6B92D9A61141004F00E82EC3</string>
<string>6B92D9A71141004F00E82EC3</string>
<string>6B92D9A81141004F00E82EC3</string>
<string>6B92D9B1114100A300E82EC3</string>
<string>6B92D9B2114100A300E82EC3</string>
<string>6B92D9BD1141017500E82EC3</string>
<string>6B92D9BE1141017500E82EC3</string>
<string>6B92D9BF1141017500E82EC3</string>
<string>6B92D9C01141017500E82EC3</string>
<string>6B92D9C11141017500E82EC3</string>
<string>6B92D9C21141017500E82EC3</string>
<string>6B92D9C31141017500E82EC3</string>
<string>6B92D9C7114101C100E82EC3</string>
<string>6B904B261144E89000A12E63</string>
<string>6B904B641144F03800A12E63</string>
<string>6B904B701144F11000A12E63</string>
<string>6B904B811144F2C000A12E63</string>
<string>6B904BA11144F39D00A12E63</string>
<string>6B904BA31144F39D00A12E63</string>
<string>6B904BAE1144F43300A12E63</string>
<string>6B904BB81144F49D00A12E63</string>
<string>6B904BB91144F49D00A12E63</string>
<string>6B904BD41144FCF000A12E63</string>
<string>6B904BD51144FCF000A12E63</string>
<string>6B904C0311452A7400A12E63</string>
<string>6B904C0411452A7400A12E63</string>
<string>6B904C0C1145340E00A12E63</string>
<string>6B904C0D1145340E00A12E63</string>
<string>6B904C0E1145340E00A12E63</string>
<string>6B904C44115211FB00A12E63</string>
<string>6B904C45115211FB00A12E63</string>
<string>6B904C46115211FB00A12E63</string>
<string>6B904C5B1152139200A12E63</string>
<string>6B904C5C1152139200A12E63</string>
<string>6B904C661152164400A12E63</string>
<string>6B904C671152164400A12E63</string>
<string>6B904C681152164400A12E63</string>
<string>6B904C691152164400A12E63</string>
<string>6B904C6A1152164400A12E63</string>
<string>6B904C6B1152164400A12E63</string>
<string>6B904C6C1152164400A12E63</string>
</array>
</dict>
<key>SplitCount</key>
@ -516,18 +500,18 @@
<key>GeometryConfiguration</key>
<dict>
<key>Frame</key>
<string>{{0, 0}, {876, 377}}</string>
<string>{{0, 0}, {876, 585}}</string>
<key>RubberWindowFrame</key>
<string>11 76 1256 702 0 0 1280 778 </string>
</dict>
<key>Module</key>
<string>PBXNavigatorGroup</string>
<key>Proportion</key>
<string>377pt</string>
<string>585pt</string>
</dict>
<dict>
<key>Proportion</key>
<string>279pt</string>
<string>71pt</string>
<key>Tabs</key>
<array>
<dict>
@ -541,7 +525,9 @@
<key>GeometryConfiguration</key>
<dict>
<key>Frame</key>
<string>{{10, 27}, {876, -27}}</string>
<string>{{10, 27}, {876, 44}}</string>
<key>RubberWindowFrame</key>
<string>11 76 1256 702 0 0 1280 778 </string>
</dict>
<key>Module</key>
<string>XCDetailModule</string>
@ -557,9 +543,7 @@
<key>GeometryConfiguration</key>
<dict>
<key>Frame</key>
<string>{{10, 27}, {876, 252}}</string>
<key>RubberWindowFrame</key>
<string>11 76 1256 702 0 0 1280 778 </string>
<string>{{10, 27}, {876, 25}}</string>
</dict>
<key>Module</key>
<string>PBXProjectFindModule</string>
@ -597,7 +581,7 @@
<key>GeometryConfiguration</key>
<dict>
<key>Frame</key>
<string>{{10, 27}, {876, 152}}</string>
<string>{{10, 27}, {876, 127}}</string>
</dict>
<key>Module</key>
<string>PBXBuildResultsModule</string>
@ -625,11 +609,11 @@
</array>
<key>TableOfContents</key>
<array>
<string>6B92D8EE1140F26D00E82EC3</string>
<string>6B904B671144F03800A12E63</string>
<string>1CA23ED40692098700951B8B</string>
<string>6B92D8EF1140F26D00E82EC3</string>
<string>6B904B681144F03800A12E63</string>
<string>6B8632A30F78115100E2684A</string>
<string>6B92D8F01140F26D00E82EC3</string>
<string>6B904B691144F03800A12E63</string>
<string>1CA23EDF0692099D00951B8B</string>
<string>1CA23EE00692099D00951B8B</string>
<string>1CA23EE10692099D00951B8B</string>
@ -778,14 +762,14 @@
</array>
<key>TableOfContents</key>
<array>
<string>6B92D9001140F44200E82EC3</string>
<string>6B904B841144F2C000A12E63</string>
<string>1CCC7628064C1048000F2A68</string>
<string>1CCC7629064C1048000F2A68</string>
<string>6B92D9011140F44200E82EC3</string>
<string>6B92D9021140F44200E82EC3</string>
<string>6B92D9031140F44200E82EC3</string>
<string>6B92D9041140F44200E82EC3</string>
<string>6B8632A30F78115100E2684A</string>
<string>6B904B851144F2C000A12E63</string>
<string>6B904B861144F2C000A12E63</string>
<string>6B904B871144F2C000A12E63</string>
<string>6B904B881144F2C000A12E63</string>
<string>6B904B891144F2C000A12E63</string>
</array>
<key>ToolbarConfigUserDefaultsMinorVersion</key>
<string>2</string>
@ -817,8 +801,8 @@
<integer>5</integer>
<key>WindowOrderList</key>
<array>
<string>6B92D9071140F44200E82EC3</string>
<string>6B92D9081140F44200E82EC3</string>
<string>6B904B901144F2ED00A12E63</string>
<string>6B904B911144F2ED00A12E63</string>
<string>/Users/memon/Code/recastnavigation/RecastDemo/Build/Xcode/Recast.xcodeproj</string>
</array>
<key>WindowString</key>

View File

@ -61,6 +61,7 @@ class NavMeshTesterTool : public SampleTool
float m_epos[3];
float m_hitPos[3];
float m_hitNormal[3];
bool m_hitResult;
float m_distanceToWall;
bool m_sposSet;
bool m_eposSet;

View File

@ -30,6 +30,11 @@ class Sample_Debug : public Sample
protected:
rcCompactHeightfield* m_chf;
rcContourSet* m_cset;
float m_ext[3];
float m_center[3];
float m_bmin[3], m_bmax[3];
dtPolyRef m_ref;
public:
Sample_Debug();

View File

@ -105,7 +105,8 @@ NavMeshTesterTool::NavMeshTesterTool() :
m_sposSet(false),
m_eposSet(false),
m_pathIterNum(0),
m_steerPointCount(0)
m_steerPointCount(0),
m_hitResult(false)
{
m_filter.includeFlags = SAMPLE_POLYFLAGS_ALL;
m_filter.excludeFlags = 0;
@ -623,13 +624,26 @@ void NavMeshTesterTool::recalc()
m_straightPath[0] = m_spos[0];
m_straightPath[1] = m_spos[1];
m_straightPath[2] = m_spos[2];
vcopy(m_hitPos, m_epos);
m_npolys = m_navMesh->raycast(m_startRef, m_spos, m_epos, &m_filter, t, m_hitNormal, m_polys, MAX_POLYS);
if (m_npolys && t < 1)
if (t > 1)
{
// No hit
vcopy(m_hitPos, m_epos);
m_hitResult = false;
}
else
{
// Hit
m_hitPos[0] = m_spos[0] + (m_epos[0] - m_spos[0]) * t;
m_hitPos[1] = m_spos[1] + (m_epos[1] - m_spos[1]) * t;
m_hitPos[2] = m_spos[2] + (m_epos[2] - m_spos[2]) * t;
if (m_npolys)
{
float h = 0;
m_navMesh->getPolyHeight(m_polys[m_npolys-1], m_hitPos, &h);
m_hitPos[1] = h;
}
m_hitResult = true;
}
vcopy(&m_straightPath[3], m_hitPos);
}
@ -812,7 +826,7 @@ void NavMeshTesterTool::handleRender()
for (int i = 1; i < m_npolys; ++i)
duDebugDrawNavMeshPoly(&dd, m_navMesh, m_polys[i], pathCol);
const unsigned int pathCol = duRGBA(64,16,0,220);
const unsigned int pathCol = m_hitResult ? duRGBA(64,16,0,220) : duRGBA(240,240,240,220);
dd.begin(DU_DRAW_LINES, 2.0f);
for (int i = 0; i < m_nstraightPath-1; ++i)
{
@ -825,13 +839,16 @@ void NavMeshTesterTool::handleRender()
dd.vertex(m_straightPath[i*3], m_straightPath[i*3+1]+0.4f, m_straightPath[i*3+2], pathCol);
dd.end();
const unsigned int hitCol = duRGBA(0,0,0,128);
dd.begin(DU_DRAW_LINES, 2.0f);
dd.vertex(m_hitPos[0], m_hitPos[1] + 0.4f, m_hitPos[2], hitCol);
dd.vertex(m_hitPos[0] + m_hitNormal[0]*agentRadius,
m_hitPos[1] + 0.4f + m_hitNormal[1]*agentRadius,
m_hitPos[2] + m_hitNormal[2]*agentRadius, hitCol);
dd.end();
if (m_hitResult)
{
const unsigned int hitCol = duRGBA(0,0,0,128);
dd.begin(DU_DRAW_LINES, 2.0f);
dd.vertex(m_hitPos[0], m_hitPos[1] + 0.4f, m_hitPos[2], hitCol);
dd.vertex(m_hitPos[0] + m_hitNormal[0]*agentRadius,
m_hitPos[1] + 0.4f + m_hitNormal[1]*agentRadius,
m_hitPos[2] + m_hitNormal[2]*agentRadius, hitCol);
dd.end();
}
}
}
else if (m_toolMode == TOOLMODE_DISTANCE_TO_WALL)

View File

@ -35,6 +35,19 @@
# define snprintf _snprintf
#endif
static int loadBin(const char* path, unsigned char** data)
{
FILE* fp = fopen(path, "rb");
if (!fp) return 0;
fseek(fp, 0, SEEK_END);
int size = ftell(fp);
fseek(fp, 0, SEEK_SET);
*data = new unsigned char[size];
fread(*data, size, 1, fp);
fclose(fp);
return size;
}
Sample_Debug::Sample_Debug() :
m_chf(0),
@ -44,13 +57,13 @@ Sample_Debug::Sample_Debug() :
// Test
m_chf = new rcCompactHeightfield;
if (!duReadCompactHeightfield(*m_chf, "test.chf"))
if (!duReadCompactHeightfield(*m_chf, "Tile_-13_-14_chf.bin"))
{
delete m_chf;
m_chf = 0;
}
if (m_chf)
/* if (m_chf)
{
unsigned short ymin = 0xffff;
unsigned short ymax = 0;
@ -68,7 +81,46 @@ Sample_Debug::Sample_Debug() :
maxSpans = rcMax(maxSpans, (int)m_chf->cells[i].count);
}
printf("maxSpans = %d\n", maxSpans);
}*/
/* const float orig[3] = {0,0,0};
m_navMesh = new dtNavMesh;
m_navMesh->init(orig, 133.333f,133.333f, 2048, 4096, 4096);
unsigned char* data = 0;
int dataSize = 0;
// Tile_-13_-14.bin is basically just the bytes that was output by Detour. It should be loaded at X: -13 and Y: -14.
dataSize = loadBin("Tile_-13_-13.bin", &data);
if (dataSize > 0)
{
m_navMesh->addTileAt(-13,-13, data, dataSize, true);
dtMeshHeader* header = (dtMeshHeader*)data;
vcopy(m_bmin, header->bmin);
vcopy(m_bmax, header->bmax);
}
dataSize = loadBin("Tile_-13_-14.bin", &data);
if (dataSize > 0)
{
m_navMesh->addTileAt(-13,-14, data, dataSize, true);
}
dataSize = loadBin("Tile_-14_-14.bin", &data);
if (dataSize > 0)
{
m_navMesh->addTileAt(-14,-14, data, dataSize, true);
}
const float ext[3] = {40,100,40};
const float center[3] = { -1667.9491f, 135.52649f, -1680.6149f };
dtQueryFilter filter;
m_ref = m_navMesh->findNearestPoly(center, ext, &filter, 0);
vcopy(m_ext, ext);
vcopy(m_center, center);*/
}
Sample_Debug::~Sample_Debug()
@ -99,6 +151,18 @@ void Sample_Debug::handleRender()
// duDebugDrawCompactHeightfieldSolid(&dd, *m_chf);
}
if (m_navMesh)
duDebugDrawNavMesh(&dd, m_navMesh, DU_DRAWNAVMESH_CLOSEDLIST|DU_DRAWNAVMESH_OFFMESHCONS);
if (m_ref)
duDebugDrawNavMeshPoly(&dd, m_navMesh, m_ref, duRGBA(255,0,0,128));
float bmin[3], bmax[3];
vsub(bmin, m_center, m_ext);
vadd(bmax, m_center, m_ext);
duDebugDrawBoxWire(&dd, bmin[0],bmin[1],bmin[2], bmax[0],bmax[1],bmax[2], duRGBA(255,255,255,128), 1.0f);
duDebugDrawCross(&dd, m_center[0], m_center[1], m_center[2], 1.0f, duRGBA(255,255,255,128), 2.0f);
if (m_cset)
duDebugDrawRawContours(&dd, *m_cset);
}
@ -114,12 +178,18 @@ void Sample_Debug::handleMeshChanged(InputGeom* geom)
const float* Sample_Debug::getBoundsMin()
{
if (m_navMesh)
return m_bmin;
if (!m_chf) return 0;
return m_chf->bmin;
}
const float* Sample_Debug::getBoundsMax()
{
if (m_navMesh)
return m_bmax;
if (!m_chf) return 0;
return m_chf->bmax;
}