[xerces-c] 01/02: XERCESC-2241 - Integer overflows in DFAContentModel class

This is an automated email from the ASF dual-hosted git repository.

scantor pushed a commit to branch xerces-3.2
in repository https://gitbox.apache.org/repos/asf/xerces-c.git

commit 1296a40db07308dbaac32494469f609b00cdfaf3
Author: Scott Cantor <[email protected]>
AuthorDate: Mon Oct 10 11:48:07 2022 -0400

    XERCESC-2241 - Integer overflows in DFAContentModel class
    
    https://issues.apache.org/jira/browse/XERCESC-2241
---
 src/xercesc/validators/common/DFAContentModel.cpp | 27 ++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/src/xercesc/validators/common/DFAContentModel.cpp b/src/xercesc/validators/common/DFAContentModel.cpp
index c14ae7b57..49efa7aed 100644
--- a/src/xercesc/validators/common/DFAContentModel.cpp
+++ b/src/xercesc/validators/common/DFAContentModel.cpp
@@ -42,6 +42,7 @@
 #include <xercesc/util/RefHashTableOf.hpp>
 #include <xercesc/util/XMLInteger.hpp>
 #include <math.h>
+#include <limits>
 
 XERCES_CPP_NAMESPACE_BEGIN
 
@@ -661,8 +662,15 @@ void DFAContentModel::buildDFA(ContentSpecNode* const curNode)
     //  in the fLeafCount member.
     //
     fLeafCount=countLeafNodes(curNode);
+    // Avoid integer overflow in below fLeafCount++ increment
+    if (fLeafCount > (std::numeric_limits<unsigned int>::max() - 1))
+        throw OutOfMemoryException();
     fEOCPos = fLeafCount++;
 
+    // Avoid integer overflow in below memory allocation
+    if (fLeafCount > (std::numeric_limits<size_t>::max() / sizeof(CMLeaf*)))
+        throw OutOfMemoryException();
+
     //  We need to build an array of references to the non-epsilon
     //  leaf nodes. We will put them in the array according to their position values
     //
@@ -1364,14 +1372,27 @@ unsigned int DFAContentModel::countLeafNodes(ContentSpecNode* const curNode)
         if(nLoopCount!=0)
         {
             count += countLeafNodes(cursor);
-            for(unsigned int i=0;i<nLoopCount;i++)
-                count += countLeafNodes(rightNode);
+            const unsigned int countRight = countLeafNodes(rightNode);
+            // Avoid integer overflow in below multiplication
+            if (countRight > (std::numeric_limits<unsigned int>::max() / nLoopCount))
+                throw OutOfMemoryException();
+            const unsigned int countRightMulLoopCount = nLoopCount * countRight;
+            // Avoid integer overflow in below addition
+            if (count > (std::numeric_limits<unsigned int>::max() - countRightMulLoopCount))
+                throw OutOfMemoryException();
+            count += countRightMulLoopCount;
             return count;
         }
         if(leftNode)
             count+=countLeafNodes(leftNode);
         if(rightNode)
-            count+=countLeafNodes(rightNode);
+        {
+            const unsigned int countRight = countLeafNodes(rightNode);
+            // Avoid integer overflow in below addition
+            if (count > (std::numeric_limits<unsigned int>::max() - countRight))
+                throw OutOfMemoryException();
+            count+=countRight;
+        }
     }
     return count;
 }