Add length checks for cryptographic primitives

This rollup patch adds additional length checks around cryptographic
primitives.

1 files changed, 11 insertions, 9 deletions

diff --git a/security/nss/lib/freebl/rsapkcs.c b/security/nss/lib/freebl/rsapkcs.c
index 875e4e28d..6f94770ad 100644
--- a/security/nss/lib/freebl/rsapkcs.c
+++ b/security/nss/lib/freebl/rsapkcs.c
@@ -115,7 +115,7 @@ rsa_FormatOneBlock(unsigned modulusLen,
 {
     unsigned char *block;
     unsigned char *bp;
-    int padLen;
+    unsigned int padLen;
     int i, j;
     SECStatus rv;
 
@@ -135,14 +135,14 @@ rsa_FormatOneBlock(unsigned modulusLen,
     switch (blockType) {
 
         /*
-       * Blocks intended for private-key operation.
-       */
+         * Blocks intended for private-key operation.
+         */
         case RSA_BlockPrivate: /* preferred method */
             /*
-         * 0x00 || BT || Pad || 0x00 || ActualData
-         *   1      1   padLen    1      data->len
-         * Pad is either all 0x00 or all 0xff bytes, depending on blockType.
-         */
+             * 0x00 || BT || Pad || 0x00 || ActualData
+             *   1      1   padLen    1      data->len
+             * Pad is either all 0x00 or all 0xff bytes, depending on blockType.
+             */
             padLen = modulusLen - data->len - 3;
             PORT_Assert(padLen >= RSA_BLOCK_MIN_PAD_LEN);
             if (padLen < RSA_BLOCK_MIN_PAD_LEN) {
@@ -162,7 +162,7 @@ rsa_FormatOneBlock(unsigned modulusLen,
             /*
              * 0x00 || BT || Pad || 0x00 || ActualData
              *   1      1   padLen    1      data->len
-             * Pad is all non-zero random bytes.
+             * Pad is 8 or more non-zero random bytes.
              *
              * Build the block left to right.
              * Fill the entire block from Pad to the end with random bytes.
@@ -236,7 +236,9 @@ rsa_FormatBlock(SECItem *result,
              * The "3" below is the first octet + the second octet + the 0x00
              * octet that always comes just before the ActualData.
              */
-            PORT_Assert(data->len <= (modulusLen - (3 + RSA_BLOCK_MIN_PAD_LEN)));
+            if (data->len > (modulusLen - (3 + RSA_BLOCK_MIN_PAD_LEN))) {
+                return SECFailure;
+            }
 
             result->data = rsa_FormatOneBlock(modulusLen, blockType, data);
             if (result->data == NULL) {