Restrict ExtractLinearSum to monotonous operation in infinite math space.

Thanks to Bruno Keith & Niklas Baumstark from the phoenhex team for
finding this issue and reporting it with a proper analysis.

1 files changed, 17 insertions, 4 deletions

diff --git a/js/src/jit/IonAnalysis.cpp b/js/src/jit/IonAnalysis.cpp
index b163d5818..d255c32a8 100644
--- a/js/src/jit/IonAnalysis.cpp
+++ b/js/src/jit/IonAnalysis.cpp
@@ -3127,6 +3127,15 @@ ExtractMathSpace(MDefinition* ins)
     MOZ_MAKE_COMPILER_ASSUME_IS_UNREACHABLE("Unknown TruncateKind");
 }
 
+static bool MonotoneAdd(int32_t lhs, int32_t rhs) {
+  return (lhs >= 0 && rhs >= 0) || (lhs <= 0 && rhs <= 0);
+}
+
+static bool MonotoneSub(int32_t lhs, int32_t rhs) {
+  return (lhs >= 0 && rhs <= 0) || (lhs <= 0 && rhs >= 0);
+}
+
+
 // Extract a linear sum from ins, if possible (otherwise giving the sum 'ins + 0').
 SimpleLinearSum
 jit::ExtractLinearSum(MDefinition* ins, MathSpace space)
@@ -3168,10 +3177,12 @@ jit::ExtractLinearSum(MDefinition* ins, MathSpace space)
     // Check if this is of the form <SUM> + n or n + <SUM>.
     if (ins->isAdd()) {
         int32_t constant;
-        if (space == MathSpace::Modulo)
+        if (space == MathSpace::Modulo) {
             constant = lsum.constant + rsum.constant;
-        else if (!SafeAdd(lsum.constant, rsum.constant, &constant))
+        } else if (!SafeAdd(lsum.constant, rsum.constant, &constant) ||
+                   !MonotoneAdd(lsum.constant, rsum.constant)) {
             return SimpleLinearSum(ins, 0);
+        }
         return SimpleLinearSum(lsum.term ? lsum.term : rsum.term, constant);
     }
 
@@ -3179,10 +3190,12 @@ jit::ExtractLinearSum(MDefinition* ins, MathSpace space)
     // Check if this is of the form <SUM> - n.
     if (lsum.term) {
         int32_t constant;
-        if (space == MathSpace::Modulo)
+        if (space == MathSpace::Modulo) {
             constant = lsum.constant - rsum.constant;
-        else if (!SafeSub(lsum.constant, rsum.constant, &constant))
+        } else if (!SafeSub(lsum.constant, rsum.constant, &constant) ||
+                   !MonotoneSub(lsum.constant, rsum.constant)) {
             return SimpleLinearSum(ins, 0);
+        }
         return SimpleLinearSum(lsum.term, constant);
     }