5 files changed, 46 insertions, 35 deletions
diff --git a/third_party/aom/aom_mem/aom_mem.c b/third_party/aom/aom_mem/aom_mem.c
index 66a0c08de..e603fc5bf 100644
--- a/third_party/aom/aom_mem/aom_mem.c
+++ b/third_party/aom/aom_mem/aom_mem.c
@@ -9,8 +9,6 @@
  * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
  */
 
-#define __AOM_MEM_C__
-
 #include "aom_mem.h"
 #include <stdio.h>
 #include <stdlib.h>
@@ -18,6 +16,17 @@
 #include "include/aom_mem_intrnl.h"
 #include "aom/aom_integer.h"
 
+#if defined(AOM_MAX_ALLOCABLE_MEMORY)
+// Returns 0 in case of overflow of nmemb * size.
+static int check_size_argument_overflow(uint64_t nmemb, uint64_t size) {
+  const uint64_t total_size = nmemb * size;
+  if (nmemb == 0) return 1;
+  if (size > AOM_MAX_ALLOCABLE_MEMORY / nmemb) return 0;
+  if (total_size != (size_t)total_size) return 0;
+  return 1;
+}
+#endif
+
 static size_t GetAlignedMallocSize(size_t size, size_t align) {
   return size + align - 1 + ADDRESS_STORAGE_SIZE;
 }
@@ -40,6 +49,9 @@ static void *GetActualMallocAddress(void *const mem) {
 void *aom_memalign(size_t align, size_t size) {
   void *x = NULL;
   const size_t aligned_size = GetAlignedMallocSize(size, align);
+#if defined(AOM_MAX_ALLOCABLE_MEMORY)
+  if (!check_size_argument_overflow(1, aligned_size)) return NULL;
+#endif
   void *const addr = malloc(aligned_size);
   if (addr) {
     x = align_addr((unsigned char *)addr + ADDRESS_STORAGE_SIZE, align);
@@ -64,11 +76,9 @@ void aom_free(void *memblk) {
   }
 }
 
-#if CONFIG_HIGHBITDEPTH
 void *aom_memset16(void *dest, int val, size_t length) {
   size_t i;
   uint16_t *dest16 = (uint16_t *)dest;
   for (i = 0; i < length; i++) *dest16++ = val;
   return dest;
 }
-#endif  // CONFIG_HIGHBITDEPTH
diff --git a/third_party/aom/aom_mem/aom_mem.cmake b/third_party/aom/aom_mem/aom_mem.cmake
index a142824c2..eaee8440b 100644
--- a/third_party/aom/aom_mem/aom_mem.cmake
+++ b/third_party/aom/aom_mem/aom_mem.cmake
@@ -1,27 +1,26 @@
-##
-## Copyright (c) 2017, Alliance for Open Media. All rights reserved
-##
-## This source code is subject to the terms of the BSD 2 Clause License and
-## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
-## was not distributed with this source code in the LICENSE file, you can
-## obtain it at www.aomedia.org/license/software. If the Alliance for Open
-## Media Patent License 1.0 was not distributed with this source code in the
-## PATENTS file, you can obtain it at www.aomedia.org/license/patent.
-##
-if (NOT AOM_AOM_MEM_AOM_MEM_CMAKE_)
+#
+# Copyright (c) 2017, Alliance for Open Media. All rights reserved
+#
+# This source code is subject to the terms of the BSD 2 Clause License and the
+# Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License was
+# not distributed with this source code in the LICENSE file, you can obtain it
+# at www.aomedia.org/license/software. If the Alliance for Open Media Patent
+# License 1.0 was not distributed with this source code in the PATENTS file, you
+# can obtain it at www.aomedia.org/license/patent.
+#
+if(AOM_AOM_MEM_AOM_MEM_CMAKE_)
+  return()
+endif() # AOM_AOM_MEM_AOM_MEM_CMAKE_
 set(AOM_AOM_MEM_AOM_MEM_CMAKE_ 1)
 
-set(AOM_MEM_SOURCES
-    "${AOM_ROOT}/aom_mem/aom_mem.c"
-    "${AOM_ROOT}/aom_mem/aom_mem.h"
-    "${AOM_ROOT}/aom_mem/include/aom_mem_intrnl.h")
+list(APPEND AOM_MEM_SOURCES "${AOM_ROOT}/aom_mem/aom_mem.c"
+            "${AOM_ROOT}/aom_mem/aom_mem.h"
+            "${AOM_ROOT}/aom_mem/include/aom_mem_intrnl.h")
 
 # Creates the aom_mem build target and makes libaom depend on it. The libaom
 # target must exist before this function is called.
-function (setup_aom_mem_targets)
+function(setup_aom_mem_targets)
   add_library(aom_mem OBJECT ${AOM_MEM_SOURCES})
   set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} aom_mem PARENT_SCOPE)
   target_sources(aom PRIVATE $<TARGET_OBJECTS:aom_mem>)
-endfunction ()
-
-endif ()  # AOM_AOM_MEM_AOM_MEM_CMAKE_
+endfunction()
diff --git a/third_party/aom/aom_mem/aom_mem.h b/third_party/aom/aom_mem/aom_mem.h
index 0d533c813..a36ee3e03 100644
--- a/third_party/aom/aom_mem/aom_mem.h
+++ b/third_party/aom/aom_mem/aom_mem.h
@@ -12,26 +12,31 @@
 #ifndef AOM_MEM_AOM_MEM_H_
 #define AOM_MEM_AOM_MEM_H_
 
-#include "aom_config.h"
+#include "aom/aom_integer.h"
+#include "config/aom_config.h"
+
 #if defined(__uClinux__)
 #include <lddk.h>
 #endif
 
-#include <stdlib.h>
-#include <stddef.h>
-
 #if defined(__cplusplus)
 extern "C" {
 #endif
 
+#ifndef AOM_MAX_ALLOCABLE_MEMORY
+#if SIZE_MAX > (1ULL << 32)
+#define AOM_MAX_ALLOCABLE_MEMORY 8589934592  // 8 GB
+#else
+// For 32-bit targets keep this below INT_MAX to avoid valgrind warnings.
+#define AOM_MAX_ALLOCABLE_MEMORY ((1ULL << 31) - (1 << 16))
+#endif
+#endif
+
 void *aom_memalign(size_t align, size_t size);
 void *aom_malloc(size_t size);
 void *aom_calloc(size_t num, size_t size);
 void aom_free(void *memblk);
-
-#if CONFIG_HIGHBITDEPTH
 void *aom_memset16(void *dest, int val, size_t length);
-#endif
 
 #include <string.h>
 
diff --git a/third_party/aom/aom_mem/aom_mem.mk b/third_party/aom/aom_mem/aom_mem.mk
deleted file mode 100644
index e9162c284..000000000
--- a/third_party/aom/aom_mem/aom_mem.mk
+++ /dev/null
@@ -1,4 +0,0 @@
-MEM_SRCS-yes += aom_mem.mk
-MEM_SRCS-yes += aom_mem.c
-MEM_SRCS-yes += aom_mem.h
-MEM_SRCS-yes += include/aom_mem_intrnl.h
diff --git a/third_party/aom/aom_mem/include/aom_mem_intrnl.h b/third_party/aom/aom_mem/include/aom_mem_intrnl.h
index 3cdfbe08d..977ebadcd 100644
--- a/third_party/aom/aom_mem/include/aom_mem_intrnl.h
+++ b/third_party/aom/aom_mem/include/aom_mem_intrnl.h
@@ -11,7 +11,8 @@
 
 #ifndef AOM_MEM_INCLUDE_AOM_MEM_INTRNL_H_
 #define AOM_MEM_INCLUDE_AOM_MEM_INTRNL_H_
-#include "./aom_config.h"
+
+#include "config/aom_config.h"
 
 #define ADDRESS_STORAGE_SIZE sizeof(size_t)