Account for offsets when creating buffers for image textures.

3 files changed, 44 insertions, 3 deletions

diff --git a/gfx/layers/ImageDataSerializer.cpp b/gfx/layers/ImageDataSerializer.cpp
index 08ed83bd9..db11e903c 100644
--- a/gfx/layers/ImageDataSerializer.cpp
+++ b/gfx/layers/ImageDataSerializer.cpp
@@ -84,6 +84,41 @@ ComputeYCbCrBufferSize(const gfx::IntSize& aYSize, const gfx::IntSize& aCbCrSize
 }
 
 uint32_t
+ComputeYCbCrBufferSize(const gfx::IntSize& aYSize, const gfx::IntSize& aCbCrSize,
+                       uint32_t aYOffset, uint32_t aCbOffset, uint32_t aCrOffset)
+{
+  MOZ_ASSERT(aYSize.height >= 0 && aYSize.width >= 0);
+
+  int32_t yStride = aYSize.width;
+  int32_t cbCrStride = aCbCrSize.width;
+  if (aYSize.height < 0 || aYSize.width < 0 || aCbCrSize.height < 0 || aCbCrSize.width < 0 ||
+      !gfx::Factory::AllowedSurfaceSize(IntSize(yStride, aYSize.height)) ||
+      !gfx::Factory::AllowedSurfaceSize(IntSize(cbCrStride, aCbCrSize.height))) {
+    return 0;
+  }
+
+  uint32_t yLength = GetAlignedStride<4>(yStride, aYSize.height);
+  uint32_t cbCrLength = GetAlignedStride<4>(cbCrStride, aCbCrSize.height);
+  if (yLength == 0 || cbCrLength == 0) {
+    return 0;
+  }
+
+  CheckedInt<uint32_t> yEnd = aYOffset;
+  yEnd += yLength;
+  CheckedInt<uint32_t> cbEnd = aCbOffset;
+  cbEnd += cbCrLength;
+  CheckedInt<uint32_t> crEnd = aCrOffset;
+  crEnd += cbCrLength;
+
+  if (!yEnd.isValid() || !cbEnd.isValid() || !crEnd.isValid() ||
+      yEnd.value() > aCbOffset || cbEnd.value() > aCrOffset) {
+    return 0;
+  }
+
+  return crEnd.value();
+}
+
+uint32_t
 ComputeYCbCrBufferSize(uint32_t aBufferSize)
 {
   return GetAlignedStride<4>(aBufferSize, 1);
diff --git a/gfx/layers/ImageDataSerializer.h b/gfx/layers/ImageDataSerializer.h
index 53a589d21..4b3194b0c 100644
--- a/gfx/layers/ImageDataSerializer.h
+++ b/gfx/layers/ImageDataSerializer.h
@@ -47,7 +47,11 @@ uint32_t ComputeYCbCrBufferSize(const gfx::IntSize& aYSize,
                                 int32_t aCbCrStride);
 uint32_t ComputeYCbCrBufferSize(const gfx::IntSize& aYSize,
                                 const gfx::IntSize& aCbCrSize);
-
+uint32_t ComputeYCbCrBufferSize(const gfx::IntSize& aYSize,
+                                const gfx::IntSize& aCbCrSize,
+                                uint32_t aYOffset,
+                                uint32_t aCbOffset,
+                                uint32_t aCrOffset);
 uint32_t ComputeYCbCrBufferSize(uint32_t aBufferSize);
 
 void ComputeYCbCrOffsets(int32_t yStride, int32_t yHeight,
diff --git a/gfx/layers/composite/TextureHost.cpp b/gfx/layers/composite/TextureHost.cpp
index c93037384..e4a2ffd86 100644
--- a/gfx/layers/composite/TextureHost.cpp
+++ b/gfx/layers/composite/TextureHost.cpp
@@ -259,7 +259,9 @@ CreateBackendIndependentTextureHost(const SurfaceDescriptor& aDesc,
             case BufferDescriptor::TYCbCrDescriptor: {
               const YCbCrDescriptor& ycbcr = desc.get_YCbCrDescriptor();
               reqSize =
-                ImageDataSerializer::ComputeYCbCrBufferSize(ycbcr.ySize(), ycbcr.cbCrSize());
+                ImageDataSerializer::ComputeYCbCrBufferSize(ycbcr.ySize(), ycbcr.cbCrSize(),
+                                                            ycbcr.yOffset(), ycbcr.cbOffset(),
+                                                            ycbcr.crOffset());
               break;
             }
             case BufferDescriptor::TRGBDescriptor: {
@@ -272,7 +274,7 @@ CreateBackendIndependentTextureHost(const SurfaceDescriptor& aDesc,
               MOZ_CRASH("GFX: Bad descriptor");
           }
 
-          if (bufSize < reqSize) {
+          if (reqSize == 0 || bufSize < reqSize) {
             NS_ERROR("A client process gave a shmem too small to fit for its descriptor!");
             return nullptr;
           }