Bug 1388020. r=nical, a=RyanVM

7 files changed, 94 insertions, 41 deletions

diff --git a/gfx/layers/composite/TextureHost.cpp b/gfx/layers/composite/TextureHost.cpp
index 8c5b8c7b7..e7d87e238 100644
--- a/gfx/layers/composite/TextureHost.cpp
+++ b/gfx/layers/composite/TextureHost.cpp
@@ -100,15 +100,9 @@ TextureHost::CreateIPDLActor(HostIPCAllocator* aAllocator,
                              TextureFlags aFlags,
                              uint64_t aSerial)
 {
-  if (aSharedData.type() == SurfaceDescriptor::TSurfaceDescriptorBuffer &&
-      aSharedData.get_SurfaceDescriptorBuffer().data().type() == MemoryOrShmem::Tuintptr_t &&
-      !aAllocator->IsSameProcess())
-  {
-    NS_ERROR("A client process is trying to peek at our address space using a MemoryTexture!");
-    return nullptr;
-  }
   TextureParent* actor = new TextureParent(aAllocator, aSerial);
   if (!actor->Init(aSharedData, aLayersBackend, aFlags)) {
+    actor->ActorDestroy(ipc::IProtocol::ActorDestroyReason::FailedConstructor);
     delete actor;
     return nullptr;
   }
@@ -210,6 +204,11 @@ TextureHost::Create(const SurfaceDescriptor& aDesc,
 
 #ifdef MOZ_X11
     case SurfaceDescriptor::TSurfaceDescriptorX11: {
+      if (!aDeallocator->IsSameProcess()) {
+        NS_ERROR("A client process is trying to peek at our address space using a X11Texture!");
+        return nullptr;
+      }
+
       const SurfaceDescriptorX11& desc = aDesc.get_SurfaceDescriptorX11();
       return MakeAndAddRef<X11TextureHost>(aFlags, desc);
     }
@@ -244,13 +243,49 @@ CreateBackendIndependentTextureHost(const SurfaceDescriptor& aDesc,
       const MemoryOrShmem& data = bufferDesc.data();
       switch (data.type()) {
         case MemoryOrShmem::TShmem: {
-          result = new ShmemTextureHost(data.get_Shmem(),
-                                        bufferDesc.desc(),
-                                        aDeallocator,
-                                        aFlags);
+          const ipc::Shmem& shmem = data.get_Shmem();
+          const BufferDescriptor& desc = bufferDesc.desc();
+          if (!shmem.IsReadable()) {
+            // We failed to map the shmem so we can't verify its size. This
+            // should not be a fatal error, so just create the texture with
+            // nothing backing it.
+            result = new ShmemTextureHost(shmem, desc, aDeallocator, aFlags);
+            break;
+          }
+
+          size_t bufSize = shmem.Size<char>();
+          size_t reqSize = SIZE_MAX;
+          switch (desc.type()) {
+            case BufferDescriptor::TYCbCrDescriptor: {
+              const YCbCrDescriptor& ycbcr = desc.get_YCbCrDescriptor();
+              reqSize =
+                ImageDataSerializer::ComputeYCbCrBufferSize(ycbcr.ySize(), ycbcr.cbCrSize());
+              break;
+            }
+            case BufferDescriptor::TRGBDescriptor: {
+              const RGBDescriptor& rgb = desc.get_RGBDescriptor();
+              reqSize = ImageDataSerializer::ComputeRGBBufferSize(rgb.size(), rgb.format());
+              break;
+            }
+            default:
+              gfxCriticalError() << "Bad buffer host descriptor " << (int)desc.type();
+              MOZ_CRASH("GFX: Bad descriptor");
+          }
+
+          if (bufSize < reqSize) {
+            NS_ERROR("A client process gave a shmem too small to fit for its descriptor!");
+            return nullptr;
+          }
+
+          result = new ShmemTextureHost(shmem, desc, aDeallocator, aFlags);
           break;
         }
         case MemoryOrShmem::Tuintptr_t: {
+          if (!aDeallocator->IsSameProcess()) {
+            NS_ERROR("A client process is trying to peek at our address space using a MemoryTexture!");
+            return nullptr;
+          }
+
           result = new MemoryTextureHost(reinterpret_cast<uint8_t*>(data.get_uintptr_t()),
                                          bufferDesc.desc(),
                                          aFlags);
@@ -268,6 +303,11 @@ CreateBackendIndependentTextureHost(const SurfaceDescriptor& aDesc,
     }
 #ifdef XP_WIN
     case SurfaceDescriptor::TSurfaceDescriptorDIB: {
+      if (!aDeallocator->IsSameProcess()) {
+        NS_ERROR("A client process is trying to peek at our address space using a DIBTexture!");
+        return nullptr;
+      }
+
       result = new DIBTextureHost(aFlags, aDesc);
       break;
     }
diff --git a/gfx/layers/composite/X11TextureHost.cpp b/gfx/layers/composite/X11TextureHost.cpp
index 7ca42426d..92b6f8e91 100644
--- a/gfx/layers/composite/X11TextureHost.cpp
+++ b/gfx/layers/composite/X11TextureHost.cpp
@@ -22,10 +22,9 @@ X11TextureHost::X11TextureHost(TextureFlags aFlags,
                                const SurfaceDescriptorX11& aDescriptor)
  : TextureHost(aFlags)
 {
-  RefPtr<gfxXlibSurface> surface = aDescriptor.OpenForeign();
-  mSurface = surface.get();
+  mSurface = aDescriptor.OpenForeign();
 
-  if (!(aFlags & TextureFlags::DEALLOCATE_CLIENT)) {
+  if (mSurface && !(aFlags & TextureFlags::DEALLOCATE_CLIENT)) {
     mSurface->TakePixmap();
   }
 }
@@ -33,7 +32,7 @@ X11TextureHost::X11TextureHost(TextureFlags aFlags,
 bool
 X11TextureHost::Lock()
 {
-  if (!mCompositor) {
+  if (!mCompositor || !mSurface) {
     return false;
   }
 
@@ -69,6 +68,9 @@ X11TextureHost::SetCompositor(Compositor* aCompositor)
 SurfaceFormat
 X11TextureHost::GetFormat() const
 {
+  if (!mSurface) {
+    return SurfaceFormat::UNKNOWN;
+  }
   gfxContentType type = mSurface->GetContentType();
 #ifdef GL_PROVIDER_GLX
   if (mCompositor->GetBackendType() == LayersBackend::LAYERS_OPENGL) {
@@ -81,6 +83,9 @@ X11TextureHost::GetFormat() const
 IntSize
 X11TextureHost::GetSize() const
 {
+  if (!mSurface) {
+    return IntSize();
+  }
   return mSurface->GetSize();
 }
 
diff --git a/gfx/layers/d3d11/TextureD3D11.cpp b/gfx/layers/d3d11/TextureD3D11.cpp
index 8fbcfd234..954242585 100644
--- a/gfx/layers/d3d11/TextureD3D11.cpp
+++ b/gfx/layers/d3d11/TextureD3D11.cpp
@@ -632,10 +632,6 @@ CreateTextureHostD3D11(const SurfaceDescriptor& aDesc,
 {
   RefPtr<TextureHost> result;
   switch (aDesc.type()) {
-    case SurfaceDescriptor::TSurfaceDescriptorBuffer: {
-      result = CreateBackendIndependentTextureHost(aDesc, aDeallocator, aFlags);
-      break;
-    }
     case SurfaceDescriptor::TSurfaceDescriptorD3D10: {
       result = new DXGITextureHostD3D11(aFlags,
                                         aDesc.get_SurfaceDescriptorD3D10());
@@ -647,7 +643,7 @@ CreateTextureHostD3D11(const SurfaceDescriptor& aDesc,
       break;
     }
     default: {
-      NS_WARNING("Unsupported SurfaceDescriptor type");
+      MOZ_ASSERT_UNREACHABLE("Unsupported SurfaceDescriptor type");
     }
   }
   return result.forget();
diff --git a/gfx/layers/opengl/MacIOSurfaceTextureHostOGL.cpp b/gfx/layers/opengl/MacIOSurfaceTextureHostOGL.cpp
index 05f8cf38f..9736618f7 100644
--- a/gfx/layers/opengl/MacIOSurfaceTextureHostOGL.cpp
+++ b/gfx/layers/opengl/MacIOSurfaceTextureHostOGL.cpp
@@ -28,6 +28,8 @@ MacIOSurfaceTextureHostOGL::~MacIOSurfaceTextureHostOGL()
 GLTextureSource*
 MacIOSurfaceTextureHostOGL::CreateTextureSourceForPlane(size_t aPlane)
 {
+  MOZ_ASSERT(mSurface);
+
   GLuint textureHandle;
   gl::GLContext* gl = mCompositor->gl();
   gl->fGenTextures(1, &textureHandle);
@@ -84,11 +86,17 @@ MacIOSurfaceTextureHostOGL::SetCompositor(Compositor* aCompositor)
 
 gfx::SurfaceFormat
 MacIOSurfaceTextureHostOGL::GetFormat() const {
+  if (!mSurface) {
+    return gfx::SurfaceFormat::UNKNOWN;
+  }
   return mSurface->GetFormat();
 }
 
 gfx::SurfaceFormat
 MacIOSurfaceTextureHostOGL::GetReadFormat() const {
+  if (!mSurface) {
+    return gfx::SurfaceFormat::UNKNOWN;
+  }
   return mSurface->GetReadFormat();
 }
 
diff --git a/gfx/layers/opengl/TextureHostOGL.cpp b/gfx/layers/opengl/TextureHostOGL.cpp
index 854160bc6..02c398b51 100644
--- a/gfx/layers/opengl/TextureHostOGL.cpp
+++ b/gfx/layers/opengl/TextureHostOGL.cpp
@@ -25,10 +25,6 @@
 #include "mozilla/layers/MacIOSurfaceTextureHostOGL.h"
 #endif
 
-#ifdef GL_PROVIDER_GLX
-#include "mozilla/layers/X11TextureHost.h"
-#endif
-
 using namespace mozilla::gl;
 using namespace mozilla::gfx;
 
@@ -44,12 +40,6 @@ CreateTextureHostOGL(const SurfaceDescriptor& aDesc,
 {
   RefPtr<TextureHost> result;
   switch (aDesc.type()) {
-    case SurfaceDescriptor::TSurfaceDescriptorBuffer: {
-      result = CreateBackendIndependentTextureHost(aDesc,
-                                                   aDeallocator, aFlags);
-      break;
-    }
-
 #ifdef MOZ_WIDGET_ANDROID
     case SurfaceDescriptor::TSurfaceTextureDescriptor: {
       const SurfaceTextureDescriptor& desc = aDesc.get_SurfaceTextureDescriptor();
@@ -79,14 +69,6 @@ CreateTextureHostOGL(const SurfaceDescriptor& aDesc,
     }
 #endif
 
-#ifdef GL_PROVIDER_GLX
-    case SurfaceDescriptor::TSurfaceDescriptorX11: {
-      const auto& desc = aDesc.get_SurfaceDescriptorX11();
-      result = new X11TextureHost(aFlags, desc);
-      break;
-    }
-#endif
-
     case SurfaceDescriptor::TSurfaceDescriptorSharedGLTexture: {
       const auto& desc = aDesc.get_SurfaceDescriptorSharedGLTexture();
       result = new GLTextureHost(aFlags, desc.texture(),
@@ -96,7 +78,10 @@ CreateTextureHostOGL(const SurfaceDescriptor& aDesc,
                                  desc.hasAlpha());
       break;
     }
-    default: return nullptr;
+    default: {
+      MOZ_ASSERT_UNREACHABLE("Unsupported SurfaceDescriptor type");
+      break;
+    }
   }
   return result.forget();
 }
diff --git a/gfx/tests/gtest/TestLayers.h b/gfx/tests/gtest/TestLayers.h
index 18e351f77..fc6b750f1 100644
--- a/gfx/tests/gtest/TestLayers.h
+++ b/gfx/tests/gtest/TestLayers.h
@@ -8,6 +8,22 @@
 
 #include "Layers.h"
 #include "nsTArray.h"
+#include "mozilla/layers/ISurfaceAllocator.h"
+
+namespace mozilla {
+namespace layers {
+
+class TestSurfaceAllocator final : public ISurfaceAllocator
+{
+public:
+  TestSurfaceAllocator() {}
+  ~TestSurfaceAllocator() override {}
+
+  bool IsSameProcess() const override { return true; }
+};
+
+} // layers
+} // mozilla
 
 /* Create layer tree from a simple layer tree description syntax.
  * Each index is either the first letter of the layer type or
diff --git a/gfx/tests/gtest/TestTextures.cpp b/gfx/tests/gtest/TestTextures.cpp
index b4856ae6b..8f413cb3b 100644
--- a/gfx/tests/gtest/TestTextures.cpp
+++ b/gfx/tests/gtest/TestTextures.cpp
@@ -5,6 +5,7 @@
 
 #include "gtest/gtest.h"
 #include "gmock/gmock.h"
+#include "TestLayers.h"
 
 #include "mozilla/gfx/2D.h"
 #include "mozilla/gfx/Tools.h"
@@ -147,7 +148,8 @@ void TestTextureClientSurface(TextureClient* texture, gfxImageSurface* surface)
   ASSERT_NE(descriptor.type(), SurfaceDescriptor::Tnull_t);
 
   // host deserialization
-  RefPtr<TextureHost> host = CreateBackendIndependentTextureHost(descriptor, nullptr,
+  RefPtr<TestSurfaceAllocator> deallocator = new TestSurfaceAllocator();
+  RefPtr<TextureHost> host = CreateBackendIndependentTextureHost(descriptor, deallocator,
                                                                  texture->GetFlags());
 
   ASSERT_TRUE(host.get() != nullptr);
@@ -191,7 +193,8 @@ void TestTextureClientYCbCr(TextureClient* client, PlanarYCbCrData& ycbcrData) {
   ASSERT_EQ(ycbcrDesc.stereoMode(), ycbcrData.mStereoMode);
 
   // host deserialization
-  RefPtr<TextureHost> textureHost = CreateBackendIndependentTextureHost(descriptor, nullptr,
+  RefPtr<TestSurfaceAllocator> deallocator = new TestSurfaceAllocator();
+  RefPtr<TextureHost> textureHost = CreateBackendIndependentTextureHost(descriptor, deallocator,
                                                                         client->GetFlags());
 
   RefPtr<BufferTextureHost> host = static_cast<BufferTextureHost*>(textureHost.get());