diff options
Diffstat (limited to 'dom/media/imagecapture')
-rw-r--r-- | dom/media/imagecapture/CaptureTask.cpp | 197 | ||||
-rw-r--r-- | dom/media/imagecapture/CaptureTask.h | 91 | ||||
-rw-r--r-- | dom/media/imagecapture/ImageCapture.cpp | 232 | ||||
-rw-r--r-- | dom/media/imagecapture/ImageCapture.h | 94 | ||||
-rw-r--r-- | dom/media/imagecapture/moz.build | 16 |
5 files changed, 630 insertions, 0 deletions
diff --git a/dom/media/imagecapture/CaptureTask.cpp b/dom/media/imagecapture/CaptureTask.cpp new file mode 100644 index 000000000..589ba5a42 --- /dev/null +++ b/dom/media/imagecapture/CaptureTask.cpp @@ -0,0 +1,197 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "CaptureTask.h" +#include "mozilla/dom/ImageCapture.h" +#include "mozilla/dom/ImageCaptureError.h" +#include "mozilla/dom/ImageEncoder.h" +#include "mozilla/dom/MediaStreamTrack.h" +#include "mozilla/dom/VideoStreamTrack.h" +#include "gfxUtils.h" +#include "nsThreadUtils.h" + +namespace mozilla { + +class CaptureTask::MediaStreamEventListener : public MediaStreamTrackListener +{ +public: + explicit MediaStreamEventListener(CaptureTask* aCaptureTask) + : mCaptureTask(aCaptureTask) {}; + + // MediaStreamListener methods. + void NotifyEnded() override + { + if(!mCaptureTask->mImageGrabbedOrTrackEnd) { + mCaptureTask->PostTrackEndEvent(); + } + } + +private: + CaptureTask* mCaptureTask; +}; + +CaptureTask::CaptureTask(dom::ImageCapture* aImageCapture) + : mImageCapture(aImageCapture) + , mEventListener(new MediaStreamEventListener(this)) + , mImageGrabbedOrTrackEnd(false) + , mPrincipalChanged(false) +{ +} + +nsresult +CaptureTask::TaskComplete(already_AddRefed<dom::Blob> aBlob, nsresult aRv) +{ + MOZ_ASSERT(NS_IsMainThread()); + + DetachTrack(); + + nsresult rv; + RefPtr<dom::Blob> blob(aBlob); + + // We have to set the parent because the blob has been generated with a valid one. + if (blob) { + blob = dom::Blob::Create(mImageCapture->GetParentObject(), blob->Impl()); + } + + if (mPrincipalChanged) { + aRv = NS_ERROR_DOM_SECURITY_ERR; + IC_LOG("MediaStream principal should not change during TakePhoto()."); + } + + if (NS_SUCCEEDED(aRv)) { + rv = mImageCapture->PostBlobEvent(blob); + } else { + rv = mImageCapture->PostErrorEvent(dom::ImageCaptureError::PHOTO_ERROR, aRv); + } + + // Ensure ImageCapture dereference on main thread here because the TakePhoto() + // sequences stopped here. + mImageCapture = nullptr; + + return rv; +} + +void +CaptureTask::AttachTrack() +{ + MOZ_ASSERT(NS_IsMainThread()); + + dom::VideoStreamTrack* track = mImageCapture->GetVideoStreamTrack(); + track->AddPrincipalChangeObserver(this); + track->AddListener(mEventListener.get()); + track->AddDirectListener(this); +} + +void +CaptureTask::DetachTrack() +{ + MOZ_ASSERT(NS_IsMainThread()); + + dom::VideoStreamTrack* track = mImageCapture->GetVideoStreamTrack(); + track->RemovePrincipalChangeObserver(this); + track->RemoveListener(mEventListener.get()); + track->RemoveDirectListener(this); +} + +void +CaptureTask::PrincipalChanged(dom::MediaStreamTrack* aMediaStreamTrack) +{ + MOZ_ASSERT(NS_IsMainThread()); + mPrincipalChanged = true; +} + +void +CaptureTask::SetCurrentFrames(const VideoSegment& aSegment) +{ + if (mImageGrabbedOrTrackEnd) { + return; + } + + // Callback for encoding complete, it calls on main thread. + class EncodeComplete : public dom::EncodeCompleteCallback + { + public: + explicit EncodeComplete(CaptureTask* aTask) : mTask(aTask) {} + + nsresult ReceiveBlob(already_AddRefed<dom::Blob> aBlob) override + { + RefPtr<dom::Blob> blob(aBlob); + mTask->TaskComplete(blob.forget(), NS_OK); + mTask = nullptr; + return NS_OK; + } + + protected: + RefPtr<CaptureTask> mTask; + }; + + VideoSegment::ConstChunkIterator iter(aSegment); + + + + while (!iter.IsEnded()) { + VideoChunk chunk = *iter; + + // Extract the first valid video frame. + VideoFrame frame; + if (!chunk.IsNull()) { + RefPtr<layers::Image> image; + if (chunk.mFrame.GetForceBlack()) { + // Create a black image. + image = VideoFrame::CreateBlackImage(chunk.mFrame.GetIntrinsicSize()); + } else { + image = chunk.mFrame.GetImage(); + } + MOZ_ASSERT(image); + mImageGrabbedOrTrackEnd = true; + + // Encode image. + nsresult rv; + nsAutoString type(NS_LITERAL_STRING("image/jpeg")); + nsAutoString options; + rv = dom::ImageEncoder::ExtractDataFromLayersImageAsync( + type, + options, + false, + image, + new EncodeComplete(this)); + if (NS_FAILED(rv)) { + PostTrackEndEvent(); + } + return; + } + iter.Next(); + } +} + +void +CaptureTask::PostTrackEndEvent() +{ + mImageGrabbedOrTrackEnd = true; + + // Got track end or finish event, stop the task. + class TrackEndRunnable : public Runnable + { + public: + explicit TrackEndRunnable(CaptureTask* aTask) + : mTask(aTask) {} + + NS_IMETHOD Run() override + { + mTask->TaskComplete(nullptr, NS_ERROR_FAILURE); + mTask = nullptr; + return NS_OK; + } + + protected: + RefPtr<CaptureTask> mTask; + }; + + IC_LOG("Got MediaStream track removed or finished event."); + NS_DispatchToMainThread(new TrackEndRunnable(this)); +} + +} // namespace mozilla diff --git a/dom/media/imagecapture/CaptureTask.h b/dom/media/imagecapture/CaptureTask.h new file mode 100644 index 000000000..2b4f0a04c --- /dev/null +++ b/dom/media/imagecapture/CaptureTask.h @@ -0,0 +1,91 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef CAPTURETASK_H +#define CAPTURETASK_H + +#include "MediaStreamGraph.h" +#include "MediaStreamListener.h" +#include "PrincipalChangeObserver.h" +#include "MediaStreamVideoSink.h" + +namespace mozilla { + +namespace dom { +class Blob; +class ImageCapture; +class MediaStreamTrack; +} // namespace dom + +/** + * CaptureTask retrieves image from MediaStream and encodes the image to jpeg in + * ImageEncoder. The whole procedures start at AttachTrack(), it will add this + * class into MediaStream and retrieves an image in MediaStreamGraph thread. + * Once the image is retrieved, it will be sent to ImageEncoder and the encoded + * blob will be sent out via encoder callback in main thread. + * + * CaptureTask holds a reference of ImageCapture to ensure ImageCapture won't be + * released during the period of the capturing process described above. + */ +class CaptureTask : public MediaStreamVideoSink, + public dom::PrincipalChangeObserver<dom::MediaStreamTrack> +{ +public: + class MediaStreamEventListener; + + // MediaStreamVideoSink methods. + void SetCurrentFrames(const VideoSegment& aSegment) override; + void ClearFrames() override {} + + // PrincipalChangeObserver<MediaStreamTrack> method. + void PrincipalChanged(dom::MediaStreamTrack* aMediaStreamTrack) override; + + // CaptureTask methods. + + // It is called when aBlob is ready to post back to script in company with + // aRv == NS_OK. If aRv is not NS_OK, it will post an error event to script. + // + // Note: + // this function should be called on main thread. + nsresult TaskComplete(already_AddRefed<dom::Blob> aBlob, nsresult aRv); + + // Add listeners into MediaStreamTrack and PrincipalChangeObserver. + // It should be on main thread only. + void AttachTrack(); + + // Remove listeners from MediaStreamTrack and PrincipalChangeObserver. + // It should be on main thread only. + void DetachTrack(); + + // CaptureTask should be created on main thread. + explicit CaptureTask(dom::ImageCapture* aImageCapture); + +protected: + virtual ~CaptureTask() {} + + // Post a runnable on main thread to end this task and call TaskComplete to post + // error event to script. It is called off-main-thread. + void PostTrackEndEvent(); + + // The ImageCapture associates with this task. This reference count should not + // change in this class unless it clears this reference after a blob or error + // event to script. + RefPtr<dom::ImageCapture> mImageCapture; + + RefPtr<MediaStreamEventListener> mEventListener; + + // True when an image is retrieved from MediaStreamGraph or MediaStreamGraph + // sends a track finish, end, or removed event. + bool mImageGrabbedOrTrackEnd; + + // True after MediaStreamTrack principal changes while waiting for a photo + // to finish and we should raise a security error. + bool mPrincipalChanged; +}; + +} // namespace mozilla + +#endif // CAPTURETASK_H diff --git a/dom/media/imagecapture/ImageCapture.cpp b/dom/media/imagecapture/ImageCapture.cpp new file mode 100644 index 000000000..243fcbcc8 --- /dev/null +++ b/dom/media/imagecapture/ImageCapture.cpp @@ -0,0 +1,232 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "ImageCapture.h" +#include "mozilla/dom/BlobEvent.h" +#include "mozilla/dom/DOMException.h" +#include "mozilla/dom/File.h" +#include "mozilla/dom/ImageCaptureError.h" +#include "mozilla/dom/ImageCaptureErrorEvent.h" +#include "mozilla/dom/ImageCaptureErrorEventBinding.h" +#include "mozilla/dom/VideoStreamTrack.h" +#include "nsIDocument.h" +#include "CaptureTask.h" +#include "MediaEngine.h" + +namespace mozilla { + +LogModule* GetICLog() +{ + static LazyLogModule log("ImageCapture"); + return log; +} + +namespace dom { + +NS_IMPL_CYCLE_COLLECTION_INHERITED(ImageCapture, DOMEventTargetHelper, + mVideoStreamTrack) + +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION_INHERITED(ImageCapture) +NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper) + +NS_IMPL_ADDREF_INHERITED(ImageCapture, DOMEventTargetHelper) +NS_IMPL_RELEASE_INHERITED(ImageCapture, DOMEventTargetHelper) + +ImageCapture::ImageCapture(VideoStreamTrack* aVideoStreamTrack, + nsPIDOMWindowInner* aOwnerWindow) + : DOMEventTargetHelper(aOwnerWindow) +{ + MOZ_ASSERT(aOwnerWindow); + MOZ_ASSERT(aVideoStreamTrack); + + mVideoStreamTrack = aVideoStreamTrack; +} + +ImageCapture::~ImageCapture() +{ + MOZ_ASSERT(NS_IsMainThread()); +} + +already_AddRefed<ImageCapture> +ImageCapture::Constructor(const GlobalObject& aGlobal, + VideoStreamTrack& aTrack, + ErrorResult& aRv) +{ + nsCOMPtr<nsPIDOMWindowInner> win = do_QueryInterface(aGlobal.GetAsSupports()); + if (!win) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + RefPtr<ImageCapture> object = new ImageCapture(&aTrack, win); + + return object.forget(); +} + +VideoStreamTrack* +ImageCapture::GetVideoStreamTrack() const +{ + return mVideoStreamTrack; +} + +nsresult +ImageCapture::TakePhotoByMediaEngine() +{ + // Callback for TakPhoto(), it also monitor the principal. If principal + // changes, it returns PHOTO_ERROR with security error. + class TakePhotoCallback : public MediaEnginePhotoCallback, + public PrincipalChangeObserver<MediaStreamTrack> + { + public: + TakePhotoCallback(VideoStreamTrack* aVideoTrack, ImageCapture* aImageCapture) + : mVideoTrack(aVideoTrack) + , mImageCapture(aImageCapture) + , mPrincipalChanged(false) + { + MOZ_ASSERT(NS_IsMainThread()); + mVideoTrack->AddPrincipalChangeObserver(this); + } + + void PrincipalChanged(MediaStreamTrack* aMediaStream) override + { + mPrincipalChanged = true; + } + + nsresult PhotoComplete(already_AddRefed<Blob> aBlob) override + { + RefPtr<Blob> blob = aBlob; + + if (mPrincipalChanged) { + return PhotoError(NS_ERROR_DOM_SECURITY_ERR); + } + return mImageCapture->PostBlobEvent(blob); + } + + nsresult PhotoError(nsresult aRv) override + { + return mImageCapture->PostErrorEvent(ImageCaptureError::PHOTO_ERROR, aRv); + } + + protected: + ~TakePhotoCallback() + { + MOZ_ASSERT(NS_IsMainThread()); + mVideoTrack->RemovePrincipalChangeObserver(this); + } + + RefPtr<VideoStreamTrack> mVideoTrack; + RefPtr<ImageCapture> mImageCapture; + bool mPrincipalChanged; + }; + + RefPtr<MediaEnginePhotoCallback> callback = + new TakePhotoCallback(mVideoStreamTrack, this); + return mVideoStreamTrack->GetSource().TakePhoto(callback); +} + +void +ImageCapture::TakePhoto(ErrorResult& aResult) +{ + // According to spec, VideoStreamTrack.readyState must be "live"; however + // gecko doesn't implement it yet (bug 910249). Instead of readyState, we + // check VideoStreamTrack.enable before bug 910249 is fixed. + // The error code should be INVALID_TRACK, but spec doesn't define it in + // ImageCaptureError. So it returns PHOTO_ERROR here before spec updates. + if (!mVideoStreamTrack->Enabled()) { + PostErrorEvent(ImageCaptureError::PHOTO_ERROR, NS_ERROR_FAILURE); + return; + } + + // Try if MediaEngine supports taking photo. + nsresult rv = TakePhotoByMediaEngine(); + + // It falls back to MediaStreamGraph image capture if MediaEngine doesn't + // support TakePhoto(). + if (rv == NS_ERROR_NOT_IMPLEMENTED) { + IC_LOG("MediaEngine doesn't support TakePhoto(), it falls back to MediaStreamGraph."); + RefPtr<CaptureTask> task = new CaptureTask(this); + + // It adds itself into MediaStreamGraph, so ImageCapture doesn't need to hold + // the reference. + task->AttachTrack(); + } +} + +nsresult +ImageCapture::PostBlobEvent(Blob* aBlob) +{ + MOZ_ASSERT(NS_IsMainThread()); + if (!CheckPrincipal()) { + // Media is not same-origin, don't allow the data out. + return PostErrorEvent(ImageCaptureError::PHOTO_ERROR, NS_ERROR_DOM_SECURITY_ERR); + } + + BlobEventInit init; + init.mBubbles = false; + init.mCancelable = false; + init.mData = aBlob; + + RefPtr<BlobEvent> blob_event = + BlobEvent::Constructor(this, NS_LITERAL_STRING("photo"), init); + + return DispatchTrustedEvent(blob_event); +} + +nsresult +ImageCapture::PostErrorEvent(uint16_t aErrorCode, nsresult aReason) +{ + MOZ_ASSERT(NS_IsMainThread()); + nsresult rv = CheckInnerWindowCorrectness(); + NS_ENSURE_SUCCESS(rv, rv); + + nsString errorMsg; + if (NS_FAILED(aReason)) { + nsCString name, message; + rv = NS_GetNameAndMessageForDOMNSResult(aReason, name, message); + if (NS_SUCCEEDED(rv)) { + CopyASCIItoUTF16(message, errorMsg); + } + } + + RefPtr<ImageCaptureError> error = + new ImageCaptureError(this, aErrorCode, errorMsg); + + ImageCaptureErrorEventInit init; + init.mBubbles = false; + init.mCancelable = false; + init.mImageCaptureError = error; + + nsCOMPtr<nsIDOMEvent> event = + ImageCaptureErrorEvent::Constructor(this, NS_LITERAL_STRING("error"), init); + + return DispatchTrustedEvent(event); +} + +bool +ImageCapture::CheckPrincipal() +{ + MOZ_ASSERT(NS_IsMainThread()); + + nsCOMPtr<nsIPrincipal> principal = mVideoStreamTrack->GetPrincipal(); + + if (!GetOwner()) { + return false; + } + nsCOMPtr<nsIDocument> doc = GetOwner()->GetExtantDoc(); + if (!doc || !principal) { + return false; + } + + bool subsumes; + if (NS_FAILED(doc->NodePrincipal()->Subsumes(principal, &subsumes))) { + return false; + } + + return subsumes; +} + +} // namespace dom +} // namespace mozilla diff --git a/dom/media/imagecapture/ImageCapture.h b/dom/media/imagecapture/ImageCapture.h new file mode 100644 index 000000000..f3256f59b --- /dev/null +++ b/dom/media/imagecapture/ImageCapture.h @@ -0,0 +1,94 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef IMAGECAPTURE_H +#define IMAGECAPTURE_H + +#include "mozilla/DOMEventTargetHelper.h" +#include "mozilla/dom/ImageCaptureBinding.h" +#include "mozilla/Logging.h" + +namespace mozilla { + +#ifndef IC_LOG +LogModule* GetICLog(); +#define IC_LOG(...) MOZ_LOG(GetICLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) +#endif + +namespace dom { + +class Blob; +class VideoStreamTrack; + +/** + * Implementation of https://dvcs.w3.org/hg/dap/raw-file/default/media-stream- + * capture/ImageCapture.html. + * The ImageCapture accepts a VideoStreamTrack as input source. The image will + * be sent back as a JPG format via Blob event. + * + * All the functions in ImageCapture are run in main thread. + * + * There are two ways to capture image, MediaEngineSource and MediaStreamGraph. + * When the implementation of MediaEngineSource supports TakePhoto(), + * it uses the platform camera to grab image. Otherwise, it falls back + * to the MediaStreamGraph way. + */ + +class ImageCapture final : public DOMEventTargetHelper +{ +public: + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(ImageCapture, DOMEventTargetHelper) + + IMPL_EVENT_HANDLER(photo) + IMPL_EVENT_HANDLER(error) + + // WebIDL members. + void TakePhoto(ErrorResult& aResult); + + // The MediaStream passed into the constructor. + VideoStreamTrack* GetVideoStreamTrack() const; + + // nsWrapperCache member + JSObject* WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto) override + { + return ImageCaptureBinding::Wrap(aCx, this, aGivenProto); + } + + // ImageCapture class members + nsPIDOMWindowInner* GetParentObject() { return GetOwner(); } + + static already_AddRefed<ImageCapture> Constructor(const GlobalObject& aGlobal, + VideoStreamTrack& aTrack, + ErrorResult& aRv); + + ImageCapture(VideoStreamTrack* aVideoStreamTrack, + nsPIDOMWindowInner* aOwnerWindow); + + // Post a Blob event to script. + nsresult PostBlobEvent(Blob* aBlob); + + // Post an error event to script. + // aErrorCode should be one of error codes defined in ImageCaptureError.h. + // aReason is the nsresult which maps to a error string in dom/base/domerr.msg. + nsresult PostErrorEvent(uint16_t aErrorCode, nsresult aReason = NS_OK); + + bool CheckPrincipal(); + +protected: + virtual ~ImageCapture(); + + // Capture image by MediaEngine. If it's not support taking photo, this function + // should return NS_ERROR_NOT_IMPLEMENTED. + nsresult TakePhotoByMediaEngine(); + + RefPtr<VideoStreamTrack> mVideoStreamTrack; +}; + +} // namespace dom +} // namespace mozilla + +#endif // IMAGECAPTURE_H diff --git a/dom/media/imagecapture/moz.build b/dom/media/imagecapture/moz.build new file mode 100644 index 000000000..f47649c5e --- /dev/null +++ b/dom/media/imagecapture/moz.build @@ -0,0 +1,16 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +EXPORTS.mozilla.dom += [ + 'ImageCapture.h' +] + +UNIFIED_SOURCES += [ + 'CaptureTask.cpp', + 'ImageCapture.cpp', +] + +FINAL_LIBRARY = 'xul' |