From 5f8de423f190bbb79a62f804151bc24824fa32d8 Mon Sep 17 00:00:00 2001 From: "Matt A. Tobin" Date: Fri, 2 Feb 2018 04:16:08 -0500 Subject: Add m-esr52 at 52.6.0 --- .../PocketSphinxSpeechRecognitionService.cpp | 357 +++++++++++++++++++++ 1 file changed, 357 insertions(+) create mode 100644 dom/media/webspeech/recognition/PocketSphinxSpeechRecognitionService.cpp (limited to 'dom/media/webspeech/recognition/PocketSphinxSpeechRecognitionService.cpp') diff --git a/dom/media/webspeech/recognition/PocketSphinxSpeechRecognitionService.cpp b/dom/media/webspeech/recognition/PocketSphinxSpeechRecognitionService.cpp new file mode 100644 index 000000000..b7aa5f2d7 --- /dev/null +++ b/dom/media/webspeech/recognition/PocketSphinxSpeechRecognitionService.cpp @@ -0,0 +1,357 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsThreadUtils.h" +#include "nsXPCOMCIDInternal.h" +#include "PocketSphinxSpeechRecognitionService.h" +#include "nsIFile.h" +#include "SpeechGrammar.h" +#include "SpeechRecognition.h" +#include "SpeechRecognitionAlternative.h" +#include "SpeechRecognitionResult.h" +#include "SpeechRecognitionResultList.h" +#include "nsIObserverService.h" +#include "MediaPrefs.h" +#include "mozilla/Services.h" +#include "nsDirectoryServiceDefs.h" +#include "nsDirectoryServiceUtils.h" +#include "nsMemory.h" + +extern "C" { +#include "pocketsphinx/pocketsphinx.h" +#include "sphinxbase/logmath.h" +#include "sphinxbase/sphinx_config.h" +#include "sphinxbase/jsgf.h" +} + +namespace mozilla { + +using namespace dom; + +class DecodeResultTask : public Runnable +{ +public: + DecodeResultTask(const nsString& hypstring, + float64 confidence, + WeakPtr recognition) + : mResult(hypstring), + mConfidence(confidence), + mRecognition(recognition), + mWorkerThread(do_GetCurrentThread()) + { + MOZ_ASSERT( + !NS_IsMainThread()); // This should be running on the worker thread + } + + NS_IMETHOD + Run() override + { + MOZ_ASSERT(NS_IsMainThread()); // This method is supposed to run on the main + // thread! + + // Declare javascript result events + RefPtr event = new SpeechEvent( + mRecognition, SpeechRecognition::EVENT_RECOGNITIONSERVICE_FINAL_RESULT); + SpeechRecognitionResultList* resultList = + new SpeechRecognitionResultList(mRecognition); + SpeechRecognitionResult* result = new SpeechRecognitionResult(mRecognition); + if (0 < mRecognition->MaxAlternatives()) { + SpeechRecognitionAlternative* alternative = + new SpeechRecognitionAlternative(mRecognition); + + alternative->mTranscript = mResult; + alternative->mConfidence = mConfidence; + + result->mItems.AppendElement(alternative); + } + resultList->mItems.AppendElement(result); + + event->mRecognitionResultList = resultList; + NS_DispatchToMainThread(event); + + // If we don't destroy the thread when we're done with it, it will hang + // around forever... bad! + // But thread->Shutdown must be called from the main thread, not from the + // thread itself. + return mWorkerThread->Shutdown(); + } + +private: + nsString mResult; + float64 mConfidence; + WeakPtr mRecognition; + nsCOMPtr mWorkerThread; +}; + +class DecodeTask : public Runnable +{ +public: + DecodeTask(WeakPtr recogntion, + const nsTArray& audiovector, ps_decoder_t* ps) + : mRecognition(recogntion), mAudiovector(audiovector), mPs(ps) + { + } + + NS_IMETHOD + Run() override + { + char const* hyp; + int rv; + int32 final; + int32 logprob; + float64 confidence; + nsAutoCString hypoValue; + + rv = ps_start_utt(mPs); + rv = ps_process_raw(mPs, &mAudiovector[0], mAudiovector.Length(), FALSE, + FALSE); + + rv = ps_end_utt(mPs); + confidence = 0; + if (rv >= 0) { + hyp = ps_get_hyp_final(mPs, &final); + if (hyp && final) { + logprob = ps_get_prob(mPs); + confidence = logmath_exp(ps_get_logmath(mPs), logprob); + hypoValue.Assign(hyp); + } + } + + nsCOMPtr resultrunnable = + new DecodeResultTask(NS_ConvertUTF8toUTF16(hypoValue), confidence, mRecognition); + return NS_DispatchToMainThread(resultrunnable); + } + +private: + WeakPtr mRecognition; + nsTArray mAudiovector; + ps_decoder_t* mPs; +}; + +NS_IMPL_ISUPPORTS(PocketSphinxSpeechRecognitionService, + nsISpeechRecognitionService, nsIObserver) + +PocketSphinxSpeechRecognitionService::PocketSphinxSpeechRecognitionService() +{ + mSpeexState = nullptr; + + // get root folder + nsCOMPtr tmpFile; + nsAutoString aStringAMPath; // am folder + nsAutoString aStringDictPath; // dict folder + + NS_GetSpecialDirectory(NS_GRE_DIR, getter_AddRefs(tmpFile)); +#if defined(XP_WIN) // for some reason, on windows NS_GRE_DIR is not bin root, + // but bin/browser + tmpFile->AppendRelativePath(NS_LITERAL_STRING("..")); +#endif + tmpFile->AppendRelativePath(NS_LITERAL_STRING("models")); + tmpFile->AppendRelativePath(NS_LITERAL_STRING("en-US")); + tmpFile->GetPath(aStringAMPath); + + NS_GetSpecialDirectory(NS_GRE_DIR, getter_AddRefs(tmpFile)); +#if defined(XP_WIN) // for some reason, on windows NS_GRE_DIR is not bin root, + // but bin/browser + tmpFile->AppendRelativePath(NS_LITERAL_STRING("..")); +#endif + tmpFile->AppendRelativePath(NS_LITERAL_STRING("models")); // + tmpFile->AppendRelativePath(NS_LITERAL_STRING("dict")); // + tmpFile->AppendRelativePath(NS_LITERAL_STRING("en-US.dic")); // + tmpFile->GetPath(aStringDictPath); + + // FOR B2G PATHS HARDCODED (APPEND /DATA ON THE BEGINING, FOR DESKTOP, ONLY + // MODELS/ RELATIVE TO ROOT + mPSConfig = cmd_ln_init(nullptr, ps_args(), TRUE, "-bestpath", "yes", "-hmm", + ToNewUTF8String(aStringAMPath), // acoustic model + "-dict", ToNewUTF8String(aStringDictPath), nullptr); + if (mPSConfig == nullptr) { + ISDecoderCreated = false; + } else { + mPSHandle = ps_init(mPSConfig); + if (mPSHandle == nullptr) { + ISDecoderCreated = false; + } else { + ISDecoderCreated = true; + } + } + + ISGrammarCompiled = false; +} + +PocketSphinxSpeechRecognitionService::~PocketSphinxSpeechRecognitionService() +{ + if (mPSConfig) { + free(mPSConfig); + } + if (mPSHandle) { + free(mPSHandle); + } + + mSpeexState = nullptr; +} + +// CALL START IN JS FALLS HERE +NS_IMETHODIMP +PocketSphinxSpeechRecognitionService::Initialize( + WeakPtr aSpeechRecognition) +{ + if (!ISDecoderCreated || !ISGrammarCompiled) { + return NS_ERROR_NOT_INITIALIZED; + } else { + mAudioVector.Clear(); + + if (mSpeexState) { + mSpeexState = nullptr; + } + + mRecognition = aSpeechRecognition; + nsCOMPtr obs = services::GetObserverService(); + obs->AddObserver(this, SPEECH_RECOGNITION_TEST_EVENT_REQUEST_TOPIC, false); + obs->AddObserver(this, SPEECH_RECOGNITION_TEST_END_TOPIC, false); + return NS_OK; + } +} + +NS_IMETHODIMP +PocketSphinxSpeechRecognitionService::ProcessAudioSegment( + AudioSegment* aAudioSegment, int32_t aSampleRate) +{ + if (!mSpeexState) { + mSpeexState = speex_resampler_init(1, aSampleRate, 16000, + SPEEX_RESAMPLER_QUALITY_MAX, nullptr); + } + aAudioSegment->ResampleChunks(mSpeexState, aSampleRate, 16000); + + AudioSegment::ChunkIterator iterator(*aAudioSegment); + + while (!iterator.IsEnded()) { + mozilla::AudioChunk& chunk = *(iterator); + MOZ_ASSERT(chunk.mBuffer); + const int16_t* buf = static_cast(chunk.mChannelData[0]); + + for (int i = 0; i < iterator->mDuration; i++) { + mAudioVector.AppendElement((int16_t)buf[i]); + } + iterator.Next(); + } + return NS_OK; +} + +NS_IMETHODIMP +PocketSphinxSpeechRecognitionService::SoundEnd() +{ + speex_resampler_destroy(mSpeexState); + mSpeexState = nullptr; + + // To create a new thread, get the thread manager + nsCOMPtr tm = do_GetService(NS_THREADMANAGER_CONTRACTID); + nsCOMPtr decodethread; + nsresult rv = tm->NewThread(0, 0, getter_AddRefs(decodethread)); + if (NS_FAILED(rv)) { + // In case of failure, call back immediately with an empty string which + // indicates failure + return NS_OK; + } + + nsCOMPtr r = + new DecodeTask(mRecognition, mAudioVector, mPSHandle); + decodethread->Dispatch(r, nsIEventTarget::DISPATCH_NORMAL); + + return NS_OK; +} + +NS_IMETHODIMP +PocketSphinxSpeechRecognitionService::ValidateAndSetGrammarList( + SpeechGrammar* aSpeechGrammar, + nsISpeechGrammarCompilationCallback* aCallback) +{ + if (!ISDecoderCreated) { + ISGrammarCompiled = false; + } else if (aSpeechGrammar) { + nsAutoString grammar; + ErrorResult rv; + aSpeechGrammar->GetSrc(grammar, rv); + + int result = ps_set_jsgf_string(mPSHandle, "name", + NS_ConvertUTF16toUTF8(grammar).get()); + + if (result != 0) { + ISGrammarCompiled = false; + } else { + ps_set_search(mPSHandle, "name"); + + ISGrammarCompiled = true; + } + } else { + ISGrammarCompiled = false; + } + + return ISGrammarCompiled ? NS_OK : NS_ERROR_NOT_INITIALIZED; +} + +NS_IMETHODIMP +PocketSphinxSpeechRecognitionService::Abort() +{ + return NS_OK; +} + +NS_IMETHODIMP +PocketSphinxSpeechRecognitionService::Observe(nsISupports* aSubject, + const char* aTopic, + const char16_t* aData) +{ + MOZ_ASSERT(MediaPrefs::WebSpeechFakeRecognitionService(), + "Got request to fake recognition service event, " + "but " TEST_PREFERENCE_FAKE_RECOGNITION_SERVICE " is not set"); + + if (!strcmp(aTopic, SPEECH_RECOGNITION_TEST_END_TOPIC)) { + nsCOMPtr obs = services::GetObserverService(); + obs->RemoveObserver(this, SPEECH_RECOGNITION_TEST_EVENT_REQUEST_TOPIC); + obs->RemoveObserver(this, SPEECH_RECOGNITION_TEST_END_TOPIC); + + return NS_OK; + } + + const nsDependentString eventName = nsDependentString(aData); + + if (eventName.EqualsLiteral("EVENT_RECOGNITIONSERVICE_ERROR")) { + mRecognition->DispatchError( + SpeechRecognition::EVENT_RECOGNITIONSERVICE_ERROR, + SpeechRecognitionErrorCode::Network, // TODO different codes? + NS_LITERAL_STRING("RECOGNITIONSERVICE_ERROR test event")); + + } else if (eventName.EqualsLiteral("EVENT_RECOGNITIONSERVICE_FINAL_RESULT")) { + RefPtr event = new SpeechEvent( + mRecognition, SpeechRecognition::EVENT_RECOGNITIONSERVICE_FINAL_RESULT); + + event->mRecognitionResultList = BuildMockResultList(); + NS_DispatchToMainThread(event); + } + + return NS_OK; +} + +SpeechRecognitionResultList* +PocketSphinxSpeechRecognitionService::BuildMockResultList() +{ + SpeechRecognitionResultList* resultList = + new SpeechRecognitionResultList(mRecognition); + SpeechRecognitionResult* result = new SpeechRecognitionResult(mRecognition); + if (0 < mRecognition->MaxAlternatives()) { + SpeechRecognitionAlternative* alternative = + new SpeechRecognitionAlternative(mRecognition); + + alternative->mTranscript = NS_LITERAL_STRING("Mock final result"); + alternative->mConfidence = 0.0f; + + result->mItems.AppendElement(alternative); + } + resultList->mItems.AppendElement(result); + + return resultList; +} + +} // namespace mozilla -- cgit v1.2.3