/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim:set ts=2 sw=2 sts=2 et cindent: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "nsThreadUtils.h" #include "nsXPCOMCIDInternal.h" #include "PocketSphinxSpeechRecognitionService.h" #include "nsIFile.h" #include "SpeechGrammar.h" #include "SpeechRecognition.h" #include "SpeechRecognitionAlternative.h" #include "SpeechRecognitionResult.h" #include "SpeechRecognitionResultList.h" #include "nsIObserverService.h" #include "MediaPrefs.h" #include "mozilla/Services.h" #include "nsDirectoryServiceDefs.h" #include "nsDirectoryServiceUtils.h" #include "nsMemory.h" extern "C" { #include "pocketsphinx/pocketsphinx.h" #include "sphinxbase/logmath.h" #include "sphinxbase/sphinx_config.h" #include "sphinxbase/jsgf.h" } namespace mozilla { using namespace dom; class DecodeResultTask : public Runnable { public: DecodeResultTask(const nsString& hypstring, float64 confidence, WeakPtr recognition) : mResult(hypstring), mConfidence(confidence), mRecognition(recognition), mWorkerThread(do_GetCurrentThread()) { MOZ_ASSERT( !NS_IsMainThread()); // This should be running on the worker thread } NS_IMETHOD Run() override { MOZ_ASSERT(NS_IsMainThread()); // This method is supposed to run on the main // thread! // Declare javascript result events RefPtr event = new SpeechEvent( mRecognition, SpeechRecognition::EVENT_RECOGNITIONSERVICE_FINAL_RESULT); SpeechRecognitionResultList* resultList = new SpeechRecognitionResultList(mRecognition); SpeechRecognitionResult* result = new SpeechRecognitionResult(mRecognition); if (0 < mRecognition->MaxAlternatives()) { SpeechRecognitionAlternative* alternative = new SpeechRecognitionAlternative(mRecognition); alternative->mTranscript = mResult; alternative->mConfidence = mConfidence; result->mItems.AppendElement(alternative); } resultList->mItems.AppendElement(result); event->mRecognitionResultList = resultList; NS_DispatchToMainThread(event); // If we don't destroy the thread when we're done with it, it will hang // around forever... bad! // But thread->Shutdown must be called from the main thread, not from the // thread itself. return mWorkerThread->Shutdown(); } private: nsString mResult; float64 mConfidence; WeakPtr mRecognition; nsCOMPtr mWorkerThread; }; class DecodeTask : public Runnable { public: DecodeTask(WeakPtr recogntion, const nsTArray& audiovector, ps_decoder_t* ps) : mRecognition(recogntion), mAudiovector(audiovector), mPs(ps) { } NS_IMETHOD Run() override { char const* hyp; int rv; int32 final; int32 logprob; float64 confidence; nsAutoCString hypoValue; rv = ps_start_utt(mPs); rv = ps_process_raw(mPs, &mAudiovector[0], mAudiovector.Length(), FALSE, FALSE); rv = ps_end_utt(mPs); confidence = 0; if (rv >= 0) { hyp = ps_get_hyp_final(mPs, &final); if (hyp && final) { logprob = ps_get_prob(mPs); confidence = logmath_exp(ps_get_logmath(mPs), logprob); hypoValue.Assign(hyp); } } nsCOMPtr resultrunnable = new DecodeResultTask(NS_ConvertUTF8toUTF16(hypoValue), confidence, mRecognition); return NS_DispatchToMainThread(resultrunnable); } private: WeakPtr mRecognition; nsTArray mAudiovector; ps_decoder_t* mPs; }; NS_IMPL_ISUPPORTS(PocketSphinxSpeechRecognitionService, nsISpeechRecognitionService, nsIObserver) PocketSphinxSpeechRecognitionService::PocketSphinxSpeechRecognitionService() { mSpeexState = nullptr; // get root folder nsCOMPtr tmpFile; nsAutoString aStringAMPath; // am folder nsAutoString aStringDictPath; // dict folder NS_GetSpecialDirectory(NS_GRE_DIR, getter_AddRefs(tmpFile)); #if defined(XP_WIN) // for some reason, on windows NS_GRE_DIR is not bin root, // but bin/browser tmpFile->AppendRelativePath(NS_LITERAL_STRING("..")); #endif tmpFile->AppendRelativePath(NS_LITERAL_STRING("models")); tmpFile->AppendRelativePath(NS_LITERAL_STRING("en-US")); tmpFile->GetPath(aStringAMPath); NS_GetSpecialDirectory(NS_GRE_DIR, getter_AddRefs(tmpFile)); #if defined(XP_WIN) // for some reason, on windows NS_GRE_DIR is not bin root, // but bin/browser tmpFile->AppendRelativePath(NS_LITERAL_STRING("..")); #endif tmpFile->AppendRelativePath(NS_LITERAL_STRING("models")); // tmpFile->AppendRelativePath(NS_LITERAL_STRING("dict")); // tmpFile->AppendRelativePath(NS_LITERAL_STRING("en-US.dic")); // tmpFile->GetPath(aStringDictPath); // FOR B2G PATHS HARDCODED (APPEND /DATA ON THE BEGINING, FOR DESKTOP, ONLY // MODELS/ RELATIVE TO ROOT mPSConfig = cmd_ln_init(nullptr, ps_args(), TRUE, "-bestpath", "yes", "-hmm", ToNewUTF8String(aStringAMPath), // acoustic model "-dict", ToNewUTF8String(aStringDictPath), nullptr); if (mPSConfig == nullptr) { ISDecoderCreated = false; } else { mPSHandle = ps_init(mPSConfig); if (mPSHandle == nullptr) { ISDecoderCreated = false; } else { ISDecoderCreated = true; } } ISGrammarCompiled = false; } PocketSphinxSpeechRecognitionService::~PocketSphinxSpeechRecognitionService() { if (mPSConfig) { free(mPSConfig); } if (mPSHandle) { free(mPSHandle); } mSpeexState = nullptr; } // CALL START IN JS FALLS HERE NS_IMETHODIMP PocketSphinxSpeechRecognitionService::Initialize( WeakPtr aSpeechRecognition) { if (!ISDecoderCreated || !ISGrammarCompiled) { return NS_ERROR_NOT_INITIALIZED; } else { mAudioVector.Clear(); if (mSpeexState) { mSpeexState = nullptr; } mRecognition = aSpeechRecognition; nsCOMPtr obs = services::GetObserverService(); obs->AddObserver(this, SPEECH_RECOGNITION_TEST_EVENT_REQUEST_TOPIC, false); obs->AddObserver(this, SPEECH_RECOGNITION_TEST_END_TOPIC, false); return NS_OK; } } NS_IMETHODIMP PocketSphinxSpeechRecognitionService::ProcessAudioSegment( AudioSegment* aAudioSegment, int32_t aSampleRate) { if (!mSpeexState) { mSpeexState = speex_resampler_init(1, aSampleRate, 16000, SPEEX_RESAMPLER_QUALITY_MAX, nullptr); } aAudioSegment->ResampleChunks(mSpeexState, aSampleRate, 16000); AudioSegment::ChunkIterator iterator(*aAudioSegment); while (!iterator.IsEnded()) { mozilla::AudioChunk& chunk = *(iterator); MOZ_ASSERT(chunk.mBuffer); const int16_t* buf = static_cast(chunk.mChannelData[0]); for (int i = 0; i < iterator->mDuration; i++) { mAudioVector.AppendElement((int16_t)buf[i]); } iterator.Next(); } return NS_OK; } NS_IMETHODIMP PocketSphinxSpeechRecognitionService::SoundEnd() { speex_resampler_destroy(mSpeexState); mSpeexState = nullptr; // To create a new thread, get the thread manager nsCOMPtr tm = do_GetService(NS_THREADMANAGER_CONTRACTID); nsCOMPtr decodethread; nsresult rv = tm->NewThread(0, 0, getter_AddRefs(decodethread)); if (NS_FAILED(rv)) { // In case of failure, call back immediately with an empty string which // indicates failure return NS_OK; } nsCOMPtr r = new DecodeTask(mRecognition, mAudioVector, mPSHandle); decodethread->Dispatch(r, nsIEventTarget::DISPATCH_NORMAL); return NS_OK; } NS_IMETHODIMP PocketSphinxSpeechRecognitionService::ValidateAndSetGrammarList( SpeechGrammar* aSpeechGrammar, nsISpeechGrammarCompilationCallback* aCallback) { if (!ISDecoderCreated) { ISGrammarCompiled = false; } else if (aSpeechGrammar) { nsAutoString grammar; ErrorResult rv; aSpeechGrammar->GetSrc(grammar, rv); int result = ps_set_jsgf_string(mPSHandle, "name", NS_ConvertUTF16toUTF8(grammar).get()); if (result != 0) { ISGrammarCompiled = false; } else { ps_set_search(mPSHandle, "name"); ISGrammarCompiled = true; } } else { ISGrammarCompiled = false; } return ISGrammarCompiled ? NS_OK : NS_ERROR_NOT_INITIALIZED; } NS_IMETHODIMP PocketSphinxSpeechRecognitionService::Abort() { return NS_OK; } NS_IMETHODIMP PocketSphinxSpeechRecognitionService::Observe(nsISupports* aSubject, const char* aTopic, const char16_t* aData) { MOZ_ASSERT(MediaPrefs::WebSpeechFakeRecognitionService(), "Got request to fake recognition service event, " "but " TEST_PREFERENCE_FAKE_RECOGNITION_SERVICE " is not set"); if (!strcmp(aTopic, SPEECH_RECOGNITION_TEST_END_TOPIC)) { nsCOMPtr obs = services::GetObserverService(); obs->RemoveObserver(this, SPEECH_RECOGNITION_TEST_EVENT_REQUEST_TOPIC); obs->RemoveObserver(this, SPEECH_RECOGNITION_TEST_END_TOPIC); return NS_OK; } const nsDependentString eventName = nsDependentString(aData); if (eventName.EqualsLiteral("EVENT_RECOGNITIONSERVICE_ERROR")) { mRecognition->DispatchError( SpeechRecognition::EVENT_RECOGNITIONSERVICE_ERROR, SpeechRecognitionErrorCode::Network, // TODO different codes? NS_LITERAL_STRING("RECOGNITIONSERVICE_ERROR test event")); } else if (eventName.EqualsLiteral("EVENT_RECOGNITIONSERVICE_FINAL_RESULT")) { RefPtr event = new SpeechEvent( mRecognition, SpeechRecognition::EVENT_RECOGNITIONSERVICE_FINAL_RESULT); event->mRecognitionResultList = BuildMockResultList(); NS_DispatchToMainThread(event); } return NS_OK; } SpeechRecognitionResultList* PocketSphinxSpeechRecognitionService::BuildMockResultList() { SpeechRecognitionResultList* resultList = new SpeechRecognitionResultList(mRecognition); SpeechRecognitionResult* result = new SpeechRecognitionResult(mRecognition); if (0 < mRecognition->MaxAlternatives()) { SpeechRecognitionAlternative* alternative = new SpeechRecognitionAlternative(mRecognition); alternative->mTranscript = NS_LITERAL_STRING("Mock final result"); alternative->mConfidence = 0.0f; result->mItems.AppendElement(alternative); } resultList->mItems.AppendElement(result); return resultList; } } // namespace mozilla