/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim: set ts=2 et sw=2 tw=80: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "nsUCSupport.h" #include "nsUTF8ToUnicode.h" #include "mozilla/CheckedInt.h" #include "mozilla/SSE.h" #include "nsCharTraits.h" #include <algorithm> #define UNICODE_BYTE_ORDER_MARK 0xFEFF static char16_t* EmitSurrogatePair(uint32_t ucs4, char16_t* aDest) { NS_ASSERTION(ucs4 > 0xFFFF, "Should be a supplementary character"); ucs4 -= 0x00010000; *aDest++ = 0xD800 | (0x000003FF & (ucs4 >> 10)); *aDest++ = 0xDC00 | (0x000003FF & ucs4); return aDest; } //---------------------------------------------------------------------- // Class nsUTF8ToUnicode [implementation] nsUTF8ToUnicode::nsUTF8ToUnicode() : nsBasicDecoderSupport() { Reset(); } //---------------------------------------------------------------------- // Subclassing of nsTableDecoderSupport class [implementation] /** * Normally the maximum length of the output of the UTF8 decoder in UTF16 * code units is the same as the length of the input in UTF8 code units, * since 1-byte, 2-byte and 3-byte UTF-8 sequences decode to a single * UTF-16 character, and 4-byte UTF-8 sequences decode to a surrogate pair. * * However, there is an edge case where the output can be longer than the * input: if the previous buffer ended with an incomplete multi-byte * sequence and this buffer does not begin with a valid continuation * byte, we will return NS_ERROR_ILLEGAL_INPUT and the caller may insert a * replacement character in the output buffer which corresponds to no * character in the input buffer. So in the worst case the destination * will need to be one code unit longer than the source. * See bug 301797. */ NS_IMETHODIMP nsUTF8ToUnicode::GetMaxLength(const char * aSrc, int32_t aSrcLength, int32_t * aDestLength) { mozilla::CheckedInt32 length = aSrcLength; length += 1; if (!length.isValid()) { return NS_ERROR_OUT_OF_MEMORY; } *aDestLength = length.value(); return NS_OK; } //---------------------------------------------------------------------- // Subclassing of nsBasicDecoderSupport class [implementation] NS_IMETHODIMP nsUTF8ToUnicode::Reset() { mUcs4 = 0; // cached Unicode character mState = 0; // cached expected number of octets after the current octet // until the beginning of the next UTF8 character sequence mBytes = 1; // cached expected number of octets in the current sequence mFirst = true; return NS_OK; } //---------------------------------------------------------------------- // Subclassing of nsBasicDecoderSupport class [implementation] // Fast ASCII -> UTF16 inner loop implementations // // Convert_ascii_run will update src and dst to the new values, and // len must be the maximum number ascii chars that it would be valid // to take from src and place into dst. (That is, the minimum of the // number of bytes left in src and the number of unichars available in // dst.) #if defined(__arm__) || defined(_M_ARM) // on ARM, do extra work to avoid byte/halfword reads/writes by // reading/writing a word at a time for as long as we can static inline void Convert_ascii_run (const char *&src, char16_t *&dst, int32_t len) { const uint32_t *src32; uint32_t *dst32; // with some alignments, we'd never actually break out of the slow loop, so // check and do the faster slow loop if ((((NS_PTR_TO_UINT32(dst) & 3) == 0) && ((NS_PTR_TO_UINT32(src) & 1) == 0)) || (((NS_PTR_TO_UINT32(dst) & 3) == 2) && ((NS_PTR_TO_UINT32(src) & 1) == 1))) { while (((NS_PTR_TO_UINT32(src) & 3) || (NS_PTR_TO_UINT32(dst) & 3)) && len > 0) { if (*src & 0x80U) return; *dst++ = (char16_t) *src++; len--; } } else { goto finish; } // then go 4 bytes at a time src32 = (const uint32_t*) src; dst32 = (uint32_t*) dst; while (len > 4) { uint32_t in = *src32++; if (in & 0x80808080U) { src32--; break; } *dst32++ = ((in & 0x000000ff) >> 0) | ((in & 0x0000ff00) << 8); *dst32++ = ((in & 0x00ff0000) >> 16) | ((in & 0xff000000) >> 8); len -= 4; } src = (const char *) src32; dst = (char16_t *) dst32; finish: while (len-- > 0 && (*src & 0x80U) == 0) { *dst++ = (char16_t) *src++; } } #else #ifdef MOZILLA_MAY_SUPPORT_SSE2 namespace mozilla { namespace SSE2 { void Convert_ascii_run(const char *&src, char16_t *&dst, int32_t len); } // namespace SSE2 } // namespace mozilla #endif static inline void Convert_ascii_run (const char *&src, char16_t *&dst, int32_t len) { #ifdef MOZILLA_MAY_SUPPORT_SSE2 if (mozilla::supports_sse2()) { mozilla::SSE2::Convert_ascii_run(src, dst, len); return; } #endif while (len-- > 0 && (*src & 0x80U) == 0) { *dst++ = (char16_t) *src++; } } #endif NS_IMETHODIMP nsUTF8ToUnicode::Convert(const char * aSrc, int32_t * aSrcLength, char16_t * aDest, int32_t * aDestLength) { uint32_t aSrcLen = (uint32_t) (*aSrcLength); uint32_t aDestLen = (uint32_t) (*aDestLength); const char *in, *inend; inend = aSrc + aSrcLen; char16_t *out, *outend; outend = aDest + aDestLen; nsresult res = NS_OK; // conversion result out = aDest; if (mState == 0xFF) { // Emit supplementary character left over from previous iteration. It is // caller's responsibility to keep a sufficient buffer. if (aDestLen < 2) { *aSrcLength = *aDestLength = 0; return NS_OK_UDEC_MOREOUTPUT; } out = EmitSurrogatePair(mUcs4, out); mUcs4 = 0; mState = 0; mBytes = 1; mFirst = false; } // alias these locally for speed int32_t mUcs4 = this->mUcs4; uint8_t mState = this->mState; uint8_t mBytes = this->mBytes; bool mFirst = this->mFirst; // Set mFirst to false now so we don't have to every time through the ASCII // branch within the loop. if (mFirst && aSrcLen && (0 == (0x80 & (*aSrc)))) mFirst = false; for (in = aSrc; ((in < inend) && (out < outend)); ++in) { uint8_t c = *in; if (0 == mState) { // When mState is zero we expect either a US-ASCII character or a // multi-octet sequence. if (c < 0x80) { // 00..7F int32_t max_loops = std::min(inend - in, outend - out); Convert_ascii_run(in, out, max_loops); --in; // match the rest of the cases mBytes = 1; } else if (c < 0xC2) { // C0/C1 // Overlong 2 octet sequence if (mErrBehavior == kOnError_Signal) { res = NS_ERROR_ILLEGAL_INPUT; break; } *out++ = UCS2_REPLACEMENT_CHAR; mFirst = false; } else if (c < 0xE0) { // C2..DF // First octet of 2 octet sequence mUcs4 = c; mUcs4 = (mUcs4 & 0x1F) << 6; mState = 1; mBytes = 2; } else if (c < 0xF0) { // E0..EF // First octet of 3 octet sequence mUcs4 = c; mUcs4 = (mUcs4 & 0x0F) << 12; mState = 2; mBytes = 3; } else if (c < 0xF5) { // F0..F4 // First octet of 4 octet sequence mUcs4 = c; mUcs4 = (mUcs4 & 0x07) << 18; mState = 3; mBytes = 4; } else { // F5..FF /* Current octet is neither in the US-ASCII range nor a legal first * octet of a multi-octet sequence. */ if (mErrBehavior == kOnError_Signal) { /* Return an error condition. Caller is responsible for flushing and * refilling the buffer and resetting state. */ res = NS_ERROR_ILLEGAL_INPUT; break; } *out++ = UCS2_REPLACEMENT_CHAR; mFirst = false; } } else { // When mState is non-zero, we expect a continuation of the multi-octet // sequence if (0x80 == (0xC0 & c)) { if (mState > 1) { // If we are here, all possibilities are: // mState == 2 && mBytes == 3 || // mState == 2 && mBytes == 4 || // mState == 3 && mBytes == 4 if ((mBytes == 3 && ((!mUcs4 && c < 0xA0) || // E0 80..9F (mUcs4 == 0xD000 && c > 0x9F))) || // ED A0..BF (mState == 3 && ((!mUcs4 && c < 0x90) || // F0 80..8F (mUcs4 == 0x100000 && c > 0x8F)))) {// F4 90..BF // illegal sequences or sequences converted into illegal ranges. in--; if (mErrBehavior == kOnError_Signal) { res = NS_ERROR_ILLEGAL_INPUT; break; } *out++ = UCS2_REPLACEMENT_CHAR; mState = 0; mFirst = false; continue; } } // Legal continuation. uint32_t shift = (mState - 1) * 6; uint32_t tmp = c; tmp = (tmp & 0x0000003FL) << shift; mUcs4 |= tmp; if (0 == --mState) { /* End of the multi-octet sequence. mUcs4 now contains the final * Unicode codepoint to be output */ if (mUcs4 > 0xFFFF) { // mUcs4 is in the range 0x10000 - 0x10FFFF. Output a UTF-16 pair if (out + 2 > outend) { // insufficient space left in the buffer. Keep mUcs4 for the // next iteration. mState = 0xFF; ++in; res = NS_OK_UDEC_MOREOUTPUT; break; } out = EmitSurrogatePair(mUcs4, out); } else if (UNICODE_BYTE_ORDER_MARK != mUcs4 || !mFirst) { // Don't output the BOM only if it is the first character *out++ = mUcs4; } //initialize UTF8 cache mUcs4 = 0; mState = 0; mBytes = 1; mFirst = false; } } else { /* ((0xC0 & c != 0x80) && (mState != 0)) * * Incomplete multi-octet sequence. Unconsume this * octet and return an error condition. Caller is responsible * for flushing and refilling the buffer and resetting state. */ in--; if (mErrBehavior == kOnError_Signal) { res = NS_ERROR_ILLEGAL_INPUT; break; } *out++ = UCS2_REPLACEMENT_CHAR; mState = 0; mFirst = false; } } } // output not finished, output buffer too short if ((NS_OK == res) && (in < inend) && (out >= outend)) res = NS_OK_UDEC_MOREOUTPUT; // last UCS4 is incomplete, make sure the caller // returns with properly aligned continuation of the buffer if ((NS_OK == res) && (mState != 0)) res = NS_OK_UDEC_MOREINPUT; *aSrcLength = in - aSrc; *aDestLength = out - aDest; this->mUcs4 = mUcs4; this->mState = mState; this->mBytes = mBytes; this->mFirst = mFirst; return(res); }