diff options
Diffstat (limited to 'gfx/2d/BlurNEON.cpp')
| -rw-r--r-- | gfx/2d/BlurNEON.cpp | 288 |
1 files changed, 288 insertions, 0 deletions
diff --git a/gfx/2d/BlurNEON.cpp b/gfx/2d/BlurNEON.cpp new file mode 100644 index 000000000..978b3cdc0 --- /dev/null +++ b/gfx/2d/BlurNEON.cpp @@ -0,0 +1,288 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "Blur.h" +#include <arm_neon.h> + +namespace mozilla { +namespace gfx { + +MOZ_ALWAYS_INLINE +uint16x4_t Divide(uint32x4_t aValues, uint32x2_t aDivisor) +{ + uint64x2_t roundingAddition = vdupq_n_u64(int64_t(1) << 31); + uint64x2_t multiplied21 = vmull_u32(vget_low_u32(aValues), aDivisor); + uint64x2_t multiplied43 = vmull_u32(vget_high_u32(aValues), aDivisor); + return vqmovn_u32(vcombine_u32(vshrn_n_u64(vaddq_u64(multiplied21, roundingAddition), 32), + vshrn_n_u64(vaddq_u64(multiplied43, roundingAddition), 32))); +} + +MOZ_ALWAYS_INLINE +uint16x4_t BlurFourPixels(const uint32x4_t& aTopLeft, const uint32x4_t& aTopRight, + const uint32x4_t& aBottomRight, const uint32x4_t& aBottomLeft, + const uint32x2_t& aDivisor) +{ + uint32x4_t values = vaddq_u32(vsubq_u32(vsubq_u32(aBottomRight, aTopRight), aBottomLeft), aTopLeft); + return Divide(values, aDivisor); +} + +MOZ_ALWAYS_INLINE +void LoadIntegralRowFromRow(uint32_t *aDest, const uint8_t *aSource, + int32_t aSourceWidth, int32_t aLeftInflation, + int32_t aRightInflation) +{ + int32_t currentRowSum = 0; + + for (int x = 0; x < aLeftInflation; x++) { + currentRowSum += aSource[0]; + aDest[x] = currentRowSum; + } + for (int x = aLeftInflation; x < (aSourceWidth + aLeftInflation); x++) { + currentRowSum += aSource[(x - aLeftInflation)]; + aDest[x] = currentRowSum; + } + for (int x = (aSourceWidth + aLeftInflation); x < (aSourceWidth + aLeftInflation + aRightInflation); x++) { + currentRowSum += aSource[aSourceWidth - 1]; + aDest[x] = currentRowSum; + } +} + +MOZ_ALWAYS_INLINE void +GenerateIntegralImage_NEON(int32_t aLeftInflation, int32_t aRightInflation, + int32_t aTopInflation, int32_t aBottomInflation, + uint32_t *aIntegralImage, size_t aIntegralImageStride, + uint8_t *aSource, int32_t aSourceStride, const IntSize &aSize) +{ + MOZ_ASSERT(!(aLeftInflation & 3)); + + uint32_t stride32bit = aIntegralImageStride / 4; + IntSize integralImageSize(aSize.width + aLeftInflation + aRightInflation, + aSize.height + aTopInflation + aBottomInflation); + + LoadIntegralRowFromRow(aIntegralImage, aSource, aSize.width, aLeftInflation, aRightInflation); + + for (int y = 1; y < aTopInflation + 1; y++) { + uint32_t *intRow = aIntegralImage + (y * stride32bit); + uint32_t *intPrevRow = aIntegralImage + (y - 1) * stride32bit; + uint32_t *intFirstRow = aIntegralImage; + + for (int x = 0; x < integralImageSize.width; x += 4) { + uint32x4_t firstRow = vld1q_u32(intFirstRow + x); + uint32x4_t previousRow = vld1q_u32(intPrevRow + x); + vst1q_u32(intRow + x, vaddq_u32(firstRow, previousRow)); + } + } + + for (int y = aTopInflation + 1; y < (aSize.height + aTopInflation); y++) { + uint32x4_t currentRowSum = vdupq_n_u32(0); + uint32_t *intRow = aIntegralImage + (y * stride32bit); + uint32_t *intPrevRow = aIntegralImage + (y - 1) * stride32bit; + uint8_t *sourceRow = aSource + aSourceStride * (y - aTopInflation); + + uint32_t pixel = sourceRow[0]; + for (int x = 0; x < aLeftInflation; x += 4) { + uint32_t temp[4]; + temp[0] = pixel; + temp[1] = temp[0] + pixel; + temp[2] = temp[1] + pixel; + temp[3] = temp[2] + pixel; + uint32x4_t sumPixels = vld1q_u32(temp); + sumPixels = vaddq_u32(sumPixels, currentRowSum); + currentRowSum = vdupq_n_u32(vgetq_lane_u32(sumPixels, 3)); + vst1q_u32(intRow + x, vaddq_u32(sumPixels, vld1q_u32(intPrevRow + x))); + } + + for (int x = aLeftInflation; x < (aSize.width + aLeftInflation); x += 4) { + // It's important to shuffle here. When we exit this loop currentRowSum + // has to be set to sumPixels, so that the following loop can get the + // correct pixel for the currentRowSum. The highest order pixel in + // currentRowSum could've originated from accumulation in the stride. + currentRowSum = vdupq_n_u32(vgetq_lane_u32(currentRowSum, 3)); + + uint32_t temp[4]; + temp[0] = *(sourceRow + (x - aLeftInflation)); + temp[1] = temp[0] + *(sourceRow + (x - aLeftInflation) + 1); + temp[2] = temp[1] + *(sourceRow + (x - aLeftInflation) + 2); + temp[3] = temp[2] + *(sourceRow + (x - aLeftInflation) + 3); + uint32x4_t sumPixels = vld1q_u32(temp); + sumPixels = vaddq_u32(sumPixels, currentRowSum); + currentRowSum = sumPixels; + vst1q_u32(intRow + x, vaddq_u32(sumPixels, vld1q_u32(intPrevRow + x))); + } + + pixel = sourceRow[aSize.width - 1]; + int x = (aSize.width + aLeftInflation); + if ((aSize.width & 3)) { + // Deal with unaligned portion. Get the correct pixel from currentRowSum, + // see explanation above. + uint32_t intCurrentRowSum = ((uint32_t*)¤tRowSum)[(aSize.width % 4) - 1]; + for (; x < integralImageSize.width; x++) { + // We could be unaligned here! + if (!(x & 3)) { + // aligned! + currentRowSum = vdupq_n_u32(intCurrentRowSum); + break; + } + intCurrentRowSum += pixel; + intRow[x] = intPrevRow[x] + intCurrentRowSum; + } + } else { + currentRowSum = vdupq_n_u32(vgetq_lane_u32(currentRowSum, 3)); + } + + for (; x < integralImageSize.width; x += 4) { + uint32_t temp[4]; + temp[0] = pixel; + temp[1] = temp[0] + pixel; + temp[2] = temp[1] + pixel; + temp[3] = temp[2] + pixel; + uint32x4_t sumPixels = vld1q_u32(temp); + sumPixels = vaddq_u32(sumPixels, currentRowSum); + currentRowSum = vdupq_n_u32(vgetq_lane_u32(sumPixels, 3)); + vst1q_u32(intRow + x, vaddq_u32(sumPixels, vld1q_u32(intPrevRow + x))); + } + } + + if (aBottomInflation) { + // Store the last valid row of our source image in the last row of + // our integral image. This will be overwritten with the correct values + // in the upcoming loop. + LoadIntegralRowFromRow(aIntegralImage + (integralImageSize.height - 1) * stride32bit, + aSource + (aSize.height - 1) * aSourceStride, aSize.width, aLeftInflation, aRightInflation); + + for (int y = aSize.height + aTopInflation; y < integralImageSize.height; y++) { + uint32_t *intRow = aIntegralImage + (y * stride32bit); + uint32_t *intPrevRow = aIntegralImage + (y - 1) * stride32bit; + uint32_t *intLastRow = aIntegralImage + (integralImageSize.height - 1) * stride32bit; + for (int x = 0; x < integralImageSize.width; x += 4) { + vst1q_u32(intRow + x, + vaddq_u32(vld1q_u32(intLastRow + x), + vld1q_u32(intPrevRow + x))); + } + } + } +} + +/** + * Attempt to do an in-place box blur using an integral image. + */ +void +AlphaBoxBlur::BoxBlur_NEON(uint8_t* aData, + int32_t aLeftLobe, + int32_t aRightLobe, + int32_t aTopLobe, + int32_t aBottomLobe, + uint32_t *aIntegralImage, + size_t aIntegralImageStride) +{ + IntSize size = GetSize(); + + MOZ_ASSERT(size.height > 0); + + // Our 'left' or 'top' lobe will include the current pixel. i.e. when + // looking at an integral image the value of a pixel at 'x,y' is calculated + // using the value of the integral image values above/below that. + aLeftLobe++; + aTopLobe++; + int32_t boxSize = (aLeftLobe + aRightLobe) * (aTopLobe + aBottomLobe); + + MOZ_ASSERT(boxSize > 0); + + if (boxSize == 1) { + return; + } + + uint32_t reciprocal = uint32_t((uint64_t(1) << 32) / boxSize); + uint32_t stride32bit = aIntegralImageStride / 4; + int32_t leftInflation = RoundUpToMultipleOf4(aLeftLobe).value(); + + GenerateIntegralImage_NEON(leftInflation, aRightLobe, aTopLobe, aBottomLobe, + aIntegralImage, aIntegralImageStride, aData, + mStride, size); + + uint32x2_t divisor = vdup_n_u32(reciprocal); + + // This points to the start of the rectangle within the IntegralImage that overlaps + // the surface being blurred. + uint32_t *innerIntegral = aIntegralImage + (aTopLobe * stride32bit) + leftInflation; + IntRect skipRect = mSkipRect; + int32_t stride = mStride; + uint8_t *data = aData; + + for (int32_t y = 0; y < size.height; y++) { + bool inSkipRectY = y > skipRect.y && y < skipRect.YMost(); + uint32_t *topLeftBase = innerIntegral + ((y - aTopLobe) * ptrdiff_t(stride32bit) - aLeftLobe); + uint32_t *topRightBase = innerIntegral + ((y - aTopLobe) * ptrdiff_t(stride32bit) + aRightLobe); + uint32_t *bottomRightBase = innerIntegral + ((y + aBottomLobe) * ptrdiff_t(stride32bit) + aRightLobe); + uint32_t *bottomLeftBase = innerIntegral + ((y + aBottomLobe) * ptrdiff_t(stride32bit) - aLeftLobe); + + int32_t x = 0; + // Process 16 pixels at a time for as long as possible. + for (; x <= size.width - 16; x += 16) { + if (inSkipRectY && x > skipRect.x && x < skipRect.XMost()) { + x = skipRect.XMost() - 16; + // Trigger early jump on coming loop iterations, this will be reset + // next line anyway. + inSkipRectY = false; + continue; + } + + uint32x4_t topLeft; + uint32x4_t topRight; + uint32x4_t bottomRight; + uint32x4_t bottomLeft; + topLeft = vld1q_u32(topLeftBase + x); + topRight = vld1q_u32(topRightBase + x); + bottomRight = vld1q_u32(bottomRightBase + x); + bottomLeft = vld1q_u32(bottomLeftBase + x); + uint16x4_t result1 = BlurFourPixels(topLeft, topRight, bottomRight, bottomLeft, divisor); + + topLeft = vld1q_u32(topLeftBase + x + 4); + topRight = vld1q_u32(topRightBase + x + 4); + bottomRight = vld1q_u32(bottomRightBase + x + 4); + bottomLeft = vld1q_u32(bottomLeftBase + x + 4); + uint16x4_t result2 = BlurFourPixels(topLeft, topRight, bottomRight, bottomLeft, divisor); + + topLeft = vld1q_u32(topLeftBase + x + 8); + topRight = vld1q_u32(topRightBase + x + 8); + bottomRight = vld1q_u32(bottomRightBase + x + 8); + bottomLeft = vld1q_u32(bottomLeftBase + x + 8); + uint16x4_t result3 = BlurFourPixels(topLeft, topRight, bottomRight, bottomLeft, divisor); + + topLeft = vld1q_u32(topLeftBase + x + 12); + topRight = vld1q_u32(topRightBase + x + 12); + bottomRight = vld1q_u32(bottomRightBase + x + 12); + bottomLeft = vld1q_u32(bottomLeftBase + x + 12); + uint16x4_t result4 = BlurFourPixels(topLeft, topRight, bottomRight, bottomLeft, divisor); + + uint8x8_t combine1 = vqmovn_u16(vcombine_u16(result1, result2)); + uint8x8_t combine2 = vqmovn_u16(vcombine_u16(result3, result4)); + uint8x16_t final = vcombine_u8(combine1, combine2); + vst1q_u8(data + stride * y + x, final); + } + + // Process the remaining pixels 4 bytes at a time. + for (; x < size.width; x += 4) { + if (inSkipRectY && x > skipRect.x && x < skipRect.XMost()) { + x = skipRect.XMost() - 4; + // Trigger early jump on coming loop iterations, this will be reset + // next line anyway. + inSkipRectY = false; + continue; + } + + uint32x4_t topLeft = vld1q_u32(topLeftBase + x); + uint32x4_t topRight = vld1q_u32(topRightBase + x); + uint32x4_t bottomRight = vld1q_u32(bottomRightBase + x); + uint32x4_t bottomLeft = vld1q_u32(bottomLeftBase + x); + uint16x4_t result = BlurFourPixels(topLeft, topRight, bottomRight, bottomLeft, divisor); + uint32x2_t final = vreinterpret_u32_u8(vmovn_u16(vcombine_u16(result, vdup_n_u16(0)))); + *(uint32_t*)(data + stride * y + x) = vget_lane_u32(final, 0); + } + } +} + +} +} + |