summaryrefslogtreecommitdiffstats
path: root/toolkit/components/lz4
diff options
context:
space:
mode:
Diffstat (limited to 'toolkit/components/lz4')
-rw-r--r--toolkit/components/lz4/lz4.cpp73
-rw-r--r--toolkit/components/lz4/lz4.js156
-rw-r--r--toolkit/components/lz4/lz4_internal.js68
-rw-r--r--toolkit/components/lz4/moz.build18
-rw-r--r--toolkit/components/lz4/tests/xpcshell/.eslintrc.js7
-rw-r--r--toolkit/components/lz4/tests/xpcshell/data/chrome.manifest1
-rw-r--r--toolkit/components/lz4/tests/xpcshell/data/compression.lzbin0 -> 23 bytes
-rw-r--r--toolkit/components/lz4/tests/xpcshell/data/worker_lz4.js146
-rw-r--r--toolkit/components/lz4/tests/xpcshell/test_lz4.js43
-rw-r--r--toolkit/components/lz4/tests/xpcshell/test_lz4_sync.js41
-rw-r--r--toolkit/components/lz4/tests/xpcshell/xpcshell.ini11
11 files changed, 564 insertions, 0 deletions
diff --git a/toolkit/components/lz4/lz4.cpp b/toolkit/components/lz4/lz4.cpp
new file mode 100644
index 000000000..34d568025
--- /dev/null
+++ b/toolkit/components/lz4/lz4.cpp
@@ -0,0 +1,73 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/Compression.h"
+
+/**
+ * LZ4 is a very fast byte-wise compression algorithm.
+ *
+ * Compared to Google's Snappy it is faster to compress and decompress and
+ * generally produces output of about the same size.
+ *
+ * Compared to zlib it compresses at about 10x the speed, decompresses at about
+ * 4x the speed and produces output of about 1.5x the size.
+ *
+ */
+
+using namespace mozilla::Compression;
+
+/**
+ * Compresses 'inputSize' bytes from 'source' into 'dest'.
+ * Destination buffer must be already allocated,
+ * and must be sized to handle worst cases situations (input data not compressible)
+ * Worst case size evaluation is provided by function LZ4_compressBound()
+ *
+ * @param inputSize is the input size. Max supported value is ~1.9GB
+ * @param return the number of bytes written in buffer dest
+ */
+extern "C" MOZ_EXPORT size_t
+workerlz4_compress(const char* source, size_t inputSize, char* dest) {
+ return LZ4::compress(source, inputSize, dest);
+}
+
+/**
+ * If the source stream is malformed, the function will stop decoding
+ * and return a negative result, indicating the byte position of the
+ * faulty instruction
+ *
+ * This function never writes outside of provided buffers, and never
+ * modifies input buffer.
+ *
+ * note : destination buffer must be already allocated.
+ * its size must be a minimum of 'outputSize' bytes.
+ * @param outputSize is the output size, therefore the original size
+ * @return true/false
+ */
+extern "C" MOZ_EXPORT int
+workerlz4_decompress(const char* source, size_t inputSize,
+ char* dest, size_t maxOutputSize,
+ size_t *bytesOutput) {
+ return LZ4::decompress(source, inputSize,
+ dest, maxOutputSize,
+ bytesOutput);
+}
+
+
+/*
+ Provides the maximum size that LZ4 may output in a "worst case"
+ scenario (input data not compressible) primarily useful for memory
+ allocation of output buffer.
+ note : this function is limited by "int" range (2^31-1)
+
+ @param inputSize is the input size. Max supported value is ~1.9GB
+ @return maximum output size in a "worst case" scenario
+*/
+extern "C" MOZ_EXPORT size_t
+workerlz4_maxCompressedSize(size_t inputSize)
+{
+ return LZ4::maxCompressedSize(inputSize);
+}
+
+
+
diff --git a/toolkit/components/lz4/lz4.js b/toolkit/components/lz4/lz4.js
new file mode 100644
index 000000000..8d4ffcf8e
--- /dev/null
+++ b/toolkit/components/lz4/lz4.js
@@ -0,0 +1,156 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+var SharedAll;
+var Primitives;
+if (typeof Components != "undefined") {
+ let Cu = Components.utils;
+ SharedAll = {};
+ Cu.import("resource://gre/modules/osfile/osfile_shared_allthreads.jsm", SharedAll);
+ Cu.import("resource://gre/modules/lz4_internal.js");
+ Cu.import("resource://gre/modules/ctypes.jsm");
+
+ this.EXPORTED_SYMBOLS = [
+ "Lz4"
+ ];
+ this.exports = {};
+} else if (typeof module != "undefined" && typeof require != "undefined") {
+ SharedAll = require("resource://gre/modules/osfile/osfile_shared_allthreads.jsm");
+ Primitives = require("resource://gre/modules/lz4_internal.js");
+} else {
+ throw new Error("Please load this module with Component.utils.import or with require()");
+}
+
+const MAGIC_NUMBER = new Uint8Array([109, 111, 122, 76, 122, 52, 48, 0]); // "mozLz4a\0"
+
+const BYTES_IN_SIZE_HEADER = ctypes.uint32_t.size;
+
+const HEADER_SIZE = MAGIC_NUMBER.byteLength + BYTES_IN_SIZE_HEADER;
+
+const EXPECTED_HEADER_TYPE = new ctypes.ArrayType(ctypes.uint8_t, HEADER_SIZE);
+const EXPECTED_SIZE_BUFFER_TYPE = new ctypes.ArrayType(ctypes.uint8_t, BYTES_IN_SIZE_HEADER);
+
+/**
+ * An error during (de)compression
+ *
+ * @param {string} operation The name of the operation ("compress", "decompress")
+ * @param {string} reason A reason to be used when matching errors. Must start
+ * with "because", e.g. "becauseInvalidContent".
+ * @param {string} message A human-readable message.
+ */
+function LZError(operation, reason, message) {
+ SharedAll.OSError.call(this);
+ this.operation = operation;
+ this[reason] = true;
+ this.message = message;
+}
+LZError.prototype = Object.create(SharedAll.OSError);
+LZError.prototype.toString = function toString() {
+ return this.message;
+};
+exports.Error = LZError;
+
+/**
+ * Compress a block to a form suitable for writing to disk.
+ *
+ * Compatibility note: For the moment, we are basing our code on lz4
+ * 1.3, which does not specify a *file* format. Therefore, we define
+ * our own format. Once lz4 defines a complete file format, we will
+ * migrate both |compressFileContent| and |decompressFileContent| to this file
+ * format. For backwards-compatibility, |decompressFileContent| will however
+ * keep the ability to decompress files provided with older versions of
+ * |compressFileContent|.
+ *
+ * Compressed files have the following layout:
+ *
+ * | MAGIC_NUMBER (8 bytes) | content size (uint32_t, little endian) | content, as obtained from lz4_compress |
+ *
+ * @param {TypedArray|void*} buffer The buffer to write to the disk.
+ * @param {object=} options An object that may contain the following fields:
+ * - {number} bytes The number of bytes to read from |buffer|. If |buffer|
+ * is an |ArrayBuffer|, |bytes| defaults to |buffer.byteLength|. If
+ * |buffer| is a |void*|, |bytes| MUST be provided.
+ * @return {Uint8Array} An array of bytes suitable for being written to the
+ * disk.
+ */
+function compressFileContent(array, options = {}) {
+ // Prepare the output array
+ let inputBytes;
+ if (SharedAll.isTypedArray(array) && !(options && "bytes" in options)) {
+ inputBytes = array.byteLength;
+ } else if (options && options.bytes) {
+ inputBytes = options.bytes;
+ } else {
+ throw new TypeError("compressFileContent requires a size");
+ }
+ let maxCompressedSize = Primitives.maxCompressedSize(inputBytes);
+ let outputArray = new Uint8Array(HEADER_SIZE + maxCompressedSize);
+
+ // Compress to output array
+ let payload = new Uint8Array(outputArray.buffer, outputArray.byteOffset + HEADER_SIZE);
+ let compressedSize = Primitives.compress(array, inputBytes, payload);
+
+ // Add headers
+ outputArray.set(MAGIC_NUMBER);
+ let view = new DataView(outputArray.buffer);
+ view.setUint32(MAGIC_NUMBER.byteLength, inputBytes, true);
+
+ return new Uint8Array(outputArray.buffer, 0, HEADER_SIZE + compressedSize);
+}
+exports.compressFileContent = compressFileContent;
+
+function decompressFileContent(array, options = {}) {
+ let bytes = SharedAll.normalizeBufferArgs(array, options.bytes || null);
+ if (bytes < HEADER_SIZE) {
+ throw new LZError("decompress", "becauseLZNoHeader",
+ `Buffer is too short (no header) - Data: ${ options.path || array }`);
+ }
+
+ // Read headers
+ let expectMagicNumber = new DataView(array.buffer, 0, MAGIC_NUMBER.byteLength);
+ for (let i = 0; i < MAGIC_NUMBER.byteLength; ++i) {
+ if (expectMagicNumber.getUint8(i) != MAGIC_NUMBER[i]) {
+ throw new LZError("decompress", "becauseLZWrongMagicNumber",
+ `Invalid header (no magic number) - Data: ${ options.path || array }`);
+ }
+ }
+
+ let sizeBuf = new DataView(array.buffer, MAGIC_NUMBER.byteLength, BYTES_IN_SIZE_HEADER);
+ let expectDecompressedSize =
+ sizeBuf.getUint8(0) +
+ (sizeBuf.getUint8(1) << 8) +
+ (sizeBuf.getUint8(2) << 16) +
+ (sizeBuf.getUint8(3) << 24);
+ if (expectDecompressedSize == 0) {
+ // The underlying algorithm cannot handle a size of 0
+ return new Uint8Array(0);
+ }
+
+ // Prepare the input buffer
+ let inputData = new DataView(array.buffer, HEADER_SIZE);
+
+ // Prepare the output buffer
+ let outputBuffer = new Uint8Array(expectDecompressedSize);
+ let decompressedBytes = (new SharedAll.Type.size_t.implementation(0));
+
+ // Decompress
+ let success = Primitives.decompress(inputData, bytes - HEADER_SIZE,
+ outputBuffer, outputBuffer.byteLength,
+ decompressedBytes.address());
+ if (!success) {
+ throw new LZError("decompress", "becauseLZInvalidContent",
+ `Invalid content: Decompression stopped at ${decompressedBytes.value} - Data: ${ options.path || array }`);
+ }
+ return new Uint8Array(outputBuffer.buffer, outputBuffer.byteOffset, decompressedBytes.value);
+}
+exports.decompressFileContent = decompressFileContent;
+
+if (typeof Components != "undefined") {
+ this.Lz4 = {
+ compressFileContent: compressFileContent,
+ decompressFileContent: decompressFileContent
+ };
+}
diff --git a/toolkit/components/lz4/lz4_internal.js b/toolkit/components/lz4/lz4_internal.js
new file mode 100644
index 000000000..d1227da6c
--- /dev/null
+++ b/toolkit/components/lz4/lz4_internal.js
@@ -0,0 +1,68 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+var Primitives = {};
+
+var SharedAll;
+if (typeof Components != "undefined") {
+ let Cu = Components.utils;
+ SharedAll = {};
+ Cu.import("resource://gre/modules/osfile/osfile_shared_allthreads.jsm", SharedAll);
+
+ this.EXPORTED_SYMBOLS = [
+ "Primitives"
+ ];
+ this.Primitives = Primitives;
+ this.exports = {};
+} else if (typeof module != "undefined" && typeof require != "undefined") {
+ SharedAll = require("resource://gre/modules/osfile/osfile_shared_allthreads.jsm");
+} else {
+ throw new Error("Please load this module with Component.utils.import or with require()");
+}
+
+var libxul = new SharedAll.Library("libxul", SharedAll.Constants.Path.libxul);
+var Type = SharedAll.Type;
+
+libxul.declareLazyFFI(Primitives, "compress",
+ "workerlz4_compress",
+ null,
+ /* return*/ Type.size_t,
+ /* const source*/ Type.void_t.in_ptr,
+ /* inputSize*/ Type.size_t,
+ /* dest*/ Type.void_t.out_ptr
+);
+
+libxul.declareLazyFFI(Primitives, "decompress",
+ "workerlz4_decompress",
+ null,
+ /* return*/ Type.int,
+ /* const source*/ Type.void_t.in_ptr,
+ /* inputSize*/ Type.size_t,
+ /* dest*/ Type.void_t.out_ptr,
+ /* maxOutputSize*/ Type.size_t,
+ /* actualOutputSize*/ Type.size_t.out_ptr
+);
+
+libxul.declareLazyFFI(Primitives, "maxCompressedSize",
+ "workerlz4_maxCompressedSize",
+ null,
+ /* return*/ Type.size_t,
+ /* inputSize*/ Type.size_t
+);
+
+if (typeof module != "undefined") {
+ module.exports = {
+ get compress() {
+ return Primitives.compress;
+ },
+ get decompress() {
+ return Primitives.decompress;
+ },
+ get maxCompressedSize() {
+ return Primitives.maxCompressedSize;
+ }
+ };
+}
diff --git a/toolkit/components/lz4/moz.build b/toolkit/components/lz4/moz.build
new file mode 100644
index 000000000..a70185930
--- /dev/null
+++ b/toolkit/components/lz4/moz.build
@@ -0,0 +1,18 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+XPCSHELL_TESTS_MANIFESTS += ['tests/xpcshell/xpcshell.ini']
+
+EXTRA_JS_MODULES += [
+ 'lz4.js',
+ 'lz4_internal.js',
+]
+
+SOURCES += [
+ 'lz4.cpp',
+]
+
+FINAL_LIBRARY = 'xul'
diff --git a/toolkit/components/lz4/tests/xpcshell/.eslintrc.js b/toolkit/components/lz4/tests/xpcshell/.eslintrc.js
new file mode 100644
index 000000000..d35787cd2
--- /dev/null
+++ b/toolkit/components/lz4/tests/xpcshell/.eslintrc.js
@@ -0,0 +1,7 @@
+"use strict";
+
+module.exports = {
+ "extends": [
+ "../../../../../testing/xpcshell/xpcshell.eslintrc.js"
+ ]
+};
diff --git a/toolkit/components/lz4/tests/xpcshell/data/chrome.manifest b/toolkit/components/lz4/tests/xpcshell/data/chrome.manifest
new file mode 100644
index 000000000..e2f9a9d8e
--- /dev/null
+++ b/toolkit/components/lz4/tests/xpcshell/data/chrome.manifest
@@ -0,0 +1 @@
+content test_lz4 ./
diff --git a/toolkit/components/lz4/tests/xpcshell/data/compression.lz b/toolkit/components/lz4/tests/xpcshell/data/compression.lz
new file mode 100644
index 000000000..a354edc03
--- /dev/null
+++ b/toolkit/components/lz4/tests/xpcshell/data/compression.lz
Binary files differ
diff --git a/toolkit/components/lz4/tests/xpcshell/data/worker_lz4.js b/toolkit/components/lz4/tests/xpcshell/data/worker_lz4.js
new file mode 100644
index 000000000..47e3ea369
--- /dev/null
+++ b/toolkit/components/lz4/tests/xpcshell/data/worker_lz4.js
@@ -0,0 +1,146 @@
+importScripts("resource://gre/modules/workers/require.js");
+importScripts("resource://gre/modules/osfile.jsm");
+
+
+function do_print(x) {
+ // self.postMessage({kind: "do_print", args: [x]});
+ dump("TEST-INFO: " + x + "\n");
+}
+
+function do_check_true(x) {
+ self.postMessage({kind: "do_check_true", args: [!!x]});
+ if (x) {
+ dump("TEST-PASS: " + x + "\n");
+ } else {
+ throw new Error("do_check_true failed");
+ }
+}
+
+function do_check_eq(a, b) {
+ let result = a == b;
+ self.postMessage({kind: "do_check_true", args: [result]});
+ if (!result) {
+ throw new Error("do_check_eq failed " + a + " != " + b);
+ }
+}
+
+function do_test_complete() {
+ self.postMessage({kind: "do_test_complete", args:[]});
+}
+
+self.onmessage = function() {
+ try {
+ run_test();
+ } catch (ex) {
+ let {message, moduleStack, moduleName, lineNumber} = ex;
+ let error = new Error(message, moduleName, lineNumber);
+ error.stack = moduleStack;
+ dump("Uncaught error: " + error + "\n");
+ dump("Full stack: " + moduleStack + "\n");
+ throw error;
+ }
+};
+
+var Lz4;
+var Internals;
+function test_import() {
+ Lz4 = require("resource://gre/modules/lz4.js");
+ Internals = require("resource://gre/modules/lz4_internal.js");
+}
+
+function test_bound() {
+ for (let k of ["compress", "decompress", "maxCompressedSize"]) {
+ try {
+ do_print("Checking the existence of " + k + "\n");
+ do_check_true(!!Internals[k]);
+ do_print(k + " exists");
+ } catch (ex) {
+ // Ignore errors
+ do_print(k + " doesn't exist!");
+ }
+ }
+}
+
+function test_reference_file() {
+ do_print("Decompress reference file");
+ let path = OS.Path.join("data", "compression.lz");
+ let data = OS.File.read(path);
+ let decompressed = Lz4.decompressFileContent(data);
+ let text = (new TextDecoder()).decode(decompressed);
+ do_check_eq(text, "Hello, lz4");
+}
+
+function compare_arrays(a, b) {
+ return Array.prototype.join.call(a) == Array.prototype.join.call(a);
+}
+
+function run_rawcompression(name, array) {
+ do_print("Raw compression test " + name);
+ let length = array.byteLength;
+ let compressedArray = new Uint8Array(Internals.maxCompressedSize(length));
+ let compressedBytes = Internals.compress(array, length, compressedArray);
+ compressedArray = new Uint8Array(compressedArray.buffer, 0, compressedBytes);
+ do_print("Raw compressed: " + length + " into " + compressedBytes);
+
+ let decompressedArray = new Uint8Array(length);
+ let decompressedBytes = new ctypes.size_t();
+ let success = Internals.decompress(compressedArray, compressedBytes,
+ decompressedArray, length,
+ decompressedBytes.address());
+ do_print("Raw decompression success? " + success);
+ do_print("Raw decompression size: " + decompressedBytes.value);
+ do_check_true(compare_arrays(array, decompressedArray));
+}
+
+function run_filecompression(name, array) {
+ do_print("File compression test " + name);
+ let compressed = Lz4.compressFileContent(array);
+ do_print("Compressed " + array.byteLength + " bytes into " + compressed.byteLength);
+
+ let decompressed = Lz4.decompressFileContent(compressed);
+ do_print("Decompressed " + compressed.byteLength + " bytes into " + decompressed.byteLength);
+ do_check_true(compare_arrays(array, decompressed));
+}
+
+function run_faileddecompression(name, array) {
+ do_print("invalid decompression test " + name);
+
+ // Ensure that raw decompression doesn't segfault
+ let length = 1 << 14;
+ let decompressedArray = new Uint8Array(length);
+ let decompressedBytes = new ctypes.size_t();
+ Internals.decompress(array, array.byteLength,
+ decompressedArray, length,
+ decompressedBytes.address());
+
+ // File decompression should fail with an acceptable exception
+ let exn = null;
+ try {
+ Lz4.decompressFileContent(array);
+ } catch (ex) {
+ exn = ex;
+ }
+ do_check_true(exn);
+ if (array.byteLength < 10) {
+ do_check_true(exn.becauseLZNoHeader);
+ } else {
+ do_check_true(exn.becauseLZWrongMagicNumber);
+ }
+}
+
+function run_test() {
+ test_import();
+ test_bound();
+ test_reference_file();
+ for (let length of [0, 1, 1024]) {
+ let array = new Uint8Array(length);
+ for (let i = 0; i < length; ++i) {
+ array[i] = i % 256;
+ }
+ let name = length + " bytes";
+ run_rawcompression(name, array);
+ run_filecompression(name, array);
+ run_faileddecompression(name, array);
+ }
+ do_test_complete();
+}
diff --git a/toolkit/components/lz4/tests/xpcshell/test_lz4.js b/toolkit/components/lz4/tests/xpcshell/test_lz4.js
new file mode 100644
index 000000000..8a8fc0b21
--- /dev/null
+++ b/toolkit/components/lz4/tests/xpcshell/test_lz4.js
@@ -0,0 +1,43 @@
+/* Any copyright is dedicated to the Public Domain.
+ http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Components.utils.import("resource://gre/modules/Promise.jsm");
+
+var WORKER_SOURCE_URI = "chrome://test_lz4/content/worker_lz4.js";
+do_load_manifest("data/chrome.manifest");
+
+function run_test() {
+ run_next_test();
+}
+
+
+add_task(function() {
+ let worker = new ChromeWorker(WORKER_SOURCE_URI);
+ let deferred = Promise.defer();
+ worker.onmessage = function(event) {
+ let data = event.data;
+ switch (data.kind) {
+ case "do_check_true":
+ try {
+ do_check_true(data.args[0]);
+ } catch (ex) {
+ // Ignore errors
+ }
+ return;
+ case "do_test_complete":
+ deferred.resolve();
+ worker.terminate();
+ break;
+ case "do_print":
+ do_print(data.args[0]);
+ }
+ };
+ worker.onerror = function(event) {
+ let error = new Error(event.message, event.filename, event.lineno);
+ worker.terminate();
+ deferred.reject(error);
+ };
+ worker.postMessage("START");
+ return deferred.promise;
+});
+
diff --git a/toolkit/components/lz4/tests/xpcshell/test_lz4_sync.js b/toolkit/components/lz4/tests/xpcshell/test_lz4_sync.js
new file mode 100644
index 000000000..61605373b
--- /dev/null
+++ b/toolkit/components/lz4/tests/xpcshell/test_lz4_sync.js
@@ -0,0 +1,41 @@
+/* Any copyright is dedicated to the Public Domain.
+ http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const Cu = Components.utils;
+Cu.import("resource://gre/modules/lz4.js");
+Cu.import("resource://gre/modules/osfile.jsm");
+
+function run_test() {
+ run_next_test();
+}
+
+function compare_arrays(a, b) {
+ return Array.prototype.join.call(a) == Array.prototype.join.call(a);
+}
+
+add_task(function*() {
+ let path = OS.Path.join("data", "compression.lz");
+ let data = yield OS.File.read(path);
+ let decompressed = Lz4.decompressFileContent(data);
+ let text = (new TextDecoder()).decode(decompressed);
+ do_check_eq(text, "Hello, lz4");
+});
+
+add_task(function*() {
+ for (let length of [0, 1, 1024]) {
+ let array = new Uint8Array(length);
+ for (let i = 0; i < length; ++i) {
+ array[i] = i % 256;
+ }
+
+ let compressed = Lz4.compressFileContent(array);
+ do_print("Compressed " + array.byteLength + " bytes into " +
+ compressed.byteLength);
+
+ let decompressed = Lz4.decompressFileContent(compressed);
+ do_print("Decompressed " + compressed.byteLength + " bytes into " +
+ decompressed.byteLength);
+
+ do_check_true(compare_arrays(array, decompressed));
+ }
+});
diff --git a/toolkit/components/lz4/tests/xpcshell/xpcshell.ini b/toolkit/components/lz4/tests/xpcshell/xpcshell.ini
new file mode 100644
index 000000000..e457f29b2
--- /dev/null
+++ b/toolkit/components/lz4/tests/xpcshell/xpcshell.ini
@@ -0,0 +1,11 @@
+[DEFAULT]
+head =
+tail =
+skip-if = toolkit == 'android'
+support-files =
+ data/worker_lz4.js
+ data/chrome.manifest
+ data/compression.lz
+
+[test_lz4.js]
+[test_lz4_sync.js]