diff options
Diffstat (limited to 'mobile/android/base/java/org/mozilla/gecko/dlc')
11 files changed, 1982 insertions, 0 deletions
diff --git a/mobile/android/base/java/org/mozilla/gecko/dlc/BaseAction.java b/mobile/android/base/java/org/mozilla/gecko/dlc/BaseAction.java new file mode 100644 index 000000000..28d6b238d --- /dev/null +++ b/mobile/android/base/java/org/mozilla/gecko/dlc/BaseAction.java @@ -0,0 +1,166 @@ +/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.dlc; + +import android.content.Context; +import android.support.annotation.IntDef; +import android.util.Log; + +import org.mozilla.gecko.AppConstants; +import org.mozilla.gecko.background.nativecode.NativeCrypto; +import org.mozilla.gecko.dlc.catalog.DownloadContent; +import org.mozilla.gecko.dlc.catalog.DownloadContentCatalog; +import org.mozilla.gecko.sync.Utils; +import org.mozilla.gecko.util.HardwareUtils; +import org.mozilla.gecko.util.IOUtils; +import org.mozilla.gecko.util.ProxySelector; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; + +public abstract class BaseAction { + private static final String LOGTAG = "GeckoDLCBaseAction"; + + /** + * Exception indicating a recoverable error has happened. Download of the content will be retried later. + */ + /* package-private */ static class RecoverableDownloadContentException extends Exception { + private static final long serialVersionUID = -2246772819507370734L; + + @IntDef({MEMORY, DISK_IO, SERVER, NETWORK}) + public @interface ErrorType {} + public static final int MEMORY = 1; + public static final int DISK_IO = 2; + public static final int SERVER = 3; + public static final int NETWORK = 4; + + private int errorType; + + public RecoverableDownloadContentException(@ErrorType int errorType, String message) { + super(message); + this.errorType = errorType; + } + + public RecoverableDownloadContentException(@ErrorType int errorType, Throwable cause) { + super(cause); + this.errorType = errorType; + } + + @ErrorType + public int getErrorType() { + return errorType; + } + + /** + * Should this error be counted as failure? If this type of error will happen multiple times in a row then this + * error will be treated as permanently and the operation will not be tried again until the content changes. + */ + public boolean shouldBeCountedAsFailure() { + if (NETWORK == errorType) { + return false; // Always retry after network errors + } + + return true; + } + } + + /** + * If this exception is thrown the content will be marked as unrecoverable, permanently failed and we will not try + * downloading it again - until a newer version of the content is available. + */ + /* package-private */ static class UnrecoverableDownloadContentException extends Exception { + private static final long serialVersionUID = 8956080754787367105L; + + public UnrecoverableDownloadContentException(String message) { + super(message); + } + + public UnrecoverableDownloadContentException(Throwable cause) { + super(cause); + } + } + + public abstract void perform(Context context, DownloadContentCatalog catalog); + + protected File getDestinationFile(Context context, DownloadContent content) + throws UnrecoverableDownloadContentException, RecoverableDownloadContentException { + if (content.isFont()) { + File destinationDirectory = new File(context.getApplicationInfo().dataDir, "fonts"); + + if (!destinationDirectory.exists() && !destinationDirectory.mkdirs()) { + throw new RecoverableDownloadContentException(RecoverableDownloadContentException.DISK_IO, + "Destination directory does not exist and cannot be created"); + } + + return new File(destinationDirectory, content.getFilename()); + } + + // Unrecoverable: We downloaded a file and we don't know what to do with it (Should not happen) + throw new UnrecoverableDownloadContentException("Can't determine destination for kind: " + content.getKind()); + } + + protected boolean verify(File file, String expectedChecksum) + throws RecoverableDownloadContentException, UnrecoverableDownloadContentException { + InputStream inputStream = null; + + try { + inputStream = new BufferedInputStream(new FileInputStream(file)); + + byte[] ctx = NativeCrypto.sha256init(); + if (ctx == null) { + throw new RecoverableDownloadContentException(RecoverableDownloadContentException.MEMORY, + "Could not create SHA-256 context"); + } + + byte[] buffer = new byte[4096]; + int read; + + while ((read = inputStream.read(buffer)) != -1) { + NativeCrypto.sha256update(ctx, buffer, read); + } + + String actualChecksum = Utils.byte2Hex(NativeCrypto.sha256finalize(ctx)); + + if (!actualChecksum.equalsIgnoreCase(expectedChecksum)) { + Log.w(LOGTAG, "Checksums do not match. Expected=" + expectedChecksum + ", Actual=" + actualChecksum); + return false; + } + + return true; + } catch (IOException e) { + // Recoverable: Just I/O discontinuation + throw new RecoverableDownloadContentException(RecoverableDownloadContentException.DISK_IO, e); + } finally { + IOUtils.safeStreamClose(inputStream); + } + } + + protected HttpURLConnection buildHttpURLConnection(String url) + throws UnrecoverableDownloadContentException, IOException { + try { + System.setProperty("http.keepAlive", "true"); + + HttpURLConnection connection = (HttpURLConnection) ProxySelector.openConnectionWithProxy(new URI(url)); + connection.setRequestProperty("User-Agent", HardwareUtils.isTablet() ? + AppConstants.USER_AGENT_FENNEC_TABLET : + AppConstants.USER_AGENT_FENNEC_MOBILE); + connection.setRequestMethod("GET"); + connection.setInstanceFollowRedirects(true); + return connection; + } catch (MalformedURLException e) { + throw new UnrecoverableDownloadContentException(e); + } catch (URISyntaxException e) { + throw new UnrecoverableDownloadContentException(e); + } + } +} diff --git a/mobile/android/base/java/org/mozilla/gecko/dlc/CleanupAction.java b/mobile/android/base/java/org/mozilla/gecko/dlc/CleanupAction.java new file mode 100644 index 000000000..e44704c6c --- /dev/null +++ b/mobile/android/base/java/org/mozilla/gecko/dlc/CleanupAction.java @@ -0,0 +1,49 @@ +/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.dlc; + +import android.content.Context; + +import org.mozilla.gecko.dlc.catalog.DownloadContent; +import org.mozilla.gecko.dlc.catalog.DownloadContentCatalog; + +import java.io.File; + +/** + * CleanupAction: Remove content that is no longer needed. + */ +public class CleanupAction extends BaseAction { + @Override + public void perform(Context context, DownloadContentCatalog catalog) { + for (DownloadContent content : catalog.getContentToDelete()) { + if (!content.isAssetArchive()) { + continue; // We do not know how to clean up this content. But this means we didn't + // download it anyways. + } + + try { + File file = getDestinationFile(context, content); + + if (!file.exists()) { + // File does not exist. As good as deleting. + catalog.remove(content); + return; + } + + if (file.delete()) { + // File has been deleted. Now remove it from the catalog. + catalog.remove(content); + } + } catch (UnrecoverableDownloadContentException e) { + // We can't recover. Pretend the content is removed. It probably never existed in + // the first place. + catalog.remove(content); + } catch (RecoverableDownloadContentException e) { + // Try again next time. + } + } + } +} diff --git a/mobile/android/base/java/org/mozilla/gecko/dlc/DownloadAction.java b/mobile/android/base/java/org/mozilla/gecko/dlc/DownloadAction.java new file mode 100644 index 000000000..8618d4699 --- /dev/null +++ b/mobile/android/base/java/org/mozilla/gecko/dlc/DownloadAction.java @@ -0,0 +1,325 @@ +/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.dlc; + +import android.content.Context; +import android.net.ConnectivityManager; +import android.net.NetworkInfo; +import android.support.v4.net.ConnectivityManagerCompat; +import android.util.Log; + +import org.mozilla.gecko.AppConstants; +import org.mozilla.gecko.dlc.catalog.DownloadContent; +import org.mozilla.gecko.dlc.catalog.DownloadContentCatalog; +import org.mozilla.gecko.util.HardwareUtils; +import org.mozilla.gecko.util.IOUtils; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.zip.GZIPInputStream; + +/** + * Download content that has been scheduled during "study" or "verify". + */ +public class DownloadAction extends BaseAction { + private static final String LOGTAG = "DLCDownloadAction"; + + private static final String CACHE_DIRECTORY = "downloadContent"; + + private static final String CDN_BASE_URL = "https://fennec-catalog.cdn.mozilla.net/"; + + public interface Callback { + void onContentDownloaded(DownloadContent content); + } + + private Callback callback; + + public DownloadAction(Callback callback) { + this.callback = callback; + } + + public void perform(Context context, DownloadContentCatalog catalog) { + Log.d(LOGTAG, "Downloading content.."); + + if (!isConnectedToNetwork(context)) { + Log.d(LOGTAG, "No connected network available. Postponing download."); + // TODO: Reschedule download (bug 1209498) + return; + } + + if (isActiveNetworkMetered(context)) { + Log.d(LOGTAG, "Network is metered. Postponing download."); + // TODO: Reschedule download (bug 1209498) + return; + } + + for (DownloadContent content : catalog.getScheduledDownloads()) { + Log.d(LOGTAG, "Downloading: " + content); + + File temporaryFile = null; + + try { + File destinationFile = getDestinationFile(context, content); + if (destinationFile.exists() && verify(destinationFile, content.getChecksum())) { + Log.d(LOGTAG, "Content already exists and is up-to-date."); + catalog.markAsDownloaded(content); + continue; + } + + temporaryFile = createTemporaryFile(context, content); + + if (!hasEnoughDiskSpace(content, destinationFile, temporaryFile)) { + Log.d(LOGTAG, "Not enough disk space to save content. Skipping download."); + continue; + } + + // TODO: Check space on disk before downloading content (bug 1220145) + final String url = createDownloadURL(content); + + if (!temporaryFile.exists() || temporaryFile.length() < content.getSize()) { + download(url, temporaryFile); + } + + if (!verify(temporaryFile, content.getDownloadChecksum())) { + Log.w(LOGTAG, "Wrong checksum after download, content=" + content.getId()); + temporaryFile.delete(); + continue; + } + + if (!content.isAssetArchive()) { + Log.e(LOGTAG, "Downloaded content is not of type 'asset-archive': " + content.getType()); + temporaryFile.delete(); + continue; + } + + extract(temporaryFile, destinationFile, content.getChecksum()); + + catalog.markAsDownloaded(content); + + Log.d(LOGTAG, "Successfully downloaded: " + content); + + if (callback != null) { + callback.onContentDownloaded(content); + } + + if (temporaryFile != null && temporaryFile.exists()) { + temporaryFile.delete(); + } + } catch (RecoverableDownloadContentException e) { + Log.w(LOGTAG, "Downloading content failed (Recoverable): " + content, e); + + if (e.shouldBeCountedAsFailure()) { + catalog.rememberFailure(content, e.getErrorType()); + } + + // TODO: Reschedule download (bug 1209498) + } catch (UnrecoverableDownloadContentException e) { + Log.w(LOGTAG, "Downloading content failed (Unrecoverable): " + content, e); + + catalog.markAsPermanentlyFailed(content); + + if (temporaryFile != null && temporaryFile.exists()) { + temporaryFile.delete(); + } + } + } + + Log.v(LOGTAG, "Done"); + } + + protected void download(String source, File temporaryFile) + throws RecoverableDownloadContentException, UnrecoverableDownloadContentException { + InputStream inputStream = null; + OutputStream outputStream = null; + + HttpURLConnection connection = null; + + try { + connection = buildHttpURLConnection(source); + + final long offset = temporaryFile.exists() ? temporaryFile.length() : 0; + if (offset > 0) { + connection.setRequestProperty("Range", "bytes=" + offset + "-"); + } + + final int status = connection.getResponseCode(); + if (status != HttpURLConnection.HTTP_OK && status != HttpURLConnection.HTTP_PARTIAL) { + // We are trying to be smart and only retry if this is an error that might resolve in the future. + // TODO: This is guesstimating at best. We want to implement failure counters (Bug 1215106). + if (status >= 500) { + // Recoverable: Server errors 5xx + throw new RecoverableDownloadContentException(RecoverableDownloadContentException.SERVER, + "(Recoverable) Download failed. Status code: " + status); + } else if (status >= 400) { + // Unrecoverable: Client errors 4xx - Unlikely that this version of the client will ever succeed. + throw new UnrecoverableDownloadContentException("(Unrecoverable) Download failed. Status code: " + status); + } else { + // HttpsUrlConnection: -1 (No valid response code) + // Informational 1xx: They have no meaning to us. + // Successful 2xx: We don't know how to handle anything but 200. + // Redirection 3xx: HttpClient should have followed redirects if possible. We should not see those errors here. + throw new UnrecoverableDownloadContentException("(Unrecoverable) Download failed. Status code: " + status); + } + } + + inputStream = new BufferedInputStream(connection.getInputStream()); + outputStream = openFile(temporaryFile, status == HttpURLConnection.HTTP_PARTIAL); + + IOUtils.copy(inputStream, outputStream); + + inputStream.close(); + outputStream.close(); + } catch (IOException e) { + // Recoverable: Just I/O discontinuation + throw new RecoverableDownloadContentException(RecoverableDownloadContentException.NETWORK, e); + } finally { + IOUtils.safeStreamClose(inputStream); + IOUtils.safeStreamClose(outputStream); + + if (connection != null) { + connection.disconnect(); + } + } + } + + protected OutputStream openFile(File file, boolean append) throws FileNotFoundException { + return new BufferedOutputStream(new FileOutputStream(file, append)); + } + + protected void extract(File sourceFile, File destinationFile, String checksum) + throws UnrecoverableDownloadContentException, RecoverableDownloadContentException { + InputStream inputStream = null; + OutputStream outputStream = null; + File temporaryFile = null; + + try { + File destinationDirectory = destinationFile.getParentFile(); + if (!destinationDirectory.exists() && !destinationDirectory.mkdirs()) { + throw new IOException("Destination directory does not exist and cannot be created"); + } + + temporaryFile = new File(destinationDirectory, destinationFile.getName() + ".tmp"); + + inputStream = new GZIPInputStream(new BufferedInputStream(new FileInputStream(sourceFile))); + outputStream = new BufferedOutputStream(new FileOutputStream(temporaryFile)); + + IOUtils.copy(inputStream, outputStream); + + inputStream.close(); + outputStream.close(); + + if (!verify(temporaryFile, checksum)) { + Log.w(LOGTAG, "Checksum of extracted file does not match."); + return; + } + + move(temporaryFile, destinationFile); + } catch (IOException e) { + // We could not extract to the destination: Keep temporary file and try again next time we run. + throw new RecoverableDownloadContentException(RecoverableDownloadContentException.DISK_IO, e); + } finally { + IOUtils.safeStreamClose(inputStream); + IOUtils.safeStreamClose(outputStream); + + if (temporaryFile != null && temporaryFile.exists()) { + temporaryFile.delete(); + } + } + } + + protected boolean isConnectedToNetwork(Context context) { + ConnectivityManager manager = (ConnectivityManager) context.getSystemService(Context.CONNECTIVITY_SERVICE); + NetworkInfo networkInfo = manager.getActiveNetworkInfo(); + + return networkInfo != null && networkInfo.isConnected(); + } + + protected boolean isActiveNetworkMetered(Context context) { + return ConnectivityManagerCompat.isActiveNetworkMetered( + (ConnectivityManager) context.getSystemService(Context.CONNECTIVITY_SERVICE)); + } + + protected String createDownloadURL(DownloadContent content) { + final String location = content.getLocation(); + + return CDN_BASE_URL + content.getLocation(); + } + + protected File createTemporaryFile(Context context, DownloadContent content) + throws RecoverableDownloadContentException { + File cacheDirectory = new File(context.getCacheDir(), CACHE_DIRECTORY); + + if (!cacheDirectory.exists() && !cacheDirectory.mkdirs()) { + // Recoverable: File system might not be mounted NOW and we didn't download anything yet anyways. + throw new RecoverableDownloadContentException(RecoverableDownloadContentException.DISK_IO, + "Could not create cache directory: " + cacheDirectory); + } + + return new File(cacheDirectory, content.getDownloadChecksum() + "-" + content.getId()); + } + + protected void move(File temporaryFile, File destinationFile) + throws RecoverableDownloadContentException, UnrecoverableDownloadContentException { + if (!temporaryFile.renameTo(destinationFile)) { + Log.d(LOGTAG, "Could not move temporary file to destination. Trying to copy.."); + copy(temporaryFile, destinationFile); + temporaryFile.delete(); + } + } + + protected void copy(File temporaryFile, File destinationFile) + throws RecoverableDownloadContentException, UnrecoverableDownloadContentException { + InputStream inputStream = null; + OutputStream outputStream = null; + + try { + File destinationDirectory = destinationFile.getParentFile(); + if (!destinationDirectory.exists() && !destinationDirectory.mkdirs()) { + throw new IOException("Destination directory does not exist and cannot be created"); + } + + inputStream = new BufferedInputStream(new FileInputStream(temporaryFile)); + outputStream = new BufferedOutputStream(new FileOutputStream(destinationFile)); + + IOUtils.copy(inputStream, outputStream); + + inputStream.close(); + outputStream.close(); + } catch (IOException e) { + // We could not copy the temporary file to its destination: Keep the temporary file and + // try again the next time we run. + throw new RecoverableDownloadContentException(RecoverableDownloadContentException.DISK_IO, e); + } finally { + IOUtils.safeStreamClose(inputStream); + IOUtils.safeStreamClose(outputStream); + } + } + + protected boolean hasEnoughDiskSpace(DownloadContent content, File destinationFile, File temporaryFile) { + final File temporaryDirectory = temporaryFile.getParentFile(); + if (temporaryDirectory.getUsableSpace() < content.getSize()) { + return false; + } + + final File destinationDirectory = destinationFile.getParentFile(); + // We need some more space to extract the file (getSize() returns the uncompressed size) + if (destinationDirectory.getUsableSpace() < content.getSize() * 2) { + return false; + } + + return true; + } +} diff --git a/mobile/android/base/java/org/mozilla/gecko/dlc/DownloadContentService.java b/mobile/android/base/java/org/mozilla/gecko/dlc/DownloadContentService.java new file mode 100644 index 000000000..3729cf2e0 --- /dev/null +++ b/mobile/android/base/java/org/mozilla/gecko/dlc/DownloadContentService.java @@ -0,0 +1,144 @@ +/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.dlc; + +import org.mozilla.gecko.AppConstants; +import org.mozilla.gecko.GeckoAppShell; +import org.mozilla.gecko.dlc.catalog.DownloadContent; +import org.mozilla.gecko.dlc.catalog.DownloadContentCatalog; +import org.mozilla.gecko.util.HardwareUtils; + +import android.app.IntentService; +import android.content.ComponentName; +import android.content.Context; +import android.content.Intent; +import android.util.Log; + +/** + * Service to handle downloadable content that did not ship with the APK. + */ +public class DownloadContentService extends IntentService { + private static final String LOGTAG = "GeckoDLCService"; + + /** + * Study: Scan the catalog for "new" content available for download. + */ + private static final String ACTION_STUDY_CATALOG = AppConstants.ANDROID_PACKAGE_NAME + ".DLC.STUDY"; + + /** + * Verify: Validate downloaded content. Does it still exist and does it have the correct checksum? + */ + private static final String ACTION_VERIFY_CONTENT = AppConstants.ANDROID_PACKAGE_NAME + ".DLC.VERIFY"; + + /** + * Download content that has been scheduled during "study" or "verify". + */ + private static final String ACTION_DOWNLOAD_CONTENT = AppConstants.ANDROID_PACKAGE_NAME + ".DLC.DOWNLOAD"; + + /** + * Sync: Synchronize catalog from a Kinto instance. + */ + private static final String ACTION_SYNCHRONIZE_CATALOG = AppConstants.ANDROID_PACKAGE_NAME + ".DLC.SYNC"; + + /** + * CleanupAction: Remove content that is no longer needed (e.g. Removed from the catalog after a sync). + */ + private static final String ACTION_CLEANUP_FILES = AppConstants.ANDROID_PACKAGE_NAME + ".DLC.CLEANUP"; + + public static void startStudy(Context context) { + Intent intent = new Intent(ACTION_STUDY_CATALOG); + intent.setComponent(new ComponentName(context, DownloadContentService.class)); + context.startService(intent); + } + + public static void startVerification(Context context) { + Intent intent = new Intent(ACTION_VERIFY_CONTENT); + intent.setComponent(new ComponentName(context, DownloadContentService.class)); + context.startService(intent); + } + + public static void startDownloads(Context context) { + Intent intent = new Intent(ACTION_DOWNLOAD_CONTENT); + intent.setComponent(new ComponentName(context, DownloadContentService.class)); + context.startService(intent); + } + + public static void startSync(Context context) { + Intent intent = new Intent(ACTION_SYNCHRONIZE_CATALOG); + intent.setComponent(new ComponentName(context, DownloadContentService.class)); + context.startService(intent); + } + + public static void startCleanup(Context context) { + Intent intent = new Intent(ACTION_CLEANUP_FILES); + intent.setComponent(new ComponentName(context, DownloadContentService.class)); + context.startService(intent); + } + + private DownloadContentCatalog catalog; + + public DownloadContentService() { + super(LOGTAG); + } + + @Override + public void onCreate() { + super.onCreate(); + + catalog = new DownloadContentCatalog(this); + } + + protected void onHandleIntent(Intent intent) { + if (!AppConstants.MOZ_ANDROID_DOWNLOAD_CONTENT_SERVICE) { + Log.w(LOGTAG, "Download content is not enabled. Stop."); + return; + } + + if (!HardwareUtils.isSupportedSystem()) { + // This service is running very early before checks in BrowserApp can prevent us from running. + Log.w(LOGTAG, "System is not supported. Stop."); + return; + } + + if (intent == null) { + return; + } + + final BaseAction action; + + switch (intent.getAction()) { + case ACTION_STUDY_CATALOG: + action = new StudyAction(); + break; + + case ACTION_DOWNLOAD_CONTENT: + action = new DownloadAction(new DownloadAction.Callback() { + @Override + public void onContentDownloaded(DownloadContent content) { + if (content.isFont()) { + GeckoAppShell.notifyObservers("Fonts:Reload", ""); + } + } + }); + break; + + case ACTION_VERIFY_CONTENT: + action = new VerifyAction(); + break; + + case ACTION_SYNCHRONIZE_CATALOG: + action = new SyncAction(); + break; + + default: + Log.e(LOGTAG, "Unknown action: " + intent.getAction()); + return; + } + + action.perform(this, catalog); + catalog.persistChanges(); + } +} diff --git a/mobile/android/base/java/org/mozilla/gecko/dlc/StudyAction.java b/mobile/android/base/java/org/mozilla/gecko/dlc/StudyAction.java new file mode 100644 index 000000000..e15a17bbe --- /dev/null +++ b/mobile/android/base/java/org/mozilla/gecko/dlc/StudyAction.java @@ -0,0 +1,81 @@ +/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.dlc; + +import android.content.Context; +import android.os.Build; +import android.text.TextUtils; +import android.util.Log; + +import org.mozilla.gecko.dlc.catalog.DownloadContent; +import org.mozilla.gecko.dlc.catalog.DownloadContentCatalog; +import org.mozilla.gecko.util.ContextUtils; + +/** + * Study: Scan the catalog for "new" content available for download. + */ +public class StudyAction extends BaseAction { + private static final String LOGTAG = "DLCStudyAction"; + + public void perform(Context context, DownloadContentCatalog catalog) { + Log.d(LOGTAG, "Studying catalog.."); + + for (DownloadContent content : catalog.getContentToStudy()) { + if (!isMatching(context, content)) { + // This content is not for this particular version of the application or system + continue; + } + + if (content.isAssetArchive() && content.isFont()) { + catalog.scheduleDownload(content); + + Log.d(LOGTAG, "Scheduled download: " + content); + } + } + + if (catalog.hasScheduledDownloads()) { + startDownloads(context); + } + + Log.v(LOGTAG, "Done"); + } + + protected boolean isMatching(Context context, DownloadContent content) { + final String androidApiPattern = content.getAndroidApiPattern(); + if (!TextUtils.isEmpty(androidApiPattern)) { + final String apiVersion = String.valueOf(Build.VERSION.SDK_INT); + if (apiVersion.matches(androidApiPattern)) { + Log.d(LOGTAG, String.format("Android API (%s) does not match pattern: %s", apiVersion, androidApiPattern)); + return false; + } + } + + final String appIdPattern = content.getAppIdPattern(); + if (!TextUtils.isEmpty(appIdPattern)) { + final String appId = context.getPackageName(); + if (!appId.matches(appIdPattern)) { + Log.d(LOGTAG, String.format("App ID (%s) does not match pattern: %s", appId, appIdPattern)); + return false; + } + } + + final String appVersionPattern = content.getAppVersionPattern(); + if (!TextUtils.isEmpty(appVersionPattern)) { + final String appVersion = ContextUtils.getCurrentPackageInfo(context).versionName; + if (!appVersion.matches(appVersionPattern)) { + Log.d(LOGTAG, String.format("App version (%s) does not match pattern: %s", appVersion, appVersionPattern)); + return false; + } + } + + // There are no patterns or all patterns have matched. + return true; + } + + protected void startDownloads(Context context) { + DownloadContentService.startDownloads(context); + } +} diff --git a/mobile/android/base/java/org/mozilla/gecko/dlc/SyncAction.java b/mobile/android/base/java/org/mozilla/gecko/dlc/SyncAction.java new file mode 100644 index 000000000..104bdad18 --- /dev/null +++ b/mobile/android/base/java/org/mozilla/gecko/dlc/SyncAction.java @@ -0,0 +1,263 @@ +/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.dlc; + +import android.content.Context; +import android.net.Uri; +import android.util.Log; + +import com.keepsafe.switchboard.SwitchBoard; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.mozilla.gecko.dlc.catalog.DownloadContent; +import org.mozilla.gecko.dlc.catalog.DownloadContentBuilder; +import org.mozilla.gecko.dlc.catalog.DownloadContentCatalog; +import org.mozilla.gecko.Experiments; +import org.mozilla.gecko.util.IOUtils; + +import java.io.BufferedInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; + +/** + * Sync: Synchronize catalog from a Kinto instance. + */ +public class SyncAction extends BaseAction { + private static final String LOGTAG = "DLCSyncAction"; + + private static final String KINTO_KEY_ID = "id"; + private static final String KINTO_KEY_DELETED = "deleted"; + private static final String KINTO_KEY_DATA = "data"; + private static final String KINTO_KEY_ATTACHMENT = "attachment"; + private static final String KINTO_KEY_ORIGINAL = "original"; + + private static final String KINTO_PARAMETER_SINCE = "_since"; + private static final String KINTO_PARAMETER_FIELDS = "_fields"; + private static final String KINTO_PARAMETER_SORT = "_sort"; + + /** + * Kinto endpoint with online version of downloadable content catalog + * + * Dev instance: + * https://kinto-ota.dev.mozaws.net/v1/buckets/dlc/collections/catalog/records + */ + private static final String CATALOG_ENDPOINT = "https://firefox.settings.services.mozilla.com/v1/buckets/fennec/collections/catalog/records"; + + @Override + public void perform(Context context, DownloadContentCatalog catalog) { + Log.d(LOGTAG, "Synchronizing catalog."); + + if (!isSyncEnabledForClient(context)) { + Log.d(LOGTAG, "Sync is not enabled for client. Skipping."); + return; + } + + boolean cleanupRequired = false; + boolean studyRequired = false; + + try { + long lastModified = catalog.getLastModified(); + + // TODO: Consider using ETag here (Bug 1257459) + JSONArray rawCatalog = fetchRawCatalog(lastModified); + + Log.d(LOGTAG, "Server returned " + rawCatalog.length() + " records (since " + lastModified + ")"); + + for (int i = 0; i < rawCatalog.length(); i++) { + JSONObject object = rawCatalog.getJSONObject(i); + String id = object.getString(KINTO_KEY_ID); + + final boolean isDeleted = object.optBoolean(KINTO_KEY_DELETED, false); + + if (!isDeleted) { + JSONObject attachment = object.getJSONObject(KINTO_KEY_ATTACHMENT); + if (attachment.isNull(KINTO_KEY_ORIGINAL)) + throw new JSONException(String.format("Old Attachment Format")); + } + + DownloadContent existingContent = catalog.getContentById(id); + + if (isDeleted) { + cleanupRequired |= deleteContent(catalog, id); + } else if (existingContent != null) { + studyRequired |= updateContent(catalog, object, existingContent); + } else { + studyRequired |= createContent(catalog, object); + } + } + } catch (UnrecoverableDownloadContentException e) { + Log.e(LOGTAG, "UnrecoverableDownloadContentException", e); + } catch (RecoverableDownloadContentException e) { + Log.e(LOGTAG, "RecoverableDownloadContentException"); + } catch (JSONException e) { + Log.e(LOGTAG, "JSONException", e); + } + + if (studyRequired) { + startStudyAction(context); + } + + if (cleanupRequired) { + startCleanupAction(context); + } + + Log.v(LOGTAG, "Done"); + } + + protected void startStudyAction(Context context) { + DownloadContentService.startStudy(context); + } + + protected void startCleanupAction(Context context) { + DownloadContentService.startCleanup(context); + } + + protected JSONArray fetchRawCatalog(long lastModified) + throws RecoverableDownloadContentException, UnrecoverableDownloadContentException { + HttpURLConnection connection = null; + + try { + Uri.Builder builder = Uri.parse(CATALOG_ENDPOINT).buildUpon(); + + if (lastModified > 0) { + builder.appendQueryParameter(KINTO_PARAMETER_SINCE, String.valueOf(lastModified)); + } + // Only select the fields we are actually going to read. + builder.appendQueryParameter(KINTO_PARAMETER_FIELDS, + "attachment.location,attachment.original.filename,attachment.original.hash,attachment.hash,type,kind,attachment.original.size,match"); + + // We want to process items in the order they have been modified. This is to ensure that + // our last_modified values are correct if we processing is interrupted and not all items + // have been processed. + builder.appendQueryParameter(KINTO_PARAMETER_SORT, "last_modified"); + + connection = buildHttpURLConnection(builder.build().toString()); + + // TODO: Read 'Alert' header and EOL message if existing (Bug 1249248) + + // TODO: Read and use 'Backoff' header if available (Bug 1249251) + + // TODO: Add support for Next-Page header (Bug 1257495) + + final int responseCode = connection.getResponseCode(); + + if (responseCode != HttpURLConnection.HTTP_OK) { + if (responseCode >= 500) { + // A Retry-After header will be added to error responses (>=500), telling the + // client how many seconds it should wait before trying again. + + // TODO: Read and obey value in "Retry-After" header (Bug 1249249) + throw new RecoverableDownloadContentException(RecoverableDownloadContentException.SERVER, "Server error (" + responseCode + ")"); + } else if (responseCode == 410) { + // A 410 Gone error response can be returned if the client version is too old, + // or the service had been replaced with a new and better service using a new + // protocol version. + + // TODO: The server is gone. Stop synchronizing the catalog from this server (Bug 1249248). + throw new UnrecoverableDownloadContentException("Server is gone (410)"); + } else if (responseCode >= 400) { + // If the HTTP status is >=400 the response contains a JSON response. + logErrorResponse(connection); + + // Unrecoverable: Client errors 4xx - Unlikely that this version of the client will ever succeed. + throw new UnrecoverableDownloadContentException("(Unrecoverable) Catalog sync failed. Status code: " + responseCode); + } else if (responseCode < 200) { + // If the HTTP status is <200 the response contains a JSON response. + logErrorResponse(connection); + + throw new RecoverableDownloadContentException(RecoverableDownloadContentException.SERVER, "Response code: " + responseCode); + } else { + // HttpsUrlConnection: -1 (No valid response code) + // Successful 2xx: We don't know how to handle anything but 200. + // Redirection 3xx: We should have followed redirects if possible. We should not see those errors here. + + throw new UnrecoverableDownloadContentException("(Unrecoverable) Download failed. Response code: " + responseCode); + } + } + + return fetchJSONResponse(connection).getJSONArray(KINTO_KEY_DATA); + } catch (JSONException | IOException e) { + throw new RecoverableDownloadContentException(RecoverableDownloadContentException.NETWORK, e); + } finally { + if (connection != null) { + connection.disconnect(); + } + } + } + + private JSONObject fetchJSONResponse(HttpURLConnection connection) throws IOException, JSONException { + InputStream inputStream = null; + + try { + inputStream = new BufferedInputStream(connection.getInputStream()); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + IOUtils.copy(inputStream, outputStream); + return new JSONObject(outputStream.toString("UTF-8")); + } finally { + IOUtils.safeStreamClose(inputStream); + } + } + + protected boolean updateContent(DownloadContentCatalog catalog, JSONObject object, DownloadContent existingContent) + throws JSONException { + DownloadContent content = existingContent.buildUpon() + .updateFromKinto(object) + .build(); + + if (existingContent.getLastModified() >= content.getLastModified()) { + Log.d(LOGTAG, "Item has not changed: " + content); + return false; + } + + catalog.update(content); + + return true; + } + + protected boolean createContent(DownloadContentCatalog catalog, JSONObject object) throws JSONException { + DownloadContent content = new DownloadContentBuilder() + .updateFromKinto(object) + .build(); + + catalog.add(content); + + return true; + } + + protected boolean deleteContent(DownloadContentCatalog catalog, String id) { + DownloadContent content = catalog.getContentById(id); + if (content == null) { + return false; + } + + catalog.markAsDeleted(content); + + return true; + } + + protected boolean isSyncEnabledForClient(Context context) { + // Sync action is behind a switchboard flag for staged rollout. + return SwitchBoard.isInExperiment(context, Experiments.DOWNLOAD_CONTENT_CATALOG_SYNC); + } + + private void logErrorResponse(HttpURLConnection connection) { + try { + JSONObject error = fetchJSONResponse(connection); + + Log.w(LOGTAG, "Server returned error response:"); + Log.w(LOGTAG, "- Code: " + error.getInt("code")); + Log.w(LOGTAG, "- Errno: " + error.getInt("errno")); + Log.w(LOGTAG, "- Error: " + error.optString("error", "-")); + Log.w(LOGTAG, "- Message: " + error.optString("message", "-")); + Log.w(LOGTAG, "- Info: " + error.optString("info", "-")); + } catch (JSONException | IOException e) { + Log.w(LOGTAG, "Could not fetch error response", e); + } + } +} diff --git a/mobile/android/base/java/org/mozilla/gecko/dlc/VerifyAction.java b/mobile/android/base/java/org/mozilla/gecko/dlc/VerifyAction.java new file mode 100644 index 000000000..e96a62eae --- /dev/null +++ b/mobile/android/base/java/org/mozilla/gecko/dlc/VerifyAction.java @@ -0,0 +1,63 @@ +/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.dlc; + +import android.content.Context; +import android.util.Log; + +import org.mozilla.gecko.dlc.catalog.DownloadContent; +import org.mozilla.gecko.dlc.catalog.DownloadContentCatalog; + +import java.io.File; + +/** + * Verify: Validate downloaded content. Does it still exist and does it have the correct checksum? + */ +public class VerifyAction extends BaseAction { + private static final String LOGTAG = "DLCVerifyAction"; + + @Override + public void perform(Context context, DownloadContentCatalog catalog) { + Log.d(LOGTAG, "Verifying catalog.."); + + for (DownloadContent content : catalog.getDownloadedContent()) { + try { + File destinationFile = getDestinationFile(context, content); + + if (!destinationFile.exists()) { + Log.d(LOGTAG, "Downloaded content does not exist anymore: " + content); + + // This file does not exist even though it is marked as downloaded in the catalog. Scheduling a + // download to fetch it again. + catalog.scheduleDownload(content); + continue; + } + + if (!verify(destinationFile, content.getChecksum())) { + catalog.scheduleDownload(content); + Log.d(LOGTAG, "Wrong checksum. Scheduling download: " + content); + continue; + } + + Log.v(LOGTAG, "Content okay: " + content); + } catch (UnrecoverableDownloadContentException e) { + Log.w(LOGTAG, "Unrecoverable exception while verifying downloaded file", e); + } catch (RecoverableDownloadContentException e) { + // That's okay, we are just verifying already existing content. No log. + } + } + + if (catalog.hasScheduledDownloads()) { + startDownloads(context); + } + + Log.v(LOGTAG, "Done"); + } + + protected void startDownloads(Context context) { + DownloadContentService.startDownloads(context); + } +} diff --git a/mobile/android/base/java/org/mozilla/gecko/dlc/catalog/DownloadContent.java b/mobile/android/base/java/org/mozilla/gecko/dlc/catalog/DownloadContent.java new file mode 100644 index 000000000..61f7992ca --- /dev/null +++ b/mobile/android/base/java/org/mozilla/gecko/dlc/catalog/DownloadContent.java @@ -0,0 +1,189 @@ +/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.dlc.catalog; + +import android.support.annotation.IntDef; +import android.support.annotation.NonNull; +import android.support.annotation.Nullable; +import android.support.annotation.StringDef; + +public class DownloadContent { + @IntDef({STATE_NONE, STATE_SCHEDULED, STATE_DOWNLOADED, STATE_FAILED, STATE_UPDATED, STATE_DELETED}) + public @interface State {} + public static final int STATE_NONE = 0; + public static final int STATE_SCHEDULED = 1; + public static final int STATE_DOWNLOADED = 2; + public static final int STATE_FAILED = 3; // Permanently failed for this version of the content + public static final int STATE_UPDATED = 4; + public static final int STATE_DELETED = 5; + + @StringDef({TYPE_ASSET_ARCHIVE}) + public @interface Type {} + public static final String TYPE_ASSET_ARCHIVE = "asset-archive"; + + @StringDef({KIND_FONT, KIND_HYPHENATION_DICTIONARY}) + public @interface Kind {} + public static final String KIND_FONT = "font"; + public static final String KIND_HYPHENATION_DICTIONARY = "hyphenation"; + + private final String id; + private final String location; + private final String filename; + private final String checksum; + private final String downloadChecksum; + private final long lastModified; + private final String type; + private final String kind; + private final long size; + private final String appVersionPattern; + private final String androidApiPattern; + private final String appIdPattern; + private int state; + private int failures; + private int lastFailureType; + + /* package-private */ DownloadContent(@NonNull String id, @NonNull String location, @NonNull String filename, + @NonNull String checksum, @NonNull String downloadChecksum, @NonNull long lastModified, + @NonNull String type, @NonNull String kind, long size, int failures, int lastFailureType, + @Nullable String appVersionPattern, @Nullable String androidApiPattern, @Nullable String appIdPattern) { + this.id = id; + this.location = location; + this.filename = filename; + this.checksum = checksum; + this.downloadChecksum = downloadChecksum; + this.lastModified = lastModified; + this.type = type; + this.kind = kind; + this.size = size; + this.state = STATE_NONE; + this.failures = failures; + this.lastFailureType = lastFailureType; + this.appVersionPattern = appVersionPattern; + this.androidApiPattern = androidApiPattern; + this.appIdPattern = appIdPattern; + } + + public String getId() { + return id; + } + + /* package-private */ void setState(@State int state) { + this.state = state; + } + + @State + public int getState() { + return state; + } + + public boolean isStateIn(@State int... states) { + for (int state : states) { + if (this.state == state) { + return true; + } + } + + return false; + } + + @Kind + public String getKind() { + return kind; + } + + @Type + public String getType() { + return type; + } + + public String getLocation() { + return location; + } + + public long getLastModified() { + return lastModified; + } + + public String getFilename() { + return filename; + } + + public String getChecksum() { + return checksum; + } + + public String getDownloadChecksum() { + return downloadChecksum; + } + + public long getSize() { + return size; + } + + public boolean isFont() { + return KIND_FONT.equals(kind); + } + + public boolean isHyphenationDictionary() { + return KIND_HYPHENATION_DICTIONARY.equals(kind); + } + + /** + *Checks whether the content to be downloaded is a known content. + *Currently it checks whether the type is "Asset Archive" and is of kind + *"Font" or "Hyphenation Dictionary". + */ + public boolean isKnownContent() { + return ((isFont() || isHyphenationDictionary()) && isAssetArchive()); + } + + public boolean isAssetArchive() { + return TYPE_ASSET_ARCHIVE.equals(type); + } + + /* package-private */ int getFailures() { + return failures; + } + + /* package-private */ int getLastFailureType() { + return lastFailureType; + } + + /* package-private */ void rememberFailure(int failureType) { + if (lastFailureType != failureType) { + lastFailureType = failureType; + failures = 1; + } else { + failures++; + } + } + + /* package-private */ void resetFailures() { + failures = 0; + lastFailureType = 0; + } + + public String getAppVersionPattern() { + return appVersionPattern; + } + + public String getAndroidApiPattern() { + return androidApiPattern; + } + + public String getAppIdPattern() { + return appIdPattern; + } + + public DownloadContentBuilder buildUpon() { + return DownloadContentBuilder.buildUpon(this); + } + + + public String toString() { + return String.format("[%s,%s] %s (%d bytes) %s", getType(), getKind(), getId(), getSize(), getChecksum()); + } +} diff --git a/mobile/android/base/java/org/mozilla/gecko/dlc/catalog/DownloadContentBootstrap.java b/mobile/android/base/java/org/mozilla/gecko/dlc/catalog/DownloadContentBootstrap.java new file mode 100644 index 000000000..40c804573 --- /dev/null +++ b/mobile/android/base/java/org/mozilla/gecko/dlc/catalog/DownloadContentBootstrap.java @@ -0,0 +1,161 @@ +/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.dlc.catalog; + +import android.support.v4.util.ArrayMap; + +import org.mozilla.gecko.AppConstants; + +import java.util.Arrays; +import java.util.List; + +/* package-private */ class DownloadContentBootstrap { + public static ArrayMap<String, DownloadContent> createInitialDownloadContentList() { + if (!AppConstants.MOZ_ANDROID_EXCLUDE_FONTS) { + // We are packaging fonts. There's nothing we want to download; + return new ArrayMap<>(); + } + + List<DownloadContent> initialList = Arrays.asList( + new DownloadContentBuilder() + .setId("c40929cf-7f4c-fa72-3dc9-12cadf56905d") + .setLocation("fennec/catalog/f63e5f92-793c-4574-a2d7-fbc50242b8cb.gz") + .setFilename("CharisSILCompact-B.ttf") + .setChecksum("699d958b492eda0cc2823535f8567d0393090e3842f6df3c36dbe7239cb80b6d") + .setDownloadChecksum("a9f9b34fed353169a88cc159b8f298cb285cce0b8b0f979c22a7d85de46f0532") + .setSize(1676072) + .setKind("font") + .setType("asset-archive") + .build(), + + new DownloadContentBuilder() + .setId("6d265876-85ed-0917-fdc8-baf583ca2cba") + .setLocation("fennec/catalog/19af6c88-09d9-4d6c-805e-cfebb8699a6c.gz") + .setFilename("CharisSILCompact-BI.ttf") + .setChecksum("82465e747b4f41471dbfd942842b2ee810749217d44b55dbc43623b89f9c7d9b") + .setDownloadChecksum("2be26671039a5e2e4d0360a948b4fa42048171133076a3bb6173d93d4b9cd55b") + .setSize(1667812) + .setKind("font") + .setType("asset-archive") + .build(), + + new DownloadContentBuilder() + .setId("8460dc6d-d129-fd1a-24b6-343dbf6531dd") + .setLocation("fennec/catalog/f35a384a-90ea-41c6-a957-bb1845de97eb.gz") + .setFilename("CharisSILCompact-I.ttf") + .setChecksum("ab3ed6f2a4d4c2095b78227bd33155d7ccd05a879c107a291912640d4d64f767") + .setDownloadChecksum("38a6469041c02624d43dfd41d2dd745e3e3211655e616188f65789a90952a1e9") + .setSize(1693988) + .setKind("font") + .setType("asset-archive") + .build(), + + new DownloadContentBuilder() + .setId("c906275c-3747-fe27-426f-6187526a6f06") + .setLocation("fennec/catalog/8c3bec92-d2df-4789-8c4a-0f523f026d96.gz") + .setFilename("CharisSILCompact-R.ttf") + .setChecksum("4ed509317f1bb441b185ea13bf1c9d19d1a0b396962efa3b5dc3190ad88f2067") + .setDownloadChecksum("7c2ec1f550c2005b75383b878f737266b5f0b1c82679dd886c8bbe30c82e340e") + .setSize(1727656) + .setKind("font") + .setType("asset-archive") + .build(), + + new DownloadContentBuilder() + .setId("ff5deecc-6ecc-d816-bb51-65face460119") + .setLocation("fennec/catalog/ea115d71-e2ac-4609-853e-c978780776b1.gz") + .setFilename("ClearSans-Bold.ttf") + .setChecksum("385d0a293c1714770e198f7c762ab32f7905a0ed9d2993f69d640bd7232b4b70") + .setDownloadChecksum("0d3c22bef90e7096f75b331bb7391de3aa43017e10d61041cd3085816db4919a") + .setSize(140136) + .setKind("font") + .setType("asset-archive") + .build(), + + new DownloadContentBuilder() + .setId("a173d1db-373b-ce42-1335-6b3285cfdebd") + .setLocation("fennec/catalog/0838e513-2d99-4e53-b58f-6b970f6548c6.gz") + .setFilename("ClearSans-BoldItalic.ttf") + .setChecksum("7bce66864e38eecd7c94b6657b9b38c35ebfacf8046bfb1247e08f07fe933198") + .setDownloadChecksum("de0903164dde1ad3768d0bd6dec949871d6ab7be08f573d9d70f38c138a22e37") + .setSize(156124) + .setKind("font") + .setType("asset-archive") + .build(), + + new DownloadContentBuilder() + .setId("e65c66df-0088-940d-ca5c-207c22118c0e") + .setLocation("fennec/catalog/7550fa42-0947-478c-a5f0-5ea1bbb6ba27.gz") + .setFilename("ClearSans-Italic.ttf") + .setChecksum("87c13c5fbae832e4f85c3bd46fcbc175978234f39be5fe51c4937be4cbff3b68") + .setDownloadChecksum("6e323db3115005dd0e96d2422db87a520f9ae426de28a342cd6cc87b55601d87") + .setSize(155672) + .setKind("font") + .setType("asset-archive") + .build(), + + new DownloadContentBuilder() + .setId("25610abb-5dc8-fd75-40e7-990507f010c4") + .setLocation("fennec/catalog/dd9bee7d-d784-476b-a3dd-69af8e516487.gz") + .setFilename("ClearSans-Light.ttf") + .setChecksum("e4885f6188e7a8587f5621c077c6c1f5e8d3739dffc8f4d055c2ba87568c750a") + .setDownloadChecksum("19d4f7c67176e9e254c61420da9c7363d9fe5e6b4bb9d61afa4b3b574280714f") + .setSize(145976) + .setKind("font") + .setType("asset-archive") + .build(), + + new DownloadContentBuilder() + .setId("ffe40339-a096-2262-c3f8-54af75c81fe6") + .setLocation("fennec/catalog/bc5ada8c-8cfc-443d-93d7-dc5f98138a07.gz") + .setFilename("ClearSans-Medium.ttf") + .setChecksum("5d0e0115f3a3ed4be3eda6d7eabb899bb9a361292802e763d53c72e00f629da1") + .setDownloadChecksum("edec86dab3ad2a97561cb41b584670262a48bed008c57bb587ee05ca47fb067f") + .setSize(148892) + .setKind("font") + .setType("asset-archive") + .build(), + + new DownloadContentBuilder() + .setId("139a94be-ac69-0264-c9cc-8f2d071fd29d") + .setLocation("fennec/catalog/0490c768-6178-49c2-af88-9f8769ff3167.gz") + .setFilename("ClearSans-MediumItalic.ttf") + .setChecksum("937dda88b26469306258527d38e42c95e27e7ebb9f05bd1d7c5d706a3c9541d7") + .setDownloadChecksum("34edbd1b325dbffe7791fba8dd2d19852eb3c2fe00cff517ea2161ddc424ee22") + .setSize(155228) + .setKind("font") + .setType("asset-archive") + .build(), + + new DownloadContentBuilder() + .setId("b887012a-01e1-7c94-fdcb-ca44d5b974a2") + .setLocation("fennec/catalog/78205bf8-c668-41b1-b68f-afd54f98713b.gz") + .setFilename("ClearSans-Regular.ttf") + .setChecksum("9b91bbdb95ffa6663da24fdaa8ee06060cd0a4d2dceaf1ffbdda00e04915ee5b") + .setDownloadChecksum("a72f1420b4da1ba9e6797adac34f08e72f94128a85e56542d5e6a8080af5f08a") + .setSize(142572) + .setKind("font") + .setType("asset-archive") + .build(), + + new DownloadContentBuilder() + .setId("c8703652-d317-0356-0bf8-95441a5b2c9b") + .setLocation("fennec/catalog/3570f44f-9440-4aa0-abd0-642eaf2a1aa0.gz") + .setFilename("ClearSans-Thin.ttf") + .setChecksum("07b0db85a3ad99afeb803f0f35631436a7b4c67ac66d0c7f77d26a47357c592a") + .setDownloadChecksum("d9f23fd8687d6743f5c281c33539fb16f163304795039959b8caf159e6d62822") + .setSize(147004) + .setKind("font") + .setType("asset-archive") + .build()); + + ArrayMap<String, DownloadContent> content = new ArrayMap<>(); + for (DownloadContent currentContent : initialList) { + content.put(currentContent.getId(), currentContent); + } + return content; + } +} diff --git a/mobile/android/base/java/org/mozilla/gecko/dlc/catalog/DownloadContentBuilder.java b/mobile/android/base/java/org/mozilla/gecko/dlc/catalog/DownloadContentBuilder.java new file mode 100644 index 000000000..243e2d4eb --- /dev/null +++ b/mobile/android/base/java/org/mozilla/gecko/dlc/catalog/DownloadContentBuilder.java @@ -0,0 +1,238 @@ +/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.dlc.catalog; + +import android.text.TextUtils; + +import org.json.JSONException; +import org.json.JSONObject; + +public class DownloadContentBuilder { + private static final String LOCAL_KEY_ID = "id"; + private static final String LOCAL_KEY_LOCATION = "location"; + private static final String LOCAL_KEY_FILENAME = "filename"; + private static final String LOCAL_KEY_CHECKSUM = "checksum"; + private static final String LOCAL_KEY_DOWNLOAD_CHECKSUM = "download_checksum"; + private static final String LOCAL_KEY_LAST_MODIFIED = "last_modified"; + private static final String LOCAL_KEY_TYPE = "type"; + private static final String LOCAL_KEY_KIND = "kind"; + private static final String LOCAL_KEY_SIZE = "size"; + private static final String LOCAL_KEY_STATE = "state"; + private static final String LOCAL_KEY_FAILURES = "failures"; + private static final String LOCAL_KEY_LAST_FAILURE_TYPE = "last_failure_type"; + private static final String LOCAL_KEY_PATTERN_APP_ID = "pattern_app_id"; + private static final String LOCAL_KEY_PATTERN_ANDROID_API = "pattern_android_api"; + private static final String LOCAL_KEY_PATTERN_APP_VERSION = "pattern_app_version"; + + private static final String KINTO_KEY_ID = "id"; + private static final String KINTO_KEY_ATTACHMENT = "attachment"; + private static final String KINTO_KEY_ORIGINAL = "original"; + private static final String KINTO_KEY_LOCATION = "location"; + private static final String KINTO_KEY_FILENAME = "filename"; + private static final String KINTO_KEY_HASH = "hash"; + private static final String KINTO_KEY_LAST_MODIFIED = "last_modified"; + private static final String KINTO_KEY_TYPE = "type"; + private static final String KINTO_KEY_KIND = "kind"; + private static final String KINTO_KEY_SIZE = "size"; + private static final String KINTO_KEY_MATCH = "match"; + private static final String KINTO_KEY_APP_ID = "appId"; + private static final String KINTO_KEY_ANDROID_API = "androidApi"; + private static final String KINTO_KEY_APP_VERSION = "appVersion"; + + private String id; + private String location; + private String filename; + private String checksum; + private String downloadChecksum; + private long lastModified; + private String type; + private String kind; + private long size; + private int state; + private int failures; + private int lastFailureType; + private String appVersionPattern; + private String androidApiPattern; + private String appIdPattern; + + public static DownloadContentBuilder buildUpon(DownloadContent content) { + DownloadContentBuilder builder = new DownloadContentBuilder(); + + builder.id = content.getId(); + builder.location = content.getLocation(); + builder.filename = content.getFilename(); + builder.checksum = content.getChecksum(); + builder.downloadChecksum = content.getDownloadChecksum(); + builder.lastModified = content.getLastModified(); + builder.type = content.getType(); + builder.kind = content.getKind(); + builder.size = content.getSize(); + builder.state = content.getState(); + builder.failures = content.getFailures(); + builder.lastFailureType = content.getLastFailureType(); + + return builder; + } + + public static DownloadContent fromJSON(JSONObject object) throws JSONException { + return new DownloadContentBuilder() + .setId(object.getString(LOCAL_KEY_ID)) + .setLocation(object.getString(LOCAL_KEY_LOCATION)) + .setFilename(object.getString(LOCAL_KEY_FILENAME)) + .setChecksum(object.getString(LOCAL_KEY_CHECKSUM)) + .setDownloadChecksum(object.getString(LOCAL_KEY_DOWNLOAD_CHECKSUM)) + .setLastModified(object.getLong(LOCAL_KEY_LAST_MODIFIED)) + .setType(object.getString(LOCAL_KEY_TYPE)) + .setKind(object.getString(LOCAL_KEY_KIND)) + .setSize(object.getLong(LOCAL_KEY_SIZE)) + .setState(object.getInt(LOCAL_KEY_STATE)) + .setFailures(object.optInt(LOCAL_KEY_FAILURES), object.optInt(LOCAL_KEY_LAST_FAILURE_TYPE)) + .setAppVersionPattern(object.optString(LOCAL_KEY_PATTERN_APP_VERSION)) + .setAppIdPattern(object.optString(LOCAL_KEY_PATTERN_APP_ID)) + .setAndroidApiPattern(object.optString(LOCAL_KEY_PATTERN_ANDROID_API)) + .build(); + } + + public static JSONObject toJSON(DownloadContent content) throws JSONException { + final JSONObject object = new JSONObject(); + object.put(LOCAL_KEY_ID, content.getId()); + object.put(LOCAL_KEY_LOCATION, content.getLocation()); + object.put(LOCAL_KEY_FILENAME, content.getFilename()); + object.put(LOCAL_KEY_CHECKSUM, content.getChecksum()); + object.put(LOCAL_KEY_DOWNLOAD_CHECKSUM, content.getDownloadChecksum()); + object.put(LOCAL_KEY_LAST_MODIFIED, content.getLastModified()); + object.put(LOCAL_KEY_TYPE, content.getType()); + object.put(LOCAL_KEY_KIND, content.getKind()); + object.put(LOCAL_KEY_SIZE, content.getSize()); + object.put(LOCAL_KEY_STATE, content.getState()); + object.put(LOCAL_KEY_PATTERN_APP_VERSION, content.getAppVersionPattern()); + object.put(LOCAL_KEY_PATTERN_APP_ID, content.getAppIdPattern()); + object.put(LOCAL_KEY_PATTERN_ANDROID_API, content.getAndroidApiPattern()); + + final int failures = content.getFailures(); + if (failures > 0) { + object.put(LOCAL_KEY_FAILURES, failures); + object.put(LOCAL_KEY_LAST_FAILURE_TYPE, content.getLastFailureType()); + } + + return object; + } + + public DownloadContent build() { + DownloadContent content = new DownloadContent(id, location, filename, checksum, + downloadChecksum, lastModified, type, kind, size, failures, lastFailureType, + appVersionPattern, androidApiPattern, appIdPattern); + content.setState(state); + + return content; + } + + public DownloadContentBuilder setId(String id) { + this.id = id; + return this; + } + + public DownloadContentBuilder setLocation(String location) { + this.location = location; + return this; + } + + public DownloadContentBuilder setFilename(String filename) { + this.filename = filename; + return this; + } + + public DownloadContentBuilder setChecksum(String checksum) { + this.checksum = checksum; + return this; + } + + public DownloadContentBuilder setDownloadChecksum(String downloadChecksum) { + this.downloadChecksum = downloadChecksum; + return this; + } + + public DownloadContentBuilder setLastModified(long lastModified) { + this.lastModified = lastModified; + return this; + } + + public DownloadContentBuilder setType(String type) { + this.type = type; + return this; + } + + public DownloadContentBuilder setKind(String kind) { + this.kind = kind; + return this; + } + + public DownloadContentBuilder setSize(long size) { + this.size = size; + return this; + } + + public DownloadContentBuilder setState(int state) { + this.state = state; + return this; + } + + /* package-private */ DownloadContentBuilder setFailures(int failures, int lastFailureType) { + this.failures = failures; + this.lastFailureType = lastFailureType; + + return this; + } + + public DownloadContentBuilder setAppVersionPattern(String appVersionPattern) { + this.appVersionPattern = appVersionPattern; + return this; + } + + public DownloadContentBuilder setAndroidApiPattern(String androidApiPattern) { + this.androidApiPattern = androidApiPattern; + return this; + } + + public DownloadContentBuilder setAppIdPattern(String appIdPattern) { + this.appIdPattern = appIdPattern; + return this; + } + + public DownloadContentBuilder updateFromKinto(JSONObject object) throws JSONException { + final String objectId = object.getString(KINTO_KEY_ID); + + if (TextUtils.isEmpty(id)) { + // New object without an id yet + id = objectId; + } else if (!id.equals(objectId)) { + throw new JSONException(String.format("Record ids do not match: Expected=%s, Actual=%s", id, objectId)); + } + + setType(object.getString(KINTO_KEY_TYPE)); + setKind(object.getString(KINTO_KEY_KIND)); + setLastModified(object.getLong(KINTO_KEY_LAST_MODIFIED)); + + JSONObject attachment = object.getJSONObject(KINTO_KEY_ATTACHMENT); + JSONObject original = attachment.getJSONObject(KINTO_KEY_ORIGINAL); + + setFilename(original.getString(KINTO_KEY_FILENAME)); + setChecksum(original.getString(KINTO_KEY_HASH)); + setSize(original.getLong(KINTO_KEY_SIZE)); + + setLocation(attachment.getString(KINTO_KEY_LOCATION)); + setDownloadChecksum(attachment.getString(KINTO_KEY_HASH)); + + JSONObject match = object.optJSONObject(KINTO_KEY_MATCH); + if (match != null) { + setAndroidApiPattern(match.optString(KINTO_KEY_ANDROID_API)); + setAppIdPattern(match.optString(KINTO_KEY_APP_ID)); + setAppVersionPattern(match.optString(KINTO_KEY_APP_VERSION)); + } + + return this; + } +} diff --git a/mobile/android/base/java/org/mozilla/gecko/dlc/catalog/DownloadContentCatalog.java b/mobile/android/base/java/org/mozilla/gecko/dlc/catalog/DownloadContentCatalog.java new file mode 100644 index 000000000..43ba4e82e --- /dev/null +++ b/mobile/android/base/java/org/mozilla/gecko/dlc/catalog/DownloadContentCatalog.java @@ -0,0 +1,303 @@ +/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.dlc.catalog; + +import android.content.Context; +import android.support.annotation.Nullable; +import android.support.v4.util.ArrayMap; +import android.support.v4.util.AtomicFile; +import android.util.Log; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.List; + +/** + * Catalog of downloadable content (DLC). + * + * Changing elements returned by the catalog should be guarded by the catalog instance to guarantee visibility when + * persisting changes. + */ +public class DownloadContentCatalog { + private static final String LOGTAG = "GeckoDLCCatalog"; + private static final String FILE_NAME = "download_content_catalog"; + + private static final String JSON_KEY_CONTENT = "content"; + + private static final int MAX_FAILURES_UNTIL_PERMANENTLY_FAILED = 10; + + private final AtomicFile file; // Guarded by 'file' + + private ArrayMap<String, DownloadContent> content; // Guarded by 'this' + private boolean hasLoadedCatalog; // Guarded by 'this + private boolean hasCatalogChanged; // Guarded by 'this' + + public DownloadContentCatalog(Context context) { + this(new AtomicFile(new File(context.getApplicationInfo().dataDir, FILE_NAME))); + + startLoadFromDisk(); + } + + // For injecting mocked AtomicFile objects during test + protected DownloadContentCatalog(AtomicFile file) { + this.content = new ArrayMap<>(); + this.file = file; + } + + public List<DownloadContent> getContentToStudy() { + return filterByState(DownloadContent.STATE_NONE, DownloadContent.STATE_UPDATED); + } + + public List<DownloadContent> getContentToDelete() { + return filterByState(DownloadContent.STATE_DELETED); + } + + public List<DownloadContent> getDownloadedContent() { + return filterByState(DownloadContent.STATE_DOWNLOADED); + } + + public List<DownloadContent> getScheduledDownloads() { + return filterByState(DownloadContent.STATE_SCHEDULED); + } + + private synchronized List<DownloadContent> filterByState(@DownloadContent.State int... filterStates) { + awaitLoadingCatalogLocked(); + + List<DownloadContent> filteredContent = new ArrayList<>(); + + for (DownloadContent currentContent : content.values()) { + if (currentContent.isStateIn(filterStates)) { + filteredContent.add(currentContent); + } + } + + return filteredContent; + } + + public boolean hasScheduledDownloads() { + return !filterByState(DownloadContent.STATE_SCHEDULED).isEmpty(); + } + + public synchronized void add(DownloadContent newContent) { + awaitLoadingCatalogLocked(); + + content.put(newContent.getId(), newContent); + hasCatalogChanged = true; + } + + public synchronized void update(DownloadContent changedContent) { + awaitLoadingCatalogLocked(); + + if (!content.containsKey(changedContent.getId())) { + Log.w(LOGTAG, "Did not find content with matching id (" + changedContent.getId() + ") to update"); + return; + } + + changedContent.setState(DownloadContent.STATE_UPDATED); + changedContent.resetFailures(); + + content.put(changedContent.getId(), changedContent); + hasCatalogChanged = true; + } + + public synchronized void remove(DownloadContent removedContent) { + awaitLoadingCatalogLocked(); + + if (!content.containsKey(removedContent.getId())) { + Log.w(LOGTAG, "Did not find content with matching id (" + removedContent.getId() + ") to remove"); + return; + } + + content.remove(removedContent.getId()); + } + + @Nullable + public synchronized DownloadContent getContentById(String id) { + return content.get(id); + } + + public synchronized long getLastModified() { + awaitLoadingCatalogLocked(); + + long lastModified = 0; + + for (DownloadContent currentContent : content.values()) { + if (currentContent.getLastModified() > lastModified) { + lastModified = currentContent.getLastModified(); + } + } + + return lastModified; + } + + public synchronized void scheduleDownload(DownloadContent content) { + content.setState(DownloadContent.STATE_SCHEDULED); + hasCatalogChanged = true; + } + + public synchronized void markAsDownloaded(DownloadContent content) { + content.setState(DownloadContent.STATE_DOWNLOADED); + content.resetFailures(); + hasCatalogChanged = true; + } + + public synchronized void markAsPermanentlyFailed(DownloadContent content) { + content.setState(DownloadContent.STATE_FAILED); + hasCatalogChanged = true; + } + + public synchronized void markAsDeleted(DownloadContent content) { + content.setState(DownloadContent.STATE_DELETED); + hasCatalogChanged = true; + } + + public synchronized void rememberFailure(DownloadContent content, int failureType) { + if (content.getFailures() >= MAX_FAILURES_UNTIL_PERMANENTLY_FAILED) { + Log.d(LOGTAG, "Maximum number of failures reached. Marking content has permanently failed."); + + markAsPermanentlyFailed(content); + } else { + content.rememberFailure(failureType); + hasCatalogChanged = true; + } + } + + public void persistChanges() { + new Thread(LOGTAG + "-Persist") { + public void run() { + writeToDisk(); + } + }.start(); + } + + private void startLoadFromDisk() { + new Thread(LOGTAG + "-Load") { + public void run() { + loadFromDisk(); + } + }.start(); + } + + private void awaitLoadingCatalogLocked() { + while (!hasLoadedCatalog) { + try { + Log.v(LOGTAG, "Waiting for catalog to be loaded"); + + wait(); + } catch (InterruptedException e) { + // Ignore + } + } + } + + protected synchronized boolean hasCatalogChanged() { + return hasCatalogChanged; + } + + protected synchronized void loadFromDisk() { + Log.d(LOGTAG, "Loading from disk"); + + if (hasLoadedCatalog) { + return; + } + + ArrayMap<String, DownloadContent> loadedContent = new ArrayMap<>(); + + try { + JSONObject catalog; + + synchronized (file) { + catalog = new JSONObject(new String(file.readFully(), "UTF-8")); + } + + JSONArray array = catalog.getJSONArray(JSON_KEY_CONTENT); + for (int i = 0; i < array.length(); i++) { + DownloadContent currentContent = DownloadContentBuilder.fromJSON(array.getJSONObject(i)); + loadedContent.put(currentContent.getId(), currentContent); + } + } catch (FileNotFoundException e) { + Log.d(LOGTAG, "Catalog file does not exist: Bootstrapping initial catalog"); + loadedContent = DownloadContentBootstrap.createInitialDownloadContentList(); + } catch (JSONException e) { + Log.w(LOGTAG, "Unable to parse catalog JSON. Re-creating catalog.", e); + // Catalog seems to be broken. Re-create catalog: + loadedContent = DownloadContentBootstrap.createInitialDownloadContentList(); + hasCatalogChanged = true; // Indicate that we want to persist the new catalog + } catch (NullPointerException e) { + // Bad content can produce an NPE in JSON code -- bug 1300139 + Log.w(LOGTAG, "Unable to parse catalog JSON. Re-creating catalog.", e); + // Catalog seems to be broken. Re-create catalog: + loadedContent = DownloadContentBootstrap.createInitialDownloadContentList(); + hasCatalogChanged = true; // Indicate that we want to persist the new catalog + } catch (UnsupportedEncodingException e) { + AssertionError error = new AssertionError("Should not happen: This device does not speak UTF-8"); + error.initCause(e); + throw error; + } catch (IOException e) { + Log.d(LOGTAG, "Can't read catalog due to IOException", e); + } + + onCatalogLoaded(loadedContent); + + notifyAll(); + + Log.d(LOGTAG, "Loaded " + content.size() + " elements"); + } + + protected void onCatalogLoaded(ArrayMap<String, DownloadContent> content) { + this.content = content; + this.hasLoadedCatalog = true; + } + + protected synchronized void writeToDisk() { + if (!hasCatalogChanged) { + Log.v(LOGTAG, "Not persisting: Catalog has not changed"); + return; + } + + Log.d(LOGTAG, "Writing to disk"); + + FileOutputStream outputStream = null; + + synchronized (file) { + try { + outputStream = file.startWrite(); + + JSONArray array = new JSONArray(); + for (DownloadContent currentContent : content.values()) { + array.put(DownloadContentBuilder.toJSON(currentContent)); + } + + JSONObject catalog = new JSONObject(); + catalog.put(JSON_KEY_CONTENT, array); + + outputStream.write(catalog.toString().getBytes("UTF-8")); + + file.finishWrite(outputStream); + + hasCatalogChanged = false; + } catch (UnsupportedEncodingException e) { + AssertionError error = new AssertionError("Should not happen: This device does not speak UTF-8"); + error.initCause(e); + throw error; + } catch (IOException | JSONException e) { + Log.e(LOGTAG, "IOException during writing catalog", e); + + if (outputStream != null) { + file.failWrite(outputStream); + } + } + } + } +} |