diff options
Diffstat (limited to 'tools/update-packaging')
116 files changed, 2365 insertions, 0 deletions
diff --git a/tools/update-packaging/Makefile.in b/tools/update-packaging/Makefile.in new file mode 100644 index 000000000..7c55958b6 --- /dev/null +++ b/tools/update-packaging/Makefile.in @@ -0,0 +1,79 @@ +# vim:set ts=8 sw=8 sts=8 noet: +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +STANDALONE_MAKEFILE := 1 + +# input location for the build, usually $(DIST) +# set this to $(DIST)/l10n-stage per override for L10n builds +PACKAGE_BASE_DIR = $(DIST) + +# Default output location for update archive +STAGE_DIR = $(ABS_DIST)/$(PKG_UPDATE_PATH) + +ifeq (cocoa,$(MOZ_WIDGET_TOOLKIT)) +ifdef UNIVERSAL_BINARY +ifneq (,$(filter %/l10n-stage,$(PACKAGE_BASE_DIR))) +PACKAGE_DIR = $(PACKAGE_BASE_DIR)/$(MOZ_PKG_DIR)/$(MOZ_MACBUNDLE_NAME) +else +PACKAGE_DIR = $(PACKAGE_BASE_DIR)/universal/$(MOZ_PKG_DIR)/$(MOZ_MACBUNDLE_NAME) +endif +else +PACKAGE_DIR = $(PACKAGE_BASE_DIR)/$(MOZ_PKG_DIR)/$(MOZ_MACBUNDLE_NAME) +endif +else +PACKAGE_DIR = $(PACKAGE_BASE_DIR)/$(MOZ_PKG_DIR) +endif + +MAR_BIN = $(DIST)/host/bin/mar$(HOST_BIN_SUFFIX) +MBSDIFF_BIN = $(DIST)/host/bin/mbsdiff$(HOST_BIN_SUFFIX) + +OVERRIDE_DEFAULT_GOAL := full-update +full-update:: complete-patch + +ifeq ($(OS_TARGET), WINNT) +MOZ_PKG_FORMAT := SFX7Z +UNPACKAGE = '$(subst $(DIST),$(ABS_DIST),$(INSTALLER_PACKAGE))' +ifdef AB_CD +UNPACKAGE = '$(PACKAGE_BASE_DIR)/$(PACKAGE)' +endif +endif + +include $(topsrcdir)/config/rules.mk +include $(topsrcdir)/toolkit/mozapps/installer/signing.mk +include $(topsrcdir)/toolkit/mozapps/installer/packager.mk + +ifdef MOZ_EXTERNAL_SIGNING_FORMAT +# We can't use sha2signcode on mar files +MOZ_EXTERNAL_SIGNING_FORMAT := $(filter-out sha2signcode,$(MOZ_EXTERNAL_SIGNING_FORMAT)) +MOZ_EXTERNAL_SIGNING_FORMAT := mar $(MOZ_EXTERNAL_SIGNING_FORMAT) +endif + +dir-stage := $(call mkdir_deps,$(STAGE_DIR)) + +complete-patch:: $(dir-stage) +ifeq ($(OS_TARGET), WINNT) + test -f $(UNPACKAGE) + $(RM) -rf '$(PACKAGE_DIR)' + cd $(PACKAGE_BASE_DIR) && $(INNER_UNMAKE_PACKAGE) +endif + MAR=$(MAR_BIN) \ + $(srcdir)/make_full_update.sh \ + '$(DIST)/$(COMPLETE_MAR)' \ + '$(PACKAGE_DIR)' +ifdef MOZ_SIGN_CMD + $(MOZ_SIGN_CMD) -f mar '$(DIST)/$(COMPLETE_MAR)' +endif + +partial-patch:: $(dir-stage) + MAR=$(MAR_BIN) \ + MBSDIFF=$(MBSDIFF_BIN) \ + $(srcdir)/make_incremental_update.sh \ + '$(STAGE_DIR)/$(PKG_UPDATE_BASENAME).partial.$(SRC_BUILD_ID)-$(DST_BUILD_ID).mar' \ + '$(SRC_BUILD)' \ + '$(DST_BUILD)' +ifdef MOZ_SIGN_CMD + $(MOZ_SIGN_CMD) -f mar '$(STAGE_DIR)/$(PKG_UPDATE_BASENAME).partial.$(SRC_BUILD_ID)-$(DST_BUILD_ID).mar' +endif diff --git a/tools/update-packaging/README b/tools/update-packaging/README new file mode 100644 index 000000000..7029ffb8e --- /dev/null +++ b/tools/update-packaging/README @@ -0,0 +1,4 @@ +This directory contains a tool for generating update packages for +the update system described here: + + http://wiki.mozilla.org/Software_Update diff --git a/tools/update-packaging/common.sh b/tools/update-packaging/common.sh new file mode 100755 index 000000000..eb358806f --- /dev/null +++ b/tools/update-packaging/common.sh @@ -0,0 +1,205 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# Code shared by update packaging scripts. +# Author: Darin Fisher +# + +# ----------------------------------------------------------------------------- +# By default just assume that these tools exist on our path +MAR=${MAR:-mar} +BZIP2=${BZIP2:-bzip2} +MBSDIFF=${MBSDIFF:-mbsdiff} + +# ----------------------------------------------------------------------------- +# Helper routines + +notice() { + echo "$*" 1>&2 +} + +get_file_size() { + info=($(ls -ln "$1")) + echo ${info[4]} +} + +copy_perm() { + reference="$1" + target="$2" + + if [ -x "$reference" ]; then + chmod 0755 "$target" + else + chmod 0644 "$target" + fi +} + +make_add_instruction() { + f="$1" + filev2="$2" + # The third param will be an empty string when a file add instruction is only + # needed in the version 2 manifest. This only happens when the file has an + # add-if-not instruction in the version 3 manifest. This is due to the + # precomplete file prior to the version 3 manifest having a remove instruction + # for this file so the file is removed before applying a complete update. + filev3="$3" + + # Used to log to the console + if [ $4 ]; then + forced=" (forced)" + else + forced= + fi + + is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') + if [ $is_extension = "1" ]; then + # Use the subdirectory of the extensions folder as the file to test + # before performing this add instruction. + testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') + notice " add-if \"$testdir\" \"$f\"" + echo "add-if \"$testdir\" \"$f\"" >> $filev2 + if [ ! $filev3 = "" ]; then + echo "add-if \"$testdir\" \"$f\"" >> $filev3 + fi + else + notice " add \"$f\"$forced" + echo "add \"$f\"" >> $filev2 + if [ ! $filev3 = "" ]; then + echo "add \"$f\"" >> $filev3 + fi + fi +} + +check_for_add_if_not_update() { + add_if_not_file_chk="$1" + + if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \ + `basename $add_if_not_file_chk` = "update-settings.ini" ]; then + ## "true" *giggle* + return 0; + fi + ## 'false'... because this is bash. Oh yay! + return 1; +} + +check_for_add_to_manifestv2() { + add_if_not_file_chk="$1" + + if [ `basename $add_if_not_file_chk` = "update-settings.ini" ]; then + ## "true" *giggle* + return 0; + fi + ## 'false'... because this is bash. Oh yay! + return 1; +} + +make_add_if_not_instruction() { + f="$1" + filev3="$2" + + notice " add-if-not \"$f\" \"$f\"" + echo "add-if-not \"$f\" \"$f\"" >> $filev3 +} + +make_patch_instruction() { + f="$1" + filev2="$2" + filev3="$3" + + is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') + if [ $is_extension = "1" ]; then + # Use the subdirectory of the extensions folder as the file to test + # before performing this add instruction. + testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') + notice " patch-if \"$testdir\" \"$f.patch\" \"$f\"" + echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev2 + echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev3 + else + notice " patch \"$f.patch\" \"$f\"" + echo "patch \"$f.patch\" \"$f\"" >> $filev2 + echo "patch \"$f.patch\" \"$f\"" >> $filev3 + fi +} + +append_remove_instructions() { + dir="$1" + filev2="$2" + filev3="$3" + + if [ -f "$dir/removed-files" ]; then + listfile="$dir/removed-files" + elif [ -f "$dir/Contents/Resources/removed-files" ]; then + listfile="$dir/Contents/Resources/removed-files" + fi + if [ -n "$listfile" ]; then + # Map spaces to pipes so that we correctly handle filenames with spaces. + files=($(cat "$listfile" | tr " " "|" | sort -r)) + num_files=${#files[*]} + for ((i=0; $i<$num_files; i=$i+1)); do + # Map pipes back to whitespace and remove carriage returns + f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r') + # Trim whitespace + f=$(echo $f) + # Exclude blank lines. + if [ -n "$f" ]; then + # Exclude comments + if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then + if [ $(echo "$f" | grep -c '\/$') = 1 ]; then + notice " rmdir \"$f\"" + echo "rmdir \"$f\"" >> $filev2 + echo "rmdir \"$f\"" >> $filev3 + elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then + # Remove the * + f=$(echo "$f" | sed -e 's:\*$::') + notice " rmrfdir \"$f\"" + echo "rmrfdir \"$f\"" >> $filev2 + echo "rmrfdir \"$f\"" >> $filev3 + else + notice " remove \"$f\"" + echo "remove \"$f\"" >> $filev2 + echo "remove \"$f\"" >> $filev3 + fi + fi + fi + done + fi +} + +# List all files in the current directory, stripping leading "./" +# Pass a variable name and it will be filled as an array. +list_files() { + count=0 + + find . -type f \ + ! -name "update.manifest" \ + ! -name "updatev2.manifest" \ + ! -name "updatev3.manifest" \ + ! -name "temp-dirlist" \ + ! -name "temp-filelist" \ + | sed 's/\.\/\(.*\)/\1/' \ + | sort -r > "temp-filelist" + while read file; do + eval "${1}[$count]=\"$file\"" + (( count++ )) + done < "temp-filelist" + rm "temp-filelist" +} + +# List all directories in the current directory, stripping leading "./" +list_dirs() { + count=0 + + find . -type d \ + ! -name "." \ + ! -name ".." \ + | sed 's/\.\/\(.*\)/\1/' \ + | sort -r > "temp-dirlist" + while read dir; do + eval "${1}[$count]=\"$dir\"" + (( count++ )) + done < "temp-dirlist" + rm "temp-dirlist" +} diff --git a/tools/update-packaging/generatesnippet.py b/tools/update-packaging/generatesnippet.py new file mode 100644 index 000000000..15b7edf58 --- /dev/null +++ b/tools/update-packaging/generatesnippet.py @@ -0,0 +1,166 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +This script generates the complete snippet for a given locale or en-US +Most of the parameters received are to generate the MAR's download URL +and determine the MAR's filename +""" +import sys, os, platform, sha +from optparse import OptionParser +from ConfigParser import ConfigParser +from stat import ST_SIZE + +def main(): + error = False + parser = OptionParser( + usage="%prog [options]") + parser.add_option("--mar-path", + action="store", + dest="marPath", + help="[Required] Specify the absolute path where the MAR file is found.") + parser.add_option("--application-ini-file", + action="store", + dest="applicationIniFile", + help="[Required] Specify the absolute path to the application.ini file.") + parser.add_option("-l", + "--locale", + action="store", + dest="locale", + help="[Required] Specify which locale we are generating the snippet for.") + parser.add_option("-p", + "--product", + action="store", + dest="product", + help="[Required] This option is used to generate the URL to download the MAR file.") + parser.add_option("--platform", + action="store", + dest="platform", + help="[Required] This option is used to indicate which target platform.") + parser.add_option("--branch", + action="store", + dest="branch", + help="This option is used to indicate which branch name to use for FTP file names.") + parser.add_option("--download-base-URL", + action="store", + dest="downloadBaseURL", + help="This option indicates under which.") + parser.add_option("-v", + "--verbose", + action="store_true", + dest="verbose", + default=False, + help="This option increases the output of the script.") + (options, args) = parser.parse_args() + for req, msg in (('marPath', "the absolute path to the where the MAR file is"), + ('applicationIniFile', "the absolute path to the application.ini file."), + ('locale', "a locale."), + ('product', "specify a product."), + ('platform', "specify the platform.")): + if not hasattr(options, req): + parser.error('You must specify %s' % msg) + + if not options.downloadBaseURL or options.downloadBaseURL == '': + options.downloadBaseURL = 'http://ftp.mozilla.org/pub/mozilla.org/%s/nightly' % options.product + + if not options.branch or options.branch == '': + options.branch = None + + snippet = generateSnippet(options.marPath, + options.applicationIniFile, + options.locale, + options.downloadBaseURL, + options.product, + options.platform, + options.branch) + f = open(os.path.join(options.marPath, 'complete.update.snippet'), 'wb') + f.write(snippet) + f.close() + + if options.verbose: + # Show in our logs what the contents of the snippet are + print snippet + +def generateSnippet(abstDistDir, applicationIniFile, locale, + downloadBaseURL, product, platform, branch): + # Let's extract information from application.ini + c = ConfigParser() + try: + c.readfp(open(applicationIniFile)) + except IOError, (stderror): + sys.exit(stderror) + buildid = c.get("App", "BuildID") + appVersion = c.get("App", "Version") + branchName = branch or c.get("App", "SourceRepository").split('/')[-1] + + marFileName = '%s-%s.%s.%s.complete.mar' % ( + product, + appVersion, + locale, + platform) + # Let's determine the hash and the size of the MAR file + # This function exits the script if the file does not exist + (completeMarHash, completeMarSize) = getFileHashAndSize( + os.path.join(abstDistDir, marFileName)) + # Construct the URL to where the MAR file will exist + interfix = '' + if locale == 'en-US': + interfix = '' + else: + interfix = '-l10n' + marDownloadURL = "%s/%s%s/%s" % (downloadBaseURL, + datedDirPath(buildid, branchName), + interfix, + marFileName) + + snippet = """complete +%(marDownloadURL)s +sha1 +%(completeMarHash)s +%(completeMarSize)s +%(buildid)s +%(appVersion)s +%(appVersion)s +""" % dict( marDownloadURL=marDownloadURL, + completeMarHash=completeMarHash, + completeMarSize=completeMarSize, + buildid=buildid, + appVersion=appVersion) + + return snippet + +def getFileHashAndSize(filepath): + sha1Hash = 'UNKNOWN' + size = 'UNKNOWN' + + try: + # open in binary mode to make sure we get consistent results + # across all platforms + f = open(filepath, "rb") + shaObj = sha.new(f.read()) + sha1Hash = shaObj.hexdigest() + size = os.stat(filepath)[ST_SIZE] + except IOError, (stderror): + sys.exit(stderror) + + return (sha1Hash, size) + +def datedDirPath(buildid, milestone): + """ + Returns a string that will look like: + 2009/12/2009-12-31-09-mozilla-central + """ + year = buildid[0:4] + month = buildid[4:6] + day = buildid[6:8] + hour = buildid[8:10] + datedDir = "%s-%s-%s-%s-%s" % (year, + month, + day, + hour, + milestone) + return "%s/%s/%s" % (year, month, datedDir) + +if __name__ == '__main__': + main() diff --git a/tools/update-packaging/make_full_update.sh b/tools/update-packaging/make_full_update.sh new file mode 100755 index 000000000..f0466144d --- /dev/null +++ b/tools/update-packaging/make_full_update.sh @@ -0,0 +1,118 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# This tool generates full update packages for the update system. +# Author: Darin Fisher +# + +. $(dirname "$0")/common.sh + +# ----------------------------------------------------------------------------- + +print_usage() { + notice "Usage: $(basename $0) [OPTIONS] ARCHIVE DIRECTORY" +} + +if [ $# = 0 ]; then + print_usage + exit 1 +fi + +if [ $1 = -h ]; then + print_usage + notice "" + notice "The contents of DIRECTORY will be stored in ARCHIVE." + notice "" + notice "Options:" + notice " -h show this help text" + notice "" + exit 1 +fi + +# ----------------------------------------------------------------------------- + +archive="$1" +targetdir="$2" +# Prevent the workdir from being inside the targetdir so it isn't included in +# the update mar. +if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then + # Remove the / + targetdir=$(echo "$targetdir" | sed -e 's:\/$::') +fi +workdir="$targetdir.work" +updatemanifestv2="$workdir/updatev2.manifest" +updatemanifestv3="$workdir/updatev3.manifest" +targetfiles="updatev2.manifest updatev3.manifest" + +mkdir -p "$workdir" + +# Generate a list of all files in the target directory. +pushd "$targetdir" +if test $? -ne 0 ; then + exit 1 +fi + +if [ ! -f "precomplete" ]; then + if [ ! -f "Contents/Resources/precomplete" ]; then + notice "precomplete file is missing!" + exit 1 + fi +fi + +list_files files + +popd + +# Add the type of update to the beginning of the update manifests. +> $updatemanifestv2 +> $updatemanifestv3 +notice "" +notice "Adding type instruction to update manifests" +notice " type complete" +echo "type \"complete\"" >> $updatemanifestv2 +echo "type \"complete\"" >> $updatemanifestv3 + +notice "" +notice "Adding file add instructions to update manifests" +num_files=${#files[*]} + +for ((i=0; $i<$num_files; i=$i+1)); do + f="${files[$i]}" + + if check_for_add_if_not_update "$f"; then + make_add_if_not_instruction "$f" "$updatemanifestv3" + if check_for_add_to_manifestv2 "$f"; then + make_add_instruction "$f" "$updatemanifestv2" "" 1 + fi + else + make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" + fi + + dir=$(dirname "$f") + mkdir -p "$workdir/$dir" + $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f" + copy_perm "$targetdir/$f" "$workdir/$f" + + targetfiles="$targetfiles \"$f\"" +done + +# Append remove instructions for any dead files. +notice "" +notice "Adding file and directory remove instructions from file 'removed-files'" +append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3" + +$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2" +$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3" + +eval "$MAR -C \"$workdir\" -c output.mar $targetfiles" +mv -f "$workdir/output.mar" "$archive" + +# cleanup +rm -fr "$workdir" + +notice "" +notice "Finished" +notice "" diff --git a/tools/update-packaging/make_incremental_update.sh b/tools/update-packaging/make_incremental_update.sh new file mode 100755 index 000000000..90372c017 --- /dev/null +++ b/tools/update-packaging/make_incremental_update.sh @@ -0,0 +1,327 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# This tool generates incremental update packages for the update system. +# Author: Darin Fisher +# + +. $(dirname "$0")/common.sh + +# ----------------------------------------------------------------------------- + +print_usage() { + notice "Usage: $(basename $0) [OPTIONS] ARCHIVE FROMDIR TODIR" + notice "" + notice "The differences between FROMDIR and TODIR will be stored in ARCHIVE." + notice "" + notice "Options:" + notice " -h show this help text" + notice " -f clobber this file in the installation" + notice " Must be a path to a file to clobber in the partial update." + notice "" +} + +check_for_forced_update() { + force_list="$1" + forced_file_chk="$2" + + local f + + if [ "$forced_file_chk" = "precomplete" ]; then + ## "true" *giggle* + return 0; + fi + + if [ "$forced_file_chk" = "Contents/Resources/precomplete" ]; then + ## "true" *giggle* + return 0; + fi + + if [ "$forced_file_chk" = "removed-files" ]; then + ## "true" *giggle* + return 0; + fi + + if [ "$forced_file_chk" = "Contents/Resources/removed-files" ]; then + ## "true" *giggle* + return 0; + fi + + if [ "$forced_file_chk" = "chrome.manifest" ]; then + ## "true" *giggle* + return 0; + fi + + if [ "$forced_file_chk" = "Contents/Resources/chrome.manifest" ]; then + ## "true" *giggle* + return 0; + fi + + if [ "${forced_file_chk##*.}" = "chk" ]; then + ## "true" *giggle* + return 0; + fi + + for f in $force_list; do + #echo comparing $forced_file_chk to $f + if [ "$forced_file_chk" = "$f" ]; then + ## "true" *giggle* + return 0; + fi + done + ## 'false'... because this is bash. Oh yay! + return 1; +} + +if [ $# = 0 ]; then + print_usage + exit 1 +fi + +requested_forced_updates='Contents/MacOS/firefox' + +while getopts "hf:" flag +do + case "$flag" in + h) print_usage; exit 0 + ;; + f) requested_forced_updates="$requested_forced_updates $OPTARG" + ;; + ?) print_usage; exit 1 + ;; + esac +done + +# ----------------------------------------------------------------------------- + +let arg_start=$OPTIND-1 +shift $arg_start + +archive="$1" +olddir="$2" +newdir="$3" +# Prevent the workdir from being inside the targetdir so it isn't included in +# the update mar. +if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then + # Remove the / + newdir=$(echo "$newdir" | sed -e 's:\/$::') +fi +workdir="$newdir.work" +updatemanifestv2="$workdir/updatev2.manifest" +updatemanifestv3="$workdir/updatev3.manifest" +archivefiles="updatev2.manifest updatev3.manifest" + +mkdir -p "$workdir" + +# Generate a list of all files in the target directory. +pushd "$olddir" +if test $? -ne 0 ; then + exit 1 +fi + +list_files oldfiles +list_dirs olddirs + +popd + +pushd "$newdir" +if test $? -ne 0 ; then + exit 1 +fi + +if [ ! -f "precomplete" ]; then + if [ ! -f "Contents/Resources/precomplete" ]; then + notice "precomplete file is missing!" + exit 1 + fi +fi + +list_dirs newdirs +list_files newfiles + +popd + +# Add the type of update to the beginning of the update manifests. +notice "" +notice "Adding type instruction to update manifests" +> $updatemanifestv2 +> $updatemanifestv3 +notice " type partial" +echo "type \"partial\"" >> $updatemanifestv2 +echo "type \"partial\"" >> $updatemanifestv3 + +notice "" +notice "Adding file patch and add instructions to update manifests" + +num_oldfiles=${#oldfiles[*]} +remove_array= +num_removes=0 + +for ((i=0; $i<$num_oldfiles; i=$i+1)); do + f="${oldfiles[$i]}" + + # If this file exists in the new directory as well, then check if it differs. + if [ -f "$newdir/$f" ]; then + + if check_for_add_if_not_update "$f"; then + # The full workdir may not exist yet, so create it if necessary. + mkdir -p `dirname "$workdir/$f"` + $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" + copy_perm "$newdir/$f" "$workdir/$f" + make_add_if_not_instruction "$f" "$updatemanifestv3" + archivefiles="$archivefiles \"$f\"" + continue 1 + fi + + if check_for_forced_update "$requested_forced_updates" "$f"; then + # The full workdir may not exist yet, so create it if necessary. + mkdir -p `dirname "$workdir/$f"` + $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" + copy_perm "$newdir/$f" "$workdir/$f" + make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1 + archivefiles="$archivefiles \"$f\"" + continue 1 + fi + + if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then + # Compute both the compressed binary diff and the compressed file, and + # compare the sizes. Then choose the smaller of the two to package. + dir=$(dirname "$workdir/$f") + mkdir -p "$dir" + notice "diffing \"$f\"" + # MBSDIFF_HOOK represents the communication interface with funsize and, + # if enabled, caches the intermediate patches for future use and + # compute avoidance + # + # An example of MBSDIFF_HOOK env variable could look like this: + # export MBSDIFF_HOOK="myscript.sh -A https://funsize/api -c /home/user" + # where myscript.sh has the following usage: + # myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \ + # PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL + # + # Note: patches are bzipped stashed in funsize to gain more speed + + # if service is not enabled then default to old behavior + if [ -z "$MBSDIFF_HOOK" ]; then + $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" + $BZIP2 -z9 "$workdir/$f.patch" + else + # if service enabled then check patch existence for retrieval + if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then + notice "file \"$f\" found in funsize, diffing skipped" + else + # if not found already - compute it and cache it for future use + $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" + $BZIP2 -z9 "$workdir/$f.patch" + $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2" + fi + fi + $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" + copy_perm "$newdir/$f" "$workdir/$f" + patchfile="$workdir/$f.patch.bz2" + patchsize=$(get_file_size "$patchfile") + fullsize=$(get_file_size "$workdir/$f") + + if [ $patchsize -lt $fullsize ]; then + make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" + mv -f "$patchfile" "$workdir/$f.patch" + rm -f "$workdir/$f" + archivefiles="$archivefiles \"$f.patch\"" + else + make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" + rm -f "$patchfile" + archivefiles="$archivefiles \"$f\"" + fi + fi + else + # remove instructions are added after add / patch instructions for + # consistency with make_incremental_updates.py + remove_array[$num_removes]=$f + (( num_removes++ )) + fi +done + +# Newly added files +notice "" +notice "Adding file add instructions to update manifests" +num_newfiles=${#newfiles[*]} + +for ((i=0; $i<$num_newfiles; i=$i+1)); do + f="${newfiles[$i]}" + + # If we've already tested this file, then skip it + for ((j=0; $j<$num_oldfiles; j=$j+1)); do + if [ "$f" = "${oldfiles[j]}" ]; then + continue 2 + fi + done + + dir=$(dirname "$workdir/$f") + mkdir -p "$dir" + + $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" + copy_perm "$newdir/$f" "$workdir/$f" + + if check_for_add_if_not_update "$f"; then + make_add_if_not_instruction "$f" "$updatemanifestv3" + else + make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" + fi + + + archivefiles="$archivefiles \"$f\"" +done + +notice "" +notice "Adding file remove instructions to update manifests" +for ((i=0; $i<$num_removes; i=$i+1)); do + f="${remove_array[$i]}" + notice " remove \"$f\"" + echo "remove \"$f\"" >> $updatemanifestv2 + echo "remove \"$f\"" >> $updatemanifestv3 +done + +# Add remove instructions for any dead files. +notice "" +notice "Adding file and directory remove instructions from file 'removed-files'" +append_remove_instructions "$newdir" "$updatemanifestv2" "$updatemanifestv3" + +notice "" +notice "Adding directory remove instructions for directories that no longer exist" +num_olddirs=${#olddirs[*]} + +for ((i=0; $i<$num_olddirs; i=$i+1)); do + f="${olddirs[$i]}" + # If this dir doesn't exist in the new directory remove it. + if [ ! -d "$newdir/$f" ]; then + notice " rmdir $f/" + echo "rmdir \"$f/\"" >> $updatemanifestv2 + echo "rmdir \"$f/\"" >> $updatemanifestv3 + fi +done + +$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2" +$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3" + +mar_command="$MAR" +if [[ -n $MOZ_PRODUCT_VERSION ]] +then + mar_command="$mar_command -V $MOZ_PRODUCT_VERSION" +fi +if [[ -n $MOZ_CHANNEL_ID ]] +then + mar_command="$mar_command -H $MOZ_CHANNEL_ID" +fi +mar_command="$mar_command -C \"$workdir\" -c output.mar" +eval "$mar_command $archivefiles" +mv -f "$workdir/output.mar" "$archive" + +# cleanup +rm -fr "$workdir" + +notice "" +notice "Finished" +notice "" diff --git a/tools/update-packaging/make_incremental_updates.py b/tools/update-packaging/make_incremental_updates.py new file mode 100755 index 000000000..d346ee2df --- /dev/null +++ b/tools/update-packaging/make_incremental_updates.py @@ -0,0 +1,560 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import shutil +import sha +from os.path import join, getsize +from stat import * +import re +import sys +import getopt +import time +import datetime +import bz2 +import string +import tempfile + +class PatchInfo: + """ Represents the meta-data associated with a patch + work_dir = working dir where files are stored for this patch + archive_files = list of files to include in this patch + manifestv2 = set of manifest version 2 patch instructions + manifestv3 = set of manifest version 3 patch instructions + file_exclusion_list = + files to exclude from this patch. names without slashes will be + excluded anywhere in the directory hiearchy. names with slashes + will only be excluded at that exact path + """ + def __init__(self, work_dir, file_exclusion_list, path_exclusion_list): + self.work_dir=work_dir + self.archive_files=[] + self.manifestv2=[] + self.manifestv3=[] + self.file_exclusion_list=file_exclusion_list + self.path_exclusion_list=path_exclusion_list + + def append_add_instruction(self, filename): + """ Appends an add instruction for this patch. + if filename starts with distribution/extensions/.*/ this will add an + add-if instruction that will add the file if the parent directory + of the file exists. This was ported from + mozilla/tools/update-packaging/common.sh's make_add_instruction. + """ + m = re.match("((?:|.*/)distribution/extensions/.*)/", filename) + if m: + # Directory immediately following extensions is used for the test + testdir = m.group(1) + print ' add-if "'+testdir+'" "'+filename+'"' + self.manifestv2.append('add-if "'+testdir+'" "'+filename+'"') + self.manifestv3.append('add-if "'+testdir+'" "'+filename+'"') + else: + print ' add "'+filename+'"' + self.manifestv2.append('add "'+filename+'"') + self.manifestv3.append('add "'+filename+'"') + + def append_add_if_not_instruction(self, filename): + """ Appends an add-if-not instruction to the version 3 manifest for this patch. + This was ported from mozilla/tools/update-packaging/common.sh's + make_add_if_not_instruction. + """ + print ' add-if-not "'+filename+'" "'+filename+'"' + self.manifestv3.append('add-if-not "'+filename+'" "'+filename+'"') + + def append_patch_instruction(self, filename, patchname): + """ Appends a patch instruction for this patch. + + filename = file to patch + patchname = patchfile to apply to file + + if filename starts with distribution/extensions/.*/ this will add a + patch-if instruction that will patch the file if the parent + directory of the file exists. This was ported from + mozilla/tools/update-packaging/common.sh's make_patch_instruction. + """ + m = re.match("((?:|.*/)distribution/extensions/.*)/", filename) + if m: + testdir = m.group(1) + print ' patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"' + self.manifestv2.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"') + self.manifestv3.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"') + else: + print ' patch "'+patchname+'" "'+filename+'"' + self.manifestv2.append('patch "'+patchname+'" "'+filename+'"') + self.manifestv3.append('patch "'+patchname+'" "'+filename+'"') + + def append_remove_instruction(self, filename): + """ Appends an remove instruction for this patch. + This was ported from + mozilla/tools/update-packaging/common.sh/make_remove_instruction + """ + if filename.endswith("/"): + print ' rmdir "'+filename+'"' + self.manifestv2.append('rmdir "'+filename+'"') + self.manifestv3.append('rmdir "'+filename+'"') + elif filename.endswith("/*"): + filename = filename[:-1] + print ' rmrfdir "'+filename+'"' + self.manifestv2.append('rmrfdir "'+filename+'"') + self.manifestv3.append('rmrfdir "'+filename+'"') + else: + print ' remove "'+filename+'"' + self.manifestv2.append('remove "'+filename+'"') + self.manifestv3.append('remove "'+filename+'"') + + def create_manifest_files(self): + """ Create the v2 manifest file in the root of the work_dir """ + manifest_file_path = os.path.join(self.work_dir,"updatev2.manifest") + manifest_file = open(manifest_file_path, "wb") + manifest_file.writelines("type \"partial\"\n") + manifest_file.writelines(string.join(self.manifestv2, '\n')) + manifest_file.writelines("\n") + manifest_file.close() + + bzip_file(manifest_file_path) + self.archive_files.append('"updatev2.manifest"') + + """ Create the v3 manifest file in the root of the work_dir """ + manifest_file_path = os.path.join(self.work_dir,"updatev3.manifest") + manifest_file = open(manifest_file_path, "wb") + manifest_file.writelines("type \"partial\"\n") + manifest_file.writelines(string.join(self.manifestv3, '\n')) + manifest_file.writelines("\n") + manifest_file.close() + + bzip_file(manifest_file_path) + self.archive_files.append('"updatev3.manifest"') + + def build_marfile_entry_hash(self, root_path): + """ Iterates through the root_path, creating a MarFileEntry for each file + and directory in that path. Excludes any filenames in the file_exclusion_list + """ + mar_entry_hash = {} + filename_set = set() + dirname_set = set() + for root, dirs, files in os.walk(root_path): + for name in files: + # filename is the relative path from root directory + partial_path = root[len(root_path)+1:] + if name not in self.file_exclusion_list: + filename = os.path.join(partial_path, name) + if "/"+filename not in self.path_exclusion_list: + mar_entry_hash[filename]=MarFileEntry(root_path, filename) + filename_set.add(filename) + + for name in dirs: + # dirname is the relative path from root directory + partial_path = root[len(root_path)+1:] + if name not in self.file_exclusion_list: + dirname = os.path.join(partial_path, name) + if "/"+dirname not in self.path_exclusion_list: + dirname = dirname+"/" + mar_entry_hash[dirname]=MarFileEntry(root_path, dirname) + dirname_set.add(dirname) + + return mar_entry_hash, filename_set, dirname_set + + +class MarFileEntry: + """Represents a file inside a Mozilla Archive Format (MAR) + abs_path = abspath to the the file + name = relative path within the mar. e.g. + foo.mar/dir/bar.txt extracted into /tmp/foo: + abs_path=/tmp/foo/dir/bar.txt + name = dir/bar.txt + """ + def __init__(self, root, name): + """root = path the the top of the mar + name = relative path within the mar""" + self.name=name.replace("\\", "/") + self.abs_path=os.path.join(root,name) + self.sha_cache=None + + def __str__(self): + return 'Name: %s FullPath: %s' %(self.name,self.abs_path) + + def calc_file_sha_digest(self, filename): + """ Returns sha digest of given filename""" + file_content = open(filename, 'r').read() + return sha.new(file_content).digest() + + def sha(self): + """ Returns sha digest of file repreesnted by this _marfile_entry""" + if not self.sha_cache: + self.sha_cache=self.calc_file_sha_digest(self.abs_path) + return self.sha_cache + +def exec_shell_cmd(cmd): + """Execs shell cmd and raises an exception if the cmd fails""" + if (os.system(cmd)): + raise Exception, "cmd failed "+cmd + + +def copy_file(src_file_abs_path, dst_file_abs_path): + """ Copies src to dst creating any parent dirs required in dst first """ + dst_file_dir=os.path.dirname(dst_file_abs_path) + if not os.path.exists(dst_file_dir): + os.makedirs(dst_file_dir) + # Copy the file over + shutil.copy2(src_file_abs_path, dst_file_abs_path) + +def bzip_file(filename): + """ Bzip's the file in place. The original file is replaced with a bzip'd version of itself + assumes the path is absolute""" + exec_shell_cmd('bzip2 -z9 "' + filename+'"') + os.rename(filename+".bz2",filename) + +def bunzip_file(filename): + """ Bzip's the file in palce. The original file is replaced with a bunzip'd version of itself. + doesn't matter if the filename ends in .bz2 or not""" + if not filename.endswith(".bz2"): + os.rename(filename, filename+".bz2") + filename=filename+".bz2" + exec_shell_cmd('bzip2 -d "' + filename+'"') + + +def extract_mar(filename, work_dir): + """ Extracts the marfile intot he work_dir + assumes work_dir already exists otherwise will throw osError""" + print "Extracting "+filename+" to "+work_dir + saved_path = os.getcwd() + try: + os.chdir(work_dir) + exec_shell_cmd("mar -x "+filename) + finally: + os.chdir(saved_path) + +def create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, patch_info): + """ Creates the partial patch file and manifest entry for the pair of files passed in + """ + if not (from_marfile_entry.sha(),to_marfile_entry.sha()) in shas: + print 'diffing "'+from_marfile_entry.name+'\"' + #bunzip to/from + bunzip_file(from_marfile_entry.abs_path) + bunzip_file(to_marfile_entry.abs_path) + + # The patch file will be created in the working directory with the + # name of the file in the mar + .patch + patch_file_abs_path = os.path.join(patch_info.work_dir,from_marfile_entry.name+".patch") + patch_file_dir=os.path.dirname(patch_file_abs_path) + if not os.path.exists(patch_file_dir): + os.makedirs(patch_file_dir) + + # Create bzip'd patch file + exec_shell_cmd("mbsdiff "+from_marfile_entry.abs_path+" "+to_marfile_entry.abs_path+" "+patch_file_abs_path) + bzip_file(patch_file_abs_path) + + # Create bzip's full file + full_file_abs_path = os.path.join(patch_info.work_dir, to_marfile_entry.name) + shutil.copy2(to_marfile_entry.abs_path, full_file_abs_path) + bzip_file(full_file_abs_path) + + if os.path.getsize(patch_file_abs_path) < os.path.getsize(full_file_abs_path): + # Patch is smaller than file. Remove the file and add patch to manifest + os.remove(full_file_abs_path) + file_in_manifest_name = from_marfile_entry.name+".patch" + file_in_manifest_abspath = patch_file_abs_path + patch_info.append_patch_instruction(to_marfile_entry.name, file_in_manifest_name) + else: + # File is smaller than patch. Remove the patch and add file to manifest + os.remove(patch_file_abs_path) + file_in_manifest_name = from_marfile_entry.name + file_in_manifest_abspath = full_file_abs_path + patch_info.append_add_instruction(file_in_manifest_name) + + shas[from_marfile_entry.sha(),to_marfile_entry.sha()] = (file_in_manifest_name,file_in_manifest_abspath) + patch_info.archive_files.append('"'+file_in_manifest_name+'"') + else: + filename, src_file_abs_path = shas[from_marfile_entry.sha(),to_marfile_entry.sha()] + # We've already calculated the patch for this pair of files. + if (filename.endswith(".patch")): + # print "skipping diff: "+from_marfile_entry.name + # Patch was smaller than file - add patch instruction to manifest + file_in_manifest_name = to_marfile_entry.name+'.patch'; + patch_info.append_patch_instruction(to_marfile_entry.name, file_in_manifest_name) + else: + # File was smaller than file - add file to manifest + file_in_manifest_name = to_marfile_entry.name + patch_info.append_add_instruction(file_in_manifest_name) + # Copy the pre-calculated file into our new patch work aread + copy_file(src_file_abs_path, os.path.join(patch_info.work_dir, file_in_manifest_name)) + patch_info.archive_files.append('"'+file_in_manifest_name+'"') + +def create_add_patch_for_file(to_marfile_entry, patch_info): + """ Copy the file to the working dir, add the add instruction, and add it to the list of archive files """ + copy_file(to_marfile_entry.abs_path, os.path.join(patch_info.work_dir, to_marfile_entry.name)) + patch_info.append_add_instruction(to_marfile_entry.name) + patch_info.archive_files.append('"'+to_marfile_entry.name+'"') + +def create_add_if_not_patch_for_file(to_marfile_entry, patch_info): + """ Copy the file to the working dir, add the add-if-not instruction, and add it to the list of archive files """ + copy_file(to_marfile_entry.abs_path, os.path.join(patch_info.work_dir, to_marfile_entry.name)) + patch_info.append_add_if_not_instruction(to_marfile_entry.name) + patch_info.archive_files.append('"'+to_marfile_entry.name+'"') + +def process_explicit_remove_files(dir_path, patch_info): + """ Looks for a 'removed-files' file in the dir_path. If the removed-files does not exist + this will throw. If found adds the removed-files + found in that file to the patch_info""" + + # Windows and linux have this file at the root of the dir + list_file_path = os.path.join(dir_path, "removed-files") + if not os.path.exists(list_file_path): + list_file_path = os.path.join(dir_path, "Contents/Resources/removed-files") + + if (os.path.exists(list_file_path)): + list_file = bz2.BZ2File(list_file_path,"r") # throws if doesn't exist + + lines = [] + for line in list_file: + lines.append(line.strip()) + list_file.close() + + lines.sort(reverse=True) + for line in lines: + # Exclude any blank and comment lines. + if line and not line.startswith("#"): + # Python on windows uses \ for path separators and the update + # manifests expects / for path separators on all platforms. + line = line.replace("\\", "/") + patch_info.append_remove_instruction(line) + +def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch_info, forced_updates, add_if_not_list): + """ Builds a partial patch by comparing the files in from_dir_path to those of to_dir_path""" + # Cannocolize the paths for safey + from_dir_path = os.path.abspath(from_dir_path) + to_dir_path = os.path.abspath(to_dir_path) + # Create a hashtable of the from and to directories + from_dir_hash,from_file_set,from_dir_set = patch_info.build_marfile_entry_hash(from_dir_path) + to_dir_hash,to_file_set,to_dir_set = patch_info.build_marfile_entry_hash(to_dir_path) + # Create a list of the forced updates + forced_list = forced_updates.strip().split('|') + # Require that the precomplete file is included in the complete update + if "precomplete" in to_file_set: + forced_list.append("precomplete") + elif "Contents/Resources/precomplete" in to_file_set: + forced_list.append("Contents/Resources/precomplete") + # The check with \ file separators allows tests for Mac to run on Windows + elif "Contents\Resources\precomplete" in to_file_set: + forced_list.append("Contents\Resources\precomplete") + else: + raise Exception, "missing precomplete file in: "+to_dir_path + + if "removed-files" in to_file_set: + forced_list.append("removed-files") + elif "Contents/Resources/removed-files" in to_file_set: + forced_list.append("Contents/Resources/removed-files") + # The check with \ file separators allows tests for Mac to run on Windows + elif "Contents\Resources\\removed-files" in to_file_set: + forced_list.append("Contents\Resources\\removed-files") + else: + raise Exception, "missing removed-files file in: "+to_dir_path + + if "chrome.manifest" in to_file_set: + forced_list.append("chrome.manifest") + elif "Contents/Resources/chrome.manifest" in to_file_set: + forced_list.append("Contents/Resources/chrome.manifest") + # The check with \ file separators allows tests for Mac to run on Windows + elif "Contents\Resources\\chrome.manifest" in to_file_set: + forced_list.append("Contents\Resources\\chrome.manifest") + else: + raise Exception, "missing chrome.manifest file in: "+to_dir_path + + # Files which exist in both sets need to be patched + patch_filenames = list(from_file_set.intersection(to_file_set)) + patch_filenames.sort(reverse=True) + for filename in patch_filenames: + from_marfile_entry = from_dir_hash[filename] + to_marfile_entry = to_dir_hash[filename] + if os.path.basename(filename) in add_if_not_list: + # This filename is in the add if not list, explicitly add-if-not + create_add_if_not_patch_for_file(to_dir_hash[filename], patch_info) + elif filename in forced_list: + print 'Forcing "'+filename+'"' + # This filename is in the forced list, explicitly add + create_add_patch_for_file(to_dir_hash[filename], patch_info) + else: + if from_marfile_entry.sha() != to_marfile_entry.sha(): + # Not the same - calculate a patch + create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, patch_info) + + # files in to_dir not in from_dir need to added + add_filenames = list(to_file_set - from_file_set) + add_filenames.sort(reverse=True) + for filename in add_filenames: + if os.path.basename(filename) in add_if_not_list: + create_add_if_not_patch_for_file(to_dir_hash[filename], patch_info) + else: + create_add_patch_for_file(to_dir_hash[filename], patch_info) + + # files in from_dir not in to_dir need to be removed + remove_filenames = list(from_file_set - to_file_set) + remove_filenames.sort(reverse=True) + for filename in remove_filenames: + patch_info.append_remove_instruction(from_dir_hash[filename].name) + + process_explicit_remove_files(to_dir_path, patch_info) + + # directories in from_dir not in to_dir need to be removed + remove_dirnames = list(from_dir_set - to_dir_set) + remove_dirnames.sort(reverse=True) + for dirname in remove_dirnames: + patch_info.append_remove_instruction(from_dir_hash[dirname].name) + + # Construct the Manifest files + patch_info.create_manifest_files() + + # And construct the mar + mar_cmd = 'mar -C '+patch_info.work_dir+' -c output.mar '+string.join(patch_info.archive_files, ' ') + exec_shell_cmd(mar_cmd) + + # Copy mar to final destination + patch_file_dir = os.path.split(patch_filename)[0] + if not os.path.exists(patch_file_dir): + os.makedirs(patch_file_dir) + shutil.copy2(os.path.join(patch_info.work_dir,"output.mar"), patch_filename) + + return patch_filename + +def usage(): + print "-h for help" + print "-f for patchlist_file" + +def get_buildid(work_dir): + """ extracts buildid from MAR + """ + ini = '%s/application.ini' % work_dir + if not os.path.exists(ini): + ini = '%s/Contents/Resources/application.ini' % work_dir + if not os.path.exists(ini): + print 'WARNING: application.ini not found, cannot find build ID' + return '' + + file = bz2.BZ2File(ini) + for line in file: + if line.find('BuildID') == 0: + return line.strip().split('=')[1] + print 'WARNING: cannot find build ID in application.ini' + return '' + +def decode_filename(filepath): + """ Breaks filename/dir structure into component parts based on regex + for example: firefox-3.0b3pre.en-US.linux-i686.complete.mar + Or linux-i686/en-US/firefox-3.0b3.complete.mar + Returns dict with keys product, version, locale, platform, type + """ + try: + m = re.search( + '(?P<product>\w+)(-)(?P<version>\w+\.\w+(\.\w+){0,2})(\.)(?P<locale>.+?)(\.)(?P<platform>.+?)(\.)(?P<type>\w+)(.mar)', + os.path.basename(filepath)) + return m.groupdict() + except Exception, exc: + try: + m = re.search( + '(?P<platform>.+?)\/(?P<locale>.+?)\/(?P<product>\w+)-(?P<version>\w+\.\w+)\.(?P<type>\w+).mar', + filepath) + return m.groupdict() + except: + raise Exception("could not parse filepath %s: %s" % (filepath, exc)) + +def create_partial_patches(patches): + """ Given the patches generates a set of partial patches""" + shas = {} + + work_dir_root = None + metadata = [] + try: + work_dir_root = tempfile.mkdtemp('-fastmode', 'tmp', os.getcwd()) + print "Building patches using work dir: %s" % (work_dir_root) + + # Iterate through every patch set in the patch file + patch_num = 1 + for patch in patches: + startTime = time.time() + + from_filename,to_filename,patch_filename,forced_updates = patch.split(",") + from_filename,to_filename,patch_filename = os.path.abspath(from_filename),os.path.abspath(to_filename),os.path.abspath(patch_filename) + + # Each patch iteration uses its own work dir + work_dir = os.path.join(work_dir_root,str(patch_num)) + os.mkdir(work_dir) + + # Extract from mar into from dir + work_dir_from = os.path.join(work_dir,"from"); + os.mkdir(work_dir_from) + extract_mar(from_filename,work_dir_from) + from_decoded = decode_filename(from_filename) + from_buildid = get_buildid(work_dir_from) + from_shasum = sha.sha(open(from_filename).read()).hexdigest() + from_size = str(os.path.getsize(to_filename)) + + # Extract to mar into to dir + work_dir_to = os.path.join(work_dir,"to") + os.mkdir(work_dir_to) + extract_mar(to_filename, work_dir_to) + to_decoded = decode_filename(from_filename) + to_buildid = get_buildid(work_dir_to) + to_shasum = sha.sha(open(to_filename).read()).hexdigest() + to_size = str(os.path.getsize(to_filename)) + + mar_extract_time = time.time() + + partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, ['update.manifest','updatev2.manifest','updatev3.manifest'],[]),forced_updates,['channel-prefs.js','update-settings.ini']) + partial_buildid = to_buildid + partial_shasum = sha.sha(open(partial_filename).read()).hexdigest() + partial_size = str(os.path.getsize(partial_filename)) + + metadata.append({ + 'to_filename': os.path.basename(to_filename), + 'from_filename': os.path.basename(from_filename), + 'partial_filename': os.path.basename(partial_filename), + 'to_buildid':to_buildid, + 'from_buildid':from_buildid, + 'to_sha1sum':to_shasum, + 'from_sha1sum':from_shasum, + 'partial_sha1sum':partial_shasum, + 'to_size':to_size, + 'from_size':from_size, + 'partial_size':partial_size, + 'to_version':to_decoded['version'], + 'from_version':from_decoded['version'], + 'locale':from_decoded['locale'], + 'platform':from_decoded['platform'], + }) + print "done with patch %s/%s time (%.2fs/%.2fs/%.2fs) (mar/patch/total)" % (str(patch_num),str(len(patches)),mar_extract_time-startTime,time.time()-mar_extract_time,time.time()-startTime) + patch_num += 1 + return metadata + finally: + # If we fail or get a ctrl-c during run be sure to clean up temp dir + if (work_dir_root and os.path.exists(work_dir_root)): + shutil.rmtree(work_dir_root) + +def main(argv): + patchlist_file = None + try: + opts, args = getopt.getopt(argv, "hf:", ["help", "patchlist_file="]) + for opt, arg in opts: + if opt in ("-h", "--help"): + usage() + sys.exit() + elif opt in ("-f", "--patchlist_file"): + patchlist_file = arg + except getopt.GetoptError: + usage() + sys.exit(2) + + if not patchlist_file: + usage() + sys.exit(2) + + patches = [] + f = open(patchlist_file, 'r') + for line in f.readlines(): + patches.append(line) + f.close() + create_partial_patches(patches) + +if __name__ == "__main__": + main(sys.argv[1:]) + diff --git a/tools/update-packaging/moz.build b/tools/update-packaging/moz.build new file mode 100644 index 000000000..28919c271 --- /dev/null +++ b/tools/update-packaging/moz.build @@ -0,0 +1,6 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + diff --git a/tools/update-packaging/test/buildrefmars.sh b/tools/update-packaging/test/buildrefmars.sh new file mode 100755 index 000000000..fd2d5384d --- /dev/null +++ b/tools/update-packaging/test/buildrefmars.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Builds all the reference mars + +if [ -f "ref.mar" ]; then + rm "ref.mar" +fi +if [ -f "ref-mac.mar" ]; then + rm "ref-mac.mar" +fi + + ../make_incremental_update.sh ref.mar `pwd`/from `pwd`/to + ../make_incremental_update.sh ref-mac.mar `pwd`/from-mac `pwd`/to-mac + +if [ -f "product-1.0.lang.platform.complete.mar" ]; then + rm "product-1.0.lang.platform.complete.mar" +fi +if [ -f "product-2.0.lang.platform.complete.mar" ]; then + rm "product-2.0.lang.platform.complete.mar" +fi +if [ -f "product-2.0.lang.mac.complete.mar" ]; then + rm "product-2.0.lang.mac.complete.mar" +fi + +./make_full_update.sh product-1.0.lang.platform.complete.mar "`pwd`/from" +./make_full_update.sh product-2.0.lang.platform.complete.mar "`pwd`/to" +./make_full_update.sh product-1.0.lang.mac.complete.mar "`pwd`/from-mac" +./make_full_update.sh product-2.0.lang.mac.complete.mar "`pwd`/to-mac" diff --git a/tools/update-packaging/test/catmanifest.sh b/tools/update-packaging/test/catmanifest.sh new file mode 100755 index 000000000..354c3ff41 --- /dev/null +++ b/tools/update-packaging/test/catmanifest.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# helper tool for testing. Cats the manifest out of a mar file + +mar="$1" +workdir="/tmp/catmanifest" + +rm -rf "$workdir" +mkdir -p "$workdir" +cp "$1" "$workdir" +cd "$workdir" +mar -x "$1" +mv updatev2.manifest updatev2.manifest.bz2 +bzip2 -d updatev2.manifest.bz2 +cat updatev2.manifest diff --git a/tools/update-packaging/test/common.sh b/tools/update-packaging/test/common.sh new file mode 100755 index 000000000..7c133724e --- /dev/null +++ b/tools/update-packaging/test/common.sh @@ -0,0 +1,202 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# Code shared by update packaging scripts. +# Author: Darin Fisher +# +# In here to use the local common.sh to allow the full mars to have unfiltered files + +# ----------------------------------------------------------------------------- +# By default just assume that these tools exist on our path +MAR=${MAR:-mar} +BZIP2=${BZIP2:-bzip2} +MBSDIFF=${MBSDIFF:-mbsdiff} + +# ----------------------------------------------------------------------------- +# Helper routines + +notice() { + echo "$*" 1>&2 +} + +get_file_size() { + info=($(ls -ln "$1")) + echo ${info[4]} +} + +copy_perm() { + reference="$1" + target="$2" + + if [ -x "$reference" ]; then + chmod 0755 "$target" + else + chmod 0644 "$target" + fi +} + +make_add_instruction() { + f="$1" + filev2="$2" + # The third param will be an empty string when a file add instruction is only + # needed in the version 2 manifest. This only happens when the file has an + # add-if-not instruction in the version 3 manifest. This is due to the + # precomplete file prior to the version 3 manifest having a remove instruction + # for this file so the file is removed before applying a complete update. + filev3="$3" + + # Used to log to the console + if [ $4 ]; then + forced=" (forced)" + else + forced= + fi + + is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') + if [ $is_extension = "1" ]; then + # Use the subdirectory of the extensions folder as the file to test + # before performing this add instruction. + testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') + notice " add-if \"$testdir\" \"$f\"" + echo "add-if \"$testdir\" \"$f\"" >> $filev2 + if [ ! $filev3 = "" ]; then + echo "add-if \"$testdir\" \"$f\"" >> $filev3 + fi + else + notice " add \"$f\"$forced" + echo "add \"$f\"" >> $filev2 + if [ ! $filev3 = "" ]; then + echo "add \"$f\"" >> $filev3 + fi + fi +} + +check_for_add_if_not_update() { + add_if_not_file_chk="$1" + + if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \ + `basename $add_if_not_file_chk` = "update-settings.ini" ]; then + ## "true" *giggle* + return 0; + fi + ## 'false'... because this is bash. Oh yay! + return 1; +} + +check_for_add_to_manifestv2() { + add_if_not_file_chk="$1" + + if [ `basename $add_if_not_file_chk` = "update-settings.ini" ]; then + ## "true" *giggle* + return 0; + fi + ## 'false'... because this is bash. Oh yay! + return 1; +} + +make_add_if_not_instruction() { + f="$1" + filev3="$2" + + notice " add-if-not \"$f\" \"$f\"" + echo "add-if-not \"$f\" \"$f\"" >> $filev3 +} + +make_patch_instruction() { + f="$1" + filev2="$2" + filev3="$3" + + is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') + if [ $is_extension = "1" ]; then + # Use the subdirectory of the extensions folder as the file to test + # before performing this add instruction. + testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') + notice " patch-if \"$testdir\" \"$f.patch\" \"$f\"" + echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev2 + echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev3 + else + notice " patch \"$f.patch\" \"$f\"" + echo "patch \"$f.patch\" \"$f\"" >> $filev2 + echo "patch \"$f.patch\" \"$f\"" >> $filev3 + fi +} + +append_remove_instructions() { + dir="$1" + filev2="$2" + filev3="$3" + + if [ -f "$dir/removed-files" ]; then + listfile="$dir/removed-files" + elif [ -f "$dir/Contents/Resources/removed-files" ]; then + listfile="$dir/Contents/Resources/removed-files" + fi + if [ -n "$listfile" ]; then + # Map spaces to pipes so that we correctly handle filenames with spaces. + files=($(cat "$listfile" | tr " " "|" | sort -r)) + num_files=${#files[*]} + for ((i=0; $i<$num_files; i=$i+1)); do + # Map pipes back to whitespace and remove carriage returns + f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r') + # Trim whitespace + f=$(echo $f) + # Exclude blank lines. + if [ -n "$f" ]; then + # Exclude comments + if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then + if [ $(echo "$f" | grep -c '\/$') = 1 ]; then + notice " rmdir \"$f\"" + echo "rmdir \"$f\"" >> $filev2 + echo "rmdir \"$f\"" >> $filev3 + elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then + # Remove the * + f=$(echo "$f" | sed -e 's:\*$::') + notice " rmrfdir \"$f\"" + echo "rmrfdir \"$f\"" >> $filev2 + echo "rmrfdir \"$f\"" >> $filev3 + else + notice " remove \"$f\"" + echo "remove \"$f\"" >> $filev2 + echo "remove \"$f\"" >> $filev3 + fi + fi + fi + done + fi +} + +# List all files in the current directory, stripping leading "./" +# Pass a variable name and it will be filled as an array. +list_files() { + count=0 + + # Removed the exclusion cases here to allow for generation of testing mars + find . -type f \ + | sed 's/\.\/\(.*\)/\1/' \ + | sort -r > "$workdir/temp-filelist" + while read file; do + eval "${1}[$count]=\"$file\"" + (( count++ )) + done < "$workdir/temp-filelist" + rm "$workdir/temp-filelist" +} + +# List all directories in the current directory, stripping leading "./" +list_dirs() { + count=0 + + find . -type d \ + ! -name "." \ + ! -name ".." \ + | sed 's/\.\/\(.*\)/\1/' \ + | sort -r > "$workdir/temp-dirlist" + while read dir; do + eval "${1}[$count]=\"$dir\"" + (( count++ )) + done < "$workdir/temp-dirlist" + rm "$workdir/temp-dirlist" +} diff --git a/tools/update-packaging/test/diffmar.sh b/tools/update-packaging/test/diffmar.sh new file mode 100755 index 000000000..771b6b0ce --- /dev/null +++ b/tools/update-packaging/test/diffmar.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Compares two mars + +marA="$1" +marB="$2" +testDir="$3" +workdir="/tmp/diffmar/$testDir" +fromdir="$workdir/0" +todir="$workdir/1" + +# On Windows, creation time can be off by a second or more between the files in +# the fromdir and todir due to them being extracted synchronously so use +# time-style and exclude seconds from the creation time. +lsargs="-algR" +unamestr=`uname` +if [ ! "$unamestr" = 'Darwin' ]; then + unamestr=`uname -o` + if [ "$unamestr" = 'Msys' -o "$unamestr" = "Cygwin" ]; then + lsargs="-algR --time-style=+%Y-%m-%d-%H:%M" + fi +fi + +rm -rf "$workdir" +mkdir -p "$fromdir" +mkdir -p "$todir" + +cp "$1" "$fromdir" +cp "$2" "$todir" + +cd "$fromdir" +mar -x "$1" +rm "$1" +mv updatev2.manifest updatev2.manifest.bz2 +bzip2 -d updatev2.manifest.bz2 +mv updatev3.manifest updatev3.manifest.bz2 +bzip2 -d updatev3.manifest.bz2 +ls $lsargs > files.txt + +cd "$todir" +mar -x "$2" +rm "$2" +mv updatev2.manifest updatev2.manifest.bz2 +bzip2 -d updatev2.manifest.bz2 +mv updatev3.manifest updatev3.manifest.bz2 +bzip2 -d updatev3.manifest.bz2 +ls $lsargs > files.txt + +echo "diffing $fromdir and $todir" +echo "on linux shell sort and python sort return different results" +echo "which can cause differences in the manifest files" +diff -ru "$fromdir" "$todir" diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from-mac/Contents/MacOS/diff-patch-larger-than-file.txt new file mode 100644 index 000000000..8098d2585 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/force.txt b/tools/update-packaging/test/from-mac/Contents/MacOS/force.txt new file mode 100644 index 000000000..0ed0d5012 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/removed.txt b/tools/update-packaging/test/from-mac/Contents/MacOS/removed.txt new file mode 100644 index 000000000..2c3f0b340 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/removed.txt @@ -0,0 +1 @@ +removed diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/same.bin b/tools/update-packaging/test/from-mac/Contents/MacOS/same.bin Binary files differnew file mode 100644 index 000000000..a9ee7258c --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/same.bin diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/update.manifest b/tools/update-packaging/test/from-mac/Contents/MacOS/update.manifest new file mode 100644 index 000000000..d3e8ed851 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/update.manifest @@ -0,0 +1 @@ +from file shouldn't go in update diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/diff-patch-larger-than-file.txt new file mode 100644 index 000000000..8098d2585 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/readme.txt b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/readme.txt new file mode 100644 index 000000000..d7c40c63c --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/readme.txt @@ -0,0 +1 @@ +This from file should be ignored diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/removed.txt b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/removed.txt new file mode 100644 index 000000000..2c3f0b340 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/removed.txt @@ -0,0 +1 @@ +removed diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/same.bin b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/same.bin Binary files differnew file mode 100644 index 000000000..a9ee7258c --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/same.bin diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/same.txt b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/same.txt new file mode 100644 index 000000000..0ed0d5012 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/update.manifest b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/update.manifest new file mode 100644 index 000000000..d3e8ed851 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/update.manifest @@ -0,0 +1 @@ +from file shouldn't go in update diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/application.ini b/tools/update-packaging/test/from-mac/Contents/Resources/application.ini new file mode 100644 index 000000000..942e91a16 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/application.ini @@ -0,0 +1,5 @@ +[App] +Vendor=Mozilla +Name=MarTest +Version=1 +BuildID=20120101010101 diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/distribution/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from-mac/Contents/Resources/distribution/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 000000000..8098d2585 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/distribution/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from-mac/Contents/Resources/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 000000000..8098d2585 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/precomplete b/tools/update-packaging/test/from-mac/Contents/Resources/precomplete new file mode 100644 index 000000000..2d9068d37 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/precomplete @@ -0,0 +1,26 @@ +remove "Contents/MacOS/{foodir/update.manifest" +remove "Contents/MacOS/{foodir/same.txt" +remove "Contents/MacOS/{foodir/same.bin" +remove "Contents/MacOS/{foodir/removed.txt" +remove "Contents/MacOS/{foodir/readme.txt" +remove "Contents/MacOS/{foodir/force.txt" +remove "Contents/MacOS/{foodir/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/update.manifest" +remove "Contents/MacOS/searchplugins/diff/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/same.txt" +remove "Contents/MacOS/same.bin" +remove "Contents/MacOS/removed.txt" +remove "Contents/MacOS/readme.txt" +remove "Contents/MacOS/force.txt" +remove "Contents/MacOS/extensions/diff/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/application.ini" +remove "Contents/Resources/precomplete" +rmdir "Contents/MacOS/{foodir/" +rmdir "Contents/MacOS/searchplugins/diff/" +rmdir "Contents/MacOS/searchplugins/" +rmdir "Contents/MacOS/extensions/diff/" +rmdir "Contents/MacOS/extensions/" +rmdir "Contents/MacOS/" +rmdir "Contents/Resources/" +rmdir "Contents/" diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/readme.txt b/tools/update-packaging/test/from-mac/Contents/Resources/readme.txt new file mode 100644 index 000000000..b1a96f1fe --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/readme.txt @@ -0,0 +1,2 @@ +This from file should be ignored + diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/removed-files b/tools/update-packaging/test/from-mac/Contents/Resources/removed-files new file mode 100644 index 000000000..5bbdac6f6 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/removed-files @@ -0,0 +1,8 @@ +Contents/Resources/removed1.txt +Contents/MacOS/removed2.bin +Contents/MacOS/recursivedir/meh/* +Contents/Resources/dir/ +Contents/MacOS/this file has spaces + + +Contents/MacOS/extra-spaces diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/removed.txt b/tools/update-packaging/test/from-mac/Contents/Resources/removed.txt new file mode 100644 index 000000000..2c3f0b340 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/removed.txt @@ -0,0 +1 @@ +removed diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/same.txt b/tools/update-packaging/test/from-mac/Contents/Resources/same.txt new file mode 100644 index 000000000..0ed0d5012 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/searchplugins/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from-mac/Contents/Resources/searchplugins/diff/diff-patch-larger-than-file.txt new file mode 100644 index 000000000..8098d2585 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/searchplugins/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/update-settings.ini b/tools/update-packaging/test/from-mac/Contents/Resources/update-settings.ini new file mode 100644 index 000000000..5fa6a9909 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/update-settings.ini @@ -0,0 +1 @@ +add-if-not from complete file diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/{foodir/channel-prefs.js b/tools/update-packaging/test/from-mac/Contents/Resources/{foodir/channel-prefs.js new file mode 100644 index 000000000..5fa6a9909 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/{foodir/channel-prefs.js @@ -0,0 +1 @@ +add-if-not from complete file diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/{foodir/force.txt b/tools/update-packaging/test/from-mac/Contents/Resources/{foodir/force.txt new file mode 100644 index 000000000..0ed0d5012 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/{foodir/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from/application.ini b/tools/update-packaging/test/from/application.ini new file mode 100644 index 000000000..942e91a16 --- /dev/null +++ b/tools/update-packaging/test/from/application.ini @@ -0,0 +1,5 @@ +[App] +Vendor=Mozilla +Name=MarTest +Version=1 +BuildID=20120101010101 diff --git a/tools/update-packaging/test/from/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from/diff-patch-larger-than-file.txt new file mode 100644 index 000000000..8098d2585 --- /dev/null +++ b/tools/update-packaging/test/from/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from/distribution/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from/distribution/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 000000000..8098d2585 --- /dev/null +++ b/tools/update-packaging/test/from/distribution/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 000000000..8098d2585 --- /dev/null +++ b/tools/update-packaging/test/from/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from/force.txt b/tools/update-packaging/test/from/force.txt new file mode 100644 index 000000000..0ed0d5012 --- /dev/null +++ b/tools/update-packaging/test/from/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from/precomplete b/tools/update-packaging/test/from/precomplete new file mode 100644 index 000000000..e27f4fc31 --- /dev/null +++ b/tools/update-packaging/test/from/precomplete @@ -0,0 +1,23 @@ +remove "{foodir/update.manifest" +remove "{foodir/same.txt" +remove "{foodir/same.bin" +remove "{foodir/removed.txt" +remove "{foodir/readme.txt" +remove "{foodir/force.txt" +remove "{foodir/diff-patch-larger-than-file.txt" +remove "update.manifest" +remove "searchplugins/diff/diff-patch-larger-than-file.txt" +remove "same.txt" +remove "same.bin" +remove "removed.txt" +remove "readme.txt" +remove "precomplete" +remove "force.txt" +remove "extensions/diff/diff-patch-larger-than-file.txt" +remove "diff-patch-larger-than-file.txt" +remove "application.ini" +rmdir "{foodir/" +rmdir "searchplugins/diff/" +rmdir "searchplugins/" +rmdir "extensions/diff/" +rmdir "extensions/" diff --git a/tools/update-packaging/test/from/readme.txt b/tools/update-packaging/test/from/readme.txt new file mode 100644 index 000000000..b1a96f1fe --- /dev/null +++ b/tools/update-packaging/test/from/readme.txt @@ -0,0 +1,2 @@ +This from file should be ignored + diff --git a/tools/update-packaging/test/from/removed-files b/tools/update-packaging/test/from/removed-files new file mode 100644 index 000000000..73b348d9c --- /dev/null +++ b/tools/update-packaging/test/from/removed-files @@ -0,0 +1,8 @@ +removed1.txt +removed2.bin +recursivedir/meh/* +dir/ +this file has spaces + + +extra-spaces diff --git a/tools/update-packaging/test/from/removed.txt b/tools/update-packaging/test/from/removed.txt new file mode 100644 index 000000000..2c3f0b340 --- /dev/null +++ b/tools/update-packaging/test/from/removed.txt @@ -0,0 +1 @@ +removed diff --git a/tools/update-packaging/test/from/same.bin b/tools/update-packaging/test/from/same.bin Binary files differnew file mode 100644 index 000000000..a9ee7258c --- /dev/null +++ b/tools/update-packaging/test/from/same.bin diff --git a/tools/update-packaging/test/from/same.txt b/tools/update-packaging/test/from/same.txt new file mode 100644 index 000000000..0ed0d5012 --- /dev/null +++ b/tools/update-packaging/test/from/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from/searchplugins/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from/searchplugins/diff/diff-patch-larger-than-file.txt new file mode 100644 index 000000000..8098d2585 --- /dev/null +++ b/tools/update-packaging/test/from/searchplugins/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from/update-settings.ini b/tools/update-packaging/test/from/update-settings.ini new file mode 100644 index 000000000..5fa6a9909 --- /dev/null +++ b/tools/update-packaging/test/from/update-settings.ini @@ -0,0 +1 @@ +add-if-not from complete file diff --git a/tools/update-packaging/test/from/update.manifest b/tools/update-packaging/test/from/update.manifest new file mode 100644 index 000000000..d3e8ed851 --- /dev/null +++ b/tools/update-packaging/test/from/update.manifest @@ -0,0 +1 @@ +from file shouldn't go in update diff --git a/tools/update-packaging/test/from/{foodir/channel-prefs.js b/tools/update-packaging/test/from/{foodir/channel-prefs.js new file mode 100644 index 000000000..5fa6a9909 --- /dev/null +++ b/tools/update-packaging/test/from/{foodir/channel-prefs.js @@ -0,0 +1 @@ +add-if-not from complete file diff --git a/tools/update-packaging/test/from/{foodir/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from/{foodir/diff-patch-larger-than-file.txt new file mode 100644 index 000000000..8098d2585 --- /dev/null +++ b/tools/update-packaging/test/from/{foodir/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from/{foodir/force.txt b/tools/update-packaging/test/from/{foodir/force.txt new file mode 100644 index 000000000..0ed0d5012 --- /dev/null +++ b/tools/update-packaging/test/from/{foodir/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from/{foodir/readme.txt b/tools/update-packaging/test/from/{foodir/readme.txt new file mode 100644 index 000000000..d7c40c63c --- /dev/null +++ b/tools/update-packaging/test/from/{foodir/readme.txt @@ -0,0 +1 @@ +This from file should be ignored diff --git a/tools/update-packaging/test/from/{foodir/removed.txt b/tools/update-packaging/test/from/{foodir/removed.txt new file mode 100644 index 000000000..2c3f0b340 --- /dev/null +++ b/tools/update-packaging/test/from/{foodir/removed.txt @@ -0,0 +1 @@ +removed diff --git a/tools/update-packaging/test/from/{foodir/same.bin b/tools/update-packaging/test/from/{foodir/same.bin Binary files differnew file mode 100644 index 000000000..a9ee7258c --- /dev/null +++ b/tools/update-packaging/test/from/{foodir/same.bin diff --git a/tools/update-packaging/test/from/{foodir/same.txt b/tools/update-packaging/test/from/{foodir/same.txt new file mode 100644 index 000000000..0ed0d5012 --- /dev/null +++ b/tools/update-packaging/test/from/{foodir/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from/{foodir/update.manifest b/tools/update-packaging/test/from/{foodir/update.manifest new file mode 100644 index 000000000..d3e8ed851 --- /dev/null +++ b/tools/update-packaging/test/from/{foodir/update.manifest @@ -0,0 +1 @@ +from file shouldn't go in update diff --git a/tools/update-packaging/test/make_full_update.sh b/tools/update-packaging/test/make_full_update.sh new file mode 100755 index 000000000..f2907da9e --- /dev/null +++ b/tools/update-packaging/test/make_full_update.sh @@ -0,0 +1,119 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# This tool generates full update packages for the update system. +# Author: Darin Fisher +# +# In here to use the local common.sh to allow the full mars to have unfiltered files + +. $(dirname "$0")/common.sh + +# ----------------------------------------------------------------------------- + +print_usage() { + notice "Usage: $(basename $0) [OPTIONS] ARCHIVE DIRECTORY" +} + +if [ $# = 0 ]; then + print_usage + exit 1 +fi + +if [ $1 = -h ]; then + print_usage + notice "" + notice "The contents of DIRECTORY will be stored in ARCHIVE." + notice "" + notice "Options:" + notice " -h show this help text" + notice "" + exit 1 +fi + +# ----------------------------------------------------------------------------- + +archive="$1" +targetdir="$2" +# Prevent the workdir from being inside the targetdir so it isn't included in +# the update mar. +if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then + # Remove the / + targetdir=$(echo "$targetdir" | sed -e 's:\/$::') +fi +workdir="$targetdir.work" +updatemanifestv2="$workdir/updatev2.manifest" +updatemanifestv3="$workdir/updatev3.manifest" +targetfiles="updatev2.manifest updatev3.manifest" + +mkdir -p "$workdir" + +# Generate a list of all files in the target directory. +pushd "$targetdir" +if test $? -ne 0 ; then + exit 1 +fi + +if [ ! -f "precomplete" ]; then + if [ ! -f "Contents/Resources/precomplete" ]; then + notice "precomplete file is missing!" + exit 1 + fi +fi + +list_files files + +popd + +# Add the type of update to the beginning of the update manifests. +> $updatemanifestv2 +> $updatemanifestv3 +notice "" +notice "Adding type instruction to update manifests" +notice " type complete" +echo "type \"complete\"" >> $updatemanifestv2 +echo "type \"complete\"" >> $updatemanifestv3 + +notice "" +notice "Adding file add instructions to update manifests" +num_files=${#files[*]} + +for ((i=0; $i<$num_files; i=$i+1)); do + f="${files[$i]}" + + if check_for_add_if_not_update "$f"; then + make_add_if_not_instruction "$f" "$updatemanifestv3" + if check_for_add_to_manifestv2 "$f"; then + make_add_instruction "$f" "$updatemanifestv2" "" 1 + fi + else + make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" + fi + + dir=$(dirname "$f") + mkdir -p "$workdir/$dir" + $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f" + copy_perm "$targetdir/$f" "$workdir/$f" + + targetfiles="$targetfiles \"$f\"" +done + +# Append remove instructions for any dead files. +notice "" +notice "Adding file and directory remove instructions from file 'removed-files'" +append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3" + +$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2" +$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3" + +eval "$MAR -C \"$workdir\" -c output.mar $targetfiles" +mv -f "$workdir/output.mar" "$archive" + +# cleanup +rm -fr "$workdir" + +notice "" +notice "Finished" +notice "" diff --git a/tools/update-packaging/test/runtests.sh b/tools/update-packaging/test/runtests.sh new file mode 100755 index 000000000..c66e54eaa --- /dev/null +++ b/tools/update-packaging/test/runtests.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +echo "testing make_incremental_updates.py" +python ../make_incremental_updates.py -f testpatchfile.txt + +echo "" +echo "diffing ref.mar and test.mar" +./diffmar.sh ref.mar test.mar test + +echo "" +echo "diffing ref-mac.mar and test-mac.mar" +./diffmar.sh ref-mac.mar test-mac.mar test-mac diff --git a/tools/update-packaging/test/testpatchfile.txt b/tools/update-packaging/test/testpatchfile.txt new file mode 100644 index 000000000..a19c831eb --- /dev/null +++ b/tools/update-packaging/test/testpatchfile.txt @@ -0,0 +1,2 @@ +product-1.0.lang.platform.complete.mar,product-2.0.lang.platform.complete.mar,test.mar,"" +product-1.0.lang.mac.complete.mar,product-2.0.lang.mac.complete.mar,test-mac.mar,"" diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/addFeedPrefs.js b/tools/update-packaging/test/to-mac/Contents/MacOS/addFeedPrefs.js new file mode 100644 index 000000000..3b2aed8e0 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/addFeedPrefs.js @@ -0,0 +1 @@ +this is a new file diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/added.txt b/tools/update-packaging/test/to-mac/Contents/MacOS/added.txt new file mode 100644 index 000000000..b242c3606 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/added.txt @@ -0,0 +1 @@ +added file diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/diff-patch-larger-than-file.bin b/tools/update-packaging/test/to-mac/Contents/MacOS/diff-patch-larger-than-file.bin Binary files differnew file mode 100644 index 000000000..a9ee7258c --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/diff-patch-larger-than-file.bin diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to-mac/Contents/MacOS/diff-patch-larger-than-file.txt new file mode 100644 index 000000000..a61ffbb5e --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +file to diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/force.txt b/tools/update-packaging/test/to-mac/Contents/MacOS/force.txt new file mode 100644 index 000000000..0ed0d5012 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/same.bin b/tools/update-packaging/test/to-mac/Contents/MacOS/same.bin Binary files differnew file mode 100644 index 000000000..a9ee7258c --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/same.bin diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/update.manifest b/tools/update-packaging/test/to-mac/Contents/MacOS/update.manifest new file mode 100644 index 000000000..73364fdca --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/update.manifest @@ -0,0 +1 @@ +to file shouldn't go in update diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/added.txt b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/added.txt new file mode 100644 index 000000000..b242c3606 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/added.txt @@ -0,0 +1 @@ +added file diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/diff-patch-larger-than-file.txt new file mode 100644 index 000000000..a61ffbb5e --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +file to diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/readme.txt b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/readme.txt new file mode 100644 index 000000000..b5f7004cc --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/readme.txt @@ -0,0 +1 @@ +This to file should be ignored diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/same.bin b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/same.bin Binary files differnew file mode 100644 index 000000000..a9ee7258c --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/same.bin diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/same.txt b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/same.txt new file mode 100644 index 000000000..0ed0d5012 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/update.manifest b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/update.manifest new file mode 100644 index 000000000..73364fdca --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/update.manifest @@ -0,0 +1 @@ +to file shouldn't go in update diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/application.ini b/tools/update-packaging/test/to-mac/Contents/Resources/application.ini new file mode 100644 index 000000000..7bdc78819 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/application.ini @@ -0,0 +1,5 @@ +[App] +Vendor=Mozilla +Name=MarTest +Version=2 +BuildID=20130101010101 diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/distribution/extensions/added/file.txt b/tools/update-packaging/test/to-mac/Contents/Resources/distribution/extensions/added/file.txt new file mode 100644 index 000000000..4bbc6747e --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/distribution/extensions/added/file.txt @@ -0,0 +1 @@ +extfile diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/distribution/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to-mac/Contents/Resources/distribution/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 000000000..b779d9648 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/distribution/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +to file diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/extensions/added/file.txt b/tools/update-packaging/test/to-mac/Contents/Resources/extensions/added/file.txt new file mode 100644 index 000000000..4bbc6747e --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/extensions/added/file.txt @@ -0,0 +1 @@ +extfile diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to-mac/Contents/Resources/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 000000000..b779d9648 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +to file diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/precomplete b/tools/update-packaging/test/to-mac/Contents/Resources/precomplete new file mode 100644 index 000000000..7af8bfd76 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/precomplete @@ -0,0 +1,33 @@ +remove "Contents/MacOS/{foodir/update.manifest" +remove "Contents/MacOS/{foodir/same.txt" +remove "Contents/MacOS/{foodir/same.bin" +remove "Contents/MacOS/{foodir/readme.txt" +remove "Contents/MacOS/{foodir/force.txt" +remove "Contents/MacOS/{foodir/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/{foodir/added.txt" +remove "Contents/MacOS/update.manifest" +remove "Contents/MacOS/searchplugins/diff/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/searchplugins/added/file.txt" +remove "Contents/MacOS/same.txt" +remove "Contents/MacOS/same.bin" +remove "Contents/MacOS/removed-files" +remove "Contents/MacOS/readme.txt" +remove "Contents/MacOS/force.txt" +remove "Contents/MacOS/extensions/diff/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/extensions/added/file.txt" +remove "Contents/MacOS/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/diff-patch-larger-than-file.bin" +remove "Contents/MacOS/application.ini" +remove "Contents/MacOS/added.txt" +remove "Contents/MacOS/addFeedPrefs.js" +remove "Contents/Resources/precomplete" +rmdir "Contents/MacOS/{foodir/" +rmdir "Contents/MacOS/searchplugins/diff/" +rmdir "Contents/MacOS/searchplugins/added/" +rmdir "Contents/MacOS/searchplugins/" +rmdir "Contents/MacOS/extensions/diff/" +rmdir "Contents/MacOS/extensions/added/" +rmdir "Contents/MacOS/extensions/" +rmdir "Contents/MacOS/" +rmdir "Contents/Resources/" +rmdir "Contents/" diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/readme.txt b/tools/update-packaging/test/to-mac/Contents/Resources/readme.txt new file mode 100644 index 000000000..b5f7004cc --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/readme.txt @@ -0,0 +1 @@ +This to file should be ignored diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/removed-files b/tools/update-packaging/test/to-mac/Contents/Resources/removed-files new file mode 100644 index 000000000..a756cc560 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/removed-files @@ -0,0 +1,14 @@ +Contents/Resources/removed1.txt +Contents/MacOS/removed2.bin +Contents/MacOS/recursivedir/meh/* +Contents/MacOS/removed3-foo.txt +Contents/Resources/dir/ +Contents/MacOS/this file has spaces +Contents/MacOS/notherdir/ + + +Contents/Resources/extra-spaces + +Contents/MacOS/lastFile + + diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/same.txt b/tools/update-packaging/test/to-mac/Contents/Resources/same.txt new file mode 100644 index 000000000..0ed0d5012 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/searchplugins/added/file.txt b/tools/update-packaging/test/to-mac/Contents/Resources/searchplugins/added/file.txt new file mode 100644 index 000000000..4bbc6747e --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/searchplugins/added/file.txt @@ -0,0 +1 @@ +extfile diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/searchplugins/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to-mac/Contents/Resources/searchplugins/diff/diff-patch-larger-than-file.txt new file mode 100644 index 000000000..b779d9648 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/searchplugins/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +to file diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/update-settings.ini b/tools/update-packaging/test/to-mac/Contents/Resources/update-settings.ini new file mode 100644 index 000000000..5fa6a9909 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/update-settings.ini @@ -0,0 +1 @@ +add-if-not from complete file diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/{foodir/channel-prefs.js b/tools/update-packaging/test/to-mac/Contents/Resources/{foodir/channel-prefs.js new file mode 100644 index 000000000..d6ada4591 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/{foodir/channel-prefs.js @@ -0,0 +1 @@ +add-if-not from partial file diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/{foodir/force.txt b/tools/update-packaging/test/to-mac/Contents/Resources/{foodir/force.txt new file mode 100644 index 000000000..0ed0d5012 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/{foodir/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to/addFeedPrefs.js b/tools/update-packaging/test/to/addFeedPrefs.js new file mode 100644 index 000000000..3b2aed8e0 --- /dev/null +++ b/tools/update-packaging/test/to/addFeedPrefs.js @@ -0,0 +1 @@ +this is a new file diff --git a/tools/update-packaging/test/to/added.txt b/tools/update-packaging/test/to/added.txt new file mode 100644 index 000000000..b242c3606 --- /dev/null +++ b/tools/update-packaging/test/to/added.txt @@ -0,0 +1 @@ +added file diff --git a/tools/update-packaging/test/to/application.ini b/tools/update-packaging/test/to/application.ini new file mode 100644 index 000000000..7bdc78819 --- /dev/null +++ b/tools/update-packaging/test/to/application.ini @@ -0,0 +1,5 @@ +[App] +Vendor=Mozilla +Name=MarTest +Version=2 +BuildID=20130101010101 diff --git a/tools/update-packaging/test/to/diff-patch-larger-than-file.bin b/tools/update-packaging/test/to/diff-patch-larger-than-file.bin Binary files differnew file mode 100644 index 000000000..a9ee7258c --- /dev/null +++ b/tools/update-packaging/test/to/diff-patch-larger-than-file.bin diff --git a/tools/update-packaging/test/to/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to/diff-patch-larger-than-file.txt new file mode 100644 index 000000000..a61ffbb5e --- /dev/null +++ b/tools/update-packaging/test/to/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +file to diff --git a/tools/update-packaging/test/to/distribution/extensions/added/file.txt b/tools/update-packaging/test/to/distribution/extensions/added/file.txt new file mode 100644 index 000000000..4bbc6747e --- /dev/null +++ b/tools/update-packaging/test/to/distribution/extensions/added/file.txt @@ -0,0 +1 @@ +extfile diff --git a/tools/update-packaging/test/to/distribution/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to/distribution/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 000000000..b779d9648 --- /dev/null +++ b/tools/update-packaging/test/to/distribution/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +to file diff --git a/tools/update-packaging/test/to/extensions/added/file.txt b/tools/update-packaging/test/to/extensions/added/file.txt new file mode 100644 index 000000000..4bbc6747e --- /dev/null +++ b/tools/update-packaging/test/to/extensions/added/file.txt @@ -0,0 +1 @@ +extfile diff --git a/tools/update-packaging/test/to/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 000000000..b779d9648 --- /dev/null +++ b/tools/update-packaging/test/to/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +to file diff --git a/tools/update-packaging/test/to/force.txt b/tools/update-packaging/test/to/force.txt new file mode 100644 index 000000000..0ed0d5012 --- /dev/null +++ b/tools/update-packaging/test/to/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to/precomplete b/tools/update-packaging/test/to/precomplete new file mode 100644 index 000000000..c2700dd97 --- /dev/null +++ b/tools/update-packaging/test/to/precomplete @@ -0,0 +1,30 @@ +remove "{foodir/update.manifest" +remove "{foodir/same.txt" +remove "{foodir/same.bin" +remove "{foodir/readme.txt" +remove "{foodir/force.txt" +remove "{foodir/diff-patch-larger-than-file.txt" +remove "{foodir/added.txt" +remove "update.manifest" +remove "searchplugins/diff/diff-patch-larger-than-file.txt" +remove "searchplugins/added/file.txt" +remove "same.txt" +remove "same.bin" +remove "removed-files" +remove "readme.txt" +remove "precomplete" +remove "force.txt" +remove "extensions/diff/diff-patch-larger-than-file.txt" +remove "extensions/added/file.txt" +remove "diff-patch-larger-than-file.txt" +remove "diff-patch-larger-than-file.bin" +remove "application.ini" +remove "added.txt" +remove "addFeedPrefs.js" +rmdir "{foodir/" +rmdir "searchplugins/diff/" +rmdir "searchplugins/added/" +rmdir "searchplugins/" +rmdir "extensions/diff/" +rmdir "extensions/added/" +rmdir "extensions/" diff --git a/tools/update-packaging/test/to/readme.txt b/tools/update-packaging/test/to/readme.txt new file mode 100644 index 000000000..b5f7004cc --- /dev/null +++ b/tools/update-packaging/test/to/readme.txt @@ -0,0 +1 @@ +This to file should be ignored diff --git a/tools/update-packaging/test/to/removed-files b/tools/update-packaging/test/to/removed-files new file mode 100644 index 000000000..4fdfff7fd --- /dev/null +++ b/tools/update-packaging/test/to/removed-files @@ -0,0 +1,14 @@ +removed1.txt +removed2.bin +recursivedir/meh/* +removed3-foo.txt +dir/ +this file has spaces +notherdir/ + + +extra-spaces + +lastFile + + diff --git a/tools/update-packaging/test/to/same.bin b/tools/update-packaging/test/to/same.bin Binary files differnew file mode 100644 index 000000000..a9ee7258c --- /dev/null +++ b/tools/update-packaging/test/to/same.bin diff --git a/tools/update-packaging/test/to/same.txt b/tools/update-packaging/test/to/same.txt new file mode 100644 index 000000000..0ed0d5012 --- /dev/null +++ b/tools/update-packaging/test/to/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to/searchplugins/added/file.txt b/tools/update-packaging/test/to/searchplugins/added/file.txt new file mode 100644 index 000000000..4bbc6747e --- /dev/null +++ b/tools/update-packaging/test/to/searchplugins/added/file.txt @@ -0,0 +1 @@ +extfile diff --git a/tools/update-packaging/test/to/searchplugins/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to/searchplugins/diff/diff-patch-larger-than-file.txt new file mode 100644 index 000000000..b779d9648 --- /dev/null +++ b/tools/update-packaging/test/to/searchplugins/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +to file diff --git a/tools/update-packaging/test/to/update-settings.ini b/tools/update-packaging/test/to/update-settings.ini new file mode 100644 index 000000000..5fa6a9909 --- /dev/null +++ b/tools/update-packaging/test/to/update-settings.ini @@ -0,0 +1 @@ +add-if-not from complete file diff --git a/tools/update-packaging/test/to/update.manifest b/tools/update-packaging/test/to/update.manifest new file mode 100644 index 000000000..73364fdca --- /dev/null +++ b/tools/update-packaging/test/to/update.manifest @@ -0,0 +1 @@ +to file shouldn't go in update diff --git a/tools/update-packaging/test/to/{foodir/added.txt b/tools/update-packaging/test/to/{foodir/added.txt new file mode 100644 index 000000000..b242c3606 --- /dev/null +++ b/tools/update-packaging/test/to/{foodir/added.txt @@ -0,0 +1 @@ +added file diff --git a/tools/update-packaging/test/to/{foodir/channel-prefs.js b/tools/update-packaging/test/to/{foodir/channel-prefs.js new file mode 100644 index 000000000..5fa6a9909 --- /dev/null +++ b/tools/update-packaging/test/to/{foodir/channel-prefs.js @@ -0,0 +1 @@ +add-if-not from complete file diff --git a/tools/update-packaging/test/to/{foodir/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to/{foodir/diff-patch-larger-than-file.txt new file mode 100644 index 000000000..a61ffbb5e --- /dev/null +++ b/tools/update-packaging/test/to/{foodir/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +file to diff --git a/tools/update-packaging/test/to/{foodir/force.txt b/tools/update-packaging/test/to/{foodir/force.txt new file mode 100644 index 000000000..0ed0d5012 --- /dev/null +++ b/tools/update-packaging/test/to/{foodir/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to/{foodir/readme.txt b/tools/update-packaging/test/to/{foodir/readme.txt new file mode 100644 index 000000000..b5f7004cc --- /dev/null +++ b/tools/update-packaging/test/to/{foodir/readme.txt @@ -0,0 +1 @@ +This to file should be ignored diff --git a/tools/update-packaging/test/to/{foodir/same.bin b/tools/update-packaging/test/to/{foodir/same.bin Binary files differnew file mode 100644 index 000000000..a9ee7258c --- /dev/null +++ b/tools/update-packaging/test/to/{foodir/same.bin diff --git a/tools/update-packaging/test/to/{foodir/same.txt b/tools/update-packaging/test/to/{foodir/same.txt new file mode 100644 index 000000000..0ed0d5012 --- /dev/null +++ b/tools/update-packaging/test/to/{foodir/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to/{foodir/update.manifest b/tools/update-packaging/test/to/{foodir/update.manifest new file mode 100644 index 000000000..73364fdca --- /dev/null +++ b/tools/update-packaging/test/to/{foodir/update.manifest @@ -0,0 +1 @@ +to file shouldn't go in update diff --git a/tools/update-packaging/test_make_incremental_updates.py b/tools/update-packaging/test_make_incremental_updates.py new file mode 100644 index 000000000..016823dee --- /dev/null +++ b/tools/update-packaging/test_make_incremental_updates.py @@ -0,0 +1,151 @@ +#!/usr/bin/python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import unittest +import make_incremental_updates as mkup +from make_incremental_updates import PatchInfo, MarFileEntry + +class TestPatchInfo(unittest.TestCase): + def setUp(self): + self.work_dir = 'work_dir' + self.file_exclusion_list = ['update.manifest','updatev2.manifest','updatev3.manifest'] + self.path_exclusion_list = ['/readme.txt'] + self.patch_info = PatchInfo(self.work_dir, self.file_exclusion_list, self.path_exclusion_list) + + def testPatchInfo(self): + self.assertEquals(self.work_dir, self.patch_info.work_dir) + self.assertEquals([], self.patch_info.archive_files) + self.assertEquals([], self.patch_info.manifestv2) + self.assertEquals([], self.patch_info.manifestv3) + self.assertEquals(self.file_exclusion_list, self.patch_info.file_exclusion_list) + self.assertEquals(self.path_exclusion_list, self.patch_info.path_exclusion_list) + + def test_append_add_instruction(self): + self.patch_info.append_add_instruction('file.test') + self.assertEquals(['add "file.test"'], self.patch_info.manifestv2) + self.assertEquals(['add "file.test"'], self.patch_info.manifestv3) + + def test_append_add_if_instruction(self): + self.patch_info.append_add_instruction('distribution/extensions/extension/file.test') + self.assertEquals(['add-if "distribution/extensions/extension" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv2) + self.assertEquals(['add-if "distribution/extensions/extension" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv3) + + def test_append_add_if_not_instruction(self): + self.patch_info.append_add_if_not_instruction('file.test') + self.assertEquals([], self.patch_info.manifestv2) + self.assertEquals(['add-if-not "file.test" "file.test"'], self.patch_info.manifestv3) + + def test_append_patch_instruction(self): + self.patch_info.append_patch_instruction('file.test', 'patchname') + self.assertEquals(['patch "patchname" "file.test"'], self.patch_info.manifestv2) + self.assertEquals(['patch "patchname" "file.test"'], self.patch_info.manifestv3) + + def test_append_patch_if_instruction(self): + self.patch_info.append_patch_instruction('distribution/extensions/extension/file.test', 'patchname') + self.assertEquals(['patch-if "distribution/extensions/extension" "patchname" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv2) + self.assertEquals(['patch-if "distribution/extensions/extension" "patchname" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv3) + + def test_append_remove_instruction(self): + self.patch_info.append_remove_instruction('file.test') + self.assertEquals(['remove "file.test"'], self.patch_info.manifestv2) + self.assertEquals(['remove "file.test"'], self.patch_info.manifestv3) + + def test_append_rmdir_instruction(self): + self.patch_info.append_remove_instruction('dirtest/') + self.assertEquals(['rmdir "dirtest/"'], self.patch_info.manifestv2) + self.assertEquals(['rmdir "dirtest/"'], self.patch_info.manifestv3) + + def test_append_rmrfdir_instruction(self): + self.patch_info.append_remove_instruction('dirtest/*') + self.assertEquals(['rmrfdir "dirtest/"'], self.patch_info.manifestv2) + self.assertEquals(['rmrfdir "dirtest/"'], self.patch_info.manifestv3) + + """ FIXME touches the filesystem, need refactoring + def test_create_manifest_file(self): + self.patch_info.create_manifest_file() + """ + + def test_build_marfile_entry_hash(self): + self.assertEquals(({}, set([]), set([])), self.patch_info.build_marfile_entry_hash('root_path')) + +""" FIXME touches the filesystem, need refactoring +class TestMarFileEntry(unittest.TestCase): + def setUp(self): + root_path = '.' + self.filename = 'file.test' + f = open(self.filename, 'w') + f.write('Test data\n') + f.close() + self.mar_file_entry = MarFileEntry(root_path, self.filename) + + def test_calc_file_sha_digest(self): + f = open('test.sha', 'r') + goodSha = f.read() + f.close() + sha = self.mar_file_entry.calc_file_sha_digest(self.filename) + self.assertEquals(goodSha, sha) + + def test_sha(self): + f = open('test.sha', 'r') + goodSha = f.read() + f.close() + sha = self.mar_file_entry.sha() + self.assertEquals(goodSha, sha) +""" + +class TestMakeIncrementalUpdates(unittest.TestCase): + def setUp(self): + work_dir = '.' + self.patch_info = PatchInfo(work_dir, ['update.manifest','updatev2.manifest','updatev3.manifest'],['/readme.txt']) + root_path = '/' + filename = 'test.file' + self.mar_file_entry = MarFileEntry(root_path, filename) + + """ FIXME makes direct shell calls, need refactoring + def test_exec_shell_cmd(self): + mkup.exec_shell_cmd('echo test') + + def test_copy_file(self): + mkup.copy_file('src_file_abs_path', 'dst_file_abs_path') + + def test_bzip_file(self): + mkup.bzip_file('filename') + + def test_bunzip_file(self): + mkup.bunzip_file('filename') + + def test_extract_mar(self): + mkup.extract_mar('filename', 'work_dir') + + def test_create_partial_patch_for_file(self): + mkup.create_partial_patch_for_file('from_marfile_entry', 'to_marfile_entry', 'shas', self.patch_info) + + def test_create_add_patch_for_file(self): + mkup.create_add_patch_for_file('to_marfile_entry', self.patch_info) + + def test_process_explicit_remove_files(self): + mkup.process_explicit_remove_files('dir_path', self.patch_info) + + def test_create_partial_patch(self): + mkup.create_partial_patch('from_dir_path', 'to_dir_path', 'patch_filename', 'shas', self.patch_info, 'forced_updates') + + def test_create_partial_patches(patches): + mkup.create_partial_patches('patches') + + """ + + """ FIXME touches the filesystem, need refactoring + def test_get_buildid(self): + mkup.get_buildid('work_dir', 'platform') + """ + + def test_decode_filename(self): + expected = {'locale': 'lang', 'platform': 'platform', 'product': 'product', 'version': '1.0', 'type': 'complete'} + self.assertEquals(expected, mkup.decode_filename('product-1.0.lang.platform.complete.mar')) + self.assertRaises(Exception, mkup.decode_filename, 'fail') + +if __name__ == '__main__': + unittest.main() diff --git a/tools/update-packaging/unwrap_full_update.pl b/tools/update-packaging/unwrap_full_update.pl new file mode 100755 index 000000000..ead1157db --- /dev/null +++ b/tools/update-packaging/unwrap_full_update.pl @@ -0,0 +1,67 @@ +#!/usr/bin/perl -w +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# This tool unpacks a full update package generated by make_full_update.sh +# Author: Benjamin Smedberg +# + +# ----------------------------------------------------------------------------- +# By default just assume that these tools exist on our path + +use Getopt::Std; + +my ($MAR, $BZIP2, $archive, @marentries, @marfiles); + +if (defined($ENV{"MAR"})) { + $MAR = $ENV{"MAR"}; +} +else { + $MAR = "mar"; +} + +if (defined($ENV{"BZIP2"})) { + $BZIP2 = $ENV{"BZIP2"}; +} +else { + $BZIP2 = "bzip2"; +} + +sub print_usage +{ + print "Usage: unwrap_full_update.pl [OPTIONS] ARCHIVE\n\n"; + print "The contents of ARCHIVE will be unpacked into the current directory.\n\n"; + print "Options:\n"; + print " -h show this help text\n"; +} + +my %opts; +getopts("h", \%opts); + +if (defined($opts{'h'}) || scalar(@ARGV) != 1) { + print_usage(); + exit 1; +} + +$archive = $ARGV[0]; +@marentries = `"$MAR" -t "$archive"`; + +$? && die("Couldn't run \"$MAR\" -t"); + +shift @marentries; + +system("$MAR -x \"$archive\"") == 0 || die "Couldn't run $MAR -x"; + +foreach (@marentries) { + tr/\n\r//d; + my @splits = split(/\t/,$_); + my $file = $splits[2]; + + system("mv \"$file\" \"$file.bz2\"") == 0 || + die "Couldn't mv \"$file\""; + system("\"$BZIP2\" -d \"$file.bz2\"") == 0 || + die "Couldn't decompress \"$file\""; +} + |