summaryrefslogtreecommitdiffstats
path: root/toolkit/mozapps/installer/find-dupes.py
diff options
context:
space:
mode:
authorMatt A. Tobin <email@mattatobin.com>2018-04-14 10:01:26 -0400
committerMatt A. Tobin <email@mattatobin.com>2018-04-14 10:01:26 -0400
commit33406af03999cfdce840f978dcf2e258c32ea8e7 (patch)
tree00ee374d9e9de1a8c450a61f90630175defd7f4f /toolkit/mozapps/installer/find-dupes.py
parentf8ad954f7d670ba7d4de0648f4b60d3507164dd2 (diff)
downloadUXP-33406af03999cfdce840f978dcf2e258c32ea8e7.tar
UXP-33406af03999cfdce840f978dcf2e258c32ea8e7.tar.gz
UXP-33406af03999cfdce840f978dcf2e258c32ea8e7.tar.lz
UXP-33406af03999cfdce840f978dcf2e258c32ea8e7.tar.xz
UXP-33406af03999cfdce840f978dcf2e258c32ea8e7.zip
Do not make duplicates fatal while packaging
Diffstat (limited to 'toolkit/mozapps/installer/find-dupes.py')
-rw-r--r--toolkit/mozapps/installer/find-dupes.py115
1 files changed, 14 insertions, 101 deletions
diff --git a/toolkit/mozapps/installer/find-dupes.py b/toolkit/mozapps/installer/find-dupes.py
index bd0561c97..34ef675f4 100644
--- a/toolkit/mozapps/installer/find-dupes.py
+++ b/toolkit/mozapps/installer/find-dupes.py
@@ -4,15 +4,8 @@
import sys
import hashlib
-import re
-from mozbuild.preprocessor import Preprocessor
-from mozbuild.util import DefinesAction
from mozpack.packager.unpack import UnpackFinder
-from mozpack.files import DeflatedFile
from collections import OrderedDict
-from StringIO import StringIO
-import argparse
-import buildconfig
'''
Find files duplicated in a given packaged directory, independently of its
@@ -20,116 +13,36 @@ package format.
'''
-def normalize_osx_path(p):
- '''
- Strips the first 3 elements of an OSX app path
-
- >>> normalize_osx_path('Nightly.app/foo/bar/baz')
- 'baz'
- '''
- bits = p.split('/')
- if len(bits) > 3 and bits[0].endswith('.app'):
- return '/'.join(bits[3:])
- return p
-
-
-def normalize_l10n_path(p):
- '''
- Normalizes localized paths to en-US
-
- >>> normalize_l10n_path('chrome/es-ES/locale/branding/brand.properties')
- 'chrome/en-US/locale/branding/brand.properties'
- >>> normalize_l10n_path('chrome/fr/locale/fr/browser/aboutHome.dtd')
- 'chrome/en-US/locale/en-US/browser/aboutHome.dtd'
- '''
- # Keep a trailing slash here! e.g. locales like 'br' can transform
- # 'chrome/br/locale/branding/' into 'chrome/en-US/locale/en-USanding/'
- p = re.sub(r'chrome/(\S+)/locale/\1/',
- 'chrome/en-US/locale/en-US/',
- p)
- p = re.sub(r'chrome/(\S+)/locale/',
- 'chrome/en-US/locale/',
- p)
- return p
-
-
-def normalize_path(p):
- return normalize_osx_path(normalize_l10n_path(p))
-
-
-def find_dupes(source, allowed_dupes, bail=True):
- allowed_dupes = set(allowed_dupes)
+def find_dupes(source):
md5s = OrderedDict()
for p, f in UnpackFinder(source):
content = f.open().read()
m = hashlib.md5(content).digest()
- if m not in md5s:
- if isinstance(f, DeflatedFile):
- compressed = f.file.compressed_size
- else:
- compressed = len(content)
- md5s[m] = (len(content), compressed, [])
- md5s[m][2].append(p)
+ if not m in md5s:
+ md5s[m] = (len(content), [])
+ md5s[m][1].append(p)
total = 0
- total_compressed = 0
num_dupes = 0
- unexpected_dupes = []
- for m, (size, compressed, paths) in sorted(md5s.iteritems(),
- key=lambda x: x[1][1]):
+ for m, (size, paths) in md5s.iteritems():
if len(paths) > 1:
- print 'Duplicates %d bytes%s%s:' % (size,
- ' (%d compressed)' % compressed if compressed != size else '',
+ print 'Duplicates %d bytes%s:' % (size,
' (%d times)' % (len(paths) - 1) if len(paths) > 2 else '')
print ''.join(' %s\n' % p for p in paths)
total += (len(paths) - 1) * size
- total_compressed += (len(paths) - 1) * compressed
num_dupes += 1
-
- unexpected_dupes.extend([p for p in paths if normalize_path(p) not in allowed_dupes])
-
if num_dupes:
- print "WARNING: Found %d duplicated files taking %d bytes (%s)" % \
- (num_dupes, total,
- '%d compressed' % total_compressed if total_compressed != total
- else 'uncompressed')
-
- if unexpected_dupes:
- errortype = "ERROR" if bail else "WARNING"
- print "%s: The following duplicated files are not allowed:" % errortype
- print "\n".join(unexpected_dupes)
- if bail:
- sys.exit(1)
+ print "WARNING: Found %d duplicated files taking %d bytes" % \
+ (num_dupes, total) + " (uncompressed)"
def main():
- parser = argparse.ArgumentParser(description='Find duplicate files in directory.')
- parser.add_argument('--warning', '-w', action='store_true',
- help='Only warn about duplicates, do not exit with an error')
- parser.add_argument('--file', '-f', action='append', dest='dupes_files', default=[],
- help='Add exceptions to the duplicate list from this file')
- parser.add_argument('-D', action=DefinesAction)
- parser.add_argument('-U', action='append', default=[])
- parser.add_argument('directory',
- help='The directory to check for duplicates in')
-
- args = parser.parse_args()
-
- allowed_dupes = []
- for filename in args.dupes_files:
- pp = Preprocessor()
- pp.context.update(buildconfig.defines)
- if args.D:
- pp.context.update(args.D)
- for undefine in args.U:
- if undefine in pp.context:
- del pp.context[undefine]
- pp.out = StringIO()
- pp.do_filter('substitution')
- pp.do_include(filename)
- allowed_dupes.extend([line.partition('#')[0].rstrip()
- for line in pp.out.getvalue().splitlines()])
+ if len(sys.argv) != 2:
+ import os
+ print >>sys.stderr, "Usage: %s directory" % \
+ os.path.basename(sys.argv[0])
+ sys.exit(1)
- find_dupes(args.directory, bail=not args.warning, allowed_dupes=allowed_dupes)
+ find_dupes(sys.argv[1])
if __name__ == "__main__":
main()