diff --git a/dist/ubuntu/build_deb.sh b/dist/ubuntu/build_deb.sh index 009dd16..99e7d0a 100644 --- a/dist/ubuntu/build_deb.sh +++ b/dist/ubuntu/build_deb.sh @@ -5,7 +5,9 @@ if [ ! -e dist/ubuntu/build_deb.sh ]; then exit 1 fi -sudo apt-get -y install debhelper maven openjdk-7-jdk devscripts +if [ -e debian ] || [ -e build ] || [ -e target ] || [ -e m2 ]; then + rm -rf debian build target m2 +fi VERSION=$(./SCYLLA-VERSION-GEN) SCYLLA_VERSION=$(cat build/SCYLLA-VERSION-FILE) @@ -13,8 +15,13 @@ SCYLLA_RELEASE=$(cat build/SCYLLA-RELEASE-FILE) if [ "$SCYLLA_VERSION" = "development" ]; then SCYLLA_VERSION=0development fi +echo $VERSION > version +./scripts/git-archive-all --extra version --force-submodules --prefix scylla-jmx ../scylla-jmx_$SCYLLA_VERSION-$SCYLLA_RELEASE.orig.tar.gz + +cp -a dist/ubuntu/debian debian cp dist/ubuntu/changelog.in debian/changelog sed -i -e "s/@@VERSION@@/$SCYLLA_VERSION/g" debian/changelog sed -i -e "s/@@RELEASE@@/$SCYLLA_RELEASE/g" debian/changelog -debuild -r fakeroot --no-tgz-check -us -uc +sudo apt-get -y install debhelper maven openjdk-7-jdk devscripts +debuild -r fakeroot -us -uc diff --git a/debian/compat b/dist/ubuntu/debian/compat similarity index 100% rename from debian/compat rename to dist/ubuntu/debian/compat diff --git a/debian/control b/dist/ubuntu/debian/control similarity index 100% rename from debian/control rename to dist/ubuntu/debian/control diff --git a/debian/copyright b/dist/ubuntu/debian/copyright similarity index 78% rename from debian/copyright rename to dist/ubuntu/debian/copyright index 87c6b65..7fc8183 100644 --- a/debian/copyright +++ b/dist/ubuntu/debian/copyright @@ -10,3 +10,7 @@ License: AGPL-3.0 Files: debian/* Copyright: Copyright (C) 2015 ScyllaDB License: AGPL-3.0 + +Files: scripts/git-archive-all +Copyright: Copyright (c) 2010 Ilya Kulakov +License: MIT diff --git a/debian/rules b/dist/ubuntu/debian/rules similarity index 100% rename from debian/rules rename to dist/ubuntu/debian/rules diff --git a/debian/scylla-jmx.upstart b/dist/ubuntu/debian/scylla-jmx.upstart similarity index 100% rename from debian/scylla-jmx.upstart rename to dist/ubuntu/debian/scylla-jmx.upstart diff --git a/dist/ubuntu/debian/source/format b/dist/ubuntu/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/dist/ubuntu/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/dist/ubuntu/debian/source/options b/dist/ubuntu/debian/source/options new file mode 100644 index 0000000..94fc8d2 --- /dev/null +++ b/dist/ubuntu/debian/source/options @@ -0,0 +1 @@ +extend-diff-ignore = ^build/ diff --git a/git-archive-all-license.txt b/git-archive-all-license.txt new file mode 100644 index 0000000..18ca7ef --- /dev/null +++ b/git-archive-all-license.txt @@ -0,0 +1,3 @@ +License: MIT + +https://github.com/Kentzo/git-archive-all diff --git a/scripts/git-archive-all b/scripts/git-archive-all new file mode 100755 index 0000000..3552c6b --- /dev/null +++ b/scripts/git-archive-all @@ -0,0 +1,494 @@ +#! /usr/bin/env python +# coding=utf-8 + +from __future__ import print_function +from __future__ import unicode_literals + +__version__ = "1.9" + +import logging +from os import extsep, path, readlink, curdir +from subprocess import CalledProcessError, Popen, PIPE +import sys +import tarfile +from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED + + +class GitArchiver(object): + """ + GitArchiver + + Scan a git repository and export all tracked files, and submodules. + Checks for .gitattributes files in each directory and uses 'export-ignore' + pattern entries for ignore files in the archive. + + >>> archiver = GitArchiver(main_repo_abspath='my/repo/path') + >>> archiver.create('output.zip') + """ + LOG = logging.getLogger('GitArchiver') + + def __init__(self, prefix='', exclude=True, force_sub=False, extra=None, main_repo_abspath=None): + """ + @param prefix: Prefix used to prepend all paths in the resulting archive. + Extra file paths are only prefixed if they are not relative. + E.g. if prefix is 'foo' and extra is ['bar', '/baz'] the resulting archive will look like this: + / + baz + foo/ + bar + @type prefix: string + + @param exclude: Determines whether archiver should follow rules specified in .gitattributes files. + @type exclude: bool + + @param force_sub: Determines whether submodules are initialized and updated before archiving. + @type force_sub: bool + + @param extra: List of extra paths to include in the resulting archive. + @type extra: list + + @param main_repo_abspath: Absolute path to the main repository (or one of subdirectories). + If given path is path to a subdirectory (but not a submodule directory!) it will be replaced + with abspath to top-level directory of the repository. + If None, current cwd is used. + @type main_repo_abspath: string + """ + if extra is None: + extra = [] + + if main_repo_abspath is None: + main_repo_abspath = path.abspath('') + elif not path.isabs(main_repo_abspath): + raise ValueError("You MUST pass absolute path to the main git repository.") + + try: + self.run_shell("[ -d .git ] || git rev-parse --git-dir > /dev/null 2>&1", main_repo_abspath) + except Exception as e: + raise ValueError("Not a git repository (or any of the parent directories).") + + main_repo_abspath = path.abspath(self.read_git_shell('git rev-parse --show-toplevel', main_repo_abspath).rstrip()) + + self.prefix = prefix + self.exclude = exclude + self.extra = extra + self.force_sub = force_sub + self.main_repo_abspath = main_repo_abspath + + def create(self, output_path, dry_run=False, output_format=None): + """ + Create the archive at output_file_path. + + Type of the archive is determined either by extension of output_file_path or by output_format. + Supported formats are: gz, zip, bz2, xz, tar, tgz, txz + + @param output_path: Output file path. + @type output_path: string + + @param dry_run: Determines whether create should do nothing but print what it would archive. + @type dry_run: bool + + @param output_format: Determines format of the output archive. If None, format is determined from extension + of output_file_path. + @type output_format: string + """ + if output_format is None: + file_name, file_ext = path.splitext(output_path) + output_format = file_ext[len(extsep):].lower() + self.LOG.debug("Output format is not explicitly set, determined format is {}.".format(output_format)) + + if not dry_run: + if output_format == 'zip': + archive = ZipFile(path.abspath(output_path), 'w') + + def add_file(file_path, arcname): + if not path.islink(file_path): + archive.write(file_path, arcname, ZIP_DEFLATED) + else: + i = ZipInfo(arcname) + i.create_system = 3 + i.external_attr = 0xA1ED0000 + archive.writestr(i, readlink(file_path)) + elif output_format in ['tar', 'bz2', 'gz', 'xz', 'tgz', 'txz']: + if output_format == 'tar': + t_mode = 'w' + elif output_format == 'tgz': + t_mode = 'w:gz' + elif output_format == 'txz': + t_mode = 'w:xz' + else: + t_mode = 'w:{}'.format(output_format) + + archive = tarfile.open(path.abspath(output_path), t_mode) + add_file = lambda file_path, arcname: archive.add(file_path, arcname) + else: + raise RuntimeError("Unknown format: {}".format(output_format)) + + def archiver(file_path, arcname): + self.LOG.debug("Compressing {} => {}...".format(file_path, arcname)) + add_file(file_path, arcname) + else: + archive = None + archiver = lambda file_path, arcname: self.LOG.info("{} => {}".format(file_path, arcname)) + + self.archive_all_files(archiver) + + if archive is not None: + archive.close() + + def get_exclude_patterns(self, repo_abspath, repo_file_paths): + """ + Returns exclude patterns for a given repo. It looks for .gitattributes files in repo_file_paths. + + Resulting dictionary will contain exclude patterns per path (relative to the repo_abspath). + E.g. {('.', 'Catalyst', 'Editions', 'Base'), ['Foo*', '*Bar']} + + @type repo_abspath: string + @param repo_abspath: Absolute path to the git repository. + + @type repo_file_paths: list + @param repo_file_paths: List of paths relative to the repo_abspath that are under git control. + + @rtype: dict + @return: Dictionary representing exclude patterns. + Keys are tuples of strings. Values are lists of strings. + Returns None if self.exclude is not set. + """ + if not self.exclude: + return None + + def read_attributes(attributes_abspath): + patterns = [] + if path.isfile(attributes_abspath): + attributes = open(attributes_abspath, 'r').readlines() + patterns = [] + for line in attributes: + tokens = line.strip().split() + if "export-ignore" in tokens[1:]: + patterns.append(tokens[0]) + return patterns + + exclude_patterns = {(): []} + + # There may be no gitattributes. + try: + global_attributes_abspath = self.read_shell("git config --get core.attributesfile", repo_abspath).rstrip() + exclude_patterns[()] = read_attributes(global_attributes_abspath) + except: + # And it's valid to not have them. + pass + + for attributes_abspath in [path.join(repo_abspath, f) for f in repo_file_paths if f.endswith(".gitattributes")]: + # Each .gitattributes affects only files within its directory. + key = tuple(self.get_path_components(repo_abspath, path.dirname(attributes_abspath))) + exclude_patterns[key] = read_attributes(attributes_abspath) + + local_attributes_abspath = path.join(repo_abspath, ".git", "info", "attributes") + key = tuple(self.get_path_components(repo_abspath, repo_abspath)) + + if key in exclude_patterns: + exclude_patterns[key].extend(read_attributes(local_attributes_abspath)) + else: + exclude_patterns[key] = read_attributes(local_attributes_abspath) + + return exclude_patterns + + def is_file_excluded(self, repo_abspath, repo_file_path, exclude_patterns): + """ + Checks whether file at a given path is excluded. + + @type repo_abspath: string + @param repo_abspath: Absolute path to the git repository. + + @type repo_file_path: string + @param repo_file_path: Path to a file within repo_abspath. + + @type exclude_patterns: dict + @param exclude_patterns: Exclude patterns with format specified for get_exclude_patterns. + + @rtype: bool + @return: True if file should be excluded. Otherwise False. + """ + if exclude_patterns is None or not len(exclude_patterns): + return False + + from fnmatch import fnmatch + + file_name = path.basename(repo_file_path) + components = self.get_path_components(repo_abspath, path.join(repo_abspath, path.dirname(repo_file_path))) + + is_excluded = False + # We should check all patterns specified in intermediate directories to the given file. + # At the end we should also check for the global patterns (key '()' or empty tuple). + while not is_excluded: + key = tuple(components) + if key in exclude_patterns: + patterns = exclude_patterns[key] + for p in patterns: + if fnmatch(file_name, p) or fnmatch(repo_file_path, p): + self.LOG.debug("Exclude pattern matched {}: {}".format(p, repo_file_path)) + is_excluded = True + + if not len(components): + break + + components.pop() + + return is_excluded + + def archive_all_files(self, archiver): + """ + Archive all files using archiver. + + @param archiver: Function that accepts 2 arguments: abspath to file on the system and relative path within archive. + """ + for file_path in self.extra: + archiver(path.abspath(file_path), path.join(self.prefix, file_path)) + + for file_path in self.walk_git_files(): + archiver(path.join(self.main_repo_abspath, file_path), path.join(self.prefix, file_path)) + + def walk_git_files(self, repo_path=''): + """ + An iterator method that yields a file path relative to main_repo_abspath + for each file that should be included in the archive. + Skips those that match the exclusion patterns found in + any discovered .gitattributes files along the way. + + Recurs into submodules as well. + + @type repo_path: string + @param repo_path: Path to the git submodule repository relative to main_repo_abspath. + + @rtype: iterator + @return: Iterator to traverse files under git control relative to main_repo_abspath. + """ + repo_abspath = path.join(self.main_repo_abspath, repo_path) + repo_file_paths = self.read_git_shell("git ls-files --cached --full-name --no-empty-directory", repo_abspath).splitlines() + exclude_patterns = self.get_exclude_patterns(repo_abspath, repo_file_paths) + + for repo_file_path in repo_file_paths: + # Git puts path in quotes if file path has unicode characters. + repo_file_path = repo_file_path.strip('"') # file path relative to current repo + file_name = path.basename(repo_file_path) + main_repo_file_path = path.join(repo_path, repo_file_path) # file path relative to the main repo + + # Only list symlinks and files that don't start with git. + if file_name.startswith(".git") or (not path.islink(main_repo_file_path) and path.isdir(main_repo_file_path)): + continue + + if self.is_file_excluded(repo_abspath, repo_file_path, exclude_patterns): + continue + + yield main_repo_file_path + + if self.force_sub: + self.run_shell("git submodule init", repo_abspath) + self.run_shell("git submodule update", repo_abspath) + + for submodule_path in self.read_shell("git submodule --quiet foreach 'pwd -P'", repo_abspath).splitlines(): + # Shell command returns absolute paths to submodules. + submodule_path = path.relpath(submodule_path, self.main_repo_abspath) + for file_path in self.walk_git_files(submodule_path): + yield file_path + + @staticmethod + def get_path_components(repo_abspath, abspath): + """ + Split given abspath into components relative to repo_abspath. + These components are primarily used as unique keys of files and folders within a repository. + + E.g. if repo_abspath is '/Documents/Hobby/ParaView/' and abspath is + '/Documents/Hobby/ParaView/Catalyst/Editions/Base/', function will return: + ['.', 'Catalyst', 'Editions', 'Base'] + + First element is always '.' (concrete symbol depends on OS). + + @param repo_abspath: Absolute path to the git repository. Normalized via os.path.normpath. + @type repo_abspath: string + + @param abspath: Absolute path to a file within repo_abspath. Normalized via os.path.normpath. + @type abspath: string + + @return: List of path components. + @rtype: list + """ + repo_abspath = path.normpath(repo_abspath) + abspath = path.normpath(abspath) + + if not path.isabs(repo_abspath): + raise ValueError("repo_abspath MUST be absolute path.") + + if not path.isabs(abspath): + raise ValueError("abspath MUST be absoulte path.") + + if not path.commonprefix([repo_abspath, abspath]): + raise ValueError("abspath (\"{}\") MUST have common prefix with repo_abspath (\"{}\")".format(abspath, repo_abspath)) + + components = [] + + while not abspath == repo_abspath: + abspath, tail = path.split(abspath) + + if tail: + components.insert(0, tail) + + components.insert(0, curdir) + return components + + @staticmethod + def run_shell(cmd, cwd=None): + """ + Runs shell command. + + @type cmd: string + @param cmd: Command to be executed. + + @type cwd: string + @param cwd: Working directory. + + @rtype: int + @return: Return code of the command. + + @raise CalledProcessError: Raises exception if return code of the command is non-zero. + """ + p = Popen(cmd, shell=True, cwd=cwd) + p.wait() + + if p.returncode: + raise CalledProcessError(returncode=p.returncode, cmd=cmd) + + return p.returncode + + @staticmethod + def read_shell(cmd, cwd=None, encoding='utf-8'): + """ + Runs shell command and reads output. + + @type cmd: string + @param cmd: Command to be executed. + + @type cwd: string + @param cwd: Working directory. + + @type encoding: string + @param encoding: Encoding used to decode bytes returned by Popen into string. + + @rtype: string + @return: Output of the command. + + @raise CalledProcessError: Raises exception if return code of the command is non-zero. + """ + p = Popen(cmd, shell=True, stdout=PIPE, cwd=cwd) + output, _ = p.communicate() + output = output.decode(encoding) + + if p.returncode: + if sys.version_info > (2,6): + raise CalledProcessError(returncode=p.returncode, cmd=cmd, output=output) + else: + raise CalledProcessError(returncode=p.returncode, cmd=cmd) + + return output + + @staticmethod + def read_git_shell(cmd, cwd=None): + """ + Runs git shell command, reads output and decodes it into unicode string + + @type cmd: string + @param cmd: Command to be executed. + + @type cwd: string + @param cwd: Working directory. + + @rtype: string + @return: Output of the command. + + @raise CalledProcessError: Raises exception if return code of the command is non-zero. + """ + p = Popen(cmd, shell=True, stdout=PIPE, cwd=cwd) + output, _ = p.communicate() + output = output.decode('unicode_escape').encode('raw_unicode_escape').decode('utf-8') + + if p.returncode: + if sys.version_info > (2,6): + raise CalledProcessError(returncode=p.returncode, cmd=cmd, output=output) + else: + raise CalledProcessError(returncode=p.returncode, cmd=cmd) + + return output + + +if __name__ == '__main__': + from optparse import OptionParser + + parser = OptionParser(usage="usage: %prog [-v] [--prefix PREFIX] [--no-exclude] [--force-submodules] [--extra EXTRA1 [EXTRA2]] [--dry-run] OUTPUT_FILE", + version="%prog {}".format(__version__)) + + parser.add_option('--prefix', + type='string', + dest='prefix', + default=None, + help="prepend PREFIX to each filename in the archive. OUTPUT_FILE name is used by default to avoid tarbomb. You can set it to '' in order to explicitly request tarbomb") + + parser.add_option('-v', '--verbose', + action='store_true', + dest='verbose', + help='enable verbose mode') + + parser.add_option('--no-exclude', + action='store_false', + dest='exclude', + default=True, + help="don't read .gitattributes files for patterns containing export-ignore attrib") + + parser.add_option('--force-submodules', + action='store_true', + dest='force_sub', + help="force a git submodule init && git submodule update at each level before iterating submodules") + + parser.add_option('--extra', + action='append', + dest='extra', + default=[], + help="any additional files to include in the archive") + + parser.add_option('--dry-run', + action='store_true', + dest='dry_run', + help="don't actually archive anything, just show what would be done") + + options, args = parser.parse_args() + + if len(args) != 1: + parser.error("You must specify exactly one output file") + + output_file_path = args[0] + + if path.isdir(output_file_path): + parser.error("You cannot use directory as output") + + # avoid tarbomb + if options.prefix is not None: + options.prefix = path.join(options.prefix, '') + else: + import re + + output_name = path.basename(output_file_path) + output_name = re.sub('(\.zip|\.tar|\.tgz|\.txz|\.gz|\.bz2|\.xz|\.tar\.gz|\.tar\.bz2|\.tar\.xz)$', '', output_name) or "Archive" + options.prefix = path.join(output_name, '') + + try: + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(logging.Formatter('%(message)s')) + GitArchiver.LOG.addHandler(handler) + GitArchiver.LOG.setLevel(logging.DEBUG if options.verbose else logging.INFO) + archiver = GitArchiver(options.prefix, + options.exclude, + options.force_sub, + options.extra) + archiver.create(output_file_path, options.dry_run) + except Exception as e: + parser.exit(2, "{}\n".format(e)) + + sys.exit(0)