dist: make ubuntu package as 'debian non-native package'

dist/ubuntu/build_deb.sh

@@ -5,7 +5,9 @@ if [ ! -e dist/ubuntu/build_deb.sh ]; then
     exit 1
 fi
 
-sudo apt-get -y install debhelper maven openjdk-7-jdk devscripts
+if [ -e debian ] || [ -e build ] || [ -e target ] || [ -e m2 ]; then
+    rm -rf debian build target m2
+fi
 
 VERSION=$(./SCYLLA-VERSION-GEN)
 SCYLLA_VERSION=$(cat build/SCYLLA-VERSION-FILE)
@@ -13,8 +15,13 @@ SCYLLA_RELEASE=$(cat build/SCYLLA-RELEASE-FILE)
 if [ "$SCYLLA_VERSION" = "development" ]; then
 	SCYLLA_VERSION=0development
 fi
+echo $VERSION > version
+./scripts/git-archive-all --extra version --force-submodules --prefix scylla-jmx ../scylla-jmx_$SCYLLA_VERSION-$SCYLLA_RELEASE.orig.tar.gz 
+
+cp -a dist/ubuntu/debian debian
 cp dist/ubuntu/changelog.in debian/changelog
 sed -i -e "s/@@VERSION@@/$SCYLLA_VERSION/g" debian/changelog
 sed -i -e "s/@@RELEASE@@/$SCYLLA_RELEASE/g" debian/changelog
 
-debuild -r fakeroot --no-tgz-check -us -uc
+sudo apt-get -y install debhelper maven openjdk-7-jdk devscripts
+debuild -r fakeroot -us -uc

dist/ubuntu/debian/copyright

@@ -10,3 +10,7 @@ License: AGPL-3.0
 Files: debian/*
 Copyright: Copyright (C) 2015 ScyllaDB
 License: AGPL-3.0
+
+Files: scripts/git-archive-all
+Copyright: Copyright (c) 2010 Ilya Kulakov
+License: MIT

dist/ubuntu/debian/source/format

@@ -0,0 +1 @@
+3.0 (quilt)

dist/ubuntu/debian/source/options

@@ -0,0 +1 @@
+extend-diff-ignore = ^build/

git-archive-all-license.txt

@@ -0,0 +1,3 @@
+License: MIT
+
+https://github.com/Kentzo/git-archive-all

scripts/git-archive-all

@@ -0,0 +1,494 @@
+#! /usr/bin/env python
+# coding=utf-8
+
+from __future__ import print_function
+from __future__ import unicode_literals
+
+__version__ = "1.9"
+
+import logging
+from os import extsep, path, readlink, curdir
+from subprocess import CalledProcessError, Popen, PIPE
+import sys
+import tarfile
+from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED
+
+
+class GitArchiver(object):
+    """
+    GitArchiver
+
+    Scan a git repository and export all tracked files, and submodules.
+    Checks for .gitattributes files in each directory and uses 'export-ignore'
+    pattern entries for ignore files in the archive.
+
+    >>> archiver = GitArchiver(main_repo_abspath='my/repo/path')
+    >>> archiver.create('output.zip')
+    """
+    LOG = logging.getLogger('GitArchiver')
+
+    def __init__(self, prefix='', exclude=True, force_sub=False, extra=None, main_repo_abspath=None):
+        """
+        @param prefix: Prefix used to prepend all paths in the resulting archive.
+            Extra file paths are only prefixed if they are not relative.
+            E.g. if prefix is 'foo' and extra is ['bar', '/baz'] the resulting archive will look like this:
+            /
+              baz
+              foo/
+                bar
+        @type prefix: string
+
+        @param exclude: Determines whether archiver should follow rules specified in .gitattributes files.
+        @type exclude:  bool
+
+        @param force_sub: Determines whether submodules are initialized and updated before archiving.
+        @type force_sub: bool
+
+        @param extra: List of extra paths to include in the resulting archive.
+        @type extra: list
+
+        @param main_repo_abspath: Absolute path to the main repository (or one of subdirectories).
+            If given path is path to a subdirectory (but not a submodule directory!) it will be replaced
+            with abspath to top-level directory of the repository.
+            If None, current cwd is used.
+        @type main_repo_abspath: string
+        """
+        if extra is None:
+            extra = []
+
+        if main_repo_abspath is None:
+            main_repo_abspath = path.abspath('')
+        elif not path.isabs(main_repo_abspath):
+            raise ValueError("You MUST pass absolute path to the main git repository.")
+
+        try:
+            self.run_shell("[ -d .git ] || git rev-parse --git-dir > /dev/null 2>&1", main_repo_abspath)
+        except Exception as e:
+            raise ValueError("Not a git repository (or any of the parent directories).")
+
+        main_repo_abspath = path.abspath(self.read_git_shell('git rev-parse --show-toplevel', main_repo_abspath).rstrip())
+
+        self.prefix = prefix
+        self.exclude = exclude
+        self.extra = extra
+        self.force_sub = force_sub
+        self.main_repo_abspath = main_repo_abspath
+
+    def create(self, output_path, dry_run=False, output_format=None):
+        """
+        Create the archive at output_file_path.
+
+        Type of the archive is determined either by extension of output_file_path or by output_format.
+        Supported formats are: gz, zip, bz2, xz, tar, tgz, txz
+
+        @param output_path: Output file path.
+        @type output_path: string
+
+        @param dry_run: Determines whether create should do nothing but print what it would archive.
+        @type dry_run: bool
+
+        @param output_format: Determines format of the output archive. If None, format is determined from extension
+            of output_file_path.
+        @type output_format: string
+        """
+        if output_format is None:
+            file_name, file_ext = path.splitext(output_path)
+            output_format = file_ext[len(extsep):].lower()
+            self.LOG.debug("Output format is not explicitly set, determined format is {}.".format(output_format))
+
+        if not dry_run:
+            if output_format == 'zip':
+                archive = ZipFile(path.abspath(output_path), 'w')
+
+                def add_file(file_path, arcname):
+                    if not path.islink(file_path):
+                        archive.write(file_path, arcname, ZIP_DEFLATED)
+                    else:
+                        i = ZipInfo(arcname)
+                        i.create_system = 3
+                        i.external_attr = 0xA1ED0000
+                        archive.writestr(i, readlink(file_path))
+            elif output_format in ['tar', 'bz2', 'gz', 'xz', 'tgz', 'txz']:
+                if output_format == 'tar':
+                    t_mode = 'w'
+                elif output_format == 'tgz':
+                    t_mode = 'w:gz'
+                elif output_format == 'txz':
+                    t_mode = 'w:xz'
+                else:
+                    t_mode = 'w:{}'.format(output_format)
+
+                archive = tarfile.open(path.abspath(output_path), t_mode)
+                add_file = lambda file_path, arcname: archive.add(file_path, arcname)
+            else:
+                raise RuntimeError("Unknown format: {}".format(output_format))
+
+            def archiver(file_path, arcname):
+                self.LOG.debug("Compressing {} => {}...".format(file_path, arcname))
+                add_file(file_path, arcname)
+        else:
+            archive = None
+            archiver = lambda file_path, arcname: self.LOG.info("{} => {}".format(file_path, arcname))
+
+        self.archive_all_files(archiver)
+
+        if archive is not None:
+            archive.close()
+
+    def get_exclude_patterns(self, repo_abspath, repo_file_paths):
+        """
+        Returns exclude patterns for a given repo. It looks for .gitattributes files in repo_file_paths.
+
+        Resulting dictionary will contain exclude patterns per path (relative to the repo_abspath).
+        E.g. {('.', 'Catalyst', 'Editions', 'Base'), ['Foo*', '*Bar']}
+
+        @type repo_abspath:     string
+        @param repo_abspath:    Absolute path to the git repository.
+
+        @type repo_file_paths:  list
+        @param repo_file_paths: List of paths relative to the repo_abspath that are under git control.
+
+        @rtype:         dict
+        @return:    Dictionary representing exclude patterns.
+                    Keys are tuples of strings. Values are lists of strings.
+                    Returns None if self.exclude is not set.
+        """
+        if not self.exclude:
+            return None
+
+        def read_attributes(attributes_abspath):
+            patterns = []
+            if path.isfile(attributes_abspath):
+                attributes = open(attributes_abspath, 'r').readlines()
+                patterns = []
+                for line in attributes:
+                    tokens = line.strip().split()
+                    if "export-ignore" in tokens[1:]:
+                        patterns.append(tokens[0])
+            return patterns
+
+        exclude_patterns = {(): []}
+
+        # There may be no gitattributes.
+        try:
+            global_attributes_abspath = self.read_shell("git config --get core.attributesfile", repo_abspath).rstrip()
+            exclude_patterns[()] = read_attributes(global_attributes_abspath)
+        except:
+            # And it's valid to not have them.
+            pass
+
+        for attributes_abspath in [path.join(repo_abspath, f) for f in repo_file_paths if f.endswith(".gitattributes")]:
+            # Each .gitattributes affects only files within its directory.
+            key = tuple(self.get_path_components(repo_abspath, path.dirname(attributes_abspath)))
+            exclude_patterns[key] = read_attributes(attributes_abspath)
+
+        local_attributes_abspath = path.join(repo_abspath, ".git", "info", "attributes")
+        key = tuple(self.get_path_components(repo_abspath, repo_abspath))
+
+        if key in exclude_patterns:
+            exclude_patterns[key].extend(read_attributes(local_attributes_abspath))
+        else:
+            exclude_patterns[key] = read_attributes(local_attributes_abspath)
+
+        return exclude_patterns
+
+    def is_file_excluded(self, repo_abspath, repo_file_path, exclude_patterns):
+        """
+        Checks whether file at a given path is excluded.
+
+        @type repo_abspath: string
+        @param repo_abspath: Absolute path to the git repository.
+
+        @type repo_file_path:   string
+        @param repo_file_path:  Path to a file within repo_abspath.
+
+        @type exclude_patterns:     dict
+        @param exclude_patterns:    Exclude patterns with format specified for get_exclude_patterns.
+
+        @rtype: bool
+        @return: True if file should be excluded. Otherwise False.
+        """
+        if exclude_patterns is None or not len(exclude_patterns):
+            return False
+
+        from fnmatch import fnmatch
+
+        file_name = path.basename(repo_file_path)
+        components = self.get_path_components(repo_abspath, path.join(repo_abspath, path.dirname(repo_file_path)))
+
+        is_excluded = False
+        # We should check all patterns specified in intermediate directories to the given file.
+        # At the end we should also check for the global patterns (key '()' or empty tuple).
+        while not is_excluded:
+            key = tuple(components)
+            if key in exclude_patterns:
+                patterns = exclude_patterns[key]
+                for p in patterns:
+                    if fnmatch(file_name, p) or fnmatch(repo_file_path, p):
+                        self.LOG.debug("Exclude pattern matched {}: {}".format(p, repo_file_path))
+                        is_excluded = True
+
+            if not len(components):
+                break
+
+            components.pop()
+
+        return is_excluded
+
+    def archive_all_files(self, archiver):
+        """
+        Archive all files using archiver.
+
+        @param archiver: Function that accepts 2 arguments: abspath to file on the system and relative path within archive.
+        """
+        for file_path in self.extra:
+            archiver(path.abspath(file_path), path.join(self.prefix, file_path))
+
+        for file_path in self.walk_git_files():
+            archiver(path.join(self.main_repo_abspath, file_path), path.join(self.prefix, file_path))
+
+    def walk_git_files(self, repo_path=''):
+        """
+        An iterator method that yields a file path relative to main_repo_abspath
+        for each file that should be included in the archive.
+        Skips those that match the exclusion patterns found in
+        any discovered .gitattributes files along the way.
+
+        Recurs into submodules as well.
+
+        @type repo_path:    string
+        @param repo_path:   Path to the git submodule repository relative to main_repo_abspath.
+
+        @rtype:     iterator
+        @return:    Iterator to traverse files under git control relative to main_repo_abspath.
+        """
+        repo_abspath = path.join(self.main_repo_abspath, repo_path)
+        repo_file_paths = self.read_git_shell("git ls-files --cached --full-name --no-empty-directory", repo_abspath).splitlines()
+        exclude_patterns = self.get_exclude_patterns(repo_abspath, repo_file_paths)
+
+        for repo_file_path in repo_file_paths:
+            # Git puts path in quotes if file path has unicode characters.
+            repo_file_path = repo_file_path.strip('"')  # file path relative to current repo
+            file_name = path.basename(repo_file_path)
+            main_repo_file_path = path.join(repo_path, repo_file_path)  # file path relative to the main repo
+
+            # Only list symlinks and files that don't start with git.
+            if file_name.startswith(".git") or (not path.islink(main_repo_file_path) and path.isdir(main_repo_file_path)):
+                continue
+
+            if self.is_file_excluded(repo_abspath, repo_file_path, exclude_patterns):
+                continue
+
+            yield main_repo_file_path
+
+        if self.force_sub:
+            self.run_shell("git submodule init", repo_abspath)
+            self.run_shell("git submodule update", repo_abspath)
+
+        for submodule_path in self.read_shell("git submodule --quiet foreach 'pwd -P'", repo_abspath).splitlines():
+            # Shell command returns absolute paths to submodules.
+            submodule_path = path.relpath(submodule_path, self.main_repo_abspath)
+            for file_path in self.walk_git_files(submodule_path):
+                yield file_path
+
+    @staticmethod
+    def get_path_components(repo_abspath, abspath):
+        """
+        Split given abspath into components relative to repo_abspath.
+        These components are primarily used as unique keys of files and folders within a repository.
+
+        E.g. if repo_abspath is '/Documents/Hobby/ParaView/' and abspath is
+        '/Documents/Hobby/ParaView/Catalyst/Editions/Base/', function will return:
+        ['.', 'Catalyst', 'Editions', 'Base']
+
+        First element is always '.' (concrete symbol depends on OS).
+
+        @param repo_abspath: Absolute path to the git repository. Normalized via os.path.normpath.
+        @type repo_abspath: string
+
+        @param abspath: Absolute path to a file within repo_abspath. Normalized via os.path.normpath.
+        @type abspath: string
+
+        @return: List of path components.
+        @rtype: list
+        """
+        repo_abspath = path.normpath(repo_abspath)
+        abspath = path.normpath(abspath)
+
+        if not path.isabs(repo_abspath):
+            raise ValueError("repo_abspath MUST be absolute path.")
+
+        if not path.isabs(abspath):
+            raise ValueError("abspath MUST be absoulte path.")
+
+        if not path.commonprefix([repo_abspath, abspath]):
+            raise ValueError("abspath (\"{}\") MUST have common prefix with repo_abspath (\"{}\")".format(abspath, repo_abspath))
+
+        components = []
+
+        while not abspath == repo_abspath:
+            abspath, tail = path.split(abspath)
+
+            if tail:
+                components.insert(0, tail)
+
+        components.insert(0, curdir)
+        return components
+
+    @staticmethod
+    def run_shell(cmd, cwd=None):
+        """
+        Runs shell command.
+
+        @type cmd:  string
+        @param cmd: Command to be executed.
+
+        @type cwd:  string
+        @param cwd: Working directory.
+
+        @rtype:     int
+        @return:    Return code of the command.
+
+        @raise CalledProcessError:  Raises exception if return code of the command is non-zero.
+        """
+        p = Popen(cmd, shell=True, cwd=cwd)
+        p.wait()
+
+        if p.returncode:
+            raise CalledProcessError(returncode=p.returncode, cmd=cmd)
+
+        return p.returncode
+
+    @staticmethod
+    def read_shell(cmd, cwd=None, encoding='utf-8'):
+        """
+        Runs shell command and reads output.
+
+        @type cmd:  string
+        @param cmd: Command to be executed.
+
+        @type cwd:  string
+        @param cwd: Working directory.
+
+        @type encoding: string
+        @param encoding: Encoding used to decode bytes returned by Popen into string.
+
+        @rtype:     string
+        @return:    Output of the command.
+
+        @raise CalledProcessError:  Raises exception if return code of the command is non-zero.
+        """
+        p = Popen(cmd, shell=True, stdout=PIPE, cwd=cwd)
+        output, _ = p.communicate()
+        output = output.decode(encoding)
+
+        if p.returncode:
+            if sys.version_info > (2,6):
+                raise CalledProcessError(returncode=p.returncode, cmd=cmd, output=output)
+            else:
+                raise CalledProcessError(returncode=p.returncode, cmd=cmd)
+
+        return output
+
+    @staticmethod
+    def read_git_shell(cmd, cwd=None):
+        """
+        Runs git shell command, reads output and decodes it into unicode string
+
+        @type cmd:  string
+        @param cmd: Command to be executed.
+
+        @type cwd:  string
+        @param cwd: Working directory.
+
+        @rtype:     string
+        @return:    Output of the command.
+
+        @raise CalledProcessError:  Raises exception if return code of the command is non-zero.
+        """
+        p = Popen(cmd, shell=True, stdout=PIPE, cwd=cwd)
+        output, _ = p.communicate()
+        output = output.decode('unicode_escape').encode('raw_unicode_escape').decode('utf-8')
+
+        if p.returncode:
+            if sys.version_info > (2,6):
+                raise CalledProcessError(returncode=p.returncode, cmd=cmd, output=output)
+            else:
+                raise CalledProcessError(returncode=p.returncode, cmd=cmd)
+
+        return output
+
+
+if __name__ == '__main__':
+    from optparse import OptionParser
+
+    parser = OptionParser(usage="usage: %prog [-v] [--prefix PREFIX] [--no-exclude] [--force-submodules] [--extra EXTRA1 [EXTRA2]] [--dry-run] OUTPUT_FILE",
+                          version="%prog {}".format(__version__))
+
+    parser.add_option('--prefix',
+                      type='string',
+                      dest='prefix',
+                      default=None,
+                      help="prepend PREFIX to each filename in the archive. OUTPUT_FILE name is used by default to avoid tarbomb. You can set it to '' in order to explicitly request tarbomb")
+
+    parser.add_option('-v', '--verbose',
+                      action='store_true',
+                      dest='verbose',
+                      help='enable verbose mode')
+
+    parser.add_option('--no-exclude',
+                      action='store_false',
+                      dest='exclude',
+                      default=True,
+                      help="don't read .gitattributes files for patterns containing export-ignore attrib")
+
+    parser.add_option('--force-submodules',
+                      action='store_true',
+                      dest='force_sub',
+                      help="force a git submodule init && git submodule update at each level before iterating submodules")
+
+    parser.add_option('--extra',
+                      action='append',
+                      dest='extra',
+                      default=[],
+                      help="any additional files to include in the archive")
+
+    parser.add_option('--dry-run',
+                      action='store_true',
+                      dest='dry_run',
+                      help="don't actually archive anything, just show what would be done")
+
+    options, args = parser.parse_args()
+
+    if len(args) != 1:
+        parser.error("You must specify exactly one output file")
+
+    output_file_path = args[0]
+
+    if path.isdir(output_file_path):
+        parser.error("You cannot use directory as output")
+
+    # avoid tarbomb
+    if options.prefix is not None:
+        options.prefix = path.join(options.prefix, '')
+    else:
+        import re
+
+        output_name = path.basename(output_file_path)
+        output_name = re.sub('(\.zip|\.tar|\.tgz|\.txz|\.gz|\.bz2|\.xz|\.tar\.gz|\.tar\.bz2|\.tar\.xz)$', '', output_name) or "Archive"
+        options.prefix = path.join(output_name, '')
+
+    try:
+        handler = logging.StreamHandler(sys.stdout)
+        handler.setFormatter(logging.Formatter('%(message)s'))
+        GitArchiver.LOG.addHandler(handler)
+        GitArchiver.LOG.setLevel(logging.DEBUG if options.verbose else logging.INFO)
+        archiver = GitArchiver(options.prefix,
+                               options.exclude,
+                               options.force_sub,
+                               options.extra)
+        archiver.create(output_file_path, options.dry_run)
+    except Exception as e:
+        parser.exit(2, "{}\n".format(e))
+
+    sys.exit(0)