Merge "relocatable package support for jmx" from Takuya

" This patchset adds relocatable package support for scylla-jmx, and also support generating .rpm/.deb from relocatable package. - Scripts are based on relocatable .rpm/.deb support patchset for main repo (not merged https://github.com/syuu1228/scylla/tree/reloc_rpmdeb_v4) - Single .rpm package provided for CentOS7/Fedora(unofficial) - Single .deb package provided for Ubuntu 14/16/18, Debian 8/9 " * 'reloc_v1' of https://github.com/syuu1228/scylla-jmx: dist/debian: use relocatable package to produce .deb dist/redhat: use relocatable package to produce .rpm reloc: add support relocatable package
2018-10-24 11:51:34 +03:00 · 2018-10-24 11:51:34 +03:00 · df2dee2402
parent ca3fa8de20 e9bfbedfed
commit df2dee2402
10 changed files with 254 additions and 116 deletions
--- a/dist/debian/build_deb.sh
+++ b/dist/debian/build_deb.sh
@ -4,15 +4,16 @@ PRODUCT=scylla

 . /etc/os-release
 print_usage() {
-    echo "build_deb.sh -target <codename>"
-    echo "  --target target distribution codename"
+    echo "build_deb.sh --reloc-pkg build/scylla-jmx-package.tar.gz"
+    echo "  --reloc-pkg specify relocatable package path"
    exit 1
 }
-TARGET=
+TARGET=stable
+RELOC_PKG=
 while [ $# -gt 0 ]; do
    case "$1" in
-        "--target")
-            TARGET=$2
+        "--reloc-pkg")
+            RELOC_PKG=$2
            shift 2
            ;;
        *)
@ -52,15 +53,25 @@ pkg_install() {
    fi
 }

-if [ ! -e dist/debian/build_deb.sh ]; then
-    echo "run build_deb.sh in top of scylla dir"
+if [ ! -e SCYLLA-RELOCATABLE-FILE ]; then
+    echo "do not directly execute build_rpm.sh, use reloc/build_rpm.sh instead."
    exit 1
 fi
+
 if [ "$(arch)" != "x86_64" ]; then
    echo "Unsupported architecture: $(arch)"
    exit 1
 fi

+if [ -z "$RELOC_PKG" ]; then
+    print_usage
+    exit 1
+fi
+if [ ! -f "$RELOC_PKG" ]; then
+    echo "$RELOC_PKG is not found."
+    exit 1
+fi
+
 if [ -e debian ] || [ -e build/release ]; then
    sudo rm -rf debian build
    mkdir build
@ -79,15 +90,15 @@ fi
 if [ ! -f /usr/bin/python ]; then
    pkg_install python
 fi
-if [ ! -f /usr/sbin/pbuilder ]; then
-    pkg_install pbuilder
-fi
-if [ ! -f /usr/bin/mvn ]; then
-    pkg_install maven
+if [ ! -f /usr/sbin/debuild ]; then
+    pkg_install devscripts
 fi
 if [ ! -f /usr/bin/dh_testdir ]; then
    pkg_install debhelper
 fi
+if [ ! -f /usr/bin/fakeroot ]; then
+    pkg_install fakeroot
+fi
 if [ ! -f /usr/bin/pystache ]; then
    if is_redhat_variant; then
        sudo yum install -y /usr/bin/pystache
@ -103,60 +114,27 @@ if [ "$ID" = "debian" ] && [ ! -f /usr/share/keyrings/ubuntu-archive-keyring.gpg
    sudo apt-get install -y ubuntu-archive-keyring
 fi

-if [ -z "$TARGET" ]; then
-    if is_debian_variant; then
-        if [ ! -f /usr/bin/lsb_release ]; then
-            pkg_install lsb-release
-        fi
-        TARGET=`lsb_release -c|awk '{print $2}'`
-    else
-        echo "Please specify target"
-        exit 1
-    fi
-fi
+RELOC_PKG_FULLPATH=$(readlink -f $RELOC_PKG)
+RELOC_PKG_BASENAME=$(basename $RELOC_PKG)
+SCYLLA_VERSION=$(cat SCYLLA-VERSION-FILE | sed 's/\.rc/~rc/')
+SCYLLA_RELEASE=$(cat SCYLLA-RELEASE-FILE)

-VERSION=$(./SCYLLA-VERSION-GEN)
-SCYLLA_VERSION=$(cat build/SCYLLA-VERSION-FILE | sed 's/\.rc/~rc/')
-SCYLLA_RELEASE=$(cat build/SCYLLA-RELEASE-FILE)
-echo $VERSION > version
-./scripts/git-archive-all --extra version --force-submodules --prefix $PRODUCT-jmx ../$PRODUCT-jmx_$SCYLLA_VERSION-$SCYLLA_RELEASE.orig.tar.gz 
+ln -fv $RELOC_PKG_FULLPATH ../$PRODUCT-jmx_$SCYLLA_VERSION-$SCYLLA_RELEASE.orig.tar.gz

-cp -a dist/debian/debian debian
+cp -al dist/debian/debian debian
 if [ "$PRODUCT" != "scylla" ]; then
    for i in debian/scylla-*;do
        mv $i ${i/scylla-/$PRODUCT-}
    done
 fi
-if is_debian $TARGET; then
-    REVISION="1~$TARGET"
-elif is_ubuntu $TARGET; then
-    REVISION="0ubuntu1~$TARGET"
-else
-   echo "Unknown distribution: $TARGET"
-fi

+REVISION="1"
 MUSTACHE_DIST="\"debian\": true, \"$TARGET\": true, \"product\": \"$PRODUCT\", \"$PRODUCT\": true"
 pystache dist/debian/changelog.mustache "{ $MUSTACHE_DIST, \"version\": \"$SCYLLA_VERSION\", \"release\": \"$SCYLLA_RELEASE\", \"revision\": \"$REVISION\", \"codename\": \"$TARGET\" }" > debian/changelog
 pystache dist/debian/rules.mustache "{ $MUSTACHE_DIST }" > debian/rules
 chmod a+rx debian/rules
 pystache dist/debian/control.mustache "{ $MUSTACHE_DIST }" > debian/control

-if [ "$TARGET" != "trusty" ]; then
-    pystache dist/common/systemd/scylla-jmx.service.mustache "{ $MUSTACHE_DIST }" > debian/scylla-jmx.service
-fi
+pystache dist/common/systemd/scylla-jmx.service.mustache "{ $MUSTACHE_DIST }" > debian/scylla-jmx.service

-sudo rm -fv /var/cache/pbuilder/$PRODUCT-jmx-$TARGET.tgz
-sudo PRODUCT=$PRODUCT DIST=$TARGET /usr/sbin/pbuilder clean --configfile ./dist/debian/pbuilderrc
-sudo PRODUCT=$PRODUCT DIST=$TARGET /usr/sbin/pbuilder create --configfile ./dist/debian/pbuilderrc
-sudo PRODUCT=$PRODUCT DIST=$TARGET /usr/sbin/pbuilder update --configfile ./dist/debian/pbuilderrc
-if [ "$TARGET" = "jessie" ]; then
-    echo "apt-get install -y -t jessie-backports ca-certificates-java" > build/jessie-pkginst.sh
-    chmod a+rx build/jessie-pkginst.sh
-    sudo PRODUCT=$PRODUCT DIST=$TARGET /usr/sbin/pbuilder execute --configfile ./dist/debian/pbuilderrc build/jessie-pkginst.sh
-elif [ "$TARGET" = "bionic" ]; then
-    echo "apt-get install -y ca-certificates-java openjdk-8-jdk-headless" > build/bionic-workaround.sh
-    echo "update-ca-certificates -f" >> build/bionic-workaround.sh
-    chmod a+rx build/bionic-workaround.sh
-    sudo PRODUCT=$PRODUCT DIST=$TARGET /usr/sbin/pbuilder execute --configfile ./dist/debian/pbuilderrc --save-after-exec build/bionic-workaround.sh
-fi
-sudo PRODUCT=$PRODUCT DIST=$TARGET pdebuild --configfile ./dist/debian/pbuilderrc --buildresult build/debs
+debuild -rfakeroot -us -uc
--- a/dist/debian/control.mustache
+++ b/dist/debian/control.mustache
@ -4,7 +4,6 @@ Homepage: http://scylladb.com
 Section: database
 Priority: optional
 Standards-Version: 3.9.5
-Build-Depends: debhelper (>= 9), maven, openjdk-8-jdk-headless

 Package: {{product}}-jmx
 Architecture: all
--- a/dist/debian/rules.mustache
+++ b/dist/debian/rules.mustache
@ -5,10 +5,8 @@ DEST = $(CURDIR)/debian/{{product}}-jmx/usr/lib/scylla/jmx
 M2_REPO= $(CURDIR)/m2

 override_dh_auto_build:
-	mvn -B -Dmaven.repo.local=$(M2_REPO) install

 override_dh_auto_clean:
-	rm -rf target m2

 override_dh_auto_install:
 	mkdir -p $(CURDIR)/debian/{{product}}-jmx/etc/default/ && \
@ -20,28 +18,18 @@ override_dh_auto_install:
 	cp $(CURDIR)/NOTICE $(DOC)

 	mkdir -p $(DEST)
-	cp $(CURDIR)/scripts/scylla-jmx $(DEST)
-	cp $(CURDIR)/target/scylla-jmx-1.0.jar $(DEST)
+	cp $(CURDIR)/scylla-jmx $(DEST)
+	cp $(CURDIR)/scylla-jmx-1.0.jar $(DEST)
 	mkdir $(DEST)/symlinks
 	ln -sf /usr/bin/java $(DEST)/symlinks/scylla-jmx

 override_dh_installinit:
-{{#trusty}}
-{{#scylla}}
-	dh_installinit --no-start --upstart-only
-{{/scylla}}
-{{^scylla}}
-	dh_installinit --no-start --name scylla-jmx --upstart-only
-{{/scylla}}
-{{/trusty}}
-{{^trusty}}
 {{#scylla}}
 	dh_installinit --no-start
 {{/scylla}}
 {{^scylla}}
 	dh_installinit --no-start --name scylla-jmx
 {{/scylla}}
-{{/trusty}}

 %:
 	dh $@
--- a/dist/redhat/build_rpm.sh
+++ b/dist/redhat/build_rpm.sh
@ -4,15 +4,15 @@ PRODUCT=scylla

 . /etc/os-release
 print_usage() {
-    echo "build_rpm.sh -target epel-7-x86_64 --configure-user"
+    echo "build_rpm.sh --reloc-pkg build/scylla-jmx-package.tar.gz"
    echo "  --target target distribution in mock cfg name"
    exit 1
 }
-TARGET=
+RELOC_PKG=
 while [ $# -gt 0 ]; do
    case "$1" in
-        "--target")
-            TARGET=$2
+        "--reloc-pkg")
+            RELOC_PKG=$2
            shift 2
            ;;
        *)
@ -33,34 +33,20 @@ pkg_install() {
    fi
 }

-
-if [ ! -e dist/redhat/build_rpm.sh ]; then
-    echo "run build_rpm.sh in top of scylla-jmx dir"
+if [ ! -e SCYLLA-RELOCATABLE-FILE ]; then
+    echo "do not directly execute build_rpm.sh, use reloc/build_rpm.sh instead."
    exit 1
 fi

-if [ "$(arch)" != "x86_64" ]; then
-    echo "Unsupported architecture: $(arch)"
+if [ -z "$RELOC_PKG" ]; then
+    print_usage
    exit 1
 fi
-if [ -z "$TARGET" ]; then
-    if [ "$ID" = "centos" -o "$ID" = "rhel" ] && [ "$VERSION_ID" = "7" ]; then
-        TARGET=./dist/redhat/mock/scylla-jmx-epel-7-x86_64.cfg
-    elif [ "$ID" = "fedora" ]; then
-        TARGET=$ID-$VERSION_ID-x86_64
-    else
-        echo "Please specify target"
-        exit 1
-    fi
+if [ ! -f "$RELOC_PKG" ]; then
+    echo "$RELOC_PKG is not found."
+    exit 1
 fi

-if [[ "$TARGET" = epel-7-x86_64 ]]; then
-    TARGET=./dist/redhat/mock/scylla-jmx-epel-7-x86_64.cfg
-fi
-
-if [ ! -f /usr/bin/mock ]; then
-    pkg_install mock
-fi
 if [ ! -f /usr/bin/git ]; then
    pkg_install git
 fi
@ -72,18 +58,15 @@ if [ ! -f /usr/bin/pystache ]; then
    fi
 fi

-VERSION=$(./SCYLLA-VERSION-GEN)
-SCYLLA_VERSION=$(cat build/SCYLLA-VERSION-FILE)
-SCYLLA_RELEASE=$(cat build/SCYLLA-RELEASE-FILE)
-git archive --format=tar --prefix=$PRODUCT-jmx-$SCYLLA_VERSION/ HEAD -o build/$PRODUCT-jmx-$VERSION.tar
-pystache dist/redhat/scylla-jmx.spec.mustache "{ \"version\": \"$SCYLLA_VERSION\", \"release\": \"$SCYLLA_RELEASE\", \"product\": \"$PRODUCT\", \"$PRODUCT\": true }" > build/scylla-jmx.spec
+SCYLLA_VERSION=$(cat SCYLLA-VERSION-FILE)
+SCYLLA_RELEASE=$(cat SCYLLA-RELEASE-FILE)
+VERSION=$SCYLLA_VERSION-$SCYLLA_RELEASE

-# mock generates files owned by root, fix this up
-fix_ownership() {
-    sudo chown "$(id -u):$(id -g)" -R "$@"
-}
+RPMBUILD=$(readlink -f ../)
+mkdir -p $RPMBUILD/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS}

-sudo mock --buildsrpm --root=$TARGET --resultdir=`pwd`/build/srpms --spec=build/scylla-jmx.spec --sources=build/$PRODUCT-jmx-$VERSION.tar
-fix_ownership build/srpms
-sudo mock --rebuild --root=$TARGET --resultdir=`pwd`/build/rpms build/srpms/$PRODUCT-jmx-$VERSION*.src.rpm
-fix_ownership build/rpms
+ln -fv $RELOC_PKG $RPMBUILD/SOURCES/
+pystache dist/redhat/scylla-jmx.spec.mustache "{ \"version\": \"$SCYLLA_VERSION\", \"release\": \"$SCYLLA_RELEASE\", \"product\": \"$PRODUCT\", \"$PRODUCT\": true }" > $RPMBUILD/SPECS/scylla-jmx.spec
+
+# this rpm can be install on both fedora / centos7, so drop distribution name from the file name
+rpmbuild -ba --define "_topdir $RPMBUILD" --undefine "dist" $RPM_JOBS_OPTS $RPMBUILD/SPECS/scylla-jmx.spec
--- a/dist/redhat/scylla-jmx.spec.mustache
+++ b/dist/redhat/scylla-jmx.spec.mustache
@ -6,10 +6,10 @@ Group:          Applications/Databases

 License:        AGPLv3
 URL:            http://www.scylladb.com/
-Source0:        %{name}-{{version}}-{{release}}.tar
+Source0:        scylla-jmx-package.tar.gz

 BuildArch:      noarch
-BuildRequires:  maven systemd-units java-1.8.0-openjdk-devel
+BuildRequires:  systemd-units
 %{?fedora:BuildRequires: python2-pystache}
 %{?rhel:BuildRequires: pystache python-setuptools}
 Requires:       {{product}}-server java-1.8.0-openjdk-headless
@ -18,14 +18,12 @@ Requires:       {{product}}-server java-1.8.0-openjdk-headless


 %prep
-%setup -q
+%setup -c


 %build
-mvn -B install
-mkdir build
 MUSTACHE_DIST="\"redhat\": true"
-pystache dist/common/systemd/scylla-jmx.service.mustache "{ $MUSTACHE_DIST }" > build/scylla-jmx.service
+pystache dist/common/systemd/scylla-jmx.service.mustache "{ $MUSTACHE_DIST }" > scylla-jmx.service

 %install
 rm -rf $RPM_BUILD_ROOT
@ -34,12 +32,12 @@ mkdir -p $RPM_BUILD_ROOT%{_unitdir}
 mkdir -p $RPM_BUILD_ROOT%{_prefix}/lib/scylla/

 install -m644 dist/common/sysconfig/scylla-jmx $RPM_BUILD_ROOT%{_sysconfdir}/sysconfig/
-install -m644 build/*.service $RPM_BUILD_ROOT%{_unitdir}/
+install -m644 scylla-jmx.service $RPM_BUILD_ROOT%{_unitdir}/
 install -d -m755 $RPM_BUILD_ROOT%{_prefix}/lib/scylla
 install -d -m755 $RPM_BUILD_ROOT%{_prefix}/lib/scylla/jmx
 install -d -m755 $RPM_BUILD_ROOT%{_prefix}/lib/scylla/jmx/symlinks
-install -m644 target/scylla-jmx-1.0.jar $RPM_BUILD_ROOT%{_prefix}/lib/scylla/jmx/
-install -m755 scripts/scylla-jmx $RPM_BUILD_ROOT%{_prefix}/lib/scylla/jmx
+install -m644 scylla-jmx-1.0.jar $RPM_BUILD_ROOT%{_prefix}/lib/scylla/jmx/
+install -m755 scylla-jmx $RPM_BUILD_ROOT%{_prefix}/lib/scylla/jmx
 ln -sf /usr/bin/java $RPM_BUILD_ROOT%{_prefix}/lib/scylla/jmx/symlinks/scylla-jmx

 %pre
--- a/install-dependencies.sh
+++ b/install-dependencies.sh
@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# This file is open source software, licensed to you under the terms
+# of the Apache License, Version 2.0 (the "License").  See the NOTICE file
+# distributed with this work for additional information regarding copyright
+# ownership.  You may not use this file except in compliance with the License.
+#
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+. /etc/os-release
+
+if [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ]; then
+    apt -y install maven openjdk-8-jdk-headless
+elif [ "$ID" = "fedora" ] || [ "$ID" = "centos" ]; then
+    yum install -y maven java-1.8.0-openjdk-devel
+fi
--- a/reloc/build_deb.sh
+++ b/reloc/build_deb.sh
@ -0,0 +1,31 @@
+#!/bin/bash -e
+
+. /etc/os-release
+print_usage() {
+    echo "build_deb.sh --reloc-pkg build/scylla-jmx-package.tar.gz"
+    echo "  --reloc-pkg specify relocatable package path"
+    exit 1
+}
+
+RELOC_PKG=$(readlink -f build/scylla-jmx-package.tar.gz)
+OPTS=""
+while [ $# -gt 0 ]; do
+    case "$1" in
+        "--reloc-pkg")
+            OPTS="$OPTS $1 $(readlink -f $2)"
+            RELOC_PKG=$2
+            shift 2
+            ;;
+        *)
+            print_usage
+            ;;
+    esac
+done
+
+if [[ ! $OPTS =~ --reloc-pkg ]]; then
+    OPTS="$OPTS --reloc-pkg $RELOC_PKG"
+fi
+mkdir -p build/debian/scylla-package
+tar -C build/debian/scylla-package -xpf $RELOC_PKG
+cd build/debian/scylla-package
+exec ./dist/debian/build_deb.sh $OPTS
--- a/reloc/build_reloc.sh
+++ b/reloc/build_reloc.sh
@ -0,0 +1,57 @@
+#!/bin/bash -e
+
+. /etc/os-release
+
+print_usage() {
+    echo "build_reloc.sh --clean --nodeps"
+    echo "  --clean clean build directory"
+    echo "  --nodeps    skip installing dependencies"
+    exit 1
+}
+
+CLEAN=
+NODEPS=
+while [ $# -gt 0 ]; do
+    case "$1" in
+        "--clean")
+            CLEAN=yes
+            shift 1
+            ;;
+        "--nodeps")
+            NODEPS=yes
+            shift 1
+            ;;
+        *)
+            print_usage
+            ;;
+    esac
+done
+
+is_redhat_variant() {
+    [ -f /etc/redhat-release ]
+}
+is_debian_variant() {
+    [ -f /etc/debian_version ]
+}
+
+
+if [ ! -e reloc/build_reloc.sh ]; then
+    echo "run build_reloc.sh in top of scylla dir"
+    exit 1
+fi
+
+if [ "$CLEAN" = "yes" ]; then
+    rm -rf build target
+fi
+
+if [ -f build/scylla-jmx-package.tar.gz ]; then
+    rm build/scylla-jmx-package.tar.gz
+fi
+
+if [ -z "$NODEPS" ]; then
+    sudo ./install-dependencies.sh
+fi
+
+mvn -B install
+./SCYLLA-VERSION-GEN
+scripts/create-relocatable-package.py build/scylla-jmx-package.tar.gz
--- a/reloc/build_rpm.sh
+++ b/reloc/build_rpm.sh
@ -0,0 +1,30 @@
+#!/bin/bash -e
+
+. /etc/os-release
+print_usage() {
+    echo "build_rpm.sh --reloc-pkg build/scylla-jmx-package.tar.gz"
+    echo "  --reloc-pkg specify relocatable package path"
+    exit 1
+}
+RELOC_PKG=$(readlink -f build/scylla-jmx-package.tar.gz)
+OPTS=""
+while [ $# -gt 0 ]; do
+    case "$1" in
+        "--reloc-pkg")
+            OPTS="$OPTS $1 $(readlink -f $2)"
+            RELOC_PKG=$2
+            shift 2
+            ;;
+        *)
+            print_usage
+            ;;
+    esac
+done
+
+if [[ ! $OPTS =~ --reloc-pkg ]]; then
+    OPTS="$OPTS --reloc-pkg $RELOC_PKG"
+fi
+mkdir -p build/redhat/scylla-package
+tar -C build/redhat/scylla-package -xpf $RELOC_PKG SCYLLA-RELEASE-FILE SCYLLA-RELOCATABLE-FILE SCYLLA-VERSION-FILE dist/redhat
+cd build/redhat/scylla-package
+exec ./dist/redhat/build_rpm.sh $OPTS
--- a/scripts/create-relocatable-package.py
+++ b/scripts/create-relocatable-package.py
@ -0,0 +1,48 @@
+#!/usr/bin/python3
+
+#
+# Copyright (C) 2018 ScyllaDB
+#
+
+#
+# This file is part of Scylla.
+#
+# Scylla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Scylla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import argparse
+import io
+import os
+import tarfile
+import pathlib
+
+
+ap = argparse.ArgumentParser(description='Create a relocatable scylla package.')
+ap.add_argument('dest',
+                help='Destination file (tar format)')
+
+args = ap.parse_args()
+
+output = args.dest
+
+ar = tarfile.open(output, mode='w|gz')
+pathlib.Path('build/SCYLLA-RELOCATABLE-FILE').touch()
+ar.add('build/SCYLLA-RELOCATABLE-FILE', arcname='SCYLLA-RELOCATABLE-FILE')
+ar.add('build/SCYLLA-RELEASE-FILE', arcname='SCYLLA-RELEASE-FILE')
+ar.add('build/SCYLLA-VERSION-FILE', arcname='SCYLLA-VERSION-FILE')
+ar.add('dist')
+ar.add('target/scylla-jmx-1.0.jar', arcname='scylla-jmx-1.0.jar')
+ar.add('scripts/scylla-jmx', arcname='scylla-jmx')
+ar.add('README.md')
+ar.add('NOTICE')