From 439377927baf5d82a5adffd390f762143582c14c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Fri, 3 Jul 2020 16:24:58 +0300 Subject: [PATCH] Deb: Misc ColumnStore Debian packaging improvements - Add 'libboost-all-dev' and 'libreadline-gplv2-dev' as they were was found to be a compulsory build dependency for columnstore plugin. - Add 'expect' as run-time dependencey for columnstore plugin as scripts use it: usr/bin/mcs_module_installer.sh: #!/usr/bin/expect usr/bin/remote_command.sh: #!/usr/bin/expect usr/bin/remote_command_verify.sh: #!/usr/bin/expect usr/bin/remote_scp_get.sh: #!/usr/bin/expect usr/bin/remote_scp_put.sh: #!/usr/bin/expect usr/bin/rsync.sh: #!/usr/bin/expect - Properly define depends on Python. No Python 2 support needs to be considered, Python 3 has been around long enough. Fixes Lintian errors E: mariadb-plugin-columnstore: python-script-but-no-python-dep usr/bin/mcs-loadbrm.py #!python E: mariadb-plugin-columnstore: python-script-but-no-python-dep usr/bin/mcs-start-storagemanager.py #!python - Partially revert undocumented and thus unjustified changes in commits d69a79da63287089efdc5f90a11ecd66ce55b471 and c0565666cfe6528b76bc53ce50d3690d13c92cf6. - Trigger ldconfig, otherwise Lintian complains: E: mariadb-plugin-columnstore: package-must-activate-ldconfig-trigger usr/lib/x86_64-linux-gnu/libwriteengineredistribute.so - Update postinst to be compatible with new server binary mariadbd name. - Properly detect systemd or fallback to sysv init in postrm script. - Only attempt to build ColumnStore on amd64 and i386. Test builds on Launchpad.net showed the CMake plugin configure step will prevent even attempts to build on other platforms. - Clean up and unify cmake build command in debian/rules. - Explicitly list files not installed. - Run 'wrap-and-sort -a -v'. - Truncate build logs on Salsa-CI to keep under 4 MB. This is now needed as the ColumnStore build is so verbose. See https://jira.mariadb.org/browse/MCOL-4111. - Update Travis-CI dependencies to match new debian/control. --- .travis.yml | 8 ++++++ debian/autobake-deb.sh | 13 ++++----- debian/control | 19 +++++++++---- debian/mariadb-plugin-columnstore.install | 12 ++++----- debian/mariadb-plugin-columnstore.postinst | 2 -- debian/mariadb-plugin-columnstore.postrm | 27 ++++++++----------- debian/mariadb-plugin-columnstore.prerm | 1 - debian/mariadb-plugin-columnstore.triggers | 1 + ...adb-plugin-cracklib-password-check.install | 2 +- debian/mariadb-plugin-gssapi-server.install | 2 +- debian/not-installed | 9 +++++-- debian/rules | 14 +++++++--- debian/salsa-ci.yml | 3 ++- 13 files changed, 69 insertions(+), 44 deletions(-) create mode 100644 debian/mariadb-plugin-columnstore.triggers diff --git a/.travis.yml b/.travis.yml index 09309534f4a..51897563235 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,7 +40,14 @@ addons: - g++-9 # ubuntu-toolchain-r/test (xenial and bionic) - gdb - libaio-dev + - libboost-atomic-dev + - libboost-chrono-dev + - libboost-date-time-dev - libboost-dev + - libboost-filesystem-dev + - libboost-regex-dev + - libboost-system-dev + - libboost-thread-dev - libcrack2-dev - libcurl3-dev - libdbd-mysql @@ -52,6 +59,7 @@ addons: - libnuma-dev - libpam0g-dev - libpcre2-dev + - libreadline-gplv2-dev - libsnappy-dev - libssl-dev - libstemmer-dev diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index e838f5964b2..5e20df3a96b 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -31,7 +31,7 @@ then sed 's|DINSTALL_MYSQLTESTDIR=share/mysql/mysql-test|DINSTALL_MYSQLTESTDIR=false|' -i debian/rules # Also skip building RocksDB, Mroonga etc to save even more time and disk space - sed 's|-DDEB|-DPLUGIN_MROONGA=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_PERFSCHEMA=NO -DPLUGIN_SPHINX=NO -DDEB|' -i debian/rules + sed 's|-DDEB|-DPLUGIN_MROONGA=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_PERFSCHEMA=NO -DPLUGIN_SPHINX=NO -DDEB|' -i debian/rules fi # Convert gcc version to numberical value. Format is Mmmpp where M is Major @@ -75,7 +75,6 @@ fi if [[ $GCCVERSION -lt 40800 ]] || [[ $(arch) =~ i[346]86 ]] || [[ $TRAVIS ]] then sed '/Package: mariadb-plugin-rocksdb/,/^$/d' -i debian/control - sed -i 's|-DPLUGIN_ROCKSDB=YES|-DPLUGIN_ROCKSDB=NO|' debian/rules fi # If libpcre2-dev is not available (before Debian Stretch and Ubuntu Xenial) @@ -95,11 +94,13 @@ then sed -i -e "/Package: libmariadbd-dev/,/^$/d" debian/control fi -if [[ $TRAVIS ]] || ! [[ $(arch) =~ 86 ]] +# Don't build nor package ColumnStore on Travis-CI +if [[ $TRAVIS ]] then - sed -i -e "/Package: mariadb-plugin-columnstore/,/^$/d" debian/control - sed -i '/flex/d' debian/control - sed -i 's|-DPLUGIN_COLUMNSTORE=YES|-DPLUGIN_COLUMNSTORE=NO|' debian/rules + sed "/Package: mariadb-plugin-columnstore/,/^$/d" -i debian/control + sed '/flex/d' -i debian/control + sed -r '/libboost-[a-z-]+-dev/d' -i debian/control + sed 's|-DPLUGIN_COLUMNSTORE=YES|-DPLUGIN_COLUMNSTORE=NO|' -i debian/rules fi # Adjust changelog, add new version diff --git a/debian/control b/debian/control index 19864c6bb1b..a3bc3b91c6e 100644 --- a/debian/control +++ b/debian/control @@ -9,10 +9,17 @@ Build-Depends: bison, dh-apparmor, dh-exec, dh-systemd, - flex, + flex [i386 amd64], gdb, libaio-dev [linux-any], + libboost-atomic-dev [i386 amd64], + libboost-chrono-dev [i386 amd64], + libboost-date-time-dev [i386 amd64], libboost-dev, + libboost-filesystem-dev [i386 amd64], + libboost-regex-dev [i386 amd64], + libboost-system-dev [i386 amd64], + libboost-thread-dev [i386 amd64], libcrack2-dev (>= 2.9.0), libcurl4-openssl-dev | libcurl4-dev, libedit-dev, @@ -24,6 +31,7 @@ Build-Depends: bison, libnuma-dev [!armhf], libpam0g-dev, libpcre2-dev, + libreadline-gplv2-dev [i386 amd64], libsnappy-dev, libssl-dev | libssl1.0-dev, libsystemd-dev [linux-any], @@ -653,11 +661,12 @@ Description: RocksDB storage engine for MariaDB Package: mariadb-plugin-columnstore Architecture: amd64 i386 -Depends: mariadb-server-10.5 (= ${binary:Version}), - net-tools, - binutils, - libsnappy1 | libsnappy1v5, +Depends: binutils, + expect, libjemalloc1 | libjemalloc2, + libsnappy1 | libsnappy1v5, + mariadb-server-10.5 (= ${binary:Version}), + net-tools, python3, ${misc:Depends}, ${shlibs:Depends} diff --git a/debian/mariadb-plugin-columnstore.install b/debian/mariadb-plugin-columnstore.install index cd4461fe308..9f44f74e663 100644 --- a/debian/mariadb-plugin-columnstore.install +++ b/debian/mariadb-plugin-columnstore.install @@ -18,7 +18,6 @@ usr/bin/ProcMon usr/bin/ServerMonitor usr/bin/StorageManager usr/bin/WriteEngineServer -usr/bin/dbmsReport.sh usr/bin/autoConfigure usr/bin/bulklogReport.sh usr/bin/clearShm @@ -42,6 +41,7 @@ usr/bin/cpimport usr/bin/cpimport.bin usr/bin/cplogger usr/bin/dbbuilder +usr/bin/dbmsReport.sh usr/bin/dbrmctl usr/bin/ddlcleanup usr/bin/disable-rep-columnstore.sh @@ -51,8 +51,12 @@ usr/bin/hardwareReport.sh usr/bin/idbmeminfo usr/bin/load_brm usr/bin/logReport.sh +usr/bin/mariadb-columnstore-start.sh +usr/bin/mariadb-columnstore-stop.sh usr/bin/mariadb-command-line.sh usr/bin/master-rep-columnstore.sh +usr/bin/mcs-loadbrm.py +usr/bin/mcs-start-storagemanager.py usr/bin/mcs-stop-controllernode.sh usr/bin/mcsGetConfig usr/bin/mcsSetConfig @@ -82,10 +86,6 @@ usr/bin/smrm usr/bin/startupTests.sh usr/bin/viewtablelock usr/bin/workernode -usr/bin/mcs-start-storagemanager.py -usr/bin/mcs-loadbrm.py -usr/bin/mariadb-columnstore-start.sh -usr/bin/mariadb-columnstore-stop.sh usr/lib/*/libalarmmanager.so usr/lib/*/libbatchloader.so usr/lib/*/libbrm.so @@ -146,9 +146,9 @@ usr/share/columnstore/mcs-dmlproc.service usr/share/columnstore/mcs-exemgr.service usr/share/columnstore/mcs-loadbrm.service usr/share/columnstore/mcs-primproc.service +usr/share/columnstore/mcs-storagemanager.service usr/share/columnstore/mcs-workernode.service usr/share/columnstore/mcs-writeengineserver.service -usr/share/columnstore/mcs-storagemanager.service usr/share/columnstore/mcstest-001.sh usr/share/columnstore/mcstest-002.sh usr/share/columnstore/mcstest-003.sh diff --git a/debian/mariadb-plugin-columnstore.postinst b/debian/mariadb-plugin-columnstore.postinst index b482002c9dc..07da2b9ee46 100644 --- a/debian/mariadb-plugin-columnstore.postinst +++ b/debian/mariadb-plugin-columnstore.postinst @@ -5,6 +5,4 @@ set -e # Install ColumnStore columnstore-post-install --rpmmode=install -echo "MariaDB ColumnStore install completed" - #DEBHELPER# diff --git a/debian/mariadb-plugin-columnstore.postrm b/debian/mariadb-plugin-columnstore.postrm index cd8decbfe76..b2e19774ec1 100644 --- a/debian/mariadb-plugin-columnstore.postrm +++ b/debian/mariadb-plugin-columnstore.postrm @@ -8,22 +8,17 @@ if [ "$1" = "purge" ]; then rm -f /etc/mysql/mariadb.conf.d/columnstore.cnf.rpmsave fi -running_systemd() { - if [ "$(ps --no-headers -o comm 1)" = "systemd" ]; then - echo 0 - else - echo 1 - fi -} - -if [ ! -z "$(pgrep -x mysqld)" ];then - systemctl cat mariadb.service > /dev/null 2>&1 - if [ $? -eq 0 ] && [ $(running_systemd) -eq 0 ]; then - systemctl restart mariadb.service > /dev/null 2>&1 - else - pkill mysqld > /dev/null 2>&1 - /usr/bin/mysqld_safe & +# Automatically restart MariaDB after ColumnStore plugin has been removed +case "$1" in + purge|remove|disappear) + if [ -d /run/systemd/system ]; then + # If systemd + deb-systemd-invoke restart mariadb.service >/dev/null + elif [ -x "/etc/init.d/mariadb" ]; then + # Fall-back to SysV init + invoke-rc.d mariadb restart || exit $? fi -fi + ;; +esac #DEBHELPER# diff --git a/debian/mariadb-plugin-columnstore.prerm b/debian/mariadb-plugin-columnstore.prerm index 10e585a887b..f6a27f14499 100644 --- a/debian/mariadb-plugin-columnstore.prerm +++ b/debian/mariadb-plugin-columnstore.prerm @@ -4,6 +4,5 @@ set -e columnstore-pre-uninstall -exit 0 #DEBHELPER# diff --git a/debian/mariadb-plugin-columnstore.triggers b/debian/mariadb-plugin-columnstore.triggers new file mode 100644 index 00000000000..dd866036784 --- /dev/null +++ b/debian/mariadb-plugin-columnstore.triggers @@ -0,0 +1 @@ +activate-noawait ldconfig diff --git a/debian/mariadb-plugin-cracklib-password-check.install b/debian/mariadb-plugin-cracklib-password-check.install index 5419f05ed3d..6592c7f461b 100644 --- a/debian/mariadb-plugin-cracklib-password-check.install +++ b/debian/mariadb-plugin-cracklib-password-check.install @@ -1,2 +1,2 @@ -etc/mysql/mariadb.conf.d/cracklib_password_check.cnf +etc/mysql/mariadb.conf.d/cracklib_password_check.cnf usr/lib/mysql/plugin/cracklib_password_check.so diff --git a/debian/mariadb-plugin-gssapi-server.install b/debian/mariadb-plugin-gssapi-server.install index 70916b49052..8261db789f7 100644 --- a/debian/mariadb-plugin-gssapi-server.install +++ b/debian/mariadb-plugin-gssapi-server.install @@ -1,2 +1,2 @@ -etc/mysql/mariadb.conf.d/auth_gssapi.cnf +etc/mysql/mariadb.conf.d/auth_gssapi.cnf usr/lib/mysql/plugin/auth_gssapi.so diff --git a/debian/not-installed b/debian/not-installed index 1c20318e2b8..58cd337212b 100644 --- a/debian/not-installed +++ b/debian/not-installed @@ -1,12 +1,17 @@ +etc/columnstore/storagemanager.cnf.example # Copy of etc/columnstore/storagemanager.cnf that is installed lib/systemd/system/mariadb.service # Installed by rules file lib/systemd/system/mariadb@.service # Installed by rules file usr/bin/mysql_config # We already have the MariaDB variant usr/bin/mytop # Mytop is distributed from a separate source package usr/bin/sst_dump # Use the one from rocksdb-tools package -usr/lib/sysusers.d/sysusers.conf -usr/lib/tmpfiles.d/tmpfiles.conf usr/lib/mysql/plugin/JavaWrappers.jar # These are only built if JNI/libjawt.so is installed from e.g. openjdk-8-jre-headless usr/lib/mysql/plugin/JdbcInterface.jar # These are only built if JNI/libjawt.so is installed from e.g. openjdk-8-jre-headless +usr/lib/sysusers.d/sysusers.conf +usr/lib/tmpfiles.d/tmpfiles.conf +usr/lib/*/libdbbc.a # ColumnStore header file +usr/lib/*/libidbboot.a # ColumnStore header file +usr/lib/*/libprocessor.a # ColumnStore header file +usr/lib/*/libwe_xml.a # ColumnStore header file usr/share/doc/mariadb-server-10.5/COPYING usr/share/doc/mariadb-server-10.5/COPYING.AGPLv3 usr/share/doc/mariadb-server-10.5/COPYING.GPLv2 diff --git a/debian/rules b/debian/rules index 61717416e35..59f4c571679 100755 --- a/debian/rules +++ b/debian/rules @@ -39,6 +39,17 @@ else TESTSUITE_FAIL_CMD:=exit 1 endif +# RocksDB cannot build on 32-bit platforms +ifeq (32,$(DEB_HOST_ARCH_BITS)) + CMAKEFLAGS += -DWITHOUT_ROCKSDB=true +endif + +# ColumnStore only attempts to build on a few platforms as dictated by CMake checks +# Also note in debian/control the CS-only build deps marked '[i386 amd64]' +ifeq ($(DEB_HOST_ARCH),$(filter $(DEB_HOST_ARCH),amd64 i386)) + CMAKEFLAGS += -DPLUGIN_COLUMNSTORE=YES +endif + # Add extra flag to avoid WolfSSL code crashing the entire mariadbd on s390x. This # can be removed once upstream has made the code s390x compatible, see # https://jira.mariadb.org/browse/MDEV-21705 and @@ -83,10 +94,7 @@ endif -DBUILD_CONFIG=mysql_release \ -DPLUGIN_TOKUDB=NO \ -DPLUGIN_CASSANDRA=NO \ - -DPLUGIN_COLUMNSTORE=YES \ - -DPLUGIN_ROCKSDB=YES \ -DPLUGIN_AWS_KEY_MANAGEMENT=NO \ - -WITH_EMBEDDED_SERVER=OFF \ -DDEB=$(DEB_VENDOR) ..' # This is needed, otherwise 'make test' will run before binaries have been built diff --git a/debian/salsa-ci.yml b/debian/salsa-ci.yml index 57da0364911..bd49f864562 100644 --- a/debian/salsa-ci.yml +++ b/debian/salsa-ci.yml @@ -33,7 +33,8 @@ build: - export CCACHE_DIR="${CCACHE_TMP_DIR}" - update-ccache-symlinks; ccache -z # Zero out ccache counters - mk-build-deps debian/control -t "apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends" -r -i - - debian/autobake-deb.sh + - while true; do sleep 600; echo "10 minutes passed" >&2; done & # Progress keeper since build is long and silent + - debian/autobake-deb.sh |& tail -n 10000 # Keep Gitlab-CI output under 4 MB - cd ..; rm -rfv *.tmp # Clean away build files not to store as artifacts - cp -v *.* ${WORKING_DIR}/ - du -shc ${WORKING_DIR}/* # Show total file size of artifacts. Must stay are under 100 MB.