Extend the Gitlab-CI pipeline to run mini benchmark

Implement new mini-benchmark script for simple CPU bound benchmark for the duration of 5 minutes. The script can be run stand-alone or as part of a CI pipeline. Extend Gitlab-CI to run mini-benchmark on every commit to catch if there are severe performance regressions. Also bump MARIADB_MAJOR_VERSION to 10.8 which is needed on the 10.8 branch.
2025-01-15 19:42:28 +01:00 · 2021-10-11 12:53:11 -07:00 · 2021-10-11 12:53:11 -07:00 · c5c61b51b6
commit c5c61b51b6
parent 1f5fc7b745
4 changed files with 262 additions and 3 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -39,7 +39,7 @@ variables:
  CMAKE_FLAGS: "-DWITH_SSL=system -DPLUGIN_COLUMNSTORE=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_S3=NO -DPLUGIN_MROONGA=NO -DPLUGIN_CONNECT=NO -DPLUGIN_MROONGA=NO -DPLUGIN_TOKUDB=NO -DPLUGIN_PERFSCHEMA=NO -DWITH_WSREP=OFF"
  # Major version dictates which branches share the same ccache. E.g. 10.6-abc
  # and 10.6-xyz will have the same cache.
-  MARIADB_MAJOR_VERSION: "10.6"
+  MARIADB_MAJOR_VERSION: "10.8"
  # NOTE! Currently ccache is only used on the Centos8 build. As each job has
  # sufficiently different environments they are unable to benefit from each
  # other's ccaches. As each build generates about 1 GB of ccache, having
@ -364,6 +364,35 @@ fedora upgrade:
      - new-installed-database.sql
      - new-upgraded-database.sql

+mini-benchmark:
+  stage: test
+  dependencies:
+    - fedora
+  script:
+    - ls -la rpm; rm -vf rpm/*.el?.*  # Delete artifacts from Centos builds
+    # Don't use cracklib, otherwise the Sysbench user password will be rejected
+    - rm -vf rpm/*cracklib*.rpm
+    # Nothing provides galera-4 on Fedora, so this step fails if built with wsrep
+    - yum install -y rpm/*.rpm
+    # Fedora does not support running services in Docker (like Debian packages do) so start it manually
+    - /usr/bin/mariadb-install-db -u mysql
+    - sudo -u mysql /usr/sbin/mariadbd & sleep 10
+    # Since we did a manual start, we also need to run upgrade manually
+    - /usr/bin/mariadb-upgrade -u root
+    - |
+      mariadb --skip-column-names -e "SELECT @@version, @@version_comment" | tee /tmp/version
+      grep $MARIADB_MAJOR_VERSION /tmp/version || echo "MariaDB didn't install properly"
+    - yum install -y sysbench procps-ng perf || yum install -y https://kojipkgs.fedoraproject.org//packages/luajit/2.0.4/3.el7/x86_64/luajit-2.0.4-3.el7.x86_64.rpm https://kojipkgs.fedoraproject.org//packages/sysbench/1.0.17/2.el7/x86_64/sysbench-1.0.17-2.el7.x86_64.rpm https://kojipkgs.fedoraproject.org//packages/ck/0.5.2/2.el7/x86_64/ck-0.5.2-2.el7.x86_64.rpm
+    - /usr/share/mysql/mini-benchmark
+    - cp -av */sysbench-run-*.log */metrics.txt ..  # Move files one level down so they can be saved as artifacts
+  artifacts:
+    when: always
+    paths:
+      - sysbench-run-*.log
+    reports:
+      metrics:
+        - metrics.txt
+
 # Once all RPM builds and tests have passed, also run the DEB builds and tests
 # @NOTE: This is likely to work well only on salsa.debian.org as the Gitlab.com
 # runners are too small for everything this stage does.
--- a/debian/mariadb-server-10.8.install
+++ b/debian/mariadb-server-10.8.install
@ -78,5 +78,6 @@ usr/share/man/man1/wsrep_sst_mysqldump.1
 usr/share/man/man1/wsrep_sst_rsync.1
 usr/share/man/man1/wsrep_sst_rsync_wan.1
 usr/share/mysql/errmsg-utf8.txt
+usr/share/mysql/mini-benchmark
 usr/share/mysql/wsrep.cnf
 usr/share/mysql/wsrep_notify
--- a/support-files/CMakeLists.txt
+++ b/support-files/CMakeLists.txt
@ -50,8 +50,8 @@ ENDIF()

 IF(UNIX AND NOT WITHOUT_SERVER)
  SET(prefix ${CMAKE_INSTALL_PREFIX})
-  FOREACH(script mysqld_multi.server mysql-log-rotate binary-configure wsrep_notify)
-    CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/${script}.sh 
+  FOREACH(script mysqld_multi.server mysql-log-rotate binary-configure wsrep_notify mini-benchmark)
+    CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/${script}.sh
                   ${CMAKE_CURRENT_BINARY_DIR}/${script} @ONLY )
    INSTALL(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${script}
      DESTINATION ${inst_location} COMPONENT Server_Scripts)
--- a/support-files/mini-benchmark.sh
+++ b/support-files/mini-benchmark.sh
@ -0,0 +1,229 @@
+#!/bin/bash
+# Abort on errors
+set -e
+
+display_help() {
+  echo "Usage: $(basename "$0") [-h] [--perf] [--perf-flamegraph]"
+  echo
+  echo "This is a very small and naive benchmark script designed to be suitable"
+  echo "for running in a CI system on every commit to detect severe performance"
+  echo "regressions."
+  echo
+  echo "optional arguments:"
+  echo "  --perf             measure CPU cycles and instruction count in for "
+  echo "                     sysbench runs"
+  echo "  --perf-flamegraph  record performance counters in perf.data.* and"
+  echo "                     generate flamegraphs automatically"
+  echo "  -h, --help         display this help and exit"
+}
+
+while :
+do
+  case "$1" in
+    -h | --help)
+      display_help
+      exit 0
+      ;;
+    --version)
+      display_version
+      exit 0
+      ;;
+    --perf)
+      PERF=true
+      shift
+      ;;
+    --perf-flamegraph)
+      PERF_RECORD=true
+      shift
+      ;;
+    -*)
+      echo "Error: Unknown option: $1" >&2
+      ## or call function display_help
+      exit 1
+      ;;
+    *)  # No more options
+      break
+      ;;
+  esac
+done
+
+# Check that the dependencies of this script are available
+if [ ! -e /usr/bin/pgrep ]
+then
+  echo "ERROR: Command 'pgrep' missing, please install package 'psproc'"
+  exit 1
+fi
+
+if [ ! -e /usr/bin/sysbench ]
+then
+  echo "ERROR: Command 'sysbench' missing, please install package 'sysbench'"
+  exit 1
+fi
+
+# If there are multiple processes, assume the last one is the actual server and
+# any potential other ones were just part of the service wrapper chain
+MARIADB_SERVER_PID="$(echo "$(pgrep -f mariadbd || pgrep -f mysqld)" | tail -n 1)"
+
+if [ -z "$MARIADB_SERVER_PID" ]
+then
+  echo "ERROR: Server 'mariadbd' or 'mysqld' is not running, please start the service"
+  exit 1
+fi
+
+if [ "$PERF" == true ] || [ "$PERF_RECORD" == true ]
+then
+  if [ ! -e /usr/bin/perf ]
+  then
+    echo "ERROR: Command 'perf' missing, please install package 'perf'"
+    exit 1
+  fi
+fi
+
+if [ "$PERF_RECORD" == true ]
+then
+  if [ ! -e /usr/bin/flamegraph.pl ]
+  then
+    echo "ERROR: Command 'flamegraph.pl' missing, please install package 'flamegraph'"
+    exit 1
+  fi
+
+  if [ ! -e /usr/bin/stackcollapse-perf.pl ]
+  then
+    echo "ERROR: Command 'stackcollapse-perf.pl' missing, please install package 'flamegraph-stackcollapse-perf'"
+    exit 1
+  fi
+
+  if [ ! -e /usr/bin/debuginfo-install ]
+  then
+    echo "ERROR: Command 'debuginfo-install' missing, please install package 'dnf-utils'"
+    exit 1
+  fi
+
+  echo "Ensure the MariaDB Server debug symbols are installed"
+  for x in $(ldd /usr/sbin/mariadbd | grep -oE " /.* ")
+  do
+    rpm -q --whatprovides --qf '%{name}' $x | cut -d : -f 1
+  done | sort -u > mariadbd-dependencies.txt
+  # shellcheck disable=SC2046
+  debuginfo-install -y mariadb-server $(cat mariadbd-dependencies.txt)
+
+  echo "Using 'perf' to record performance counters in perf.data files"
+  PERF="perf record -g --freq=99 --output=perf.data --timestamp-filename --pid=$MARIADB_SERVER_PID --"
+
+elif [ -e /usr/bin/perf ]
+then
+  # If flamegraphs were not requested, log normal perf counters if possible
+  echo "Using 'perf' to log basic performance counters for benchmark"
+fi
+  PERF="perf stat -p $MARIADB_SERVER_PID --"
+
+# Run sysbench on another CPU if system has more than one available
+if [ "$(nproc)" -gt 1 ]
+then
+  TASKSET_SYSBENCH='taskset -c 1'
+else
+  TASKSET_SYSBENCH=''
+fi
+
+echo "System hardware information:"
+lscpu
+free -m
+df -h .
+uname -a
+echo
+
+echo "Set highest priority for MariaDB Server process ID $MARIADB_SERVER_PID"
+renice --priority -20 --pid "$MARIADB_SERVER_PID"
+
+echo "Set CPU affinity 0 for MariaDB Server process ID $MARIADB_SERVER_PID"
+taskset -cp 0 "$MARIADB_SERVER_PID"
+
+mariadb -e "
+  CREATE DATABASE IF NOT EXISTS sbtest;
+  CREATE USER IF NOT EXISTS sbtest@localhost;
+  GRANT ALL PRIVILEGES ON sbtest.* TO sbtest@localhost"
+
+sysbench oltp_read_write prepare --tables=20 --table-size=100000 | tee sysbench-prepare.log
+sync && sleep 1 # Ensure writes were propagated to disk
+
+# Save results of this run in a subdirectory so that they are not overwritten by
+# the next run
+TIMESTAMP="$(date -Iseconds)"
+mkdir "mini-benchmark-$TIMESTAMP"
+cd "mini-benchmark-$TIMESTAMP" || exit 1
+
+# Run benchmark with increasing thread counts. The MariaDB Server will be using
+# around 300 MB of RAM and mostly reading and writing in RAM, so I/O usage is
+# also low. The benchmark will most likely be CPU bound to due to the load
+# profile, and also guaranteed to be CPU bound because of being limited to a
+# single CPU with 'tasksel'.
+for t in 1 2 4 8 16
+do
+  # Prepend command with perf if defined
+  # Output stderr to stdout as perf outpus everything in stderr
+  $PERF $TASKSET_SYSBENCH sysbench oltp_read_write run --threads=$t --time=60 --report-interval=10 2>&1 | tee sysbench-run-$t.log
+done
+
+sysbench oltp_read_write cleanup --tables=20 | tee sysbench-cleanup.log
+
+# Store results from 4 thread run in a Gitlab-CI compatible metrics file
+grep -oE '[a-z]+:[ ]+[0-9.]+' sysbench-run-4.log | sed -r 's/\s+/ /g' | tail -n 15 > metrics.txt
+
+echo # Newline improves readability
+echo "== SUMMARY =="
+
+# Print performance counter summary if they were logged
+if grep --quiet cycles sysbench-run-*.log
+then
+  grep -e cycles sysbench-run-*.log | sort -k 2
+  echo "Total: $(grep -h -e cycles sysbench-run-*.log | sort -k 1 | awk '{s+=$1}END{print s}')"
+  echo # Newline improves readability
+  grep -e instructions sysbench-run-*.log | sort -k 2
+  echo "Total: $(grep -h -e instructions sysbench-run-*.log | sort -k 1 | awk '{s+=$1}END{print s}')"
+  echo # Newline improves readability
+
+  # Final verdict based on cpu cycle count
+  RESULT="$(grep -h -e cycles sysbench-run-*.log | sort -k 1 | awk '{s+=$1}END{print s}')"
+  if [ "$RESULT" -gt 850000000000 ]
+  then
+    echo # Newline improves readability
+    echo "Benchmark exceeded 8.5 billion cpu cycles, performance most likely regressed!"
+    exit 1
+  fi
+fi
+
+# List all sysbench status lines at once
+grep -h thds sysbench-run-*.log | sort -k 5 -h
+
+echo # Newline improves readability
+echo "Highest count for queries per second:"
+sort -k 9 -h sysbench-run-*.log | tail -n 1
+
+if [ "$PERF_RECORD" == true ]
+then
+  for f in perf.data.*
+  do
+    perf script -i $f | stackcollapse-perf.pl | flamegraph.pl --width 3000 > $f.svg
+  done
+  echo "Flamegraphs stored in folder mini-benchmark-$TIMESTAMP/"
+fi
+
+# Fallback if CPU cycle count not availalbe: final verdict based on peak QPS
+RESULT="$(sort -k 9 -h sysbench-run-*.log | tail -n 1 | grep -oE "qps: [0-9]+" | grep -oE "[0-9]+")"
+case $RESULT in
+  ''|*[!0-9]*)
+    echo "ERROR: Benchmark result invalid, not an integer."
+    exit 1
+    ;;
+
+  *)
+    if [ "$RESULT" -lt 13000 ]
+    then
+      echo # Newline improves readability
+      echo "Benchmark did not reach 13000+ qps, performance most likely regressed!"
+      exit 1
+    else
+      echo "Banchmark passed with $RESULT queries per second as peak value"
+    fi
+    ;;
+esac