hack localtests/test.sh to run sysbench

Author: meiji163

localtests/sysbench/create.sql

@@ -0,0 +1,9 @@
+DROP TABLE IF EXISTS `sbtest1`;
+CREATE TABLE `sbtest1` (
+  `id` int NOT NULL AUTO_INCREMENT,
+  `k` int NOT NULL DEFAULT '0',
+  `c` char(120) NOT NULL DEFAULT '',
+  `pad` char(60) NOT NULL DEFAULT '',
+  PRIMARY KEY (`id`),
+  KEY `k_1` (`k`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

localtests/sysbench/generate_load

@@ -0,0 +1 @@
+#!/usr/bin/env bash

localtests/test.sh

@@ -14,6 +14,7 @@ ghost_binary=""
 docker=false
 storage_engine=innodb
 exec_command_file=/tmp/gh-ost-test.bash
+generate_load_file=/tmp/gh-ost-generate-load.bash
 ghost_structure_output_file=/tmp/gh-ost-test.ghost.structure.sql
 orig_content_output_file=/tmp/gh-ost-test.orig.content.csv
 ghost_content_output_file=/tmp/gh-ost-test.ghost.content.csv
@@ -24,33 +25,35 @@ master_port=
 replica_host=
 replica_port=
 original_sql_mode=
-docker=false
+gen_load_pid=0
 
 OPTIND=1
-while getopts "b:s:d" OPTION
-do
+while getopts "b:s:d" OPTION; do
   case $OPTION in
-    b)
-      ghost_binary="$OPTARG";;
-    s)
-      storage_engine="$OPTARG";;
-    d)
-      docker=true;;
+  b)
+    ghost_binary="$OPTARG"
+    ;;
+  s)
+    storage_engine="$OPTARG"
+    ;;
+  d)
+    docker=true
+    ;;
   esac
 done
-shift $((OPTIND-1))
+shift $((OPTIND - 1))
 
 test_pattern="${1:-.}"
 
 verify_master_and_replica() {
-  if [ "$(gh-ost-test-mysql-master -e "select 1" -ss)" != "1" ] ; then
+  if [ "$(gh-ost-test-mysql-master -e "select 1" -ss)" != "1" ]; then
     echo "Cannot verify gh-ost-test-mysql-master"
     exit 1
   fi
-  read master_host master_port <<< $(gh-ost-test-mysql-master -e "select @@hostname, @@port" -ss)
+  read master_host master_port <<<$(gh-ost-test-mysql-master -e "select @@hostname, @@port" -ss)
   [ "$master_host" == "$(hostname)" ] && master_host="127.0.0.1"
   echo "# master verified at $master_host:$master_port"
-  if ! gh-ost-test-mysql-master -e "set global event_scheduler := 1" ; then
+  if ! gh-ost-test-mysql-master -e "set global event_scheduler := 1"; then
     echo "Cannot enable event_scheduler on master"
     exit 1
   fi
@@ -60,22 +63,22 @@ verify_master_and_replica() {
   echo "Gracefully sleeping for 3 seconds while replica is setting up..."
   sleep 3
 
-  if [ "$(gh-ost-test-mysql-replica -e "select 1" -ss)" != "1" ] ; then
+  if [ "$(gh-ost-test-mysql-replica -e "select 1" -ss)" != "1" ]; then
     echo "Cannot verify gh-ost-test-mysql-replica"
     exit 1
   fi
-  if [ "$(gh-ost-test-mysql-replica -e "select @@global.binlog_format" -ss)" != "ROW" ] ; then
+  if [ "$(gh-ost-test-mysql-replica -e "select @@global.binlog_format" -ss)" != "ROW" ]; then
     echo "Expecting test replica to have binlog_format=ROW"
     exit 1
   fi
-  read replica_host replica_port <<< $(gh-ost-test-mysql-replica -e "select @@hostname, @@port" -ss)
+  read replica_host replica_port <<<$(gh-ost-test-mysql-replica -e "select @@hostname, @@port" -ss)
   [ "$replica_host" == "$(hostname)" ] && replica_host="127.0.0.1"
   echo "# replica verified at $replica_host:$replica_port"
 }
 
 exec_cmd() {
   echo "$@"
-  command "$@" 1> $test_logfile 2>&1
+  command "$@" 1>$test_logfile 2>&1
   return $?
 }
 
@@ -84,7 +87,7 @@ echo_dot() {
 }
 
 start_replication() {
-  mysql_version="$(gh-ost-test-mysql-replica -e  "select @@version")"
+  mysql_version="$(gh-ost-test-mysql-replica -e "select @@version")"
   if [[ $mysql_version =~ "8.4" ]]; then
     seconds_behind_source="Seconds_Behind_Source"
     replica_terminology="replica"
@@ -95,9 +98,9 @@ start_replication() {
   gh-ost-test-mysql-replica -e "stop $replica_terminology; start $replica_terminology;"
 
   num_attempts=0
-  while gh-ost-test-mysql-replica -e "show $replica_terminology status\G" | grep $seconds_behind_source | grep -q NULL ; do
-    ((num_attempts=num_attempts+1))
-    if [ $num_attempts -gt 10 ] ; then
+  while gh-ost-test-mysql-replica -e "show $replica_terminology status\G" | grep $seconds_behind_source | grep -q NULL; do
+    ((num_attempts = num_attempts + 1))
+    if [ $num_attempts -gt 10 ]; then
       echo
       echo "ERROR replication failure"
       exit 1
@@ -107,6 +110,31 @@ start_replication() {
   done
 }
 
+generate_load_cmd() {
+  local mysql_host="$1"
+  local mysql_port="$2"
+  cmd="sysbench oltp_write_only \
+    --mysql-host="$mysql_host" \
+    --mysql-port="$mysql_port" \
+    --mysql-user=root \
+    --mysql-password=opensesame \
+    --mysql-db=test \
+    --table-size=100000 \
+    --tables=1 \
+    --threads=8 \
+    --time=60 \
+    --report-interval=10 \
+    --rate=200 \
+    run"
+  echo $cmd
+}
+
+cleanup() {
+  if [ $gen_load_pid -gt 0 ]; then
+    kill $gen_load_pid
+  fi
+}
+
 test_single() {
   local test_name
   test_name="$1"
@@ -118,14 +146,14 @@ test_single() {
     replica_port="3308"
   fi
 
-  if [ -f $tests_path/$test_name/ignore_versions ] ; then
+  if [ -f $tests_path/$test_name/ignore_versions ]; then
     ignore_versions=$(cat $tests_path/$test_name/ignore_versions)
     mysql_version=$(gh-ost-test-mysql-master -s -s -e "select @@version")
     mysql_version_comment=$(gh-ost-test-mysql-master -s -s -e "select @@version_comment")
-    if echo "$mysql_version" | egrep -q "^${ignore_versions}" ; then
+    if echo "$mysql_version" | egrep -q "^${ignore_versions}"; then
       echo -n "Skipping: $test_name"
       return 0
-    elif echo "$mysql_version_comment" | egrep -i -q "^${ignore_versions}" ; then
+    elif echo "$mysql_version_comment" | egrep -i -q "^${ignore_versions}"; then
       echo -n "Skipping: $test_name"
       return 0
     fi
@@ -137,56 +165,73 @@ test_single() {
   start_replication
   echo_dot
 
-  if [ -f $tests_path/$test_name/sql_mode ] ; then
+  if [ -f $tests_path/$test_name/sql_mode ]; then
     gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'"
     gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'"
   fi
 
-  gh-ost-test-mysql-master --default-character-set=utf8mb4 test < $tests_path/$test_name/create.sql
+  gh-ost-test-mysql-master --default-character-set=utf8mb4 test <$tests_path/$test_name/create.sql
   test_create_result=$?
 
-  if [ $test_create_result -ne 0 ] ; then
+  if [ $test_create_result -ne 0 ]; then
     echo
     echo "ERROR $test_name create failure. cat $tests_path/$test_name/create.sql:"
     cat $tests_path/$test_name/create.sql
     return 1
   fi
 
   extra_args=""
-  if [ -f $tests_path/$test_name/extra_args ] ; then
+  if [ -f $tests_path/$test_name/extra_args ]; then
     extra_args=$(cat $tests_path/$test_name/extra_args)
   fi
   orig_columns="*"
   ghost_columns="*"
   order_by=""
-  if [ -f $tests_path/$test_name/orig_columns ] ; then
+  if [ -f $tests_path/$test_name/orig_columns ]; then
     orig_columns=$(cat $tests_path/$test_name/orig_columns)
   fi
-  if [ -f $tests_path/$test_name/ghost_columns ] ; then
+  if [ -f $tests_path/$test_name/ghost_columns ]; then
     ghost_columns=$(cat $tests_path/$test_name/ghost_columns)
   fi
-  if [ -f $tests_path/$test_name/order_by ] ; then
+  if [ -f $tests_path/$test_name/order_by ]; then
     order_by="order by $(cat $tests_path/$test_name/order_by)"
   fi
   # graceful sleep for replica to catch up
   echo_dot
   sleep 1
-  #
+
+  table_name="gh_ost_test"
+
+  # run gh-ost with sysbench write load
+  trap cleanup EXIT INT TERM
+  if [[ "$test_name" == "sysbench" ]]; then
+    table_name="sbtest1"
+    load_cmd="$(generate_load_cmd $master_host $master_port)"
+    echo $load_cmd >$generate_load_file
+    bash $generate_load_file 1>$test_logfile 2>&1 &
+    gen_load_pid=$!
+    echo
+    echo "Started sysbench (PID $gen_load_pid)"
+    echo $load_cmd
+  fi
+
+  ghost_table_name="_${table_name}_gho"
+
   cmd="$ghost_binary \
     --user=gh-ost \
     --password=gh-ost \
     --host=$replica_host \
     --port=$replica_port \
     --assume-master-host=${master_host}:${master_port}
     --database=test \
-    --table=gh_ost_test \
+    --table=${table_name} \
     --storage-engine=${storage_engine} \
     --alter='engine=${storage_engine}' \
     --exact-rowcount \
     --assume-rbr \
     --initially-drop-old-table \
     --initially-drop-ghost-table \
-    --throttle-query='select timestampdiff(second, min(last_update), now()) < 5 from _gh_ost_test_ghc' \
+    --throttle-query='select timestampdiff(second, min(last_update), now()) < 5 from _${table_name}_ghc' \
     --throttle-flag-file=$throttle_flag_file \
     --serve-socket-file=/tmp/gh-ost.test.sock \
     --initially-drop-socket-file \
@@ -198,32 +243,33 @@ test_single() {
     --stack \
     --execute ${extra_args[@]}"
   echo_dot
-  echo $cmd > $exec_command_file
+  echo $cmd >$exec_command_file
   echo_dot
-  bash $exec_command_file 1> $test_logfile 2>&1
+
+  bash $exec_command_file 1>$test_logfile 2>&1
 
   execution_result=$?
 
-  if [ -f $tests_path/$test_name/sql_mode ] ; then
+  if [ -f $tests_path/$test_name/sql_mode ]; then
     gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.sql_mode='${original_sql_mode}'"
     gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.sql_mode='${original_sql_mode}'"
   fi
 
-  if [ -f $tests_path/$test_name/destroy.sql ] ; then
-    gh-ost-test-mysql-master --default-character-set=utf8mb4 test < $tests_path/$test_name/destroy.sql
+  if [ -f $tests_path/$test_name/destroy.sql ]; then
+    gh-ost-test-mysql-master --default-character-set=utf8mb4 test <$tests_path/$test_name/destroy.sql
   fi
 
-  if [ -f $tests_path/$test_name/expect_failure ] ; then
-    if [ $execution_result -eq 0 ] ; then
+  if [ -f $tests_path/$test_name/expect_failure ]; then
+    if [ $execution_result -eq 0 ]; then
       echo
       echo "ERROR $test_name execution was expected to exit on error but did not. cat $test_logfile"
       return 1
     fi
-    if [ -s $tests_path/$test_name/expect_failure ] ; then
+    if [ -s $tests_path/$test_name/expect_failure ]; then
       # 'expect_failure' file has content. We expect to find this content in the log.
       expected_error_message="$(cat $tests_path/$test_name/expect_failure)"
-      if grep -q "$expected_error_message" $test_logfile ; then
-          return 0
+      if grep -q "$expected_error_message" $test_logfile; then
+        return 0
       fi
       echo
       echo "ERROR $test_name execution was expected to exit with error message '${expected_error_message}' but did not. cat $test_logfile"
@@ -233,18 +279,18 @@ test_single() {
     return 0
   fi
 
-  if [ $execution_result -ne 0 ] ; then
+  if [ $execution_result -ne 0 ]; then
     echo
     echo "ERROR $test_name execution failure. cat $test_logfile:"
     cat $test_logfile
     return 1
   fi
 
-  gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "show create table _gh_ost_test_gho\G" -ss > $ghost_structure_output_file
+  gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "show create table ${ghost_table_name}\G" -ss >$ghost_structure_output_file
 
-  if [ -f $tests_path/$test_name/expect_table_structure ] ; then
+  if [ -f $tests_path/$test_name/expect_table_structure ]; then
     expected_table_structure="$(cat $tests_path/$test_name/expect_table_structure)"
-    if ! grep -q "$expected_table_structure" $ghost_structure_output_file ; then
+    if ! grep -q "$expected_table_structure" $ghost_structure_output_file; then
       echo
       echo "ERROR $test_name: table structure was expected to include ${expected_table_structure} but did not. cat $ghost_structure_output_file:"
       cat $ghost_structure_output_file
@@ -253,14 +299,14 @@ test_single() {
   fi
 
   echo_dot
-  gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${orig_columns} from gh_ost_test ${order_by}" -ss > $orig_content_output_file
-  gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${ghost_columns} from _gh_ost_test_gho ${order_by}" -ss > $ghost_content_output_file
+  gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${orig_columns} from ${table_name} ${order_by}" -ss >$orig_content_output_file
+  gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${ghost_columns} from ${ghost_table_name} ${order_by}" -ss >$ghost_content_output_file
   orig_checksum=$(cat $orig_content_output_file | md5sum)
   ghost_checksum=$(cat $ghost_content_output_file | md5sum)
 
-  if [ "$orig_checksum" != "$ghost_checksum" ] ; then
-    gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${orig_columns} from gh_ost_test" -ss > $orig_content_output_file
-    gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${ghost_columns} from _gh_ost_test_gho" -ss > $ghost_content_output_file
+  if [ "$orig_checksum" != "$ghost_checksum" ]; then
+    gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${orig_columns} from ${table_name}" -ss >$orig_content_output_file
+    gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${ghost_columns} from ${ghost_table_name}" -ss >$ghost_content_output_file
     echo "ERROR $test_name: checksum mismatch"
     echo "---"
     diff $orig_content_output_file $ghost_content_output_file
@@ -275,14 +321,14 @@ build_binary() {
   echo "Building"
   rm -f $default_ghost_binary
   [ "$ghost_binary" == "" ] && ghost_binary="$default_ghost_binary"
-  if [ -f "$ghost_binary" ] ; then
+  if [ -f "$ghost_binary" ]; then
     echo "Using binary: $ghost_binary"
     return 0
   fi
 
   go build -o $ghost_binary go/cmd/gh-ost/main.go
 
-  if [ $? -ne 0 ] ; then
+  if [ $? -ne 0 ]; then
     echo "Build failure"
     exit 1
   fi
@@ -293,22 +339,22 @@ test_all() {
   test_dirs=$(find "$tests_path" -mindepth 1 -maxdepth 1 ! -path . -type d | grep "$test_pattern" | sort)
   while read -r test_dir; do
     test_name=$(basename "$test_dir")
-    if ! test_single "$test_name" ; then
+    if ! test_single "$test_name"; then
       create_statement=$(gh-ost-test-mysql-replica test -t -e "show create table _gh_ost_test_gho \G")
-      echo "$create_statement" >> $test_logfile
+      echo "$create_statement" >>$test_logfile
       echo "+ FAIL"
       return 1
     else
       echo
       echo "+ pass"
     fi
-    mysql_version="$(gh-ost-test-mysql-replica -e  "select @@version")"
+    mysql_version="$(gh-ost-test-mysql-replica -e "select @@version")"
     replica_terminology="slave"
     if [[ $mysql_version =~ "8.4" ]]; then
       replica_terminology="replica"
     fi
     gh-ost-test-mysql-replica -e "start $replica_terminology"
-  done <<< "$test_dirs"
+  done <<<"$test_dirs"
 }
 
 verify_master_and_replica