Skip to content

Commit dccb0c9

Browse files
raghunandanbhatraghunandanbhat
committed
MDEV-38111: SIGSEGV when multiple servers use the same Vault KV storage for encrypted tables
Problem: A data race between InnoDB background threads reading the cached keys and the thread executing FLUSH command clearing it without acquiring a lock. This non-synchronized memory write caused InnoDB threads that were concurrently reading the cache to access freed memory, leading to a crash. Fix: Acquire the lock before clearing the latest version cahce. This ensures the cache clearing operation is serialized, preventing concurrent access and resolving the data race.
1 parent e03ffbc commit dccb0c9

File tree

4 files changed

+120
-0
lines changed

4 files changed

+120
-0
lines changed

‎plugin/hashicorp_key_management/hashicorp_key_management_plugin.cc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,8 +129,10 @@ class HCData
129129
}
130130
void cache_clean ()
131131
{
132+
mtx.lock();
132133
latest_version_cache.clear();
133134
key_info_cache.clear();
135+
mtx.unlock();
134136
}
135137
int fill_info_schema(THD *thd, TABLE *table);
136138
private:

‎plugin/hashicorp_key_management/mysql-test/vault/include/have_hashicorp_key_management_plugin.opt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,4 @@
44
--loose-hashicorp-key-management-token="$VAULT_TOKEN"
55
--loose-hashicorp-key-management-timeout=60
66
--loose-hashicorp-key-management-check-kv-version=off
7+
--innodb_tablespaces_encryption

‎plugin/hashicorp_key_management/mysql-test/vault/r/MDEV-38111.result‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
#
2+
# MDEV-38111: SIGSEGV when multiple servers use the same Vault KV storage for encrypted tables
3+
#
4+
# restart the server with encryption
5+
CREATE OR REPLACE TABLE t1 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=1;
6+
CREATE OR REPLACE TABLE t2 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=2;
7+
CREATE OR REPLACE TABLE t2 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=1;
8+
FLUSH HASHICORP_KEY_MANAGEMENT_CACHE;
9+
CREATE TABLE t3 (c1 INT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO;
10+
DROP TABLE t3;
11+
CREATE OR REPLACE TABLE t4 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=2;
12+
FLUSH HASHICORP_KEY_MANAGEMENT_CACHE;
13+
CREATE OR REPLACE TABLE t2 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=1;
14+
FLUSH HASHICORP_KEY_MANAGEMENT_CACHE;
15+
CREATE TABLE t3 (c1 INT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO;
16+
DROP TABLE t3;
17+
CREATE OR REPLACE TABLE t4 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=2;
18+
FLUSH HASHICORP_KEY_MANAGEMENT_CACHE;
19+
CREATE OR REPLACE TABLE t2 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=1;
20+
FLUSH HASHICORP_KEY_MANAGEMENT_CACHE;
21+
CREATE TABLE t3 (c1 INT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO;
22+
DROP TABLE t3;
23+
CREATE OR REPLACE TABLE t4 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=2;
24+
FLUSH HASHICORP_KEY_MANAGEMENT_CACHE;
25+
CREATE OR REPLACE TABLE t2 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=1;
26+
FLUSH HASHICORP_KEY_MANAGEMENT_CACHE;
27+
CREATE TABLE t3 (c1 INT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO;
28+
DROP TABLE t3;
29+
CREATE OR REPLACE TABLE t4 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=2;
30+
FLUSH HASHICORP_KEY_MANAGEMENT_CACHE;
31+
CREATE OR REPLACE TABLE t2 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=1;
32+
FLUSH HASHICORP_KEY_MANAGEMENT_CACHE;
33+
CREATE TABLE t3 (c1 INT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO;
34+
DROP TABLE t3;
35+
CREATE OR REPLACE TABLE t4 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=2;
36+
FLUSH HASHICORP_KEY_MANAGEMENT_CACHE;
37+
CREATE OR REPLACE TABLE t2 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=1;
38+
FLUSH HASHICORP_KEY_MANAGEMENT_CACHE;
39+
CREATE TABLE t3 (c1 INT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO;
40+
DROP TABLE t3;
41+
CREATE OR REPLACE TABLE t4 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=2;
42+
FLUSH HASHICORP_KEY_MANAGEMENT_CACHE;
43+
CREATE OR REPLACE TABLE t2 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=1;
44+
FLUSH HASHICORP_KEY_MANAGEMENT_CACHE;
45+
CREATE TABLE t3 (c1 INT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO;
46+
DROP TABLE t3;
47+
CREATE OR REPLACE TABLE t4 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=2;
48+
FLUSH HASHICORP_KEY_MANAGEMENT_CACHE;
49+
CREATE OR REPLACE TABLE t2 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=1;
50+
FLUSH HASHICORP_KEY_MANAGEMENT_CACHE;
51+
CREATE TABLE t3 (c1 INT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO;
52+
DROP TABLE t3;
53+
CREATE OR REPLACE TABLE t4 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=2;
54+
FLUSH HASHICORP_KEY_MANAGEMENT_CACHE;
55+
CREATE OR REPLACE TABLE t2 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=1;
56+
FLUSH HASHICORP_KEY_MANAGEMENT_CACHE;
57+
CREATE TABLE t3 (c1 INT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO;
58+
DROP TABLE t3;
59+
CREATE OR REPLACE TABLE t4 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=2;
60+
FLUSH HASHICORP_KEY_MANAGEMENT_CACHE;
61+
CREATE OR REPLACE TABLE t2 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=1;
62+
FLUSH HASHICORP_KEY_MANAGEMENT_CACHE;
63+
CREATE TABLE t3 (c1 INT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO;
64+
DROP TABLE t3;
65+
CREATE OR REPLACE TABLE t4 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=2;
66+
FLUSH HASHICORP_KEY_MANAGEMENT_CACHE;
67+
DROP TABLE t1, t2, t4;
68+
# Wait max 10 min for key encryption threads to encrypt all spaces
69+
# End of 12.3 tests

‎plugin/hashicorp_key_management/mysql-test/vault/t/MDEV-38111.test‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
--source include/have_innodb.inc
2+
--source hashicorp_plugin.inc
3+
4+
--echo #
5+
--echo # MDEV-38111: SIGSEGV when multiple servers use the same Vault KV storage for encrypted tables
6+
--echo #
7+
8+
--exec vault secrets disable flush_race_test > /dev/null
9+
--exec vault secrets enable -path /flush_race_test -version=2 kv > /dev/null
10+
--exec vault kv put /flush_race_test/1 data="59af70fb2f9f9e0acbff9af9c81dec91" > /dev/null
11+
--exec vault kv put /flush_race_test/2 data="526540c9854d26cea56a463e4ccff3ec" > /dev/null
12+
13+
--echo # restart the server with encryption
14+
--let $vault_defaults=--plugin-load-add=hashicorp_key_management --hashicorp_key_management=force --hashicorp-key-management-check-kv-version=on --hashicorp-key-management-vault-url=$VAULT_ADDR/v1/flush_race_test/ --hashicorp-key-management-token=$VAULT_TOKEN
15+
--let $innodb_defaults=--innodb-encrypt-tables=ON --innodb-encrypt-log=ON --innodb-tablespaces-encryption=ON --innodb-encryption-threads=4 --innodb-encryption-rotate-key-age=1
16+
--let $restart_parameters=$vault_defaults $innodb_defaults
17+
--let $restart_noprint=2
18+
--source include/restart_mysqld.inc
19+
20+
CREATE OR REPLACE TABLE t1 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=1;
21+
CREATE OR REPLACE TABLE t2 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=2;
22+
23+
--let $i = 10
24+
while ($i > 0)
25+
{
26+
CREATE OR REPLACE TABLE t2 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=1;
27+
FLUSH HASHICORP_KEY_MANAGEMENT_CACHE;
28+
CREATE TABLE t3 (c1 INT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO;
29+
DROP TABLE t3;
30+
CREATE OR REPLACE TABLE t4 (c1 INT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=2;
31+
FLUSH HASHICORP_KEY_MANAGEMENT_CACHE;
32+
33+
--dec $i
34+
}
35+
36+
# Cleanup
37+
DROP TABLE t1, t2, t4;
38+
39+
--let $tables_count= `select count(*) + @@global.innodb_undo_tablespaces + 1 from information_schema.tables where engine = 'InnoDB'`
40+
41+
--echo # Wait max 10 min for key encryption threads to encrypt all spaces
42+
--let $wait_timeout= 600
43+
--let $wait_condition=SELECT COUNT(*) >= $tables_count FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
44+
--source include/wait_condition.inc
45+
46+
--exec vault secrets disable flush_race_test > /dev/null
47+
48+
--echo # End of 12.3 tests

0 commit comments

Comments
 (0)