diff --git a/src/cpp/src/kvs.cpp b/src/cpp/src/kvs.cpp index 1fc39005..f8d82db5 100644 --- a/src/cpp/src/kvs.cpp +++ b/src/cpp/src/kvs.cpp @@ -17,6 +17,7 @@ #include #include #include +#include // TODO Default Value Handling TBD // TODO Add Score Logging @@ -47,7 +48,8 @@ Kvs::Kvs(Kvs&& other) noexcept object would also be okay*/ , writer(std::move(other.writer)), - logger(std::move(other.logger)) + logger(std::move(other.logger)), + max_storage_bytes(other.max_storage_bytes) { { std::lock_guard lock(other.kvs_mutex); @@ -81,6 +83,7 @@ Kvs& Kvs::operator=(Kvs&& other) noexcept parser = std::move(other.parser); writer = std::move(other.writer); logger = std::move(other.logger); + max_storage_bytes = other.max_storage_bytes; } return *this; } @@ -218,7 +221,8 @@ score::Result> Kvs::open_json(const score:: score::Result Kvs::open(const InstanceId& instance_id, OpenNeedDefaults need_defaults, OpenNeedKvs need_kvs, - const std::string&& dir) + const std::string&& dir, + std::optional max_storage_bytes) { score::Result result = score::MakeUnexpected(ErrorCode::UnmappedError); /* Redundant initialization needed, since Resul would call @@ -230,6 +234,7 @@ score::Result Kvs::open(const InstanceId& instance_id, const score::filesystem::Path filename_kvs = filename_prefix.Native() + "_0"; Kvs kvs; /* Create KVS instance */ + kvs.max_storage_bytes = max_storage_bytes; /* Store maximum storage limit */ auto default_res = kvs.open_json( filename_default, need_defaults == OpenNeedDefaults::Required ? OpenJsonNeedFile::Required : OpenJsonNeedFile::Optional); @@ -328,6 +333,7 @@ score::Result Kvs::key_exists(const std::string_view key) return result; } + /* Retrieve the value associated with a key*/ score::Result Kvs::get_value(const std::string_view key) { @@ -472,6 +478,54 @@ score::ResultBlank Kvs::remove_key(const std::string_view key) return result; } +score::Result Kvs::get_file_size(const score::filesystem::Path& file_path) { + std::error_code ec; + const auto size = std::filesystem::file_size(file_path.CStr(), ec); + + if (ec) { + // Check if the error is "file not found" + if (ec == std::errc::no_such_file_or_directory) { + // File does not exist, its size is 0. This is not an error. + return 0; + } + logger->LogError() << "Error: Could not get size of file " << file_path << ": " << ec.message(); + return score::MakeUnexpected(ErrorCode::PhysicalStorageFailure); + } + + return size; +} + +/* Helper Function to get current storage size of all persisted files (defaults and historical snapshots) */ +score::Result Kvs::get_current_storage_size() { + size_t total_size = 0; + const std::array file_extensions = {".json", ".hash"}; + + // Add the size of the default files + const std::string default_suffix = "_default"; + for (const char* extension : file_extensions) { + const score::filesystem::Path file_path = filename_prefix.Native() + default_suffix + extension; + auto size_result = get_file_size(file_path); + if (!size_result) { + return size_result; // Propagate error directly + } + total_size += size_result.value(); + } + + // Add the size of current and historical snapshots that will be kept after rotation (0 to N-1). + for (size_t snapshot_index = 0; snapshot_index < KVS_MAX_SNAPSHOTS; ++snapshot_index) { + const std::string snapshot_suffix = "_" + to_string(snapshot_index); + + for (const char* extension : file_extensions) { + const score::filesystem::Path file_path = filename_prefix.Native() + snapshot_suffix + extension; + auto size_result = get_file_size(file_path); + if (!size_result) { + return size_result; // Propagate error directly + } + total_size += size_result.value(); + } + } + return total_size; +} /* Helper: write data to a file and ensure it reaches physical storage.*/ score::ResultBlank Kvs::write_and_sync(const std::string& path, const void* data, std::size_t size) { @@ -550,13 +604,10 @@ score::ResultBlank Kvs::write_json_data(const std::string& buf) return result; } -/* Flush the key-value store*/ -score::ResultBlank Kvs::flush() -{ - score::ResultBlank result = score::MakeUnexpected(ErrorCode::UnmappedError); - /* Create JSON Object */ +score::Result Kvs::serialize_and_check() { score::json::Object root_obj; - bool error = false; + + // 1. Serialize the current KVS map to a buffer { std::unique_lock lock(kvs_mutex, std::try_to_lock); if (lock.owns_lock()) @@ -564,52 +615,71 @@ score::ResultBlank Kvs::flush() for (const auto& [key, value] : kvs) { auto conv = kvsvalue_to_any(value); - if (!conv) - { - result = score::MakeUnexpected(static_cast(*conv.error())); - error = true; - break; - } - else - { - root_obj.emplace(key, std::move(conv.value()) /*emplace in map uses move operator*/ - ); + if (!conv) { + return score::MakeUnexpected(static_cast(*conv.error())); } + root_obj.emplace(key, std::move(conv.value())); } - } - else - { - result = score::MakeUnexpected(ErrorCode::MutexLockFailed); - error = true; + } else { + return score::MakeUnexpected(ErrorCode::MutexLockFailed); } } - if (!error) - { - /* Serialize Buffer */ - auto buf_res = writer->ToBuffer(root_obj); - if (!buf_res) - { - result = score::MakeUnexpected(ErrorCode::JsonGeneratorError); - } - else - { - /* Rotate Snapshots */ - auto rotate_result = snapshot_rotate(); - if (!rotate_result) - { - result = rotate_result; - } - else - { - /* Write JSON Data */ - std::string buf = std::move(buf_res.value()); - result = write_json_data(buf); - } - } + auto buf_res = writer->ToBuffer(root_obj); + if (!buf_res) { + return score::MakeUnexpected(ErrorCode::JsonGeneratorError); } + const std::string& buf = buf_res.value(); - return result; + // 2. Get the size of all other persisted files + auto current_size_res = get_current_storage_size(); + if (!current_size_res) { + return score::MakeUnexpected(static_cast(*current_size_res.error())); + } + + // 3. Calculate the potential total size + const size_t total_size = current_size_res.value() + buf.size() + HASH_FILE_SIZE; + + // 4. Check against the limit + if (this->max_storage_bytes.has_value() && total_size > this->max_storage_bytes.value()) { + logger->LogError() << "error: KVS storage limit would be exceeded. total_size:" << total_size + << " max_storage_bytes:" << this->max_storage_bytes.value(); + return score::MakeUnexpected(ErrorCode::OutOfStorageSpace); + } + + return buf; +} + +/* Flush the key-value store*/ +score::ResultBlank Kvs::flush() { + auto result = serialize_and_check(); + if (!result) { + return score::MakeUnexpected(static_cast(*result.error())); + } + + auto rotate_result = snapshot_rotate(); + if (!rotate_result) { + return rotate_result; + } + + return write_json_data(result.value()); +} + +/* Performs a 'dry run' to check the potential storage size */ +score::Result Kvs::calculate_potential_size() { + auto result = serialize_and_check(); + if (!result) { + return score::MakeUnexpected(static_cast(*result.error())); + } + + // Re-calculate size to return it, as serialize_and_check only returns the buffer + const std::string& buf = result.value(); + auto current_size_res = get_current_storage_size(); + if (!current_size_res) { + return score::MakeUnexpected(static_cast(*current_size_res.error())); + } + + return current_size_res.value() + buf.size() + HASH_FILE_SIZE; } /* Retrieve the snapshot count*/ diff --git a/src/cpp/src/kvs.hpp b/src/cpp/src/kvs.hpp index cdccf1a0..9b9bc1c8 100644 --- a/src/cpp/src/kvs.hpp +++ b/src/cpp/src/kvs.hpp @@ -27,12 +27,14 @@ #include #include #include +#include #define KVS_MAX_SNAPSHOTS 3 +static constexpr size_t HASH_FILE_SIZE = 4; namespace score::mw::per::kvs -{ +{ struct InstanceId { size_t id; @@ -172,7 +174,8 @@ class Kvs final static score::Result open(const InstanceId& instance_id, OpenNeedDefaults need_defaults, OpenNeedKvs need_kvs, - const std::string&& dir); + const std::string&& dir, + std::optional max_storage_bytes); /** * @brief Resets a key-value-storage to its initial state @@ -352,30 +355,47 @@ class Kvs final */ score::Result get_hash_filename(const SnapshotId& snapshot_id) const; - private: - /* Private constructor to prevent direct instantiation */ - Kvs(); + /** + * @brief Performs a 'dry run' to check if the current in-memory store would + * exceed the storage limit upon flushing. + * + * This function serializes the current key-value data to a temporary buffer + * and calculates the potential total storage size. It checks this size against + * the configured `max_storage_bytes` limit. + * + * @return A score::Result object containing either: + * - On success: The estimated total size (size_t) that the KVS would occupy after a flush. + * - On failure: An `OutOfStorageSpace` error if the limit would be exceeded, + * or another ErrorCode for other failures (e.g., serialization). + */ + score::Result calculate_potential_size(); + + private: + /* Private constructor to prevent direct instantiation */ + Kvs(); - /* Internal storage and configuration details.*/ - std::mutex kvs_mutex; - std::unordered_map kvs; + /* Internal storage and configuration details.*/ + std::mutex kvs_mutex; + std::unordered_map kvs; - /* Optional default values */ - std::unordered_map default_values; + /* Optional default values */ + std::unordered_map default_values; - /* Filename prefix */ - score::filesystem::Path filename_prefix; + /* Filename prefix */ + score::filesystem::Path filename_prefix; - /* Filesystem handling */ - std::unique_ptr filesystem; + /* Filesystem handling */ + std::unique_ptr filesystem; - /* Json handling */ - std::unique_ptr parser; - std::unique_ptr writer; + /* Json handling */ + std::unique_ptr parser; + std::unique_ptr writer; - /* Logging */ - std::unique_ptr logger; + /* Logging */ + std::unique_ptr logger; + /* Maximum storage limit in bytes */ + std::optional max_storage_bytes; /* Private Methods */ score::ResultBlank snapshot_rotate(); score::Result> parse_json_data(const std::string& data); @@ -383,6 +403,10 @@ class Kvs final OpenJsonNeedFile need_file); score::ResultBlank write_json_data(const std::string& buf); score::ResultBlank write_and_sync(const std::string& path, const void* data, std::size_t size); + + score::Result serialize_and_check(); + score::Result get_file_size(const score::filesystem::Path& file_path); + score::Result get_current_storage_size(); }; } /* namespace score::mw::per::kvs */ diff --git a/src/cpp/src/kvsbuilder.cpp b/src/cpp/src/kvsbuilder.cpp index a248ebfe..c1c4b138 100644 --- a/src/cpp/src/kvsbuilder.cpp +++ b/src/cpp/src/kvsbuilder.cpp @@ -20,7 +20,8 @@ KvsBuilder::KvsBuilder(const InstanceId& instance_id) : instance_id(instance_id), need_defaults(false), need_kvs(false), - directory("./data_folder/") /* Default Directory */ + directory("./data_folder/"), /* Default Directory */ + maximum_size(KVS_DEFAULT_MAX_SIZE_BYTES) /* Default max size in bytes */ { } @@ -42,6 +43,12 @@ KvsBuilder& KvsBuilder::dir(std::string&& dir_path) return *this; } +KvsBuilder& KvsBuilder::max_size(std::optional max_bytes) +{ + this->maximum_size = max_bytes; + return *this; +} + score::Result KvsBuilder::build() { score::Result result = score::MakeUnexpected(ErrorCode::UnmappedError); @@ -55,7 +62,8 @@ score::Result KvsBuilder::build() result = Kvs::open(instance_id, need_defaults ? OpenNeedDefaults::Required : OpenNeedDefaults::Optional, need_kvs ? OpenNeedKvs::Required : OpenNeedKvs::Optional, - std::move(directory)); + std::move(directory), + maximum_size); return result; } diff --git a/src/cpp/src/kvsbuilder.hpp b/src/cpp/src/kvsbuilder.hpp index 2f6e1f6e..35d69cb5 100644 --- a/src/cpp/src/kvsbuilder.hpp +++ b/src/cpp/src/kvsbuilder.hpp @@ -15,10 +15,13 @@ #include "kvs.hpp" #include +#include namespace score::mw::per::kvs { +constexpr size_t KVS_DEFAULT_MAX_SIZE_BYTES = 1000; + /** * @class KvsBuilder * @brief Builder for opening a KVS object. @@ -89,6 +92,13 @@ class KvsBuilder final */ KvsBuilder& dir(std::string&& dir_path); + /** + * @brief Configure the maximum storage size for the KVS in bytes. + * @param max_bytes The maximum allowed total storage size in bytes. + * @return Reference to this builder (for chaining). + */ + KvsBuilder& max_size(std::optional max_bytes); + /** * @brief Builds and opens the Kvs instance with the configured options. * @@ -103,8 +113,10 @@ class KvsBuilder final bool need_defaults; ///< Whether default values are required bool need_kvs; ///< Whether an existing KVS is required std::string directory; ///< Directory where to store the KVS Files + std::optional maximum_size; ///< Maximum total storage size for the KVS in bytes }; } /* namespace score::mw::per::kvs */ #endif /* SCORE_LIB_KVS_KVSBUILDER_HPP */ + diff --git a/src/cpp/tests/test_kvs.cpp b/src/cpp/tests/test_kvs.cpp index c3d5d080..edcca575 100644 --- a/src/cpp/tests/test_kvs.cpp +++ b/src/cpp/tests/test_kvs.cpp @@ -18,12 +18,12 @@ TEST(kvs_constructor, move_constructor) /* create object A */ auto result_a = - Kvs::open(InstanceId(instance_b), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + Kvs::open(InstanceId(instance_b), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result_a); Kvs kvs_a = std::move(result_a.value()); /* create object B */ - auto result_b = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result_b = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result_b); Kvs kvs_b = std::move(result_b.value()); @@ -72,7 +72,7 @@ TEST(kvs_TEST, parse_json_data_sucess) prepare_environment(); auto kvs = - Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(kvs); auto mock_parser = std::make_unique(); @@ -101,7 +101,7 @@ TEST(kvs_TEST, parse_json_data_failure) /* Json Parser Failure */ auto kvs = - Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(kvs); auto mock_parser = std::make_unique(); @@ -156,7 +156,7 @@ TEST(kvs_open_json, open_json_success) prepare_environment(); auto kvs = - Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(kvs); auto result = kvs->open_json(score::filesystem::Path(kvs_prefix), OpenJsonNeedFile::Required); @@ -240,7 +240,7 @@ TEST(kvs_open_json, open_json_hash_invalid) TEST(kvs_reset, reset_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Check Data existing */ @@ -259,7 +259,7 @@ TEST(kvs_reset, reset_failure) prepare_environment(); /* Mutex locked */ - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); std::unique_lock lock(result.value().kvs_mutex); @@ -274,7 +274,7 @@ TEST(kvs_get_all_keys, get_all_keys_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Check Data existing */ @@ -301,7 +301,7 @@ TEST(kvs_get_all_keys, get_all_keys_failure) prepare_environment(); /* Mutex locked */ - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); std::unique_lock lock(result.value().kvs_mutex); @@ -316,7 +316,7 @@ TEST(kvs_key_exists, key_exists_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Check Data existing */ @@ -339,7 +339,7 @@ TEST(kvs_key_exists, key_exists_failure) prepare_environment(); /* Mutex locked */ - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); std::unique_lock lock(result.value().kvs_mutex); @@ -354,7 +354,7 @@ TEST(kvs_get_value, get_value_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Check Data existing */ @@ -383,7 +383,7 @@ TEST(kvs_get_value, get_value_failure) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Check if non-existing key returns error */ @@ -392,7 +392,7 @@ TEST(kvs_get_value, get_value_failure) EXPECT_EQ(get_value_result.error(), ErrorCode::KeyNotFound); /* Mutex locked */ - result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); std::unique_lock lock(result.value().kvs_mutex); get_value_result = result.value().get_value("kvs"); @@ -406,7 +406,7 @@ TEST(kvs_get_default_value, get_default_value_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Check Data existing */ @@ -426,7 +426,7 @@ TEST(kvs_get_default_value, get_default_value_failure) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Check if non-existing key returns error */ @@ -441,7 +441,7 @@ TEST(kvs_reset_key, reset_key_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); ASSERT_TRUE(result.value().kvs.count("kvs")); /* Check Data existing */ @@ -466,7 +466,7 @@ TEST(kvs_reset_key, reset_key_failure) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Reset a non-existing key */ @@ -475,7 +475,7 @@ TEST(kvs_reset_key, reset_key_failure) EXPECT_EQ(reset_key_result.error(), ErrorCode::KeyDefaultNotFound); /* Reset a key without default value */ - result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); result.value().default_values.clear(); // Clear default values to ensure no default value // exists for "kvs" @@ -484,7 +484,7 @@ TEST(kvs_reset_key, reset_key_failure) EXPECT_EQ(reset_key_result.error(), ErrorCode::KeyDefaultNotFound); /* Mutex locked */ - result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); std::unique_lock lock(result.value().kvs_mutex); reset_key_result = result.value().reset_key("kvs"); @@ -498,7 +498,7 @@ TEST(kvs_is_value_default, is_value_default) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); auto kvs{std::move(result.value())}; @@ -533,7 +533,7 @@ TEST(kvs_is_value_default, is_value_default) TEST(kvs_set_value, set_value_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Set a new value */ @@ -557,7 +557,7 @@ TEST(kvs_set_value, set_value_failure) prepare_environment(); /* Mutex locked */ - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); std::unique_lock lock(result.value().kvs_mutex); @@ -571,7 +571,7 @@ TEST(kvs_set_value, set_value_failure) TEST(kvs_remove_key, remove_key_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Check Data existing */ @@ -589,7 +589,7 @@ TEST(kvs_remove_key, remove_key_failure) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Remove a non-existing key */ @@ -598,7 +598,7 @@ TEST(kvs_remove_key, remove_key_failure) EXPECT_EQ(remove_key_result.error(), ErrorCode::KeyNotFound); /* Mutex locked */ - result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); std::unique_lock lock(result.value().kvs_mutex); remove_key_result = result.value().remove_key("kvs"); @@ -622,7 +622,7 @@ TEST(kvs_write_json_data, write_json_data_success) system(("rm -rf " + kvs_prefix + ".hash").c_str()); auto kvs = - Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(kvs); kvs->filename_prefix = score::filesystem::Path(filename_prefix); /* Set the filename prefix to the test prefix */ @@ -658,7 +658,7 @@ TEST(kvs_write_json_data, write_json_data_filesystem_failure) system(("rm -rf " + kvs_prefix + ".hash").c_str()); auto kvs = - Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(kvs); /* Mock Filesystem */ @@ -691,7 +691,7 @@ TEST(kvs_write_json_data, write_json_data_permissions_failure) system(("rm -rf " + kvs_prefix + ".hash").c_str()); auto kvs = - Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(kvs); /* Test writing to a non-writable hash file */ @@ -723,7 +723,7 @@ TEST(kvs_snapshot_rotate, snapshot_rotate_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Create empty Test-Snapshot Files */ @@ -753,7 +753,7 @@ TEST(kvs_snapshot_rotate, snapshot_rotate_max_snapshots) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Create empty Test-Snapshot Files */ @@ -779,7 +779,7 @@ TEST(kvs_snapshot_rotate, snapshot_rotate_failure_renaming_json) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Create empty Test-Snapshot Files */ @@ -804,7 +804,7 @@ TEST(kvs_snapshot_rotate, snapshot_rotate_failure_renaming_hash) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Create empty Test-Snapshot Files */ @@ -828,7 +828,7 @@ TEST(kvs_snapshot_rotate, snapshot_rotate_failure_mutex) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Mutex locked */ @@ -847,7 +847,7 @@ TEST(kvs_flush, flush_success_data) system(("rm -rf " + kvs_prefix + ".json").c_str()); system(("rm -rf " + kvs_prefix + ".hash").c_str()); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); result.value().kvs.clear(); /* Clear KVS to ensure no data is written */ @@ -873,7 +873,7 @@ TEST(kvs_flush, flush_success_snapshot_rotate) system(("rm -rf " + kvs_prefix + ".json").c_str()); system(("rm -rf " + kvs_prefix + ".hash").c_str()); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); EXPECT_FALSE(std::filesystem::exists(filename_prefix + "_1.json")); EXPECT_FALSE(std::filesystem::exists(filename_prefix + "_1.hash")); @@ -894,7 +894,7 @@ TEST(kvs_flush, flush_failure_mutex) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); std::unique_lock lock(result.value().kvs_mutex); @@ -912,7 +912,7 @@ TEST(kvs_flush, flush_failure_rotate_snapshots) std::string permissions_dir = data_dir + "permissions/"; std::filesystem::create_directories(permissions_dir); auto result = - Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(permissions_dir)); + Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(permissions_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); std::filesystem::permissions( @@ -929,7 +929,7 @@ TEST(kvs_flush, flush_failure_kvsvalue_invalid) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); BrokenKvsValue invalid; @@ -946,7 +946,7 @@ TEST(kvs_flush, flush_failure_json_writer) { prepare_environment(); - auto kvs = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto kvs = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(kvs); auto mock_writer = std::make_unique(); /* Force error in writer.ToBuffer */ @@ -966,7 +966,7 @@ TEST(kvs_snapshot_count, snapshot_count_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Create empty Test-Snapshot Files */ @@ -990,7 +990,7 @@ TEST(kvs_snapshot_count, snapshot_count_invalid) { prepare_environment(); - auto kvs = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto kvs = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(kvs); /* Mock Filesystem */ @@ -1011,7 +1011,7 @@ TEST(kvs_snapshot_restore, snapshot_restore_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Create empty Test-Snapshot Files -> Data received by the JsonParser should be the data listed @@ -1051,7 +1051,7 @@ TEST(kvs_snapshot_restore, snapshot_restore_failure_invalid_snapshot_id) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Restore Snapshot ID 0 -> Current KVS*/ @@ -1071,7 +1071,7 @@ TEST(kvs_snapshot_restore, snapshot_restore_failure_open_json) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Create empty Test-Snapshot Files */ @@ -1089,7 +1089,7 @@ TEST(kvs_snapshot_restore, snapshot_restore_failure_mutex) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); std::unique_lock lock(result.value().kvs_mutex); @@ -1104,7 +1104,7 @@ TEST(kvs_snapshot_restore, snapshot_restore_failure_snapshot_count) { prepare_environment(); - auto kvs = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto kvs = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(kvs); /* Mock Filesystem */ @@ -1126,7 +1126,7 @@ TEST(kvs_snapshot_max_count, snapshot_max_count) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); EXPECT_EQ(result.value().snapshot_max_count(), KVS_MAX_SNAPSHOTS); @@ -1137,7 +1137,7 @@ TEST(kvs_get_filename, get_kvs_filename_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Generate Testfiles */ @@ -1160,7 +1160,7 @@ TEST(kvs_get_filename, get_kvs_filename_failure) { prepare_environment(); - auto kvs = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto kvs = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(kvs); /* Testfiles not available */ @@ -1188,7 +1188,7 @@ TEST(kvs_get_filename, get_hashname_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Generate Testfiles */ @@ -1212,7 +1212,7 @@ TEST(kvs_get_filename, get_hashname_failure) { prepare_environment(); - auto kvs = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto kvs = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(kvs); /* Testfiles not available */ @@ -1235,3 +1235,91 @@ TEST(kvs_get_filename, get_hashname_failure) cleanup_environment(); } + + +TEST(kvs, flush_fails_when_storage_limit_exceeded) { + // Use a unique directory for this test to avoid conflicts + const std::string test_dir = "./kvs_storage_test/"; + // Ensure the directory is clean before starting + std::filesystem::remove_all(test_dir); + + // 1. Setup KVS + KvsBuilder builder(instance_id); + builder.dir("./kvs_storage_test/"); + auto open_res = builder.build(); + ASSERT_TRUE(open_res); + Kvs kvs = std::move(open_res.value()); + + // 2. Add data that is close to the limit. + // There is overhead for the JSON structure (key, type info, braces, etc.) and the hash file (4 bytes). + // We will make the data payload a bit smaller than the max to account for this. + size_t overhead_estimate = 100; + size_t data_size = KVS_DEFAULT_MAX_SIZE_BYTES - overhead_estimate; + std::string large_data(data_size, 'a'); + + auto set_res1 = kvs.set_value("large_data", KvsValue(large_data.c_str())); + ASSERT_TRUE(set_res1); + + // 3. Flush the first batch of data and assert it succeeds. + auto flush_res1 = kvs.flush(); + ASSERT_TRUE(flush_res1); + + // 4. Add a little more data, which should push the total size over the limit. + auto set_res2 = kvs.set_value("extra_data", KvsValue("this should not fit")); + ASSERT_TRUE(set_res2); + + // 5. The second flush should fail because the storage limit is exceeded. + auto flush_res2 = kvs.flush(); + ASSERT_FALSE(flush_res2); + ASSERT_EQ(static_cast(*flush_res2.error()), ErrorCode::OutOfStorageSpace); + + // Cleanup the test directory + std::filesystem::remove_all(test_dir); +} + + +TEST(kvs_check_size, check_size_scenarios) { + const std::string test_dir = "./kvs_check_size_test/"; + std::filesystem::remove_all(test_dir); + + // --- SCENARIO 1: Success on data within limits --- + { + KvsBuilder builder(InstanceId(1)); + builder.dir(std::string(test_dir)); + auto open_res = builder.build(); + ASSERT_TRUE(open_res); + Kvs kvs = std::move(open_res.value()); + + // Add a small amount of data + auto set_res = kvs.set_value("key", KvsValue("some_data")); + ASSERT_TRUE(set_res); + + // Check the size + auto check_res = kvs.calculate_potential_size(); + ASSERT_TRUE(check_res) << "check_size should succeed for data within limits"; + // Check if the returned size is plausible + EXPECT_GT(check_res.value(), 0); + EXPECT_LT(check_res.value(), KVS_DEFAULT_MAX_SIZE_BYTES); + } + + // --- SCENARIO 2: Failure on data exceeding limits --- + { + KvsBuilder builder(InstanceId(2)); + builder.dir(std::string(test_dir)); + auto open_res = builder.build(); + ASSERT_TRUE(open_res); + Kvs kvs = std::move(open_res.value()); + + // Add data that will certainly exceed the storage limit + std::string large_data(KVS_DEFAULT_MAX_SIZE_BYTES, 'x'); + auto set_res = kvs.set_value("oversized_key", KvsValue(large_data.c_str())); + ASSERT_TRUE(set_res); + + // Check the size, expecting a failure + auto check_res = kvs.calculate_potential_size(); + ASSERT_FALSE(check_res) << "check_size should fail when storage limit is exceeded"; + EXPECT_EQ(static_cast(*check_res.error()), ErrorCode::OutOfStorageSpace); + } + + std::filesystem::remove_all(test_dir); +}