Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 1 | // Copyright 2020 The Pigweed Authors |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not |
| 4 | // use this file except in compliance with the License. You may obtain a copy of |
| 5 | // the License at |
| 6 | // |
| 7 | // https://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 12 | // License for the specific language governing permissions and limitations under |
| 13 | // the License. |
| 14 | |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 15 | #include <cstdlib> |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 16 | #include <random> |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 17 | #include <set> |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 18 | #include <string> |
| 19 | #include <string_view> |
| 20 | #include <unordered_map> |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 21 | #include <unordered_set> |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 22 | |
| 23 | #define DUMP_KVS_CONTENTS 0 |
| 24 | |
| 25 | #if DUMP_KVS_CONTENTS |
| 26 | #include <iostream> |
| 27 | #endif // DUMP_KVS_CONTENTS |
| 28 | |
| 29 | #include "gtest/gtest.h" |
| 30 | #include "pw_kvs/crc16_checksum.h" |
David Rogers | 5cc5ce8 | 2020-03-13 19:19:03 -0700 | [diff] [blame] | 31 | #include "pw_kvs/flash_partition_with_stats.h" |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 32 | #include "pw_kvs/in_memory_fake_flash.h" |
Wyatt Hepler | bdd8e5a | 2020-02-20 19:27:26 -0800 | [diff] [blame] | 33 | #include "pw_kvs/internal/entry.h" |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 34 | #include "pw_kvs/key_value_store.h" |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 35 | #include "pw_log/log.h" |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 36 | #include "pw_span/span.h" |
| 37 | |
| 38 | namespace pw::kvs { |
| 39 | namespace { |
| 40 | |
| 41 | using std::byte; |
| 42 | |
Wyatt Hepler | 38ce30f | 2020-02-19 11:48:31 -0800 | [diff] [blame] | 43 | constexpr size_t kMaxEntries = 256; |
| 44 | constexpr size_t kMaxUsableSectors = 256; |
| 45 | |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 46 | constexpr std::string_view kChars = |
| 47 | "abcdefghijklmnopqrstuvwxyz" |
| 48 | "ABCDEFGHIJKLMNOPQRSTUVWXYZ" |
| 49 | "0123456789"; |
| 50 | |
| 51 | struct TestParameters { |
| 52 | size_t sector_size; |
| 53 | size_t sector_count; |
| 54 | size_t sector_alignment; |
David Rogers | f3884eb | 2020-03-08 19:21:40 -0700 | [diff] [blame] | 55 | size_t redundancy; |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 56 | size_t partition_start_sector; |
| 57 | size_t partition_sector_count; |
| 58 | size_t partition_alignment; |
| 59 | }; |
| 60 | |
David Rogers | c8fe1f5 | 2020-02-27 14:04:08 -0800 | [diff] [blame] | 61 | enum Options { |
| 62 | kNone, |
| 63 | kReinit, |
| 64 | kReinitWithFullGC, |
| 65 | kReinitWithPartialGC, |
| 66 | }; |
| 67 | |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 68 | template <typename T> |
| 69 | std::set<T> difference(const std::set<T> lhs, const std::set<T> rhs) { |
| 70 | std::set<T> diff; |
| 71 | std::set_difference(lhs.begin(), |
| 72 | lhs.end(), |
| 73 | rhs.begin(), |
| 74 | rhs.end(), |
| 75 | std::inserter(diff, diff.begin())); |
| 76 | |
| 77 | return diff; |
| 78 | } |
| 79 | |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 80 | template <const TestParameters& kParams> |
| 81 | class KvsTester { |
| 82 | public: |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 83 | KvsTester() |
| 84 | : partition_(&flash_, |
| 85 | kParams.partition_start_sector, |
| 86 | kParams.partition_sector_count, |
| 87 | kParams.partition_alignment), |
Wyatt Hepler | 22d0d9f | 2020-03-05 14:57:11 -0800 | [diff] [blame] | 88 | kvs_(&partition_, {.magic = 0xBAD'C0D3, .checksum = nullptr}) { |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 89 | EXPECT_EQ(Status::OK, partition_.Erase()); |
| 90 | Status result = kvs_.Init(); |
| 91 | EXPECT_EQ(Status::OK, result); |
| 92 | |
| 93 | if (!result.ok()) { |
| 94 | std::abort(); |
| 95 | } |
| 96 | } |
| 97 | |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 98 | ~KvsTester() { CompareContents(); } |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 99 | |
Wyatt Hepler | e2a36a2 | 2020-02-20 17:01:38 -0800 | [diff] [blame] | 100 | void Test_RandomValidInputs(int iterations, |
| 101 | uint_fast32_t seed, |
David Rogers | c8fe1f5 | 2020-02-27 14:04:08 -0800 | [diff] [blame] | 102 | Options options) { |
Wyatt Hepler | e2a36a2 | 2020-02-20 17:01:38 -0800 | [diff] [blame] | 103 | std::mt19937 random(seed); |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 104 | std::uniform_int_distribution<unsigned> distro; |
| 105 | auto random_int = [&] { return distro(random); }; |
| 106 | |
| 107 | auto random_string = [&](size_t length) { |
| 108 | std::string value; |
| 109 | for (size_t i = 0; i < length; ++i) { |
| 110 | value.push_back(kChars[random_int() % kChars.size()]); |
| 111 | } |
| 112 | return value; |
| 113 | }; |
| 114 | |
David Rogers | 5cc5ce8 | 2020-03-13 19:19:03 -0700 | [diff] [blame] | 115 | partition_.ResetCounters(); |
| 116 | |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 117 | for (int i = 0; i < iterations; ++i) { |
David Rogers | c8fe1f5 | 2020-02-27 14:04:08 -0800 | [diff] [blame] | 118 | if (options != kNone && random_int() % 10 == 0) { |
Wyatt Hepler | e2a36a2 | 2020-02-20 17:01:38 -0800 | [diff] [blame] | 119 | Init(); |
| 120 | } |
| 121 | |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 122 | // One out of 4 times, delete a key. |
| 123 | if (random_int() % 4 == 0) { |
| 124 | // Either delete a non-existent key or delete an existing one. |
| 125 | if (empty() || random_int() % 8 == 0) { |
| 126 | Delete("not_a_key" + std::to_string(random_int())); |
| 127 | } else { |
Wyatt Hepler | e541e07 | 2020-02-14 09:10:53 -0800 | [diff] [blame] | 128 | Delete(RandomPresentKey()); |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 129 | } |
| 130 | } else { |
| 131 | std::string key; |
| 132 | |
| 133 | // Either add a new key or replace an existing one. |
David Rogers | f3884eb | 2020-03-08 19:21:40 -0700 | [diff] [blame] | 134 | // TODO: Using %2 (or any less than 16) fails with redundancy due to KVS |
| 135 | // filling up and not being able to write the second redundant entry, |
| 136 | // returning error. After re-init() the new key is picked up, resulting |
| 137 | // in a mis-match between KVS and the test map. |
| 138 | if (empty() || random_int() % 16 == 0) { |
Wyatt Hepler | 1fc1104 | 2020-02-19 17:17:51 -0800 | [diff] [blame] | 139 | key = random_string(random_int() % |
| 140 | (internal::Entry::kMaxKeyLength + 1)); |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 141 | } else { |
Wyatt Hepler | e541e07 | 2020-02-14 09:10:53 -0800 | [diff] [blame] | 142 | key = RandomPresentKey(); |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 143 | } |
| 144 | |
| 145 | Put(key, random_string(random_int() % kMaxValueLength)); |
| 146 | } |
David Rogers | cd87c32 | 2020-02-27 14:04:08 -0800 | [diff] [blame] | 147 | |
David Rogers | c8fe1f5 | 2020-02-27 14:04:08 -0800 | [diff] [blame] | 148 | if (options == kReinitWithFullGC && random_int() % 250 == 0) { |
David Rogers | cd87c32 | 2020-02-27 14:04:08 -0800 | [diff] [blame] | 149 | GCFull(); |
David Rogers | c8fe1f5 | 2020-02-27 14:04:08 -0800 | [diff] [blame] | 150 | } else if (options == kReinitWithPartialGC && random_int() % 40 == 0) { |
David Rogers | cd87c32 | 2020-02-27 14:04:08 -0800 | [diff] [blame] | 151 | GCPartial(); |
| 152 | } |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 153 | } |
David Rogers | 5cc5ce8 | 2020-03-13 19:19:03 -0700 | [diff] [blame] | 154 | |
| 155 | // Only save for tests that have enough data to be interesting. |
| 156 | if (partition_.sector_count() > 2 && partition_.total_erase_count() > 20) { |
| 157 | pw::StringBuffer<64> label; |
| 158 | label << "Random"; |
| 159 | label << partition_.sector_count(); |
| 160 | label << "Sector"; |
| 161 | label << iterations; |
| 162 | label << ((options != kNone) ? "Reinit" : ""); |
| 163 | label << ((options == kReinitWithFullGC) ? "FullGC" : ""); |
| 164 | label << ((options == kReinitWithPartialGC) ? "PartialGC" : ""); |
| 165 | label << ((kvs_.redundancy() > 1) ? "Redundant" : ""); |
| 166 | |
| 167 | partition_.SaveStorageStats(kvs_, label.data()); |
| 168 | } |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 169 | } |
| 170 | |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 171 | void Test_Put() { |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 172 | Put("base_key", "base_value"); |
| 173 | for (int i = 0; i < 100; ++i) { |
| 174 | Put("other_key", std::to_string(i)); |
| 175 | } |
| 176 | for (int i = 0; i < 100; ++i) { |
| 177 | Put("key_" + std::to_string(i), std::to_string(i)); |
| 178 | } |
| 179 | } |
| 180 | |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 181 | void Test_PutAndDelete_RelocateDeletedEntriesShouldStayDeleted() { |
| 182 | for (int i = 0; i < 100; ++i) { |
| 183 | std::string str = "key_" + std::to_string(i); |
| 184 | Put(str, std::string(kMaxValueLength, '?')); |
| 185 | Delete(str); |
| 186 | } |
| 187 | } |
| 188 | |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 189 | private: |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 190 | void CompareContents() { |
| 191 | #if DUMP_KVS_CONTENTS |
| 192 | std::set<std::string> map_keys, kvs_keys; |
| 193 | |
| 194 | std::cout << "/==============================================\\\n"; |
| 195 | std::cout << "KVS EXPECTED CONTENTS\n"; |
| 196 | std::cout << "------------------------------------------------\n"; |
| 197 | std::cout << "Entries: " << map_.size() << '\n'; |
| 198 | std::cout << "------------------------------------------------\n"; |
| 199 | for (const auto& [key, value] : map_) { |
| 200 | std::cout << key << " = " << value << '\n'; |
| 201 | map_keys.insert(key); |
| 202 | } |
| 203 | std::cout << "\\===============================================/\n"; |
| 204 | |
| 205 | std::cout << "/==============================================\\\n"; |
| 206 | std::cout << "KVS ACTUAL CONTENTS\n"; |
| 207 | std::cout << "------------------------------------------------\n"; |
| 208 | std::cout << "Entries: " << kvs_.size() << '\n'; |
| 209 | std::cout << "------------------------------------------------\n"; |
| 210 | for (const auto& item : kvs_) { |
| 211 | std::cout << item.key() << " = " << item.ValueSize().size() << " B\n"; |
| 212 | kvs_keys.insert(std::string(item.key())); |
| 213 | } |
| 214 | std::cout << "\\===============================================/\n"; |
| 215 | |
| 216 | auto missing_from_kvs = difference(map_keys, kvs_keys); |
| 217 | |
| 218 | if (!missing_from_kvs.empty()) { |
| 219 | std::cout << "MISSING FROM KVS: " << missing_from_kvs.size() << '\n'; |
| 220 | for (auto& key : missing_from_kvs) { |
| 221 | std::cout << key << '\n'; |
| 222 | } |
| 223 | } |
| 224 | |
| 225 | auto missing_from_map = difference(kvs_keys, map_keys); |
| 226 | if (!missing_from_map.empty()) { |
| 227 | std::cout << "MISSING FROM MAP: " << missing_from_map.size() << '\n'; |
| 228 | for (auto& key : missing_from_map) { |
| 229 | std::cout << key << '\n'; |
| 230 | } |
| 231 | } |
| 232 | #endif // DUMP_KVS_CONTENTS |
| 233 | |
| 234 | EXPECT_EQ(map_.size(), kvs_.size()); |
| 235 | |
| 236 | size_t count = 0; |
| 237 | |
| 238 | for (auto& item : kvs_) { |
| 239 | count += 1; |
| 240 | |
| 241 | auto map_entry = map_.find(std::string(item.key())); |
| 242 | if (map_entry == map_.end()) { |
Wyatt Hepler | 7bc766e | 2020-03-10 10:06:20 -0700 | [diff] [blame] | 243 | PW_LOG_CRITICAL( |
| 244 | "Entry %s missing from map%s", |
| 245 | item.key(), |
| 246 | deleted_.count(item.key()) > 0u ? " [was deleted previously]" : ""); |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 247 | } else if (map_entry != map_.end()) { |
| 248 | EXPECT_EQ(map_entry->first, item.key()); |
| 249 | |
| 250 | char value[kMaxValueLength + 1] = {}; |
| 251 | EXPECT_EQ(Status::OK, |
| 252 | item.Get(as_writable_bytes(span(value))).status()); |
| 253 | EXPECT_EQ(map_entry->second, std::string(value)); |
| 254 | } |
| 255 | } |
| 256 | |
| 257 | EXPECT_EQ(count, map_.size()); |
| 258 | } |
| 259 | |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 260 | // Adds a key to the KVS, if there is room for it. |
| 261 | void Put(const std::string& key, const std::string& value) { |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 262 | StartOperation("Put", key); |
| 263 | EXPECT_LE(value.size(), kMaxValueLength); |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 264 | |
| 265 | Status result = kvs_.Put(key, as_bytes(span(value))); |
| 266 | |
Wyatt Hepler | 1fc1104 | 2020-02-19 17:17:51 -0800 | [diff] [blame] | 267 | if (key.empty() || key.size() > internal::Entry::kMaxKeyLength) { |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 268 | EXPECT_EQ(Status::INVALID_ARGUMENT, result); |
Wyatt Hepler | 38ce30f | 2020-02-19 11:48:31 -0800 | [diff] [blame] | 269 | } else if (map_.size() == kvs_.max_size()) { |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 270 | EXPECT_EQ(Status::RESOURCE_EXHAUSTED, result); |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 271 | } else if (result == Status::RESOURCE_EXHAUSTED) { |
| 272 | EXPECT_FALSE(map_.empty()); |
| 273 | } else if (result.ok()) { |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 274 | map_[key] = value; |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 275 | deleted_.erase(key); |
| 276 | } else { |
| 277 | PW_LOG_CRITICAL("Put: unhandled result %s", result.str()); |
| 278 | std::abort(); |
| 279 | } |
| 280 | |
| 281 | FinishOperation("Put", result, key); |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 282 | } |
| 283 | |
| 284 | // Deletes a key from the KVS if it is present. |
| 285 | void Delete(const std::string& key) { |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 286 | StartOperation("Delete", key); |
| 287 | |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 288 | Status result = kvs_.Delete(key); |
| 289 | |
Wyatt Hepler | 1fc1104 | 2020-02-19 17:17:51 -0800 | [diff] [blame] | 290 | if (key.empty() || key.size() > internal::Entry::kMaxKeyLength) { |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 291 | EXPECT_EQ(Status::INVALID_ARGUMENT, result); |
| 292 | } else if (map_.count(key) == 0) { |
| 293 | EXPECT_EQ(Status::NOT_FOUND, result); |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 294 | } else if (result.ok()) { |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 295 | map_.erase(key); |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 296 | |
| 297 | if (deleted_.count(key) > 0u) { |
| 298 | PW_LOG_CRITICAL("Deleted key that was already deleted %s", key.c_str()); |
| 299 | std::abort(); |
| 300 | } |
| 301 | |
| 302 | deleted_.insert(key); |
Wyatt Hepler | e2a36a2 | 2020-02-20 17:01:38 -0800 | [diff] [blame] | 303 | } else if (result == Status::RESOURCE_EXHAUSTED) { |
| 304 | PW_LOG_WARN("Delete: RESOURCE_EXHAUSTED could not delete key %s", |
| 305 | key.c_str()); |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 306 | } else { |
Wyatt Hepler | e2a36a2 | 2020-02-20 17:01:38 -0800 | [diff] [blame] | 307 | PW_LOG_CRITICAL("Delete: unhandled result \"%s\"", result.str()); |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 308 | std::abort(); |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 309 | } |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 310 | FinishOperation("Delete", result, key); |
| 311 | } |
| 312 | |
Wyatt Hepler | e2a36a2 | 2020-02-20 17:01:38 -0800 | [diff] [blame] | 313 | void Init() { |
Wyatt Hepler | 7bc766e | 2020-03-10 10:06:20 -0700 | [diff] [blame] | 314 | StartOperation("Init"); |
Wyatt Hepler | e2a36a2 | 2020-02-20 17:01:38 -0800 | [diff] [blame] | 315 | Status status = kvs_.Init(); |
| 316 | EXPECT_EQ(Status::OK, status); |
Wyatt Hepler | 7bc766e | 2020-03-10 10:06:20 -0700 | [diff] [blame] | 317 | FinishOperation("Init", status); |
Wyatt Hepler | e2a36a2 | 2020-02-20 17:01:38 -0800 | [diff] [blame] | 318 | } |
| 319 | |
David Rogers | cd87c32 | 2020-02-27 14:04:08 -0800 | [diff] [blame] | 320 | void GCFull() { |
Wyatt Hepler | 7bc766e | 2020-03-10 10:06:20 -0700 | [diff] [blame] | 321 | StartOperation("GCFull"); |
David Rogers | 9abe3c7 | 2020-03-24 19:03:13 -0700 | [diff] [blame] | 322 | Status status = kvs_.FullMaintenance(); |
David Rogers | cd87c32 | 2020-02-27 14:04:08 -0800 | [diff] [blame] | 323 | EXPECT_EQ(Status::OK, status); |
| 324 | KeyValueStore::StorageStats post_stats = kvs_.GetStorageStats(); |
| 325 | EXPECT_EQ(post_stats.reclaimable_bytes, 0U); |
Wyatt Hepler | 7bc766e | 2020-03-10 10:06:20 -0700 | [diff] [blame] | 326 | FinishOperation("GCFull", status); |
David Rogers | cd87c32 | 2020-02-27 14:04:08 -0800 | [diff] [blame] | 327 | } |
| 328 | |
| 329 | void GCPartial() { |
Wyatt Hepler | 7bc766e | 2020-03-10 10:06:20 -0700 | [diff] [blame] | 330 | StartOperation("GCPartial"); |
David Rogers | cd87c32 | 2020-02-27 14:04:08 -0800 | [diff] [blame] | 331 | KeyValueStore::StorageStats pre_stats = kvs_.GetStorageStats(); |
David Rogers | 9abe3c7 | 2020-03-24 19:03:13 -0700 | [diff] [blame] | 332 | Status status = kvs_.PartialMaintenance(); |
David Rogers | cd87c32 | 2020-02-27 14:04:08 -0800 | [diff] [blame] | 333 | KeyValueStore::StorageStats post_stats = kvs_.GetStorageStats(); |
| 334 | if (pre_stats.reclaimable_bytes != 0) { |
David Rogers | a2562b5 | 2020-03-05 15:30:05 -0800 | [diff] [blame] | 335 | EXPECT_EQ(Status::OK, status); |
David Rogers | cd87c32 | 2020-02-27 14:04:08 -0800 | [diff] [blame] | 336 | EXPECT_LT(post_stats.reclaimable_bytes, pre_stats.reclaimable_bytes); |
| 337 | } else { |
David Rogers | a2562b5 | 2020-03-05 15:30:05 -0800 | [diff] [blame] | 338 | EXPECT_EQ(Status::NOT_FOUND, status); |
David Rogers | cd87c32 | 2020-02-27 14:04:08 -0800 | [diff] [blame] | 339 | EXPECT_EQ(post_stats.reclaimable_bytes, 0U); |
| 340 | } |
Wyatt Hepler | 7bc766e | 2020-03-10 10:06:20 -0700 | [diff] [blame] | 341 | FinishOperation("GCPartial", status); |
David Rogers | cd87c32 | 2020-02-27 14:04:08 -0800 | [diff] [blame] | 342 | } |
| 343 | |
Wyatt Hepler | 486ac57 | 2020-03-12 15:15:22 -0700 | [diff] [blame] | 344 | // Logs that an operation started and checks that the KVS matches the map. If |
| 345 | // a key is provided, that is included in the logs. |
Wyatt Hepler | 7bc766e | 2020-03-10 10:06:20 -0700 | [diff] [blame] | 346 | void StartOperation(const std::string& operation, |
| 347 | const std::string& key = "") { |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 348 | count_ += 1; |
Wyatt Hepler | 7bc766e | 2020-03-10 10:06:20 -0700 | [diff] [blame] | 349 | if (key.empty()) { |
| 350 | PW_LOG_DEBUG("[%3u] START %s", count_, operation.c_str()); |
| 351 | } else { |
| 352 | PW_LOG_DEBUG( |
| 353 | "[%3u] START %s for '%s'", count_, operation.c_str(), key.c_str()); |
| 354 | } |
| 355 | AbortIfMismatched("Pre-" + operation); |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 356 | } |
| 357 | |
Wyatt Hepler | 486ac57 | 2020-03-12 15:15:22 -0700 | [diff] [blame] | 358 | // Logs that an operation finished and checks that the KVS matches the map. |
| 359 | // If a key is provided, that is included in the logs. |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 360 | void FinishOperation(const std::string& operation, |
| 361 | Status result, |
Wyatt Hepler | 7bc766e | 2020-03-10 10:06:20 -0700 | [diff] [blame] | 362 | const std::string& key = "") { |
| 363 | if (key.empty()) { |
| 364 | PW_LOG_DEBUG( |
| 365 | "[%3u] FINISH %s <%s>", count_, operation.c_str(), result.str()); |
| 366 | } else { |
| 367 | PW_LOG_DEBUG("[%3u] FINISH %s <%s> for '%s'", |
| 368 | count_, |
| 369 | operation.c_str(), |
| 370 | result.str(), |
| 371 | key.c_str()); |
| 372 | } |
| 373 | AbortIfMismatched(operation); |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 374 | } |
| 375 | |
| 376 | bool empty() const { return map_.empty(); } |
| 377 | |
Wyatt Hepler | e541e07 | 2020-02-14 09:10:53 -0800 | [diff] [blame] | 378 | std::string RandomPresentKey() const { |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 379 | return map_.empty() ? "" : map_.begin()->second; |
| 380 | } |
| 381 | |
Wyatt Hepler | 7bc766e | 2020-03-10 10:06:20 -0700 | [diff] [blame] | 382 | void AbortIfMismatched(const std::string& stage) { |
| 383 | if (kvs_.size() != map_.size()) { |
| 384 | PW_LOG_CRITICAL("%s: size mismatch", stage.c_str()); |
| 385 | CompareContents(); |
| 386 | std::abort(); |
| 387 | } |
| 388 | } |
| 389 | |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 390 | static constexpr size_t kMaxValueLength = 64; |
| 391 | |
David Rogers | f3884eb | 2020-03-08 19:21:40 -0700 | [diff] [blame] | 392 | static FakeFlashBuffer<kParams.sector_size, |
| 393 | (kParams.sector_count * kParams.redundancy)> |
| 394 | flash_; |
David Rogers | 5cc5ce8 | 2020-03-13 19:19:03 -0700 | [diff] [blame] | 395 | |
| 396 | FlashPartitionWithStatsBuffer<kMaxEntries> partition_; |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 397 | |
David Rogers | f3884eb | 2020-03-08 19:21:40 -0700 | [diff] [blame] | 398 | KeyValueStoreBuffer<kMaxEntries, kMaxUsableSectors, kParams.redundancy> kvs_; |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 399 | std::unordered_map<std::string, std::string> map_; |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 400 | std::unordered_set<std::string> deleted_; |
| 401 | unsigned count_ = 0; |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 402 | }; |
| 403 | |
| 404 | template <const TestParameters& kParams> |
David Rogers | f3884eb | 2020-03-08 19:21:40 -0700 | [diff] [blame] | 405 | FakeFlashBuffer<kParams.sector_size, |
| 406 | (kParams.sector_count * kParams.redundancy)> |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 407 | KvsTester<kParams>::flash_ = |
David Rogers | f3884eb | 2020-03-08 19:21:40 -0700 | [diff] [blame] | 408 | FakeFlashBuffer<kParams.sector_size, |
| 409 | (kParams.sector_count * kParams.redundancy)>( |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 410 | kParams.sector_alignment); |
| 411 | |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 412 | #define _TEST(fixture, test, ...) \ |
Wyatt Hepler | e2a36a2 | 2020-02-20 17:01:38 -0800 | [diff] [blame] | 413 | _TEST_VARIANT(fixture, test, test, __VA_ARGS__) |
| 414 | |
| 415 | #define _TEST_VARIANT(fixture, test, variant, ...) \ |
| 416 | TEST_F(fixture, test##variant) { tester_.Test_##test(__VA_ARGS__); } |
Wyatt Hepler | 0af6ad9 | 2020-02-13 15:54:46 -0800 | [diff] [blame] | 417 | |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 418 | // Defines a test fixture that runs all tests against a flash with the specified |
| 419 | // parameters. |
David Rogers | f3884eb | 2020-03-08 19:21:40 -0700 | [diff] [blame] | 420 | #define RUN_TESTS_WITH_PARAMETERS(name, ...) \ |
| 421 | class name : public ::testing::Test { \ |
| 422 | protected: \ |
| 423 | static constexpr TestParameters kParams = {__VA_ARGS__}; \ |
| 424 | \ |
| 425 | KvsTester<kParams> tester_; \ |
| 426 | }; \ |
| 427 | /* Run each test defined in the KvsTester class with these parameters. */ \ |
| 428 | _TEST(name, Put); \ |
| 429 | _TEST(name, PutAndDelete_RelocateDeletedEntriesShouldStayDeleted); \ |
| 430 | _TEST_VARIANT(name, RandomValidInputs, 1, 1000, 6006411, kNone); \ |
| 431 | _TEST_VARIANT(name, RandomValidInputs, 1WithReinit, 500, 6006411, kReinit); \ |
| 432 | _TEST_VARIANT(name, RandomValidInputs, 2, 100, 123, kNone); \ |
| 433 | _TEST_VARIANT(name, RandomValidInputs, 2WithReinit, 100, 123, kReinit); \ |
| 434 | _TEST_VARIANT(name, \ |
| 435 | RandomValidInputs, \ |
| 436 | 1ReinitFullGC, \ |
| 437 | 300, \ |
| 438 | 6006411, \ |
| 439 | kReinitWithFullGC); \ |
| 440 | _TEST_VARIANT( \ |
| 441 | name, RandomValidInputs, 2ReinitFullGC, 300, 123, kReinitWithFullGC); \ |
| 442 | _TEST_VARIANT(name, \ |
| 443 | RandomValidInputs, \ |
| 444 | 1ReinitPartialGC, \ |
| 445 | 100, \ |
| 446 | 6006411, \ |
| 447 | kReinitWithPartialGC); \ |
| 448 | _TEST_VARIANT(name, \ |
| 449 | RandomValidInputs, \ |
| 450 | 2ReinitPartialGC, \ |
| 451 | 200, \ |
| 452 | 123, \ |
| 453 | kReinitWithPartialGC); \ |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 454 | static_assert(true, "Don't forget a semicolon!") |
| 455 | |
| 456 | RUN_TESTS_WITH_PARAMETERS(Basic, |
| 457 | .sector_size = 4 * 1024, |
| 458 | .sector_count = 4, |
| 459 | .sector_alignment = 16, |
David Rogers | f3884eb | 2020-03-08 19:21:40 -0700 | [diff] [blame] | 460 | .redundancy = 1, |
| 461 | .partition_start_sector = 0, |
| 462 | .partition_sector_count = 4, |
| 463 | .partition_alignment = 16); |
| 464 | |
| 465 | RUN_TESTS_WITH_PARAMETERS(BasicRedundant, |
| 466 | .sector_size = 4 * 1024, |
| 467 | .sector_count = 4, |
| 468 | .sector_alignment = 16, |
| 469 | .redundancy = 2, |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 470 | .partition_start_sector = 0, |
| 471 | .partition_sector_count = 4, |
| 472 | .partition_alignment = 16); |
| 473 | |
Wyatt Hepler | e2a36a2 | 2020-02-20 17:01:38 -0800 | [diff] [blame] | 474 | RUN_TESTS_WITH_PARAMETERS(LotsOfSmallSectors, |
| 475 | .sector_size = 160, |
| 476 | .sector_count = 100, |
| 477 | .sector_alignment = 32, |
David Rogers | f3884eb | 2020-03-08 19:21:40 -0700 | [diff] [blame] | 478 | .redundancy = 1, |
| 479 | .partition_start_sector = 5, |
| 480 | .partition_sector_count = 95, |
| 481 | .partition_alignment = 32); |
| 482 | |
| 483 | RUN_TESTS_WITH_PARAMETERS(LotsOfSmallSectorsRedundant, |
| 484 | .sector_size = 160, |
| 485 | .sector_count = 100, |
| 486 | .sector_alignment = 32, |
| 487 | .redundancy = 2, |
Wyatt Hepler | e2a36a2 | 2020-02-20 17:01:38 -0800 | [diff] [blame] | 488 | .partition_start_sector = 5, |
| 489 | .partition_sector_count = 95, |
| 490 | .partition_alignment = 32); |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 491 | |
Wyatt Hepler | e2a36a2 | 2020-02-20 17:01:38 -0800 | [diff] [blame] | 492 | RUN_TESTS_WITH_PARAMETERS(OnlyTwoSectors, |
| 493 | .sector_size = 4 * 1024, |
| 494 | .sector_count = 20, |
| 495 | .sector_alignment = 16, |
David Rogers | f3884eb | 2020-03-08 19:21:40 -0700 | [diff] [blame] | 496 | .redundancy = 1, |
Wyatt Hepler | e2a36a2 | 2020-02-20 17:01:38 -0800 | [diff] [blame] | 497 | .partition_start_sector = 18, |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 498 | .partition_sector_count = 2, |
Wyatt Hepler | e2a36a2 | 2020-02-20 17:01:38 -0800 | [diff] [blame] | 499 | .partition_alignment = 64); |
Wyatt Hepler | 495b6ee | 2020-02-12 18:58:01 -0800 | [diff] [blame] | 500 | |
| 501 | } // namespace |
| 502 | } // namespace pw::kvs |