1 /** 2 * Utilities for loading generic configuration files consisting of key/value pairs. 3 * 4 * Authors: 5 * Mike Bierlee, m.bierlee@lostmoment.com 6 * Copyright: 2022-2025 Mike Bierlee 7 * License: 8 * This software is licensed under the terms of the MIT license. 9 * The full terms of the license can be found in the LICENSE file. 10 */ 11 12 module mirage.keyvalue; 13 14 import mirage.config : ConfigFactory, ConfigDictionary, ConfigNode, ValueNode, ObjectNode, ConfigCreationException; 15 16 import std.string : lineSplitter, strip, startsWith, endsWith, split, indexOf, join; 17 import std.array : array; 18 import std.exception : enforce; 19 import std.conv : to; 20 import std.typecons : Flag; 21 22 alias SupportHashtagComments = Flag!"SupportHashtagComments"; 23 alias SupportSemicolonComments = Flag!"SupportSemicolonComments"; 24 alias SupportExclamationComments = Flag!"SupportExclamationComments"; 25 alias SupportSections = Flag!"SupportSections"; 26 alias NormalizeQuotedValues = Flag!"NormalizeQuotedValues"; 27 alias SupportEqualsSeparator = Flag!"SupportEqualsSeparator"; 28 alias SupportColonSeparator = Flag!"SupportColonSeparator"; 29 alias SupportKeysWithoutValues = Flag!"SupportKeysWithoutValues"; 30 alias SupportMultilineValues = Flag!"SupportMultilineValues"; 31 32 /** 33 * A generic reusable key/value config factory that can be configured to parse 34 * the specifics of certain key/value formats. 35 */ 36 class KeyValueConfigFactory( 37 SupportHashtagComments supportHashtagComments = SupportHashtagComments.no, 38 SupportSemicolonComments supportSemicolonComments = SupportSemicolonComments.no, 39 SupportExclamationComments supportExclamationComments = SupportExclamationComments.no, 40 SupportSections supportSections = SupportSections.no, 41 NormalizeQuotedValues normalizeQuotedValues = NormalizeQuotedValues.no, 42 SupportEqualsSeparator supportEqualsSeparator = SupportEqualsSeparator.no, 43 SupportColonSeparator supportColonSeparator = SupportColonSeparator.no, 44 SupportKeysWithoutValues supportKeysWithoutValues = SupportKeysWithoutValues.no, 45 SupportMultilineValues supportMultilineValues = SupportMultilineValues.no 46 ) : ConfigFactory { 47 48 /** 49 * Parse a configuration file following the configured key/value conventions. 50 * 51 * Params: 52 * contents = Text contents of the config to be parsed. 53 * Returns: The parsed configuration. 54 */ 55 override ConfigDictionary parseConfig(string contents) { 56 enforce!ConfigCreationException(contents !is null, "Contents cannot be null."); 57 enforce!ConfigCreationException(supportEqualsSeparator || supportColonSeparator, "No key/value separator is supported. Factory must set one either SupportEqualsSeparator or SupportColonSeparator"); 58 59 auto lines = contents.lineSplitter().array; 60 auto properties = new ConfigDictionary(); 61 auto section = ""; 62 string key = null; 63 string valueBuffer = ""; 64 65 foreach (size_t index, string line; lines) { 66 auto processedLine = line; 67 68 void replaceComments(bool isTypeSupported, char commentToken) { 69 if (isTypeSupported) { 70 auto commentPosition = processedLine.indexOf(commentToken); 71 if (commentPosition >= 0) { 72 processedLine = processedLine[0 .. commentPosition]; 73 } 74 } 75 } 76 77 replaceComments(supportHashtagComments, '#'); 78 replaceComments(supportSemicolonComments, ';'); 79 replaceComments(supportExclamationComments, '!'); 80 81 processedLine = processedLine.strip; 82 83 if (supportSections && 84 key is null && 85 processedLine.startsWith('[') && processedLine.endsWith(']')) { 86 auto parsedSection = processedLine[1 .. $ - 1]; 87 if (parsedSection.startsWith('.')) { 88 section ~= parsedSection; 89 } else { 90 section = parsedSection; 91 } 92 93 continue; 94 } 95 96 if (processedLine.length == 0) { 97 continue; 98 } 99 100 string value; 101 102 if (key is null) { 103 char keyValueSplitter; 104 if (supportEqualsSeparator && processedLine.indexOf('=') >= 0) { 105 keyValueSplitter = '='; 106 } else if (supportColonSeparator && processedLine.indexOf(':') >= 0) { 107 keyValueSplitter = ':'; 108 } 109 110 auto parts = processedLine.split(keyValueSplitter); 111 112 enforce!ConfigCreationException(parts.length <= 2, "Line has too many equals signs and cannot be parsed (L" ~ index 113 .to!string ~ "): " ~ processedLine); 114 enforce!ConfigCreationException(supportKeysWithoutValues || parts.length == 2, "Missing value assignment (L" ~ index 115 .to!string ~ "): " ~ processedLine); 116 117 key = [section, parts[0].strip].join('.'); 118 value = supportKeysWithoutValues && parts.length == 1 ? "" : parts[1].strip; 119 } else { 120 value = processedLine; 121 } 122 123 if (supportMultilineValues && value.endsWith('\\')) { 124 valueBuffer ~= value[0 .. $ - 1]; 125 continue; 126 } 127 128 auto fullValue = valueBuffer ~ value; 129 if (normalizeQuotedValues && 130 fullValue.length > 1 && 131 (fullValue.startsWith('"') || fullValue.startsWith('\'')) && 132 (fullValue.endsWith('"') || fullValue.endsWith('\''))) { 133 fullValue = fullValue[1 .. $ - 1]; 134 } 135 136 properties.set(key, fullValue); 137 key = null; 138 valueBuffer = ""; 139 } 140 141 return properties; 142 } 143 } 144 145 version (unittest) { 146 import std.exception : assertThrown; 147 import std.process : environment; 148 149 class TestKeyValueConfigFactory : KeyValueConfigFactory!( 150 SupportHashtagComments.no, 151 SupportSemicolonComments.no, 152 SupportExclamationComments.no, 153 SupportSections.no, 154 NormalizeQuotedValues.no, 155 SupportEqualsSeparator.yes, 156 SupportColonSeparator.no, 157 SupportKeysWithoutValues.no, 158 SupportMultilineValues.no 159 ) { 160 } 161 162 @("Parse standard key/value config") 163 unittest { 164 auto config = new TestKeyValueConfigFactory().parseConfig(" 165 bla=one 166 di.bla=two 167 "); 168 169 assert(config.get("bla") == "one"); 170 assert(config.get("di.bla") == "two"); 171 } 172 173 @("Parse and ignore comments") 174 unittest { 175 auto config = new KeyValueConfigFactory!( 176 SupportHashtagComments.yes, 177 SupportSemicolonComments.yes, 178 SupportExclamationComments.yes, 179 SupportSections.no, 180 NormalizeQuotedValues.no, 181 SupportEqualsSeparator.yes, 182 SupportColonSeparator.no 183 )().parseConfig(" 184 # this is a comment 185 ; this is another comment 186 ! this then is also a comment! 187 iamset=true 188 "); 189 190 assert(config.get!bool("iamset")); 191 } 192 193 @("Fail to parse when there are too many equals signs") 194 unittest { 195 assertThrown!ConfigCreationException(new TestKeyValueConfigFactory() 196 .parseConfig("one=two=three")); 197 } 198 199 @("Fail to parse when value assignment is missing") 200 unittest { 201 assertThrown!ConfigCreationException(new TestKeyValueConfigFactory() 202 .parseConfig("answertolife")); 203 } 204 205 @("Succeed to parse when value assignment is missing and SupportKeysWithoutValues = yes") 206 unittest { 207 auto config = new KeyValueConfigFactory!( 208 SupportHashtagComments.no, 209 SupportSemicolonComments.no, 210 SupportExclamationComments.no, 211 SupportSections.no, 212 NormalizeQuotedValues.no, 213 SupportEqualsSeparator.yes, 214 SupportColonSeparator.no, 215 SupportKeysWithoutValues.yes 216 )().parseConfig("answertolife"); 217 218 assert(config.get("answertolife") == ""); 219 } 220 221 @("Substitute env vars") 222 unittest { 223 environment["MIRAGE_TEST_ENVY"] = "Much"; 224 auto config = new TestKeyValueConfigFactory().parseConfig("envy=$MIRAGE_TEST_ENVY"); 225 226 assert(config.get("envy") == "Much"); 227 } 228 229 @("Use value from other key") 230 unittest { 231 auto config = new TestKeyValueConfigFactory().parseConfig(" 232 one=money 233 two=${one} 234 "); 235 236 assert(config.get("two") == "money"); 237 } 238 239 @("Values and keys are trimmed") 240 unittest { 241 auto config = new TestKeyValueConfigFactory().parseConfig(" 242 one = money 243 "); 244 245 assert(config.get("one") == "money"); 246 } 247 248 @("Remove end-of-line comments") 249 unittest { 250 auto config = new KeyValueConfigFactory!( 251 SupportHashtagComments.yes, 252 SupportSemicolonComments.yes, 253 SupportExclamationComments.yes, 254 SupportSections.no, 255 NormalizeQuotedValues.no, 256 SupportEqualsSeparator.yes, 257 SupportColonSeparator.no 258 )().parseConfig(" 259 server=localhost #todo: change me. default=localhost when not set. 260 port=9876; I think this port = right? 261 timeout=36000 ! pretty long! 262 "); 263 264 assert(config.get("server") == "localhost"); 265 assert(config.get("port") == "9876"); 266 assert(config.get("timeout") == "36000"); 267 } 268 269 @("Support sections when enabled") 270 unittest { 271 auto config = new KeyValueConfigFactory!( 272 SupportHashtagComments.no, 273 SupportSemicolonComments.yes, 274 SupportExclamationComments.no, 275 SupportSections.yes, 276 NormalizeQuotedValues.no, 277 SupportEqualsSeparator.yes, 278 SupportColonSeparator.no 279 )().parseConfig(" 280 applicationName = test me! 281 282 [server] 283 host=localhost 284 port=2873 285 286 [.toaster] 287 color=chrome 288 289 [server.middleware] ; Stuff that handles the http protocol 290 protocolServer = netty 291 292 [database.driver] 293 id=PostgresDriver 294 "); 295 296 assert(config.get("applicationName") == "test me!"); 297 assert(config.get("server.host") == "localhost"); 298 assert(config.get("server.port") == "2873"); 299 assert(config.get("server.toaster.color") == "chrome"); 300 assert(config.get("server.middleware.protocolServer") == "netty"); 301 assert(config.get("database.driver.id") == "PostgresDriver"); 302 } 303 304 @("Values with quotes are normalized and return the value within") 305 unittest { 306 auto config = new KeyValueConfigFactory!( 307 SupportHashtagComments.yes, 308 SupportSemicolonComments.no, 309 SupportExclamationComments.no, 310 SupportSections.no, 311 NormalizeQuotedValues.yes, 312 SupportEqualsSeparator.yes, 313 SupportColonSeparator.no 314 )().parseConfig(" 315 baboon = \"ape\" 316 monkey = 'ape' 317 human = ape 318 excessiveWhitespace = ' ' 319 breaksWithComments = ' # Don't do this ' 320 "); 321 322 assert(config.get("baboon") == "ape"); 323 assert(config.get("monkey") == "ape"); 324 assert(config.get("human") == "ape"); 325 assert(config.get("excessiveWhitespace") == " "); 326 assert(config.get("breaksWithComments") == "'"); 327 } 328 329 @("Support colon as key/value separator") 330 unittest { 331 auto config = new KeyValueConfigFactory!( 332 SupportHashtagComments.no, 333 SupportSemicolonComments.no, 334 SupportExclamationComments.no, 335 SupportSections.no, 336 NormalizeQuotedValues.no, 337 SupportEqualsSeparator.yes, 338 SupportColonSeparator.yes 339 )().parseConfig(" 340 one = here 341 two: also here 342 "); 343 344 assert(config.get("one") == "here"); 345 assert(config.get("two") == "also here"); 346 347 assertThrown!ConfigCreationException(new KeyValueConfigFactory!()().parseConfig("a=b")); // No separator is configured 348 } 349 350 @("Support multiline values") 351 unittest { 352 auto config = new KeyValueConfigFactory!( 353 SupportHashtagComments.yes, 354 SupportSemicolonComments.no, 355 SupportExclamationComments.no, 356 SupportSections.yes, 357 NormalizeQuotedValues.yes, 358 SupportEqualsSeparator.yes, 359 SupportColonSeparator.no, 360 SupportKeysWithoutValues.yes, 361 SupportMultilineValues.yes 362 )().parseConfig(" 363 sentence = the quick \\ 364 'brown fox' \\ # comments 365 [jump]\\ 366 \\ 367 ed over \\ #are not part of the 368 the lazy \\ 369 '[dog]' #value 370 371 not part of the sentence 372 "); 373 374 assert(config.get("sentence") == "the quick 'brown fox' [jump]ed over the lazy '[dog]'"); 375 } 376 377 @("Normalize multiline values with quotes") 378 unittest { 379 auto config = new KeyValueConfigFactory!( 380 SupportHashtagComments.no, 381 SupportSemicolonComments.no, 382 SupportExclamationComments.no, 383 SupportSections.no, 384 NormalizeQuotedValues.yes, 385 SupportEqualsSeparator.yes, 386 SupportColonSeparator.no, 387 SupportKeysWithoutValues.no, 388 SupportMultilineValues.yes 389 )().parseConfig(" 390 doubles = \"Well then there I was \\ 391 doing my thing.\" 392 singles = 'When suddenly \\ 393 a shark bit me \\ 394 from the sky' 395 "); 396 397 assert(config.get("doubles") == "Well then there I was doing my thing."); 398 assert(config.get("singles") == "When suddenly a shark bit me from the sky"); 399 } 400 401 }