1 /**
2  * Utilities for loading generic configuration files consisting of key/value pairs.
3  *
4  * Authors:
5  *  Mike Bierlee, m.bierlee@lostmoment.com
6  * Copyright: 2022-2025 Mike Bierlee
7  * License:
8  *  This software is licensed under the terms of the MIT license.
9  *  The full terms of the license can be found in the LICENSE file.
10  */
11 
12 module mirage.keyvalue;
13 
14 import mirage.config : ConfigFactory, ConfigDictionary, ConfigNode, ValueNode, ObjectNode, ConfigCreationException;
15 
16 import std.string : lineSplitter, strip, startsWith, endsWith, split, indexOf, join;
17 import std.array : array;
18 import std.exception : enforce;
19 import std.conv : to;
20 import std.typecons : Flag;
21 
22 alias SupportHashtagComments = Flag!"SupportHashtagComments";
23 alias SupportSemicolonComments = Flag!"SupportSemicolonComments";
24 alias SupportExclamationComments = Flag!"SupportExclamationComments";
25 alias SupportSections = Flag!"SupportSections";
26 alias NormalizeQuotedValues = Flag!"NormalizeQuotedValues";
27 alias SupportEqualsSeparator = Flag!"SupportEqualsSeparator";
28 alias SupportColonSeparator = Flag!"SupportColonSeparator";
29 alias SupportKeysWithoutValues = Flag!"SupportKeysWithoutValues";
30 alias SupportMultilineValues = Flag!"SupportMultilineValues";
31 
32 /** 
33  * A generic reusable key/value config factory that can be configured to parse
34  * the specifics of certain key/value formats.
35  */
36 class KeyValueConfigFactory(
37     SupportHashtagComments supportHashtagComments = SupportHashtagComments.no,
38     SupportSemicolonComments supportSemicolonComments = SupportSemicolonComments.no,
39     SupportExclamationComments supportExclamationComments = SupportExclamationComments.no,
40     SupportSections supportSections = SupportSections.no,
41     NormalizeQuotedValues normalizeQuotedValues = NormalizeQuotedValues.no,
42     SupportEqualsSeparator supportEqualsSeparator = SupportEqualsSeparator.no,
43     SupportColonSeparator supportColonSeparator = SupportColonSeparator.no,
44     SupportKeysWithoutValues supportKeysWithoutValues = SupportKeysWithoutValues.no,
45     SupportMultilineValues supportMultilineValues = SupportMultilineValues.no
46 ) : ConfigFactory {
47 
48     /**
49      * Parse a configuration file following the configured key/value conventions.
50      *
51      * Params:
52      *   contents = Text contents of the config to be parsed.
53      * Returns: The parsed configuration.
54      */
55     override ConfigDictionary parseConfig(string contents) {
56         enforce!ConfigCreationException(contents !is null, "Contents cannot be null.");
57         enforce!ConfigCreationException(supportEqualsSeparator || supportColonSeparator, "No key/value separator is supported. Factory must set one either SupportEqualsSeparator or SupportColonSeparator");
58 
59         auto lines = contents.lineSplitter().array;
60         auto properties = new ConfigDictionary();
61         auto section = "";
62         string key = null;
63         string valueBuffer = "";
64 
65         foreach (size_t index, string line; lines) {
66             auto processedLine = line;
67 
68             void replaceComments(bool isTypeSupported, char commentToken) {
69                 if (isTypeSupported) {
70                     auto commentPosition = processedLine.indexOf(commentToken);
71                     if (commentPosition >= 0) {
72                         processedLine = processedLine[0 .. commentPosition];
73                     }
74                 }
75             }
76 
77             replaceComments(supportHashtagComments, '#');
78             replaceComments(supportSemicolonComments, ';');
79             replaceComments(supportExclamationComments, '!');
80 
81             processedLine = processedLine.strip;
82 
83             if (supportSections &&
84                 key is null &&
85                 processedLine.startsWith('[') && processedLine.endsWith(']')) {
86                 auto parsedSection = processedLine[1 .. $ - 1];
87                 if (parsedSection.startsWith('.')) {
88                     section ~= parsedSection;
89                 } else {
90                     section = parsedSection;
91                 }
92 
93                 continue;
94             }
95 
96             if (processedLine.length == 0) {
97                 continue;
98             }
99 
100             string value;
101 
102             if (key is null) {
103                 char keyValueSplitter;
104                 if (supportEqualsSeparator && processedLine.indexOf('=') >= 0) {
105                     keyValueSplitter = '=';
106                 } else if (supportColonSeparator && processedLine.indexOf(':') >= 0) {
107                     keyValueSplitter = ':';
108                 }
109 
110                 auto parts = processedLine.split(keyValueSplitter);
111 
112                 enforce!ConfigCreationException(parts.length <= 2, "Line has too many equals signs and cannot be parsed (L" ~ index
113                         .to!string ~ "): " ~ processedLine);
114                 enforce!ConfigCreationException(supportKeysWithoutValues || parts.length == 2, "Missing value assignment (L" ~ index
115                         .to!string ~ "): " ~ processedLine);
116 
117                 key = [section, parts[0].strip].join('.');
118                 value = supportKeysWithoutValues && parts.length == 1 ? "" : parts[1].strip;
119             } else {
120                 value = processedLine;
121             }
122 
123             if (supportMultilineValues && value.endsWith('\\')) {
124                 valueBuffer ~= value[0 .. $ - 1];
125                 continue;
126             }
127 
128             auto fullValue = valueBuffer ~ value;
129             if (normalizeQuotedValues &&
130                 fullValue.length > 1 &&
131                 (fullValue.startsWith('"') || fullValue.startsWith('\'')) &&
132                 (fullValue.endsWith('"') || fullValue.endsWith('\''))) {
133                 fullValue = fullValue[1 .. $ - 1];
134             }
135 
136             properties.set(key, fullValue);
137             key = null;
138             valueBuffer = "";
139         }
140 
141         return properties;
142     }
143 }
144 
145 version (unittest) {
146     import std.exception : assertThrown;
147     import std.process : environment;
148 
149     class TestKeyValueConfigFactory : KeyValueConfigFactory!(
150         SupportHashtagComments.no,
151         SupportSemicolonComments.no,
152         SupportExclamationComments.no,
153         SupportSections.no,
154         NormalizeQuotedValues.no,
155         SupportEqualsSeparator.yes,
156         SupportColonSeparator.no,
157         SupportKeysWithoutValues.no,
158         SupportMultilineValues.no
159     ) {
160     }
161 
162     @("Parse standard key/value config")
163     unittest {
164         auto config = new TestKeyValueConfigFactory().parseConfig("
165             bla=one
166             di.bla=two
167         ");
168 
169         assert(config.get("bla") == "one");
170         assert(config.get("di.bla") == "two");
171     }
172 
173     @("Parse and ignore comments")
174     unittest {
175         auto config = new KeyValueConfigFactory!(
176             SupportHashtagComments.yes,
177             SupportSemicolonComments.yes,
178             SupportExclamationComments.yes,
179             SupportSections.no,
180             NormalizeQuotedValues.no,
181             SupportEqualsSeparator.yes,
182             SupportColonSeparator.no
183         )().parseConfig("
184             # this is a comment
185             ; this is another comment
186             ! this then is also a comment!
187             iamset=true
188         ");
189 
190         assert(config.get!bool("iamset"));
191     }
192 
193     @("Fail to parse when there are too many equals signs")
194     unittest {
195         assertThrown!ConfigCreationException(new TestKeyValueConfigFactory()
196                 .parseConfig("one=two=three"));
197     }
198 
199     @("Fail to parse when value assignment is missing")
200     unittest {
201         assertThrown!ConfigCreationException(new TestKeyValueConfigFactory()
202                 .parseConfig("answertolife"));
203     }
204 
205     @("Succeed to parse when value assignment is missing and SupportKeysWithoutValues = yes")
206     unittest {
207         auto config = new KeyValueConfigFactory!(
208             SupportHashtagComments.no,
209             SupportSemicolonComments.no,
210             SupportExclamationComments.no,
211             SupportSections.no,
212             NormalizeQuotedValues.no,
213             SupportEqualsSeparator.yes,
214             SupportColonSeparator.no,
215             SupportKeysWithoutValues.yes
216         )().parseConfig("answertolife");
217 
218         assert(config.get("answertolife") == "");
219     }
220 
221     @("Substitute env vars")
222     unittest {
223         environment["MIRAGE_TEST_ENVY"] = "Much";
224         auto config = new TestKeyValueConfigFactory().parseConfig("envy=$MIRAGE_TEST_ENVY");
225 
226         assert(config.get("envy") == "Much");
227     }
228 
229     @("Use value from other key")
230     unittest {
231         auto config = new TestKeyValueConfigFactory().parseConfig("
232             one=money
233             two=${one}
234         ");
235 
236         assert(config.get("two") == "money");
237     }
238 
239     @("Values and keys are trimmed")
240     unittest {
241         auto config = new TestKeyValueConfigFactory().parseConfig("
242             one    =       money
243         ");
244 
245         assert(config.get("one") == "money");
246     }
247 
248     @("Remove end-of-line comments")
249     unittest {
250         auto config = new KeyValueConfigFactory!(
251             SupportHashtagComments.yes,
252             SupportSemicolonComments.yes,
253             SupportExclamationComments.yes,
254             SupportSections.no,
255             NormalizeQuotedValues.no,
256             SupportEqualsSeparator.yes,
257             SupportColonSeparator.no
258         )().parseConfig("
259             server=localhost #todo: change me. default=localhost when not set.
260             port=9876; I think this port = right?
261             timeout=36000 ! pretty long!
262         ");
263 
264         assert(config.get("server") == "localhost");
265         assert(config.get("port") == "9876");
266         assert(config.get("timeout") == "36000");
267     }
268 
269     @("Support sections when enabled")
270     unittest {
271         auto config = new KeyValueConfigFactory!(
272             SupportHashtagComments.no,
273             SupportSemicolonComments.yes,
274             SupportExclamationComments.no,
275             SupportSections.yes,
276             NormalizeQuotedValues.no,
277             SupportEqualsSeparator.yes,
278             SupportColonSeparator.no
279         )().parseConfig("
280             applicationName = test me!
281 
282             [server]
283             host=localhost
284             port=2873
285 
286             [.toaster]
287             color=chrome
288 
289             [server.middleware] ; Stuff that handles the http protocol
290             protocolServer = netty
291 
292             [database.driver]
293             id=PostgresDriver
294         ");
295 
296         assert(config.get("applicationName") == "test me!");
297         assert(config.get("server.host") == "localhost");
298         assert(config.get("server.port") == "2873");
299         assert(config.get("server.toaster.color") == "chrome");
300         assert(config.get("server.middleware.protocolServer") == "netty");
301         assert(config.get("database.driver.id") == "PostgresDriver");
302     }
303 
304     @("Values with quotes are normalized and return the value within")
305     unittest {
306         auto config = new KeyValueConfigFactory!(
307             SupportHashtagComments.yes,
308             SupportSemicolonComments.no,
309             SupportExclamationComments.no,
310             SupportSections.no,
311             NormalizeQuotedValues.yes,
312             SupportEqualsSeparator.yes,
313             SupportColonSeparator.no
314         )().parseConfig("
315             baboon = \"ape\"
316             monkey = 'ape'
317             human = ape
318             excessiveWhitespace = '             '
319             breaksWithComments = '   # Don't do this    '
320         ");
321 
322         assert(config.get("baboon") == "ape");
323         assert(config.get("monkey") == "ape");
324         assert(config.get("human") == "ape");
325         assert(config.get("excessiveWhitespace") == "             ");
326         assert(config.get("breaksWithComments") == "'");
327     }
328 
329     @("Support colon as key/value separator")
330     unittest {
331         auto config = new KeyValueConfigFactory!(
332             SupportHashtagComments.no,
333             SupportSemicolonComments.no,
334             SupportExclamationComments.no,
335             SupportSections.no,
336             NormalizeQuotedValues.no,
337             SupportEqualsSeparator.yes,
338             SupportColonSeparator.yes
339         )().parseConfig("
340             one = here
341             two: also here
342         ");
343 
344         assert(config.get("one") == "here");
345         assert(config.get("two") == "also here");
346 
347         assertThrown!ConfigCreationException(new KeyValueConfigFactory!()().parseConfig("a=b")); // No separator is configured
348     }
349 
350     @("Support multiline values")
351     unittest {
352         auto config = new KeyValueConfigFactory!(
353             SupportHashtagComments.yes,
354             SupportSemicolonComments.no,
355             SupportExclamationComments.no,
356             SupportSections.yes,
357             NormalizeQuotedValues.yes,
358             SupportEqualsSeparator.yes,
359             SupportColonSeparator.no,
360             SupportKeysWithoutValues.yes,
361             SupportMultilineValues.yes
362         )().parseConfig("
363             sentence = the quick \\
364                        'brown fox' \\ # comments
365                        [jump]\\
366                 \\
367                        ed over \\ #are not part of the
368                        the lazy \\
369             '[dog]'   #value
370 
371             not part of the sentence
372         ");
373 
374         assert(config.get("sentence") == "the quick 'brown fox' [jump]ed over the lazy '[dog]'");
375     }
376 
377     @("Normalize multiline values with quotes")
378     unittest {
379         auto config = new KeyValueConfigFactory!(
380             SupportHashtagComments.no,
381             SupportSemicolonComments.no,
382             SupportExclamationComments.no,
383             SupportSections.no,
384             NormalizeQuotedValues.yes,
385             SupportEqualsSeparator.yes,
386             SupportColonSeparator.no,
387             SupportKeysWithoutValues.no,
388             SupportMultilineValues.yes
389         )().parseConfig("
390             doubles = \"Well then there I was \\
391                         doing my thing.\"
392             singles = 'When suddenly \\
393                 a shark bit me \\
394                             from the sky'
395         ");
396 
397         assert(config.get("doubles") == "Well then there I was doing my thing.");
398         assert(config.get("singles") == "When suddenly a shark bit me from the sky");
399     }
400 
401 }