From 02ca32ae0a5bc290918d2b2a3288e385b9cc6b11 Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Fri, 19 Feb 2021 17:10:51 -0500 Subject: external & build dependences in src tree - external & build dependences boost licensed - ext_depends (external depends) - D-YAML - tinyendian - d2sqlite3 - imageformats - build_depends - dub2nix --- .../D-YAML/examples/constructor/dub.json | 10 + .../D-YAML/examples/constructor/input.yaml | 8 + src/ext_depends/D-YAML/examples/constructor/main.d | 91 ++++++ .../D-YAML/examples/getting_started/dub.json | 10 + .../D-YAML/examples/getting_started/input.yaml | 2 + .../D-YAML/examples/getting_started/main.d | 18 ++ .../D-YAML/examples/representer/dub.json | 10 + src/ext_depends/D-YAML/examples/representer/main.d | 43 +++ src/ext_depends/D-YAML/examples/resolver/dub.json | 10 + .../D-YAML/examples/resolver/input.yaml | 8 + src/ext_depends/D-YAML/examples/resolver/main.d | 36 +++ src/ext_depends/D-YAML/examples/tojson/dub.json | 8 + .../D-YAML/examples/tojson/source/app.d | 54 ++++ .../D-YAML/examples/yaml_bench/dub.json | 10 + .../D-YAML/examples/yaml_bench/yaml_bench.d | 179 ++++++++++++ .../D-YAML/examples/yaml_gen/config.yaml | 46 +++ src/ext_depends/D-YAML/examples/yaml_gen/dub.json | 10 + .../D-YAML/examples/yaml_gen/yaml_gen.d | 316 +++++++++++++++++++++ .../D-YAML/examples/yaml_stats/dub.json | 10 + .../D-YAML/examples/yaml_stats/small.yaml | 4 + .../D-YAML/examples/yaml_stats/yaml_stats.d | 106 +++++++ 21 files changed, 989 insertions(+) create mode 100644 src/ext_depends/D-YAML/examples/constructor/dub.json create mode 100644 src/ext_depends/D-YAML/examples/constructor/input.yaml create mode 100644 src/ext_depends/D-YAML/examples/constructor/main.d create mode 100644 src/ext_depends/D-YAML/examples/getting_started/dub.json create mode 100644 src/ext_depends/D-YAML/examples/getting_started/input.yaml create mode 100644 src/ext_depends/D-YAML/examples/getting_started/main.d create mode 100644 src/ext_depends/D-YAML/examples/representer/dub.json create mode 100644 src/ext_depends/D-YAML/examples/representer/main.d create mode 100644 src/ext_depends/D-YAML/examples/resolver/dub.json create mode 100644 src/ext_depends/D-YAML/examples/resolver/input.yaml create mode 100644 src/ext_depends/D-YAML/examples/resolver/main.d create mode 100644 src/ext_depends/D-YAML/examples/tojson/dub.json create mode 100644 src/ext_depends/D-YAML/examples/tojson/source/app.d create mode 100644 src/ext_depends/D-YAML/examples/yaml_bench/dub.json create mode 100644 src/ext_depends/D-YAML/examples/yaml_bench/yaml_bench.d create mode 100644 src/ext_depends/D-YAML/examples/yaml_gen/config.yaml create mode 100644 src/ext_depends/D-YAML/examples/yaml_gen/dub.json create mode 100644 src/ext_depends/D-YAML/examples/yaml_gen/yaml_gen.d create mode 100644 src/ext_depends/D-YAML/examples/yaml_stats/dub.json create mode 100644 src/ext_depends/D-YAML/examples/yaml_stats/small.yaml create mode 100644 src/ext_depends/D-YAML/examples/yaml_stats/yaml_stats.d (limited to 'src/ext_depends/D-YAML/examples') diff --git a/src/ext_depends/D-YAML/examples/constructor/dub.json b/src/ext_depends/D-YAML/examples/constructor/dub.json new file mode 100644 index 0000000..1aca4ee --- /dev/null +++ b/src/ext_depends/D-YAML/examples/constructor/dub.json @@ -0,0 +1,10 @@ +{ + "name": "constructor", + "targetType": "executable", + "sourceFiles": ["main.d"], + "mainSourceFile": "main.d", + "dependencies": + { + "dyaml": { "version" : "*"} + } +} diff --git a/src/ext_depends/D-YAML/examples/constructor/input.yaml b/src/ext_depends/D-YAML/examples/constructor/input.yaml new file mode 100644 index 0000000..8d56926 --- /dev/null +++ b/src/ext_depends/D-YAML/examples/constructor/input.yaml @@ -0,0 +1,8 @@ +scalar-red: !color FF0000 +scalar-orange: !color FFFF00 +mapping-red: !color-mapping {r: 255, g: 0, b: 0} +mapping-orange: + !color-mapping + r: 255 + g: 255 + b: 0 diff --git a/src/ext_depends/D-YAML/examples/constructor/main.d b/src/ext_depends/D-YAML/examples/constructor/main.d new file mode 100644 index 0000000..50826f1 --- /dev/null +++ b/src/ext_depends/D-YAML/examples/constructor/main.d @@ -0,0 +1,91 @@ +import std.stdio; +import std.string; +import dyaml; + +struct Color +{ + ubyte red; + ubyte green; + ubyte blue; + + this(ubyte r, ubyte g, ubyte b) @safe + { + red = r; + green = g; + blue = b; + } + + this(const Node node, string tag) @safe + { + if (tag == "!color-mapping") + { + //Will throw if a value is missing, is not an integer, or is out of range. + red = node["r"].as!ubyte; + green = node["g"].as!ubyte; + blue = node["b"].as!ubyte; + } + else + { + string value = node.as!string; + + if(value.length != 6) + { + throw new Exception("Invalid color: " ~ value); + } + //We don't need to check for uppercase chars this way. + value = value.toLower(); + + //Get value of a hex digit. + uint hex(char c) + { + import std.ascii; + if(!std.ascii.isHexDigit(c)) + { + throw new Exception("Invalid color: " ~ value); + } + + if(std.ascii.isDigit(c)) + { + return c - '0'; + } + return c - 'a' + 10; + } + + red = cast(ubyte)(16 * hex(value[0]) + hex(value[1])); + green = cast(ubyte)(16 * hex(value[2]) + hex(value[3])); + blue = cast(ubyte)(16 * hex(value[4]) + hex(value[5])); + } + } +} + +void main(string[] args) +{ + auto red = Color(255, 0, 0); + auto orange = Color(255, 255, 0); + + string path = "input.yaml"; + if (args.length > 1) + { + path = args[1]; + } + + try + { + auto root = Loader.fromFile(path).load(); + + if(root["scalar-red"].as!Color == red && + root["mapping-red"].as!Color == red && + root["scalar-orange"].as!Color == orange && + root["mapping-orange"].as!Color == orange) + { + writeln("SUCCESS"); + return; + } + } + catch(YAMLException e) + { + writeln(e.msg); + } + + writeln("FAILURE"); +} diff --git a/src/ext_depends/D-YAML/examples/getting_started/dub.json b/src/ext_depends/D-YAML/examples/getting_started/dub.json new file mode 100644 index 0000000..844c006 --- /dev/null +++ b/src/ext_depends/D-YAML/examples/getting_started/dub.json @@ -0,0 +1,10 @@ +{ + "name": "getting-started", + "targetType": "executable", + "sourceFiles": ["main.d"], + "mainSourceFile": "main.d", + "dependencies": + { + "dyaml": { "version" : "*" } + } +} diff --git a/src/ext_depends/D-YAML/examples/getting_started/input.yaml b/src/ext_depends/D-YAML/examples/getting_started/input.yaml new file mode 100644 index 0000000..8b093ae --- /dev/null +++ b/src/ext_depends/D-YAML/examples/getting_started/input.yaml @@ -0,0 +1,2 @@ +Hello World : [Hello, World] +Answer : 42 diff --git a/src/ext_depends/D-YAML/examples/getting_started/main.d b/src/ext_depends/D-YAML/examples/getting_started/main.d new file mode 100644 index 0000000..2cc242c --- /dev/null +++ b/src/ext_depends/D-YAML/examples/getting_started/main.d @@ -0,0 +1,18 @@ +import std.stdio; +import dyaml; + +void main() +{ + //Read the input. + Node root = Loader.fromFile("input.yaml").load(); + + //Display the data read. + foreach(string word; root["Hello World"]) + { + writeln(word); + } + writeln("The answer is ", root["Answer"].as!int); + + //Dump the loaded document to output.yaml. + dumper().dump(File("output.yaml", "w").lockingTextWriter, root); +} diff --git a/src/ext_depends/D-YAML/examples/representer/dub.json b/src/ext_depends/D-YAML/examples/representer/dub.json new file mode 100644 index 0000000..c46c42f --- /dev/null +++ b/src/ext_depends/D-YAML/examples/representer/dub.json @@ -0,0 +1,10 @@ +{ + "name": "representer", + "targetType": "executable", + "sourceFiles": ["main.d"], + "mainSourceFile": "main.d", + "dependencies": + { + "dyaml": { "version" : "*" } + } +} diff --git a/src/ext_depends/D-YAML/examples/representer/main.d b/src/ext_depends/D-YAML/examples/representer/main.d new file mode 100644 index 0000000..18ec4c5 --- /dev/null +++ b/src/ext_depends/D-YAML/examples/representer/main.d @@ -0,0 +1,43 @@ +import std.stdio; +import dyaml; + +struct Color +{ + ubyte red; + ubyte green; + ubyte blue; + + Node opCast(T: Node)() const + { + static immutable hex = "0123456789ABCDEF"; + + //Using the color format from the Constructor example. + string scalar; + foreach(channel; [red, green, blue]) + { + scalar ~= hex[channel / 16]; + scalar ~= hex[channel % 16]; + } + + //Representing as a scalar, with custom tag to specify this data type. + return Node(scalar, "!color"); + } +} + +void main() +{ + try + { + auto dumper = dumper(); + + auto document = Node([Color(255, 0, 0), + Color(0, 255, 0), + Color(0, 0, 255)]); + + dumper.dump(File("output.yaml", "w").lockingTextWriter, document); + } + catch(YAMLException e) + { + writeln(e.msg); + } +} diff --git a/src/ext_depends/D-YAML/examples/resolver/dub.json b/src/ext_depends/D-YAML/examples/resolver/dub.json new file mode 100644 index 0000000..30b55d8 --- /dev/null +++ b/src/ext_depends/D-YAML/examples/resolver/dub.json @@ -0,0 +1,10 @@ +{ + "name": "resolver", + "targetType": "executable", + "sourceFiles": ["main.d"], + "mainSourceFile": "main.d", + "dependencies": + { + "dyaml": { "version" : "*" } + } +} diff --git a/src/ext_depends/D-YAML/examples/resolver/input.yaml b/src/ext_depends/D-YAML/examples/resolver/input.yaml new file mode 100644 index 0000000..99b9a5a --- /dev/null +++ b/src/ext_depends/D-YAML/examples/resolver/input.yaml @@ -0,0 +1,8 @@ +scalar-red: FF0000 +scalar-orange: FFFF00 +mapping-red: !color-mapping {r: 255, g: 0, b: 0} +mapping-orange: + !color-mapping + r: 255 + g: 255 + b: 0 diff --git a/src/ext_depends/D-YAML/examples/resolver/main.d b/src/ext_depends/D-YAML/examples/resolver/main.d new file mode 100644 index 0000000..f3c08a1 --- /dev/null +++ b/src/ext_depends/D-YAML/examples/resolver/main.d @@ -0,0 +1,36 @@ +import std.regex; +import std.stdio; +import dyaml; + +int main(string[] args) +{ + string path = "input.yaml"; + if (args.length > 1) + { + path = args[1]; + } + + try + { + + auto loader = Loader.fromFile("input.yaml"); + loader.resolver.addImplicitResolver("!color", regex("[0-9a-fA-F]{6}"), + "0123456789abcdefABCDEF"); + + auto root = loader.load(); + + if(root["scalar-red"].tag == "!color" && + root["scalar-orange"].tag == "!color") + { + writeln("SUCCESS"); + return 0; + } + } + catch(YAMLException e) + { + writeln(e.msg); + } + + writeln("FAILURE"); + return 1; +} diff --git a/src/ext_depends/D-YAML/examples/tojson/dub.json b/src/ext_depends/D-YAML/examples/tojson/dub.json new file mode 100644 index 0000000..ba014b8 --- /dev/null +++ b/src/ext_depends/D-YAML/examples/tojson/dub.json @@ -0,0 +1,8 @@ +{ + "name": "tojson", + "targetType": "executable", + "dependencies": + { + "dyaml": "*" + } +} diff --git a/src/ext_depends/D-YAML/examples/tojson/source/app.d b/src/ext_depends/D-YAML/examples/tojson/source/app.d new file mode 100644 index 0000000..654274f --- /dev/null +++ b/src/ext_depends/D-YAML/examples/tojson/source/app.d @@ -0,0 +1,54 @@ +module dyaml.tojson; +import std.datetime; +import std.json; +import std.stdio; +import dyaml; + +void main() +{ + auto doc = Loader.fromFile(stdin).load(); + auto json = doc.toJSON; + writeln(json.toPrettyString); +} + +JSONValue toJSON(Node node) +{ + JSONValue output; + final switch (node.type) + { + case NodeType.sequence: + output = JSONValue(string[].init); + foreach (Node seqNode; node) + { + output.array ~= seqNode.toJSON(); + } + break; + case NodeType.mapping: + output = JSONValue(string[string].init); + foreach (Node keyNode, Node valueNode; node) + { + output[keyNode.as!string] = valueNode.toJSON(); + } + break; + case NodeType.string: + output = node.as!string; + break; + case NodeType.integer: + output = node.as!long; + break; + case NodeType.decimal: + output = node.as!real; + break; + case NodeType.boolean: + output = node.as!bool; + break; + case NodeType.timestamp: + output = node.as!SysTime.toISOExtString(); + break; + case NodeType.merge: + case NodeType.null_: + case NodeType.binary: + case NodeType.invalid: + } + return output; +} diff --git a/src/ext_depends/D-YAML/examples/yaml_bench/dub.json b/src/ext_depends/D-YAML/examples/yaml_bench/dub.json new file mode 100644 index 0000000..b4e778f --- /dev/null +++ b/src/ext_depends/D-YAML/examples/yaml_bench/dub.json @@ -0,0 +1,10 @@ +{ + "name": "benchmark", + "targetType": "executable", + "sourceFiles": ["yaml_bench.d"], + "mainSourceFile": "yaml_bench.d", + "dependencies": + { + "dyaml": { "version" : "*" } + } +} diff --git a/src/ext_depends/D-YAML/examples/yaml_bench/yaml_bench.d b/src/ext_depends/D-YAML/examples/yaml_bench/yaml_bench.d new file mode 100644 index 0000000..bb8446e --- /dev/null +++ b/src/ext_depends/D-YAML/examples/yaml_bench/yaml_bench.d @@ -0,0 +1,179 @@ + +module dyaml.yaml_bench; +//Benchmark that loads, and optionally extracts data from and/or emits a YAML file. + +import std.algorithm; +import std.conv; +import std.datetime.systime; +import std.datetime.stopwatch; +import std.file; +import std.getopt; +import std.range; +import std.stdio; +import std.string; +import dyaml; + +///Get data out of every node. +void extract(ref Node document) @safe +{ + void crawl(ref Node root) @safe + { + final switch (root.nodeID) + { + case NodeID.scalar: + switch(root.tag) + { + case "tag:yaml.org,2002:null": auto value = root.as!YAMLNull; break; + case "tag:yaml.org,2002:bool": auto value = root.as!bool; break; + case "tag:yaml.org,2002:int": auto value = root.as!long; break; + case "tag:yaml.org,2002:float": auto value = root.as!real; break; + case "tag:yaml.org,2002:binary": auto value = root.as!(ubyte[]); break; + case "tag:yaml.org,2002:timestamp": auto value = root.as!SysTime; break; + case "tag:yaml.org,2002:str": auto value = root.as!string; break; + default: writeln("Unrecognozed tag: ", root.tag); + } + break; + case NodeID.sequence: + foreach(ref Node node; root) + { + crawl(node); + } + break; + case NodeID.mapping: + foreach(ref Node key, ref Node value; root) + { + crawl(key); + crawl(value); + } + break; + case NodeID.invalid: + assert(0); + } + } + + crawl(document); +} + +void main(string[] args) //@safe +{ + import std.array : array; + bool get = false; + bool dump = false; + bool reload = false; + bool quiet = false; + bool verbose = false; + bool scanOnly = false; + uint runs = 1; + + auto help = getopt( + args, + "get|g", "Extract data from the file (using Node.as()).", &get, + "dump|d", "Dump the loaded data (to YAML_FILE.dump).", &dump, + "runs|r", "Repeat parsing the file NUM times.", &runs, + "reload", "Reload the file from the diskl on every repeat By default,"~ + " the file is loaded to memory once and repeatedly parsed from memory.", &reload, + "quiet|q", "Don't print anything.", &quiet, + "verbose|v", "Print even more.", &verbose, + "scan-only|s", "Do not execute the entire parsing process, only scanning. Overrides '--dump'", &scanOnly + ); + + if (help.helpWanted || (args.length < 2)) + { + defaultGetoptPrinter( + "D:YAML benchmark\n"~ + "Copyright (C) 2011-2018 Ferdinand Majerech, Cameron \"Herringway\" Ross\n"~ + "Usage: yaml_bench [OPTION ...] [YAML_FILE]\n\n"~ + "Loads and optionally extracts data and/or dumps a YAML file.\n", + help.options + ); + return; + } + + string file = args[1]; + + auto stopWatch = StopWatch(AutoStart.yes); + void[] fileInMemory; + if(!reload) { fileInMemory = std.file.read(file); } + void[] fileWorkingCopy = fileInMemory.dup; + auto loadTime = stopWatch.peek(); + stopWatch.reset(); + try + { + // Instead of constructing a resolver/constructor with each Loader, + // construct them once to remove noise when profiling. + auto resolver = Resolver.withDefaultResolvers; + + auto constructTime = stopWatch.peek(); + + Node[] nodes; + + void runLoaderBenchmark() //@safe + { + // Loading the file rewrites the loaded buffer, so if we don't reload from + // disk, we need to use a copy of the originally loaded file. + if(reload) { fileInMemory = std.file.read(file); } + else { fileWorkingCopy[] = fileInMemory[]; } + void[] fileToLoad = reload ? fileInMemory : fileWorkingCopy; + + auto loader = Loader.fromBuffer(fileToLoad); + if(scanOnly) + { + loader.scanBench(); + return; + } + + loader.resolver = resolver; + nodes = loader.array; + } + void runDumpBenchmark() @safe + { + if(dump) + { + dumper().dump(File(file ~ ".dump", "w").lockingTextWriter, nodes); + } + } + void runGetBenchmark() @safe + { + if(get) foreach(ref node; nodes) + { + extract(node); + } + } + auto totalTime = benchmark!(runLoaderBenchmark, runDumpBenchmark, runGetBenchmark)(runs); + if (!quiet) + { + auto enabledOptions = + only( + get ? "Get" : "", + dump ? "Dump" : "", + reload ? "Reload" : "", + scanOnly ? "Scan Only": "" + ).filter!(x => x != ""); + if (!enabledOptions.empty) + { + writefln!"Options enabled: %-(%s, %)"(enabledOptions); + } + if (verbose) + { + if (!reload) + { + writeln("Time to load file: ", loadTime); + } + writeln("Time to set up resolver: ", constructTime); + } + writeln("Runs: ", runs); + foreach(time, func, enabled; lockstep(totalTime[], only("Loader", "Dumper", "Get"), only(true, dump, get))) + { + if (enabled) + { + writeln("Average time spent on ", func, ": ", time / runs); + writeln("Total time spent on ", func, ": ", time); + } + } + } + } + catch(YAMLException e) + { + writeln("ERROR: ", e.msg); + } +} diff --git a/src/ext_depends/D-YAML/examples/yaml_gen/config.yaml b/src/ext_depends/D-YAML/examples/yaml_gen/config.yaml new file mode 100644 index 0000000..5484686 --- /dev/null +++ b/src/ext_depends/D-YAML/examples/yaml_gen/config.yaml @@ -0,0 +1,46 @@ +root-type: map +documents: 2 +complex-keys: false +collection-keys: false +min-nodes-per-document: 4096 +encoding: utf-8 +indent: 4 +text-width: 40 + +#Note: setting collection probabilities too high can lead to stack overflow as +#we end up with extremely deeply nested structures + +string: + probability: 20 + alphabet: " abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_0123456789ábćčďéěǵǧȟíǐǰḱǩĺľḿńňóǒôäṕŕřśšť" + range: {min: 1, max: 40, dist: cubic} +int: + probability: 10 + range: {min: -10000000, max: 10000000, dist: linear} +float: + probability: 10 + range: {min: -10000000.0, max: 10000000.0, dist: linear} +bool: + probability: 10 +timestamp: + probability: 10 + round-chance: 0.9 + range: {min: 0, max: 1231200000000000000, dist: linear} +binary: + probability: 4 + range: {min: 1, max: 400, dist: quadratic} +map: + probability: 2 + range: {min: 1, max: 20, dist: cubic} +omap: + probability: 1 + range: {min: 1, max: 20, dist: cubic} +pairs: + probability: 1 + range: {min: 1, max: 20, dist: cubic} +seq: + probability: 2 + range: {min: 1, max: 20, dist: cubic} +set: + probability: 1 + range: {min: 1, max: 20, dist: cubic} diff --git a/src/ext_depends/D-YAML/examples/yaml_gen/dub.json b/src/ext_depends/D-YAML/examples/yaml_gen/dub.json new file mode 100644 index 0000000..1f5a2d1 --- /dev/null +++ b/src/ext_depends/D-YAML/examples/yaml_gen/dub.json @@ -0,0 +1,10 @@ +{ + "name": "yaml_gen", + "targetType": "executable", + "sourceFiles": ["yaml_gen.d"], + "mainSourceFile": "yaml_gen.d", + "dependencies": + { + "dyaml": { "version" : "*" } + } +} diff --git a/src/ext_depends/D-YAML/examples/yaml_gen/yaml_gen.d b/src/ext_depends/D-YAML/examples/yaml_gen/yaml_gen.d new file mode 100644 index 0000000..b970c31 --- /dev/null +++ b/src/ext_depends/D-YAML/examples/yaml_gen/yaml_gen.d @@ -0,0 +1,316 @@ + +///Random YAML generator. Used to generate benchmarking inputs. + +import std.algorithm; +import std.conv; +import std.datetime; +import std.math; +import std.random; +import std.stdio; +import std.string; +import dyaml; + + +Node config; +Node function(bool)[string] generators; +auto typesScalar = ["string", "int", "float", "bool", "timestamp", "binary"]; +auto typesScalarKey = ["string", "int", "float", "timestamp"]; +auto typesCollection = ["map","omap", "pairs", "seq", "set"]; +ulong minNodesDocument; +ulong totalNodes; + +static this() +{ + generators["string"] = &genString; + generators["int"] = &genInt; + generators["float"] = &genFloat; + generators["bool"] = &genBool; + generators["timestamp"] = &genTimestamp; + generators["binary"] = &genBinary; + generators["map"] = &genMap; + generators["omap"] = &genOmap; + generators["pairs"] = &genPairs; + generators["seq"] = &genSeq; + generators["set"] = &genSet; +} + +real randomNormalized(const string distribution = "linear") +{ + auto generator = Random(unpredictableSeed()); + const r = uniform!"[]"(0.0L, 1.0L, generator); + switch(distribution) + { + case "linear": + return r; + case "quadratic": + return r * r; + case "cubic": + return r * r * r; + default: + writeln("Unknown random distribution: ", distribution, + ", falling back to linear"); + return randomNormalized("linear"); + } +} + +long randomLong(const long min, const long max, const string distribution = "linear") +{ + return min + cast(long)round((max - min) * randomNormalized(distribution)); +} + +real randomReal(const real min, const real max, const string distribution = "linear") +{ + return min + (max - min) * randomNormalized(distribution); +} + +dchar randomChar(const dstring chars) +{ + return chars[randomLong(0, chars.length - 1)]; +} + +string randomType(string[] types) +{ + auto probabilities = new uint[types.length]; + foreach(index, type; types) + { + probabilities[index] = config[type]["probability"].as!uint; + } + return types[dice(probabilities)]; +} + +Node genString(bool root = false) +{ + auto range = config["string"]["range"]; + + auto alphabet = config["string"]["alphabet"].as!dstring; + + const chars = randomLong(range["min"].as!uint, range["max"].as!uint, + range["dist"].as!string); + + dchar[] result = new dchar[chars]; + result[0] = randomChar(alphabet); + foreach(i; 1 .. chars) + { + result[i] = randomChar(alphabet); + } + + return Node(result.to!string); +} + +Node genInt(bool root = false) +{ + auto range = config["int"]["range"]; + + const result = randomLong(range["min"].as!int, range["max"].as!int, + range["dist"].as!string); + + return Node(result); +} + +Node genFloat(bool root = false) +{ + auto range = config["float"]["range"]; + + const result = randomReal(range["min"].as!real, range["max"].as!real, + range["dist"].as!string); + + return Node(result); +} + +Node genBool(bool root = false) +{ + return Node([true, false][randomLong(0, 1)]); +} + +Node genTimestamp(bool root = false) +{ + auto range = config["timestamp"]["range"]; + + auto hnsecs = randomLong(range["min"].as!ulong, range["max"].as!ulong, + range["dist"].as!string); + + if(randomNormalized() <= config["timestamp"]["round-chance"].as!real) + { + hnsecs -= hnsecs % 10000000; + } + + return Node(SysTime(hnsecs)); +} + +Node genBinary(bool root = false) +{ + auto range = config["binary"]["range"]; + + const bytes = randomLong(range["min"].as!uint, range["max"].as!uint, + range["dist"].as!string); + + ubyte[] result = new ubyte[bytes]; + foreach(i; 0 .. bytes) + { + result[i] = cast(ubyte)randomLong(0, 255); + } + + return Node(result); +} + +Node nodes(const bool root, Node range, const string tag, const bool set = false) +{ + auto types = config["collection-keys"].as!bool ? typesCollection : []; + types ~= (set ? typesScalarKey : typesScalar); + + Node[] nodes; + if(root) + { + while(!(totalNodes >= minNodesDocument)) + { + nodes.assumeSafeAppend; + nodes ~= generateNode(randomType(types)); + } + } + else + { + const elems = randomLong(range["min"].as!uint, range["max"].as!uint, + range["dist"].as!string); + + nodes = new Node[elems]; + foreach(i; 0 .. elems) + { + nodes[i] = generateNode(randomType(types)); + } + } + + return Node(nodes, tag); +} + +Node genSeq(bool root = false) +{ + return nodes(root, config["seq"]["range"], "tag:yaml.org,2002:seq"); +} + +Node genSet(bool root = false) +{ + return nodes(root, config["seq"]["range"], "tag:yaml.org,2002:set", true); +} + +Node pairs(bool root, bool complex, Node range, string tag) +{ + Node[] keys, values; + + if(root) + { + while(!(totalNodes >= minNodesDocument)) + { + const key = generateNode(randomType(typesScalarKey ~ (complex ? typesCollection : []))); + // Maps can't contain duplicate keys + if(tag.endsWith("map") && keys.canFind(key)) { continue; } + keys.assumeSafeAppend; + values.assumeSafeAppend; + keys ~= key; + values ~= generateNode(randomType(typesScalar ~ typesCollection)); + } + } + else + { + const pairs = randomLong(range["min"].as!uint, range["max"].as!uint, + range["dist"].as!string); + + keys = new Node[pairs]; + values = new Node[pairs]; + outer: foreach(i; 0 .. pairs) + { + auto key = generateNode(randomType(typesScalarKey ~ (complex ? typesCollection : []))); + // Maps can't contain duplicate keys + while(tag.endsWith("map") && keys[0 .. i].canFind(key)) + { + key = generateNode(randomType(typesScalarKey ~ (complex ? typesCollection : []))); + } + keys[i] = key; + values[i] = generateNode(randomType(typesScalar ~ typesCollection)); + } + } + + return Node(keys, values, tag); +} + +Node genMap(bool root = false) +{ + Node range = config["map"]["range"]; + const complex = config["complex-keys"].as!bool; + + return pairs(root, complex, range, "tag:yaml.org,2002:map"); +} + +Node genOmap(bool root = false) +{ + Node range = config["omap"]["range"]; + const complex = config["complex-keys"].as!bool; + + return pairs(root, complex, range, "tag:yaml.org,2002:omap"); +} + +Node genPairs(bool root = false) +{ + Node range = config["pairs"]["range"]; + const complex = config["complex-keys"].as!bool; + + return pairs(root, complex, range, "tag:yaml.org,2002:pairs"); +} + +Node generateNode(const string type, bool root = false) +{ + ++totalNodes; + return generators[type](root); +} + +Node[] generate(const string configFileName) +{ + config = Loader.fromFile(configFileName).load(); + + minNodesDocument = config["min-nodes-per-document"].as!long; + + Node[] result; + foreach(i; 0 .. config["documents"].as!uint) + { + result ~= generateNode(config["root-type"].as!string, true); + totalNodes = 0; + } + + return result; +} + + +void main(string[] args) +{ + //Help message. + if(args.length == 1) + { + writeln("Usage: yaml_gen FILE [CONFIG_FILE]\n"); + writeln("Generates a random YAML file and writes it to FILE."); + writeln("If provided, CONFIG_FILE overrides the default config file."); + return; + } + + string configFile = args.length >= 3 ? args[2] : "config.yaml"; + + try + { + //Generate and dump the nodes. + Node[] generated = generate(configFile); + + auto dumper = dumper(); + auto encoding = config["encoding"]; + + dumper.indent = config["indent"].as!uint; + dumper.textWidth = config["text-width"].as!uint; + switch(encoding.as!string) + { + case "utf-16": dumper.dump!wchar(File(args[1], "w").lockingTextWriter, generated); break; + case "utf-32": dumper.dump!dchar(File(args[1], "w").lockingTextWriter, generated); break; + default: dumper.dump!char(File(args[1], "w").lockingTextWriter, generated); break; + } + } + catch(YAMLException e) + { + writeln("ERROR: ", e.msg); + } +} diff --git a/src/ext_depends/D-YAML/examples/yaml_stats/dub.json b/src/ext_depends/D-YAML/examples/yaml_stats/dub.json new file mode 100644 index 0000000..c86f091 --- /dev/null +++ b/src/ext_depends/D-YAML/examples/yaml_stats/dub.json @@ -0,0 +1,10 @@ +{ + "name": "yaml_stats", + "targetType": "executable", + "sourceFiles": ["yaml_stats.d"], + "mainSourceFile": "yaml_stats.d", + "dependencies": + { + "dyaml": { "version" : "*" } + } +} diff --git a/src/ext_depends/D-YAML/examples/yaml_stats/small.yaml b/src/ext_depends/D-YAML/examples/yaml_stats/small.yaml new file mode 100644 index 0000000..4f5c0ea --- /dev/null +++ b/src/ext_depends/D-YAML/examples/yaml_stats/small.yaml @@ -0,0 +1,4 @@ +- 1 +- 2 : 'a' + 3 : 'b' +- 4 : [1.0, 2.1, 3.2] diff --git a/src/ext_depends/D-YAML/examples/yaml_stats/yaml_stats.d b/src/ext_depends/D-YAML/examples/yaml_stats/yaml_stats.d new file mode 100644 index 0000000..b3f6c17 --- /dev/null +++ b/src/ext_depends/D-YAML/examples/yaml_stats/yaml_stats.d @@ -0,0 +1,106 @@ + +///Example D:YAML application that displays statistics about YAML documents. + +import std.stdio; +import std.string; +import dyaml; + + +///Collects statistics about a YAML document and returns them as string. +string statistics(ref Node document) +{ + size_t nodes; + size_t scalars, sequences, mappings; + size_t seqItems, mapPairs; + + size_t[string] tags; + + void crawl(ref Node root) + { + ++nodes; + if((root.tag in tags) is null) + { + tags[root.tag] = 0; + } + ++tags[root.tag]; + final switch (root.nodeID) + { + case NodeID.scalar: + ++scalars; + return; + case NodeID.sequence: + ++sequences; + seqItems += root.length; + foreach(ref Node node; root) + { + crawl(node); + } + return; + case NodeID.mapping: + ++mappings; + mapPairs += root.length; + foreach(ref Node key, ref Node value; root) + { + crawl(key); + crawl(value); + } + return; + case NodeID.invalid: + assert(0); + } + } + + crawl(document); + + string tagStats = "\nTag statistics:\n"; + foreach(tag, count; tags) + { + tagStats ~= format("\n%s : %s", tag, count); + } + + return format( "\nNodes: %s" ~ + "\n\nScalars: %s" ~ + "\nSequences: %s" ~ + "\nMappings: %s" ~ + "\n\nAverage sequence length: %s" ~ + "\nAverage mapping length: %s" ~ + "\n\n%s", + nodes, scalars, sequences, mappings, + sequences == 0.0 ? 0.0 : cast(real)seqItems / sequences, + mappings == 0.0 ? 0.0 : cast(real)mapPairs / mappings, + tagStats); +} + +void main(string[] args) +{ + //Help message + if(args.length == 1) + { + writeln("Usage: yaml_stats [YAML_FILE ...]\n"); + writeln("Analyzes YAML files with provided filenames and displays statistics."); + return; + } + + //Print stats about every document in every file. + foreach(file; args[1 .. $]) + { + writeln("\nFile ", file); + writeln("------------------------------------------------------------"); + try + { + auto loader = Loader.fromFile(file); + + size_t idx = 0; + foreach(ref document; loader) + { + writeln("\nDocument ", idx++); + writeln("----------------------------------------"); + writeln(statistics(document)); + } + } + catch(YAMLException e) + { + writeln("ERROR: ", e.msg); + } + } +} -- cgit v1.2.3