1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
|
// Copyright Ferdinand Majerech 2011-2014.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
/// YAML tokens.
/// Code based on PyYAML: http://www.pyyaml.org
module dyaml.token;
import std.conv;
import dyaml.encoding;
import dyaml.exception;
import dyaml.reader;
import dyaml.style;
package:
/// Token types.
enum TokenID : ubyte
{
// Invalid (uninitialized) token
invalid = 0,
directive,
documentStart,
documentEnd,
streamStart,
streamEnd,
blockSequenceStart,
blockMappingStart,
blockEnd,
flowSequenceStart,
flowMappingStart,
flowSequenceEnd,
flowMappingEnd,
key,
value,
blockEntry,
flowEntry,
alias_,
anchor,
tag,
scalar
}
/// Specifies the type of a tag directive token.
enum DirectiveType : ubyte
{
// YAML version directive.
yaml,
// Tag directive.
tag,
// Any other directive is "reserved" for future YAML versions.
reserved
}
/// Token produced by scanner.
///
/// 32 bytes on 64-bit.
struct Token
{
@disable int opCmp(ref Token);
// 16B
/// Value of the token, if any.
///
/// Values are char[] instead of string, as Parser may still change them in a few
/// cases. Parser casts values to strings when producing Events.
char[] value;
// 4B
/// Start position of the token in file/stream.
Mark startMark;
// 4B
/// End position of the token in file/stream.
Mark endMark;
// 1B
/// Token type.
TokenID id;
// 1B
/// Style of scalar token, if this is a scalar token.
ScalarStyle style;
// 1B
/// Encoding, if this is a stream start token.
Encoding encoding;
// 1B
/// Type of directive for directiveToken.
DirectiveType directive;
// 4B
/// Used to split value into 2 substrings for tokens that need 2 values (tagToken)
uint valueDivider;
/// Get string representation of the token ID.
@property string idString() @safe pure const {return id.to!string;}
}
/// Construct a directive token.
///
/// Params: start = Start position of the token.
/// end = End position of the token.
/// value = Value of the token.
/// directive = Directive type (YAML or TAG in YAML 1.1).
/// nameEnd = Position of the end of the name
Token directiveToken(const Mark start, const Mark end, char[] value,
DirectiveType directive, const uint nameEnd) @safe pure nothrow @nogc
{
return Token(value, start, end, TokenID.directive, ScalarStyle.init, Encoding.init,
directive, nameEnd);
}
/// Construct a simple (no value) token with specified type.
///
/// Params: id = Type of the token.
/// start = Start position of the token.
/// end = End position of the token.
Token simpleToken(TokenID id)(const Mark start, const Mark end)
{
return Token(null, start, end, id);
}
/// Construct a stream start token.
///
/// Params: start = Start position of the token.
/// end = End position of the token.
/// encoding = Encoding of the stream.
Token streamStartToken(const Mark start, const Mark end, const Encoding encoding) @safe pure nothrow @nogc
{
return Token(null, start, end, TokenID.streamStart, ScalarStyle.invalid, encoding);
}
/// Aliases for construction of simple token types.
alias streamEndToken = simpleToken!(TokenID.streamEnd);
alias blockSequenceStartToken = simpleToken!(TokenID.blockSequenceStart);
alias blockMappingStartToken = simpleToken!(TokenID.blockMappingStart);
alias blockEndToken = simpleToken!(TokenID.blockEnd);
alias keyToken = simpleToken!(TokenID.key);
alias valueToken = simpleToken!(TokenID.value);
alias blockEntryToken = simpleToken!(TokenID.blockEntry);
alias flowEntryToken = simpleToken!(TokenID.flowEntry);
/// Construct a simple token with value with specified type.
///
/// Params: id = Type of the token.
/// start = Start position of the token.
/// end = End position of the token.
/// value = Value of the token.
/// valueDivider = A hack for TagToken to store 2 values in value; the first
/// value goes up to valueDivider, the second after it.
Token simpleValueToken(TokenID id)(const Mark start, const Mark end, char[] value,
const uint valueDivider = uint.max)
{
return Token(value, start, end, id, ScalarStyle.invalid, Encoding.init,
DirectiveType.init, valueDivider);
}
/// Alias for construction of tag token.
alias tagToken = simpleValueToken!(TokenID.tag);
alias aliasToken = simpleValueToken!(TokenID.alias_);
alias anchorToken = simpleValueToken!(TokenID.anchor);
/// Construct a scalar token.
///
/// Params: start = Start position of the token.
/// end = End position of the token.
/// value = Value of the token.
/// style = Style of the token.
Token scalarToken(const Mark start, const Mark end, char[] value, const ScalarStyle style) @safe pure nothrow @nogc
{
return Token(value, start, end, TokenID.scalar, style);
}
|
