gered/tree-sitter-toml
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

feat: flatten table/table-array structure (#2)

Browse source 
Input:

```toml
[table]
key = "value"
```

Output:

```sh
# before
(root
  (table
    (key)
    (pair (key) (string)))

# after
(root
  (table (key))
  (pair (key) (string)))
```
This commit is contained in:
Ika 2019-08-20 14:59:19 +08:00 committed by GitHub
parent 43595130ac
commit a3812033af
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 2121 additions and 2749 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

99
corpus/custom.txt
View file

@ -45,11 +45,13 @@ invalid_both = 1 .0 e 2
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
(root
(ERROR (ERROR
(dotted_key (dotted_key
(key) (ERROR) (key) (key) (ERROR) (key)
) )
) )
)
================================================================================ ================================================================================
INVALID - offset date time - whitespaces between its components are not allowed INVALID - offset date time - whitespaces between its components are not allowed
@ -60,9 +62,11 @@ invalid2 = 1979-05-27 T 07:32:00 Z
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
(root
(ERROR (ERROR
(key) (ERROR) (local_time) (local_time) (key) (ERROR) (local_time) (local_time)
) )
)
================================================================================ ================================================================================
INVALID - local date time - whitespaces between its components are not allowed INVALID - local date time - whitespaces between its components are not allowed
@ -125,14 +129,14 @@ key = {
(pair (pair
(key) (key)
(inline_table (inline_table
(MISSING "}") (ERROR)
)
)
(pair (pair
(key) (boolean) (key) (boolean)
) )
(ERROR) (ERROR)
) )
)
)
================================================================================ ================================================================================
INVALID - array of tables - multiline string for header keys are not allowed INVALID - array of tables - multiline string for header keys are not allowed
@ -154,9 +158,96 @@ key
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
(root
(ERROR (ERROR
(ERROR (key) (key)
)
)
================================================================================
VALID - table - empty content separated by comments
================================================================================
[table]
# comment
--------------------------------------------------------------------------------
(root
(table
(key) (key)
) )
(comment)
)
================================================================================
VALID - array of tables - empty content separated by comments
================================================================================
[[array-table]]
# comment
--------------------------------------------------------------------------------
(root
(table_array
(key) (key)
) )
(comment)
)
================================================================================
VALID - table - two pairs separated by comments
================================================================================
[table]
a = 1
# comment
b = 2
--------------------------------------------------------------------------------
(root
(table
(key)
)
(pair
(key) (integer)
)
(comment)
(pair
(key) (integer)
)
)
================================================================================
VALID - array of tables - two pairs separated by comments
================================================================================
[[array-table]]
a = 1
# comment
b = 2
--------------------------------------------------------------------------------
(root
(table_array
(key)
)
(pair
(key) (integer)
)
(comment)
(pair
(key) (integer)
)
)

44
corpus/spec.txt
View file

@ -37,12 +37,8 @@ key = # INVALID
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
(root (root
(pair (ERROR
(key) (key) (comment)
(comment)
(integer
(MISSING "integer_token1")
)
) )
) )
@ -782,15 +778,16 @@ key2 = 456
(root (root
(table (table
(key) (key)
)
(pair (pair
(key) (string) (key) (string)
) )
(pair (pair
(key) (integer) (key) (integer)
) )
)
(table (table
(key) (key)
)
(pair (pair
(key) (string) (key) (string)
) )
@ -798,7 +795,6 @@ key2 = 456
(key) (integer) (key) (integer)
) )
) )
)
================================================================================ ================================================================================
VALID - table - header with dotted key VALID - table - header with dotted key
@ -814,6 +810,7 @@ type.name = "pug"
(dotted_key (dotted_key
(key) (key) (key) (key)
) )
)
(pair (pair
(dotted_key (dotted_key
(key) (key) (key) (key)
@ -821,7 +818,6 @@ type.name = "pug"
(string) (string)
) )
) )
)
================================================================================ ================================================================================
VALID - table- header with whitespaces VALID - table- header with whitespaces
@ -920,17 +916,17 @@ c = 2
(comment) (comment)
(table (table
(key) (key)
)
(pair (pair
(key) (integer) (key) (integer)
) )
)
(table (table
(key) (key)
)
(pair (pair
(key) (integer) (key) (integer)
) )
) )
)
================================================================================ ================================================================================
VALID - table - overlapped header key (semantically INVALID) VALID - table - overlapped header key (semantically INVALID)
@ -950,19 +946,19 @@ c = 2
(comment) (comment)
(table (table
(key) (key)
)
(pair (pair
(key) (integer) (key) (integer)
) )
)
(table (table
(dotted_key (dotted_key
(key) (key) (key) (key)
) )
)
(pair (pair
(key) (integer) (key) (integer)
) )
) )
)
================================================================================ ================================================================================
VALID - inline table - basic VALID - inline table - basic
@ -1030,18 +1026,19 @@ color = "gray"
(root (root
(table_array (table_array
(key) (key)
)
(pair (pair
(key) (string) (key) (string)
) )
(pair (pair
(key) (integer) (key) (integer)
) )
)
(table_array (table_array
(key) (key)
) )
(table_array (table_array
(key) (key)
)
(pair (pair
(key) (string) (key) (string)
) )
@ -1052,7 +1049,6 @@ color = "gray"
(key) (string) (key) (string)
) )
) )
)
================================================================================ ================================================================================
VALID - array of tables - nested arrays of tables VALID - array of tables - nested arrays of tables
@ -1082,52 +1078,52 @@ VALID - array of tables - nested arrays of tables
(root (root
(table_array (table_array
(key) (key)
)
(pair (pair
(key) (string) (key) (string)
) )
)
(table (table
(dotted_key (dotted_key
(key) (key) (key) (key)
) )
)
(pair (pair
(key) (string) (key) (string)
) )
(pair (pair
(key) (string) (key) (string)
) )
)
(table_array (table_array
(dotted_key (dotted_key
(key) (key) (key) (key)
) )
)
(pair (pair
(key) (string) (key) (string)
) )
)
(table_array (table_array
(dotted_key (dotted_key
(key) (key) (key) (key)
) )
)
(pair (pair
(key) (string) (key) (string)
) )
)
(table_array (table_array
(key) (key)
)
(pair (pair
(key) (string) (key) (string)
) )
)
(table_array (table_array
(dotted_key (dotted_key
(key) (key) (key) (key)
) )
)
(pair (pair
(key) (string) (key) (string)
) )
) )
)
================================================================================ ================================================================================
VALID - array of tables - append to statically defined array (semantically INVALID) VALID - array of tables - append to statically defined array (semantically INVALID)
@ -1171,25 +1167,25 @@ VALID - array of tables - append to table (semantically INVALID)
(comment) (comment)
(table_array (table_array
(key) (key)
)
(pair (pair
(key) (string) (key) (string)
) )
)
(table_array (table_array
(dotted_key (dotted_key
(key) (key) (key) (key)
) )
)
(pair (pair
(key) (string) (key) (string)
) )
(comment) (comment)
)
(table (table
(dotted_key (dotted_key
(key) (key) (key) (key)
) )
)
(pair (pair
(key) (string) (key) (string)
) )
) )
)

51
grammar.js
View file

@ -23,38 +23,24 @@ const rfc3339_offset = /([zZ])|([+-]([01][0-9]|2[0-3]):[0-5][0-9])/;
module.exports = grammar({ module.exports = grammar({
name: "toml", name: "toml",
externals: $ => [$._eof], externals: $ => [$._line_ending_or_eof],
extras: $ => [$.comment, /[ \t]/], extras: $ => [$.comment, /[ \t]/],
rules: { rules: {
root: $ => root: $ => repeat(choice($.pair, $.table, $.table_array, $._newline)),
seq(
repeat($._newline),
choice(
$._eof,
seq(
choice($.pair, $._loose_pair, $._pairs, $.table, $.table_array),
repeat(choice($.table, $.table_array))
)
)
),
comment: $ => /#.*/, comment: $ => /#.*/,
_newline: $ => newline, _newline: $ => newline,
_newline_or_eof: $ => choice($._newline, $._eof),
...table_like("table", "[", "]"), table: $ =>
...table_like("table_array", "[[", "]]"), seq("[", choice($.dotted_key, $.key), "]", $._line_ending_or_eof),
pair: $ => seq($._inline_pair, $._newline_or_eof), table_array: $ =>
seq("[[", choice($.dotted_key, $.key), "]]", $._line_ending_or_eof),
pair: $ => seq($._inline_pair, $._line_ending_or_eof),
_inline_pair: $ => seq(choice($.dotted_key, $.key), "=", $._inline_value), _inline_pair: $ => seq(choice($.dotted_key, $.key), "=", $._inline_value),
_loose_pair: $ => seq(choice($._loose_pair, $.pair), $._newline),
_pairs: $ =>
seq(
choice($.pair, $._loose_pair, $._pairs),
choice($.pair, $._loose_pair)
),
key: $ => choice($._bare_key, $._quoted_key), key: $ => choice($._bare_key, $._quoted_key),
dotted_key: $ => seq(choice($.dotted_key, $.key), ".", $.key), dotted_key: $ => seq(choice($.dotted_key, $.key), ".", $.key),
@ -212,24 +198,3 @@ module.exports = grammar({
), ),
}, },
}); });
function table_like(name, header_start, header_end) {
const header_name = `_${name}_header`;
const loose_header_name = `_loose_${name}_header`;
return {
[name]: $ =>
seq(
choice($[header_name], $[loose_header_name]),
optional(choice($.pair, $._loose_pair, $._pairs))
),
[header_name]: $ =>
seq(
header_start,
choice($.dotted_key, $.key),
header_end,
$._newline_or_eof
),
[loose_header_name]: $ =>
seq(choice($[loose_header_name], $[header_name]), $._newline),
};
}

253
src/grammar.json generated
View file

@ -2,40 +2,14 @@
"name": "toml", "name": "toml",
"rules": { "rules": {
"root": { "root": {
"type": "SEQ",
"members": [
{
"type": "REPEAT", "type": "REPEAT",
"content": { "content": {
"type": "SYMBOL",
"name": "_newline"
}
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_eof"
},
{
"type": "SEQ",
"members": [
{
"type": "CHOICE", "type": "CHOICE",
"members": [ "members": [
{ {
"type": "SYMBOL", "type": "SYMBOL",
"name": "pair" "name": "pair"
}, },
{
"type": "SYMBOL",
"name": "_loose_pair"
},
{
"type": "SYMBOL",
"name": "_pairs"
},
{ {
"type": "SYMBOL", "type": "SYMBOL",
"name": "table" "name": "table"
@ -43,30 +17,13 @@
{ {
"type": "SYMBOL", "type": "SYMBOL",
"name": "table_array" "name": "table_array"
}
]
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "table"
}, },
{ {
"type": "SYMBOL", "type": "SYMBOL",
"name": "table_array" "name": "_newline"
} }
] ]
} }
}
]
}
]
}
]
}, },
"comment": { "comment": {
"type": "PATTERN", "type": "PATTERN",
@ -76,63 +33,7 @@
"type": "PATTERN", "type": "PATTERN",
"value": "(\\r?\\n)+" "value": "(\\r?\\n)+"
}, },
"_newline_or_eof": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_newline"
},
{
"type": "SYMBOL",
"name": "_eof"
}
]
},
"table": { "table": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_table_header"
},
{
"type": "SYMBOL",
"name": "_loose_table_header"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "pair"
},
{
"type": "SYMBOL",
"name": "_loose_pair"
},
{
"type": "SYMBOL",
"name": "_pairs"
}
]
},
{
"type": "BLANK"
}
]
}
]
},
"_table_header": {
"type": "SEQ", "type": "SEQ",
"members": [ "members": [
{ {
@ -158,76 +59,11 @@
}, },
{ {
"type": "SYMBOL", "type": "SYMBOL",
"name": "_newline_or_eof" "name": "_line_ending_or_eof"
}
]
},
"_loose_table_header": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_loose_table_header"
},
{
"type": "SYMBOL",
"name": "_table_header"
}
]
},
{
"type": "SYMBOL",
"name": "_newline"
} }
] ]
}, },
"table_array": { "table_array": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_table_array_header"
},
{
"type": "SYMBOL",
"name": "_loose_table_array_header"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "pair"
},
{
"type": "SYMBOL",
"name": "_loose_pair"
},
{
"type": "SYMBOL",
"name": "_pairs"
}
]
},
{
"type": "BLANK"
}
]
}
]
},
"_table_array_header": {
"type": "SEQ", "type": "SEQ",
"members": [ "members": [
{ {
@ -253,29 +89,7 @@
}, },
{ {
"type": "SYMBOL", "type": "SYMBOL",
"name": "_newline_or_eof" "name": "_line_ending_or_eof"
}
]
},
"_loose_table_array_header": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_loose_table_array_header"
},
{
"type": "SYMBOL",
"name": "_table_array_header"
}
]
},
{
"type": "SYMBOL",
"name": "_newline"
} }
] ]
}, },
@ -288,7 +102,7 @@
}, },
{ {
"type": "SYMBOL", "type": "SYMBOL",
"name": "_newline_or_eof" "name": "_line_ending_or_eof"
} }
] ]
}, },
@ -318,63 +132,6 @@
} }
] ]
}, },
"_loose_pair": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_loose_pair"
},
{
"type": "SYMBOL",
"name": "pair"
}
]
},
{
"type": "SYMBOL",
"name": "_newline"
}
]
},
"_pairs": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "pair"
},
{
"type": "SYMBOL",
"name": "_loose_pair"
},
{
"type": "SYMBOL",
"name": "_pairs"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "pair"
},
{
"type": "SYMBOL",
"name": "_loose_pair"
}
]
}
]
},
"key": { "key": {
"type": "CHOICE", "type": "CHOICE",
"members": [ "members": [
@ -1035,7 +792,7 @@
"externals": [ "externals": [
{ {
"type": "SYMBOL", "type": "SYMBOL",
"name": "_eof" "name": "_line_ending_or_eof"
} }
], ],
"inline": [], "inline": [],

8
src/node-types.json generated
View file

@ -236,10 +236,6 @@
{ {
"type": "key", "type": "key",
"named": true "named": true
},
{
"type": "pair",
"named": true
} }
] ]
} }
@ -259,10 +255,6 @@
{ {
"type": "key", "type": "key",
"named": true "named": true
},
{
"type": "pair",
"named": true
} }
] ]
} }

4227
src/parser.c generated
View file

File diff suppressed because it is too large Load diff

20
src/scanner.c
View file

@ -1,7 +1,7 @@
#include <tree_sitter/parser.h> #include <tree_sitter/parser.h>
enum TokenType { enum TokenType {
EOF LINE_ENDING_OR_EOF
}; };
void *tree_sitter_toml_external_scanner_create() { return NULL; } void *tree_sitter_toml_external_scanner_create() { return NULL; }
@ -15,10 +15,20 @@ bool tree_sitter_toml_external_scanner_scan(
TSLexer *lexer, TSLexer *lexer,
const bool *valid_symbols const bool *valid_symbols
) { ) {
if (lexer->lookahead != 0) { if (lexer->lookahead == 0 || lexer->lookahead == '\n') {
return false; lexer->result_symbol = LINE_ENDING_OR_EOF;
}
lexer->result_symbol = EOF;
lexer->mark_end(lexer); lexer->mark_end(lexer);
return true; return true;
} }
if (lexer->lookahead == '\r') {
lexer->mark_end(lexer);
lexer->advance(lexer, false);
if (lexer->lookahead == '\n') {
lexer->result_symbol = LINE_ENDING_OR_EOF;
return true;
}
}
return false;
}