gered/tree-sitter-toml
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

feat: flatten table/table-array structure (#2)

Browse source 
Input:

```toml
[table]
key = "value"
```

Output:

```sh
# before
(root
  (table
    (key)
    (pair (key) (string)))

# after
(root
  (table (key))
  (pair (key) (string)))
```
This commit is contained in:
Ika 2019-08-20 14:59:19 +08:00 committed by GitHub
parent 43595130ac
commit a3812033af
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 2121 additions and 2749 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

115
corpus/custom.txt
View file

@ -45,9 +45,11 @@ invalid_both = 1 .0 e 2
--------------------------------------------------------------------------------
(ERROR
(dotted_key
(key) (ERROR) (key)
(root
(ERROR
(dotted_key
(key) (ERROR) (key)
)
)
)
@ -60,8 +62,10 @@ invalid2 = 1979-05-27 T 07:32:00 Z
--------------------------------------------------------------------------------
(ERROR
(key) (ERROR) (local_time) (local_time)
(root
(ERROR
(key) (ERROR) (local_time) (local_time)
)
)
================================================================================
@ -125,13 +129,13 @@ key = {
(pair
(key)
(inline_table
(MISSING "}")
(ERROR)
(pair
(key) (boolean)
)
(ERROR)
)
)
(pair
(key) (boolean)
)
(ERROR)
)
================================================================================
@ -154,9 +158,96 @@ key
--------------------------------------------------------------------------------
(ERROR
(root
(ERROR
(key) (key)
)
)
================================================================================
VALID - table - empty content separated by comments
================================================================================
[table]
# comment
--------------------------------------------------------------------------------
(root
(table
(key)
)
(key)
(comment)
)
================================================================================
VALID - array of tables - empty content separated by comments
================================================================================
[[array-table]]
# comment
--------------------------------------------------------------------------------
(root
(table_array
(key)
)
(comment)
)
================================================================================
VALID - table - two pairs separated by comments
================================================================================
[table]
a = 1
# comment
b = 2
--------------------------------------------------------------------------------
(root
(table
(key)
)
(pair
(key) (integer)
)
(comment)
(pair
(key) (integer)
)
)
================================================================================
VALID - array of tables - two pairs separated by comments
================================================================================
[[array-table]]
a = 1
# comment
b = 2
--------------------------------------------------------------------------------
(root
(table_array
(key)
)
(pair
(key) (integer)
)
(comment)
(pair
(key) (integer)
)
)

158
corpus/spec.txt
View file

@ -37,12 +37,8 @@ key = # INVALID
--------------------------------------------------------------------------------
(root
(pair
(key)
(comment)
(integer
(MISSING "integer_token1")
)
(ERROR
(key) (comment)
)
)
@ -782,21 +778,21 @@ key2 = 456
(root
(table
(key)
(pair
(key) (string)
)
(pair
(key) (integer)
)
)
(pair
(key) (string)
)
(pair
(key) (integer)
)
(table
(key)
(pair
(key) (string)
)
(pair
(key) (integer)
)
)
(pair
(key) (string)
)
(pair
(key) (integer)
)
)
@ -814,12 +810,12 @@ type.name = "pug"
(dotted_key
(key) (key)
)
(pair
(dotted_key
(key) (key)
)
(string)
)
(pair
(dotted_key
(key) (key)
)
(string)
)
)
@ -920,15 +916,15 @@ c = 2
(comment)
(table
(key)
(pair
(key) (integer)
)
)
(pair
(key) (integer)
)
(table
(key)
(pair
(key) (integer)
)
)
(pair
(key) (integer)
)
)
@ -950,17 +946,17 @@ c = 2
(comment)
(table
(key)
(pair
(key) (integer)
)
)
(pair
(key) (integer)
)
(table
(dotted_key
(key) (key)
)
(pair
(key) (integer)
)
)
(pair
(key) (integer)
)
)
@ -1030,27 +1026,27 @@ color = "gray"
(root
(table_array
(key)
(pair
(key) (string)
)
(pair
(key) (integer)
)
)
(pair
(key) (string)
)
(pair
(key) (integer)
)
(table_array
(key)
)
(table_array
(key)
(pair
(key) (string)
)
(pair
(key) (integer)
)
(pair
(key) (string)
)
)
(pair
(key) (string)
)
(pair
(key) (integer)
)
(pair
(key) (string)
)
)
@ -1082,50 +1078,50 @@ VALID - array of tables - nested arrays of tables
(root
(table_array
(key)
(pair
(key) (string)
)
)
(pair
(key) (string)
)
(table
(dotted_key
(key) (key)
)
(pair
(key) (string)
)
(pair
(key) (string)
)
)
(pair
(key) (string)
)
(pair
(key) (string)
)
(table_array
(dotted_key
(key) (key)
)
(pair
(key) (string)
)
)
(pair
(key) (string)
)
(table_array
(dotted_key
(key) (key)
)
(pair
(key) (string)
)
)
(pair
(key) (string)
)
(table_array
(key)
(pair
(key) (string)
)
)
(pair
(key) (string)
)
(table_array
(dotted_key
(key) (key)
)
(pair
(key) (string)
)
)
(pair
(key) (string)
)
)
@ -1171,25 +1167,25 @@ VALID - array of tables - append to table (semantically INVALID)
(comment)
(table_array
(key)
(pair
(key) (string)
)
)
(pair
(key) (string)
)
(table_array
(dotted_key
(key) (key)
)
(pair
(key) (string)
)
(comment)
)
(pair
(key) (string)
)
(comment)
(table
(dotted_key
(key) (key)
)
(pair
(key) (string)
)
)
(pair
(key) (string)
)
)

51
grammar.js
View file

@ -23,38 +23,24 @@ const rfc3339_offset = /([zZ])|([+-]([01][0-9]|2[0-3]):[0-5][0-9])/;
module.exports = grammar({
name: "toml",
externals: $ => [$._eof],
externals: $ => [$._line_ending_or_eof],
extras: $ => [$.comment, /[ \t]/],
rules: {
root: $ =>
seq(
repeat($._newline),
choice(
$._eof,
seq(
choice($.pair, $._loose_pair, $._pairs, $.table, $.table_array),
repeat(choice($.table, $.table_array))
)
)
),
root: $ => repeat(choice($.pair, $.table, $.table_array, $._newline)),
comment: $ => /#.*/,
_newline: $ => newline,
_newline_or_eof: $ => choice($._newline, $._eof),
...table_like("table", "[", "]"),
...table_like("table_array", "[[", "]]"),
table: $ =>
seq("[", choice($.dotted_key, $.key), "]", $._line_ending_or_eof),
pair: $ => seq($._inline_pair, $._newline_or_eof),
table_array: $ =>
seq("[[", choice($.dotted_key, $.key), "]]", $._line_ending_or_eof),
pair: $ => seq($._inline_pair, $._line_ending_or_eof),
_inline_pair: $ => seq(choice($.dotted_key, $.key), "=", $._inline_value),
_loose_pair: $ => seq(choice($._loose_pair, $.pair), $._newline),
_pairs: $ =>
seq(
choice($.pair, $._loose_pair, $._pairs),
choice($.pair, $._loose_pair)
),
key: $ => choice($._bare_key, $._quoted_key),
dotted_key: $ => seq(choice($.dotted_key, $.key), ".", $.key),
@ -212,24 +198,3 @@ module.exports = grammar({
),
},
});
function table_like(name, header_start, header_end) {
const header_name = `_${name}_header`;
const loose_header_name = `_loose_${name}_header`;
return {
[name]: $ =>
seq(
choice($[header_name], $[loose_header_name]),
optional(choice($.pair, $._loose_pair, $._pairs))
),
[header_name]: $ =>
seq(
header_start,
choice($.dotted_key, $.key),
header_end,
$._newline_or_eof
),
[loose_header_name]: $ =>
seq(choice($[loose_header_name], $[header_name]), $._newline),
};
}

289
src/grammar.json generated
View file

@ -2,71 +2,28 @@
"name": "toml",
"rules": {
"root": {
"type": "SEQ",
"members": [
{
"type": "REPEAT",
"content": {
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "pair"
},
{
"type": "SYMBOL",
"name": "table"
},
{
"type": "SYMBOL",
"name": "table_array"
},
{
"type": "SYMBOL",
"name": "_newline"
}
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_eof"
},
{
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "pair"
},
{
"type": "SYMBOL",
"name": "_loose_pair"
},
{
"type": "SYMBOL",
"name": "_pairs"
},
{
"type": "SYMBOL",
"name": "table"
},
{
"type": "SYMBOL",
"name": "table_array"
}
]
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "table"
},
{
"type": "SYMBOL",
"name": "table_array"
}
]
}
}
]
}
]
}
]
]
}
},
"comment": {
"type": "PATTERN",
@ -76,63 +33,7 @@
"type": "PATTERN",
"value": "(\\r?\\n)+"
},
"_newline_or_eof": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_newline"
},
{
"type": "SYMBOL",
"name": "_eof"
}
]
},
"table": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_table_header"
},
{
"type": "SYMBOL",
"name": "_loose_table_header"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "pair"
},
{
"type": "SYMBOL",
"name": "_loose_pair"
},
{
"type": "SYMBOL",
"name": "_pairs"
}
]
},
{
"type": "BLANK"
}
]
}
]
},
"_table_header": {
"type": "SEQ",
"members": [
{
@ -158,76 +59,11 @@
},
{
"type": "SYMBOL",
"name": "_newline_or_eof"
}
]
},
"_loose_table_header": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_loose_table_header"
},
{
"type": "SYMBOL",
"name": "_table_header"
}
]
},
{
"type": "SYMBOL",
"name": "_newline"
"name": "_line_ending_or_eof"
}
]
},
"table_array": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_table_array_header"
},
{
"type": "SYMBOL",
"name": "_loose_table_array_header"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "pair"
},
{
"type": "SYMBOL",
"name": "_loose_pair"
},
{
"type": "SYMBOL",
"name": "_pairs"
}
]
},
{
"type": "BLANK"
}
]
}
]
},
"_table_array_header": {
"type": "SEQ",
"members": [
{
@ -253,29 +89,7 @@
},
{
"type": "SYMBOL",
"name": "_newline_or_eof"
}
]
},
"_loose_table_array_header": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_loose_table_array_header"
},
{
"type": "SYMBOL",
"name": "_table_array_header"
}
]
},
{
"type": "SYMBOL",
"name": "_newline"
"name": "_line_ending_or_eof"
}
]
},
@ -288,7 +102,7 @@
},
{
"type": "SYMBOL",
"name": "_newline_or_eof"
"name": "_line_ending_or_eof"
}
]
},
@ -318,63 +132,6 @@
}
]
},
"_loose_pair": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_loose_pair"
},
{
"type": "SYMBOL",
"name": "pair"
}
]
},
{
"type": "SYMBOL",
"name": "_newline"
}
]
},
"_pairs": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "pair"
},
{
"type": "SYMBOL",
"name": "_loose_pair"
},
{
"type": "SYMBOL",
"name": "_pairs"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "pair"
},
{
"type": "SYMBOL",
"name": "_loose_pair"
}
]
}
]
},
"key": {
"type": "CHOICE",
"members": [
@ -1035,7 +792,7 @@
"externals": [
{
"type": "SYMBOL",
"name": "_eof"
"name": "_line_ending_or_eof"
}
],
"inline": [],

8
src/node-types.json generated
View file

@ -236,10 +236,6 @@
{
"type": "key",
"named": true
},
{
"type": "pair",
"named": true
}
]
}
@ -259,10 +255,6 @@
{
"type": "key",
"named": true
},
{
"type": "pair",
"named": true
}
]
}

4227
src/parser.c generated
View file

File diff suppressed because it is too large Load diff

22
src/scanner.c
View file

@ -1,7 +1,7 @@
#include <tree_sitter/parser.h>
enum TokenType {
EOF
LINE_ENDING_OR_EOF
};
void *tree_sitter_toml_external_scanner_create() { return NULL; }
@ -15,10 +15,20 @@ bool tree_sitter_toml_external_scanner_scan(
TSLexer *lexer,
const bool *valid_symbols
) {
if (lexer->lookahead != 0) {
return false;
if (lexer->lookahead == 0 || lexer->lookahead == '\n') {
lexer->result_symbol = LINE_ENDING_OR_EOF;
lexer->mark_end(lexer);
return true;
}
lexer->result_symbol = EOF;
lexer->mark_end(lexer);
return true;
if (lexer->lookahead == '\r') {
lexer->mark_end(lexer);
lexer->advance(lexer, false);
if (lexer->lookahead == '\n') {
lexer->result_symbol = LINE_ENDING_OR_EOF;
return true;
}
}
return false;
}