gered/tree-sitter-yaml
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: gered:master
Branches Tags
gered:tree-sitter-language
gered:master
gered:v0.6.1
gered:v0.6.0
..
compare: gered:tree-sitter-language
Branches Tags
gered:tree-sitter-language
gered:master
gered:v0.6.1
gered:v0.6.0

1 commit
master ... tree-sitte

Author SHA1 Message Date
Gered 9ffe10a1b8 update rust bindings to use tree-sitter-language 2024-11-10 11:52:35 -05:00
4 changed files with 397 additions and 423 deletions
Show stats Expand all files Collapse all files

5
Cargo.toml
View file

@ -21,7 +21,10 @@ include = ["bindings/rust/*", "grammar.js", "queries/*", "src/*"]
path = "bindings/rust/lib.rs"
[dependencies]
tree-sitter = ">=0.21.0"
tree-sitter-language = "0.1.2"
[dev-dependencies]
tree-sitter = ">=0.22"
[build-dependencies]
cc = "1.0.92"

16
bindings/rust/lib.rs generated
View file

@ -11,7 +11,7 @@
//! - item2
//! "#;
//! let mut parser = tree_sitter::Parser::new();
//! parser.set_language(&tree_sitter_yaml::language()).expect("Error loading YAML grammar");
//! parser.set_language(&tree_sitter_yaml::LANGUAGE.into()).expect("Error loading YAML grammar");
//! let tree = parser.parse(code, None).unwrap();
//! assert!(!tree.root_node().has_error());
//! ```
@ -21,18 +21,16 @@
//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
//! [tree-sitter]: https://tree-sitter.github.io/
use tree_sitter::Language;
use tree_sitter_language::LanguageFn;
extern "C" {
fn tree_sitter_yaml() -> Language;
fn tree_sitter_yaml() -> *const ();
}
/// Get the tree-sitter [Language][] for this grammar.
/// The tree-sitter [`LanguageFn`][LanguageFn] for this grammar.
///
/// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
pub fn language() -> Language {
unsafe { tree_sitter_yaml() }
}
/// [LanguageFn]: https://docs.rs/tree-sitter-language/*/tree_sitter_language/struct.LanguageFn.html
pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_yaml) };
/// The content of the [`node-types.json`][] file for this grammar.
///
@ -48,7 +46,7 @@ mod tests {
fn test_can_load_grammar() {
let mut parser = tree_sitter::Parser::new();
parser
.set_language(&super::language())
.set_language(&super::LANGUAGE.into())
.expect("Error loading YAML grammar");
}
}

55
src/scanner.c
View file

@ -2,7 +2,7 @@
#include "tree_sitter/parser.h"
#define _str(x) #x
#define _file(x) _str(schema.x.c)
#define _file(x) _str(./schema.x.c)
#ifndef YAML_SCHEMA
#define YAML_SCHEMA core
@ -152,27 +152,20 @@ typedef struct {
} Scanner;
static unsigned serialize(Scanner *scanner, char *buffer) {
size_t size = 0;
*(int16_t *)&buffer[size] = scanner->row;
size += sizeof(int16_t);
*(int16_t *)&buffer[size] = scanner->col;
size += sizeof(int16_t);
*(int16_t *)&buffer[size] = scanner->blk_imp_row;
size += sizeof(int16_t);
*(int16_t *)&buffer[size] = scanner->blk_imp_col;
size += sizeof(int16_t);
*(int16_t *)&buffer[size] = scanner->blk_imp_tab;
size += sizeof(int16_t);
size_t i = 0;
buffer[i++] = (char)scanner->row;
buffer[i++] = (char)scanner->col;
buffer[i++] = (char)scanner->blk_imp_row;
buffer[i++] = (char)scanner->blk_imp_col;
buffer[i++] = (char)scanner->blk_imp_tab;
int16_t *typ_itr = scanner->ind_typ_stk.contents + 1;
int16_t *typ_end = scanner->ind_typ_stk.contents + scanner->ind_typ_stk.size;
int16_t *len_itr = scanner->ind_len_stk.contents + 1;
for (; typ_itr != typ_end && size < TREE_SITTER_SERIALIZATION_BUFFER_SIZE; ++typ_itr, ++len_itr) {
*(int16_t *)&buffer[size] = *typ_itr;
size += sizeof(int16_t);
*(int16_t *)&buffer[size] = *len_itr;
size += sizeof(int16_t);
for (; typ_itr != typ_end && i < TREE_SITTER_SERIALIZATION_BUFFER_SIZE; ++typ_itr, ++len_itr) {
buffer[i++] = (char)*typ_itr;
buffer[i++] = (char)*len_itr;
}
return size;
return i;
}
static void deserialize(Scanner *scanner, const char *buffer, unsigned length) {
@ -186,24 +179,16 @@ static void deserialize(Scanner *scanner, const char *buffer, unsigned length) {
array_delete(&scanner->ind_len_stk);
array_push(&scanner->ind_len_stk, -1);
if (length > 0) {
size_t size = 0;
scanner->row = *(int16_t *)&buffer[size];
size += sizeof(int16_t);
scanner->col = *(int16_t *)&buffer[size];
size += sizeof(int16_t);
scanner->blk_imp_row = *(int16_t *)&buffer[size];
size += sizeof(int16_t);
scanner->blk_imp_col = *(int16_t *)&buffer[size];
size += sizeof(int16_t);
scanner->blk_imp_tab = *(int16_t *)&buffer[size];
size += sizeof(int16_t);
while (size < length) {
array_push(&scanner->ind_typ_stk, *(int16_t *)&buffer[size]);
size += sizeof(int16_t);
array_push(&scanner->ind_len_stk, *(int16_t *)&buffer[size]);
size += sizeof(int16_t);
size_t i = 0;
scanner->row = (int16_t)buffer[i++];
scanner->col = (int16_t)buffer[i++];
scanner->blk_imp_row = (int16_t)buffer[i++];
scanner->blk_imp_col = (int16_t)buffer[i++];
scanner->blk_imp_tab = (int16_t)buffer[i++];
while (i < length) {
array_push(&scanner->ind_typ_stk, (int16_t)buffer[i++]);
array_push(&scanner->ind_len_stk, (int16_t)buffer[i++]);
}
assert(size == length);
}
}

744
test/corpus/08_blocks.txt
View file

@ -19,25 +19,25 @@ Block Scalar Header
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_sequence
(block_sequence_item
(block_node
(block_scalar
(comment))))
(block_sequence_item
(block_node
(block_scalar
(comment))))
(block_sequence_item
(block_node
(block_scalar
(comment))))
(block_sequence_item
(block_node
(block_scalar
(comment))))))))
(document
(block_node
(block_sequence
(block_sequence_item
(block_node
(block_scalar
(comment))))
(block_sequence_item
(block_node
(block_scalar
(comment))))
(block_sequence_item
(block_node
(block_scalar
(comment))))
(block_sequence_item
(block_node
(block_scalar
(comment))))))))
================================================================================
Block Indentation Indicator
@ -57,21 +57,36 @@ Block Indentation Indicator
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_sequence
(block_sequence_item
(block_node
(block_scalar)))
(block_sequence_item
(block_node
(block_scalar)))
(block_sequence_item
(block_node
(block_scalar)))
(block_sequence_item
(block_node
(block_scalar)))))))
(document
(block_node
(block_sequence
(block_sequence_item
(block_node
(block_scalar)))
(block_sequence_item
(block_node
(block_scalar)))
(block_sequence_item
(block_node
(block_scalar)))
(block_sequence_item
(block_node
(block_scalar)))))))
================================================================================
Invalid Block Scalar Indentation Indicators
:error
================================================================================
- |
text
- >
text
text
- |2
text
--------------------------------------------------------------------------------
================================================================================
Chomping Final Line Break
@ -89,27 +104,27 @@ keep: |+
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))))))
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))))))
================================================================================
Chomping Trailing Lines
@ -144,35 +159,35 @@ keep: |+
--------------------------------------------------------------------------------
(stream
(comment)
(comment)
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(comment)
(comment)
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(comment)
(comment)
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(comment)
(comment)))))
(comment)
(comment)
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(comment)
(comment)
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(comment)
(comment)
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(comment)
(comment)))))
================================================================================
Empty Scalar Chomping
@ -190,27 +205,27 @@ keep: |+
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))))))
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))))))
================================================================================
Literal Scalar
@ -227,9 +242,9 @@ Literal Scalar
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_scalar))))
(document
(block_node
(block_scalar))))
================================================================================
Literal Content
@ -251,10 +266,10 @@ Literal Content
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_scalar)))
(comment))
(document
(block_node
(block_scalar)))
(comment))
================================================================================
Folded Scalar
@ -271,9 +286,9 @@ Folded Scalar
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_scalar))))
(document
(block_node
(block_scalar))))
================================================================================
Folded Lines
@ -304,10 +319,10 @@ Folded Lines
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_scalar)))
(comment))
(document
(block_node
(block_scalar)))
(comment))
================================================================================
More Indented Lines
@ -336,10 +351,10 @@ More Indented Lines
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_scalar)))
(comment))
(document
(block_node
(block_scalar)))
(comment))
================================================================================
Empty Separation Lines
@ -370,10 +385,10 @@ Empty Separation Lines
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_scalar)))
(comment))
(document
(block_node
(block_scalar)))
(comment))
================================================================================
Final Empty Lines
@ -399,10 +414,10 @@ Final Empty Lines
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_scalar)))
(comment))
(document
(block_node
(block_scalar)))
(comment))
================================================================================
Block Sequence
@ -416,29 +431,29 @@ block sequence:
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(block_sequence_item
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(block_sequence_item
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar))))))))))))))
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar))))))))))))))
================================================================================
Block Sequence Entry Types
@ -453,38 +468,38 @@ Block Sequence Entry Types
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_sequence
(block_sequence_item)
(comment)
(block_sequence_item
(block_node
(block_scalar)))
(block_sequence_item
(block_node
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(comment)
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(comment))))
(block_sequence_item
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar))))
(comment))))))))
(document
(block_node
(block_sequence
(block_sequence_item)
(comment)
(block_sequence_item
(block_node
(block_scalar)))
(block_sequence_item
(block_node
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(comment)
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(comment))))
(block_sequence_item
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar))))
(comment))))))))
================================================================================
Block Mappings
@ -496,22 +511,22 @@ block mapping:
--------------------------------------------------------------------------------
(stream
(document
(block_node
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar)))))))))))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar)))))))))))
================================================================================
Explicit Block Mapping Entries
@ -528,29 +543,29 @@ Explicit Block Mapping Entries
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar))))
(comment)
(block_mapping_pair
(block_node
(block_scalar))
(block_node
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(comment)
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(comment))))))))
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar))))
(comment)
(block_mapping_pair
(block_node
(block_scalar))
(block_node
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(comment)
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(comment))))))))
================================================================================
Implicit Block Mapping Entries
@ -563,27 +578,27 @@ plain key: in-line value
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar))))
(block_mapping_pair)
(comment)
(block_mapping_pair
(flow_node
(double_quote_scalar))
(block_node
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar)))))))))))
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar))))
(block_mapping_pair)
(comment)
(block_mapping_pair
(flow_node
(double_quote_scalar))
(block_node
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar)))))))))))
================================================================================
Compact Block Mappings
@ -598,41 +613,41 @@ Compact Block Mappings
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_sequence
(block_sequence_item
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar)))))))
(block_sequence_item
(block_node
(block_mapping
(block_mapping_pair
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar))))))
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar))))))))))))))
(document
(block_node
(block_sequence
(block_sequence_item
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar)))))))
(block_sequence_item
(block_node
(block_mapping
(block_mapping_pair
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar))))))
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar))))))))))))))
================================================================================
Block Node Types
@ -651,27 +666,27 @@ Block Node Types
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_sequence
(block_sequence_item
(flow_node
(double_quote_scalar)))
(block_sequence_item
(block_node
(block_scalar)))
(block_sequence_item
(block_node
(tag)
(comment)
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar)))))))))))
(document
(block_node
(block_sequence
(block_sequence_item
(flow_node
(double_quote_scalar)))
(block_sequence_item
(block_node
(block_scalar)))
(block_sequence_item
(block_node
(tag)
(comment)
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar)))))))))))
================================================================================
Block Scalar Nodes
@ -687,22 +702,22 @@ folded:
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(tag)
(block_scalar)))))))
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(tag)
(block_scalar)))))))
================================================================================
Block Collection Nodes
@ -717,66 +732,39 @@ mapping: !!map
--------------------------------------------------------------------------------
(stream
(document
(block_node
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(tag)
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(block_sequence_item
(block_node
(tag)
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))))))))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(tag)
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(tag)
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(block_sequence_item
(block_node
(tag)
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))))))))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(tag)
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar)))))))))))
================================================================================
Long string scalar (> SCHAR_MAX in length)
================================================================================
long:
abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdef1234567890!@#$%^&*()_+{}|:"<>?`-=[]\;',./almostat128chars!:
{}
---
(stream
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(flow_mapping))))))))))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar)))))))))))