Compare commits

..

1 commit

Author SHA1 Message Date
Amaan Qureshi 7b03feefd3 fix: do not perform lossy casts on row/col info
Some checks failed
CI / Test parsers (macos-14) (push) Has been cancelled
CI / Test parsers (ubuntu-latest) (push) Has been cancelled
CI / Test parsers (windows-latest) (push) Has been cancelled
Fuzz / Fuzz scanner (push) Has been cancelled
2024-05-08 11:44:24 -04:00
4 changed files with 423 additions and 397 deletions

View file

@ -21,10 +21,7 @@ include = ["bindings/rust/*", "grammar.js", "queries/*", "src/*"]
path = "bindings/rust/lib.rs"
[dependencies]
tree-sitter-language = "0.1.2"
[dev-dependencies]
tree-sitter = ">=0.22"
tree-sitter = ">=0.21.0"
[build-dependencies]
cc = "1.0.92"

16
bindings/rust/lib.rs generated
View file

@ -11,7 +11,7 @@
//! - item2
//! "#;
//! let mut parser = tree_sitter::Parser::new();
//! parser.set_language(&tree_sitter_yaml::LANGUAGE.into()).expect("Error loading YAML grammar");
//! parser.set_language(&tree_sitter_yaml::language()).expect("Error loading YAML grammar");
//! let tree = parser.parse(code, None).unwrap();
//! assert!(!tree.root_node().has_error());
//! ```
@ -21,16 +21,18 @@
//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
//! [tree-sitter]: https://tree-sitter.github.io/
use tree_sitter_language::LanguageFn;
use tree_sitter::Language;
extern "C" {
fn tree_sitter_yaml() -> *const ();
fn tree_sitter_yaml() -> Language;
}
/// The tree-sitter [`LanguageFn`][LanguageFn] for this grammar.
/// Get the tree-sitter [Language][] for this grammar.
///
/// [LanguageFn]: https://docs.rs/tree-sitter-language/*/tree_sitter_language/struct.LanguageFn.html
pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_yaml) };
/// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
pub fn language() -> Language {
unsafe { tree_sitter_yaml() }
}
/// The content of the [`node-types.json`][] file for this grammar.
///
@ -46,7 +48,7 @@ mod tests {
fn test_can_load_grammar() {
let mut parser = tree_sitter::Parser::new();
parser
.set_language(&super::LANGUAGE.into())
.set_language(&super::language())
.expect("Error loading YAML grammar");
}
}

View file

@ -2,7 +2,7 @@
#include "tree_sitter/parser.h"
#define _str(x) #x
#define _file(x) _str(./schema.x.c)
#define _file(x) _str(schema.x.c)
#ifndef YAML_SCHEMA
#define YAML_SCHEMA core
@ -152,20 +152,27 @@ typedef struct {
} Scanner;
static unsigned serialize(Scanner *scanner, char *buffer) {
size_t i = 0;
buffer[i++] = (char)scanner->row;
buffer[i++] = (char)scanner->col;
buffer[i++] = (char)scanner->blk_imp_row;
buffer[i++] = (char)scanner->blk_imp_col;
buffer[i++] = (char)scanner->blk_imp_tab;
size_t size = 0;
*(int16_t *)&buffer[size] = scanner->row;
size += sizeof(int16_t);
*(int16_t *)&buffer[size] = scanner->col;
size += sizeof(int16_t);
*(int16_t *)&buffer[size] = scanner->blk_imp_row;
size += sizeof(int16_t);
*(int16_t *)&buffer[size] = scanner->blk_imp_col;
size += sizeof(int16_t);
*(int16_t *)&buffer[size] = scanner->blk_imp_tab;
size += sizeof(int16_t);
int16_t *typ_itr = scanner->ind_typ_stk.contents + 1;
int16_t *typ_end = scanner->ind_typ_stk.contents + scanner->ind_typ_stk.size;
int16_t *len_itr = scanner->ind_len_stk.contents + 1;
for (; typ_itr != typ_end && i < TREE_SITTER_SERIALIZATION_BUFFER_SIZE; ++typ_itr, ++len_itr) {
buffer[i++] = (char)*typ_itr;
buffer[i++] = (char)*len_itr;
for (; typ_itr != typ_end && size < TREE_SITTER_SERIALIZATION_BUFFER_SIZE; ++typ_itr, ++len_itr) {
*(int16_t *)&buffer[size] = *typ_itr;
size += sizeof(int16_t);
*(int16_t *)&buffer[size] = *len_itr;
size += sizeof(int16_t);
}
return i;
return size;
}
static void deserialize(Scanner *scanner, const char *buffer, unsigned length) {
@ -179,16 +186,24 @@ static void deserialize(Scanner *scanner, const char *buffer, unsigned length) {
array_delete(&scanner->ind_len_stk);
array_push(&scanner->ind_len_stk, -1);
if (length > 0) {
size_t i = 0;
scanner->row = (int16_t)buffer[i++];
scanner->col = (int16_t)buffer[i++];
scanner->blk_imp_row = (int16_t)buffer[i++];
scanner->blk_imp_col = (int16_t)buffer[i++];
scanner->blk_imp_tab = (int16_t)buffer[i++];
while (i < length) {
array_push(&scanner->ind_typ_stk, (int16_t)buffer[i++]);
array_push(&scanner->ind_len_stk, (int16_t)buffer[i++]);
size_t size = 0;
scanner->row = *(int16_t *)&buffer[size];
size += sizeof(int16_t);
scanner->col = *(int16_t *)&buffer[size];
size += sizeof(int16_t);
scanner->blk_imp_row = *(int16_t *)&buffer[size];
size += sizeof(int16_t);
scanner->blk_imp_col = *(int16_t *)&buffer[size];
size += sizeof(int16_t);
scanner->blk_imp_tab = *(int16_t *)&buffer[size];
size += sizeof(int16_t);
while (size < length) {
array_push(&scanner->ind_typ_stk, *(int16_t *)&buffer[size]);
size += sizeof(int16_t);
array_push(&scanner->ind_len_stk, *(int16_t *)&buffer[size]);
size += sizeof(int16_t);
}
assert(size == length);
}
}

View file

@ -19,25 +19,25 @@ Block Scalar Header
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_sequence
(block_sequence_item
(block_node
(block_scalar
(comment))))
(block_sequence_item
(block_node
(block_scalar
(comment))))
(block_sequence_item
(block_node
(block_scalar
(comment))))
(block_sequence_item
(block_node
(block_scalar
(comment))))))))
(document
(block_node
(block_sequence
(block_sequence_item
(block_node
(block_scalar
(comment))))
(block_sequence_item
(block_node
(block_scalar
(comment))))
(block_sequence_item
(block_node
(block_scalar
(comment))))
(block_sequence_item
(block_node
(block_scalar
(comment))))))))
================================================================================
Block Indentation Indicator
@ -57,36 +57,21 @@ Block Indentation Indicator
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_sequence
(block_sequence_item
(block_node
(block_scalar)))
(block_sequence_item
(block_node
(block_scalar)))
(block_sequence_item
(block_node
(block_scalar)))
(block_sequence_item
(block_node
(block_scalar)))))))
================================================================================
Invalid Block Scalar Indentation Indicators
:error
================================================================================
- |
text
- >
text
text
- |2
text
--------------------------------------------------------------------------------
(document
(block_node
(block_sequence
(block_sequence_item
(block_node
(block_scalar)))
(block_sequence_item
(block_node
(block_scalar)))
(block_sequence_item
(block_node
(block_scalar)))
(block_sequence_item
(block_node
(block_scalar)))))))
================================================================================
Chomping Final Line Break
@ -104,27 +89,27 @@ keep: |+
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))))))
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))))))
================================================================================
Chomping Trailing Lines
@ -159,35 +144,35 @@ keep: |+
--------------------------------------------------------------------------------
(stream
(comment)
(comment)
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(comment)
(comment)
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(comment)
(comment)
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(comment)
(comment)))))
(comment)
(comment)
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(comment)
(comment)
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(comment)
(comment)
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(comment)
(comment)))))
================================================================================
Empty Scalar Chomping
@ -205,27 +190,27 @@ keep: |+
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))))))
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))))))
================================================================================
Literal Scalar
@ -242,9 +227,9 @@ Literal Scalar
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_scalar))))
(document
(block_node
(block_scalar))))
================================================================================
Literal Content
@ -266,10 +251,10 @@ Literal Content
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_scalar)))
(comment))
(document
(block_node
(block_scalar)))
(comment))
================================================================================
Folded Scalar
@ -286,9 +271,9 @@ Folded Scalar
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_scalar))))
(document
(block_node
(block_scalar))))
================================================================================
Folded Lines
@ -319,10 +304,10 @@ Folded Lines
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_scalar)))
(comment))
(document
(block_node
(block_scalar)))
(comment))
================================================================================
More Indented Lines
@ -351,10 +336,10 @@ More Indented Lines
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_scalar)))
(comment))
(document
(block_node
(block_scalar)))
(comment))
================================================================================
Empty Separation Lines
@ -385,10 +370,10 @@ Empty Separation Lines
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_scalar)))
(comment))
(document
(block_node
(block_scalar)))
(comment))
================================================================================
Final Empty Lines
@ -414,10 +399,10 @@ Final Empty Lines
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_scalar)))
(comment))
(document
(block_node
(block_scalar)))
(comment))
================================================================================
Block Sequence
@ -431,29 +416,29 @@ block sequence:
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(block_sequence_item
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar))))))))))))))
(string_scalar)))
(block_node
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(block_sequence_item
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar))))))))))))))
================================================================================
Block Sequence Entry Types
@ -468,38 +453,38 @@ Block Sequence Entry Types
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_sequence
(block_sequence_item)
(comment)
(block_sequence_item
(block_node
(block_scalar)))
(block_sequence_item
(block_node
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(comment)
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(comment))))
(block_sequence_item
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar))))
(comment))))))))
(document
(block_node
(block_sequence
(block_sequence_item)
(comment)
(block_sequence_item
(block_node
(block_scalar)))
(block_sequence_item
(block_node
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(comment)
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(comment))))
(block_sequence_item
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar))))
(comment))))))))
================================================================================
Block Mappings
@ -511,22 +496,22 @@ block mapping:
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar)))))))))))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar)))))))))))
================================================================================
Explicit Block Mapping Entries
@ -543,29 +528,29 @@ Explicit Block Mapping Entries
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar))))
(comment)
(block_mapping_pair
(block_node
(block_scalar))
(block_node
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(comment)
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(comment))))))))
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar))))
(comment)
(block_mapping_pair
(block_node
(block_scalar))
(block_node
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(comment)
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(comment))))))))
================================================================================
Implicit Block Mapping Entries
@ -578,27 +563,27 @@ plain key: in-line value
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar))))
(block_mapping_pair)
(comment)
(block_mapping_pair
(flow_node
(double_quote_scalar))
(block_node
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar)))))))))))
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar))))
(block_mapping_pair)
(comment)
(block_mapping_pair
(flow_node
(double_quote_scalar))
(block_node
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar)))))))))))
================================================================================
Compact Block Mappings
@ -613,41 +598,41 @@ Compact Block Mappings
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_sequence
(block_sequence_item
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar)))))))
(block_sequence_item
(block_node
(block_mapping
(block_mapping_pair
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar))))))
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar))))))))))))))
(document
(block_node
(block_sequence
(block_sequence_item
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar)))))))
(block_sequence_item
(block_node
(block_mapping
(block_mapping_pair
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar))))))
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar))))))))))))))
================================================================================
Block Node Types
@ -666,27 +651,27 @@ Block Node Types
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_sequence
(block_sequence_item
(flow_node
(double_quote_scalar)))
(block_sequence_item
(block_node
(block_scalar)))
(block_sequence_item
(block_node
(tag)
(comment)
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar)))))))))))
(document
(block_node
(block_sequence
(block_sequence_item
(flow_node
(double_quote_scalar)))
(block_sequence_item
(block_node
(block_scalar)))
(block_sequence_item
(block_node
(tag)
(comment)
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar)))))))))))
================================================================================
Block Scalar Nodes
@ -702,22 +687,22 @@ folded:
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(tag)
(block_scalar)))))))
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_scalar)))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(tag)
(block_scalar)))))))
================================================================================
Block Collection Nodes
@ -732,39 +717,66 @@ mapping: !!map
--------------------------------------------------------------------------------
(stream
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(tag)
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(block_sequence_item
(block_node
(tag)
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))))))))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(tag)
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar)))))))))))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(tag)
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))
(block_sequence_item
(block_node
(tag)
(block_sequence
(block_sequence_item
(flow_node
(plain_scalar
(string_scalar))))))))))
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(tag)
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(plain_scalar
(string_scalar)))))))))))
================================================================================
Long string scalar (> SCHAR_MAX in length)
================================================================================
long:
abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdef1234567890!@#$%^&*()_+{}|:"<>?`-=[]\;',./almostat128chars!:
{}
---
(stream
(document
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(block_node
(block_mapping
(block_mapping_pair
(flow_node
(plain_scalar
(string_scalar)))
(flow_node
(flow_mapping))))))))))