fix: do not perform lossy casts on row/col info

.editorconfig

@@ -17,7 +17,7 @@ indent_size = 2
 
 [*.{c,cc,h}]
 indent_style = space
-indent_size = 2
+indent_size = 4
 
 [*.rs]
 indent_style = space

.gitattributes

@@ -4,8 +4,8 @@ src/*.json linguist-generated
 src/parser.c linguist-generated
 src/tree_sitter/* linguist-generated
 
-src/schema.generated.c linguist-generated
+src/schema.*.c linguist-generated
-schema/src/** linguist-generated
+schema/*/src/** linguist-generated
 
 bindings/** linguist-generated
 binding.gyp linguist-generated

.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    commit-message:
+      prefix: "ci"

.github/workflows/ci.yml

@@ -0,0 +1,57 @@
+name: CI
+
+on:
+  push:
+    branches: [master]
+    paths:
+      - grammar.js
+      - src/**
+      - test/**
+      - bindings/**
+      - binding.gyp
+  pull_request:
+    paths:
+      - grammar.js
+      - src/**
+      - test/**
+      - bindings/**
+      - binding.gyp
+
+concurrency:
+  group: ${{github.workflow}}-${{github.ref}}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Test parsers
+    runs-on: ${{matrix.os}}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-14]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Set up tree-sitter
+        uses: tree-sitter/setup-action/cli@v1
+      - name: Run tests
+        uses: tree-sitter/parser-test-action@v2
+        with:
+          test-rust: ${{runner.os == 'Linux'}}
+      - name: Set up test suite
+        shell: bash
+        run: |-
+          printf '%s\n' examples/*.yaml > "$RUNNER_TEMP/test-files"
+          while read -r file; do
+            printf '%s\n' "$file" >> "$RUNNER_TEMP/test-files"
+            if [[ -f ${file/in.yaml/error} ]]; then
+              printf '%s\n' "$file" >> "$RUNNER_TEMP/invalid-files"
+            fi
+          done < <(find -L examples/yaml-test-suite/name -name in.yaml)
+      - name: Parse test suite
+        uses: tree-sitter/parse-action@v4
+        with:
+          files-list: ${{runner.temp}}/test-files
+          invalid-files-list: ${{runner.temp}}/invalid-files

.github/workflows/fuzz.yml

@@ -0,0 +1,26 @@
+name: Fuzz
+
+on:
+  push:
+    branches: [master]
+    paths:
+      - src/schema.core.c
+      - src/scanner.c
+  pull_request:
+    paths:
+      - src/schema.core.c
+      - src/scanner.c
+
+concurrency:
+  group: ${{github.workflow}}-${{github.ref}}
+  cancel-in-progress: true
+
+jobs:
+  fuzz:
+    name: Fuzz scanner
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Run fuzzer
+        uses: tree-sitter/fuzz-action@v4

.github/workflows/publish.yml

@@ -0,0 +1,21 @@
+name: Publish package
+
+on:
+  push:
+    tags: ["*"]
+
+jobs:
+  npm:
+    uses: tree-sitter/workflows/.github/workflows/package-npm.yml@main
+    with:
+      package-name: "@tree-sitter-grammars/tree-sitter-yaml"
+    secrets:
+      NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
+  crates:
+    uses: tree-sitter/workflows/.github/workflows/package-crates.yml@main
+    secrets:
+      CARGO_REGISTRY_TOKEN: ${{secrets.CARGO_TOKEN}}
+  pypi:
+    uses: tree-sitter/workflows/.github/workflows/package-pypi.yml@main
+    secrets:
+      PYPI_API_TOKEN: ${{secrets.PYPI_TOKEN}}

.gitignore

@@ -29,9 +29,6 @@ dist/
 *.dll
 *.pc
 
-# Example dirs
-/examples/*/
-
 # Grammar volatiles
 dsl.d.ts
 *.wasm

.gitmodules

@@ -0,0 +1,5 @@
+[submodule "yaml-test-suite"]
+	path = examples/yaml-test-suite
+	url = https://github.com/yaml/yaml-test-suite
+	branch = data
+	shallow = true

.travis.yml

@@ -1,13 +0,0 @@
-language: rust
-
-rust:
-  - stable
-
-script:
-  - if [ ! -d "./tree-sitter/target/release" ]; then bash ./scripts/setup-tree-sitter.sh; fi
-  - ./tree-sitter/target/release/tree-sitter test
-
-cache:
-  cargo: true
-  directories:
-    - ./tree-sitter

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "tree-sitter-yaml"
 description = "YAML grammar for tree-sitter"
-version = "0.6.0"
+version = "0.6.1"
 license = "MIT"
 readme = "README.md"
 keywords = ["incremental", "parsing", "tree-sitter", "yaml"]

Makefile

@@ -1,4 +1,4 @@
-VERSION := 0.6.0
+VERSION := 0.6.1
 
 LANGUAGE_NAME := tree-sitter-yaml
 

README.md

@@ -11,7 +11,7 @@ A tree-sitter parser for YAML files.
 
 ## References
 
-- [YAML version 1.2](https://yaml.org/spec/1.2/spec.html)
+- [YAML version 1.2](https://yaml.org/spec/1.2.2/)
 
 [ci]: https://img.shields.io/github/actions/workflow/status/tree-sitter-grammars/tree-sitter-yaml/ci.yml?logo=github&label=CI
 [discord]: https://img.shields.io/discord/1063097320771698699?logo=discord&label=discord

binding.gyp

@@ -13,8 +13,17 @@
         "src/parser.c",
         "src/scanner.c",
       ],
-      "cflags_c": [
+      "conditions": [
-        "-std=c11",
+        ["OS!='win'", {
+          "cflags_c": [
+            "-std=c11",
+          ],
+        }, { # OS == "win"
+          "cflags_c": [
+            "/std:c11",
+            "/utf-8",
+          ],
+        }],
       ],
     }
   ]

bindings/rust/build.rs

@@ -2,8 +2,14 @@ fn main() {
     let src_dir = std::path::Path::new("src");
 
     let mut c_config = cc::Build::new();
-    c_config.flag("-Wno-unused-but-set-variable").flag("-Wno-unused-value").flag("-Wno-implicit-fallthrough");
     c_config.std("c11").include(src_dir);
+    c_config
+        .flag_if_supported("-Wno-unused-but-set-variable")
+        .flag_if_supported("-Wno-unused-value")
+        .flag_if_supported("-Wno-implicit-fallthrough");
+
+    #[cfg(target_env = "msvc")]
+    c_config.flag("-utf-8");
 
     let parser_path = src_dir.join("parser.c");
     c_config.file(&parser_path);

bindings/rust/lib.rs

@@ -39,12 +39,8 @@ pub fn language() -> Language {
 /// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types
 pub const NODE_TYPES: &str = include_str!("../../src/node-types.json");
 
-// Uncomment these to include any queries that this grammar contains
+/// The highlight queries for this grammar.
-
+pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
-// pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
-// pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
-// pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
-// pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
 
 #[cfg(test)]
 mod tests {

examples/invoice.yaml

@@ -0,0 +1,29 @@
+--- !<tag:clarkevans.com,2002:invoice>
+invoice: 34843
+date   : 2001-01-23
+bill-to: &id001
+    given  : Chris
+    family : Dumars
+    address:
+        lines: |
+            458 Walkman Dr.
+            Suite #292
+        city    : Royal Oak
+        state   : MI
+        postal  : 48046
+ship-to: *id001
+product:
+    - sku         : BL394D
+      quantity    : 4
+      description : Basketball
+      price       : 450.00
+    - sku         : BL4438H
+      quantity    : 1
+      description : Super Hoop
+      price       : 2392.00
+tax  : 251.42
+total: 4443.52
+comments:
+    Late afternoon is best.
+    Backup contact is Nancy
+    Billsmer @ 338-4338.

examples/log-file.yaml

@@ -0,0 +1,29 @@
+---
+Time: 2001-11-23 15:01:42 -5
+User: ed
+Warning:
+  This is an error message
+  for the log file
+---
+Time: 2001-11-23 15:02:31 -5
+User: ed
+Warning:
+  A slightly different error
+  message.
+---
+Date: 2001-11-23 15:03:17 -5
+User: ed
+Fatal:
+  Unknown variable "bar"
+Stack:
+  - file: TopClass.py
+    line: 23
+    code: |
+      x = MoreObject("345\n")
+  - file: MoreClass.py
+    line: 58
+    code: |-
+      foo = bar
+
+
+

examples/yaml-test-suite

@@ -0,0 +1 @@
+Subproject commit 6ad3d2c62885d82fc349026c136ef560838fdf3d

grammar.js

@@ -64,6 +64,8 @@ module.exports = grammar({
 
     $._bl,
     $.comment,
+
+    $._err_rec,
   ],
 
   extras: $ => [$.comment],

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@tree-sitter-grammars/tree-sitter-yaml",
-  "version": "0.6.0",
+  "version": "0.6.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@tree-sitter-grammars/tree-sitter-yaml",
-      "version": "0.6.0",
+      "version": "0.6.1",
       "hasInstallScript": true,
       "license": "MIT",
       "dependencies": {
@@ -14,8 +14,8 @@
         "node-gyp-build": "^4.8.0"
       },
       "devDependencies": {
-        "prebuildify": "^6.0.0",
+        "prebuildify": "^6.0.1",
-        "tree-sitter-cli": "^0.22.2"
+        "tree-sitter-cli": "^0.22.5"
       },
       "peerDependencies": {
         "tree-sitter": "^0.21.1"
@@ -96,15 +96,6 @@
         "once": "^1.4.0"
       }
     },
-    "node_modules/execspawn": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/execspawn/-/execspawn-1.0.1.tgz",
-      "integrity": "sha512-s2k06Jy9i8CUkYe0+DxRlvtkZoOkwwfhB+Xxo5HGUtrISVW2m98jO2tr67DGRFxZwkjQqloA3v/tNtjhBRBieg==",
-      "dev": true,
-      "dependencies": {
-        "util-extend": "^1.0.1"
-      }
-    },
     "node_modules/fs-constants": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
@@ -225,12 +216,11 @@
       }
     },
     "node_modules/prebuildify": {
-      "version": "6.0.0",
+      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/prebuildify/-/prebuildify-6.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/prebuildify/-/prebuildify-6.0.1.tgz",
-      "integrity": "sha512-DEvK4C3tcimIp7Pzqbs036n9i6CTKGp1XVEpMnr4wV3enKU5sBogPP+lP3KZw7993i42bXnsd5eIxAXQ566Cqw==",
+      "integrity": "sha512-8Y2oOOateom/s8dNBsGIcnm6AxPmLH4/nanQzL5lQMU+sC0CMhzARZHizwr36pUPLdvBnOkCNQzxg4djuFSgIw==",
       "dev": true,
       "dependencies": {
-        "execspawn": "^1.0.1",
         "minimist": "^1.2.5",
         "mkdirp-classic": "^0.5.3",
         "node-abi": "^3.3.0",
@@ -350,9 +340,9 @@
       }
     },
     "node_modules/tree-sitter-cli": {
-      "version": "0.22.2",
+      "version": "0.22.5",
-      "resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.22.2.tgz",
+      "resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.22.5.tgz",
-      "integrity": "sha512-ecqccEp27XMFXgjLMEEU71vK9JCWAC7fqSTTxcs5P1tnEnaaf4GkHz/wfo4lJ9l3rfxcTDPxN84tHAoitIQqdA==",
+      "integrity": "sha512-c3VT46Bc3a6pEd0JAwufbqEw9Q2FRLDp5E230hGvnr+Hivw+Y6jyeP+3T89KDptvn48MOPVmbgaLm69xYgLVTw==",
       "dev": true,
       "hasInstallScript": true,
       "bin": {
@@ -365,12 +355,6 @@
       "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",
       "dev": true
     },
-    "node_modules/util-extend": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/util-extend/-/util-extend-1.0.3.tgz",
-      "integrity": "sha512-mLs5zAK+ctllYBj+iAQvlDCwoxU/WDOUaJkcFudeiAX6OajC6BKXJUa9a+tbtkC11dz2Ufb7h0lyvIOVn4LADA==",
-      "dev": true
-    },
     "node_modules/wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tree-sitter-grammars/tree-sitter-yaml",
-  "version": "0.6.0",
+  "version": "0.6.1",
   "license": "MIT",
   "description": "YAML grammar for tree-sitter",
   "repository": "github:tree-sitter-grammars/tree-sitter-yaml",
@@ -37,8 +37,8 @@
     "node-gyp-build": "^4.8.0"
   },
   "devDependencies": {
-    "prebuildify": "^6.0.0",
+    "prebuildify": "^6.0.1",
-    "tree-sitter-cli": "^0.22.2"
+    "tree-sitter-cli": "^0.22.5"
   },
   "peerDependencies": {
     "tree-sitter": "^0.21.1"
@@ -50,7 +50,7 @@
   },
   "scripts": {
     "build": "tree-sitter generate --no-bindings",
-    "postbuild": "node schema/update-schema.js",
+    "postbuild": "npm run --prefix schema/core build",
     "test": "tree-sitter test",
     "install": "node-gyp-build",
     "prebuildify": "prebuildify --napi --strip"
@@ -66,8 +66,9 @@
         "yaml"
       ],
       "injection-regex": "^ya?ml$",
+      "highlights": "queries/highlights.scm",
       "external-files": [
-        "src/schema.generated.c"
+        "src/schema.core.c"
       ]
     }
   ]

pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "tree-sitter-yaml"
 description = "YAML grammar for tree-sitter"
-version = "0.6.0"
+version = "0.6.1"
 keywords = ["incremental", "parsing", "tree-sitter", "yaml"]
 classifiers = [
   "Intended Audience :: Developers",

queries/highlights.scm

@@ -0,0 +1,79 @@
+(boolean_scalar) @boolean
+
+(null_scalar) @constant.builtin
+
+[
+  (double_quote_scalar)
+  (single_quote_scalar)
+  (block_scalar)
+  (string_scalar)
+] @string
+
+[
+  (integer_scalar)
+  (float_scalar)
+] @number
+
+(comment) @comment
+
+[
+  (anchor_name)
+  (alias_name)
+] @label
+
+(tag) @type
+
+[
+  (yaml_directive)
+  (tag_directive)
+  (reserved_directive)
+] @attribute
+
+(block_mapping_pair
+  key: (flow_node
+    [
+      (double_quote_scalar)
+      (single_quote_scalar)
+    ] @property))
+
+(block_mapping_pair
+  key: (flow_node
+    (plain_scalar
+      (string_scalar) @property)))
+
+(flow_mapping
+  (_
+    key: (flow_node
+      [
+        (double_quote_scalar)
+        (single_quote_scalar)
+      ] @property)))
+
+(flow_mapping
+  (_
+    key: (flow_node
+      (plain_scalar
+        (string_scalar) @property))))
+
+[
+  ","
+  "-"
+  ":"
+  ">"
+  "?"
+  "|"
+] @punctuation.delimiter
+
+[
+  "["
+  "]"
+  "{"
+  "}"
+] @punctuation.bracket
+
+[
+  "*"
+  "&"
+  "---"
+  "..."
+] @punctuation.special

schema/core/grammar.js

@@ -1,6 +1,4 @@
-/**
+/** @see {@link Core Schema|https://yaml.org/spec/1.2.2/#103-core-schema} */
- * @see {@link Core Schema|https://yaml.org/spec/1.2/spec.html#schema/core}
- */
 
 /// <reference types="tree-sitter-cli/dsl" />
 

schema/core/package.json

@@ -1,6 +1,7 @@
 {
   "private": true,
   "scripts": {
-    "build": "tree-sitter generate --no-bindings"
+    "build": "tree-sitter generate --no-bindings",
+    "postbuild": "node ../update-schema.js core"
   }
 }

schema/core/src/parser.c

@@ -89,17 +89,19 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) {
   switch (state) {
     case 0:
       if (eof) ADVANCE(34);
-      if (lookahead == '+' ||
+      ADVANCE_MAP(
-          lookahead == '-') ADVANCE(1);
+        '.', 6,
-      if (lookahead == '.') ADVANCE(6);
+        '0', 37,
-      if (lookahead == '0') ADVANCE(37);
+        'F', 2,
-      if (lookahead == 'F') ADVANCE(2);
+        'N', 16,
-      if (lookahead == 'N') ADVANCE(16);
+        'T', 13,
-      if (lookahead == 'T') ADVANCE(13);
+        'f', 17,
-      if (lookahead == 'f') ADVANCE(17);
+        'n', 29,
-      if (lookahead == 'n') ADVANCE(29);
+        't', 26,
-      if (lookahead == 't') ADVANCE(26);
+        '~', 35,
-      if (lookahead == '~') ADVANCE(35);
+        '+', 1,
+        '-', 1,
+      );
       if (('1' <= lookahead && lookahead <= '9')) ADVANCE(38);
       END_STATE();
     case 1:
@@ -313,7 +315,7 @@ static const TSParseActionEntry ts_parse_actions[] = {
   [1] = {.entry = {.count = 1, .reusable = false}}, RECOVER(),
   [3] = {.entry = {.count = 1, .reusable = true}}, SHIFT(2),
   [5] = {.entry = {.count = 1, .reusable = false}}, SHIFT(2),
-  [7] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_scalar, 1),
+  [7] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_scalar, 1, 0, 0),
   [9] = {.entry = {.count = 1, .reusable = true}},  ACCEPT_INPUT(),
 };
 
@@ -328,7 +330,7 @@ extern "C" {
 #define TS_PUBLIC __attribute__((visibility("default")))
 #endif
 
-TS_PUBLIC const TSLanguage *tree_sitter_core_schema() {
+TS_PUBLIC const TSLanguage *tree_sitter_core_schema(void) {
   static const TSLanguage language = {
     .version = LANGUAGE_VERSION,
     .symbol_count = SYMBOL_COUNT,

schema/core/src/tree_sitter/parser.h

@@ -86,6 +86,11 @@ typedef union {
   } entry;
 } TSParseActionEntry;
 
+typedef struct {
+  int32_t start;
+  int32_t end;
+} TSCharacterRange;
+
 struct TSLanguage {
   uint32_t version;
   uint32_t symbol_count;
@@ -125,6 +130,24 @@ struct TSLanguage {
   const TSStateId *primary_state_ids;
 };
 
+static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
+  uint32_t index = 0;
+  uint32_t size = len - index;
+  while (size > 1) {
+    uint32_t half_size = size / 2;
+    uint32_t mid_index = index + half_size;
+    TSCharacterRange *range = &ranges[mid_index];
+    if (lookahead >= range->start && lookahead <= range->end) {
+      return true;
+    } else if (lookahead > range->end) {
+      index = mid_index;
+    }
+    size -= half_size;
+  }
+  TSCharacterRange *range = &ranges[index];
+  return (lookahead >= range->start && lookahead <= range->end);
+}
+
 /*
  *  Lexer Macros
  */
@@ -154,6 +177,17 @@ struct TSLanguage {
     goto next_state;         \
   }
 
+#define ADVANCE_MAP(...)                                              \
+  {                                                                   \
+    static const uint16_t map[] = { __VA_ARGS__ };                    \
+    for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) {  \
+      if (map[i] == lookahead) {                                      \
+        state = map[i + 1];                                           \
+        goto next_state;                                              \
+      }                                                               \
+    }                                                                 \
+  }
+
 #define SKIP(state_value) \
   {                       \
     skip = true;          \
@@ -203,14 +237,15 @@ struct TSLanguage {
     }                                 \
   }}
 
-#define REDUCE(symbol_val, child_count_val, ...) \
+#define REDUCE(symbol_name, children, precedence, prod_id) \
-  {{                                             \
+  {{                                                       \
-    .reduce = {                                  \
+    .reduce = {                                            \
-      .type = TSParseActionTypeReduce,           \
+      .type = TSParseActionTypeReduce,                     \
-      .symbol = symbol_val,                      \
+      .symbol = symbol_name,                               \
-      .child_count = child_count_val,            \
+      .child_count = children,                             \
-      __VA_ARGS__                                \
+      .dynamic_precedence = precedence,                    \
-    },                                           \
+      .production_id = prod_id                             \
+    },                                                     \
   }}
 
 #define RECOVER()                    \

schema/json/grammar.js

@@ -0,0 +1,21 @@
+/** @see {@link JSON Schema|https://yaml.org/spec/1.2.2/#102-json-schema} */
+
+/// <reference types="tree-sitter-cli/dsl" />
+
+module.exports = grammar({
+  name: "json_schema",
+
+  extras: _ => [],
+
+  rules: {
+    scalar: $ => choice($.null, $.bool, $.int, $.float),
+
+    null: _ => token("null"),
+
+    bool: _ => token(choice("true", "false")),
+
+    int: _ => token(/-?(0|[1-9][0-9]*)/),
+
+    float: _ => token(/-?(0|[1-9][0-9]*)(\.[0-9]*)?([eE][-+]?[0-9]+)?/),
+  },
+});

schema/json/package.json

@@ -0,0 +1,7 @@
+{
+  "private": true,
+  "scripts": {
+    "build": "tree-sitter generate --no-bindings",
+    "postbuild": "node ../update-schema.js json"
+  }
+}

schema/json/src/grammar.json

@@ -0,0 +1,69 @@
+{
+  "name": "json_schema",
+  "rules": {
+    "scalar": {
+      "type": "CHOICE",
+      "members": [
+        {
+          "type": "SYMBOL",
+          "name": "null"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "bool"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "int"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "float"
+        }
+      ]
+    },
+    "null": {
+      "type": "TOKEN",
+      "content": {
+        "type": "STRING",
+        "value": "null"
+      }
+    },
+    "bool": {
+      "type": "TOKEN",
+      "content": {
+        "type": "CHOICE",
+        "members": [
+          {
+            "type": "STRING",
+            "value": "true"
+          },
+          {
+            "type": "STRING",
+            "value": "false"
+          }
+        ]
+      }
+    },
+    "int": {
+      "type": "TOKEN",
+      "content": {
+        "type": "PATTERN",
+        "value": "-?(0|[1-9][0-9]*)"
+      }
+    },
+    "float": {
+      "type": "TOKEN",
+      "content": {
+        "type": "PATTERN",
+        "value": "-?(0|[1-9][0-9]*)(\\.[0-9]*)?([eE][-+]?[0-9]+)?"
+      }
+    }
+  },
+  "extras": [],
+  "conflicts": [],
+  "precedences": [],
+  "externals": [],
+  "inline": [],
+  "supertypes": []
+}

schema/json/src/node-types.json

@@ -0,0 +1,45 @@
+[
+  {
+    "type": "scalar",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": false,
+      "required": true,
+      "types": [
+        {
+          "type": "bool",
+          "named": true
+        },
+        {
+          "type": "float",
+          "named": true
+        },
+        {
+          "type": "int",
+          "named": true
+        },
+        {
+          "type": "null",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "bool",
+    "named": true
+  },
+  {
+    "type": "float",
+    "named": true
+  },
+  {
+    "type": "int",
+    "named": true
+  },
+  {
+    "type": "null",
+    "named": true
+  }
+]

schema/json/src/parser.c

@@ -0,0 +1,262 @@
+#include "tree_sitter/parser.h"
+
+#if defined(__GNUC__) || defined(__clang__)
+#pragma GCC diagnostic ignored "-Wmissing-field-initializers"
+#endif
+
+#define LANGUAGE_VERSION 14
+#define STATE_COUNT 4
+#define LARGE_STATE_COUNT 2
+#define SYMBOL_COUNT 6
+#define ALIAS_COUNT 0
+#define TOKEN_COUNT 5
+#define EXTERNAL_TOKEN_COUNT 0
+#define FIELD_COUNT 0
+#define MAX_ALIAS_SEQUENCE_LENGTH 1
+#define PRODUCTION_ID_COUNT 1
+
+enum ts_symbol_identifiers {
+  sym_null = 1,
+  sym_bool = 2,
+  sym_int = 3,
+  sym_float = 4,
+  sym_scalar = 5,
+};
+
+static const char * const ts_symbol_names[] = {
+  [ts_builtin_sym_end] = "end",
+  [sym_null] = "null",
+  [sym_bool] = "bool",
+  [sym_int] = "int",
+  [sym_float] = "float",
+  [sym_scalar] = "scalar",
+};
+
+static const TSSymbol ts_symbol_map[] = {
+  [ts_builtin_sym_end] = ts_builtin_sym_end,
+  [sym_null] = sym_null,
+  [sym_bool] = sym_bool,
+  [sym_int] = sym_int,
+  [sym_float] = sym_float,
+  [sym_scalar] = sym_scalar,
+};
+
+static const TSSymbolMetadata ts_symbol_metadata[] = {
+  [ts_builtin_sym_end] = {
+    .visible = false,
+    .named = true,
+  },
+  [sym_null] = {
+    .visible = true,
+    .named = true,
+  },
+  [sym_bool] = {
+    .visible = true,
+    .named = true,
+  },
+  [sym_int] = {
+    .visible = true,
+    .named = true,
+  },
+  [sym_float] = {
+    .visible = true,
+    .named = true,
+  },
+  [sym_scalar] = {
+    .visible = true,
+    .named = true,
+  },
+};
+
+static const TSSymbol ts_alias_sequences[PRODUCTION_ID_COUNT][MAX_ALIAS_SEQUENCE_LENGTH] = {
+  [0] = {0},
+};
+
+static const uint16_t ts_non_terminal_alias_map[] = {
+  0,
+};
+
+static const TSStateId ts_primary_state_ids[STATE_COUNT] = {
+  [0] = 0,
+  [1] = 1,
+  [2] = 2,
+  [3] = 3,
+};
+
+static bool ts_lex(TSLexer *lexer, TSStateId state) {
+  START_LEXER();
+  eof = lexer->eof(lexer);
+  switch (state) {
+    case 0:
+      if (eof) ADVANCE(13);
+      if (lookahead == '-') ADVANCE(1);
+      if (lookahead == '0') ADVANCE(16);
+      if (lookahead == 'f') ADVANCE(2);
+      if (lookahead == 'n') ADVANCE(10);
+      if (lookahead == 't') ADVANCE(7);
+      if (('1' <= lookahead && lookahead <= '9')) ADVANCE(17);
+      END_STATE();
+    case 1:
+      if (lookahead == '0') ADVANCE(16);
+      if (('1' <= lookahead && lookahead <= '9')) ADVANCE(17);
+      END_STATE();
+    case 2:
+      if (lookahead == 'a') ADVANCE(4);
+      END_STATE();
+    case 3:
+      if (lookahead == 'e') ADVANCE(15);
+      END_STATE();
+    case 4:
+      if (lookahead == 'l') ADVANCE(8);
+      END_STATE();
+    case 5:
+      if (lookahead == 'l') ADVANCE(14);
+      END_STATE();
+    case 6:
+      if (lookahead == 'l') ADVANCE(5);
+      END_STATE();
+    case 7:
+      if (lookahead == 'r') ADVANCE(9);
+      END_STATE();
+    case 8:
+      if (lookahead == 's') ADVANCE(3);
+      END_STATE();
+    case 9:
+      if (lookahead == 'u') ADVANCE(3);
+      END_STATE();
+    case 10:
+      if (lookahead == 'u') ADVANCE(6);
+      END_STATE();
+    case 11:
+      if (lookahead == '+' ||
+          lookahead == '-') ADVANCE(12);
+      if (('0' <= lookahead && lookahead <= '9')) ADVANCE(19);
+      END_STATE();
+    case 12:
+      if (('0' <= lookahead && lookahead <= '9')) ADVANCE(19);
+      END_STATE();
+    case 13:
+      ACCEPT_TOKEN(ts_builtin_sym_end);
+      END_STATE();
+    case 14:
+      ACCEPT_TOKEN(sym_null);
+      END_STATE();
+    case 15:
+      ACCEPT_TOKEN(sym_bool);
+      END_STATE();
+    case 16:
+      ACCEPT_TOKEN(sym_int);
+      if (lookahead == '.') ADVANCE(18);
+      if (lookahead == 'E' ||
+          lookahead == 'e') ADVANCE(11);
+      END_STATE();
+    case 17:
+      ACCEPT_TOKEN(sym_int);
+      if (lookahead == '.') ADVANCE(18);
+      if (lookahead == 'E' ||
+          lookahead == 'e') ADVANCE(11);
+      if (('0' <= lookahead && lookahead <= '9')) ADVANCE(17);
+      END_STATE();
+    case 18:
+      ACCEPT_TOKEN(sym_float);
+      if (lookahead == 'E' ||
+          lookahead == 'e') ADVANCE(11);
+      if (('0' <= lookahead && lookahead <= '9')) ADVANCE(18);
+      END_STATE();
+    case 19:
+      ACCEPT_TOKEN(sym_float);
+      if (('0' <= lookahead && lookahead <= '9')) ADVANCE(19);
+      END_STATE();
+    default:
+      return false;
+  }
+}
+
+static const TSLexMode ts_lex_modes[STATE_COUNT] = {
+  [0] = {.lex_state = 0},
+  [1] = {.lex_state = 0},
+  [2] = {.lex_state = 0},
+  [3] = {.lex_state = 0},
+};
+
+static const uint16_t ts_parse_table[LARGE_STATE_COUNT][SYMBOL_COUNT] = {
+  [0] = {
+    [ts_builtin_sym_end] = ACTIONS(1),
+    [sym_null] = ACTIONS(1),
+    [sym_bool] = ACTIONS(1),
+    [sym_int] = ACTIONS(1),
+    [sym_float] = ACTIONS(1),
+  },
+  [1] = {
+    [sym_scalar] = STATE(3),
+    [sym_null] = ACTIONS(3),
+    [sym_bool] = ACTIONS(3),
+    [sym_int] = ACTIONS(5),
+    [sym_float] = ACTIONS(5),
+  },
+};
+
+static const uint16_t ts_small_parse_table[] = {
+  [0] = 1,
+    ACTIONS(7), 1,
+      ts_builtin_sym_end,
+  [4] = 1,
+    ACTIONS(9), 1,
+      ts_builtin_sym_end,
+};
+
+static const uint32_t ts_small_parse_table_map[] = {
+  [SMALL_STATE(2)] = 0,
+  [SMALL_STATE(3)] = 4,
+};
+
+static const TSParseActionEntry ts_parse_actions[] = {
+  [0] = {.entry = {.count = 0, .reusable = false}},
+  [1] = {.entry = {.count = 1, .reusable = false}}, RECOVER(),
+  [3] = {.entry = {.count = 1, .reusable = true}}, SHIFT(2),
+  [5] = {.entry = {.count = 1, .reusable = false}}, SHIFT(2),
+  [7] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_scalar, 1, 0, 0),
+  [9] = {.entry = {.count = 1, .reusable = true}},  ACCEPT_INPUT(),
+};
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#ifdef TREE_SITTER_HIDE_SYMBOLS
+#define TS_PUBLIC
+#elif defined(_WIN32)
+#define TS_PUBLIC __declspec(dllexport)
+#else
+#define TS_PUBLIC __attribute__((visibility("default")))
+#endif
+
+TS_PUBLIC const TSLanguage *tree_sitter_json_schema(void) {
+  static const TSLanguage language = {
+    .version = LANGUAGE_VERSION,
+    .symbol_count = SYMBOL_COUNT,
+    .alias_count = ALIAS_COUNT,
+    .token_count = TOKEN_COUNT,
+    .external_token_count = EXTERNAL_TOKEN_COUNT,
+    .state_count = STATE_COUNT,
+    .large_state_count = LARGE_STATE_COUNT,
+    .production_id_count = PRODUCTION_ID_COUNT,
+    .field_count = FIELD_COUNT,
+    .max_alias_sequence_length = MAX_ALIAS_SEQUENCE_LENGTH,
+    .parse_table = &ts_parse_table[0][0],
+    .small_parse_table = ts_small_parse_table,
+    .small_parse_table_map = ts_small_parse_table_map,
+    .parse_actions = ts_parse_actions,
+    .symbol_names = ts_symbol_names,
+    .symbol_metadata = ts_symbol_metadata,
+    .public_symbol_map = ts_symbol_map,
+    .alias_map = ts_non_terminal_alias_map,
+    .alias_sequences = &ts_alias_sequences[0][0],
+    .lex_modes = ts_lex_modes,
+    .lex_fn = ts_lex,
+    .primary_state_ids = ts_primary_state_ids,
+  };
+  return &language;
+}
+#ifdef __cplusplus
+}
+#endif

schema/json/src/tree_sitter/alloc.h

@@ -0,0 +1,54 @@
+#ifndef TREE_SITTER_ALLOC_H_
+#define TREE_SITTER_ALLOC_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+// Allow clients to override allocation functions
+#ifdef TREE_SITTER_REUSE_ALLOCATOR
+
+extern void *(*ts_current_malloc)(size_t);
+extern void *(*ts_current_calloc)(size_t, size_t);
+extern void *(*ts_current_realloc)(void *, size_t);
+extern void (*ts_current_free)(void *);
+
+#ifndef ts_malloc
+#define ts_malloc  ts_current_malloc
+#endif
+#ifndef ts_calloc
+#define ts_calloc  ts_current_calloc
+#endif
+#ifndef ts_realloc
+#define ts_realloc ts_current_realloc
+#endif
+#ifndef ts_free
+#define ts_free    ts_current_free
+#endif
+
+#else
+
+#ifndef ts_malloc
+#define ts_malloc  malloc
+#endif
+#ifndef ts_calloc
+#define ts_calloc  calloc
+#endif
+#ifndef ts_realloc
+#define ts_realloc realloc
+#endif
+#ifndef ts_free
+#define ts_free    free
+#endif
+
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // TREE_SITTER_ALLOC_H_

schema/json/src/tree_sitter/array.h

@@ -0,0 +1,290 @@
+#ifndef TREE_SITTER_ARRAY_H_
+#define TREE_SITTER_ARRAY_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "./alloc.h"
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef _MSC_VER
+#pragma warning(disable : 4101)
+#elif defined(__GNUC__) || defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#endif
+
+#define Array(T)       \
+  struct {             \
+    T *contents;       \
+    uint32_t size;     \
+    uint32_t capacity; \
+  }
+
+/// Initialize an array.
+#define array_init(self) \
+  ((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
+
+/// Create an empty array.
+#define array_new() \
+  { NULL, 0, 0 }
+
+/// Get a pointer to the element at a given `index` in the array.
+#define array_get(self, _index) \
+  (assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index])
+
+/// Get a pointer to the first element in the array.
+#define array_front(self) array_get(self, 0)
+
+/// Get a pointer to the last element in the array.
+#define array_back(self) array_get(self, (self)->size - 1)
+
+/// Clear the array, setting its size to zero. Note that this does not free any
+/// memory allocated for the array's contents.
+#define array_clear(self) ((self)->size = 0)
+
+/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
+/// less than the array's current capacity, this function has no effect.
+#define array_reserve(self, new_capacity) \
+  _array__reserve((Array *)(self), array_elem_size(self), new_capacity)
+
+/// Free any memory allocated for this array. Note that this does not free any
+/// memory allocated for the array's contents.
+#define array_delete(self) _array__delete((Array *)(self))
+
+/// Push a new `element` onto the end of the array.
+#define array_push(self, element)                            \
+  (_array__grow((Array *)(self), 1, array_elem_size(self)), \
+   (self)->contents[(self)->size++] = (element))
+
+/// Increase the array's size by `count` elements.
+/// New elements are zero-initialized.
+#define array_grow_by(self, count) \
+  do { \
+    if ((count) == 0) break; \
+    _array__grow((Array *)(self), count, array_elem_size(self)); \
+    memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
+    (self)->size += (count); \
+  } while (0)
+
+/// Append all elements from one array to the end of another.
+#define array_push_all(self, other)                                       \
+  array_extend((self), (other)->size, (other)->contents)
+
+/// Append `count` elements to the end of the array, reading their values from the
+/// `contents` pointer.
+#define array_extend(self, count, contents)                    \
+  _array__splice(                                               \
+    (Array *)(self), array_elem_size(self), (self)->size, \
+    0, count,  contents                                        \
+  )
+
+/// Remove `old_count` elements from the array starting at the given `index`. At
+/// the same index, insert `new_count` new elements, reading their values from the
+/// `new_contents` pointer.
+#define array_splice(self, _index, old_count, new_count, new_contents)  \
+  _array__splice(                                                       \
+    (Array *)(self), array_elem_size(self), _index,                \
+    old_count, new_count, new_contents                                 \
+  )
+
+/// Insert one `element` into the array at the given `index`.
+#define array_insert(self, _index, element) \
+  _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
+
+/// Remove one element from the array at the given `index`.
+#define array_erase(self, _index) \
+  _array__erase((Array *)(self), array_elem_size(self), _index)
+
+/// Pop the last element off the array, returning the element by value.
+#define array_pop(self) ((self)->contents[--(self)->size])
+
+/// Assign the contents of one array to another, reallocating if necessary.
+#define array_assign(self, other) \
+  _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
+
+/// Swap one array with another
+#define array_swap(self, other) \
+  _array__swap((Array *)(self), (Array *)(other))
+
+/// Get the size of the array contents
+#define array_elem_size(self) (sizeof *(self)->contents)
+
+/// Search a sorted array for a given `needle` value, using the given `compare`
+/// callback to determine the order.
+///
+/// If an existing element is found to be equal to `needle`, then the `index`
+/// out-parameter is set to the existing value's index, and the `exists`
+/// out-parameter is set to true. Otherwise, `index` is set to an index where
+/// `needle` should be inserted in order to preserve the sorting, and `exists`
+/// is set to false.
+#define array_search_sorted_with(self, compare, needle, _index, _exists) \
+  _array__search_sorted(self, 0, compare, , needle, _index, _exists)
+
+/// Search a sorted array for a given `needle` value, using integer comparisons
+/// of a given struct field (specified with a leading dot) to determine the order.
+///
+/// See also `array_search_sorted_with`.
+#define array_search_sorted_by(self, field, needle, _index, _exists) \
+  _array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists)
+
+/// Insert a given `value` into a sorted array, using the given `compare`
+/// callback to determine the order.
+#define array_insert_sorted_with(self, compare, value) \
+  do { \
+    unsigned _index, _exists; \
+    array_search_sorted_with(self, compare, &(value), &_index, &_exists); \
+    if (!_exists) array_insert(self, _index, value); \
+  } while (0)
+
+/// Insert a given `value` into a sorted array, using integer comparisons of
+/// a given struct field (specified with a leading dot) to determine the order.
+///
+/// See also `array_search_sorted_by`.
+#define array_insert_sorted_by(self, field, value) \
+  do { \
+    unsigned _index, _exists; \
+    array_search_sorted_by(self, field, (value) field, &_index, &_exists); \
+    if (!_exists) array_insert(self, _index, value); \
+  } while (0)
+
+// Private
+
+typedef Array(void) Array;
+
+/// This is not what you're looking for, see `array_delete`.
+static inline void _array__delete(Array *self) {
+  if (self->contents) {
+    ts_free(self->contents);
+    self->contents = NULL;
+    self->size = 0;
+    self->capacity = 0;
+  }
+}
+
+/// This is not what you're looking for, see `array_erase`.
+static inline void _array__erase(Array *self, size_t element_size,
+                                uint32_t index) {
+  assert(index < self->size);
+  char *contents = (char *)self->contents;
+  memmove(contents + index * element_size, contents + (index + 1) * element_size,
+          (self->size - index - 1) * element_size);
+  self->size--;
+}
+
+/// This is not what you're looking for, see `array_reserve`.
+static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
+  if (new_capacity > self->capacity) {
+    if (self->contents) {
+      self->contents = ts_realloc(self->contents, new_capacity * element_size);
+    } else {
+      self->contents = ts_malloc(new_capacity * element_size);
+    }
+    self->capacity = new_capacity;
+  }
+}
+
+/// This is not what you're looking for, see `array_assign`.
+static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
+  _array__reserve(self, element_size, other->size);
+  self->size = other->size;
+  memcpy(self->contents, other->contents, self->size * element_size);
+}
+
+/// This is not what you're looking for, see `array_swap`.
+static inline void _array__swap(Array *self, Array *other) {
+  Array swap = *other;
+  *other = *self;
+  *self = swap;
+}
+
+/// This is not what you're looking for, see `array_push` or `array_grow_by`.
+static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
+  uint32_t new_size = self->size + count;
+  if (new_size > self->capacity) {
+    uint32_t new_capacity = self->capacity * 2;
+    if (new_capacity < 8) new_capacity = 8;
+    if (new_capacity < new_size) new_capacity = new_size;
+    _array__reserve(self, element_size, new_capacity);
+  }
+}
+
+/// This is not what you're looking for, see `array_splice`.
+static inline void _array__splice(Array *self, size_t element_size,
+                                 uint32_t index, uint32_t old_count,
+                                 uint32_t new_count, const void *elements) {
+  uint32_t new_size = self->size + new_count - old_count;
+  uint32_t old_end = index + old_count;
+  uint32_t new_end = index + new_count;
+  assert(old_end <= self->size);
+
+  _array__reserve(self, element_size, new_size);
+
+  char *contents = (char *)self->contents;
+  if (self->size > old_end) {
+    memmove(
+      contents + new_end * element_size,
+      contents + old_end * element_size,
+      (self->size - old_end) * element_size
+    );
+  }
+  if (new_count > 0) {
+    if (elements) {
+      memcpy(
+        (contents + index * element_size),
+        elements,
+        new_count * element_size
+      );
+    } else {
+      memset(
+        (contents + index * element_size),
+        0,
+        new_count * element_size
+      );
+    }
+  }
+  self->size += new_count - old_count;
+}
+
+/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
+/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`.
+#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \
+  do { \
+    *(_index) = start; \
+    *(_exists) = false; \
+    uint32_t size = (self)->size - *(_index); \
+    if (size == 0) break; \
+    int comparison; \
+    while (size > 1) { \
+      uint32_t half_size = size / 2; \
+      uint32_t mid_index = *(_index) + half_size; \
+      comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \
+      if (comparison <= 0) *(_index) = mid_index; \
+      size -= half_size; \
+    } \
+    comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \
+    if (comparison == 0) *(_exists) = true; \
+    else if (comparison < 0) *(_index) += 1; \
+  } while (0)
+
+/// Helper macro for the `_sorted_by` routines below. This takes the left (existing)
+/// parameter by reference in order to work with the generic sorting function above.
+#define _compare_int(a, b) ((int)*(a) - (int)(b))
+
+#ifdef _MSC_VER
+#pragma warning(default : 4101)
+#elif defined(__GNUC__) || defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_ARRAY_H_

schema/json/src/tree_sitter/parser.h

@@ -0,0 +1,265 @@
+#ifndef TREE_SITTER_PARSER_H_
+#define TREE_SITTER_PARSER_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#define ts_builtin_sym_error ((TSSymbol)-1)
+#define ts_builtin_sym_end 0
+#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
+
+#ifndef TREE_SITTER_API_H_
+typedef uint16_t TSStateId;
+typedef uint16_t TSSymbol;
+typedef uint16_t TSFieldId;
+typedef struct TSLanguage TSLanguage;
+#endif
+
+typedef struct {
+  TSFieldId field_id;
+  uint8_t child_index;
+  bool inherited;
+} TSFieldMapEntry;
+
+typedef struct {
+  uint16_t index;
+  uint16_t length;
+} TSFieldMapSlice;
+
+typedef struct {
+  bool visible;
+  bool named;
+  bool supertype;
+} TSSymbolMetadata;
+
+typedef struct TSLexer TSLexer;
+
+struct TSLexer {
+  int32_t lookahead;
+  TSSymbol result_symbol;
+  void (*advance)(TSLexer *, bool);
+  void (*mark_end)(TSLexer *);
+  uint32_t (*get_column)(TSLexer *);
+  bool (*is_at_included_range_start)(const TSLexer *);
+  bool (*eof)(const TSLexer *);
+};
+
+typedef enum {
+  TSParseActionTypeShift,
+  TSParseActionTypeReduce,
+  TSParseActionTypeAccept,
+  TSParseActionTypeRecover,
+} TSParseActionType;
+
+typedef union {
+  struct {
+    uint8_t type;
+    TSStateId state;
+    bool extra;
+    bool repetition;
+  } shift;
+  struct {
+    uint8_t type;
+    uint8_t child_count;
+    TSSymbol symbol;
+    int16_t dynamic_precedence;
+    uint16_t production_id;
+  } reduce;
+  uint8_t type;
+} TSParseAction;
+
+typedef struct {
+  uint16_t lex_state;
+  uint16_t external_lex_state;
+} TSLexMode;
+
+typedef union {
+  TSParseAction action;
+  struct {
+    uint8_t count;
+    bool reusable;
+  } entry;
+} TSParseActionEntry;
+
+typedef struct {
+  int32_t start;
+  int32_t end;
+} TSCharacterRange;
+
+struct TSLanguage {
+  uint32_t version;
+  uint32_t symbol_count;
+  uint32_t alias_count;
+  uint32_t token_count;
+  uint32_t external_token_count;
+  uint32_t state_count;
+  uint32_t large_state_count;
+  uint32_t production_id_count;
+  uint32_t field_count;
+  uint16_t max_alias_sequence_length;
+  const uint16_t *parse_table;
+  const uint16_t *small_parse_table;
+  const uint32_t *small_parse_table_map;
+  const TSParseActionEntry *parse_actions;
+  const char * const *symbol_names;
+  const char * const *field_names;
+  const TSFieldMapSlice *field_map_slices;
+  const TSFieldMapEntry *field_map_entries;
+  const TSSymbolMetadata *symbol_metadata;
+  const TSSymbol *public_symbol_map;
+  const uint16_t *alias_map;
+  const TSSymbol *alias_sequences;
+  const TSLexMode *lex_modes;
+  bool (*lex_fn)(TSLexer *, TSStateId);
+  bool (*keyword_lex_fn)(TSLexer *, TSStateId);
+  TSSymbol keyword_capture_token;
+  struct {
+    const bool *states;
+    const TSSymbol *symbol_map;
+    void *(*create)(void);
+    void (*destroy)(void *);
+    bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist);
+    unsigned (*serialize)(void *, char *);
+    void (*deserialize)(void *, const char *, unsigned);
+  } external_scanner;
+  const TSStateId *primary_state_ids;
+};
+
+static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
+  uint32_t index = 0;
+  uint32_t size = len - index;
+  while (size > 1) {
+    uint32_t half_size = size / 2;
+    uint32_t mid_index = index + half_size;
+    TSCharacterRange *range = &ranges[mid_index];
+    if (lookahead >= range->start && lookahead <= range->end) {
+      return true;
+    } else if (lookahead > range->end) {
+      index = mid_index;
+    }
+    size -= half_size;
+  }
+  TSCharacterRange *range = &ranges[index];
+  return (lookahead >= range->start && lookahead <= range->end);
+}
+
+/*
+ *  Lexer Macros
+ */
+
+#ifdef _MSC_VER
+#define UNUSED __pragma(warning(suppress : 4101))
+#else
+#define UNUSED __attribute__((unused))
+#endif
+
+#define START_LEXER()           \
+  bool result = false;          \
+  bool skip = false;            \
+  UNUSED                        \
+  bool eof = false;             \
+  int32_t lookahead;            \
+  goto start;                   \
+  next_state:                   \
+  lexer->advance(lexer, skip);  \
+  start:                        \
+  skip = false;                 \
+  lookahead = lexer->lookahead;
+
+#define ADVANCE(state_value) \
+  {                          \
+    state = state_value;     \
+    goto next_state;         \
+  }
+
+#define ADVANCE_MAP(...)                                              \
+  {                                                                   \
+    static const uint16_t map[] = { __VA_ARGS__ };                    \
+    for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) {  \
+      if (map[i] == lookahead) {                                      \
+        state = map[i + 1];                                           \
+        goto next_state;                                              \
+      }                                                               \
+    }                                                                 \
+  }
+
+#define SKIP(state_value) \
+  {                       \
+    skip = true;          \
+    state = state_value;  \
+    goto next_state;      \
+  }
+
+#define ACCEPT_TOKEN(symbol_value)     \
+  result = true;                       \
+  lexer->result_symbol = symbol_value; \
+  lexer->mark_end(lexer);
+
+#define END_STATE() return result;
+
+/*
+ *  Parse Table Macros
+ */
+
+#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
+
+#define STATE(id) id
+
+#define ACTIONS(id) id
+
+#define SHIFT(state_value)            \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .state = (state_value)          \
+    }                                 \
+  }}
+
+#define SHIFT_REPEAT(state_value)     \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .state = (state_value),         \
+      .repetition = true              \
+    }                                 \
+  }}
+
+#define SHIFT_EXTRA()                 \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .extra = true                   \
+    }                                 \
+  }}
+
+#define REDUCE(symbol_name, children, precedence, prod_id) \
+  {{                                                       \
+    .reduce = {                                            \
+      .type = TSParseActionTypeReduce,                     \
+      .symbol = symbol_name,                               \
+      .child_count = children,                             \
+      .dynamic_precedence = precedence,                    \
+      .production_id = prod_id                             \
+    },                                                     \
+  }}
+
+#define RECOVER()                    \
+  {{                                 \
+    .type = TSParseActionTypeRecover \
+  }}
+
+#define ACCEPT_INPUT()              \
+  {{                                \
+    .type = TSParseActionTypeAccept \
+  }}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_PARSER_H_

schema/update-schema.js

@@ -6,7 +6,9 @@ const { writeFileSync } = require("fs");
 const { readFile } = require("fs/promises");
 const { join } = require("path");
 
-readFile(join(__dirname, process.argv[2] ?? "core", "src", "parser.c"), "utf8").then(input => {
+const schema = process.argv[2] ?? "core";
+
+readFile(join(__dirname, schema, "src", "parser.c"), "utf8").then(input => {
   const cases = extractCases(input);
   const enums = ["RS_STR"];
   const content = "switch (sch_stt) " + block([
@@ -14,6 +16,9 @@ readFile(join(__dirname, process.argv[2] ?? "core", "src", "parser.c"), "utf8").
     cases
       .map(([key, { content }]) => `${(key === "default" ? "default:" : `case ${key}:`)}\n${indent(content)}`)
       .join("\n  END_STATE();\n")
+      .replace(/\s+ADVANCE_MAP\(([^]+?)\);\n/, (_, map) => {
+        return map.replace(/'(.)', (\d+),/g, "if (lookahead == '$1') ADVANCE($2);");
+      })
       .replace(/ADVANCE\((\d+)\);/g, (_, state) => {
         const stateCase = cases.find(([key]) => key === state);
         if (stateCase) {
@@ -37,7 +42,7 @@ readFile(join(__dirname, process.argv[2] ?? "core", "src", "parser.c"), "utf8").
       .replace(/lookahead/g, "cur_chr"),
   ]);
   writeFileSync(
-    join(__dirname, "..", "src", "schema.generated.c"),
+    join(__dirname, "..", "src", `schema.${schema}.c`),
     [
       "#include <stdlib.h>",
       "#define SCH_STT_FRZ -1",

setup.py

@@ -39,7 +39,7 @@ setup(
                 "src/scanner.c",
             ],
             extra_compile_args=(
-                ["-std=c11"] if system() != "Windows" else []
+                ["-std=c11"] if system() != "Windows" else ["/std:c11"]
             ),
             define_macros=[
                 ("Py_LIMITED_API", "0x03080000"),

src/grammar.json

@@ -6950,6 +6950,10 @@
     {
       "type": "SYMBOL",
       "name": "comment"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_err_rec"
     }
   ],
   "inline": [

src/scanner.c

@@ -1,58 +1,67 @@
 #include "tree_sitter/array.h"
 #include "tree_sitter/parser.h"
 
-#include "./schema.generated.c"
+#define _str(x) #x
+#define _file(x) _str(schema.x.c)
+
+#ifndef YAML_SCHEMA
+#define YAML_SCHEMA core
+#endif
+
+#include _file(YAML_SCHEMA)
 
 // clang-format off
 
 typedef enum {
-  END_OF_FILE,
+    END_OF_FILE,
 
-  S_DIR_YML_BGN,  R_DIR_YML_VER,
+    S_DIR_YML_BGN,  R_DIR_YML_VER,
-  S_DIR_TAG_BGN,  R_DIR_TAG_HDL,  R_DIR_TAG_PFX,
+    S_DIR_TAG_BGN,  R_DIR_TAG_HDL,  R_DIR_TAG_PFX,
-  S_DIR_RSV_BGN,  R_DIR_RSV_PRM,
+    S_DIR_RSV_BGN,  R_DIR_RSV_PRM,
-  S_DRS_END,
+    S_DRS_END,
-  S_DOC_END,
+    S_DOC_END,
-  R_BLK_SEQ_BGN,  BR_BLK_SEQ_BGN, B_BLK_SEQ_BGN,
+    R_BLK_SEQ_BGN,  BR_BLK_SEQ_BGN, B_BLK_SEQ_BGN,
-  R_BLK_KEY_BGN,  BR_BLK_KEY_BGN, B_BLK_KEY_BGN,
+    R_BLK_KEY_BGN,  BR_BLK_KEY_BGN, B_BLK_KEY_BGN,
-  R_BLK_VAL_BGN,  BR_BLK_VAL_BGN, B_BLK_VAL_BGN,
+    R_BLK_VAL_BGN,  BR_BLK_VAL_BGN, B_BLK_VAL_BGN,
-  R_BLK_IMP_BGN,
+    R_BLK_IMP_BGN,
-  R_BLK_LIT_BGN,  BR_BLK_LIT_BGN,
+    R_BLK_LIT_BGN,  BR_BLK_LIT_BGN,
-  R_BLK_FLD_BGN,  BR_BLK_FLD_BGN,
+    R_BLK_FLD_BGN,  BR_BLK_FLD_BGN,
-                  BR_BLK_STR_CTN,
+    BR_BLK_STR_CTN,
-  R_FLW_SEQ_BGN,  BR_FLW_SEQ_BGN, B_FLW_SEQ_BGN,
+    R_FLW_SEQ_BGN,  BR_FLW_SEQ_BGN, B_FLW_SEQ_BGN,
-  R_FLW_SEQ_END,  BR_FLW_SEQ_END, B_FLW_SEQ_END,
+    R_FLW_SEQ_END,  BR_FLW_SEQ_END, B_FLW_SEQ_END,
-  R_FLW_MAP_BGN,  BR_FLW_MAP_BGN, B_FLW_MAP_BGN,
+    R_FLW_MAP_BGN,  BR_FLW_MAP_BGN, B_FLW_MAP_BGN,
-  R_FLW_MAP_END,  BR_FLW_MAP_END, B_FLW_MAP_END,
+    R_FLW_MAP_END,  BR_FLW_MAP_END, B_FLW_MAP_END,
-  R_FLW_SEP_BGN,  BR_FLW_SEP_BGN,
+    R_FLW_SEP_BGN,  BR_FLW_SEP_BGN,
-  R_FLW_KEY_BGN,  BR_FLW_KEY_BGN,
+    R_FLW_KEY_BGN,  BR_FLW_KEY_BGN,
-  R_FLW_JSV_BGN,  BR_FLW_JSV_BGN,
+    R_FLW_JSV_BGN,  BR_FLW_JSV_BGN,
-  R_FLW_NJV_BGN,  BR_FLW_NJV_BGN,
+    R_FLW_NJV_BGN,  BR_FLW_NJV_BGN,
-  R_DQT_STR_BGN,  BR_DQT_STR_BGN, B_DQT_STR_BGN,
+    R_DQT_STR_BGN,  BR_DQT_STR_BGN, B_DQT_STR_BGN,
-  R_DQT_STR_CTN,  BR_DQT_STR_CTN,
+    R_DQT_STR_CTN,  BR_DQT_STR_CTN,
-  R_DQT_ESC_NWL,  BR_DQT_ESC_NWL,
+    R_DQT_ESC_NWL,  BR_DQT_ESC_NWL,
-  R_DQT_ESC_SEQ,  BR_DQT_ESC_SEQ,
+    R_DQT_ESC_SEQ,  BR_DQT_ESC_SEQ,
-  R_DQT_STR_END,  BR_DQT_STR_END,
+    R_DQT_STR_END,  BR_DQT_STR_END,
-  R_SQT_STR_BGN,  BR_SQT_STR_BGN, B_SQT_STR_BGN,
+    R_SQT_STR_BGN,  BR_SQT_STR_BGN, B_SQT_STR_BGN,
-  R_SQT_STR_CTN,  BR_SQT_STR_CTN,
+    R_SQT_STR_CTN,  BR_SQT_STR_CTN,
-  R_SQT_ESC_SQT,  BR_SQT_ESC_SQT,
+    R_SQT_ESC_SQT,  BR_SQT_ESC_SQT,
-  R_SQT_STR_END,  BR_SQT_STR_END,
+    R_SQT_STR_END,  BR_SQT_STR_END,
 
-  R_SGL_PLN_NUL_BLK, BR_SGL_PLN_NUL_BLK, B_SGL_PLN_NUL_BLK, R_SGL_PLN_NUL_FLW, BR_SGL_PLN_NUL_FLW,
+    R_SGL_PLN_NUL_BLK, BR_SGL_PLN_NUL_BLK, B_SGL_PLN_NUL_BLK, R_SGL_PLN_NUL_FLW, BR_SGL_PLN_NUL_FLW,
-  R_SGL_PLN_BOL_BLK, BR_SGL_PLN_BOL_BLK, B_SGL_PLN_BOL_BLK, R_SGL_PLN_BOL_FLW, BR_SGL_PLN_BOL_FLW,
+    R_SGL_PLN_BOL_BLK, BR_SGL_PLN_BOL_BLK, B_SGL_PLN_BOL_BLK, R_SGL_PLN_BOL_FLW, BR_SGL_PLN_BOL_FLW,
-  R_SGL_PLN_INT_BLK, BR_SGL_PLN_INT_BLK, B_SGL_PLN_INT_BLK, R_SGL_PLN_INT_FLW, BR_SGL_PLN_INT_FLW,
+    R_SGL_PLN_INT_BLK, BR_SGL_PLN_INT_BLK, B_SGL_PLN_INT_BLK, R_SGL_PLN_INT_FLW, BR_SGL_PLN_INT_FLW,
-  R_SGL_PLN_FLT_BLK, BR_SGL_PLN_FLT_BLK, B_SGL_PLN_FLT_BLK, R_SGL_PLN_FLT_FLW, BR_SGL_PLN_FLT_FLW,
+    R_SGL_PLN_FLT_BLK, BR_SGL_PLN_FLT_BLK, B_SGL_PLN_FLT_BLK, R_SGL_PLN_FLT_FLW, BR_SGL_PLN_FLT_FLW,
-  R_SGL_PLN_STR_BLK, BR_SGL_PLN_STR_BLK, B_SGL_PLN_STR_BLK, R_SGL_PLN_STR_FLW, BR_SGL_PLN_STR_FLW,
+    R_SGL_PLN_STR_BLK, BR_SGL_PLN_STR_BLK, B_SGL_PLN_STR_BLK, R_SGL_PLN_STR_FLW, BR_SGL_PLN_STR_FLW,
 
-  R_MTL_PLN_STR_BLK,  BR_MTL_PLN_STR_BLK,
+    R_MTL_PLN_STR_BLK,  BR_MTL_PLN_STR_BLK,
-  R_MTL_PLN_STR_FLW,  BR_MTL_PLN_STR_FLW,
+    R_MTL_PLN_STR_FLW,  BR_MTL_PLN_STR_FLW,
 
-  R_TAG,     BR_TAG,     B_TAG,
+    R_TAG,     BR_TAG,     B_TAG,
-  R_ACR_BGN, BR_ACR_BGN, B_ACR_BGN, R_ACR_CTN,
+    R_ACR_BGN, BR_ACR_BGN, B_ACR_BGN, R_ACR_CTN,
-  R_ALS_BGN, BR_ALS_BGN, B_ALS_BGN, R_ALS_CTN,
+    R_ALS_BGN, BR_ALS_BGN, B_ALS_BGN, R_ALS_CTN,
 
-  BL,
+    BL,
-  COMMENT,
+    COMMENT,
+
+    ERR_REC,
 } TokenType;
 
 // clang-format on
@@ -117,11 +126,11 @@ typedef enum {
     }
 
 #define SGL_PLN_SYM(POS, CTX)                                                                                          \
-    (scanner->rlt_sch == RS_NULL   ? POS##_SGL_PLN_NUL_##CTX                                                            \
+    (scanner->rlt_sch == RS_NULL    ? POS##_SGL_PLN_NUL_##CTX                                                          \
-     : scanner->rlt_sch == RS_BOOL ? POS##_SGL_PLN_BOL_##CTX                                                            \
+     : scanner->rlt_sch == RS_BOOL  ? POS##_SGL_PLN_BOL_##CTX                                                          \
-     : scanner->rlt_sch == RS_INT ? POS##_SGL_PLN_INT_##CTX                                                            \
+     : scanner->rlt_sch == RS_INT   ? POS##_SGL_PLN_INT_##CTX                                                          \
-     : scanner->rlt_sch == RS_FLOAT ? POS##_SGL_PLN_FLT_##CTX                                                            \
+     : scanner->rlt_sch == RS_FLOAT ? POS##_SGL_PLN_FLT_##CTX                                                          \
-                                  : POS##_SGL_PLN_STR_##CTX)
+                                    : POS##_SGL_PLN_STR_##CTX)
 
 typedef struct {
     int16_t row;
@@ -143,20 +152,27 @@ typedef struct {
 } Scanner;
 
 static unsigned serialize(Scanner *scanner, char *buffer) {
-    size_t i = 0;
+    size_t size = 0;
-    buffer[i++] = (char)scanner->row;
+    *(int16_t *)&buffer[size] = scanner->row;
-    buffer[i++] = (char)scanner->col;
+    size += sizeof(int16_t);
-    buffer[i++] = (char)scanner->blk_imp_row;
+    *(int16_t *)&buffer[size] = scanner->col;
-    buffer[i++] = (char)scanner->blk_imp_col;
+    size += sizeof(int16_t);
-    buffer[i++] = (char)scanner->blk_imp_tab;
+    *(int16_t *)&buffer[size] = scanner->blk_imp_row;
+    size += sizeof(int16_t);
+    *(int16_t *)&buffer[size] = scanner->blk_imp_col;
+    size += sizeof(int16_t);
+    *(int16_t *)&buffer[size] = scanner->blk_imp_tab;
+    size += sizeof(int16_t);
     int16_t *typ_itr = scanner->ind_typ_stk.contents + 1;
     int16_t *typ_end = scanner->ind_typ_stk.contents + scanner->ind_typ_stk.size;
     int16_t *len_itr = scanner->ind_len_stk.contents + 1;
-    for (; typ_itr != typ_end && i < TREE_SITTER_SERIALIZATION_BUFFER_SIZE; ++typ_itr, ++len_itr) {
+    for (; typ_itr != typ_end && size < TREE_SITTER_SERIALIZATION_BUFFER_SIZE; ++typ_itr, ++len_itr) {
-        buffer[i++] = (char)*typ_itr;
+        *(int16_t *)&buffer[size] = *typ_itr;
-        buffer[i++] = (char)*len_itr;
+        size += sizeof(int16_t);
+        *(int16_t *)&buffer[size] = *len_itr;
+        size += sizeof(int16_t);
     }
-    return i;
+    return size;
 }
 
 static void deserialize(Scanner *scanner, const char *buffer, unsigned length) {
@@ -170,16 +186,24 @@ static void deserialize(Scanner *scanner, const char *buffer, unsigned length) {
     array_delete(&scanner->ind_len_stk);
     array_push(&scanner->ind_len_stk, -1);
     if (length > 0) {
-        size_t i = 0;
+        size_t size = 0;
-        scanner->row = (int16_t)buffer[i++];
+        scanner->row = *(int16_t *)&buffer[size];
-        scanner->col = (int16_t)buffer[i++];
+        size += sizeof(int16_t);
-        scanner->blk_imp_row = (int16_t)buffer[i++];
+        scanner->col = *(int16_t *)&buffer[size];
-        scanner->blk_imp_col = (int16_t)buffer[i++];
+        size += sizeof(int16_t);
-        scanner->blk_imp_tab = (int16_t)buffer[i++];
+        scanner->blk_imp_row = *(int16_t *)&buffer[size];
-        while (i < length) {
+        size += sizeof(int16_t);
-            array_push(&scanner->ind_typ_stk, (int16_t)buffer[i++]);
+        scanner->blk_imp_col = *(int16_t *)&buffer[size];
-            array_push(&scanner->ind_len_stk, (int16_t)buffer[i++]);
+        size += sizeof(int16_t);
+        scanner->blk_imp_tab = *(int16_t *)&buffer[size];
+        size += sizeof(int16_t);
+        while (size < length) {
+            array_push(&scanner->ind_typ_stk, *(int16_t *)&buffer[size]);
+            size += sizeof(int16_t);
+            array_push(&scanner->ind_len_stk, *(int16_t *)&buffer[size]);
+            size += sizeof(int16_t);
         }
+        assert(size == length);
     }
 }
 
@@ -246,7 +270,9 @@ static inline bool is_wht(int32_t c) { return is_wsp(c) || is_nwl(c) || c == 0;
 
 static inline bool is_ns_dec_digit(int32_t c) { return c >= '0' && c <= '9'; }
 
-static inline bool is_ns_hex_digit(int32_t c) { return is_ns_dec_digit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); }
+static inline bool is_ns_hex_digit(int32_t c) {
+    return is_ns_dec_digit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
+}
 
 static inline bool is_ns_word_char(int32_t c) {
     return c == '-' || (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
@@ -792,7 +818,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
     bool allow_comment = !(valid_symbols[R_DQT_STR_CTN] || valid_symbols[BR_DQT_STR_CTN] ||
                            valid_symbols[R_SQT_STR_CTN] || valid_symbols[BR_SQT_STR_CTN]);
     int16_t *ind_ptr = scanner->ind_len_stk.contents + scanner->ind_len_stk.size - 1;
-    int16_t *ind_end = scanner->ind_len_stk.contents;
+    int16_t *ind_end = scanner->ind_len_stk.contents - 1;
     int16_t cur_ind = *ind_ptr--;
     int16_t prt_ind = ind_ptr == ind_end ? -1 : *ind_ptr;
     int16_t cur_ind_typ = *array_back(&scanner->ind_typ_stk);
@@ -1346,7 +1372,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
         return false;
     }
 
-    return false;
+    return !valid_symbols[ERR_REC];
 }
 
 void *tree_sitter_yaml_external_scanner_create() {
@@ -1357,6 +1383,8 @@ void *tree_sitter_yaml_external_scanner_create() {
 
 void tree_sitter_yaml_external_scanner_destroy(void *payload) {
     Scanner *scanner = (Scanner *)payload;
+    array_delete(&scanner->ind_len_stk);
+    array_delete(&scanner->ind_typ_stk);
     ts_free(scanner);
 }
 

src/schema.core.c

@@ -15,18 +15,18 @@ static int8_t adv_sch_stt(int8_t sch_stt, int32_t cur_chr, ResultSchema *rlt_sch
     case SCH_STT_FRZ:
       break;
     case 0:
-      if (cur_chr == '+' ||
+        if (cur_chr == '.') {*rlt_sch = RS_STR; return 6;}
-          cur_chr == '-') {*rlt_sch = RS_STR; return 1;}
+        if (cur_chr == '0') {*rlt_sch = RS_INT; return 37;}
-      if (cur_chr == '.') {*rlt_sch = RS_STR; return 6;}
+        if (cur_chr == 'F') {*rlt_sch = RS_STR; return 2;}
-      if (cur_chr == '0') {*rlt_sch = RS_INT; return 37;}
+        if (cur_chr == 'N') {*rlt_sch = RS_STR; return 16;}
-      if (cur_chr == 'F') {*rlt_sch = RS_STR; return 2;}
+        if (cur_chr == 'T') {*rlt_sch = RS_STR; return 13;}
-      if (cur_chr == 'N') {*rlt_sch = RS_STR; return 16;}
+        if (cur_chr == 'f') {*rlt_sch = RS_STR; return 17;}
-      if (cur_chr == 'T') {*rlt_sch = RS_STR; return 13;}
+        if (cur_chr == 'n') {*rlt_sch = RS_STR; return 29;}
-      if (cur_chr == 'f') {*rlt_sch = RS_STR; return 17;}
+        if (cur_chr == 't') {*rlt_sch = RS_STR; return 26;}
-      if (cur_chr == 'n') {*rlt_sch = RS_STR; return 29;}
+        if (cur_chr == '~') {*rlt_sch = RS_NULL; return 35;}
-      if (cur_chr == 't') {*rlt_sch = RS_STR; return 26;}
+        if (cur_chr == '+') {*rlt_sch = RS_STR; return 1;}
-      if (cur_chr == '~') {*rlt_sch = RS_NULL; return 35;}
+        if (cur_chr == '-') {*rlt_sch = RS_STR; return 1;}
-      if (('1' <= cur_chr && cur_chr <= '9')) {*rlt_sch = RS_INT; return 38;}
+        if (('1' <= cur_chr && cur_chr <= '9')) {*rlt_sch = RS_INT; return 38;}
       break;
     case 1:
       if (cur_chr == '.') {*rlt_sch = RS_STR; return 7;}

src/schema.json.c

@@ -0,0 +1,102 @@
+#include <stdlib.h>
+
+#define SCH_STT_FRZ -1
+
+typedef enum {
+  RS_STR,
+  RS_INT,
+  RS_BOOL,
+  RS_NULL,
+  RS_FLOAT,
+} ResultSchema;
+
+static int8_t adv_sch_stt(int8_t sch_stt, int32_t cur_chr, ResultSchema *rlt_sch) {
+  switch (sch_stt) {
+    case SCH_STT_FRZ:
+      break;
+    case 0:
+      if (cur_chr == '-') {*rlt_sch = RS_STR; return 1;}
+      if (cur_chr == '0') {*rlt_sch = RS_INT; return 16;}
+      if (cur_chr == 'f') {*rlt_sch = RS_STR; return 2;}
+      if (cur_chr == 'n') {*rlt_sch = RS_STR; return 10;}
+      if (cur_chr == 't') {*rlt_sch = RS_STR; return 7;}
+      if (('1' <= cur_chr && cur_chr <= '9')) {*rlt_sch = RS_INT; return 17;}
+      break;
+    case 1:
+      if (cur_chr == '0') {*rlt_sch = RS_INT; return 16;}
+      if (('1' <= cur_chr && cur_chr <= '9')) {*rlt_sch = RS_INT; return 17;}
+      break;
+    case 2:
+      if (cur_chr == 'a') {*rlt_sch = RS_STR; return 4;}
+      break;
+    case 3:
+      if (cur_chr == 'e') {*rlt_sch = RS_BOOL; return 15;}
+      break;
+    case 4:
+      if (cur_chr == 'l') {*rlt_sch = RS_STR; return 8;}
+      break;
+    case 5:
+      if (cur_chr == 'l') {*rlt_sch = RS_NULL; return 14;}
+      break;
+    case 6:
+      if (cur_chr == 'l') {*rlt_sch = RS_STR; return 5;}
+      break;
+    case 7:
+      if (cur_chr == 'r') {*rlt_sch = RS_STR; return 9;}
+      break;
+    case 8:
+      if (cur_chr == 's') {*rlt_sch = RS_STR; return 3;}
+      break;
+    case 9:
+      if (cur_chr == 'u') {*rlt_sch = RS_STR; return 3;}
+      break;
+    case 10:
+      if (cur_chr == 'u') {*rlt_sch = RS_STR; return 6;}
+      break;
+    case 11:
+      if (cur_chr == '+' ||
+          cur_chr == '-') {*rlt_sch = RS_STR; return 12;}
+      if (('0' <= cur_chr && cur_chr <= '9')) {*rlt_sch = RS_FLOAT; return 19;}
+      break;
+    case 12:
+      if (('0' <= cur_chr && cur_chr <= '9')) {*rlt_sch = RS_FLOAT; return 19;}
+      break;
+    case 13:
+      abort();
+      break;
+    case 14:
+      *rlt_sch = RS_NULL;
+      break;
+    case 15:
+      *rlt_sch = RS_BOOL;
+      break;
+    case 16:
+      *rlt_sch = RS_INT;
+      if (cur_chr == '.') {*rlt_sch = RS_FLOAT; return 18;}
+      if (cur_chr == 'E' ||
+          cur_chr == 'e') {*rlt_sch = RS_STR; return 11;}
+      break;
+    case 17:
+      *rlt_sch = RS_INT;
+      if (cur_chr == '.') {*rlt_sch = RS_FLOAT; return 18;}
+      if (cur_chr == 'E' ||
+          cur_chr == 'e') {*rlt_sch = RS_STR; return 11;}
+      if (('0' <= cur_chr && cur_chr <= '9')) {*rlt_sch = RS_INT; return 17;}
+      break;
+    case 18:
+      *rlt_sch = RS_FLOAT;
+      if (cur_chr == 'E' ||
+          cur_chr == 'e') {*rlt_sch = RS_STR; return 11;}
+      if (('0' <= cur_chr && cur_chr <= '9')) {*rlt_sch = RS_FLOAT; return 18;}
+      break;
+    case 19:
+      *rlt_sch = RS_FLOAT;
+      if (('0' <= cur_chr && cur_chr <= '9')) {*rlt_sch = RS_FLOAT; return 19;}
+      break;
+    default:
+      *rlt_sch = RS_STR;
+      return SCH_STT_FRZ;
+  }
+  if (cur_chr != '\r' && cur_chr != '\n' && cur_chr != ' ' && cur_chr != 0) *rlt_sch = RS_STR;
+  return SCH_STT_FRZ;
+}

src/tree_sitter/parser.h

@@ -86,6 +86,11 @@ typedef union {
   } entry;
 } TSParseActionEntry;
 
+typedef struct {
+  int32_t start;
+  int32_t end;
+} TSCharacterRange;
+
 struct TSLanguage {
   uint32_t version;
   uint32_t symbol_count;
@@ -125,6 +130,24 @@ struct TSLanguage {
   const TSStateId *primary_state_ids;
 };
 
+static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
+  uint32_t index = 0;
+  uint32_t size = len - index;
+  while (size > 1) {
+    uint32_t half_size = size / 2;
+    uint32_t mid_index = index + half_size;
+    TSCharacterRange *range = &ranges[mid_index];
+    if (lookahead >= range->start && lookahead <= range->end) {
+      return true;
+    } else if (lookahead > range->end) {
+      index = mid_index;
+    }
+    size -= half_size;
+  }
+  TSCharacterRange *range = &ranges[index];
+  return (lookahead >= range->start && lookahead <= range->end);
+}
+
 /*
  *  Lexer Macros
  */
@@ -154,6 +177,17 @@ struct TSLanguage {
     goto next_state;         \
   }
 
+#define ADVANCE_MAP(...)                                              \
+  {                                                                   \
+    static const uint16_t map[] = { __VA_ARGS__ };                    \
+    for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) {  \
+      if (map[i] == lookahead) {                                      \
+        state = map[i + 1];                                           \
+        goto next_state;                                              \
+      }                                                               \
+    }                                                                 \
+  }
+
 #define SKIP(state_value) \
   {                       \
     skip = true;          \
@@ -203,14 +237,15 @@ struct TSLanguage {
     }                                 \
   }}
 
-#define REDUCE(symbol_val, child_count_val, ...) \
+#define REDUCE(symbol_name, children, precedence, prod_id) \
-  {{                                             \
+  {{                                                       \
-    .reduce = {                                  \
+    .reduce = {                                            \
-      .type = TSParseActionTypeReduce,           \
+      .type = TSParseActionTypeReduce,                     \
-      .symbol = symbol_val,                      \
+      .symbol = symbol_name,                               \
-      .child_count = child_count_val,            \
+      .child_count = children,                             \
-      __VA_ARGS__                                \
+      .dynamic_precedence = precedence,                    \
-    },                                           \
+      .production_id = prod_id                             \
+    },                                                     \
   }}
 
 #define RECOVER()                    \

test/corpus/05_characters.txt

@@ -0,0 +1,326 @@
+================================================================================
+Byte Order Mark
+================================================================================
+# Comment only.
+
+--------------------------------------------------------------------------------
+
+(stream
+ (comment))
+
+================================================================================
+Invalid Byte Order Mark
+:error
+================================================================================
+- Invalid use of BOM
+
+- Inside a document.
+
+--------------------------------------------------------------------------------
+
+================================================================================
+Block Structure Indicators
+================================================================================
+sequence:
+- one
+- two
+mapping:
+  ? sky
+  : blue
+  sea : green
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (block_node
+      (block_sequence
+       (block_sequence_item
+        (flow_node
+         (plain_scalar
+          (string_scalar))))
+       (block_sequence_item
+        (flow_node
+         (plain_scalar
+          (string_scalar)))))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (block_node
+      (block_mapping
+       (block_mapping_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (flow_node
+         (plain_scalar
+          (string_scalar))))
+       (block_mapping_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (flow_node
+         (plain_scalar
+          (string_scalar)))))))))))
+
+================================================================================
+Flow Collection Indicators
+================================================================================
+sequence: [ one, two, ]
+mapping: { sky: blue, sea: green }
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (flow_sequence
+       (flow_node
+        (plain_scalar
+         (string_scalar)))
+       (flow_node
+        (plain_scalar
+         (string_scalar))))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (flow_mapping
+       (flow_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (flow_node
+         (plain_scalar
+          (string_scalar))))
+       (flow_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (flow_node
+         (plain_scalar
+          (string_scalar)))))))))))
+
+================================================================================
+Comment Indicator
+================================================================================
+# Comment only.
+
+--------------------------------------------------------------------------------
+
+(stream
+ (comment))
+
+================================================================================
+Node Property Indicators
+================================================================================
+anchored: !local &anchor value
+alias: *anchor
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (tag)
+      (anchor
+       (anchor_name))
+      (plain_scalar
+       (string_scalar))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (alias
+       (alias_name))))))))
+
+================================================================================
+Block Scalar Indicators
+================================================================================
+literal: |
+  some
+  text
+folded: >
+  some
+  text
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (block_node
+      (block_scalar)))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (block_node
+      (block_scalar)))))))
+
+================================================================================
+Quoted Scalar Indicators
+================================================================================
+single: 'text'
+double: "text"
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (single_quote_scalar)))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (double_quote_scalar)))))))
+
+================================================================================
+Directive Indicator
+================================================================================
+%YAML 1.2
+--- text
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (yaml_directive
+   (yaml_version))
+  (flow_node
+   (plain_scalar
+    (string_scalar)))))
+
+================================================================================
+Invalid use of Reserved Indicators
+:error
+================================================================================
+commercial-at: @text
+grave-accent: `text
+
+--------------------------------------------------------------------------------
+
+================================================================================
+Line Break Characters
+================================================================================
+|
+  Line break (no glyph)
+  Line break (glyphed)
+
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_scalar))))
+
+================================================================================
+Tabs and Spaces
+================================================================================
+# Tabs and spaces
+quoted: "Quoted 	"
+block:	|
+  void main() {
+  	printf("Hello, world!\n");
+  }
+
+--------------------------------------------------------------------------------
+
+  (stream
+   (comment)
+   (document
+    (block_node
+     (block_mapping
+      (block_mapping_pair
+       (flow_node
+        (plain_scalar
+         (string_scalar)))
+       (flow_node
+        (double_quote_scalar)))
+      (block_mapping_pair
+       (flow_node
+        (plain_scalar
+         (string_scalar)))
+       (block_node
+        (block_scalar)))))))
+
+================================================================================
+Escaped Characters
+================================================================================
+"Fun with \\
+\" \a \b \e \f
+\n \r \t \v \0
+\  \_ \N \L \P
+\x41 \u0041 \U00000041"
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (flow_node
+   (double_quote_scalar
+    (escape_sequence)
+    (escape_sequence)
+    (escape_sequence)
+    (escape_sequence)
+    (escape_sequence)
+    (escape_sequence)
+    (escape_sequence)
+    (escape_sequence)
+    (escape_sequence)
+    (escape_sequence)
+    (escape_sequence)
+    (escape_sequence)
+    (escape_sequence)
+    (escape_sequence)
+    (escape_sequence)
+    (escape_sequence)
+    (escape_sequence)
+    (escape_sequence)
+    (escape_sequence)))))
+
+================================================================================
+Invalid Escaped Characters
+:error
+================================================================================
+Bad escapes:
+  "\c
+  \xq-"
+
+--------------------------------------------------------------------------------
+

test/corpus/06_structures.txt

@@ -0,0 +1,763 @@
+============================================================================||||
+Indentation Spaces
+============================================================================||||
+  # Leading comment line spaces are
+   # neither content nor indentation.
+
+Not indented:
+ By one space: |
+    By four
+      spaces
+ Flow style: [    # Leading spaces
+   By two,        # in flow style
+  Also by two,    # are neither
+  	Still by two   # content nor
+    ]             # indentation.
+
+----------------------------------------------------------------------------||||
+
+(stream
+ (comment)
+ (comment)
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (block_node
+      (block_mapping
+       (block_mapping_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (block_node
+         (block_scalar)))
+       (block_mapping_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (flow_node
+         (flow_sequence
+          (comment)
+          (flow_node
+           (plain_scalar
+            (string_scalar)))
+          (comment)
+          (flow_node
+           (plain_scalar
+            (string_scalar)))
+          (comment)
+          (flow_node
+           (plain_scalar
+            (string_scalar)))
+          (comment))))
+(comment))))))))
+
+============================================================================||||
+Indentation Indicators
+============================================================================||||
+? a
+: -	b
+  -  -	c
+     - d
+
+----------------------------------------------------------------------------||||
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (block_node
+      (block_sequence
+       (block_sequence_item
+        (flow_node
+         (plain_scalar
+          (string_scalar))))
+       (block_sequence_item
+        (block_node
+         (block_sequence
+          (block_sequence_item
+           (flow_node
+            (plain_scalar
+             (string_scalar))))
+          (block_sequence_item
+           (flow_node
+            (plain_scalar
+             (string_scalar))))))))))))))
+
+============================================================================||||
+Separation Spaces
+============================================================================||||
+- foo:	 bar
+- - baz
+  -	baz
+
+----------------------------------------------------------------------------||||
+
+(stream
+ (document
+  (block_node
+   (block_sequence
+    (block_sequence_item
+     (block_node
+      (block_mapping
+       (block_mapping_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (flow_node
+         (plain_scalar
+          (string_scalar)))))))
+    (block_sequence_item
+     (block_node
+      (block_sequence
+       (block_sequence_item
+        (flow_node
+         (plain_scalar
+          (string_scalar))))
+       (block_sequence_item
+        (flow_node
+         (plain_scalar
+          (string_scalar)))))))))))
+
+============================================================================||||
+Line Prefixes
+============================================================================||||
+plain: text
+  lines
+quoted: "text
+  	lines"
+block: |
+  text
+   	lines
+
+----------------------------------------------------------------------------||||
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (plain_scalar
+       (string_scalar))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (double_quote_scalar)))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (block_node
+      (block_scalar)))))))
+
+============================================================================||||
+Empty Lines
+============================================================================||||
+Folding:
+  "Empty line
+
+  as a line feed"
+Chomping: |
+  Clipped empty lines
+
+
+----------------------------------------------------------------------------||||
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (double_quote_scalar)))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (block_node
+      (block_scalar)))))))
+
+============================================================================||||
+Line Folding
+============================================================================||||
+>-
+  trimmed
+
+
+
+
+
+
+
+  as
+
+  space
+
+----------------------------------------------------------------------------||||
+
+(stream
+ (document
+  (block_node
+   (block_scalar))))
+
+============================================================================||||
+Block Folding
+============================================================================||||
+>
+  foo
+
+
+
+  	 bar
+
+
+
+  baz
+
+
+----------------------------------------------------------------------------||||
+
+(stream
+ (document
+  (block_node
+   (block_scalar))))
+
+============================================================================||||
+Flow Folding
+============================================================================||||
+"
+
+  foo
+
+
+
+  	 bar
+
+
+
+  baz
+
+"
+
+----------------------------------------------------------------------------||||
+
+(stream
+ (document
+  (flow_node
+   (double_quote_scalar))))
+
+============================================================================||||
+Separated Comment
+============================================================================||||
+key:    # Comment
+
+  value
+----------------------------------------------------------------------------||||
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (comment)
+     (flow_node
+      (plain_scalar
+       (string_scalar))))))))
+
+============================================================================||||
+Comment Lines
+============================================================================||||
+  # Comment
+
+
+
+
+
+
+----------------------------------------------------------------------------||||
+
+(stream
+ (comment))
+
+============================================================================||||
+Multi-Line Comments
+============================================================================||||
+key:    # Comment
+
+        # lines
+
+  value
+
+
+
+
+----------------------------------------------------------------------------||||
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (comment)
+     (comment)
+     (flow_node
+      (plain_scalar
+       (string_scalar))))))))
+
+============================================================================||||
+Separation Spaces
+============================================================================||||
+{ first: Sammy, last: Sosa }:
+
+# Statistics:
+  hr:  # Home runs
+     65
+  avg: # Average
+   0.278
+
+----------------------------------------------------------------------------||||
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (flow_mapping
+       (flow_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (flow_node
+         (plain_scalar
+          (string_scalar))))
+       (flow_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (flow_node
+         (plain_scalar
+          (string_scalar))))))
+     (comment)
+     (block_node
+      (block_mapping
+       (block_mapping_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (comment)
+        (flow_node
+         (plain_scalar
+          (integer_scalar))))
+       (block_mapping_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (comment)
+        (flow_node
+         (plain_scalar
+          (float_scalar)))))))))))
+
+============================================================================||||
+Reserved Directives
+============================================================================||||
+%FOO  bar baz # Should be ignored
+               # with a warning.
+--- "foo"
+
+----------------------------------------------------------------------------||||
+
+(stream
+ (document
+  (reserved_directive
+   (directive_name)
+   (directive_parameter)
+   (directive_parameter))
+  (comment)
+  (comment)
+  (flow_node
+   (double_quote_scalar))))
+
+============================================================================||||
+“YAML” directive
+============================================================================||||
+%YAML 1.3 # Attempt parsing
+           # with a warning
+---
+"foo"
+
+----------------------------------------------------------------------------||||
+
+(stream
+ (document
+  (yaml_directive
+   (yaml_version))
+  (comment)
+  (comment)
+  (flow_node
+   (double_quote_scalar))))
+
+============================================================================||||
+Invalid Repeated YAML directive
+:error
+============================================================================||||
+%YAML 1.2
+%YAML 1.1
+foo
+
+----------------------------------------------------------------------------||||
+
+============================================================================||||
+“TAG” directive
+============================================================================||||
+%TAG !yaml! tag:yaml.org,2002:
+---
+!yaml!str "foo"
+
+----------------------------------------------------------------------------||||
+
+(stream
+ (document
+  (tag_directive
+   (tag_handle)
+   (tag_prefix))
+  (flow_node
+   (tag)
+   (double_quote_scalar))))
+
+============================================================================||||
+Invalid Repeated TAG directive
+:error
+============================================================================||||
+%TAG ! !foo
+%TAG ! !foo
+bar
+
+----------------------------------------------------------------------------||||
+
+============================================================================||||
+Primary Tag Handle
+============================================================================||||
+# Private
+!foo "bar"
+...
+# Global
+%TAG ! tag:example.com,2000:app/
+---
+!foo "bar"
+
+----------------------------------------------------------------------------||||
+
+(stream
+ (comment)
+ (document
+  (flow_node
+   (tag)
+   (double_quote_scalar)))
+ (comment)
+ (document
+  (tag_directive
+   (tag_handle)
+   (tag_prefix))
+  (flow_node
+   (tag)
+   (double_quote_scalar))))
+
+============================================================================||||
+Secondary Tag Handle
+============================================================================||||
+%TAG !! tag:example.com,2000:app/
+---
+!!int 1 - 3 # Interval, not integer
+
+----------------------------------------------------------------------------||||
+
+(stream
+ (document
+  (tag_directive
+   (tag_handle)
+   (tag_prefix))
+  (flow_node
+   (tag)
+   (plain_scalar
+    (string_scalar))))
+ (comment))
+
+============================================================================||||
+Tag Handles
+============================================================================||||
+%TAG !e! tag:example.com,2000:app/
+---
+!e!foo "bar"
+
+----------------------------------------------------------------------------||||
+
+(stream
+(document
+(tag_directive
+(tag_handle)
+(tag_prefix))
+(flow_node
+(tag)
+(double_quote_scalar))))
+
+============================================================================||||
+Local Tag Prefix
+============================================================================||||
+%TAG !m! !my-
+--- # Bulb here
+!m!light fluorescent
+...
+%TAG !m! !my-
+--- # Color here
+!m!light green
+
+----------------------------------------------------------------------------||||
+
+(stream
+(document
+(tag_directive
+(tag_handle)
+(tag_prefix))
+(comment)
+(flow_node
+(tag)
+(plain_scalar
+(string_scalar))))
+(document
+(tag_directive
+(tag_handle)
+(tag_prefix))
+(comment)
+(flow_node
+(tag)
+(plain_scalar
+(string_scalar)))))
+
+============================================================================||||
+Global Tag Prefix
+============================================================================||||
+%TAG !e! tag:example.com,2000:app/
+---
+- !e!foo "bar"
+
+----------------------------------------------------------------------------||||
+
+(stream
+(document
+(tag_directive
+(tag_handle)
+(tag_prefix))
+(block_node
+(block_sequence
+(block_sequence_item
+(flow_node
+(tag)
+(double_quote_scalar)))))))
+
+============================================================================||||
+Node Properties
+============================================================================||||
+!!str &a1 "foo":
+  !!str bar
+&a2 baz : *a1
+
+----------------------------------------------------------------------------||||
+
+(stream
+(document
+(block_node
+(block_mapping
+(block_mapping_pair
+(flow_node
+(tag)
+(anchor
+(anchor_name))
+(double_quote_scalar))
+(flow_node
+(tag)
+(plain_scalar
+(string_scalar))))
+(block_mapping_pair
+(flow_node
+(anchor
+(anchor_name))
+(plain_scalar
+(string_scalar)))
+(flow_node
+(alias
+(alias_name))))))))
+
+============================================================================||||
+Verbatim Tags
+============================================================================||||
+!<tag:yaml.org,2002:str> foo :
+  !<!bar> baz
+
+----------------------------------------------------------------------------||||
+
+(stream
+(document
+(block_node
+(block_mapping
+(block_mapping_pair
+(flow_node
+(tag)
+(plain_scalar
+(string_scalar)))
+(flow_node
+(tag)
+(plain_scalar
+(string_scalar))))))))
+
+============================================================================||||
+Invalid Verbatim Tags
+============================================================================||||
+- !<!> foo
+- !<$:?> bar
+
+----------------------------------------------------------------------------||||
+
+(stream
+(document
+(block_node
+(block_sequence
+(block_sequence_item
+(flow_node
+(tag)
+(plain_scalar
+(string_scalar))))
+(block_sequence_item
+(flow_node
+(tag)
+(plain_scalar
+(string_scalar))))))))
+
+============================================================================||||
+Tag Shorthands
+============================================================================||||
+%TAG !e! tag:example.com,2000:app/
+---
+- !local foo
+- !!str bar
+- !e!tag%21 baz
+
+----------------------------------------------------------------------------||||
+
+(stream
+(document
+(tag_directive
+(tag_handle)
+(tag_prefix))
+(block_node
+(block_sequence
+(block_sequence_item
+(flow_node
+(tag)
+(plain_scalar
+(string_scalar))))
+(block_sequence_item
+(flow_node
+(tag)
+(plain_scalar
+(string_scalar))))
+(block_sequence_item
+(flow_node
+(tag)
+(plain_scalar
+(string_scalar))))))))
+
+============================================================================||||
+Invalid Tag Shorthands
+============================================================================||||
+%TAG !e! tag:example,2000:app/
+---
+- !e! foo
+- !h!bar baz
+
+----------------------------------------------------------------------------||||
+
+(ERROR
+(tag_directive
+(tag_handle)
+(tag_prefix))
+(block_sequence_item))
+
+============================================================================||||
+Non-Specific Tags
+============================================================================||||
+# Assuming conventional resolution:
+- "12"
+- 12
+- ! 12
+
+----------------------------------------------------------------------------||||
+
+(stream
+(comment)
+(document
+(block_node
+(block_sequence
+(block_sequence_item
+(flow_node
+(double_quote_scalar)))
+(block_sequence_item
+(flow_node
+(plain_scalar
+(integer_scalar))))
+(block_sequence_item
+(flow_node
+(tag)
+(plain_scalar
+(integer_scalar))))))))
+
+============================================================================||||
+Node Anchors
+============================================================================||||
+First occurrence: &anchor Value
+Second occurrence: *anchor
+
+----------------------------------------------------------------------------||||
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (anchor
+       (anchor_name))
+      (plain_scalar
+       (string_scalar))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (alias
+       (alias_name))))))))

test/corpus/07_flows.txt

@@ -0,0 +1,729 @@
+================================================================================
+Alias Nodes
+================================================================================
+First occurrence: &anchor Foo
+Second occurrence: *anchor
+Override anchor: &anchor Bar
+Reuse anchor: *anchor
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (anchor
+       (anchor_name))
+      (plain_scalar
+       (string_scalar))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (alias
+       (alias_name))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (anchor
+       (anchor_name))
+      (plain_scalar
+       (string_scalar))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (alias
+       (alias_name))))))))
+
+================================================================================
+Empty Content
+================================================================================
+{
+  foo : !!str,
+  !!str : bar,
+}
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (flow_node
+   (flow_mapping
+    (flow_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (tag)))
+    (flow_pair
+     (flow_node
+      (tag))
+     (flow_node
+      (plain_scalar
+       (string_scalar))))))))
+
+================================================================================
+Completely Empty Flow Nodes
+================================================================================
+{
+  ? foo :,
+  : bar,
+}
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (flow_node
+   (flow_mapping
+    (flow_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar))))
+    (flow_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar))))))))
+
+================================================================================
+Double Quoted Implicit Keys
+================================================================================
+"implicit block key" : [
+  "implicit flow key" : value,
+ ]
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (double_quote_scalar))
+     (flow_node
+      (flow_sequence
+       (flow_pair
+        (flow_node
+         (double_quote_scalar))
+        (flow_node
+         (plain_scalar
+          (string_scalar)))))))))))
+
+================================================================================
+Double Quoted Line Breaks
+================================================================================
+"folded
+
+to a space,
+
+
+
+to a line feed, or 	\
+
+ \ 	non-content"
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (flow_node
+   (double_quote_scalar
+    (escape_sequence)
+    (escape_sequence)))))
+
+================================================================================
+Double Quoted Lines
+================================================================================
+" 1st non-empty
+
+
+
+ 2nd non-empty
+	3rd non-empty "
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (flow_node
+   (double_quote_scalar))))
+
+================================================================================
+Single Quoted Characters
+================================================================================
+ 'here''s to "quotes"'
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (flow_node
+   (single_quote_scalar
+    (escape_sequence)))))
+
+================================================================================
+Single Quoted Implicit Keys
+================================================================================
+'implicit block key' : [
+  'implicit flow key' : value,
+ ]
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (single_quote_scalar))
+     (flow_node
+      (flow_sequence
+       (flow_pair
+        (flow_node
+         (single_quote_scalar))
+        (flow_node
+         (plain_scalar
+          (string_scalar)))))))))))
+
+================================================================================
+Single Quoted Lines
+================================================================================
+' 1st non-empty
+
+
+
+ 2nd non-empty
+	3rd non-empty '
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (flow_node
+   (single_quote_scalar))))
+
+================================================================================
+Plain Characters
+================================================================================
+# Outside flow collection:
+- ::vector
+- ": - ()"
+- Up, up, and away!
+- -123
+- http://example.com/foo#bar
+# Inside flow collection:
+- [ ::vector,
+  ": - ()",
+  "Up, up and away!",
+  -123,
+  http://example.com/foo#bar ]
+
+--------------------------------------------------------------------------------
+
+(stream
+ (comment)
+ (document
+  (block_node
+   (block_sequence
+    (block_sequence_item
+     (flow_node
+      (plain_scalar
+       (string_scalar))))
+    (block_sequence_item
+     (flow_node
+      (double_quote_scalar)))
+    (block_sequence_item
+     (flow_node
+      (plain_scalar
+       (string_scalar))))
+    (block_sequence_item
+     (flow_node
+      (plain_scalar
+       (integer_scalar))))
+    (block_sequence_item
+     (flow_node
+      (plain_scalar
+       (string_scalar))))
+    (comment)
+(block_sequence_item
+ (flow_node
+  (flow_sequence
+   (flow_node
+    (plain_scalar
+     (string_scalar)))
+   (flow_node
+    (double_quote_scalar))
+   (flow_node
+    (double_quote_scalar))
+   (flow_node
+    (plain_scalar
+     (integer_scalar)))
+   (flow_node
+    (plain_scalar
+     (string_scalar))))))))))
+
+================================================================================
+Plain Implicit Keys
+================================================================================
+implicit block key : [
+  implicit flow key : value,
+ ]
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (flow_sequence
+       (flow_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (flow_node
+         (plain_scalar
+          (string_scalar)))))))))))
+
+================================================================================
+Plain Lines
+================================================================================
+1st non-empty
+
+
+
+ 2nd non-empty
+	3rd non-empty
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (flow_node
+   (plain_scalar
+    (string_scalar)))))
+
+================================================================================
+Flow Sequence
+================================================================================
+- [ one, two, ]
+- [three ,four]
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_sequence
+    (block_sequence_item
+     (flow_node
+      (flow_sequence
+       (flow_node
+        (plain_scalar
+         (string_scalar)))
+       (flow_node
+        (plain_scalar
+         (string_scalar))))))
+    (block_sequence_item
+     (flow_node
+      (flow_sequence
+       (flow_node
+        (plain_scalar
+         (string_scalar)))
+       (flow_node
+        (plain_scalar
+         (string_scalar))))))))))
+
+================================================================================
+Flow Sequence Entries
+================================================================================
+[
+"double
+ quoted", 'single
+           quoted',
+plain
+ text, [ nested ],
+single: pair,
+]
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (flow_node
+   (flow_sequence
+    (flow_node
+     (double_quote_scalar))
+    (flow_node
+     (single_quote_scalar))
+    (flow_node
+     (plain_scalar
+      (string_scalar)))
+    (flow_node
+     (flow_sequence
+      (flow_node
+       (plain_scalar
+        (string_scalar)))))
+    (flow_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (plain_scalar
+       (string_scalar))))))))
+
+================================================================================
+Flow Mappings
+================================================================================
+- { one : two , three: four , }
+- {five: six,seven : eight}
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_sequence
+    (block_sequence_item
+     (flow_node
+      (flow_mapping
+       (flow_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (flow_node
+         (plain_scalar
+          (string_scalar))))
+       (flow_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (flow_node
+         (plain_scalar
+          (string_scalar)))))))
+    (block_sequence_item
+     (flow_node
+      (flow_mapping
+       (flow_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (flow_node
+         (plain_scalar
+          (string_scalar))))
+       (flow_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (flow_node
+         (plain_scalar
+          (string_scalar)))))))))))
+
+================================================================================
+Flow Mapping Entries
+================================================================================
+{
+? explicit: entry,
+implicit: entry,
+?
+}
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (flow_node
+   (flow_mapping
+    (flow_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (plain_scalar
+       (string_scalar))))
+    (flow_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (plain_scalar
+       (string_scalar))))
+    (flow_pair)))))
+
+================================================================================
+Flow Mapping Separate Values
+================================================================================
+{
+unquoted : "separate",
+http://foo.com,
+omitted value:,
+: omitted key,
+}
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (flow_node
+   (flow_mapping
+    (flow_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (double_quote_scalar)))
+    (flow_node
+     (plain_scalar
+      (string_scalar)))
+    (flow_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar))))
+    (flow_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar))))))))
+
+================================================================================
+Flow Mapping Adjacent Values
+================================================================================
+{
+"adjacent":value,
+"readable": value,
+"empty":
+}
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (flow_node
+   (flow_mapping
+    (flow_pair
+     (flow_node
+      (double_quote_scalar))
+     (flow_node
+      (plain_scalar
+       (string_scalar))))
+    (flow_pair
+     (flow_node
+      (double_quote_scalar))
+     (flow_node
+      (plain_scalar
+       (string_scalar))))
+    (flow_pair
+     (flow_node
+      (double_quote_scalar)))))))
+
+================================================================================
+Single Pair Flow Mappings
+================================================================================
+[
+foo: bar
+]
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (flow_node
+   (flow_sequence
+    (flow_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (plain_scalar
+       (string_scalar))))))))
+
+================================================================================
+Single Pair Explicit Entry
+================================================================================
+[
+? foo
+ bar : baz
+]
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (flow_node
+   (flow_sequence
+    (flow_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (plain_scalar
+       (string_scalar))))))))
+
+================================================================================
+Single Pair Implicit Entries
+================================================================================
+- [ YAML : separate ]
+- [ : empty key entry ]
+- [ {JSON: like}:adjacent ]
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_sequence
+    (block_sequence_item
+     (flow_node
+      (flow_sequence
+       (flow_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (flow_node
+         (plain_scalar
+          (string_scalar)))))))
+    (block_sequence_item
+     (flow_node
+      (flow_sequence
+       (flow_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))))))
+    (block_sequence_item
+     (flow_node
+      (flow_sequence
+       (flow_pair
+        (flow_node
+         (flow_mapping
+          (flow_pair
+           (flow_node
+            (plain_scalar
+             (string_scalar)))
+           (flow_node
+            (plain_scalar
+             (string_scalar))))))
+        (flow_node
+         (plain_scalar
+          (string_scalar)))))))))))
+
+================================================================================
+Invalid Implicit Keys
+================================================================================
+[ foo
+ bar: invalid,
+ "foo...>1K characters...bar": invalid ]
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (flow_node
+   (flow_sequence
+    (flow_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (plain_scalar
+       (string_scalar))))
+    (flow_pair
+     (flow_node
+      (double_quote_scalar))
+     (flow_node
+      (plain_scalar
+       (string_scalar))))))))
+
+================================================================================
+Flow Content
+================================================================================
+- [ a, b ]
+- { a: b }
+- "a"
+- 'b'
+- c
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_sequence
+    (block_sequence_item
+     (flow_node
+      (flow_sequence
+       (flow_node
+        (plain_scalar
+         (string_scalar)))
+       (flow_node
+        (plain_scalar
+         (string_scalar))))))
+    (block_sequence_item
+     (flow_node
+      (flow_mapping
+       (flow_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (flow_node
+         (plain_scalar
+          (string_scalar)))))))
+    (block_sequence_item
+     (flow_node
+      (double_quote_scalar)))
+    (block_sequence_item
+     (flow_node
+      (single_quote_scalar)))
+    (block_sequence_item
+     (flow_node
+      (plain_scalar
+       (string_scalar))))))))
+
+================================================================================
+Flow Nodes
+================================================================================
+- !!str "a"
+- 'b'
+- &anchor "c"
+- *anchor
+- !!str
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_sequence
+    (block_sequence_item
+     (flow_node
+      (tag)
+      (double_quote_scalar)))
+    (block_sequence_item
+     (flow_node
+      (single_quote_scalar)))
+    (block_sequence_item
+     (flow_node
+      (anchor
+       (anchor_name))
+      (double_quote_scalar)))
+    (block_sequence_item
+     (flow_node
+      (alias
+       (alias_name))))
+    (block_sequence_item
+     (flow_node
+      (tag)))))))

test/corpus/08_blocks.txt

@@ -0,0 +1,782 @@
+================================================================================
+Block Scalar Header
+================================================================================
+- | # Empty header
+
+ literal
+- >1 # Indentation indicator
+
+  folded
+- |+ # Chomping indicator
+
+ keep
+
+- >1- # Both indicators
+
+  strip
+
+
+--------------------------------------------------------------------------------
+
+(stream
+  (document
+    (block_node
+      (block_sequence
+        (block_sequence_item
+          (block_node
+            (block_scalar
+              (comment))))
+        (block_sequence_item
+          (block_node
+            (block_scalar
+              (comment))))
+        (block_sequence_item
+          (block_node
+            (block_scalar
+              (comment))))
+        (block_sequence_item
+          (block_node
+            (block_scalar
+              (comment))))))))
+
+================================================================================
+Block Indentation Indicator
+================================================================================
+- |
+ detected
+- >
+
+
+  # detected
+- |1
+  explicit
+- >
+
+ detected
+
+--------------------------------------------------------------------------------
+
+(stream
+  (document
+    (block_node
+      (block_sequence
+        (block_sequence_item
+          (block_node
+            (block_scalar)))
+        (block_sequence_item
+          (block_node
+            (block_scalar)))
+        (block_sequence_item
+          (block_node
+            (block_scalar)))
+        (block_sequence_item
+          (block_node
+            (block_scalar)))))))
+
+================================================================================
+Chomping Final Line Break
+================================================================================
+strip: |-
+  text
+
+clip: |
+  text
+
+keep: |+
+  text
+
+
+--------------------------------------------------------------------------------
+
+(stream
+  (document
+    (block_node
+      (block_mapping
+        (block_mapping_pair
+          (flow_node
+            (plain_scalar
+              (string_scalar)))
+          (block_node
+            (block_scalar)))
+        (block_mapping_pair
+          (flow_node
+            (plain_scalar
+              (string_scalar)))
+          (block_node
+            (block_scalar)))
+        (block_mapping_pair
+          (flow_node
+            (plain_scalar
+              (string_scalar)))
+          (block_node
+            (block_scalar)))))))
+
+================================================================================
+Chomping Trailing Lines
+================================================================================
+ # Strip
+  # Comments:
+strip: |-
+  # text
+
+  ⇓
+ # Clip
+  # comments:
+
+
+clip: |
+  # text
+
+
+
+ # Keep
+  # comments:
+
+
+keep: |+
+  # text
+
+
+
+ # Trail
+  # comments.
+
+--------------------------------------------------------------------------------
+
+(stream
+  (comment)
+  (comment)
+  (document
+    (block_node
+      (block_mapping
+        (block_mapping_pair
+          (flow_node
+            (plain_scalar
+              (string_scalar)))
+          (block_node
+            (block_scalar)))
+        (comment)
+        (comment)
+        (block_mapping_pair
+          (flow_node
+            (plain_scalar
+              (string_scalar)))
+          (block_node
+            (block_scalar)))
+        (comment)
+        (comment)
+        (block_mapping_pair
+          (flow_node
+            (plain_scalar
+              (string_scalar)))
+          (block_node
+            (block_scalar)))
+        (comment)
+        (comment)))))
+
+================================================================================
+Empty Scalar Chomping
+================================================================================
+strip: >-
+
+
+clip: >
+
+
+keep: |+
+
+
+
+--------------------------------------------------------------------------------
+
+(stream
+  (document
+    (block_node
+      (block_mapping
+        (block_mapping_pair
+          (flow_node
+            (plain_scalar
+              (string_scalar)))
+          (block_node
+            (block_scalar)))
+        (block_mapping_pair
+          (flow_node
+            (plain_scalar
+              (string_scalar)))
+          (block_node
+            (block_scalar)))
+        (block_mapping_pair
+          (flow_node
+            (plain_scalar
+              (string_scalar)))
+          (block_node
+            (block_scalar)))))))
+
+================================================================================
+Literal Scalar
+================================================================================
+|
+
+ literal
+
+ 	text
+
+
+
+
+--------------------------------------------------------------------------------
+
+(stream
+  (document
+    (block_node
+      (block_scalar))))
+
+================================================================================
+Literal Content
+================================================================================
+|
+
+
+  literal
+
+
+
+
+  text
+
+
+
+ # Comment
+
+--------------------------------------------------------------------------------
+
+(stream
+  (document
+    (block_node
+      (block_scalar)))
+  (comment))
+
+================================================================================
+Folded Scalar
+================================================================================
+>
+
+ folded
+
+ text
+
+
+
+
+--------------------------------------------------------------------------------
+
+(stream
+  (document
+    (block_node
+      (block_scalar))))
+
+================================================================================
+Folded Lines
+================================================================================
+>
+
+ folded
+
+ line
+
+
+
+ next
+ line
+
+   * bullet
+
+   * list
+   * lines
+
+ last
+
+ line
+
+
+# Comment
+
+--------------------------------------------------------------------------------
+
+(stream
+  (document
+    (block_node
+      (block_scalar)))
+  (comment))
+
+================================================================================
+More Indented Lines
+================================================================================
+>
+
+ folded
+ line
+
+ next
+ line
+   * bullet
+
+
+
+   * list
+
+   * lines
+
+
+ last
+ line
+
+# Comment
+
+--------------------------------------------------------------------------------
+
+(stream
+  (document
+    (block_node
+      (block_scalar)))
+  (comment))
+
+================================================================================
+Empty Separation Lines
+================================================================================
+>
+
+
+ folded
+ line
+
+
+
+ next
+ line
+
+   * bullet
+
+   * list
+   * line
+
+
+
+ last
+ line
+
+# Comment
+
+--------------------------------------------------------------------------------
+
+(stream
+  (document
+    (block_node
+      (block_scalar)))
+  (comment))
+
+================================================================================
+Final Empty Lines
+================================================================================
+>
+ folded
+ line
+
+ next
+ line
+   * bullet
+
+   * list
+   * line
+
+ last
+ line
+
+
+
+# Comment
+
+--------------------------------------------------------------------------------
+
+(stream
+  (document
+    (block_node
+      (block_scalar)))
+  (comment))
+
+================================================================================
+Block Sequence
+================================================================================
+block sequence:
+  - one
+
+  - two : three
+
+
+--------------------------------------------------------------------------------
+
+(stream
+  (document
+    (block_node
+      (block_mapping
+        (block_mapping_pair
+          (flow_node
+            (plain_scalar
+              (string_scalar)))
+          (block_node
+            (block_sequence
+              (block_sequence_item
+                (flow_node
+                  (plain_scalar
+                    (string_scalar))))
+              (block_sequence_item
+                (block_node
+                  (block_mapping
+                    (block_mapping_pair
+                      (flow_node
+                        (plain_scalar
+                          (string_scalar)))
+                      (flow_node
+                        (plain_scalar
+                          (string_scalar))))))))))))))
+
+================================================================================
+Block Sequence Entry Types
+================================================================================
+- # Empty
+- |
+ block node
+- - one # Compact
+  - two # sequence
+- one: two # Compact mapping
+
+--------------------------------------------------------------------------------
+
+(stream
+  (document
+    (block_node
+      (block_sequence
+        (block_sequence_item)
+        (comment)
+        (block_sequence_item
+          (block_node
+            (block_scalar)))
+        (block_sequence_item
+          (block_node
+            (block_sequence
+              (block_sequence_item
+                (flow_node
+                  (plain_scalar
+                    (string_scalar))))
+              (comment)
+              (block_sequence_item
+                (flow_node
+                  (plain_scalar
+                    (string_scalar))))
+              (comment))))
+        (block_sequence_item
+          (block_node
+            (block_mapping
+              (block_mapping_pair
+                (flow_node
+                  (plain_scalar
+                    (string_scalar)))
+                (flow_node
+                  (plain_scalar
+                    (string_scalar))))
+              (comment))))))))
+
+================================================================================
+Block Mappings
+================================================================================
+block mapping:
+ key: value
+
+
+--------------------------------------------------------------------------------
+
+(stream
+  (document
+    (block_node
+      (block_mapping
+        (block_mapping_pair
+          (flow_node
+            (plain_scalar
+              (string_scalar)))
+          (block_node
+            (block_mapping
+              (block_mapping_pair
+                (flow_node
+                  (plain_scalar
+                    (string_scalar)))
+                (flow_node
+                  (plain_scalar
+                    (string_scalar)))))))))))
+
+================================================================================
+Explicit Block Mapping Entries
+================================================================================
+? explicit key # Empty value
+
+? |
+  block key
+
+: - one # Explicit compact
+  - two # block value
+
+
+--------------------------------------------------------------------------------
+
+(stream
+  (document
+    (block_node
+      (block_mapping
+        (block_mapping_pair
+          (flow_node
+            (plain_scalar
+              (string_scalar))))
+        (comment)
+        (block_mapping_pair
+          (block_node
+            (block_scalar))
+          (block_node
+            (block_sequence
+              (block_sequence_item
+                (flow_node
+                  (plain_scalar
+                    (string_scalar))))
+              (comment)
+              (block_sequence_item
+                (flow_node
+                  (plain_scalar
+                    (string_scalar))))
+              (comment))))))))
+
+================================================================================
+Implicit Block Mapping Entries
+================================================================================
+plain key: in-line value
+: # Both empty
+"quoted key":
+- entry
+
+--------------------------------------------------------------------------------
+
+(stream
+  (document
+    (block_node
+      (block_mapping
+        (block_mapping_pair
+          (flow_node
+            (plain_scalar
+              (string_scalar)))
+          (flow_node
+            (plain_scalar
+              (string_scalar))))
+        (block_mapping_pair)
+        (comment)
+        (block_mapping_pair
+          (flow_node
+            (double_quote_scalar))
+          (block_node
+            (block_sequence
+              (block_sequence_item
+                (flow_node
+                  (plain_scalar
+                    (string_scalar)))))))))))
+
+================================================================================
+Compact Block Mappings
+================================================================================
+- sun: yellow
+
+- ? earth: blue
+
+  : moon: white
+
+
+--------------------------------------------------------------------------------
+
+(stream
+  (document
+    (block_node
+      (block_sequence
+        (block_sequence_item
+          (block_node
+            (block_mapping
+              (block_mapping_pair
+                (flow_node
+                  (plain_scalar
+                    (string_scalar)))
+                (flow_node
+                  (plain_scalar
+                    (string_scalar)))))))
+        (block_sequence_item
+          (block_node
+            (block_mapping
+              (block_mapping_pair
+                (block_node
+                  (block_mapping
+                    (block_mapping_pair
+                      (flow_node
+                        (plain_scalar
+                          (string_scalar)))
+                      (flow_node
+                        (plain_scalar
+                          (string_scalar))))))
+                (block_node
+                  (block_mapping
+                    (block_mapping_pair
+                      (flow_node
+                        (plain_scalar
+                          (string_scalar)))
+                      (flow_node
+                        (plain_scalar
+                          (string_scalar))))))))))))))
+
+================================================================================
+Block Node Types
+================================================================================
+-
+
+  "flow in block"
+
+- >
+ Block scalar
+
+- !!map # Block collection
+  foo : bar
+
+
+--------------------------------------------------------------------------------
+
+(stream
+  (document
+    (block_node
+      (block_sequence
+        (block_sequence_item
+          (flow_node
+            (double_quote_scalar)))
+        (block_sequence_item
+          (block_node
+            (block_scalar)))
+        (block_sequence_item
+          (block_node
+            (tag)
+            (comment)
+            (block_mapping
+              (block_mapping_pair
+                (flow_node
+                  (plain_scalar
+                    (string_scalar)))
+                (flow_node
+                  (plain_scalar
+                    (string_scalar)))))))))))
+
+================================================================================
+Block Scalar Nodes
+================================================================================
+literal: |2
+  value
+folded:
+
+   !foo
+  >1
+ value
+
+--------------------------------------------------------------------------------
+
+(stream
+  (document
+    (block_node
+      (block_mapping
+        (block_mapping_pair
+          (flow_node
+            (plain_scalar
+              (string_scalar)))
+          (block_node
+            (block_scalar)))
+        (block_mapping_pair
+          (flow_node
+            (plain_scalar
+              (string_scalar)))
+          (block_node
+            (tag)
+            (block_scalar)))))))
+
+================================================================================
+Block Collection Nodes
+================================================================================
+sequence: !!seq
+- entry
+- !!seq
+ - nested
+mapping: !!map
+ foo: bar
+
+--------------------------------------------------------------------------------
+
+(stream
+  (document
+    (block_node
+      (block_mapping
+        (block_mapping_pair
+          (flow_node
+            (plain_scalar
+              (string_scalar)))
+          (block_node
+            (tag)
+            (block_sequence
+              (block_sequence_item
+                (flow_node
+                  (plain_scalar
+                    (string_scalar))))
+              (block_sequence_item
+                (block_node
+                  (tag)
+                  (block_sequence
+                    (block_sequence_item
+                      (flow_node
+                        (plain_scalar
+                          (string_scalar))))))))))
+        (block_mapping_pair
+          (flow_node
+            (plain_scalar
+              (string_scalar)))
+          (block_node
+            (tag)
+            (block_mapping
+              (block_mapping_pair
+                (flow_node
+                  (plain_scalar
+                    (string_scalar)))
+                (flow_node
+                  (plain_scalar
+                    (string_scalar)))))))))))
+
+================================================================================
+Long string scalar (> SCHAR_MAX in length)
+================================================================================
+
+long:
+  abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdef1234567890!@#$%^&*()_+{}|:"<>?`-=[]\;',./almostat128chars!:
+    {}
+
+---
+
+(stream
+  (document
+    (block_node
+      (block_mapping
+        (block_mapping_pair
+          (flow_node
+            (plain_scalar
+              (string_scalar)))
+          (block_node
+            (block_mapping
+              (block_mapping_pair
+                (flow_node
+                  (plain_scalar
+                    (string_scalar)))
+                (flow_node
+                  (flow_mapping))))))))))

test/corpus/09_streams.txt

@@ -0,0 +1,145 @@
+============================================================================||||
+Document Prefix
+============================================================================||||
+# Comment
+# lines
+Document
+
+----------------------------------------------------------------------------||||
+
+(stream
+ (comment)
+ (comment)
+ (document
+  (flow_node
+   (plain_scalar
+    (string_scalar)))))
+
+============================================================================||||
+Document Markers
+============================================================================||||
+%YAML 1.2
+---
+Document
+... # Suffix
+
+----------------------------------------------------------------------------||||
+
+(stream
+ (document
+  (yaml_directive
+   (yaml_version))
+  (flow_node
+   (plain_scalar
+    (string_scalar))))
+ (comment))
+
+============================================================================||||
+Bare Documents
+============================================================================||||
+Bare
+document
+...
+# No document
+...
+|
+%!PS-Adobe-2.0 # Not the first line
+
+----------------------------------------------------------------------------||||
+
+(stream
+ (document
+  (flow_node
+   (plain_scalar
+    (string_scalar))))
+ (comment)
+ (document)
+ (document
+  (block_node
+   (block_scalar))))
+
+============================================================================||||
+Explicit Documents
+============================================================================||||
+---
+{ matches
+% : 20 }
+...
+---
+# Empty
+...
+
+----------------------------------------------------------------------------||||
+
+(stream
+ (document
+  (flow_node
+   (flow_mapping
+    (flow_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (plain_scalar
+       (integer_scalar)))))))
+ (document
+  (comment)))
+
+============================================================================||||
+Directives Documents
+============================================================================||||
+%YAML 1.2
+--- |
+%!PS-Adobe-2.0
+...
+%YAML1.2
+---
+# Empty
+...
+
+----------------------------------------------------------------------------||||
+
+(stream
+ (document
+  (yaml_directive
+   (yaml_version))
+  (block_node
+   (block_scalar)))
+ (document
+  (reserved_directive
+   (directive_name))
+  (comment)))
+
+============================================================================||||
+Stream
+============================================================================||||
+Document
+---
+# Empty
+...
+%YAML 1.2
+---
+matches %: 20
+
+----------------------------------------------------------------------------||||
+
+(stream
+ (document
+  (flow_node
+   (plain_scalar
+    (string_scalar))))
+ (document
+  (comment))
+ (document
+  (yaml_directive
+   (yaml_version))
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (plain_scalar
+       (integer_scalar))))))))
+

test/corpus/10_schemas.txt

@@ -0,0 +1,508 @@
+================================================================================
+!!map Examples
+================================================================================
+Block style: !!map
+  Clark : Evans
+  Ingy  : döt Net
+  Oren  : Ben-Kiki
+
+Flow style: !!map { Clark: Evans, Ingy: döt Net, Oren: Ben-Kiki }
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (block_node
+      (tag)
+      (block_mapping
+       (block_mapping_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (flow_node
+         (plain_scalar
+          (string_scalar))))
+       (block_mapping_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (flow_node
+         (plain_scalar
+          (string_scalar))))
+       (block_mapping_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (flow_node
+         (plain_scalar
+          (string_scalar)))))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (tag)
+      (flow_mapping
+       (flow_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (flow_node
+         (plain_scalar
+          (string_scalar))))
+       (flow_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (flow_node
+         (plain_scalar
+          (string_scalar))))
+       (flow_pair
+        (flow_node
+         (plain_scalar
+          (string_scalar)))
+        (flow_node
+         (plain_scalar
+          (string_scalar)))))))))))
+
+================================================================================
+!!seq Examples
+================================================================================
+Block style: !!seq
+- Clark Evans
+- Ingy döt Net
+- Oren Ben-Kiki
+
+Flow style: !!seq [ Clark Evans, Ingy döt Net, Oren Ben-Kiki ]
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (block_node
+      (tag)
+      (block_sequence
+       (block_sequence_item
+        (flow_node
+         (plain_scalar
+          (string_scalar))))
+       (block_sequence_item
+        (flow_node
+         (plain_scalar
+          (string_scalar))))
+       (block_sequence_item
+        (flow_node
+         (plain_scalar
+          (string_scalar)))))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (tag)
+      (flow_sequence
+       (flow_node
+        (plain_scalar
+         (string_scalar)))
+       (flow_node
+        (plain_scalar
+         (string_scalar)))
+       (flow_node
+        (plain_scalar
+         (string_scalar))))))))))
+
+================================================================================
+!!str Examples
+================================================================================
+Block style: !!str |-
+  String: just a theory.
+
+Flow style: !!str "String: just a theory."
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (block_node
+      (tag)
+      (block_scalar)))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (tag)
+      (double_quote_scalar)))))))
+
+================================================================================
+!!null Examples
+================================================================================
+!!null null: value for null key
+key with null value: !!null null
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (tag)
+      (plain_scalar
+       (null_scalar)))
+     (flow_node
+      (plain_scalar
+       (string_scalar))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (tag)
+      (plain_scalar
+       (null_scalar))))))))
+
+================================================================================
+!!bool Examples
+================================================================================
+YAML is a superset of JSON: !!bool true
+Pluto is a planet: !!bool false
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (tag)
+      (plain_scalar
+       (boolean_scalar))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (tag)
+      (plain_scalar
+       (boolean_scalar))))))))
+
+================================================================================
+!!int Examples
+================================================================================
+negative: !!int -12
+zero: !!int 0
+positive: !!int 34
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (tag)
+      (plain_scalar
+       (integer_scalar))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (tag)
+      (plain_scalar
+       (integer_scalar))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (tag)
+      (plain_scalar
+       (integer_scalar))))))))
+
+================================================================================
+!!float Examples
+================================================================================
+negative: !!float -1
+zero: !!float 0
+positive: !!float 2.3e4
+infinity: !!float .inf
+not a number: !!float .nan
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (tag)
+      (plain_scalar
+       (integer_scalar))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (tag)
+      (plain_scalar
+       (integer_scalar))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (tag)
+      (plain_scalar
+       (float_scalar))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (tag)
+      (plain_scalar
+       (float_scalar))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (tag)
+      (plain_scalar
+       (float_scalar))))))))
+
+================================================================================
+JSON Tag Resolution
+================================================================================
+A null: null
+Booleans: [ true, false ]
+Integers: [ 0, -0, 3, -19 ]
+Floats: [ 0., -0.0, 12e03, -2E+05 ]
+Invalid: [ True, Null, 0o7, 0x3A, +12.3 ]
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (plain_scalar
+       (null_scalar))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (flow_sequence
+       (flow_node
+        (plain_scalar
+         (boolean_scalar)))
+       (flow_node
+        (plain_scalar
+         (boolean_scalar))))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (flow_sequence
+       (flow_node
+        (plain_scalar
+         (integer_scalar)))
+       (flow_node
+        (plain_scalar
+         (integer_scalar)))
+       (flow_node
+        (plain_scalar
+         (integer_scalar)))
+       (flow_node
+        (plain_scalar
+         (integer_scalar))))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (flow_sequence
+       (flow_node
+        (plain_scalar
+         (float_scalar)))
+       (flow_node
+        (plain_scalar
+         (float_scalar)))
+       (flow_node
+        (plain_scalar
+         (float_scalar)))
+       (flow_node
+        (plain_scalar
+         (float_scalar))))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (flow_sequence
+       (flow_node
+        (plain_scalar
+         (boolean_scalar)))
+       (flow_node
+        (plain_scalar
+         (null_scalar)))
+       (flow_node
+        (plain_scalar
+         (integer_scalar)))
+       (flow_node
+        (plain_scalar
+         (integer_scalar)))
+       (flow_node
+        (plain_scalar
+         (float_scalar))))))))))
+
+================================================================================
+Core Tag Resolution
+================================================================================
+A null: null
+Also a null: # Empty
+Not a null: ""
+Booleans: [ true, True, false, FALSE ]
+Integers: [ 0, 0o7, 0x3A, -19 ]
+Floats: [ 0., -0.0, .5, +12e03, -2E+05 ]
+Also floats: [ .inf, -.Inf, +.INF, .NAN ]
+
+--------------------------------------------------------------------------------
+
+(stream
+ (document
+  (block_node
+   (block_mapping
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (plain_scalar
+       (null_scalar))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar))))
+    (comment)
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (double_quote_scalar)))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (flow_sequence
+       (flow_node
+        (plain_scalar
+         (boolean_scalar)))
+       (flow_node
+        (plain_scalar
+         (boolean_scalar)))
+       (flow_node
+        (plain_scalar
+         (boolean_scalar)))
+       (flow_node
+        (plain_scalar
+         (boolean_scalar))))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (flow_sequence
+       (flow_node
+        (plain_scalar
+         (integer_scalar)))
+       (flow_node
+        (plain_scalar
+         (integer_scalar)))
+       (flow_node
+        (plain_scalar
+         (integer_scalar)))
+       (flow_node
+        (plain_scalar
+         (integer_scalar))))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (flow_sequence
+       (flow_node
+        (plain_scalar
+         (float_scalar)))
+       (flow_node
+        (plain_scalar
+         (float_scalar)))
+       (flow_node
+        (plain_scalar
+         (float_scalar)))
+       (flow_node
+        (plain_scalar
+         (float_scalar)))
+       (flow_node
+        (plain_scalar
+         (float_scalar))))))
+    (block_mapping_pair
+     (flow_node
+      (plain_scalar
+       (string_scalar)))
+     (flow_node
+      (flow_sequence
+       (flow_node
+        (plain_scalar
+         (float_scalar)))
+       (flow_node
+        (plain_scalar
+         (float_scalar)))
+       (flow_node
+        (plain_scalar
+         (float_scalar)))
+       (flow_node
+        (plain_scalar
+         (float_scalar))))))))))

test/corpus/schema.txt

@@ -1,354 +0,0 @@
-================================================================================
-Example 10.4. !!null Examples
-================================================================================
-!!null null: value for null key
-key with null value: !!null null
-
---------------------------------------------------------------------------------
-
-(stream
-  (document
-    (block_node
-      (block_mapping
-        (block_mapping_pair
-          key: (flow_node
-            (tag)
-            (plain_scalar
-              (null_scalar)))
-          value: (flow_node
-            (plain_scalar
-              (string_scalar))))
-        (block_mapping_pair
-          key: (flow_node
-            (plain_scalar
-              (string_scalar)))
-          value: (flow_node
-            (tag)
-            (plain_scalar
-              (null_scalar))))))))
-
-================================================================================
-Example 10.5. !!bool Examples
-================================================================================
-YAML is a superset of JSON: !!bool true
-Pluto is a planet: !!bool false
-
---------------------------------------------------------------------------------
-
-(stream
-  (document
-    (block_node
-      (block_mapping
-        (block_mapping_pair
-          key: (flow_node
-            (plain_scalar
-              (string_scalar)))
-          value: (flow_node
-            (tag)
-            (plain_scalar
-              (boolean_scalar))))
-        (block_mapping_pair
-          key: (flow_node
-            (plain_scalar
-              (string_scalar)))
-          value: (flow_node
-            (tag)
-            (plain_scalar
-              (boolean_scalar))))))))
-
-================================================================================
-Example 10.6. !!int Examples
-================================================================================
-negative: !!int -12
-zero: !!int 0
-positive: !!int 34
-
---------------------------------------------------------------------------------
-
-(stream
-  (document
-    (block_node
-      (block_mapping
-        (block_mapping_pair
-          key: (flow_node
-            (plain_scalar
-              (string_scalar)))
-          value: (flow_node
-            (tag)
-            (plain_scalar
-              (integer_scalar))))
-        (block_mapping_pair
-          key: (flow_node
-            (plain_scalar
-              (string_scalar)))
-          value: (flow_node
-            (tag)
-            (plain_scalar
-              (integer_scalar))))
-        (block_mapping_pair
-          key: (flow_node
-            (plain_scalar
-              (string_scalar)))
-          value: (flow_node
-            (tag)
-            (plain_scalar
-              (integer_scalar))))))))
-
-================================================================================
-Example 10.7. !!float Examples
-================================================================================
-negative: !!float -1
-zero: !!float 0
-positive: !!float 2.3e4
-infinity: !!float .inf
-not a number: !!float .nan
-
---------------------------------------------------------------------------------
-
-(stream
-  (document
-    (block_node
-      (block_mapping
-        (block_mapping_pair
-          key: (flow_node
-            (plain_scalar
-              (string_scalar)))
-          value: (flow_node
-            (tag)
-            (plain_scalar
-              (integer_scalar))))
-        (block_mapping_pair
-          key: (flow_node
-            (plain_scalar
-              (string_scalar)))
-          value: (flow_node
-            (tag)
-            (plain_scalar
-              (integer_scalar))))
-        (block_mapping_pair
-          key: (flow_node
-            (plain_scalar
-              (string_scalar)))
-          value: (flow_node
-            (tag)
-            (plain_scalar
-              (float_scalar))))
-        (block_mapping_pair
-          key: (flow_node
-            (plain_scalar
-              (string_scalar)))
-          value: (flow_node
-            (tag)
-            (plain_scalar
-              (float_scalar))))
-        (block_mapping_pair
-          key: (flow_node
-            (plain_scalar
-              (string_scalar)))
-          value: (flow_node
-            (tag)
-            (plain_scalar
-              (float_scalar))))))))
-
-================================================================================
-Example 10.8. JSON Tag Resolution
-================================================================================
-A null: null
-Booleans: [ true, false ]
-Integers: [ 0, -0, 3, -19 ]
-Floats: [ 0., -0.0, 12e03, -2E+05 ]
-Invalid: [ True, Null, 0o7, 0x3A, +12.3 ]
-
---------------------------------------------------------------------------------
-
-(stream
-  (document
-    (block_node
-      (block_mapping
-        (block_mapping_pair
-          key: (flow_node
-            (plain_scalar
-              (string_scalar)))
-          value: (flow_node
-            (plain_scalar
-              (null_scalar))))
-        (block_mapping_pair
-          key: (flow_node
-            (plain_scalar
-              (string_scalar)))
-          value: (flow_node
-            (flow_sequence
-              (flow_node
-                (plain_scalar
-                  (boolean_scalar)))
-              (flow_node
-                (plain_scalar
-                  (boolean_scalar))))))
-        (block_mapping_pair
-          key: (flow_node
-            (plain_scalar
-              (string_scalar)))
-          value: (flow_node
-            (flow_sequence
-              (flow_node
-                (plain_scalar
-                  (integer_scalar)))
-              (flow_node
-                (plain_scalar
-                  (integer_scalar)))
-              (flow_node
-                (plain_scalar
-                  (integer_scalar)))
-              (flow_node
-                (plain_scalar
-                  (integer_scalar))))))
-        (block_mapping_pair
-          key: (flow_node
-            (plain_scalar
-              (string_scalar)))
-          value: (flow_node
-            (flow_sequence
-              (flow_node
-                (plain_scalar
-                  (float_scalar)))
-              (flow_node
-                (plain_scalar
-                  (float_scalar)))
-              (flow_node
-                (plain_scalar
-                  (float_scalar)))
-              (flow_node
-                (plain_scalar
-                  (float_scalar))))))
-        (block_mapping_pair
-          key: (flow_node
-            (plain_scalar
-              (string_scalar)))
-          value: (flow_node
-            (flow_sequence
-              (flow_node
-                (plain_scalar
-                  (boolean_scalar)))
-              (flow_node
-                (plain_scalar
-                  (null_scalar)))
-              (flow_node
-                (plain_scalar
-                  (integer_scalar)))
-              (flow_node
-                (plain_scalar
-                  (integer_scalar)))
-              (flow_node
-                (plain_scalar
-                  (float_scalar))))))))))
-
-================================================================================
-Example 10.9. Core Tag Resolution
-================================================================================
-A null: null
-Also a null: # Empty
-Not a null: ""
-Booleans: [ true, True, false, FALSE ]
-Integers: [ 0, 0o7, 0x3A, -19 ]
-Floats: [ 0., -0.0, .5, +12e03, -2E+05 ]
-Also floats: [ .inf, -.Inf, +.INF, .NAN ]
-
---------------------------------------------------------------------------------
-
-(stream
-  (document
-    (block_node
-      (block_mapping
-        (block_mapping_pair
-          key: (flow_node
-            (plain_scalar
-              (string_scalar)))
-          value: (flow_node
-            (plain_scalar
-              (null_scalar))))
-        (block_mapping_pair
-          key: (flow_node
-            (plain_scalar
-              (string_scalar))))
-        (comment)
-        (block_mapping_pair
-          key: (flow_node
-            (plain_scalar
-              (string_scalar)))
-          value: (flow_node
-            (double_quote_scalar)))
-        (block_mapping_pair
-          key: (flow_node
-            (plain_scalar
-              (string_scalar)))
-          value: (flow_node
-            (flow_sequence
-              (flow_node
-                (plain_scalar
-                  (boolean_scalar)))
-              (flow_node
-                (plain_scalar
-                  (boolean_scalar)))
-              (flow_node
-                (plain_scalar
-                  (boolean_scalar)))
-              (flow_node
-                (plain_scalar
-                  (boolean_scalar))))))
-        (block_mapping_pair
-          key: (flow_node
-            (plain_scalar
-              (string_scalar)))
-          value: (flow_node
-            (flow_sequence
-              (flow_node
-                (plain_scalar
-                  (integer_scalar)))
-              (flow_node
-                (plain_scalar
-                  (integer_scalar)))
-              (flow_node
-                (plain_scalar
-                  (integer_scalar)))
-              (flow_node
-                (plain_scalar
-                  (integer_scalar))))))
-        (block_mapping_pair
-          key: (flow_node
-            (plain_scalar
-              (string_scalar)))
-          value: (flow_node
-            (flow_sequence
-              (flow_node
-                (plain_scalar
-                  (float_scalar)))
-              (flow_node
-                (plain_scalar
-                  (float_scalar)))
-              (flow_node
-                (plain_scalar
-                  (float_scalar)))
-              (flow_node
-                (plain_scalar
-                  (float_scalar)))
-              (flow_node
-                (plain_scalar
-                  (float_scalar))))))
-        (block_mapping_pair
-          key: (flow_node
-            (plain_scalar
-              (string_scalar)))
-          value: (flow_node
-            (flow_sequence
-              (flow_node
-                (plain_scalar
-                  (float_scalar)))
-              (flow_node
-                (plain_scalar
-                  (float_scalar)))
-              (flow_node
-                (plain_scalar
-                  (float_scalar)))
-              (flow_node
-                (plain_scalar
-                  (float_scalar))))))))))

test/highlight/core.yaml

@@ -0,0 +1,32 @@
+A null: null
+# <- @property
+#     ^ @punctuation.delimiter
+#       ^^^^ @constant.builtin
+Also a null: # Empty
+#            ^^^^^^^ @comment
+Not a null: ""
+#           ^^ @string
+Booleans: [ true, True, false, FALSE ]
+#         ^ @punctuation.bracket
+#                                    ^ @punctuation.bracket
+#               ^ @punctuation.delimiter
+#           ^^^^ @boolean
+#                 ^^^^ @boolean
+#                       ^^^^^ @boolean
+#                              ^^^^^ @boolean
+Integers: [ 0, 0o7, 0x3A, -19 ]
+#           ^ @number
+#              ^^^ @number
+#                   ^^^^ @number
+#                         ^^^ @number
+Floats: [ 0., -0.0, .5, +12e03, -2E+05 ]
+#         ^^ @number
+#             ^^^^ @number
+#                   ^^ @number
+#                       ^^^^^^ @number
+#                               ^^^^^^ @number
+Also floats: [ .inf, -.Inf, +.INF, .NAN ]
+#              ^^^^ @number
+#                    ^^^^^ @number
+#                           ^^^^^ @number
+#                                  ^^^^ @number