[WIP] Start extracting overloaded foreach commands

corpus/foreach.txt

@@ -9,7 +9,7 @@ endforeach()
 (source_file
  (command_invocation
   (foreach_loop
-   (foreach_items
+   (foreach_iter
     loop_var: (variable)
    )
   )
@@ -27,7 +27,7 @@ endforeach(var)
 (source_file
  (command_invocation
   (foreach_loop
-   (foreach_items
+   (foreach_iter
     loop_var: (variable)
    )
    (variable)
@@ -46,7 +46,53 @@ endforeach()
 (source_file
  (command_invocation
   (foreach_loop
-   (foreach_items
+   (foreach_iter
+    loop_var: (variable)
+    (seperation (space))
+    (arguments
+     (argument (unquoted_argument))
+    )
+   )
+  )
+ )
+ )
+
+=========
+Two items
+=========
+
+foreach(var item item)
+endforeach()
+
+---
+(source_file
+ (command_invocation
+  (foreach_loop
+   (foreach_iter
+    loop_var: (variable)
+    (seperation (space))
+    (arguments
+     (argument (unquoted_argument))
+     (seperation (space))
+     (argument (unquoted_argument))
+    )
+   )
+  )
+ )
+ )
+
+=====================
+Iter containing RANGE
+=====================
+
+foreach(var RANGE1)
+endforeach()
+
+---
+(source_file
+ (command_invocation
+  (foreach_loop
+   (foreach_iter
     loop_var: (variable)
     (seperation (space))
     (arguments
@@ -72,7 +118,6 @@ endforeach()
     (foreach_range_stop
      loop_var: (variable)
      (seperation (space))
-     (seperation (space))
      stop: (integer)
     )
    )
@@ -95,9 +140,7 @@ endforeach()
     (foreach_range_full
      loop_var: (variable)
      (seperation (space))
-     (seperation (space))
      start: (integer)
-     (seperation (space))
      stop: (integer)
     )
    )
@@ -106,3 +149,52 @@ endforeach()
  )
 
 
+=========================
+Range lists with one list
+=========================
+
+foreach(var IN LISTS A)
+endforeach()
+
+---
+(source_file
+ (command_invocation
+  (foreach_loop
+   (foreach_lists_items
+    loop_var: (variable)
+    (seperation (space))
+    (seperation (space))
+    (seperation (space))
+    (arguments
+     (argument (unquoted_argument))
+    )
+   )
+  )
+ )
+ )
+
+==========================
+Range lists with two lists
+==========================
+
+foreach(var IN LISTS A B)
+endforeach()
+
+---
+(source_file
+ (command_invocation
+  (foreach_loop
+   (foreach_lists_items
+    loop_var: (variable)
+    (seperation (space))
+    (seperation (space))
+    (seperation (space))
+    (arguments
+     (argument (unquoted_argument))
+     (seperation (space))
+     (argument (unquoted_argument))
+    )
+   )
+  )
+ )
+ )

grammar.js

@@ -7,7 +7,7 @@ module.exports = grammar({
     line_ending: ($) => $.newline,
     seperation: ($) => choice($.space, $.line_ending),
     space: ($) => /[ \t]+/,
-    newline: ($) => /\n/,
+    newline: ($) => /\n+/,
     identifier: ($) => /[A-Za-z_][A-Za-z0-9_]*/,
     integer: ($) => /[+-]*\d+/,
 
@@ -45,7 +45,7 @@ module.exports = grammar({
         repeat($.seperation),
         "(",
         repeat($.seperation),
-        choice($.foreach_items, $.foreach_range, $.foreach_lists_items, $.foreach_zip_lists),
+        choice($.foreach_range, $.foreach_lists_items, $.foreach_iter),
         ")",
         repeat($.command_invocation),
         repeat($.space),
@@ -55,32 +55,38 @@ module.exports = grammar({
         optional($.variable),
         ")"
       ),
-    foreach_items: ($) =>
-      seq(field("loop_var", $.variable), repeat($.seperation), optional($.arguments)),
-    foreach_lists_items: ($) => seq("b"),
-    foreach_zip_lists: ($) => seq("c"),
 
     foreach_range: ($) => choice($.foreach_range_stop, $.foreach_range_full),
     foreach_range_stop: ($) =>
       seq(
         field("loop_var", $.variable),
         repeat1($.seperation),
-        "RANGE",
+        seq("RANGE", optional($.seperation)),
-        repeat1($.seperation),
+        optional(field("stop", $.integer))
-        field("stop", $.integer)
       ),
     foreach_range_full: ($) =>
       seq(
         field("loop_var", $.variable),
         repeat1($.seperation),
-        "RANGE",
+        /RANGE[ \t\n]+/,
-        repeat1($.seperation),
         field("start", $.integer),
-        repeat1($.seperation),
         field("stop", $.integer),
         optional(seq(repeat1($.seperation), field("step", $.integer)))
       ),
 
+    foreach_lists_items: ($) =>
+      seq(
+        field("loop_var", $.variable),
+        "IN",
+        repeat(seq(repeat1($.seperation), choice($.foreach_lists)))
+      ),
+    foreach_lists: ($) =>
+      prec.left(seq("LISTS", optional(seq(repeat1($.seperation), $.variable)))),
+    // foreach_items: ($) => prec.left(seq("ITEMS", repeat(seq(repeat1($.seperation), $.argument)))),
+
+    foreach_iter: ($) =>
+      seq(field("loop_var", $.variable), optional(seq(repeat1($.seperation), $.arguments))),
+
     normal_command: ($) =>
       seq(
         repeat($.space),

queries/highlights.scm

@@ -1,3 +1,8 @@
-(argument) @variable.parameter
+(bracket_content) @parameter
-(normal_var) @variable.builtin
+(quoted_element) @parameter
-(command_invocation (identifier) @function)
+(unquoted_argument) @parameter
+(variable) @variable.builtin
+(command_invocation [
+  (normal_command)
+  (foreach_loop)
+] @function)

src/grammar.json

@@ -31,7 +31,7 @@
     },
     "newline": {
       "type": "PATTERN",
-      "value": "\\n"
+      "value": "\\n+"
     },
     "identifier": {
       "type": "PATTERN",
@@ -417,10 +417,6 @@
         {
           "type": "CHOICE",
           "members": [
-            {
-              "type": "SYMBOL",
-              "name": "foreach_items"
-            },
             {
               "type": "SYMBOL",
               "name": "foreach_range"
@@ -431,7 +427,7 @@
             },
             {
               "type": "SYMBOL",
-              "name": "foreach_zip_lists"
+              "name": "foreach_iter"
             }
           ]
         },
@@ -486,56 +482,6 @@
         }
       ]
     },
-    "foreach_items": {
-      "type": "SEQ",
-      "members": [
-        {
-          "type": "FIELD",
-          "name": "loop_var",
-          "content": {
-            "type": "SYMBOL",
-            "name": "variable"
-          }
-        },
-        {
-          "type": "REPEAT",
-          "content": {
-            "type": "SYMBOL",
-            "name": "seperation"
-          }
-        },
-        {
-          "type": "CHOICE",
-          "members": [
-            {
-              "type": "SYMBOL",
-              "name": "arguments"
-            },
-            {
-              "type": "BLANK"
-            }
-          ]
-        }
-      ]
-    },
-    "foreach_lists_items": {
-      "type": "SEQ",
-      "members": [
-        {
-          "type": "STRING",
-          "value": "b"
-        }
-      ]
-    },
-    "foreach_zip_lists": {
-      "type": "SEQ",
-      "members": [
-        {
-          "type": "STRING",
-          "value": "c"
-        }
-      ]
-    },
     "foreach_range": {
       "type": "CHOICE",
       "members": [
@@ -568,23 +514,41 @@
           }
         },
         {
-          "type": "STRING",
+          "type": "SEQ",
-          "value": "RANGE"
+          "members": [
+            {
+              "type": "STRING",
+              "value": "RANGE"
+            },
+            {
+              "type": "CHOICE",
+              "members": [
+                {
+                  "type": "SYMBOL",
+                  "name": "seperation"
+                },
+                {
+                  "type": "BLANK"
+                }
+              ]
+            }
+          ]
         },
         {
-          "type": "REPEAT1",
+          "type": "CHOICE",
-          "content": {
+          "members": [
-            "type": "SYMBOL",
+            {
-            "name": "seperation"
+              "type": "FIELD",
-          }
+              "name": "stop",
-        },
+              "content": {
-        {
+                "type": "SYMBOL",
-          "type": "FIELD",
+                "name": "integer"
-          "name": "stop",
+              }
-          "content": {
+            },
-            "type": "SYMBOL",
+            {
-            "name": "integer"
+              "type": "BLANK"
-          }
+            }
+          ]
         }
       ]
     },
@@ -607,15 +571,8 @@
           }
         },
         {
-          "type": "STRING",
+          "type": "PATTERN",
-          "value": "RANGE"
+          "value": "RANGE[ \\t\\n]+"
-        },
-        {
-          "type": "REPEAT1",
-          "content": {
-            "type": "SYMBOL",
-            "name": "seperation"
-          }
         },
         {
           "type": "FIELD",
@@ -625,13 +582,6 @@
             "name": "integer"
           }
         },
-        {
-          "type": "REPEAT1",
-          "content": {
-            "type": "SYMBOL",
-            "name": "seperation"
-          }
-        },
         {
           "type": "FIELD",
           "name": "stop",
@@ -670,6 +620,121 @@
         }
       ]
     },
+    "foreach_lists_items": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "FIELD",
+          "name": "loop_var",
+          "content": {
+            "type": "SYMBOL",
+            "name": "variable"
+          }
+        },
+        {
+          "type": "STRING",
+          "value": "IN"
+        },
+        {
+          "type": "REPEAT",
+          "content": {
+            "type": "SEQ",
+            "members": [
+              {
+                "type": "REPEAT1",
+                "content": {
+                  "type": "SYMBOL",
+                  "name": "seperation"
+                }
+              },
+              {
+                "type": "CHOICE",
+                "members": [
+                  {
+                    "type": "SYMBOL",
+                    "name": "foreach_lists"
+                  }
+                ]
+              }
+            ]
+          }
+        }
+      ]
+    },
+    "foreach_lists": {
+      "type": "PREC_LEFT",
+      "value": 0,
+      "content": {
+        "type": "SEQ",
+        "members": [
+          {
+            "type": "STRING",
+            "value": "LISTS"
+          },
+          {
+            "type": "CHOICE",
+            "members": [
+              {
+                "type": "SEQ",
+                "members": [
+                  {
+                    "type": "REPEAT1",
+                    "content": {
+                      "type": "SYMBOL",
+                      "name": "seperation"
+                    }
+                  },
+                  {
+                    "type": "SYMBOL",
+                    "name": "variable"
+                  }
+                ]
+              },
+              {
+                "type": "BLANK"
+              }
+            ]
+          }
+        ]
+      }
+    },
+    "foreach_iter": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "FIELD",
+          "name": "loop_var",
+          "content": {
+            "type": "SYMBOL",
+            "name": "variable"
+          }
+        },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "SEQ",
+              "members": [
+                {
+                  "type": "REPEAT1",
+                  "content": {
+                    "type": "SYMBOL",
+                    "name": "seperation"
+                  }
+                },
+                {
+                  "type": "SYMBOL",
+                  "name": "arguments"
+                }
+              ]
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
+        }
+      ]
+    },
     "normal_command": {
       "type": "SEQ",
       "members": [

src/node-types.json

@@ -116,7 +116,7 @@
     "fields": {}
   },
   {
-    "type": "foreach_items",
+    "type": "foreach_iter",
     "named": true,
     "fields": {
       "loop_var": {
@@ -145,10 +145,54 @@
       ]
     }
   },
+  {
+    "type": "foreach_lists",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": false,
+      "types": [
+        {
+          "type": "seperation",
+          "named": true
+        },
+        {
+          "type": "variable",
+          "named": true
+        }
+      ]
+    }
+  },
   {
     "type": "foreach_lists_items",
     "named": true,
-    "fields": {}
+    "fields": {
+      "loop_var": {
+        "multiple": false,
+        "required": true,
+        "types": [
+          {
+            "type": "variable",
+            "named": true
+          }
+        ]
+      }
+    },
+    "children": {
+      "multiple": true,
+      "required": false,
+      "types": [
+        {
+          "type": "foreach_lists",
+          "named": true
+        },
+        {
+          "type": "seperation",
+          "named": true
+        }
+      ]
+    }
   },
   {
     "type": "foreach_loop",
@@ -163,7 +207,7 @@
           "named": true
         },
         {
-          "type": "foreach_items",
+          "type": "foreach_iter",
           "named": true
         },
         {
@@ -174,10 +218,6 @@
           "type": "foreach_range",
           "named": true
         },
-        {
-          "type": "foreach_zip_lists",
-          "named": true
-        },
         {
           "type": "seperation",
           "named": true
@@ -284,7 +324,7 @@
       },
       "stop": {
         "multiple": false,
-        "required": true,
+        "required": false,
         "types": [
           {
             "type": "integer",
@@ -304,11 +344,6 @@
       ]
     }
   },
-  {
-    "type": "foreach_zip_lists",
-    "named": true,
-    "fields": {}
-  },
   {
     "type": "line_ending",
     "named": true,
@@ -523,6 +558,14 @@
     "type": "=",
     "named": false
   },
+  {
+    "type": "IN",
+    "named": false
+  },
+  {
+    "type": "LISTS",
+    "named": false
+  },
   {
     "type": "RANGE",
     "named": false
@@ -551,14 +594,6 @@
     "type": "]",
     "named": false
   },
-  {
-    "type": "b",
-    "named": false
-  },
-  {
-    "type": "c",
-    "named": false
-  },
   {
     "type": "endforeach",
     "named": false