dunham/tree-sitter-newt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

case/let and some cleanup of the cst

Browse Source
This commit is contained in:
Steve Dunham
2026-03-08 10:06:53 -07:00
parent eaf020bb91
commit 1a9e76b3e9
9 changed files with 31995 additions and 22415 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.editorconfig
View File

@@ -2,6 +2,8 @@ root = true
[*]
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
[*.{json,toml,yml,gyp}]
indent_style = space

63
grammar.js
View File

@@ -41,40 +41,57 @@ module.exports = grammar({
seq("/-", /([^-]|-+[^/])-/, "/"),
),
),
_arr: ($) => choice("->", "→"),
_arr: _ => choice("->", "→"),
number: $ => /\d+/,
lamExpr: $ => seq(
choice("\\", "λ"),
repeat1($.identifier),
"=>",
$.typeExpr
$._typeExpr
),
// hole, parenTypeExpression, record update
_atom: $ => choice($.identifier, $.string, $.character, $.number, $.recUpdate, seq("(", $.typeExpr, ")")),
_parg: $ => choice(seq("{{", $.typeExpr, "}}"), seq("{", $.typeExpr, "}"), $._atom),
_atom: $ => choice($.identifier, $.string, $.character, $.number, $.recUpdate, seq("(", $._typeExpr, ")")),
_parg: $ => choice(seq("{{", $._typeExpr, "}}"), seq("{", $._typeExpr, "}"), $._atom),
recUpdate: $ => seq("[", sep(";", seq($.identifier, choice(":=", "$="), $.term)), "]"),
_appExpr: $ => (seq($._atom, repeat($._parg))),
qname: ($) => sep1(".", $.identifier),
string: $ => /"[^"]*"/,
character: $ => /'(\\)?.'/,
doCaseLet: $ => seq("let", "(", $.term, ")", "=", $.typeExpr, repeat($.orAlt)),
string: _ => /"[^"]*"/,
character: _ => /'(\\)?.'/,
doLet: $ => seq("let", $.identifier, "=", $._typeExpr),
doCaseLet: $ => seq("let", "(", $.term, ")", "=", $._typeExpr, repeat($.orAlt)),
caseAlt: $ => seq($.term, "=>", $.term),
orAlt: $ => seq("|", $.caseAlt),
// layout was causing trouble here. I kinda wanted to ditch it, but there
// could be a shift/reduce thing in the real parser
_doArrow: $ => seq("<-", $.typeExpr, repeat($.orAlt)),
_doArrow: $ => seq("<-", $._typeExpr, repeat($.orAlt)),
doArrow: $ => seq($.term, optional($._doArrow)),
doLet: $ => seq("let", $.identifier, "=", $.term),
_doExpr: $ => choice(
$.doCaseLet,
$.doLet,
$.doArrow),
doBlock: $ => seq("do", layout($, $._doExpr)),
ifThen: ($) => seq("if", $.term, "then", $.term, "else", $.term),
caseExpr: $ => seq(
"case",
$._typeExpr,
"of",
layout($,$.caseAlt)
),
caseLet: $ => seq(
// what do we do with "in" - it makes an end without a start...
"let", "(", $._typeExpr,")","=",repeat($.orAlt),"in"
),
letAssign: $ => seq($.identifier, "=", $._typeExpr),
letStmt: $ => seq(
"let",
layout($,$.letAssign),
"in",
$._typeExpr),
_term2: ($) =>
choice(
// caseExpr
// caseLet
$.caseExpr,
$.caseLet,
$.letStmt,
// caseLamExpr
$.lamExpr,
$.doBlock,
@@ -91,14 +108,14 @@ module.exports = grammar({
binder: ($) =>
choice(
// repeat($.identifier) has a conflict
seq("(", alias(optional("0"), "quantity"), $.identifier, ":", $.typeExpr, ")"),
seq("{{", $.typeExpr, "}}"),
seq("{", alias(optional("0"), "quantity"), repeat1($.identifier), ":", $.typeExpr, "}"),
seq("(", alias(optional("0"), "quantity"), $.identifier, ":", $._typeExpr, ")"),
seq("{{", $._typeExpr, "}}"),
seq("{", alias(optional("0"), "quantity"), repeat1($.identifier), ":", $._typeExpr, "}"),
),
forall: ($) => seq("∀", repeat1($.identifier), ".", $.typeExpr),
binders: ($) => seq(choice(repeat1($.binder)), $._arr, $.typeExpr),
typeExpr: ($) => prec.right(choice($.forall, $.binders, seq($.term, optional(seq($._arr, $.typeExpr))))),
forall: ($) => seq(choice("∀", "forall"), repeat1($.identifier), ".", $._typeExpr),
binders: ($) => seq(choice(repeat1($.binder)), $._arr, $._typeExpr),
_typeExpr: ($) => prec.right(choice($.forall, $.binders, seq($.term, optional(seq($._arr, $._typeExpr))))),
// pitype: ($) =>
// seq(
@@ -106,9 +123,9 @@ module.exports = grammar({
// repeat(seq(repeat1(choice($.identifier, $.binder)), $._arr)),
// $.identifier,
// ),
sigDecl: ($) => seq($.identifier, ":", $.typeExpr),
sigDecl: ($) => seq($.identifier, ":", $._typeExpr),
whereClause: $ => seq("where", layout($, choice($.sigDecl, $.defDecl))),
defDecl: ($) => seq(alias($._appExpr, $.lhs), "=", $.typeExpr, optional($.whereClause)),
defDecl: ($) => seq(alias($._appExpr, $.lhs), "=", $._typeExpr, optional($.whereClause)),
shortDataDecl: $ => seq(
"data",
alias($.identifier, "typeName"),
@@ -121,7 +138,7 @@ module.exports = grammar({
"data",
alias($.identifier, "typeName"),
":",
$.typeExpr,
$._typeExpr,
// the layout here can be empty (so no start tag)
// optional doesn't seem to help, so we have an error at void
optional(seq("where", optional(layout($, $.sigDecl)))),
@@ -133,14 +150,14 @@ module.exports = grammar({
alias($.identifier, "name"),
optional(seq("uses", "(", repeat1($.identifier), ")")),
":",
$.typeExpr,
$._typeExpr,
":=",
$.jsLitString
),
ptypeDecl: $ => seq(
"ptype",
alias($.identifier, $.name),
optional(seq(":", $.typeExpr))
optional(seq(":", $._typeExpr))
),
importDef: ($) => seq("import", $.qname),
mixfixDecl: $ => seq(
@@ -165,7 +182,7 @@ module.exports = grammar({
),
instanceDecl: $ => seq(
"instance",
$.typeExpr,
$._typeExpr,
"where",
layout($, choice($.sigDecl, $.defDecl))
),

4
queries/highlights.scm
View File

@@ -2,14 +2,14 @@
"let"
; "in"
"where"
; "case" "of"
"case" "of"
"data"
;"U"
"do"
"ptype" "pfunc"
"module"
; "infixl" "infixr" "infix"
"∀" ; "forall"
"∀" "forall"
"import"
"uses" "derive"
"class" "instance" "record" "constructor"

3
queries/injections.scm Normal file
View File

@@ -0,0 +1,3 @@
(jsLitString (jsStringFragment) @injection.content
(#set! injection.language "javascript"))

250
src/grammar.json generated
View File

@@ -91,7 +91,7 @@
},
{
"type": "SYMBOL",
"name": "typeExpr"
"name": "_typeExpr"
}
]
},
@@ -127,7 +127,7 @@
},
{
"type": "SYMBOL",
"name": "typeExpr"
"name": "_typeExpr"
},
{
"type": "STRING",
@@ -149,7 +149,7 @@
},
{
"type": "SYMBOL",
"name": "typeExpr"
"name": "_typeExpr"
},
{
"type": "STRING",
@@ -166,7 +166,7 @@
},
{
"type": "SYMBOL",
"name": "typeExpr"
"name": "_typeExpr"
},
{
"type": "STRING",
@@ -319,6 +319,27 @@
"type": "PATTERN",
"value": "'(\\\\)?.'"
},
"doLet": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "let"
},
{
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "STRING",
"value": "="
},
{
"type": "SYMBOL",
"name": "_typeExpr"
}
]
},
"doCaseLet": {
"type": "SEQ",
"members": [
@@ -344,7 +365,7 @@
},
{
"type": "SYMBOL",
"name": "typeExpr"
"name": "_typeExpr"
},
{
"type": "REPEAT",
@@ -394,7 +415,7 @@
},
{
"type": "SYMBOL",
"name": "typeExpr"
"name": "_typeExpr"
},
{
"type": "REPEAT",
@@ -426,27 +447,6 @@
}
]
},
"doLet": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "let"
},
{
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "STRING",
"value": "="
},
{
"type": "SYMBOL",
"name": "term"
}
]
},
"_doExpr": {
"type": "CHOICE",
"members": [
@@ -531,9 +531,166 @@
}
]
},
"caseExpr": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "case"
},
{
"type": "SYMBOL",
"name": "_typeExpr"
},
{
"type": "STRING",
"value": "of"
},
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "start"
},
{
"type": "REPEAT",
"content": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "semi"
},
{
"type": "SYMBOL",
"name": "caseAlt"
}
]
}
},
{
"type": "SYMBOL",
"name": "end"
}
]
}
]
},
"caseLet": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "let"
},
{
"type": "STRING",
"value": "("
},
{
"type": "SYMBOL",
"name": "_typeExpr"
},
{
"type": "STRING",
"value": ")"
},
{
"type": "STRING",
"value": "="
},
{
"type": "REPEAT",
"content": {
"type": "SYMBOL",
"name": "orAlt"
}
},
{
"type": "STRING",
"value": "in"
}
]
},
"letAssign": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "STRING",
"value": "="
},
{
"type": "SYMBOL",
"name": "_typeExpr"
}
]
},
"letStmt": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "let"
},
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "start"
},
{
"type": "REPEAT",
"content": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "semi"
},
{
"type": "SYMBOL",
"name": "letAssign"
}
]
}
},
{
"type": "SYMBOL",
"name": "end"
}
]
},
{
"type": "STRING",
"value": "in"
},
{
"type": "SYMBOL",
"name": "_typeExpr"
}
]
},
"_term2": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "caseExpr"
},
{
"type": "SYMBOL",
"name": "caseLet"
},
{
"type": "SYMBOL",
"name": "letStmt"
},
{
"type": "SYMBOL",
"name": "lamExpr"
@@ -627,7 +784,7 @@
},
{
"type": "SYMBOL",
"name": "typeExpr"
"name": "_typeExpr"
},
{
"type": "STRING",
@@ -644,7 +801,7 @@
},
{
"type": "SYMBOL",
"name": "typeExpr"
"name": "_typeExpr"
},
{
"type": "STRING",
@@ -689,7 +846,7 @@
},
{
"type": "SYMBOL",
"name": "typeExpr"
"name": "_typeExpr"
},
{
"type": "STRING",
@@ -703,8 +860,17 @@
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "∀"
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "∀"
},
{
"type": "STRING",
"value": "forall"
}
]
},
{
"type": "REPEAT1",
@@ -719,7 +885,7 @@
},
{
"type": "SYMBOL",
"name": "typeExpr"
"name": "_typeExpr"
}
]
},
@@ -744,11 +910,11 @@
},
{
"type": "SYMBOL",
"name": "typeExpr"
"name": "_typeExpr"
}
]
},
"typeExpr": {
"_typeExpr": {
"type": "PREC_RIGHT",
"value": 0,
"content": {
@@ -781,7 +947,7 @@
},
{
"type": "SYMBOL",
"name": "typeExpr"
"name": "_typeExpr"
}
]
},
@@ -808,7 +974,7 @@
},
{
"type": "SYMBOL",
"name": "typeExpr"
"name": "_typeExpr"
}
]
},
@@ -877,7 +1043,7 @@
},
{
"type": "SYMBOL",
"name": "typeExpr"
"name": "_typeExpr"
},
{
"type": "CHOICE",
@@ -1003,7 +1169,7 @@
},
{
"type": "SYMBOL",
"name": "typeExpr"
"name": "_typeExpr"
},
{
"type": "CHOICE",
@@ -1153,7 +1319,7 @@
},
{
"type": "SYMBOL",
"name": "typeExpr"
"name": "_typeExpr"
},
{
"type": "STRING",
@@ -1193,7 +1359,7 @@
},
{
"type": "SYMBOL",
"name": "typeExpr"
"name": "_typeExpr"
}
]
},
@@ -1427,7 +1593,7 @@
},
{
"type": "SYMBOL",
"name": "typeExpr"
"name": "_typeExpr"
},
{
"type": "STRING",

347
src/node-types.json generated
View File

@@ -7,12 +7,20 @@
"multiple": true,
"required": true,
"types": [
{
"type": "binders",
"named": true
},
{
"type": "forall",
"named": true
},
{
"type": "identifier",
"named": true
},
{
"type": "typeExpr",
"type": "term",
"named": true
}
]
@@ -31,7 +39,15 @@
"named": true
},
{
"type": "typeExpr",
"type": "binders",
"named": true
},
{
"type": "forall",
"named": true
},
{
"type": "term",
"named": true
}
]
@@ -52,6 +68,72 @@
]
}
},
{
"type": "caseExpr",
"named": true,
"fields": {},
"children": {
"multiple": true,
"required": true,
"types": [
{
"type": "binders",
"named": true
},
{
"type": "caseAlt",
"named": true
},
{
"type": "end",
"named": true
},
{
"type": "forall",
"named": true
},
{
"type": "semi",
"named": true
},
{
"type": "start",
"named": true
},
{
"type": "term",
"named": true
}
]
}
},
{
"type": "caseLet",
"named": true,
"fields": {},
"children": {
"multiple": true,
"required": true,
"types": [
{
"type": "binders",
"named": true
},
{
"type": "forall",
"named": true
},
{
"type": "orAlt",
"named": true
},
{
"type": "term",
"named": true
}
]
}
},
{
"type": "classDecl",
"named": true,
@@ -99,10 +181,18 @@
"multiple": true,
"required": true,
"types": [
{
"type": "binders",
"named": true
},
{
"type": "end",
"named": true
},
{
"type": "forall",
"named": true
},
{
"type": "semi",
"named": true
@@ -116,7 +206,7 @@
"named": true
},
{
"type": "typeExpr",
"type": "term",
"named": true
}
]
@@ -130,12 +220,20 @@
"multiple": true,
"required": true,
"types": [
{
"type": "binders",
"named": true
},
{
"type": "forall",
"named": true
},
{
"type": "lhs",
"named": true
},
{
"type": "typeExpr",
"type": "term",
"named": true
},
{
@@ -168,6 +266,14 @@
"multiple": true,
"required": true,
"types": [
{
"type": "binders",
"named": true
},
{
"type": "forall",
"named": true
},
{
"type": "orAlt",
"named": true
@@ -175,10 +281,6 @@
{
"type": "term",
"named": true
},
{
"type": "typeExpr",
"named": true
}
]
}
@@ -226,6 +328,14 @@
"multiple": true,
"required": true,
"types": [
{
"type": "binders",
"named": true
},
{
"type": "forall",
"named": true
},
{
"type": "orAlt",
"named": true
@@ -233,10 +343,6 @@
{
"type": "term",
"named": true
},
{
"type": "typeExpr",
"named": true
}
]
}
@@ -249,6 +355,14 @@
"multiple": true,
"required": true,
"types": [
{
"type": "binders",
"named": true
},
{
"type": "forall",
"named": true
},
{
"type": "identifier",
"named": true
@@ -283,12 +397,20 @@
"multiple": true,
"required": true,
"types": [
{
"type": "binders",
"named": true
},
{
"type": "forall",
"named": true
},
{
"type": "identifier",
"named": true
},
{
"type": "typeExpr",
"type": "term",
"named": true
}
]
@@ -332,6 +454,10 @@
"multiple": true,
"required": true,
"types": [
{
"type": "binders",
"named": true
},
{
"type": "defDecl",
"named": true
@@ -340,6 +466,10 @@
"type": "end",
"named": true
},
{
"type": "forall",
"named": true
},
{
"type": "semi",
"named": true
@@ -353,7 +483,7 @@
"named": true
},
{
"type": "typeExpr",
"type": "term",
"named": true
}
]
@@ -382,12 +512,86 @@
"multiple": true,
"required": true,
"types": [
{
"type": "binders",
"named": true
},
{
"type": "forall",
"named": true
},
{
"type": "identifier",
"named": true
},
{
"type": "typeExpr",
"type": "term",
"named": true
}
]
}
},
{
"type": "letAssign",
"named": true,
"fields": {},
"children": {
"multiple": true,
"required": true,
"types": [
{
"type": "binders",
"named": true
},
{
"type": "forall",
"named": true
},
{
"type": "identifier",
"named": true
},
{
"type": "term",
"named": true
}
]
}
},
{
"type": "letStmt",
"named": true,
"fields": {},
"children": {
"multiple": true,
"required": true,
"types": [
{
"type": "binders",
"named": true
},
{
"type": "end",
"named": true
},
{
"type": "forall",
"named": true
},
{
"type": "letAssign",
"named": true
},
{
"type": "semi",
"named": true
},
{
"type": "start",
"named": true
},
{
"type": "term",
"named": true
}
]
@@ -401,10 +605,18 @@
"multiple": true,
"required": true,
"types": [
{
"type": "binders",
"named": true
},
{
"type": "character",
"named": true
},
{
"type": "forall",
"named": true
},
{
"type": "identifier",
"named": true
@@ -422,7 +634,7 @@
"named": true
},
{
"type": "typeExpr",
"type": "term",
"named": true
}
]
@@ -537,6 +749,14 @@
"multiple": true,
"required": true,
"types": [
{
"type": "binders",
"named": true
},
{
"type": "forall",
"named": true
},
{
"type": "identifier",
"named": true
@@ -546,7 +766,7 @@
"named": true
},
{
"type": "typeExpr",
"type": "term",
"named": true
}
]
@@ -560,12 +780,20 @@
"multiple": true,
"required": true,
"types": [
{
"type": "binders",
"named": true
},
{
"type": "forall",
"named": true
},
{
"type": "name",
"named": true
},
{
"type": "typeExpr",
"type": "term",
"named": true
}
]
@@ -652,10 +880,18 @@
"multiple": true,
"required": false,
"types": [
{
"type": "binders",
"named": true
},
{
"type": "character",
"named": true
},
{
"type": "forall",
"named": true
},
{
"type": "identifier",
"named": true
@@ -673,7 +909,7 @@
"named": true
},
{
"type": "typeExpr",
"type": "term",
"named": true
}
]
@@ -687,12 +923,20 @@
"multiple": true,
"required": true,
"types": [
{
"type": "binders",
"named": true
},
{
"type": "forall",
"named": true
},
{
"type": "identifier",
"named": true
},
{
"type": "typeExpr",
"type": "term",
"named": true
}
]
@@ -722,6 +966,18 @@
"multiple": true,
"required": true,
"types": [
{
"type": "binders",
"named": true
},
{
"type": "caseExpr",
"named": true
},
{
"type": "caseLet",
"named": true
},
{
"type": "character",
"named": true
@@ -734,6 +990,10 @@
"type": "dollar",
"named": true
},
{
"type": "forall",
"named": true
},
{
"type": "identifier",
"named": true
@@ -746,6 +1006,10 @@
"type": "lamExpr",
"named": true
},
{
"type": "letStmt",
"named": true
},
{
"type": "number",
"named": true
@@ -758,36 +1022,9 @@
"type": "string",
"named": true
},
{
"type": "typeExpr",
"named": true
}
]
}
},
{
"type": "typeExpr",
"named": true,
"fields": {},
"children": {
"multiple": true,
"required": true,
"types": [
{
"type": "binders",
"named": true
},
{
"type": "forall",
"named": true
},
{
"type": "term",
"named": true
},
{
"type": "typeExpr",
"named": true
}
]
}
@@ -887,6 +1124,10 @@
"type": "`",
"named": false
},
{
"type": "case",
"named": false
},
{
"type": "character",
"named": true
@@ -932,6 +1173,10 @@
"type": "end",
"named": true
},
{
"type": "forall",
"named": false
},
{
"type": "identifier",
"named": true
@@ -944,6 +1189,10 @@
"type": "import",
"named": false
},
{
"type": "in",
"named": false
},
{
"type": "infixl",
"named": false
@@ -980,6 +1229,10 @@
"type": "number",
"named": true
},
{
"type": "of",
"named": false
},
{
"type": "pfunc",
"named": false

53694
src/parser.c generated
View File

File diff suppressed because it is too large Load Diff

22
src/scanner.c
View File

@@ -53,6 +53,13 @@ static int32_t peek(State *state) {
#define PEEK lexer->lookahead
#define PEEK_WS (PEEK == ' ' || PEEK == '\n' || PEEK == '\t')
static bool isAtIn(TSLexer *lexer) {
if (PEEK != 'i') return false;
lexer->mark_end(lexer);
lexer->advance(lexer, false);
return PEEK == 'n';
}
/**
* The custom scanner is responsible for the virtual indent, outdent, and semi tokens.
* Additionally it handles whitespace. This allows us to give the virtual tokens priority over
@@ -95,15 +102,12 @@ bool tree_sitter_newt_external_scanner_scan(State *state, TSLexer *lexer,
// even if it's not expected (I think this is important)
// on the editor side there is a `then` expected vs outdented `then`, but
// maybe GLR can detect a "stray" END token?
if (syms[VIRT_END] || true) {
if (col < cur) {
fprintf(stderr, "end [%d %d %d %d] %d %d\n", syms[0], syms[1], syms[2],
syms[3], col, cur);
pop(state);
lexer->result_symbol = VIRT_END;
return true;
}
if (ws && (col < cur || PEEK == '|' || isAtIn(lexer))) {
fprintf(stderr, "end [%d %d %d %d] %d %d\n", syms[0], syms[1], syms[2],
syms[3], col, cur);
pop(state);
lexer->result_symbol = VIRT_END;
return true;
}
// but we can't do that for semi?
if (syms[VIRT_SEMI]) {

1
tree-sitter.json
View File

@@ -13,6 +13,7 @@
"class-name": "TreeSitterNewt"
}
],
"highlights": "queries/highlights.scm",
"metadata": {
"version": "0.1.0",
"license": "MIT",