Updates to highlighting in playground

playground/src/cmeditor.ts

@@ -1,11 +1,16 @@
 import { AbstractEditor, EditorDelegate, Marker } from "./types";
 import { basicSetup } from "codemirror";
-import { defaultKeymap, indentMore, indentLess, toggleLineComment } from "@codemirror/commands";
+import {
+  indentMore,
+  indentLess,
+  toggleLineComment,
+} from "@codemirror/commands";
 import { EditorView, hoverTooltip, keymap, Tooltip } from "@codemirror/view";
 import { Compartment, Prec } from "@codemirror/state";
 import { oneDark } from "@codemirror/theme-one-dark";
 import { linter } from "@codemirror/lint";
 import {
+  bracketMatching,
   indentService,
   LanguageSupport,
   StreamLanguage,
@@ -13,9 +18,6 @@ import {
 } from "@codemirror/language";
 import { ABBREV } from "./abbrev.js";
 
-// maybe use https://github.com/codemirror/legacy-modes/blob/main/mode/simple-mode.js instead.
-// @codemirror/legacy-modes/mode/simple-mode.js
-
 // prettier flattened this...
 const keywords = [
   "let",
@@ -61,39 +63,75 @@ const keywords = [
   "|",
 ];
 
+// a stack of tokenizers, current is first
+// we need to push / pop {} so we can parse strings correctly
 interface State {
-  tokenizer(stream: StringStream, state: State): string | null;
+  tokenizers: Tokenizer[];
 }
+type Tokenizer = (stream: StringStream, state: State) => string | null;
 
 function tokenizer(stream: StringStream, state: State): string | null {
-  if (stream.eatSpace()) return null
+  if (stream.eatSpace()) return null;
   if (stream.match("--")) {
     stream.skipToEnd();
     return "comment";
   }
-  if (stream.match(/^[/]-/)) {
+  // maybe keyword?
-    state.tokenizer = commentTokenizer;
+  if (stream.match(/{/)) {
-    return state.tokenizer(stream, state);
+    state.tokenizers.unshift(tokenizer);
+    return null;
+  }
+  if (stream.match(/}/) && state.tokenizers.length > 1) {
+    state.tokenizers.shift();
+    return state.tokenizers[0] === stringTokenizer ? "keyword" : null;
+  }
+  if (stream.match(/^[/]-/)) {
+    state.tokenizers.unshift(commentTokenizer);
+    return state.tokenizers[0](stream, state);
+  }
+  if (stream.match(/"/)) {
+    state.tokenizers.unshift(stringTokenizer);
+    return stringTokenizer(stream, state);
   }
-
   // TODO match tokenizer better..
   if (stream.match(/[^\\(){}[\],.@;\s][^()\\{}\[\],.@;\s]*/)) {
     let word = stream.current();
     if (keywords.includes(word)) return "keyword";
-    if (word[0] >= "A" && word[0] <= "Z") return "typename";
+    if (word[0] >= "A" && word[0] <= "Z") return "typeName";
-    return "identifier";
+    return "variableName";
   }
   // unhandled
-  stream.next()
+  stream.next();
   return null;
 }
+
+function stringTokenizer(stream: StringStream, state: State) {
+  while (true) {
+    if (stream.current() && stream.match(/^\\{/, false)) {
+      return "string";
+    }
+    if (stream.match(/^\\{/)) {
+      state.tokenizers.unshift(tokenizer);
+      return "keyword";
+    }
+    let ch = stream.next();
+    if (!ch) return "string";
+    if (ch === '"') {
+      state.tokenizers.shift();
+      return "string";
+    }
+  }
+}
+
+// We have a tokenizer for this because codemirror processes a line at a time.
+// So we may need to end the line in `comment` state and see the -/ later
 function commentTokenizer(stream: StringStream, state: State): string | null {
   console.log("ctok");
   let dash = false;
   let ch;
   while ((ch = stream.next())) {
     if (dash && ch === "/") {
-      state.tokenizer = tokenizer;
+      state.tokenizers.shift();
       return "comment";
     }
     dash = ch === "-";
@@ -101,17 +139,18 @@ function commentTokenizer(stream: StringStream, state: State): string | null {
   return "comment";
 }
 
-const newtLanguage2 = StreamLanguage.define({
+const newtLanguage2: StreamLanguage<State> = StreamLanguage.define({
-  startState: () => ({ tokenizer }),
+  startState: () => ({ tokenizers: [tokenizer] }),
   token(stream, st) {
-    return st.tokenizer(stream, st);
+    return st.tokenizers[0](stream, st);
   },
   languageData: {
     commentTokens: {
-      line: "--"
+      line: "--",
     },
+    // The real list would include almost every character.
     wordChars: "!#$%^&*_+-=<>|",
-  }
+  },
 });
 
 function newt() {
@@ -134,16 +173,17 @@ export class CMEditor implements AbstractEditor {
         linter((view) => this.delegate.lint(view)),
         // For indent on return
         indentService.of((ctx, pos) => {
-          let line = ctx.lineAt(pos)
+          let line = ctx.lineAt(pos);
-          if (!line || !line.from) return null
+          if (!line || !line.from) return null;
           let prevLine = ctx.lineAt(line.from - 1);
           if (prevLine.text.trimEnd().match(/\b(of|where|do)\s*$/)) {
-              let pindent = prevLine.text.match(/^\s*/)?.[0].length ?? 0
+            let pindent = prevLine.text.match(/^\s*/)?.[0].length ?? 0;
-              return pindent + 2
+            return pindent + 2;
           }
-          return null
+          return null;
         }),
-        Prec.highest(keymap.of([
+        Prec.highest(
+          keymap.of([
             {
               key: "Tab",
               preventDefault: true,
@@ -157,21 +197,20 @@ export class CMEditor implements AbstractEditor {
             {
               key: "c-c c-s",
               run: () => {
-              console.log("C-c C-s")
+                console.log("C-c C-s");
-              return false
+                return false;
-            }
+              },
             },
             {
               key: "Shift-Tab",
               preventDefault: true,
               run: indentLess,
             },
-        ])),
+          ])
+        ),
         EditorView.updateListener.of((update) => {
           let doc = update.state.doc;
-          console.log('update', update)
           update.changes.iterChanges((fromA, toA, fromB, toB, inserted) => {
-            console.log('inserted', inserted)
             if (" ')\\_".includes(inserted.toString()) || inserted.lines > 1) {
               console.log("changes", update.changes, update.changes.desc);
               let line = doc.lineAt(fromA);
@@ -203,7 +242,8 @@ export class CMEditor implements AbstractEditor {
               let col = range.from - cursor.from;
               let word = this.view.state.doc.sliceString(range.from, range.to);
               let entry = this.delegate.getEntry(word, line, col);
-            if (!entry) entry = this.delegate.getEntry('_'+word+'_', line, col);
+              if (!entry)
+                entry = this.delegate.getEntry("_" + word + "_", line, col);
               console.log("entry for", word, "is", entry);
               if (entry) {
                 let rval: Tooltip = {

playground/style.css

@@ -122,3 +122,6 @@ svg.icon path {
 .cm-editor .cm-content {
   font-family: 'Comic Code', monospace;
 }
+.cm-editor {
+  height: 100%;
+}

tests/Tree.newt

@@ -75,6 +75,9 @@ insert : {l u h} -> Intv l u -> T23 l u h -> TooBig l u h + T23 l u h
 insert (intv x lx xu) (leaf lu) = inl (x , (leaf lx , leaf xu))
 insert (intv x lx xu) (node2 y tly tyu) = case cmp x y of
   -- u := N y is not solved at this time
+  -- The problem looks like
+  -- %v13 <= %v8 =?= N %v13 <<= (?m104 ...)
+  -- This might work if ?m104 is solved another way, so perhaps it could be postponed
   inl xy => case insert (intv {_} {N y} x lx xy) tly of
     inl (z , (tlz , tzy)) => inr (node3 z y tlz tzy tyu)
     inr tly' => inr (node2 y tly' tyu)