Initial LSP implementation/vscode support

.gitignore

@@ -19,3 +19,4 @@ src/Revision.newt
 .lsp
 .vscode
 bootstrap/serializer.js
+/newt-vscode-lsp/src/newt.js

Makefile

@@ -58,3 +58,17 @@ clean:
 audit: .PHONY
 	(cd playground && npm audit)
 	(cd newt-vscode && npm audit)
+
+lsp.js: ${SRCS}
+	node newt.js src/LSP.newt -o lsp.js
+
+newt-vscode-lsp/src/newt.js: lsp.js .PHONY
+	echo "import fs from 'fs'\nlet mods = { fs }\nlet require = key => mods[key]\n" > $@
+	# HACK
+	perl -p -e "s/(const LSP_(?:updateFile|checkFile|hoverInfo))/export \$$1/" lsp.js >> $@
+
+newt-vscode-lsp/dist/lsp.js: newt-vscode-lsp/src/lsp.ts newt-vscode-lsp/src/newt.js
+	(cd newt-vscode-lsp && node esbuild.js)
+
+lsp: newt-vscode-lsp/dist/lsp.js
+

newt-vscode-lsp/.eslintrc.json

@@ -0,0 +1,30 @@
+{
+    "root": true,
+    "parser": "@typescript-eslint/parser",
+    "parserOptions": {
+        "ecmaVersion": 6,
+        "sourceType": "module"
+    },
+    "plugins": [
+        "@typescript-eslint"
+    ],
+    "rules": {
+        "@typescript-eslint/naming-convention": [
+            "warn",
+            {
+                "selector": "import",
+                "format": [ "camelCase", "PascalCase" ]
+            }
+        ],
+        "@typescript-eslint/semi": "warn",
+        "curly": "off",
+        "eqeqeq": "warn",
+        "no-throw-literal": "warn",
+        "semi": "off"
+    },
+    "ignorePatterns": [
+        "out",
+        "dist",
+        "**/*.d.ts"
+    ]
+}

newt-vscode-lsp/.gitignore

@@ -0,0 +1,4 @@
+dist
+node_modules
+*.vsix
+

newt-vscode-lsp/.vscodeignore

@@ -0,0 +1,5 @@
+.vscode/**
+.vscode-test/**
+.gitignore
+vsc-extension-quickstart.md
+node_modules

newt-vscode-lsp/CHANGELOG.md

@@ -0,0 +1,9 @@
+# Change Log
+
+All notable changes to the "newt-vscode" extension will be documented in this file.
+
+Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how to structure this file.
+
+## [Unreleased]
+
+- Initial release

newt-vscode-lsp/LICENSE

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <https://unlicense.org>

newt-vscode-lsp/README.md

@@ -0,0 +1,3 @@
+# newt-vscode README
+
+newt extension for vscode

newt-vscode-lsp/esbuild.js

@@ -0,0 +1,58 @@
+const esbuild = require("esbuild");
+
+const production = process.argv.includes('--production');
+const watch = process.argv.includes('--watch');
+
+/**
+ * @type {import('esbuild').Plugin}
+ */
+const esbuildProblemMatcherPlugin = {
+	name: 'esbuild-problem-matcher',
+
+	setup(build) {
+		build.onStart(() => {
+			console.log('[watch] build started');
+		});
+		build.onEnd((result) => {
+			result.errors.forEach(({ text, location }) => {
+				console.error(`✘ [ERROR] ${text}`);
+				console.error(`    ${location.file}:${location.line}:${location.column}:`);
+			});
+			console.log('[watch] build finished');
+		});
+	},
+};
+
+async function main() {
+	const ctx = await esbuild.context({
+		entryPoints: [
+			'src/extension.ts',
+			'src/lsp.ts'
+		],
+		bundle: true,
+		format: 'cjs',
+		minify: production,
+		sourcemap: !production,
+		sourcesContent: false,
+		platform: 'node',
+		// outfile: 'dist/extension.js',
+		outdir: 'dist',
+		external: ['vscode'],
+		logLevel: 'silent',
+		plugins: [
+			/* add to the end of plugins array */
+			esbuildProblemMatcherPlugin,
+		],
+	});
+	if (watch) {
+		await ctx.watch();
+	} else {
+		await ctx.rebuild();
+		await ctx.dispose();
+	}
+}
+
+main().catch(e => {
+	console.error(e);
+	process.exit(1);
+});

newt-vscode-lsp/language-configuration.json

@@ -0,0 +1,53 @@
+{
+  // see singleton in Tokenizer.idr
+  "wordPattern": "[^()\\{}\\[\\],.@\\s]+",
+  "comments": {
+    // symbol used for single line comment. Remove this entry if your language does not support line comments
+    "lineComment": "--",
+    // symbols used for start and end a block comment. Remove this entry if your language does not support block comments
+    "blockComment": ["/-", "-/"]
+  },
+  // symbols used as brackets
+  "brackets": [
+    ["{", "}"],
+    ["{{", "}}"],
+    ["[", "]"],
+    ["(", ")"]
+  ],
+  // symbols that are auto closed when typing
+  "autoClosingPairs": [
+    ["{", "}"],
+    ["[", "]"],
+    ["(", ")"],
+    ["\"", "\""],
+    // ["'", "'"],  causes problems with foo''
+    ["/-", "-/"]
+  ],
+  // symbols that can be used to surround a selection
+  "surroundingPairs": [
+    ["{", "}"],
+    ["[", "]"],
+    ["(", ")"],
+    ["\"", "\""],
+    ["'", "'"]
+  ],
+  "onEnterRules": [
+    {
+      "beforeText": "\\b(where|of|do)\\s*$",
+      "action": { "indent": "indent" }
+    },
+    {
+      "beforeText": "/-",
+      "afterText": "-/",
+      "action": {
+        "indent": "indentOutdent"
+      }
+    },
+    {
+      "beforeText": "^\\s+$",
+      "action": {
+        "indent": "outdent"
+      }
+    }
+  ]
+}

newt-vscode-lsp/package.json

@@ -0,0 +1,105 @@
+{
+  "name": "newt-vscode",
+  "publisher": "dunhamsteve",
+  "displayName": "newt-vscode",
+  "description": "newt language support with LSP",
+  "version": "0.0.1",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dunhamsteve/newt"
+  },
+  "engines": {
+    "vscode": "^1.91.0"
+  },
+  "categories": [
+    "Programming Languages"
+  ],
+  "activationEvents": [],
+  "main": "./dist/extension.js",
+  "contributes": {
+    "languages": [
+      {
+        "id": "newt",
+        "aliases": [
+          "newt"
+        ],
+        "extensions": [
+          "newt"
+        ],
+        "configuration": "./language-configuration.json"
+      }
+    ],
+    "grammars": [
+      {
+        "language": "newt",
+        "scopeName": "source.newt",
+        "path": "./syntaxes/newt.tmLanguage.json"
+      },
+      {
+        "path": "./syntaxes/inject.json",
+        "scopeName": "newt.injection",
+        "injectTo": [
+          "text.html.markdown"
+        ],
+        "embeddedLanguages": {
+          "meta.embedded.block.idris": "newt"
+        }
+      }
+    ],
+    "commands": [
+      {
+        "command": "newt-vscode.check",
+        "title": "Check newt file"
+      }
+    ],
+    "configuration": {
+      "type": "object",
+      "title": "Newt Configuration",
+      "properties": {
+        "newt.path": {
+          "type": "string",
+          "default": "node bootstrap/newt.js",
+          "description": "Command to run newt"
+        },
+        "newt.lspPath": {
+          "type": "string",
+          "default": null,
+          "description": "path to LSP script (run in node)"
+        }
+      }
+    }
+  },
+  "scripts": {
+    "vscode:prepublish": "npm run package",
+    "compile": "npm run check-types && npm run lint && node esbuild.js",
+    "watch": "npm-run-all -p watch:*",
+    "watch:esbuild": "node esbuild.js --watch",
+    "esbuild": "node esbuild.js",
+    "package": "echo npm run check-types && npm run lint && node esbuild.js --production",
+    "compile-tests": "tsc -p . --outDir out",
+    "watch-tests": "tsc -p . -w --outDir out",
+    "pretest": "npm run compile-tests && npm run compile && npm run lint",
+    "check-types": "tsc --noEmit",
+    "lint": "eslint src --ext ts",
+    "test": "vscode-test"
+  },
+  "devDependencies": {
+    "@types/mocha": "^10.0.7",
+    "@types/node": "25.x",
+    "@types/vscode": "^1.90.0",
+    "@typescript-eslint/eslint-plugin": "^7.14.1",
+    "@typescript-eslint/parser": "^7.11.0",
+    "@vscode/test-cli": "^0.0.9",
+    "@vscode/test-electron": "^2.4.0",
+    "esbuild": "^0.25.0",
+    "eslint": "^8.57.0",
+    "npm-run-all": "^4.1.5",
+    "typescript": "^5.4.5"
+  },
+  "dependencies": {
+    "vscode-languageclient": "^9.0.1",
+    "vscode-languageserver": "^9.0.1",
+    "vscode-languageserver-textdocument": "^1.0.12"
+  }
+}

newt-vscode-lsp/src/abbrev.ts

@@ -0,0 +1,36 @@
+// This needs to be fleshed out a lot, I've been adding them as needed
+// There is a mix of agda, lean, and my own shortcuts here
+export const ABBREV: Record<string, string> = {
+  "\\x": "×",
+  "\\r": "→",
+  "\\all": "∀",
+  "\\\\": "\\",
+  "\\==": "≡",
+  "\\circ": "∘",
+  "\\oplus": "⊕",
+  "\\otimes": "⊗",
+  // lean
+  "\\1": "₁",
+  "\\2": "₂",
+  "\\<": "⟨",
+  "\\>": "⟩",
+  // agda
+  "\\_0": "₀",
+  "\\_1": "₁",
+  "\\_2": "₂",
+  "\\_3": "₃",
+  // lean has \n here, which is a royal pain
+  "\\neg": "¬",
+  "\\bN": "ℕ",
+  "\\bZ": "ℤ",
+  "\\GG": "Γ",
+  "\\Gi": "ι",
+  "\\Gl": "λ",
+  "\\Gs": "σ",
+  "\\Gt": "τ",
+  "\\GD": "Δ",
+  "\\GS": "Σ",
+  "\\GP": "∏",
+  "\\[[": "⟦",
+  "\\]]": "⟧",
+};

newt-vscode-lsp/src/extension.ts

@@ -0,0 +1,95 @@
+import * as vscode from "vscode";
+import { ABBREV } from "./abbrev";
+import {
+  LanguageClient,
+  LanguageClientOptions,
+  ServerOptions,
+  TransportKind
+} from 'vscode-languageclient/node';
+
+// They put this at module level for deactivate below.
+let client: LanguageClient
+
+export function activate(context: vscode.ExtensionContext) {
+  const serverModule = context.asAbsolutePath('./dist/lsp.js')
+  console.log('*** servervModule', serverModule)
+  const config = vscode.workspace.getConfiguration("newt");
+  const cmd = config.get<string | null>("lspPath");
+
+  let serverOptions: ServerOptions
+  if (cmd) {
+    serverOptions = {
+      run: { command: "node", args: [cmd], transport: TransportKind.pipe },
+      debug: { command: "node", args: [cmd], transport: TransportKind.pipe },
+    }
+  } else {
+    serverOptions = {
+      run: { module: serverModule, transport: TransportKind.ipc },
+      debug: { module: serverModule, transport: TransportKind.ipc },
+    }
+  }
+
+  const clientOptions: LanguageClientOptions = {
+    documentSelector: [{ scheme: 'file', language: 'newt' }],
+    synchronize: {
+      fileEvents: vscode.workspace.createFileSystemWatcher('*.newt')
+    }
+  }
+
+  client = new LanguageClient(
+    'NewtLanguageServer',
+    'Newt Language Server',
+    serverOptions,
+    clientOptions
+  )
+
+  client.start();
+  console.log('CLIENT started')
+
+  vscode.workspace.onDidChangeTextDocument((event) => {
+    const editor = vscode.window.activeTextEditor;
+    if (!editor || event.document !== editor.document) return;
+
+    const changes = event.contentChanges;
+    if (changes.length === 0) return;
+
+    // TODO - agda input mode does the replacement as soon as possible
+    // but if the sequence is a prefix, it will change for subsequent characters
+    // The latter would require keeping state, but if we don't allow sequences to prefix
+    // each other, we could activate quicker.
+    const lastChange = changes[changes.length - 1];
+    const text = lastChange.text;
+    // Check if the last change is a potential shortcut trigger
+    if (!text || !(" ')\\_".includes(text) || text.startsWith('\n'))) return;
+
+    const document = editor.document;
+    const position = lastChange.range.end;
+    const lineText = document.lineAt(position.line).text;
+    const start = Math.max(0, position.character - 10);
+    const snippet = lineText.slice(start, position.character);
+    const m = snippet.match(/(\\[^ ]+)$/);
+    if (m) {
+      const cand = m[0];
+      console.log("cand", cand);
+      const replacement = ABBREV[cand];
+      console.log("repl", replacement);
+      if (replacement) {
+        const range = new vscode.Range(
+          position.line,
+          position.character - cand.length,
+          position.line,
+          position.character
+        );
+        editor.edit((editBuilder) => {
+          editBuilder.replace(range, replacement);
+        });
+      }
+    }
+  });
+  return;
+
+}
+
+export function deactivate() {
+  if (client) client.stop();
+}

newt-vscode-lsp/src/lsp.ts

@@ -0,0 +1,53 @@
+/**
+ * WIP
+ *
+ * Wraps newt.js (compiled from src/LSP.newt with some tweaks to `export`) with the
+ * vscode LSP server module.
+ */
+
+import { LSP_checkFile, LSP_updateFile, LSP_hoverInfo } from './newt.js'
+
+import {
+  createConnection,
+  TextDocuments,
+  ProposedFeatures,
+  Hover,
+  InitializeParams,
+  InitializeResult,
+  TextDocumentSyncKind,
+} from "vscode-languageserver/node";
+import { TextDocument } from "vscode-languageserver-textdocument";
+
+const connection = createConnection(ProposedFeatures.all);
+const documents = new TextDocuments(TextDocument);
+
+documents.onDidChangeContent(async (change) => {
+  console.log('DIDCHANGE', change.document.uri)
+  const uri = change.document.uri;
+  const text = change.document.getText();
+  LSP_updateFile(uri, text);
+  const diagnostics = LSP_checkFile(uri);
+  console.log(`Got ${JSON.stringify(diagnostics, null, '  ')}`)
+  connection.sendDiagnostics({ uri, diagnostics })
+});
+
+connection.onHover((params): Hover | null => {
+  const uri = params.textDocument.uri;
+  const pos = params.position;
+  console.log('HOVER', uri, pos)
+  let value = LSP_hoverInfo(uri, pos.line, pos.character)
+  if (!value) return null
+  console.log('HOVER is ', value)
+  return { contents: { kind: "plaintext", value } };
+});
+
+connection.onInitialize((_params: InitializeParams): InitializeResult => ({
+  capabilities: {
+    textDocumentSync: TextDocumentSyncKind.Incremental,
+    hoverProvider: true,
+  },
+}));
+
+documents.listen(connection);
+connection.listen();
+console.log('STARTED')

newt-vscode-lsp/src/newt.d.ts

@@ -0,0 +1,5 @@
+import { Diagnostic } from "vscode-languageserver";
+
+export function LSP_updateFile(name: string, content: string): (eta: any) => any;
+export function LSP_checkFile(name: string): Diagnostic[];
+export function LSP_hoverInfo(name: string, row: number, col: number): string|null;

newt-vscode-lsp/syntaxes/inject.json

@@ -0,0 +1,44 @@
+{
+  "scopeName": "newt.injection",
+  "injectionSelector": "L:text.html.markdown",
+  "patterns": [
+    {
+      "include": "#fenced_code_block_newt"
+    }
+  ],
+  "repository": {
+    "fenced_code_block_newt": {
+      "begin": "(^|\\G)(\\s*)(`{3,}|~{3,})\\s*(?i:(newt)((\\s+|:|,|\\{|\\?)[^`]*)?$)",
+      "name": "markup.fenced_code.block.markdown",
+      "end": "(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$",
+      "beginCaptures": {
+        "3": {
+          "name": "punctuation.definition.markdown"
+        },
+        "4": {
+          "name": "fenced_code.block.language.markdown"
+        },
+        "5": {
+          "name": "fenced_code.block.language.attributes.markdown"
+        }
+      },
+      "endCaptures": {
+        "3": {
+          "name": "punctuation.definition.markdown"
+        }
+      },
+      "patterns": [
+        {
+          "begin": "(^|\\G)(\\s*)(.*)",
+          "while": "(^|\\G)(?!\\s*([`~]{3,})\\s*$)",
+          "contentName": "meta.embedded.block.newt",
+          "patterns": [
+            {
+              "include": "source.newt"
+            }
+          ]
+        }
+      ]
+    }
+  }
+}

newt-vscode-lsp/syntaxes/newt.tmLanguage.json

@@ -0,0 +1,47 @@
+{
+  "$schema": "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json",
+  "name": "newt",
+  "scopeName": "source.newt",
+  "patterns": [
+    {
+      "name": "invalid.illegal.trace",
+      "match": "\\b(trace|strace|fatalError)\\b"
+    },
+    {
+      "name": "comment.block.newt",
+      "begin": "/-",
+      "end": "-/",
+      "contentName": "comment.block.newt"
+    },
+    {
+      "name": "comment.line.newt",
+      "begin": "--",
+      "end": "\\n"
+    },
+    {
+      "name": "keyword.newt",
+      "match": "\\b(λ|=>|<-|->|→|:=|\\$|data|record|constructor|where|do|class|uses|instance|case|of|let|if|then|else|forall|∀|in|U|module|import|ptype|pfunc|infix|infixl|infixr)\\b"
+    },
+    {
+      "name": "string.js",
+      "begin": "`",
+      "end": "`",
+      "patterns": [{ "include": "source.js" }]
+    },
+    {
+      "name": "character",
+      "match": "'\\\\?.'"
+    },
+    {
+      "name": "string.double.newt",
+      "begin": "\"",
+      "end": "\"",
+      "patterns": [
+        {
+          "name": "constant.character.escape.newt",
+          "match": "\\\\[^{]"
+        }
+      ]
+    }
+  ]
+}

newt-vscode-lsp/tsconfig.json

@@ -0,0 +1,18 @@
+{
+	"compilerOptions": {
+		"module": "Node16",
+		"target": "ES2022",
+		"lib": [ "ES2022" ],
+		"sourceMap": true,
+		// so node can run this stuff
+		"allowImportingTsExtensions": true,
+		// required by previous, but we use esbuild anyway
+		"noEmit": true,
+		"rootDir": "src",
+		"strict": true   /* enable all strict type-checking options */
+		/* Additional Checks */
+		// "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */
+		// "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */
+		// "noUnusedParameters": true,  /* Report errors on unused parameters. */
+	}
+}

src/Commands.newt

@@ -0,0 +1,49 @@
+-- For shared code between REPL and LSP
+module Commands
+
+import Prelude
+import Lib.ProcessModule
+import Lib.Types
+import Lib.TopContext
+import Lib.Common
+import Data.List1
+import Lib.Tokenizer
+import Lib.Token
+import Lib.Elab
+
+-- For now we cheat and assume capitalized directories are a module component
+decomposeName : String → String × List String
+decomposeName fn =
+    go Nil $ Lin <>< split (fst $ splitFileName fn) "/"
+    where
+        go : List String → SnocList String → String × List String
+        go acc Lin = (".", acc)
+        go acc (xs :< x) = if isUpper $ strIndex x 0
+            then go (x :: acc) xs
+            else (joinBy "/" (xs :< x <>> Nil), acc)
+
+-- The cheap version of type at point, find the token, lookup in global context
+-- Later we will either get good FC for entries or scan them all and build a cache.
+getHoverInfo : FileSource → List String → Int → Int → M (Maybe String)
+getHoverInfo repo modns row col = do
+    mod <- processModule emptyFC repo Nil modns
+    -- not necessarily loaded into top... (Maybe push this down into that branch of processModule)
+    modifyTop [ defs := mod.modDefs; metaCtx := mod.modMetaCtx; ops := mod.ctxOps; imported := mod.modDeps ]
+    top <- getTop
+
+    -- Find the token at the point
+    let lines = split mod.modSource "\n"
+    let line = fromMaybe "" (getAt' row lines)
+    let (Right toks) = tokenise "" line | Left _ => pure Nothing
+    let (Just name) = getTok toks       | _ => pure Nothing
+
+    -- Lookup the name
+    let (Just e) = lookupRaw name top   | _ => pure Nothing
+    pure $ Just "\{show e.name} : \{rpprint Nil e.type}"
+
+    where
+        getTok : List BTok → Maybe String
+        getTok Nil = Nothing
+        getTok (tok :: toks) =
+            if tok.bounds.startCol <= col && (col <= tok.bounds.endCol)
+            then Just $ value tok else getTok toks

src/LSP.newt

@@ -0,0 +1,144 @@
+module LSP
+
+import Prelude
+-- TODO pull this into its own file
+import Lib.Common
+import Lib.Types
+import Lib.TopContext
+import Lib.Tokenizer
+import Lib.Parser
+import Lib.Parser.Impl
+import Lib.ProcessModule
+import Data.SortedMap
+import Data.IORef
+import Node
+import Commands
+import Data.List1
+
+pfunc js_castArray : Array JSObject → JSObject := `x => x`
+pfunc js_castInt : Int → JSObject := `x => x`
+pfunc js_castBool : Bool → JSObject := `x => x`
+pfunc js_castStr : String → JSObject := `x => x`
+pfunc js_null : JSObject := `null`
+pfunc js_castObj : Array (String × JSObject) → JSObject := `(data) => {
+    let rval = {}
+    for (let x of data) rval[x.h2] = x.h3
+    return rval
+}`
+
+-- need case split
+jsonToJObject : Json → JSObject
+jsonToJObject (JsonInt x) = js_castInt x
+jsonToJObject (JsonNull) = js_null
+jsonToJObject (JsonArray xs) = js_castArray $ listToArray $ map jsonToJObject xs
+jsonToJObject (JsonBool x) = js_castBool x
+jsonToJObject (JsonStr x) = js_castStr x
+-- IMPERROR - if I leave off the `map` I get an error that is hard to sort out
+jsonToJObject (JsonObj xs) = js_castObj $ listToArray $ map (mapSnd jsonToJObject) xs
+
+record LSPState where
+    topContext : TopContext
+    baseDir : String
+    files : SortedMap String String
+
+state : IORef LSPState
+state = unsafePerformIO $ newIORef $ MkLSPState emptyTop "" emptyMap
+
+resetState : String → IO Unit
+resetState base = do
+    putStrLn "Reset base to \{base}"
+    writeIORef state $ MkLSPState emptyTop base emptyMap
+
+lspFileSource : FileSource
+lspFileSource = MkFileSource $ \fc fn => do
+    st <- readIORef state
+    let fn = st.baseDir ++ "/" ++ fn
+    let (Nothing) = lookupMap' fn st.files
+        | Just src => pure (fn,src)
+    let fn' = case split fn "file://" of
+        x :: fn :: _ => fn
+        _ => fn
+    (Right src) <- liftIO {M} $ readFile fn'
+        | Left err => throwError $ E fc "error reading \{fn}: \{show err}"
+    pure (fn,src)
+
+updateFile : String → String → Unit
+updateFile fn src = unsafePerformIO $ do
+    st <- readIORef state
+    modifyIORef state $ \a => [ files $= updateMap fn src ] a
+    let st = the LSPState $ [ files $= updateMap fn src ] st
+    -- module relative to base
+
+    let (Right toks) = tokenise fn src | Left err => writeIORef state st
+    let (Right ((nameFC, modName), _, _)) = partialParse fn parseModHeader emptyMap toks
+        | Left (err,toks) => writeIORef state st
+
+    Right (ctx,_) <- (invalidateModule $ split modName ".").runM st.topContext
+        | _ => writeIORef state st
+    -- TODO It doesn't have record type, but eta expanding resolves this.  See if there is a quick fix.
+    -- modifyIORef state [ topContext := ctx ]
+    modifyIORef state $ \a => [ topContext := ctx ] a
+
+hoverInfo : String → Int → Int → JSObject
+hoverInfo uri line col = unsafePerformIO $ do
+    let (base,modns) = decomposeName uri
+    putStrLn "Hover \{uri} base \{base} mod \{joinBy "." modns}"
+    st <- readIORef state
+    if (st.baseDir /= base)
+        then resetState base
+        else pure MkUnit
+    Right (_, Just msg) <- (getHoverInfo lspFileSource modns line col).runM st.topContext
+        | Right _  => do
+            putStrLn $ "Nothing to see here"
+            pure $ jsonToJObject JsonNull
+        | Left err => do
+            putStrLn $ showError "" err
+            pure $ jsonToJObject JsonNull
+    pure $ jsonToJObject $ JsonStr msg
+
+errorToDiag : Error -> Json
+errorToDiag (E (MkFC fn (MkBounds sr sc er ec)) msg) =
+  JsonObj
+    $  ("severity", JsonInt 1)
+    -- PARSER `$` is winning over `,`, which is not what I'm expecting  Maybe `,` should be special...
+    :: ("range", (JsonObj $ ("start", range sr sc) :: ("end", range er (ec + 1)) :: Nil))
+    :: ("message", JsonStr msg)
+    :: ("source", JsonStr "newt") -- what is this key for?
+    :: Nil
+  where
+    range : Int → Int → Json
+    range l c = JsonObj $ ("line", JsonInt l) :: ("character", JsonInt c) :: Nil
+-- These shouldn't escape
+errorToDiag (Postpone fc qn msg) = errorToDiag $ E fc "Postpone \{show qn} \{msg}"
+
+
+checkFile : String → JSObject
+checkFile fn = unsafePerformIO $ do
+    let (base,modns) = decomposeName fn
+    putStrLn "Checking \{fn} base \{base} mod \{joinBy "." modns}"
+    st <- readIORef state
+    if (st.baseDir /= base)
+        then resetState base
+        else pure MkUnit
+    (Right (top, json)) <- (do
+        modifyTop [ errors := Nil ]
+        putStrLn "add prim"
+        addPrimitives
+        putStrLn "processModule"
+        _ <- processModule emptyFC lspFileSource Nil modns
+        pure MkUnit
+        -- pull out errors and infos
+        top <- getTop
+        pure $ map (errorToDiag) top.errors
+        ).runM st.topContext
+        | Left err => do
+            putStrLn $ showError "" err
+            pure $ jsonToJObject $ JsonArray $ errorToDiag err :: Nil
+    -- Cache loaded modules
+    modifyIORef state $ \a => [ topContext := top ] a
+    pure $ jsonToJObject $ JsonArray json
+
+-- This seems like a hack, but it works.
+-- Dummy main function with references to force functions into ouput file.
+-- but we don't get `export` on it..
+pfunc main uses (updateFile checkFile hoverInfo) : IO Unit  := `() => {}`

src/Lib/Common.newt

@@ -81,6 +81,7 @@ data Json : U where
   JsonBool : Bool -> Json
   JsonInt : Int -> Json
   JsonArray : List Json -> Json
+  JsonNull : Json
 
 
 renderJson : Json -> String
@@ -88,6 +89,7 @@ renderJson (JsonObj xs) = "{" ++ joinBy "," (map renderPair xs) ++ "}"
   where
     renderPair : (String × Json) -> String
     renderPair (k,v) = quoteString k ++ ":" ++ renderJson v
+renderJson (JsonNull) = "null"
 renderJson (JsonStr str) = quoteString str
 renderJson (JsonBool x) = ite x "true" "false"
 renderJson (JsonInt i) = cast i

src/Lib/Elab.newt

@@ -27,6 +27,7 @@ collectDecl ((FunDef fc nm cl) :: rest@(FunDef _ nm' cl' :: xs)) =
   else (FunDef fc nm cl :: collectDecl rest)
 collectDecl (x :: xs) = x :: collectDecl xs
 
+-- TODO Move this, so we don't need to import all of Elab
 rpprint : List String → Tm → String
 rpprint names tm = render 90 $ pprint names tm
 

src/Lib/ProcessModule.newt

@@ -1,7 +1,6 @@
 module Lib.ProcessModule
 
 import Prelude
-import Serialize
 import Lib.Types
 import Lib.Common
 import Lib.Syntax
@@ -21,7 +20,14 @@ addPrimitives = do
   processDecl primNS (PType emptyFC "String" Nothing)
   processDecl primNS (PType emptyFC "Char" Nothing)
   setDef (QN primNS "PiType") emptyFC (Erased emptyFC) (PrimFn "(h0, h1) => ({ tag: \"PiType\", h0, h1 });" (S (S Z)) Nil) Nil
-
+  top <- getTop
+  let modules = updateMap primNS (MkModCtx "" top.defs (MC emptyMap Nil 0 CheckAll) top.ops Nil) top.modules
+  modifyTop [ modules := modules
+      ; imported := primNS :: Nil
+      ; hints := emptyMap
+      ; ns := Nil
+      ; defs := emptyMap
+      ]
 
 record FileSource where
   getFile : FC → String → M (String × String)
@@ -31,7 +37,6 @@ parseDecls fn ops Nil acc = pure (acc <>> Nil, ops)
 parseDecls fn ops toks@(first :: _) acc =
   case partialParse fn (sameLevel parseDecl) ops toks of
     Left (err, toks) => do
-      putStrLn $ showError "" err
       addError err
       parseDecls fn ops (recover toks) acc
     Right (decl,ops,toks) => parseDecls fn ops toks (acc :< decl)
@@ -43,14 +48,6 @@ parseDecls fn ops toks@(first :: _) acc =
       then (tok :: toks)
       else recover toks
 
-moduleHash : String → List (List String) → M String
-moduleHash src imports = do
-  srcHash <- liftIO $ checksum src
-  top <- getTop
-  let mods = mapMaybe (\x => lookupMap' x top.modules) imports
-  let modHashes = map (\x => x.csum) mods
-  liftIO $ checksum $ fastConcat $ srcHash :: modHashes
-
 importToName : Import → List String
 importToName  (MkImport fc (_,name)) = split name "."
 
@@ -60,11 +57,14 @@ importHints (entry :: entries) = do
   when (elem Hint entry.eflags) $ \ _ => addHint entry.name
   importHints entries
 
-processModule : FC → FileSource → List String → List String → M String
+-- HACK this is returning src to help render errors..
+-- Maybe return module, put src and errors in module, add error for import with error, callers can sort out what they want to do?
+-- The issue here is command line newt wants to report all errors (we can print that though?)  LSP wants something more subtle
+processModule : FC → FileSource → List String → (stack : List String) → M ModContext
 processModule importFC repo stk modns = do
   top <- getTop
   let name = joinBy "." modns
-  let (Nothing) = lookupMap modns top.modules | _ => pure ""
+  let (Nothing) = lookupMap' modns top.modules | Just mod => pure mod
 
   let fn = joinBy "/" modns ++ ".newt"
   -- TODO now we can pass in the module name...
@@ -91,21 +91,8 @@ processModule importFC repo stk modns = do
       processModule nameFC repo (name :: stk) imp
       pure $ imp
   let imported = snoc imported primNS
-  srcSum <- liftIO $ checksum src
-  csum <- moduleHash srcSum imported
 
   putStrLn "module \{modName}"
-  top <- getTop
-  -- TODO we need a flag on this so `make newt3.js` properly tests self-compile
-  (Nothing) <- loadModule modns csum
-    | Just mod => do
-      let modules = updateMap modns mod top.modules
-
-      -- FIXME - we don't want stray operators in a module.
-      -- inject module ops into top
-      let ops = foldMap const top.ops $ toList mod.ctxOps
-      modifyTop [modules := modules; ops := ops ]
-      pure src  -- why am I returning this?
 
   log 1 $ \ _ => "MODNS " ++ show modns
   top <- getTop
@@ -131,21 +118,20 @@ processModule importFC repo stk modns = do
   -- update modules with result, leave the rest of context in case this is top file
   top <- getTop
 
-  let mod = MkModCtx csum top.defs top.metaCtx top.ops importNames
+  let mod = MkModCtx src top.defs top.metaCtx top.ops importNames
-  if stk /= Nil && length' top.errors == 0
-    then dumpModule modns src mod
-    else pure MkUnit
 
   let modules = updateMap modns mod top.modules
   modifyTop [modules := modules]
 
   logMetas $ reverse $ listValues top.metaCtx.metas
-  let (Nil) = top.errors
+  -- FIXME module context should hold errors, to report in replay
-    | errors => throwError $ E importFC "Failed to compile module \{joinBy "." modns}"
+  pure mod
-  pure src
   where
     tryProcessDecl : String → List String → Decl → M Unit
     tryProcessDecl src ns decl = do
       (Left err) <- tryError $ processDecl ns decl | _ => pure MkUnit
-      putStrLn $ showError src err
       addError err
+
+-- TODO clear dependents too.
+invalidateModule : List String -> M Unit
+invalidateModule modname = modifyTop [modules $= deleteMap modname]

src/Lib/TopContext.newt

@@ -46,11 +46,8 @@ lookupRaw raw top =
 instance Show TopContext where
   show top = "\nContext:\n [\{ joinBy "\n" $ map (show ∘ snd) $ toList top.defs}]"
 
--- TODO need to get class dependencies working
+emptyTop : TopContext
-emptyTop : ∀ io. {{Monad io}} {{HasIO io}} -> io TopContext
+emptyTop = MkTop emptyMap Nil emptyMap Nil emptyMap (MC emptyMap Nil 0 CheckAll) 0 Nil emptyMap
-emptyTop = do
-  let mcctx = MC emptyMap Nil 0 CheckAll
-  pure $ MkTop emptyMap Nil emptyMap Nil emptyMap mcctx 0 Nil emptyMap
 
 
 setFlag : QName → FC → EFlag → M Unit

src/Lib/Types.newt

@@ -396,7 +396,7 @@ instance Show TopEntry where
 
 record ModContext where
   constructor MkModCtx
-  csum : String
+  modSource : String
   modDefs : SortedMap QName TopEntry
   -- Do we need this if everything solved is zonked?
   modMetaCtx : MetaContext

src/Main.newt

@@ -18,7 +18,6 @@ import Lib.Types
 import Lib.Syntax
 import Lib.ReplParser
 import Node
-import Serialize
 import Revision
 
 dirFileSource : String → FileSource
@@ -73,8 +72,7 @@ showErrors fn src = do
         throwError $ E (MkFC fn $ MkBounds 0 0 0 0) "Compile failed"
   pure MkUnit
 
-invalidateModule : List String -> M Unit
+
-invalidateModule modname = modifyTop [modules $= deleteMap modname]
 
 -- processFile called on the top level file
 -- it sets up everything and then recurses into processModule
@@ -98,21 +96,12 @@ processFile fn = do
   base <- getBaseDir fn nameFC modns
   addPrimitives
 
-  top <- getTop
-  let modules = updateMap primNS (MkModCtx "" top.defs (MC emptyMap Nil 0 CheckAll) top.ops Nil) top.modules
-  modifyTop [ modules := modules
-            ; imported := primNS :: Nil
-            ; hints := emptyMap
-            ; ns := Nil
-            ; defs := emptyMap
-            ]
-
   invalidateModule modns
   let repo = dirFileSource base
-  src <- processModule emptyFC repo Nil modns
+  mod <- processModule emptyFC repo Nil modns
   top <- getTop
 
-  showErrors fn src
+  showErrors fn mod.modSource
   pure MkUnit
 
 cmdLine : List String -> M (Maybe String × List String)
@@ -142,7 +131,7 @@ browseTop qn@(QN ns x) = do
     go : List TopEntry → M Unit
     go Nil = pure MkUnit
     go (e :: es) = do
-      putStrLn "\{show e.name} : \{rpprint Nil e.type}"
+      putStrLn "\{show e.fc} \{show e.name} : \{rpprint Nil e.type}"
       go es
 
 replHeader : M Unit
@@ -206,6 +195,7 @@ main' = do
             replHeader
             runRepl
   (out, files) <- cmdLine args
+
   traverse processFile files
 
   when (elem "--top" args) $ \ _ => do
@@ -218,8 +208,6 @@ main' = do
 
 main : IO Unit
 main = do
-  -- we'll need to reset for each file, etc.
+  (Right _)  <- .runM main' emptyTop
-  ctx <- emptyTop
-  (Right _)  <- .runM main' ctx
     | Left err => exitFailure "ERROR at \{show $ getFC err}: \{errorMsg err}"
   putStrLn "done"

src/Prelude.newt

@@ -955,3 +955,4 @@ foldlM : ∀ m a e. {{Monad m}} → (a → e → m a) → a → List e → m a
 foldlM f a xs = foldl (\ ma b => ma >>= flip f b) (pure a) xs
 
 pfunc unsafePerformIO : ∀ a. IO a → a := `(a, f) => f().h1 `
+

tests/BadAlt.newt.fail

@@ -7,4 +7,4 @@ ERROR at tests/BadAlt.newt:6:6--6:13: Prelude._:<_ not a constructor for (Prelud
   foo (xs :< x) = x
        ^^^^^^^
 
-ERROR at :1:1--1:2: Failed to compile module BadAlt
+ERROR at tests/BadAlt.newt:1:1--1:2: Compile failed

tests/Duplicate.newt

@@ -1,6 +1,7 @@
 module Duplicate
 
 -- duplicate name should fail
+-- FIXME FC is wrong here
 data Either : U -> U -> U where
   Left : {a b : U} -> a -> Either a b
   Left : {a b : U} -> b -> Either a b

tests/Duplicate.newt.fail

@@ -1,9 +1,9 @@
 *** Process tests/Duplicate.newt
 module Duplicate
-ERROR at tests/Duplicate.newt:4:1--4:5: Duplicate.Left is already defined at tests/Duplicate.newt:4:1--4:5
+ERROR at tests/Duplicate.newt:5:1--5:5: Duplicate.Left is already defined at tests/Duplicate.newt:5:1--5:5
-  
   -- duplicate name should fail
+  -- FIXME FC is wrong here
   data Either : U -> U -> U where
   ^^^^
 
-ERROR at :1:1--1:2: Failed to compile module Duplicate
+ERROR at tests/Duplicate.newt:1:1--1:2: Compile failed

tests/ErrMsg2.newt.fail

@@ -3,4 +3,8 @@ module Prelude
 module ErrMsg2
 ERROR at tests/ErrMsg2.newt:6:13--6:15: Expected '=>' at Keyword:->
   
-ERROR at :1:1--1:2: Failed to compile module ErrMsg2
+  foo : Nat → (Nat → Nat)
+  foo x = \ x -> x
+              ^^
+
+ERROR at tests/ErrMsg2.newt:1:1--1:2: Compile failed

tests/ErrorDup.newt.fail

@@ -1,9 +1,9 @@
 *** Process tests/ErrorDup.newt
 module ErrorDup
-ERROR at tests/ErrorDup.newt:5:6--5:9: Nat already declared
+ERROR at tests/ErrorDup.newt:9:7--9:10: Nat already declared
-  data Nat = Z | S Nat
+  record Nat where
   
-  data Nat = Z | S Nat
+  class Nat where
         ^^^
 
 ERROR at tests/ErrorDup.newt:7:8--7:11: Nat already declared
@@ -12,10 +12,10 @@ ERROR at tests/ErrorDup.newt:7:8--7:11: Nat already declared
   record Nat where
          ^^^
 
-ERROR at tests/ErrorDup.newt:9:7--9:10: Nat already declared
+ERROR at tests/ErrorDup.newt:5:6--5:9: Nat already declared
-  record Nat where
+  data Nat = Z | S Nat
   
-  class Nat where
+  data Nat = Z | S Nat
        ^^^
 
-ERROR at :1:1--1:2: Failed to compile module ErrorDup
+ERROR at tests/ErrorDup.newt:1:1--1:2: Compile failed

tests/LitConCase.newt.fail

@@ -6,4 +6,4 @@ ERROR at tests/LitConCase.newt:7:5--7:11: expected Prim.Int
   foo MkUnit = MkUnit
       ^^^^^^
 
-ERROR at :1:1--1:2: Failed to compile module LitConCase
+ERROR at tests/LitConCase.newt:1:1--1:2: Compile failed

tests/Possible.newt.fail

@@ -7,4 +7,4 @@ ERROR at tests/Possible.newt:6:5--6:8: possible constructors: [Prelude.Z, Prelud
   foo ()
       ^^^
 
-ERROR at :1:1--1:2: Failed to compile module Possible
+ERROR at tests/Possible.newt:1:1--1:2: Compile failed

tests/Quantity.newt.fail

@@ -7,4 +7,4 @@ ERROR at tests/Quantity.newt:11:15--11:16: used erased value x$0 (FIXME FC may b
   bar {x} = foo x
                 ^
 
-ERROR at :1:1--1:2: Failed to compile module Quantity
+ERROR at tests/Quantity.newt:1:1--1:2: Compile failed