From dcf162684edcb46cbf03432e1ba5654f61b17a1c Mon Sep 17 00:00:00 2001 From: Steve Dunham Date: Fri, 13 Feb 2026 09:28:42 -0800 Subject: [PATCH] Drop module serializer --- Makefile | 3 - serializer.ts | 269 --------------------------------------------- src/Serialize.newt | 69 ------------ 3 files changed, 341 deletions(-) delete mode 100644 serializer.ts delete mode 100644 src/Serialize.newt diff --git a/Makefile b/Makefile index 09534ab..1a594c8 100644 --- a/Makefile +++ b/Makefile @@ -16,15 +16,12 @@ src/Revision.newt: .PHONY rm -f src/Revision.newt.new newt.js: ${SRCS} src/Revision.newt - -rm build/* >/dev/null $(RUNJS) bootstrap/newt.js src/Main.newt -o newt.js newt2.js: newt.js - -rm build/* $(RUNJS) newt.js src/Main.newt -o newt2.js newt3.js: newt2.js - -rm build/* time $(RUNJS) newt2.js src/Main.newt -o newt3.js cmp newt2.js newt3.js diff --git a/serializer.ts b/serializer.ts deleted file mode 100644 index 5356136..0000000 --- a/serializer.ts +++ /dev/null @@ -1,269 +0,0 @@ -// Experimental serializer / deserializer for modules -// not completely wired in yet, serialization is running, deserialization is not - -const END = 0; -const LIST = 1; -const TUPLE = 2; -const INDUCT = 3; -const STRING = 4; -const NUMBER = 5; -const NULL = 6; -const TRUE = 7; -const FALSE = 8; -const te = new TextEncoder(); - -class DeserializationStream { - pos = 0; - buf: Uint8Array; - - constructor(buf: Uint8Array) { - this.buf = buf; - } - - readByte() { - return this.buf[this.pos++]; - } - - readVarint() { - let shift = 0; - let result = 0; - while (true) { - const byte = this.readByte(); - result |= (byte & 0x7f) << shift; - if ((byte & 0x80) === 0) break; - shift += 7; - } - return result; - } - - readSignedVarint() { - const n = this.readVarint(); - return (n >>> 1) ^ -(n & 1); - } - - readString() { - const length = this.readVarint(); - const bytes = this.buf.slice(this.pos, this.pos + length); - this.pos += length; - return new TextDecoder().decode(bytes); - } -} - -export class DecFile { - pool: string[] = [""]; - buf: DeserializationStream; - static decode(encoded: Uint8Array) { - return new DecFile(encoded).read() - } - constructor(data: Uint8Array) { - this.buf = new DeserializationStream(data); - this.readPool(); - } - - readPool() { - while (true) { - let str = this.buf.readString(); - if (!str.length) break - this.pool.push(str); - } - } - - read(): any { - const type = this.buf.readByte(); - switch (type) { - case NULL: - return null; - case LIST: { - const list: any[] = []; - while (this.buf.buf[this.buf.pos] !== END) { - list.push(this.read()); - } - this.buf.pos++; - let rval: any = { tag: "Nil", 'h0': null }; - while (list.length) - rval = { tag: "_::_", h0: null, h1: list.pop(), h2: rval }; - return rval; - } - case TUPLE: { - const tuple: any[] = []; - while (this.buf.buf[this.buf.pos] !== END) { - tuple.push(this.read()); - } - this.buf.pos++; - let rval: any = tuple.pop(); - while (tuple.length) - rval = { tag: "_,_", h0: null, h1: null, h2: tuple.pop(), h3: rval }; - return rval; - } - case STRING: - return this.pool[this.buf.readVarint()]; - case NUMBER: - return this.buf.readSignedVarint(); - case INDUCT: - const tag = this.buf.readVarint() - const obj: any = { tag }; - let i = 0; - while (this.buf.buf[this.buf.pos] !== END) { - obj[`h${i++}`] = this.read(); - } - this.buf.pos++; - return obj; - case TRUE: - return true; - case FALSE: - return false; - default: - debugger - throw new Error(`Unknown type: ${type}`); - } - } -} - -class SerializationStream { - pos = 0; - buf = new Uint8Array(1024 * 1024); - - ensure(size: number) { - if (this.buf.length - this.pos < size) { - const tmp = new Uint8Array(this.buf.length * 1.5); - tmp.set(this.buf); - this.buf = tmp; - } - } - - writeByte(n: number) { - this.ensure(1); - this.buf[this.pos++] = n % 256; - } - - writeVarint(n: number) { - while (n > 127) { - this.writeByte((n & 0x7f) | 0x80); - n >>= 7; - } - this.writeByte(n & 0x7f); - } - - writeSignedVarint(n: number) { - const zigzag = (n << 1) ^ (n >> 31); - this.writeVarint(zigzag); - } - - writeString(s: string) { - let data = te.encode(s); - this.ensure(data.byteLength + 4); - this.writeVarint(data.byteLength); - this.buf.set(data, this.pos); - this.pos += data.byteLength; - } - toUint8Array() { - return this.buf.slice(0, this.pos); - } -} - -export class EncFile { - poollen = 1; - pool = new SerializationStream(); - buf = new SerializationStream(); - pmap: Map = new Map(); - - constructor() { - this.pmap.set("",0); - } - static encode(data: any) { - let f = new EncFile() - f.write(data) - f.pool.writeVarint(0) - return f.toUint8Array() - } - - writeString(s: string) { - let n = this.pmap.get(s); - if (n === undefined) { - n = this.poollen++; - this.pool.writeString(s); - this.pmap.set(s,n); - } - this.buf.writeVarint(n); - } - - write(a: any) { - // shouldn't happen? - if (a == null) { - this.buf.writeByte(NULL); - } else if (a.tag == "_::_") { - this.buf.writeByte(LIST); - for (; a.tag === "_::_"; a = a.h2) { - this.write(a.h1); - } - this.buf.writeByte(END); - } else if (a.tag == "_,_") { - this.buf.writeByte(TUPLE); - for (; a.tag === "_,_"; a = a.h3) { - this.write(a.h2); - } - this.write(a); - this.buf.writeByte(END); - } else if (typeof a === "string") { - this.buf.writeByte(STRING); - this.writeString(a); - } else if (typeof a === "number") { - this.buf.writeByte(NUMBER); - this.buf.writeSignedVarint(a); - } else if (a === true) { - this.buf.writeByte(TRUE); - } else if (a === false) { - this.buf.writeByte(FALSE); - } else if (a.tag !== undefined) { - this.buf.writeByte(INDUCT); - this.buf.writeVarint(a.tag); - // Sometimes keys are skipped - let end = 0 - for (let k in a) { - if (k[0] == 'h') end = Math.max(end, +k.slice(1)) - } - for (let i = 0; i <= end; i++) { - let key = 'h' + i - let v = a[key] ?? null - this.write(v); - } - this.buf.writeByte(END); - } else { - throw new Error(`handle ${typeof a} ${a} ${Object.keys(a)}`); - } - } - toUint8Array() { - const poolArray = this.pool.toUint8Array(); - const bufArray = this.buf.toUint8Array(); - const rval = new Uint8Array(poolArray.length + bufArray.length); - rval.set(poolArray); - rval.set(bufArray, poolArray.length); - return rval; - } -} - -// This was for testing -function deepEqual(a: any, b: any): boolean { - if (a === b) return true; - if (typeof a !== typeof b) return false; - if (a == null || b == null) return false; - if (typeof a !== "object") return false; - - if (Array.isArray(a)) { - if (!Array.isArray(b) || a.length !== b.length) return false; - for (let i = 0; i < a.length; i++) { - if (!deepEqual(a[i], b[i])) return false; - } - return true; - } - - const keysA = Object.keys(a); - const keysB = Object.keys(b); - if (keysA.length !== keysB.length) return false; - - for (const key of keysA) { - if (!deepEqual(a[key], b[key])) return false; - } - - return true; -} diff --git a/src/Serialize.newt b/src/Serialize.newt deleted file mode 100644 index 361a867..0000000 --- a/src/Serialize.newt +++ /dev/null @@ -1,69 +0,0 @@ -module Serialize - -import Prelude -import Node -import Lib.Common -import Lib.Types -import Data.SortedMap - -ModFile : U -ModFile = (String × List TopEntry × List (String × OpDef) × List (QName × MetaEntry) × List (List String)) - -pfunc checksum uses (MkIORes) : String → IO String := `(a) => (w) => { - const arr = new TextEncoder().encode(a); - // djb2 hash - let val = 5381 - for (let i = 0; i < arr.length; i++) { - val = ((val * 33) + arr[i]) | 0 - } - return Prelude_MkIORes(""+val, w); -}` - --- this was an experiment, prepping for dumping module information -pfunc dumpModFile uses (MkIORes MkUnit): String → ModFile → IO Unit := `(fn,a) => (w) => { - let fs = require('fs') - try { - let {EncFile} = require('./serializer') - let enc = EncFile.encode(a) - fs.writeFileSync(fn, enc) - } catch (e) {} - return Prelude_MkIORes(Prelude_MkUnit, w) -}` - - --- for now, include src and use that to see if something changed -dumpModule : List String → String → ModContext → M Unit -dumpModule qn src mod = do - let fn = "build/\{joinBy "." qn}.newtmod" - let csum = mod.csum - let defs = listValues mod.modDefs - let ops = toList mod.ctxOps - let mctx = toList mod.modMetaCtx.metas - let deps = mod.modDeps - liftIO $ dumpModFile fn (csum,defs,ops,mctx, deps) - -pfunc readModFile uses (MkIORes Just Nothing): String → IO (Maybe ModFile) := `(fn) => (w) => { - let fs = require('fs') - try { - let {DecFile} = require('./serializer') - let data = fs.readFileSync(fn) - let dec = DecFile.decode(data) - return Prelude_MkIORes(Prelude_Just(dec), w) - } catch (e) { - return Prelude_MkIORes(Prelude_Nothing(), w) - } -}` - -loadModule : List String → String → M (Maybe ModContext) -loadModule qn src = do - let fn = "build/\{joinBy "." qn}.newtmod" - (Just (csum, defs, ops, mctx, deps)) <- liftIO {M} $ readModFile fn - | _ => pure Nothing - - let ops = mapFromList ops - let defs = mapFromList $ map (\ entry => (entry.name, entry)) defs - -- REVIEW can we ignore those last two inside a module - let mctx = MC (mapFromList mctx) Nil 0 NoCheck - if csum == src - then pure $ Just $ MkModCtx csum defs mctx ops deps - else pure Nothing