Drop module serializer
This commit is contained in:
3
Makefile
3
Makefile
@@ -16,15 +16,12 @@ src/Revision.newt: .PHONY
|
||||
rm -f src/Revision.newt.new
|
||||
|
||||
newt.js: ${SRCS} src/Revision.newt
|
||||
-rm build/* >/dev/null
|
||||
$(RUNJS) bootstrap/newt.js src/Main.newt -o newt.js
|
||||
|
||||
newt2.js: newt.js
|
||||
-rm build/*
|
||||
$(RUNJS) newt.js src/Main.newt -o newt2.js
|
||||
|
||||
newt3.js: newt2.js
|
||||
-rm build/*
|
||||
time $(RUNJS) newt2.js src/Main.newt -o newt3.js
|
||||
cmp newt2.js newt3.js
|
||||
|
||||
|
||||
269
serializer.ts
269
serializer.ts
@@ -1,269 +0,0 @@
|
||||
// Experimental serializer / deserializer for modules
|
||||
// not completely wired in yet, serialization is running, deserialization is not
|
||||
|
||||
const END = 0;
|
||||
const LIST = 1;
|
||||
const TUPLE = 2;
|
||||
const INDUCT = 3;
|
||||
const STRING = 4;
|
||||
const NUMBER = 5;
|
||||
const NULL = 6;
|
||||
const TRUE = 7;
|
||||
const FALSE = 8;
|
||||
const te = new TextEncoder();
|
||||
|
||||
class DeserializationStream {
|
||||
pos = 0;
|
||||
buf: Uint8Array;
|
||||
|
||||
constructor(buf: Uint8Array) {
|
||||
this.buf = buf;
|
||||
}
|
||||
|
||||
readByte() {
|
||||
return this.buf[this.pos++];
|
||||
}
|
||||
|
||||
readVarint() {
|
||||
let shift = 0;
|
||||
let result = 0;
|
||||
while (true) {
|
||||
const byte = this.readByte();
|
||||
result |= (byte & 0x7f) << shift;
|
||||
if ((byte & 0x80) === 0) break;
|
||||
shift += 7;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
readSignedVarint() {
|
||||
const n = this.readVarint();
|
||||
return (n >>> 1) ^ -(n & 1);
|
||||
}
|
||||
|
||||
readString() {
|
||||
const length = this.readVarint();
|
||||
const bytes = this.buf.slice(this.pos, this.pos + length);
|
||||
this.pos += length;
|
||||
return new TextDecoder().decode(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
export class DecFile {
|
||||
pool: string[] = [""];
|
||||
buf: DeserializationStream;
|
||||
static decode(encoded: Uint8Array) {
|
||||
return new DecFile(encoded).read()
|
||||
}
|
||||
constructor(data: Uint8Array) {
|
||||
this.buf = new DeserializationStream(data);
|
||||
this.readPool();
|
||||
}
|
||||
|
||||
readPool() {
|
||||
while (true) {
|
||||
let str = this.buf.readString();
|
||||
if (!str.length) break
|
||||
this.pool.push(str);
|
||||
}
|
||||
}
|
||||
|
||||
read(): any {
|
||||
const type = this.buf.readByte();
|
||||
switch (type) {
|
||||
case NULL:
|
||||
return null;
|
||||
case LIST: {
|
||||
const list: any[] = [];
|
||||
while (this.buf.buf[this.buf.pos] !== END) {
|
||||
list.push(this.read());
|
||||
}
|
||||
this.buf.pos++;
|
||||
let rval: any = { tag: "Nil", 'h0': null };
|
||||
while (list.length)
|
||||
rval = { tag: "_::_", h0: null, h1: list.pop(), h2: rval };
|
||||
return rval;
|
||||
}
|
||||
case TUPLE: {
|
||||
const tuple: any[] = [];
|
||||
while (this.buf.buf[this.buf.pos] !== END) {
|
||||
tuple.push(this.read());
|
||||
}
|
||||
this.buf.pos++;
|
||||
let rval: any = tuple.pop();
|
||||
while (tuple.length)
|
||||
rval = { tag: "_,_", h0: null, h1: null, h2: tuple.pop(), h3: rval };
|
||||
return rval;
|
||||
}
|
||||
case STRING:
|
||||
return this.pool[this.buf.readVarint()];
|
||||
case NUMBER:
|
||||
return this.buf.readSignedVarint();
|
||||
case INDUCT:
|
||||
const tag = this.buf.readVarint()
|
||||
const obj: any = { tag };
|
||||
let i = 0;
|
||||
while (this.buf.buf[this.buf.pos] !== END) {
|
||||
obj[`h${i++}`] = this.read();
|
||||
}
|
||||
this.buf.pos++;
|
||||
return obj;
|
||||
case TRUE:
|
||||
return true;
|
||||
case FALSE:
|
||||
return false;
|
||||
default:
|
||||
debugger
|
||||
throw new Error(`Unknown type: ${type}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class SerializationStream {
|
||||
pos = 0;
|
||||
buf = new Uint8Array(1024 * 1024);
|
||||
|
||||
ensure(size: number) {
|
||||
if (this.buf.length - this.pos < size) {
|
||||
const tmp = new Uint8Array(this.buf.length * 1.5);
|
||||
tmp.set(this.buf);
|
||||
this.buf = tmp;
|
||||
}
|
||||
}
|
||||
|
||||
writeByte(n: number) {
|
||||
this.ensure(1);
|
||||
this.buf[this.pos++] = n % 256;
|
||||
}
|
||||
|
||||
writeVarint(n: number) {
|
||||
while (n > 127) {
|
||||
this.writeByte((n & 0x7f) | 0x80);
|
||||
n >>= 7;
|
||||
}
|
||||
this.writeByte(n & 0x7f);
|
||||
}
|
||||
|
||||
writeSignedVarint(n: number) {
|
||||
const zigzag = (n << 1) ^ (n >> 31);
|
||||
this.writeVarint(zigzag);
|
||||
}
|
||||
|
||||
writeString(s: string) {
|
||||
let data = te.encode(s);
|
||||
this.ensure(data.byteLength + 4);
|
||||
this.writeVarint(data.byteLength);
|
||||
this.buf.set(data, this.pos);
|
||||
this.pos += data.byteLength;
|
||||
}
|
||||
toUint8Array() {
|
||||
return this.buf.slice(0, this.pos);
|
||||
}
|
||||
}
|
||||
|
||||
export class EncFile {
|
||||
poollen = 1;
|
||||
pool = new SerializationStream();
|
||||
buf = new SerializationStream();
|
||||
pmap: Map<string, number> = new Map();
|
||||
|
||||
constructor() {
|
||||
this.pmap.set("",0);
|
||||
}
|
||||
static encode(data: any) {
|
||||
let f = new EncFile()
|
||||
f.write(data)
|
||||
f.pool.writeVarint(0)
|
||||
return f.toUint8Array()
|
||||
}
|
||||
|
||||
writeString(s: string) {
|
||||
let n = this.pmap.get(s);
|
||||
if (n === undefined) {
|
||||
n = this.poollen++;
|
||||
this.pool.writeString(s);
|
||||
this.pmap.set(s,n);
|
||||
}
|
||||
this.buf.writeVarint(n);
|
||||
}
|
||||
|
||||
write(a: any) {
|
||||
// shouldn't happen?
|
||||
if (a == null) {
|
||||
this.buf.writeByte(NULL);
|
||||
} else if (a.tag == "_::_") {
|
||||
this.buf.writeByte(LIST);
|
||||
for (; a.tag === "_::_"; a = a.h2) {
|
||||
this.write(a.h1);
|
||||
}
|
||||
this.buf.writeByte(END);
|
||||
} else if (a.tag == "_,_") {
|
||||
this.buf.writeByte(TUPLE);
|
||||
for (; a.tag === "_,_"; a = a.h3) {
|
||||
this.write(a.h2);
|
||||
}
|
||||
this.write(a);
|
||||
this.buf.writeByte(END);
|
||||
} else if (typeof a === "string") {
|
||||
this.buf.writeByte(STRING);
|
||||
this.writeString(a);
|
||||
} else if (typeof a === "number") {
|
||||
this.buf.writeByte(NUMBER);
|
||||
this.buf.writeSignedVarint(a);
|
||||
} else if (a === true) {
|
||||
this.buf.writeByte(TRUE);
|
||||
} else if (a === false) {
|
||||
this.buf.writeByte(FALSE);
|
||||
} else if (a.tag !== undefined) {
|
||||
this.buf.writeByte(INDUCT);
|
||||
this.buf.writeVarint(a.tag);
|
||||
// Sometimes keys are skipped
|
||||
let end = 0
|
||||
for (let k in a) {
|
||||
if (k[0] == 'h') end = Math.max(end, +k.slice(1))
|
||||
}
|
||||
for (let i = 0; i <= end; i++) {
|
||||
let key = 'h' + i
|
||||
let v = a[key] ?? null
|
||||
this.write(v);
|
||||
}
|
||||
this.buf.writeByte(END);
|
||||
} else {
|
||||
throw new Error(`handle ${typeof a} ${a} ${Object.keys(a)}`);
|
||||
}
|
||||
}
|
||||
toUint8Array() {
|
||||
const poolArray = this.pool.toUint8Array();
|
||||
const bufArray = this.buf.toUint8Array();
|
||||
const rval = new Uint8Array(poolArray.length + bufArray.length);
|
||||
rval.set(poolArray);
|
||||
rval.set(bufArray, poolArray.length);
|
||||
return rval;
|
||||
}
|
||||
}
|
||||
|
||||
// This was for testing
|
||||
function deepEqual(a: any, b: any): boolean {
|
||||
if (a === b) return true;
|
||||
if (typeof a !== typeof b) return false;
|
||||
if (a == null || b == null) return false;
|
||||
if (typeof a !== "object") return false;
|
||||
|
||||
if (Array.isArray(a)) {
|
||||
if (!Array.isArray(b) || a.length !== b.length) return false;
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
if (!deepEqual(a[i], b[i])) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
const keysA = Object.keys(a);
|
||||
const keysB = Object.keys(b);
|
||||
if (keysA.length !== keysB.length) return false;
|
||||
|
||||
for (const key of keysA) {
|
||||
if (!deepEqual(a[key], b[key])) return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
module Serialize
|
||||
|
||||
import Prelude
|
||||
import Node
|
||||
import Lib.Common
|
||||
import Lib.Types
|
||||
import Data.SortedMap
|
||||
|
||||
ModFile : U
|
||||
ModFile = (String × List TopEntry × List (String × OpDef) × List (QName × MetaEntry) × List (List String))
|
||||
|
||||
pfunc checksum uses (MkIORes) : String → IO String := `(a) => (w) => {
|
||||
const arr = new TextEncoder().encode(a);
|
||||
// djb2 hash
|
||||
let val = 5381
|
||||
for (let i = 0; i < arr.length; i++) {
|
||||
val = ((val * 33) + arr[i]) | 0
|
||||
}
|
||||
return Prelude_MkIORes(""+val, w);
|
||||
}`
|
||||
|
||||
-- this was an experiment, prepping for dumping module information
|
||||
pfunc dumpModFile uses (MkIORes MkUnit): String → ModFile → IO Unit := `(fn,a) => (w) => {
|
||||
let fs = require('fs')
|
||||
try {
|
||||
let {EncFile} = require('./serializer')
|
||||
let enc = EncFile.encode(a)
|
||||
fs.writeFileSync(fn, enc)
|
||||
} catch (e) {}
|
||||
return Prelude_MkIORes(Prelude_MkUnit, w)
|
||||
}`
|
||||
|
||||
|
||||
-- for now, include src and use that to see if something changed
|
||||
dumpModule : List String → String → ModContext → M Unit
|
||||
dumpModule qn src mod = do
|
||||
let fn = "build/\{joinBy "." qn}.newtmod"
|
||||
let csum = mod.csum
|
||||
let defs = listValues mod.modDefs
|
||||
let ops = toList mod.ctxOps
|
||||
let mctx = toList mod.modMetaCtx.metas
|
||||
let deps = mod.modDeps
|
||||
liftIO $ dumpModFile fn (csum,defs,ops,mctx, deps)
|
||||
|
||||
pfunc readModFile uses (MkIORes Just Nothing): String → IO (Maybe ModFile) := `(fn) => (w) => {
|
||||
let fs = require('fs')
|
||||
try {
|
||||
let {DecFile} = require('./serializer')
|
||||
let data = fs.readFileSync(fn)
|
||||
let dec = DecFile.decode(data)
|
||||
return Prelude_MkIORes(Prelude_Just(dec), w)
|
||||
} catch (e) {
|
||||
return Prelude_MkIORes(Prelude_Nothing(), w)
|
||||
}
|
||||
}`
|
||||
|
||||
loadModule : List String → String → M (Maybe ModContext)
|
||||
loadModule qn src = do
|
||||
let fn = "build/\{joinBy "." qn}.newtmod"
|
||||
(Just (csum, defs, ops, mctx, deps)) <- liftIO {M} $ readModFile fn
|
||||
| _ => pure Nothing
|
||||
|
||||
let ops = mapFromList ops
|
||||
let defs = mapFromList $ map (\ entry => (entry.name, entry)) defs
|
||||
-- REVIEW can we ignore those last two inside a module
|
||||
let mctx = MC (mapFromList mctx) Nil 0 NoCheck
|
||||
if csum == src
|
||||
then pure $ Just $ MkModCtx csum defs mctx ops deps
|
||||
else pure Nothing
|
||||
Reference in New Issue
Block a user