From 81839bb4fa039ce9c22ab7baba772b8f0533ab9a Mon Sep 17 00:00:00 2001 From: Ladd Date: Mon, 23 Dec 2024 17:29:38 -0600 Subject: [PATCH] added lossless view --- README.md | 1 + __tests__/lossless.ts | 76 ++++ __tests__/peer-address.ts | 2 +- markdown/001-meeting.md | 167 ++++++++ markdown/002-brainstorming.md | 26 ++ markdown/003-design.md | 62 +++ markdown/004-library.md | 65 +++ markdown/005-before-second-mtg.md | 9 + markdown/006-lossless-representation.md | 12 + markdown/006-lossless.md | 90 ++++ package-lock.json | 460 ++++++--------------- package.json | 4 + src/{collection-layer.ts => collection.ts} | 0 src/deltas.ts | 7 +- src/example-app.ts | 110 +---- src/http-api.ts | 116 ++++++ src/lossless.ts | 88 ++++ src/object-layer.ts | 8 +- src/typed-collection.ts | 11 + src/types.ts | 19 +- 20 files changed, 897 insertions(+), 436 deletions(-) create mode 100644 __tests__/lossless.ts create mode 100644 markdown/001-meeting.md create mode 100644 markdown/002-brainstorming.md create mode 100644 markdown/003-design.md create mode 100644 markdown/004-library.md create mode 100644 markdown/005-before-second-mtg.md create mode 100644 markdown/006-lossless-representation.md create mode 100644 markdown/006-lossless.md rename src/{collection-layer.ts => collection.ts} (100%) create mode 100644 src/http-api.ts create mode 100644 src/lossless.ts create mode 100644 src/typed-collection.ts diff --git a/README.md b/README.md index 7d592dc..8680956 100644 --- a/README.md +++ b/README.md @@ -58,3 +58,4 @@ curl -s http://localhost:3000/peers | jq curl -s http://localhost:3000/deltas/count | jq curl -s http://localhost:3000/deltas | jq ``` + diff --git a/__tests__/lossless.ts b/__tests__/lossless.ts new file mode 100644 index 0000000..6f68b70 --- /dev/null +++ b/__tests__/lossless.ts @@ -0,0 +1,76 @@ +import {Lossless} from '../src/lossless'; +import {Delta} from '../src/types'; + +describe('Lossless', () => { + it('creates a lossless view of neo in the matrix', () => { + const delta: Delta = { + creator: 'a', + host: 'h', + pointers: [{ + localContext: "actor", + target: "keanu", + targetContext: "roles" + }, { + localContext: "role", + target: "neo", + targetContext: "actor" + }, { + localContext: "film", + target: "the_matrix", + targetContext: "cast" + }, { + localContext: "base_salary", + target: 1000000 + }, { + localContext: "salary_currency", + target: "usd" + }] + }; + + const lossless = new Lossless(); + + lossless.ingestDelta(delta); + + expect(lossless.view()).toEqual({ + keanu: { + roles: [{ + creator: "a", + host: "h", + pointers: [ + {actor: "keanu"}, + {role: "neo"}, + {film: "the_matrix"}, + {base_salary: 1000000}, + {salary_currency: "usd"}, + ], + }], + }, + neo: { + actor: [{ + creator: "a", + host: "h", + pointers: [ + {actor: "keanu"}, + {role: "neo"}, + {film: "the_matrix"}, + {base_salary: 1000000}, + {salary_currency: "usd"}, + ], + }], + }, + the_matrix: { + cast: [{ + creator: "a", + host: "h", + pointers: [ + {actor: "keanu"}, + {role: "neo"}, + {film: "the_matrix"}, + {base_salary: 1000000}, + {salary_currency: "usd"}, + ], + }], + } + }); + }); +}); diff --git a/__tests__/peer-address.ts b/__tests__/peer-address.ts index 74093f2..907efb3 100644 --- a/__tests__/peer-address.ts +++ b/__tests__/peer-address.ts @@ -3,7 +3,7 @@ import {PeerAddress} from '../src/types'; describe('PeerAddress', () => { it('toString()', () => { const addr = new PeerAddress('localhost', 1000); - expect(addr.toString()).toBe("localhost:1000"); + expect(addr.toAddrString()).toBe("localhost:1000"); }); it('fromString()', () => { const addr = PeerAddress.fromString("localhost:1000"); diff --git a/markdown/001-meeting.md b/markdown/001-meeting.md new file mode 100644 index 0000000..40e3a1b --- /dev/null +++ b/markdown/001-meeting.md @@ -0,0 +1,167 @@ +2024-12-14 2-4pm CST + +myk: small IP considerations due to 15 years ago some work done for a company + +databases hold on to state +e.g. a query can be answered by this information +table ops can be optimized with relational algebra +p. good ontology +so why change? what needs improvement? +e.g. schema changes +it's possible but it introduces complexity +ex. different departments in same company, modeling same things differently in parallel +last write wins +effects of decisions stack; you're stuck with what's already been done + +relational means records are defined in reference to other records +nosql, no schema, can change things easily +but, hard to produce coherent materialized view + +at some point data lakes became a thing, which works but means your eng. team is now part of your db. + +changes cost more over time +early changes are impactful +ideally we want a flexible ontology + + +need to be able to see both the dao and the ten thousand things + +append only stream of immutable deltas - crdt +assemble a sequence of deltas to obtain a materialized view + +-- q: how to index for query optimization? +a: a succession of more specialized caches / layers of view composition + +-- q: graph? +a: hypergraph -- domain nodes never reference each other, only connected by the deltas that reference them + +now instead of relational tables, that information is pulled into deltas + +does that make sense, asks myk? + +-- q: why is it called a delta? +because it represents a change in state, and metadata associated with that change + +-- q: how to operationalize the management? to define useful ontoligies and pragmatic procedures for usage + +-- q: computational objects? + +databases could subscribe to one another + +-- q: we've talked about indexing, how are we formulating queries? +a db is a func that persists information +a subtype is a query which is a func that returns information +function application, connect by hyperedges, +can have a function that's a query edge + +binding between persistence tier and query tier is loose + +stuff they tried e.g. storing keys and grouping them, etc... not necessarily what we want + + +say a db is forked, 2 copies extended separately for a year, then you want to merge them + +hopping back to computation +myk's WTF moment about this system +say we define a function that takes t-shirt specs and places an order, invoking a remote API +what if I don't have a remote API, just a friend with a t-shirt printer? +friend can be the implementation of that function +I just send them a form to complete, with shipping info etc +Now there's a human in my functional dependency chain + +Normally with a db there's the notion of canonical source of truth +this system doesn't have that + +-- q: trust model, threat model + +provenance? + +-- q: distributed system considerations + + + +what if schemas have diverged? +in a traditional graph you only get what you select + +what if, instead of edge nodes, embeddings? +embedding like in llm context +e.g. embedding of word "friends" +gives you the _numbers_ associated with that term +semantic map +coordinates in semantic space + + +-- q: as an attacker could I hijack deltas? +depends on our trust model + + +agnostic to implementation +high level structure + +strange loop conference +turning the database inside out + +-- q: details of content of each delta: +set of bindings +assertions targeting particular properties of entities +each binding gets a name + +-- q: each delta implies a context? + + + +suggested path: write a set of assertions +those serve as the specification + +initial rollout possibility +toy model that may or may not grow +each participant gets their own data store +should be able to choose to share certain deltas with certain people? + +-- q: + + + +store +retrieve +send +receive +compute + +you could invoke a function by inserting some deltas into the graph, is a concept we're pointing to + +e.g. a "publish" function that notifies another user +can share with the other user so they can activate that behavior +smalltalk type messaging structure on top of the database +note dimensions of attack surface + +layers: +primitives - what's a delta, schema, materialized view, lossy view +delta store - allows you to persiste deltas and query over the delta stream +materialized view store - lossy snapshot(s) +lossy bindings - e.g. graphql, define what a user looks like, that gets application bindings +e.g. where your resolvers come in, so that your fields aren't all arrays, i.e. you resolve conflicts + +-- idea: diff tools +comparing, merging suggestions + +-- idea: operations encoded as deltas, that agents can execute + + +tangent: absential properties +things that are absent from a model can have significance +causal absence-- something that is absent but which caused a thing to be the way it is + + +schema +we'd keep a bucket of myk as a user, that combinatorially combines associated schemas + +every network that you have access to is within your query space + +so... some data types could be more collaborative + +adding a field to a delta doesn't add it to a materialized view automatically-- but that's a good thing + + + + diff --git a/markdown/002-brainstorming.md b/markdown/002-brainstorming.md new file mode 100644 index 0000000..c4e6e86 --- /dev/null +++ b/markdown/002-brainstorming.md @@ -0,0 +1,26 @@ +# What problems are we trying to solve? + +## As an individual I want to be able to write code that can run in a reliable fashion, to provide functionality to myself or others. +- Listen for incoming events +- Store data +- Retrieve data +- Emit events +- Perform computations + +## As an individual I want to be able to form and/or participate in an organization, which +- provides a context for interactions among individuals +- enables interactions among functions written and run by different participants + +## As an individual I want to be able to communicate with other participants, in order to +- make plans +- communicate ideas and information +- attempt to resolve disagreements or establish consensus + + +# What problems are we not trying to solve? + +## No single source of truth. + +## Not a currency trading platform. + + diff --git a/markdown/003-design.md b/markdown/003-design.md new file mode 100644 index 0000000..0ce5671 --- /dev/null +++ b/markdown/003-design.md @@ -0,0 +1,62 @@ +node should be able to store and retrieve data, send and receive messages, perform computations + +what is an appropriate runtime for this? +erlang? + +nodejs? + +custom - rust, c + +bash, +php, +lua + +what are the main features we want/need? + +key value storage +socket listening +message handling +storage management +policy declarations/implementations +communications layer +confidence levels, derived confidence levels + +possible architecture +--- + +tinc vpn +each node has a private key +network must seed with some out-of-band public key exchanges +once the network is seeded, participating nodes can propagate new public keys to be onboarded +simple daemon should run on each node +if it has a static public IP, it advertises it +If it has a dns record, it may advertise it +Peers may discover it's IP through STUN/TURN type arrangement - tinc may be handling that + +What if we use HTTP for the protocol? So e.g. curl can be used for manual testing. + +nodejs with leveldb could be a reasonable way to start a prototype +maybe try to use typescript for good measure + +rust or erlang do seem stronger than nodejs + +in any case we probably want to run our node and its dependencies in containers +do we want to use kubernetes? helm? + +tinc could be good for facilitating operations among the dev team + +node maintains a set of functions and the streams that feed them +executes the functions that it's configured to execute +within the contexts they're configured to use +perhaps creating new deltas representing the results of ingesting some deltas + +so we want to be able to ask a node: +what have you seen? +What have you accepted/how confident are you about xyz + + + + + + + diff --git a/markdown/004-library.md b/markdown/004-library.md new file mode 100644 index 0000000..fe85791 --- /dev/null +++ b/markdown/004-library.md @@ -0,0 +1,65 @@ +When a node wants to perform a write operation, what's the sequence? +Persist the write locally + Think about how this should look from data/metadata perspective -- +Propagate the write to peers + Keep track of confidence in extent of propagation +`*` Read-your-write will only be satisfied after a certian amount of propagation + Views of the record must be able to clearly see its provisionality + + + +Protocol among peers +ZeroMQ can provide messaging fabric +Nodes can request and provide certain things +Query +Subscribe + +Execute a function and encode the result as a delta targeting a given entity +That gets shared via the generic fabric of propagating deltas + +So a fundamental data structure that we need is for the filtering of deltas +I like JSON-Logic for this sort of use case. +Other syntaxes can be transformed into json logic if we want to suppor them. + +A functional unit then would consist of a filter expression +Shall it then also include a resultant expression? Or more broadly an expression representing its logic, +which could include various actions and sequences + + +Address the functions on a ring and orchestrate overlapping coverage of responsibility + +How will we store and access the journal? +Low level approach-- Read from disk and network, process into memory. +Pre-built technologies? LevelDB +Can use LevelDB to store deltas +Structure can correspond to our desired ontology +Layers for +- primitives - what's a delta, schema, materialized view, lossy view +- delta store - allows you to persiste deltas and query over the delta stream +- materialized view store - lossy snapshot(s) +- lossy bindings - e.g. graphql, define what a user looks like, that gets application bindings + e.g. where your resolvers come in, so that your fields aren't all arrays, i.e. you resolve conflicts + +Protocol involving ZeroMQ-- +Pub/sub for deltas +- Can we subdivide / convey our filter expressions? +Req/Reply for ... +- Catching up / obtaining complete set of deltas (*since some snapshot?) +Fanout/in +Pipeline + +Routing-- +You receive a message +Someone sent it as part of a process of attempting to... +- propagate deltas +- issue a request +Is it signed by the sending peer? +- We can probably safely prioritize these +Does the sending peer express confidence? +What is my derived confidence in the records + +In general it seems we have this tradeoff available between computing everything ad hoc as needed from the most complete possible set of source data, applying all relevant filters in each scenario + +Connect to multiple peers +Ask each what you might be missing + diff --git a/markdown/005-before-second-mtg.md b/markdown/005-before-second-mtg.md new file mode 100644 index 0000000..88bd885 --- /dev/null +++ b/markdown/005-before-second-mtg.md @@ -0,0 +1,9 @@ +Code + +Questions + +Ideas + +Requests for input + + diff --git a/markdown/006-lossless-representation.md b/markdown/006-lossless-representation.md new file mode 100644 index 0000000..fff973e --- /dev/null +++ b/markdown/006-lossless-representation.md @@ -0,0 +1,12 @@ + + +> myk: +> I think so far this seems mostly on point, but I'd focus on building the bridge between Domain Entity (lossy representation) <-> Lossless Representation <-> Delta[] I think +> the tricky stuff comes in with, like, how do you take an undifferentiated stream of deltas, a query and a schema +> and filter / merge those deltas into the lossless tree structure you need in order to then reduce into a lossy domain node +> if that part of the flow works then the rest becomes fairly self-evident +> a "lossless representation" is basically a DAG/Tree that starts with a root node whose properties each contain the deltas that assign values to them, where the delta may have a pointer up to "this" and then down to some related domain node, which gets interpolated into the tree instead of just referenced, and it has its properties contain the deltas that target it, etc +> so you need both the ID of the root node (the thing being targeted by one or more deltas) as well as the scehma to apply to determine which contexts on that target to include (target_context effectively becomes a property on the domain entity, right?), as well as which schema to apply to included referenced entities, etc. +> so it's what keeps you from returning the whole stream of deltas, while still allowing you to follow arbitrary edges + + diff --git a/markdown/006-lossless.md b/markdown/006-lossless.md new file mode 100644 index 0000000..9807436 --- /dev/null +++ b/markdown/006-lossless.md @@ -0,0 +1,90 @@ + + +> myk: +> I think so far this seems mostly on point, but I'd focus on building the bridge between Domain Entity (lossy representation) <-> Lossless Representation <-> Delta[] I think +> the tricky stuff comes in with, like, how do you take an undifferentiated stream of deltas, a query and a schema +> and filter / merge those deltas into the lossless tree structure you need in order to then reduce into a lossy domain node +> if that part of the flow works then the rest becomes fairly self-evident +> a "lossless representation" is basically a DAG/Tree that starts with a root node whose properties each contain the deltas that assign values to them, where the delta may have a pointer up to "this" and then down to some related domain node, which gets interpolated into the tree instead of just referenced, and it has its properties contain the deltas that target it, etc +> so you need both the ID of the root node (the thing being targeted by one or more deltas) as well as the scehma to apply to determine which contexts on that target to include (target_context effectively becomes a property on the domain entity, right?), as well as which schema to apply to included referenced entities, etc. +> so it's what keeps you from returning the whole stream of deltas, while still allowing you to follow arbitrary edges + +Example delta: + + pointers: [{ + localContext: "actor", + target: keanu, + targetContext: "roles" + }, { + localContext: "role", + target: neo, + targetContext: "actor" + }, { + localContext: "film", + target: the_matrix, + targetContext: "cast" + }, { + localContext: "base_salary", + target: 1000000 + }, { + localContext: "salary_currency", + target: "usd" + }] + +Lossless transformation: + + { + keanu: { + roles: [{ + pointers: [ + { actor: keanu }, + { role: neo }, + { film: the_matrix }, + { base_salary: 1000000 }, + { salary_currency: "usd" }, + ], + }], + }, + neo: { + actor: [{ + pointers: [ + { actor: keanu }, + { role: neo }, + { film: the_matrix }, + { base_salary: 1000000 }, + { salary_currency: "usd" }, + ], + }], + }, + the_matrix: { + cast: [{ + pointers: [ + { actor: keanu }, + { role: neo }, + { film: the_matrix }, + { base_salary: 1000000 }, + { salary_currency: "usd" }, + ], + }], + } + } + +Lossy transformation: + + { + keanu: { + identities: ["actor"], + films: [the_matrix], + roles: [{film: the_matrix, role: neo, base_salary: 1000000, salary_currency: "usd"}], + }, + neo: { + identities: ["role"], + films: [the_matrix], + actors: [{film: the_matrix, base_salary: 1000000, salary_currency: "usd"}], + }, + the_matrix: { + identities: ["film"], + roles: [neo], + actors: [{role: neo, base_salary: 1000000, salary_currency: "usd"}], + }, + } diff --git a/package-lock.json b/package-lock.json index 4a8f052..3fb72e2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,12 +10,15 @@ "license": "Unlicense", "dependencies": { "@types/bluebird": "^3.5.42", + "@types/debug": "^4.1.12", "@types/json-logic-js": "^2.0.8", "@types/object-hash": "^3.0.6", + "debug": "^4.4.0", "express": "^4.21.2", "json-logic-js": "^2.0.5", "level": "^9.0.0", "object-hash": "^3.0.0", + "showdown": "^2.1.0", "zeromq": "^6.1.2" }, "devDependencies": { @@ -23,6 +26,7 @@ "@types/express": "^5.0.0", "@types/jest": "^29.5.14", "@types/node": "^22.10.2", + "@types/showdown": "^2.0.6", "eslint": "^9.17.0", "eslint-config-airbnb-base-typescript": "^1.1.0", "jest": "^29.7.0", @@ -138,24 +142,6 @@ "url": "https://opencollective.com/babel" } }, - "node_modules/@babel/core/node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/@babel/core/node_modules/json5": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", @@ -169,13 +155,6 @@ "node": ">=6" } }, - "node_modules/@babel/core/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT" - }, "node_modules/@babel/generator": { "version": "7.26.3", "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.26.3.tgz", @@ -585,24 +564,6 @@ "node": ">=6.9.0" } }, - "node_modules/@babel/traverse/node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/@babel/traverse/node_modules/globals": { "version": "11.12.0", "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", @@ -613,13 +574,6 @@ "node": ">=4" } }, - "node_modules/@babel/traverse/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT" - }, "node_modules/@babel/types": { "version": "7.26.3", "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.3.tgz", @@ -742,31 +696,6 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, - "node_modules/@eslint/config-array/node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@eslint/config-array/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT" - }, "node_modules/@eslint/core": { "version": "0.9.1", "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.9.1.tgz", @@ -804,31 +733,6 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/@eslint/eslintrc/node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@eslint/eslintrc/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT" - }, "node_modules/@eslint/js": { "version": "9.17.0", "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.17.0.tgz", @@ -916,33 +820,6 @@ "node": ">=10.10.0" } }, - "node_modules/@humanwhocodes/config-array/node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", - "dev": true, - "license": "MIT", - "peer": true, - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@humanwhocodes/config-array/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT", - "peer": true - }, "node_modules/@humanwhocodes/module-importer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", @@ -1584,6 +1461,15 @@ "@types/node": "*" } }, + "node_modules/@types/debug": { + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", + "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", + "license": "MIT", + "dependencies": { + "@types/ms": "*" + } + }, "node_modules/@types/estree": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", @@ -1700,6 +1586,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/ms": { + "version": "0.7.34", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz", + "integrity": "sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==", + "license": "MIT" + }, "node_modules/@types/node": { "version": "22.10.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.2.tgz", @@ -1752,6 +1644,13 @@ "@types/send": "*" } }, + "node_modules/@types/showdown": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/showdown/-/showdown-2.0.6.tgz", + "integrity": "sha512-pTvD/0CIeqe4x23+YJWlX2gArHa8G0J0Oh6GKaVXV7TAeickpkkZiNOgFcFcmLQ5lB/K0qBJL1FtRYltBfbGCQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/stack-utils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", @@ -1958,23 +1857,6 @@ "balanced-match": "^1.0.0" } }, - "node_modules/@typescript-eslint/parser/node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", - "dev": true, - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/@typescript-eslint/parser/node_modules/minimatch": { "version": "9.0.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", @@ -1990,12 +1872,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/@typescript-eslint/parser/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true - }, "node_modules/@typescript-eslint/parser/node_modules/semver": { "version": "7.6.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", @@ -2114,23 +1990,6 @@ "balanced-match": "^1.0.0" } }, - "node_modules/@typescript-eslint/type-utils/node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", - "dev": true, - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/@typescript-eslint/type-utils/node_modules/minimatch": { "version": "9.0.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", @@ -2146,12 +2005,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/@typescript-eslint/type-utils/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true - }, "node_modules/@typescript-eslint/type-utils/node_modules/semver": { "version": "7.6.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", @@ -2206,31 +2059,6 @@ } } }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT" - }, "node_modules/@typescript-eslint/typescript-estree/node_modules/semver": { "version": "7.6.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", @@ -2349,23 +2177,6 @@ "balanced-match": "^1.0.0" } }, - "node_modules/@typescript-eslint/utils/node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", - "dev": true, - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/@typescript-eslint/utils/node_modules/minimatch": { "version": "9.0.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", @@ -2381,12 +2192,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/@typescript-eslint/utils/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true - }, "node_modules/@typescript-eslint/utils/node_modules/semver": { "version": "7.6.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", @@ -2987,6 +2792,21 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/body-parser/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/body-parser/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -3354,6 +3174,15 @@ "node": ">= 0.8" } }, + "node_modules/commander": { + "version": "9.5.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", + "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || >=14" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -3526,11 +3355,20 @@ } }, "node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "license": "MIT", "dependencies": { - "ms": "2.0.0" + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } } }, "node_modules/dedent": { @@ -4108,24 +3946,6 @@ } } }, - "node_modules/eslint-config-airbnb-base-typescript/node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/eslint-config-airbnb-base-typescript/node_modules/doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -4316,13 +4136,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/eslint-config-airbnb-base-typescript/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT" - }, "node_modules/eslint-import-resolver-node": { "version": "0.3.9", "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.9.tgz", @@ -4347,14 +4160,6 @@ "ms": "^2.1.1" } }, - "node_modules/eslint-import-resolver-node/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT", - "peer": true - }, "node_modules/eslint-module-utils": { "version": "2.12.0", "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.12.0.tgz", @@ -4385,14 +4190,6 @@ "ms": "^2.1.1" } }, - "node_modules/eslint-module-utils/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT", - "peer": true - }, "node_modules/eslint-plugin-import": { "version": "2.31.0", "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.31.0.tgz", @@ -4439,14 +4236,6 @@ "ms": "^2.1.1" } }, - "node_modules/eslint-plugin-import/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT", - "peer": true - }, "node_modules/eslint-scope": { "version": "8.2.0", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.2.0.tgz", @@ -4477,31 +4266,6 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/eslint/node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/eslint/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT" - }, "node_modules/espree": { "version": "10.3.0", "resolved": "https://registry.npmjs.org/espree/-/espree-10.3.0.tgz", @@ -4683,6 +4447,21 @@ "url": "https://opencollective.com/express" } }, + "node_modules/express/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/express/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, "node_modules/extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -4841,6 +4620,21 @@ "node": ">= 0.8" } }, + "node_modules/finalhandler/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/finalhandler/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, "node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -6079,31 +5873,6 @@ "node": ">=10" } }, - "node_modules/istanbul-lib-source-maps/node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/istanbul-lib-source-maps/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT" - }, "node_modules/istanbul-reports": { "version": "3.1.7", "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.7.tgz", @@ -7249,9 +7018,10 @@ } }, "node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" }, "node_modules/napi-macros": { "version": "2.2.2", @@ -8204,6 +7974,21 @@ "node": ">= 0.8.0" } }, + "node_modules/send/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/send/node_modules/debug/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, "node_modules/send/node_modules/encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -8212,11 +7997,6 @@ "node": ">= 0.8" } }, - "node_modules/send/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - }, "node_modules/serve-static": { "version": "1.16.2", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", @@ -8298,6 +8078,22 @@ "node": ">=8" } }, + "node_modules/showdown": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/showdown/-/showdown-2.1.0.tgz", + "integrity": "sha512-/6NVYu4U819R2pUIk79n67SYgJHWCce0a5xTP979WbNp0FL9MN1I1QK662IDU1b6JzKTvmhgI7T7JYIxBi3kMQ==", + "license": "MIT", + "dependencies": { + "commander": "^9.0.0" + }, + "bin": { + "showdown": "bin/showdown.js" + }, + "funding": { + "type": "individual", + "url": "https://www.paypal.me/tiviesantos" + } + }, "node_modules/side-channel": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", diff --git a/package.json b/package.json index bb056bb..11353cf 100644 --- a/package.json +++ b/package.json @@ -15,12 +15,15 @@ "license": "Unlicense", "dependencies": { "@types/bluebird": "^3.5.42", + "@types/debug": "^4.1.12", "@types/json-logic-js": "^2.0.8", "@types/object-hash": "^3.0.6", + "debug": "^4.4.0", "express": "^4.21.2", "json-logic-js": "^2.0.5", "level": "^9.0.0", "object-hash": "^3.0.0", + "showdown": "^2.1.0", "zeromq": "^6.1.2" }, "devDependencies": { @@ -28,6 +31,7 @@ "@types/express": "^5.0.0", "@types/jest": "^29.5.14", "@types/node": "^22.10.2", + "@types/showdown": "^2.0.6", "eslint": "^9.17.0", "eslint-config-airbnb-base-typescript": "^1.1.0", "jest": "^29.7.0", diff --git a/src/collection-layer.ts b/src/collection.ts similarity index 100% rename from src/collection-layer.ts rename to src/collection.ts diff --git a/src/deltas.ts b/src/deltas.ts index 39bf8fd..53c28c1 100644 --- a/src/deltas.ts +++ b/src/deltas.ts @@ -1,9 +1,8 @@ import EventEmitter from 'node:events'; -import {REQUEST_BIND_HOST, REQUEST_BIND_PORT} from './config'; -import {publishSock, subscribeSock} from './pub-sub'; -import {Decision, Delta, PeerAddress} from './types'; -import {myRequestAddr} from './peers'; import objectHash from 'object-hash'; +import {myRequestAddr} from './peers'; +import {publishSock, subscribeSock} from './pub-sub'; +import {Decision, Delta, PeerAddress, Properties} from './types'; export const deltaStream = new EventEmitter(); diff --git a/src/example-app.ts b/src/example-app.ts index 5caee79..35b2bf8 100644 --- a/src/example-app.ts +++ b/src/example-app.ts @@ -1,22 +1,19 @@ -// We can start to use deltas to express relational data in a given context - -import express from "express"; -import {Collection} from "./collection-layer"; -import {HTTP_API_ENABLE, HTTP_API_ADDR, HTTP_API_PORT, SEED_PEERS} from "./config"; -import {deltasAccepted, deltasProposed, runDeltas} from "./deltas"; +import {Collection} from "./collection"; +import {HTTP_API_ENABLE} from "./config"; +import {runDeltas} from "./deltas"; +import {runHttpApi} from "./http-api"; import {Entity} from "./object-layer"; -import {askAllPeersForDeltas, peers, subscribeToSeeds} from "./peers"; +import {askAllPeersForDeltas, subscribeToSeeds} from "./peers"; import {bindPublish, } from "./pub-sub"; import {bindReply, runRequestHandlers} from "./request-reply"; -import {Delta, PeerAddress} from "./types"; +import {TypedCollection} from "./typed-collection"; // As an app we want to be able to write and read data. // The data is whatever shape we define it to be in a given context. // So we want access to an API that is integrated with our declarations of // e.g. entities and their properties. -// This implies at least one layer on top of the underlying primitive deltas. -type UserProperties = { +type User = { id?: string; name: string; nameLong?: string; @@ -24,78 +21,15 @@ type UserProperties = { age: number; }; -class Users { - db = new Collection(); - create(properties: UserProperties): Entity { - // We provide undefined for the id, to let the database generate it - // This call returns the id - const user = this.db.put(undefined, properties); - console.log(`Users.create(${user.id}, ${JSON.stringify(properties)}`); - return user; - } - upsert(properties: UserProperties): Entity { - const user = this.db.put(properties.id, properties); - console.log(`Users.upsert(${user.id}, ${JSON.stringify(properties)}`); - return user; - } - getOne(id: string): Entity | undefined { - return this.db.get(id); - } - getIds(): string[] { - return this.db.getIds(); - } -} - (async () => { - const users = new Users(); - - const app = express() - app.get("/ids", (req: express.Request, res: express.Response) => { - res.json({ids: users.getIds()}); - }); - - app.get("/deltas", (req: express.Request, res: express.Response) => { - // TODO: streaming - res.json(deltasAccepted); - }); - - app.get("/deltas/count", (req: express.Request, res: express.Response) => { - res.json(deltasAccepted.length); - }); - - app.get("/peers", (req: express.Request, res: express.Response) => { - res.json(peers.map(({reqAddr, publishAddr, isSelf, isSeedPeer}) => { - const deltasAcceptedCount = deltasAccepted - .filter((delta: Delta) => { - return delta.receivedFrom?.addr == reqAddr.addr && - delta.receivedFrom?.port == reqAddr.port; - }) - .length; - const peerInfo = { - reqAddr: reqAddr.toAddrString(), - publishAddr: publishAddr?.toAddrString(), - isSelf, - isSeedPeer, - deltaCount: { - accepted: deltasAcceptedCount - } - }; - return peerInfo; - })); - }); - - app.get("/peers/count", (req: express.Request, res: express.Response) => { - res.json(peers.length); - }); - - if (HTTP_API_ENABLE) { - app.listen(HTTP_API_PORT, HTTP_API_ADDR, () => { - console.log(`HTTP API bound to http://${HTTP_API_ADDR}:${HTTP_API_PORT}`); - }); - } + const users = new TypedCollection(); await bindPublish(); await bindReply(); + if (HTTP_API_ENABLE) { + runHttpApi({users}); + } + runDeltas(); runRequestHandlers(); await new Promise((resolve) => setTimeout(resolve, 500)); @@ -104,26 +38,26 @@ class Users { askAllPeersForDeltas(); await new Promise((resolve) => setTimeout(resolve, 1000)); - const taliesin = users.upsert({ + users.onUpdate((u: Entity) => { + console.log('User updated:', u); + }); + + users.onCreate((u: Entity) => { + console.log('New user!:', u); + }); + + const taliesin = users.put(undefined, { // id: 'taliesin-1', name: 'Taliesin', nameLong: 'Taliesin (Ladd)', age: Math.floor(Math.random() * 1000) }); - users.db.onUpdate((u: Entity) => { - console.log('User updated:', u); - }); - - users.db.onCreate((u: Entity) => { - console.log('New user!:', u); - }); - // TODO: Allow configuration regarding read/write concern i.e. // if we perform a read immediately do we see the value we wrote? // Intuition says yes, we want that-- but how do we expose the propagation status? - const result = users.getOne(taliesin.id); + const result = users.get(taliesin.id); const matches: boolean = JSON.stringify(result) === JSON.stringify(taliesin); if (matches) { console.log('Result matches expected: ' + JSON.stringify(taliesin)); diff --git a/src/http-api.ts b/src/http-api.ts new file mode 100644 index 0000000..f84ee9b --- /dev/null +++ b/src/http-api.ts @@ -0,0 +1,116 @@ +import express from "express"; +import {HTTP_API_ADDR, HTTP_API_PORT} from "./config"; +import {deltasAccepted} from "./deltas"; +import {peers} from "./peers"; +import {Delta} from "./types"; +import {readdirSync, readFileSync} from "fs"; +import Debug from "debug"; +import {Collection} from "./collection"; +import {Converter} from "showdown"; +import path from "path"; +const debug = Debug('http-api'); + +type CollectionsToServe = { + [key: string]: Collection; +}; + +const docConverter = new Converter({ + simpleLineBreaks: true, + completeHTMLDocument: true +}); +const htmlDocFromMarkdown = (md: string): string => docConverter.makeHtml(md); + +export function runHttpApi(collections?: CollectionsToServe) { + const app = express(); + app.use(express.json()); + + // Convert markdown to HTML and serve it + const mdFiles = readdirSync('./markdown/') + .filter((f) => f.endsWith('.md')) + .map((name) => path.parse(name).name); + + debug('mdFiles:', mdFiles); + + app.get('/html', (_req: express.Request, res: express.Response) => { + let md = `# Files\n\n`; + for (const name of mdFiles) { + md += `- [${name}](./${name})\n`; + } + const html = htmlDocFromMarkdown(md); + res.setHeader('content-type', 'text/html').send(html); + }); + + for (const name of mdFiles) { + const md = readFileSync(`./markdown/${name}.md`).toString(); + const html = htmlDocFromMarkdown(md); + app.get(`/html/${name}`, (_req: express.Request, res: express.Response) => { + res.setHeader('content-type', 'text/html').send(html); + }); + } + + // Set up API routes + + if (collections) { + for (const [name, collection] of Object.entries(collections)) { + debug(`collection: ${name}`); + + app.get(`/${name}/ids`, (_req: express.Request, res: express.Response) => { + res.json({ids: collection.getIds()}); + }); + + app.put(`/${name}`, (req: express.Request, res: express.Response) => { + const {body: properties} = req; + const ent = collection.put(undefined, properties); + res.json(ent); + }); + + app.put(`/${name}/:id`, (req: express.Request, res: express.Response) => { + const {body: properties, params: {id}} = req; + if (properties.id && properties.id !== id) { + res.status(400).json({error: "ID Mismatch", param: id, property: properties.id}); + return; + } + const ent = collection.put(id, properties); + res.json(ent); + }); + } + } + + app.get("/deltas", (_req: express.Request, res: express.Response) => { + // TODO: streaming + res.json(deltasAccepted); + }); + + app.get("/deltas/count", (_req: express.Request, res: express.Response) => { + res.json(deltasAccepted.length); + }); + + app.get("/peers", (_req: express.Request, res: express.Response) => { + res.json(peers.map(({reqAddr, publishAddr, isSelf, isSeedPeer}) => { + const deltasAcceptedCount = deltasAccepted + .filter((delta: Delta) => { + return delta.receivedFrom?.addr == reqAddr.addr && + delta.receivedFrom?.port == reqAddr.port; + }) + .length; + const peerInfo = { + reqAddr: reqAddr.toAddrString(), + publishAddr: publishAddr?.toAddrString(), + isSelf, + isSeedPeer, + deltaCount: { + accepted: deltasAcceptedCount + } + }; + return peerInfo; + })); + }); + + app.get("/peers/count", (_req: express.Request, res: express.Response) => { + res.json(peers.length); + }); + + app.listen(HTTP_API_PORT, HTTP_API_ADDR, () => { + debug(`HTTP API bound to http://${HTTP_API_ADDR}:${HTTP_API_PORT}`); + }); +} diff --git a/src/lossless.ts b/src/lossless.ts new file mode 100644 index 0000000..f612780 --- /dev/null +++ b/src/lossless.ts @@ -0,0 +1,88 @@ +// Deltas target entities. +// We can maintain a record of all the targeted entities, and the deltas that targeted them + +import {Delta, PropertyTypes} from "./types"; + +type DomainEntityID = string; +type PropertyID = string; + +export type LosslessView = {[key: string]: {[key: string]: Delta[]}}; +export type CollapsedPointer = {[key: string]: PropertyTypes}; +export type CollapsedDelta = Omit & { + pointers: CollapsedPointer[]; +}; + +class DomainEntityMap extends Map {}; + +class DomainEntityProperty { + id: PropertyID; + deltas = new Set(); + + constructor(id: PropertyID) { + this.id = id; + } +} + +class DomainEntity { + id: DomainEntityID; + properties = new Map(); + + constructor(id: DomainEntityID) { + this.id = id; + } + + addDelta(delta: Delta) { + const targetContexts = delta.pointers + .filter(({target}) => target === this.id) + .map(({targetContext}) => targetContext) + .filter((targetContext) => typeof targetContext === 'string'); + for (const targetContext of targetContexts) { + let property = this.properties.get(targetContext); + if (!property) { + property = new DomainEntityProperty(targetContext); + this.properties.set(targetContext, property); + } + property.deltas.add(delta); + } + } +} + +export class Lossless { + domainEntities = new DomainEntityMap(); + + ingestDelta(delta: Delta) { + const targets = delta.pointers + .filter(({targetContext}) => !!targetContext) + .map(({target}) => target) + .filter((target) => typeof target === 'string') + for (const target of targets) { + let ent = this.domainEntities.get(target); + if (!ent) { + ent = new DomainEntity(target); + this.domainEntities.set(target, ent); + } + ent.addDelta(delta); + } + } + + view() { + const view: {[key: DomainEntityID]: {[key: PropertyID]: CollapsedDelta[]}} = {}; + for (const ent of this.domainEntities.values()) { + const obj: {[key: PropertyID]: CollapsedDelta[]} = {}; + view[ent.id] = obj; + for (const prop of ent.properties.values()) { + obj[prop.id] = obj[prop.id] || []; + for (const delta of prop.deltas) { + const collapsedDelta: CollapsedDelta = { + ...delta, + pointers: delta.pointers.map(({localContext, target}) => ({ + [localContext]: target + })) + }; + obj[prop.id].push(collapsedDelta); + } + } + } + return view; + } +} diff --git a/src/object-layer.ts b/src/object-layer.ts index 4be0268..4ab91d5 100644 --- a/src/object-layer.ts +++ b/src/object-layer.ts @@ -11,16 +11,16 @@ import { CREATOR, HOST_ID } from "./config"; import { Delta, PropertyTypes } from "./types"; export type EntityProperties = { - [key: string]: PropertyTypes + [key: string]: PropertyTypes; }; export class Entity { id: string; - properties: EntityProperties; + properties: EntityProperties = {}; ahead = 0; + constructor(id: string) { this.id = id; - this.properties = {}; } } @@ -28,6 +28,7 @@ export class Entity { export class EntityPropertiesDeltaBuilder { delta: Delta; + constructor(entityId: string) { this.delta = { creator: CREATOR, @@ -39,6 +40,7 @@ export class EntityPropertiesDeltaBuilder { }] }; } + add(localContext: string, target: PropertyTypes) { this.delta.pointers.push({localContext, target}); } diff --git a/src/typed-collection.ts b/src/typed-collection.ts new file mode 100644 index 0000000..fec9742 --- /dev/null +++ b/src/typed-collection.ts @@ -0,0 +1,11 @@ +import { Collection } from './collection'; +import {Entity, EntityProperties} from './object-layer'; + +export class TypedCollection extends Collection { + put(id: string | undefined, properties: T): Entity { + return super.put(id, properties); + } + get(id: string): Entity | undefined { + return super.get(id); + } +} diff --git a/src/types.ts b/src/types.ts index f2e6792..c83e6ce 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,14 +1,16 @@ +export type PointerTarget = string | number | undefined; + export type Pointer = { - localContext: string, - target: string | number | undefined, - targetContext?: string + localContext: string; + target: PointerTarget; + targetContext?: string; }; export type Delta = { - creator: string, - host: string, - pointers: Pointer[], - receivedFrom?: PeerAddress, + creator: string; + host: string; + pointers: Pointer[]; + receivedFrom?: PeerAddress; } export type DeltaContext = Delta & { @@ -29,7 +31,6 @@ export enum Decision { Defer }; - export type JSONLogic = object; export type FilterExpr = JSONLogic; @@ -38,6 +39,8 @@ export type FilterGenerator = () => FilterExpr; export type PropertyTypes = string | number | undefined; +export type Properties = {[key: string]: PropertyTypes}; + export class PeerAddress { addr: string; port: number;