added lossless view

This commit is contained in:
Ladd Hoffman 2024-12-23 17:29:38 -06:00
parent f8b501859b
commit 81839bb4fa
20 changed files with 897 additions and 436 deletions

View File

@ -58,3 +58,4 @@ curl -s http://localhost:3000/peers | jq
curl -s http://localhost:3000/deltas/count | jq
curl -s http://localhost:3000/deltas | jq
```

76
__tests__/lossless.ts Normal file
View File

@ -0,0 +1,76 @@
import {Lossless} from '../src/lossless';
import {Delta} from '../src/types';
describe('Lossless', () => {
it('creates a lossless view of neo in the matrix', () => {
const delta: Delta = {
creator: 'a',
host: 'h',
pointers: [{
localContext: "actor",
target: "keanu",
targetContext: "roles"
}, {
localContext: "role",
target: "neo",
targetContext: "actor"
}, {
localContext: "film",
target: "the_matrix",
targetContext: "cast"
}, {
localContext: "base_salary",
target: 1000000
}, {
localContext: "salary_currency",
target: "usd"
}]
};
const lossless = new Lossless();
lossless.ingestDelta(delta);
expect(lossless.view()).toEqual({
keanu: {
roles: [{
creator: "a",
host: "h",
pointers: [
{actor: "keanu"},
{role: "neo"},
{film: "the_matrix"},
{base_salary: 1000000},
{salary_currency: "usd"},
],
}],
},
neo: {
actor: [{
creator: "a",
host: "h",
pointers: [
{actor: "keanu"},
{role: "neo"},
{film: "the_matrix"},
{base_salary: 1000000},
{salary_currency: "usd"},
],
}],
},
the_matrix: {
cast: [{
creator: "a",
host: "h",
pointers: [
{actor: "keanu"},
{role: "neo"},
{film: "the_matrix"},
{base_salary: 1000000},
{salary_currency: "usd"},
],
}],
}
});
});
});

View File

@ -3,7 +3,7 @@ import {PeerAddress} from '../src/types';
describe('PeerAddress', () => {
it('toString()', () => {
const addr = new PeerAddress('localhost', 1000);
expect(addr.toString()).toBe("localhost:1000");
expect(addr.toAddrString()).toBe("localhost:1000");
});
it('fromString()', () => {
const addr = PeerAddress.fromString("localhost:1000");

167
markdown/001-meeting.md Normal file
View File

@ -0,0 +1,167 @@
2024-12-14 2-4pm CST
myk: small IP considerations due to 15 years ago some work done for a company
databases hold on to state
e.g. a query can be answered by this information
table ops can be optimized with relational algebra
p. good ontology
so why change? what needs improvement?
e.g. schema changes
it's possible but it introduces complexity
ex. different departments in same company, modeling same things differently in parallel
last write wins
effects of decisions stack; you're stuck with what's already been done
relational means records are defined in reference to other records
nosql, no schema, can change things easily
but, hard to produce coherent materialized view
at some point data lakes became a thing, which works but means your eng. team is now part of your db.
changes cost more over time
early changes are impactful
ideally we want a flexible ontology
need to be able to see both the dao and the ten thousand things
append only stream of immutable deltas - crdt
assemble a sequence of deltas to obtain a materialized view
-- q: how to index for query optimization?
a: a succession of more specialized caches / layers of view composition
-- q: graph?
a: hypergraph -- domain nodes never reference each other, only connected by the deltas that reference them
now instead of relational tables, that information is pulled into deltas
does that make sense, asks myk?
-- q: why is it called a delta?
because it represents a change in state, and metadata associated with that change
-- q: how to operationalize the management? to define useful ontoligies and pragmatic procedures for usage
-- q: computational objects?
databases could subscribe to one another
-- q: we've talked about indexing, how are we formulating queries?
a db is a func that persists information
a subtype is a query which is a func that returns information
function application, connect by hyperedges,
can have a function that's a query edge
binding between persistence tier and query tier is loose
stuff they tried e.g. storing keys and grouping them, etc... not necessarily what we want
say a db is forked, 2 copies extended separately for a year, then you want to merge them
hopping back to computation
myk's WTF moment about this system
say we define a function that takes t-shirt specs and places an order, invoking a remote API
what if I don't have a remote API, just a friend with a t-shirt printer?
friend can be the implementation of that function
I just send them a form to complete, with shipping info etc
Now there's a human in my functional dependency chain
Normally with a db there's the notion of canonical source of truth
this system doesn't have that
-- q: trust model, threat model
provenance?
-- q: distributed system considerations
what if schemas have diverged?
in a traditional graph you only get what you select
what if, instead of edge nodes, embeddings?
embedding like in llm context
e.g. embedding of word "friends"
gives you the _numbers_ associated with that term
semantic map
coordinates in semantic space
-- q: as an attacker could I hijack deltas?
depends on our trust model
agnostic to implementation
high level structure
strange loop conference
turning the database inside out
-- q: details of content of each delta:
set of bindings
assertions targeting particular properties of entities
each binding gets a name
-- q: each delta implies a context?
suggested path: write a set of assertions
those serve as the specification
initial rollout possibility
toy model that may or may not grow
each participant gets their own data store
should be able to choose to share certain deltas with certain people?
-- q:
store
retrieve
send
receive
compute
you could invoke a function by inserting some deltas into the graph, is a concept we're pointing to
e.g. a "publish" function that notifies another user
can share with the other user so they can activate that behavior
smalltalk type messaging structure on top of the database
note dimensions of attack surface
layers:
primitives - what's a delta, schema, materialized view, lossy view
delta store - allows you to persiste deltas and query over the delta stream
materialized view store - lossy snapshot(s)
lossy bindings - e.g. graphql, define what a user looks like, that gets application bindings
e.g. where your resolvers come in, so that your fields aren't all arrays, i.e. you resolve conflicts
-- idea: diff tools
comparing, merging suggestions
-- idea: operations encoded as deltas, that agents can execute
tangent: absential properties
things that are absent from a model can have significance
causal absence-- something that is absent but which caused a thing to be the way it is
schema
we'd keep a bucket of myk as a user, that combinatorially combines associated schemas
every network that you have access to is within your query space
so... some data types could be more collaborative
adding a field to a delta doesn't add it to a materialized view automatically-- but that's a good thing

View File

@ -0,0 +1,26 @@
# What problems are we trying to solve?
## As an individual I want to be able to write code that can run in a reliable fashion, to provide functionality to myself or others.
- Listen for incoming events
- Store data
- Retrieve data
- Emit events
- Perform computations
## As an individual I want to be able to form and/or participate in an organization, which
- provides a context for interactions among individuals
- enables interactions among functions written and run by different participants
## As an individual I want to be able to communicate with other participants, in order to
- make plans
- communicate ideas and information
- attempt to resolve disagreements or establish consensus
# What problems are we not trying to solve?
## No single source of truth.
## Not a currency trading platform.

62
markdown/003-design.md Normal file
View File

@ -0,0 +1,62 @@
node should be able to store and retrieve data, send and receive messages, perform computations
what is an appropriate runtime for this?
erlang?
nodejs?
custom - rust, c
bash,
php,
lua
what are the main features we want/need?
key value storage
socket listening
message handling
storage management
policy declarations/implementations
communications layer
confidence levels, derived confidence levels
possible architecture
---
tinc vpn
each node has a private key
network must seed with some out-of-band public key exchanges
once the network is seeded, participating nodes can propagate new public keys to be onboarded
simple daemon should run on each node
if it has a static public IP, it advertises it
If it has a dns record, it may advertise it
Peers may discover it's IP through STUN/TURN type arrangement - tinc may be handling that
What if we use HTTP for the protocol? So e.g. curl can be used for manual testing.
nodejs with leveldb could be a reasonable way to start a prototype
maybe try to use typescript for good measure
rust or erlang do seem stronger than nodejs
in any case we probably want to run our node and its dependencies in containers
do we want to use kubernetes? helm?
tinc could be good for facilitating operations among the dev team
node maintains a set of functions and the streams that feed them
executes the functions that it's configured to execute
within the contexts they're configured to use
perhaps creating new deltas representing the results of ingesting some deltas
so we want to be able to ask a node:
what have you seen?
What have you accepted/how confident are you about xyz

65
markdown/004-library.md Normal file
View File

@ -0,0 +1,65 @@
When a node wants to perform a write operation, what's the sequence?
Persist the write locally
Think about how this should look from data/metadata perspective --
Propagate the write to peers
Keep track of confidence in extent of propagation
`*` Read-your-write will only be satisfied after a certian amount of propagation
Views of the record must be able to clearly see its provisionality
Protocol among peers
ZeroMQ can provide messaging fabric
Nodes can request and provide certain things
Query
Subscribe
Execute a function and encode the result as a delta targeting a given entity
That gets shared via the generic fabric of propagating deltas
So a fundamental data structure that we need is for the filtering of deltas
I like JSON-Logic for this sort of use case.
Other syntaxes can be transformed into json logic if we want to suppor them.
A functional unit then would consist of a filter expression
Shall it then also include a resultant expression? Or more broadly an expression representing its logic,
which could include various actions and sequences
Address the functions on a ring and orchestrate overlapping coverage of responsibility
How will we store and access the journal?
Low level approach-- Read from disk and network, process into memory.
Pre-built technologies? LevelDB
Can use LevelDB to store deltas
Structure can correspond to our desired ontology
Layers for
- primitives - what's a delta, schema, materialized view, lossy view
- delta store - allows you to persiste deltas and query over the delta stream
- materialized view store - lossy snapshot(s)
- lossy bindings - e.g. graphql, define what a user looks like, that gets application bindings
e.g. where your resolvers come in, so that your fields aren't all arrays, i.e. you resolve conflicts
Protocol involving ZeroMQ--
Pub/sub for deltas
- Can we subdivide / convey our filter expressions?
Req/Reply for ...
- Catching up / obtaining complete set of deltas (*since some snapshot?)
Fanout/in
Pipeline
Routing--
You receive a message
Someone sent it as part of a process of attempting to...
- propagate deltas
- issue a request
Is it signed by the sending peer?
- We can probably safely prioritize these
Does the sending peer express confidence?
What is my derived confidence in the records
In general it seems we have this tradeoff available between computing everything ad hoc as needed from the most complete possible set of source data, applying all relevant filters in each scenario
Connect to multiple peers
Ask each what you might be missing

View File

@ -0,0 +1,9 @@
Code
Questions
Ideas
Requests for input

View File

@ -0,0 +1,12 @@
> myk:
> I think so far this seems mostly on point, but I'd focus on building the bridge between Domain Entity (lossy representation) <-> Lossless Representation <-> Delta[] I think
> the tricky stuff comes in with, like, how do you take an undifferentiated stream of deltas, a query and a schema
> and filter / merge those deltas into the lossless tree structure you need in order to then reduce into a lossy domain node
> if that part of the flow works then the rest becomes fairly self-evident
> a "lossless representation" is basically a DAG/Tree that starts with a root node whose properties each contain the deltas that assign values to them, where the delta may have a pointer up to "this" and then down to some related domain node, which gets interpolated into the tree instead of just referenced, and it has its properties contain the deltas that target it, etc
> so you need both the ID of the root node (the thing being targeted by one or more deltas) as well as the scehma to apply to determine which contexts on that target to include (target_context effectively becomes a property on the domain entity, right?), as well as which schema to apply to included referenced entities, etc.
> so it's what keeps you from returning the whole stream of deltas, while still allowing you to follow arbitrary edges

90
markdown/006-lossless.md Normal file
View File

@ -0,0 +1,90 @@
> myk:
> I think so far this seems mostly on point, but I'd focus on building the bridge between Domain Entity (lossy representation) <-> Lossless Representation <-> Delta[] I think
> the tricky stuff comes in with, like, how do you take an undifferentiated stream of deltas, a query and a schema
> and filter / merge those deltas into the lossless tree structure you need in order to then reduce into a lossy domain node
> if that part of the flow works then the rest becomes fairly self-evident
> a "lossless representation" is basically a DAG/Tree that starts with a root node whose properties each contain the deltas that assign values to them, where the delta may have a pointer up to "this" and then down to some related domain node, which gets interpolated into the tree instead of just referenced, and it has its properties contain the deltas that target it, etc
> so you need both the ID of the root node (the thing being targeted by one or more deltas) as well as the scehma to apply to determine which contexts on that target to include (target_context effectively becomes a property on the domain entity, right?), as well as which schema to apply to included referenced entities, etc.
> so it's what keeps you from returning the whole stream of deltas, while still allowing you to follow arbitrary edges
Example delta:
pointers: [{
localContext: "actor",
target: keanu,
targetContext: "roles"
}, {
localContext: "role",
target: neo,
targetContext: "actor"
}, {
localContext: "film",
target: the_matrix,
targetContext: "cast"
}, {
localContext: "base_salary",
target: 1000000
}, {
localContext: "salary_currency",
target: "usd"
}]
Lossless transformation:
{
keanu: {
roles: [{
pointers: [
{ actor: keanu },
{ role: neo },
{ film: the_matrix },
{ base_salary: 1000000 },
{ salary_currency: "usd" },
],
}],
},
neo: {
actor: [{
pointers: [
{ actor: keanu },
{ role: neo },
{ film: the_matrix },
{ base_salary: 1000000 },
{ salary_currency: "usd" },
],
}],
},
the_matrix: {
cast: [{
pointers: [
{ actor: keanu },
{ role: neo },
{ film: the_matrix },
{ base_salary: 1000000 },
{ salary_currency: "usd" },
],
}],
}
}
Lossy transformation:
{
keanu: {
identities: ["actor"],
films: [the_matrix],
roles: [{film: the_matrix, role: neo, base_salary: 1000000, salary_currency: "usd"}],
},
neo: {
identities: ["role"],
films: [the_matrix],
actors: [{film: the_matrix, base_salary: 1000000, salary_currency: "usd"}],
},
the_matrix: {
identities: ["film"],
roles: [neo],
actors: [{role: neo, base_salary: 1000000, salary_currency: "usd"}],
},
}

460
package-lock.json generated
View File

@ -10,12 +10,15 @@
"license": "Unlicense",
"dependencies": {
"@types/bluebird": "^3.5.42",
"@types/debug": "^4.1.12",
"@types/json-logic-js": "^2.0.8",
"@types/object-hash": "^3.0.6",
"debug": "^4.4.0",
"express": "^4.21.2",
"json-logic-js": "^2.0.5",
"level": "^9.0.0",
"object-hash": "^3.0.0",
"showdown": "^2.1.0",
"zeromq": "^6.1.2"
},
"devDependencies": {
@ -23,6 +26,7 @@
"@types/express": "^5.0.0",
"@types/jest": "^29.5.14",
"@types/node": "^22.10.2",
"@types/showdown": "^2.0.6",
"eslint": "^9.17.0",
"eslint-config-airbnb-base-typescript": "^1.1.0",
"jest": "^29.7.0",
@ -138,24 +142,6 @@
"url": "https://opencollective.com/babel"
}
},
"node_modules/@babel/core/node_modules/debug": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
"integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
"dev": true,
"license": "MIT",
"dependencies": {
"ms": "^2.1.3"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/@babel/core/node_modules/json5": {
"version": "2.2.3",
"resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
@ -169,13 +155,6 @@
"node": ">=6"
}
},
"node_modules/@babel/core/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"dev": true,
"license": "MIT"
},
"node_modules/@babel/generator": {
"version": "7.26.3",
"resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.26.3.tgz",
@ -585,24 +564,6 @@
"node": ">=6.9.0"
}
},
"node_modules/@babel/traverse/node_modules/debug": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
"integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
"dev": true,
"license": "MIT",
"dependencies": {
"ms": "^2.1.3"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/@babel/traverse/node_modules/globals": {
"version": "11.12.0",
"resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz",
@ -613,13 +574,6 @@
"node": ">=4"
}
},
"node_modules/@babel/traverse/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"dev": true,
"license": "MIT"
},
"node_modules/@babel/types": {
"version": "7.26.3",
"resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.3.tgz",
@ -742,31 +696,6 @@
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
}
},
"node_modules/@eslint/config-array/node_modules/debug": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
"integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
"dev": true,
"license": "MIT",
"dependencies": {
"ms": "^2.1.3"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/@eslint/config-array/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"dev": true,
"license": "MIT"
},
"node_modules/@eslint/core": {
"version": "0.9.1",
"resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.9.1.tgz",
@ -804,31 +733,6 @@
"url": "https://opencollective.com/eslint"
}
},
"node_modules/@eslint/eslintrc/node_modules/debug": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
"integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
"dev": true,
"license": "MIT",
"dependencies": {
"ms": "^2.1.3"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/@eslint/eslintrc/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"dev": true,
"license": "MIT"
},
"node_modules/@eslint/js": {
"version": "9.17.0",
"resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.17.0.tgz",
@ -916,33 +820,6 @@
"node": ">=10.10.0"
}
},
"node_modules/@humanwhocodes/config-array/node_modules/debug": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
"integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"ms": "^2.1.3"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/@humanwhocodes/config-array/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"dev": true,
"license": "MIT",
"peer": true
},
"node_modules/@humanwhocodes/module-importer": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz",
@ -1584,6 +1461,15 @@
"@types/node": "*"
}
},
"node_modules/@types/debug": {
"version": "4.1.12",
"resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz",
"integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==",
"license": "MIT",
"dependencies": {
"@types/ms": "*"
}
},
"node_modules/@types/estree": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz",
@ -1700,6 +1586,12 @@
"dev": true,
"license": "MIT"
},
"node_modules/@types/ms": {
"version": "0.7.34",
"resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz",
"integrity": "sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==",
"license": "MIT"
},
"node_modules/@types/node": {
"version": "22.10.2",
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.2.tgz",
@ -1752,6 +1644,13 @@
"@types/send": "*"
}
},
"node_modules/@types/showdown": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/@types/showdown/-/showdown-2.0.6.tgz",
"integrity": "sha512-pTvD/0CIeqe4x23+YJWlX2gArHa8G0J0Oh6GKaVXV7TAeickpkkZiNOgFcFcmLQ5lB/K0qBJL1FtRYltBfbGCQ==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/stack-utils": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz",
@ -1958,23 +1857,6 @@
"balanced-match": "^1.0.0"
}
},
"node_modules/@typescript-eslint/parser/node_modules/debug": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
"integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
"dev": true,
"dependencies": {
"ms": "^2.1.3"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/@typescript-eslint/parser/node_modules/minimatch": {
"version": "9.0.5",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz",
@ -1990,12 +1872,6 @@
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/@typescript-eslint/parser/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"dev": true
},
"node_modules/@typescript-eslint/parser/node_modules/semver": {
"version": "7.6.3",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz",
@ -2114,23 +1990,6 @@
"balanced-match": "^1.0.0"
}
},
"node_modules/@typescript-eslint/type-utils/node_modules/debug": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
"integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
"dev": true,
"dependencies": {
"ms": "^2.1.3"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/@typescript-eslint/type-utils/node_modules/minimatch": {
"version": "9.0.5",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz",
@ -2146,12 +2005,6 @@
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/@typescript-eslint/type-utils/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"dev": true
},
"node_modules/@typescript-eslint/type-utils/node_modules/semver": {
"version": "7.6.3",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz",
@ -2206,31 +2059,6 @@
}
}
},
"node_modules/@typescript-eslint/typescript-estree/node_modules/debug": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
"integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
"dev": true,
"license": "MIT",
"dependencies": {
"ms": "^2.1.3"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/@typescript-eslint/typescript-estree/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"dev": true,
"license": "MIT"
},
"node_modules/@typescript-eslint/typescript-estree/node_modules/semver": {
"version": "7.6.3",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz",
@ -2349,23 +2177,6 @@
"balanced-match": "^1.0.0"
}
},
"node_modules/@typescript-eslint/utils/node_modules/debug": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
"integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
"dev": true,
"dependencies": {
"ms": "^2.1.3"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/@typescript-eslint/utils/node_modules/minimatch": {
"version": "9.0.5",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz",
@ -2381,12 +2192,6 @@
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/@typescript-eslint/utils/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"dev": true
},
"node_modules/@typescript-eslint/utils/node_modules/semver": {
"version": "7.6.3",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz",
@ -2987,6 +2792,21 @@
"npm": "1.2.8000 || >= 1.4.16"
}
},
"node_modules/body-parser/node_modules/debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"license": "MIT",
"dependencies": {
"ms": "2.0.0"
}
},
"node_modules/body-parser/node_modules/ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT"
},
"node_modules/brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
@ -3354,6 +3174,15 @@
"node": ">= 0.8"
}
},
"node_modules/commander": {
"version": "9.5.0",
"resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz",
"integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==",
"license": "MIT",
"engines": {
"node": "^12.20.0 || >=14"
}
},
"node_modules/concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@ -3526,11 +3355,20 @@
}
},
"node_modules/debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
"integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
"license": "MIT",
"dependencies": {
"ms": "2.0.0"
"ms": "^2.1.3"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/dedent": {
@ -4108,24 +3946,6 @@
}
}
},
"node_modules/eslint-config-airbnb-base-typescript/node_modules/debug": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
"integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
"dev": true,
"license": "MIT",
"dependencies": {
"ms": "^2.1.3"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/eslint-config-airbnb-base-typescript/node_modules/doctrine": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz",
@ -4316,13 +4136,6 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/eslint-config-airbnb-base-typescript/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"dev": true,
"license": "MIT"
},
"node_modules/eslint-import-resolver-node": {
"version": "0.3.9",
"resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.9.tgz",
@ -4347,14 +4160,6 @@
"ms": "^2.1.1"
}
},
"node_modules/eslint-import-resolver-node/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"dev": true,
"license": "MIT",
"peer": true
},
"node_modules/eslint-module-utils": {
"version": "2.12.0",
"resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.12.0.tgz",
@ -4385,14 +4190,6 @@
"ms": "^2.1.1"
}
},
"node_modules/eslint-module-utils/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"dev": true,
"license": "MIT",
"peer": true
},
"node_modules/eslint-plugin-import": {
"version": "2.31.0",
"resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.31.0.tgz",
@ -4439,14 +4236,6 @@
"ms": "^2.1.1"
}
},
"node_modules/eslint-plugin-import/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"dev": true,
"license": "MIT",
"peer": true
},
"node_modules/eslint-scope": {
"version": "8.2.0",
"resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.2.0.tgz",
@ -4477,31 +4266,6 @@
"url": "https://opencollective.com/eslint"
}
},
"node_modules/eslint/node_modules/debug": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
"integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
"dev": true,
"license": "MIT",
"dependencies": {
"ms": "^2.1.3"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/eslint/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"dev": true,
"license": "MIT"
},
"node_modules/espree": {
"version": "10.3.0",
"resolved": "https://registry.npmjs.org/espree/-/espree-10.3.0.tgz",
@ -4683,6 +4447,21 @@
"url": "https://opencollective.com/express"
}
},
"node_modules/express/node_modules/debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"license": "MIT",
"dependencies": {
"ms": "2.0.0"
}
},
"node_modules/express/node_modules/ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT"
},
"node_modules/extend": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
@ -4841,6 +4620,21 @@
"node": ">= 0.8"
}
},
"node_modules/finalhandler/node_modules/debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"license": "MIT",
"dependencies": {
"ms": "2.0.0"
}
},
"node_modules/finalhandler/node_modules/ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT"
},
"node_modules/find-up": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz",
@ -6079,31 +5873,6 @@
"node": ">=10"
}
},
"node_modules/istanbul-lib-source-maps/node_modules/debug": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
"integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
"dev": true,
"license": "MIT",
"dependencies": {
"ms": "^2.1.3"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/istanbul-lib-source-maps/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"dev": true,
"license": "MIT"
},
"node_modules/istanbul-reports": {
"version": "3.1.7",
"resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.7.tgz",
@ -7249,9 +7018,10 @@
}
},
"node_modules/ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"license": "MIT"
},
"node_modules/napi-macros": {
"version": "2.2.2",
@ -8204,6 +7974,21 @@
"node": ">= 0.8.0"
}
},
"node_modules/send/node_modules/debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"license": "MIT",
"dependencies": {
"ms": "2.0.0"
}
},
"node_modules/send/node_modules/debug/node_modules/ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT"
},
"node_modules/send/node_modules/encodeurl": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
@ -8212,11 +7997,6 @@
"node": ">= 0.8"
}
},
"node_modules/send/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
},
"node_modules/serve-static": {
"version": "1.16.2",
"resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz",
@ -8298,6 +8078,22 @@
"node": ">=8"
}
},
"node_modules/showdown": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/showdown/-/showdown-2.1.0.tgz",
"integrity": "sha512-/6NVYu4U819R2pUIk79n67SYgJHWCce0a5xTP979WbNp0FL9MN1I1QK662IDU1b6JzKTvmhgI7T7JYIxBi3kMQ==",
"license": "MIT",
"dependencies": {
"commander": "^9.0.0"
},
"bin": {
"showdown": "bin/showdown.js"
},
"funding": {
"type": "individual",
"url": "https://www.paypal.me/tiviesantos"
}
},
"node_modules/side-channel": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz",

View File

@ -15,12 +15,15 @@
"license": "Unlicense",
"dependencies": {
"@types/bluebird": "^3.5.42",
"@types/debug": "^4.1.12",
"@types/json-logic-js": "^2.0.8",
"@types/object-hash": "^3.0.6",
"debug": "^4.4.0",
"express": "^4.21.2",
"json-logic-js": "^2.0.5",
"level": "^9.0.0",
"object-hash": "^3.0.0",
"showdown": "^2.1.0",
"zeromq": "^6.1.2"
},
"devDependencies": {
@ -28,6 +31,7 @@
"@types/express": "^5.0.0",
"@types/jest": "^29.5.14",
"@types/node": "^22.10.2",
"@types/showdown": "^2.0.6",
"eslint": "^9.17.0",
"eslint-config-airbnb-base-typescript": "^1.1.0",
"jest": "^29.7.0",

View File

@ -1,9 +1,8 @@
import EventEmitter from 'node:events';
import {REQUEST_BIND_HOST, REQUEST_BIND_PORT} from './config';
import {publishSock, subscribeSock} from './pub-sub';
import {Decision, Delta, PeerAddress} from './types';
import {myRequestAddr} from './peers';
import objectHash from 'object-hash';
import {myRequestAddr} from './peers';
import {publishSock, subscribeSock} from './pub-sub';
import {Decision, Delta, PeerAddress, Properties} from './types';
export const deltaStream = new EventEmitter();

View File

@ -1,22 +1,19 @@
// We can start to use deltas to express relational data in a given context
import express from "express";
import {Collection} from "./collection-layer";
import {HTTP_API_ENABLE, HTTP_API_ADDR, HTTP_API_PORT, SEED_PEERS} from "./config";
import {deltasAccepted, deltasProposed, runDeltas} from "./deltas";
import {Collection} from "./collection";
import {HTTP_API_ENABLE} from "./config";
import {runDeltas} from "./deltas";
import {runHttpApi} from "./http-api";
import {Entity} from "./object-layer";
import {askAllPeersForDeltas, peers, subscribeToSeeds} from "./peers";
import {askAllPeersForDeltas, subscribeToSeeds} from "./peers";
import {bindPublish, } from "./pub-sub";
import {bindReply, runRequestHandlers} from "./request-reply";
import {Delta, PeerAddress} from "./types";
import {TypedCollection} from "./typed-collection";
// As an app we want to be able to write and read data.
// The data is whatever shape we define it to be in a given context.
// So we want access to an API that is integrated with our declarations of
// e.g. entities and their properties.
// This implies at least one layer on top of the underlying primitive deltas.
type UserProperties = {
type User = {
id?: string;
name: string;
nameLong?: string;
@ -24,78 +21,15 @@ type UserProperties = {
age: number;
};
class Users {
db = new Collection();
create(properties: UserProperties): Entity {
// We provide undefined for the id, to let the database generate it
// This call returns the id
const user = this.db.put(undefined, properties);
console.log(`Users.create(${user.id}, ${JSON.stringify(properties)}`);
return user;
}
upsert(properties: UserProperties): Entity {
const user = this.db.put(properties.id, properties);
console.log(`Users.upsert(${user.id}, ${JSON.stringify(properties)}`);
return user;
}
getOne(id: string): Entity | undefined {
return this.db.get(id);
}
getIds(): string[] {
return this.db.getIds();
}
}
(async () => {
const users = new Users();
const app = express()
app.get("/ids", (req: express.Request, res: express.Response) => {
res.json({ids: users.getIds()});
});
app.get("/deltas", (req: express.Request, res: express.Response) => {
// TODO: streaming
res.json(deltasAccepted);
});
app.get("/deltas/count", (req: express.Request, res: express.Response) => {
res.json(deltasAccepted.length);
});
app.get("/peers", (req: express.Request, res: express.Response) => {
res.json(peers.map(({reqAddr, publishAddr, isSelf, isSeedPeer}) => {
const deltasAcceptedCount = deltasAccepted
.filter((delta: Delta) => {
return delta.receivedFrom?.addr == reqAddr.addr &&
delta.receivedFrom?.port == reqAddr.port;
})
.length;
const peerInfo = {
reqAddr: reqAddr.toAddrString(),
publishAddr: publishAddr?.toAddrString(),
isSelf,
isSeedPeer,
deltaCount: {
accepted: deltasAcceptedCount
}
};
return peerInfo;
}));
});
app.get("/peers/count", (req: express.Request, res: express.Response) => {
res.json(peers.length);
});
if (HTTP_API_ENABLE) {
app.listen(HTTP_API_PORT, HTTP_API_ADDR, () => {
console.log(`HTTP API bound to http://${HTTP_API_ADDR}:${HTTP_API_PORT}`);
});
}
const users = new TypedCollection<User>();
await bindPublish();
await bindReply();
if (HTTP_API_ENABLE) {
runHttpApi({users});
}
runDeltas();
runRequestHandlers();
await new Promise((resolve) => setTimeout(resolve, 500));
@ -104,26 +38,26 @@ class Users {
askAllPeersForDeltas();
await new Promise((resolve) => setTimeout(resolve, 1000));
const taliesin = users.upsert({
users.onUpdate((u: Entity) => {
console.log('User updated:', u);
});
users.onCreate((u: Entity) => {
console.log('New user!:', u);
});
const taliesin = users.put(undefined, {
// id: 'taliesin-1',
name: 'Taliesin',
nameLong: 'Taliesin (Ladd)',
age: Math.floor(Math.random() * 1000)
});
users.db.onUpdate((u: Entity) => {
console.log('User updated:', u);
});
users.db.onCreate((u: Entity) => {
console.log('New user!:', u);
});
// TODO: Allow configuration regarding read/write concern i.e.
// if we perform a read immediately do we see the value we wrote?
// Intuition says yes, we want that-- but how do we expose the propagation status?
const result = users.getOne(taliesin.id);
const result = users.get(taliesin.id);
const matches: boolean = JSON.stringify(result) === JSON.stringify(taliesin);
if (matches) {
console.log('Result matches expected: ' + JSON.stringify(taliesin));

116
src/http-api.ts Normal file
View File

@ -0,0 +1,116 @@
import express from "express";
import {HTTP_API_ADDR, HTTP_API_PORT} from "./config";
import {deltasAccepted} from "./deltas";
import {peers} from "./peers";
import {Delta} from "./types";
import {readdirSync, readFileSync} from "fs";
import Debug from "debug";
import {Collection} from "./collection";
import {Converter} from "showdown";
import path from "path";
const debug = Debug('http-api');
type CollectionsToServe = {
[key: string]: Collection;
};
const docConverter = new Converter({
simpleLineBreaks: true,
completeHTMLDocument: true
});
const htmlDocFromMarkdown = (md: string): string => docConverter.makeHtml(md);
export function runHttpApi(collections?: CollectionsToServe) {
const app = express();
app.use(express.json());
// Convert markdown to HTML and serve it
const mdFiles = readdirSync('./markdown/')
.filter((f) => f.endsWith('.md'))
.map((name) => path.parse(name).name);
debug('mdFiles:', mdFiles);
app.get('/html', (_req: express.Request, res: express.Response) => {
let md = `# Files\n\n`;
for (const name of mdFiles) {
md += `- [${name}](./${name})\n`;
}
const html = htmlDocFromMarkdown(md);
res.setHeader('content-type', 'text/html').send(html);
});
for (const name of mdFiles) {
const md = readFileSync(`./markdown/${name}.md`).toString();
const html = htmlDocFromMarkdown(md);
app.get(`/html/${name}`, (_req: express.Request, res: express.Response) => {
res.setHeader('content-type', 'text/html').send(html);
});
}
// Set up API routes
if (collections) {
for (const [name, collection] of Object.entries(collections)) {
debug(`collection: ${name}`);
app.get(`/${name}/ids`, (_req: express.Request, res: express.Response) => {
res.json({ids: collection.getIds()});
});
app.put(`/${name}`, (req: express.Request, res: express.Response) => {
const {body: properties} = req;
const ent = collection.put(undefined, properties);
res.json(ent);
});
app.put(`/${name}/:id`, (req: express.Request, res: express.Response) => {
const {body: properties, params: {id}} = req;
if (properties.id && properties.id !== id) {
res.status(400).json({error: "ID Mismatch", param: id, property: properties.id});
return;
}
const ent = collection.put(id, properties);
res.json(ent);
});
}
}
app.get("/deltas", (_req: express.Request, res: express.Response) => {
// TODO: streaming
res.json(deltasAccepted);
});
app.get("/deltas/count", (_req: express.Request, res: express.Response) => {
res.json(deltasAccepted.length);
});
app.get("/peers", (_req: express.Request, res: express.Response) => {
res.json(peers.map(({reqAddr, publishAddr, isSelf, isSeedPeer}) => {
const deltasAcceptedCount = deltasAccepted
.filter((delta: Delta) => {
return delta.receivedFrom?.addr == reqAddr.addr &&
delta.receivedFrom?.port == reqAddr.port;
})
.length;
const peerInfo = {
reqAddr: reqAddr.toAddrString(),
publishAddr: publishAddr?.toAddrString(),
isSelf,
isSeedPeer,
deltaCount: {
accepted: deltasAcceptedCount
}
};
return peerInfo;
}));
});
app.get("/peers/count", (_req: express.Request, res: express.Response) => {
res.json(peers.length);
});
app.listen(HTTP_API_PORT, HTTP_API_ADDR, () => {
debug(`HTTP API bound to http://${HTTP_API_ADDR}:${HTTP_API_PORT}`);
});
}

88
src/lossless.ts Normal file
View File

@ -0,0 +1,88 @@
// Deltas target entities.
// We can maintain a record of all the targeted entities, and the deltas that targeted them
import {Delta, PropertyTypes} from "./types";
type DomainEntityID = string;
type PropertyID = string;
export type LosslessView = {[key: string]: {[key: string]: Delta[]}};
export type CollapsedPointer = {[key: string]: PropertyTypes};
export type CollapsedDelta = Omit<Delta, 'pointers'> & {
pointers: CollapsedPointer[];
};
class DomainEntityMap extends Map<DomainEntityID, DomainEntity> {};
class DomainEntityProperty {
id: PropertyID;
deltas = new Set<Delta>();
constructor(id: PropertyID) {
this.id = id;
}
}
class DomainEntity {
id: DomainEntityID;
properties = new Map<PropertyID, DomainEntityProperty>();
constructor(id: DomainEntityID) {
this.id = id;
}
addDelta(delta: Delta) {
const targetContexts = delta.pointers
.filter(({target}) => target === this.id)
.map(({targetContext}) => targetContext)
.filter((targetContext) => typeof targetContext === 'string');
for (const targetContext of targetContexts) {
let property = this.properties.get(targetContext);
if (!property) {
property = new DomainEntityProperty(targetContext);
this.properties.set(targetContext, property);
}
property.deltas.add(delta);
}
}
}
export class Lossless {
domainEntities = new DomainEntityMap();
ingestDelta(delta: Delta) {
const targets = delta.pointers
.filter(({targetContext}) => !!targetContext)
.map(({target}) => target)
.filter((target) => typeof target === 'string')
for (const target of targets) {
let ent = this.domainEntities.get(target);
if (!ent) {
ent = new DomainEntity(target);
this.domainEntities.set(target, ent);
}
ent.addDelta(delta);
}
}
view() {
const view: {[key: DomainEntityID]: {[key: PropertyID]: CollapsedDelta[]}} = {};
for (const ent of this.domainEntities.values()) {
const obj: {[key: PropertyID]: CollapsedDelta[]} = {};
view[ent.id] = obj;
for (const prop of ent.properties.values()) {
obj[prop.id] = obj[prop.id] || [];
for (const delta of prop.deltas) {
const collapsedDelta: CollapsedDelta = {
...delta,
pointers: delta.pointers.map(({localContext, target}) => ({
[localContext]: target
}))
};
obj[prop.id].push(collapsedDelta);
}
}
}
return view;
}
}

View File

@ -11,16 +11,16 @@ import { CREATOR, HOST_ID } from "./config";
import { Delta, PropertyTypes } from "./types";
export type EntityProperties = {
[key: string]: PropertyTypes
[key: string]: PropertyTypes;
};
export class Entity {
id: string;
properties: EntityProperties;
properties: EntityProperties = {};
ahead = 0;
constructor(id: string) {
this.id = id;
this.properties = {};
}
}
@ -28,6 +28,7 @@ export class Entity {
export class EntityPropertiesDeltaBuilder {
delta: Delta;
constructor(entityId: string) {
this.delta = {
creator: CREATOR,
@ -39,6 +40,7 @@ export class EntityPropertiesDeltaBuilder {
}]
};
}
add(localContext: string, target: PropertyTypes) {
this.delta.pointers.push({localContext, target});
}

11
src/typed-collection.ts Normal file
View File

@ -0,0 +1,11 @@
import { Collection } from './collection';
import {Entity, EntityProperties} from './object-layer';
export class TypedCollection<T extends EntityProperties> extends Collection {
put(id: string | undefined, properties: T): Entity {
return super.put(id, properties);
}
get(id: string): Entity | undefined {
return super.get(id);
}
}

View File

@ -1,14 +1,16 @@
export type PointerTarget = string | number | undefined;
export type Pointer = {
localContext: string,
target: string | number | undefined,
targetContext?: string
localContext: string;
target: PointerTarget;
targetContext?: string;
};
export type Delta = {
creator: string,
host: string,
pointers: Pointer[],
receivedFrom?: PeerAddress,
creator: string;
host: string;
pointers: Pointer[];
receivedFrom?: PeerAddress;
}
export type DeltaContext = Delta & {
@ -29,7 +31,6 @@ export enum Decision {
Defer
};
export type JSONLogic = object;
export type FilterExpr = JSONLogic;
@ -38,6 +39,8 @@ export type FilterGenerator = () => FilterExpr;
export type PropertyTypes = string | number | undefined;
export type Properties = {[key: string]: PropertyTypes};
export class PeerAddress {
addr: string;
port: number;