refactored to use lossless/lossy view mechanisms for resolving get requests

.gitignore

@@ -1,4 +1,5 @@
 dist/
 node_modules/
+coverage/
 *.swp
 *.swo

README.md

@@ -16,6 +16,110 @@
 
 If we express views and filter rules as JSON-Logic, we can easily include them in records.
 
+# Development / Demo
+
+## Setup
+
+Install [`nvm`](https://nvm.sh)
+
+Clone repo
+```bash
+git clone https://gitea.dgov.io/ladd/rhizome
+```
+
+Use `nvm` to install and activate the target nodejs version
+```bash
+nvm install
+```
+
+Install nodejs packages
+```bash
+npm install
+```
+
+## Build
+
+Compile Typescript
+```bash
+npm run build
+```
+
+During development, it's useful to run the compiler in watch mode:
+```bash
+npm run build:watch
+```
+
+## Run tests
+
+```bash
+npm run test
+```
+
+## Run test coverage report
+
+```bash
+npm run test:coverage
+```
+
+## Run multiple live nodes locally as separate processes
+To demonstrate the example application, you can open multiple terminals, and in each terminal execute something like the following:
+
+```bash
+export DEBUG="*,-express"
+export RHIZOME_REQUEST_BIND_PORT=4000
+export RHIZOME_PUBLISH_BIND_PORT=4001
+export RHIZOME_SEED_PEERS='localhost:4002, localhost:4004'
+export RHIZOME_HTTP_API_PORT=3000
+export RHIZOME_PEER_ID=peer1
+npm run example-app
+```
+
+```bash
+export DEBUG="*,-express"
+export RHIZOME_REQUEST_BIND_PORT=4002
+export RHIZOME_PUBLISH_BIND_PORT=4003
+export RHIZOME_SEED_PEERS='localhost:4000, localhost:4004'
+export RHIZOME_HTTP_API_PORT=3001
+export RHIZOME_PEER_ID=peer2
+npm run example-app
+```
+
+```bash
+export DEBUG="*,-express"
+export RHIZOME_REQUEST_BIND_PORT=4004
+export RHIZOME_PUBLISH_BIND_PORT=4005
+export RHIZOME_SEED_PEERS='localhost:4000, localhost:4002'
+export RHIZOME_HTTP_API_PORT=3002
+export RHIZOME_PEER_ID=peer3
+npm run example-app
+```
+
+In a separate terminal, you can use `curl` to interact with an instance.
+
+`jq` is helpful for formatting the json responses.
+
+Query the number of peers seen by a given node (including itself)
+```bash
+curl -s  http://localhost:3000/peers/count | jq
+```
+
+Query the list of peers seen by a given node (including itself)
+```bash
+curl -s  http://localhost:3000/peers | jq
+```
+
+Query the number of deltas ingested by this node
+```bash
+curl -s  http://localhost:3000/deltas/count | jq
+```
+
+Query the list of deltas ingested by this node
+```bash
+curl -s  http://localhost:3000/deltas | jq
+```
+
+# More About Concepts
+
 ## Clocks?
 
 Do we want to involve a time synchronization protocol? e.g. ntpd
@@ -78,94 +182,3 @@ Considerations imposed by Tinc would include
 * IP addressing
 * public key management
 
-# Development / Demo
-
-## Setup
-
-Install [`nvm`](https://nvm.sh)
-
-Clone repo
-```bash
-git clone https://gitea.dgov.io/ladd/rhizome
-```
-
-Use `nvm` to install and activate the target nodejs version
-```bash
-nvm install
-```
-
-Install nodejs packages
-```bash
-npm install
-```
-
-## Build
-
-Compile Typescript
-```bash
-npm run build
-```
-
-During development, it's useful to run the compiler in watch mode:
-```bash
-npm run build:watch
-```
-
-## Run
-
-To demonstrate the example application, you can open multiple terminals, and in each terminal execute something like the following:
-
-```bash
-export DEBUG="*,-express"
-export RHIZOME_REQUEST_BIND_PORT=4000
-export RHIZOME_PUBLISH_BIND_PORT=4001
-export RHIZOME_SEED_PEERS='localhost:4002, localhost:4004'
-export RHIZOME_HTTP_API_PORT=3000
-export RHIZOME_PEER_ID=peer1
-npm run example-app
-```
-
-```bash
-export DEBUG="*,-express"
-export RHIZOME_REQUEST_BIND_PORT=4002
-export RHIZOME_PUBLISH_BIND_PORT=4003
-export RHIZOME_SEED_PEERS='localhost:4000, localhost:4004'
-export RHIZOME_HTTP_API_PORT=3001
-export RHIZOME_PEER_ID=peer2
-npm run example-app
-```
-
-```bash
-export DEBUG="*,-express"
-export RHIZOME_REQUEST_BIND_PORT=4004
-export RHIZOME_PUBLISH_BIND_PORT=4005
-export RHIZOME_SEED_PEERS='localhost:4000, localhost:4002'
-export RHIZOME_HTTP_API_PORT=3002
-export RHIZOME_PEER_ID=peer3
-npm run example-app
-```
-
-In a separate terminal, you can use `curl` to interact with an instance.
-
-`jq` is helpful for formatting the json responses.
-
-Query the number of peers seen by a given node (including itself)
-```bash
-curl -s  http://localhost:3000/peers/count | jq
-```
-
-Query the list of peers seen by a given node (including itself)
-```bash
-curl -s  http://localhost:3000/peers | jq
-```
-
-Query the number of deltas ingested by this node
-```bash
-curl -s  http://localhost:3000/deltas/count | jq
-```
-
-Query the list of deltas ingested by this node
-```bash
-curl -s  http://localhost:3000/deltas | jq
-```
-

__tests__/lossless.ts

@@ -135,7 +135,22 @@ describe('Lossless', () => {
         return creator === 'A' && host === 'H';
       };
 
-      expect(lossless.view(filter)).toEqual({
+      expect(lossless.view(undefined, filter)).toEqual({
+        ace: {
+          referencedAs: ["1"],
+          properties: {
+            value: [{
+              creator: 'A',
+              host: 'H',
+              pointers: [
+                {"1": "ace"},
+              ]
+            }]
+          }
+        }
+      });
+
+      expect(lossless.view(["ace"], filter)).toEqual({
         ace: {
           referencedAs: ["1"],
           properties: {

__tests__/lossy.ts

@@ -1,7 +1,6 @@
-import Debug from "debug";
 import {Lossless, LosslessViewMany} from "../src/lossless";
 import {Lossy, firstValueFromLosslessViewOne, valueFromCollapsedDelta} from "../src/lossy";
-const debug = Debug('test:lossy');
+import {PointerTarget} from "../src/types";
 
 describe('Lossy', () => {
   describe('se a provided function to resolve entity views', () => {
@@ -36,9 +35,9 @@ describe('Lossy', () => {
 
     it('example summary', () => {
       type Role = {
-        actor: string,
+        actor: PointerTarget,
-        film: string,
+        film: PointerTarget,
-        role: string
+        role: PointerTarget
       };
 
       type Summary = {
@@ -47,14 +46,12 @@ describe('Lossy', () => {
 
       const resolver = (losslessView: LosslessViewMany): Summary => {
         const roles: Role[] = [];
-        debug('resolving roles');
         for (const [id, ent] of Object.entries(losslessView)) {
           if (ent.referencedAs.includes("role")) {
             const {delta, value: actor} = firstValueFromLosslessViewOne(ent, "actor") ?? {};
             if (!delta) continue; // TODO: panic
             if (!actor) continue; // TODO: panic
             const film = valueFromCollapsedDelta(delta, "film");
-            debug(`role ${id}`, {actor, film});
             if (!film) continue; // TODO: panic
             roles.push({
               role: id,

__tests__/run/001-single-node.ts

@@ -17,9 +17,8 @@ describe('Run', () => {
     await app.stop();
   });
 
-  it('can put a new user', async () => {
+  it('can put a new user and fetch it', async () => {
-    const {httpAddr, httpPort} = app.config;
+    const res = await fetch(`${app.apiUrl}/user`, {
-    const res = await fetch(`http://${httpAddr}:${httpPort}/users`, {
       method: 'PUT',
       headers: {'Content-Type': 'application/json'},
       body: JSON.stringify({
@@ -38,5 +37,18 @@ describe('Run', () => {
         age: 263
       }
     });
+
+    await new Promise((resolve) => setTimeout(resolve, 100));
+
+    const res2 = await fetch(`${app.apiUrl}/user/peon-1`);
+    const data2 = await res2.json();
+    expect(data2).toMatchObject({
+      id: "peon-1",
+      properties: {
+        name: "Peon",
+        age: 263
+      }
+    });
+
   });
 });

__tests__/run/002-two-nodes.ts

@@ -28,7 +28,7 @@ describe('Run', () => {
     debug('apps[0].apiUrl', apps[0].apiUrl);
     debug('apps[1].apiUrl', apps[1].apiUrl);
 
-    const res = await fetch(`${apps[0].apiUrl}/users`, {
+    const res = await fetch(`${apps[0].apiUrl}/user`, {
       method: 'PUT',
       headers: {'Content-Type': 'application/json'},
       body: JSON.stringify({
@@ -50,7 +50,7 @@ describe('Run', () => {
 
     await new Promise((resolve) => setTimeout(resolve, 100));
 
-    const res2 = await fetch(`${apps[1].apiUrl}/users/peon-1`);
+    const res2 = await fetch(`${apps[1].apiUrl}/user/peon-1`);
     const data2 = await res2.json();
     debug('data2', data2);
     expect(data2).toMatchObject({

markdown/coverage_report.md

@@ -0,0 +1,38 @@
+
+> rhizome-node@1.0.0 test
+> jest --coverage
+
+PASS __tests__/lossy.ts
+PASS __tests__/lossless.ts
+PASS __tests__/peer-address.ts
+PASS __tests__/run/002-two-nodes.ts
+PASS __tests__/run/001-single-node.ts
+----------------------|---------|----------|---------|---------|----------------------------------------------------
+File                  | % Stmts | % Branch | % Funcs | % Lines | Uncovered Line #s                                  
+----------------------|---------|----------|---------|---------|----------------------------------------------------
+All files             |   86.86 |    62.41 |    82.7 |   87.38 |                                                    
+ src                  |   87.94 |    68.06 |   82.45 |   88.09 |                                                    
+  collection.ts       |   88.31 |    71.42 |   66.66 |   90.54 | 62-65,114-122,172                                  
+  config.ts           |   94.44 |    89.65 |      50 |   94.44 | 22                                                 
+  deltas.ts           |   64.44 |       50 |   76.92 |   64.44 | 27-30,42-46,55-56,64-73                            
+  entity.ts           |     100 |      100 |     100 |     100 |                                                    
+  http-api.ts         |    59.7 |    13.04 |   38.88 |    59.7 | 32,37,44-60,66,79-80,85-92,100,121,129-130,145-151 
+  lossless.ts         |   98.27 |    91.66 |     100 |     100 | 96                                                 
+  lossy.ts            |     100 |    85.71 |     100 |     100 | 38                                                 
+  node.ts             |     100 |      100 |     100 |     100 |                                                    
+  peers.ts            |   96.82 |      100 |     100 |   96.61 | 125-126                                            
+  pub-sub.ts          |     100 |      100 |     100 |     100 |                                                    
+  request-reply.ts    |   95.65 |        0 |     100 |   95.34 | 46,59                                              
+  typed-collection.ts |     100 |      100 |     100 |     100 |                                                    
+  types.ts            |     100 |      100 |     100 |     100 |                                                    
+ src/util             |   74.54 |    31.81 |   82.35 |      78 |                                                    
+  md-files.ts         |   74.54 |    31.81 |   82.35 |      78 | 52-56,90-94,108-115                                
+ util                 |     100 |      100 |     100 |     100 |                                                    
+  app.ts              |     100 |      100 |     100 |     100 |                                                    
+----------------------|---------|----------|---------|---------|----------------------------------------------------
+
+Test Suites: 5 passed, 5 total
+Tests:       7 passed, 7 total
+Snapshots:   0 total
+Time:        3.709 s, estimated 5 s
+Ran all test suites.

package.json

@@ -7,7 +7,8 @@
     "build": "tsc",
     "build:watch": "tsc --watch",
     "lint": "eslint",
-    "test": "jest"
+    "test": "jest",
+    "test:coverage": "./scripts/coverage.sh"
   },
   "jest": {
     "testEnvironment": "node",

scripts/coverage.sh

@@ -0,0 +1,16 @@
+#!/bin/env bash
+
+force=false
+
+while [[ -n "$1" ]]; do
+  case "$1" in
+    -f | --force)
+      force=true
+      ;;
+  esac
+  shift
+done
+
+dest="./markdown/coverage_report.md"
+
+npm run test -- --coverage 2>&1 | tee "$dest"

src/collection.ts

@@ -3,44 +3,45 @@
 // It should enable operations like removing a property removes the value from the entities in the collection
 // It could then be further extended with e.g. table semantics like filter, sort, join
 
+import Debug from 'debug';
 import {randomUUID} from "node:crypto";
 import EventEmitter from "node:events";
+import {Entity} from "./entity";
+import {Lossless, LosslessViewMany} from "./lossless";
+import {firstValueFromLosslessViewOne, Lossy, LossyViewMany, LossyViewOne} from "./lossy";
 import {RhizomeNode} from "./node";
-import {Entity, EntityProperties, EntityPropertiesDeltaBuilder} from "./object-layer";
 import {Delta} from "./types";
-
+const debug = Debug('collection');
-// type Property = {
-//   name: string,
-//   type: number | string;
-// }
-
-// class EntityType {
-//   name: string;
-//   properties?: Property[];
-//   constructor(name: string) {
-//     this.name = name;
-//   }
-// }
 
 export class Collection {
   rhizomeNode?: RhizomeNode;
   name: string;
   entities = new Map<string, Entity>();
   eventStream = new EventEmitter();
+  lossless = new Lossless(); // TODO: Really just need one global Lossless instance
 
   constructor(name: string) {
     this.name = name;
   }
 
+  // Instead of trying to update our final view of the entity with every incoming delta,
+  // let's try this: 
+  // - keep a lossless view (of everything)
+  // - build a lossy view when needed
+  // This approach is simplistic, but can then be optimized and enhanced.
+
   rhizomeConnect(rhizomeNode: RhizomeNode) {
     this.rhizomeNode = rhizomeNode;
 
     rhizomeNode.deltaStream.subscribeDeltas((delta: Delta) => {
       // TODO: Make sure this is the kind of delta we're looking for
-      this.applyDelta(delta);
+      debug(`collection ${this.name} received delta ${JSON.stringify(delta)}`);
+      this.lossless.ingestDelta(delta);
     });
 
     rhizomeNode.httpApi.serveCollection(this);
+
+    debug(`connected ${this.name} to rhizome`);
   }
 
   // Applies the javascript rules for updating object values,
@@ -55,7 +56,6 @@ export class Collection {
       entity.id = entityId;
       eventType = 'create';
     }
-    const deltaBulider = new EntityPropertiesDeltaBuilder(this.rhizomeNode!, entityId);
 
     if (!properties) {
       // Let's interpret this as entity deletion
@@ -74,69 +74,50 @@ export class Collection {
         }
         if (local && changed) {
           // If this is a change, let's generate a delta
-          deltaBulider.add(key, value);
+          if (!this.rhizomeNode) throw new Error(`${this.name} collection not connected to rhizome`);
+          const delta: Delta = {
+            creator: this.rhizomeNode.config.creator,
+            host: this.rhizomeNode.config.peerId,
+            pointers: [{
+              localContext: this.name,
+              target: entityId,
+              targetContext: key
+            }, {
+              localContext: key,
+              target: value
+            }]
+          };
+          deltas?.push(delta);
+
           // We append to the array the caller may provide
           // We can update this count as we receive network confirmation for deltas
           entity.ahead += 1;
         }
         anyChanged = anyChanged || changed;
       });
-      // We've noted that we may be ahead of the server, let's update our
-      // local image of this entity.
-      //* In principle, this system can recreate past or alternative states.
-      //* At worst, by replaying all the deltas up to a particular point.
-      //* Some sort of checkpointing strategy would probably be helpful.
-      //* Furthermore, if we can implement reversible transformations,
-      //* it would then be efficient to calculate the state of the system with 
-      //* specific deltas removed. We could use it to extract a measurement
-      //* of the effects of some deltas' inclusion or exclusion, the
-      //* evaluation of which may lend evidence to some possible arguments.
 
       this.entities.set(entityId, entity);
+
       if (anyChanged) {
-        deltas?.push(deltaBulider.delta);
         eventType = eventType || 'update';
       }
     }
     if (eventType) {
+      // TODO: Reconcile this with lossy view approach
       this.eventStream.emit(eventType, entity);
     }
     return entity;
   }
-  // We can update our local image of the entity, but we should annotate it
-  // to indicate that we have not yet received any confirmation of this delta
-  // having been propagated.
-  // Later when we receive deltas regarding this entity we can detect when
-  // we have received back an image that matches our target.
-
-  // So we need a function to generate one or more deltas for each call to put/
-  // maybe we stage them and wait for a call to commit() that initiates the
-  // assembly and transmission of one or more deltas
-
-  applyDelta(delta: Delta) {
-    // TODO: handle delta representing entity deletion
-    const idPtr = delta.pointers.find(({localContext}) => localContext === 'id');
-    if (!idPtr) {
-      console.error('encountered delta with no entity id', delta);
-      return;
-    }
-    const properties: EntityProperties = {};
-    delta.pointers.filter(({localContext}) => localContext !== 'id')
-      .forEach(({localContext: key, target: value}) => {
-        properties[key] = value;
-      }, {});
-    const entityId = idPtr.target as string;
-    // TODO: Handle the scenario where this update has been superceded by a newer one locally
-    this.updateEntity(entityId, properties);
-  }
 
   onCreate(cb: (entity: Entity) => void) {
+    // TODO: Reconcile this with lossy view approach
     this.eventStream.on('create', (entity: Entity) => {
       cb(entity);
     });
   }
 
   onUpdate(cb: (entity: Entity) => void) {
+    // TODO: Reconcile this with lossy view approach
     this.eventStream.on('update', (entity: Entity) => {
       cb(entity);
     });
@@ -145,25 +126,46 @@ export class Collection {
   put(entityId: string | undefined, properties: object): Entity {
     const deltas: Delta[] = [];
     const entity = this.updateEntity(entityId, properties, true, deltas);
+
+    debug(`put ${entityId} generated deltas:`, JSON.stringify(deltas));
+
+    // updateEntity may have generated some deltas for us to store and publish
     deltas.forEach(async (delta: Delta) => {
+
+      // record this delta just as if we had received it from a peer
       delta.receivedFrom = this.rhizomeNode!.myRequestAddr;
       this.rhizomeNode!.deltaStream.deltasAccepted.push(delta);
+
+      // publish the delta to our subscribed peers
       await this.rhizomeNode!.deltaStream.publishDelta(delta);
+      debug(`published delta ${JSON.stringify(delta)}`);
+
+      // ingest the delta as though we had received it from a peer
+      this.lossless.ingestDelta(delta);
     });
     return entity;
   }
 
-  del(entityId: string) {
+  get(id: string): LossyViewOne | undefined {
-    const deltas: Delta[] = [];
+    // Now with lossy view approach, instead of just returning what we already have,
-    this.updateEntity(entityId, undefined, true, deltas);
+    // let's compute our view now.
-    deltas.forEach(async (delta: Delta) => {
+    // return this.entities.get(id);
-      this.rhizomeNode!.deltaStream.deltasAccepted.push(delta);
+    const lossy = new Lossy(this.lossless);
-      await this.rhizomeNode!.deltaStream.publishDelta(delta);
+    const resolver = (losslessView: LosslessViewMany) => {
-    });
+      const lossyView: LossyViewMany = {};
+      debug('lossless view', JSON.stringify(losslessView));
+      for (const [id, ent] of Object.entries(losslessView)) {
+        lossyView[id] = {id, properties: {}};
+        for (const key of Object.keys(ent.properties)) {
+          const {value} = firstValueFromLosslessViewOne(ent, key) || {};
+          debug(`[ ${key} ] = ${value}`);
+          lossyView[id].properties[key] = value;
         }
-
+      }
-  get(id: string): Entity | undefined {
+      return lossyView;
-    return this.entities.get(id);
+    };
+    const res = lossy.resolve(resolver, [id]) as LossyViewMany;;
+    return res[id];
   }
 
   getIds(): string[] {

src/context.ts

@@ -0,0 +1,24 @@
+// A delta represents an assertion from a given (perspective/POV/context).
+// So we want it to be fluent to express these in the local context,
+// and propagated as deltas in a configurable manner; i.e. configurable batches or immediate
+
+// import {Delta} from './types';
+
+export class Entity {
+}
+
+export class Context {
+}
+
+export class Assertion {
+}
+
+export class View {
+}
+
+export class User {
+  
+}
+
+export function example() {
+}

src/entity.ts

@@ -0,0 +1,25 @@
+// The goal here is to provide a translation for
+// entities and their properties
+// to and from (sequences of) deltas.
+
+// How can our caller define the entities and their properties?
+// - As typescript types?
+// - As typescript interfaces?
+// - As typescript classes?
+
+import {PropertyTypes} from "./types";
+
+export type EntityProperties = {
+  [key: string]: PropertyTypes;
+};
+
+export class Entity {
+  id: string;
+  properties: EntityProperties = {};
+  ahead = 0;
+
+  constructor(id: string) {
+    this.id = id;
+  }
+}
+

src/example-app.ts

@@ -1,6 +1,6 @@
 import Debug from 'debug';
 import {RhizomeNode} from "./node";
-import {Entity} from "./object-layer";
+import {Entity} from "./entity";
 import {TypedCollection} from "./typed-collection";
 const debug = Debug('example-app');
 

src/http-api.ts

@@ -24,42 +24,7 @@ export class HttpApi {
   }
 
   start() {
-    // Scan and watch for markdown files
+    // --------------- deltas ----------------
-    this.mdFiles.readDir();
-    this.mdFiles.readReadme();
-    this.mdFiles.watchDir();
-    this.mdFiles.watchReadme();
-
-    // Serve README
-    this.router.get('/html/README', (_req: express.Request, res: express.Response) => {
-      const html = this.mdFiles.getReadmeHTML();
-      res.setHeader('content-type', 'text/html').send(html);
-    });
-
-    // Serve markdown files as html
-    this.router.get('/html/:name', (req: express.Request, res: express.Response) => {
-      let html = this.mdFiles.getHtml(req.params.name);
-      if (!html) {
-        res.status(404);
-        html = htmlDocFromMarkdown('# 404\n\n## [Index](/html)');
-      }
-      res.setHeader('content-type', 'text/html');
-      res.send(html);
-    });
-
-    // Serve index
-    {
-      let md = `# Files\n\n`;
-      md += `[README](/html/README)\n\n`;
-      for (const name of this.mdFiles.list()) {
-        md += `- [${name}](./${name})\n`;
-      }
-      const html = htmlDocFromMarkdown(md);
-
-      this.router.get('/html', (_req: express.Request, res: express.Response) => {
-        res.setHeader('content-type', 'text/html').send(html);
-      });
-    }
 
     // Serve list of all deltas accepted
     // TODO: This won't scale well
@@ -72,6 +37,8 @@ export class HttpApi {
       res.json(this.rhizomeNode.deltaStream.deltasAccepted.length);
     });
 
+    // --------------- peers ----------------
+
     // Get the list of peers seen by this node (including itself)
     this.router.get("/peers", (_req: express.Request, res: express.Response) => {
       res.json(this.rhizomeNode.peers.peers.map(({reqAddr, publishAddr, isSelf, isSeedPeer}) => {
@@ -99,6 +66,43 @@ export class HttpApi {
       res.json(this.rhizomeNode.peers.peers.length);
     });
 
+    // ----------------- html ---------------------
+
+    // Scan and watch for markdown files
+    this.mdFiles.readDir();
+    this.mdFiles.readReadme();
+    this.mdFiles.watchDir();
+    this.mdFiles.watchReadme();
+
+    // Serve README
+    this.router.get('/html/README', (_req: express.Request, res: express.Response) => {
+      const html = this.mdFiles.getReadmeHTML();
+      res.setHeader('content-type', 'text/html').send(html);
+    });
+
+    // Serve markdown files as html
+    this.router.get('/html/:name', (req: express.Request, res: express.Response) => {
+      const {name} = req.params;
+      let html = this.mdFiles.getHtml(name);
+      if (!html) {
+        res.status(404);
+        html = htmlDocFromMarkdown(`# 404 Not Found: ${name}\n\n ## [Index](/html)`);
+      }
+      res.setHeader('content-type', 'text/html');
+      res.send(html);
+    });
+
+    // Serve index
+    {
+      const html = this.mdFiles.generateIndex();
+
+      this.router.get('/html', (_req: express.Request, res: express.Response) => {
+        res.setHeader('content-type', 'text/html').send(html);
+      });
+    }
+
+    // ------------------- server ---------------------
+
     const {httpAddr, httpPort} = this.rhizomeNode.config;
     this.server = this.app.listen({
       port: httpPort,

src/lossless.ts

@@ -1,39 +1,32 @@
 // Deltas target entities.
 // We can maintain a record of all the targeted entities, and the deltas that targeted them
 
-import {Delta, DeltaFilter, PropertyTypes} from "./types";
+import Debug from 'debug';
-
+import {Delta, DeltaFilter, DomainEntityID, Properties, PropertyID, PropertyTypes} from "./types";
-type DomainEntityID = string;
+const debug = Debug('lossless');
-type PropertyID = string;
 
 export type CollapsedPointer = {[key: string]: PropertyTypes};
+
 export type CollapsedDelta = Omit<Delta, 'pointers'> & {
   pointers: CollapsedPointer[];
 };
+
 export type LosslessViewOne = {
   referencedAs: string[];
   properties: {
     [key: PropertyID]: CollapsedDelta[]
   }
 };
+
 export type LosslessViewMany = {
   [key: DomainEntityID]: LosslessViewOne;
 };
 
 class DomainEntityMap extends Map<DomainEntityID, DomainEntity> {};
 
-class DomainEntityProperty {
-  id: PropertyID;
-  deltas = new Set<Delta>();
-
-  constructor(id: PropertyID) {
-    this.id = id;
-  }
-}
-
 class DomainEntity {
   id: DomainEntityID;
-  properties = new Map<PropertyID, DomainEntityProperty>();
+  properties = new Map<PropertyID, Set<Delta>>();
 
   constructor(id: DomainEntityID) {
     this.id = id;
@@ -44,15 +37,29 @@ class DomainEntity {
       .filter(({target}) => target === this.id)
       .map(({targetContext}) => targetContext)
       .filter((targetContext) => typeof targetContext === 'string');
+
     for (const targetContext of targetContexts) {
-      let property = this.properties.get(targetContext);
+      let propertyDeltas = this.properties.get(targetContext);
-      if (!property) {
+      if (!propertyDeltas) {
-        property = new DomainEntityProperty(targetContext);
+        propertyDeltas = new Set<Delta>();
-        this.properties.set(targetContext, property);
+        this.properties.set(targetContext, propertyDeltas);
       }
-      property.deltas.add(delta);
+
+      debug(`adding delta for entity ${this.id}`);
+      propertyDeltas.add(delta);
     }
   }
+
+  toJSON() {
+    const properties: {[key: PropertyID]: number} = {};
+    for (const [key, deltas] of this.properties.entries()) {
+      properties[key] = deltas.size;
+    }
+    return {
+      id: this.id,
+      properties
+    };
+  }
 }
 
 export class Lossless {
@@ -63,47 +70,70 @@ export class Lossless {
       .filter(({targetContext}) => !!targetContext)
       .map(({target}) => target)
       .filter((target) => typeof target === 'string')
+
     for (const target of targets) {
       let ent = this.domainEntities.get(target);
+
       if (!ent) {
         ent = new DomainEntity(target);
         this.domainEntities.set(target, ent);
       }
+
+      debug('before add, domain entity:', JSON.stringify(ent));
+
       ent.addDelta(delta);
+
+      debug('after add, domain entity:', JSON.stringify(ent));
     }
   }
 
   //TODO: json logic -- view(deltaFilter?: FilterExpr) {
-  view(deltaFilter?: DeltaFilter): LosslessViewMany {
+  view(entityIds?: DomainEntityID[], deltaFilter?: DeltaFilter): LosslessViewMany {
     const view: LosslessViewMany = {};
-    for (const ent of this.domainEntities.values()) {
+    entityIds = entityIds ?? Array.from(this.domainEntities.keys());
+    for (const id of entityIds) {
+      const ent = this.domainEntities.get(id);
+      if (!ent) continue;
+
+      debug(`domain entity ${id}`, JSON.stringify(ent));
+
       const referencedAs = new Set<string>();
-      view[ent.id] = {
+      const properties: {
-        referencedAs: [],
+        [key: PropertyID]: CollapsedDelta[]
-        properties: {}
+      } = {};
-      };
+
-      for (const prop of ent.properties.values()) {
+      for (const [key, deltas] of ent.properties.entries()) {
-        view[ent.id].properties[prop.id] = view[ent.id].properties[prop.id] || [];
+        properties[key] = properties[key] || [];
-        for (const delta of prop.deltas) {
+
+        for (const delta of deltas) {
+
           if (deltaFilter) {
             const include = deltaFilter(delta);
             if (!include) continue;
           }
+
           const pointers: CollapsedPointer[] = [];
+
           for (const {localContext, target} of delta.pointers) {
             pointers.push({[localContext]: target});
             if (target === ent.id) {
               referencedAs.add(localContext);
             }
           }
+
           const collapsedDelta: CollapsedDelta = {
             ...delta,
             pointers
           };
-          view[ent.id].referencedAs = Array.from(referencedAs.values());
+
-          view[ent.id].properties[prop.id].push(collapsedDelta);
+          properties[key].push(collapsedDelta);
         }
       }
+
+      view[ent.id] = {
+        referencedAs: Array.from(referencedAs.values()),
+        properties
+      };
     }
     return view;
   }

src/lossy.ts

@@ -5,24 +5,36 @@
 // We can achieve this via functional expression, encoded as JSON-Logic.
 // Fields in the output can be described as transformations
 
+import Debug from 'debug';
 import {CollapsedDelta, Lossless, LosslessViewMany, LosslessViewOne} from "./lossless";
-import {DeltaFilter} from "./types";
+import {DeltaFilter, DomainEntityID, Properties} from "./types";
+const debug = Debug('lossy');
 
-type Resolver = (losslessView: LosslessViewMany) => unknown;
+export type LossyViewOne = {
+  id: DomainEntityID;
+  properties: Properties;
+};
+
+export type LossyViewMany = {
+  [key: DomainEntityID]: LossyViewOne;
+};
+
+type Resolver = (losslessView: LosslessViewMany) => LossyViewMany | unknown;
 
 // Extract a particular value from a delta's pointers
-export function valueFromCollapsedDelta(delta: CollapsedDelta, key: string): string | undefined {
+export function valueFromCollapsedDelta(delta: CollapsedDelta, key: string): string | number | undefined {
-  const pointers = delta.pointers;
+  for (const pointer of delta.pointers) {
-  for (const pointer of pointers || []) {
+    for (const [k, value] of Object.entries(pointer)) {
-    const [[k, value]] = Object.entries(pointer);
+      if (k === key && (typeof value === "string" || typeof value === "number")) {
-    if (k === key && typeof value === "string") {
         return value;
       }
     }
+  }
 }
 
 // Example function for resolving a value for an entity by taking the first value we find
-export function firstValueFromLosslessViewOne(ent: LosslessViewOne, key: string): {delta: CollapsedDelta, value: string} | undefined {
+export function firstValueFromLosslessViewOne(ent: LosslessViewOne, key: string): {delta: CollapsedDelta, value: string | number} | undefined {
+  debug(`trying to get value for ${key} from ${JSON.stringify(ent.properties[key])}`);
   for (const delta of ent.properties[key] || []) {
     const value = valueFromCollapsedDelta(delta, key);
     if (value) return {delta, value};
@@ -36,8 +48,8 @@ export class Lossy {
     this.lossless = lossless;
   }
 
-  resolve(fn: Resolver, deltaFilter?: DeltaFilter) {
+  resolve(fn: Resolver, entityIds?: DomainEntityID[], deltaFilter?: DeltaFilter) {
-    return fn(this.lossless.view(deltaFilter));
+    return fn(this.lossless.view(entityIds, deltaFilter));
   }
 }
 

src/node.ts

@@ -67,15 +67,28 @@ export class RhizomeNode {
   }
 
   async start() {
+    // Start ZeroMQ publish and reply sockets
     this.pubSub.start();
     this.requestReply.start();
+
+    // Start HTTP server
     if (this.config.httpEnable) {
       this.httpApi.start();
     }
+
+    // Wait a short time for sockets to initialize
     await new Promise((resolve) => setTimeout(resolve, 500));
+
+    // Subscribe to seed peers
     this.peers.subscribeToSeeds();
+
+    // Wait a short time for sockets to initialize
     await new Promise((resolve) => setTimeout(resolve, 500));
+
+    // Ask all peers for all deltas
     this.peers.askAllPeersForDeltas();
+
+    // Wait to receive all deltas
     await new Promise((resolve) => setTimeout(resolve, 1000));
   }
 

src/object-layer.ts

@@ -1,48 +0,0 @@
-// The goal here is to provide a translation for
-// entities and their properties
-// to and from (sequences of) deltas.
-
-// How can our caller define the entities and their properties?
-// - As typescript types?
-// - As typescript interfaces?
-// - As typescript classes?
-
-import {RhizomeNode} from "./node";
-import {Delta, PropertyTypes} from "./types";
-
-export type EntityProperties = {
-  [key: string]: PropertyTypes;
-};
-
-export class Entity {
-  id: string;
-  properties: EntityProperties = {};
-  ahead = 0;
-
-  constructor(id: string) {
-    this.id = id;
-  }
-}
-
-// TODO: Use leveldb for storing view snapshots
-
-export class EntityPropertiesDeltaBuilder {
-  delta: Delta;
-
-  constructor(rhizomeNode: RhizomeNode, entityId: string) {
-    this.delta = {
-      creator: rhizomeNode.config.creator,
-      host: rhizomeNode.config.peerId,
-      pointers: [{
-        localContext: 'id',
-        target: entityId,
-        targetContext: 'properties'
-      }]
-    };
-  }
-
-  add(localContext: string, target: PropertyTypes) {
-    this.delta.pointers.push({localContext, target});
-  }
-}
-

src/typed-collection.ts

@@ -1,12 +1,13 @@
 import { Collection } from './collection';
-import {Entity, EntityProperties} from './object-layer';
+import {Entity, EntityProperties} from './entity';
+import {LossyViewOne} from './lossy';
 
 export class TypedCollection<T extends EntityProperties> extends Collection {
   put(id: string | undefined, properties: T): Entity {
     return super.put(id, properties);
   }
 
-  get(id: string): Entity | undefined {
+  get(id: string): LossyViewOne | undefined {
     return super.get(id);
   }
 }

src/types.ts

@@ -41,7 +41,10 @@ export type DeltaFilter = (delta: Delta) => boolean;
 
 export type PropertyTypes = string | number | undefined;
 
-export type Properties = {[key: string]: PropertyTypes};
+export type DomainEntityID = string;
+export type PropertyID = string;
+
+export type Properties = {[key: PropertyID]: PropertyTypes};
 
 export class PeerAddress {
   addr: string;

src/util/md-files.ts

@@ -11,7 +11,10 @@ const docConverter = new Converter({
   tasklists: true
 });
 
-export const htmlDocFromMarkdown = (md: string): string => docConverter.makeHtml(md);
+export type Markdown = string;
+export type Html = string;
+
+export const htmlDocFromMarkdown = (md: Markdown): Html => docConverter.makeHtml(md);
 
 type mdFileInfo = {
   name: string,
@@ -27,7 +30,15 @@ export class MDFiles {
 
   readFile(name: string) {
     const md = readFileSync(join('./markdown', `${name}.md`)).toString();
-    const html = htmlDocFromMarkdown(md);
+    let m = "";
+
+    // Add title and render the markdown
+    m += `# File: [${name}](/html/${name})\n\n---\n\n${md}`;
+
+    // Add footer with the nav menu
+    m += `\n\n---\n\n${this.generateIndex()}`;
+
+    const html = htmlDocFromMarkdown(m);
     this.files.set(name, {name, md, html});
   }
 
@@ -49,6 +60,15 @@ export class MDFiles {
     return Array.from(this.files.keys());
   }
 
+  generateIndex(): Markdown {
+    let md = `# [Index](/html)\n\n`;
+    md += `[README](/html/README)\n\n`;
+    for (const name of this.list()) {
+      md += `- [${name}](/html/${name})\n`;
+    }
+    return htmlDocFromMarkdown(md);
+  }
+
   readDir() {
     // Read list of markdown files from directory and
     // render each markdown file as html

util/app.ts

@@ -25,7 +25,7 @@ export class App extends RhizomeNode {
       ...config,
     });
 
-    const users = new TypedCollection<User>("users");
+    const users = new TypedCollection<User>("user");
     users.rhizomeConnect(this);
 
     const {httpAddr, httpPort} = this.config;