ladd
/
rhizome
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed test run/002

This commit is contained in:
Ladd Hoffman 2024-12-25 19:27:36 -06:00
parent 28691d677a
commit 9d9a1e1f08
10 changed files with 175 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
__tests__/run/001-single-node.ts
View File

@ -1,4 +1,4 @@
import {App} from "../../util/app";
import {App} from "../util/app";
describe('Run', () => {
let app: App;
@ -23,16 +23,18 @@ describe('Run', () => {
method: 'PUT',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({
name: "Peon",
id: "peon-1",
properties: {
name: "Peon",
age: 263
}
})
});
const data = await res.json();
expect(data).toMatchObject({
id: "peon-1",
properties: {
name: "Peon",
id: "peon-1",
age: 263
}
});

30
__tests__/run/002-two-nodes.ts
View File

@ -1,4 +1,6 @@
import {App} from '../../util/app';
import Debug from 'debug';
import {App} from '../util/app';
const debug = Debug('test:two');
describe('Run', () => {
const apps: App[] = [];
@ -6,10 +8,14 @@ describe('Run', () => {
beforeAll(async () => {
apps[0] = new App({
httpEnable: true,
peerId: 'app0',
});
apps[1] = new App({
httpEnable: true,
peerId: 'app1',
});
apps[0].config.seedPeers.push(apps[1].myRequestAddr);
apps[1].config.seedPeers.push(apps[0].myRequestAddr);
await Promise.all(apps.map((app) => app.start()));
});
@ -19,40 +25,38 @@ describe('Run', () => {
});
it('can create a record on node 0 and read it on node 1', async () => {
debug('apps[0].apiUrl', apps[0].apiUrl);
debug('apps[1].apiUrl', apps[1].apiUrl);
const res = await fetch(`${apps[0].apiUrl}/users`, {
method: 'PUT',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({
name: "Peon",
id: "peon-1",
properties: {
name: "Peon",
age: 263
}
})
});
const data = await res.json();
expect(data).toMatchObject({
id: "peon-1",
properties: {
name: "Peon",
id: "peon-1",
age: 263
}
});
await new Promise((resolve) => setTimeout(resolve, 100));
const res2 = await fetch(`${apps[0].apiUrl}/users`, {
method: 'PUT',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({
name: "Peon",
id: "peon-1",
age: 263
})
});
const res2 = await fetch(`${apps[1].apiUrl}/users/peon-1`);
const data2 = await res2.json();
debug('data2', data2);
expect(data2).toMatchObject({
id: "peon-1",
properties: {
name: "Peon",
id: "peon-1",
age: 263
}
});

7
util/app.ts → __tests__/util/app.ts
View File

@ -1,5 +1,5 @@
import {RhizomeNode, RhizomeNodeConfig} from "../src/node";
import {TypedCollection} from "../src/typed-collection";
import {RhizomeNode, RhizomeNodeConfig} from "../../src/node";
import {TypedCollection} from "../../src/typed-collection";
type User = {
id?: string;
@ -28,7 +28,8 @@ export class App extends RhizomeNode {
const users = new TypedCollection<User>("users");
users.rhizomeConnect(this);
this.apiUrl = `http://${this.config.httpAddr}:${this.config.httpPort}`;
const {httpAddr, httpPort} = this.config;
this.apiUrl = `http://${httpAddr}:${httpPort}`;
}
}

3
package.json
View File

@ -26,7 +26,8 @@
"level": "^9.0.0",
"object-hash": "^3.0.0",
"showdown": "^2.1.0",
"zeromq": "^6.1.2"
"zeromq": "^6.1.2",
"util": "./util/"
},
"devDependencies": {
"@eslint/js": "^9.17.0",

1
src/collection.ts
View File

@ -66,6 +66,7 @@ export class Collection {
} else {
let anyChanged = false;
Object.entries(properties).forEach(([key, value]) => {
if (key === 'id') return;
let changed = false;
if (entity.properties && entity.properties[key] !== value) {
entity.properties[key] = value;

150
src/http-api.ts
View File

@ -1,124 +1,26 @@
import Debug from "debug";
import express from "express";
import {FSWatcher} from "fs";
import {readdirSync, readFileSync, watch} from "fs";
import express, {Express, Router} from "express";
import {Server} from "http";
import path, {join} from "path";
import {Converter} from "showdown";
import {Collection} from "./collection";
import {RhizomeNode} from "./node";
import {Delta} from "./types";
import {htmlDocFromMarkdown, MDFiles} from "./util/md-files";
const debug = Debug('http-api');
const docConverter = new Converter({
completeHTMLDocument: true,
// simpleLineBreaks: true,
tables: true,
tasklists: true
});
const htmlDocFromMarkdown = (md: string): string => docConverter.makeHtml(md);
type mdFileInfo = {
name: string,
md: string,
html: string
};
class MDFiles {
files = new Map<string, mdFileInfo>();
readme?: mdFileInfo;
dirWatcher?: FSWatcher;
readmeWatcher?: FSWatcher;
readFile(name: string) {
const md = readFileSync(join('./markdown', `${name}.md`)).toString();
const html = htmlDocFromMarkdown(md);
this.files.set(name, {name, md, html});
}
readReadme() {
const md = readFileSync('./README.md').toString();
const html = htmlDocFromMarkdown(md);
this.readme = {name: 'README', md, html};
}
getReadmeHTML() {
return this.readme?.html;
}
getHtml(name: string): string | undefined {
return this.files.get(name)?.html;
}
list(): string[] {
return Array.from(this.files.keys());
}
readDir() {
// Read list of markdown files from directory and
// render each markdown file as html
readdirSync('./markdown/')
.filter((f) => f.endsWith('.md'))
.map((name) => path.parse(name).name)
.forEach((name) => this.readFile(name));
}
watchDir() {
this.dirWatcher = watch('./markdown', null, (eventType, filename) => {
if (!filename) return;
if (!filename.endsWith(".md")) return;
const name = path.parse(filename).name;
switch (eventType) {
case 'rename': {
debug(`file ${name} renamed`);
// Remove it from memory and re-scan everything
this.files.delete(name);
this.readDir();
break;
}
case 'change': {
debug(`file ${name} changed`);
// Re-read this file
this.readFile(name)
break;
}
}
});
}
watchReadme() {
this.readmeWatcher = watch('./README.md', null, (eventType, filename) => {
if (!filename) return;
switch (eventType) {
case 'change': {
debug(`README file changed`);
// Re-read this file
this.readReadme()
break;
}
}
});
}
close() {
this.dirWatcher?.close();
this.readmeWatcher?.close();
}
}
export class HttpApi {
rhizomeNode: RhizomeNode;
app = express();
app: Express;
router: Router;
mdFiles = new MDFiles();
server?: Server;
constructor(rhizomeNode: RhizomeNode) {
this.rhizomeNode = rhizomeNode;
this.app = express();
this.router = Router();
this.app.use(express.json());
this.app.use(this.router);
}
start() {
@ -129,13 +31,13 @@ export class HttpApi {
this.mdFiles.watchReadme();
// Serve README
this.app.get('/html/README', (_req: express.Request, res: express.Response) => {
this.router.get('/html/README', (_req: express.Request, res: express.Response) => {
const html = this.mdFiles.getReadmeHTML();
res.setHeader('content-type', 'text/html').send(html);
});
// Serve markdown files as html
this.app.get('/html/:name', (req: express.Request, res: express.Response) => {
this.router.get('/html/:name', (req: express.Request, res: express.Response) => {
let html = this.mdFiles.getHtml(req.params.name);
if (!html) {
res.status(404);
@ -154,24 +56,24 @@ export class HttpApi {
}
const html = htmlDocFromMarkdown(md);
this.app.get('/html', (_req: express.Request, res: express.Response) => {
this.router.get('/html', (_req: express.Request, res: express.Response) => {
res.setHeader('content-type', 'text/html').send(html);
});
}
// Serve list of all deltas accepted
// TODO: This won't scale well
this.app.get("/deltas", (_req: express.Request, res: express.Response) => {
this.router.get("/deltas", (_req: express.Request, res: express.Response) => {
res.json(this.rhizomeNode.deltaStream.deltasAccepted);
});
// Get the number of deltas ingested by this node
this.app.get("/deltas/count", (_req: express.Request, res: express.Response) => {
this.router.get("/deltas/count", (_req: express.Request, res: express.Response) => {
res.json(this.rhizomeNode.deltaStream.deltasAccepted.length);
});
// Get the list of peers seen by this node (including itself)
this.app.get("/peers", (_req: express.Request, res: express.Response) => {
this.router.get("/peers", (_req: express.Request, res: express.Response) => {
res.json(this.rhizomeNode.peers.peers.map(({reqAddr, publishAddr, isSelf, isSeedPeer}) => {
const deltasAcceptedCount = this.rhizomeNode.deltaStream.deltasAccepted
.filter((delta: Delta) => {
@ -193,12 +95,16 @@ export class HttpApi {
});
// Get the number of peers seen by this node (including itself)
this.app.get("/peers/count", (_req: express.Request, res: express.Response) => {
this.router.get("/peers/count", (_req: express.Request, res: express.Response) => {
res.json(this.rhizomeNode.peers.peers.length);
});
const {httpAddr, httpPort} = this.rhizomeNode.config;
this.server = this.app.listen(httpPort, httpAddr, () => {
this.server = this.app.listen({
port: httpPort,
host: httpAddr,
exclusive: true
}, () => {
debug(`HTTP API bound to ${httpAddr}:${httpPort}`);
});
}
@ -207,27 +113,31 @@ export class HttpApi {
const {name} = collection;
// Get the ID of all domain entities
this.app.get(`/${name}/ids`, (_req: express.Request, res: express.Response) => {
this.router.get(`/${name}/ids`, (_req: express.Request, res: express.Response) => {
res.json({ids: collection.getIds()});
});
// Get a single domain entity by ID
this.app.get(`/${name}/:id`, (req: express.Request, res: express.Response) => {
this.router.get(`/${name}/:id`, (req: express.Request, res: express.Response) => {
const {params: {id}} = req;
const ent = collection.get(id);
if (!ent) {
res.status(404).send({error: "Not Found"});
return;
}
res.json(ent);
});
// Add a new domain entity
// TODO: schema validation
this.app.put(`/${name}`, (req: express.Request, res: express.Response) => {
const {body: properties} = req;
const ent = collection.put(properties.id, properties);
this.router.put(`/${name}`, (req: express.Request, res: express.Response) => {
const {body: {id, properties}} = req;
const ent = collection.put(id, properties);
res.json(ent);
});
// Update a domain entity
this.app.put(`/${name}/:id`, (req: express.Request, res: express.Response) => {
this.router.put(`/${name}/:id`, (req: express.Request, res: express.Response) => {
const {body: properties, params: {id}} = req;
if (properties.id && properties.id !== id) {
res.status(400).json({error: "ID Mismatch", param: id, property: properties.id});

12
src/peers.ts
View File

@ -7,7 +7,7 @@ import {PeerRequest, RequestSocket, ResponseSocket} from "./request-reply";
import {Delta, PeerAddress} from "./types";
const debug = Debug('peers');
export enum PeerMethods {
export enum RequestMethods {
GetPublishAddress,
AskForDeltas
}
@ -28,7 +28,7 @@ class Peer {
this.isSeedPeer = !!SEED_PEERS.find((seedPeer) => reqAddr.isEqual(seedPeer));
}
async request(method: PeerMethods): Promise<Message> {
async request(method: RequestMethods): Promise<Message> {
if (!this.reqSock) {
this.reqSock = new RequestSocket(this.reqAddr);
}
@ -38,7 +38,7 @@ class Peer {
async subscribeDeltas() {
if (!this.publishAddr) {
debug(`requesting publish addr from peer ${this.reqAddr.toAddrString()}`);
const res = await this.request(PeerMethods.GetPublishAddress);
const res = await this.request(RequestMethods.GetPublishAddress);
this.publishAddr = PeerAddress.fromString(res.toString());
debug(`received publish addr ${this.publishAddr.toAddrString()} from peer ${this.reqAddr.toAddrString()}`);
}
@ -63,7 +63,7 @@ class Peer {
// Third pass should find a way to reduce the number of deltas transmitted.
// TODO: requestTimeout
const res = await this.request(PeerMethods.AskForDeltas);
const res = await this.request(RequestMethods.AskForDeltas);
const deltas = JSON.parse(res.toString());
return deltas;
}
@ -82,12 +82,12 @@ export class Peers {
this.rhizomeNode.requestReply.registerRequestHandler(async (req: PeerRequest, res: ResponseSocket) => {
debug('inspecting peer request');
switch (req.method) {
case PeerMethods.GetPublishAddress: {
case RequestMethods.GetPublishAddress: {
debug('it\'s a request for our publish address');
await res.send(this.rhizomeNode.myPublishAddr.toAddrString());
break;
}
case PeerMethods.AskForDeltas: {
case RequestMethods.AskForDeltas: {
debug('it\'s a request for deltas');
// TODO: stream these rather than
// trying to write them all in one message

6
src/request-reply.ts
View File

@ -1,13 +1,13 @@
import {Request, Reply, Message} from 'zeromq';
import {EventEmitter} from 'node:events';
import {PeerMethods} from './peers';
import {RequestMethods} from './peers';
import Debug from 'debug';
import {RhizomeNode} from './node';
import {PeerAddress} from './types';
const debug = Debug('request-reply');
export type PeerRequest = {
method: PeerMethods;
method: RequestMethods;
};
export type RequestHandler = (req: PeerRequest, res: ResponseSocket) => void;
@ -22,7 +22,7 @@ export class RequestSocket {
debug(`Request socket connecting to ${addrStr}`);
}
async request(method: PeerMethods): Promise<Message> {
async request(method: RequestMethods): Promise<Message> {
const req: PeerRequest = {
method
};

105
src/util/md-files.ts Normal file
View File

@ -0,0 +1,105 @@
import Debug from "debug";
import {FSWatcher, readdirSync, readFileSync, watch} from "fs";
import path, {join} from "path";
import {Converter} from "showdown";
const debug = Debug('md-files');
const docConverter = new Converter({
completeHTMLDocument: true,
// simpleLineBreaks: true,
tables: true,
tasklists: true
});
export const htmlDocFromMarkdown = (md: string): string => docConverter.makeHtml(md);
type mdFileInfo = {
name: string,
md: string,
html: string
};
export class MDFiles {
files = new Map<string, mdFileInfo>();
readme?: mdFileInfo;
dirWatcher?: FSWatcher;
readmeWatcher?: FSWatcher;
readFile(name: string) {
const md = readFileSync(join('./markdown', `${name}.md`)).toString();
const html = htmlDocFromMarkdown(md);
this.files.set(name, {name, md, html});
}
readReadme() {
const md = readFileSync('./README.md').toString();
const html = htmlDocFromMarkdown(md);
this.readme = {name: 'README', md, html};
}
getReadmeHTML() {
return this.readme?.html;
}
getHtml(name: string): string | undefined {
return this.files.get(name)?.html;
}
list(): string[] {
return Array.from(this.files.keys());
}
readDir() {
// Read list of markdown files from directory and
// render each markdown file as html
readdirSync('./markdown/')
.filter((f) => f.endsWith('.md'))
.map((name) => path.parse(name).name)
.forEach((name) => this.readFile(name));
}
watchDir() {
this.dirWatcher = watch('./markdown', null, (eventType, filename) => {
if (!filename) return;
if (!filename.endsWith(".md")) return;
const name = path.parse(filename).name;
switch (eventType) {
case 'rename': {
debug(`file ${name} renamed`);
// Remove it from memory and re-scan everything
this.files.delete(name);
this.readDir();
break;
}
case 'change': {
debug(`file ${name} changed`);
// Re-read this file
this.readFile(name)
break;
}
}
});
}
watchReadme() {
this.readmeWatcher = watch('./README.md', null, (eventType, filename) => {
if (!filename) return;
switch (eventType) {
case 'change': {
debug(`README file changed`);
// Re-read this file
this.readReadme()
break;
}
}
});
}
close() {
this.dirWatcher?.close();
this.readmeWatcher?.close();
}
}

3
tsconfig.json
View File

@ -6,9 +6,6 @@
"moduleResolution": "Node",
"sourceMap": true,
"baseUrl": ".",
"paths": {
"@/*": ["src/*"]
},
"outDir": "dist",
"importsNotUsedAsValues": "remove",
"strict": true,