From 0b6d147f9c004b99a04a41a37f066fe7b668ee6b Mon Sep 17 00:00:00 2001 From: Ladd Hoffman Date: Sat, 27 Apr 2024 15:59:36 -0500 Subject: [PATCH] matrix event import is working --- backend/src/api/import-from-matrix.js | 82 +++++++++++++++++++ backend/src/api/import-from-ss.js | 20 ++--- backend/src/api/index.js | 10 ++- backend/src/api/read.js | 8 +- backend/src/api/write.js | 29 +++---- backend/src/contract-listeners/proposals.js | 24 ++++-- backend/src/index.js | 14 +++- backend/src/matrix-bot/index.js | 3 + backend/src/matrix-bot/outbound-queue.js | 1 + backend/src/matrix-bot/register-identity.js | 0 frontend/src/App.css | 5 ++ frontend/src/WebApp.jsx | 5 +- frontend/src/Widget.jsx | 2 + frontend/src/components/ImportMatrixEvent.jsx | 36 ++++++++ frontend/src/utils/Post.js | 20 +++-- 15 files changed, 204 insertions(+), 55 deletions(-) create mode 100644 backend/src/api/import-from-matrix.js delete mode 100644 backend/src/matrix-bot/register-identity.js create mode 100644 frontend/src/components/ImportMatrixEvent.jsx diff --git a/backend/src/api/import-from-matrix.js b/backend/src/api/import-from-matrix.js new file mode 100644 index 0000000..979369c --- /dev/null +++ b/backend/src/api/import-from-matrix.js @@ -0,0 +1,82 @@ +const { getClient } = require('../matrix-bot'); + +const { matrixUserToAuthorAddress } = require('../util/db'); +const write = require('./write'); +const { dao } = require('../util/contracts'); + +const addPostWithRetry = async (authors, hash, citations, retryDelay = 5000) => { + try { + await dao.addPost(authors, hash, citations); + } catch (e) { + if (e.code === 'REPLACEMENT_UNDERPRICED') { + console.log('retry delay (sec):', retryDelay / 1000); + await Promise.delay(retryDelay); + return addPostWithRetry(authors, hash, citations, retryDelay * 2); + } if (e.reason === 'A post with this contentId already exists') { + return { alreadyAdded: true }; + } + throw e; + } + return { alreadyAdded: false }; +}; + +module.exports = async (req, res) => { + const { + body: { + eventUri, + }, + } = req; + + if (!eventUri) { + res.status(400).end(); + return; + } + console.log(`importFromMatrix: event ${eventUri}`); + + // URI format: + // https://matrix.to/#/${roomId}/${eventId}?via= + const uriRegex = /#\/(![A-Za-z0-9:._-]+)\/(\$[A-Za-z0-9._-]+)(\?.*)$/; + const [, roomId, eventId] = uriRegex.exec(new URL(eventUri).hash); + console.log('roomId', roomId); + console.log('eventId', eventId); + + const client = getClient(); + const event = await client.getEvent(roomId, eventId); + console.log('event', event); + + let authorAddress; + try { + authorAddress = await matrixUserToAuthorAddress.get(event.sender); + } catch (e) { + // Matrix user has not registered their author address + res.send(`Author address not registered for matrix user ${event.sender}`); + return; + } + + // We want to add a post representing this matrix message. + // We can't sign it on behalf of the author. + // That means we need to support posts without signatures. + const authors = [{ authorAddress, weightPPM: 1000000 }]; + // TODO: Take citations as input to this API call, referencing other posts or matrix events + const citations = []; + const content = `Matrix event URI: ${eventUri}`; + const embeddedData = { + roomId, + eventId, + }; + + const { hash } = await write({ + authors, citations, content, embeddedData, + }); + + // Now we want to add a post on-chain + const { alreadyAdded } = await addPostWithRetry(authors, hash, citations); + + if (alreadyAdded) { + console.log(`Post already added for matrix event ${eventUri}`); + } else { + console.log(`Added post to blockchain for matrix event ${eventUri}`); + } + + res.json({ postId: hash, alreadyAdded }); +}; diff --git a/backend/src/api/import-from-ss.js b/backend/src/api/import-from-ss.js index 5cedbe2..ddd21af 100644 --- a/backend/src/api/import-from-ss.js +++ b/backend/src/api/import-from-ss.js @@ -1,12 +1,11 @@ const axios = require('axios'); const ethers = require('ethers'); const crypto = require('crypto'); -const objectHash = require('object-hash'); const Promise = require('bluebird'); -const verifySignature = require('../util/verify-signature'); -const { authorAddresses, authorPrivKeys, forum } = require('../util/db'); +const { authorAddresses, authorPrivKeys } = require('../util/db'); const { dao } = require('../util/contracts'); +const write = require('./write'); // Each post allocates 30% of its reputation to citations const PPM_TO_CITATIONS = 300000; @@ -111,17 +110,8 @@ HREF ${paper.url}`; contentToSign += `\n\n${JSON.stringify(embeddedData, null, 2)}`; } const signature = firstAuthorWallet.signMessageSync(contentToSign); - const verified = verifySignature({ - authors, content, signature, embeddedData, - }); - if (!verified) { - throw new Error('Signature verification failed'); - } - const hash = objectHash({ - authors, content, signature, embeddedData, - }); return { - hash, authors, content, signature, embeddedData, + authors, content, signature, embeddedData, }; }; @@ -172,11 +162,11 @@ const importPaper = async (paper) => { // Create a post for this paper const { - hash, authors, content, signature, embeddedData, + authors, content, signature, embeddedData, } = await generatePost(paper); // Write the new post to our database - await forum.put(hash, { + const hash = await write({ authors, content, signature, embeddedData, citations, }); diff --git a/backend/src/api/index.js b/backend/src/api/index.js index aa86b06..f830515 100644 --- a/backend/src/api/index.js +++ b/backend/src/api/index.js @@ -5,13 +5,19 @@ require('express-async-errors'); const read = require('./read'); const write = require('./write'); const importFromSS = require('./import-from-ss'); +const importFromMatrix = require('./import-from-matrix'); const app = express(); const port = process.env.API_LISTEN_PORT || 3000; app.use(express.json()); -app.post('/write', write); +app.post('/write', async (req, res) => { + const { hash, data } = await write(req.body); + console.log('write', hash); + console.log(data); + res.send(hash); +}); app.get('/read/:hash', async (req, res) => { const { hash } = req.params; @@ -22,6 +28,8 @@ app.get('/read/:hash', async (req, res) => { app.post('/importFromSemanticScholar', importFromSS); +app.post('/importFromMatrix', importFromMatrix); + app.get('*', (req, res) => { console.log(`404 req.path: ${req.path}`); res.status(404).json({ errorCode: 404 }); diff --git a/backend/src/api/read.js b/backend/src/api/read.js index e739e8f..13a140c 100644 --- a/backend/src/api/read.js +++ b/backend/src/api/read.js @@ -21,9 +21,11 @@ const read = async (hash) => { throw new Error('hash mismatch'); } - // Verify signature - if (!verifySignature(data)) { - throw new Error('signature verificaition failed'); + if (signature) { + // Verify signature + if (!verifySignature(data)) { + throw new Error('signature verificaition failed'); + } } return { diff --git a/backend/src/api/write.js b/backend/src/api/write.js index f51b322..2258a53 100644 --- a/backend/src/api/write.js +++ b/backend/src/api/write.js @@ -3,18 +3,19 @@ const objectHash = require('object-hash'); const verifySignature = require('../util/verify-signature'); const { forum } = require('../util/db'); -module.exports = async (req, res) => { - const { - body: { - authors, content, signature, embeddedData, citations, - }, - } = req; +const write = async ({ + authors, content, citations, embeddedData, signature, +}) => { + if (signature) { // Check author signature - if (!verifySignature({ - authors, content, signature, embeddedData, - })) { - res.status(403).end(); - return; + if (!verifySignature({ + authors, content, signature, embeddedData, + })) { + const err = new Error(); + err.status = 403; + err.message = 'Signature verification failed'; + throw err; + } } // Compute content hash @@ -24,12 +25,12 @@ module.exports = async (req, res) => { const hash = objectHash({ authors, content, signature, embeddedData, }); - console.log('write', hash); - console.log(data); // Store content await forum.put(hash, data); // Return hash - res.send(hash); + return { hash, data }; }; + +module.exports = write; diff --git a/backend/src/contract-listeners/proposals.js b/backend/src/contract-listeners/proposals.js index eb40972..71173c9 100644 --- a/backend/src/contract-listeners/proposals.js +++ b/backend/src/contract-listeners/proposals.js @@ -4,6 +4,7 @@ const { sendNewProposalEvent } = require('../matrix-bot/outbound-queue'); // Subscribe to proposal events const start = () => { + console.log('registering proposal listener'); proposals.on('NewProposal', async (proposalIndex) => { console.log('New Proposal, index', proposalIndex); @@ -11,17 +12,22 @@ const start = () => { console.log('postId:', proposal.postId); // Read post from database - const post = await read(proposal.postId); - console.log('post.content:', post.content); + try { + const post = await read(proposal.postId); + console.log('post.content:', post.content); - // Send matrix room event - let message = `Proposal ${proposalIndex}\n\n${post.content}`; - if (post.embeddedData && Object.entries(post.embeddedData).length) { - message += `\n\n${JSON.stringify(post.embeddedData, null, 2)}`; + // Send matrix room event + let message = `Proposal ${proposalIndex}\n\n${post.content}`; + if (post.embeddedData && Object.entries(post.embeddedData).length) { + message += `\n\n${JSON.stringify(post.embeddedData, null, 2)}`; + } + + // The outbound queue handles deduplication + sendNewProposalEvent(proposalIndex, message); + } catch (e) { + // Post for proposal not found + console.error(`error: post for proposal ${proposalIndex} not found`); } - - // The outbound queue handles deduplication - sendNewProposalEvent(proposalIndex, message); }); }; diff --git a/backend/src/index.js b/backend/src/index.js index 72a792e..8f45379 100644 --- a/backend/src/index.js +++ b/backend/src/index.js @@ -1,5 +1,13 @@ require('dotenv').config(); -require('./api').start(); -require('./matrix-bot').start(); -require('./contract-listeners').start(); +const api = require('./api'); +const matrixBot = require('./matrix-bot'); +const contractListeners = require('./contract-listeners'); + +api.start(); + +if (process.env.ENABLE_MATRIX !== 'false') { + matrixBot.start(); +} + +contractListeners.start(); diff --git a/backend/src/matrix-bot/index.js b/backend/src/matrix-bot/index.js index 3da0416..d2b0f5b 100644 --- a/backend/src/matrix-bot/index.js +++ b/backend/src/matrix-bot/index.js @@ -50,6 +50,9 @@ const start = async () => { }); }; +const getClient = () => client; + module.exports = { start, + getClient, }; diff --git a/backend/src/matrix-bot/outbound-queue.js b/backend/src/matrix-bot/outbound-queue.js index f3885c0..4d1a69a 100644 --- a/backend/src/matrix-bot/outbound-queue.js +++ b/backend/src/matrix-bot/outbound-queue.js @@ -12,6 +12,7 @@ const setTargetRoomId = (roomId) => { }; const processOutboundQueue = async ({ type, ...args }) => { + console.log('processing outbound queue item'); if (!targetRoomId) return; switch (type) { case 'NewProposal': { diff --git a/backend/src/matrix-bot/register-identity.js b/backend/src/matrix-bot/register-identity.js deleted file mode 100644 index e69de29..0000000 diff --git a/frontend/src/App.css b/frontend/src/App.css index 05c5864..9d73dd8 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -4,4 +4,9 @@ .input-paper-id { width: 30em !important; +} + +.input-event-uri { + width: 75em !important; + } \ No newline at end of file diff --git a/frontend/src/WebApp.jsx b/frontend/src/WebApp.jsx index b57fb4e..ae632eb 100644 --- a/frontend/src/WebApp.jsx +++ b/frontend/src/WebApp.jsx @@ -25,6 +25,7 @@ import Proposals from './components/Proposals'; import ImportPaper from './components/ImportPaper'; import ImportPapersByAuthor from './components/ImportPapersByAuthor'; import getAddressName from './utils/get-address-name'; +import ImportMatrixEvent from './components/ImportMatrixEvent'; function WebApp() { const { @@ -566,9 +567,11 @@ function WebApp() { -

Semantic Scholar Import

+

Semantic Scholar

+

Matrix

+ diff --git a/frontend/src/Widget.jsx b/frontend/src/Widget.jsx index 1ed9d23..6289908 100644 --- a/frontend/src/Widget.jsx +++ b/frontend/src/Widget.jsx @@ -30,6 +30,7 @@ import Proposals from './components/Proposals'; import ImportPaper from './components/ImportPaper'; import ImportPapersByAuthor from './components/ImportPapersByAuthor'; import getAddressName from './utils/get-address-name'; +import ImportMatrixEvent from './components/ImportMatrixEvent'; function Widget() { const { @@ -608,6 +609,7 @@ function Widget() {

Semantic Scholar Import

+ diff --git a/frontend/src/components/ImportMatrixEvent.jsx b/frontend/src/components/ImportMatrixEvent.jsx new file mode 100644 index 0000000..1147d8b --- /dev/null +++ b/frontend/src/components/ImportMatrixEvent.jsx @@ -0,0 +1,36 @@ +import { useState } from 'react'; +import Button from 'react-bootstrap/Button'; +import Form from 'react-bootstrap/Form'; +import axios from 'axios'; + +function ImportMatrixEvent() { + const [eventUri, setEventUri] = useState(); + const [status, setStatus] = useState(''); + + const handleImport = async () => { + setStatus(`Importing event ${eventUri}...`); + const { data } = await axios.post('/api/importFromMatrix', { eventUri }) + .catch((error) => { + setStatus(`Error: ${error.response?.data ?? error.message}`); + }); + setStatus(`Response: ${JSON.stringify(data, null, 2)}`); + }; + + return ( + <> +

Import Matrix Event

+ + Event URI + setEventUri(e.target.value)} + /> + + +

{status}

+ + ); +} + +export default ImportMatrixEvent; diff --git a/frontend/src/utils/Post.js b/frontend/src/utils/Post.js index 7697fd0..f779b49 100644 --- a/frontend/src/utils/Post.js +++ b/frontend/src/utils/Post.js @@ -32,15 +32,17 @@ class Post { if (hash !== derivedHash) { throw new Error('Hash mismatch'); } - // Verify signature - let contentToVerify = content; - if (embeddedData && Object.entries(embeddedData).length) { - contentToVerify += `\n\n${JSON.stringify(embeddedData, null, 2)}`; - } - const recovered = recoverPersonalSignature({ data: contentToVerify, signature }); - const authorAddresses = authors.map((author) => author.authorAddress.toLowerCase()); - if (!authorAddresses.includes(recovered.toLowerCase())) { - throw new Error('Signer is not among the authors'); + if (signature) { + // Verify signature + let contentToVerify = content; + if (embeddedData && Object.entries(embeddedData).length) { + contentToVerify += `\n\n${JSON.stringify(embeddedData, null, 2)}`; + } + const recovered = recoverPersonalSignature({ data: contentToVerify, signature }); + const authorAddresses = authors.map((author) => author.authorAddress.toLowerCase()); + if (!authorAddresses.includes(recovered.toLowerCase())) { + throw new Error('Signer is not among the authors'); + } } return new Post({ content, authors, signature, hash, embeddedData, citations,