matrix event import is working

backend/src/api/import-from-matrix.js

@@ -0,0 +1,82 @@
+const { getClient } = require('../matrix-bot');
+
+const { matrixUserToAuthorAddress } = require('../util/db');
+const write = require('./write');
+const { dao } = require('../util/contracts');
+
+const addPostWithRetry = async (authors, hash, citations, retryDelay = 5000) => {
+  try {
+    await dao.addPost(authors, hash, citations);
+  } catch (e) {
+    if (e.code === 'REPLACEMENT_UNDERPRICED') {
+      console.log('retry delay (sec):', retryDelay / 1000);
+      await Promise.delay(retryDelay);
+      return addPostWithRetry(authors, hash, citations, retryDelay * 2);
+    } if (e.reason === 'A post with this contentId already exists') {
+      return { alreadyAdded: true };
+    }
+    throw e;
+  }
+  return { alreadyAdded: false };
+};
+
+module.exports = async (req, res) => {
+  const {
+    body: {
+      eventUri,
+    },
+  } = req;
+
+  if (!eventUri) {
+    res.status(400).end();
+    return;
+  }
+  console.log(`importFromMatrix: event ${eventUri}`);
+
+  // URI format:
+  // https://matrix.to/#/${roomId}/${eventId}?via=
+  const uriRegex = /#\/(![A-Za-z0-9:._-]+)\/(\$[A-Za-z0-9._-]+)(\?.*)$/;
+  const [, roomId, eventId] = uriRegex.exec(new URL(eventUri).hash);
+  console.log('roomId', roomId);
+  console.log('eventId', eventId);
+
+  const client = getClient();
+  const event = await client.getEvent(roomId, eventId);
+  console.log('event', event);
+
+  let authorAddress;
+  try {
+    authorAddress = await matrixUserToAuthorAddress.get(event.sender);
+  } catch (e) {
+    // Matrix user has not registered their author address
+    res.send(`Author address not registered for matrix user ${event.sender}`);
+    return;
+  }
+
+  // We want to add a post representing this matrix message.
+  // We can't sign it on behalf of the author.
+  // That means we need to support posts without signatures.
+  const authors = [{ authorAddress, weightPPM: 1000000 }];
+  // TODO: Take citations as input to this API call, referencing other posts or matrix events
+  const citations = [];
+  const content = `Matrix event URI: ${eventUri}`;
+  const embeddedData = {
+    roomId,
+    eventId,
+  };
+
+  const { hash } = await write({
+    authors, citations, content, embeddedData,
+  });
+
+  // Now we want to add a post on-chain
+  const { alreadyAdded } = await addPostWithRetry(authors, hash, citations);
+
+  if (alreadyAdded) {
+    console.log(`Post already added for matrix event ${eventUri}`);
+  } else {
+    console.log(`Added post to blockchain for matrix event ${eventUri}`);
+  }
+
+  res.json({ postId: hash, alreadyAdded });
+};

backend/src/api/import-from-ss.js

@@ -1,12 +1,11 @@
 const axios = require('axios');
 const ethers = require('ethers');
 const crypto = require('crypto');
-const objectHash = require('object-hash');
 const Promise = require('bluebird');
 
-const verifySignature = require('../util/verify-signature');
-const { authorAddresses, authorPrivKeys, forum } = require('../util/db');
+const { authorAddresses, authorPrivKeys } = require('../util/db');
 const { dao } = require('../util/contracts');
+const write = require('./write');
 
 // Each post allocates 30% of its reputation to citations
 const PPM_TO_CITATIONS = 300000;
@@ -111,17 +110,8 @@ HREF ${paper.url}`;
     contentToSign += `\n\n${JSON.stringify(embeddedData, null, 2)}`;
   }
   const signature = firstAuthorWallet.signMessageSync(contentToSign);
-  const verified = verifySignature({
-    authors, content, signature, embeddedData,
-  });
-  if (!verified) {
-    throw new Error('Signature verification failed');
-  }
-  const hash = objectHash({
-    authors, content, signature, embeddedData,
-  });
   return {
-    hash, authors, content, signature, embeddedData,
+    authors, content, signature, embeddedData,
   };
 };
 
@@ -172,11 +162,11 @@ const importPaper = async (paper) => {
 
   // Create a post for this paper
   const {
-    hash, authors, content, signature, embeddedData,
+    authors, content, signature, embeddedData,
   } = await generatePost(paper);
 
   // Write the new post to our database
-  await forum.put(hash, {
+  const hash = await write({
     authors, content, signature, embeddedData, citations,
   });
 

backend/src/api/index.js

@@ -5,13 +5,19 @@ require('express-async-errors');
 const read = require('./read');
 const write = require('./write');
 const importFromSS = require('./import-from-ss');
+const importFromMatrix = require('./import-from-matrix');
 
 const app = express();
 const port = process.env.API_LISTEN_PORT || 3000;
 
 app.use(express.json());
 
-app.post('/write', write);
+app.post('/write', async (req, res) => {
+  const { hash, data } = await write(req.body);
+  console.log('write', hash);
+  console.log(data);
+  res.send(hash);
+});
 
 app.get('/read/:hash', async (req, res) => {
   const { hash } = req.params;
@@ -22,6 +28,8 @@ app.get('/read/:hash', async (req, res) => {
 
 app.post('/importFromSemanticScholar', importFromSS);
 
+app.post('/importFromMatrix', importFromMatrix);
+
 app.get('*', (req, res) => {
   console.log(`404 req.path: ${req.path}`);
   res.status(404).json({ errorCode: 404 });

backend/src/api/read.js

@@ -21,9 +21,11 @@ const read = async (hash) => {
     throw new Error('hash mismatch');
   }
 
-  // Verify signature
-  if (!verifySignature(data)) {
-    throw new Error('signature verificaition failed');
+  if (signature) {
+    // Verify signature
+    if (!verifySignature(data)) {
+      throw new Error('signature verificaition failed');
+    }
   }
 
   return {

backend/src/api/write.js

@@ -3,18 +3,19 @@ const objectHash = require('object-hash');
 const verifySignature = require('../util/verify-signature');
 const { forum } = require('../util/db');
 
-module.exports = async (req, res) => {
-  const {
-    body: {
-      authors, content, signature, embeddedData, citations,
-    },
-  } = req;
+const write = async ({
+  authors, content, citations, embeddedData, signature,
+}) => {
+  if (signature) {
   // Check author signature
-  if (!verifySignature({
-    authors, content, signature, embeddedData,
-  })) {
-    res.status(403).end();
-    return;
+    if (!verifySignature({
+      authors, content, signature, embeddedData,
+    })) {
+      const err = new Error();
+      err.status = 403;
+      err.message = 'Signature verification failed';
+      throw err;
+    }
   }
 
   // Compute content hash
@@ -24,12 +25,12 @@ module.exports = async (req, res) => {
   const hash = objectHash({
     authors, content, signature, embeddedData,
   });
-  console.log('write', hash);
-  console.log(data);
 
   // Store content
   await forum.put(hash, data);
 
   // Return hash
-  res.send(hash);
+  return { hash, data };
 };
+
+module.exports = write;

backend/src/contract-listeners/proposals.js

@@ -4,6 +4,7 @@ const { sendNewProposalEvent } = require('../matrix-bot/outbound-queue');
 
 // Subscribe to proposal events
 const start = () => {
+  console.log('registering proposal listener');
   proposals.on('NewProposal', async (proposalIndex) => {
     console.log('New Proposal, index', proposalIndex);
 
@@ -11,17 +12,22 @@ const start = () => {
     console.log('postId:', proposal.postId);
 
     // Read post from database
-    const post = await read(proposal.postId);
-    console.log('post.content:', post.content);
+    try {
+      const post = await read(proposal.postId);
+      console.log('post.content:', post.content);
 
-    // Send matrix room event
-    let message = `Proposal ${proposalIndex}\n\n${post.content}`;
-    if (post.embeddedData && Object.entries(post.embeddedData).length) {
-      message += `\n\n${JSON.stringify(post.embeddedData, null, 2)}`;
+      // Send matrix room event
+      let message = `Proposal ${proposalIndex}\n\n${post.content}`;
+      if (post.embeddedData && Object.entries(post.embeddedData).length) {
+        message += `\n\n${JSON.stringify(post.embeddedData, null, 2)}`;
+      }
+
+      // The outbound queue handles deduplication
+      sendNewProposalEvent(proposalIndex, message);
+    } catch (e) {
+    // Post for proposal not found
+      console.error(`error: post for proposal ${proposalIndex} not found`);
     }
-
-    // The outbound queue handles deduplication
-    sendNewProposalEvent(proposalIndex, message);
   });
 };
 

backend/src/index.js

@@ -1,5 +1,13 @@
 require('dotenv').config();
 
-require('./api').start();
-require('./matrix-bot').start();
-require('./contract-listeners').start();
+const api = require('./api');
+const matrixBot = require('./matrix-bot');
+const contractListeners = require('./contract-listeners');
+
+api.start();
+
+if (process.env.ENABLE_MATRIX !== 'false') {
+  matrixBot.start();
+}
+
+contractListeners.start();

backend/src/matrix-bot/index.js

@@ -50,6 +50,9 @@ const start = async () => {
   });
 };
 
+const getClient = () => client;
+
 module.exports = {
   start,
+  getClient,
 };

backend/src/matrix-bot/outbound-queue.js

@@ -12,6 +12,7 @@ const setTargetRoomId = (roomId) => {
 };
 
 const processOutboundQueue = async ({ type, ...args }) => {
+  console.log('processing outbound queue item');
   if (!targetRoomId) return;
   switch (type) {
     case 'NewProposal': {

frontend/src/App.css

@@ -4,4 +4,9 @@
 
 .input-paper-id {
   width: 30em !important;
+}
+
+.input-event-uri {
+  width: 75em !important;
+
 }

frontend/src/WebApp.jsx

@@ -25,6 +25,7 @@ import Proposals from './components/Proposals';
 import ImportPaper from './components/ImportPaper';
 import ImportPapersByAuthor from './components/ImportPapersByAuthor';
 import getAddressName from './utils/get-address-name';
+import ImportMatrixEvent from './components/ImportMatrixEvent';
 
 function WebApp() {
   const {
@@ -566,9 +567,11 @@ function WebApp() {
               <Proposals />
             </Tab>
             <Tab eventKey="import" title="Import">
-              <h1>Semantic Scholar Import</h1>
+              <h1>Semantic Scholar</h1>
               <ImportPaper />
               <ImportPapersByAuthor />
+              <h1>Matrix</h1>
+              <ImportMatrixEvent />
             </Tab>
           </Tabs>
         </>

frontend/src/Widget.jsx

@@ -30,6 +30,7 @@ import Proposals from './components/Proposals';
 import ImportPaper from './components/ImportPaper';
 import ImportPapersByAuthor from './components/ImportPapersByAuthor';
 import getAddressName from './utils/get-address-name';
+import ImportMatrixEvent from './components/ImportMatrixEvent';
 
 function Widget() {
   const {
@@ -608,6 +609,7 @@ function Widget() {
               <h1>Semantic Scholar Import</h1>
               <ImportPaper />
               <ImportPapersByAuthor />
+              <ImportMatrixEvent />
             </Tab>
           </Tabs>
         </>

frontend/src/components/ImportMatrixEvent.jsx

@@ -0,0 +1,36 @@
+import { useState } from 'react';
+import Button from 'react-bootstrap/Button';
+import Form from 'react-bootstrap/Form';
+import axios from 'axios';
+
+function ImportMatrixEvent() {
+  const [eventUri, setEventUri] = useState();
+  const [status, setStatus] = useState('');
+
+  const handleImport = async () => {
+    setStatus(`Importing event ${eventUri}...`);
+    const { data } = await axios.post('/api/importFromMatrix', { eventUri })
+      .catch((error) => {
+        setStatus(`Error: ${error.response?.data ?? error.message}`);
+      });
+    setStatus(`Response: ${JSON.stringify(data, null, 2)}`);
+  };
+
+  return (
+    <>
+      <h2>Import Matrix Event</h2>
+      <Form.Group>
+        <Form.Label>Event URI</Form.Label>
+        <Form.Control
+          as="input"
+          className="input-event-uri mb-3"
+          onChange={(e) => setEventUri(e.target.value)}
+        />
+      </Form.Group>
+      <Button className="mb-2" onClick={handleImport}>Import</Button>
+      <p>{status}</p>
+    </>
+  );
+}
+
+export default ImportMatrixEvent;

frontend/src/utils/Post.js

@@ -32,15 +32,17 @@ class Post {
     if (hash !== derivedHash) {
       throw new Error('Hash mismatch');
     }
-    // Verify signature
-    let contentToVerify = content;
-    if (embeddedData && Object.entries(embeddedData).length) {
-      contentToVerify += `\n\n${JSON.stringify(embeddedData, null, 2)}`;
-    }
-    const recovered = recoverPersonalSignature({ data: contentToVerify, signature });
-    const authorAddresses = authors.map((author) => author.authorAddress.toLowerCase());
-    if (!authorAddresses.includes(recovered.toLowerCase())) {
-      throw new Error('Signer is not among the authors');
+    if (signature) {
+      // Verify signature
+      let contentToVerify = content;
+      if (embeddedData && Object.entries(embeddedData).length) {
+        contentToVerify += `\n\n${JSON.stringify(embeddedData, null, 2)}`;
+      }
+      const recovered = recoverPersonalSignature({ data: contentToVerify, signature });
+      const authorAddresses = authors.map((author) => author.authorAddress.toLowerCase());
+      if (!authorAddresses.includes(recovered.toLowerCase())) {
+        throw new Error('Signer is not among the authors');
+      }
     }
     return new Post({
       content, authors, signature, hash, embeddedData, citations,