successful semantic scholar import

README.md

@@ -101,7 +101,7 @@ Clone this repository to a directory on your machine
 
 1. Run the daemon
 
-        node index.js
+        node src/index.js
 
 ### Hardhat
 

backend/.env.example

@@ -1,4 +1,5 @@
 PORT=3000
 DATA_DIR="./data"
 SEMANTIC_SCHOLAR_API_KEY=
-NETWORK="localhost"
+ETH_NETWORK="localhost"
+ETH_PRIVATE_KEY=

backend/Dockerfile

@@ -2,8 +2,8 @@ FROM node
 
 WORKDIR /app
 
-ADD package.json package-lock.json index.js /app/
+ADD package.json package-lock.json src/ /app/
 
 RUN npm ci
 
-ENTRYPOINT ["node", "index.js"]
+ENTRYPOINT ["node", "src/index.js"]

backend/package-lock.json

@@ -11,6 +11,7 @@
       "dependencies": {
         "@metamask/eth-sig-util": "^7.0.1",
         "axios": "^1.6.8",
+        "bluebird": "^3.7.2",
         "dotenv": "^16.4.5",
         "ethers": "^6.12.0",
         "express": "^4.18.2",
@@ -764,6 +765,11 @@
         }
       ]
     },
+    "node_modules/bluebird": {
+      "version": "3.7.2",
+      "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
+      "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
+    },
     "node_modules/body-parser": {
       "version": "1.20.1",
       "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz",

backend/package.json

@@ -2,7 +2,7 @@
   "name": "backend",
   "version": "1.0.0",
   "description": "",
-  "main": "index.js",
+  "main": "src/index.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
   },
@@ -11,6 +11,7 @@
   "dependencies": {
     "@metamask/eth-sig-util": "^7.0.1",
     "axios": "^1.6.8",
+    "bluebird": "^3.7.2",
     "dotenv": "^16.4.5",
     "ethers": "^6.12.0",
     "express": "^4.18.2",

backend/src/import-from-ss.js

@@ -2,53 +2,97 @@ const axios = require('axios');
 const ethers = require('ethers');
 const crypto = require('crypto');
 const objectHash = require('object-hash');
+const Promise = require('bluebird');
 
+require('dotenv').config();
+
+const verifySignature = require('./verify-signature');
 const { getContractAddressByNetworkName } = require('./contract-config');
 const { authorAddresses, authorPrivKeys, forum } = require('./db');
+const DAOArtifact = require('../contractArtifacts/DAO.json');
 
-const getContract = (name) => ethers.getContractAt(
-  name,
-  getContractAddressByNetworkName(process.env.NETWORK, name),
+const network = process.env.ETH_NETWORK;
+console.log('network:', network);
+const getProvider = () => {
+  switch (network) {
+    case 'localhost':
+      return ethers.getDefaultProvider('http://localhost:8545');
+    default:
+      throw new Error('Unknown network');
+  }
+};
+const signer = new ethers.Wallet(process.env.ETH_PRIVATE_KEY, getProvider());
+const getContract = (name) => new ethers.Contract(
+  getContractAddressByNetworkName(process.env.ETH_NETWORK, name),
+  DAOArtifact.abi,
+  signer,
 );
 
-const fetchPaperInfo = async (paperId) => {
-  const paper = await axios.get(`https://api.semanticscholar.org/graph/v1/paper/${paperId}`, {
+const fetchPaperInfo = async (paperId, retryDelay = 5000) => {
+  const url = `https://api.semanticscholar.org/graph/v1/paper/${paperId}?fields=title,url,authors,references`;
+  console.log('url:', url);
+  let retry = false;
+  let paper;
+  const response = await axios.get(url, {
     headers: {
       'api-key': process.env.SEMANTIC_SCHOLAR_API_KEY,
     },
+  }).catch(async (error) => {
+    if (error.response?.status === 429) {
+      // Rate limit
+      retry = true;
+      return;
+    }
+    // Some other error occurred
+    throw new Error(error);
   });
+  if (retry) {
+    console.log('retry delay (sec):', retryDelay / 1000);
+    await new Promise((resolve) => {
+      setTimeout(resolve, retryDelay);
+    });
+    paper = await fetchPaperInfo(paperId, retryDelay * 2);
+  } else {
+    paper = response.data;
+  }
   return paper;
 };
 
-const getAuthorsInfo = async (paper) => Promise.all(paper.authors.map(async ({ authorId }) => {
+const getAuthorsInfo = async (paper) => Promise.mapSeries(
+  paper.authors.filter((x) => !!x.authorId),
+  async ({ authorId }) => {
   // Check if we already have an account for each author
-  let authorAddress;
-  let authorPrivKey;
-  try {
-    authorAddress = await authorAddresses.get(authorId);
-  } catch (e) {
+    let authorAddress;
+    let authorPrivKey;
+    try {
+      authorAddress = await authorAddresses.get(authorId);
+    } catch (e) {
     // Probably not found
-  }
-  if (authorAddress) {
+    }
+    if (authorAddress) {
     // This should always succeed, so we don't use try/catch here
-    authorPrivKey = await authorPrivKeys.get(authorAddress);
-  } else {
+      authorPrivKey = await authorPrivKeys.get(authorAddress);
+    } else {
     // Generate and store a new account
-    const id = crypto.randomBytes(32).toString('hex');
-    authorPrivKey = `0x${id}`;
-    const wallet = new ethers.Wallet(authorPrivKey);
-    authorAddress = wallet.address;
-    await authorAddress.put(authorId, authorAddress);
-    await authorPrivKeys.put(authorAddress, authorPrivKey);
-  }
-  return {
-    authorAddress,
-    authorPrivKey,
-  };
-}));
+      const id = crypto.randomBytes(32).toString('hex');
+      authorPrivKey = `0x${id}`;
+      const wallet = new ethers.Wallet(authorPrivKey);
+      authorAddress = wallet.address;
+      await authorAddresses.put(authorId, authorAddress);
+      await authorPrivKeys.put(authorAddress, authorPrivKey);
+    }
+    return {
+      authorAddress,
+      authorPrivKey,
+    };
+  },
+);
 
 const generatePost = async (paper) => {
-  const authorsInfo = getAuthorsInfo(paper);
+  const authorsInfo = await getAuthorsInfo(paper);
+  if (!authorsInfo.length) {
+    throw new Error('Paper has no authors with id');
+  }
   const firstAuthorWallet = new ethers.Wallet(authorsInfo[0].authorPrivKey);
   const eachAuthorWeightPercent = Math.floor(100 / authorsInfo.length);
   const authors = authorsInfo.map(({ authorAddress }) => ({
@@ -56,20 +100,30 @@ const generatePost = async (paper) => {
     authorAddress,
   }));
     // Make sure author weights sum to 100
-  const totalAuthorsWeight = authors.reduce((t, { weightPercent }) => t + weightPercent);
+  const totalAuthorsWeight = authors.reduce((t, { weightPercent }) => t + weightPercent, 0);
   authors[0].weightPercent += 100 - totalAuthorsWeight;
 
   const content = `Semantic Scholar paper ${paper.paperId}
 ${paper.title}
 HREF ${paper.url}`;
 
-  // Note that for now we leave embedded data empty, but the stub is here in case we want to use it
-  const embeddedData = {};
+  const embeddedData = {
+    semanticScholarPaperId: paper.paperId,
+  };
   let contentToSign = content;
   if (embeddedData && Object.entries(embeddedData).length) {
-    contentToSign += `\n\nDATA\n${JSON.stringify(embeddedData, null, 2)}`;
+    contentToSign += `\n\n${JSON.stringify(embeddedData, null, 2)}`;
   }
   const signature = firstAuthorWallet.signMessageSync(contentToSign);
+  console.log({
+    authors, content, signature, embeddedData,
+  });
+  const verified = verifySignature({
+    authors, content, signature, embeddedData,
+  });
+  if (!verified) {
+    throw new Error('Signature verification failed');
+  }
   const hash = objectHash({
     authors, content, signature, embeddedData,
   });
@@ -90,33 +144,37 @@ module.exports = async (req, res) => {
 
   // Read the paper info from SS
   const paper = await fetchPaperInfo(paperId);
+  console.log('references count:', paper.references.length);
 
-  const citations = [];
-
-  if (paper.references) {
-    const eachCitationWeightPercent = Math.floor(30 / paper.references.length);
-    paper.references.forEach(async ({ paperId: citedPaperId }) => {
-      // We need to fetch this paper so we can
-      // We need to generate the post we would add to the forum, sign, and hash it.
+  const eachCitationWeightPercent = Math.floor(30 / paper.references.length);
+  const citations = await Promise.mapSeries(
+    paper.references.filter((x) => !!x.paperId),
+    async ({ paperId: citedPaperId }) => {
+      // We need to fetch this paper so we can generate the post we WOULD add to the forum.
+      // That way, if we later add the cited paper to the blockchain it will have the correct hash.
       // The forum allows dangling citations to support this use case.
       const citedPaper = await fetchPaperInfo(citedPaperId);
-      const citedPaperInfo = await generatePost(citedPaper);
-
-      citations.push({
+      const citedPost = await generatePost(citedPaper);
+      return {
         weightPercent: eachCitationWeightPercent,
-        targetPostId: citedPaperInfo.hash,
-      });
-    });
-    // Make sure citation weights sum to 100
-    const totalCitationWeight = citations.reduce((t, { weightPercent }) => t + weightPercent);
-    citations[0].weightPercent += 100 - totalCitationWeight;
-  }
+        targetPostId: citedPost.hash,
+      };
+    },
+  );
+
+  // Make sure citation weights sum to 100
+  const totalCitationWeight = citations.reduce((t, { weightPercent }) => t + weightPercent, 0);
+  citations[0].weightPercent += 100 - totalCitationWeight;
 
   // Create a post for this paper
   const {
     hash, authors, content, signature, embeddedData,
   } = await generatePost(paper);
 
+  console.log({
+    hash, authors, content, signature, embeddedData, citations,
+  });
+
   // Write the new post to our database
   await forum.put(hash, {
     authors, content, signature, embeddedData, citations,
@@ -125,9 +183,7 @@ module.exports = async (req, res) => {
   // Add the post to the form (on-chain)
   await dao.addPost(authors, hash, citations);
 
-  console.log({
-    authors, content, signature, embeddedData, citations,
-  });
+  console.log(`Added post to blockchain for paper ${paperId}`);
 
   res.end();
 };

backend/src/index.js

@@ -1,8 +1,8 @@
 const express = require('express');
 
-const read = require('./src/read');
-const write = require('./src/write');
-const importFromSS = require('./src/import-from-ss');
+const read = require('./read');
+const write = require('./write');
+const importFromSS = require('./import-from-ss');
 
 require('dotenv').config();
 

backend/src/verify-signature.js

@@ -5,12 +5,13 @@ const verifySignature = ({
 }) => {
   let contentToVerify = content;
   if (embeddedData && Object.entries(embeddedData).length) {
-    contentToVerify += `\n\nDATA\n${JSON.stringify(embeddedData, null, 2)}`;
+    contentToVerify += `\n\n${JSON.stringify(embeddedData, null, 2)}`;
   }
   try {
     const account = recoverPersonalSignature({ data: contentToVerify, signature });
-    const authorAddresses = authors.map((author) => author.authorAddress);
-    if (!authorAddresses.includes(account)) {
+    console.log(`recovered account: ${account}`);
+    const authorAddresses = authors.map((author) => author.authorAddress.toLowerCase());
+    if (!authorAddresses.includes(account.toLowerCase())) {
       console.log('error: signer is not among the authors');
       return false;
     }

ethereum/scripts/util/deploy-contract.js

@@ -15,9 +15,12 @@ const deployContract = async (name, args = [], isCore = false) => {
   contractAddresses[network][name] = contract.target;
 
   const from = `./artifacts/contracts/${isCore ? 'core/' : ''}${name}.sol/${name}.json`;
-  const to = `../frontend/src/assets/${name}.json`;
-  fs.copyFileSync(from, to);
-  console.log(`Copied ${fs.realpathSync(from)} to ${fs.realpathSync(to)}`);
+  const toFrontend = `../frontend/contractArtifacts/${name}.json`;
+  const toBackend = `../backend/contractArtifacts/${name}.json`;
+  fs.copyFileSync(from, toFrontend);
+  console.log(`Copied ${fs.realpathSync(from)} to ${fs.realpathSync(toFrontend)}`);
+  fs.copyFileSync(from, toBackend);
+  console.log(`Copied ${fs.realpathSync(from)} to ${fs.realpathSync(toBackend)}`);
 
   writeContractAddresses(contractAddresses);
 };

frontend/src/App.jsx

@@ -17,9 +17,9 @@ import './App.css';
 import useList from './utils/List';
 import { getContractAddressByChainId } from './utils/contract-config';
 import Web3Context from './contexts/Web3Context';
-import DAOArtifact from './assets/DAO.json';
-import Work1Artifact from './assets/Work1.json';
-import OnboardingArtifact from './assets/Onboarding.json';
+import DAOArtifact from '../contractArtifacts/DAO.json';
+import Work1Artifact from '../contractArtifacts/Work1.json';
+import OnboardingArtifact from '../contractArtifacts/Onboarding.json';
 import WorkContract from './components/work-contracts/WorkContract';
 import AddPostModal from './components/posts/AddPostModal';
 import ViewPostModal from './components/posts/ViewPostModal';

frontend/src/components/Proposals.jsx

@@ -7,7 +7,7 @@ import Button from 'react-bootstrap/esm/Button';
 import Stack from 'react-bootstrap/esm/Stack';
 import useList from '../utils/List';
 import Web3Context from '../contexts/Web3Context';
-import ProposalsArtifact from '../assets/Proposals.json';
+import ProposalsArtifact from '../../contractArtifacts/Proposals.json';
 import { getContractAddressByChainId } from '../utils/contract-config';
 import AddPostModal from './posts/AddPostModal';
 import ViewPostModal from './posts/ViewPostModal';

frontend/src/components/posts/AddPostModal.jsx

@@ -38,7 +38,7 @@ function AddPostModal({
   }, [provider, DAO, account, content, setShow, postToBlockchain, onSubmit]);
 
   return (
-    <Modal show={show} onHide={handleClose}>
+    <Modal className="modal" show={show} onHide={handleClose}>
       <Modal.Header closeButton>
         <Modal.Title>{title}</Modal.Title>
       </Modal.Header>

frontend/src/components/posts/ViewPostModal.jsx

@@ -7,41 +7,57 @@ function ViewPostModal({
   show, setShow, title, post,
 }) {
   const handleClose = () => setShow(false);
-  const { content, authors, embeddedData } = post;
+  const {
+    content, authors, embeddedData, citations,
+  } = post;
 
   const embeddedDataJson = JSON.stringify(embeddedData, null, 2);
 
   return (
-    <Modal show={show} onHide={handleClose}>
+    <Modal className="modal-lg" show={show} onHide={handleClose}>
       <Modal.Header closeButton>
         <Modal.Title>
           {title}
         </Modal.Title>
       </Modal.Header>
       <Modal.Body>
-        <h6>
-          Authors:
-          <Stack>
-            {authors?.map(({ authorAddress, weightPercent }) => (
-              <div key={authorAddress}>
-                {authorAddress}
-                {' '}
-                {weightPercent.toString()}
-                %
-              </div>
-            ))}
-          </Stack>
-
-        </h6>
+        <h5>Authors</h5>
+        <Stack>
+          {authors?.map(({ authorAddress, weightPercent }) => (
+            <div key={authorAddress}>
+              {authorAddress}
+              {' '}
+              {weightPercent.toString()}
+              %
+            </div>
+          ))}
+        </Stack>
         <hr />
         <p className="post-content">
           {content}
         </p>
-        {embeddedData && Object.entries(embeddedData).length && (
+        <hr />
+        {embeddedData && Object.entries(embeddedData).length > 0 && (
           <pre>
             {embeddedDataJson}
           </pre>
         )}
+        {citations && citations.length > 0 && (
+          <>
+            <hr />
+            <h5>Citations</h5>
+            <Stack>
+              {citations.map(({ weightPercent, targetPostId }) => (
+                <div key={targetPostId}>
+                  {targetPostId}
+                  {' '}
+                  {weightPercent.toString()}
+                  %
+                </div>
+              ))}
+            </Stack>
+          </>
+        )}
       </Modal.Body>
       <Modal.Footer>
         <Button variant="secondary" onClick={handleClose}>

frontend/src/components/work-contracts/ProposePriceChangeModal.jsx

@@ -9,7 +9,7 @@ import PropTypes from 'prop-types';
 import Web3 from 'web3';
 import Web3Context from '../../contexts/Web3Context';
 import Post from '../../utils/Post';
-import ProposalsArtifact from '../../assets/Proposals.json';
+import ProposalsArtifact from '../../../contractArtifacts/Proposals.json';
 import { getContractAddressByChainId } from '../../utils/contract-config';
 import WorkContractContext from '../../contexts/WorkContractContext';
 
@@ -34,7 +34,7 @@ function ProposePriceChangeModal({
   const handleClose = () => setShow(false);
 
   const handleSubmit = useCallback(async () => {
-    const post = new Post({ content });
+    const post = new Post({ content, authors: [{ weightPercent: 100, authorAddress: account }] });
     // Include price as embedded data
     post.embeddedData = { proposedPrice };
     // Include metamask signature

frontend/src/utils/Post.js

@@ -15,8 +15,8 @@ class Post {
     this.content = content;
     this.signature = signature;
     this.hash = hash;
-    this.embeddedData = embeddedData;
-    this.citations = citations;
+    this.embeddedData = embeddedData ?? {};
+    this.citations = citations ?? [];
   }
 
   // Read from API
@@ -39,8 +39,8 @@ class Post {
       contentToVerify += `\n\n${JSON.stringify(embeddedData, null, 2)}`;
     }
     const recovered = recoverPersonalSignature({ data: contentToVerify, signature });
-    const authorAddresses = authors.map((author) => author.authorAddress);
-    if (!authorAddresses.includes(recovered)) {
+    const authorAddresses = authors.map((author) => author.authorAddress.toLowerCase());
+    if (!authorAddresses.includes(recovered.toLowerCase())) {
       throw new Error('Signer is not among the authors');
     }
     return new Post({
@@ -48,13 +48,6 @@ class Post {
     });
   }
 
-  static deriveEmbeddedData(content) {
-    const dataStart = content.search(/^\{/);
-    const dataStr = content.substring(dataStart);
-    const embeddedData = JSON.parse(dataStr);
-    return embeddedData;
-  }
-
   // Include MetaMask signature
   async sign(web3Provider, account) {
     const author = this.authors?.find(({ authorAddress }) => authorAddress === account);
@@ -63,7 +56,7 @@ class Post {
     }
     let contentToSign = this.content;
     if (this.embeddedData && Object.entries(this.embeddedData).length) {
-      contentToSign += `\n\nDATA\n${JSON.stringify(this.embeddedData, null, 2)}`;
+      contentToSign += `\n\n${JSON.stringify(this.embeddedData, null, 2)}`;
     }
     const msg = `0x${Buffer.from(contentToSign, 'utf8').toString('hex')}`;
     this.signature = await web3Provider.request({