add feature: import papers by author
Gitea Actions Demo / Explore-Gitea-Actions (push) Failing after 31s Details

This commit is contained in:
Ladd Hoffman 2024-04-22 13:52:04 -05:00
parent 1f50a8acd7
commit 8b99e71f42
2 changed files with 51 additions and 26 deletions

View File

@ -37,11 +37,9 @@ const getContract = (name) => new ethers.Contract(
);
const dao = getContract('DAO');
const fetchPaperInfo = async (paperId, retryDelay = 5000) => {
const url = `https://api.semanticscholar.org/graph/v1/paper/${paperId}?`
+ 'fields=title,url,authors,references.title,references.url,references.authors';
const fetchWithRetry = async (url, retryDelay = 5000) => {
let retry = false;
let paper;
let responseData;
let requestError;
const response = await axios.get(url, {
headers: {
@ -63,11 +61,25 @@ const fetchPaperInfo = async (paperId, retryDelay = 5000) => {
await new Promise((resolve) => {
setTimeout(resolve, retryDelay);
});
paper = await fetchPaperInfo(paperId, retryDelay * 2);
responseData = await fetchWithRetry(url, retryDelay * 2);
} else {
paper = response.data;
responseData = response.data;
}
return paper;
return responseData;
};
const fetchPaperInfo = async (paperId) => {
const url = `https://api.semanticscholar.org/graph/v1/paper/${paperId}?`
+ 'fields=title,url,authors,references.title,references.url,references.authors';
return fetchWithRetry(url);
};
const fetchAuthorPapers = async (authorId) => {
const url = `https://api.semanticscholar.org/graph/v1/author/${authorId}/papers?`
+ 'fields=title,url,authors,references.title,references.url,references.authors';
const responseData = await fetchWithRetry(url);
// TODO: pagination
return responseData.data;
};
const getOrCreateAuthor = async (paper) => Promise.mapSeries(
@ -141,20 +153,9 @@ HREF ${paper.url}`;
};
};
module.exports = async (req, res) => {
const {
body: {
paperId,
},
} = req;
console.log(`importFromSS ${paperId}`);
// Read the paper info from SS
const paper = await fetchPaperInfo(paperId);
const importPaper = async (paper) => {
console.log('references count:', paper.references.length);
const { paperId } = paper;
const references = paper.references.filter((x) => !!x.paperId);
const eachCitationWeightPercent = Math.floor(PPM_TO_CITATIONS / references.length);
const citations = await Promise.mapSeries(
@ -172,8 +173,10 @@ module.exports = async (req, res) => {
);
// Make sure citation weights sum to the designated total
const totalCitationWeight = citations.reduce((t, { weightPPM }) => t + weightPPM, 0);
citations[0].weightPPM += PPM_TO_CITATIONS - totalCitationWeight;
if (citations.length) {
const totalCitationWeight = citations.reduce((t, { weightPPM }) => t + weightPPM, 0);
citations[0].weightPPM += PPM_TO_CITATIONS - totalCitationWeight;
}
// Create a post for this paper
const {
@ -191,12 +194,34 @@ module.exports = async (req, res) => {
} catch (e) {
if (e.reason === 'A post with this contentId already exists') {
console.log(`Post already added for paper ${paperId}`);
res.json({ alreadyAdded: true, postId: hash });
return;
return { paperId, postId: hash, alreadyAdded: true };
}
throw e;
}
console.log(`Added post to blockchain for paper ${paperId}`);
res.json({ postId: hash });
return { paperId, postId: hash };
};
module.exports = async (req, res) => {
const {
body: {
paperId,
authorId,
},
} = req;
if (paperId) {
console.log(`importFromSS: paper ${paperId}`);
const paper = await fetchPaperInfo(paperId);
const result = await importPaper(paper);
res.json(result);
} else if (authorId) {
console.log(`importFromSS: author ${authorId}`);
const papers = await fetchAuthorPapers(authorId);
console.log('papers count:', papers.length);
const result = await Promise.mapSeries(papers, importPaper);
console.log(`Added posts for ${result.length} papers by author ${authorId}`);
res.json(result);
}
};

View File

@ -24,7 +24,7 @@ app.get('*', (req, res) => {
app.use((err, req, res, next) => {
const status = err.response?.status ?? 500;
const message = err.response?.data?.error ?? err.message;
console.error(`error: ${message}`);
console.error(`error: ${message}`, err);
res.status(status).send(message);
next();
});