Collect contribution data for pull request reviews

This commit is contained in:
galfert 2021-01-12 15:32:06 +01:00
parent ad0e34b990
commit 93d7c8944b
No known key found for this signature in database
GPG Key ID: 942831AA5471AB49
8 changed files with 562 additions and 4 deletions

2
.env.example Normal file
View File

@ -0,0 +1,2 @@
GITEA_TOKEN=your-token-here
GITHUB_TOKEN=your-token-here

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
node_modules
wallet.json
.env

150
package-lock.json generated
View File

@ -928,6 +928,44 @@
"resolved": "https://registry.npmjs.org/class-is/-/class-is-1.1.0.tgz",
"integrity": "sha512-rhjH9AG1fvabIDoGRVH587413LPjTZgmDF9fOFCbFJQV4yuocX1mHxxvXI4g3cGwbVY9wAYIoKlg1N79frJKQw=="
},
"cliui": {
"version": "7.0.4",
"resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz",
"integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==",
"requires": {
"string-width": "^4.2.0",
"strip-ansi": "^6.0.0",
"wrap-ansi": "^7.0.0"
},
"dependencies": {
"ansi-regex": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
"integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg=="
},
"strip-ansi": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
"integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
"requires": {
"ansi-regex": "^5.0.0"
}
}
}
},
"color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
"requires": {
"color-name": "~1.1.4"
}
},
"color-name": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
},
"colors": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/colors/-/colors-1.1.2.tgz",
@ -1117,6 +1155,11 @@
"repeating": "^2.0.0"
}
},
"dotenv": {
"version": "8.2.0",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz",
"integrity": "sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw=="
},
"drbg.js": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/drbg.js/-/drbg.js-1.0.1.tgz",
@ -1146,6 +1189,11 @@
"minimalistic-crypto-utils": "^1.0.0"
}
},
"emoji-regex": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
},
"encodeurl": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
@ -1156,6 +1204,11 @@
"resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz",
"integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA=="
},
"escalade": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
"integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw=="
},
"escape-html": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
@ -1434,6 +1487,11 @@
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
},
"get-caller-file": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="
},
"get-iterator": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/get-iterator/-/get-iterator-1.0.2.tgz",
@ -1905,6 +1963,11 @@
"number-is-nan": "^1.0.0"
}
},
"is-fullwidth-code-point": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="
},
"is-ip": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/is-ip/-/is-ip-3.1.0.tgz",
@ -3146,6 +3209,11 @@
}
}
},
"require-directory": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
"integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I="
},
"revalidator": {
"version": "0.1.8",
"resolved": "https://registry.npmjs.org/revalidator/-/revalidator-0.1.8.tgz",
@ -3369,6 +3437,31 @@
"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
"integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow="
},
"string-width": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz",
"integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==",
"requires": {
"emoji-regex": "^8.0.0",
"is-fullwidth-code-point": "^3.0.0",
"strip-ansi": "^6.0.0"
},
"dependencies": {
"ansi-regex": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
"integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg=="
},
"strip-ansi": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
"integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
"requires": {
"ansi-regex": "^5.0.0"
}
}
}
},
"string_decoder": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz",
@ -3560,6 +3653,39 @@
"resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz",
"integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc="
},
"wrap-ansi": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
"integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
"requires": {
"ansi-styles": "^4.0.0",
"string-width": "^4.1.0",
"strip-ansi": "^6.0.0"
},
"dependencies": {
"ansi-regex": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
"integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg=="
},
"ansi-styles": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
"requires": {
"color-convert": "^2.0.1"
}
},
"strip-ansi": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
"integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
"requires": {
"ansi-regex": "^5.0.0"
}
}
}
},
"wrappy": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
@ -3585,6 +3711,30 @@
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
"integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68="
},
"y18n": {
"version": "5.0.5",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.5.tgz",
"integrity": "sha512-hsRUr4FFrvhhRH12wOdfs38Gy7k2FFzB9qgN9v3aLykRq0dRcdcpz5C9FxdS2NuhOrI/628b/KSTJ3rwHysYSg=="
},
"yargs": {
"version": "16.2.0",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz",
"integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==",
"requires": {
"cliui": "^7.0.2",
"escalade": "^3.1.1",
"get-caller-file": "^2.0.5",
"require-directory": "^2.1.1",
"string-width": "^4.2.0",
"y18n": "^5.0.5",
"yargs-parser": "^20.2.2"
}
},
"yargs-parser": {
"version": "20.2.4",
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.4.tgz",
"integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA=="
}
}
}

View File

@ -7,13 +7,15 @@
"test": "echo \"Error: no test specified\" && exit 1"
},
"bin": {
"create-wallet": "scripts/create-wallet.js"
"create-wallet": "scripts/create-wallet.js",
"review-kredits": "scripts/review-kredits.js"
},
"dependencies": {
"@ethersproject/experimental": "5.0.0",
"cors": "^2.8.5",
"dotenv": "^8.2.0",
"eth-provider": "^0.2.2",
"ethers": "^5.0.5",
"@ethersproject/experimental": "5.0.0",
"express": "^4.17.1",
"express-session": "^1.16.2",
"grant-express": "^4.6.1",
@ -22,7 +24,8 @@
"kredits-contracts": "^6.0.0",
"node-cron": "^2.0.3",
"node-fetch": "^2.3.0",
"prompt": "^1.0.0"
"prompt": "^1.0.0",
"yargs": "^16.2.0"
},
"repository": {
"type": "git",
@ -36,7 +39,8 @@
"author": "Kosmos Developers <mail@kosmos.org>",
"contributors": [
"Sebastian Kippe <sebastian@kip.pe>",
"Michael Bumann <hello@michaelbumann.com>"
"Michael Bumann <hello@michaelbumann.com>",
"Garret Alfert <alfert@wevelop.de>"
],
"license": "MIT",
"bugs": {

25
repos.json Normal file
View File

@ -0,0 +1,25 @@
{
"github": [
"67P/botka",
"67P/hal8000",
"67P/hubot-kredits",
"67P/hubot-remotestorage-logger",
"67P/hyperchannel",
"67P/kosmos-schemas",
"67P/kredits-contracts",
"67P/kredits-signup",
"67P/kredits-web",
"67P/remotestorage-module-kosmos",
"67P/waves",
"sockethub/sockethub"
],
"gitea": [
"kosmos/akkounts",
"kosmos/chef",
"kosmos/gitea.kosmos.org",
"kosmos/ipfs-cookbook",
"kosmos/kredits-ipfs-pinner",
"kosmos/website",
"kosmos/wormhole"
]
}

View File

@ -0,0 +1,98 @@
const fetch = require('node-fetch');
module.exports = class GiteaReviews {
token = null;
kreditsAmounts = null;
pageLimit = 100;
constructor (token, kreditsAmounts) {
this.token = token;
this.kreditsAmounts = kreditsAmounts;
}
async request (path) {
return fetch(
`https://gitea.kosmos.org/api/v1${path}`,
{
headers: {
'accepts': 'application/json',
'Authorization': `token ${this.token}`
}
}
).then(response => response.json());
}
async getReviewContributions (repos, startDate, endDate) {
let pulls = [];
let reviewContributions = {}
await Promise.all(repos.map(async (repo) => {
let page = 1;
let result;
do {
try {
result = await this.request(`/repos/${repo}/pulls?state=closed&limit=${this.pageLimit}&page=${page}`);
} catch(error) {
console.log(`failed to fetch PRs for repo ${repo}:`, error.message);
continue;
}
if (!result || result.length === 0) {
continue;
}
let pullRequests = result.filter(pr => {
if (!pr.merged) return false; // only interested in merged PRs
// check if the PR has been merged in the given timeframe
const mergeDate = new Date(pr.merged_at);
if (mergeDate < startDate || mergeDate > endDate) return false;
// check if the PR has a kredits label
return pr.labels.some(label => label.name.match(/kredits-[123]/));
});
await Promise.all(pullRequests.map(async (pr) => {
let reviews;
try {
reviews = await this.request(`/repos/${repo}/pulls/${pr.number}/reviews`);
} catch(error) {
console.log(`failed to fetch reviews for repo ${repo}, PR ${pr.number}:`, error.message);
return;
}
if (!reviews || reviews.length === 0) {
return;
}
reviews = reviews.filter(review => {
return ['APPROVED', 'REJECTED'].includes(review.state);
});
reviews.forEach(review => {
if (typeof reviewContributions[review.user.login] === 'undefined') {
reviewContributions[review.user.login] = [];
}
let kreditsLabel = pr.labels.find(label => label.name.match(/kredits-[123]/));
reviewContributions[review.user.login].push({
pr,
prNumber: pr.number,
review,
reviewState: review.state,
kredits: this.kreditsAmounts[kreditsLabel.name]
});
});
}));
page++;
} while (result && result.length > 0);
}));
return reviewContributions;
}
}

View File

@ -0,0 +1,99 @@
const fetch = require('node-fetch');
module.exports = class GithubReviews {
token = null;
kreditsAmounts = null;
pageLimit = 100;
constructor (token, kreditsAmounts) {
this.token = token;
this.kreditsAmounts = kreditsAmounts;
}
async request (path) {
return fetch(
`https://api.github.com${path}`,
{
headers: {
'Accept': 'application/vnd.github.v3+json',
'User-Agent': 'Kosmos Kredits for reviews',
'Authorization': `token ${this.token}`
}
}
).then(response => response.json());
}
async getReviewContributions (repos, startDate, endDate) {
let pulls = [];
let reviewContributions = {}
await Promise.all(repos.map(async (repo) => {
let page = 1;
let result;
do {
try {
result = await this.request(`/repos/${repo}/pulls?state=closed&perPage=${this.pageLimit}&page=${page}`);
} catch(error) {
console.log(`failed to fetch PRs for repo ${repo}:`, error.message);
continue;
}
if (!result || result.length === 0) {
continue;
}
let pullRequests = result.filter(pr => {
if (!pr.merged_at) return false; // only interested in merged PRs
// check if the PR has been merged in the given timeframe
const mergeDate = new Date(pr.merged_at);
if (mergeDate < startDate || mergeDate > endDate) return false;
// check if the PR has a kredits label
return pr.labels.some(label => label.name.match(/kredits-[123]/));
});
await Promise.all(pullRequests.map(async (pr) => {
let reviews;
try {
reviews = await this.request(`/repos/${repo}/pulls/${pr.number}/reviews`);
} catch(error) {
console.log(`failed to fetch reviews for repo ${repo}, PR ${pr.number}:`, error.message);
return;
}
if (!reviews || reviews.length === 0) {
return;
}
reviews = reviews.filter(review => {
return ['APPROVED', 'REJECTED'].includes(review.state);
});
reviews.forEach(review => {
if (typeof reviewContributions[review.user.login] === 'undefined') {
reviewContributions[review.user.login] = [];
}
let kreditsLabel = pr.labels.find(label => label.name.match(/kredits-[123]/));
reviewContributions[review.user.login].push({
pr,
prNumber: pr.number,
review,
reviewState: review.state,
kredits: this.kreditsAmounts[kreditsLabel.name]
});
});
}));
page++;
} while (result && result.length > 0);
}));
return reviewContributions;
}
}

179
scripts/review-kredits.js Executable file
View File

@ -0,0 +1,179 @@
#!/usr/bin/env node
require('dotenv').config({ path: '../.env' });
const GiteaReviews = require('./lib/gitea-reviews');
const GithubReviews = require('./lib/github-reviews');
const ethers = require('ethers');
const Kredits = require('kredits-contracts');
const util = require('util');
const yargs = require('yargs/yargs')
const { hideBin } = require('yargs/helpers')
const providerUrl = process.env.KREDITS_PROVIDER_URL;
const daoAddress = process.env.KREDITS_DAO_ADDRESS;
const ipfsConfig = {
host: process.env.IPFS_API_HOST || 'localhost',
port: process.env.IPFS_API_PORT || '5001',
protocol: process.env.IPFS_API_PROTOCOL || 'http'
};
const kreditsAmounts = {
'kredits-1': 100,
'kredits-2': 300,
'kredits-3': 1000
};
const repos = require('../repos.json');
const argv = yargs(hideBin(process.argv))
.option('start', {
alias: 's',
description: 'Include reviews for PRs merged after this date'
})
.option('end', {
alias: 'e',
description: 'Include reviews for PRs merged before this date'
})
.option('dry', {
alias: 'd',
type: 'boolean',
description: 'Only list contribution details without creating them'
})
.help()
.version()
.demandOption('start', 'Please provide a start date')
.default('end', function now () {
return (new Date()).toISOString().split('.')[0]+"Z";
})
.example([
['$0 --start 2020-11-01 --end 2020-11-30T23:59:59Z', 'Create contributions for reviews of pull requests merged in November 2020'],
['$0 --start 2021-01-01', 'Create contributions for reviews of pull requests merged from Januar 2021 until now'],
])
.argv
const startTimestamp = Date.parse(argv.start);
const endTimestamp = Date.parse(argv.end);
if (isNaN(startTimestamp)) {
console.log('The provided start date is invalid');
process.exit(1);
}
if (isNaN(endTimestamp)) {
console.log('The provided end date is invalid');
process.exit(1);
}
const startDate = new Date(startTimestamp);
const endDate = new Date(endTimestamp);
async function getAllReviews(repos, startDate, endDate) {
const githubReviews = new GithubReviews(process.env.GITHUB_TOKEN, kreditsAmounts);
const giteaReviews = new GiteaReviews(process.env.GITEA_TOKEN, kreditsAmounts);
return Promise.all([
githubReviews.getReviewContributions(repos.github, startDate, endDate),
giteaReviews.getReviewContributions(repos.gitea, startDate, endDate)
]).then(reviews => {
// console.log(util.inspect(reviews[0], { depth: 3, colors: true }));
return { github: reviews[0], gitea: reviews[1] }
});
}
async function initializeKredits () {
//
// Ethereum provider/node setup
//
let ethProvider;
if (providerUrl) {
ethProvider = new ethers.providers.JsonRpcProvider(providerUrl);
} else {
ethProvider = new ethers.getDefaultProvider('rinkeby');
}
//
// Kredits contracts setup
//
const opts = { ipfsConfig };
if (daoAddress) {
opts.addresses = { Kernel: daoAddress };
}
let kredits;
try {
kredits = await new Kredits(ethProvider, null, opts).init();
} catch(error) {
console.log('Could not set up kredits:', error);
process.exit(1);
}
return kredits;
}
async function generateContributionData(reviews, Contributor) {
const contributors = await Contributor.all();
const contributionData = {};
const now = (new Date()).toISOString().split('.')[0]+"Z";
[date, time] = now.split('T');
function addContributionDataForPlatform(platform) {
for (const [username, platformReviews] of Object.entries(reviews[platform])) {
const contributor = contributors.find(c => {
return c[`${platform}_username`] === username;
});
if (!contributor) {
console.log(`Could not find contributor for ${platform} user "${username}"`);
continue;
}
const urls = platformReviews.map(review => review.pr.html_url);
const kreditsAmount = platformReviews.reduce((amount, review) => {
return review.kredits + amount;
}, 0);
if (typeof contributionData[contributor.name] !== 'undefined') {
contributionData[contributor.name].amount += kreditsAmount;
contributionData[contributor.name].details.pullRequests.push(...urls);
} else {
contributionData[contributor.name] = {
contributorId: contributor.id,
contributorIpfsHash: contributor.ipfsHash,
date,
time,
amount: kreditsAmount,
kind: 'dev',
description: 'PR reviews', // TODO add timeframe to description
details: {
'pullRequests': urls
}
}
}
}
}
addContributionDataForPlatform('gitea');
addContributionDataForPlatform('github');
return contributionData;
}
Promise.all([initializeKredits(), getAllReviews(repos, startDate, endDate)]).then((values) => {
const kredits = values[0];
const reviews = values[1];
generateContributionData(reviews, kredits.Contributor).then(contributionData => {
if (argv.dry) {
console.log('contributions:');
console.log(util.inspect(contributionData, { depth: 3, colors: true }));
}
// TODO create contributions
});
});