Compare commits
38 Commits
Author | SHA1 | Date |
---|---|---|
Râu Cao | bcb46c2b2b | |
Râu Cao | 7e7416a8fd | |
Râu Cao | b8fa61a434 | |
Râu Cao | 1aa2fac3cc | |
Râu Cao | b7d1600c59 | |
Râu Cao | ed50e84662 | |
Râu Cao | c7fe50ffa9 | |
Râu Cao | 83b3045afd | |
Râu Cao | 3c7bcf038e | |
Râu Cao | ca0b870315 | |
Râu Cao | 0a21c57251 | |
Râu Cao | 1401adbf0c | |
Râu Cao | 2401d8dff2 | |
Râu Cao | 45c0aed4a5 | |
Râu Cao | f25fcaa1b2 | |
Râu Cao | 7c8f0814b9 | |
Râu Cao | b566a432a9 | |
Greg | b7a7214f08 | |
Greg | 6829a13a57 | |
Râu Cao | 328905814d | |
Râu Cao | 03ee17f00e | |
Râu Cao | 9dffb15f45 | |
Râu Cao | c21512fa0b | |
Râu Cao | 4888d1be78 | |
Râu Cao | 0ac7228bfe | |
Râu Cao | 77c2cea4a4 | |
Râu Cao | 3c02591061 | |
Râu Cao | 9d7760edef | |
Râu Cao | 4a3988137e | |
Râu Cao | e12e798735 | |
Râu Cao | fee6b5c121 | |
Râu Cao | d8dda84fd6 | |
Basti | ba7c98b564 | |
bumi | 3f03368e25 | |
Basti | a3d21ebb30 | |
Basti | b11dbb9fde | |
gregkare | bfb51d48bc | |
Basti | f2f884a317 |
|
@ -0,0 +1,14 @@
|
||||||
|
name-template: 'v$RESOLVED_VERSION'
|
||||||
|
tag-template: 'v$RESOLVED_VERSION'
|
||||||
|
version-resolver:
|
||||||
|
major:
|
||||||
|
labels:
|
||||||
|
- release/major
|
||||||
|
minor:
|
||||||
|
labels:
|
||||||
|
- release/minor
|
||||||
|
- feature
|
||||||
|
patch:
|
||||||
|
labels:
|
||||||
|
- release/patch
|
||||||
|
default: patch
|
|
@ -0,0 +1,11 @@
|
||||||
|
name: Release Drafter
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
types: [closed]
|
||||||
|
jobs:
|
||||||
|
release_drafter_job:
|
||||||
|
name: Update release notes draft
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Release Drafter
|
||||||
|
uses: https://github.com/raucao/gitea-release-drafter@dev
|
26
README.md
26
README.md
|
@ -1,6 +1,8 @@
|
||||||
|
[![npm](https://img.shields.io/npm/v/@kredits/ipfs-pinner.svg)](https://www.npmjs.com/package/@kredits/ipfs-pinner)
|
||||||
|
|
||||||
# Kredits IPFS Pinner
|
# Kredits IPFS Pinner
|
||||||
|
|
||||||
This tool pins the IPFS hashes of a Kredits organisation on an IPFS node.
|
This tool pins the IPFS documents of a Kredits organization on any IPFS node.
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
|
@ -9,11 +11,11 @@ config flags, in case it is not running on localhost with default ports.)
|
||||||
|
|
||||||
### With NPX magic
|
### With NPX magic
|
||||||
|
|
||||||
npx @kosmos/kredits-ipfs-pinner
|
npx @kredits/ipfs-pinner
|
||||||
|
|
||||||
### Global install
|
### Global install
|
||||||
|
|
||||||
npm install -g @kosmos/kredits-ipfs-pinner
|
npm install -g @kredits/ipfs-pinner
|
||||||
kredits-ipfs-pinner
|
kredits-ipfs-pinner
|
||||||
|
|
||||||
### From repo
|
### From repo
|
||||||
|
@ -25,3 +27,21 @@ Clone the Git repository, then use npm scripts from its root directory.
|
||||||
Or with debug output:
|
Or with debug output:
|
||||||
|
|
||||||
npm run debug
|
npm run debug
|
||||||
|
|
||||||
|
### CLI options
|
||||||
|
|
||||||
|
Display progress bars:
|
||||||
|
|
||||||
|
kredits-ipfs-pinner --progress
|
||||||
|
|
||||||
|
Or with npm:
|
||||||
|
|
||||||
|
npm start -- --progress
|
||||||
|
|
||||||
|
Keep running and watch for new data/documents:
|
||||||
|
|
||||||
|
kredits-ipfs-pinner --watch
|
||||||
|
|
||||||
|
Multiple options at once:
|
||||||
|
|
||||||
|
kredits-ipfs-pinner --progress --watch
|
||||||
|
|
|
@ -0,0 +1,6 @@
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"ID": "QmRw21aC3TroRVdZhrE2Qh4W6PBA67kbE8p3fNfsVcfW8D",
|
||||||
|
"Addrs": ["/dns4/draco.kosmos.org/tcp/4001"]
|
||||||
|
}
|
||||||
|
]
|
90
index.js
90
index.js
|
@ -1,21 +1,33 @@
|
||||||
#!/usr/bin/env node
|
#!/usr/bin/env node
|
||||||
|
|
||||||
const Kredits = require('kredits-contracts');
|
const fs = require('fs');
|
||||||
|
const debug = require('debug')('ipfs-pinner');
|
||||||
|
const Kredits = require('@kredits/contracts');
|
||||||
const IpfsPinner = require('./lib/ipfs-pinner');
|
const IpfsPinner = require('./lib/ipfs-pinner');
|
||||||
|
const defaultPeers = JSON.parse(fs.readFileSync('./config/peers.json'));
|
||||||
|
|
||||||
const argv = require('yargs')
|
const argv = require('yargs')
|
||||||
.default({ network: 'rinkeby', apm: 'open.aragonpm.eth', host: 'localhost', port: '5001', protocol: 'http', monitor: true })
|
.default({
|
||||||
.boolean('monitor')
|
rpcUrl: 'https://rsk-testnet.kosmos.org',
|
||||||
|
host: 'localhost',
|
||||||
|
port: '5001',
|
||||||
|
protocol: 'http',
|
||||||
|
watch: false,
|
||||||
|
progress: false,
|
||||||
|
bootstrapNode: `${defaultPeers[0].Addrs[0]}/ipfs/${defaultPeers[0].ID}`
|
||||||
|
})
|
||||||
|
.boolean('watch')
|
||||||
|
.boolean('progress')
|
||||||
.describe({
|
.describe({
|
||||||
network: 'Ethereum network to connect to',
|
rpcUrl: 'Web3/EVM node RPC URL; alternative to --network',
|
||||||
rpcUrl: 'Ethereum node RPC URL; alternative to --network',
|
|
||||||
daoAddress: 'Optional Kredits DAO address',
|
|
||||||
host: 'IPFS API host',
|
host: 'IPFS API host',
|
||||||
port: 'IPFS API port',
|
port: 'IPFS API port',
|
||||||
protocol: 'IPFS API protocol',
|
protocol: 'IPFS API protocol',
|
||||||
monitor: 'Monitor contract events for new IPFS documents'
|
watch: 'Monitor contract events for new IPFS documents',
|
||||||
|
progress: 'Show progress bars',
|
||||||
|
bootstrapNode: 'IPFS node address to connect to before fetching documents'
|
||||||
})
|
})
|
||||||
.example('$0 --network rinkeby --host localhost', 'Pins all existing IPFS documents to the IPFS API running on localhost and monitors for new events.')
|
.example('$0 --host localhost', 'Pins all existing IPFS documents to the IPFS API running on localhost and monitors for new events')
|
||||||
.argv;
|
.argv;
|
||||||
|
|
||||||
const ipfsConfig = {
|
const ipfsConfig = {
|
||||||
|
@ -24,33 +36,55 @@ const ipfsConfig = {
|
||||||
protocol: argv.protocol
|
protocol: argv.protocol
|
||||||
};
|
};
|
||||||
|
|
||||||
console.log(`Using IPFS:`, ipfsConfig);
|
debug(`IPFS node:`, ipfsConfig);
|
||||||
|
|
||||||
(async () => {
|
(async () => {
|
||||||
try {
|
const kredits = await Kredits.for(
|
||||||
const kredits = await Kredits.for(
|
{ rpcUrl: argv.rpcUrl },
|
||||||
{ network: argv.network, rpcUrl: argv.rpcUrl },
|
{ ipfsConfig: ipfsConfig }
|
||||||
{ apm: argv.apm, ipfsConfig: ipfsConfig, addresses: { Kernel: argv.daoAddress } }
|
).init().catch(e => {
|
||||||
).init();
|
console.log('Failed to initialize Kredits:');
|
||||||
|
console.log(e.message);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
|
||||||
// check the connection to the IPFS client
|
// TODO redesign IPFS wrapper API and do not use an internal attribute
|
||||||
// TODO redesign IPFS wrapper API and do not use an internal attribute
|
const ipfsApi = kredits.ipfs._ipfsAPI;
|
||||||
await kredits.ipfs._ipfsAPI.id();
|
let connectBootstrapNode = true;
|
||||||
|
|
||||||
const ipfsPinner = new IpfsPinner(kredits);
|
await ipfsApi.id().then(res => {
|
||||||
|
if (res.id === argv.bootstrapNode.split('/')[argv.bootstrapNode.split('/').length-1]) {
|
||||||
|
connectBootstrapNode = false;
|
||||||
|
}
|
||||||
|
}).catch(e => {
|
||||||
|
console.log('Failed to initialize IPFS:');
|
||||||
|
console.log(e.message);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
|
||||||
ipfsPinner.pinAll().then(pins => {
|
if (connectBootstrapNode) {
|
||||||
console.log(`Pinned ${pins.length} existing documents`);
|
debug(`Connecting to known IPFS node ${argv.bootstrapNode}`);
|
||||||
});
|
await ipfsApi.swarm.connect(argv.bootstrapNode);
|
||||||
|
}
|
||||||
|
|
||||||
ipfsPinner.monitor(pin => {
|
const ipfsPinner = new IpfsPinner(kredits, {
|
||||||
|
progress: argv.progress
|
||||||
|
});
|
||||||
|
|
||||||
|
await ipfsPinner.pinAll().then(cids => {
|
||||||
|
console.log(`\nSuccessfully pinned ${cids.length} documents`)
|
||||||
|
});
|
||||||
|
|
||||||
|
if (argv.watch) {
|
||||||
|
console.log('\nWatching contract events for new documents...');
|
||||||
|
|
||||||
|
ipfsPinner.watch(pin => {
|
||||||
console.log('Pinned a new document:', pin[0]["hash"]);
|
console.log('Pinned a new document:', pin[0]["hash"]);
|
||||||
});
|
});
|
||||||
|
} else {
|
||||||
console.log(`Subscribed to DAO: ${kredits.Kernel.contract.address}`);
|
process.exit(0);
|
||||||
} catch(e) {
|
|
||||||
console.log('Failed to start');
|
|
||||||
console.log(e);
|
|
||||||
process.exit(1);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO Add new deployment/DAO/org ID or all contract proxy addresses
|
||||||
|
// console.log(`Subscribed to DAO: ${kredits.Kernel.contract.address}`);
|
||||||
})();
|
})();
|
||||||
|
|
|
@ -1,21 +1,61 @@
|
||||||
const debug = require('debug')('ipfs-pinner');
|
const debug = require('debug')('ipfs-pinner');
|
||||||
|
const cliProgress = require('cli-progress');
|
||||||
|
|
||||||
|
function sleep(ms) {
|
||||||
|
return new Promise(resolve => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
|
||||||
class IpfsPinner {
|
class IpfsPinner {
|
||||||
constructor (kredits, ipfsApi) {
|
constructor (kredits, options={}) {
|
||||||
this.kredits = kredits;
|
this.kredits = kredits;
|
||||||
this.ipfsApi = ipfsApi || this.kredits.ipfs;
|
this.ipfsApi = this.kredits.ipfs;
|
||||||
|
this.progressBars = !!options.progress && !process.env.DEBUG;
|
||||||
|
|
||||||
|
if (this.progressBars) {
|
||||||
|
this.multibar = new cliProgress.MultiBar({
|
||||||
|
stopOnComplete: true,
|
||||||
|
clearOnComplete: false,
|
||||||
|
hideCursor: false,
|
||||||
|
etaBuffer: 30,
|
||||||
|
format: '{entity} [{bar}] {percentage}% | ETA: {eta_formatted} | {value}/{total}'
|
||||||
|
}, cliProgress.Presets.shades_grey);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async pinAll () {
|
async pinAll () {
|
||||||
const contributorHashes = await this._pinAllFromContract(this.kredits.Contributor);
|
console.log('Pinning IPFS documents for all known items...\n')
|
||||||
const contributionHashes = await this._pinAllFromContract(this.kredits.Contribution);
|
const cids = [];
|
||||||
const proposalHashes = await this._pinAllFromContract(this.kredits.Proposal);
|
const promises = [];
|
||||||
|
const contracts = [
|
||||||
|
this.kredits.Contributor,
|
||||||
|
this.kredits.Contribution,
|
||||||
|
this.kredits.Reimbursement
|
||||||
|
]
|
||||||
|
|
||||||
return contributorHashes.concat(contributionHashes)
|
for (const contract of contracts) {
|
||||||
.concat(proposalHashes);
|
debug(`Pinning data from ${contract.constructor.name}...`);
|
||||||
|
const itemCount = await contract.count;
|
||||||
|
debug(`${contract.constructor.name} item count:`, itemCount);
|
||||||
|
let bar;
|
||||||
|
|
||||||
|
if (this.progressBars) {
|
||||||
|
bar = this.multibar.create(itemCount, 0);
|
||||||
|
bar.update(0, {entity: `${contract.constructor.name}s`.padEnd(14)});
|
||||||
|
}
|
||||||
|
|
||||||
|
promises.push(this._pinAllFromContract(contract, itemCount, bar)
|
||||||
|
.then(res => { cids.push(...res); }));
|
||||||
|
}
|
||||||
|
|
||||||
|
await Promise.all(promises);
|
||||||
|
|
||||||
|
// Avoid console output race condition with progress bars finishing update
|
||||||
|
if (this.progressBars) await sleep(1000);
|
||||||
|
|
||||||
|
return cids;
|
||||||
}
|
}
|
||||||
|
|
||||||
monitor (callback) {
|
watch (callback) {
|
||||||
this.kredits.Contribution.on('ContributionAdded', (id) => {
|
this.kredits.Contribution.on('ContributionAdded', (id) => {
|
||||||
this.kredits.Contribution.getData(id)
|
this.kredits.Contribution.getData(id)
|
||||||
.then(data => { return this.ipfsApi.pin(data); })
|
.then(data => { return this.ipfsApi.pin(data); })
|
||||||
|
@ -31,21 +71,50 @@ class IpfsPinner {
|
||||||
.then(data => { return this.ipfsApi.pin(data); })
|
.then(data => { return this.ipfsApi.pin(data); })
|
||||||
.then(callback);
|
.then(callback);
|
||||||
});
|
});
|
||||||
}
|
this.kredits.Reimbursement.on('ReimbursementAdded', (id) => {
|
||||||
|
this.kredits.Reimbursement.getData(id)
|
||||||
_pinAllFromContract (contract) {
|
.then(data => { return this.ipfsApi.pin(data); })
|
||||||
debug(`Pinning data from ${contract.constructor.name}...`);
|
.then(callback);
|
||||||
return contract.count.then(count => {
|
|
||||||
let promises = [...Array(count).keys()].map(i => {
|
|
||||||
let id = i + 1; // 0 => 1 - ids start with 1 and not with 0
|
|
||||||
debug(`Loading ${contract.constructor.name} #${id}`);
|
|
||||||
return contract.getData(id).then(data => {
|
|
||||||
debug(`Pinning ${contract.constructor.name} #${id}`);
|
|
||||||
return this.ipfsApi.pin(data);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
return Promise.all(promises);
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async _pinAllFromContract (contract, itemCount, progressBar) {
|
||||||
|
const ipfsApi = this.ipfsApi;
|
||||||
|
const progressBars = this.progressBars;
|
||||||
|
const ids = [...Array(itemCount).keys()].map(i => i+1);
|
||||||
|
const cids = [];
|
||||||
|
const batchSize = 20;
|
||||||
|
let position = 0;
|
||||||
|
|
||||||
|
async function loadAndPin(id) {
|
||||||
|
let cid;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const data = await contract.getData(id);
|
||||||
|
debug(`Loaded ${contract.constructor.name} #${id}`);
|
||||||
|
cid = await ipfsApi.pin(data);
|
||||||
|
debug(`Pinned ${contract.constructor.name} #${id} at ${cid}`);
|
||||||
|
} catch(e) {
|
||||||
|
debug(`Error while trying to load and pin ${contract.constructor.name} #${id}:`)
|
||||||
|
debug(e);
|
||||||
|
debug(`\nTrying again...`);
|
||||||
|
loadAndPin(id);
|
||||||
|
} finally {
|
||||||
|
cids.push(cid);
|
||||||
|
if (progressBars) { progressBar.increment(); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
while (position < itemCount) {
|
||||||
|
const batchIds = ids.slice(position, position + batchSize);
|
||||||
|
|
||||||
|
await Promise.all(batchIds.map(async id => loadAndPin(id)));
|
||||||
|
|
||||||
|
position += batchSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
return cids;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = IpfsPinner;
|
module.exports = IpfsPinner;
|
||||||
|
|
File diff suppressed because it is too large
Load Diff
11
package.json
11
package.json
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "@kosmos/kredits-ipfs-pinner",
|
"name": "@kredits/ipfs-pinner",
|
||||||
"version": "1.0.3",
|
"version": "2.3.0",
|
||||||
"description": "Pins IPFS data of a Kredits organization on an IPFS node",
|
"description": "Pins IPFS data of a Kredits organization on an IPFS node",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
@ -20,9 +20,10 @@
|
||||||
],
|
],
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"debug": "^4.1.1",
|
"@kredits/contracts": "^7.4.0",
|
||||||
"kredits-contracts": "^5.4.0",
|
"cli-progress": "^3.11.2",
|
||||||
"yargs": "^13.2.4"
|
"debug": "^4.3.4",
|
||||||
|
"yargs": "^17.6.0"
|
||||||
},
|
},
|
||||||
"repository": {
|
"repository": {
|
||||||
"type": "git",
|
"type": "git",
|
||||||
|
|
Loading…
Reference in New Issue