Merge pull request 'Add CLI option to display progress bars' (#11) from feature/progress_bars into chore/upgrade_kredits
Reviewed-on: #11
This commit is contained in:
commit
6829a13a57
66
index.js
66
index.js
|
|
@ -12,16 +12,19 @@ const argv = require('yargs')
|
|||
host: 'localhost',
|
||||
port: '5001',
|
||||
protocol: 'http',
|
||||
monitor: true,
|
||||
watch: false,
|
||||
progress: false,
|
||||
bootstrapNode: `${defaultPeers[0].Addrs[0]}/ipfs/${defaultPeers[0].ID}`
|
||||
})
|
||||
.boolean('monitor')
|
||||
.boolean('watch')
|
||||
.boolean('progress')
|
||||
.describe({
|
||||
rpcUrl: 'Web3/EVM node RPC URL; alternative to --network',
|
||||
host: 'IPFS API host',
|
||||
port: 'IPFS API port',
|
||||
protocol: 'IPFS API protocol',
|
||||
monitor: 'Monitor contract events for new IPFS documents',
|
||||
watch: 'Monitor contract events for new IPFS documents',
|
||||
progress: 'Show progress bars',
|
||||
bootstrapNode: 'IPFS node address to connect to before fetching documents'
|
||||
})
|
||||
.example('$0 --host localhost', 'Pins all existing IPFS documents to the IPFS API running on localhost and monitors for new events')
|
||||
|
|
@ -33,39 +36,48 @@ const ipfsConfig = {
|
|||
protocol: argv.protocol
|
||||
};
|
||||
|
||||
console.log(`Using IPFS:`, ipfsConfig);
|
||||
debug(`IPFS node:`, ipfsConfig);
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
const kredits = await Kredits.for(
|
||||
{ rpcUrl: argv.rpcUrl },
|
||||
{ ipfsConfig: ipfsConfig }
|
||||
).init();
|
||||
const kredits = await Kredits.for(
|
||||
{ rpcUrl: argv.rpcUrl },
|
||||
{ ipfsConfig: ipfsConfig }
|
||||
).init().catch(e => {
|
||||
console.log('Failed to initialize Kredits:');
|
||||
console.log(e.message);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
// Check the connection to the IPFS client
|
||||
// TODO redesign IPFS wrapper API and do not use an internal attribute
|
||||
const ipfsApi = kredits.ipfs._ipfsAPI;
|
||||
// TODO redesign IPFS wrapper API and do not use an internal attribute
|
||||
const ipfsApi = kredits.ipfs._ipfsAPI;
|
||||
|
||||
await ipfsApi.id();
|
||||
await ipfsApi.id().catch(e => {
|
||||
console.log('Failed to initialize IPFS:');
|
||||
console.log(e.message);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
debug(`Connecting to known IPFS node ${argv.bootstrapNode}`);
|
||||
await ipfsApi.swarm.connect(argv.bootstrapNode);
|
||||
debug(`Connecting to known IPFS node ${argv.bootstrapNode}`);
|
||||
await ipfsApi.swarm.connect(argv.bootstrapNode);
|
||||
|
||||
const ipfsPinner = new IpfsPinner(kredits);
|
||||
const ipfsPinner = new IpfsPinner(kredits, {
|
||||
progress: argv.progress
|
||||
});
|
||||
|
||||
ipfsPinner.pinAll().then(pins => {
|
||||
console.log(`Pinned ${pins.length} existing documents`);
|
||||
});
|
||||
await ipfsPinner.pinAll().then(cids => {
|
||||
console.log(`\nSuccessfully pinned ${cids.length} documents`)
|
||||
});
|
||||
|
||||
ipfsPinner.monitor(pin => {
|
||||
if (argv.watch) {
|
||||
console.log('\nWatching contract events for new documents...');
|
||||
|
||||
ipfsPinner.watch(pin => {
|
||||
console.log('Pinned a new document:', pin[0]["hash"]);
|
||||
});
|
||||
|
||||
// TODO Add new deployment/DAO/org ID or all contract proxy addresses
|
||||
// console.log(`Subscribed to DAO: ${kredits.Kernel.contract.address}`);
|
||||
} catch(e) {
|
||||
console.log('Failed to start');
|
||||
console.log(e);
|
||||
process.exit(1);
|
||||
} else {
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// TODO Add new deployment/DAO/org ID or all contract proxy addresses
|
||||
// console.log(`Subscribed to DAO: ${kredits.Kernel.contract.address}`);
|
||||
})();
|
||||
|
|
|
|||
|
|
@ -1,32 +1,62 @@
|
|||
const debug = require('debug')('ipfs-pinner');
|
||||
const cliProgress = require('cli-progress');
|
||||
|
||||
async function promiseAllInBatches(task, items, batchSize) {
|
||||
let position = 0;
|
||||
let results = [];
|
||||
while (position < items.length) {
|
||||
const itemsForBatch = items.slice(position, position + batchSize);
|
||||
results = [...results, ...await Promise.allSettled(itemsForBatch.map(item => task(item)))];
|
||||
position += batchSize;
|
||||
}
|
||||
return results;
|
||||
function sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
class IpfsPinner {
|
||||
constructor (kredits, ipfsApi) {
|
||||
constructor (kredits, options={}) {
|
||||
this.kredits = kredits;
|
||||
this.ipfsApi = ipfsApi || this.kredits.ipfs;
|
||||
this.ipfsApi = this.kredits.ipfs;
|
||||
this.progressBars = !!options.progress && !process.env.DEBUG;
|
||||
|
||||
if (this.progressBars) {
|
||||
this.multibar = new cliProgress.MultiBar({
|
||||
stopOnComplete: true,
|
||||
clearOnComplete: false,
|
||||
hideCursor: false,
|
||||
etaBuffer: 30,
|
||||
format: '{entity} [{bar}] {percentage}% | ETA: {eta_formatted} | {value}/{total}'
|
||||
}, cliProgress.Presets.shades_grey);
|
||||
}
|
||||
}
|
||||
|
||||
async pinAll () {
|
||||
const contributorHashes = await this._pinAllFromContract(this.kredits.Contributor);
|
||||
const contributionHashes = await this._pinAllFromContract(this.kredits.Contribution);
|
||||
const reimbursementHashes = await this._pinAllFromContract(this.kredits.Reimbursement);
|
||||
console.log('Pinning IPFS documents for all known items...\n')
|
||||
const cids = [];
|
||||
const promises = [];
|
||||
const contracts = [
|
||||
this.kredits.Contributor,
|
||||
this.kredits.Contribution,
|
||||
// TODO uncomment once we have data here
|
||||
// this.kredits.Reimbursement
|
||||
]
|
||||
|
||||
return contributorHashes.concat(contributionHashes)
|
||||
.concat(reimbursementHashes);
|
||||
for (const contract of contracts) {
|
||||
debug(`Pinning data from ${contract.constructor.name}...`);
|
||||
const itemCount = await contract.count;
|
||||
debug(`${contract.constructor.name} item count:`, itemCount);
|
||||
let bar;
|
||||
|
||||
if (this.progressBars) {
|
||||
bar = this.multibar.create(itemCount, 0);
|
||||
bar.update(0, {entity: `${contract.constructor.name}s`.padEnd(14)});
|
||||
}
|
||||
|
||||
promises.push(this._pinAllFromContract(contract, itemCount, bar)
|
||||
.then(res => { cids.push(...res); }));
|
||||
}
|
||||
|
||||
await Promise.all(promises);
|
||||
|
||||
// Avoid console output race condition with progress bars finishing update
|
||||
if (this.progressBars) await sleep(1000);
|
||||
|
||||
return cids;
|
||||
}
|
||||
|
||||
monitor (callback) {
|
||||
watch (callback) {
|
||||
this.kredits.Contribution.on('ContributionAdded', (id) => {
|
||||
this.kredits.Contribution.getData(id)
|
||||
.then(data => { return this.ipfsApi.pin(data); })
|
||||
|
|
@ -44,22 +74,40 @@ class IpfsPinner {
|
|||
});
|
||||
}
|
||||
|
||||
async _pinAllFromContract (contract) {
|
||||
debug(`Pinning data from ${contract.constructor.name}...`);
|
||||
const count = await contract.count;
|
||||
debug('Item count:', count);
|
||||
const ids = [...Array(count).keys()].map(i => i+1);
|
||||
async _pinAllFromContract (contract, itemCount, progressBar) {
|
||||
const ipfsApi = this.ipfsApi;
|
||||
const progressBars = this.progressBars;
|
||||
const ids = [...Array(itemCount).keys()].map(i => i+1);
|
||||
const cids = [];
|
||||
const batchSize = 20;
|
||||
let position = 0;
|
||||
|
||||
async function loadAndPin (id) {
|
||||
debug(`Loading ${contract.constructor.name} #${id}`);
|
||||
return contract.getData(id).then(data => {
|
||||
debug(`Pinning ${contract.constructor.name} #${id}`);
|
||||
return this.ipfsApi.pin(data).then(cid => cids.push(cid));
|
||||
});
|
||||
async function loadAndPin(id) {
|
||||
let cid;
|
||||
|
||||
try {
|
||||
const data = await contract.getData(id);
|
||||
debug(`Loaded ${contract.constructor.name} #${id}`);
|
||||
cid = await ipfsApi.pin(data);
|
||||
debug(`Pinned ${contract.constructor.name} #${id} at ${cid}`);
|
||||
} catch(e) {
|
||||
debug(`Error while trying to load an pin ${contract.constructor.name} #${id}:`)
|
||||
debug(e);
|
||||
debug(`\nTrying again...`);
|
||||
loadAndPin(id);
|
||||
} finally {
|
||||
cids.push(cid);
|
||||
if (progressBars) { progressBar.increment(); }
|
||||
}
|
||||
}
|
||||
|
||||
await promiseAllInBatches(loadAndPin.bind(this), ids, 100);
|
||||
while (position < itemCount) {
|
||||
const batchIds = ids.slice(position, position + batchSize);
|
||||
|
||||
await Promise.all(batchIds.map(async id => loadAndPin(id)));
|
||||
|
||||
position += batchSize;
|
||||
}
|
||||
|
||||
return cids;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,8 @@
|
|||
"version": "1.2.0",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@kredits/contracts": "git+https://gitea.kosmos.org/kredits/contracts#6e0ec87",
|
||||
"@kredits/contracts": "7.0.0",
|
||||
"cli-progress": "^3.11.2",
|
||||
"debug": "^4.3.4",
|
||||
"yargs": "^17.6.0"
|
||||
},
|
||||
|
|
@ -720,9 +721,9 @@
|
|||
"integrity": "sha512-yOTK5WiXFDNAitPByMabE365aEEzFHgSUSgAssbJWt7BZ80HQSVu8XWrQiTbFbCkoIBmXwPP/RoxgXJQVgZTFQ=="
|
||||
},
|
||||
"node_modules/@kredits/contracts": {
|
||||
"version": "7.0.0-beta.0",
|
||||
"resolved": "git+https://gitea.kosmos.org/kredits/contracts#6e0ec8741e61b51fb5c9c636da4e8d3610d090ac",
|
||||
"license": "MIT",
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@kredits/contracts/-/contracts-7.0.0.tgz",
|
||||
"integrity": "sha512-UITEkP3njFNI2WS7v5ivGE3ruFwdWPWuJZrhBXBEAZbtmr1t/p1K7jkmmjyLDUeKXJ/udMlH6oQMCgh7P/aHNg==",
|
||||
"dependencies": {
|
||||
"@kosmos/schemas": "^3.1.0",
|
||||
"ethers": "^5.4.7",
|
||||
|
|
@ -930,6 +931,17 @@
|
|||
"cborg": "cli.js"
|
||||
}
|
||||
},
|
||||
"node_modules/cli-progress": {
|
||||
"version": "3.11.2",
|
||||
"resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.11.2.tgz",
|
||||
"integrity": "sha512-lCPoS6ncgX4+rJu5bS3F/iCz17kZ9MPZ6dpuTtI0KXKABkhyXIdYB3Inby1OpaGti3YlI3EeEkM9AuWpelJrVA==",
|
||||
"dependencies": {
|
||||
"string-width": "^4.2.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/cliui": {
|
||||
"version": "8.0.1",
|
||||
"resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz",
|
||||
|
|
@ -2233,8 +2245,9 @@
|
|||
"integrity": "sha512-yOTK5WiXFDNAitPByMabE365aEEzFHgSUSgAssbJWt7BZ80HQSVu8XWrQiTbFbCkoIBmXwPP/RoxgXJQVgZTFQ=="
|
||||
},
|
||||
"@kredits/contracts": {
|
||||
"version": "git+https://gitea.kosmos.org/kredits/contracts#6e0ec8741e61b51fb5c9c636da4e8d3610d090ac",
|
||||
"from": "@kredits/contracts@git+https://gitea.kosmos.org/kredits/contracts#6e0ec87",
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@kredits/contracts/-/contracts-7.0.0.tgz",
|
||||
"integrity": "sha512-UITEkP3njFNI2WS7v5ivGE3ruFwdWPWuJZrhBXBEAZbtmr1t/p1K7jkmmjyLDUeKXJ/udMlH6oQMCgh7P/aHNg==",
|
||||
"requires": {
|
||||
"@kosmos/schemas": "^3.1.0",
|
||||
"ethers": "^5.4.7",
|
||||
|
|
@ -2402,6 +2415,14 @@
|
|||
"resolved": "https://registry.npmjs.org/cborg/-/cborg-1.9.5.tgz",
|
||||
"integrity": "sha512-fLBv8wmqtlXqy1Yu+pHzevAIkW6k2K0ZtMujNzWphLsA34vzzg9BHn+5GmZqOJkSA9V7EMKsWrf6K976c1QMjQ=="
|
||||
},
|
||||
"cli-progress": {
|
||||
"version": "3.11.2",
|
||||
"resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.11.2.tgz",
|
||||
"integrity": "sha512-lCPoS6ncgX4+rJu5bS3F/iCz17kZ9MPZ6dpuTtI0KXKABkhyXIdYB3Inby1OpaGti3YlI3EeEkM9AuWpelJrVA==",
|
||||
"requires": {
|
||||
"string-width": "^4.2.3"
|
||||
}
|
||||
},
|
||||
"cliui": {
|
||||
"version": "8.0.1",
|
||||
"resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz",
|
||||
|
|
|
|||
|
|
@ -20,7 +20,8 @@
|
|||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@kredits/contracts": "git+https://gitea.kosmos.org/kredits/contracts#6e0ec87",
|
||||
"@kredits/contracts": "7.0.0",
|
||||
"cli-progress": "^3.11.2",
|
||||
"debug": "^4.3.4",
|
||||
"yargs": "^17.6.0"
|
||||
},
|
||||
|
|
|
|||
Loading…
Reference in New Issue