diff --git a/index.js b/index.js index c528e50..5e89146 100755 --- a/index.js +++ b/index.js @@ -12,16 +12,19 @@ const argv = require('yargs') host: 'localhost', port: '5001', protocol: 'http', - monitor: true, + watch: false, + progress: false, bootstrapNode: `${defaultPeers[0].Addrs[0]}/ipfs/${defaultPeers[0].ID}` }) - .boolean('monitor') + .boolean('watch') + .boolean('progress') .describe({ rpcUrl: 'Web3/EVM node RPC URL; alternative to --network', host: 'IPFS API host', port: 'IPFS API port', protocol: 'IPFS API protocol', - monitor: 'Monitor contract events for new IPFS documents', + watch: 'Monitor contract events for new IPFS documents', + progress: 'Show progress bars', bootstrapNode: 'IPFS node address to connect to before fetching documents' }) .example('$0 --host localhost', 'Pins all existing IPFS documents to the IPFS API running on localhost and monitors for new events') @@ -33,39 +36,48 @@ const ipfsConfig = { protocol: argv.protocol }; -console.log(`Using IPFS:`, ipfsConfig); +debug(`IPFS node:`, ipfsConfig); (async () => { - try { - const kredits = await Kredits.for( - { rpcUrl: argv.rpcUrl }, - { ipfsConfig: ipfsConfig } - ).init(); + const kredits = await Kredits.for( + { rpcUrl: argv.rpcUrl }, + { ipfsConfig: ipfsConfig } + ).init().catch(e => { + console.log('Failed to initialize Kredits:'); + console.log(e.message); + process.exit(1); + }); - // Check the connection to the IPFS client - // TODO redesign IPFS wrapper API and do not use an internal attribute - const ipfsApi = kredits.ipfs._ipfsAPI; + // TODO redesign IPFS wrapper API and do not use an internal attribute + const ipfsApi = kredits.ipfs._ipfsAPI; - await ipfsApi.id(); + await ipfsApi.id().catch(e => { + console.log('Failed to initialize IPFS:'); + console.log(e.message); + process.exit(1); + }); - debug(`Connecting to known IPFS node ${argv.bootstrapNode}`); - await ipfsApi.swarm.connect(argv.bootstrapNode); + debug(`Connecting to known IPFS node ${argv.bootstrapNode}`); + await ipfsApi.swarm.connect(argv.bootstrapNode); - const ipfsPinner = new IpfsPinner(kredits); + const ipfsPinner = new IpfsPinner(kredits, { + progress: argv.progress + }); - ipfsPinner.pinAll().then(pins => { - console.log(`Pinned ${pins.length} existing documents`); - }); + await ipfsPinner.pinAll().then(cids => { + console.log(`\nSuccessfully pinned ${cids.length} documents`) + }); - ipfsPinner.monitor(pin => { + if (argv.watch) { + console.log('\nWatching contract events for new documents...'); + + ipfsPinner.watch(pin => { console.log('Pinned a new document:', pin[0]["hash"]); }); - - // TODO Add new deployment/DAO/org ID or all contract proxy addresses - // console.log(`Subscribed to DAO: ${kredits.Kernel.contract.address}`); - } catch(e) { - console.log('Failed to start'); - console.log(e); - process.exit(1); + } else { + process.exit(0); } + + // TODO Add new deployment/DAO/org ID or all contract proxy addresses + // console.log(`Subscribed to DAO: ${kredits.Kernel.contract.address}`); })(); diff --git a/lib/ipfs-pinner.js b/lib/ipfs-pinner.js index fdf34f1..9e0aebc 100644 --- a/lib/ipfs-pinner.js +++ b/lib/ipfs-pinner.js @@ -1,32 +1,62 @@ const debug = require('debug')('ipfs-pinner'); +const cliProgress = require('cli-progress'); -async function promiseAllInBatches(task, items, batchSize) { - let position = 0; - let results = []; - while (position < items.length) { - const itemsForBatch = items.slice(position, position + batchSize); - results = [...results, ...await Promise.allSettled(itemsForBatch.map(item => task(item)))]; - position += batchSize; - } - return results; +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); } class IpfsPinner { - constructor (kredits, ipfsApi) { + constructor (kredits, options={}) { this.kredits = kredits; - this.ipfsApi = ipfsApi || this.kredits.ipfs; + this.ipfsApi = this.kredits.ipfs; + this.progressBars = !!options.progress && !process.env.DEBUG; + + if (this.progressBars) { + this.multibar = new cliProgress.MultiBar({ + stopOnComplete: true, + clearOnComplete: false, + hideCursor: false, + etaBuffer: 30, + format: '{entity} [{bar}] {percentage}% | ETA: {eta_formatted} | {value}/{total}' + }, cliProgress.Presets.shades_grey); + } } async pinAll () { - const contributorHashes = await this._pinAllFromContract(this.kredits.Contributor); - const contributionHashes = await this._pinAllFromContract(this.kredits.Contribution); - const reimbursementHashes = await this._pinAllFromContract(this.kredits.Reimbursement); + console.log('Pinning IPFS documents for all known items...\n') + const cids = []; + const promises = []; + const contracts = [ + this.kredits.Contributor, + this.kredits.Contribution, + // TODO uncomment once we have data here + // this.kredits.Reimbursement + ] - return contributorHashes.concat(contributionHashes) - .concat(reimbursementHashes); + for (const contract of contracts) { + debug(`Pinning data from ${contract.constructor.name}...`); + const itemCount = await contract.count; + debug(`${contract.constructor.name} item count:`, itemCount); + let bar; + + if (this.progressBars) { + bar = this.multibar.create(itemCount, 0); + bar.update(0, {entity: `${contract.constructor.name}s`.padEnd(14)}); + } + + promises.push(this._pinAllFromContract(contract, itemCount, bar) + .then(res => { cids.push(...res); })); + } + + await Promise.all(promises); + + // Avoid console output race condition with progress bars finishing update + if (this.progressBars) await sleep(1000); + + return cids; } - monitor (callback) { + watch (callback) { this.kredits.Contribution.on('ContributionAdded', (id) => { this.kredits.Contribution.getData(id) .then(data => { return this.ipfsApi.pin(data); }) @@ -44,22 +74,40 @@ class IpfsPinner { }); } - async _pinAllFromContract (contract) { - debug(`Pinning data from ${contract.constructor.name}...`); - const count = await contract.count; - debug('Item count:', count); - const ids = [...Array(count).keys()].map(i => i+1); + async _pinAllFromContract (contract, itemCount, progressBar) { + const ipfsApi = this.ipfsApi; + const progressBars = this.progressBars; + const ids = [...Array(itemCount).keys()].map(i => i+1); const cids = []; + const batchSize = 20; + let position = 0; - async function loadAndPin (id) { - debug(`Loading ${contract.constructor.name} #${id}`); - return contract.getData(id).then(data => { - debug(`Pinning ${contract.constructor.name} #${id}`); - return this.ipfsApi.pin(data).then(cid => cids.push(cid)); - }); + async function loadAndPin(id) { + let cid; + + try { + const data = await contract.getData(id); + debug(`Loaded ${contract.constructor.name} #${id}`); + cid = await ipfsApi.pin(data); + debug(`Pinned ${contract.constructor.name} #${id} at ${cid}`); + } catch(e) { + debug(`Error while trying to load an pin ${contract.constructor.name} #${id}:`) + debug(e); + debug(`\nTrying again...`); + loadAndPin(id); + } finally { + cids.push(cid); + if (progressBars) { progressBar.increment(); } + } } - await promiseAllInBatches(loadAndPin.bind(this), ids, 100); + while (position < itemCount) { + const batchIds = ids.slice(position, position + batchSize); + + await Promise.all(batchIds.map(async id => loadAndPin(id))); + + position += batchSize; + } return cids; } diff --git a/package-lock.json b/package-lock.json index 856dba2..cc171c0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,8 @@ "version": "1.2.0", "license": "MIT", "dependencies": { - "@kredits/contracts": "git+https://gitea.kosmos.org/kredits/contracts#6e0ec87", + "@kredits/contracts": "7.0.0", + "cli-progress": "^3.11.2", "debug": "^4.3.4", "yargs": "^17.6.0" }, @@ -720,9 +721,9 @@ "integrity": "sha512-yOTK5WiXFDNAitPByMabE365aEEzFHgSUSgAssbJWt7BZ80HQSVu8XWrQiTbFbCkoIBmXwPP/RoxgXJQVgZTFQ==" }, "node_modules/@kredits/contracts": { - "version": "7.0.0-beta.0", - "resolved": "git+https://gitea.kosmos.org/kredits/contracts#6e0ec8741e61b51fb5c9c636da4e8d3610d090ac", - "license": "MIT", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@kredits/contracts/-/contracts-7.0.0.tgz", + "integrity": "sha512-UITEkP3njFNI2WS7v5ivGE3ruFwdWPWuJZrhBXBEAZbtmr1t/p1K7jkmmjyLDUeKXJ/udMlH6oQMCgh7P/aHNg==", "dependencies": { "@kosmos/schemas": "^3.1.0", "ethers": "^5.4.7", @@ -930,6 +931,17 @@ "cborg": "cli.js" } }, + "node_modules/cli-progress": { + "version": "3.11.2", + "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.11.2.tgz", + "integrity": "sha512-lCPoS6ncgX4+rJu5bS3F/iCz17kZ9MPZ6dpuTtI0KXKABkhyXIdYB3Inby1OpaGti3YlI3EeEkM9AuWpelJrVA==", + "dependencies": { + "string-width": "^4.2.3" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", @@ -2233,8 +2245,9 @@ "integrity": "sha512-yOTK5WiXFDNAitPByMabE365aEEzFHgSUSgAssbJWt7BZ80HQSVu8XWrQiTbFbCkoIBmXwPP/RoxgXJQVgZTFQ==" }, "@kredits/contracts": { - "version": "git+https://gitea.kosmos.org/kredits/contracts#6e0ec8741e61b51fb5c9c636da4e8d3610d090ac", - "from": "@kredits/contracts@git+https://gitea.kosmos.org/kredits/contracts#6e0ec87", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@kredits/contracts/-/contracts-7.0.0.tgz", + "integrity": "sha512-UITEkP3njFNI2WS7v5ivGE3ruFwdWPWuJZrhBXBEAZbtmr1t/p1K7jkmmjyLDUeKXJ/udMlH6oQMCgh7P/aHNg==", "requires": { "@kosmos/schemas": "^3.1.0", "ethers": "^5.4.7", @@ -2402,6 +2415,14 @@ "resolved": "https://registry.npmjs.org/cborg/-/cborg-1.9.5.tgz", "integrity": "sha512-fLBv8wmqtlXqy1Yu+pHzevAIkW6k2K0ZtMujNzWphLsA34vzzg9BHn+5GmZqOJkSA9V7EMKsWrf6K976c1QMjQ==" }, + "cli-progress": { + "version": "3.11.2", + "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.11.2.tgz", + "integrity": "sha512-lCPoS6ncgX4+rJu5bS3F/iCz17kZ9MPZ6dpuTtI0KXKABkhyXIdYB3Inby1OpaGti3YlI3EeEkM9AuWpelJrVA==", + "requires": { + "string-width": "^4.2.3" + } + }, "cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", diff --git a/package.json b/package.json index 7dc75cb..3f30c8f 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,8 @@ ], "license": "MIT", "dependencies": { - "@kredits/contracts": "git+https://gitea.kosmos.org/kredits/contracts#6e0ec87", + "@kredits/contracts": "7.0.0", + "cli-progress": "^3.11.2", "debug": "^4.3.4", "yargs": "^17.6.0" },