Compare commits
42 Commits
Author | SHA1 | Date |
---|---|---|
Râu Cao | bcb46c2b2b | |
Râu Cao | 7e7416a8fd | |
Râu Cao | b8fa61a434 | |
Râu Cao | 1aa2fac3cc | |
Râu Cao | b7d1600c59 | |
Râu Cao | ed50e84662 | |
Râu Cao | c7fe50ffa9 | |
Râu Cao | 83b3045afd | |
Râu Cao | 3c7bcf038e | |
Râu Cao | ca0b870315 | |
Râu Cao | 0a21c57251 | |
Râu Cao | 1401adbf0c | |
Râu Cao | 2401d8dff2 | |
Râu Cao | 45c0aed4a5 | |
Râu Cao | f25fcaa1b2 | |
Râu Cao | 7c8f0814b9 | |
Râu Cao | b566a432a9 | |
Greg | b7a7214f08 | |
Greg | 6829a13a57 | |
Râu Cao | 328905814d | |
Râu Cao | 03ee17f00e | |
Râu Cao | 9dffb15f45 | |
Râu Cao | c21512fa0b | |
Râu Cao | 4888d1be78 | |
Râu Cao | 0ac7228bfe | |
Râu Cao | 77c2cea4a4 | |
Râu Cao | 3c02591061 | |
Râu Cao | 9d7760edef | |
Râu Cao | 4a3988137e | |
Râu Cao | e12e798735 | |
Râu Cao | fee6b5c121 | |
Râu Cao | d8dda84fd6 | |
Basti | ba7c98b564 | |
bumi | 3f03368e25 | |
Basti | a3d21ebb30 | |
Basti | b11dbb9fde | |
gregkare | bfb51d48bc | |
Basti | f2f884a317 | |
Basti | 2a3c021737 | |
Basti | c4c95d9b38 | |
Râu Cao | 2d08cc929f | |
Basti | 5d36ca05a5 |
|
@ -0,0 +1,14 @@
|
|||
name-template: 'v$RESOLVED_VERSION'
|
||||
tag-template: 'v$RESOLVED_VERSION'
|
||||
version-resolver:
|
||||
major:
|
||||
labels:
|
||||
- release/major
|
||||
minor:
|
||||
labels:
|
||||
- release/minor
|
||||
- feature
|
||||
patch:
|
||||
labels:
|
||||
- release/patch
|
||||
default: patch
|
|
@ -0,0 +1,11 @@
|
|||
name: Release Drafter
|
||||
on:
|
||||
pull_request:
|
||||
types: [closed]
|
||||
jobs:
|
||||
release_drafter_job:
|
||||
name: Update release notes draft
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Release Drafter
|
||||
uses: https://github.com/raucao/gitea-release-drafter@dev
|
26
README.md
26
README.md
|
@ -1,6 +1,8 @@
|
|||
[![npm](https://img.shields.io/npm/v/@kredits/ipfs-pinner.svg)](https://www.npmjs.com/package/@kredits/ipfs-pinner)
|
||||
|
||||
# Kredits IPFS Pinner
|
||||
|
||||
This tool pins the IPFS hashes of a Kredits organisation on an IPFS node.
|
||||
This tool pins the IPFS documents of a Kredits organization on any IPFS node.
|
||||
|
||||
## Usage
|
||||
|
||||
|
@ -9,11 +11,11 @@ config flags, in case it is not running on localhost with default ports.)
|
|||
|
||||
### With NPX magic
|
||||
|
||||
npx @67p/kredits-ipfs-pinner
|
||||
npx @kredits/ipfs-pinner
|
||||
|
||||
### Global install
|
||||
|
||||
npm install -g @67p/kredits-ipfs-pinner
|
||||
npm install -g @kredits/ipfs-pinner
|
||||
kredits-ipfs-pinner
|
||||
|
||||
### From repo
|
||||
|
@ -25,3 +27,21 @@ Clone the Git repository, then use npm scripts from its root directory.
|
|||
Or with debug output:
|
||||
|
||||
npm run debug
|
||||
|
||||
### CLI options
|
||||
|
||||
Display progress bars:
|
||||
|
||||
kredits-ipfs-pinner --progress
|
||||
|
||||
Or with npm:
|
||||
|
||||
npm start -- --progress
|
||||
|
||||
Keep running and watch for new data/documents:
|
||||
|
||||
kredits-ipfs-pinner --watch
|
||||
|
||||
Multiple options at once:
|
||||
|
||||
kredits-ipfs-pinner --progress --watch
|
||||
|
|
|
@ -0,0 +1,6 @@
|
|||
[
|
||||
{
|
||||
"ID": "QmRw21aC3TroRVdZhrE2Qh4W6PBA67kbE8p3fNfsVcfW8D",
|
||||
"Addrs": ["/dns4/draco.kosmos.org/tcp/4001"]
|
||||
}
|
||||
]
|
91
index.js
91
index.js
|
@ -1,20 +1,33 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
const fs = require('fs');
|
||||
const debug = require('debug')('ipfs-pinner');
|
||||
const Kredits = require('kredits-contracts');
|
||||
const Kredits = require('@kredits/contracts');
|
||||
const IpfsPinner = require('./lib/ipfs-pinner');
|
||||
const defaultPeers = JSON.parse(fs.readFileSync('./config/peers.json'));
|
||||
|
||||
const argv = require('yargs')
|
||||
.default({ network: 'rinkeby', apm: 'open.aragonpm.eth', host: 'localhost', port: '5001', protocol: 'http', monitor: true })
|
||||
.boolean('monitor')
|
||||
.default({
|
||||
rpcUrl: 'https://rsk-testnet.kosmos.org',
|
||||
host: 'localhost',
|
||||
port: '5001',
|
||||
protocol: 'http',
|
||||
watch: false,
|
||||
progress: false,
|
||||
bootstrapNode: `${defaultPeers[0].Addrs[0]}/ipfs/${defaultPeers[0].ID}`
|
||||
})
|
||||
.boolean('watch')
|
||||
.boolean('progress')
|
||||
.describe({
|
||||
network: 'Ethereum network to connect to',
|
||||
rpcUrl: 'Ethereum node RPC URL; alternative to --network',
|
||||
daoAddress: 'Optional Kredits DAO address',
|
||||
rpcUrl: 'Web3/EVM node RPC URL; alternative to --network',
|
||||
host: 'IPFS API host',
|
||||
port: 'IPFS API port',
|
||||
protocol: 'IPFS API protocol',
|
||||
monitor: 'Monitor contract events for new IPFS documents'
|
||||
watch: 'Monitor contract events for new IPFS documents',
|
||||
progress: 'Show progress bars',
|
||||
bootstrapNode: 'IPFS node address to connect to before fetching documents'
|
||||
})
|
||||
.example('$0 --network rinkeby --host localhost', 'Pins all existing IPFS documents to the IPFS API running on localhost and monitors for new events.')
|
||||
.example('$0 --host localhost', 'Pins all existing IPFS documents to the IPFS API running on localhost and monitors for new events')
|
||||
.argv;
|
||||
|
||||
const ipfsConfig = {
|
||||
|
@ -23,33 +36,55 @@ const ipfsConfig = {
|
|||
protocol: argv.protocol
|
||||
};
|
||||
|
||||
console.log(`Using IPFS:`, ipfsConfig);
|
||||
debug(`IPFS node:`, ipfsConfig);
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
const kredits = await Kredits.for(
|
||||
{ network: argv.network, rpcUrl: argv.rpcUrl },
|
||||
{ apm: argv.apm, ipfsConfig: ipfsConfig, addresses: { Kernel: argv.daoAddress } }
|
||||
).init();
|
||||
const kredits = await Kredits.for(
|
||||
{ rpcUrl: argv.rpcUrl },
|
||||
{ ipfsConfig: ipfsConfig }
|
||||
).init().catch(e => {
|
||||
console.log('Failed to initialize Kredits:');
|
||||
console.log(e.message);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
// check the connection to the IPFS client
|
||||
// TODO redesign IPFS wrapper API and do not use an internal attribute
|
||||
await kredits.ipfs._ipfsAPI.id();
|
||||
// TODO redesign IPFS wrapper API and do not use an internal attribute
|
||||
const ipfsApi = kredits.ipfs._ipfsAPI;
|
||||
let connectBootstrapNode = true;
|
||||
|
||||
const ipfsPinner = new IpfsPinner(kredits);
|
||||
await ipfsApi.id().then(res => {
|
||||
if (res.id === argv.bootstrapNode.split('/')[argv.bootstrapNode.split('/').length-1]) {
|
||||
connectBootstrapNode = false;
|
||||
}
|
||||
}).catch(e => {
|
||||
console.log('Failed to initialize IPFS:');
|
||||
console.log(e.message);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
ipfsPinner.pinAll().then(pins => {
|
||||
console.log(`Pinned ${pins.length} existing documents`);
|
||||
});
|
||||
if (connectBootstrapNode) {
|
||||
debug(`Connecting to known IPFS node ${argv.bootstrapNode}`);
|
||||
await ipfsApi.swarm.connect(argv.bootstrapNode);
|
||||
}
|
||||
|
||||
ipfsPinner.monitor(pin => {
|
||||
const ipfsPinner = new IpfsPinner(kredits, {
|
||||
progress: argv.progress
|
||||
});
|
||||
|
||||
await ipfsPinner.pinAll().then(cids => {
|
||||
console.log(`\nSuccessfully pinned ${cids.length} documents`)
|
||||
});
|
||||
|
||||
if (argv.watch) {
|
||||
console.log('\nWatching contract events for new documents...');
|
||||
|
||||
ipfsPinner.watch(pin => {
|
||||
console.log('Pinned a new document:', pin[0]["hash"]);
|
||||
});
|
||||
|
||||
console.log(`Subscribed to DAO: ${kredits.Kernel.contract.address}`);
|
||||
} catch(e) {
|
||||
console.log('Failed to start');
|
||||
console.log(e);
|
||||
process.exit(1);
|
||||
} else {
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// TODO Add new deployment/DAO/org ID or all contract proxy addresses
|
||||
// console.log(`Subscribed to DAO: ${kredits.Kernel.contract.address}`);
|
||||
})();
|
||||
|
|
|
@ -1,21 +1,61 @@
|
|||
const debug = require('debug')('ipfs-pinner');
|
||||
const cliProgress = require('cli-progress');
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
class IpfsPinner {
|
||||
constructor (kredits, ipfsApi) {
|
||||
constructor (kredits, options={}) {
|
||||
this.kredits = kredits;
|
||||
this.ipfsApi = ipfsApi || this.kredits.ipfs;
|
||||
this.ipfsApi = this.kredits.ipfs;
|
||||
this.progressBars = !!options.progress && !process.env.DEBUG;
|
||||
|
||||
if (this.progressBars) {
|
||||
this.multibar = new cliProgress.MultiBar({
|
||||
stopOnComplete: true,
|
||||
clearOnComplete: false,
|
||||
hideCursor: false,
|
||||
etaBuffer: 30,
|
||||
format: '{entity} [{bar}] {percentage}% | ETA: {eta_formatted} | {value}/{total}'
|
||||
}, cliProgress.Presets.shades_grey);
|
||||
}
|
||||
}
|
||||
|
||||
async pinAll () {
|
||||
const contributorHashes = await this._pinAllFromContract(this.kredits.Contributor);
|
||||
const contributionHashes = await this._pinAllFromContract(this.kredits.Contribution);
|
||||
const proposalHashes = await this._pinAllFromContract(this.kredits.Proposal);
|
||||
console.log('Pinning IPFS documents for all known items...\n')
|
||||
const cids = [];
|
||||
const promises = [];
|
||||
const contracts = [
|
||||
this.kredits.Contributor,
|
||||
this.kredits.Contribution,
|
||||
this.kredits.Reimbursement
|
||||
]
|
||||
|
||||
return contributorHashes.concat(contributionHashes)
|
||||
.concat(proposalHashes);
|
||||
for (const contract of contracts) {
|
||||
debug(`Pinning data from ${contract.constructor.name}...`);
|
||||
const itemCount = await contract.count;
|
||||
debug(`${contract.constructor.name} item count:`, itemCount);
|
||||
let bar;
|
||||
|
||||
if (this.progressBars) {
|
||||
bar = this.multibar.create(itemCount, 0);
|
||||
bar.update(0, {entity: `${contract.constructor.name}s`.padEnd(14)});
|
||||
}
|
||||
|
||||
promises.push(this._pinAllFromContract(contract, itemCount, bar)
|
||||
.then(res => { cids.push(...res); }));
|
||||
}
|
||||
|
||||
await Promise.all(promises);
|
||||
|
||||
// Avoid console output race condition with progress bars finishing update
|
||||
if (this.progressBars) await sleep(1000);
|
||||
|
||||
return cids;
|
||||
}
|
||||
|
||||
monitor (callback) {
|
||||
watch (callback) {
|
||||
this.kredits.Contribution.on('ContributionAdded', (id) => {
|
||||
this.kredits.Contribution.getData(id)
|
||||
.then(data => { return this.ipfsApi.pin(data); })
|
||||
|
@ -31,21 +71,50 @@ class IpfsPinner {
|
|||
.then(data => { return this.ipfsApi.pin(data); })
|
||||
.then(callback);
|
||||
});
|
||||
}
|
||||
|
||||
_pinAllFromContract (contract) {
|
||||
debug(`Pinning data from ${contract.constructor.name}...`);
|
||||
return contract.count.then(count => {
|
||||
let promises = [...Array(count).keys()].map(i => {
|
||||
let id = i + 1; // 0 => 1 - ids start with 1 and not with 0
|
||||
debug(`Loading ${contract.constructor.name} #${id}`);
|
||||
return contract.getData(id).then(data => {
|
||||
debug(`Pinning ${contract.constructor.name} #${id}`);
|
||||
return this.ipfsApi.pin(data);
|
||||
});
|
||||
});
|
||||
return Promise.all(promises);
|
||||
this.kredits.Reimbursement.on('ReimbursementAdded', (id) => {
|
||||
this.kredits.Reimbursement.getData(id)
|
||||
.then(data => { return this.ipfsApi.pin(data); })
|
||||
.then(callback);
|
||||
});
|
||||
}
|
||||
|
||||
async _pinAllFromContract (contract, itemCount, progressBar) {
|
||||
const ipfsApi = this.ipfsApi;
|
||||
const progressBars = this.progressBars;
|
||||
const ids = [...Array(itemCount).keys()].map(i => i+1);
|
||||
const cids = [];
|
||||
const batchSize = 20;
|
||||
let position = 0;
|
||||
|
||||
async function loadAndPin(id) {
|
||||
let cid;
|
||||
|
||||
try {
|
||||
const data = await contract.getData(id);
|
||||
debug(`Loaded ${contract.constructor.name} #${id}`);
|
||||
cid = await ipfsApi.pin(data);
|
||||
debug(`Pinned ${contract.constructor.name} #${id} at ${cid}`);
|
||||
} catch(e) {
|
||||
debug(`Error while trying to load and pin ${contract.constructor.name} #${id}:`)
|
||||
debug(e);
|
||||
debug(`\nTrying again...`);
|
||||
loadAndPin(id);
|
||||
} finally {
|
||||
cids.push(cid);
|
||||
if (progressBars) { progressBar.increment(); }
|
||||
}
|
||||
}
|
||||
|
||||
while (position < itemCount) {
|
||||
const batchIds = ids.slice(position, position + batchSize);
|
||||
|
||||
await Promise.all(batchIds.map(async id => loadAndPin(id)));
|
||||
|
||||
position += batchSize;
|
||||
}
|
||||
|
||||
return cids;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = IpfsPinner;
|
||||
|
|
File diff suppressed because it is too large
Load Diff
11
package.json
11
package.json
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@67p/kredits-ipfs-pinner",
|
||||
"version": "1.0.2",
|
||||
"name": "@kredits/ipfs-pinner",
|
||||
"version": "2.3.0",
|
||||
"description": "Pins IPFS data of a Kredits organization on an IPFS node",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
|
@ -20,9 +20,10 @@
|
|||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"debug": "^4.1.1",
|
||||
"kredits-contracts": "^5.4.0",
|
||||
"yargs": "^13.2.4"
|
||||
"@kredits/contracts": "^7.4.0",
|
||||
"cli-progress": "^3.11.2",
|
||||
"debug": "^4.3.4",
|
||||
"yargs": "^17.6.0"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
|
|
Loading…
Reference in New Issue