Cache data in browser storage, and load from local cache when present #189

Merged
raucao merged 8 commits from feature/data_loading into master 2020-05-31 08:44:37 +00:00
18 changed files with 753 additions and 77 deletions
+11 -5
View File
@@ -1,6 +1,5 @@
import EmberObject, { computed } from '@ember/object';
import { isEmpty, isPresent } from '@ember/utils';
import bignumber from 'kredits-web/utils/cps/bignumber';
import moment from 'moment';
export default EmberObject.extend({
@@ -9,11 +8,15 @@ export default EmberObject.extend({
id: null,
contributorId: null,
amount: null,
confirmedAt: bignumber('confirmedAtBlock', 'toNumber'),
confirmedAt: null,
vetoed: null,
ipfsHash: null,
creatorAccount: null,
// contributor model instance
contributor: null,
// TODO contributor who submitted the contribution
// submittedBy: null,
// IPFS
kind: null,
@@ -22,7 +25,6 @@ export default EmberObject.extend({
url: null,
date: null,
time: null,
ipfsData: '',
pendingTx: null,
@@ -41,6 +43,10 @@ export default EmberObject.extend({
hasPendingChanges: computed('pendingTx', function() {
return isPresent(this.pendingTx);
})
}),
serialize () {
return JSON.stringify(this);
}
});
+8 -7
View File
@@ -1,14 +1,12 @@
import EmberObject from '@ember/object';
import bignumber from 'kredits-web/utils/cps/bignumber';
import kreditsValue from 'kredits-web/utils/cps/kredits';
export default EmberObject.extend({
// Contract
id: bignumber('idRaw', 'toString'),
id: null,
account: null,
balance: kreditsValue('balanceRaw'),
totalKreditsEarned: bignumber('totalKreditsEarnedRaw', 'toNumber'),
contributionsCount: bignumber('contributionsCountRaw', 'toNumber'),
balance: 0,
totalKreditsEarned: 0,
contributionsCount: 0,
isCore: false,
ipfsHash: null,
@@ -20,6 +18,9 @@ export default EmberObject.extend({
github_uid: null,
wiki_username: null,
zoom_display_name: null,
ipfsData: ''
serialize () {
return JSON.stringify(this);
}
});
+7
View File
@@ -1,5 +1,6 @@
import { inject as service } from '@ember/service';
import Route from '@ember/routing/route';
import { schedule } from '@ember/runloop';
export default Route.extend({
kredits: service(),
@@ -21,6 +22,12 @@ export default Route.extend({
return this.kredits.loadInitialData()
.then(() => {
this.kredits.addContractEventHandlers();
})
.then(() => {
if (this.kredits.contributorsNeedFetch) {
schedule('afterRender', this.kredits, this.kredits.fetchContributors);
}
schedule('afterRender', this.kredits, this.kredits.fetchContributions);
});
}
});
+27
View File
@@ -0,0 +1,27 @@
import Service from '@ember/service';
import * as localforage from 'localforage';
import config from 'kredits-web/config/environment';
function createStore(name) {
const networkName = config.web3RequiredNetwork || 'custom';
return localforage.createInstance({ name: `kredits:${networkName}:${name}` });
}
export default class BrowserCacheService extends Service {
constructor() {
super(...arguments);
this.stores = {
contributors: createStore('contributors'),
contributions: createStore('contributions')
}
}
get contributors() {
return this.stores.contributors;
}
get contributions() {
return this.stores.contributions;
}
}
+83 -13
View File
@@ -7,8 +7,11 @@ import EmberObject from '@ember/object';
import { computed } from '@ember/object';
import { alias, notEmpty } from '@ember/object/computed';
import { isEmpty, isPresent } from '@ember/utils';
import { inject as service } from '@ember/service';
import groupBy from 'kredits-web/utils/group-by';
import processContributorData from 'kredits-web/utils/process-contributor-data';
import processContributionData from 'kredits-web/utils/process-contribution-data';
import formatKredits from 'kredits-web/utils/format-kredits';
import config from 'kredits-web/config/environment';
@@ -17,6 +20,9 @@ import Contribution from 'kredits-web/models/contribution'
export default Service.extend({
browserCache: service(),
contributorsNeedFetch: false,
currentBlock: null,
currentUserAccounts: null, // default to not having an account. this is the wen web3 is loaded.
currentUser: null,
@@ -169,12 +175,23 @@ export default Service.extend({
.then(total => total.toNumber());
}),
async loadInitialData () {
const numCachedContributors = await this.browserCache.contributors.length();
if (numCachedContributors > 0) {
await this.loadContributorsFromCache();
this.set('contributorsNeedFetch', true);
} else {
await this.fetchContributors();
}
loadInitialData () {
return this.getContributors()
.then(contributors => this.contributors.pushObjects(contributors))
.then(() => this.getContributions())
.then(contributions => this.contributions.pushObjects(contributions))
const numCachedContributions = await this.browserCache.contributions.length();
if (numCachedContributions > 0) {
await this.loadContributionsFromCache();
} else {
await this.fetchContributions({ page: { size: 30 } });
}
return Promise.resolve();
},
addContributor (attributes) {
@@ -205,15 +222,39 @@ export default Service.extend({
});
},
getContributors () {
fetchContributors () {
console.debug(`[kredits] Fetching all contributors from the network`);
return this.kredits.Contributor.all()
.then(contributors => {
return contributors.map(contributor => {
return Contributor.create(contributor);
return contributors.map(data => {
const contributor = Contributor.create(processContributorData(data));
const loadedContributor = this.contributors.findBy('id', contributor.id);
if (loadedContributor) { this.contributors.removeObject(loadedContributor); }
this.contributors.pushObject(contributor);
return contributor;
});
})
.then(() => {
return this.cacheLoadedContributors();
});
},
async cacheLoadedContributors () {
for (const c of this.contributors) {
await this.browserCache.contributors.setItem(c.id, c.serialize());
}
console.debug(`[kredits] Cached ${this.contributors.length} contributors in browser storage`);
return Promise.resolve();
},
async loadContributorsFromCache () {
return this.browserCache.contributors.iterate((value/*, key , iterationNumber */) => {
this.contributors.pushObject(Contributor.create(JSON.parse(value)));
}).then((/* result */) => {
console.debug(`[kredits] Loaded ${this.contributors.length} contributors from cache`);
});
},
addContribution (attributes) {
console.debug('[kredits] add contribution', attributes);
@@ -229,16 +270,45 @@ export default Service.extend({
});
},
getContributions () {
return this.kredits.Contribution.all({page: {size: 200}})
fetchContributions (options = { page: { size: 200 } }) {
console.debug(`[kredits] Fetching contributions from the network`);
return this.kredits.Contribution.all(options)
.then(contributions => {
return contributions.map(contribution => {
contribution.contributor = this.contributors.findBy('id', contribution.contributorId.toString());
return Contribution.create(contribution);
return contributions.map(data => {
const contribution = Contribution.create(processContributionData(data));
contribution.set('contributor', this.contributors.findBy('id', data.contributorId.toString()));
const loadedContribution = this.contributions.findBy('id', contribution.id);
if (loadedContribution) { this.contributions.removeObject(loadedContribution); }
this.contributions.pushObject(contribution);
return contribution;
});
})
.then(contributions => {
const cacheWrites = contributions.map(c => {
return this.browserCache.contributions.setItem(c.id.toString(), c.serialize());
});
return Promise.all(cacheWrites).then(() => {
console.debug(`[kredits] Cached ${contributions.length} contributions in browser storage`);
});
});
},
async cacheLoadedContributions () {
for (const c of this.contributions) {
await this.browserCache.contributions.setItem(c.id, c.serialize());
}
console.debug(`[kredits] Cached ${this.contributions.length} contributions in browser storage`);
return Promise.resolve();
},
async loadContributionsFromCache () {
return this.browserCache.contributions.iterate((value/*, key , iterationNumber */) => {
this.contributions.pushObject(Contribution.create(JSON.parse(value)));
}).then((/* result */) => {
console.debug(`[kredits] Loaded ${this.contributions.length} contributions from cache`);
});
},
veto (contributionId) {
console.debug('[kredits] veto against', contributionId);
const contribution = this.contributions.findBy('id', contributionId);
-20
View File
@@ -1,20 +0,0 @@
import { computed } from '@ember/object';
import ethers from 'ethers';
export default function(dependentKey, converterMethod) {
return computed(dependentKey, {
get () {
let value = this.get(dependentKey);
if (value && ethers.utils.BigNumber.isBigNumber(value)) {
return value[converterMethod]();
} else {
return value;
}
},
set (key, value) {
const bnValue = ethers.utils.bigNumberify(value);
this.set(dependentKey, bnValue);
return bnValue[converterMethod]();
}
});
}
-17
View File
@@ -1,17 +0,0 @@
import { computed } from '@ember/object';
import ethers from 'ethers';
import formatKredits from 'kredits-web/utils/format-kredits';
export default function(dependentKey, options = {}) {
return computed(dependentKey, {
get () {
const value = this.get(dependentKey);
return formatKredits(value, options);
},
set (key, value) {
const bnValue = ethers.utils.bigNumberify(value);
this.set(dependentKey, bnValue);
return formatKredits(bnValue, options);
}
});
}
+21
View File
@@ -0,0 +1,21 @@
export default function processContributionData(data) {
const processed = {}
if (data.confirmedAtBlock && (typeof data.confirmedAtBlock.toNumber === 'function')) {
processed.confirmedAt = data.confirmedAtBlock.toNumber();
bumi commented 2020-05-30 10:35:09 +00:00 (Migrated from github.com)
Review

was wondering if the rename from confirmedAtBlock to confirmedAt is so needed because now we call it differently in the kredits-contracts and kredits-web

was wondering if the rename from `confirmedAtBlock` to `confirmedAt` is so needed because now we call it differently in the kredits-contracts and kredits-web
raucao commented 2020-05-31 08:44:07 +00:00 (Migrated from github.com)
Review

It's not a rename, because it's the same property that returned the number before as well. The confirmedAtBlock is the one from the contracts, which isn't formatted as a number.

It's not a rename, because it's the same property that returned the number before as well. The `confirmedAtBlock` is the one from the contracts, which isn't formatted as a number.
} else if (data.confirmedAt !== 'undefined') {
processed.confirmedAt = data.confirmedAt;
}
const otherProperties = [
'id', 'contributorId', 'amount', 'vetoed',
'ipfsHash', 'kind', 'description', 'details',
'url', 'date', 'time', 'pendingTx'
];
otherProperties.forEach(prop => {
processed[prop] = data[prop];
});
return processed;
}
+19
View File
@@ -0,0 +1,19 @@
export default function processContributorData(data) {
const processed = {
id: data.id.toString(),
balance: data.balanceInt,
totalKreditsEarned: data.totalKreditsEarned,
contributionsCount: data.contributionsCount?.toNumber()
}
const otherProperties = [
'account', 'accounts', 'ipfsHash', 'isCore', 'kind', 'name', 'url',
'github_username', 'github_uid', 'wiki_username', 'zoom_display_name'
];
otherProperties.forEach(prop => {
processed[prop] = data[prop];
});
return processed;
}
+24
View File
@@ -15545,6 +15545,12 @@
"integrity": "sha512-MzbUSahkTW1u7JpKKjY7LCARd1fU5W2rLdxlM4kdkayuCwZImjkpluF9CM1aLewYJguPDqewLam18Y6AU69A8A==",
"dev": true
},
"immediate": {
"version": "3.0.6",
"resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz",
"integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=",
"dev": true
},
"import-fresh": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz",
@@ -16768,6 +16774,15 @@
}
}
},
"lie": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/lie/-/lie-3.1.1.tgz",
"integrity": "sha1-mkNrLMd0bKWd56QfpGmz77dr2H4=",
"dev": true,
"requires": {
"immediate": "~3.0.5"
}
},
"linkify-it": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-2.2.0.tgz",
@@ -16849,6 +16864,15 @@
"integrity": "sha512-9M2KvGT6duzGMgkOcTkWb+PR/Q2Oe54df/tLgHGVmFpAmtqJ553xJh6N63iFYI2yjo2PeJXbS5skHi/QpJq4vA==",
"dev": true
},
"localforage": {
"version": "1.7.3",
"resolved": "https://registry.npmjs.org/localforage/-/localforage-1.7.3.tgz",
"integrity": "sha512-1TulyYfc4udS7ECSBT2vwJksWbkwwTX8BzeUIiq8Y07Riy7bDAAnxDaPU/tWyOVmQAcWJIEIFP9lPfBGqVoPgQ==",
"dev": true,
"requires": {
"lie": "3.1.1"
}
},
"locate-character": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/locate-character/-/locate-character-2.0.5.tgz",
+1
View File
@@ -67,6 +67,7 @@
"kosmos-schemas": "^2.1.0",
"kredits-contracts": "^5.5.0",
"loader.js": "^4.7.0",
"localforage": "^1.7.3",
"ndjson": "github:hugomrdias/ndjson#feat/readable-stream3",
"npm-run-all": "^4.1.5",
"qunit-dom": "^1.2.0",
File diff suppressed because one or more lines are too long
+13 -11
View File
@@ -1,23 +1,25 @@
import Model from 'kredits-web/models/contribution';
import contributors from '../../tests/fixtures/contributors';
import processContributionData from 'kredits-web/utils/process-contribution-data';
const items = [];
const data = [
{ id: 1, contributorId: 1, confirmedAtBlock: 1000, claimed: false, vetoed: false, amount: 1500, kind: 'dev' },
{ id: 2, contributorId: 1, confirmedAtBlock: 1000, claimed: false, vetoed: false, amount: 5000, kind: 'ops' },
{ id: 3, contributorId: 2, confirmedAtBlock: 1000, claimed: false, vetoed: false, amount: 1500, kind: 'ops' },
{ id: 4, contributorId: 2, confirmedAtBlock: 1000, claimed: false, vetoed: false, amount: 1500, kind: 'docs' },
{ id: 5, contributorId: 1, confirmedAtBlock: 1000, claimed: false, vetoed: false, amount: 5000, kind: 'design' },
{ id: 6, contributorId: 1, confirmedAtBlock: 1000, claimed: false, vetoed: true, amount: 500, kind: 'dev' },
{ id: 7, contributorId: 3, confirmedAtBlock: 2000, claimed: false, vetoed: false, amount: 5000, kind: 'dev' },
{ id: 8, contributorId: 1, confirmedAtBlock: 2000, claimed: false, vetoed: false, amount: 1500, kind: 'community' },
{ id: 9, contributorId: 3, confirmedAtBlock: 2000, claimed: false, vetoed: true, amount: 1500, kind: 'docs' },
{ id: 1, contributorId: 1, confirmedAt: 1000, claimed: false, vetoed: false, amount: 1500, kind: 'dev' },
{ id: 2, contributorId: 1, confirmedAt: 1000, claimed: false, vetoed: false, amount: 5000, kind: 'ops' },
{ id: 3, contributorId: 2, confirmedAt: 1000, claimed: false, vetoed: false, amount: 1500, kind: 'ops' },
{ id: 4, contributorId: 2, confirmedAt: 1000, claimed: false, vetoed: false, amount: 1500, kind: 'docs' },
{ id: 5, contributorId: 1, confirmedAt: 1000, claimed: false, vetoed: false, amount: 5000, kind: 'design' },
{ id: 6, contributorId: 1, confirmedAt: 1000, claimed: false, vetoed: true, amount: 500, kind: 'dev' },
{ id: 7, contributorId: 3, confirmedAt: 2000, claimed: false, vetoed: false, amount: 5000, kind: 'dev' },
{ id: 8, contributorId: 1, confirmedAt: 2000, claimed: false, vetoed: false, amount: 1500, kind: 'community' },
{ id: 9, contributorId: 3, confirmedAt: 2000, claimed: false, vetoed: true, amount: 1500, kind: 'docs' },
];
data.forEach(attrs => {
attrs.contributor = contributors.findBy('id', attrs.contributorId.toString());
items.push(Model.create(attrs))
const c = Model.create(processContributionData(attrs));
c.set('contributor', contributors.findBy('id', attrs.contributorId.toString()));
items.push(c);
});
export default items;
+64
View File
@@ -0,0 +1,64 @@
export default {
"0": 1,
"1": "0x7E8f313C56F809188313aa274Fa67EE58c31515d",
"2": "0x99b8afd7b266e19990924a8be9099e81054b70c36b20937228a77a5cf75723b8",
"3": 18,
"4": 32,
"5": true,
"6": {
"_hex": "0x09979c0838e8fc880000"
},
"7": 53500,
"8": {
"_hex": "0x49"
},
"9": true,
"id": 1,
"account": "0x7E8f313C56F809188313aa274Fa67EE58c31515d",
"hashDigest": "0x99b8afd7b266e19990924a8be9099e81054b70c36b20937228a77a5cf75723b8",
"hashFunction": 18,
"hashSize": 32,
"isCore": true,
"balance": {
"_hex": "0x09979c0838e8fc880000"
},
"totalKreditsEarned": 53500,
"contributionsCount": {
"_hex": "0x49",
"toNumber": function() { return 73; }
},
"exists": true,
"balanceInt": 45298,
"ipfsHash": "QmYgiRd1FZ7JjDGBwkmLQNF6XQuyF8AWoFDRvUWhhmxiEj",
"name": "Bumi",
"kind": "person",
"url": "https://michaelbumann.com",
"accounts": [
{
"site": "github.com",
"uid": 318,
"username": "bumi",
"url": "https://github.com/bumi"
},
{
"site": "gitea.kosmos.org",
"username": "bumi",
"url": "https://gitea.kosmos.org/bumi"
},
{
"site": "wiki.kosmos.org",
"username": "Bumi",
"url": "https://wiki.kosmos.org/User:Bumi"
},
{
"site": "zoom.us",
"username": "bumi"
}
],
"github_uid": 318,
"github_username": "bumi",
"gitea_username": "bumi",
"wiki_username": "Bumi",
"zoom_display_name": "bumi",
"ipfsData": "{\n \"@context\": \"https://schema.kosmos.org\",\n \"@type\": \"Contributor\",\n \"kind\": \"person\",\n \"name\": \"Bumi\",\n \"accounts\": [\n {\n \"site\": \"github.com\",\n \"uid\": 318,\n \"username\": \"bumi\",\n \"url\": \"https://github.com/bumi\"\n },\n {\n \"site\": \"gitea.kosmos.org\",\n \"username\": \"bumi\",\n \"url\": \"https://gitea.kosmos.org/bumi\"\n },\n {\n \"site\": \"wiki.kosmos.org\",\n \"username\": \"Bumi\",\n \"url\": \"https://wiki.kosmos.org/User:Bumi\"\n },\n {\n \"site\": \"zoom.us\",\n \"username\": \"bumi\"\n }\n ],\n \"url\": \"https://michaelbumann.com\"\n}"
}
+8 -4
View File
@@ -1,13 +1,17 @@
import Contributor from 'kredits-web/models/contributor';
import processContributorData from 'kredits-web/utils/process-contributor-data';
const contributors = [];
const data = [
{ id: 1, name: 'Bumi', totalKreditsEarned: 11500, github_uid: 318 },
{ id: 2, name: 'Râu Cao', totalKreditsEarned: 3000, github_uid: 842 },
{ id: 3, name: 'Manuel', totalKreditsEarned: 0, github_uid: 54812 }
{ id: '1', name: 'Bumi', totalKreditsEarned: 11500, github_uid: 318 },
{ id: '2', name: 'Râu Cao', totalKreditsEarned: 3000, github_uid: 842 },
{ id: '3', name: 'Manuel', totalKreditsEarned: 0, github_uid: 54812 }
];
data.forEach(attrs => contributors.push(Contributor.create(attrs)));
data.forEach(attrs => {
const c = Contributor.create(processContributorData(attrs));
contributors.push(c);
});
export default contributors;
+17
View File
@@ -0,0 +1,17 @@
import { module, test } from 'qunit';
import { setupTest } from 'ember-qunit';
module('Unit | Service | browser-cache', function(hooks) {
setupTest(hooks);
test('it exists', function(assert) {
let service = this.owner.lookup('service:browser-cache');
assert.ok(service);
});
test('creates kredits data stores', function(assert) {
let cache = this.owner.lookup('service:browser-cache');
assert.equal(cache.contributors._config.name, "kredits:rinkeby:contributors");
assert.equal(cache.contributions._config.name, "kredits:rinkeby:contributions");
});
});
@@ -0,0 +1,28 @@
import { module, test } from 'qunit';
import processContributionData from 'kredits-web/utils/process-contribution-data';
import testData from '../../fixtures/contribution-data';
module('Unit | Utility | process-contribution-data', function() {
let result = processContributionData(testData);
test('formats the data correctly', function(assert) {
assert.ok(typeof result.confirmedAt === 'number');
});
test('copies other properties', function(assert) {
[
'id', 'contributorId', 'amount', 'vetoed',
'ipfsHash', 'kind', 'description', 'details',
'url', 'date', 'time', 'pendingTx'
].forEach(p => {
assert.ok(Object.prototype.hasOwnProperty.call(result, p), `copies property ${p}`);
})
});
test('does not copy unnecessary properties', function(assert) {
['exists', '5'].forEach(p => {
assert.notOk(Object.prototype.hasOwnProperty.call(result, p));
})
});
});
@@ -0,0 +1,31 @@
import { module, test } from 'qunit';
import processContributorData from 'kredits-web/utils/process-contributor-data';
import testData from '../../fixtures/contributor-data';
module('Unit | Utility | process-contributor-data', function() {
let result = processContributorData(testData);
test('formats the data correctly', function(assert) {
// TODO use integers everywhere for IDs
assert.ok(typeof result.id === 'string');
assert.ok(typeof result.balance === 'number');
assert.ok(typeof result.totalKreditsEarned === 'number');
assert.ok(typeof result.contributionsCount === 'number');
});
test('copies other properties', function(assert) {
[
'account', 'accounts', 'ipfsHash', 'isCore', 'kind', 'name', 'url',
'github_username', 'github_uid', 'wiki_username', 'zoom_display_name'
].forEach(p => {
assert.ok(Object.prototype.hasOwnProperty.call(result, p), `copies property ${p}`);
})
});
test('does not copy unnecessary properties', function(assert) {
['exists', '5'].forEach(p => {
assert.notOk(Object.prototype.hasOwnProperty.call(result, p));
})
});
});