Merge pull request #189 from 67P/feature/data_loading

Cache data in browser storage, and load from local cache when present

app/models/contribution.js

@@ -1,6 +1,5 @@
 import EmberObject, { computed } from '@ember/object';
 import { isEmpty, isPresent } from '@ember/utils';
-import bignumber from 'kredits-web/utils/cps/bignumber';
 import moment from 'moment';
 
 export default EmberObject.extend({
@@ -9,11 +8,15 @@ export default EmberObject.extend({
   id: null,
   contributorId: null,
   amount: null,
-  confirmedAt: bignumber('confirmedAtBlock', 'toNumber'),
+  confirmedAt: null,
   vetoed: null,
   ipfsHash: null,
 
-  creatorAccount: null,
+  // contributor model instance
+  contributor: null,
+
+  // TODO contributor who submitted the contribution
+  // submittedBy: null,
 
   // IPFS
   kind: null,
@@ -22,7 +25,6 @@ export default EmberObject.extend({
   url: null,
   date: null,
   time: null,
-  ipfsData: '',
 
   pendingTx: null,
 
@@ -41,6 +43,10 @@ export default EmberObject.extend({
 
   hasPendingChanges: computed('pendingTx', function() {
     return isPresent(this.pendingTx);
-  })
+  }),
+
+  serialize () {
+    return JSON.stringify(this);
+  }
 
 });

app/models/contributor.js

@@ -1,14 +1,12 @@
 import EmberObject from '@ember/object';
-import bignumber from 'kredits-web/utils/cps/bignumber';
-import kreditsValue from 'kredits-web/utils/cps/kredits';
 
 export default EmberObject.extend({
   // Contract
-  id: bignumber('idRaw', 'toString'),
+  id: null,
   account: null,
-  balance: kreditsValue('balanceRaw'),
+  balance: 0,
-  totalKreditsEarned: bignumber('totalKreditsEarnedRaw', 'toNumber'),
+  totalKreditsEarned: 0,
-  contributionsCount: bignumber('contributionsCountRaw', 'toNumber'),
+  contributionsCount: 0,
   isCore: false,
   ipfsHash: null,
 
@@ -20,6 +18,9 @@ export default EmberObject.extend({
   github_uid: null,
   wiki_username: null,
   zoom_display_name: null,
-  ipfsData: ''
+
+  serialize () {
+    return JSON.stringify(this);
+  }
 
 });

app/routes/application.js

@@ -1,5 +1,6 @@
 import { inject as service } from '@ember/service';
 import Route from '@ember/routing/route';
+import { schedule } from '@ember/runloop';
 
 export default Route.extend({
   kredits: service(),
@@ -21,6 +22,12 @@ export default Route.extend({
     return this.kredits.loadInitialData()
       .then(() => {
         this.kredits.addContractEventHandlers();
+      })
+      .then(() => {
+        if (this.kredits.contributorsNeedFetch) {
+          schedule('afterRender', this.kredits, this.kredits.fetchContributors);
+        }
+        schedule('afterRender', this.kredits, this.kredits.fetchContributions);
       });
   }
 });

app/services/browser-cache.js

@@ -0,0 +1,27 @@
+import Service from '@ember/service';
+import * as localforage from 'localforage';
+import config from 'kredits-web/config/environment';
+
+function createStore(name) {
+  const networkName = config.web3RequiredNetwork || 'custom';
+  return localforage.createInstance({ name: `kredits:${networkName}:${name}` });
+}
+
+export default class BrowserCacheService extends Service {
+
+  constructor() {
+    super(...arguments);
+    this.stores = {
+      contributors:  createStore('contributors'),
+      contributions: createStore('contributions')
+    }
+  }
+
+  get contributors() {
+    return this.stores.contributors;
+  }
+
+  get contributions() {
+    return this.stores.contributions;
+  }
+}

app/services/kredits.js

@@ -7,8 +7,11 @@ import EmberObject from '@ember/object';
 import { computed } from '@ember/object';
 import { alias, notEmpty } from '@ember/object/computed';
 import { isEmpty, isPresent } from '@ember/utils';
+import { inject as service } from '@ember/service';
 
 import groupBy from 'kredits-web/utils/group-by';
+import processContributorData from 'kredits-web/utils/process-contributor-data';
+import processContributionData from 'kredits-web/utils/process-contribution-data';
 import formatKredits from 'kredits-web/utils/format-kredits';
 
 import config from 'kredits-web/config/environment';
@@ -17,6 +20,9 @@ import Contribution from 'kredits-web/models/contribution'
 
 export default Service.extend({
 
+  browserCache: service(),
+  contributorsNeedFetch: false,
+
   currentBlock: null,
   currentUserAccounts: null, // default to not having an account. this is the wen web3 is loaded.
   currentUser: null,
@@ -169,12 +175,23 @@ export default Service.extend({
       .then(total => total.toNumber());
   }),
 
+  async loadInitialData () {
+    const numCachedContributors = await this.browserCache.contributors.length();
+    if (numCachedContributors > 0) {
+      await this.loadContributorsFromCache();
+      this.set('contributorsNeedFetch', true);
+    } else {
+      await this.fetchContributors();
+    }
 
-  loadInitialData () {
+    const numCachedContributions = await this.browserCache.contributions.length();
-    return this.getContributors()
+    if (numCachedContributions > 0) {
-               .then(contributors => this.contributors.pushObjects(contributors))
+      await this.loadContributionsFromCache();
-               .then(() => this.getContributions())
+    } else {
-               .then(contributions => this.contributions.pushObjects(contributions))
+      await this.fetchContributions({ page: { size: 30 } });
+    }
+
+    return Promise.resolve();
   },
 
   addContributor (attributes) {
@@ -205,15 +222,39 @@ export default Service.extend({
       });
   },
 
-  getContributors () {
+  fetchContributors () {
+    console.debug(`[kredits] Fetching all contributors from the network`);
     return this.kredits.Contributor.all()
       .then(contributors => {
-        return contributors.map(contributor => {
+        return contributors.map(data => {
-          return Contributor.create(contributor);
+          const contributor = Contributor.create(processContributorData(data));
+          const loadedContributor = this.contributors.findBy('id', contributor.id);
+          if (loadedContributor) { this.contributors.removeObject(loadedContributor); }
+          this.contributors.pushObject(contributor);
+          return contributor;
         });
+      })
+      .then(() => {
+        return this.cacheLoadedContributors();
       });
   },
 
+  async cacheLoadedContributors () {
+    for (const c of this.contributors) {
+      await this.browserCache.contributors.setItem(c.id, c.serialize());
+    }
+    console.debug(`[kredits] Cached ${this.contributors.length} contributors in browser storage`);
+    return Promise.resolve();
+  },
+
+  async loadContributorsFromCache () {
+    return this.browserCache.contributors.iterate((value/*, key , iterationNumber */) => {
+      this.contributors.pushObject(Contributor.create(JSON.parse(value)));
+    }).then((/* result */) => {
+      console.debug(`[kredits] Loaded ${this.contributors.length} contributors from cache`);
+    });
+  },
+
   addContribution (attributes) {
     console.debug('[kredits] add contribution', attributes);
 
@@ -229,16 +270,45 @@ export default Service.extend({
       });
   },
 
-  getContributions () {
+  fetchContributions (options = { page: { size: 200 } }) {
-    return this.kredits.Contribution.all({page: {size: 200}})
+    console.debug(`[kredits] Fetching contributions from the network`);
+    return this.kredits.Contribution.all(options)
       .then(contributions => {
-        return contributions.map(contribution => {
+        return contributions.map(data => {
-          contribution.contributor = this.contributors.findBy('id', contribution.contributorId.toString());
+          const contribution = Contribution.create(processContributionData(data));
-          return Contribution.create(contribution);
+          contribution.set('contributor', this.contributors.findBy('id', data.contributorId.toString()));
+          const loadedContribution = this.contributions.findBy('id', contribution.id);
+          if (loadedContribution) { this.contributions.removeObject(loadedContribution); }
+          this.contributions.pushObject(contribution);
+          return contribution;
+        });
+      })
+      .then(contributions => {
+        const cacheWrites = contributions.map(c => {
+          return this.browserCache.contributions.setItem(c.id.toString(), c.serialize());
+        });
+        return Promise.all(cacheWrites).then(() => {
+          console.debug(`[kredits] Cached ${contributions.length} contributions in browser storage`);
         });
       });
   },
 
+  async cacheLoadedContributions () {
+    for (const c of this.contributions) {
+      await this.browserCache.contributions.setItem(c.id, c.serialize());
+    }
+    console.debug(`[kredits] Cached ${this.contributions.length} contributions in browser storage`);
+    return Promise.resolve();
+  },
+
+  async loadContributionsFromCache () {
+    return this.browserCache.contributions.iterate((value/*, key , iterationNumber */) => {
+      this.contributions.pushObject(Contribution.create(JSON.parse(value)));
+    }).then((/* result */) => {
+      console.debug(`[kredits] Loaded ${this.contributions.length} contributions from cache`);
+    });
+  },
+
   veto (contributionId) {
     console.debug('[kredits] veto against', contributionId);
     const contribution = this.contributions.findBy('id', contributionId);

app/utils/cps/bignumber.js

@@ -1,20 +0,0 @@
-import { computed } from '@ember/object';
-import ethers from 'ethers';
-
-export default function(dependentKey, converterMethod) {
-  return computed(dependentKey, {
-    get () {
-      let value = this.get(dependentKey);
-      if (value && ethers.utils.BigNumber.isBigNumber(value)) {
-        return value[converterMethod]();
-      } else {
-        return value;
-      }
-    },
-    set (key, value) {
-      const bnValue = ethers.utils.bigNumberify(value);
-      this.set(dependentKey, bnValue);
-      return bnValue[converterMethod]();
-    }
-  });
-}

app/utils/cps/kredits.js

@@ -1,17 +0,0 @@
-import { computed } from '@ember/object';
-import ethers from 'ethers';
-import formatKredits from 'kredits-web/utils/format-kredits';
-
-export default function(dependentKey, options = {}) {
-  return computed(dependentKey, {
-    get () {
-      const value = this.get(dependentKey);
-      return formatKredits(value, options);
-    },
-    set (key, value) {
-      const bnValue = ethers.utils.bigNumberify(value);
-      this.set(dependentKey, bnValue);
-      return formatKredits(bnValue, options);
-    }
-  });
-}

app/utils/process-contribution-data.js

@@ -0,0 +1,21 @@
+export default function processContributionData(data) {
+  const processed = {}
+
+  if (data.confirmedAtBlock && (typeof data.confirmedAtBlock.toNumber === 'function')) {
+    processed.confirmedAt = data.confirmedAtBlock.toNumber();
+  } else if (data.confirmedAt !== 'undefined') {
+    processed.confirmedAt = data.confirmedAt;
+  }
+
+  const otherProperties = [
+    'id', 'contributorId', 'amount', 'vetoed',
+    'ipfsHash', 'kind', 'description', 'details',
+    'url', 'date', 'time', 'pendingTx'
+  ];
+
+  otherProperties.forEach(prop => {
+    processed[prop] = data[prop];
+  });
+
+  return processed;
+}

app/utils/process-contributor-data.js

@@ -0,0 +1,19 @@
+export default function processContributorData(data) {
+  const processed = {
+    id: data.id.toString(),
+    balance: data.balanceInt,
+    totalKreditsEarned: data.totalKreditsEarned,
+    contributionsCount: data.contributionsCount?.toNumber()
+  }
+
+  const otherProperties = [
+    'account', 'accounts', 'ipfsHash', 'isCore', 'kind', 'name', 'url',
+    'github_username', 'github_uid', 'wiki_username', 'zoom_display_name'
+  ];
+
+  otherProperties.forEach(prop => {
+    processed[prop] = data[prop];
+  });
+
+  return processed;
+}

package-lock.json

@@ -15545,6 +15545,12 @@
       "integrity": "sha512-MzbUSahkTW1u7JpKKjY7LCARd1fU5W2rLdxlM4kdkayuCwZImjkpluF9CM1aLewYJguPDqewLam18Y6AU69A8A==",
       "dev": true
     },
+    "immediate": {
+      "version": "3.0.6",
+      "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz",
+      "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=",
+      "dev": true
+    },
     "import-fresh": {
       "version": "3.2.1",
       "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz",
@@ -16768,6 +16774,15 @@
         }
       }
     },
+    "lie": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/lie/-/lie-3.1.1.tgz",
+      "integrity": "sha1-mkNrLMd0bKWd56QfpGmz77dr2H4=",
+      "dev": true,
+      "requires": {
+        "immediate": "~3.0.5"
+      }
+    },
     "linkify-it": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-2.2.0.tgz",
@@ -16849,6 +16864,15 @@
       "integrity": "sha512-9M2KvGT6duzGMgkOcTkWb+PR/Q2Oe54df/tLgHGVmFpAmtqJ553xJh6N63iFYI2yjo2PeJXbS5skHi/QpJq4vA==",
       "dev": true
     },
+    "localforage": {
+      "version": "1.7.3",
+      "resolved": "https://registry.npmjs.org/localforage/-/localforage-1.7.3.tgz",
+      "integrity": "sha512-1TulyYfc4udS7ECSBT2vwJksWbkwwTX8BzeUIiq8Y07Riy7bDAAnxDaPU/tWyOVmQAcWJIEIFP9lPfBGqVoPgQ==",
+      "dev": true,
+      "requires": {
+        "lie": "3.1.1"
+      }
+    },
     "locate-character": {
       "version": "2.0.5",
       "resolved": "https://registry.npmjs.org/locate-character/-/locate-character-2.0.5.tgz",

package.json

@@ -67,6 +67,7 @@
     "kosmos-schemas": "^2.1.0",
     "kredits-contracts": "^5.5.0",
     "loader.js": "^4.7.0",
+    "localforage": "^1.7.3",
     "ndjson": "github:hugomrdias/ndjson#feat/readable-stream3",
     "npm-run-all": "^4.1.5",
     "qunit-dom": "^1.2.0",

tests/fixtures/contributions.js

@@ -1,23 +1,25 @@
 import Model from 'kredits-web/models/contribution';
 import contributors from '../../tests/fixtures/contributors';
+import processContributionData from 'kredits-web/utils/process-contribution-data';
 
 const items = [];
 
 const data = [
-  { id: 1, contributorId: 1, confirmedAtBlock: 1000, claimed: false, vetoed: false, amount: 1500, kind: 'dev' },
+  { id: 1, contributorId: 1, confirmedAt: 1000, claimed: false, vetoed: false, amount: 1500, kind: 'dev' },
-  { id: 2, contributorId: 1, confirmedAtBlock: 1000, claimed: false, vetoed: false, amount: 5000, kind: 'ops' },
+  { id: 2, contributorId: 1, confirmedAt: 1000, claimed: false, vetoed: false, amount: 5000, kind: 'ops' },
-  { id: 3, contributorId: 2, confirmedAtBlock: 1000, claimed: false, vetoed: false, amount: 1500, kind: 'ops' },
+  { id: 3, contributorId: 2, confirmedAt: 1000, claimed: false, vetoed: false, amount: 1500, kind: 'ops' },
-  { id: 4, contributorId: 2, confirmedAtBlock: 1000, claimed: false, vetoed: false, amount: 1500, kind: 'docs' },
+  { id: 4, contributorId: 2, confirmedAt: 1000, claimed: false, vetoed: false, amount: 1500, kind: 'docs' },
-  { id: 5, contributorId: 1, confirmedAtBlock: 1000, claimed: false, vetoed: false, amount: 5000, kind: 'design' },
+  { id: 5, contributorId: 1, confirmedAt: 1000, claimed: false, vetoed: false, amount: 5000, kind: 'design' },
-  { id: 6, contributorId: 1, confirmedAtBlock: 1000, claimed: false, vetoed: true, amount: 500, kind: 'dev' },
+  { id: 6, contributorId: 1, confirmedAt: 1000, claimed: false, vetoed: true, amount: 500, kind: 'dev' },
-  { id: 7, contributorId: 3, confirmedAtBlock: 2000, claimed: false, vetoed: false, amount: 5000, kind: 'dev' },
+  { id: 7, contributorId: 3, confirmedAt: 2000, claimed: false, vetoed: false, amount: 5000, kind: 'dev' },
-  { id: 8, contributorId: 1, confirmedAtBlock: 2000, claimed: false, vetoed: false, amount: 1500, kind: 'community' },
+  { id: 8, contributorId: 1, confirmedAt: 2000, claimed: false, vetoed: false, amount: 1500, kind: 'community' },
-  { id: 9, contributorId: 3, confirmedAtBlock: 2000, claimed: false, vetoed: true, amount: 1500, kind: 'docs' },
+  { id: 9, contributorId: 3, confirmedAt: 2000, claimed: false, vetoed: true, amount: 1500, kind: 'docs' },
 ];
 
 data.forEach(attrs => {
-  attrs.contributor = contributors.findBy('id', attrs.contributorId.toString());
+  const c = Model.create(processContributionData(attrs));
-  items.push(Model.create(attrs))
+  c.set('contributor', contributors.findBy('id', attrs.contributorId.toString()));
+  items.push(c);
 });
 
 export default items;

tests/fixtures/contributor-data.js

@@ -0,0 +1,64 @@
+export default {
+  "0": 1,
+  "1": "0x7E8f313C56F809188313aa274Fa67EE58c31515d",
+  "2": "0x99b8afd7b266e19990924a8be9099e81054b70c36b20937228a77a5cf75723b8",
+  "3": 18,
+  "4": 32,
+  "5": true,
+  "6": {
+    "_hex": "0x09979c0838e8fc880000"
+  },
+  "7": 53500,
+  "8": {
+    "_hex": "0x49"
+  },
+  "9": true,
+  "id": 1,
+  "account": "0x7E8f313C56F809188313aa274Fa67EE58c31515d",
+  "hashDigest": "0x99b8afd7b266e19990924a8be9099e81054b70c36b20937228a77a5cf75723b8",
+  "hashFunction": 18,
+  "hashSize": 32,
+  "isCore": true,
+  "balance": {
+    "_hex": "0x09979c0838e8fc880000"
+  },
+  "totalKreditsEarned": 53500,
+  "contributionsCount": {
+    "_hex": "0x49",
+    "toNumber": function() { return 73; }
+  },
+  "exists": true,
+  "balanceInt": 45298,
+  "ipfsHash": "QmYgiRd1FZ7JjDGBwkmLQNF6XQuyF8AWoFDRvUWhhmxiEj",
+  "name": "Bumi",
+  "kind": "person",
+  "url": "https://michaelbumann.com",
+  "accounts": [
+    {
+      "site": "github.com",
+      "uid": 318,
+      "username": "bumi",
+      "url": "https://github.com/bumi"
+    },
+    {
+      "site": "gitea.kosmos.org",
+      "username": "bumi",
+      "url": "https://gitea.kosmos.org/bumi"
+    },
+    {
+      "site": "wiki.kosmos.org",
+      "username": "Bumi",
+      "url": "https://wiki.kosmos.org/User:Bumi"
+    },
+    {
+      "site": "zoom.us",
+      "username": "bumi"
+    }
+  ],
+  "github_uid": 318,
+  "github_username": "bumi",
+  "gitea_username": "bumi",
+  "wiki_username": "Bumi",
+  "zoom_display_name": "bumi",
+  "ipfsData": "{\n  \"@context\": \"https://schema.kosmos.org\",\n  \"@type\": \"Contributor\",\n  \"kind\": \"person\",\n  \"name\": \"Bumi\",\n  \"accounts\": [\n    {\n      \"site\": \"github.com\",\n      \"uid\": 318,\n      \"username\": \"bumi\",\n      \"url\": \"https://github.com/bumi\"\n    },\n    {\n      \"site\": \"gitea.kosmos.org\",\n      \"username\": \"bumi\",\n      \"url\": \"https://gitea.kosmos.org/bumi\"\n    },\n    {\n      \"site\": \"wiki.kosmos.org\",\n      \"username\": \"Bumi\",\n      \"url\": \"https://wiki.kosmos.org/User:Bumi\"\n    },\n    {\n      \"site\": \"zoom.us\",\n      \"username\": \"bumi\"\n    }\n  ],\n  \"url\": \"https://michaelbumann.com\"\n}"
+}

tests/fixtures/contributors.js

@@ -1,13 +1,17 @@
 import Contributor from 'kredits-web/models/contributor';
+import processContributorData from 'kredits-web/utils/process-contributor-data';
 
 const contributors = [];
 
 const data = [
-  { id: 1, name: 'Bumi', totalKreditsEarned: 11500, github_uid: 318 },
+  { id: '1', name: 'Bumi', totalKreditsEarned: 11500, github_uid: 318 },
-  { id: 2, name: 'Râu Cao', totalKreditsEarned: 3000, github_uid: 842 },
+  { id: '2', name: 'Râu Cao', totalKreditsEarned: 3000, github_uid: 842 },
-  { id: 3, name: 'Manuel', totalKreditsEarned: 0, github_uid: 54812 }
+  { id: '3', name: 'Manuel', totalKreditsEarned: 0, github_uid: 54812 }
 ];
 
-data.forEach(attrs => contributors.push(Contributor.create(attrs)));
+data.forEach(attrs => {
+  const c = Contributor.create(processContributorData(attrs));
+  contributors.push(c);
+});
 
 export default contributors;

tests/unit/services/browser-cache-test.js

@@ -0,0 +1,17 @@
+import { module, test } from 'qunit';
+import { setupTest } from 'ember-qunit';
+
+module('Unit | Service | browser-cache', function(hooks) {
+  setupTest(hooks);
+
+  test('it exists', function(assert) {
+    let service = this.owner.lookup('service:browser-cache');
+    assert.ok(service);
+  });
+
+  test('creates kredits data stores', function(assert) {
+    let cache = this.owner.lookup('service:browser-cache');
+    assert.equal(cache.contributors._config.name, "kredits:rinkeby:contributors");
+    assert.equal(cache.contributions._config.name, "kredits:rinkeby:contributions");
+  });
+});

tests/unit/utils/process-contribution-data-test.js

@@ -0,0 +1,28 @@
+import { module, test } from 'qunit';
+import processContributionData from 'kredits-web/utils/process-contribution-data';
+import testData from '../../fixtures/contribution-data';
+
+module('Unit | Utility | process-contribution-data', function() {
+
+  let result = processContributionData(testData);
+
+  test('formats the data correctly', function(assert) {
+    assert.ok(typeof result.confirmedAt === 'number');
+  });
+
+  test('copies other properties', function(assert) {
+    [
+      'id', 'contributorId', 'amount', 'vetoed',
+      'ipfsHash', 'kind', 'description', 'details',
+      'url', 'date', 'time', 'pendingTx'
+    ].forEach(p => {
+      assert.ok(Object.prototype.hasOwnProperty.call(result, p), `copies property ${p}`);
+    })
+  });
+
+  test('does not copy unnecessary properties', function(assert) {
+    ['exists', '5'].forEach(p => {
+      assert.notOk(Object.prototype.hasOwnProperty.call(result, p));
+    })
+  });
+});

tests/unit/utils/process-contributor-data-test.js

@@ -0,0 +1,31 @@
+import { module, test } from 'qunit';
+import processContributorData from 'kredits-web/utils/process-contributor-data';
+import testData from '../../fixtures/contributor-data';
+
+module('Unit | Utility | process-contributor-data', function() {
+
+  let result = processContributorData(testData);
+
+  test('formats the data correctly', function(assert) {
+    // TODO use integers everywhere for IDs
+    assert.ok(typeof result.id === 'string');
+    assert.ok(typeof result.balance === 'number');
+    assert.ok(typeof result.totalKreditsEarned === 'number');
+    assert.ok(typeof result.contributionsCount === 'number');
+  });
+
+  test('copies other properties', function(assert) {
+    [
+      'account', 'accounts', 'ipfsHash', 'isCore', 'kind', 'name', 'url',
+      'github_username', 'github_uid', 'wiki_username', 'zoom_display_name'
+    ].forEach(p => {
+      assert.ok(Object.prototype.hasOwnProperty.call(result, p), `copies property ${p}`);
+    })
+  });
+
+  test('does not copy unnecessary properties', function(assert) {
+    ['exists', '5'].forEach(p => {
+      assert.notOk(Object.prototype.hasOwnProperty.call(result, p));
+    })
+  });
+});