2.1.1

Reviewed-on: #10

.drone.yml

@@ -0,0 +1,23 @@
+---
+kind: pipeline
+name: node 14
+
+steps:
+- name: test
+  image: node:14
+  commands:
+  - npm install
+  - npm run build
+  - npm test
+
+---
+kind: pipeline
+name: node 16
+
+steps:
+- name: test
+  image: node:16
+  commands:
+  - npm install
+  - npm run build
+  - npm test

README.md

@@ -1,22 +1,40 @@
-[![Release](https://img.shields.io/npm/v/remotestorage-module-chat-messages.svg?style=flat)](https://github.com/67P/remotestorage-module-chat-messages/releases)
+[![Release](https://img.shields.io/npm/v/remotestorage-module-chat-messages.svg?style=flat)](https://www.npmjs.com/package/remotestorage-module-chat-messages)
 
 # remoteStorage Module: Chat Messages
 
 Stores chat messages in daily archive documents.
 
-Please feel free to open GitHub issues for questions, feature requests,
-protocol proposals, and whatever else you like.
+## Usage
 
-## Protocols
+Open a daily archive and write messages to it:
 
-### Currently supported
+```js
+const RemoteStorage = require("remotestoragejs");
+const ChatMessages  = require("remotestorage-module-chat-messages");
+const remoteStorage = new RemoteStorage({ modules: [ ChatMessages ] });
 
-* IRC
-* XMPP
+const archive = new remoteStorage.chatMessages.DailyArchive({
+  service: {
+    protocol: 'IRC',
+    domain: 'irc.libera.chat'
+  },
+  channelName: '#kosmos',
+  date: new Date(),
+  isPublic: true // Channel logs will be written to public folder
+});
 
-### Planned
+const messages = [
+  { "date": "2015-06-05T17:35:28.454Z", "user": "jimmy", "text": "knock knock" },
+  { "date": "2015-06-05T17:36:05.123Z", "user": "walter", "text": "who's there?" }
+];
 
-* Mattermost
-* Matrix
-* Slack
-* ...
+archive.addMessages(messages);
+```
+
+See the inline source code documentation (JSDoc) for usage details and function
+arguments. For a real-world integration example, see
+[hubot-remotestorage-logger](https://github.com/67P/hubot-remotestorage-logger/).
+
+## Support, bugs, feedback, questions
+
+Come and chat with us: https://wiki.kosmos.org/Main_Page#Chat

dist/build.js.LICENSE.txt

@@ -0,0 +1 @@
+/*! regenerator-runtime -- Copyright (c) 2014-present, Facebook, Inc. -- license (MIT): https://github.com/facebook/regenerator/blob/main/LICENSE */

package.json

@@ -1,26 +1,31 @@
 {
   "name": "remotestorage-module-chat-messages",
-  "version": "1.0.1",
+  "version": "2.1.1",
   "description": "Stores chat messages in daily archive files",
   "main": "./dist/build.js",
   "scripts": {
     "build": "NODE_ENV=production webpack",
     "dev": "webpack -w",
     "start": "npm run dev",
-    "test": "echo \"Error: no test specified\" && exit 1",
+    "test": "mocha tests/",
     "version": "npm run build && git add dist/"
   },
   "author": "Kosmos Contributors <mail@kosmos.org> (https://kosmos.org)",
   "license": "MIT",
+  "homepage": "https://gitea.kosmos.org/kosmos/rs-module-chat-messages",
   "repository": {
     "type": "git",
-    "url": "https://github.com/67P/remotestorage-module-chat-messages.git"
+    "url": "https://gitea.kosmos.org/kosmos/rs-module-chat-messages.git"
   },
   "devDependencies": {
-    "@babel/core": "^7.14.8",
-    "@babel/preset-env": "^7.14.9",
-    "babel-loader": "^8.2.2",
-    "webpack": "^5.48.0",
-    "webpack-cli": "^4.8.0"
+    "@babel/core": "^7.18.10",
+    "@babel/preset-env": "^7.18.10",
+    "babel-loader": "^8.2.5",
+    "chai": "^4.3.6",
+    "mocha": "^10.0.0",
+    "regenerator-runtime": "^0.13.9",
+    "sinon": "^14.0.0",
+    "webpack": "^5.74.0",
+    "webpack-cli": "^4.10.0"
   }
 }

src/chat-messages.js

@@ -1,3 +1,5 @@
+import 'regenerator-runtime/runtime';
+
 function pad (num) {
   num = String(num);
   if (num.length === 1) { num = "0" + num; }
@@ -12,15 +14,23 @@ function parseDate (date) {
   };
 };
 
+function lowestNumberInListing(items) {
+  const sortedNumbers = Object.keys(items)
+                              .map(i => parseInt(i))
+                              .filter(i => !Number.isNaN(i))
+                              .sort();
+  return sortedNumbers[0];
+}
+
 const ChatMessages = function (privateClient, publicClient) {
   /**
-   * Schema: chat-messages/daily
+   * Schema: chat-messages/daily-archive
    *
    * Represents one calendar day of chat messages
    *
    * @example
    * {
-   *   "@context": "https://kosmos.org/ns/v1",
+   *   "@context": "https://kosmos.org/ns/v2",
    *   "@id": "chat-messages/irc.libera.chat/channels/kosmos/",
    *   "@type": "ChatChannel",
    *   "service": {
@@ -48,8 +58,7 @@ const ChatMessages = function (privateClient, publicClient) {
     "properties": {
       "@context": {
         "type": "string",
-        "default": "https://kosmos.org/ns/v1",
-        "enum": ["https://kosmos.org/ns/v1"]
+        "default": "https://kosmos.org/ns/v2/chat-channel"
       },
       "@id": {
         "type": "string",
@@ -141,8 +150,53 @@ const ChatMessages = function (privateClient, publicClient) {
     "required": []
   };
 
-  privateClient.declareType("daily-archive", "https://kosmos.org/ns/v1", archiveSchema);
-  publicClient.declareType("daily-archive", "https://kosmos.org/ns/v1", archiveSchema);
+  privateClient.declareType("daily-archive", "https://kosmos.org/ns/v2/chat-channel", archiveSchema);
+  publicClient.declareType("daily-archive", "https://kosmos.org/ns/v2/chat-channel", archiveSchema);
+
+  /**
+   * Schema: chat-messages/daily-archive-meta
+   *
+   * Stores meta information about the daily archives
+   *
+   * @example
+   * {
+   *   "@context": "https://kosmos.org/ns/v2",
+   *   "@id": "chat-messages/irc.libera.chat/channels/kosmos/meta",
+   *   "@type": "ChatChannelMeta",
+   *   "first": "2009/01/03",
+   *   "last": "2021/11/05"
+   *   }
+   * }
+   */
+  const archiveMetaSchema = {
+    "type": "object",
+    "properties": {
+      "@context": {
+        "type": "string",
+        "default": "https://kosmos.org/ns/v2/chat-channel-meta"
+      },
+      "@id": {
+        "type": "string",
+      },
+      "@type": {
+        "type": "string",
+        "default": "ChatChannelMeta",
+        "enum": ["ChatChannelMeta"]
+      },
+      "first": {
+        "type": "string",
+        "pattern": "^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$"
+      },
+      "last": {
+        "type": "string",
+        "pattern": "^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$"
+      }
+    },
+    "required": ["@id", "first", "last"]
+  };
+
+  privateClient.declareType("daily-archive-meta", "https://kosmos.org/ns/v2/chat-channel-meta", archiveMetaSchema);
+  publicClient.declareType("daily-archive-meta", "https://kosmos.org/ns/v2/chat-channel-meta", archiveMetaSchema);
 
   /**
    * A daily archive stores chat messages by calendar day.
@@ -158,7 +212,7 @@ const ChatMessages = function (privateClient, publicClient) {
      * @param {string} options.channelName      - Name of room/channel (e.g. "#kosmos")
      * @param {string} [options.channelType]    - Type of channel ("room" or "person")
      * @param {date} options.date               - Date of archive day
-     * @param {boolean} options.isPublic        - Store logs in public folder (defaults to false)
+     * @param {boolean} [options.isPublic]      - Store logs in public folder (defaults to false)
      * @param {string} [options.previous]       - Date of previous log file as `YYYY/MM/DD`. Looked up automatically when not given
      * @param {string} [options.next]           - Date of next log file as `YYYY/MM/DD`. looked up automatically when not given
      *
@@ -257,17 +311,27 @@ const ChatMessages = function (privateClient, publicClient) {
       this.dateId = this.parsedDate.year+'/'+this.parsedDate.month+'/'+this.parsedDate.day;
 
       /**
-       * @property {string} path - Document path of the archive file
+       * @property {string} channelPath - Base directory path of the channel archives
        */
       if (this.channelType === "room") {
         // Normal chatroom
-        const channelName = this.channelName.replace(/#/,'');
-        this.path = `${this.service.domain}/channels/${channelName}/${this.dateId}`;
+        const channelName = this.channelName.replace(/^#/,'');
+        this.channelPath = `${this.service.domain}/channels/${channelName}`;
       } else {
-        // User direct message
-        this.path = `${this.service.domain}/users/${this.channelName}/${this.dateId}`;
+        // User direct messages
+        this.channelPath = `${this.service.domain}/users/${this.channelName}`;
       }
 
+      /**
+       * @property {string} path - Path of the archive document
+       */
+      this.path = `${this.channelPath}/${this.dateId}`;
+
+      /**
+       * @property {string} metaPath - Path of the channel's metadata document
+       */
+      this.metaPath = `${this.channelPath}/meta`;
+
       /**
        * @property {object} client - Public or private remoteStorgage.js BaseClient
        */
@@ -349,6 +413,8 @@ const ChatMessages = function (privateClient, publicClient) {
      * @returns {Promise}
      */
     remove () {
+      // TODO when removing, if previous is set, but not next, it means the
+      // removed file is the last archive. Thus, set "last" to previous file.
       return this.client.remove(this.path);
     }
 
@@ -359,8 +425,8 @@ const ChatMessages = function (privateClient, publicClient) {
      *
      * @private
      */
-    _updateDocument (archive, messages) {
-      console.debug('[chat-messages] Updating archive document', archive);
+    async _updateDocument (archive, messages) {
+      console.debug('[chat-messages] Updating archive document');
 
       if (Array.isArray(messages)) {
         messages.forEach(function(message) {
@@ -380,12 +446,12 @@ const ChatMessages = function (privateClient, publicClient) {
      *
      * @private
      */
-    _createDocument (messages) {
+    async _createDocument (messages) {
       console.debug('[chat-messages] Creating new archive document');
       const archive = this._buildArchiveObject();
 
       if (Array.isArray(messages)) {
-        messages.forEach((message) => {
+        messages.forEach(message => {
           archive.today.messages.push(message);
         });
       } else {
@@ -397,16 +463,22 @@ const ChatMessages = function (privateClient, publicClient) {
         // That includes setting 'next' in the previous log file
         if (this.previous) { archive.today.previous = this.previous; }
         if (this.next)     { archive.today.next = this.next; }
-        return this._sync(archive);
       } else {
         // Find and update previous archive, set 'previous' on this one
-        return this._updatePreviousArchive().then((previous) => {
-          if (typeof previous === 'object') {
-            archive.today.previous = previous.today['@id'];
-          }
-          return this._sync(archive);
-        });
+        const previous = await this._updatePreviousArchive();
+        if (typeof previous === 'object') {
+          archive.today.previous = previous.today['@id'];
+        }
       }
+
+      await this._sync(archive);
+
+      // TODO only write meta doc if argument is set on addMessages. This way
+      // we can avoid race conditions when syncing remote chat messages all at
+      // once for multiple days
+      await this._updateArchiveMetaDocument();
+
+      return;
     }
 
     /*
@@ -417,7 +489,7 @@ const ChatMessages = function (privateClient, publicClient) {
      * @private
      */
     _buildArchiveObject () {
-      const roomName = this.channelName.replace(/#/,'');
+      const roomName = this.channelName.replace(/^#/,'');
 
       const archive = {
         "@id": "chat-messages/"+this.service.domain+"/channels/"+roomName+"/",
@@ -461,7 +533,7 @@ const ChatMessages = function (privateClient, publicClient) {
           const path = this.path.substring(0, this.path.length-this.dateId.length)+archive.today['@id'];
 
           return this.client.storeObject('daily-archive', path, archive).then(() => {
-            console.debug('[chat-messages] Previous archive written to remote storage', path, archive);
+            console.debug('[chat-messages] Previous archive written to remote storage at', path);
             return archive;
           });
         } else {
@@ -537,6 +609,57 @@ const ChatMessages = function (privateClient, publicClient) {
       });
     }
 
+    async _updateArchiveMetaDocument () {
+      const meta = await this.client.getObject(this.metaPath);
+      if (typeof meta !== 'object') {
+        return this._createArchiveMetaDocument();
+      }
+
+      // Only update document if current date is newer than known "last"
+      if (Date.parse(meta.last.replace(/\//g,'-')) < Date.parse(this.date)) {
+        console.debug('[chat-messages]', 'Updating meta document for channel');
+        meta.last = this.dateId;
+        await this.client.storeObject('daily-archive-meta', this.metaPath, meta);
+      }
+
+      return;
+    }
+
+    async _createArchiveMetaDocument () {
+      console.debug('[chat-messages]', 'Creating new meta document for channel');
+      // When creating a new meta doc, we need to find the oldest archive,
+      // because older versions of the module did not write a meta doc.
+      const first = await this._findFirstArchive();
+      const roomName = this.channelName.replace(/^#/,'');
+
+      const meta = {
+        '@id': `chat-messages/${this.service.domain}/channels/${roomName}/meta`,
+        '@type': 'ChatChannelMeta',
+        first: first,
+        last: this.dateId // TODO might have to search for last?
+      };
+
+      return this.client.storeObject('daily-archive-meta', this.metaPath, meta)
+                        .then(() => console.debug('[chat-messages]', 'Meta document written to remote storage'))
+                        .catch(e => {
+                          console.log('[chat-messages]', `Failed to store ${this.metaPath}`);
+                          console.error(e);
+                        });
+    }
+
+    async _findFirstArchive () {
+      console.debug('[chat-messages]', 'Finding first archive for channel');
+      const years   = await this.client.getListing(`${this.channelPath}/`);
+      const year    = lowestNumberInListing(years);
+      const months  = await this.client.getListing(`${this.channelPath}/${year}/`);
+      const month   = lowestNumberInListing(months);
+      const days    = await this.client.getListing(`${this.channelPath}/${year}/${pad(month)}/`);
+      const day     = lowestNumberInListing(days);
+      const firstId =  `${year}/${pad(month)}/${pad(day)}`;
+      console.debug('[chat-messages]', 'First is', firstId);
+      return firstId;
+    }
+
     /*
      * Write archive document
      *
@@ -544,10 +667,10 @@ const ChatMessages = function (privateClient, publicClient) {
      *
      * @private
      */
-    _sync (obj) {
-      console.debug('[chat-messages] Writing archive object', obj);
+    async _sync (archive) {
+      console.debug(`[chat-messages] Writing archive object with ${archive.today.messages.length} messages`);
 
-      return this.client.storeObject('daily-archive', this.path, obj).then(function(){
+      return this.client.storeObject('daily-archive', this.path, archive).then(function(){
         console.debug('[chat-messages] Archive written to remote storage');
         return true;
       },function(error){

tests/chat-messages-spec.js

@@ -0,0 +1,114 @@
+const expect = require('chai').expect;
+const sandbox = require("sinon").createSandbox();
+const ChatMessages = require('../dist/build');
+
+const rsClient = {
+  declareType: function() {},
+  getObject: function() {},
+  getListing: function() {},
+  storeObject: function() {},
+  remove: function() {}
+}
+
+describe('ChatMessages', function () {
+
+  describe('constructor', function () {
+    let chatMessages;
+
+    before(function() {
+      chatMessages = new ChatMessages.builder(rsClient, rsClient);
+    });
+
+    it('behaves like a remoteStorage module', function () {
+      expect(chatMessages).to.be.an('object');
+      expect(chatMessages.exports).to.be.an('object');
+    });
+
+    it('exports the desired functionality', function () {
+      expect(chatMessages.exports.DailyArchive).to.be.a('function');
+    });
+  });
+
+  describe('DailyArchive', function () {
+    let archive;
+
+    before(function() {
+      chatMessages = (new ChatMessages.builder(rsClient, rsClient)).exports;
+
+      archive = new chatMessages.DailyArchive({
+        service: { protocol: 'IRC', domain: 'irc.libera.chat' },
+        channelName: '#kosmos',
+        date: new Date('2022-08-11')
+      });
+    });
+
+    describe('constructor', function () {
+      it('creates an archive instance with the desired properties', function () {
+        expect(archive).to.be.an('object');
+        expect(archive.service.protocol).to.eq('IRC');
+        expect(archive.service.domain).to.eq('irc.libera.chat');
+        expect(archive.channelName).to.eq('#kosmos');
+        expect(archive.channelType).to.eq('room');
+        expect(archive.date).to.be.a('date');
+        expect(archive.parsedDate.year).to.eq(2022);
+        expect(archive.parsedDate.month).to.eq('08');
+        expect(archive.parsedDate.day).to.eq('11');
+        expect(archive.dateId).to.eq('2022/08/11');
+        expect(archive.isPublic).to.eq(false);
+        expect(archive.channelPath).to.eq('irc.libera.chat/channels/kosmos');
+        expect(archive.path).to.eq('irc.libera.chat/channels/kosmos/2022/08/11');
+        expect(archive.metaPath).to.eq('irc.libera.chat/channels/kosmos/meta');
+        expect(archive.client).to.eq(rsClient);
+        expect(archive.previous).to.be.an('undefined');
+        expect(archive.next).to.be.an('undefined');
+      });
+    });
+
+    describe('#_updateArchiveMetaDocument', function () {
+      describe('meta up to date', function () {
+        before(function() {
+          sandbox.stub(archive.client, 'getObject').withArgs(archive.metaPath)
+                 .returns({
+                   '@id': `chat-messages/irc.libera.chat/channels/kosmos/meta`,
+                   '@type': 'ChatChannelMeta',
+                   first: '2021/01/01', last: '2022/08/11'
+                 })
+          sandbox.stub(archive.client, 'storeObject');
+        });
+
+        it('does not store a new archive', async function () {
+          await archive._updateArchiveMetaDocument();
+          sandbox.assert.notCalled(archive.client.storeObject);
+        });
+
+        after(function() { sandbox.restore() });
+      });
+
+      describe('meta needs updating', function () {
+        before(function() {
+          sandbox.stub(archive.client, 'getObject').withArgs(archive.metaPath)
+                 .returns({
+                   '@id': archive.metaPath,
+                   '@type': 'ChatChannelMeta',
+                   first: '2021/01/01', last: '2022/08/10'
+                 })
+          sandbox.stub(archive.client, 'storeObject');
+        });
+
+        it('stores a new archive', async function () {
+          await archive._updateArchiveMetaDocument();
+          sandbox.assert.calledWithMatch(
+            archive.client.storeObject,
+            'daily-archive-meta', archive.metaPath, {
+              '@id': archive.metaPath, '@type': 'ChatChannelMeta',
+              first: '2021/01/01', last: '2022/08/11'
+            }
+          );
+        });
+
+        after(function() { sandbox.restore() });
+      });
+    });
+  });
+
+});

webpack.config.js

@@ -9,7 +9,9 @@ module.exports = {
     filename: 'build.js',
     library: 'ChatMessages',
     libraryTarget: 'umd',
-    libraryExport: 'default'
+    libraryExport: 'default',
+    umdNamedDefine: true,
+    globalObject: 'this'
   },
   mode: isProd ? 'production' : 'development',
   devtool: isProd ? 'source-map' : 'eval-source-map',