Compare commits

...

19 Commits

Author SHA1 Message Date
Râu Cao
929188bc2b
2.1.1
All checks were successful
continuous-integration/drone/push Build is passing
2022-08-12 16:08:34 +01:00
f4388bb202 Merge pull request 'Set up CI, fix updating archive meta documents' (#10) from dev/testing into master
All checks were successful
continuous-integration/drone/push Build is passing
Reviewed-on: #10
2022-08-12 15:04:54 +00:00
Râu Cao
dafccf23e4
Remove extra newline
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is passing
2022-08-11 11:11:56 +01:00
Râu Cao
185d9c71df
Expect correct arguments for storing meta doc
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is passing
2022-08-11 10:51:13 +01:00
Râu Cao
8744307ee7
Fix updating of archive meta document
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is passing
It's brainfart typo fixing time!
2022-08-11 10:41:48 +01:00
Râu Cao
1821d3cb64
Add Drone config
Some checks failed
continuous-integration/drone/push Build is failing
continuous-integration/drone/pr Build is failing
2022-08-11 10:38:20 +01:00
Râu Cao
1738dd8ad9
Update build 2022-08-11 10:35:43 +01:00
Râu Cao
9fb8829901
Fix string replacement
Fixes only the first occurence of `/` being replaced when parsing the
current meta doc's `last` date.
2022-08-11 10:33:57 +01:00
Râu Cao
e663a46242
Add spec for updating meta document 2022-08-11 10:33:13 +01:00
Râu Cao
f48933751d
Add initial specs 2022-08-11 10:05:15 +01:00
Râu Cao
43871634be
Update dependencies 2022-08-11 08:31:40 +01:00
Râu Cao
4d30536b18
Update package lock 2022-08-11 08:27:27 +01:00
189cd1f86d
2.1.0 2021-11-13 21:59:53 +01:00
8db448c4a0
Make log less verbose 2021-11-13 21:59:34 +01:00
84613ed3a3 Merge pull request 'Add/store channel archive metadata document' (#8) from feature/7-archive_metadata into master
Reviewed-on: #8
2021-11-13 20:56:18 +00:00
f83ef92adc
Remove superfluous sort function
#8 (comment)

Co-authored-by: Garret Alfert <alfert@wevelop.de>
2021-11-13 21:53:56 +01:00
54352e09e8
Store meta document with first/last archive ID 2021-11-08 12:55:42 +01:00
cd93cd4167
Add archive metadata schema, RS type 2021-11-04 13:18:55 +01:00
9f477ba14b
Update badge link 2021-09-13 14:31:04 +02:00
9 changed files with 6491 additions and 917 deletions

23
.drone.yml Normal file
View File

@ -0,0 +1,23 @@
---
kind: pipeline
name: node 14
steps:
- name: test
image: node:14
commands:
- npm install
- npm run build
- npm test
---
kind: pipeline
name: node 16
steps:
- name: test
image: node:16
commands:
- npm install
- npm run build
- npm test

View File

@ -1,4 +1,4 @@
[![Release](https://img.shields.io/npm/v/remotestorage-module-chat-messages.svg?style=flat)](https://github.com/67P/remotestorage-module-chat-messages/releases) [![Release](https://img.shields.io/npm/v/remotestorage-module-chat-messages.svg?style=flat)](https://www.npmjs.com/package/remotestorage-module-chat-messages)
# remoteStorage Module: Chat Messages # remoteStorage Module: Chat Messages

3
dist/build.js vendored

File diff suppressed because one or more lines are too long

1
dist/build.js.LICENSE.txt vendored Normal file
View File

@ -0,0 +1 @@
/*! regenerator-runtime -- Copyright (c) 2014-present, Facebook, Inc. -- license (MIT): https://github.com/facebook/regenerator/blob/main/LICENSE */

2
dist/build.js.map vendored

File diff suppressed because one or more lines are too long

7072
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -1,13 +1,13 @@
{ {
"name": "remotestorage-module-chat-messages", "name": "remotestorage-module-chat-messages",
"version": "2.0.0", "version": "2.1.1",
"description": "Stores chat messages in daily archive files", "description": "Stores chat messages in daily archive files",
"main": "./dist/build.js", "main": "./dist/build.js",
"scripts": { "scripts": {
"build": "NODE_ENV=production webpack", "build": "NODE_ENV=production webpack",
"dev": "webpack -w", "dev": "webpack -w",
"start": "npm run dev", "start": "npm run dev",
"test": "echo \"Error: no test specified\" && exit 1", "test": "mocha tests/",
"version": "npm run build && git add dist/" "version": "npm run build && git add dist/"
}, },
"author": "Kosmos Contributors <mail@kosmos.org> (https://kosmos.org)", "author": "Kosmos Contributors <mail@kosmos.org> (https://kosmos.org)",
@ -18,10 +18,14 @@
"url": "https://gitea.kosmos.org/kosmos/rs-module-chat-messages.git" "url": "https://gitea.kosmos.org/kosmos/rs-module-chat-messages.git"
}, },
"devDependencies": { "devDependencies": {
"@babel/core": "^7.14.8", "@babel/core": "^7.18.10",
"@babel/preset-env": "^7.14.9", "@babel/preset-env": "^7.18.10",
"babel-loader": "^8.2.2", "babel-loader": "^8.2.5",
"webpack": "^5.48.0", "chai": "^4.3.6",
"webpack-cli": "^4.8.0" "mocha": "^10.0.0",
"regenerator-runtime": "^0.13.9",
"sinon": "^14.0.0",
"webpack": "^5.74.0",
"webpack-cli": "^4.10.0"
} }
} }

View File

@ -1,3 +1,5 @@
import 'regenerator-runtime/runtime';
function pad (num) { function pad (num) {
num = String(num); num = String(num);
if (num.length === 1) { num = "0" + num; } if (num.length === 1) { num = "0" + num; }
@ -12,9 +14,17 @@ function parseDate (date) {
}; };
}; };
function lowestNumberInListing(items) {
const sortedNumbers = Object.keys(items)
.map(i => parseInt(i))
.filter(i => !Number.isNaN(i))
.sort();
return sortedNumbers[0];
}
const ChatMessages = function (privateClient, publicClient) { const ChatMessages = function (privateClient, publicClient) {
/** /**
* Schema: chat-messages/daily * Schema: chat-messages/daily-archive
* *
* Represents one calendar day of chat messages * Represents one calendar day of chat messages
* *
@ -48,8 +58,7 @@ const ChatMessages = function (privateClient, publicClient) {
"properties": { "properties": {
"@context": { "@context": {
"type": "string", "type": "string",
"default": "https://kosmos.org/ns/v2", "default": "https://kosmos.org/ns/v2/chat-channel"
"enum": ["https://kosmos.org/ns/v2"]
}, },
"@id": { "@id": {
"type": "string", "type": "string",
@ -141,8 +150,53 @@ const ChatMessages = function (privateClient, publicClient) {
"required": [] "required": []
}; };
privateClient.declareType("daily-archive", "https://kosmos.org/ns/v2", archiveSchema); privateClient.declareType("daily-archive", "https://kosmos.org/ns/v2/chat-channel", archiveSchema);
publicClient.declareType("daily-archive", "https://kosmos.org/ns/v2", archiveSchema); publicClient.declareType("daily-archive", "https://kosmos.org/ns/v2/chat-channel", archiveSchema);
/**
* Schema: chat-messages/daily-archive-meta
*
* Stores meta information about the daily archives
*
* @example
* {
* "@context": "https://kosmos.org/ns/v2",
* "@id": "chat-messages/irc.libera.chat/channels/kosmos/meta",
* "@type": "ChatChannelMeta",
* "first": "2009/01/03",
* "last": "2021/11/05"
* }
* }
*/
const archiveMetaSchema = {
"type": "object",
"properties": {
"@context": {
"type": "string",
"default": "https://kosmos.org/ns/v2/chat-channel-meta"
},
"@id": {
"type": "string",
},
"@type": {
"type": "string",
"default": "ChatChannelMeta",
"enum": ["ChatChannelMeta"]
},
"first": {
"type": "string",
"pattern": "^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$"
},
"last": {
"type": "string",
"pattern": "^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$"
}
},
"required": ["@id", "first", "last"]
};
privateClient.declareType("daily-archive-meta", "https://kosmos.org/ns/v2/chat-channel-meta", archiveMetaSchema);
publicClient.declareType("daily-archive-meta", "https://kosmos.org/ns/v2/chat-channel-meta", archiveMetaSchema);
/** /**
* A daily archive stores chat messages by calendar day. * A daily archive stores chat messages by calendar day.
@ -257,17 +311,27 @@ const ChatMessages = function (privateClient, publicClient) {
this.dateId = this.parsedDate.year+'/'+this.parsedDate.month+'/'+this.parsedDate.day; this.dateId = this.parsedDate.year+'/'+this.parsedDate.month+'/'+this.parsedDate.day;
/** /**
* @property {string} path - Document path of the archive file * @property {string} channelPath - Base directory path of the channel archives
*/ */
if (this.channelType === "room") { if (this.channelType === "room") {
// Normal chatroom // Normal chatroom
const channelName = this.channelName.replace(/#/,''); const channelName = this.channelName.replace(/^#/,'');
this.path = `${this.service.domain}/channels/${channelName}/${this.dateId}`; this.channelPath = `${this.service.domain}/channels/${channelName}`;
} else { } else {
// User direct message // User direct messages
this.path = `${this.service.domain}/users/${this.channelName}/${this.dateId}`; this.channelPath = `${this.service.domain}/users/${this.channelName}`;
} }
/**
* @property {string} path - Path of the archive document
*/
this.path = `${this.channelPath}/${this.dateId}`;
/**
* @property {string} metaPath - Path of the channel's metadata document
*/
this.metaPath = `${this.channelPath}/meta`;
/** /**
* @property {object} client - Public or private remoteStorgage.js BaseClient * @property {object} client - Public or private remoteStorgage.js BaseClient
*/ */
@ -349,6 +413,8 @@ const ChatMessages = function (privateClient, publicClient) {
* @returns {Promise} * @returns {Promise}
*/ */
remove () { remove () {
// TODO when removing, if previous is set, but not next, it means the
// removed file is the last archive. Thus, set "last" to previous file.
return this.client.remove(this.path); return this.client.remove(this.path);
} }
@ -359,7 +425,7 @@ const ChatMessages = function (privateClient, publicClient) {
* *
* @private * @private
*/ */
_updateDocument (archive, messages) { async _updateDocument (archive, messages) {
console.debug('[chat-messages] Updating archive document'); console.debug('[chat-messages] Updating archive document');
if (Array.isArray(messages)) { if (Array.isArray(messages)) {
@ -380,12 +446,12 @@ const ChatMessages = function (privateClient, publicClient) {
* *
* @private * @private
*/ */
_createDocument (messages) { async _createDocument (messages) {
console.debug('[chat-messages] Creating new archive document'); console.debug('[chat-messages] Creating new archive document');
const archive = this._buildArchiveObject(); const archive = this._buildArchiveObject();
if (Array.isArray(messages)) { if (Array.isArray(messages)) {
messages.forEach((message) => { messages.forEach(message => {
archive.today.messages.push(message); archive.today.messages.push(message);
}); });
} else { } else {
@ -397,16 +463,22 @@ const ChatMessages = function (privateClient, publicClient) {
// That includes setting 'next' in the previous log file // That includes setting 'next' in the previous log file
if (this.previous) { archive.today.previous = this.previous; } if (this.previous) { archive.today.previous = this.previous; }
if (this.next) { archive.today.next = this.next; } if (this.next) { archive.today.next = this.next; }
return this._sync(archive);
} else { } else {
// Find and update previous archive, set 'previous' on this one // Find and update previous archive, set 'previous' on this one
return this._updatePreviousArchive().then((previous) => { const previous = await this._updatePreviousArchive();
if (typeof previous === 'object') { if (typeof previous === 'object') {
archive.today.previous = previous.today['@id']; archive.today.previous = previous.today['@id'];
} }
return this._sync(archive);
});
} }
await this._sync(archive);
// TODO only write meta doc if argument is set on addMessages. This way
// we can avoid race conditions when syncing remote chat messages all at
// once for multiple days
await this._updateArchiveMetaDocument();
return;
} }
/* /*
@ -417,7 +489,7 @@ const ChatMessages = function (privateClient, publicClient) {
* @private * @private
*/ */
_buildArchiveObject () { _buildArchiveObject () {
const roomName = this.channelName.replace(/#/,''); const roomName = this.channelName.replace(/^#/,'');
const archive = { const archive = {
"@id": "chat-messages/"+this.service.domain+"/channels/"+roomName+"/", "@id": "chat-messages/"+this.service.domain+"/channels/"+roomName+"/",
@ -461,7 +533,7 @@ const ChatMessages = function (privateClient, publicClient) {
const path = this.path.substring(0, this.path.length-this.dateId.length)+archive.today['@id']; const path = this.path.substring(0, this.path.length-this.dateId.length)+archive.today['@id'];
return this.client.storeObject('daily-archive', path, archive).then(() => { return this.client.storeObject('daily-archive', path, archive).then(() => {
console.debug('[chat-messages] Previous archive written to remote storage', path, archive); console.debug('[chat-messages] Previous archive written to remote storage at', path);
return archive; return archive;
}); });
} else { } else {
@ -537,6 +609,57 @@ const ChatMessages = function (privateClient, publicClient) {
}); });
} }
async _updateArchiveMetaDocument () {
const meta = await this.client.getObject(this.metaPath);
if (typeof meta !== 'object') {
return this._createArchiveMetaDocument();
}
// Only update document if current date is newer than known "last"
if (Date.parse(meta.last.replace(/\//g,'-')) < Date.parse(this.date)) {
console.debug('[chat-messages]', 'Updating meta document for channel');
meta.last = this.dateId;
await this.client.storeObject('daily-archive-meta', this.metaPath, meta);
}
return;
}
async _createArchiveMetaDocument () {
console.debug('[chat-messages]', 'Creating new meta document for channel');
// When creating a new meta doc, we need to find the oldest archive,
// because older versions of the module did not write a meta doc.
const first = await this._findFirstArchive();
const roomName = this.channelName.replace(/^#/,'');
const meta = {
'@id': `chat-messages/${this.service.domain}/channels/${roomName}/meta`,
'@type': 'ChatChannelMeta',
first: first,
last: this.dateId // TODO might have to search for last?
};
return this.client.storeObject('daily-archive-meta', this.metaPath, meta)
.then(() => console.debug('[chat-messages]', 'Meta document written to remote storage'))
.catch(e => {
console.log('[chat-messages]', `Failed to store ${this.metaPath}`);
console.error(e);
});
}
async _findFirstArchive () {
console.debug('[chat-messages]', 'Finding first archive for channel');
const years = await this.client.getListing(`${this.channelPath}/`);
const year = lowestNumberInListing(years);
const months = await this.client.getListing(`${this.channelPath}/${year}/`);
const month = lowestNumberInListing(months);
const days = await this.client.getListing(`${this.channelPath}/${year}/${pad(month)}/`);
const day = lowestNumberInListing(days);
const firstId = `${year}/${pad(month)}/${pad(day)}`;
console.debug('[chat-messages]', 'First is', firstId);
return firstId;
}
/* /*
* Write archive document * Write archive document
* *
@ -544,10 +667,10 @@ const ChatMessages = function (privateClient, publicClient) {
* *
* @private * @private
*/ */
_sync (obj) { async _sync (archive) {
console.debug(`[chat-messages] Writing archive object with ${obj.today.messages.length} messages`); console.debug(`[chat-messages] Writing archive object with ${archive.today.messages.length} messages`);
return this.client.storeObject('daily-archive', this.path, obj).then(function(){ return this.client.storeObject('daily-archive', this.path, archive).then(function(){
console.debug('[chat-messages] Archive written to remote storage'); console.debug('[chat-messages] Archive written to remote storage');
return true; return true;
},function(error){ },function(error){

114
tests/chat-messages-spec.js Normal file
View File

@ -0,0 +1,114 @@
const expect = require('chai').expect;
const sandbox = require("sinon").createSandbox();
const ChatMessages = require('../dist/build');
const rsClient = {
declareType: function() {},
getObject: function() {},
getListing: function() {},
storeObject: function() {},
remove: function() {}
}
describe('ChatMessages', function () {
describe('constructor', function () {
let chatMessages;
before(function() {
chatMessages = new ChatMessages.builder(rsClient, rsClient);
});
it('behaves like a remoteStorage module', function () {
expect(chatMessages).to.be.an('object');
expect(chatMessages.exports).to.be.an('object');
});
it('exports the desired functionality', function () {
expect(chatMessages.exports.DailyArchive).to.be.a('function');
});
});
describe('DailyArchive', function () {
let archive;
before(function() {
chatMessages = (new ChatMessages.builder(rsClient, rsClient)).exports;
archive = new chatMessages.DailyArchive({
service: { protocol: 'IRC', domain: 'irc.libera.chat' },
channelName: '#kosmos',
date: new Date('2022-08-11')
});
});
describe('constructor', function () {
it('creates an archive instance with the desired properties', function () {
expect(archive).to.be.an('object');
expect(archive.service.protocol).to.eq('IRC');
expect(archive.service.domain).to.eq('irc.libera.chat');
expect(archive.channelName).to.eq('#kosmos');
expect(archive.channelType).to.eq('room');
expect(archive.date).to.be.a('date');
expect(archive.parsedDate.year).to.eq(2022);
expect(archive.parsedDate.month).to.eq('08');
expect(archive.parsedDate.day).to.eq('11');
expect(archive.dateId).to.eq('2022/08/11');
expect(archive.isPublic).to.eq(false);
expect(archive.channelPath).to.eq('irc.libera.chat/channels/kosmos');
expect(archive.path).to.eq('irc.libera.chat/channels/kosmos/2022/08/11');
expect(archive.metaPath).to.eq('irc.libera.chat/channels/kosmos/meta');
expect(archive.client).to.eq(rsClient);
expect(archive.previous).to.be.an('undefined');
expect(archive.next).to.be.an('undefined');
});
});
describe('#_updateArchiveMetaDocument', function () {
describe('meta up to date', function () {
before(function() {
sandbox.stub(archive.client, 'getObject').withArgs(archive.metaPath)
.returns({
'@id': `chat-messages/irc.libera.chat/channels/kosmos/meta`,
'@type': 'ChatChannelMeta',
first: '2021/01/01', last: '2022/08/11'
})
sandbox.stub(archive.client, 'storeObject');
});
it('does not store a new archive', async function () {
await archive._updateArchiveMetaDocument();
sandbox.assert.notCalled(archive.client.storeObject);
});
after(function() { sandbox.restore() });
});
describe('meta needs updating', function () {
before(function() {
sandbox.stub(archive.client, 'getObject').withArgs(archive.metaPath)
.returns({
'@id': archive.metaPath,
'@type': 'ChatChannelMeta',
first: '2021/01/01', last: '2022/08/10'
})
sandbox.stub(archive.client, 'storeObject');
});
it('stores a new archive', async function () {
await archive._updateArchiveMetaDocument();
sandbox.assert.calledWithMatch(
archive.client.storeObject,
'daily-archive-meta', archive.metaPath, {
'@id': archive.metaPath, '@type': 'ChatChannelMeta',
first: '2021/01/01', last: '2022/08/11'
}
);
});
after(function() { sandbox.restore() });
});
});
});
});