Compare commits

...

40 Commits

Author SHA1 Message Date
Râu Cao
929188bc2b
2.1.1
All checks were successful
continuous-integration/drone/push Build is passing
2022-08-12 16:08:34 +01:00
f4388bb202 Merge pull request 'Set up CI, fix updating archive meta documents' (#10) from dev/testing into master
All checks were successful
continuous-integration/drone/push Build is passing
Reviewed-on: #10
2022-08-12 15:04:54 +00:00
Râu Cao
dafccf23e4
Remove extra newline
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is passing
2022-08-11 11:11:56 +01:00
Râu Cao
185d9c71df
Expect correct arguments for storing meta doc
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is passing
2022-08-11 10:51:13 +01:00
Râu Cao
8744307ee7
Fix updating of archive meta document
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is passing
It's brainfart typo fixing time!
2022-08-11 10:41:48 +01:00
Râu Cao
1821d3cb64
Add Drone config
Some checks failed
continuous-integration/drone/push Build is failing
continuous-integration/drone/pr Build is failing
2022-08-11 10:38:20 +01:00
Râu Cao
1738dd8ad9
Update build 2022-08-11 10:35:43 +01:00
Râu Cao
9fb8829901
Fix string replacement
Fixes only the first occurence of `/` being replaced when parsing the
current meta doc's `last` date.
2022-08-11 10:33:57 +01:00
Râu Cao
e663a46242
Add spec for updating meta document 2022-08-11 10:33:13 +01:00
Râu Cao
f48933751d
Add initial specs 2022-08-11 10:05:15 +01:00
Râu Cao
43871634be
Update dependencies 2022-08-11 08:31:40 +01:00
Râu Cao
4d30536b18
Update package lock 2022-08-11 08:27:27 +01:00
189cd1f86d
2.1.0 2021-11-13 21:59:53 +01:00
8db448c4a0
Make log less verbose 2021-11-13 21:59:34 +01:00
84613ed3a3 Merge pull request 'Add/store channel archive metadata document' (#8) from feature/7-archive_metadata into master
Reviewed-on: #8
2021-11-13 20:56:18 +00:00
f83ef92adc
Remove superfluous sort function
#8 (comment)

Co-authored-by: Garret Alfert <alfert@wevelop.de>
2021-11-13 21:53:56 +01:00
54352e09e8
Store meta document with first/last archive ID 2021-11-08 12:55:42 +01:00
cd93cd4167
Add archive metadata schema, RS type 2021-11-04 13:18:55 +01:00
9f477ba14b
Update badge link 2021-09-13 14:31:04 +02:00
328be7ed75
2.0.0 2021-09-13 14:08:10 +02:00
cf8c43bede Merge pull request 'Add proper README' (#6) from docs/readme into master
Reviewed-on: #6
2021-09-13 12:06:56 +00:00
188b45e778
Add proper README
* Document usage
* Update communication options for support/issues
2021-09-13 14:03:41 +02:00
d331af7256 Merge pull request 'Various pre-release fixes' (#5) from dev/prerelease_fixes into master
Reviewed-on: #5
2021-09-13 11:43:27 +00:00
9ba618f38e
Update repo URL, add website 2021-09-13 13:40:24 +02:00
0b517400b1 Remove noise from logs 2021-09-13 13:18:25 +02:00
15af899e1d Document isPublic argument as optional 2021-09-13 13:17:51 +02:00
bf896a076c Update context version 2021-09-13 13:17:32 +02:00
a5a547d40d
Fix build config to work with node.js 2021-09-13 13:16:40 +02:00
21cb5a02c8 Merge pull request 'Update data model, documentation' (#4) from feature/2-update_data_model into master
Reviewed-on: #4
2021-09-10 12:34:40 +00:00
c42e0a37a8
Update build 2021-09-05 15:59:10 +02:00
2a73816e56 Merge branch 'master' into feature/2-update_data_model 2021-09-05 13:58:28 +00:00
74680ad42b Merge pull request 'Update JS syntax' (#3) from chore/modern_js into master
Reviewed-on: #3
2021-09-05 13:57:09 +00:00
dd1bfd08ea
Update data model, documentation
* Port NaturalDocs code documentation to JSDoc
* server --> service,
  server.name --> service.domain,
  server.type --> service.protocol
* Remove ircURI and xmppURI in favor of service.protocol
* Add channel type ("room" or "person")
* Change document path to use service domain instead of custom id/name
2021-09-05 15:56:17 +02:00
e803a1e799
Move source to src directory 2021-09-05 14:05:06 +02:00
54f5d903d0
Update JS syntax 2021-09-04 14:11:37 +02:00
a270d5f9d6 Merge pull request 'Update development environment (Webpack)' (#1) from chore/update_dev_env into master
Reviewed-on: #1
2021-09-04 11:46:25 +00:00
603fd507bf
Update development environment (Webpack) 2021-09-04 13:45:15 +02:00
6b95652a76
Update package meta 2021-09-04 12:57:47 +02:00
89c4ee229a 1.0.1 2017-10-24 17:04:21 +02:00
990b081b3d Use standard logger instead of RS.log 2017-10-24 17:03:54 +02:00
11 changed files with 8778 additions and 616 deletions

23
.drone.yml Normal file
View File

@ -0,0 +1,23 @@
---
kind: pipeline
name: node 14
steps:
- name: test
image: node:14
commands:
- npm install
- npm run build
- npm test
---
kind: pipeline
name: node 16
steps:
- name: test
image: node:16
commands:
- npm install
- npm run build
- npm test

View File

@ -1,22 +1,40 @@
[![Release](https://img.shields.io/npm/v/remotestorage-module-chat-messages.svg?style=flat)](https://github.com/67P/remotestorage-module-chat-messages/releases)
[![Release](https://img.shields.io/npm/v/remotestorage-module-chat-messages.svg?style=flat)](https://www.npmjs.com/package/remotestorage-module-chat-messages)
# remoteStorage Module: Chat Messages
Stores chat messages in daily archive documents.
Please feel free to open GitHub issues for questions, feature requests,
protocol proposals, and whatever else you like.
## Usage
## Protocols
Open a daily archive and write messages to it:
### Currently supported
```js
const RemoteStorage = require("remotestoragejs");
const ChatMessages = require("remotestorage-module-chat-messages");
const remoteStorage = new RemoteStorage({ modules: [ ChatMessages ] });
* IRC
* XMPP
const archive = new remoteStorage.chatMessages.DailyArchive({
service: {
protocol: 'IRC',
domain: 'irc.libera.chat'
},
channelName: '#kosmos',
date: new Date(),
isPublic: true // Channel logs will be written to public folder
});
### Planned
const messages = [
{ "date": "2015-06-05T17:35:28.454Z", "user": "jimmy", "text": "knock knock" },
{ "date": "2015-06-05T17:36:05.123Z", "user": "walter", "text": "who's there?" }
];
* Mattermost
* Matrix
* Slack
* ...
archive.addMessages(messages);
```
See the inline source code documentation (JSDoc) for usage details and function
arguments. For a real-world integration example, see
[hubot-remotestorage-logger](https://github.com/67P/hubot-remotestorage-logger/).
## Support, bugs, feedback, questions
Come and chat with us: https://wiki.kosmos.org/Main_Page#Chat

4
dist/build.js vendored

File diff suppressed because one or more lines are too long

1
dist/build.js.LICENSE.txt vendored Normal file
View File

@ -0,0 +1 @@
/*! regenerator-runtime -- Copyright (c) 2014-present, Facebook, Inc. -- license (MIT): https://github.com/facebook/regenerator/blob/main/LICENSE */

2
dist/build.js.map vendored

File diff suppressed because one or more lines are too long

574
index.js
View File

@ -1,574 +0,0 @@
var ChatMessages = function (privateClient, publicClient) {
/**
* Schema: chat-messages/daily
*
* Represents one day of chat messages
*
* Example:
*
* (start code)
* {
* "@context": "https://kosmos.org/ns/v1",
* "@id": "chat-messages/freenode/channels/kosmos/",
* "@type": "ChatChannel",
* "name": "#kosmos",
* "ircURI": "irc://irc.freenode.net/kosmos",
* "today": {
* "@id": "2015/01/01",
* "@type": "ChatLog",
* "messageType": "InstantMessage",
* "previous": "2014/12/31",
* "next": "2015/01/02",
* "messages": [
* { "date": "2015-06-05T17:35:28.454Z", "user": "hal8000", "text": "knock knock" },
* { "date": "2015-06-05T17:37:42.123Z", "user": "raucao", "text": "who's there?" },
* { "date": "2015-06-05T17:55:01.235Z", "user": "hal8000", "text": "HAL" }
* ]
* }
* }
* (end code)
*/
const archiveSchema = {
"type": "object",
"properties": {
"@context": {
"type": "string",
"default": "https://kosmos.org/ns/v1",
"enum": ["https://kosmos.org/ns/v1"]
},
"@id": {
"type": "string",
"required": true
},
"@type": {
"type": "string",
"default": "ChatChannel",
"enum": ["ChatChannel"]
},
"name": {
"type": "string",
"required": true
},
"ircURI": {
"type": "string",
"format": "uri"
},
"xmppURI": {
"type": "string",
"format": "uri"
},
"today": {
"type": "object",
"properties": {
"@id": {
"type": "string",
"pattern": "^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$",
"required": true
},
"@type": {
"type": "string",
"default": "ChatLog",
"pattern": "^ChatLog$"
},
"messageType": {
"type": "string",
"default": "InstantMessage",
"pattern": "^InstantMessage$"
},
"previous": {
"type": "string",
"pattern": "^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$"
},
"next": {
"type": "string",
"pattern": "^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$"
},
"messages": {
"type": "array",
"required": true,
"items": {
"type": "object",
"properties": {
"date": {
"type": "string",
"format": "date-time"
},
"user": {
"type": "string"
},
"text": {
"type": "string"
},
"type": "string",
"default": "text",
"enum": [
"text",
"join",
"leave",
"action"
]
}
}
}
}
}
},
"required": []
};
privateClient.declareType("daily-archive", "https://kosmos.org/ns/v1", archiveSchema);
publicClient.declareType("daily-archive", "https://kosmos.org/ns/v1", archiveSchema);
/**
* Class: DailyArchive
*
* A daily archive stores IRC messages by day.
*
* Parameters (object):
* server - Chat server info (see <DailyArchive.server>)
* channelName - Name of room/channel
* date - Date of archive day
* isPublic - Store logs in public folder (defaults to false)
* previous - Date of previous log file as YYYY/MM/DD;
* looked up automatically when not given
* next - Date of next log file as YYYY/MM/DD;
* looked up automatically when not given
*
* Example for IRC:
*
* (start code)
* var archive = new chatMessages.DailyArchive({
* server: {
* type: 'irc',
* name: 'freenode',
* ircURI: 'irc://irc.freenode.net'
* },
* channelName: '#kosmos',
* date: new Date(),
* isPublic: true
* });
* (end code)
*
* Example for XMPP:
*
* (start code)
* var archive = new chatMessages.DailyArchive({
* server: {
* type: 'xmpp',
* name: '5apps',
* xmppMUC: 'muc.5apps.com'
* },
* channelName: 'watercooler',
* date: new Date(),
* isPublic: false
* });
* (end code)
*/
var DailyArchive = function DailyArchive(options) {
//
// Defaults
//
options.isPublic = options.isPublic || false;
//
// Validate options
//
if (typeof options !== "object") {
throw "options must be an object";
}
if (typeof options.server !== "object" ||
typeof options.server.type !== "string" ||
typeof options.server.name !== "string") {
throw "server must be an object containing at least server \"type\" and \"name\"";
}
if (typeof options.channelName !== "string") {
throw "channelName must be a string";
}
if (!(options.date instanceof Date)) {
throw "date must be a date object";
}
if (typeof options.isPublic !== "boolean") {
throw "isPublic must be a boolean value";
}
/**
* Property: server
*
* Contains information about the chat server/network
*
* Properties:
* type - Type of server/protocol (e.g. "irc", "xmpp", "campfire", "slack")
* name - Shortname/id/alias of network/server (e.g. "freenode", "mycompanyname")
* ircURI - (optional) IRC URI of network (e.g. "irc://irc.freenode.net/")
* xmppMUC - (optional) XMPP MUC service host (e.g. "conference.jabber.org")
*/
this.server = options.server;
/**
* Property: channelName
*
* Name of the IRC channel (e.g. "#kosmos")
*/
this.channelName = options.channelName;
/**
* Property: date
*
* Date of the archive's content
*/
this.date = options.date;
/**
* Property: isPublic
*
* `true` for public archives, `false` for private ones
*/
this.isPublic = options.isPublic;
/**
* Property: parsedDate
*
* Object containing padded year, month and day of date
*/
this.parsedDate = parseDate(this.date);
/**
* Property: dateId
*
* Date string in the form of YYYY/MM/DD
*/
this.dateId = this.parsedDate.year+'/'+this.parsedDate.month+'/'+this.parsedDate.day;
/**
* Property: path
*
* Document path of the archive file
*/
switch (this.server.type) {
case 'irc':
if (this.channelName.match(/^#/)) {
// normal chatroom
var channelName = this.channelName.replace(/^#/,'');
this.path = `${this.server.name}/channels/${channelName}/${this.dateId}`;
} else {
// user direct message
this.path = `${this.server.name}/users/${this.channelName}/${this.dateId}`;
}
break;
default:
this.path = `${this.server.name}/${this.channelName}/${this.dateId}`;
break;
}
/**
* Property: client
*
* Public or private BaseClient, depending on isPublic
*/
this.client = this.isPublic ? publicClient : privateClient;
/**
* Property: previous
*
* Date of previous log file as YYYY/MM/DD
*/
this.previous = options.previous;
/**
* Property: next
*
* Date of next log file as YYYY/MM/DD
*/
this.next = options.next;
};
DailyArchive.prototype = {
/*
* Method: addMessage
*
* Parameters (object):
* timestamp - Timestamp of the message
* from - The sender of the message
* text - The message itself
* type - Type of message (one of text, join, leave, action)
*/
addMessage: function addMessage(message) {
if (this.isPublic && !this.channelName.match(/^#/)) {
return Promise.resolve(false);
}
message.type = message.type || 'text';
return this.client.getObject(this.path).then((archive) => {
if (typeof archive === 'object') {
return this._updateDocument(archive, message);
} else {
return this._createDocument(message);
}
});
},
/*
* Method: addMessages
*
* Like <addMessage>, but for multiple messages at once. Useful for bulk
* imports of messages.
*
* Parameters:
* messages - Array of message objects (see params for addMessage)
* overwrite - If true, creates a new archive file and overwrites the
* old one. Defaults to false.
*/
addMessages: function addMessage(messages, overwrite) {
if (this.isPublic && !this.channelName.match(/^#/)) {
return Promise.resolve(false);
}
overwrite = overwrite || false;
messages.forEach(function(message) {
message.type = message.type || 'text';
});
if (overwrite) {
return this._createDocument(messages);
} else {
return this.client.getObject(this.path).then((archive) => {
if (typeof archive === 'object') {
return this._updateDocument(archive, messages);
} else {
return this._createDocument(messages);
}
});
}
},
/*
* Method: remove
*
* Deletes the entire archive document from storage
*/
remove: function() {
return this.client.remove(this.path);
},
/*
* Method: _updateDocument
*
* Updates and writes an existing archive document
*/
_updateDocument: function(archive, messages) {
RemoteStorage.log('[chat-messages] Updating archive document', archive);
if (Array.isArray(messages)) {
messages.forEach(function(message) {
archive.today.messages.push(message);
});
} else {
archive.today.messages.push(messages);
}
return this._sync(archive);
},
/*
* Method: _createDocument
*
* Creates and writes a new archive document
*/
_createDocument: function(messages) {
RemoteStorage.log('[chat-messages] Creating new archive document');
let archive = this._buildArchiveObject();
if (Array.isArray(messages)) {
messages.forEach((message) => {
archive.today.messages.push(message);
});
} else {
archive.today.messages.push(messages);
}
if (this.previous || this.next) {
// The app is handling previous/next keys itself
// That includes setting 'next' in the previous log file
if (this.previous) { archive.today.previous = this.previous; }
if (this.next) { archive.today.next = this.next; }
return this._sync(archive);
} else {
// Find and update previous archive, set 'previous' on this one
return this._updatePreviousArchive().then((previous) => {
if (typeof previous === 'object') {
archive.today.previous = previous.today['@id'];
}
return this._sync(archive);
});
}
},
/*
* Method: _buildArchiveObject
*
* Builds the object to be stored in remote storage
*/
_buildArchiveObject: function() {
let roomName = this.channelName.replace(/#/,'');
let archive = {
"@id": "chat-messages/"+this.server.name+"/channels/"+roomName+"/",
"@type": "ChatChannel",
"name": this.channelName,
"today": {
"@id": this.dateId,
"@type": "ChatLog",
"messageType": "InstantMessage",
"messages": []
}
};
switch (this.server.type) {
case 'irc':
if (!this.channelName.match(/^#/)) {
archive["@id"] = "chat-messages/"+this.server.name+"/users/"+this.channelName+"/";
}
archive["ircURI"] = this.server.ircURI+"/"+roomName;
break;
case 'xmpp':
archive["xmppURI"] = `xmpp:${this.channelName}@${this.server.xmppMUC}`;
break;
}
return archive;
},
/*
* Method: _updatePreviousArchive
*
* Finds the previous archive document and updates its today.next value
*/
_updatePreviousArchive: function() {
return this._findPreviousArchive().then((archive) => {
if (typeof archive === 'object' && archive.today) {
archive.today.next = this.dateId;
let path = this.path.substring(0, this.path.length-this.dateId.length)+archive.today['@id'];
return this.client.storeObject('daily-archive', path, archive).then(() => {
RemoteStorage.log('[chat-messages] Previous archive written to remote storage', path, archive);
return archive;
});
} else {
RemoteStorage.log('[chat-messages] Previous archive not found');
return false;
}
});
},
/*
* Method: _findPreviousArchive
*
* Returns the previous archive document
*/
_findPreviousArchive: function() {
const monthPath = this.path.substring(0, this.path.length-2);
const yearPath = this.path.substring(0, this.path.length-5);
const basePath = this.path.substring(0, this.path.length-10);
return this.client.getListing(monthPath).then((listing) => {
let days = Object.keys(listing).map((i) => parseInt(i)).map((i) => {
return (i < parseInt(this.parsedDate.day)) ? i : null;
}).filter(function(i){ return i != null; });
if (days.length > 0) {
let day = pad(Math.max(...days).toString());
return this.client.getObject(monthPath+day);
}
// Find last day in previous month
return this.client.getListing(yearPath).then((listing) => {
let months = Object.keys(listing).map((i) => parseInt(i.substr(0,2))).map((i) => {
return (i < parseInt(this.parsedDate.month)) ? i : null;
}).filter(function(i){ return i != null; });
if (months.length > 0) {
let month = pad(Math.max(...months).toString());
return this.client.getListing(yearPath+month+'/').then((listing) => {
let days = Object.keys(listing).map((i) => parseInt(i));
let day = pad(Math.max(...days).toString());
return this.client.getObject(yearPath+month+'/'+day);
});
} else {
// Find last month and day in previous year
return this.client.getListing(basePath).then((listing) => {
let years = Object.keys(listing).map((i) => parseInt(i.substr(0,4))).map((i) => {
return (i < parseInt(this.parsedDate.year)) ? i : null;
}).filter(function(i){ return i != null; });
if (years.length > 0) {
let year = Math.max(...years).toString();
return this.client.getListing(basePath+year+'/').then((listing) => {
let months = Object.keys(listing).map((i) => parseInt(i.substr(0,2)));
let month = pad(Math.max(...months).toString());
return this.client.getListing(basePath+year+'/'+month+'/').then((listing) => {
let days = Object.keys(listing).map((i) => parseInt(i));
let day = pad(Math.max(...days).toString());
return this.client.getObject(basePath+year+'/'+month+'/'+day);
});
});
} else {
return false;
}
});
}
});
});
},
/*
* Method: _sync
*
* Write archive document
*/
_sync: function(obj) {
RemoteStorage.log('[chat-messages] Writing archive object', obj);
return this.client.storeObject('daily-archive', this.path, obj).then(function(){
RemoteStorage.log('[chat-messages] Archive written to remote storage');
return true;
},function(error){
console.log('[chat-messages] Error trying to store object', error);
return error;
});
}
};
var pad = function(num) {
num = String(num);
if (num.length === 1) { num = "0" + num; }
return num;
};
var parseDate = function(date) {
return {
year: date.getUTCFullYear(),
month: pad( date.getUTCMonth() + 1 ),
day: pad( date.getUTCDate() )
};
};
var exports = {
DailyArchive: DailyArchive,
privateClient: privateClient,
publicClient: publicClient
};
// Return public functions
return { exports: exports };
};
export default { name: 'chat-messages', builder: ChatMessages };

7873
package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

View File

@ -1,27 +1,31 @@
{
"name": "remotestorage-module-chat-messages",
"version": "1.0.0",
"version": "2.1.1",
"description": "Stores chat messages in daily archive files",
"main": "./dist/build.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"build": "NODE_ENV=production webpack",
"dev": "webpack -w",
"build": "NODE_ENV=production webpack -p",
"start": "npm run dev",
"test": "mocha tests/",
"version": "npm run build && git add dist/"
},
"author": "Kosmos Developers <mail@kosmos.org> (https://kosmos.org)",
"contributors": [
"Sebastian Kippe <sebastian@kip.pe>"
],
"author": "Kosmos Contributors <mail@kosmos.org> (https://kosmos.org)",
"license": "MIT",
"homepage": "https://gitea.kosmos.org/kosmos/rs-module-chat-messages",
"repository": {
"type": "git",
"url": "https://github.com/67P/remotestorage-module-chat-messages.git"
"url": "https://gitea.kosmos.org/kosmos/rs-module-chat-messages.git"
},
"devDependencies": {
"babel-core": "^6.18.2",
"babel-loader": "^6.2.7",
"babel-preset-es2015": "^6.18.0",
"webpack": "^1.13.2"
"@babel/core": "^7.18.10",
"@babel/preset-env": "^7.18.10",
"babel-loader": "^8.2.5",
"chai": "^4.3.6",
"mocha": "^10.0.0",
"regenerator-runtime": "^0.13.9",
"sinon": "^14.0.0",
"webpack": "^5.74.0",
"webpack-cli": "^4.10.0"
}
}

692
src/chat-messages.js Normal file
View File

@ -0,0 +1,692 @@
import 'regenerator-runtime/runtime';
function pad (num) {
num = String(num);
if (num.length === 1) { num = "0" + num; }
return num;
};
function parseDate (date) {
return {
year: date.getUTCFullYear(),
month: pad( date.getUTCMonth() + 1 ),
day: pad( date.getUTCDate() )
};
};
function lowestNumberInListing(items) {
const sortedNumbers = Object.keys(items)
.map(i => parseInt(i))
.filter(i => !Number.isNaN(i))
.sort();
return sortedNumbers[0];
}
const ChatMessages = function (privateClient, publicClient) {
/**
* Schema: chat-messages/daily-archive
*
* Represents one calendar day of chat messages
*
* @example
* {
* "@context": "https://kosmos.org/ns/v2",
* "@id": "chat-messages/irc.libera.chat/channels/kosmos/",
* "@type": "ChatChannel",
* "service": {
* "domain": "irc.libera.chat",
* "protocol": "IRC",
* },
* "name": "#kosmos",
* "type": "room",
* "today": {
* "@id": "2015/01/01",
* "@type": "ChatLog",
* "messageType": "InstantMessage",
* "previous": "2014/12/31",
* "next": "2015/01/02",
* "messages": [
* { "date": "2015-06-05T17:35:28.454Z", "user": "hal8000", "text": "knock knock" },
* { "date": "2015-06-05T17:37:42.123Z", "user": "raucao", "text": "who's there?" },
* { "date": "2015-06-05T17:55:01.235Z", "user": "hal8000", "text": "HAL" }
* ]
* }
* }
*/
const archiveSchema = {
"type": "object",
"properties": {
"@context": {
"type": "string",
"default": "https://kosmos.org/ns/v2/chat-channel"
},
"@id": {
"type": "string",
"required": true
},
"@type": {
"type": "string",
"default": "ChatChannel",
"enum": ["ChatChannel"]
},
"service": {
"type": "object",
"properties": {
"domain": {
"type": "string",
"required": true
},
"protocol": {
"type": "string",
"required": true
}
}
},
"name": {
"type": "string",
"required": true
},
"type": {
"type": "string",
"required": true,
"enum": [ "room", "person" ]
},
"today": {
"type": "object",
"properties": {
"@id": {
"type": "string",
"pattern": "^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$",
"required": true
},
"@type": {
"type": "string",
"default": "ChatLog",
"pattern": "^ChatLog$"
},
"messageType": {
"type": "string",
"default": "InstantMessage",
"pattern": "^InstantMessage$"
},
"previous": {
"type": "string",
"pattern": "^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$"
},
"next": {
"type": "string",
"pattern": "^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$"
},
"messages": {
"type": "array",
"required": true,
"items": {
"type": "object",
"properties": {
"date": {
"type": "string",
"format": "date-time"
},
"user": {
"type": "string"
},
"text": {
"type": "string"
},
"type": "string",
"default": "text",
"enum": [
"text",
"join",
"leave",
"action"
]
}
}
}
}
}
},
"required": []
};
privateClient.declareType("daily-archive", "https://kosmos.org/ns/v2/chat-channel", archiveSchema);
publicClient.declareType("daily-archive", "https://kosmos.org/ns/v2/chat-channel", archiveSchema);
/**
* Schema: chat-messages/daily-archive-meta
*
* Stores meta information about the daily archives
*
* @example
* {
* "@context": "https://kosmos.org/ns/v2",
* "@id": "chat-messages/irc.libera.chat/channels/kosmos/meta",
* "@type": "ChatChannelMeta",
* "first": "2009/01/03",
* "last": "2021/11/05"
* }
* }
*/
const archiveMetaSchema = {
"type": "object",
"properties": {
"@context": {
"type": "string",
"default": "https://kosmos.org/ns/v2/chat-channel-meta"
},
"@id": {
"type": "string",
},
"@type": {
"type": "string",
"default": "ChatChannelMeta",
"enum": ["ChatChannelMeta"]
},
"first": {
"type": "string",
"pattern": "^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$"
},
"last": {
"type": "string",
"pattern": "^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$"
}
},
"required": ["@id", "first", "last"]
};
privateClient.declareType("daily-archive-meta", "https://kosmos.org/ns/v2/chat-channel-meta", archiveMetaSchema);
publicClient.declareType("daily-archive-meta", "https://kosmos.org/ns/v2/chat-channel-meta", archiveMetaSchema);
/**
* A daily archive stores chat messages by calendar day.
*
* @class
*/
class DailyArchive {
/**
* @param {object} options
* @param {object} options.service
* @param {string} options.service.protocol - Type of chat service/protocol (e.g. "IRC", "XMPP", "Campfire", "Slack")
* @param {string} options.service.domain - Domain of the chat service (e.g. "irc.libera.chat", "kosmos.chat")
* @param {string} options.channelName - Name of room/channel (e.g. "#kosmos")
* @param {string} [options.channelType] - Type of channel ("room" or "person")
* @param {date} options.date - Date of archive day
* @param {boolean} [options.isPublic] - Store logs in public folder (defaults to false)
* @param {string} [options.previous] - Date of previous log file as `YYYY/MM/DD`. Looked up automatically when not given
* @param {string} [options.next] - Date of next log file as `YYYY/MM/DD`. looked up automatically when not given
*
* @example
* // IRC archive:
* const archive = new chatMessages.DailyArchive({
* service: {
* protocol: 'IRC',
* domain: 'irc.libera.chat',
* },
* channelName: '#kosmos-dev',
* channelType: 'room',
* date: new Date(),
* isPublic: true
* });
*
* // XMPP archive:
* const archive = new chatMessages.DailyArchive({
* service: {
* protocol: 'XMPP',
* domain: 'kosmos.chat',
* },
* channelName: 'kosmos-dev',
* channelType: 'room',
* date: new Date(),
* isPublic: false
* });
*
*/
constructor (options) {
//
// Defaults
//
options.isPublic = options.isPublic || false;
options.channelType = options.channelType || "room";
//
// Validate options
//
if (typeof options !== "object") {
throw "options must be an object";
}
if (typeof options.service !== "object" ||
typeof options.service.protocol !== "string" ||
typeof options.service.domain !== "string") {
throw "service must be an object containing at least service \"protocol\" and \"domain\"";
}
if (typeof options.channelName !== "string") {
throw "channelName must be a string";
}
if (!(options.date instanceof Date)) {
throw "date must be a date object";
}
if (typeof options.isPublic !== "boolean") {
throw "isPublic must be a boolean value";
}
/**
* @property {object} service
* @property {string} service.protocol - Type of chat service/protocol (e.g. "IRC", "XMPP", "campfire", "slack")
* @property {string} service.domain - Domain of the chat service (e.g. "irc.libera.chat", "kosmos.chat")
*/
this.service = options.service;
/**
* @property {string} channelName - Name of channel (e.g. "#kosmos")
*/
this.channelName = options.channelName;
/**
* @property {string} channelType - Type of channel ("room" or "person")
*/
this.channelType = options.channelType;
/**
* @property {string} date - Gregorian calendar date of the archive's content
*/
this.date = options.date;
/**
* @property {boolean} isPublic - `true` for public archives, `false` for private ones
*/
this.isPublic = options.isPublic || false;
/**
* @property {object} parsedDate - Contains padded year, month and day of date
* @property {string} year
* @property {string} month
* @property {string} day
*/
this.parsedDate = parseDate(this.date);
/**
* @property {string} dateId - Date string in the form of YYYY/MM/DD
*/
this.dateId = this.parsedDate.year+'/'+this.parsedDate.month+'/'+this.parsedDate.day;
/**
* @property {string} channelPath - Base directory path of the channel archives
*/
if (this.channelType === "room") {
// Normal chatroom
const channelName = this.channelName.replace(/^#/,'');
this.channelPath = `${this.service.domain}/channels/${channelName}`;
} else {
// User direct messages
this.channelPath = `${this.service.domain}/users/${this.channelName}`;
}
/**
* @property {string} path - Path of the archive document
*/
this.path = `${this.channelPath}/${this.dateId}`;
/**
* @property {string} metaPath - Path of the channel's metadata document
*/
this.metaPath = `${this.channelPath}/meta`;
/**
* @property {object} client - Public or private remoteStorgage.js BaseClient
*/
this.client = this.isPublic ? publicClient : privateClient;
/**
* @property {string} previous - Date of previous log file as YYYY/MM/DD
*/
this.previous = options.previous;
/**
* @property {string} next - Date of next log file as YYYY/MM/DD
*/
this.next = options.next;
}
/*
* @param {object} message
* @param {string} message.timestamp - Timestamp of the message
* @param {string} message.from - The sender of the message
* @param {string} message.text - The message itself
* @param {string} message.type - Type of message (one of text, join, leave, action)
* @param {string} [message.id] - Unique ID of message. TODO implement
*
* @returns {Promise}
*/
addMessage (message) {
if (this.isPublic && !this.channelName.match(/^#/)) {
return Promise.resolve(false);
}
message.type = message.type || 'text';
return this.client.getObject(this.path).then((archive) => {
if (typeof archive === 'object') {
return this._updateDocument(archive, message);
} else {
return this._createDocument(message);
}
});
}
/*
* Like <addMessage>, but for multiple messages at once. Useful for bulk
* imports of messages.
*
* @param {Array} messages - Array of message objects (see params for addMessage)
* @param {boolean} overwrite - If true, creates a new archive file and overwrites the old one. Defaults to false.
*
* @returns {Promise}
*/
addMessages (messages, overwrite) {
if (this.isPublic && !this.channelName.match(/^#/)) {
return Promise.resolve(false);
}
overwrite = overwrite || false;
messages.forEach(function(message) {
message.type = message.type || 'text';
});
if (overwrite) {
return this._createDocument(messages);
} else {
return this.client.getObject(this.path).then((archive) => {
if (typeof archive === 'object') {
return this._updateDocument(archive, messages);
} else {
return this._createDocument(messages);
}
});
}
}
/*
* Deletes the entire archive document from storage
*
* @returns {Promise}
*/
remove () {
// TODO when removing, if previous is set, but not next, it means the
// removed file is the last archive. Thus, set "last" to previous file.
return this.client.remove(this.path);
}
/*
* Updates and writes an existing archive document
*
* @returns {Promise}
*
* @private
*/
async _updateDocument (archive, messages) {
console.debug('[chat-messages] Updating archive document');
if (Array.isArray(messages)) {
messages.forEach(function(message) {
archive.today.messages.push(message);
});
} else {
archive.today.messages.push(messages);
}
return this._sync(archive);
}
/*
* Creates and writes a new archive document
*
* @returns {Promise}
*
* @private
*/
async _createDocument (messages) {
console.debug('[chat-messages] Creating new archive document');
const archive = this._buildArchiveObject();
if (Array.isArray(messages)) {
messages.forEach(message => {
archive.today.messages.push(message);
});
} else {
archive.today.messages.push(messages);
}
if (this.previous || this.next) {
// The app is handling previous/next keys itself
// That includes setting 'next' in the previous log file
if (this.previous) { archive.today.previous = this.previous; }
if (this.next) { archive.today.next = this.next; }
} else {
// Find and update previous archive, set 'previous' on this one
const previous = await this._updatePreviousArchive();
if (typeof previous === 'object') {
archive.today.previous = previous.today['@id'];
}
}
await this._sync(archive);
// TODO only write meta doc if argument is set on addMessages. This way
// we can avoid race conditions when syncing remote chat messages all at
// once for multiple days
await this._updateArchiveMetaDocument();
return;
}
/*
* Builds the object to be stored in remote storage
*
* @returns {object}
*
* @private
*/
_buildArchiveObject () {
const roomName = this.channelName.replace(/^#/,'');
const archive = {
"@id": "chat-messages/"+this.service.domain+"/channels/"+roomName+"/",
"@type": "ChatChannel",
"service": this.service,
"name": this.channelName,
"type": this.channelType,
"today": {
"@id": this.dateId,
"@type": "ChatLog",
"messageType": "InstantMessage",
"messages": []
}
};
switch (this.service.protocol) {
case 'IRC':
if (!this.channelName.match(/^#/)) {
archive["@id"] = "chat-messages/"+this.service.domain+"/users/"+this.channelName+"/";
}
break;
case 'XMPP':
// XMPP-specific adjustments
break;
}
return archive;
}
/*
* Finds the previous archive document and updates its today.next value
*
* @returns {boolean|Promise}
*
* @private
*/
_updatePreviousArchive () {
return this._findPreviousArchive().then((archive) => {
if (typeof archive === 'object' && archive.today) {
archive.today.next = this.dateId;
const path = this.path.substring(0, this.path.length-this.dateId.length)+archive.today['@id'];
return this.client.storeObject('daily-archive', path, archive).then(() => {
console.debug('[chat-messages] Previous archive written to remote storage at', path);
return archive;
});
} else {
console.debug('[chat-messages] Previous archive not found');
return false;
}
});
}
/*
* Returns the previous archive document
*
* @returns {Promise}
*
* @private
*/
_findPreviousArchive () {
const monthPath = this.path.substring(0, this.path.length-2);
const yearPath = this.path.substring(0, this.path.length-5);
const basePath = this.path.substring(0, this.path.length-10);
return this.client.getListing(monthPath).then((listing) => {
const days = Object.keys(listing).map((i) => parseInt(i)).map((i) => {
return (i < parseInt(this.parsedDate.day)) ? i : null;
}).filter(function(i){ return i != null; });
if (days.length > 0) {
const day = pad(Math.max(...days).toString());
return this.client.getObject(monthPath+day);
}
// Find last day in previous month
return this.client.getListing(yearPath).then((listing) => {
const months = Object.keys(listing).map((i) => parseInt(i.substr(0,2))).map((i) => {
return (i < parseInt(this.parsedDate.month)) ? i : null;
}).filter(function(i){ return i != null; });
if (months.length > 0) {
const month = pad(Math.max(...months).toString());
return this.client.getListing(yearPath+month+'/').then((listing) => {
const days = Object.keys(listing).map((i) => parseInt(i));
const day = pad(Math.max(...days).toString());
return this.client.getObject(yearPath+month+'/'+day);
});
} else {
// Find last month and day in previous year
return this.client.getListing(basePath).then((listing) => {
const years = Object.keys(listing).map((i) => parseInt(i.substr(0,4))).map((i) => {
return (i < parseInt(this.parsedDate.year)) ? i : null;
}).filter(function(i){ return i != null; });
if (years.length > 0) {
const year = Math.max(...years).toString();
return this.client.getListing(basePath+year+'/').then((listing) => {
const months = Object.keys(listing).map((i) => parseInt(i.substr(0,2)));
const month = pad(Math.max(...months).toString());
return this.client.getListing(basePath+year+'/'+month+'/').then((listing) => {
const days = Object.keys(listing).map((i) => parseInt(i));
const day = pad(Math.max(...days).toString());
return this.client.getObject(basePath+year+'/'+month+'/'+day);
});
});
} else {
return false;
}
});
}
});
});
}
async _updateArchiveMetaDocument () {
const meta = await this.client.getObject(this.metaPath);
if (typeof meta !== 'object') {
return this._createArchiveMetaDocument();
}
// Only update document if current date is newer than known "last"
if (Date.parse(meta.last.replace(/\//g,'-')) < Date.parse(this.date)) {
console.debug('[chat-messages]', 'Updating meta document for channel');
meta.last = this.dateId;
await this.client.storeObject('daily-archive-meta', this.metaPath, meta);
}
return;
}
async _createArchiveMetaDocument () {
console.debug('[chat-messages]', 'Creating new meta document for channel');
// When creating a new meta doc, we need to find the oldest archive,
// because older versions of the module did not write a meta doc.
const first = await this._findFirstArchive();
const roomName = this.channelName.replace(/^#/,'');
const meta = {
'@id': `chat-messages/${this.service.domain}/channels/${roomName}/meta`,
'@type': 'ChatChannelMeta',
first: first,
last: this.dateId // TODO might have to search for last?
};
return this.client.storeObject('daily-archive-meta', this.metaPath, meta)
.then(() => console.debug('[chat-messages]', 'Meta document written to remote storage'))
.catch(e => {
console.log('[chat-messages]', `Failed to store ${this.metaPath}`);
console.error(e);
});
}
async _findFirstArchive () {
console.debug('[chat-messages]', 'Finding first archive for channel');
const years = await this.client.getListing(`${this.channelPath}/`);
const year = lowestNumberInListing(years);
const months = await this.client.getListing(`${this.channelPath}/${year}/`);
const month = lowestNumberInListing(months);
const days = await this.client.getListing(`${this.channelPath}/${year}/${pad(month)}/`);
const day = lowestNumberInListing(days);
const firstId = `${year}/${pad(month)}/${pad(day)}`;
console.debug('[chat-messages]', 'First is', firstId);
return firstId;
}
/*
* Write archive document
*
* @returns {Promise}
*
* @private
*/
async _sync (archive) {
console.debug(`[chat-messages] Writing archive object with ${archive.today.messages.length} messages`);
return this.client.storeObject('daily-archive', this.path, archive).then(function(){
console.debug('[chat-messages] Archive written to remote storage');
return true;
},function(error){
console.warn('[chat-messages] Error trying to store object', error);
return error;
});
}
};
return {
exports: {
DailyArchive,
privateClient,
publicClient
}
};
};
export default { name: 'chat-messages', builder: ChatMessages };

114
tests/chat-messages-spec.js Normal file
View File

@ -0,0 +1,114 @@
const expect = require('chai').expect;
const sandbox = require("sinon").createSandbox();
const ChatMessages = require('../dist/build');
const rsClient = {
declareType: function() {},
getObject: function() {},
getListing: function() {},
storeObject: function() {},
remove: function() {}
}
describe('ChatMessages', function () {
describe('constructor', function () {
let chatMessages;
before(function() {
chatMessages = new ChatMessages.builder(rsClient, rsClient);
});
it('behaves like a remoteStorage module', function () {
expect(chatMessages).to.be.an('object');
expect(chatMessages.exports).to.be.an('object');
});
it('exports the desired functionality', function () {
expect(chatMessages.exports.DailyArchive).to.be.a('function');
});
});
describe('DailyArchive', function () {
let archive;
before(function() {
chatMessages = (new ChatMessages.builder(rsClient, rsClient)).exports;
archive = new chatMessages.DailyArchive({
service: { protocol: 'IRC', domain: 'irc.libera.chat' },
channelName: '#kosmos',
date: new Date('2022-08-11')
});
});
describe('constructor', function () {
it('creates an archive instance with the desired properties', function () {
expect(archive).to.be.an('object');
expect(archive.service.protocol).to.eq('IRC');
expect(archive.service.domain).to.eq('irc.libera.chat');
expect(archive.channelName).to.eq('#kosmos');
expect(archive.channelType).to.eq('room');
expect(archive.date).to.be.a('date');
expect(archive.parsedDate.year).to.eq(2022);
expect(archive.parsedDate.month).to.eq('08');
expect(archive.parsedDate.day).to.eq('11');
expect(archive.dateId).to.eq('2022/08/11');
expect(archive.isPublic).to.eq(false);
expect(archive.channelPath).to.eq('irc.libera.chat/channels/kosmos');
expect(archive.path).to.eq('irc.libera.chat/channels/kosmos/2022/08/11');
expect(archive.metaPath).to.eq('irc.libera.chat/channels/kosmos/meta');
expect(archive.client).to.eq(rsClient);
expect(archive.previous).to.be.an('undefined');
expect(archive.next).to.be.an('undefined');
});
});
describe('#_updateArchiveMetaDocument', function () {
describe('meta up to date', function () {
before(function() {
sandbox.stub(archive.client, 'getObject').withArgs(archive.metaPath)
.returns({
'@id': `chat-messages/irc.libera.chat/channels/kosmos/meta`,
'@type': 'ChatChannelMeta',
first: '2021/01/01', last: '2022/08/11'
})
sandbox.stub(archive.client, 'storeObject');
});
it('does not store a new archive', async function () {
await archive._updateArchiveMetaDocument();
sandbox.assert.notCalled(archive.client.storeObject);
});
after(function() { sandbox.restore() });
});
describe('meta needs updating', function () {
before(function() {
sandbox.stub(archive.client, 'getObject').withArgs(archive.metaPath)
.returns({
'@id': archive.metaPath,
'@type': 'ChatChannelMeta',
first: '2021/01/01', last: '2022/08/10'
})
sandbox.stub(archive.client, 'storeObject');
});
it('stores a new archive', async function () {
await archive._updateArchiveMetaDocument();
sandbox.assert.calledWithMatch(
archive.client.storeObject,
'daily-archive-meta', archive.metaPath, {
'@id': archive.metaPath, '@type': 'ChatChannelMeta',
first: '2021/01/01', last: '2022/08/11'
}
);
});
after(function() { sandbox.restore() });
});
});
});
});

View File

@ -1,24 +1,33 @@
var webpack = require('webpack');
var isProd = (process.env.NODE_ENV === 'production');
// minimize only in production
var plugins = isProd ? [new webpack.optimize.UglifyJsPlugin({minimize: true })] : []
/* global __dirname */
const isProd = (process.env.NODE_ENV === 'production');
const path = require('path');
module.exports = {
entry: './index.js',
// source map not in production
devtool: !isProd && 'source-map',
entry: ['./src/chat-messages.js'],
output: {
filename: __dirname + '/dist/build.js',
libraryTarget: 'umd'
path: path.resolve(__dirname, 'dist'),
filename: 'build.js',
library: 'ChatMessages',
libraryTarget: 'umd',
libraryExport: 'default',
umdNamedDefine: true,
globalObject: 'this'
},
mode: isProd ? 'production' : 'development',
devtool: isProd ? 'source-map' : 'eval-source-map',
module: {
loaders: [
{ test: /\.js$/, exclude: '/node_modules|dist/', loader: 'babel?presets[]=es2015' },
rules: [
{
test: /\.js$/,
exclude: /node_modules/,
use: {
loader: 'babel-loader',
options: {
presets: ['@babel/preset-env']
}
}
},
]
},
resolve: {
extensions: ['', '.js']
},
plugins: plugins
// plugins: plugins
};