From d23ced1fba0e4bc7ecbc00d0a2376f34bab80509 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 10 Oct 2016 20:23:33 +0800 Subject: Update to move authorship calculation code to note model and support update authorship after making revision of docs --- lib/models/note.js | 177 ++++++++++++++++++++++++++++++++++++++++++++++++++++- lib/realtime.js | 101 +----------------------------- 2 files changed, 177 insertions(+), 101 deletions(-) (limited to 'lib') diff --git a/lib/models/note.js b/lib/models/note.js index d1c073e9..17988f74 100644 --- a/lib/models/note.js +++ b/lib/models/note.js @@ -11,11 +11,16 @@ var shortId = require('shortid'); var Sequelize = require("sequelize"); var async = require('async'); var moment = require('moment'); +var DiffMatchPatch = require('diff-match-patch'); +var dmp = new DiffMatchPatch(); // core var config = require("../config.js"); var logger = require("../logger.js"); +//ot +var ot = require("../ot/index.js"); + // permission types var permissionTypes = ["freely", "editable", "locked", "private"]; @@ -115,6 +120,7 @@ module.exports = function (sequelize, DataTypes) { var fsModifiedTime = moment(fs.statSync(filePath).mtime); var dbModifiedTime = moment(note.lastchangeAt || note.createdAt); var body = fs.readFileSync(filePath, 'utf8'); + var contentLength = body.length; var title = Note.parseNoteTitle(body); body = LZString.compressToBase64(body); title = LZString.compressToBase64(title); @@ -126,7 +132,20 @@ module.exports = function (sequelize, DataTypes) { }).then(function (note) { sequelize.models.Revision.saveNoteRevision(note, function (err, revision) { if (err) return _callback(err, null); - return callback(null, note.id); + // update authorship on after making revision of docs + var patch = dmp.patch_fromText(LZString.decompressFromBase64(revision.patch)); + var operations = Note.transformPatchToOperations(patch, contentLength); + var authorship = note.authorship ? JSON.parse(LZString.decompressFromBase64(note.authorship)) : []; + for (var i = 0; i < operations.length; i++) { + authorship = Note.updateAuthorshipByOperation(operations[i], null, authorship); + } + note.update({ + authorship: authorship + }).then(function (note) { + return callback(null, note.id); + }).catch(function (err) { + return _callback(err, null); + }); }); }).catch(function (err) { return _callback(err, null); @@ -247,6 +266,162 @@ module.exports = function (sequelize, DataTypes) { _meta.slideOptions = meta.slideOptions; } return _meta; + }, + updateAuthorshipByOperation: function (operation, userId, authorships) { + var index = 0; + var timestamp = Date.now(); + for (var i = 0; i < operation.length; i++) { + var op = operation[i]; + if (ot.TextOperation.isRetain(op)) { + index += op; + } else if (ot.TextOperation.isInsert(op)) { + var opStart = index; + var opEnd = index + op.length; + var inserted = false; + // authorship format: [userId, startPos, endPos, createdAt, updatedAt] + if (authorships.length <= 0) authorships.push([userId, opStart, opEnd, timestamp, timestamp]); + else { + for (var j = 0; j < authorships.length; j++) { + var authorship = authorships[j]; + if (!inserted) { + var nextAuthorship = authorships[j + 1] || -1; + if (nextAuthorship != -1 && nextAuthorship[1] >= opEnd || j >= authorships.length - 1) { + if (authorship[1] < opStart && authorship[2] > opStart) { + // divide + var postLength = authorship[2] - opStart; + authorship[2] = opStart; + authorship[4] = timestamp; + authorships.splice(j + 1, 0, [userId, opStart, opEnd, timestamp, timestamp]); + authorships.splice(j + 2, 0, [authorship[0], opEnd, opEnd + postLength, authorship[3], timestamp]); + j += 2; + inserted = true; + } else if (authorship[1] >= opStart) { + authorships.splice(j, 0, [userId, opStart, opEnd, timestamp, timestamp]); + j += 1; + inserted = true; + } else if (authorship[2] <= opStart) { + authorships.splice(j + 1, 0, [userId, opStart, opEnd, timestamp, timestamp]); + j += 1; + inserted = true; + } + } + } + if (authorship[1] >= opStart) { + authorship[1] += op.length; + authorship[2] += op.length; + } + } + } + index += op.length; + } else if (ot.TextOperation.isDelete(op)) { + var opStart = index; + var opEnd = index - op; + if (operation.length == 1) { + authorships = []; + } else if (authorships.length > 0) { + for (var j = 0; j < authorships.length; j++) { + var authorship = authorships[j]; + if (authorship[1] >= opStart && authorship[1] <= opEnd && authorship[2] >= opStart && authorship[2] <= opEnd) { + authorships.splice(j, 1); + j -= 1; + } else if (authorship[1] < opStart && authorship[1] < opEnd && authorship[2] > opStart && authorship[2] > opEnd) { + authorship[2] += op; + authorship[4] = timestamp; + } else if (authorship[2] >= opStart && authorship[2] <= opEnd) { + authorship[2] = opStart; + authorship[4] = timestamp; + } else if (authorship[1] >= opStart && authorship[1] <= opEnd) { + authorship[1] = opEnd; + authorship[4] = timestamp; + } + if (authorship[1] >= opEnd) { + authorship[1] += op; + authorship[2] += op; + } + } + } + index += op; + } + } + // merge + for (var j = 0; j < authorships.length; j++) { + var authorship = authorships[j]; + for (var k = j + 1; k < authorships.length; k++) { + var nextAuthorship = authorships[k]; + if (nextAuthorship && authorship[0] === nextAuthorship[0] && authorship[2] === nextAuthorship[1]) { + var minTimestamp = Math.min(authorship[3], nextAuthorship[3]); + var maxTimestamp = Math.max(authorship[3], nextAuthorship[3]); + authorships.splice(j, 1, [authorship[0], authorship[1], nextAuthorship[2], minTimestamp, maxTimestamp]); + authorships.splice(k, 1); + j -= 1; + break; + } + } + } + // clear + for (var j = 0; j < authorships.length; j++) { + var authorship = authorships[j]; + if (!authorship[0]) { + authorships.splice(j, 1); + j -= 1; + } + } + return authorships; + }, + transformPatchToOperations: function (patch, contentLength) { + var operations = []; + if (patch.length > 0) { + // calculate original content length + for (var j = patch.length - 1; j >= 0; j--) { + var p = patch[j]; + for (var i = 0; i < p.diffs.length; i++) { + var diff = p.diffs[i]; + switch(diff[0]) { + case 1: // insert + contentLength -= diff[1].length; + break; + case -1: // delete + contentLength += diff[1].length; + break; + } + } + } + // generate operations + var bias = 0; + var lengthBias = 0; + for (var j = 0; j < patch.length; j++) { + var operation = []; + var p = patch[j]; + var currIndex = p.start1; + var currLength = contentLength - bias; + for (var i = 0; i < p.diffs.length; i++) { + var diff = p.diffs[i]; + switch(diff[0]) { + case 0: // retain + if (i == 0) // first + operation.push(currIndex + diff[1].length); + else if (i != p.diffs.length - 1) // mid + operation.push(diff[1].length); + else // last + operation.push(currLength + lengthBias - currIndex); + currIndex += diff[1].length; + break; + case 1: // insert + operation.push(diff[1]); + lengthBias += diff[1].length; + currIndex += diff[1].length; + break; + case -1: // delete + operation.push(-diff[1].length); + bias += diff[1].length; + currIndex += diff[1].length; + break; + } + } + operations.push(operation); + } + } + return operations; } }, hooks: { diff --git a/lib/realtime.js b/lib/realtime.js index 0c68a256..9c3d5b2e 100644 --- a/lib/realtime.js +++ b/lib/realtime.js @@ -642,106 +642,7 @@ function operationCallback(socket, operation) { } } // save authorship - var index = 0; - var authorships = note.authorship; - var timestamp = Date.now(); - for (var i = 0; i < operation.length; i++) { - var op = operation[i]; - if (ot.TextOperation.isRetain(op)) { - index += op; - } else if (ot.TextOperation.isInsert(op)) { - var opStart = index; - var opEnd = index + op.length; - var inserted = false; - // authorship format: [userId, startPos, endPos, createdAt, updatedAt] - if (authorships.length <= 0) authorships.push([userId, opStart, opEnd, timestamp, timestamp]); - else { - for (var j = 0; j < authorships.length; j++) { - var authorship = authorships[j]; - if (!inserted) { - var nextAuthorship = authorships[j + 1] || -1; - if (nextAuthorship != -1 && nextAuthorship[1] >= opEnd || j >= authorships.length - 1) { - if (authorship[1] < opStart && authorship[2] > opStart) { - // divide - var postLength = authorship[2] - opStart; - authorship[2] = opStart; - authorship[4] = timestamp; - authorships.splice(j + 1, 0, [userId, opStart, opEnd, timestamp, timestamp]); - authorships.splice(j + 2, 0, [authorship[0], opEnd, opEnd + postLength, authorship[3], timestamp]); - j += 2; - inserted = true; - } else if (authorship[1] >= opStart) { - authorships.splice(j, 0, [userId, opStart, opEnd, timestamp, timestamp]); - j += 1; - inserted = true; - } else if (authorship[2] <= opStart) { - authorships.splice(j + 1, 0, [userId, opStart, opEnd, timestamp, timestamp]); - j += 1; - inserted = true; - } - } - } - if (authorship[1] >= opStart) { - authorship[1] += op.length; - authorship[2] += op.length; - } - } - } - index += op.length; - } else if (ot.TextOperation.isDelete(op)) { - var opStart = index; - var opEnd = index - op; - if (operation.length == 1) { - authorships = []; - } else if (authorships.length > 0) { - for (var j = 0; j < authorships.length; j++) { - var authorship = authorships[j]; - if (authorship[1] >= opStart && authorship[1] <= opEnd && authorship[2] >= opStart && authorship[2] <= opEnd) { - authorships.splice(j, 1); - j -= 1; - } else if (authorship[1] < opStart && authorship[1] < opEnd && authorship[2] > opStart && authorship[2] > opEnd) { - authorship[2] += op; - authorship[4] = timestamp; - } else if (authorship[2] >= opStart && authorship[2] <= opEnd) { - authorship[2] = opStart; - authorship[4] = timestamp; - } else if (authorship[1] >= opStart && authorship[1] <= opEnd) { - authorship[1] = opEnd; - authorship[4] = timestamp; - } - if (authorship[1] >= opEnd) { - authorship[1] += op; - authorship[2] += op; - } - } - } - index += op; - } - } - // merge - for (var j = 0; j < authorships.length; j++) { - var authorship = authorships[j]; - for (var k = j + 1; k < authorships.length; k++) { - var nextAuthorship = authorships[k]; - if (nextAuthorship && authorship[0] === nextAuthorship[0] && authorship[2] === nextAuthorship[1]) { - var minTimestamp = Math.min(authorship[3], nextAuthorship[3]); - var maxTimestamp = Math.max(authorship[3], nextAuthorship[3]); - authorships.splice(j, 1, [authorship[0], authorship[1], nextAuthorship[2], minTimestamp, maxTimestamp]); - authorships.splice(k, 1); - j -= 1; - break; - } - } - } - // clear - for (var j = 0; j < authorships.length; j++) { - var authorship = authorships[j]; - if (!authorship[0]) { - authorships.splice(j, 1); - j -= 1; - } - } - note.authorship = authorships; + note.authorship = models.Note.updateAuthorshipByOperation(operation, userId, note.authorship); } function connection(socket) { -- cgit v1.2.3 From 1cae0c5b7f63bd9049fbfbc891f77d1b2af3ec99 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 10 Oct 2016 20:25:48 +0800 Subject: Update to prevent duplicate socket push in queue in order to lower down server loading --- lib/realtime.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'lib') diff --git a/lib/realtime.js b/lib/realtime.js index 9c3d5b2e..ae624c24 100644 --- a/lib/realtime.js +++ b/lib/realtime.js @@ -327,6 +327,15 @@ function emitRefresh(socket) { socket.emit('refresh', out); } +function isDuplicatedInSocketQueue(queue, socket) { + for (var i = 0; i < queue.length; i++) { + if (queue[i] && queue[i].id == socket.id) { + return true; + } + } + return false; +} + function clearSocketQueue(queue, socket) { for (var i = 0; i < queue.length; i++) { if (!queue[i] || queue[i].id == socket.id) { @@ -654,6 +663,8 @@ function connection(socket) { if (!noteId) { return failConnection(404, 'note id not found', socket); } + + if (isDuplicatedInSocketQueue(socket, connectionSocketQueue)) return; // store noteId in this socket session socket.noteId = noteId; @@ -830,6 +841,7 @@ function connection(socket) { //when a new client disconnect socket.on('disconnect', function () { + if (isDuplicatedInSocketQueue(socket, disconnectSocketQueue)) return; disconnectSocketQueue.push(socket); disconnect(socket); }); -- cgit v1.2.3 From 3175616573a9346f8ae2731a8a963b28c42224c3 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 10 Oct 2016 20:32:20 +0800 Subject: Update to support showing owner on the infobar --- lib/response.js | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib') diff --git a/lib/response.js b/lib/response.js index 796f951f..15b81176 100755 --- a/lib/response.js +++ b/lib/response.js @@ -205,6 +205,9 @@ function showPublishNote(req, res, next) { url: origin, body: text, useCDN: config.usecdn, + owner: note.owner ? note.owner.id : null, + ownerprofile: note.owner ? models.User.parseProfile(note.owner.profile) : null, + lastchangeuser: note.lastchangeuser ? note.lastchangeuser.id : null, lastchangeuserprofile: note.lastchangeuser ? models.User.parseProfile(note.lastchangeuser.profile) : null, robots: meta.robots || false, //default allow robots GA: meta.GA, @@ -576,6 +579,9 @@ function showPublishSlide(req, res, next) { slides: slides, meta: JSON.stringify(obj.meta || {}), useCDN: config.usecdn, + owner: note.owner ? note.owner.id : null, + ownerprofile: note.owner ? models.User.parseProfile(note.owner.profile) : null, + lastchangeuser: note.lastchangeuser ? note.lastchangeuser.id : null, lastchangeuserprofile: note.lastchangeuser ? models.User.parseProfile(note.lastchangeuser.profile) : null, robots: meta.robots || false, //default allow robots GA: meta.GA, -- cgit v1.2.3 From 55ac4dcccb683322e12ee507d784a20a30e376e1 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 10 Oct 2016 20:33:48 +0800 Subject: Update to allow CORS as API on revision actions --- lib/response.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'lib') diff --git a/lib/response.js b/lib/response.js index 15b81176..043f4f7d 100755 --- a/lib/response.js +++ b/lib/response.js @@ -335,6 +335,13 @@ function actionRevision(req, res, note) { if (!content) { return response.errorNotFound(res); } + res.set({ + 'Access-Control-Allow-Origin': '*', //allow CORS as API + 'Access-Control-Allow-Headers': 'Range', + 'Access-Control-Expose-Headers': 'Cache-Control, Content-Encoding, Content-Range', + 'Cache-Control': 'private', // only cache by client + 'X-Robots-Tag': 'noindex, nofollow' // prevent crawling + }); res.send(content); }); } else { @@ -349,6 +356,13 @@ function actionRevision(req, res, note) { var out = { revision: data }; + res.set({ + 'Access-Control-Allow-Origin': '*', //allow CORS as API + 'Access-Control-Allow-Headers': 'Range', + 'Access-Control-Expose-Headers': 'Cache-Control, Content-Encoding, Content-Range', + 'Cache-Control': 'private', // only cache by client + 'X-Robots-Tag': 'noindex, nofollow' // prevent crawling + }); res.send(out); }); } -- cgit v1.2.3 From 11a8c0f9cfbf9d94a5acc320991509660e5e74b2 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 10 Oct 2016 20:38:14 +0800 Subject: Workaround cheerio text method shouldn't preserve html tags on fetching note title --- lib/models/note.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/models/note.js b/lib/models/note.js index 17988f74..3478538f 100644 --- a/lib/models/note.js +++ b/lib/models/note.js @@ -13,6 +13,7 @@ var async = require('async'); var moment = require('moment'); var DiffMatchPatch = require('diff-match-patch'); var dmp = new DiffMatchPatch(); +var S = require('string'); // core var config = require("../config.js"); @@ -234,7 +235,7 @@ module.exports = function (sequelize, DataTypes) { var $ = cheerio.load(md.render(body)); var h1s = $("h1"); if (h1s.length > 0 && h1s.first().text().split('\n').length == 1) - title = h1s.first().text(); + title = S(h1s.first().text()).stripTags().s; } if (!title) title = "Untitled"; return title; -- cgit v1.2.3 From a090008d4ae7eb69d0403b2dec43605bc981b19a Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 10 Oct 2016 20:40:45 +0800 Subject: Update to make OT socket io handle error better, use delay to avoid wrong reversion on client --- lib/ot/editor-socketio-server.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/ot/editor-socketio-server.js b/lib/ot/editor-socketio-server.js index 45ed5036..d062fa19 100755 --- a/lib/ot/editor-socketio-server.js +++ b/lib/ot/editor-socketio-server.js @@ -55,7 +55,15 @@ EditorSocketIOServer.prototype.addClient = function (socket) { if (typeof self.operationCallback === 'function') self.operationCallback(socket, operation); } catch (err) { - socket.disconnect(true); + setTimeout(function() { + var docOut = { + str: self.document, + revision: self.operations.length, + clients: self.users, + force: true + }; + socket.emit('doc', LZString.compressToUTF16(JSON.stringify(docOut))); + }, 100); } }); }); -- cgit v1.2.3 From b54b3cbe6957b46d3b689b0108b0b767b7da261b Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 10 Oct 2016 20:48:56 +0800 Subject: Add more comments in the code and remove unused code file --- lib/realtime.js | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/realtime.js b/lib/realtime.js index ae624c24..c9c6543c 100644 --- a/lib/realtime.js +++ b/lib/realtime.js @@ -518,13 +518,16 @@ function disconnect(socket) { var noteId = socket.noteId; var note = notes[noteId]; if (note) { + // delete user in users delete note.users[socket.id]; + // remove sockets in the note socks do { var index = note.socks.indexOf(socket); if (index != -1) { note.socks.splice(index, 1); } } while (index != -1); + // remove note in notes if no user inside if (Object.keys(note.users).length <= 0) { if (note.server.isDirty) { updateNote(note, function (err, _note) { -- cgit v1.2.3 From af77bb8f59adade4e99886dfc37f716f074294fb Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 10 Oct 2016 20:51:46 +0800 Subject: Update to add cache to history --- lib/history.js | 228 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 lib/history.js (limited to 'lib') diff --git a/lib/history.js b/lib/history.js new file mode 100644 index 00000000..119b7510 --- /dev/null +++ b/lib/history.js @@ -0,0 +1,228 @@ +//history +//external modules +var async = require('async'); +var moment = require('moment'); + +//core +var config = require("./config.js"); +var logger = require("./logger.js"); +var response = require("./response.js"); +var models = require("./models"); + +//public +var History = { + historyGet: historyGet, + historyPost: historyPost, + historyDelete: historyDelete, + isReady: isReady, + updateHistory: updateHistory +}; + +var caches = {}; +//update when the history is dirty +var updater = setInterval(function () { + var deleted = []; + async.each(Object.keys(caches), function (key, callback) { + var cache = caches[key]; + if (cache.isDirty) { + if (config.debug) logger.info("history updater found dirty history: " + key); + var history = parseHistoryToArray(cache.history); + finishUpdateHistory(key, history, function (err, count) { + if (err) return callback(err, null); + if (!count) return callback(null, null); + cache.isDirty = false; + cache.updateAt = Date.now(); + return callback(null, null); + }); + } else { + if (moment().isAfter(moment(cache.updateAt).add(5, 'minutes'))) { + deleted.push(key); + } + return callback(null, null); + } + }, function (err) { + if (err) return logger.error('history updater error', err); + }); + // delete specified caches + for (var i = 0, l = deleted.length; i < l; i++) { + caches[deleted[i]].history = {}; + delete caches[deleted[i]]; + } +}, 1000); + +function finishUpdateHistory(userid, history, callback) { + models.User.update({ + history: JSON.stringify(history) + }, { + where: { + id: userid + } + }).then(function (count) { + return callback(null, count); + }).catch(function (err) { + return callback(err, null); + }); +} + +function isReady() { + var dirtyCount = 0; + async.each(Object.keys(caches), function (key, callback) { + if (caches[key].isDirty) dirtyCount++; + return callback(null, null); + }, function (err) { + if (err) return logger.error('history ready check error', err); + }); + return dirtyCount > 0 ? false : true; +} + +function getHistory(userid, callback) { + if (caches[userid]) { + return callback(null, caches[userid].history); + } else { + models.User.findOne({ + where: { + id: userid + } + }).then(function (user) { + if (!user) + return callback(null, null); + var history = []; + if (user.history) + history = JSON.parse(user.history); + if (config.debug) + logger.info('read history success: ' + user.id); + setHistory(userid, history); + return callback(null, history); + }).catch(function (err) { + logger.error('read history failed: ' + err); + return callback(err, null); + }); + } +} + +function setHistory(userid, history) { + if (Array.isArray(history)) history = parseHistoryToObject(history); + if (!caches[userid]) { + caches[userid] = { + history: {}, + isDirty: false, + updateAt: Date.now() + }; + } + caches[userid].history = history; +} + +function updateHistory(userid, noteId, document) { + var t0 = new Date().getTime(); + if (userid && noteId && document) { + getHistory(userid, function (err, history) { + if (err || !history) return; + if (!caches[userid].history[noteId]) { + caches[userid].history[noteId] = {}; + } + var noteHistory = caches[userid].history[noteId]; + var noteInfo = models.Note.parseNoteInfo(document); + noteHistory.id = noteId; + noteHistory.text = noteInfo.title; + noteHistory.time = moment().format('MMMM Do YYYY, h:mm:ss a'); + noteHistory.tags = noteInfo.tags; + caches[userid].isDirty = true; + var t1 = new Date().getTime(); + console.warn(t1 - t0); + }); + } +} + +function parseHistoryToArray(history) { + var _history = []; + Object.keys(history).forEach(function (key) { + var item = history[key]; + _history.push(item); + }); + return _history; +} + +function parseHistoryToObject(history) { + var _history = {}; + for (var i = 0, l = history.length; i < l; i++) { + var item = history[i]; + _history[item.id] = item; + } + return _history; +} + +function historyGet(req, res) { + if (req.isAuthenticated()) { + getHistory(req.user.id, function (err, history) { + if (err) return response.errorInternalError(res); + if (!history) return response.errorNotFound(res); + res.send({ + history: parseHistoryToArray(history) + }); + }); + } else { + return response.errorForbidden(res); + } +} + +function historyPost(req, res) { + if (req.isAuthenticated()) { + var noteId = req.params.noteId; + if (!noteId) { + if (typeof req.body['history'] === 'undefined') return response.errorBadRequest(res); + if (config.debug) + logger.info('SERVER received history from [' + req.user.id + ']: ' + req.body.history); + try { + var history = JSON.parse(req.body.history); + } catch (err) { + return response.errorBadRequest(res); + } + if (Array.isArray(history)) { + setHistory(req.user.id, history); + caches[req.user.id].isDirty = true; + res.end(); + } else { + return response.errorBadRequest(res); + } + } else { + if (typeof req.body['pinned'] === 'undefined') return response.errorBadRequest(res); + getHistory(req.user.id, function (err, history) { + if (err) return response.errorInternalError(res); + if (!history) return response.errorNotFound(res); + if (!caches[req.user.id].history[noteId]) return response.errorNotFound(res); + if (req.body.pinned === 'true' || req.body.pinned === 'false') { + caches[req.user.id].history[noteId].pinned = (req.body.pinned === 'true'); + caches[req.user.id].isDirty = true; + res.end(); + } else { + return response.errorBadRequest(res); + } + }); + } + } else { + return response.errorForbidden(res); + } +} + +function historyDelete(req, res) { + if (req.isAuthenticated()) { + var noteId = req.params.noteId; + if (!noteId) { + setHistory(req.user.id, []); + caches[req.user.id].isDirty = true; + res.end(); + } else { + getHistory(req.user.id, function (err, history) { + if (err) return response.errorInternalError(res); + if (!history) return response.errorNotFound(res); + delete caches[req.user.id].history[noteId]; + caches[req.user.id].isDirty = true; + res.end(); + }); + } + } else { + return response.errorForbidden(res); + } +} + +module.exports = History; \ No newline at end of file -- cgit v1.2.3 From 1d2a9826af247883ff8e67b346263e5e2cd49791 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 10 Oct 2016 20:52:31 +0800 Subject: Update to improve history api error and bad request handling --- lib/response.js | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/response.js b/lib/response.js index 043f4f7d..fa97f157 100755 --- a/lib/response.js +++ b/lib/response.js @@ -33,6 +33,9 @@ var response = { errorNotFound: function (res) { responseError(res, "404", "Not Found", "oops."); }, + errorBadRequest: function (res) { + responseError(res, "400", "Bad Request", "something not right."); + }, errorInternalError: function (res) { responseError(res, "500", "Internal Error", "wtf."); }, -- cgit v1.2.3 From 36a1900ce3496f2d71ae4c41609dc52d24636be2 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 10 Oct 2016 20:55:33 +0800 Subject: Update to make note history count in server-side when user logged --- lib/models/note.js | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- lib/realtime.js | 14 ++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/models/note.js b/lib/models/note.js index 3478538f..08ef083d 100644 --- a/lib/models/note.js +++ b/lib/models/note.js @@ -218,8 +218,23 @@ module.exports = function (sequelize, DataTypes) { return callback(null, null); }); }, + parseNoteInfo: function (body) { + var meta = null; + try { + var obj = metaMarked(body); + body = obj.markdown; + meta = obj.meta; + } catch (err) { + //na + } + if (!meta) meta = {}; + var $ = cheerio.load(md.render(body)); + return { + title: Note.extractNoteTitle(meta, $), + tags: Note.extractNoteTags(meta, $) + }; + }, parseNoteTitle: function (body) { - var title = ""; var meta = null; try { var obj = metaMarked(body); @@ -229,10 +244,14 @@ module.exports = function (sequelize, DataTypes) { //na } if (!meta) meta = {}; + var $ = cheerio.load(md.render(body)); + return Note.extractNoteTitle(meta, $); + }, + extractNoteTitle: function (meta, $) { + var title = ""; if (meta.title && (typeof meta.title == "string" || typeof meta.title == "number")) { title = meta.title; } else { - var $ = cheerio.load(md.render(body)); var h1s = $("h1"); if (h1s.length > 0 && h1s.first().text().split('\n').length == 1) title = S(h1s.first().text()).stripTags().s; @@ -250,6 +269,40 @@ module.exports = function (sequelize, DataTypes) { title = !title || title == "Untitled" ? "HackMD - Collaborative markdown notes" : title + " - HackMD"; return title; }, + extractNoteTags: function (meta, $) { + var tags = []; + var rawtags = []; + if (meta.tags && (typeof meta.tags == "string" || typeof meta.tags == "number")) { + var metaTags = ('' + meta.tags).split(','); + for (var i = 0; i < metaTags.length; i++) { + var text = metaTags[i].trim(); + if (text) rawtags.push(text); + } + } else { + var h6s = $("h6"); + h6s.each(function (key, value) { + if (/^tags/gmi.test($(value).text())) { + var codes = $(value).find("code"); + for (var i = 0; i < codes.length; i++) { + var text = $(codes[i]).html().trim(); + if (text) rawtags.push(text); + } + } + }); + } + for (var i = 0; i < rawtags.length; i++) { + var found = false; + for (var j = 0; j < tags.length; j++) { + if (tags[j] == rawtags[i]) { + found = true; + break; + } + } + if (!found) + tags.push(rawtags[i]); + } + return tags; + }, parseMeta: function (meta) { var _meta = {}; if (meta) { diff --git a/lib/realtime.js b/lib/realtime.js index c9c6543c..a05c1f41 100644 --- a/lib/realtime.js +++ b/lib/realtime.js @@ -13,6 +13,7 @@ var moment = require('moment'); //core var config = require("./config.js"); var logger = require("./logger.js"); +var history = require("./history.js"); var models = require("./models"); //ot @@ -390,6 +391,12 @@ function finishConnection(socket, note, user) { note.server.setName(socket, user.name); note.server.setColor(socket, user.color); + // update user note history + setTimeout(function () { + var noteId = note.alias ? note.alias : LZString.compressToBase64(note.id); + history.updateHistory(user.userid, noteId, note.server.document); + }, 0); + emitOnlineUsers(socket); emitRefresh(socket); @@ -468,6 +475,7 @@ function startConnection(socket) { notes[noteId] = { id: noteId, + alias: note.alias, owner: owner, ownerprofile: ownerprofile, permission: note.permission, @@ -652,6 +660,12 @@ function operationCallback(socket, operation) { return logger.error('operation callback failed: ' + err); }); } + // update user note history + setTimeout(function() { + var noteId = note.alias ? note.alias : LZString.compressToBase64(note.id); + history.updateHistory(userId, noteId, note.server.document); + }, 0); + } // save authorship note.authorship = models.Note.updateAuthorshipByOperation(operation, userId, note.authorship); -- cgit v1.2.3 From d6d2cf978aed8f7abeb93d0393c9d04a4a06bd4e Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 10 Oct 2016 20:56:22 +0800 Subject: Update to send note title on emit check and refresh event --- lib/realtime.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/realtime.js b/lib/realtime.js index a05c1f41..c6c9ffeb 100644 --- a/lib/realtime.js +++ b/lib/realtime.js @@ -64,6 +64,7 @@ function secure(socket, next) { function emitCheck(note) { var out = { + title: note.title, updatetime: note.updatetime, lastchangeuser: note.lastchangeuser, lastchangeuserprofile: note.lastchangeuserprofile, @@ -149,7 +150,7 @@ function updateNote(note, callback) { function finishUpdateNote(note, _note, callback) { if (!note || !note.server) return callback(null, null); var body = note.server.document; - var title = models.Note.parseNoteTitle(body); + var title = note.title = models.Note.parseNoteTitle(body); title = LZString.compressToBase64(title); body = LZString.compressToBase64(body); var values = { @@ -313,6 +314,7 @@ function emitRefresh(socket) { if (!noteId || !notes[noteId]) return; var note = notes[noteId]; var out = { + title: note.title, docmaxlength: config.documentmaxlength, owner: note.owner, ownerprofile: note.ownerprofile, @@ -476,6 +478,7 @@ function startConnection(socket) { notes[noteId] = { id: noteId, alias: note.alias, + title: LZString.decompressFromBase64(note.title), owner: owner, ownerprofile: ownerprofile, permission: note.permission, -- cgit v1.2.3 From 12d5ed43a7376e0ca361160698f07066218d6ed2 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 10 Oct 2016 21:04:24 +0800 Subject: Update to support delete note --- .../20161009040430-support-delete-note.js | 11 ++++++++ lib/models/note.js | 1 + lib/realtime.js | 29 ++++++++++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 lib/migrations/20161009040430-support-delete-note.js (limited to 'lib') diff --git a/lib/migrations/20161009040430-support-delete-note.js b/lib/migrations/20161009040430-support-delete-note.js new file mode 100644 index 00000000..f478b6fe --- /dev/null +++ b/lib/migrations/20161009040430-support-delete-note.js @@ -0,0 +1,11 @@ +'use strict'; + +module.exports = { + up: function (queryInterface, Sequelize) { + queryInterface.addColumn('Notes', 'deletedAt', Sequelize.DATE); + }, + + down: function (queryInterface, Sequelize) { + queryInterface.removeColumn('Notes', 'deletedAt', Sequelize.DATE); + } +}; diff --git a/lib/models/note.js b/lib/models/note.js index 08ef083d..6efa5d4f 100644 --- a/lib/models/note.js +++ b/lib/models/note.js @@ -67,6 +67,7 @@ module.exports = function (sequelize, DataTypes) { type: DataTypes.DATE } }, { + paranoid: true, classMethods: { associate: function (models) { Note.belongsTo(models.User, { diff --git a/lib/realtime.js b/lib/realtime.js index c6c9ffeb..0fcd2eaa 100644 --- a/lib/realtime.js +++ b/lib/realtime.js @@ -796,6 +796,35 @@ function connection(socket) { } }); + // delete a note + socket.on('delete', function () { + //need login to do more actions + if (socket.request.user && socket.request.user.logged_in) { + var noteId = socket.noteId; + if (!noteId || !notes[noteId]) return; + var note = notes[noteId]; + //Only owner can delete note + if (note.owner && note.owner == socket.request.user.id) { + models.Note.destroy({ + where: { + id: noteId + } + }).then(function (count) { + if (!count) return; + for (var i = 0, l = note.socks.length; i < l; i++) { + var sock = note.socks[i]; + if (typeof sock !== 'undefined' && sock) { + sock.emit('delete'); + return sock.disconnect(true); + } + } + }).catch(function (err) { + return logger.error('delete note failed: ' + err); + }); + } + } + }); + //reveiced when user logout or changed socket.on('user changed', function () { logger.info('user changed'); -- cgit v1.2.3 From b734eb9c85fc6b2cfe534206f0f1b01656d9acc3 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 10 Oct 2016 21:05:49 +0800 Subject: Try to fix memory leaks by clear OT server before disconnect note --- lib/realtime.js | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/realtime.js b/lib/realtime.js index 0fcd2eaa..ea3735a6 100644 --- a/lib/realtime.js +++ b/lib/realtime.js @@ -543,6 +543,9 @@ function disconnect(socket) { if (note.server.isDirty) { updateNote(note, function (err, _note) { if (err) return logger.error('disconnect note failed: ' + err); + // clear server before delete to avoid memory leaks + note.server.document = ""; + note.server.operations = []; delete note.server; delete notes[noteId]; if (config.debug) { -- cgit v1.2.3 From dfc8aeeba07966bc1ea116e20e3b2d0af27f6c47 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 10 Oct 2016 21:16:58 +0800 Subject: Add more environment variables for server configuration, update related section in README.md --- lib/config.js | 48 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/config.js b/lib/config.js index 588128a7..46afea9e 100644 --- a/lib/config.js +++ b/lib/config.js @@ -7,16 +7,17 @@ var config = require(path.join(__dirname, '..', 'config.json'))[env]; var debug = process.env.DEBUG ? (process.env.DEBUG === 'true') : ((typeof config.debug === 'boolean') ? config.debug : (env === 'development')); // url -var domain = process.env.DOMAIN || config.domain || ''; -var urlpath = process.env.URL_PATH || config.urlpath || ''; -var port = process.env.PORT || config.port || 3000; -var alloworigin = config.alloworigin || ['localhost']; +var domain = process.env.DOMAIN || process.env.HMD_DOMAIN || config.domain || ''; +var urlpath = process.env.URL_PATH || process.env.HMD_URL_PATH || config.urlpath || ''; +var port = process.env.PORT || process.env.HMD_PORT || config.port || 3000; +var alloworigin = process.env.HMD_ALLOW_ORIGIN ? process.env.HMD_ALLOW_ORIGIN.split(',') : (config.alloworigin || ['localhost']); var usessl = !!config.usessl; -var protocolusessl = (config.usessl === true && typeof config.protocolusessl === 'undefined') ? true : !!config.protocolusessl; -var urladdport = !!config.urladdport; +var protocolusessl = (usessl === true && typeof process.env.HMD_PROTOCOL_USESSL === 'undefined' && typeof config.protocolusessl === 'undefined') + ? true : (process.env.HMD_PROTOCOL_USESSL ? (process.env.HMD_PROTOCOL_USESSL === 'true') : !!config.protocolusessl); +var urladdport = process.env.HMD_URL_ADDPORT ? (process.env.HMD_URL_ADDPORT === 'true') : !!config.urladdport; -var usecdn = !!config.usecdn; +var usecdn = process.env.HMD_USECDN ? (process.env.HMD_USECDN === 'true') : !!config.usecdn; // db var db = config.db || { @@ -56,13 +57,32 @@ var heartbeattimeout = config.heartbeattimeout || 5000; var documentmaxlength = config.documentmaxlength || 100000; // auth -var facebook = config.facebook || false; -var twitter = config.twitter || false; -var github = config.github || false; -var gitlab = config.gitlab || false; -var dropbox = config.dropbox || false; -var google = config.google || false; -var imgur = config.imgur || false; +var facebook = (process.env.HMD_FACEBOOK_CLIENTID && process.env.HMD_FACEBOOK_CLIENTSECRET) ? { + clientID: process.env.HMD_FACEBOOK_CLIENTID, + clientSecret: process.env.HMD_FACEBOOK_CLIENTSECRET +} : config.facebook || false; +var twitter = (process.env.HMD_TWITTER_CONSUMERKEY && process.env.HMD_TWITTER_CONSUMERSECRET) ? { + consumerKey: process.env.HMD_TWITTER_CONSUMERKEY, + consumerSecret: process.env.HMD_TWITTER_CONSUMERSECRET +} : config.twitter || false; +var github = (process.env.HMD_GITHUB_CLIENTID && process.env.HMD_GITHUB_CLIENTSECRET) ? { + clientID: process.env.HMD_GITHUB_CLIENTID, + clientSecret: process.env.HMD_GITHUB_CLIENTSECRET +} : config.github || false; +var gitlab = (process.env.HMD_GITLAB_CLIENTID && process.env.HMD_GITLAB_CLIENTSECRET) ? { + baseURL: process.env.HMD_GITLAB_BASEURL, + clientID: process.env.HMD_GITLAB_CLIENTID, + clientSecret: process.env.HMD_GITLAB_CLIENTSECRET +} : config.gitlab || false; +var dropbox = (process.env.HMD_DROPBOX_CLIENTID && process.env.HMD_DROPBOX_CLIENTSECRET) ? { + clientID: process.env.HMD_DROPBOX_CLIENTID, + clientSecret: process.env.HMD_DROPBOX_CLIENTSECRET +} : config.dropbox || false; +var google = (process.env.HMD_GOOGLE_CLIENTID && process.env.HMD_GOOGLE_CLIENTSECRET) ? { + clientID: process.env.HMD_GOOGLE_CLIENTID, + clientSecret: process.env.HMD_GOOGLE_CLIENTSECRET +} : config.google || false; +var imgur = process.env.HMD_IMGUR_CLIENTID || config.imgur || false; function getserverurl() { var url = ''; -- cgit v1.2.3 From bc74c1f0cb4e146d7944cea37fb1afbe6772fd71 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Tue, 11 Oct 2016 00:55:38 +0800 Subject: Fix doc updating revision not stringify and compress authorship before save --- lib/models/note.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/models/note.js b/lib/models/note.js index 6efa5d4f..7fdc5645 100644 --- a/lib/models/note.js +++ b/lib/models/note.js @@ -142,7 +142,7 @@ module.exports = function (sequelize, DataTypes) { authorship = Note.updateAuthorshipByOperation(operations[i], null, authorship); } note.update({ - authorship: authorship + authorship: LZString.compressToBase64(JSON.stringify(authorship)) }).then(function (note) { return callback(null, note.id); }).catch(function (err) { -- cgit v1.2.3 From 510b1254323ad03a2ac21a93a4450d7fb67ab9a8 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Tue, 11 Oct 2016 01:22:08 +0800 Subject: Fix new note with empty content not saving to history and remove debug code --- lib/history.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/history.js b/lib/history.js index 119b7510..c3754e0b 100644 --- a/lib/history.js +++ b/lib/history.js @@ -113,8 +113,7 @@ function setHistory(userid, history) { } function updateHistory(userid, noteId, document) { - var t0 = new Date().getTime(); - if (userid && noteId && document) { + if (userid && noteId && typeof document !== 'undefined') { getHistory(userid, function (err, history) { if (err || !history) return; if (!caches[userid].history[noteId]) { @@ -127,8 +126,6 @@ function updateHistory(userid, noteId, document) { noteHistory.time = moment().format('MMMM Do YYYY, h:mm:ss a'); noteHistory.tags = noteInfo.tags; caches[userid].isDirty = true; - var t1 = new Date().getTime(); - console.warn(t1 - t0); }); } } -- cgit v1.2.3 From 9a15cad42de2112c9c433b3199f39aca440f1581 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Tue, 11 Oct 2016 11:01:05 +0800 Subject: Mark as 0.4.5 --- lib/config.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/config.js b/lib/config.js index 46afea9e..63114975 100644 --- a/lib/config.js +++ b/lib/config.js @@ -97,8 +97,8 @@ function getserverurl() { return url; } -var version = '0.4.4'; -var minimumCompatibleVersion = '0.4.4'; +var version = '0.4.5'; +var minimumCompatibleVersion = '0.4.5'; var maintenance = true; var cwd = path.join(__dirname, '..'); -- cgit v1.2.3 From c06b2f483898669b224321728321b7a3a8e9e37c Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Tue, 11 Oct 2016 16:46:50 +0800 Subject: Fix history time should save in UNIX timestamp to avoid time offset issue --- lib/history.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/history.js b/lib/history.js index c3754e0b..4a3bbe1e 100644 --- a/lib/history.js +++ b/lib/history.js @@ -123,7 +123,7 @@ function updateHistory(userid, noteId, document) { var noteInfo = models.Note.parseNoteInfo(document); noteHistory.id = noteId; noteHistory.text = noteInfo.title; - noteHistory.time = moment().format('MMMM Do YYYY, h:mm:ss a'); + noteHistory.time = moment().valueOf(); noteHistory.tags = noteInfo.tags; caches[userid].isDirty = true; }); -- cgit v1.2.3