From c904083d1f5064d2e786e0bc6ee3804b91805d24 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 2 Jan 2017 10:52:47 +0800 Subject: Remove manual LZString compression for partial socket io event data --- lib/ot/editor-socketio-server.js | 8 ++------ lib/realtime.js | 4 ---- 2 files changed, 2 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/ot/editor-socketio-server.js b/lib/ot/editor-socketio-server.js index d062fa19..7b204539 100755 --- a/lib/ot/editor-socketio-server.js +++ b/lib/ot/editor-socketio-server.js @@ -7,7 +7,6 @@ var Server = require('./server'); var Selection = require('./selection'); var util = require('util'); -var LZString = require('lz-string'); var logger = require('../logger'); function EditorSocketIOServer(document, operations, docId, mayWrite, operationCallback) { @@ -40,10 +39,8 @@ EditorSocketIOServer.prototype.addClient = function (socket) { revision: this.operations.length, clients: this.users }; - socket.emit('doc', LZString.compressToUTF16(JSON.stringify(docOut))); + socket.emit('doc', docOut); socket.on('operation', function (revision, operation, selection) { - operation = LZString.decompressFromUTF16(operation); - operation = JSON.parse(operation); socket.origin = 'operation'; self.mayWrite(socket, function (mayWrite) { if (!mayWrite) { @@ -62,7 +59,7 @@ EditorSocketIOServer.prototype.addClient = function (socket) { clients: self.users, force: true }; - socket.emit('doc', LZString.compressToUTF16(JSON.stringify(docOut))); + socket.emit('doc', docOut); }, 100); } }); @@ -129,7 +126,6 @@ EditorSocketIOServer.prototype.onGetOperations = function (socket, base, head) { var operations = this.operations.slice(base, head).map(function (op) { return op.wrapped.toJSON(); }); - operations = LZString.compressToUTF16(JSON.stringify(operations)); socket.emit('operations', head, operations); }; diff --git a/lib/realtime.js b/lib/realtime.js index c243ffc1..c66fea0a 100644 --- a/lib/realtime.js +++ b/lib/realtime.js @@ -71,7 +71,6 @@ function emitCheck(note) { authors: note.authors, authorship: note.authorship }; - out = LZString.compressToUTF16(JSON.stringify(out)); realtime.io.to(note.id).emit('check', out); } @@ -301,7 +300,6 @@ function emitOnlineUsers(socket) { var out = { users: users }; - out = LZString.compressToUTF16(JSON.stringify(out)); realtime.io.to(noteId).emit('online users', out); } @@ -330,7 +328,6 @@ function emitRefresh(socket) { createtime: note.createtime, updatetime: note.updatetime }; - out = LZString.compressToUTF16(JSON.stringify(out)); socket.emit('refresh', out); } @@ -863,7 +860,6 @@ function connection(socket) { var out = { users: users }; - out = LZString.compressToUTF16(JSON.stringify(out)); socket.emit('online users', out); }); -- cgit v1.2.3 From f6d8e3ab00370a78c0c788ad1e37a7ff77a53555 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 2 Jan 2017 10:59:53 +0800 Subject: Remove LZString compression for data storage --- lib/models/note.js | 17 ++++++----------- lib/models/revision.js | 11 +++++------ lib/realtime.js | 10 ++++------ lib/response.js | 14 +++++++------- lib/workers/dmpWorker.js | 13 ++++++------- 5 files changed, 28 insertions(+), 37 deletions(-) (limited to 'lib') diff --git a/lib/models/note.js b/lib/models/note.js index 5727046c..81de991f 100644 --- a/lib/models/note.js +++ b/lib/models/note.js @@ -124,8 +124,6 @@ module.exports = function (sequelize, DataTypes) { var body = fs.readFileSync(filePath, 'utf8'); var contentLength = body.length; var title = Note.parseNoteTitle(body); - body = LZString.compressToBase64(body); - title = LZString.compressToBase64(title); if (fsModifiedTime.isAfter(dbModifiedTime) && note.content !== body) { note.update({ title: title, @@ -135,14 +133,14 @@ module.exports = function (sequelize, DataTypes) { sequelize.models.Revision.saveNoteRevision(note, function (err, revision) { if (err) return _callback(err, null); // update authorship on after making revision of docs - var patch = dmp.patch_fromText(LZString.decompressFromBase64(revision.patch)); + var patch = dmp.patch_fromText(revision.patch); var operations = Note.transformPatchToOperations(patch, contentLength); - var authorship = note.authorship ? JSON.parse(LZString.decompressFromBase64(note.authorship)) : []; + var authorship = note.authorship; for (var i = 0; i < operations.length; i++) { authorship = Note.updateAuthorshipByOperation(operations[i], null, authorship); } note.update({ - authorship: LZString.compressToBase64(JSON.stringify(authorship)) + authorship: JSON.stringify(authorship) }).then(function (note) { return callback(null, note.id); }).catch(function (err) { @@ -264,10 +262,7 @@ module.exports = function (sequelize, DataTypes) { return markdown.substr(0, 100).replace(/(?:\r\n|\r|\n)/g, ' '); }, decodeTitle: function (title) { - var decodedTitle = LZString.decompressFromBase64(title); - if (decodedTitle) title = decodedTitle; - else title = 'Untitled'; - return title; + return title ? title : 'Untitled'; }, generateWebTitle: function (title) { title = !title || title == "Untitled" ? "HackMD - Collaborative markdown notes" : title + " - HackMD"; @@ -496,8 +491,8 @@ module.exports = function (sequelize, DataTypes) { if (Note.checkFileExist(filePath)) { var fsCreatedTime = moment(fs.statSync(filePath).ctime); body = fs.readFileSync(filePath, 'utf8'); - note.title = LZString.compressToBase64(Note.parseNoteTitle(body)); - note.content = LZString.compressToBase64(body); + note.title = Note.parseNoteTitle(body); + note.content = body; if (filePath !== config.defaultnotepath) { note.createdAt = fsCreatedTime; } diff --git a/lib/models/revision.js b/lib/models/revision.js index 8b8eba94..6f44cf1d 100644 --- a/lib/models/revision.js +++ b/lib/models/revision.js @@ -2,7 +2,6 @@ // external modules var Sequelize = require("sequelize"); -var LZString = require('lz-string'); var async = require('async'); var moment = require('moment'); var childProcess = require('child_process'); @@ -214,7 +213,7 @@ module.exports = function (sequelize, DataTypes) { Revision.create({ noteId: note.id, lastContent: note.content, - length: LZString.decompressFromBase64(note.content).length, + length: note.content.length, authorship: note.authorship }).then(function (revision) { Revision.finishSaveNoteRevision(note, revision, callback); @@ -223,8 +222,8 @@ module.exports = function (sequelize, DataTypes) { }); } else { var latestRevision = revisions[0]; - var lastContent = LZString.decompressFromBase64(latestRevision.content || latestRevision.lastContent); - var content = LZString.decompressFromBase64(note.content); + var lastContent = latestRevision.content || latestRevision.lastContent; + var content = note.content; sendDmpWorker({ msg: 'create patch', lastDoc: lastContent, @@ -244,9 +243,9 @@ module.exports = function (sequelize, DataTypes) { } else { Revision.create({ noteId: note.id, - patch: LZString.compressToBase64(patch), + patch: patch, content: note.content, - length: LZString.decompressFromBase64(note.content).length, + length: note.content.length, authorship: note.authorship }).then(function (revision) { // clear last revision content to reduce db size diff --git a/lib/realtime.js b/lib/realtime.js index c66fea0a..a662deeb 100644 --- a/lib/realtime.js +++ b/lib/realtime.js @@ -152,12 +152,10 @@ function finishUpdateNote(note, _note, callback) { if (!note || !note.server) return callback(null, null); var body = note.server.document; var title = note.title = models.Note.parseNoteTitle(body); - title = LZString.compressToBase64(title); - body = LZString.compressToBase64(body); var values = { title: title, content: body, - authorship: LZString.compressToBase64(JSON.stringify(note.authorship)), + authorship: note.authorship, lastchangeuserId: note.lastchangeuser, lastchangeAt: Date.now() }; @@ -459,7 +457,7 @@ function startConnection(socket) { var lastchangeuser = note.lastchangeuserId; var lastchangeuserprofile = note.lastchangeuser ? models.User.getProfile(note.lastchangeuser) : null; - var body = LZString.decompressFromBase64(note.content); + var body = note.content; var createtime = note.createdAt; var updatetime = note.lastchangeAt; var server = new ot.EditorSocketIOServer(body, [], noteId, ifMayEdit, operationCallback); @@ -479,7 +477,7 @@ function startConnection(socket) { notes[noteId] = { id: noteId, alias: note.alias, - title: LZString.decompressFromBase64(note.title), + title: note.title, owner: owner, ownerprofile: ownerprofile, permission: note.permission, @@ -491,7 +489,7 @@ function startConnection(socket) { updatetime: moment(updatetime).valueOf(), server: server, authors: authors, - authorship: note.authorship ? JSON.parse(LZString.decompressFromBase64(note.authorship)) : [] + authorship: note.authorship }; return finishConnection(socket, notes[noteId], users[socket.id]); diff --git a/lib/response.js b/lib/response.js index 2b38cf25..f591d5ed 100755 --- a/lib/response.js +++ b/lib/response.js @@ -75,7 +75,7 @@ function showIndex(req, res, next) { } function responseHackMD(res, note) { - var body = LZString.decompressFromBase64(note.content); + var body = note.content; var meta = null; try { meta = models.Note.parseMeta(metaMarked(body).meta); @@ -191,7 +191,7 @@ function showPublishNote(req, res, next) { if (!note) { return response.errorNotFound(res); } - var body = LZString.decompressFromBase64(note.content); + var body = note.content; var meta = null; var markdown = null; try { @@ -248,7 +248,7 @@ function actionSlide(req, res, note) { } function actionDownload(req, res, note) { - var body = LZString.decompressFromBase64(note.content); + var body = note.content; var title = models.Note.decodeTitle(note.title); var filename = title; filename = encodeURIComponent(filename); @@ -265,7 +265,7 @@ function actionDownload(req, res, note) { } function actionInfo(req, res, note) { - var body = LZString.decompressFromBase64(note.content); + var body = note.content; var meta = null; var markdown = null; try { @@ -297,7 +297,7 @@ function actionInfo(req, res, note) { } function actionPDF(req, res, note) { - var body = LZString.decompressFromBase64(note.content); + var body = note.content; try { body = metaMarked(body).markdown; } catch(err) { @@ -479,7 +479,7 @@ function githubActionGist(req, res, note) { if (!error && httpResponse.statusCode == 200) { var access_token = body.access_token; if (access_token) { - var content = LZString.decompressFromBase64(note.content); + var content = note.content; var title = models.Note.decodeTitle(note.title); var filename = title.replace('/', ' ') + '.md'; var gist = { @@ -579,7 +579,7 @@ function showPublishSlide(req, res, next) { if (!note) { return response.errorNotFound(res); } - var body = LZString.decompressFromBase64(note.content); + var body = note.content; var meta = null; var markdown = null; try { diff --git a/lib/workers/dmpWorker.js b/lib/workers/dmpWorker.js index fae36191..5b4b6aa2 100644 --- a/lib/workers/dmpWorker.js +++ b/lib/workers/dmpWorker.js @@ -1,5 +1,4 @@ // external modules -var LZString = require('lz-string'); var DiffMatchPatch = require('diff-match-patch'); var dmp = new DiffMatchPatch(); @@ -80,10 +79,10 @@ function getRevision(revisions, count) { for (var i = 0; i < count; i++) { var revision = revisions[i]; if (i == 0) { - startContent = LZString.decompressFromBase64(revision.content || revision.lastContent); + startContent = revision.content || revision.lastContent; } if (i != count - 1) { - var patch = dmp.patch_fromText(LZString.decompressFromBase64(revision.patch)); + var patch = dmp.patch_fromText(revision.patch); applyPatches = applyPatches.concat(patch); } lastPatch = revision.patch; @@ -105,11 +104,11 @@ function getRevision(revisions, count) { for (var i = l; i >= count - 1; i--) { var revision = revisions[i]; if (i == l) { - startContent = LZString.decompressFromBase64(revision.lastContent); + startContent = revision.lastContent; authorship = revision.authorship; } if (revision.patch) { - var patch = dmp.patch_fromText(LZString.decompressFromBase64(revision.patch)); + var patch = dmp.patch_fromText(revision.patch); applyPatches = applyPatches.concat(patch); } lastPatch = revision.patch; @@ -123,8 +122,8 @@ function getRevision(revisions, count) { } var data = { content: finalContent, - patch: dmp.patch_fromText(LZString.decompressFromBase64(lastPatch)), - authorship: authorship ? JSON.parse(LZString.decompressFromBase64(authorship)) : null + patch: dmp.patch_fromText(lastPatch), + authorship: authorship }; var ms_end = (new Date()).getTime(); if (config.debug) { -- cgit v1.2.3 From 99628a5662b0abcb396e78aad0f631071d9d1f61 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 2 Jan 2017 11:00:08 +0800 Subject: Fix to not use diff_cleanupSemantic, bug report refer to https://code.google.com/p/google-diff-match-patch/issues/detail?id=67 --- lib/workers/dmpWorker.js | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/workers/dmpWorker.js b/lib/workers/dmpWorker.js index 5b4b6aa2..8e69636e 100644 --- a/lib/workers/dmpWorker.js +++ b/lib/workers/dmpWorker.js @@ -57,7 +57,6 @@ process.on('message', function(data) { function createPatch(lastDoc, currDoc) { var ms_start = (new Date()).getTime(); var diff = dmp.diff_main(lastDoc, currDoc); - dmp.diff_cleanupSemantic(diff); var patch = dmp.patch_make(lastDoc, diff); patch = dmp.patch_toText(patch); var ms_end = (new Date()).getTime(); -- cgit v1.2.3 From c3a96ff112c8f7ea31af97fbc3319e782f4490e2 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 2 Jan 2017 11:00:40 +0800 Subject: Fix migration script of revision lacks of definition of primary key --- lib/migrations/20160607060246-support-revision.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/migrations/20160607060246-support-revision.js b/lib/migrations/20160607060246-support-revision.js index 9721d7fc..fa647d93 100644 --- a/lib/migrations/20160607060246-support-revision.js +++ b/lib/migrations/20160607060246-support-revision.js @@ -4,7 +4,10 @@ module.exports = { up: function (queryInterface, Sequelize) { queryInterface.addColumn('Notes', 'savedAt', Sequelize.DATE); queryInterface.createTable('Revisions', { - id: Sequelize.UUID, + id: { + type: Sequelize.UUID, + primaryKey: true + }, noteId: Sequelize.UUID, patch: Sequelize.TEXT, lastContent: Sequelize.TEXT, -- cgit v1.2.3 From d9e19b602968551fcda4ee806d767a04f6a11490 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 2 Jan 2017 11:05:05 +0800 Subject: Update to remove null byte before saving to DB and remove null byte on changes --- lib/models/index.js | 7 +++++++ lib/models/note.js | 15 ++++++++++++--- lib/models/revision.js | 20 ++++++++++++++++---- 3 files changed, 35 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/models/index.js b/lib/models/index.js index de6cd13c..6d0fd3c3 100644 --- a/lib/models/index.js +++ b/lib/models/index.js @@ -20,6 +20,13 @@ if (config.dburl) else sequelize = new Sequelize(dbconfig.database, dbconfig.username, dbconfig.password, dbconfig); +// [Postgres] Handling NULL bytes +// https://github.com/sequelize/sequelize/issues/6485 +function stripNullByte(value) { + return value ? value.replace(/\u0000/g, "") : value; +} +sequelize.stripNullByte = stripNullByte; + var db = {}; fs diff --git a/lib/models/note.js b/lib/models/note.js index 81de991f..37d26ec0 100644 --- a/lib/models/note.js +++ b/lib/models/note.js @@ -52,13 +52,22 @@ module.exports = function (sequelize, DataTypes) { defaultValue: 0 }, title: { - type: DataTypes.TEXT + type: DataTypes.TEXT, + set: function (value) { + this.setDataValue('title', sequelize.stripNullByte(value)); + } }, content: { - type: DataTypes.TEXT + type: DataTypes.TEXT, + set: function (value) { + this.setDataValue('content', sequelize.stripNullByte(value)); + } }, authorship: { - type: DataTypes.TEXT + type: DataTypes.TEXT, + set: function (value) { + this.setDataValue('authorship', JSON.stringify(value)); + } }, lastchangeAt: { type: DataTypes.DATE diff --git a/lib/models/revision.js b/lib/models/revision.js index 6f44cf1d..adc651f1 100644 --- a/lib/models/revision.js +++ b/lib/models/revision.js @@ -59,19 +59,31 @@ module.exports = function (sequelize, DataTypes) { defaultValue: Sequelize.UUIDV4 }, patch: { - type: DataTypes.TEXT + type: DataTypes.TEXT, + set: function (value) { + this.setDataValue('patch', sequelize.stripNullByte(value)); + } }, lastContent: { - type: DataTypes.TEXT + type: DataTypes.TEXT, + set: function (value) { + this.setDataValue('lastContent', sequelize.stripNullByte(value)); + } }, content: { - type: DataTypes.TEXT + type: DataTypes.TEXT, + set: function (value) { + this.setDataValue('content', sequelize.stripNullByte(value)); + } }, length: { type: DataTypes.INTEGER }, authorship: { - type: DataTypes.TEXT + type: DataTypes.TEXT, + set: function (value) { + this.setDataValue('authorship', value ? JSON.stringify(value) : value); + } } }, { classMethods: { -- cgit v1.2.3 From b1ec3ba748dc7b58015e97448958e78b512f6df0 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 2 Jan 2017 11:05:36 +0800 Subject: Refactor data processing to model definition --- lib/models/index.js | 6 ++++++ lib/models/note.js | 9 +++++++++ lib/models/revision.js | 12 ++++++++++++ 3 files changed, 27 insertions(+) (limited to 'lib') diff --git a/lib/models/index.js b/lib/models/index.js index 6d0fd3c3..e83956e5 100644 --- a/lib/models/index.js +++ b/lib/models/index.js @@ -27,6 +27,12 @@ function stripNullByte(value) { } sequelize.stripNullByte = stripNullByte; +function processData(data, _default, process) { + if (data === undefined) return data; + else return data === null ? _default : (process ? process(data) : data); +} +sequelize.processData = processData; + var db = {}; fs diff --git a/lib/models/note.js b/lib/models/note.js index 37d26ec0..5c63dc1a 100644 --- a/lib/models/note.js +++ b/lib/models/note.js @@ -53,18 +53,27 @@ module.exports = function (sequelize, DataTypes) { }, title: { type: DataTypes.TEXT, + get: function () { + return sequelize.processData(this.getDataValue('title'), ""); + }, set: function (value) { this.setDataValue('title', sequelize.stripNullByte(value)); } }, content: { type: DataTypes.TEXT, + get: function () { + return sequelize.processData(this.getDataValue('content'), ""); + }, set: function (value) { this.setDataValue('content', sequelize.stripNullByte(value)); } }, authorship: { type: DataTypes.TEXT, + get: function () { + return sequelize.processData(this.getDataValue('authorship'), [], JSON.parse); + }, set: function (value) { this.setDataValue('authorship', JSON.stringify(value)); } diff --git a/lib/models/revision.js b/lib/models/revision.js index adc651f1..c7360fed 100644 --- a/lib/models/revision.js +++ b/lib/models/revision.js @@ -60,18 +60,27 @@ module.exports = function (sequelize, DataTypes) { }, patch: { type: DataTypes.TEXT, + get: function () { + return sequelize.processData(this.getDataValue('patch'), ""); + }, set: function (value) { this.setDataValue('patch', sequelize.stripNullByte(value)); } }, lastContent: { type: DataTypes.TEXT, + get: function () { + return sequelize.processData(this.getDataValue('lastContent'), ""); + }, set: function (value) { this.setDataValue('lastContent', sequelize.stripNullByte(value)); } }, content: { type: DataTypes.TEXT, + get: function () { + return sequelize.processData(this.getDataValue('content'), ""); + }, set: function (value) { this.setDataValue('content', sequelize.stripNullByte(value)); } @@ -81,6 +90,9 @@ module.exports = function (sequelize, DataTypes) { }, authorship: { type: DataTypes.TEXT, + get: function () { + return sequelize.processData(this.getDataValue('authorship'), [], JSON.parse); + }, set: function (value) { this.setDataValue('authorship', value ? JSON.stringify(value) : value); } -- cgit v1.2.3 From 10a8448c6a9cdde0ad8cadf3dc250de159eacbf2 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 2 Jan 2017 11:13:41 +0800 Subject: Fix yaml metadata description not able to show --- lib/response.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/response.js b/lib/response.js index f591d5ed..54e2a337 100755 --- a/lib/response.js +++ b/lib/response.js @@ -209,7 +209,7 @@ function showPublishNote(req, res, next) { var origin = config.serverurl; var data = { title: title, - description: meta.description || markdown ? models.Note.generateDescription(markdown) : null, + description: meta.description || (markdown ? models.Note.generateDescription(markdown) : null), viewcount: note.viewcount, createtime: createtime, updatetime: updatetime, @@ -281,7 +281,7 @@ function actionInfo(req, res, note) { var title = models.Note.decodeTitle(note.title); var data = { title: meta.title || title, - description: meta.description || markdown ? models.Note.generateDescription(markdown) : null, + description: meta.description || (markdown ? models.Note.generateDescription(markdown) : null), viewcount: note.viewcount, createtime: createtime, updatetime: updatetime @@ -597,7 +597,7 @@ function showPublishSlide(req, res, next) { var origin = config.serverurl; var data = { title: title, - description: meta.description || markdown ? models.Note.generateDescription(markdown) : null, + description: meta.description || (markdown ? models.Note.generateDescription(markdown) : null), viewcount: note.viewcount, createtime: createtime, updatetime: updatetime, -- cgit v1.2.3 From bd4335964d678dbde153946a4998a47fed63d48e Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 2 Jan 2017 11:19:01 +0800 Subject: Mark as 0.5.0 --- lib/config.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/config.js b/lib/config.js index f8df0a73..53497f1f 100644 --- a/lib/config.js +++ b/lib/config.js @@ -111,8 +111,8 @@ function getserverurl() { return url; } -var version = '0.4.6'; -var minimumCompatibleVersion = '0.4.5'; +var version = '0.5.0'; +var minimumCompatibleVersion = '0.5.0'; var maintenance = true; var cwd = path.join(__dirname, '..'); -- cgit v1.2.3