Merge branch 'master' into frontend-next

9 files changed, 101 insertions, 61 deletions

diff --git a/lib/config.js b/lib/config.js
index f8df0a73..53497f1f 100644
--- a/lib/config.js
+++ b/lib/config.js
@@ -111,8 +111,8 @@ function getserverurl() {
     return url;
 }
 
-var version = '0.4.6';
-var minimumCompatibleVersion = '0.4.5';
+var version = '0.5.0';
+var minimumCompatibleVersion = '0.5.0';
 var maintenance = true;
 var cwd = path.join(__dirname, '..');
 
diff --git a/lib/migrations/20160607060246-support-revision.js b/lib/migrations/20160607060246-support-revision.js
index 9721d7fc..fa647d93 100644
--- a/lib/migrations/20160607060246-support-revision.js
+++ b/lib/migrations/20160607060246-support-revision.js
@@ -4,7 +4,10 @@ module.exports = {
   up: function (queryInterface, Sequelize) {
     queryInterface.addColumn('Notes', 'savedAt', Sequelize.DATE);
     queryInterface.createTable('Revisions', {
-      id: Sequelize.UUID,
+      id: {
+        type: Sequelize.UUID,
+        primaryKey: true
+      },
       noteId: Sequelize.UUID,
       patch: Sequelize.TEXT,
       lastContent: Sequelize.TEXT,
diff --git a/lib/models/index.js b/lib/models/index.js
index de6cd13c..e83956e5 100644
--- a/lib/models/index.js
+++ b/lib/models/index.js
@@ -20,6 +20,19 @@ if (config.dburl)
 else
     sequelize = new Sequelize(dbconfig.database, dbconfig.username, dbconfig.password, dbconfig);
 
+// [Postgres] Handling NULL bytes
+// https://github.com/sequelize/sequelize/issues/6485
+function stripNullByte(value) {
+    return value ? value.replace(/\u0000/g, "") : value;
+}
+sequelize.stripNullByte = stripNullByte;
+
+function processData(data, _default, process) {
+    if (data === undefined) return data;
+    else return data === null ? _default : (process ? process(data) : data);
+}
+sequelize.processData = processData;
+
 var db = {};
 
 fs
diff --git a/lib/models/note.js b/lib/models/note.js
index 5727046c..5c63dc1a 100644
--- a/lib/models/note.js
+++ b/lib/models/note.js
@@ -52,13 +52,31 @@ module.exports = function (sequelize, DataTypes) {
             defaultValue: 0
         },
         title: {
-            type: DataTypes.TEXT
+            type: DataTypes.TEXT,
+            get: function () {
+                return sequelize.processData(this.getDataValue('title'), "");
+            },
+            set: function (value) {
+                this.setDataValue('title', sequelize.stripNullByte(value));
+            }
         },
         content: {
-            type: DataTypes.TEXT
+            type: DataTypes.TEXT,
+            get: function () {
+                return sequelize.processData(this.getDataValue('content'), "");
+            },
+            set: function (value) {
+                this.setDataValue('content', sequelize.stripNullByte(value));
+            }
         },
         authorship: {
-            type: DataTypes.TEXT
+            type: DataTypes.TEXT,
+            get: function () {
+                return sequelize.processData(this.getDataValue('authorship'), [], JSON.parse);
+            },
+            set: function (value) {
+                this.setDataValue('authorship', JSON.stringify(value));
+            }
         },
         lastchangeAt: {
             type: DataTypes.DATE
@@ -124,8 +142,6 @@ module.exports = function (sequelize, DataTypes) {
                                     var body = fs.readFileSync(filePath, 'utf8');
                                     var contentLength = body.length;
                                     var title = Note.parseNoteTitle(body);
-                                    body = LZString.compressToBase64(body);
-                                    title = LZString.compressToBase64(title);
                                     if (fsModifiedTime.isAfter(dbModifiedTime) && note.content !== body) {
                                         note.update({
                                             title: title,
@@ -135,14 +151,14 @@ module.exports = function (sequelize, DataTypes) {
                                             sequelize.models.Revision.saveNoteRevision(note, function (err, revision) {
                                                 if (err) return _callback(err, null);
                                                 // update authorship on after making revision of docs
-                                                var patch = dmp.patch_fromText(LZString.decompressFromBase64(revision.patch));
+                                                var patch = dmp.patch_fromText(revision.patch);
                                                 var operations = Note.transformPatchToOperations(patch, contentLength);
-                                                var authorship = note.authorship ? JSON.parse(LZString.decompressFromBase64(note.authorship)) : [];
+                                                var authorship = note.authorship;
                                                 for (var i = 0; i < operations.length; i++) {
                                                     authorship = Note.updateAuthorshipByOperation(operations[i], null, authorship);
                                                 }
                                                 note.update({
-                                                    authorship: LZString.compressToBase64(JSON.stringify(authorship))
+                                                    authorship: JSON.stringify(authorship)
                                                 }).then(function (note) {
                                                     return callback(null, note.id);
                                                 }).catch(function (err) {
@@ -264,10 +280,7 @@ module.exports = function (sequelize, DataTypes) {
                 return markdown.substr(0, 100).replace(/(?:\r\n|\r|\n)/g, ' ');
             },
             decodeTitle: function (title) {
-                var decodedTitle = LZString.decompressFromBase64(title);
-                if (decodedTitle) title = decodedTitle;
-                else title = 'Untitled';
-                return title;
+                return title ? title : 'Untitled';
             },
             generateWebTitle: function (title) {
                 title = !title || title == "Untitled" ? "HackMD - Collaborative markdown notes" : title + " - HackMD";
@@ -496,8 +509,8 @@ module.exports = function (sequelize, DataTypes) {
                     if (Note.checkFileExist(filePath)) {
                         var fsCreatedTime = moment(fs.statSync(filePath).ctime);
                         body = fs.readFileSync(filePath, 'utf8');
-                        note.title = LZString.compressToBase64(Note.parseNoteTitle(body));
-                        note.content = LZString.compressToBase64(body);
+                        note.title = Note.parseNoteTitle(body);
+                        note.content = body;
                         if (filePath !== config.defaultnotepath) {
                             note.createdAt = fsCreatedTime;
                         }
diff --git a/lib/models/revision.js b/lib/models/revision.js
index 8b8eba94..c7360fed 100644
--- a/lib/models/revision.js
+++ b/lib/models/revision.js
@@ -2,7 +2,6 @@
 
 // external modules
 var Sequelize = require("sequelize");
-var LZString = require('lz-string');
 var async = require('async');
 var moment = require('moment');
 var childProcess = require('child_process');
@@ -60,19 +59,43 @@ module.exports = function (sequelize, DataTypes) {
             defaultValue: Sequelize.UUIDV4
         },
         patch: {
-            type: DataTypes.TEXT
+            type: DataTypes.TEXT,
+            get: function () {
+                return sequelize.processData(this.getDataValue('patch'), "");
+            },
+            set: function (value) {
+                this.setDataValue('patch', sequelize.stripNullByte(value));
+            }
         },
         lastContent: {
-            type: DataTypes.TEXT
+            type: DataTypes.TEXT,
+            get: function () {
+                return sequelize.processData(this.getDataValue('lastContent'), "");
+            },
+            set: function (value) {
+                this.setDataValue('lastContent', sequelize.stripNullByte(value));
+            }
         },
         content: {
-            type: DataTypes.TEXT
+            type: DataTypes.TEXT,
+            get: function () {
+                return sequelize.processData(this.getDataValue('content'), "");
+            },
+            set: function (value) {
+                this.setDataValue('content', sequelize.stripNullByte(value));
+            }
         },
         length: {
             type: DataTypes.INTEGER
         },
         authorship: {
-            type: DataTypes.TEXT
+            type: DataTypes.TEXT,
+            get: function () {
+                return sequelize.processData(this.getDataValue('authorship'), [], JSON.parse);
+            },
+            set: function (value) {
+                this.setDataValue('authorship', value ? JSON.stringify(value) : value);
+            }
         }
     }, {
         classMethods: {
@@ -214,7 +237,7 @@ module.exports = function (sequelize, DataTypes) {
                         Revision.create({
                             noteId: note.id,
                             lastContent: note.content,
-                            length: LZString.decompressFromBase64(note.content).length,
+                            length: note.content.length,
                             authorship: note.authorship
                         }).then(function (revision) {
                             Revision.finishSaveNoteRevision(note, revision, callback);
@@ -223,8 +246,8 @@ module.exports = function (sequelize, DataTypes) {
                         });
                     } else {
                         var latestRevision = revisions[0];
-                        var lastContent = LZString.decompressFromBase64(latestRevision.content || latestRevision.lastContent);
-                        var content = LZString.decompressFromBase64(note.content);
+                        var lastContent = latestRevision.content || latestRevision.lastContent;
+                        var content = note.content;
                         sendDmpWorker({
                             msg: 'create patch',
                             lastDoc: lastContent,
@@ -244,9 +267,9 @@ module.exports = function (sequelize, DataTypes) {
                             } else {
                                 Revision.create({
                                     noteId: note.id,
-                                    patch: LZString.compressToBase64(patch),
+                                    patch: patch,
                                     content: note.content,
-                                    length: LZString.decompressFromBase64(note.content).length,
+                                    length: note.content.length,
                                     authorship: note.authorship
                                 }).then(function (revision) {
                                     // clear last revision content to reduce db size
diff --git a/lib/ot/editor-socketio-server.js b/lib/ot/editor-socketio-server.js
index d062fa19..7b204539 100755
--- a/lib/ot/editor-socketio-server.js
+++ b/lib/ot/editor-socketio-server.js
@@ -7,7 +7,6 @@ var Server = require('./server');
 var Selection = require('./selection');
 var util = require('util');
 
-var LZString = require('lz-string');
 var logger = require('../logger');
 
 function EditorSocketIOServer(document, operations, docId, mayWrite, operationCallback) {
@@ -40,10 +39,8 @@ EditorSocketIOServer.prototype.addClient = function (socket) {
         revision: this.operations.length,
         clients: this.users
     };
-    socket.emit('doc', LZString.compressToUTF16(JSON.stringify(docOut)));
+    socket.emit('doc', docOut);
     socket.on('operation', function (revision, operation, selection) {
-        operation = LZString.decompressFromUTF16(operation);
-        operation = JSON.parse(operation);
         socket.origin = 'operation';
         self.mayWrite(socket, function (mayWrite) {
             if (!mayWrite) {
@@ -62,7 +59,7 @@ EditorSocketIOServer.prototype.addClient = function (socket) {
                         clients: self.users,
                         force: true
                     };
-                    socket.emit('doc', LZString.compressToUTF16(JSON.stringify(docOut)));
+                    socket.emit('doc', docOut);
                 }, 100);
             }
         });
@@ -129,7 +126,6 @@ EditorSocketIOServer.prototype.onGetOperations = function (socket, base, head) {
     var operations = this.operations.slice(base, head).map(function (op) {
         return op.wrapped.toJSON();
     });
-    operations = LZString.compressToUTF16(JSON.stringify(operations));
     socket.emit('operations', head, operations);
 };
 
diff --git a/lib/realtime.js b/lib/realtime.js
index c243ffc1..a662deeb 100644
--- a/lib/realtime.js
+++ b/lib/realtime.js
@@ -71,7 +71,6 @@ function emitCheck(note) {
         authors: note.authors,
         authorship: note.authorship
     };
-    out = LZString.compressToUTF16(JSON.stringify(out));
     realtime.io.to(note.id).emit('check', out);
 }
 
@@ -153,12 +152,10 @@ function finishUpdateNote(note, _note, callback) {
     if (!note || !note.server) return callback(null, null);
     var body = note.server.document;
     var title = note.title = models.Note.parseNoteTitle(body);
-    title = LZString.compressToBase64(title);
-    body = LZString.compressToBase64(body);
     var values = {
         title: title,
         content: body,
-        authorship: LZString.compressToBase64(JSON.stringify(note.authorship)),
+        authorship: note.authorship,
         lastchangeuserId: note.lastchangeuser,
         lastchangeAt: Date.now()
     };
@@ -301,7 +298,6 @@ function emitOnlineUsers(socket) {
     var out = {
         users: users
     };
-    out = LZString.compressToUTF16(JSON.stringify(out));
     realtime.io.to(noteId).emit('online users', out);
 }
 
@@ -330,7 +326,6 @@ function emitRefresh(socket) {
         createtime: note.createtime,
         updatetime: note.updatetime
     };
-    out = LZString.compressToUTF16(JSON.stringify(out));
     socket.emit('refresh', out);
 }
 
@@ -462,7 +457,7 @@ function startConnection(socket) {
             var lastchangeuser = note.lastchangeuserId;
             var lastchangeuserprofile = note.lastchangeuser ? models.User.getProfile(note.lastchangeuser) : null;
 
-            var body = LZString.decompressFromBase64(note.content);
+            var body = note.content;
             var createtime = note.createdAt;
             var updatetime = note.lastchangeAt;
             var server = new ot.EditorSocketIOServer(body, [], noteId, ifMayEdit, operationCallback);
@@ -482,7 +477,7 @@ function startConnection(socket) {
             notes[noteId] = {
                 id: noteId,
                 alias: note.alias,
-                title: LZString.decompressFromBase64(note.title),
+                title: note.title,
                 owner: owner,
                 ownerprofile: ownerprofile,
                 permission: note.permission,
@@ -494,7 +489,7 @@ function startConnection(socket) {
                 updatetime: moment(updatetime).valueOf(),
                 server: server,
                 authors: authors,
-                authorship: note.authorship ? JSON.parse(LZString.decompressFromBase64(note.authorship)) : []
+                authorship: note.authorship
             };
 
             return finishConnection(socket, notes[noteId], users[socket.id]);
@@ -863,7 +858,6 @@ function connection(socket) {
         var out = {
             users: users
         };
-        out = LZString.compressToUTF16(JSON.stringify(out));
         socket.emit('online users', out);
     });
 
diff --git a/lib/response.js b/lib/response.js
index 2b38cf25..54e2a337 100755
--- a/lib/response.js
+++ b/lib/response.js
@@ -75,7 +75,7 @@ function showIndex(req, res, next) {
 }
 
 function responseHackMD(res, note) {
-    var body = LZString.decompressFromBase64(note.content);
+    var body = note.content;
     var meta = null;
     try {
         meta = models.Note.parseMeta(metaMarked(body).meta);
@@ -191,7 +191,7 @@ function showPublishNote(req, res, next) {
             if (!note) {
                 return response.errorNotFound(res);
             }
-            var body = LZString.decompressFromBase64(note.content);
+            var body = note.content;
             var meta = null;
             var markdown = null;
             try {
@@ -209,7 +209,7 @@ function showPublishNote(req, res, next) {
             var origin = config.serverurl;
             var data = {
                 title: title,
-                description: meta.description || markdown ? models.Note.generateDescription(markdown) : null,
+                description: meta.description || (markdown ? models.Note.generateDescription(markdown) : null),
                 viewcount: note.viewcount,
                 createtime: createtime,
                 updatetime: updatetime,
@@ -248,7 +248,7 @@ function actionSlide(req, res, note) {
 }
 
 function actionDownload(req, res, note) {
-    var body = LZString.decompressFromBase64(note.content);
+    var body = note.content;
     var title = models.Note.decodeTitle(note.title);
     var filename = title;
     filename = encodeURIComponent(filename);
@@ -265,7 +265,7 @@ function actionDownload(req, res, note) {
 }
 
 function actionInfo(req, res, note) {
-    var body = LZString.decompressFromBase64(note.content);
+    var body = note.content;
     var meta = null;
     var markdown = null;
     try {
@@ -281,7 +281,7 @@ function actionInfo(req, res, note) {
     var title = models.Note.decodeTitle(note.title);
     var data = {
         title: meta.title || title,
-        description: meta.description || markdown ? models.Note.generateDescription(markdown) : null,
+        description: meta.description || (markdown ? models.Note.generateDescription(markdown) : null),
         viewcount: note.viewcount,
         createtime: createtime,
         updatetime: updatetime
@@ -297,7 +297,7 @@ function actionInfo(req, res, note) {
 }
 
 function actionPDF(req, res, note) {
-    var body = LZString.decompressFromBase64(note.content);
+    var body = note.content;
     try {
         body = metaMarked(body).markdown;
     } catch(err) {
@@ -479,7 +479,7 @@ function githubActionGist(req, res, note) {
             if (!error && httpResponse.statusCode == 200) {
                 var access_token = body.access_token;
                 if (access_token) {
-                    var content = LZString.decompressFromBase64(note.content);
+                    var content = note.content;
                     var title = models.Note.decodeTitle(note.title);
                     var filename = title.replace('/', ' ') + '.md';
                     var gist = {
@@ -579,7 +579,7 @@ function showPublishSlide(req, res, next) {
             if (!note) {
                 return response.errorNotFound(res);
             }
-            var body = LZString.decompressFromBase64(note.content);
+            var body = note.content;
             var meta = null;
             var markdown = null;
             try {
@@ -597,7 +597,7 @@ function showPublishSlide(req, res, next) {
             var origin = config.serverurl;
             var data = {
                 title: title,
-                description: meta.description || markdown ? models.Note.generateDescription(markdown) : null,
+                description: meta.description || (markdown ? models.Note.generateDescription(markdown) : null),
                 viewcount: note.viewcount,
                 createtime: createtime,
                 updatetime: updatetime,
diff --git a/lib/workers/dmpWorker.js b/lib/workers/dmpWorker.js
index fae36191..8e69636e 100644
--- a/lib/workers/dmpWorker.js
+++ b/lib/workers/dmpWorker.js
@@ -1,5 +1,4 @@
 // external modules
-var LZString = require('lz-string');
 var DiffMatchPatch = require('diff-match-patch');
 var dmp = new DiffMatchPatch();
 
@@ -58,7 +57,6 @@ process.on('message', function(data) {
 function createPatch(lastDoc, currDoc) {
     var ms_start = (new Date()).getTime();
     var diff = dmp.diff_main(lastDoc, currDoc);
-    dmp.diff_cleanupSemantic(diff);
     var patch = dmp.patch_make(lastDoc, diff);
     patch = dmp.patch_toText(patch);
     var ms_end = (new Date()).getTime();
@@ -80,10 +78,10 @@ function getRevision(revisions, count) {
         for (var i = 0; i < count; i++) {
             var revision = revisions[i];
             if (i == 0) {
-                startContent = LZString.decompressFromBase64(revision.content || revision.lastContent);
+                startContent = revision.content || revision.lastContent;
             }
             if (i != count - 1) {
-                var patch = dmp.patch_fromText(LZString.decompressFromBase64(revision.patch));
+                var patch = dmp.patch_fromText(revision.patch);
                 applyPatches = applyPatches.concat(patch);
             }
             lastPatch = revision.patch;
@@ -105,11 +103,11 @@ function getRevision(revisions, count) {
         for (var i = l; i >= count - 1; i--) {
             var revision = revisions[i];
             if (i == l) {
-                startContent = LZString.decompressFromBase64(revision.lastContent);
+                startContent = revision.lastContent;
                 authorship = revision.authorship;
             }
             if (revision.patch) {
-                var patch = dmp.patch_fromText(LZString.decompressFromBase64(revision.patch));
+                var patch = dmp.patch_fromText(revision.patch);
                 applyPatches = applyPatches.concat(patch);
             }
             lastPatch = revision.patch;
@@ -123,8 +121,8 @@ function getRevision(revisions, count) {
     }
     var data = {
         content: finalContent,
-        patch: dmp.patch_fromText(LZString.decompressFromBase64(lastPatch)),
-        authorship: authorship ? JSON.parse(LZString.decompressFromBase64(authorship)) : null
+        patch: dmp.patch_fromText(lastPatch),
+        authorship: authorship
     };
     var ms_end = (new Date()).getTime();
     if (config.debug) {