From f6d8e3ab00370a78c0c788ad1e37a7ff77a53555 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 2 Jan 2017 10:59:53 +0800 Subject: Remove LZString compression for data storage --- lib/models/note.js | 17 ++++++----------- lib/models/revision.js | 11 +++++------ 2 files changed, 11 insertions(+), 17 deletions(-) (limited to 'lib/models') diff --git a/lib/models/note.js b/lib/models/note.js index 5727046c..81de991f 100644 --- a/lib/models/note.js +++ b/lib/models/note.js @@ -124,8 +124,6 @@ module.exports = function (sequelize, DataTypes) { var body = fs.readFileSync(filePath, 'utf8'); var contentLength = body.length; var title = Note.parseNoteTitle(body); - body = LZString.compressToBase64(body); - title = LZString.compressToBase64(title); if (fsModifiedTime.isAfter(dbModifiedTime) && note.content !== body) { note.update({ title: title, @@ -135,14 +133,14 @@ module.exports = function (sequelize, DataTypes) { sequelize.models.Revision.saveNoteRevision(note, function (err, revision) { if (err) return _callback(err, null); // update authorship on after making revision of docs - var patch = dmp.patch_fromText(LZString.decompressFromBase64(revision.patch)); + var patch = dmp.patch_fromText(revision.patch); var operations = Note.transformPatchToOperations(patch, contentLength); - var authorship = note.authorship ? JSON.parse(LZString.decompressFromBase64(note.authorship)) : []; + var authorship = note.authorship; for (var i = 0; i < operations.length; i++) { authorship = Note.updateAuthorshipByOperation(operations[i], null, authorship); } note.update({ - authorship: LZString.compressToBase64(JSON.stringify(authorship)) + authorship: JSON.stringify(authorship) }).then(function (note) { return callback(null, note.id); }).catch(function (err) { @@ -264,10 +262,7 @@ module.exports = function (sequelize, DataTypes) { return markdown.substr(0, 100).replace(/(?:\r\n|\r|\n)/g, ' '); }, decodeTitle: function (title) { - var decodedTitle = LZString.decompressFromBase64(title); - if (decodedTitle) title = decodedTitle; - else title = 'Untitled'; - return title; + return title ? title : 'Untitled'; }, generateWebTitle: function (title) { title = !title || title == "Untitled" ? "HackMD - Collaborative markdown notes" : title + " - HackMD"; @@ -496,8 +491,8 @@ module.exports = function (sequelize, DataTypes) { if (Note.checkFileExist(filePath)) { var fsCreatedTime = moment(fs.statSync(filePath).ctime); body = fs.readFileSync(filePath, 'utf8'); - note.title = LZString.compressToBase64(Note.parseNoteTitle(body)); - note.content = LZString.compressToBase64(body); + note.title = Note.parseNoteTitle(body); + note.content = body; if (filePath !== config.defaultnotepath) { note.createdAt = fsCreatedTime; } diff --git a/lib/models/revision.js b/lib/models/revision.js index 8b8eba94..6f44cf1d 100644 --- a/lib/models/revision.js +++ b/lib/models/revision.js @@ -2,7 +2,6 @@ // external modules var Sequelize = require("sequelize"); -var LZString = require('lz-string'); var async = require('async'); var moment = require('moment'); var childProcess = require('child_process'); @@ -214,7 +213,7 @@ module.exports = function (sequelize, DataTypes) { Revision.create({ noteId: note.id, lastContent: note.content, - length: LZString.decompressFromBase64(note.content).length, + length: note.content.length, authorship: note.authorship }).then(function (revision) { Revision.finishSaveNoteRevision(note, revision, callback); @@ -223,8 +222,8 @@ module.exports = function (sequelize, DataTypes) { }); } else { var latestRevision = revisions[0]; - var lastContent = LZString.decompressFromBase64(latestRevision.content || latestRevision.lastContent); - var content = LZString.decompressFromBase64(note.content); + var lastContent = latestRevision.content || latestRevision.lastContent; + var content = note.content; sendDmpWorker({ msg: 'create patch', lastDoc: lastContent, @@ -244,9 +243,9 @@ module.exports = function (sequelize, DataTypes) { } else { Revision.create({ noteId: note.id, - patch: LZString.compressToBase64(patch), + patch: patch, content: note.content, - length: LZString.decompressFromBase64(note.content).length, + length: note.content.length, authorship: note.authorship }).then(function (revision) { // clear last revision content to reduce db size -- cgit v1.2.3 From d9e19b602968551fcda4ee806d767a04f6a11490 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 2 Jan 2017 11:05:05 +0800 Subject: Update to remove null byte before saving to DB and remove null byte on changes --- lib/models/index.js | 7 +++++++ lib/models/note.js | 15 ++++++++++++--- lib/models/revision.js | 20 ++++++++++++++++---- 3 files changed, 35 insertions(+), 7 deletions(-) (limited to 'lib/models') diff --git a/lib/models/index.js b/lib/models/index.js index de6cd13c..6d0fd3c3 100644 --- a/lib/models/index.js +++ b/lib/models/index.js @@ -20,6 +20,13 @@ if (config.dburl) else sequelize = new Sequelize(dbconfig.database, dbconfig.username, dbconfig.password, dbconfig); +// [Postgres] Handling NULL bytes +// https://github.com/sequelize/sequelize/issues/6485 +function stripNullByte(value) { + return value ? value.replace(/\u0000/g, "") : value; +} +sequelize.stripNullByte = stripNullByte; + var db = {}; fs diff --git a/lib/models/note.js b/lib/models/note.js index 81de991f..37d26ec0 100644 --- a/lib/models/note.js +++ b/lib/models/note.js @@ -52,13 +52,22 @@ module.exports = function (sequelize, DataTypes) { defaultValue: 0 }, title: { - type: DataTypes.TEXT + type: DataTypes.TEXT, + set: function (value) { + this.setDataValue('title', sequelize.stripNullByte(value)); + } }, content: { - type: DataTypes.TEXT + type: DataTypes.TEXT, + set: function (value) { + this.setDataValue('content', sequelize.stripNullByte(value)); + } }, authorship: { - type: DataTypes.TEXT + type: DataTypes.TEXT, + set: function (value) { + this.setDataValue('authorship', JSON.stringify(value)); + } }, lastchangeAt: { type: DataTypes.DATE diff --git a/lib/models/revision.js b/lib/models/revision.js index 6f44cf1d..adc651f1 100644 --- a/lib/models/revision.js +++ b/lib/models/revision.js @@ -59,19 +59,31 @@ module.exports = function (sequelize, DataTypes) { defaultValue: Sequelize.UUIDV4 }, patch: { - type: DataTypes.TEXT + type: DataTypes.TEXT, + set: function (value) { + this.setDataValue('patch', sequelize.stripNullByte(value)); + } }, lastContent: { - type: DataTypes.TEXT + type: DataTypes.TEXT, + set: function (value) { + this.setDataValue('lastContent', sequelize.stripNullByte(value)); + } }, content: { - type: DataTypes.TEXT + type: DataTypes.TEXT, + set: function (value) { + this.setDataValue('content', sequelize.stripNullByte(value)); + } }, length: { type: DataTypes.INTEGER }, authorship: { - type: DataTypes.TEXT + type: DataTypes.TEXT, + set: function (value) { + this.setDataValue('authorship', value ? JSON.stringify(value) : value); + } } }, { classMethods: { -- cgit v1.2.3 From b1ec3ba748dc7b58015e97448958e78b512f6df0 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 2 Jan 2017 11:05:36 +0800 Subject: Refactor data processing to model definition --- lib/models/index.js | 6 ++++++ lib/models/note.js | 9 +++++++++ lib/models/revision.js | 12 ++++++++++++ 3 files changed, 27 insertions(+) (limited to 'lib/models') diff --git a/lib/models/index.js b/lib/models/index.js index 6d0fd3c3..e83956e5 100644 --- a/lib/models/index.js +++ b/lib/models/index.js @@ -27,6 +27,12 @@ function stripNullByte(value) { } sequelize.stripNullByte = stripNullByte; +function processData(data, _default, process) { + if (data === undefined) return data; + else return data === null ? _default : (process ? process(data) : data); +} +sequelize.processData = processData; + var db = {}; fs diff --git a/lib/models/note.js b/lib/models/note.js index 37d26ec0..5c63dc1a 100644 --- a/lib/models/note.js +++ b/lib/models/note.js @@ -53,18 +53,27 @@ module.exports = function (sequelize, DataTypes) { }, title: { type: DataTypes.TEXT, + get: function () { + return sequelize.processData(this.getDataValue('title'), ""); + }, set: function (value) { this.setDataValue('title', sequelize.stripNullByte(value)); } }, content: { type: DataTypes.TEXT, + get: function () { + return sequelize.processData(this.getDataValue('content'), ""); + }, set: function (value) { this.setDataValue('content', sequelize.stripNullByte(value)); } }, authorship: { type: DataTypes.TEXT, + get: function () { + return sequelize.processData(this.getDataValue('authorship'), [], JSON.parse); + }, set: function (value) { this.setDataValue('authorship', JSON.stringify(value)); } diff --git a/lib/models/revision.js b/lib/models/revision.js index adc651f1..c7360fed 100644 --- a/lib/models/revision.js +++ b/lib/models/revision.js @@ -60,18 +60,27 @@ module.exports = function (sequelize, DataTypes) { }, patch: { type: DataTypes.TEXT, + get: function () { + return sequelize.processData(this.getDataValue('patch'), ""); + }, set: function (value) { this.setDataValue('patch', sequelize.stripNullByte(value)); } }, lastContent: { type: DataTypes.TEXT, + get: function () { + return sequelize.processData(this.getDataValue('lastContent'), ""); + }, set: function (value) { this.setDataValue('lastContent', sequelize.stripNullByte(value)); } }, content: { type: DataTypes.TEXT, + get: function () { + return sequelize.processData(this.getDataValue('content'), ""); + }, set: function (value) { this.setDataValue('content', sequelize.stripNullByte(value)); } @@ -81,6 +90,9 @@ module.exports = function (sequelize, DataTypes) { }, authorship: { type: DataTypes.TEXT, + get: function () { + return sequelize.processData(this.getDataValue('authorship'), [], JSON.parse); + }, set: function (value) { this.setDataValue('authorship', value ? JSON.stringify(value) : value); } -- cgit v1.2.3