From baa0418fb54fb8f158267f8e8b5f248232dc0a8f Mon Sep 17 00:00:00 2001 From: Max Wu Date: Mon, 26 Feb 2018 16:43:29 +0800 Subject: Remove and replace all note id compression in LZString with base64url Signed-off-by: Max Wu --- lib/models/note.js | 27 +++++++++++++++++++++++++++ lib/realtime.js | 3 +-- lib/response.js | 11 +++++------ 3 files changed, 33 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/models/note.js b/lib/models/note.js index 484f1a8c..e199a3db 100644 --- a/lib/models/note.js +++ b/lib/models/note.js @@ -3,6 +3,7 @@ var fs = require('fs') var path = require('path') var LZString = require('lz-string') +var base64url = require('base64url') var md = require('markdown-it')() var metaMarked = require('meta-marked') var cheerio = require('cheerio') @@ -114,6 +115,22 @@ module.exports = function (sequelize, DataTypes) { return false } }, + encodeNoteId: function (id) { + // remove dashes in UUID and encode in url-safe base64 + return base64url.encode(id.replace(/-/g, '')) + }, + decodeNoteId: function (encodedId) { + // decode from url-safe base64 + let id = base64url.decode(encodedId) + // add dashes between the UUID string parts + let idParts = [] + idParts.push(id.substr(0, 8)) + idParts.push(id.substr(8, 4)) + idParts.push(id.substr(12, 4)) + idParts.push(id.substr(16, 4)) + idParts.push(id.substr(20, 12)) + return idParts.join('-') + }, checkNoteIdValid: function (id) { var uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i var result = id.match(uuidRegex) @@ -190,6 +207,16 @@ module.exports = function (sequelize, DataTypes) { return _callback(err, null) }) }, + parseNoteIdByBase64Url: function (_callback) { + // try to parse note id by base64url + try { + var id = Note.decodeNoteId(noteId) + if (id && Note.checkNoteIdValid(id)) { return callback(null, id) } else { return _callback(null, null) } + } catch (err) { + return _callback(err, null) + } + }, + // parse note id by LZString is deprecated, here for compability parseNoteIdByLZString: function (_callback) { // try to parse note id by LZString Base64 try { diff --git a/lib/realtime.js b/lib/realtime.js index d6ba62b2..5ee9f8fd 100644 --- a/lib/realtime.js +++ b/lib/realtime.js @@ -5,7 +5,6 @@ var cookie = require('cookie') var cookieParser = require('cookie-parser') var url = require('url') var async = require('async') -var LZString = require('lz-string') var randomcolor = require('randomcolor') var Chance = require('chance') var chance = new Chance() @@ -703,7 +702,7 @@ function operationCallback (socket, operation) { } function updateHistory (userId, note, time) { - var noteId = note.alias ? note.alias : LZString.compressToBase64(note.id) + var noteId = note.alias ? note.alias : models.Note.encodeNoteId(note.id) if (note.server) history.updateHistory(userId, noteId, note.server.document, time) } diff --git a/lib/response.js b/lib/response.js index 41e8c336..25b9fafc 100644 --- a/lib/response.js +++ b/lib/response.js @@ -3,7 +3,6 @@ // external modules var fs = require('fs') var markdownpdf = require('markdown-pdf') -var LZString = require('lz-string') var shortId = require('shortid') var querystring = require('querystring') var request = require('request') @@ -124,7 +123,7 @@ function newNote (req, res, next) { alias: req.alias ? req.alias : null, content: req.body ? req.body : '' }).then(function (note) { - return res.redirect(config.serverurl + '/' + LZString.compressToBase64(note.id)) + return res.redirect(config.serverurl + '/' + models.Note.encodeNoteId(note.id)) }).catch(function (err) { logger.error(err) return response.errorInternalError(res) @@ -179,7 +178,7 @@ function showNote (req, res, next) { findNote(req, res, function (note) { // force to use note id var noteId = req.params.noteId - var id = LZString.compressToBase64(note.id) + var id = models.Note.encodeNoteId(note.id) if ((note.alias && noteId !== note.alias) || (!note.alias && noteId !== id)) { return res.redirect(config.serverurl + '/' + (note.alias || id)) } return responseHackMD(res, note) }) @@ -321,7 +320,7 @@ function actionPDF (req, res, note) { function actionGist (req, res, note) { var data = { client_id: config.github.clientID, - redirect_uri: config.serverurl + '/auth/github/callback/' + LZString.compressToBase64(note.id) + '/gist', + redirect_uri: config.serverurl + '/auth/github/callback/' + models.Note.encodeNoteId(note.id) + '/gist', scope: 'gist', state: shortId.generate() } @@ -418,7 +417,7 @@ function publishNoteActions (req, res, next) { var action = req.params.action switch (action) { case 'edit': - res.redirect(config.serverurl + '/' + (note.alias ? note.alias : LZString.compressToBase64(note.id))) + res.redirect(config.serverurl + '/' + (note.alias ? note.alias : models.Note.encodeNoteId(note.id))) break default: res.redirect(config.serverurl + '/s/' + note.shortid) @@ -432,7 +431,7 @@ function publishSlideActions (req, res, next) { var action = req.params.action switch (action) { case 'edit': - res.redirect(config.serverurl + '/' + (note.alias ? note.alias : LZString.compressToBase64(note.id))) + res.redirect(config.serverurl + '/' + (note.alias ? note.alias : models.Note.encodeNoteId(note.id))) break default: res.redirect(config.serverurl + '/p/' + note.shortid) -- cgit v1.2.3 From 44298baa935916c61d8402122ed5801b1d973acd Mon Sep 17 00:00:00 2001 From: Max Wu Date: Mon, 26 Feb 2018 16:46:59 +0800 Subject: Add migration for LZString compressed note id in history Signed-off-by: Max Wu --- lib/history.js | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/history.js b/lib/history.js index f46ff49f..f3d4440e 100644 --- a/lib/history.js +++ b/lib/history.js @@ -1,6 +1,7 @@ 'use strict' // history // external modules +var LZString = require('lz-string') // core var config = require('./config') @@ -27,7 +28,26 @@ function getHistory (userid, callback) { } var history = {} if (user.history) { - history = parseHistoryToObject(JSON.parse(user.history)) + history = JSON.parse(user.history) + // migrate LZString encoded note id to base64url encoded note id + for (let i = 0, l = history.length; i < l; i++) { + let item = history[i] + // try to parse in base64url + let id = models.Note.decodeNoteId(item.id) + if (!id || !models.Note.checkNoteIdValid(id)) { + // try to parse in LZString if it can't be parsed in base64url + try { + id = LZString.decompressFromBase64(item.id) + } catch (err) { + id = null + } + if (id && models.Note.checkNoteIdValid(id)) { + // replace the note id to base64url encoded note id + history[i].id = models.Note.encodeNoteId(id) + } + } + } + history = parseHistoryToObject(history) } if (config.debug) { logger.info('read history success: ' + user.id) -- cgit v1.2.3 From fe429e9ac17b73638835b2ec1c5033043c5f9942 Mon Sep 17 00:00:00 2001 From: Max Wu Date: Tue, 27 Feb 2018 20:57:31 +0800 Subject: Update to use buffer in encode/decode note id Signed-off-by: Max Wu --- lib/models/note.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/models/note.js b/lib/models/note.js index e199a3db..119d72c3 100644 --- a/lib/models/note.js +++ b/lib/models/note.js @@ -117,11 +117,13 @@ module.exports = function (sequelize, DataTypes) { }, encodeNoteId: function (id) { // remove dashes in UUID and encode in url-safe base64 - return base64url.encode(id.replace(/-/g, '')) + let str = id.replace(/-/g, '') + let hexStr = Buffer.from(str, 'hex') + return base64url.encode(hexStr) }, decodeNoteId: function (encodedId) { // decode from url-safe base64 - let id = base64url.decode(encodedId) + let id = base64url.toBuffer(encodedId).toString('hex') // add dashes between the UUID string parts let idParts = [] idParts.push(id.substr(0, 8)) -- cgit v1.2.3 From 16cb842b946d55668318c08cf2e0aed001b9f855 Mon Sep 17 00:00:00 2001 From: Max Wu Date: Sat, 10 Mar 2018 16:51:00 +0800 Subject: Improve history migration performance Signed-off-by: Max Wu --- lib/history.js | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/history.js b/lib/history.js index f3d4440e..c7d2472c 100644 --- a/lib/history.js +++ b/lib/history.js @@ -31,20 +31,14 @@ function getHistory (userid, callback) { history = JSON.parse(user.history) // migrate LZString encoded note id to base64url encoded note id for (let i = 0, l = history.length; i < l; i++) { - let item = history[i] - // try to parse in base64url - let id = models.Note.decodeNoteId(item.id) - if (!id || !models.Note.checkNoteIdValid(id)) { - // try to parse in LZString if it can't be parsed in base64url - try { - id = LZString.decompressFromBase64(item.id) - } catch (err) { - id = null - } + try { + let id = LZString.decompressFromBase64(history[i].id) if (id && models.Note.checkNoteIdValid(id)) { - // replace the note id to base64url encoded note id history[i].id = models.Note.encodeNoteId(id) } + } catch (err) { + // most error here comes from LZString, ignore + logger.error(err) } } history = parseHistoryToObject(history) -- cgit v1.2.3 From c7657ae81e23102cedd31543ee111d2736dc3b22 Mon Sep 17 00:00:00 2001 From: Max Wu Date: Sat, 10 Mar 2018 16:52:24 +0800 Subject: Fix parseNoteId order to fix some edge case that LZString note url could be parsed by base64url note url and thus return wrong note id Signed-off-by: Max Wu --- lib/models/note.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/models/note.js b/lib/models/note.js index 119d72c3..dc4d187b 100644 --- a/lib/models/note.js +++ b/lib/models/note.js @@ -209,20 +209,20 @@ module.exports = function (sequelize, DataTypes) { return _callback(err, null) }) }, - parseNoteIdByBase64Url: function (_callback) { - // try to parse note id by base64url + // parse note id by LZString is deprecated, here for compability + parseNoteIdByLZString: function (_callback) { + // try to parse note id by LZString Base64 try { - var id = Note.decodeNoteId(noteId) + var id = LZString.decompressFromBase64(noteId) if (id && Note.checkNoteIdValid(id)) { return callback(null, id) } else { return _callback(null, null) } } catch (err) { return _callback(err, null) } }, - // parse note id by LZString is deprecated, here for compability - parseNoteIdByLZString: function (_callback) { - // try to parse note id by LZString Base64 + parseNoteIdByBase64Url: function (_callback) { + // try to parse note id by base64url try { - var id = LZString.decompressFromBase64(noteId) + var id = Note.decodeNoteId(noteId) if (id && Note.checkNoteIdValid(id)) { return callback(null, id) } else { return _callback(null, null) } } catch (err) { return _callback(err, null) -- cgit v1.2.3 From 5e975cbe690048e144fb6bd99c5b239a3e764445 Mon Sep 17 00:00:00 2001 From: Max Wu Date: Sun, 11 Mar 2018 02:52:24 +0800 Subject: Fix to log instead of throwing error on parse note id Signed-off-by: Max Wu --- lib/models/note.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/models/note.js b/lib/models/note.js index dc4d187b..d615bcf7 100644 --- a/lib/models/note.js +++ b/lib/models/note.js @@ -216,7 +216,8 @@ module.exports = function (sequelize, DataTypes) { var id = LZString.decompressFromBase64(noteId) if (id && Note.checkNoteIdValid(id)) { return callback(null, id) } else { return _callback(null, null) } } catch (err) { - return _callback(err, null) + logger.error(err) + return _callback(null, null) } }, parseNoteIdByBase64Url: function (_callback) { @@ -225,7 +226,8 @@ module.exports = function (sequelize, DataTypes) { var id = Note.decodeNoteId(noteId) if (id && Note.checkNoteIdValid(id)) { return callback(null, id) } else { return _callback(null, null) } } catch (err) { - return _callback(err, null) + logger.error(err) + return _callback(null, null) } }, parseNoteIdByShortId: function (_callback) { -- cgit v1.2.3