libretic/etherpad-lite
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #3559 from raybellis/async-PR

Browse source 
With this commit, that closes #3540, we pay the first big slice of our technical
debt. In this line of work we streamlined the code base, reducing its size by
15-20% and making it more understandable at the same time.

The changes were audited and tested collaboratively and are deemed sufficiently
stable for being merged.

Known issues:
- plugin compatibility is still not perfect
- the error handling path needs to be improved

This is an important day for Etherpad: thanks, Ray!
This commit is contained in:
muxator 2019-03-07 02:04:29 +01:00 committed by GitHub
commit 4c45ac3cb1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
40 changed files with 3984 additions and 5844 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

213
bin/checkAllPads.js
View file

@ -1,145 +1,94 @@
/*
This is a debug tool. It checks all revisions for data corruption
*/
* This is a debug tool. It checks all revisions for data corruption
*/
if(process.argv.length != 2)
{
if (process.argv.length != 2) {
console.error("Use: node bin/checkAllPads.js");
process.exit(1);
}
//initialize the variables
var db, settings, padManager;
var npm = require("../src/node_modules/npm");
var async = require("../src/node_modules/async");
// load and initialize NPM
let npm = require('../src/node_modules/npm');
npm.load({}, async function() {
var Changeset = require("../src/static/js/Changeset");
try {
// initialize the database
let settings = require('../src/node/utils/Settings');
let db = require('../src/node/db/DB');
await db.init();
async.series([
//load npm
function(callback) {
npm.load({}, callback);
},
//load modules
function(callback) {
settings = require('../src/node/utils/Settings');
db = require('../src/node/db/DB');
// load modules
let Changeset = require('../src/static/js/Changeset');
let padManager = require('../src/node/db/PadManager');
//initialize the database
db.init(callback);
},
//load pads
function (callback)
{
padManager = require('../src/node/db/PadManager');
padManager.listAllPads(function(err, res)
{
padIds = res.padIDs;
callback(err);
});
},
function (callback)
{
async.forEach(padIds, function(padId, callback)
{
padManager.getPad(padId, function(err, pad) {
if (err) {
callback(err);
}
//check if the pad has a pool
if(pad.pool === undefined )
{
console.error("[" + pad.id + "] Missing attribute pool");
callback();
return;
}
// get all pads
let res = await padManager.listAllPads();
//create an array with key kevisions
//key revisions always save the full pad atext
var head = pad.getHeadRevisionNumber();
var keyRevisions = [];
for(var i=0;i<head;i+=100)
{
keyRevisions.push(i);
}
//run trough all key revisions
async.forEachSeries(keyRevisions, function(keyRev, callback)
{
//create an array of revisions we need till the next keyRevision or the End
var revisionsNeeded = [];
for(var i=keyRev;i<=keyRev+100 && i<=head; i++)
{
revisionsNeeded.push(i);
}
//this array will hold all revision changesets
var revisions = [];
//run trough all needed revisions and get them from the database
async.forEach(revisionsNeeded, function(revNum, callback)
{
db.db.get("pad:"+pad.id+":revs:" + revNum, function(err, revision)
{
revisions[revNum] = revision;
callback(err);
});
}, function(err)
{
if(err)
{
callback(err);
return;
}
for (let padId of res.padIDs) {
//check if the revision exists
if (revisions[keyRev] == null) {
console.error("[" + pad.id + "] Missing revision " + keyRev);
callback();
return;
}
//check if there is a atext in the keyRevisions
if(revisions[keyRev].meta === undefined || revisions[keyRev].meta.atext === undefined)
{
console.error("[" + pad.id + "] Missing atext in revision " + keyRev);
callback();
return;
}
var apool = pad.pool;
var atext = revisions[keyRev].meta.atext;
for(var i=keyRev+1;i<=keyRev+100 && i<=head; i++)
{
try
{
//console.log("[" + pad.id + "] check revision " + i);
var cs = revisions[i].changeset;
atext = Changeset.applyToAText(cs, atext, apool);
}
catch(e)
{
console.error("[" + pad.id + "] Bad changeset at revision " + i + " - " + e.message);
callback();
return;
}
}
callback();
});
}, callback);
});
}, callback);
}
], function (err)
{
if(err) throw err;
else
{
console.log("finished");
process.exit(0);
let pad = await padManager.getPad(padId);
// check if the pad has a pool
if (pad.pool === undefined) {
console.error("[" + pad.id + "] Missing attribute pool");
continue;
}
// create an array with key kevisions
// key revisions always save the full pad atext
let head = pad.getHeadRevisionNumber();
let keyRevisions = [];
for (let rev = 0; rev < head; rev += 100) {
keyRevisions.push(rev);
}
// run through all key revisions
for (let keyRev of keyRevisions) {
// create an array of revisions we need till the next keyRevision or the End
var revisionsNeeded = [];
for (let rev = keyRev ; rev <= keyRev + 100 && rev <= head; rev++) {
revisionsNeeded.push(rev);
}
// this array will hold all revision changesets
var revisions = [];
// run through all needed revisions and get them from the database
for (let revNum of revisionsNeeded) {
let revision = await db.get("pad:" + pad.id + ":revs:" + revNum);
revisions[revNum] = revision;
}
// check if the revision exists
if (revisions[keyRev] == null) {
console.error("[" + pad.id + "] Missing revision " + keyRev);
continue;
}
// check if there is a atext in the keyRevisions
if (revisions[keyRev].meta === undefined || revisions[keyRev].meta.atext === undefined) {
console.error("[" + pad.id + "] Missing atext in revision " + keyRev);
continue;
}
let apool = pad.pool;
let atext = revisions[keyRev].meta.atext;
for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) {
try {
let cs = revisions[rev].changeset;
atext = Changeset.applyToAText(cs, atext, apool);
} catch (e) {
console.error("[" + pad.id + "] Bad changeset at revision " + i + " - " + e.message);
}
}
}
console.log("finished");
process.exit(0);
}
} catch (err) {
console.trace(err);
process.exit(1);
}
});

206
bin/checkPad.js
View file

@ -1,141 +1,95 @@
/*
This is a debug tool. It checks all revisions for data corruption
*/
* This is a debug tool. It checks all revisions for data corruption
*/
if(process.argv.length != 3)
{
if (process.argv.length != 3) {
console.error("Use: node bin/checkPad.js $PADID");
process.exit(1);
}
//get the padID
var padId = process.argv[2];
//initialize the variables
var db, settings, padManager;
var npm = require("../src/node_modules/npm");
var async = require("../src/node_modules/async");
// get the padID
const padId = process.argv[2];
var Changeset = require("ep_etherpad-lite/static/js/Changeset");
// load and initialize NPM;
let npm = require('../src/node_modules/npm');
npm.load({}, async function() {
async.series([
//load npm
function(callback) {
npm.load({}, function(er) {
callback(er);
})
},
//load modules
function(callback) {
settings = require('../src/node/utils/Settings');
db = require('../src/node/db/DB');
try {
// initialize database
let settings = require('../src/node/utils/Settings');
let db = require('../src/node/db/DB');
await db.init();
//initialize the database
db.init(callback);
},
//get the pad
function (callback)
{
padManager = require('../src/node/db/PadManager');
padManager.doesPadExists(padId, function(err, exists)
{
if(!exists)
{
console.error("Pad does not exist");
// load modules
let Changeset = require('ep_etherpad-lite/static/js/Changeset');
let padManager = require('../src/node/db/PadManager');
let exists = await padManager.doesPadExists(padId);
if (!exists) {
console.error("Pad does not exist");
process.exit(1);
}
// get the pad
let pad = await padManager.getPad(padId);
// create an array with key revisions
// key revisions always save the full pad atext
let head = pad.getHeadRevisionNumber();
let keyRevisions = [];
for (let rev = 0; rev < head; rev += 100) {
keyRevisions.push(rev);
}
// run through all key revisions
for (let keyRev of keyRevisions) {
// create an array of revisions we need till the next keyRevision or the End
let revisionsNeeded = [];
for (let rev = keyRev; rev <= keyRev + 100 && rev <= head; rev++) {
revisionsNeeded.push(rev);
}
// this array will hold all revision changesets
var revisions = [];
// run through all needed revisions and get them from the database
for (let revNum of revisionsNeeded) {
let revision = await db.get("pad:" + padId + ":revs:" + revNum);
revisions[revNum] = revision;
}
// check if the pad has a pool
if (pad.pool === undefined ) {
console.error("Attribute pool is missing");
process.exit(1);
}
padManager.getPad(padId, function(err, _pad)
{
pad = _pad;
callback(err);
});
});
},
function (callback)
{
//create an array with key revisions
//key revisions always save the full pad atext
var head = pad.getHeadRevisionNumber();
var keyRevisions = [];
for(var i=0;i<head;i+=100)
{
keyRevisions.push(i);
}
//run trough all key revisions
async.forEachSeries(keyRevisions, function(keyRev, callback)
{
//create an array of revisions we need till the next keyRevision or the End
var revisionsNeeded = [];
for(var i=keyRev;i<=keyRev+100 && i<=head; i++)
{
revisionsNeeded.push(i);
// check if there is an atext in the keyRevisions
if (revisions[keyRev] === undefined || revisions[keyRev].meta === undefined || revisions[keyRev].meta.atext === undefined) {
console.error("No atext in key revision " + keyRev);
continue;
}
//this array will hold all revision changesets
var revisions = [];
//run trough all needed revisions and get them from the database
async.forEach(revisionsNeeded, function(revNum, callback)
{
db.db.get("pad:"+padId+":revs:" + revNum, function(err, revision)
{
revisions[revNum] = revision;
callback(err);
});
}, function(err)
{
if(err)
{
callback(err);
return;
let apool = pad.pool;
let atext = revisions[keyRev].meta.atext;
for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) {
try {
// console.log("check revision " + rev);
let cs = revisions[rev].changeset;
atext = Changeset.applyToAText(cs, atext, apool);
} catch(e) {
console.error("Bad changeset at revision " + rev + " - " + e.message);
continue;
}
//check if the pad has a pool
if(pad.pool === undefined )
{
console.error("Attribute pool is missing");
process.exit(1);
}
//check if there is an atext in the keyRevisions
if(revisions[keyRev] === undefined || revisions[keyRev].meta === undefined || revisions[keyRev].meta.atext === undefined)
{
console.error("No atext in key revision " + keyRev);
callback();
return;
}
var apool = pad.pool;
var atext = revisions[keyRev].meta.atext;
for(var i=keyRev+1;i<=keyRev+100 && i<=head; i++)
{
try
{
//console.log("check revision " + i);
var cs = revisions[i].changeset;
atext = Changeset.applyToAText(cs, atext, apool);
}
catch(e)
{
console.error("Bad changeset at revision " + i + " - " + e.message);
callback();
return;
}
}
callback();
});
}, callback);
}
], function (err)
{
if(err) throw err;
else
{
console.log("finished");
process.exit(0);
}
console.log("finished");
process.exit(0);
}
} catch (e) {
console.trace(e);
process.exit(1);
}
});

82
bin/deletePad.js
View file

@ -1,63 +1,41 @@
/*
A tool for deleting pads from the CLI, because sometimes a brick is required to fix a window.
*/
* A tool for deleting pads from the CLI, because sometimes a brick is required
* to fix a window.
*/
if(process.argv.length != 3)
{
if (process.argv.length != 3) {
console.error("Use: node deletePad.js $PADID");
process.exit(1);
}
//get the padID
var padId = process.argv[2];
var db, padManager, pad, settings;
var neededDBValues = ["pad:"+padId];
// get the padID
let padId = process.argv[2];
var npm = require("../src/node_modules/npm");
var async = require("../src/node_modules/async");
let npm = require('../src/node_modules/npm');
async.series([
// load npm
function(callback) {
npm.load({}, function(er) {
if(er)
{
console.error("Could not load NPM: " + er)
process.exit(1);
}
else
{
callback();
}
})
},
// load modules
function(callback) {
settings = require('../src/node/utils/Settings');
db = require('../src/node/db/DB');
callback();
},
// initialize the database
function (callback)
{
db.init(callback);
},
// delete the pad and its links
function (callback)
{
padManager = require('../src/node/db/PadManager');
padManager.removePad(padId, function(err){
callback(err);
});
callback();
npm.load({}, async function(er) {
if (er) {
console.error("Could not load NPM: " + er)
process.exit(1);
}
], function (err)
{
if(err) throw err;
else
{
console.log("Finished deleting padId: "+padId);
process.exit();
try {
let settings = require('../src/node/utils/Settings');
let db = require('../src/node/db/DB');
await db.init();
padManager = require('../src/node/db/PadManager');
await padManager.removePad(padId);
console.log("Finished deleting padId: " + padId);
process.exit(0);
} catch (e) {
if (err.name === "apierror") {
console.error(e);
} else {
console.trace(e);
}
process.exit(1);
}
});

162
bin/extractPadData.js
View file

@ -1,109 +1,75 @@
/*
This is a debug tool. It helps to extract all datas of a pad and move it from an productive environment and to a develop environment to reproduce bugs there. It outputs a dirtydb file
*/
* This is a debug tool. It helps to extract all datas of a pad and move it from
* a productive environment and to a develop environment to reproduce bugs
* there. It outputs a dirtydb file
*/
if(process.argv.length != 3)
{
if (process.argv.length != 3) {
console.error("Use: node extractPadData.js $PADID");
process.exit(1);
}
//get the padID
var padId = process.argv[2];
var db, dirty, padManager, pad, settings;
var neededDBValues = ["pad:"+padId];
// get the padID
let padId = process.argv[2];
var npm = require("../node_modules/ep_etherpad-lite/node_modules/npm");
var async = require("../node_modules/ep_etherpad-lite/node_modules/async");
let npm = require('../src/node_modules/npm');
async.series([
// load npm
function(callback) {
npm.load({}, function(er) {
if(er)
{
console.error("Could not load NPM: " + er)
process.exit(1);
}
else
{
callback();
}
})
},
// load modules
function(callback) {
settings = require('../node_modules/ep_etherpad-lite/node/utils/Settings');
db = require('../node_modules/ep_etherpad-lite/node/db/DB');
dirty = require("../node_modules/ep_etherpad-lite/node_modules/ueberDB/node_modules/dirty")(padId + ".db");
callback();
},
//initialize the database
function (callback)
{
db.init(callback);
},
//get the pad
function (callback)
{
padManager = require('../node_modules/ep_etherpad-lite/node/db/PadManager');
padManager.getPad(padId, function(err, _pad)
{
pad = _pad;
callback(err);
});
},
function (callback)
{
//add all authors
var authors = pad.getAllAuthors();
for(var i=0;i<authors.length;i++)
{
neededDBValues.push("globalAuthor:" + authors[i]);
}
//add all revisions
var revHead = pad.head;
for(var i=0;i<=revHead;i++)
{
neededDBValues.push("pad:"+padId+":revs:" + i);
}
//get all chat values
var chatHead = pad.chatHead;
for(var i=0;i<=chatHead;i++)
{
neededDBValues.push("pad:"+padId+":chat:" + i);
}
//get and set all values
async.forEach(neededDBValues, function(dbkey, callback)
{
db.db.db.wrappedDB.get(dbkey, function(err, dbvalue)
{
if(err) { callback(err); return}
if(dbvalue && typeof dbvalue != 'object'){
dbvalue=JSON.parse(dbvalue); // if it's not json then parse it as json
}
dirty.set(dbkey, dbvalue, callback);
});
}, callback);
npm.load({}, async function(er) {
if (er) {
console.error("Could not load NPM: " + er)
process.exit(1);
}
], function (err)
{
if(err) throw err;
else
{
console.log("finished");
process.exit();
try {
// initialize database
let settings = require('../src/node/utils/Settings');
let db = require('../src/node/db/DB');
await db.init();
// load extra modules
let dirtyDB = require('../src/node_modules/dirty');
let padManager = require('../src/node/db/PadManager');
let util = require('util');
// initialize output database
let dirty = dirtyDB(padId + '.db');
// Promise wrapped get and set function
let wrapped = db.db.db.wrappedDB;
let get = util.promisify(wrapped.get.bind(wrapped));
let set = util.promisify(dirty.set.bind(dirty));
// array in which required key values will be accumulated
let neededDBValues = ['pad:' + padId];
// get the actual pad object
let pad = await padManager.getPad(padId);
// add all authors
neededDBValues.push(...pad.getAllAuthors().map(author => 'globalAuthor:' + author));
// add all revisions
for (let rev = 0; rev <= pad.head; ++rev) {
neededDBValues.push('pad:' + padId + ':revs:' + rev);
}
// add all chat values
for (let chat = 0; chat <= pad.chatHead; ++chat) {
neededDBValues.push('pad:' + padId + ':chat:' + chat);
}
for (let dbkey of neededDBValues) {
let dbvalue = await get(dbkey);
if (dbvalue && typeof dbvalue !== 'object') {
dbvalue = JSON.parse(dbvalue);
}
await set(dbkey, dbvalue);
}
console.log('finished');
process.exit(0);
} catch (er) {
console.error(er);
process.exit(1);
}
});
//get the pad object
//get all revisions of this pad
//get all authors related to this pad
//get the readonly link related to this pad
//get the chat entries related to this pad

144
bin/repairPad.js
View file

@ -1,106 +1,78 @@
/*
This is a repair tool. It extracts all datas of a pad, removes and inserts them again.
*/
* This is a repair tool. It extracts all datas of a pad, removes and inserts them again.
*/
console.warn("WARNING: This script must not be used while etherpad is running!");
if(process.argv.length != 3)
{
if (process.argv.length != 3) {
console.error("Use: node bin/repairPad.js $PADID");
process.exit(1);
}
//get the padID
// get the padID
var padId = process.argv[2];
var db, padManager, pad, settings;
var neededDBValues = ["pad:"+padId];
let npm = require("../src/node_modules/npm");
npm.load({}, async function(er) {
if (er) {
console.error("Could not load NPM: " + er)
process.exit(1);
}
var npm = require("../src/node_modules/npm");
var async = require("../src/node_modules/async");
try {
// intialize database
let settings = require('../src/node/utils/Settings');
let db = require('../src/node/db/DB');
await db.init();
async.series([
// load npm
function(callback) {
npm.load({}, function(er) {
if(er)
{
console.error("Could not load NPM: " + er)
process.exit(1);
}
else
{
callback();
}
})
},
// load modules
function(callback) {
settings = require('../src/node/utils/Settings');
db = require('../src/node/db/DB');
callback();
},
//initialize the database
function (callback)
{
db.init(callback);
},
//get the pad
function (callback)
{
padManager = require('../src/node/db/PadManager');
padManager.getPad(padId, function(err, _pad)
{
pad = _pad;
callback(err);
});
},
function (callback)
{
//add all authors
var authors = pad.getAllAuthors();
for(var i=0;i<authors.length;i++)
{
neededDBValues.push("globalAuthor:" + authors[i]);
// get the pad
let padManager = require('../src/node/db/PadManager');
let pad = await padManager.getPad(padId);
// accumulate the required keys
let neededDBValues = ["pad:" + padId];
// add all authors
neededDBValues.push(...pad.getAllAuthors().map(author => "globalAuthor:"));
// add all revisions
for (let rev = 0; rev <= pad.head; ++rev) {
neededDBValues.push("pad:" + padId + ":revs:" + rev);
}
//add all revisions
var revHead = pad.head;
for(var i=0;i<=revHead;i++)
{
neededDBValues.push("pad:"+padId+":revs:" + i);
// add all chat values
for (let chat = 0; chat <= pad.chatHead; ++chat) {
neededDBValues.push("pad:" + padId + ":chat:" + chat);
}
//get all chat values
var chatHead = pad.chatHead;
for(var i=0;i<=chatHead;i++)
{
neededDBValues.push("pad:"+padId+":chat:" + i);
}
callback();
},
function (callback) {
db = db.db;
//
// NB: this script doesn't actually does what's documented
// since the `value` fields in the following `.forEach`
// block are just the array index numbers
//
// the script therefore craps out now before it can do
// any damage.
//
// See gitlab issue #3545
//
console.info("aborting [gitlab #3545]");
process.exit(1);
// now fetch and reinsert every key
neededDBValues.forEach(function(key, value) {
console.debug("Key: "+key+", value: "+value);
console.log("Key: " + key+ ", value: " + value);
db.remove(key);
db.set(key, value);
});
callback();
}
], function (err)
{
if(err) throw err;
else
{
console.info("finished");
process.exit();
process.exit(0);
} catch (er) {
if (er.name === "apierror") {
console.error(er);
} else {
console.trace(er);
}
}
});
//get the pad object
//get all revisions of this pad
//get all authors related to this pad
//get the readonly link related to this pad
//get the chat entries related to this pad
//remove all keys from database and insert them again

1149
src/node/db/API.js
View file

File diff suppressed because it is too large Load diff

242
src/node/db/AuthorManager.js
View file

@ -18,211 +18,189 @@
* limitations under the License.
*/
var ERR = require("async-stacktrace");
var db = require("./DB").db;
var db = require("./DB");
var customError = require("../utils/customError");
var randomString = require('ep_etherpad-lite/static/js/pad_utils').randomString;
exports.getColorPalette = function(){
return ["#ffc7c7", "#fff1c7", "#e3ffc7", "#c7ffd5", "#c7ffff", "#c7d5ff", "#e3c7ff", "#ffc7f1", "#ffa8a8", "#ffe699", "#cfff9e", "#99ffb3", "#a3ffff", "#99b3ff", "#cc99ff", "#ff99e5", "#e7b1b1", "#e9dcAf", "#cde9af", "#bfedcc", "#b1e7e7", "#c3cdee", "#d2b8ea", "#eec3e6", "#e9cece", "#e7e0ca", "#d3e5c7", "#bce1c5", "#c1e2e2", "#c1c9e2", "#cfc1e2", "#e0bdd9", "#baded3", "#a0f8eb", "#b1e7e0", "#c3c8e4", "#cec5e2", "#b1d5e7", "#cda8f0", "#f0f0a8", "#f2f2a6", "#f5a8eb", "#c5f9a9", "#ececbb", "#e7c4bc", "#daf0b2", "#b0a0fd", "#bce2e7", "#cce2bb", "#ec9afe", "#edabbd", "#aeaeea", "#c4e7b1", "#d722bb", "#f3a5e7", "#ffa8a8", "#d8c0c5", "#eaaedd", "#adc6eb", "#bedad1", "#dee9af", "#e9afc2", "#f8d2a0", "#b3b3e6"];
exports.getColorPalette = function() {
return [
"#ffc7c7", "#fff1c7", "#e3ffc7", "#c7ffd5", "#c7ffff", "#c7d5ff", "#e3c7ff", "#ffc7f1",
"#ffa8a8", "#ffe699", "#cfff9e", "#99ffb3", "#a3ffff", "#99b3ff", "#cc99ff", "#ff99e5",
"#e7b1b1", "#e9dcAf", "#cde9af", "#bfedcc", "#b1e7e7", "#c3cdee", "#d2b8ea", "#eec3e6",
"#e9cece", "#e7e0ca", "#d3e5c7", "#bce1c5", "#c1e2e2", "#c1c9e2", "#cfc1e2", "#e0bdd9",
"#baded3", "#a0f8eb", "#b1e7e0", "#c3c8e4", "#cec5e2", "#b1d5e7", "#cda8f0", "#f0f0a8",
"#f2f2a6", "#f5a8eb", "#c5f9a9", "#ececbb", "#e7c4bc", "#daf0b2", "#b0a0fd", "#bce2e7",
"#cce2bb", "#ec9afe", "#edabbd", "#aeaeea", "#c4e7b1", "#d722bb", "#f3a5e7", "#ffa8a8",
"#d8c0c5", "#eaaedd", "#adc6eb", "#bedad1", "#dee9af", "#e9afc2", "#f8d2a0", "#b3b3e6"
];
};
/**
* Checks if the author exists
*/
exports.doesAuthorExists = function (authorID, callback)
exports.doesAuthorExist = async function(authorID)
{
//check if the database entry of this author exists
db.get("globalAuthor:" + authorID, function (err, author)
{
if(ERR(err, callback)) return;
callback(null, author != null);
});
let author = await db.get("globalAuthor:" + authorID);
return author !== null;
}
/* exported for backwards compatibility */
exports.doesAuthorExists = exports.doesAuthorExist;
/**
* Returns the AuthorID for a token.
* @param {String} token The token
* @param {Function} callback callback (err, author)
*/
exports.getAuthor4Token = function (token, callback)
exports.getAuthor4Token = async function(token)
{
mapAuthorWithDBKey("token2author", token, function(err, author)
{
if(ERR(err, callback)) return;
//return only the sub value authorID
callback(null, author ? author.authorID : author);
});
let author = await mapAuthorWithDBKey("token2author", token);
// return only the sub value authorID
return author ? author.authorID : author;
}
/**
* Returns the AuthorID for a mapper.
* @param {String} token The mapper
* @param {String} name The name of the author (optional)
* @param {Function} callback callback (err, author)
*/
exports.createAuthorIfNotExistsFor = function (authorMapper, name, callback)
exports.createAuthorIfNotExistsFor = async function(authorMapper, name)
{
mapAuthorWithDBKey("mapper2author", authorMapper, function(err, author)
{
if(ERR(err, callback)) return;
let author = await mapAuthorWithDBKey("mapper2author", authorMapper);
//set the name of this author
if(name)
exports.setAuthorName(author.authorID, name);
if (name) {
// set the name of this author
await exports.setAuthorName(author.authorID, name);
}
//return the authorID
callback(null, author);
});
}
return author;
};
/**
* Returns the AuthorID for a mapper. We can map using a mapperkey,
* so far this is token2author and mapper2author
* @param {String} mapperkey The database key name for this mapper
* @param {String} mapper The mapper
* @param {Function} callback callback (err, author)
*/
function mapAuthorWithDBKey (mapperkey, mapper, callback)
async function mapAuthorWithDBKey (mapperkey, mapper)
{
//try to map to an author
db.get(mapperkey + ":" + mapper, function (err, author)
{
if(ERR(err, callback)) return;
// try to map to an author
let author = await db.get(mapperkey + ":" + mapper);
//there is no author with this mapper, so create one
if(author == null)
{
exports.createAuthor(null, function(err, author)
{
if(ERR(err, callback)) return;
if (author === null) {
// there is no author with this mapper, so create one
let author = await exports.createAuthor(null);
//create the token2author relation
db.set(mapperkey + ":" + mapper, author.authorID);
// create the token2author relation
await db.set(mapperkey + ":" + mapper, author.authorID);
//return the author
callback(null, author);
});
// return the author
return author;
}
return;
}
// there is an author with this mapper
// update the timestamp of this author
await db.setSub("globalAuthor:" + author, ["timestamp"], Date.now());
//there is a author with this mapper
//update the timestamp of this author
db.setSub("globalAuthor:" + author, ["timestamp"], Date.now());
//return the author
callback(null, {authorID: author});
});
// return the author
return { authorID: author};
}
/**
* Internal function that creates the database entry for an author
* @param {String} name The name of the author
*/
exports.createAuthor = function(name, callback)
exports.createAuthor = function(name)
{
//create the new author name
var author = "a." + randomString(16);
// create the new author name
let author = "a." + randomString(16);
//create the globalAuthors db entry
var authorObj = {"colorId" : Math.floor(Math.random()*(exports.getColorPalette().length)), "name": name, "timestamp": Date.now()};
// create the globalAuthors db entry
let authorObj = {
"colorId": Math.floor(Math.random() * (exports.getColorPalette().length)),
"name": name,
"timestamp": Date.now()
};
//set the global author db entry
// set the global author db entry
// NB: no await, since we're not waiting for the DB set to finish
db.set("globalAuthor:" + author, authorObj);
callback(null, {authorID: author});
return { authorID: author };
}
/**
* Returns the Author Obj of the author
* @param {String} author The id of the author
* @param {Function} callback callback(err, authorObj)
*/
exports.getAuthor = function (author, callback)
exports.getAuthor = function(author)
{
db.get("globalAuthor:" + author, callback);
// NB: result is already a Promise
return db.get("globalAuthor:" + author);
}
/**
* Returns the color Id of the author
* @param {String} author The id of the author
* @param {Function} callback callback(err, colorId)
*/
exports.getAuthorColorId = function (author, callback)
exports.getAuthorColorId = function(author)
{
db.getSub("globalAuthor:" + author, ["colorId"], callback);
return db.getSub("globalAuthor:" + author, ["colorId"]);
}
/**
* Sets the color Id of the author
* @param {String} author The id of the author
* @param {String} colorId The color id of the author
* @param {Function} callback (optional)
*/
exports.setAuthorColorId = function (author, colorId, callback)
exports.setAuthorColorId = function(author, colorId)
{
db.setSub("globalAuthor:" + author, ["colorId"], colorId, callback);
return db.setSub("globalAuthor:" + author, ["colorId"], colorId);
}
/**
* Returns the name of the author
* @param {String} author The id of the author
* @param {Function} callback callback(err, name)
*/
exports.getAuthorName = function (author, callback)
exports.getAuthorName = function(author)
{
db.getSub("globalAuthor:" + author, ["name"], callback);
return db.getSub("globalAuthor:" + author, ["name"]);
}
/**
* Sets the name of the author
* @param {String} author The id of the author
* @param {String} name The name of the author
* @param {Function} callback (optional)
*/
exports.setAuthorName = function (author, name, callback)
exports.setAuthorName = function(author, name)
{
db.setSub("globalAuthor:" + author, ["name"], name, callback);
return db.setSub("globalAuthor:" + author, ["name"], name);
}
/**
* Returns an array of all pads this author contributed to
* @param {String} author The id of the author
* @param {Function} callback (optional)
*/
exports.listPadsOfAuthor = function (authorID, callback)
exports.listPadsOfAuthor = async function(authorID)
{
/* There are two other places where this array is manipulated:
* (1) When the author is added to a pad, the author object is also updated
* (2) When a pad is deleted, each author of that pad is also updated
*/
//get the globalAuthor
db.get("globalAuthor:" + authorID, function(err, author)
{
if(ERR(err, callback)) return;
//author does not exists
if(author == null)
{
callback(new customError("authorID does not exist","apierror"))
return;
}
// get the globalAuthor
let author = await db.get("globalAuthor:" + authorID);
//everything is fine, return the pad IDs
var pads = [];
if(author.padIDs != null)
{
for (var padId in author.padIDs)
{
pads.push(padId);
}
}
callback(null, {padIDs: pads});
});
if (author === null) {
// author does not exist
throw new customError("authorID does not exist", "apierror");
}
// everything is fine, return the pad IDs
let padIDs = Object.keys(author.padIDs || {});
return { padIDs };
}
/**
@ -230,26 +208,27 @@ exports.listPadsOfAuthor = function (authorID, callback)
* @param {String} author The id of the author
* @param {String} padID The id of the pad the author contributes to
*/
exports.addPad = function (authorID, padID)
exports.addPad = async function(authorID, padID)
{
//get the entry
db.get("globalAuthor:" + authorID, function(err, author)
{
if(ERR(err)) return;
if(author == null) return;
// get the entry
let author = await db.get("globalAuthor:" + authorID);
//the entry doesn't exist so far, let's create it
if(author.padIDs == null)
{
author.padIDs = {};
}
if (author === null) return;
//add the entry for this pad
author.padIDs[padID] = 1;// anything, because value is not used
/*
* ACHTUNG: padIDs can also be undefined, not just null, so it is not possible
* to perform a strict check here
*/
if (!author.padIDs) {
// the entry doesn't exist so far, let's create it
author.padIDs = {};
}
//save the new element back
db.set("globalAuthor:" + authorID, author);
});
// add the entry for this pad
author.padIDs[padID] = 1; // anything, because value is not used
// save the new element back
db.set("globalAuthor:" + authorID, author);
}
/**
@ -257,18 +236,15 @@ exports.addPad = function (authorID, padID)
* @param {String} author The id of the author
* @param {String} padID The id of the pad the author contributes to
*/
exports.removePad = function (authorID, padID)
exports.removePad = async function(authorID, padID)
{
db.get("globalAuthor:" + authorID, function (err, author)
{
if(ERR(err)) return;
if(author == null) return;
let author = await db.get("globalAuthor:" + authorID);
if(author.padIDs != null)
{
//remove pad from author
delete author.padIDs[padID];
db.set("globalAuthor:" + authorID, author);
}
});
if (author === null) return;
if (author.padIDs !== null) {
// remove pad from author
delete author.padIDs[padID];
db.set("globalAuthor:" + authorID, author);
}
}

60
src/node/db/DB.js
View file

@ -1,5 +1,5 @@
/**
* The DB Module provides a database initalized with the settings
* The DB Module provides a database initalized with the settings
* provided by the settings module
*/
@ -22,9 +22,10 @@
var ueberDB = require("ueberdb2");
var settings = require("../utils/Settings");
var log4js = require('log4js');
const util = require("util");
//set database settings
var db = new ueberDB.database(settings.dbType, settings.dbSettings, null, log4js.getLogger("ueberDB"));
// set database settings
let db = new ueberDB.database(settings.dbType, settings.dbSettings, null, log4js.getLogger("ueberDB"));
/**
* The UeberDB Object that provides the database functions
@ -33,25 +34,40 @@ exports.db = null;
/**
* Initalizes the database with the settings provided by the settings module
* @param {Function} callback
* @param {Function} callback
*/
exports.init = function(callback)
{
//initalize the database async
db.init(function(err)
{
//there was an error while initializing the database, output it and stop
if(err)
{
console.error("ERROR: Problem while initalizing the database");
console.error(err.stack ? err.stack : err);
process.exit(1);
}
//everything ok
else
{
exports.db = db;
callback(null);
}
exports.init = function() {
// initalize the database async
return new Promise((resolve, reject) => {
db.init(function(err) {
if (err) {
// there was an error while initializing the database, output it and stop
console.error("ERROR: Problem while initalizing the database");
console.error(err.stack ? err.stack : err);
process.exit(1);
} else {
// everything ok, set up Promise-based methods
['get', 'set', 'findKeys', 'getSub', 'setSub', 'remove', 'doShutdown'].forEach(fn => {
exports[fn] = util.promisify(db[fn].bind(db));
});
// set up wrappers for get and getSub that can't return "undefined"
let get = exports.get;
exports.get = async function(key) {
let result = await get(key);
return (result === undefined) ? null : result;
};
let getSub = exports.getSub;
exports.getSub = async function(key, sub) {
let result = await getSub(key, sub);
return (result === undefined) ? null : result;
};
// exposed for those callers that need the underlying raw API
exports.db = db;
resolve();
}
});
});
}

420
src/node/db/GroupManager.js
View file

@ -17,319 +17,167 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
var ERR = require("async-stacktrace");
var customError = require("../utils/customError");
var randomString = require('ep_etherpad-lite/static/js/pad_utils').randomString;
var db = require("./DB").db;
var async = require("async");
var db = require("./DB");
var padManager = require("./PadManager");
var sessionManager = require("./SessionManager");
exports.listAllGroups = function(callback) {
db.get("groups", function (err, groups) {
if(ERR(err, callback)) return;
// there are no groups
if(groups == null) {
callback(null, {groupIDs: []});
return;
}
var groupIDs = [];
for ( var groupID in groups ) {
groupIDs.push(groupID);
}
callback(null, {groupIDs: groupIDs});
});
}
exports.deleteGroup = function(groupID, callback)
exports.listAllGroups = async function()
{
var group;
let groups = await db.get("groups");
groups = groups || {};
async.series([
//ensure group exists
function (callback)
{
//try to get the group entry
db.get("group:" + groupID, function (err, _group)
{
if(ERR(err, callback)) return;
//group does not exist
if(_group == null)
{
callback(new customError("groupID does not exist","apierror"));
return;
}
//group exists, everything is fine
group = _group;
callback();
});
},
//iterate trough all pads of this groups and delete them
function(callback)
{
//collect all padIDs in an array, that allows us to use async.forEach
var padIDs = [];
for(var i in group.pads)
{
padIDs.push(i);
}
//loop trough all pads and delete them
async.forEach(padIDs, function(padID, callback)
{
padManager.getPad(padID, function(err, pad)
{
if(ERR(err, callback)) return;
pad.remove(callback);
});
}, callback);
},
//iterate trough group2sessions and delete all sessions
function(callback)
{
//try to get the group entry
db.get("group2sessions:" + groupID, function (err, group2sessions)
{
if(ERR(err, callback)) return;
//skip if there is no group2sessions entry
if(group2sessions == null) {callback(); return}
//collect all sessions in an array, that allows us to use async.forEach
var sessions = [];
for(var i in group2sessions.sessionsIDs)
{
sessions.push(i);
}
//loop trough all sessions and delete them
async.forEach(sessions, function(session, callback)
{
sessionManager.deleteSession(session, callback);
}, callback);
});
},
//remove group and group2sessions entry
function(callback)
{
db.remove("group2sessions:" + groupID);
db.remove("group:" + groupID);
callback();
},
//unlist the group
function(callback)
{
exports.listAllGroups(function(err, groups) {
if(ERR(err, callback)) return;
groups = groups? groups.groupIDs : [];
// it's not listed
if(groups.indexOf(groupID) == -1) {
callback();
return;
}
groups.splice(groups.indexOf(groupID), 1);
// store empty groupe list
if(groups.length == 0) {
db.set("groups", {});
callback();
return;
}
// regenerate group list
var newGroups = {};
async.forEach(groups, function(group, cb) {
newGroups[group] = 1;
cb();
},function() {
db.set("groups", newGroups);
callback();
});
});
}
], function(err)
{
if(ERR(err, callback)) return;
callback();
});
}
exports.doesGroupExist = function(groupID, callback)
{
//try to get the group entry
db.get("group:" + groupID, function (err, group)
{
if(ERR(err, callback)) return;
callback(null, group != null);
});
let groupIDs = Object.keys(groups);
return { groupIDs };
}
exports.createGroup = function(callback)
exports.deleteGroup = async function(groupID)
{
//search for non existing groupID
var groupID = "g." + randomString(16);
//create the group
db.set("group:" + groupID, {pads: {}});
//list the group
exports.listAllGroups(function(err, groups) {
if(ERR(err, callback)) return;
groups = groups? groups.groupIDs : [];
groups.push(groupID);
// regenerate group list
var newGroups = {};
async.forEach(groups, function(group, cb) {
newGroups[group] = 1;
cb();
},function() {
db.set("groups", newGroups);
callback(null, {groupID: groupID});
});
});
}
let group = await db.get("group:" + groupID);
// ensure group exists
if (group == null) {
// group does not exist
throw new customError("groupID does not exist", "apierror");
}
// iterate through all pads of this group and delete them (in parallel)
await Promise.all(Object.keys(group.pads).map(padID => {
return padManager.getPad(padID).then(pad => pad.remove());
}));
// iterate through group2sessions and delete all sessions
let group2sessions = await db.get("group2sessions:" + groupID);
let sessions = group2sessions ? group2sessions.sessionsIDs : {};
// loop through all sessions and delete them (in parallel)
await Promise.all(Object.keys(sessions).map(session => {
return sessionManager.deleteSession(session);
}));
// remove group and group2sessions entry
await db.remove("group2sessions:" + groupID);
await db.remove("group:" + groupID);
// unlist the group
let groups = await exports.listAllGroups();
groups = groups ? groups.groupIDs : [];
let index = groups.indexOf(groupID);
if (index === -1) {
// it's not listed
exports.createGroupIfNotExistsFor = function(groupMapper, callback)
{
//ensure mapper is optional
if(typeof groupMapper != "string")
{
callback(new customError("groupMapper is no string","apierror"));
return;
}
//try to get a group for this mapper
db.get("mapper2group:"+groupMapper, function(err, groupID)
{
function createGroupForMapper(cb) {
exports.createGroup(function(err, responseObj)
{
if(ERR(err, cb)) return;
//create the mapper entry for this group
db.set("mapper2group:"+groupMapper, responseObj.groupID);
cb(null, responseObj);
});
}
if(ERR(err, callback)) return;
// remove from the list
groups.splice(index, 1);
// regenerate group list
var newGroups = {};
groups.forEach(group => newGroups[group] = 1);
await db.set("groups", newGroups);
}
exports.doesGroupExist = async function(groupID)
{
// try to get the group entry
let group = await db.get("group:" + groupID);
return (group != null);
}
exports.createGroup = async function()
{
// search for non existing groupID
var groupID = "g." + randomString(16);
// create the group
await db.set("group:" + groupID, {pads: {}});
// list the group
let groups = await exports.listAllGroups();
groups = groups? groups.groupIDs : [];
groups.push(groupID);
// regenerate group list
var newGroups = {};
groups.forEach(group => newGroups[group] = 1);
await db.set("groups", newGroups);
return { groupID };
}
exports.createGroupIfNotExistsFor = async function(groupMapper)
{
// ensure mapper is optional
if (typeof groupMapper !== "string") {
throw new customError("groupMapper is not a string", "apierror");
}
// try to get a group for this mapper
let groupID = await db.get("mapper2group:" + groupMapper);
if (groupID) {
// there is a group for this mapper
if(groupID) {
exports.doesGroupExist(groupID, function(err, exists) {
if(ERR(err, callback)) return;
if(exists) return callback(null, {groupID: groupID});
let exists = await exports.doesGroupExist(groupID);
// hah, the returned group doesn't exist, let's create one
createGroupForMapper(callback)
})
if (exists) return { groupID };
}
return;
}
// hah, the returned group doesn't exist, let's create one
let result = await exports.createGroup();
//there is no group for this mapper, let's create a group
createGroupForMapper(callback)
});
// create the mapper entry for this group
await db.set("mapper2group:" + groupMapper, result.groupID);
return result;
}
exports.createGroupPad = function(groupID, padName, text, callback)
exports.createGroupPad = async function(groupID, padName, text)
{
//create the padID
var padID = groupID + "$" + padName;
// create the padID
let padID = groupID + "$" + padName;
async.series([
//ensure group exists
function (callback)
{
exports.doesGroupExist(groupID, function(err, exists)
{
if(ERR(err, callback)) return;
//group does not exist
if(exists == false)
{
callback(new customError("groupID does not exist","apierror"));
return;
}
// ensure group exists
let groupExists = await exports.doesGroupExist(groupID);
//group exists, everything is fine
callback();
});
},
//ensure pad does not exists
function (callback)
{
padManager.doesPadExists(padID, function(err, exists)
{
if(ERR(err, callback)) return;
//pad exists already
if(exists == true)
{
callback(new customError("padName does already exist","apierror"));
return;
}
if (!groupExists) {
throw new customError("groupID does not exist", "apierror");
}
//pad does not exist, everything is fine
callback();
});
},
//create the pad
function (callback)
{
padManager.getPad(padID, text, function(err)
{
if(ERR(err, callback)) return;
callback();
});
},
//create an entry in the group for this pad
function (callback)
{
db.setSub("group:" + groupID, ["pads", padID], 1);
callback();
}
], function(err)
{
if(ERR(err, callback)) return;
callback(null, {padID: padID});
});
// ensure pad doesn't exist already
let padExists = await padManager.doesPadExists(padID);
if (padExists) {
// pad exists already
throw new customError("padName does already exist", "apierror");
}
// create the pad
await padManager.getPad(padID, text);
//create an entry in the group for this pad
await db.setSub("group:" + groupID, ["pads", padID], 1);
return { padID };
}
exports.listPads = function(groupID, callback)
exports.listPads = async function(groupID)
{
exports.doesGroupExist(groupID, function(err, exists)
{
if(ERR(err, callback)) return;
//group does not exist
if(exists == false)
{
callback(new customError("groupID does not exist","apierror"));
return;
}
let exists = await exports.doesGroupExist(groupID);
//group exists, let's get the pads
db.getSub("group:" + groupID, ["pads"], function(err, result)
{
if(ERR(err, callback)) return;
var pads = [];
for ( var padId in result ) {
pads.push(padId);
}
callback(null, {padIDs: pads});
});
});
// ensure the group exists
if (!exists) {
throw new customError("groupID does not exist", "apierror");
}
// group exists, let's get the pads
let result = await db.getSub("group:" + groupID, ["pads"]);
let padIDs = Object.keys(result);
return { padIDs };
}

764
src/node/db/Pad.js
View file

@ -3,11 +3,9 @@
*/
var ERR = require("async-stacktrace");
var Changeset = require("ep_etherpad-lite/static/js/Changeset");
var AttributePool = require("ep_etherpad-lite/static/js/AttributePool");
var db = require("./DB").db;
var async = require("async");
var db = require("./DB");
var settings = require('../utils/Settings');
var authorManager = require("./AuthorManager");
var padManager = require("./PadManager");
@ -19,7 +17,7 @@ var crypto = require("crypto");
var randomString = require("../utils/randomstring");
var hooks = require('ep_etherpad-lite/static/js/pluginfw/hooks');
//serialization/deserialization attributes
// serialization/deserialization attributes
var attributeBlackList = ["id"];
var jsonableList = ["pool"];
@ -32,8 +30,7 @@ exports.cleanText = function (txt) {
};
var Pad = function Pad(id) {
let Pad = function Pad(id) {
this.atext = Changeset.makeAText("\n");
this.pool = new AttributePool();
this.head = -1;
@ -60,7 +57,7 @@ Pad.prototype.getSavedRevisionsNumber = function getSavedRevisionsNumber() {
Pad.prototype.getSavedRevisionsList = function getSavedRevisionsList() {
var savedRev = new Array();
for(var rev in this.savedRevisions){
for (var rev in this.savedRevisions) {
savedRev.push(this.savedRevisions[rev].revNum);
}
savedRev.sort(function(a, b) {
@ -74,8 +71,9 @@ Pad.prototype.getPublicStatus = function getPublicStatus() {
};
Pad.prototype.appendRevision = function appendRevision(aChangeset, author) {
if(!author)
if (!author) {
author = '';
}
var newAText = Changeset.applyToAText(aChangeset, this.atext, this.pool);
Changeset.copyAText(newAText, this.atext);
@ -88,21 +86,22 @@ Pad.prototype.appendRevision = function appendRevision(aChangeset, author) {
newRevData.meta.author = author;
newRevData.meta.timestamp = Date.now();
//ex. getNumForAuthor
if(author != '')
// ex. getNumForAuthor
if (author != '') {
this.pool.putAttrib(['author', author || '']);
}
if(newRev % 100 == 0)
{
if (newRev % 100 == 0) {
newRevData.meta.atext = this.atext;
}
db.set("pad:"+this.id+":revs:"+newRev, newRevData);
db.set("pad:" + this.id + ":revs:" + newRev, newRevData);
this.saveToDatabase();
// set the author to pad
if(author)
if (author) {
authorManager.addPad(author, this.id);
}
if (this.head == 0) {
hooks.callAll("padCreate", {'pad':this, 'author': author});
@ -111,49 +110,47 @@ Pad.prototype.appendRevision = function appendRevision(aChangeset, author) {
}
};
//save all attributes to the database
Pad.prototype.saveToDatabase = function saveToDatabase(){
// save all attributes to the database
Pad.prototype.saveToDatabase = function saveToDatabase() {
var dbObject = {};
for(var attr in this){
if(typeof this[attr] === "function") continue;
if(attributeBlackList.indexOf(attr) !== -1) continue;
for (var attr in this) {
if (typeof this[attr] === "function") continue;
if (attributeBlackList.indexOf(attr) !== -1) continue;
dbObject[attr] = this[attr];
if(jsonableList.indexOf(attr) !== -1){
if (jsonableList.indexOf(attr) !== -1) {
dbObject[attr] = dbObject[attr].toJsonable();
}
}
db.set("pad:"+this.id, dbObject);
db.set("pad:" + this.id, dbObject);
}
// get time of last edit (changeset application)
Pad.prototype.getLastEdit = function getLastEdit(callback){
Pad.prototype.getLastEdit = function getLastEdit() {
var revNum = this.getHeadRevisionNumber();
db.getSub("pad:"+this.id+":revs:"+revNum, ["meta", "timestamp"], callback);
return db.getSub("pad:" + this.id + ":revs:" + revNum, ["meta", "timestamp"]);
}
Pad.prototype.getRevisionChangeset = function getRevisionChangeset(revNum, callback) {
db.getSub("pad:"+this.id+":revs:"+revNum, ["changeset"], callback);
};
Pad.prototype.getRevisionChangeset = function getRevisionChangeset(revNum) {
return db.getSub("pad:" + this.id + ":revs:" + revNum, ["changeset"]);
}
Pad.prototype.getRevisionAuthor = function getRevisionAuthor(revNum, callback) {
db.getSub("pad:"+this.id+":revs:"+revNum, ["meta", "author"], callback);
};
Pad.prototype.getRevisionAuthor = function getRevisionAuthor(revNum) {
return db.getSub("pad:" + this.id + ":revs:" + revNum, ["meta", "author"]);
}
Pad.prototype.getRevisionDate = function getRevisionDate(revNum, callback) {
db.getSub("pad:"+this.id+":revs:"+revNum, ["meta", "timestamp"], callback);
};
Pad.prototype.getRevisionDate = function getRevisionDate(revNum) {
return db.getSub("pad:" + this.id + ":revs:" + revNum, ["meta", "timestamp"]);
}
Pad.prototype.getAllAuthors = function getAllAuthors() {
var authors = [];
for(var key in this.pool.numToAttrib)
{
if(this.pool.numToAttrib[key][0] == "author" && this.pool.numToAttrib[key][1] != "")
{
for(var key in this.pool.numToAttrib) {
if (this.pool.numToAttrib[key][0] == "author" && this.pool.numToAttrib[key][1] != "") {
authors.push(this.pool.numToAttrib[key][1]);
}
}
@ -161,120 +158,77 @@ Pad.prototype.getAllAuthors = function getAllAuthors() {
return authors;
};
Pad.prototype.getInternalRevisionAText = function getInternalRevisionAText(targetRev, callback) {
var _this = this;
Pad.prototype.getInternalRevisionAText = async function getInternalRevisionAText(targetRev) {
let keyRev = this.getKeyRevisionNumber(targetRev);
var keyRev = this.getKeyRevisionNumber(targetRev);
var atext;
var changesets = [];
//find out which changesets are needed
var neededChangesets = [];
var curRev = keyRev;
while (curRev < targetRev)
{
curRev++;
neededChangesets.push(curRev);
// find out which changesets are needed
let neededChangesets = [];
for (let curRev = keyRev; curRev < targetRev; ) {
neededChangesets.push(++curRev);
}
async.series([
//get all needed data out of the database
function(callback)
{
async.parallel([
//get the atext of the key revision
function (callback)
{
db.getSub("pad:"+_this.id+":revs:"+keyRev, ["meta", "atext"], function(err, _atext)
{
if(ERR(err, callback)) return;
try {
atext = Changeset.cloneAText(_atext);
} catch (e) {
return callback(e);
}
// get all needed data out of the database
callback();
});
},
//get all needed changesets
function (callback)
{
async.forEach(neededChangesets, function(item, callback)
{
_this.getRevisionChangeset(item, function(err, changeset)
{
if(ERR(err, callback)) return;
changesets[item] = changeset;
callback();
});
}, callback);
}
], callback);
},
//apply all changesets to the key changeset
function(callback)
{
var apool = _this.apool();
var curRev = keyRev;
// start to get the atext of the key revision
let p_atext = db.getSub("pad:" + this.id + ":revs:" + keyRev, ["meta", "atext"]);
while (curRev < targetRev)
{
curRev++;
var cs = changesets[curRev];
try{
atext = Changeset.applyToAText(cs, atext, apool);
}catch(e) {
return callback(e)
}
}
callback(null);
}
], function(err)
{
if(ERR(err, callback)) return;
callback(null, atext);
});
};
Pad.prototype.getRevision = function getRevisionChangeset(revNum, callback) {
db.get("pad:"+this.id+":revs:"+revNum, callback);
};
Pad.prototype.getAllAuthorColors = function getAllAuthorColors(callback){
var authors = this.getAllAuthors();
var returnTable = {};
var colorPalette = authorManager.getColorPalette();
async.forEach(authors, function(author, callback){
authorManager.getAuthorColorId(author, function(err, colorId){
if(err){
return callback(err);
}
//colorId might be a hex color or an number out of the palette
returnTable[author]=colorPalette[colorId] || colorId;
callback();
// get all needed changesets
let changesets = [];
await Promise.all(neededChangesets.map(item => {
return this.getRevisionChangeset(item).then(changeset => {
changesets[item] = changeset;
});
}, function(err){
callback(err, returnTable);
});
};
}));
// we should have the atext by now
let atext = await p_atext;
atext = Changeset.cloneAText(atext);
// apply all changesets to the key changeset
let apool = this.apool();
for (let curRev = keyRev; curRev < targetRev; ) {
let cs = changesets[++curRev];
atext = Changeset.applyToAText(cs, atext, apool);
}
return atext;
}
Pad.prototype.getRevision = function getRevisionChangeset(revNum) {
return db.get("pad:" + this.id + ":revs:" + revNum);
}
Pad.prototype.getAllAuthorColors = async function getAllAuthorColors() {
let authors = this.getAllAuthors();
let returnTable = {};
let colorPalette = authorManager.getColorPalette();
await Promise.all(authors.map(author => {
return authorManager.getAuthorColorId(author).then(colorId => {
// colorId might be a hex color or an number out of the palette
returnTable[author] = colorPalette[colorId] || colorId;
});
}));
return returnTable;
}
Pad.prototype.getValidRevisionRange = function getValidRevisionRange(startRev, endRev) {
startRev = parseInt(startRev, 10);
var head = this.getHeadRevisionNumber();
endRev = endRev ? parseInt(endRev, 10) : head;
if(isNaN(startRev) || startRev < 0 || startRev > head) {
if (isNaN(startRev) || startRev < 0 || startRev > head) {
startRev = null;
}
if(isNaN(endRev) || endRev < startRev) {
if (isNaN(endRev) || endRev < startRev) {
endRev = null;
} else if(endRev > head) {
} else if (endRev > head) {
endRev = head;
}
if(startRev !== null && endRev !== null) {
if (startRev !== null && endRev !== null) {
return { startRev: startRev , endRev: endRev }
}
return null;
@ -289,12 +243,12 @@ Pad.prototype.text = function text() {
};
Pad.prototype.setText = function setText(newText) {
//clean the new text
// clean the new text
newText = exports.cleanText(newText);
var oldText = this.text();
//create the changeset
// create the changeset
// We want to ensure the pad still ends with a \n, but otherwise keep
// getText() and setText() consistent.
var changeset;
@ -304,155 +258,105 @@ Pad.prototype.setText = function setText(newText) {
changeset = Changeset.makeSplice(oldText, 0, oldText.length-1, newText);
}
//append the changeset
// append the changeset
this.appendRevision(changeset);
};
Pad.prototype.appendText = function appendText(newText) {
//clean the new text
// clean the new text
newText = exports.cleanText(newText);
var oldText = this.text();
//create the changeset
// create the changeset
var changeset = Changeset.makeSplice(oldText, oldText.length, 0, newText);
//append the changeset
// append the changeset
this.appendRevision(changeset);
};
Pad.prototype.appendChatMessage = function appendChatMessage(text, userId, time) {
this.chatHead++;
//save the chat entry in the database
db.set("pad:"+this.id+":chat:"+this.chatHead, {"text": text, "userId": userId, "time": time});
// save the chat entry in the database
db.set("pad:" + this.id + ":chat:" + this.chatHead, { "text": text, "userId": userId, "time": time });
this.saveToDatabase();
};
Pad.prototype.getChatMessage = function getChatMessage(entryNum, callback) {
var _this = this;
var entry;
Pad.prototype.getChatMessage = async function getChatMessage(entryNum) {
// get the chat entry
let entry = await db.get("pad:" + this.id + ":chat:" + entryNum);
async.series([
//get the chat entry
function(callback)
{
db.get("pad:"+_this.id+":chat:"+entryNum, function(err, _entry)
{
if(ERR(err, callback)) return;
entry = _entry;
callback();
});
},
//add the authorName
function(callback)
{
//this chat message doesn't exist, return null
if(entry == null)
{
callback();
return;
}
//get the authorName
authorManager.getAuthorName(entry.userId, function(err, authorName)
{
if(ERR(err, callback)) return;
entry.userName = authorName;
callback();
});
}
], function(err)
{
if(ERR(err, callback)) return;
callback(null, entry);
});
};
Pad.prototype.getChatMessages = function getChatMessages(start, end, callback) {
//collect the numbers of chat entries and in which order we need them
var neededEntries = [];
var order = 0;
for(var i=start;i<=end; i++)
{
neededEntries.push({entryNum:i, order: order});
order++;
// get the authorName if the entry exists
if (entry != null) {
entry.userName = await authorManager.getAuthorName(entry.userId);
}
var _this = this;
//get all entries out of the database
var entries = [];
async.forEach(neededEntries, function(entryObject, callback)
{
_this.getChatMessage(entryObject.entryNum, function(err, entry)
{
if(ERR(err, callback)) return;
entries[entryObject.order] = entry;
callback();
});
}, function(err)
{
if(ERR(err, callback)) return;
//sort out broken chat entries
//it looks like in happend in the past that the chat head was
//incremented, but the chat message wasn't added
var cleanedEntries = [];
for(var i=0;i<entries.length;i++)
{
if(entries[i]!=null)
cleanedEntries.push(entries[i]);
else
console.warn("WARNING: Found broken chat entry in pad " + _this.id);
}
callback(null, cleanedEntries);
});
return entry;
};
Pad.prototype.init = function init(text, callback) {
var _this = this;
Pad.prototype.getChatMessages = async function getChatMessages(start, end) {
//replace text with default text if text isn't set
if(text == null)
{
// collect the numbers of chat entries and in which order we need them
let neededEntries = [];
for (let order = 0, entryNum = start; entryNum <= end; ++order, ++entryNum) {
neededEntries.push({ entryNum, order });
}
// get all entries out of the database
let entries = [];
await Promise.all(neededEntries.map(entryObject => {
return this.getChatMessage(entryObject.entryNum).then(entry => {
entries[entryObject.order] = entry;
});
}));
// sort out broken chat entries
// it looks like in happened in the past that the chat head was
// incremented, but the chat message wasn't added
let cleanedEntries = entries.filter(entry => {
let pass = (entry != null);
if (!pass) {
console.warn("WARNING: Found broken chat entry in pad " + this.id);
}
return pass;
});
return cleanedEntries;
}
Pad.prototype.init = async function init(text) {
// replace text with default text if text isn't set
if (text == null) {
text = settings.defaultPadText;
}
//try to load the pad
db.get("pad:"+this.id, function(err, value)
{
if(ERR(err, callback)) return;
// try to load the pad
let value = await db.get("pad:" + this.id);
//if this pad exists, load it
if(value != null)
{
//copy all attr. To a transfrom via fromJsonable if necassary
for(var attr in value){
if(jsonableList.indexOf(attr) !== -1){
_this[attr] = _this[attr].fromJsonable(value[attr]);
} else {
_this[attr] = value[attr];
}
// if this pad exists, load it
if (value != null) {
// copy all attr. To a transfrom via fromJsonable if necassary
for (var attr in value) {
if (jsonableList.indexOf(attr) !== -1) {
this[attr] = this[attr].fromJsonable(value[attr]);
} else {
this[attr] = value[attr];
}
}
//this pad doesn't exist, so create it
else
{
var firstChangeset = Changeset.makeSplice("\n", 0, 0, exports.cleanText(text));
} else {
// this pad doesn't exist, so create it
let firstChangeset = Changeset.makeSplice("\n", 0, 0, exports.cleanText(text));
_this.appendRevision(firstChangeset, '');
}
this.appendRevision(firstChangeset, '');
}
hooks.callAll("padLoad", {'pad':_this});
callback(null);
});
};
hooks.callAll("padLoad", { 'pad': this });
}
Pad.prototype.copy = function copy(destinationID, force, callback) {
var sourceID = this.id;
var _this = this;
var destGroupID;
Pad.prototype.copy = async function copy(destinationID, force) {
let sourceID = this.id;
// allow force to be a string
if (typeof force === "string") {
@ -467,247 +371,139 @@ Pad.prototype.copy = function copy(destinationID, force, callback) {
// padMessageHandler.kickSessionsFromPad(sourceID);
// flush the source pad:
_this.saveToDatabase();
this.saveToDatabase();
async.series([
// if it's a group pad, let's make sure the group exists.
function(callback)
{
if (destinationID.indexOf("$") === -1)
{
callback();
return;
}
// if it's a group pad, let's make sure the group exists.
let destGroupID;
if (destinationID.indexOf("$") >= 0) {
destGroupID = destinationID.split("$")[0]
groupManager.doesGroupExist(destGroupID, function (err, exists)
{
if(ERR(err, callback)) return;
destGroupID = destinationID.split("$")[0]
let groupExists = await groupManager.doesGroupExist(destGroupID);
//group does not exist
if(exists == false)
{
callback(new customError("groupID does not exist for destinationID","apierror"));
return;
}
//everything is fine, continue
callback();
});
},
// if the pad exists, we should abort, unless forced.
function(callback)
{
padManager.doesPadExists(destinationID, function (err, exists)
{
if(ERR(err, callback)) return;
/*
* this is the negation of a truthy comparison. Has been left in this
* wonky state to keep the old (possibly buggy) behaviour
*/
if (!(exists == true))
{
callback();
return;
}
if (!force)
{
console.error("erroring out without force");
callback(new customError("destinationID already exists","apierror"));
console.error("erroring out without force - after");
return;
}
// exists and forcing
padManager.getPad(destinationID, function(err, pad) {
if (ERR(err, callback)) return;
pad.remove(callback);
});
});
},
// copy the 'pad' entry
function(callback)
{
db.get("pad:"+sourceID, function(err, pad) {
db.set("pad:"+destinationID, pad);
});
callback();
},
//copy all relations
function(callback)
{
async.parallel([
//copy all chat messages
function(callback)
{
var chatHead = _this.chatHead;
for(var i=0;i<=chatHead;i++)
{
db.get("pad:"+sourceID+":chat:"+i, function (err, chat) {
if (ERR(err, callback)) return;
db.set("pad:"+destinationID+":chat:"+i, chat);
});
}
callback();
},
//copy all revisions
function(callback)
{
var revHead = _this.head;
for(var i=0;i<=revHead;i++)
{
db.get("pad:"+sourceID+":revs:"+i, function (err, rev) {
if (ERR(err, callback)) return;
db.set("pad:"+destinationID+":revs:"+i, rev);
});
}
callback();
},
//add the new pad to all authors who contributed to the old one
function(callback)
{
var authorIDs = _this.getAllAuthors();
authorIDs.forEach(function (authorID)
{
authorManager.addPad(authorID, destinationID);
});
callback();
},
// parallel
], callback);
},
function(callback) {
// Group pad? Add it to the group's list
if(destGroupID) db.setSub("group:" + destGroupID, ["pads", destinationID], 1);
// Initialize the new pad (will update the listAllPads cache)
setTimeout(function(){
padManager.getPad(destinationID, null, callback) // this runs too early.
},10);
},
// let the plugins know the pad was copied
function(callback) {
hooks.callAll('padCopy', { 'originalPad': _this, 'destinationID': destinationID });
callback();
// group does not exist
if (!groupExists) {
throw new customError("groupID does not exist for destinationID", "apierror");
}
// series
], function(err)
{
if(ERR(err, callback)) return;
callback(null, {padID: destinationID});
}
// if the pad exists, we should abort, unless forced.
let exists = await padManager.doesPadExist(destinationID);
if (exists) {
if (!force) {
console.error("erroring out without force");
throw new customError("destinationID already exists", "apierror");
return;
}
// exists and forcing
let pad = await padManager.getPad(destinationID);
await pad.remove(callback);
}
// copy the 'pad' entry
let pad = await db.get("pad:" + sourceID);
db.set("pad:" + destinationID, pad);
// copy all relations in parallel
let promises = [];
// copy all chat messages
let chatHead = this.chatHead;
for (let i = 0; i <= chatHead; ++i) {
let p = db.get("pad:" + sourceID + ":chat:" + i).then(chat => {
return db.set("pad:" + destinationID + ":chat:" + i, chat);
});
promises.push(p);
}
// copy all revisions
let revHead = this.head;
for (let i = 0; i <= revHead; ++i) {
let p = db.get("pad:" + sourceID + ":revs:" + i).then(rev => {
return db.set("pad:" + destinationID + ":revs:" + i, rev);
});
promises.push(p);
}
// add the new pad to all authors who contributed to the old one
this.getAllAuthors().forEach(authorID => {
authorManager.addPad(authorID, destinationID);
});
};
Pad.prototype.remove = function remove(callback) {
// wait for the above to complete
await Promise.all(promises);
// Group pad? Add it to the group's list
if (destGroupID) {
await db.setSub("group:" + destGroupID, ["pads", destinationID], 1);
}
// delay still necessary?
await new Promise(resolve => setTimeout(resolve, 10));
// Initialize the new pad (will update the listAllPads cache)
await padManager.getPad(destinationID, null); // this runs too early.
// let the plugins know the pad was copied
hooks.callAll('padCopy', { 'originalPad': this, 'destinationID': destinationID });
return { padID: destinationID };
}
Pad.prototype.remove = async function remove() {
var padID = this.id;
var _this = this;
//kick everyone from this pad
// kick everyone from this pad
padMessageHandler.kickSessionsFromPad(padID);
async.series([
//delete all relations
function(callback)
{
async.parallel([
//is it a group pad? -> delete the entry of this pad in the group
function(callback)
{
if(padID.indexOf("$") === -1)
{
// it isn't a group pad, nothing to do here
callback();
return;
}
// delete all relations - the original code used async.parallel but
// none of the operations except getting the group depended on callbacks
// so the database operations here are just started and then left to
// run to completion
// is it a group pad? -> delete the entry of this pad in the group
if (padID.indexOf("$") >= 0) {
// it is a group pad
var groupID = padID.substring(0,padID.indexOf("$"));
// it is a group pad
let groupID = padID.substring(0, padID.indexOf("$"));
let group = await db.get("group:" + groupID);
db.get("group:" + groupID, function (err, group)
{
if(ERR(err, callback)) return;
// remove the pad entry
delete group.pads[padID];
//remove the pad entry
delete group.pads[padID];
// set the new value
db.set("group:" + groupID, group);
}
//set the new value
db.set("group:" + groupID, group);
// remove the readonly entries
let readonlyID = readOnlyManager.getReadOnlyId(padID);
callback();
});
},
//remove the readonly entries
function(callback)
{
readOnlyManager.getReadOnlyId(padID, function(err, readonlyID)
{
if(ERR(err, callback)) return;
db.remove("pad2readonly:" + padID);
db.remove("readonly2pad:" + readonlyID);
db.remove("pad2readonly:" + padID);
db.remove("readonly2pad:" + readonlyID);
// delete all chat messages
for (let i = 0, n = this.chatHead; i <= n; ++i) {
db.remove("pad:" + padID + ":chat:" + i);
}
callback();
});
},
//delete all chat messages
function(callback)
{
var chatHead = _this.chatHead;
// delete all revisions
for (let i = 0, n = this.head; i <= n; ++i) {
db.remove("pad:" + padID + ":revs:" + i);
}
for(var i=0;i<=chatHead;i++)
{
db.remove("pad:"+padID+":chat:"+i);
}
callback();
},
//delete all revisions
function(callback)
{
var revHead = _this.head;
for(var i=0;i<=revHead;i++)
{
db.remove("pad:"+padID+":revs:"+i);
}
callback();
},
//remove pad from all authors who contributed
function(callback)
{
var authorIDs = _this.getAllAuthors();
authorIDs.forEach(function (authorID)
{
authorManager.removePad(authorID, padID);
});
callback();
}
], callback);
},
//delete the pad entry and delete pad from padManager
function(callback)
{
padManager.removePad(padID);
hooks.callAll("padRemove", {'padID':padID});
callback();
}
], function(err)
{
if(ERR(err, callback)) return;
callback();
// remove pad from all authors who contributed
this.getAllAuthors().forEach(authorID => {
authorManager.removePad(authorID, padID);
});
};
//set in db
// delete the pad entry and delete pad from padManager
padManager.removePad(padID);
hooks.callAll("padRemove", { padID });
}
// set in db
Pad.prototype.setPublicStatus = function setPublicStatus(publicStatus) {
this.publicStatus = publicStatus;
this.saveToDatabase();
@ -727,14 +523,14 @@ Pad.prototype.isPasswordProtected = function isPasswordProtected() {
};
Pad.prototype.addSavedRevision = function addSavedRevision(revNum, savedById, label) {
//if this revision is already saved, return silently
for(var i in this.savedRevisions){
if(this.savedRevisions[i] && this.savedRevisions[i].revNum === revNum){
// if this revision is already saved, return silently
for (var i in this.savedRevisions) {
if (this.savedRevisions[i] && this.savedRevisions[i].revNum === revNum) {
return;
}
}
//build the saved revision object
// build the saved revision object
var savedRevision = {};
savedRevision.revNum = revNum;
savedRevision.savedById = savedById;
@ -742,7 +538,7 @@ Pad.prototype.addSavedRevision = function addSavedRevision(revNum, savedById, la
savedRevision.timestamp = Date.now();
savedRevision.id = randomString(10);
//save this new saved revision
// save this new saved revision
this.savedRevisions.push(savedRevision);
this.saveToDatabase();
};
@ -753,19 +549,17 @@ Pad.prototype.getSavedRevisions = function getSavedRevisions() {
/* Crypto helper methods */
function hash(password, salt)
{
function hash(password, salt) {
var shasum = crypto.createHash('sha512');
shasum.update(password + salt);
return shasum.digest("hex") + "$" + salt;
}
function generateSalt()
{
function generateSalt() {
return randomString(86);
}
function compare(hashStr, password)
{
function compare(hashStr, password) {
return hash(password, hashStr.split("$")[1]) === hashStr;
}

280
src/node/db/PadManager.js
View file

@ -18,12 +18,11 @@
* limitations under the License.
*/
var ERR = require("async-stacktrace");
var customError = require("../utils/customError");
var Pad = require("../db/Pad").Pad;
var db = require("./DB").db;
var db = require("./DB");
/**
/**
* A cache of all loaded Pads.
*
* Provides "get" and "set" functions,
@ -35,12 +34,11 @@ var db = require("./DB").db;
* that's defined somewhere more sensible.
*/
var globalPads = {
get: function (name) { return this[':'+name]; },
set: function (name, value)
{
get: function(name) { return this[':'+name]; },
set: function(name, value) {
this[':'+name] = value;
},
remove: function (name) {
remove: function(name) {
delete this[':'+name];
}
};
@ -50,183 +48,151 @@ var globalPads = {
*
* Updated without db access as new pads are created/old ones removed.
*/
var padList = {
let padList = {
list: [],
sorted : false,
initiated: false,
init: function(cb)
{
db.findKeys("pad:*", "*:*:*", function(err, dbData)
{
if(ERR(err, cb)) return;
if(dbData != null){
padList.initiated = true
dbData.forEach(function(val){
padList.addPad(val.replace(/pad:/,""),false);
});
cb && cb()
init: async function() {
let dbData = await db.findKeys("pad:*", "*:*:*");
if (dbData != null) {
this.initiated = true;
for (let val of dbData) {
this.addPad(val.replace(/pad:/,""), false);
}
});
}
return this;
},
load: function(cb) {
if(this.initiated) cb && cb()
else this.init(cb)
load: async function() {
if (!this.initiated) {
return this.init();
}
return this;
},
/**
* Returns all pads in alphabetical order as array.
*/
getPads: function(cb){
this.load(function() {
if(!padList.sorted){
padList.list = padList.list.sort();
padList.sorted = true;
}
cb && cb(padList.list);
})
getPads: async function() {
await this.load();
if (!this.sorted) {
this.list.sort();
this.sorted = true;
}
return this.list;
},
addPad: function(name)
{
if(!this.initiated) return;
if(this.list.indexOf(name) == -1){
addPad: function(name) {
if (!this.initiated) return;
if (this.list.indexOf(name) == -1) {
this.list.push(name);
this.sorted=false;
this.sorted = false;
}
},
removePad: function(name)
{
if(!this.initiated) return;
removePad: function(name) {
if (!this.initiated) return;
var index = this.list.indexOf(name);
if(index>-1){
this.list.splice(index,1);
this.sorted=false;
if (index > -1) {
this.list.splice(index, 1);
this.sorted = false;
}
}
};
//initialises the allknowing data structure
// initialises the all-knowing data structure
/**
* Returns a Pad Object with the callback
* @param id A String with the id of the pad
* @param {Function} callback
*/
exports.getPad = async function(id, text)
{
// check if this is a valid padId
if (!exports.isValidPadId(id)) {
throw new customError(id + " is not a valid padId", "apierror");
}
// check if this is a valid text
if (text != null) {
// check if text is a string
if (typeof text != "string") {
throw new customError("text is not a string", "apierror");
}
// check if text is less than 100k chars
if (text.length > 100000) {
throw new customError("text must be less than 100k chars", "apierror");
}
}
let pad = globalPads.get(id);
// return pad if it's already loaded
if (pad != null) {
return pad;
}
// try to load pad
pad = new Pad(id);
// initalize the pad
await pad.init(text);
globalPads.set(id, pad);
padList.addPad(id);
return pad;
}
exports.listAllPads = async function()
{
let padIDs = await padList.getPads();
return { padIDs };
}
// checks if a pad exists
exports.doesPadExist = async function(padId)
{
let value = await db.get("pad:" + padId);
return (value != null && value.atext);
}
// alias for backwards compatibility
exports.doesPadExists = exports.doesPadExist;
/**
* An array of padId transformations. These represent changes in pad name policy over
* time, and allow us to "play back" these changes so legacy padIds can be found.
*/
var padIdTransforms = [
const padIdTransforms = [
[/\s+/g, '_'],
[/:+/g, '_']
];
/**
* Returns a Pad Object with the callback
* @param id A String with the id of the pad
* @param {Function} callback
*/
exports.getPad = function(id, text, callback)
{
//check if this is a valid padId
if(!exports.isValidPadId(id))
{
callback(new customError(id + " is not a valid padId","apierror"));
return;
}
//make text an optional parameter
if(typeof text == "function")
{
callback = text;
text = null;
}
//check if this is a valid text
if(text != null)
{
//check if text is a string
if(typeof text != "string")
{
callback(new customError("text is not a string","apierror"));
return;
// returns a sanitized padId, respecting legacy pad id formats
exports.sanitizePadId = async function sanitizePadId(padId) {
for (let i = 0, n = padIdTransforms.length; i < n; ++i) {
let exists = await exports.doesPadExist(padId);
if (exists) {
return padId;
}
//check if text is less than 100k chars
if(text.length > 100000)
{
callback(new customError("text must be less than 100k chars","apierror"));
return;
}
}
var pad = globalPads.get(id);
//return pad if its already loaded
if(pad != null)
{
callback(null, pad);
return;
let [from, to] = padIdTransforms[i];
padId = padId.replace(from, to);
}
//try to load pad
pad = new Pad(id);
//initalize the pad
pad.init(text, function(err)
{
if(ERR(err, callback)) return;
globalPads.set(id, pad);
padList.addPad(id);
callback(null, pad);
});
}
exports.listAllPads = function(cb)
{
padList.getPads(function(list) {
cb && cb(null, {padIDs: list});
});
}
//checks if a pad exists
exports.doesPadExists = function(padId, callback)
{
db.get("pad:"+padId, function(err, value)
{
if(ERR(err, callback)) return;
if(value != null && value.atext){
callback(null, true);
}
else
{
callback(null, false);
}
});
}
//returns a sanitized padId, respecting legacy pad id formats
exports.sanitizePadId = function(padId, callback) {
var transform_index = arguments[2] || 0;
//we're out of possible transformations, so just return it
if(transform_index >= padIdTransforms.length)
{
callback(padId);
return;
}
//check if padId exists
exports.doesPadExists(padId, function(junk, exists)
{
if(exists)
{
callback(padId);
return;
}
//get the next transformation *that's different*
var transformedPadId = padId;
while(transformedPadId == padId && transform_index < padIdTransforms.length)
{
transformedPadId = padId.replace(padIdTransforms[transform_index][0], padIdTransforms[transform_index][1]);
transform_index += 1;
}
//check the next transform
exports.sanitizePadId(transformedPadId, callback, transform_index);
});
// we're out of possible transformations, so just return it
return padId;
}
exports.isValidPadId = function(padId)
@ -237,13 +203,13 @@ exports.isValidPadId = function(padId)
/**
* Removes the pad from database and unloads it.
*/
exports.removePad = function(padId){
db.remove("pad:"+padId);
exports.removePad = function(padId) {
db.remove("pad:" + padId);
exports.unloadPad(padId);
padList.removePad(padId);
}
//removes a pad from the cache
// removes a pad from the cache
exports.unloadPad = function(padId)
{
globalPads.remove(padId);

85
src/node/db/ReadOnlyManager.js
View file

@ -19,80 +19,47 @@
*/
var ERR = require("async-stacktrace");
var db = require("./DB").db;
var async = require("async");
var db = require("./DB");
var randomString = require("../utils/randomstring");
/**
* returns a read only id for a pad
* @param {String} padId the id of the pad
*/
exports.getReadOnlyId = function (padId, callback)
{
var readOnlyId;
async.waterfall([
//check if there is a pad2readonly entry
function(callback)
{
db.get("pad2readonly:" + padId, callback);
},
function(dbReadOnlyId, callback)
{
//there is no readOnly Entry in the database, let's create one
if(dbReadOnlyId == null)
{
readOnlyId = "r." + randomString(16);
db.set("pad2readonly:" + padId, readOnlyId);
db.set("readonly2pad:" + readOnlyId, padId);
}
//there is a readOnly Entry in the database, let's take this one
else
{
readOnlyId = dbReadOnlyId;
}
callback();
}
], function(err)
{
if(ERR(err, callback)) return;
//return the results
callback(null, readOnlyId);
})
exports.getReadOnlyId = async function (padId)
{
// check if there is a pad2readonly entry
let readOnlyId = await db.get("pad2readonly:" + padId);
// there is no readOnly Entry in the database, let's create one
if (readOnlyId == null) {
readOnlyId = "r." + randomString(16);
db.set("pad2readonly:" + padId, readOnlyId);
db.set("readonly2pad:" + readOnlyId, padId);
}
return readOnlyId;
}
/**
* returns a the padId for a read only id
* returns the padId for a read only id
* @param {String} readOnlyId read only id
*/
exports.getPadId = function(readOnlyId, callback)
exports.getPadId = function(readOnlyId)
{
db.get("readonly2pad:" + readOnlyId, callback);
return db.get("readonly2pad:" + readOnlyId);
}
/**
* returns a the padId and readonlyPadId in an object for any id
* returns the padId and readonlyPadId in an object for any id
* @param {String} padIdOrReadonlyPadId read only id or real pad id
*/
exports.getIds = function(id, callback) {
if (id.indexOf("r.") == 0)
exports.getPadId(id, function (err, value) {
if(ERR(err, callback)) return;
callback(null, {
readOnlyPadId: id,
padId: value, // Might be null, if this is an unknown read-only id
readonly: true
});
});
else
exports.getReadOnlyId(id, function (err, value) {
callback(null, {
readOnlyPadId: value,
padId: id,
readonly: false
});
});
exports.getIds = async function(id) {
let readonly = (id.indexOf("r.") === 0);
// Might be null, if this is an unknown read-only id
let readOnlyPadId = readonly ? id : await exports.getReadOnlyId(id);
let padId = readonly ? await exports.getPadId(id) : id;
return { readOnlyPadId, padId, readonly };
}

496
src/node/db/SecurityManager.js
View file

@ -18,9 +18,6 @@
* limitations under the License.
*/
var ERR = require("async-stacktrace");
var async = require("async");
var authorManager = require("./AuthorManager");
var hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks.js");
var padManager = require("./PadManager");
@ -34,296 +31,231 @@ var authLogger = log4js.getLogger("auth");
* @param padID the pad the user wants to access
* @param sessionCookie the session the user has (set via api)
* @param token the token of the author (randomly generated at client side, used for public pads)
* @param password the password the user has given to access this pad, can be null
* @param callback will be called with (err, {accessStatus: grant|deny|wrongPassword|needPassword, authorID: a.xxxxxx})
*/
exports.checkAccess = function (padID, sessionCookie, token, password, callback)
{
var statusObject;
if(!padID) {
callback(null, {accessStatus: "deny"});
return;
* @param password the password the user has given to access this pad, can be null
* @return {accessStatus: grant|deny|wrongPassword|needPassword, authorID: a.xxxxxx})
*/
exports.checkAccess = async function(padID, sessionCookie, token, password)
{
// immutable object
let deny = Object.freeze({ accessStatus: "deny" });
if (!padID) {
return deny;
}
// allow plugins to deny access
var deniedByHook = hooks.callAll("onAccessCheck", {'padID': padID, 'password': password, 'token': token, 'sessionCookie': sessionCookie}).indexOf(false) > -1;
if(deniedByHook)
{
callback(null, {accessStatus: "deny"});
return;
if (deniedByHook) {
return deny;
}
// a valid session is required (api-only mode)
if(settings.requireSession)
{
// without sessionCookie, access is denied
if(!sessionCookie)
{
callback(null, {accessStatus: "deny"});
return;
// start to get author for this token
let p_tokenAuthor = authorManager.getAuthor4Token(token);
// start to check if pad exists
let p_padExists = padManager.doesPadExist(padID);
if (settings.requireSession) {
// a valid session is required (api-only mode)
if (!sessionCookie) {
// without sessionCookie, access is denied
return deny;
}
}
// a session is not required, so we'll check if it's a public pad
else
{
// it's not a group pad, means we can grant access
if(padID.indexOf("$") == -1)
{
//get author for this token
authorManager.getAuthor4Token(token, function(err, author)
{
if(ERR(err, callback)) return;
// assume user has access
statusObject = {accessStatus: "grant", authorID: author};
} else {
// a session is not required, so we'll check if it's a public pad
if (padID.indexOf("$") === -1) {
// it's not a group pad, means we can grant access
// assume user has access
let authorID = await p_tokenAuthor;
let statusObject = { accessStatus: "grant", authorID };
if (settings.editOnly) {
// user can't create pads
if(settings.editOnly)
{
// check if pad exists
padManager.doesPadExists(padID, function(err, exists)
{
if(ERR(err, callback)) return;
// pad doesn't exist - user can't have access
if(!exists) statusObject.accessStatus = "deny";
// grant or deny access, with author of token
callback(null, statusObject);
});
return;
}
let padExists = await p_padExists;
// user may create new pads - no need to check anything
// grant access, with author of token
callback(null, statusObject);
});
//don't continue
return;
}
}
var groupID = padID.split("$")[0];
var padExists = false;
var validSession = false;
var sessionAuthor;
var tokenAuthor;
var isPublic;
var isPasswordProtected;
var passwordStatus = password == null ? "notGiven" : "wrong"; // notGiven, correct, wrong
async.series([
//get basic informations from the database
function(callback)
{
async.parallel([
//does pad exists
function(callback)
{
padManager.doesPadExists(padID, function(err, exists)
{
if(ERR(err, callback)) return;
padExists = exists;
callback();
});
},
//get information about all sessions contained in this cookie
function(callback)
{
if (!sessionCookie)
{
callback();
return;
}
var sessionIDs = sessionCookie.split(',');
async.forEach(sessionIDs, function(sessionID, callback)
{
sessionManager.getSessionInfo(sessionID, function(err, sessionInfo)
{
//skip session if it doesn't exist
if(err && err.message == "sessionID does not exist")
{
authLogger.debug("Auth failed: unknown session");
callback();
return;
}
if(ERR(err, callback)) return;
var now = Math.floor(Date.now()/1000);
//is it for this group?
if(sessionInfo.groupID != groupID)
{
authLogger.debug("Auth failed: wrong group");
callback();
return;
}
//is validUntil still ok?
if(sessionInfo.validUntil <= now)
{
authLogger.debug("Auth failed: validUntil");
callback();
return;
}
// There is a valid session
validSession = true;
sessionAuthor = sessionInfo.authorID;
callback();
});
}, callback);
},
//get author for token
function(callback)
{
//get author for this token
authorManager.getAuthor4Token(token, function(err, author)
{
if(ERR(err, callback)) return;
tokenAuthor = author;
callback();
});
}
], callback);
},
//get more informations of this pad, if avaiable
function(callback)
{
//skip this if the pad doesn't exists
if(padExists == false)
{
callback();
return;
}
padManager.getPad(padID, function(err, pad)
{
if(ERR(err, callback)) return;
//is it a public pad?
isPublic = pad.getPublicStatus();
//is it password protected?
isPasswordProtected = pad.isPasswordProtected();
//is password correct?
if(isPasswordProtected && password && pad.isCorrectPassword(password))
{
passwordStatus = "correct";
}
callback();
});
},
function(callback)
{
//- a valid session for this group is avaible AND pad exists
if(validSession && padExists)
{
//- the pad is not password protected
if(!isPasswordProtected)
{
//--> grant access
statusObject = {accessStatus: "grant", authorID: sessionAuthor};
}
//- the setting to bypass password validation is set
else if(settings.sessionNoPassword)
{
//--> grant access
statusObject = {accessStatus: "grant", authorID: sessionAuthor};
}
//- the pad is password protected and password is correct
else if(isPasswordProtected && passwordStatus == "correct")
{
//--> grant access
statusObject = {accessStatus: "grant", authorID: sessionAuthor};
}
//- the pad is password protected but wrong password given
else if(isPasswordProtected && passwordStatus == "wrong")
{
//--> deny access, ask for new password and tell them that the password is wrong
statusObject = {accessStatus: "wrongPassword"};
}
//- the pad is password protected but no password given
else if(isPasswordProtected && passwordStatus == "notGiven")
{
//--> ask for password
statusObject = {accessStatus: "needPassword"};
}
else
{
throw new Error("Ops, something wrong happend");
}
}
//- a valid session for this group avaible but pad doesn't exists
else if(validSession && !padExists)
{
//--> grant access
statusObject = {accessStatus: "grant", authorID: sessionAuthor};
//--> deny access if user isn't allowed to create the pad
if(settings.editOnly)
{
authLogger.debug("Auth failed: valid session & pad does not exist");
if (!padExists) {
// pad doesn't exist - user can't have access
statusObject.accessStatus = "deny";
}
}
// there is no valid session avaiable AND pad exists
else if(!validSession && padExists)
{
//-- its public and not password protected
if(isPublic && !isPasswordProtected)
{
//--> grant access, with author of token
statusObject = {accessStatus: "grant", authorID: tokenAuthor};
}
//- its public and password protected and password is correct
else if(isPublic && isPasswordProtected && passwordStatus == "correct")
{
//--> grant access, with author of token
statusObject = {accessStatus: "grant", authorID: tokenAuthor};
}
//- its public and the pad is password protected but wrong password given
else if(isPublic && isPasswordProtected && passwordStatus == "wrong")
{
//--> deny access, ask for new password and tell them that the password is wrong
statusObject = {accessStatus: "wrongPassword"};
}
//- its public and the pad is password protected but no password given
else if(isPublic && isPasswordProtected && passwordStatus == "notGiven")
{
//--> ask for password
statusObject = {accessStatus: "needPassword"};
}
//- its not public
else if(!isPublic)
{
authLogger.debug("Auth failed: invalid session & pad is not public");
//--> deny access
statusObject = {accessStatus: "deny"};
}
else
{
throw new Error("Ops, something wrong happend");
}
}
// there is no valid session avaiable AND pad doesn't exists
else
{
authLogger.debug("Auth failed: invalid session & pad does not exist");
//--> deny access
statusObject = {accessStatus: "deny"};
}
callback();
// user may create new pads - no need to check anything
// grant access, with author of token
return statusObject;
}
], function(err)
{
if(ERR(err, callback)) return;
callback(null, statusObject);
});
};
}
let validSession = false;
let sessionAuthor;
let isPublic;
let isPasswordProtected;
let passwordStatus = password == null ? "notGiven" : "wrong"; // notGiven, correct, wrong
// get information about all sessions contained in this cookie
if (sessionCookie) {
let groupID = padID.split("$")[0];
let sessionIDs = sessionCookie.split(',');
// was previously iterated in parallel using async.forEach
let sessionInfos = await Promise.all(sessionIDs.map(sessionID => {
return sessionManager.getSessionInfo(sessionID);
}));
// seperated out the iteration of sessioninfos from the (parallel) fetches from the DB
for (let sessionInfo of sessionInfos) {
try {
// is it for this group?
if (sessionInfo.groupID != groupID) {
authLogger.debug("Auth failed: wrong group");
continue;
}
// is validUntil still ok?
let now = Math.floor(Date.now() / 1000);
if (sessionInfo.validUntil <= now) {
authLogger.debug("Auth failed: validUntil");
continue;
}
// fall-through - there is a valid session
validSession = true;
sessionAuthor = sessionInfo.authorID;
break;
} catch (err) {
// skip session if it doesn't exist
if (err.message == "sessionID does not exist") {
authLogger.debug("Auth failed: unknown session");
} else {
throw err;
}
}
}
}
let padExists = await p_padExists;
if (padExists) {
let pad = await padManager.getPad(padID);
// is it a public pad?
isPublic = pad.getPublicStatus();
// is it password protected?
isPasswordProtected = pad.isPasswordProtected();
// is password correct?
if (isPasswordProtected && password && pad.isCorrectPassword(password)) {
passwordStatus = "correct";
}
}
// - a valid session for this group is avaible AND pad exists
if (validSession && padExists) {
let authorID = sessionAuthor;
let grant = Object.freeze({ accessStatus: "grant", authorID });
if (!isPasswordProtected) {
// - the pad is not password protected
// --> grant access
return grant;
}
if (settings.sessionNoPassword) {
// - the setting to bypass password validation is set
// --> grant access
return grant;
}
if (isPasswordProtected && passwordStatus === "correct") {
// - the pad is password protected and password is correct
// --> grant access
return grant;
}
if (isPasswordProtected && passwordStatus === "wrong") {
// - the pad is password protected but wrong password given
// --> deny access, ask for new password and tell them that the password is wrong
return { accessStatus: "wrongPassword" };
}
if (isPasswordProtected && passwordStatus === "notGiven") {
// - the pad is password protected but no password given
// --> ask for password
return { accessStatus: "needPassword" };
}
throw new Error("Oops, something wrong happend");
}
if (validSession && !padExists) {
// - a valid session for this group avaible but pad doesn't exist
// --> grant access by default
let accessStatus = "grant";
let authorID = sessionAuthor;
// --> deny access if user isn't allowed to create the pad
if (settings.editOnly) {
authLogger.debug("Auth failed: valid session & pad does not exist");
accessStatus = "deny";
}
return { accessStatus, authorID };
}
if (!validSession && padExists) {
// there is no valid session avaiable AND pad exists
let authorID = await p_tokenAuthor;
let grant = Object.freeze({ accessStatus: "grant", authorID });
if (isPublic && !isPasswordProtected) {
// -- it's public and not password protected
// --> grant access, with author of token
return grant;
}
if (isPublic && isPasswordProtected && passwordStatus === "correct") {
// - it's public and password protected and password is correct
// --> grant access, with author of token
return grant;
}
if (isPublic && isPasswordProtected && passwordStatus === "wrong") {
// - it's public and the pad is password protected but wrong password given
// --> deny access, ask for new password and tell them that the password is wrong
return { accessStatus: "wrongPassword" };
}
if (isPublic && isPasswordProtected && passwordStatus === "notGiven") {
// - it's public and the pad is password protected but no password given
// --> ask for password
return { accessStatus: "needPassword" };
}
if (!isPublic) {
// - it's not public
authLogger.debug("Auth failed: invalid session & pad is not public");
// --> deny access
return { accessStatus: "deny" };
}
throw new Error("Oops, something wrong happend");
}
// there is no valid session avaiable AND pad doesn't exist
authLogger.debug("Auth failed: invalid session & pad does not exist");
return { accessStatus: "deny" };
}

495
src/node/db/SessionManager.js
View file

@ -17,361 +17,208 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
var ERR = require("async-stacktrace");
var customError = require("../utils/customError");
var randomString = require("../utils/randomstring");
var db = require("./DB").db;
var async = require("async");
var groupMangager = require("./GroupManager");
var authorMangager = require("./AuthorManager");
exports.doesSessionExist = function(sessionID, callback)
var db = require("./DB");
var groupManager = require("./GroupManager");
var authorManager = require("./AuthorManager");
exports.doesSessionExist = async function(sessionID)
{
//check if the database entry of this session exists
db.get("session:" + sessionID, function (err, session)
{
if(ERR(err, callback)) return;
callback(null, session != null);
});
let session = await db.get("session:" + sessionID);
return (session !== null);
}
/**
* Creates a new session between an author and a group
*/
exports.createSession = function(groupID, authorID, validUntil, callback)
exports.createSession = async function(groupID, authorID, validUntil)
{
var sessionID;
// check if the group exists
let groupExists = await groupManager.doesGroupExist(groupID);
if (!groupExists) {
throw new customError("groupID does not exist", "apierror");
}
async.series([
//check if group exists
function(callback)
{
groupMangager.doesGroupExist(groupID, function(err, exists)
{
if(ERR(err, callback)) return;
//group does not exist
if(exists == false)
{
callback(new customError("groupID does not exist","apierror"));
}
//everything is fine, continue
else
{
callback();
}
});
},
//check if author exists
function(callback)
{
authorMangager.doesAuthorExists(authorID, function(err, exists)
{
if(ERR(err, callback)) return;
//author does not exist
if(exists == false)
{
callback(new customError("authorID does not exist","apierror"));
}
//everything is fine, continue
else
{
callback();
}
});
},
//check validUntil and create the session db entry
function(callback)
{
//check if rev is a number
if(typeof validUntil != "number")
{
//try to parse the number
if(isNaN(parseInt(validUntil)))
{
callback(new customError("validUntil is not a number","apierror"));
return;
}
// check if the author exists
let authorExists = await authorManager.doesAuthorExist(authorID);
if (!authorExists) {
throw new customError("authorID does not exist", "apierror");
}
validUntil = parseInt(validUntil);
}
//ensure this is not a negativ number
if(validUntil < 0)
{
callback(new customError("validUntil is a negativ number","apierror"));
return;
}
//ensure this is not a float value
if(!is_int(validUntil))
{
callback(new customError("validUntil is a float value","apierror"));
return;
}
//check if validUntil is in the future
if(Math.floor(Date.now()/1000) > validUntil)
{
callback(new customError("validUntil is in the past","apierror"));
return;
}
//generate sessionID
sessionID = "s." + randomString(16);
//set the session into the database
db.set("session:" + sessionID, {"groupID": groupID, "authorID": authorID, "validUntil": validUntil});
callback();
},
//set the group2sessions entry
function(callback)
{
//get the entry
db.get("group2sessions:" + groupID, function(err, group2sessions)
{
if(ERR(err, callback)) return;
//the entry doesn't exist so far, let's create it
if(group2sessions == null || group2sessions.sessionIDs == null)
{
group2sessions = {sessionIDs : {}};
}
//add the entry for this session
group2sessions.sessionIDs[sessionID] = 1;
//save the new element back
db.set("group2sessions:" + groupID, group2sessions);
callback();
});
},
//set the author2sessions entry
function(callback)
{
//get the entry
db.get("author2sessions:" + authorID, function(err, author2sessions)
{
if(ERR(err, callback)) return;
//the entry doesn't exist so far, let's create it
if(author2sessions == null || author2sessions.sessionIDs == null)
{
author2sessions = {sessionIDs : {}};
}
//add the entry for this session
author2sessions.sessionIDs[sessionID] = 1;
//save the new element back
db.set("author2sessions:" + authorID, author2sessions);
callback();
});
}
], function(err)
{
if(ERR(err, callback)) return;
//return error and sessionID
callback(null, {sessionID: sessionID});
})
// try to parse validUntil if it's not a number
if (typeof validUntil !== "number") {
validUntil = parseInt(validUntil);
}
// check it's a valid number
if (isNaN(validUntil)) {
throw new customError("validUntil is not a number", "apierror");
}
// ensure this is not a negative number
if (validUntil < 0) {
throw new customError("validUntil is a negative number", "apierror");
}
// ensure this is not a float value
if (!is_int(validUntil)) {
throw new customError("validUntil is a float value", "apierror");
}
// check if validUntil is in the future
if (validUntil < Math.floor(Date.now() / 1000)) {
throw new customError("validUntil is in the past", "apierror");
}
// generate sessionID
let sessionID = "s." + randomString(16);
// set the session into the database
await db.set("session:" + sessionID, {"groupID": groupID, "authorID": authorID, "validUntil": validUntil});
// get the entry
let group2sessions = await db.get("group2sessions:" + groupID);
/*
* In some cases, the db layer could return "undefined" as well as "null".
* Thus, it is not possible to perform strict null checks on group2sessions.
* In a previous version of this code, a strict check broke session
* management.
*
* See: https://github.com/ether/etherpad-lite/issues/3567#issuecomment-468613960
*/
if (!group2sessions || !group2sessions.sessionIDs) {
// the entry doesn't exist so far, let's create it
group2sessions = {sessionIDs : {}};
}
// add the entry for this session
group2sessions.sessionIDs[sessionID] = 1;
// save the new element back
await db.set("group2sessions:" + groupID, group2sessions);
// get the author2sessions entry
let author2sessions = await db.get("author2sessions:" + authorID);
if (author2sessions == null || author2sessions.sessionIDs == null) {
// the entry doesn't exist so far, let's create it
author2sessions = {sessionIDs : {}};
}
// add the entry for this session
author2sessions.sessionIDs[sessionID] = 1;
//save the new element back
await db.set("author2sessions:" + authorID, author2sessions);
return { sessionID };
}
exports.getSessionInfo = function(sessionID, callback)
exports.getSessionInfo = async function(sessionID)
{
//check if the database entry of this session exists
db.get("session:" + sessionID, function (err, session)
{
if(ERR(err, callback)) return;
//session does not exists
if(session == null)
{
callback(new customError("sessionID does not exist","apierror"))
}
//everything is fine, return the sessioninfos
else
{
callback(null, session);
}
});
// check if the database entry of this session exists
let session = await db.get("session:" + sessionID);
if (session == null) {
// session does not exist
throw new customError("sessionID does not exist", "apierror");
}
// everything is fine, return the sessioninfos
return session;
}
/**
* Deletes a session
*/
exports.deleteSession = function(sessionID, callback)
exports.deleteSession = async function(sessionID)
{
var authorID, groupID;
var group2sessions, author2sessions;
// ensure that the session exists
let session = await db.get("session:" + sessionID);
if (session == null) {
throw new customError("sessionID does not exist", "apierror");
}
async.series([
function(callback)
{
//get the session entry
db.get("session:" + sessionID, function (err, session)
{
if(ERR(err, callback)) return;
//session does not exists
if(session == null)
{
callback(new customError("sessionID does not exist","apierror"))
}
//everything is fine, return the sessioninfos
else
{
authorID = session.authorID;
groupID = session.groupID;
callback();
}
});
},
//get the group2sessions entry
function(callback)
{
db.get("group2sessions:" + groupID, function (err, _group2sessions)
{
if(ERR(err, callback)) return;
group2sessions = _group2sessions;
callback();
});
},
//get the author2sessions entry
function(callback)
{
db.get("author2sessions:" + authorID, function (err, _author2sessions)
{
if(ERR(err, callback)) return;
author2sessions = _author2sessions;
callback();
});
},
//remove the values from the database
function(callback)
{
//remove the session
db.remove("session:" + sessionID);
//remove session from group2sessions
if(group2sessions != null) { // Maybe the group was already deleted
delete group2sessions.sessionIDs[sessionID];
db.set("group2sessions:" + groupID, group2sessions);
}
// everything is fine, use the sessioninfos
let groupID = session.groupID;
let authorID = session.authorID;
//remove session from author2sessions
if(author2sessions != null) { // Maybe the author was already deleted
delete author2sessions.sessionIDs[sessionID];
db.set("author2sessions:" + authorID, author2sessions);
}
callback();
}
], function(err)
{
if(ERR(err, callback)) return;
callback();
})
// get the group2sessions and author2sessions entries
let group2sessions = await db.get("group2sessions:" + groupID);
let author2sessions = await db.get("author2sessions:" + authorID);
// remove the session
await db.remove("session:" + sessionID);
// remove session from group2sessions
if (group2sessions != null) { // Maybe the group was already deleted
delete group2sessions.sessionIDs[sessionID];
await db.set("group2sessions:" + groupID, group2sessions);
}
// remove session from author2sessions
if (author2sessions != null) { // Maybe the author was already deleted
delete author2sessions.sessionIDs[sessionID];
await db.set("author2sessions:" + authorID, author2sessions);
}
}
exports.listSessionsOfGroup = function(groupID, callback)
exports.listSessionsOfGroup = async function(groupID)
{
groupMangager.doesGroupExist(groupID, function(err, exists)
{
if(ERR(err, callback)) return;
//group does not exist
if(exists == false)
{
callback(new customError("groupID does not exist","apierror"));
}
//everything is fine, continue
else
{
listSessionsWithDBKey("group2sessions:" + groupID, callback);
}
});
// check that the group exists
let exists = await groupManager.doesGroupExist(groupID);
if (!exists) {
throw new customError("groupID does not exist", "apierror");
}
let sessions = await listSessionsWithDBKey("group2sessions:" + groupID);
return sessions;
}
exports.listSessionsOfAuthor = function(authorID, callback)
{
authorMangager.doesAuthorExists(authorID, function(err, exists)
{
if(ERR(err, callback)) return;
//group does not exist
if(exists == false)
{
callback(new customError("authorID does not exist","apierror"));
}
//everything is fine, continue
else
{
listSessionsWithDBKey("author2sessions:" + authorID, callback);
}
});
}
//this function is basicly the code listSessionsOfAuthor and listSessionsOfGroup has in common
function listSessionsWithDBKey (dbkey, callback)
exports.listSessionsOfAuthor = async function(authorID)
{
var sessions;
// check that the author exists
let exists = await authorManager.doesAuthorExist(authorID)
if (!exists) {
throw new customError("authorID does not exist", "apierror");
}
async.series([
function(callback)
{
//get the group2sessions entry
db.get(dbkey, function(err, sessionObject)
{
if(ERR(err, callback)) return;
sessions = sessionObject ? sessionObject.sessionIDs : null;
callback();
});
},
function(callback)
{
//collect all sessionIDs in an arrary
var sessionIDs = [];
for (var i in sessions)
{
sessionIDs.push(i);
}
//foreach trough the sessions and get the sessioninfos
async.forEach(sessionIDs, function(sessionID, callback)
{
exports.getSessionInfo(sessionID, function(err, sessionInfo)
{
if (err == "apierror: sessionID does not exist")
{
console.warn(`Found bad session ${sessionID} in ${dbkey}`);
}
else if(ERR(err, callback))
{
return;
}
sessions[sessionID] = sessionInfo;
callback();
});
}, callback);
}
], function(err)
{
if(ERR(err, callback)) return;
callback(null, sessions);
});
let sessions = await listSessionsWithDBKey("author2sessions:" + authorID);
return sessions;
}
//checks if a number is an int
// this function is basically the code listSessionsOfAuthor and listSessionsOfGroup has in common
// required to return null rather than an empty object if there are none
async function listSessionsWithDBKey(dbkey)
{
// get the group2sessions entry
let sessionObject = await db.get(dbkey);
let sessions = sessionObject ? sessionObject.sessionIDs : null;
// iterate through the sessions and get the sessioninfos
for (let sessionID in sessions) {
try {
let sessionInfo = await exports.getSessionInfo(sessionID);
sessions[sessionID] = sessionInfo;
} catch (err) {
if (err == "apierror: sessionID does not exist") {
console.warn(`Found bad session ${sessionID} in ${dbkey}`);
sessions[sessionID] = null;
} else {
throw err;
}
}
}
return sessions;
}
// checks if a number is an int
function is_int(value)
{
return (parseFloat(value) == parseInt(value)) && !isNaN(value)
{
return (parseFloat(value) == parseInt(value)) && !isNaN(value);
}

98
src/node/db/SessionStore.js
View file

@ -1,7 +1,10 @@
/*
/*
* Stores session data in the database
* Source; https://github.com/edy-b/SciFlowWriter/blob/develop/available_plugins/ep_sciflowwriter/db/DirtyStore.js
* This is not used for authors that are created via the API at current
*
* RPB: this module was not migrated to Promises, because it is only used via
* express-session, which can't actually use promises anyway.
*/
var Store = require('ep_etherpad-lite/node_modules/express-session').Store,
@ -13,11 +16,12 @@ var SessionStore = module.exports = function SessionStore() {};
SessionStore.prototype.__proto__ = Store.prototype;
SessionStore.prototype.get = function(sid, fn){
SessionStore.prototype.get = function(sid, fn) {
messageLogger.debug('GET ' + sid);
var self = this;
db.get("sessionstorage:" + sid, function (err, sess)
{
db.get("sessionstorage:" + sid, function(err, sess) {
if (sess) {
sess.cookie.expires = 'string' == typeof sess.cookie.expires ? new Date(sess.cookie.expires) : sess.cookie.expires;
if (!sess.cookie.expires || new Date() < sess.cookie.expires) {
@ -31,50 +35,64 @@ SessionStore.prototype.get = function(sid, fn){
});
};
SessionStore.prototype.set = function(sid, sess, fn){
SessionStore.prototype.set = function(sid, sess, fn) {
messageLogger.debug('SET ' + sid);
db.set("sessionstorage:" + sid, sess);
process.nextTick(function(){
if(fn) fn();
});
if (fn) {
process.nextTick(fn);
}
};
SessionStore.prototype.destroy = function(sid, fn){
SessionStore.prototype.destroy = function(sid, fn) {
messageLogger.debug('DESTROY ' + sid);
db.remove("sessionstorage:" + sid);
process.nextTick(function(){
if(fn) fn();
});
if (fn) {
process.nextTick(fn);
}
};
SessionStore.prototype.all = function(fn){
messageLogger.debug('ALL');
var sessions = [];
db.forEach(function(key, value){
if (key.substr(0,15) === "sessionstorage:") {
sessions.push(value);
}
});
fn(null, sessions);
};
/*
* RPB: the following methods are optional requirements for a compatible session
* store for express-session, but in any case appear to depend on a
* non-existent feature of ueberdb2
*/
if (db.forEach) {
SessionStore.prototype.all = function(fn) {
messageLogger.debug('ALL');
SessionStore.prototype.clear = function(fn){
messageLogger.debug('CLEAR');
db.forEach(function(key, value){
if (key.substr(0,15) === "sessionstorage:") {
db.db.remove("session:" + key);
}
});
if(fn) fn();
};
var sessions = [];
SessionStore.prototype.length = function(fn){
messageLogger.debug('LENGTH');
var i = 0;
db.forEach(function(key, value){
if (key.substr(0,15) === "sessionstorage:") {
i++;
}
});
fn(null, i);
db.forEach(function(key, value) {
if (key.substr(0,15) === "sessionstorage:") {
sessions.push(value);
}
});
fn(null, sessions);
};
SessionStore.prototype.clear = function(fn) {
messageLogger.debug('CLEAR');
db.forEach(function(key, value) {
if (key.substr(0,15) === "sessionstorage:") {
db.remove("session:" + key);
}
});
if (fn) fn();
};
SessionStore.prototype.length = function(fn) {
messageLogger.debug('LENGTH');
var i = 0;
db.forEach(function(key, value) {
if (key.substr(0,15) === "sessionstorage:") {
i++;
}
});
fn(null, i);
}
};

134
src/node/handler/APIHandler.js
View file

@ -19,7 +19,6 @@
*/
var absolutePaths = require('../utils/AbsolutePaths');
var ERR = require("async-stacktrace");
var fs = require("fs");
var api = require("../db/API");
var log4js = require('log4js');
@ -32,19 +31,17 @@ var apiHandlerLogger = log4js.getLogger('APIHandler');
//ensure we have an apikey
var apikey = null;
var apikeyFilename = absolutePaths.makeAbsolute(argv.apikey || "./APIKEY.txt");
try
{
try {
apikey = fs.readFileSync(apikeyFilename,"utf8");
apiHandlerLogger.info(`Api key file read from: "${apikeyFilename}"`);
}
catch(e)
{
} catch(e) {
apiHandlerLogger.info(`Api key file "${apikeyFilename}" not found. Creating with random contents.`);
apikey = randomString(32);
fs.writeFileSync(apikeyFilename,apikey,"utf8");
}
//a list of all functions
// a list of all functions
var version = {};
version["1"] = Object.assign({},
@ -152,110 +149,73 @@ exports.version = version;
* @req express request object
* @res express response object
*/
exports.handle = function(apiVersion, functionName, fields, req, res)
exports.handle = async function(apiVersion, functionName, fields, req, res)
{
//check if this is a valid apiversion
var isKnownApiVersion = false;
for(var knownApiVersion in version)
{
if(knownApiVersion == apiVersion)
{
isKnownApiVersion = true;
break;
}
}
//say goodbye if this is an unknown API version
if(!isKnownApiVersion)
{
// say goodbye if this is an unknown API version
if (!(apiVersion in version)) {
res.statusCode = 404;
res.send({code: 3, message: "no such api version", data: null});
return;
}
//check if this is a valid function name
var isKnownFunctionname = false;
for(var knownFunctionname in version[apiVersion])
{
if(knownFunctionname == functionName)
{
isKnownFunctionname = true;
break;
}
}
//say goodbye if this is a unknown function
if(!isKnownFunctionname)
{
// say goodbye if this is an unknown function
if (!(functionName in version[apiVersion])) {
// no status code?!
res.send({code: 3, message: "no such function", data: null});
return;
}
//check the api key!
// check the api key!
fields["apikey"] = fields["apikey"] || fields["api_key"];
if(fields["apikey"] != apikey.trim())
{
if (fields["apikey"] !== apikey.trim()) {
res.statusCode = 401;
res.send({code: 4, message: "no or wrong API Key", data: null});
return;
}
//sanitize any pad id's before continuing
if(fields["padID"])
{
padManager.sanitizePadId(fields["padID"], function(padId)
{
fields["padID"] = padId;
callAPI(apiVersion, functionName, fields, req, res);
});
// sanitize any padIDs before continuing
if (fields["padID"]) {
fields["padID"] = await padManager.sanitizePadId(fields["padID"]);
}
else if(fields["padName"])
{
padManager.sanitizePadId(fields["padName"], function(padId)
{
fields["padName"] = padId;
callAPI(apiVersion, functionName, fields, req, res);
});
}
else
{
callAPI(apiVersion, functionName, fields, req, res);
// there was an 'else' here before - removed it to ensure
// that this sanitize step can't be circumvented by forcing
// the first branch to be taken
if (fields["padName"]) {
fields["padName"] = await padManager.sanitizePadId(fields["padName"]);
}
// no need to await - callAPI returns a promise
return callAPI(apiVersion, functionName, fields, req, res);
}
//calls the api function
function callAPI(apiVersion, functionName, fields, req, res)
// calls the api function
async function callAPI(apiVersion, functionName, fields, req, res)
{
//put the function parameters in an array
// put the function parameters in an array
var functionParams = version[apiVersion][functionName].map(function (field) {
return fields[field]
})
//add a callback function to handle the response
functionParams.push(function(err, data)
{
// no error happend, everything is fine
if(err == null)
{
if(!data)
data = null;
res.send({code: 0, message: "ok", data: data});
}
// parameters were wrong and the api stopped execution, pass the error
else if(err.name == "apierror")
{
res.send({code: 1, message: err.message, data: null});
}
//an unknown error happend
else
{
res.send({code: 2, message: "internal error", data: null});
ERR(err);
}
});
//call the api function
api[functionName].apply(this, functionParams);
try {
// call the api function
let data = await api[functionName].apply(this, functionParams);
if (!data) {
data = null;
}
res.send({code: 0, message: "ok", data: data});
} catch (err) {
if (err.name == "apierror") {
// parameters were wrong and the api stopped execution, pass the error
res.send({code: 1, message: err.message, data: null});
} else {
// an unknown error happened
res.send({code: 2, message: "internal error", data: null});
throw err;
}
}
}

219
src/node/handler/ExportHandler.js
View file

@ -19,163 +19,122 @@
* limitations under the License.
*/
var ERR = require("async-stacktrace");
var exporthtml = require("../utils/ExportHtml");
var exporttxt = require("../utils/ExportTxt");
var exportEtherpad = require("../utils/ExportEtherpad");
var async = require("async");
var fs = require("fs");
var settings = require('../utils/Settings');
var os = require('os');
var hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks");
var TidyHtml = require('../utils/TidyHtml');
const util = require("util");
var convertor = null;
const fsp_writeFile = util.promisify(fs.writeFile);
const fsp_unlink = util.promisify(fs.unlink);
//load abiword only if its enabled
if(settings.abiword != null)
let convertor = null;
// load abiword only if it is enabled
if (settings.abiword != null) {
convertor = require("../utils/Abiword");
}
// Use LibreOffice if an executable has been defined in the settings
if(settings.soffice != null)
if (settings.soffice != null) {
convertor = require("../utils/LibreOffice");
}
const tempDirectory = os.tmpdir();
/**
* do a requested export
*/
exports.doExport = function(req, res, padId, type)
async function doExport(req, res, padId, type)
{
var fileName = padId;
// allow fileName to be overwritten by a hook, the type type is kept static for security reasons
hooks.aCallFirst("exportFileName", padId,
function(err, hookFileName){
// if fileName is set then set it to the padId, note that fileName is returned as an array.
if(hookFileName.length) fileName = hookFileName;
let hookFileName = await hooks.aCallFirst("exportFileName", padId);
//tell the browser that this is a downloadable file
res.attachment(fileName + "." + type);
// if fileName is set then set it to the padId, note that fileName is returned as an array.
if (hookFileName.length) {
fileName = hookFileName;
}
//if this is a plain text export, we can do this directly
// We have to over engineer this because tabs are stored as attributes and not plain text
if(type == "etherpad"){
exportEtherpad.getPadRaw(padId, function(err, pad){
if(!err){
res.send(pad);
// return;
}
});
}
else if(type == "txt")
{
exporttxt.getPadTXTDocument(padId, req.params.rev, function(err, txt)
{
if(!err) {
res.send(txt);
}
});
}
else
{
var html;
var randNum;
var srcFile, destFile;
// tell the browser that this is a downloadable file
res.attachment(fileName + "." + type);
async.series([
//render the html document
function(callback)
{
exporthtml.getPadHTMLDocument(padId, req.params.rev, function(err, _html)
{
if(ERR(err, callback)) return;
html = _html;
callback();
});
},
//decide what to do with the html export
function(callback)
{
//if this is a html export, we can send this from here directly
if(type == "html")
{
// do any final changes the plugin might want to make
hooks.aCallFirst("exportHTMLSend", html, function(err, newHTML){
if(newHTML.length) html = newHTML;
res.send(html);
callback("stop");
});
}
else //write the html export to a file
{
randNum = Math.floor(Math.random()*0xFFFFFFFF);
srcFile = tempDirectory + "/etherpad_export_" + randNum + ".html";
fs.writeFile(srcFile, html, callback);
}
},
// if this is a plain text export, we can do this directly
// We have to over engineer this because tabs are stored as attributes and not plain text
if (type === "etherpad") {
let pad = await exportEtherpad.getPadRaw(padId);
res.send(pad);
} else if (type === "txt") {
let txt = await exporttxt.getPadTXTDocument(padId, req.params.rev);
res.send(txt);
} else {
// render the html document
let html = await exporthtml.getPadHTMLDocument(padId, req.params.rev);
// Tidy up the exported HTML
function(callback)
{
//ensure html can be collected by the garbage collector
html = null;
// decide what to do with the html export
TidyHtml.tidy(srcFile, callback);
},
//send the convert job to the convertor (abiword, libreoffice, ..)
function(callback)
{
destFile = tempDirectory + "/etherpad_export_" + randNum + "." + type;
// Allow plugins to overwrite the convert in export process
hooks.aCallAll("exportConvert", {srcFile: srcFile, destFile: destFile, req: req, res: res}, function(err, result){
if(!err && result.length > 0){
// console.log("export handled by plugin", destFile);
handledByPlugin = true;
callback();
}else{
convertor.convertFile(srcFile, destFile, type, callback);
}
});
},
//send the file
function(callback)
{
res.sendFile(destFile, null, callback);
},
//clean up temporary files
function(callback)
{
async.parallel([
function(callback)
{
fs.unlink(srcFile, callback);
},
function(callback)
{
//100ms delay to accomidate for slow windows fs
if(os.type().indexOf("Windows") > -1)
{
setTimeout(function()
{
fs.unlink(destFile, callback);
}, 100);
}
else
{
fs.unlink(destFile, callback);
}
}
], callback);
}
], function(err)
{
if(err && err != "stop") ERR(err);
})
}
// if this is a html export, we can send this from here directly
if (type === "html") {
// do any final changes the plugin might want to make
let newHTML = await hooks.aCallFirst("exportHTMLSend", html);
if (newHTML.length) html = newHTML;
res.send(html);
throw "stop";
}
);
};
// else write the html export to a file
let randNum = Math.floor(Math.random()*0xFFFFFFFF);
let srcFile = tempDirectory + "/etherpad_export_" + randNum + ".html";
await fsp_writeFile(srcFile, html);
// Tidy up the exported HTML
// ensure html can be collected by the garbage collector
html = null;
await TidyHtml.tidy(srcFile);
// send the convert job to the convertor (abiword, libreoffice, ..)
let destFile = tempDirectory + "/etherpad_export_" + randNum + "." + type;
// Allow plugins to overwrite the convert in export process
let result = await hooks.aCallAll("exportConvert", { srcFile, destFile, req, res });
if (result.length > 0) {
// console.log("export handled by plugin", destFile);
handledByPlugin = true;
} else {
// @TODO no Promise interface for convertors (yet)
await new Promise((resolve, reject) => {
convertor.convertFile(srcFile, destFile, type, function(err) {
err ? reject("convertFailed") : resolve();
});
});
}
// send the file
let sendFile = util.promisify(res.sendFile);
await res.sendFile(destFile, null);
// clean up temporary files
await fsp_unlink(srcFile);
// 100ms delay to accommodate for slow windows fs
if (os.type().indexOf("Windows") > -1) {
await new Promise(resolve => setTimeout(resolve, 100));
}
await fsp_unlink(destFile);
}
}
exports.doExport = function(req, res, padId, type)
{
doExport(req, res, padId, type).catch(err => {
if (err !== "stop") {
throw err;
}
});
}

458
src/node/handler/ImportHandler.js
View file

@ -20,10 +20,8 @@
* limitations under the License.
*/
var ERR = require("async-stacktrace")
, padManager = require("../db/PadManager")
var padManager = require("../db/PadManager")
, padMessageHandler = require("./PadMessageHandler")
, async = require("async")
, fs = require("fs")
, path = require("path")
, settings = require('../utils/Settings')
@ -32,303 +30,241 @@ var ERR = require("async-stacktrace")
, importHtml = require("../utils/ImportHtml")
, importEtherpad = require("../utils/ImportEtherpad")
, log4js = require("log4js")
, hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks.js");
, hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks.js")
, util = require("util");
var convertor = null;
var exportExtension = "htm";
let fsp_exists = util.promisify(fs.exists);
let fsp_rename = util.promisify(fs.rename);
let fsp_readFile = util.promisify(fs.readFile);
let fsp_unlink = util.promisify(fs.unlink)
//load abiword only if its enabled and if soffice is disabled
if(settings.abiword != null && settings.soffice === null)
let convertor = null;
let exportExtension = "htm";
// load abiword only if it is enabled and if soffice is disabled
if (settings.abiword != null && settings.soffice === null) {
convertor = require("../utils/Abiword");
}
//load soffice only if its enabled
if(settings.soffice != null) {
// load soffice only if it is enabled
if (settings.soffice != null) {
convertor = require("../utils/LibreOffice");
exportExtension = "html";
}
const tmpDirectory = os.tmpdir();
/**
* do a requested import
*/
exports.doImport = function(req, res, padId)
*/
async function doImport(req, res, padId)
{
var apiLogger = log4js.getLogger("ImportHandler");
//pipe to a file
//convert file to html via abiword or soffice
//set html in the pad
var srcFile, destFile
, pad
, text
, importHandledByPlugin
, directDatabaseAccess
, useConvertor;
// pipe to a file
// convert file to html via abiword or soffice
// set html in the pad
var randNum = Math.floor(Math.random()*0xFFFFFFFF);
// setting flag for whether to use convertor or not
useConvertor = (convertor != null);
let useConvertor = (convertor != null);
async.series([
//save the uploaded file to /tmp
function(callback) {
var form = new formidable.IncomingForm();
form.keepExtensions = true;
form.uploadDir = tmpDirectory;
form.parse(req, function(err, fields, files) {
//the upload failed, stop at this point
if(err || files.file === undefined) {
if(err) console.warn("Uploading Error: " + err.stack);
callback("uploadFailed");
let form = new formidable.IncomingForm();
form.keepExtensions = true;
form.uploadDir = tmpDirectory;
return;
// locally wrapped Promise, since form.parse requires a callback
let srcFile = await new Promise((resolve, reject) => {
form.parse(req, function(err, fields, files) {
if (err || files.file === undefined) {
// the upload failed, stop at this point
if (err) {
console.warn("Uploading Error: " + err.stack);
}
//everything ok, continue
//save the path of the uploaded file
srcFile = files.file.path;
callback();
});
},
//ensure this is a file ending we know, else we change the file ending to .txt
//this allows us to accept source code files like .c or .java
function(callback) {
var fileEnding = path.extname(srcFile).toLowerCase()
, knownFileEndings = [".txt", ".doc", ".docx", ".pdf", ".odt", ".html", ".htm", ".etherpad", ".rtf"]
, fileEndingKnown = (knownFileEndings.indexOf(fileEnding) > -1);
//if the file ending is known, continue as normal
if(fileEndingKnown) {
callback();
return;
reject("uploadFailed");
}
resolve(files.file.path);
});
});
//we need to rename this file with a .txt ending
if(settings.allowUnknownFileEnds === true){
var oldSrcFile = srcFile;
srcFile = path.join(path.dirname(srcFile),path.basename(srcFile, fileEnding)+".txt");
fs.rename(oldSrcFile, srcFile, callback);
}else{
console.warn("Not allowing unknown file type to be imported", fileEnding);
callback("uploadFailed");
}
},
function(callback){
destFile = path.join(tmpDirectory, "etherpad_import_" + randNum + "." + exportExtension);
// ensure this is a file ending we know, else we change the file ending to .txt
// this allows us to accept source code files like .c or .java
let fileEnding = path.extname(srcFile).toLowerCase()
, knownFileEndings = [".txt", ".doc", ".docx", ".pdf", ".odt", ".html", ".htm", ".etherpad", ".rtf"]
, fileEndingUnknown = (knownFileEndings.indexOf(fileEnding) < 0);
// Logic for allowing external Import Plugins
hooks.aCallAll("import", {srcFile: srcFile, destFile: destFile}, function(err, result){
if(ERR(err, callback)) return callback();
if(result.length > 0){ // This feels hacky and wrong..
importHandledByPlugin = true;
}
callback();
});
},
function(callback) {
var fileEnding = path.extname(srcFile).toLowerCase()
var fileIsNotEtherpad = (fileEnding !== ".etherpad");
if (fileEndingUnknown) {
// the file ending is not known
if (fileIsNotEtherpad) {
callback();
if (settings.allowUnknownFileEnds === true) {
// we need to rename this file with a .txt ending
let oldSrcFile = srcFile;
return;
}
srcFile = path.join(path.dirname(srcFile), path.basename(srcFile, fileEnding) + ".txt");
await fs.rename(oldSrcFile, srcFile);
} else {
console.warn("Not allowing unknown file type to be imported", fileEnding);
throw "uploadFailed";
}
}
// we do this here so we can see if the pad has quit ea few edits
padManager.getPad(padId, function(err, _pad){
var headCount = _pad.head;
if(headCount >= 10){
apiLogger.warn("Direct database Import attempt of a pad that already has content, we wont be doing this")
return callback("padHasData");
}
let destFile = path.join(tmpDirectory, "etherpad_import_" + randNum + "." + exportExtension);
fs.readFile(srcFile, "utf8", function(err, _text){
directDatabaseAccess = true;
importEtherpad.setPadRaw(padId, _text, function(err){
callback();
});
});
});
},
//convert file to html
function(callback) {
if (importHandledByPlugin || directDatabaseAccess) {
callback();
// Logic for allowing external Import Plugins
let result = await hooks.aCallAll("import", { srcFile, destFile });
let importHandledByPlugin = (result.length > 0); // This feels hacky and wrong..
return;
}
let fileIsEtherpad = (fileEnding === ".etherpad");
let fileIsHTML = (fileEnding === ".html" || fileEnding === ".htm");
let fileIsTXT = (fileEnding === ".txt");
var fileEnding = path.extname(srcFile).toLowerCase();
var fileIsHTML = (fileEnding === ".html" || fileEnding === ".htm");
var fileIsTXT = (fileEnding === ".txt");
if (fileIsTXT) useConvertor = false; // Don't use convertor for text files
// See https://github.com/ether/etherpad-lite/issues/2572
if (fileIsHTML || (useConvertor === false)) {
// if no convertor only rename
fs.rename(srcFile, destFile, callback);
return;
}
if (fileIsEtherpad) {
// we do this here so we can see if the pad has quite a few edits
let _pad = await padManager.getPad(padId);
let headCount = _pad.head;
convertor.convertFile(srcFile, destFile, exportExtension, function(err) {
//catch convert errors
if(err) {
console.warn("Converting Error:", err);
return callback("convertFailed");
}
if (headCount >= 10) {
apiLogger.warn("Direct database Import attempt of a pad that already has content, we won't be doing this");
throw "padHasData";
}
callback();
});
},
function(callback) {
if (useConvertor || directDatabaseAccess) {
callback();
const fsp_readFile = util.promisify(fs.readFile);
let _text = await fsp_readFile(srcFile, "utf8");
req.directDatabaseAccess = true;
await importEtherpad.setPadRaw(padId, _text);
}
return;
}
// convert file to html if necessary
if (!importHandledByPlugin && !req.directDatabaseAccess) {
if (fileIsTXT) {
// Don't use convertor for text files
useConvertor = false;
}
// Read the file with no encoding for raw buffer access.
fs.readFile(destFile, function(err, buf) {
if (err) throw err;
var isAscii = true;
// Check if there are only ascii chars in the uploaded file
for (var i=0, len=buf.length; i<len; i++) {
if (buf[i] > 240) {
isAscii=false;
break;
// See https://github.com/ether/etherpad-lite/issues/2572
if (fileIsHTML || !useConvertor) {
// if no convertor only rename
fs.renameSync(srcFile, destFile);
} else {
// @TODO - no Promise interface for convertors (yet)
await new Promise((resolve, reject) => {
convertor.convertFile(srcFile, destFile, exportExtension, function(err) {
// catch convert errors
if (err) {
console.warn("Converting Error:", err);
reject("convertFailed");
}
}
if (!isAscii) {
callback("uploadFailed");
return;
}
callback();
});
},
//get the pad object
function(callback) {
padManager.getPad(padId, function(err, _pad){
if(ERR(err, callback)) return;
pad = _pad;
callback();
});
},
//read the text
function(callback) {
if (directDatabaseAccess) {
callback();
return;
}
fs.readFile(destFile, "utf8", function(err, _text){
if(ERR(err, callback)) return;
text = _text;
// Title needs to be stripped out else it appends it to the pad..
text = text.replace("<title>", "<!-- <title>");
text = text.replace("</title>","</title>-->");
//node on windows has a delay on releasing of the file lock.
//We add a 100ms delay to work around this
if(os.type().indexOf("Windows") > -1){
setTimeout(function() {callback();}, 100);
} else {
callback();
}
});
},
//change text of the pad and broadcast the changeset
function(callback) {
if(!directDatabaseAccess){
var fileEnding = path.extname(srcFile).toLowerCase();
if (importHandledByPlugin || useConvertor || fileEnding == ".htm" || fileEnding == ".html") {
importHtml.setPadHTML(pad, text, function(e){
if(e) apiLogger.warn("Error importing, possibly caused by malformed HTML");
});
} else {
pad.setText(text);
}
}
// Load the Pad into memory then brodcast updates to all clients
padManager.unloadPad(padId);
padManager.getPad(padId, function(err, _pad){
var pad = _pad;
padManager.unloadPad(padId);
// direct Database Access means a pad user should perform a switchToPad
// and not attempt to recieve updated pad data..
if (directDatabaseAccess) {
callback();
return;
}
padMessageHandler.updatePadClients(pad, function(){
callback();
resolve();
});
});
},
//clean up temporary files
function(callback) {
if (directDatabaseAccess) {
callback();
return;
}
try {
fs.unlinkSync(srcFile);
} catch (e) {
console.log(e);
}
try {
fs.unlinkSync(destFile);
} catch (e) {
console.log(e);
}
callback();
}
], function(err) {
var status = "ok";
//check for known errors and replace the status
if(err == "uploadFailed" || err == "convertFailed" || err == "padHasData")
{
}
if (!useConvertor && !req.directDatabaseAccess) {
// Read the file with no encoding for raw buffer access.
let buf = await fsp_readFile(destFile);
// Check if there are only ascii chars in the uploaded file
let isAscii = ! Array.prototype.some.call(buf, c => (c > 240));
if (!isAscii) {
throw "uploadFailed";
}
}
// get the pad object
let pad = await padManager.getPad(padId);
// read the text
let text;
if (!req.directDatabaseAccess) {
text = await fsp_readFile(destFile, "utf8");
// Title needs to be stripped out else it appends it to the pad..
text = text.replace("<title>", "<!-- <title>");
text = text.replace("</title>","</title>-->");
// node on windows has a delay on releasing of the file lock.
// We add a 100ms delay to work around this
if (os.type().indexOf("Windows") > -1){
await new Promise(resolve => setTimeout(resolve, 100));
}
}
// change text of the pad and broadcast the changeset
if (!req.directDatabaseAccess) {
if (importHandledByPlugin || useConvertor || fileIsHTML) {
try {
importHtml.setPadHTML(pad, text);
} catch (e) {
apiLogger.warn("Error importing, possibly caused by malformed HTML");
}
} else {
pad.setText(text);
}
}
// Load the Pad into memory then broadcast updates to all clients
padManager.unloadPad(padId);
pad = await padManager.getPad(padId);
padManager.unloadPad(padId);
// direct Database Access means a pad user should perform a switchToPad
// and not attempt to receive updated pad data
if (req.directDatabaseAccess) {
return;
}
// tell clients to update
await padMessageHandler.updatePadClients(pad);
// clean up temporary files
/*
* TODO: directly delete the file and handle the eventual error. Checking
* before for existence is prone to race conditions, and does not handle any
* errors anyway.
*/
if (await fsp_exists(srcFile)) {
fsp_unlink(srcFile);
}
if (await fsp_exists(destFile)) {
fsp_unlink(destFile);
}
}
exports.doImport = function (req, res, padId)
{
/**
* NB: abuse the 'req' object by storing an additional
* 'directDatabaseAccess' property on it so that it can
* be passed back in the HTML below.
*
* this is necessary because in the 'throw' paths of
* the function above there's no other way to return
* a value to the caller.
*/
let status = "ok";
doImport(req, res, padId).catch(err => {
// check for known errors and replace the status
if (err == "uploadFailed" || err == "convertFailed" || err == "padHasData") {
status = err;
err = null;
} else {
throw err;
}
ERR(err);
//close the connection
}).then(() => {
// close the connection
res.send(
"<head> \
<script type='text/javascript' src='../../static/js/jquery.js'></script> \
</head> \
<script> \
$(window).load(function(){ \
var impexp = window.parent.padimpexp.handleFrameCall('" + directDatabaseAccess +"', '" + status + "'); \
}) \
</script>"
"<head> \
<script type='text/javascript' src='../../static/js/jquery.js'></script> \
</head> \
<script> \
$(window).load(function(){ \
var impexp = window.parent.padimpexp.handleFrameCall('" + req.directDatabaseAccess +"', '" + status + "'); \
}) \
</script>"
);
});
}

2075
src/node/handler/PadMessageHandler.js
View file

File diff suppressed because it is too large Load diff

132
src/node/handler/SocketIORouter.js
View file

@ -1,5 +1,5 @@
/**
* This is the Socket.IO Router. It routes the Messages between the
* This is the Socket.IO Router. It routes the Messages between the
* components of the Server. The components are at the moment: pad and timeslider
*/
@ -19,7 +19,6 @@
* limitations under the License.
*/
var ERR = require("async-stacktrace");
var log4js = require('log4js');
var messageLogger = log4js.getLogger("message");
var securityManager = require("../db/SecurityManager");
@ -31,20 +30,20 @@ var settings = require('../utils/Settings');
* Saves all components
* key is the component name
* value is the component module
*/
*/
var components = {};
var socket;
/**
* adds a component
*/
exports.addComponent = function(moduleName, module)
{
//save the component
// save the component
components[moduleName] = module;
//give the module the socket
// give the module the socket
module.setSocketIO(socket);
}
@ -52,115 +51,102 @@ exports.addComponent = function(moduleName, module)
* sets the socket.io and adds event functions for routing
*/
exports.setSocketIO = function(_socket) {
//save this socket internaly
// save this socket internaly
socket = _socket;
socket.sockets.on('connection', function(client)
{
// Broken: See http://stackoverflow.com/questions/4647348/send-message-to-specific-client-with-socket-io-and-node-js
// Fixed by having a persistant object, ideally this would actually be in the database layer
// TODO move to database layer
if(settings.trustProxy && client.handshake.headers['x-forwarded-for'] !== undefined){
if (settings.trustProxy && client.handshake.headers['x-forwarded-for'] !== undefined) {
remoteAddress[client.id] = client.handshake.headers['x-forwarded-for'];
}
else{
} else {
remoteAddress[client.id] = client.handshake.address;
}
var clientAuthorized = false;
//wrap the original send function to log the messages
// wrap the original send function to log the messages
client._send = client.send;
client.send = function(message) {
messageLogger.debug("to " + client.id + ": " + stringifyWithoutPassword(message));
client._send(message);
}
//tell all components about this connect
for(var i in components) {
components[i].handleConnect(client);
}
client.on('message', function(message)
{
if(message.protocolVersion && message.protocolVersion != 2) {
// tell all components about this connect
for (let i in components) {
components[i].handleConnect(client);
}
client.on('message', async function(message) {
if (message.protocolVersion && message.protocolVersion != 2) {
messageLogger.warn("Protocolversion header is not correct:" + stringifyWithoutPassword(message));
return;
}
//client is authorized, everything ok
if(clientAuthorized) {
if (clientAuthorized) {
// client is authorized, everything ok
handleMessage(client, message);
} else { //try to authorize the client
if(message.padId !== undefined && message.sessionID !== undefined && message.token !== undefined && message.password !== undefined) {
var checkAccessCallback = function(err, statusObject) {
ERR(err);
//access was granted, mark the client as authorized and handle the message
if(statusObject.accessStatus == "grant") {
clientAuthorized = true;
handleMessage(client, message);
}
//no access, send the client a message that tell him why
else {
messageLogger.warn("Authentication try failed:" + stringifyWithoutPassword(message));
client.json.send({accessStatus: statusObject.accessStatus});
}
};
if (message.padId.indexOf("r.") === 0) {
readOnlyManager.getPadId(message.padId, function(err, value) {
ERR(err);
securityManager.checkAccess (value, message.sessionID, message.token, message.password, checkAccessCallback);
});
} else {
//this message has everything to try an authorization
securityManager.checkAccess (message.padId, message.sessionID, message.token, message.password, checkAccessCallback);
} else {
// try to authorize the client
if (message.padId !== undefined && message.sessionID !== undefined && message.token !== undefined && message.password !== undefined) {
// check for read-only pads
let padId = message.padId;
if (padId.indexOf("r.") === 0) {
padId = await readOnlyManager.getPadId(message.padId);
}
} else { //drop message
messageLogger.warn("Dropped message cause of bad permissions:" + stringifyWithoutPassword(message));
let { accessStatus } = await securityManager.checkAccess(padId, message.sessionID, message.token, message.password);
if (accessStatus === "grant") {
// access was granted, mark the client as authorized and handle the message
clientAuthorized = true;
handleMessage(client, message);
} else {
// no access, send the client a message that tells him why
messageLogger.warn("Authentication try failed:" + stringifyWithoutPassword(message));
client.json.send({ accessStatus });
}
} else {
// drop message
messageLogger.warn("Dropped message because of bad permissions:" + stringifyWithoutPassword(message));
}
}
});
client.on('disconnect', function()
{
//tell all components about this disconnect
for(var i in components)
{
client.on('disconnect', function() {
// tell all components about this disconnect
for (let i in components) {
components[i].handleDisconnect(client);
}
});
});
}
//try to handle the message of this client
// try to handle the message of this client
function handleMessage(client, message)
{
if(message.component && components[message.component]) {
//check if component is registered in the components array
if(components[message.component]) {
if (message.component && components[message.component]) {
// check if component is registered in the components array
if (components[message.component]) {
messageLogger.debug("from " + client.id + ": " + stringifyWithoutPassword(message));
components[message.component].handleMessage(client, message);
}
} else {
messageLogger.error("Can't route the message:" + stringifyWithoutPassword(message));
}
}
}
//returns a stringified representation of a message, removes the password
//this ensures there are no passwords in the log
// returns a stringified representation of a message, removes the password
// this ensures there are no passwords in the log
function stringifyWithoutPassword(message)
{
var newMessage = {};
for(var i in message)
{
if(i == "password" && message[i] != null)
newMessage["password"] = "xxx";
else
newMessage[i]=message[i];
let newMessage = Object.assign({}, message);
if (newMessage.password != null) {
newMessage.password = "xxx";
}
return JSON.stringify(newMessage);
}

124
src/node/hooks/express/adminplugins.js
View file

@ -5,7 +5,7 @@ var plugins = require('ep_etherpad-lite/static/js/pluginfw/plugins');
var _ = require('underscore');
var semver = require('semver');
exports.expressCreateServer = function (hook_name, args, cb) {
exports.expressCreateServer = function(hook_name, args, cb) {
args.app.get('/admin/plugins', function(req, res) {
var plugins = require("ep_etherpad-lite/static/js/pluginfw/plugins");
var render_args = {
@ -13,91 +13,99 @@ exports.expressCreateServer = function (hook_name, args, cb) {
search_results: {},
errors: [],
};
res.send( eejs.require("ep_etherpad-lite/templates/admin/plugins.html", render_args) );
res.send(eejs.require("ep_etherpad-lite/templates/admin/plugins.html", render_args));
});
args.app.get('/admin/plugins/info', function(req, res) {
var gitCommit = settings.getGitCommit();
var epVersion = settings.getEpVersion();
res.send( eejs.require("ep_etherpad-lite/templates/admin/plugins-info.html",
{
gitCommit: gitCommit,
epVersion: epVersion
})
);
res.send(eejs.require("ep_etherpad-lite/templates/admin/plugins-info.html", {
gitCommit: gitCommit,
epVersion: epVersion
}));
});
}
exports.socketio = function (hook_name, args, cb) {
exports.socketio = function(hook_name, args, cb) {
var io = args.io.of("/pluginfw/installer");
io.on('connection', function (socket) {
io.on('connection', function(socket) {
if (!socket.conn.request.session || !socket.conn.request.session.user || !socket.conn.request.session.user.is_admin) return;
socket.on("getInstalled", function (query) {
socket.on("getInstalled", function(query) {
// send currently installed plugins
var installed = Object.keys(plugins.plugins).map(function(plugin) {
return plugins.plugins[plugin].package
})
});
socket.emit("results:installed", {installed: installed});
});
socket.on("checkUpdates", function() {
socket.on("checkUpdates", async function() {
// Check plugins for updates
installer.getAvailablePlugins(/*maxCacheAge:*/60*10, function(er, results) {
if(er) {
console.warn(er);
socket.emit("results:updatable", {updatable: {}});
return;
}
try {
let results = await installer.getAvailablePlugins(/*maxCacheAge:*/ 60 * 10);
var updatable = _(plugins.plugins).keys().filter(function(plugin) {
if(!results[plugin]) return false;
var latestVersion = results[plugin].version
var currentVersion = plugins.plugins[plugin].package.version
return semver.gt(latestVersion, currentVersion)
if (!results[plugin]) return false;
var latestVersion = results[plugin].version;
var currentVersion = plugins.plugins[plugin].package.version;
return semver.gt(latestVersion, currentVersion);
});
socket.emit("results:updatable", {updatable: updatable});
});
})
socket.on("getAvailable", function (query) {
installer.getAvailablePlugins(/*maxCacheAge:*/false, function (er, results) {
if(er) {
console.error(er)
results = {}
}
socket.emit("results:available", results);
});
} catch (er) {
console.warn(er);
socket.emit("results:updatable", {updatable: {}});
}
});
socket.on("search", function (query) {
installer.search(query.searchTerm, /*maxCacheAge:*/60*10, function (er, results) {
if(er) {
console.error(er)
results = {}
}
socket.on("getAvailable", async function(query) {
try {
let results = await installer.getAvailablePlugins(/*maxCacheAge:*/ false);
socket.emit("results:available", results);
} catch (er) {
console.error(er);
socket.emit("results:available", {});
}
});
socket.on("search", async function(query) {
try {
let results = await installer.search(query.searchTerm, /*maxCacheAge:*/ 60 * 10);
var res = Object.keys(results)
.map(function(pluginName) {
return results[pluginName]
return results[pluginName];
})
.filter(function(plugin) {
return !plugins.plugins[plugin.name]
return !plugins.plugins[plugin.name];
});
res = sortPluginList(res, query.sortBy, query.sortDir)
.slice(query.offset, query.offset+query.limit);
socket.emit("results:search", {results: res, query: query});
});
} catch (er) {
console.error(er);
socket.emit("results:search", {results: {}, query: query});
}
});
socket.on("install", function (plugin_name) {
installer.install(plugin_name, function (er) {
if(er) console.warn(er)
socket.on("install", function(plugin_name) {
installer.install(plugin_name, function(er) {
if (er) console.warn(er);
socket.emit("finished:install", {plugin: plugin_name, code: er? er.code : null, error: er? er.message : null});
});
});
socket.on("uninstall", function (plugin_name) {
installer.uninstall(plugin_name, function (er) {
if(er) console.warn(er)
socket.on("uninstall", function(plugin_name) {
installer.uninstall(plugin_name, function(er) {
if (er) console.warn(er);
socket.emit("finished:uninstall", {plugin: plugin_name, error: er? er.message : null});
});
});
@ -106,11 +114,15 @@ exports.socketio = function (hook_name, args, cb) {
function sortPluginList(plugins, property, /*ASC?*/dir) {
return plugins.sort(function(a, b) {
if (a[property] < b[property])
return dir? -1 : 1;
if (a[property] > b[property])
return dir? 1 : -1;
if (a[property] < b[property]) {
return dir? -1 : 1;
}
if (a[property] > b[property]) {
return dir? 1 : -1;
}
// a must be equal to b
return 0;
})
});
}

17
src/node/hooks/express/errorhandling.js
View file

@ -11,20 +11,23 @@ exports.gracefulShutdown = function(err) {
console.error(err);
}
//ensure there is only one graceful shutdown running
if(exports.onShutdown) return;
// ensure there is only one graceful shutdown running
if (exports.onShutdown) {
return;
}
exports.onShutdown = true;
console.log("graceful shutdown...");
//do the db shutdown
db.db.doShutdown(function() {
// do the db shutdown
db.doShutdown().then(function() {
console.log("db sucessfully closed.");
process.exit(0);
});
setTimeout(function(){
setTimeout(function() {
process.exit(1);
}, 3000);
}
@ -35,14 +38,14 @@ exports.expressCreateServer = function (hook_name, args, cb) {
exports.app = args.app;
// Handle errors
args.app.use(function(err, req, res, next){
args.app.use(function(err, req, res, next) {
// if an error occurs Connect will pass it down
// through these "error-handling" middleware
// allowing you to respond however you like
res.status(500).send({ error: 'Sorry, something bad happened!' });
console.error(err.stack? err.stack : err.toString());
stats.meter('http500').mark()
})
});
/*
* Connect graceful shutdown with sigint and uncaught exception

43
src/node/hooks/express/importexport.js
View file

@ -5,15 +5,14 @@ var importHandler = require('../../handler/ImportHandler');
var padManager = require("../../db/PadManager");
exports.expressCreateServer = function (hook_name, args, cb) {
args.app.get('/p/:pad/:rev?/export/:type', function(req, res, next) {
args.app.get('/p/:pad/:rev?/export/:type', async function(req, res, next) {
var types = ["pdf", "doc", "txt", "html", "odt", "etherpad"];
//send a 404 if we don't support this filetype
if (types.indexOf(req.params.type) == -1) {
next();
return;
return next();
}
//if abiword is disabled, and this is a format we only support with abiword, output a message
// if abiword is disabled, and this is a format we only support with abiword, output a message
if (settings.exportAvailable() == "no" &&
["odt", "pdf", "doc"].indexOf(req.params.type) !== -1) {
res.send("This export is not enabled at this Etherpad instance. Set the path to Abiword or SOffice in settings.json to enable this feature");
@ -22,30 +21,26 @@ exports.expressCreateServer = function (hook_name, args, cb) {
res.header("Access-Control-Allow-Origin", "*");
hasPadAccess(req, res, function() {
if (await hasPadAccess(req, res)) {
console.log('req.params.pad', req.params.pad);
padManager.doesPadExists(req.params.pad, function(err, exists)
{
if(!exists) {
return next();
}
let exists = await padManager.doesPadExists(req.params.pad);
if (!exists) {
return next();
}
exportHandler.doExport(req, res, req.params.pad, req.params.type);
});
});
exportHandler.doExport(req, res, req.params.pad, req.params.type);
}
});
//handle import requests
args.app.post('/p/:pad/import', function(req, res, next) {
hasPadAccess(req, res, function() {
padManager.doesPadExists(req.params.pad, function(err, exists)
{
if(!exists) {
return next();
}
// handle import requests
args.app.post('/p/:pad/import', async function(req, res, next) {
if (await hasPadAccess(req, res)) {
let exists = await padManager.doesPadExists(req.params.pad);
if (!exists) {
return next();
}
importHandler.doImport(req, res, req.params.pad);
});
});
importHandler.doImport(req, res, req.params.pad);
}
});
}

68
src/node/hooks/express/padreadonly.js
View file

@ -1,64 +1,26 @@
var async = require('async');
var ERR = require("async-stacktrace");
var readOnlyManager = require("../../db/ReadOnlyManager");
var hasPadAccess = require("../../padaccess");
var exporthtml = require("../../utils/ExportHtml");
exports.expressCreateServer = function (hook_name, args, cb) {
//serve read only pad
args.app.get('/ro/:id', function(req, res)
{
var html;
var padId;
// serve read only pad
args.app.get('/ro/:id', async function(req, res) {
async.series([
//translate the read only pad to a padId
function(callback)
{
readOnlyManager.getPadId(req.params.id, function(err, _padId)
{
if(ERR(err, callback)) return;
// translate the read only pad to a padId
let padId = await readOnlyManager.getPadId(req.params.id);
if (padId == null) {
res.status(404).send('404 - Not Found');
return;
}
padId = _padId;
// we need that to tell hasPadAcess about the pad
req.params.pad = padId;
//we need that to tell hasPadAcess about the pad
req.params.pad = padId;
callback();
});
},
//render the html document
function(callback)
{
//return if the there is no padId
if(padId == null)
{
callback("notfound");
return;
}
hasPadAccess(req, res, function()
{
//render the html document
exporthtml.getPadHTMLDocument(padId, null, function(err, _html)
{
if(ERR(err, callback)) return;
html = _html;
callback();
});
});
}
], function(err)
{
//throw any unexpected error
if(err && err != "notfound")
ERR(err);
if(err == "notfound")
res.status(404).send('404 - Not Found');
else
res.send(html);
});
if (await hasPadAccess(req, res)) {
// render the html document
html = await exporthtml.getPadHTMLDocument(padId, null);
res.send(html);
}
});
}

41
src/node/hooks/express/padurlsanitize.js
View file

@ -2,31 +2,28 @@ var padManager = require('../../db/PadManager');
var url = require('url');
exports.expressCreateServer = function (hook_name, args, cb) {
//redirects browser to the pad's sanitized url if needed. otherwise, renders the html
args.app.param('pad', function (req, res, next, padId) {
//ensure the padname is valid and the url doesn't end with a /
if(!padManager.isValidPadId(padId) || /\/$/.test(req.url))
{
// redirects browser to the pad's sanitized url if needed. otherwise, renders the html
args.app.param('pad', async function (req, res, next, padId) {
// ensure the padname is valid and the url doesn't end with a /
if (!padManager.isValidPadId(padId) || /\/$/.test(req.url)) {
res.status(404).send('Such a padname is forbidden');
return;
}
padManager.sanitizePadId(padId, function(sanitizedPadId) {
//the pad id was sanitized, so we redirect to the sanitized version
if(sanitizedPadId != padId)
{
var real_url = sanitizedPadId;
real_url = encodeURIComponent(real_url);
var query = url.parse(req.url).query;
if ( query ) real_url += '?' + query;
res.header('Location', real_url);
res.status(302).send('You should be redirected to <a href="' + real_url + '">' + real_url + '</a>');
}
//the pad id was fine, so just render it
else
{
next();
}
});
let sanitizedPadId = await padManager.sanitizePadId(padId);
if (sanitizedPadId === padId) {
// the pad id was fine, so just render it
next();
} else {
// the pad id was sanitized, so we redirect to the sanitized version
var real_url = sanitizedPadId;
real_url = encodeURIComponent(real_url);
var query = url.parse(req.url).query;
if ( query ) real_url += '?' + query;
res.header('Location', real_url);
res.status(302).send('You should be redirected to <a href="' + real_url + '">' + real_url + '</a>');
}
});
}

92
src/node/hooks/express/tests.js
View file

@ -1,40 +1,33 @@
var path = require("path")
, npm = require("npm")
, fs = require("fs")
, async = require("async");
, util = require("util");
exports.expressCreateServer = function (hook_name, args, cb) {
args.app.get('/tests/frontend/specs_list.js', function(req, res){
async.parallel({
coreSpecs: function(callback){
exports.getCoreTests(callback);
},
pluginSpecs: function(callback){
exports.getPluginTests(callback);
}
},
function(err, results){
var files = results.coreSpecs; // push the core specs to a file object
files = files.concat(results.pluginSpecs); // add the plugin Specs to the core specs
console.debug("Sent browser the following test specs:", files.sort());
res.send("var specs_list = " + JSON.stringify(files.sort()) + ";\n");
});
args.app.get('/tests/frontend/specs_list.js', async function(req, res) {
let [coreTests, pluginTests] = await Promise.all([
exports.getCoreTests(),
exports.getPluginTests()
]);
// merge the two sets of results
let files = [].concat(coreTests, pluginTests).sort();
console.debug("Sent browser the following test specs:", files);
res.send("var specs_list = " + JSON.stringify(files) + ";\n");
});
// path.join seems to normalize by default, but we'll just be explicit
var rootTestFolder = path.normalize(path.join(npm.root, "../tests/frontend/"));
var url2FilePath = function(url){
var url2FilePath = function(url) {
var subPath = url.substr("/tests/frontend".length);
if (subPath == ""){
if (subPath == "") {
subPath = "index.html"
}
subPath = subPath.split("?")[0];
var filePath = path.normalize(path.join(rootTestFolder, subPath));
// make sure we jail the paths to the test folder, otherwise serve index
if (filePath.indexOf(rootTestFolder) !== 0) {
filePath = path.join(rootTestFolder, "index.html");
@ -46,13 +39,13 @@ exports.expressCreateServer = function (hook_name, args, cb) {
var specFilePath = url2FilePath(req.url);
var specFileName = path.basename(specFilePath);
fs.readFile(specFilePath, function(err, content){
if(err){ return res.send(500); }
fs.readFile(specFilePath, function(err, content) {
if (err) { return res.send(500); }
content = "describe(" + JSON.stringify(specFileName) + ", function(){ " + content + " });";
res.send(content);
});
});
});
args.app.get('/tests/frontend/*', function (req, res) {
@ -62,30 +55,33 @@ exports.expressCreateServer = function (hook_name, args, cb) {
args.app.get('/tests/frontend', function (req, res) {
res.redirect('/tests/frontend/');
});
}
exports.getPluginTests = function(callback){
var pluginSpecs = [];
var plugins = fs.readdirSync('node_modules');
plugins.forEach(function(plugin){
if(fs.existsSync("node_modules/"+plugin+"/static/tests/frontend/specs")){ // if plugins exists
var specFiles = fs.readdirSync("node_modules/"+plugin+"/static/tests/frontend/specs/");
async.forEach(specFiles, function(spec){ // for each specFile push it to pluginSpecs
pluginSpecs.push("/static/plugins/"+plugin+"/static/tests/frontend/specs/" + spec);
},
function(err){
// blow up if something bad happens!
});
}
});
callback(null, pluginSpecs);
}
exports.getCoreTests = function(callback){
fs.readdir('tests/frontend/specs', function(err, coreSpecs){ // get the core test specs
if(err){ return res.send(500); }
callback(null, coreSpecs);
});
}
const readdir = util.promisify(fs.readdir);
exports.getPluginTests = async function(callback) {
const moduleDir = "node_modules/";
const specPath = "/static/tests/frontend/specs/";
const staticDir = "/static/plugins/";
let pluginSpecs = [];
let plugins = await readdir(moduleDir);
let promises = plugins
.map(plugin => [ plugin, moduleDir + plugin + specPath] )
.filter(([plugin, specDir]) => fs.existsSync(specDir)) // check plugin exists
.map(([plugin, specDir]) => {
return readdir(specDir)
.then(specFiles => specFiles.map(spec => {
pluginSpecs.push(staticDir + plugin + specPath + spec);
}));
});
return Promise.all(promises).then(() => pluginSpecs);
}
exports.getCoreTests = function() {
// get the core test specs
return readdir('tests/frontend/specs');
}

23
src/node/padaccess.js
View file

@ -1,17 +1,20 @@
var ERR = require("async-stacktrace");
var securityManager = require('./db/SecurityManager');
//checks for padAccess
module.exports = function (req, res, callback) {
securityManager.checkAccess(req.params.pad, req.cookies.sessionID, req.cookies.token, req.cookies.password, function(err, accessObj) {
if(ERR(err, callback)) return;
// checks for padAccess
module.exports = async function (req, res) {
try {
let accessObj = await securityManager.checkAccess(req.params.pad, req.cookies.sessionID, req.cookies.token, req.cookies.password);
//there is access, continue
if(accessObj.accessStatus == "grant") {
callback();
//no access
if (accessObj.accessStatus === "grant") {
// there is access, continue
return true;
} else {
// no access
res.status(403).send("403 - Can't touch this");
return false;
}
});
} catch (err) {
// @TODO - send internal server error here?
throw err;
}
}

88
src/node/server.js
View file

@ -1,7 +1,7 @@
#!/usr/bin/env node
/**
* This module is started with bin/run.sh. It sets up a Express HTTP and a Socket.IO Server.
* Static file Requests are answered directly from this module, Socket.IO messages are passed
* This module is started with bin/run.sh. It sets up a Express HTTP and a Socket.IO Server.
* Static file Requests are answered directly from this module, Socket.IO messages are passed
* to MessageHandler and minfied requests are passed to minified.
*/
@ -22,7 +22,6 @@
*/
var log4js = require('log4js')
, async = require('async')
, NodeVersion = require('./utils/NodeVersion')
;
@ -46,57 +45,40 @@ NodeVersion.enforceMinNodeVersion('8.9.0');
*/
var stats = require('./stats');
stats.gauge('memoryUsage', function() {
return process.memoryUsage().rss
})
return process.memoryUsage().rss;
});
var settings
, db
, plugins
, hooks;
/*
* no use of let or await here because it would cause startup
* to fail completely on very early versions of NodeJS
*/
var npm = require("npm/lib/npm.js");
async.waterfall([
// load npm
function(callback) {
npm.load({}, function(er) {
callback(er)
npm.load({}, function() {
var settings = require('./utils/Settings');
var db = require('./db/DB');
var plugins = require("ep_etherpad-lite/static/js/pluginfw/plugins");
var hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks");
hooks.plugins = plugins;
db.init()
.then(plugins.update)
.then(function() {
console.info("Installed plugins: " + plugins.formatPluginsWithVersion());
console.debug("Installed parts:\n" + plugins.formatParts());
console.debug("Installed hooks:\n" + plugins.formatHooks());
// Call loadSettings hook
hooks.aCallAll("loadSettings", { settings: settings });
// initalize the http server
hooks.callAll("createServer", {});
})
},
// load everything
function(callback) {
settings = require('./utils/Settings');
db = require('./db/DB');
plugins = require("ep_etherpad-lite/static/js/pluginfw/plugins");
hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks");
hooks.plugins = plugins;
callback();
},
//initalize the database
function (callback)
{
db.init(callback);
},
function(callback) {
plugins.update(callback)
},
function (callback) {
console.info("Installed plugins: " + plugins.formatPluginsWithVersion());
console.debug("Installed parts:\n" + plugins.formatParts());
console.debug("Installed hooks:\n" + plugins.formatHooks());
// Call loadSettings hook
hooks.aCallAll("loadSettings", { settings: settings });
callback();
},
//initalize the http server
function (callback)
{
hooks.callAll("createServer", {});
callback(null);
}
]);
.catch(function(e) {
console.error("exception thrown: " + e.message);
if (e.stack) {
console.log(e.stack);
}
process.exit(1);
});
});

90
src/node/utils/ExportEtherpad.js
View file

@ -15,58 +15,48 @@
*/
var async = require("async");
var db = require("../db/DB").db;
var ERR = require("async-stacktrace");
let db = require("../db/DB");
exports.getPadRaw = function(padId, callback){
async.waterfall([
function(cb){
db.get("pad:"+padId, cb);
},
function(padcontent,cb){
var records = ["pad:"+padId];
for (var i = 0; i <= padcontent.head; i++) {
records.push("pad:"+padId+":revs:" + i);
}
exports.getPadRaw = async function(padId) {
for (var i = 0; i <= padcontent.chatHead; i++) {
records.push("pad:"+padId+":chat:" + i);
}
let padKey = "pad:" + padId;
let padcontent = await db.get(padKey);
var data = {};
async.forEachSeries(Object.keys(records), function(key, r){
// For each piece of info about a pad.
db.get(records[key], function(err, entry){
data[records[key]] = entry;
// Get the Pad Authors
if(entry.pool && entry.pool.numToAttrib){
var authors = entry.pool.numToAttrib;
async.forEachSeries(Object.keys(authors), function(k, c){
if(authors[k][0] === "author"){
var authorId = authors[k][1];
// Get the author info
db.get("globalAuthor:"+authorId, function(e, authorEntry){
if(authorEntry && authorEntry.padIDs) authorEntry.padIDs = padId;
if(!e) data["globalAuthor:"+authorId] = authorEntry;
});
}
// console.log("authorsK", authors[k]);
c(null);
});
}
r(null); // callback;
});
}, function(err){
cb(err, data);
})
let records = [ padKey ];
for (let i = 0; i <= padcontent.head; i++) {
records.push(padKey + ":revs:" + i);
}
], function(err, data){
callback(null, data);
});
for (let i = 0; i <= padcontent.chatHead; i++) {
records.push(padKey + ":chat:" + i);
}
let data = {};
for (let key of records) {
// For each piece of info about a pad.
let entry = data[key] = await db.get(key);
// Get the Pad Authors
if (entry.pool && entry.pool.numToAttrib) {
let authors = entry.pool.numToAttrib;
for (let k of Object.keys(authors)) {
if (authors[k][0] === "author") {
let authorId = authors[k][1];
// Get the author info
let authorEntry = await db.get("globalAuthor:" + authorId);
if (authorEntry) {
data["globalAuthor:" + authorId] = authorEntry;
if (authorEntry.padIDs) {
authorEntry.padIDs = padId;
}
}
}
}
}
}
return data;
}

91
src/node/utils/ExportHtml.js
View file

@ -14,11 +14,8 @@
* limitations under the License.
*/
var async = require("async");
var Changeset = require("ep_etherpad-lite/static/js/Changeset");
var padManager = require("../db/PadManager");
var ERR = require("async-stacktrace");
var _ = require('underscore');
var Security = require('ep_etherpad-lite/static/js/security');
var hooks = require('ep_etherpad-lite/static/js/pluginfw/hooks');
@ -26,45 +23,17 @@ var eejs = require('ep_etherpad-lite/node/eejs');
var _analyzeLine = require('./ExportHelper')._analyzeLine;
var _encodeWhitespace = require('./ExportHelper')._encodeWhitespace;
function getPadHTML(pad, revNum, callback)
async function getPadHTML(pad, revNum)
{
var atext = pad.atext;
var html;
async.waterfall([
let atext = pad.atext;
// fetch revision atext
function (callback)
{
if (revNum != undefined)
{
pad.getInternalRevisionAText(revNum, function (err, revisionAtext)
{
if(ERR(err, callback)) return;
atext = revisionAtext;
callback();
});
}
else
{
callback(null);
}
},
if (revNum != undefined) {
atext = await pad.getInternalRevisionAText(revNum);
}
// convert atext to html
function (callback)
{
html = getHTMLFromAtext(pad, atext);
callback(null);
}],
// run final callback
function (err)
{
if(ERR(err, callback)) return;
callback(null, html);
});
return getHTMLFromAtext(pad, atext);
}
exports.getPadHTML = getPadHTML;
@ -81,15 +50,16 @@ function getHTMLFromAtext(pad, atext, authorColors)
// prepare tags stored as ['tag', true] to be exported
hooks.aCallAll("exportHtmlAdditionalTags", pad, function(err, newProps){
newProps.forEach(function (propName, i){
newProps.forEach(function (propName, i) {
tags.push(propName);
props.push(propName);
});
});
// prepare tags stored as ['tag', 'value'] to be exported. This will generate HTML
// with tags like <span data-tag="value">
hooks.aCallAll("exportHtmlAdditionalTagsWithData", pad, function(err, newProps){
newProps.forEach(function (propName, i){
newProps.forEach(function (propName, i) {
tags.push('span data-' + propName[0] + '="' + propName[1] + '"');
props.push(propName);
});
@ -453,38 +423,31 @@ function getHTMLFromAtext(pad, atext, authorColors)
hooks.aCallAll("getLineHTMLForExport", context);
pieces.push(context.lineContent, "<br>");
}
}
}
return pieces.join('');
}
exports.getPadHTMLDocument = function (padId, revNum, callback)
exports.getPadHTMLDocument = async function (padId, revNum)
{
padManager.getPad(padId, function (err, pad)
{
if(ERR(err, callback)) return;
let pad = await padManager.getPad(padId);
var stylesForExportCSS = "";
// Include some Styles into the Head for Export
hooks.aCallAll("stylesForExport", padId, function(err, stylesForExport){
stylesForExport.forEach(function(css){
stylesForExportCSS += css;
});
getPadHTML(pad, revNum, function (err, html)
{
if(ERR(err, callback)) return;
var exportedDoc = eejs.require("ep_etherpad-lite/templates/export_html.html", {
body: html,
padId: Security.escapeHTML(padId),
extraCSS: stylesForExportCSS
});
callback(null, exportedDoc);
});
});
// Include some Styles into the Head for Export
let stylesForExportCSS = "";
let stylesForExport = await hooks.aCallAll("stylesForExport", padId);
stylesForExport.forEach(function(css){
stylesForExportCSS += css;
});
};
let html = await getPadHTML(pad, revNum);
return eejs.require("ep_etherpad-lite/templates/export_html.html", {
body: html,
padId: Security.escapeHTML(padId),
extraCSS: stylesForExportCSS
});
}
// copied from ACE
var _REGEX_WORDCHAR = /[\u0030-\u0039\u0041-\u005A\u0061-\u007A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u1FFF\u3040-\u9FFF\uF900-\uFDFF\uFE70-\uFEFE\uFF10-\uFF19\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFDC]/;

182
src/node/utils/ExportTxt.js
View file

@ -18,54 +18,22 @@
* limitations under the License.
*/
var async = require("async");
var Changeset = require("ep_etherpad-lite/static/js/Changeset");
var padManager = require("../db/PadManager");
var ERR = require("async-stacktrace");
var _analyzeLine = require('./ExportHelper')._analyzeLine;
// This is slightly different than the HTML method as it passes the output to getTXTFromAText
function getPadTXT(pad, revNum, callback)
var getPadTXT = async function(pad, revNum)
{
var atext = pad.atext;
var html;
async.waterfall([
// fetch revision atext
let atext = pad.atext;
function (callback)
{
if (revNum != undefined)
{
pad.getInternalRevisionAText(revNum, function (err, revisionAtext)
{
if(ERR(err, callback)) return;
atext = revisionAtext;
callback();
});
}
else
{
callback(null);
}
},
if (revNum != undefined) {
// fetch revision atext
atext = await pad.getInternalRevisionAText(revNum);
}
// convert atext to html
function (callback)
{
html = getTXTFromAtext(pad, atext); // only this line is different to the HTML function
callback(null);
}],
// run final callback
function (err)
{
if(ERR(err, callback)) return;
callback(null, html);
});
return getTXTFromAtext(pad, atext);
}
// This is different than the functionality provided in ExportHtml as it provides formatting
@ -80,17 +48,14 @@ function getTXTFromAtext(pad, atext, authorColors)
var anumMap = {};
var css = "";
props.forEach(function (propName, i)
{
props.forEach(function(propName, i) {
var propTrueNum = apool.putAttrib([propName, true], true);
if (propTrueNum >= 0)
{
if (propTrueNum >= 0) {
anumMap[propTrueNum] = i;
}
});
function getLineTXT(text, attribs)
{
function getLineTXT(text, attribs) {
var propVals = [false, false, false];
var ENTER = 1;
var STAY = 2;
@ -106,94 +71,77 @@ function getTXTFromAtext(pad, atext, authorColors)
var idx = 0;
function processNextChars(numChars)
{
if (numChars <= 0)
{
function processNextChars(numChars) {
if (numChars <= 0) {
return;
}
var iter = Changeset.opIterator(Changeset.subattribution(attribs, idx, idx + numChars));
idx += numChars;
while (iter.hasNext())
{
while (iter.hasNext()) {
var o = iter.next();
var propChanged = false;
Changeset.eachAttribNumber(o.attribs, function (a)
{
if (a in anumMap)
{
Changeset.eachAttribNumber(o.attribs, function(a) {
if (a in anumMap) {
var i = anumMap[a]; // i = 0 => bold, etc.
if (!propVals[i])
{
if (!propVals[i]) {
propVals[i] = ENTER;
propChanged = true;
}
else
{
} else {
propVals[i] = STAY;
}
}
});
for (var i = 0; i < propVals.length; i++)
{
if (propVals[i] === true)
{
for (var i = 0; i < propVals.length; i++) {
if (propVals[i] === true) {
propVals[i] = LEAVE;
propChanged = true;
}
else if (propVals[i] === STAY)
{
propVals[i] = true; // set it back
} else if (propVals[i] === STAY) {
// set it back
propVals[i] = true;
}
}
// now each member of propVal is in {false,LEAVE,ENTER,true}
// according to what happens at start of span
if (propChanged)
{
if (propChanged) {
// leaving bold (e.g.) also leaves italics, etc.
var left = false;
for (var i = 0; i < propVals.length; i++)
{
for (var i = 0; i < propVals.length; i++) {
var v = propVals[i];
if (!left)
{
if (v === LEAVE)
{
if (!left) {
if (v === LEAVE) {
left = true;
}
}
else
{
if (v === true)
{
propVals[i] = STAY; // tag will be closed and re-opened
} else {
if (v === true) {
// tag will be closed and re-opened
propVals[i] = STAY;
}
}
}
var tags2close = [];
for (var i = propVals.length - 1; i >= 0; i--)
{
if (propVals[i] === LEAVE)
{
for (var i = propVals.length - 1; i >= 0; i--) {
if (propVals[i] === LEAVE) {
//emitCloseTag(i);
tags2close.push(i);
propVals[i] = false;
}
else if (propVals[i] === STAY)
{
} else if (propVals[i] === STAY) {
//emitCloseTag(i);
tags2close.push(i);
}
}
for (var i = 0; i < propVals.length; i++)
{
if (propVals[i] === ENTER || propVals[i] === STAY)
{
for (var i = 0; i < propVals.length; i++) {
if (propVals[i] === ENTER || propVals[i] === STAY) {
propVals[i] = true;
}
}
@ -201,9 +149,9 @@ function getTXTFromAtext(pad, atext, authorColors)
} // end if (propChanged)
var chars = o.chars;
if (o.lines)
{
chars--; // exclude newline at end of line, if present
if (o.lines) {
// exclude newline at end of line, if present
chars--;
}
var s = taker.take(chars);
@ -220,19 +168,19 @@ function getTXTFromAtext(pad, atext, authorColors)
} // end iteration over spans in line
var tags2close = [];
for (var i = propVals.length - 1; i >= 0; i--)
{
if (propVals[i])
{
for (var i = propVals.length - 1; i >= 0; i--) {
if (propVals[i]) {
tags2close.push(i);
propVals[i] = false;
}
}
} // end processNextChars
processNextChars(text.length - idx);
return(assem.toString());
} // end getLineHTML
var pieces = [css];
// Need to deal with constraints imposed on HTML lists; can
@ -242,42 +190,38 @@ function getTXTFromAtext(pad, atext, authorColors)
// so we want to do something reasonable there. We also
// want to deal gracefully with blank lines.
// => keeps track of the parents level of indentation
for (var i = 0; i < textLines.length; i++)
{
for (var i = 0; i < textLines.length; i++) {
var line = _analyzeLine(textLines[i], attribLines[i], apool);
var lineContent = getLineTXT(line.text, line.aline);
if(line.listTypeName == "bullet"){
if (line.listTypeName == "bullet") {
lineContent = "* " + lineContent; // add a bullet
}
if(line.listLevel > 0){
for (var j = line.listLevel - 1; j >= 0; j--){
if (line.listLevel > 0) {
for (var j = line.listLevel - 1; j >= 0; j--) {
pieces.push('\t');
}
if(line.listTypeName == "number"){
if (line.listTypeName == "number") {
pieces.push(line.listLevel + ". ");
// This is bad because it doesn't truly reflect what the user
// sees because browsers do magic on nested <ol><li>s
}
pieces.push(lineContent, '\n');
}else{
} else {
pieces.push(lineContent, '\n');
}
}
return pieces.join('');
}
exports.getTXTFromAtext = getTXTFromAtext;
exports.getPadTXTDocument = function (padId, revNum, callback)
exports.getPadTXTDocument = async function(padId, revNum)
{
padManager.getPad(padId, function (err, pad)
{
if(ERR(err, callback)) return;
getPadTXT(pad, revNum, function (err, html)
{
if(ERR(err, callback)) return;
callback(null, html);
});
});
};
let pad = await padManager.getPad(padId);
return getPadTXT(pad, revNum);
}

70
src/node/utils/ImportEtherpad.js
View file

@ -15,60 +15,56 @@
*/
var log4js = require('log4js');
var async = require("async");
var db = require("../db/DB").db;
const db = require("../db/DB");
exports.setPadRaw = function(padId, records, callback){
exports.setPadRaw = function(padId, records)
{
records = JSON.parse(records);
async.eachSeries(Object.keys(records), function(key, cb){
var value = records[key]
Object.keys(records).forEach(async function(key) {
let value = records[key];
if(!value){
return setImmediate(cb);
if (!value) {
return;
}
// Author data
if(value.padIDs){
// rewrite author pad ids
let newKey;
if (value.padIDs) {
// Author data - rewrite author pad ids
value.padIDs[padId] = 1;
var newKey = key;
newKey = key;
// Does this author already exist?
db.get(key, function(err, author){
if(author){
// Yes, add the padID to the author..
if( Object.prototype.toString.call(author) === '[object Array]'){
author.padIDs.push(padId);
}
value = author;
}else{
// No, create a new array with the author info in
value.padIDs = [padId];
let author = await db.get(key);
if (author) {
// Yes, add the padID to the author
if (Object.prototype.toString.call(author) === '[object Array]') {
author.padIDs.push(padId);
}
});
// Not author data, probably pad data
}else{
// we can split it to look to see if its pad data
var oldPadId = key.split(":");
// we know its pad data..
if(oldPadId[0] === "pad"){
value = author;
} else {
// No, create a new array with the author info in
value.padIDs = [ padId ];
}
} else {
// Not author data, probably pad data
// we can split it to look to see if it's pad data
let oldPadId = key.split(":");
// we know it's pad data
if (oldPadId[0] === "pad") {
// so set the new pad id for the author
oldPadId[1] = padId;
// and create the value
var newKey = oldPadId.join(":"); // create the new key
newKey = oldPadId.join(":"); // create the new key
}
}
// Write the value to the server
db.set(newKey, value);
setImmediate(cb);
}, function(){
callback(null, true);
// Write the value to the server
await db.set(newKey, value);
});
}

31
src/node/utils/ImportHtml.js
View file

@ -19,7 +19,7 @@ var Changeset = require("ep_etherpad-lite/static/js/Changeset");
var contentcollector = require("ep_etherpad-lite/static/js/contentcollector");
var cheerio = require("cheerio");
function setPadHTML(pad, html, callback)
exports.setPadHTML = function(pad, html)
{
var apiLogger = log4js.getLogger("ImportHtml");
@ -36,19 +36,22 @@ function setPadHTML(pad, html, callback)
// Convert a dom tree into a list of lines and attribute liens
// using the content collector object
var cc = contentcollector.makeContentCollector(true, null, pad.pool);
try{ // we use a try here because if the HTML is bad it will blow up
try {
// we use a try here because if the HTML is bad it will blow up
cc.collectContent(doc);
}catch(e){
} catch(e) {
apiLogger.warn("HTML was not properly formed", e);
return callback(e); // We don't process the HTML because it was bad..
// don't process the HTML because it was bad
throw e;
}
var result = cc.finish();
apiLogger.debug('Lines:');
var i;
for (i = 0; i < result.lines.length; i += 1)
{
for (i = 0; i < result.lines.length; i++) {
apiLogger.debug('Line ' + (i + 1) + ' text: ' + result.lines[i]);
apiLogger.debug('Line ' + (i + 1) + ' attributes: ' + result.lineAttribs[i]);
}
@ -59,18 +62,15 @@ function setPadHTML(pad, html, callback)
apiLogger.debug(newText);
var newAttribs = result.lineAttribs.join('|1+1') + '|1+1';
function eachAttribRun(attribs, func /*(startInNewText, endInNewText, attribs)*/ )
{
function eachAttribRun(attribs, func /*(startInNewText, endInNewText, attribs)*/ ) {
var attribsIter = Changeset.opIterator(attribs);
var textIndex = 0;
var newTextStart = 0;
var newTextEnd = newText.length;
while (attribsIter.hasNext())
{
while (attribsIter.hasNext()) {
var op = attribsIter.next();
var nextIndex = textIndex + op.chars;
if (!(nextIndex <= newTextStart || textIndex >= newTextEnd))
{
if (!(nextIndex <= newTextStart || textIndex >= newTextEnd)) {
func(Math.max(newTextStart, textIndex), Math.min(newTextEnd, nextIndex), op.attribs);
}
textIndex = nextIndex;
@ -81,17 +81,14 @@ function setPadHTML(pad, html, callback)
var builder = Changeset.builder(1);
// assemble each line into the builder
eachAttribRun(newAttribs, function(start, end, attribs)
{
eachAttribRun(newAttribs, function(start, end, attribs) {
builder.insert(newText.substring(start, end), attribs);
});
// the changeset is ready!
var theChangeset = builder.toString();
apiLogger.debug('The changeset: ' + theChangeset);
pad.setText("\n");
pad.appendRevision(theChangeset);
callback(null);
}
exports.setPadHTML = setPadHTML;

58
src/node/utils/TidyHtml.js
View file

@ -6,36 +6,38 @@ var log4js = require('log4js');
var settings = require('./Settings');
var spawn = require('child_process').spawn;
exports.tidy = function(srcFile, callback) {
exports.tidy = function(srcFile) {
var logger = log4js.getLogger('TidyHtml');
// Don't do anything if Tidy hasn't been enabled
if (!settings.tidyHtml) {
logger.debug('tidyHtml has not been configured yet, ignoring tidy request');
return callback(null);
}
return new Promise((resolve, reject) => {
var errMessage = '';
// Spawn a new tidy instance that cleans up the file inline
logger.debug('Tidying ' + srcFile);
var tidy = spawn(settings.tidyHtml, ['-modify', srcFile]);
// Keep track of any error messages
tidy.stderr.on('data', function (data) {
errMessage += data.toString();
});
// Wait until Tidy is done
tidy.on('close', function(code) {
// Tidy returns a 0 when no errors occur and a 1 exit code when
// the file could be tidied but a few warnings were generated
if (code === 0 || code === 1) {
logger.debug('Tidied ' + srcFile + ' successfully');
return callback(null);
} else {
logger.error('Failed to tidy ' + srcFile + '\n' + errMessage);
return callback('Tidy died with exit code ' + code);
// Don't do anything if Tidy hasn't been enabled
if (!settings.tidyHtml) {
logger.debug('tidyHtml has not been configured yet, ignoring tidy request');
return resolve(null);
}
var errMessage = '';
// Spawn a new tidy instance that cleans up the file inline
logger.debug('Tidying ' + srcFile);
var tidy = spawn(settings.tidyHtml, ['-modify', srcFile]);
// Keep track of any error messages
tidy.stderr.on('data', function (data) {
errMessage += data.toString();
});
tidy.on('close', function(code) {
// Tidy returns a 0 when no errors occur and a 1 exit code when
// the file could be tidied but a few warnings were generated
if (code === 0 || code === 1) {
logger.debug('Tidied ' + srcFile + ' successfully');
resolve(null);
} else {
logger.error('Failed to tidy ' + srcFile + '\n' + errMessage);
reject('Tidy died with exit code ' + code);
}
});
});
};
}

573
src/node/utils/padDiff.js
View file

@ -1,336 +1,267 @@
var Changeset = require("../../static/js/Changeset");
var async = require("async");
var exportHtml = require('./ExportHtml');
function PadDiff (pad, fromRev, toRev){
//check parameters
if(!pad || !pad.id || !pad.atext || !pad.pool)
{
function PadDiff (pad, fromRev, toRev) {
// check parameters
if (!pad || !pad.id || !pad.atext || !pad.pool) {
throw new Error('Invalid pad');
}
var range = pad.getValidRevisionRange(fromRev, toRev);
if(!range) { throw new Error('Invalid revision range.' +
if (!range) {
throw new Error('Invalid revision range.' +
' startRev: ' + fromRev +
' endRev: ' + toRev); }
' endRev: ' + toRev);
}
this._pad = pad;
this._fromRev = range.startRev;
this._toRev = range.endRev;
this._html = null;
this._authors = [];
}
PadDiff.prototype._isClearAuthorship = function(changeset){
//unpack
PadDiff.prototype._isClearAuthorship = function(changeset) {
// unpack
var unpacked = Changeset.unpack(changeset);
//check if there is nothing in the charBank
if(unpacked.charBank !== "")
// check if there is nothing in the charBank
if (unpacked.charBank !== "") {
return false;
//check if oldLength == newLength
if(unpacked.oldLen !== unpacked.newLen)
}
// check if oldLength == newLength
if (unpacked.oldLen !== unpacked.newLen) {
return false;
//lets iterator over the operators
}
// lets iterator over the operators
var iterator = Changeset.opIterator(unpacked.ops);
//get the first operator, this should be a clear operator
// get the first operator, this should be a clear operator
var clearOperator = iterator.next();
//check if there is only one operator
if(iterator.hasNext() === true)
// check if there is only one operator
if (iterator.hasNext() === true) {
return false;
//check if this operator doesn't change text
if(clearOperator.opcode !== "=")
}
// check if this operator doesn't change text
if (clearOperator.opcode !== "=") {
return false;
//check that this operator applys to the complete text
//if the text ends with a new line, its exactly one character less, else it has the same length
if(clearOperator.chars !== unpacked.oldLen-1 && clearOperator.chars !== unpacked.oldLen)
}
// check that this operator applys to the complete text
// if the text ends with a new line, its exactly one character less, else it has the same length
if (clearOperator.chars !== unpacked.oldLen-1 && clearOperator.chars !== unpacked.oldLen) {
return false;
}
var attributes = [];
Changeset.eachAttribNumber(changeset, function(attrNum){
Changeset.eachAttribNumber(changeset, function(attrNum) {
attributes.push(attrNum);
});
//check that this changeset uses only one attribute
if(attributes.length !== 1)
// check that this changeset uses only one attribute
if (attributes.length !== 1) {
return false;
}
var appliedAttribute = this._pad.pool.getAttrib(attributes[0]);
//check if the applied attribute is an anonymous author attribute
if(appliedAttribute[0] !== "author" || appliedAttribute[1] !== "")
// check if the applied attribute is an anonymous author attribute
if (appliedAttribute[0] !== "author" || appliedAttribute[1] !== "") {
return false;
}
return true;
};
PadDiff.prototype._createClearAuthorship = function(rev, callback){
var self = this;
this._pad.getInternalRevisionAText(rev, function(err, atext){
if(err){
return callback(err);
}
//build clearAuthorship changeset
var builder = Changeset.builder(atext.text.length);
builder.keepText(atext.text, [['author','']], self._pad.pool);
var changeset = builder.toString();
callback(null, changeset);
});
};
PadDiff.prototype._createClearStartAtext = function(rev, callback){
var self = this;
//get the atext of this revision
this._pad.getInternalRevisionAText(rev, function(err, atext){
if(err){
return callback(err);
}
//create the clearAuthorship changeset
self._createClearAuthorship(rev, function(err, changeset){
if(err){
return callback(err);
}
try {
//apply the clearAuthorship changeset
var newAText = Changeset.applyToAText(changeset, atext, self._pad.pool);
} catch(err) {
return callback(err)
}
callback(null, newAText);
});
});
};
PadDiff.prototype._getChangesetsInBulk = function(startRev, count, callback) {
var self = this;
//find out which revisions we need
var revisions = [];
for(var i=startRev;i<(startRev+count) && i<=this._pad.head;i++){
PadDiff.prototype._createClearAuthorship = async function(rev) {
let atext = await this._pad.getInternalRevisionAText(rev);
// build clearAuthorship changeset
var builder = Changeset.builder(atext.text.length);
builder.keepText(atext.text, [['author','']], this._pad.pool);
var changeset = builder.toString();
return changeset;
}
PadDiff.prototype._createClearStartAtext = async function(rev) {
// get the atext of this revision
let atext = this._pad.getInternalRevisionAText(rev);
// create the clearAuthorship changeset
let changeset = await this._createClearAuthorship(rev);
// apply the clearAuthorship changeset
let newAText = Changeset.applyToAText(changeset, atext, this._pad.pool);
return newAText;
}
PadDiff.prototype._getChangesetsInBulk = async function(startRev, count) {
// find out which revisions we need
let revisions = [];
for (let i = startRev; i < (startRev + count) && i <= this._pad.head; i++) {
revisions.push(i);
}
var changesets = [], authors = [];
//get all needed revisions
async.forEach(revisions, function(rev, callback){
self._pad.getRevision(rev, function(err, revision){
if(err){
return callback(err);
}
var arrayNum = rev-startRev;
// get all needed revisions (in parallel)
let changesets = [], authors = [];
await Promise.all(revisions.map(rev => {
return this._pad.getRevision(rev).then(revision => {
let arrayNum = rev - startRev;
changesets[arrayNum] = revision.changeset;
authors[arrayNum] = revision.meta.author;
callback();
});
}, function(err){
callback(err, changesets, authors);
});
};
}));
return { changesets, authors };
}
PadDiff.prototype._addAuthors = function(authors) {
var self = this;
//add to array if not in the array
authors.forEach(function(author){
if(self._authors.indexOf(author) == -1){
// add to array if not in the array
authors.forEach(function(author) {
if (self._authors.indexOf(author) == -1) {
self._authors.push(author);
}
});
};
PadDiff.prototype._createDiffAtext = function(callback) {
var self = this;
var bulkSize = 100;
//get the cleaned startAText
self._createClearStartAtext(self._fromRev, function(err, atext){
if(err) { return callback(err); }
var superChangeset = null;
var rev = self._fromRev + 1;
//async while loop
async.whilst(
//loop condition
function () { return rev <= self._toRev; },
//loop body
function (callback) {
//get the bulk
self._getChangesetsInBulk(rev,bulkSize,function(err, changesets, authors){
var addedAuthors = [];
//run trough all changesets
for(var i=0;i<changesets.length && (rev+i)<=self._toRev;i++){
var changeset = changesets[i];
//skip clearAuthorship Changesets
if(self._isClearAuthorship(changeset)){
continue;
}
changeset = self._extendChangesetWithAuthor(changeset, authors[i], self._pad.pool);
//add this author to the authorarray
addedAuthors.push(authors[i]);
//compose it with the superChangset
if(superChangeset === null){
superChangeset = changeset;
} else {
superChangeset = Changeset.composeWithDeletions(superChangeset, changeset, self._pad.pool);
}
}
//add the authors to the PadDiff authorArray
self._addAuthors(addedAuthors);
//lets continue with the next bulk
rev += bulkSize;
callback();
});
},
//after the loop has ended
function (err) {
//if there are only clearAuthorship changesets, we don't get a superChangeset, so we can skip this step
if(superChangeset){
var deletionChangeset = self._createDeletionChangeset(superChangeset,atext,self._pad.pool);
try {
//apply the superChangeset, which includes all addings
atext = Changeset.applyToAText(superChangeset,atext,self._pad.pool);
//apply the deletionChangeset, which adds a deletions
atext = Changeset.applyToAText(deletionChangeset,atext,self._pad.pool);
} catch(err) {
return callback(err)
}
}
callback(err, atext);
PadDiff.prototype._createDiffAtext = async function() {
let bulkSize = 100;
// get the cleaned startAText
let atext = await this._createClearStartAtext(this._fromRev);
let superChangeset = null;
let rev = this._fromRev + 1;
for (let rev = this._fromRev + 1; rev <= this._toRev; rev += bulkSize) {
// get the bulk
let { changesets, authors } = await this._getChangesetsInBulk(rev, bulkSize);
let addedAuthors = [];
// run through all changesets
for (let i = 0; i < changesets.length && (rev + i) <= this._toRev; ++i) {
let changeset = changesets[i];
// skip clearAuthorship Changesets
if (this._isClearAuthorship(changeset)) {
continue;
}
changeset = this._extendChangesetWithAuthor(changeset, authors[i], this._pad.pool);
// add this author to the authorarray
addedAuthors.push(authors[i]);
// compose it with the superChangset
if (superChangeset === null) {
superChangeset = changeset;
} else {
superChangeset = Changeset.composeWithDeletions(superChangeset, changeset, this._pad.pool);
}
);
});
};
PadDiff.prototype.getHtml = function(callback){
//cache the html
if(this._html != null){
return callback(null, this._html);
}
var self = this;
var atext, html, authorColors;
async.series([
//get the diff atext
function(callback){
self._createDiffAtext(function(err, _atext){
if(err){
return callback(err);
}
atext = _atext;
callback();
});
},
//get the authorColor table
function(callback){
self._pad.getAllAuthorColors(function(err, _authorColors){
if(err){
return callback(err);
}
authorColors = _authorColors;
callback();
});
},
//convert the atext to html
function(callback){
html = exportHtml.getHTMLFromAtext(self._pad, atext, authorColors);
self._html = html;
callback();
}
], function(err){
callback(err, html);
});
};
PadDiff.prototype.getAuthors = function(callback){
var self = this;
//check if html was already produced, if not produce it, this generates the author array at the same time
if(self._html == null){
self.getHtml(function(err){
if(err){
return callback(err);
}
callback(null, self._authors);
});
} else {
callback(null, self._authors);
// add the authors to the PadDiff authorArray
this._addAuthors(addedAuthors);
}
};
PadDiff.prototype._extendChangesetWithAuthor = function(changeset, author, apool) {
//unpack
var unpacked = Changeset.unpack(changeset);
// if there are only clearAuthorship changesets, we don't get a superChangeset, so we can skip this step
if (superChangeset) {
let deletionChangeset = this._createDeletionChangeset(superChangeset, atext, this._pad.pool);
// apply the superChangeset, which includes all addings
atext = Changeset.applyToAText(superChangeset, atext, this._pad.pool);
// apply the deletionChangeset, which adds a deletions
atext = Changeset.applyToAText(deletionChangeset, atext, this._pad.pool);
}
return atext;
}
PadDiff.prototype.getHtml = async function() {
// cache the html
if (this._html != null) {
return this._html;
}
// get the diff atext
let atext = await this._createDiffAtext();
// get the authorColor table
let authorColors = await this._pad.getAllAuthorColors();
// convert the atext to html
this._html = exportHtml.getHTMLFromAtext(this._pad, atext, authorColors);
return this._html;
}
PadDiff.prototype.getAuthors = async function() {
// check if html was already produced, if not produce it, this generates the author array at the same time
if (this._html == null) {
await this.getHtml();
}
return self._authors;
}
PadDiff.prototype._extendChangesetWithAuthor = function(changeset, author, apool) {
// unpack
var unpacked = Changeset.unpack(changeset);
var iterator = Changeset.opIterator(unpacked.ops);
var assem = Changeset.opAssembler();
//create deleted attribs
// create deleted attribs
var authorAttrib = apool.putAttrib(["author", author || ""]);
var deletedAttrib = apool.putAttrib(["removed", true]);
var attribs = "*" + Changeset.numToString(authorAttrib) + "*" + Changeset.numToString(deletedAttrib);
//iteratore over the operators of the changeset
while(iterator.hasNext()){
// iteratore over the operators of the changeset
while(iterator.hasNext()) {
var operator = iterator.next();
//this is a delete operator, extend it with the author
if(operator.opcode === "-"){
if (operator.opcode === "-") {
// this is a delete operator, extend it with the author
operator.attribs = attribs;
}
//this is operator changes only attributes, let's mark which author did that
else if(operator.opcode === "=" && operator.attribs){
} else if (operator.opcode === "=" && operator.attribs) {
// this is operator changes only attributes, let's mark which author did that
operator.attribs+="*"+Changeset.numToString(authorAttrib);
}
//append the new operator to our assembler
// append the new operator to our assembler
assem.append(operator);
}
//return the modified changeset
// return the modified changeset
return Changeset.pack(unpacked.oldLen, unpacked.newLen, assem.toString(), unpacked.charBank);
};
//this method is 80% like Changeset.inverse. I just changed so instead of reverting, it adds deletions and attribute changes to to the atext.
// this method is 80% like Changeset.inverse. I just changed so instead of reverting, it adds deletions and attribute changes to to the atext.
PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) {
var lines = Changeset.splitTextLines(startAText.text);
var alines = Changeset.splitAttributionLines(startAText.attribs, startAText.text);
// lines and alines are what the exports is meant to apply to.
// They may be arrays or objects with .get(i) and .length methods.
// They include final newlines on lines.
function lines_get(idx) {
if (lines.get) {
return lines.get(idx);
@ -338,7 +269,7 @@ PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) {
return lines[idx];
}
}
function alines_get(idx) {
if (alines.get) {
return alines.get(idx);
@ -346,19 +277,19 @@ PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) {
return alines[idx];
}
}
var curLine = 0;
var curChar = 0;
var curLineOpIter = null;
var curLineOpIterLine;
var curLineNextOp = Changeset.newOp('+');
var unpacked = Changeset.unpack(cs);
var csIter = Changeset.opIterator(unpacked.ops);
var builder = Changeset.builder(unpacked.newLen);
function consumeAttribRuns(numChars, func /*(len, attribs, endsLine)*/ ) {
if ((!curLineOpIter) || (curLineOpIterLine != curLine)) {
// create curLineOpIter and advance it to curChar
curLineOpIter = Changeset.opIterator(alines_get(curLine));
@ -375,7 +306,7 @@ PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) {
}
}
}
while (numChars > 0) {
if ((!curLineNextOp.chars) && (!curLineOpIter.hasNext())) {
curLine++;
@ -384,22 +315,25 @@ PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) {
curLineNextOp.chars = 0;
curLineOpIter = Changeset.opIterator(alines_get(curLine));
}
if (!curLineNextOp.chars) {
curLineOpIter.next(curLineNextOp);
}
var charsToUse = Math.min(numChars, curLineNextOp.chars);
func(charsToUse, curLineNextOp.attribs, charsToUse == curLineNextOp.chars && curLineNextOp.lines > 0);
numChars -= charsToUse;
curLineNextOp.chars -= charsToUse;
curChar += charsToUse;
}
if ((!curLineNextOp.chars) && (!curLineOpIter.hasNext())) {
curLine++;
curChar = 0;
}
}
function skip(N, L) {
if (L) {
curLine += L;
@ -412,27 +346,29 @@ PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) {
}
}
}
function nextText(numChars) {
var len = 0;
var assem = Changeset.stringAssembler();
var firstString = lines_get(curLine).substring(curChar);
len += firstString.length;
assem.append(firstString);
var lineNum = curLine + 1;
while (len < numChars) {
var nextString = lines_get(lineNum);
len += nextString.length;
assem.append(nextString);
lineNum++;
}
return assem.toString().substring(0, numChars);
}
function cachedStrFunc(func) {
var cache = {};
return function (s) {
if (!cache[s]) {
cache[s] = func(s);
@ -440,57 +376,59 @@ PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) {
return cache[s];
};
}
var attribKeys = [];
var attribValues = [];
//iterate over all operators of this changeset
// iterate over all operators of this changeset
while (csIter.hasNext()) {
var csOp = csIter.next();
if (csOp.opcode == '=') {
if (csOp.opcode == '=') {
var textBank = nextText(csOp.chars);
// decide if this equal operator is an attribution change or not. We can see this by checkinf if attribs is set.
// If the text this operator applies to is only a star, than this is a false positive and should be ignored
if (csOp.attribs && textBank != "*") {
var deletedAttrib = apool.putAttrib(["removed", true]);
var authorAttrib = apool.putAttrib(["author", ""]);
attribKeys.length = 0;
attribValues.length = 0;
Changeset.eachAttribNumber(csOp.attribs, function (n) {
attribKeys.push(apool.getAttribKey(n));
attribValues.push(apool.getAttribValue(n));
if(apool.getAttribKey(n) === "author"){
if (apool.getAttribKey(n) === "author") {
authorAttrib = n;
}
});
var undoBackToAttribs = cachedStrFunc(function (attribs) {
var backAttribs = [];
for (var i = 0; i < attribKeys.length; i++) {
var appliedKey = attribKeys[i];
var appliedValue = attribValues[i];
var oldValue = Changeset.attribsAttributeValue(attribs, appliedKey, apool);
if (appliedValue != oldValue) {
backAttribs.push([appliedKey, oldValue]);
}
}
return Changeset.makeAttribsString('=', backAttribs, apool);
});
var oldAttribsAddition = "*" + Changeset.numToString(deletedAttrib) + "*" + Changeset.numToString(authorAttrib);
var textLeftToProcess = textBank;
while(textLeftToProcess.length > 0){
//process till the next line break or process only one line break
while(textLeftToProcess.length > 0) {
// process till the next line break or process only one line break
var lengthToProcess = textLeftToProcess.indexOf("\n");
var lineBreak = false;
switch(lengthToProcess){
case -1:
switch(lengthToProcess) {
case -1:
lengthToProcess=textLeftToProcess.length;
break;
case 0:
@ -498,27 +436,28 @@ PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) {
lengthToProcess=1;
break;
}
//get the text we want to procceed in this step
// get the text we want to procceed in this step
var processText = textLeftToProcess.substr(0, lengthToProcess);
textLeftToProcess = textLeftToProcess.substr(lengthToProcess);
if(lineBreak){
builder.keep(1, 1); //just skip linebreaks, don't do a insert + keep for a linebreak
//consume the attributes of this linebreak
consumeAttribRuns(1, function(){});
if (lineBreak) {
builder.keep(1, 1); // just skip linebreaks, don't do a insert + keep for a linebreak
// consume the attributes of this linebreak
consumeAttribRuns(1, function() {});
} else {
//add the old text via an insert, but add a deletion attribute + the author attribute of the author who deleted it
// add the old text via an insert, but add a deletion attribute + the author attribute of the author who deleted it
var textBankIndex = 0;
consumeAttribRuns(lengthToProcess, function (len, attribs, endsLine) {
//get the old attributes back
// get the old attributes back
var attribs = (undoBackToAttribs(attribs) || "") + oldAttribsAddition;
builder.insert(processText.substr(textBankIndex, len), attribs);
textBankIndex += len;
});
builder.keep(lengthToProcess, 0);
}
}
@ -531,16 +470,16 @@ PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) {
} else if (csOp.opcode == '-') {
var textBank = nextText(csOp.chars);
var textBankIndex = 0;
consumeAttribRuns(csOp.chars, function (len, attribs, endsLine) {
builder.insert(textBank.substr(textBankIndex, len), attribs + csOp.attribs);
textBankIndex += len;
});
}
}
return Changeset.checkRep(builder.toString());
};
//export the constructor
// export the constructor
module.exports = PadDiff;

2
src/package.json
View file

@ -48,6 +48,7 @@
"languages4translatewiki": "0.1.3",
"log4js": "0.6.35",
"measured-core": "1.11.2",
"nodeify": "^1.0.1",
"npm": "6.4.1",
"object.values": "^1.0.4",
"request": "2.88.0",
@ -86,4 +87,3 @@
"version": "1.7.5",
"license": "Apache-2.0"
}

30
src/static/js/pluginfw/hooks.js
View file

@ -78,7 +78,7 @@ exports.callAll = function (hook_name, args) {
}
}
exports.aCallAll = function (hook_name, args, cb) {
function aCallAll(hook_name, args, cb) {
if (!args) args = {};
if (!cb) cb = function () {};
if (exports.plugins.hooks[hook_name] === undefined) return cb(null, []);
@ -93,6 +93,19 @@ exports.aCallAll = function (hook_name, args, cb) {
);
}
/* return a Promise if cb is not supplied */
exports.aCallAll = function (hook_name, args, cb) {
if (cb === undefined) {
return new Promise(function(resolve, reject) {
aCallAll(hook_name, args, function(err, res) {
return err ? reject(err) : resolve(res);
});
});
} else {
return aCallAll(hook_name, args, cb);
}
}
exports.callFirst = function (hook_name, args) {
if (!args) args = {};
if (exports.plugins.hooks[hook_name] === undefined) return [];
@ -101,7 +114,7 @@ exports.callFirst = function (hook_name, args) {
});
}
exports.aCallFirst = function (hook_name, args, cb) {
function aCallFirst(hook_name, args, cb) {
if (!args) args = {};
if (!cb) cb = function () {};
if (exports.plugins.hooks[hook_name] === undefined) return cb(null, []);
@ -114,6 +127,19 @@ exports.aCallFirst = function (hook_name, args, cb) {
);
}
/* return a Promise if cb is not supplied */
exports.aCallFirst = function (hook_name, args, cb) {
if (cb === undefined) {
return new Promise(function(resolve, reject) {
aCallFirst(hook_name, args, function(err, res) {
return err ? reject(err) : resolve(res);
});
});
} else {
return aCallFirst(hook_name, args, cb);
}
}
exports.callAllStr = function(hook_name, args, sep, pre, post) {
if (sep == undefined) sep = '';
if (pre == undefined) pre = '';

119
src/static/js/pluginfw/installer.js
View file

@ -4,12 +4,14 @@ var npm = require("npm");
var request = require("request");
var npmIsLoaded = false;
var withNpm = function (npmfn) {
if(npmIsLoaded) return npmfn();
npm.load({}, function (er) {
var withNpm = function(npmfn) {
if (npmIsLoaded) return npmfn();
npm.load({}, function(er) {
if (er) return npmfn(er);
npmIsLoaded = true;
npm.on("log", function (message) {
npm.on("log", function(message) {
console.log('npm: ',message)
});
npmfn();
@ -17,42 +19,57 @@ var withNpm = function (npmfn) {
}
var tasks = 0
function wrapTaskCb(cb) {
tasks++
tasks++;
return function() {
cb && cb.apply(this, arguments);
tasks--;
if(tasks == 0) onAllTasksFinished();
if (tasks == 0) onAllTasksFinished();
}
}
function onAllTasksFinished() {
hooks.aCallAll("restartServer", {}, function () {});
hooks.aCallAll("restartServer", {}, function() {});
}
/*
* We cannot use arrow functions in this file, because code in /src/static
* can end up being loaded in browsers, and we still support IE11.
*/
exports.uninstall = function(plugin_name, cb) {
cb = wrapTaskCb(cb);
withNpm(function (er) {
withNpm(function(er) {
if (er) return cb && cb(er);
npm.commands.uninstall([plugin_name], function (er) {
npm.commands.uninstall([plugin_name], function(er) {
if (er) return cb && cb(er);
hooks.aCallAll("pluginUninstall", {plugin_name: plugin_name}, function (er, data) {
if (er) return cb(er);
plugins.update(cb);
});
hooks.aCallAll("pluginUninstall", {plugin_name: plugin_name})
.then(plugins.update)
.then(function() { cb(null) })
.catch(function(er) { cb(er) });
});
});
};
/*
* We cannot use arrow functions in this file, because code in /src/static
* can end up being loaded in browsers, and we still support IE11.
*/
exports.install = function(plugin_name, cb) {
cb = wrapTaskCb(cb)
withNpm(function (er) {
cb = wrapTaskCb(cb);
withNpm(function(er) {
if (er) return cb && cb(er);
npm.commands.install([plugin_name], function (er) {
npm.commands.install([plugin_name], function(er) {
if (er) return cb && cb(er);
hooks.aCallAll("pluginInstall", {plugin_name: plugin_name}, function (er, data) {
if (er) return cb(er);
plugins.update(cb);
});
hooks.aCallAll("pluginInstall", {plugin_name: plugin_name})
.then(plugins.update)
.then(function() { cb(null) })
.catch(function(er) { cb(er) });
});
});
};
@ -60,44 +77,58 @@ exports.install = function(plugin_name, cb) {
exports.availablePlugins = null;
var cacheTimestamp = 0;
exports.getAvailablePlugins = function(maxCacheAge, cb) {
request("https://static.etherpad.org/plugins.json", function(er, response, plugins){
if (er) return cb && cb(er);
if(exports.availablePlugins && maxCacheAge && Math.round(+new Date/1000)-cacheTimestamp <= maxCacheAge) {
return cb && cb(null, exports.availablePlugins)
exports.getAvailablePlugins = function(maxCacheAge) {
var nowTimestamp = Math.round(Date.now() / 1000);
return new Promise(function(resolve, reject) {
// check cache age before making any request
if (exports.availablePlugins && maxCacheAge && (nowTimestamp - cacheTimestamp) <= maxCacheAge) {
return resolve(exports.availablePlugins);
}
try {
plugins = JSON.parse(plugins);
} catch (err) {
console.error('error parsing plugins.json:', err);
plugins = [];
}
exports.availablePlugins = plugins;
cacheTimestamp = Math.round(+new Date/1000);
cb && cb(null, plugins)
request("https://static.etherpad.org/plugins.json", function(er, response, plugins) {
if (er) return reject(er);
try {
plugins = JSON.parse(plugins);
} catch (err) {
console.error('error parsing plugins.json:', err);
plugins = [];
}
exports.availablePlugins = plugins;
cacheTimestamp = nowTimestamp;
resolve(plugins);
});
});
};
exports.search = function(searchTerm, maxCacheAge, cb) {
exports.getAvailablePlugins(maxCacheAge, function(er, results) {
if(er) return cb && cb(er);
exports.search = function(searchTerm, maxCacheAge) {
return exports.getAvailablePlugins(maxCacheAge).then(function(results) {
var res = {};
if (searchTerm)
if (searchTerm) {
searchTerm = searchTerm.toLowerCase();
for (var pluginName in results) { // for every available plugin
}
for (var pluginName in results) {
// for every available plugin
if (pluginName.indexOf(plugins.prefix) != 0) continue; // TODO: Also search in keywords here!
if(searchTerm && !~results[pluginName].name.toLowerCase().indexOf(searchTerm)
if (searchTerm && !~results[pluginName].name.toLowerCase().indexOf(searchTerm)
&& (typeof results[pluginName].description != "undefined" && !~results[pluginName].description.toLowerCase().indexOf(searchTerm) )
){
if(typeof results[pluginName].description === "undefined"){
) {
if (typeof results[pluginName].description === "undefined") {
console.debug('plugin without Description: %s', results[pluginName].name);
}
continue;
}
res[pluginName] = results[pluginName];
}
cb && cb(null, res)
})
return res;
});
};

162
src/static/js/pluginfw/plugins.js
View file

@ -1,7 +1,6 @@
var npm = require("npm/lib/npm.js");
var readInstalled = require("./read-installed.js");
var path = require("path");
var async = require("async");
var fs = require("fs");
var tsort = require("./tsort");
var util = require("util");
@ -15,6 +14,7 @@ exports.plugins = {};
exports.parts = [];
exports.hooks = {};
// @TODO RPB this appears to be unused
exports.ensure = function (cb) {
if (!exports.loaded)
exports.update(cb);
@ -53,106 +53,94 @@ exports.formatHooks = function (hook_set_name) {
return "<dl>" + res.join("\n") + "</dl>";
};
exports.callInit = function (cb) {
exports.callInit = function () {
const fsp_stat = util.promisify(fs.stat);
const fsp_writeFile = util.promisify(fs.writeFile);
var hooks = require("./hooks");
async.map(
Object.keys(exports.plugins),
function (plugin_name, cb) {
var plugin = exports.plugins[plugin_name];
fs.stat(path.normalize(path.join(plugin.package.path, ".ep_initialized")), function (err, stats) {
if (err) {
async.waterfall([
function (cb) { fs.writeFile(path.normalize(path.join(plugin.package.path, ".ep_initialized")), 'done', cb); },
function (cb) { hooks.aCallAll("init_" + plugin_name, {}, cb); },
cb,
]);
} else {
cb();
}
});
},
function () { cb(); }
);
let p = Object.keys(exports.plugins).map(function (plugin_name) {
let plugin = exports.plugins[plugin_name];
let ep_init = path.normalize(path.join(plugin.package.path, ".ep_initialized"));
return fsp_stat(ep_init).catch(async function() {
await fsp_writeFile(ep_init, "done");
await hooks.aCallAll("init_" + plugin_name, {});
});
});
return Promise.all(p);
}
exports.pathNormalization = function (part, hook_fn_name) {
return path.normalize(path.join(path.dirname(exports.plugins[part.plugin].package.path), hook_fn_name));
}
exports.update = function (cb) {
exports.getPackages(function (er, packages) {
var parts = [];
var plugins = {};
// Load plugin metadata ep.json
async.forEach(
Object.keys(packages),
function (plugin_name, cb) {
loadPlugin(packages, plugin_name, plugins, parts, cb);
},
function (err) {
if (err) cb(err);
exports.plugins = plugins;
exports.parts = sortParts(parts);
exports.hooks = pluginUtils.extractHooks(exports.parts, "hooks", exports.pathNormalization);
exports.loaded = true;
exports.callInit(cb);
}
);
});
};
exports.update = async function () {
let packages = await exports.getPackages();
var parts = [];
var plugins = {};
exports.getPackages = function (cb) {
// Load plugin metadata ep.json
let p = Object.keys(packages).map(function (plugin_name) {
return loadPlugin(packages, plugin_name, plugins, parts);
});
return Promise.all(p).then(function() {
exports.plugins = plugins;
exports.parts = sortParts(parts);
exports.hooks = pluginUtils.extractHooks(exports.parts, "hooks", exports.pathNormalization);
exports.loaded = true;
}).then(exports.callInit);
}
exports.getPackages = async function () {
// Load list of installed NPM packages, flatten it to a list, and filter out only packages with names that
var dir = path.resolve(npm.dir, '..');
readInstalled(dir, function (er, data) {
if (er) cb(er, null);
var packages = {};
function flatten(deps) {
_.chain(deps).keys().each(function (name) {
if (name.indexOf(exports.prefix) === 0) {
packages[name] = _.clone(deps[name]);
// Delete anything that creates loops so that the plugin
// list can be sent as JSON to the web client
delete packages[name].dependencies;
delete packages[name].parent;
}
// I don't think we need recursion
//if (deps[name].dependencies !== undefined) flatten(deps[name].dependencies);
});
}
var tmp = {};
tmp[data.name] = data;
flatten(tmp[data.name].dependencies);
cb(null, packages);
});
let data = await util.promisify(readInstalled)(dir);
var packages = {};
function flatten(deps) {
_.chain(deps).keys().each(function (name) {
if (name.indexOf(exports.prefix) === 0) {
packages[name] = _.clone(deps[name]);
// Delete anything that creates loops so that the plugin
// list can be sent as JSON to the web client
delete packages[name].dependencies;
delete packages[name].parent;
}
// I don't think we need recursion
//if (deps[name].dependencies !== undefined) flatten(deps[name].dependencies);
});
}
var tmp = {};
tmp[data.name] = data;
flatten(tmp[data.name].dependencies);
return packages;
};
function loadPlugin(packages, plugin_name, plugins, parts, cb) {
async function loadPlugin(packages, plugin_name, plugins, parts) {
let fsp_readFile = util.promisify(fs.readFile);
var plugin_path = path.resolve(packages[plugin_name].path, "ep.json");
fs.readFile(
plugin_path,
function (er, data) {
if (er) {
console.error("Unable to load plugin definition file " + plugin_path);
return cb();
}
try {
var plugin = JSON.parse(data);
plugin['package'] = packages[plugin_name];
plugins[plugin_name] = plugin;
_.each(plugin.parts, function (part) {
part.plugin = plugin_name;
part.full_name = plugin_name + "/" + part.name;
parts[part.full_name] = part;
});
} catch (ex) {
console.error("Unable to parse plugin definition file " + plugin_path + ": " + ex.toString());
}
cb();
try {
let data = await fsp_readFile(plugin_path);
try {
var plugin = JSON.parse(data);
plugin['package'] = packages[plugin_name];
plugins[plugin_name] = plugin;
_.each(plugin.parts, function (part) {
part.plugin = plugin_name;
part.full_name = plugin_name + "/" + part.name;
parts[part.full_name] = part;
});
} catch (ex) {
console.error("Unable to parse plugin definition file " + plugin_path + ": " + ex.toString());
}
);
} catch (er) {
console.error("Unable to load plugin definition file " + plugin_path);
}
}
function partsToParentChildList(parts) {

10
tests/backend/specs/api/tidy.js
View file

@ -1,10 +1,12 @@
var assert = require('assert')
os = require('os'),
fs = require('fs'),
path = require('path'),
TidyHtml = null,
Settings = null;
var npm = require("../../../../src/node_modules/npm/lib/npm.js");
var nodeify = require('../../../../src/node_modules/nodeify');
describe('tidyHtml', function() {
before(function(done) {
@ -16,6 +18,10 @@ describe('tidyHtml', function() {
});
});
function tidy(file, callback) {
return nodeify(TidyHtml.tidy(file), callback);
}
it('Tidies HTML', function(done) {
// If the user hasn't configured Tidy, we skip this tests as it's required for this test
if (!Settings.tidyHtml) {
@ -27,7 +33,7 @@ describe('tidyHtml', function() {
var tmpFile = path.join(tmpDir, 'tmp_' + (Math.floor(Math.random() * 1000000)) + '.html')
fs.writeFileSync(tmpFile, '<html><body><p>a paragraph</p><li>List without outer UL</li>trailing closing p</p></body></html>');
TidyHtml.tidy(tmpFile, function(err){
tidy(tmpFile, function(err){
assert.ok(!err);
// Read the file again
@ -56,7 +62,7 @@ describe('tidyHtml', function() {
this.skip();
}
TidyHtml.tidy('/some/none/existing/file.html', function(err) {
tidy('/some/none/existing/file.html', function(err) {
assert.ok(err);
return done();
});