libretic
/
etherpad-lite
5
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

bugfix, lint and refactor all bin scripts (#4617) 

* bugfix, lint and refactor all bin scripts

* for squash: throw Error(message) rather than log(message); throw Error()

* for squash: Exit non-0 on unhandled Promise rejection

Many of the recent lint changes have converted normal functions to
async functions, and an error thrown in an async function does not
cause Node.js to exit by default.

* for squash: fix `require()` paths

* for squash: remove erroneous `Object.keys()` call

* for squash: fix missing `continue` statements

* for squash: Fix HTTP method for deleteSession

* for squash: delete erroneous throw

Throw is only for errors, not successful completion.

* for squash: redo migrateDirtyDBtoRealDB.js to fix async bugs

* for squash: fix erroneous use of `for..of`

* for squash: Add line break between statements

* for squash: put closing paren on same line as last arg

* for squash: Move `log()` back up where it was

to minimize the diff to develop

* for squash: indentation fixes

* for squash: typo fix

* for squash: wrap long lines

* for squash: use `util.callbackify` to silence promise/no-callback-in-promise warning

* for squash: use double quotes to improve readability

Co-authored-by: Richard Hansen <rhansen@rhansen.org>
This commit is contained in:
John McLear 2021-01-18 08:53:15 +00:00 committed by GitHub
parent c0d9881a62
commit 2fdc737355
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 458 additions and 466 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
bin/checkAllPads.js
View File

@ -1,28 +1,31 @@
'use strict';
/*
* This is a debug tool. It checks all revisions for data corruption
*/
if (process.argv.length != 2) {
console.error('Use: node bin/checkAllPads.js');
process.exit(1);
}
// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
process.on('unhandledRejection', (err) => { throw err; });
if (process.argv.length !== 2) throw new Error('Use: node bin/checkAllPads.js');
// load and initialize NPM
const npm = require('../src/node_modules/npm');
const npm = require('ep_etherpad-lite/node_modules/npm');
npm.load({}, async () => {
try {
// initialize the database
const settings = require('../src/node/utils/Settings');
const db = require('../src/node/db/DB');
require('ep_etherpad-lite/node/utils/Settings');
const db = require('ep_etherpad-lite/node/db/DB');
await db.init();
// load modules
const Changeset = require('../src/static/js/Changeset');
const padManager = require('../src/node/db/PadManager');
const Changeset = require('ep_etherpad-lite/static/js/Changeset');
const padManager = require('ep_etherpad-lite/node/db/PadManager');
let revTestedCount = 0;
// get all pads
const res = await padManager.listAllPads();
for (const padId of res.padIDs) {
const pad = await padManager.getPad(padId);
@ -31,7 +34,6 @@ npm.load({}, async () => {
console.error(`[${pad.id}] Missing attribute pool`);
continue;
}
// create an array with key kevisions
// key revisions always save the full pad atext
const head = pad.getHeadRevisionNumber();
@ -71,21 +73,23 @@ npm.load({}, async () => {
const apool = pad.pool;
let atext = revisions[keyRev].meta.atext;
for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) {
try {
const cs = revisions[rev].changeset;
atext = Changeset.applyToAText(cs, atext, apool);
revTestedCount++;
} catch (e) {
console.error(`[${pad.id}] Bad changeset at revision ${i} - ${e.message}`);
console.error(`[${pad.id}] Bad changeset at revision ${rev} - ${e.message}`);
}
}
}
console.log('finished');
process.exit(0);
}
if (revTestedCount === 0) {
throw new Error('No revisions tested');
}
console.log(`Finished: Tested ${revTestedCount} revisions`);
} catch (err) {
console.trace(err);
process.exit(1);
throw err;
}
});

47
bin/checkPad.js
View File

@ -1,33 +1,33 @@
'use strict';
/*
* This is a debug tool. It checks all revisions for data corruption
*/
if (process.argv.length != 3) {
console.error('Use: node bin/checkPad.js $PADID');
process.exit(1);
}
// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
process.on('unhandledRejection', (err) => { throw err; });
if (process.argv.length !== 3) throw new Error('Use: node bin/checkPad.js $PADID');
// get the padID
const padId = process.argv[2];
let checkRevisionCount = 0;
// load and initialize NPM;
const npm = require('../src/node_modules/npm');
const npm = require('ep_etherpad-lite/node_modules/npm');
npm.load({}, async () => {
try {
// initialize database
const settings = require('../src/node/utils/Settings');
const db = require('../src/node/db/DB');
require('ep_etherpad-lite/node/utils/Settings');
const db = require('ep_etherpad-lite/node/db/DB');
await db.init();
// load modules
const Changeset = require('ep_etherpad-lite/static/js/Changeset');
const padManager = require('../src/node/db/PadManager');
const padManager = require('ep_etherpad-lite/node/db/PadManager');
const exists = await padManager.doesPadExists(padId);
if (!exists) {
console.error('Pad does not exist');
process.exit(1);
}
if (!exists) throw new Error('Pad does not exist');
// get the pad
const pad = await padManager.getPad(padId);
@ -41,7 +41,8 @@ npm.load({}, async () => {
}
// run through all key revisions
for (const keyRev of keyRevisions) {
for (let keyRev of keyRevisions) {
keyRev = parseInt(keyRev);
// create an array of revisions we need till the next keyRevision or the End
const revisionsNeeded = [];
for (let rev = keyRev; rev <= keyRev + 100 && rev <= head; rev++) {
@ -58,13 +59,12 @@ npm.load({}, async () => {
}
// check if the pad has a pool
if (pad.pool === undefined) {
console.error('Attribute pool is missing');
process.exit(1);
}
if (pad.pool === undefined) throw new Error('Attribute pool is missing');
// check if there is an atext in the keyRevisions
if (revisions[keyRev] === undefined || revisions[keyRev].meta === undefined || revisions[keyRev].meta.atext === undefined) {
if (revisions[keyRev] === undefined ||
revisions[keyRev].meta === undefined ||
revisions[keyRev].meta.atext === undefined) {
console.error(`No atext in key revision ${keyRev}`);
continue;
}
@ -73,8 +73,8 @@ npm.load({}, async () => {
let atext = revisions[keyRev].meta.atext;
for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) {
checkRevisionCount++;
try {
// console.log("check revision " + rev);
const cs = revisions[rev].changeset;
atext = Changeset.applyToAText(cs, atext, apool);
} catch (e) {
@ -82,11 +82,10 @@ npm.load({}, async () => {
continue;
}
}
console.log('finished');
process.exit(0);
console.log(`Finished: Checked ${checkRevisionCount} revisions`);
}
} catch (e) {
console.trace(e);
process.exit(1);
} catch (err) {
console.trace(err);
throw err;
}
});

176
bin/checkPadDeltas.js
View File

@ -1,111 +1,107 @@
'use strict';
/*
* This is a debug tool. It checks all revisions for data corruption
*/
if (process.argv.length != 3) {
console.error('Use: node bin/checkPadDeltas.js $PADID');
process.exit(1);
}
// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
process.on('unhandledRejection', (err) => { throw err; });
if (process.argv.length !== 3) throw new Error('Use: node bin/checkPadDeltas.js $PADID');
// get the padID
const padId = process.argv[2];
// load and initialize NPM;
const expect = require('expect.js');
const diff = require('diff');
var async = require('async');
const npm = require('../src/node_modules/npm');
var async = require('ep_etherpad-lite/node_modules/async');
const Changeset = require('ep_etherpad-lite/static/js/Changeset');
const expect = require('../tests/frontend/lib/expect');
const diff = require('ep_etherpad-lite/node_modules/diff');
const npm = require('ep_etherpad-lite/node_modules/npm');
npm.load({}, async () => {
try {
// initialize database
const settings = require('../src/node/utils/Settings');
const db = require('../src/node/db/DB');
await db.init();
// initialize database
require('ep_etherpad-lite/node/utils/Settings');
const db = require('ep_etherpad-lite/node/db/DB');
await db.init();
// load modules
const Changeset = require('ep_etherpad-lite/static/js/Changeset');
const padManager = require('../src/node/db/PadManager');
// load modules
const Changeset = require('ep_etherpad-lite/static/js/Changeset');
const padManager = require('ep_etherpad-lite/node/db/PadManager');
const exists = await padManager.doesPadExists(padId);
if (!exists) {
console.error('Pad does not exist');
process.exit(1);
const exists = await padManager.doesPadExists(padId);
if (!exists) throw new Error('Pad does not exist');
// get the pad
const pad = await padManager.getPad(padId);
// create an array with key revisions
// key revisions always save the full pad atext
const head = pad.getHeadRevisionNumber();
const keyRevisions = [];
for (let i = 0; i < head; i += 100) {
keyRevisions.push(i);
}
// create an array with all revisions
const revisions = [];
for (let i = 0; i <= head; i++) {
revisions.push(i);
}
let atext = Changeset.makeAText('\n');
// run trough all revisions
for (const revNum of revisions) {
// console.log('Fetching', revNum)
const revision = await db.get(`pad:${padId}:revs:${revNum}`);
// check if there is a atext in the keyRevisions
if (~keyRevisions.indexOf(revNum) &&
(revision === undefined ||
revision.meta === undefined ||
revision.meta.atext === undefined)) {
console.error(`No atext in key revision ${revNum}`);
continue;
}
// get the pad
const pad = await padManager.getPad(padId);
// create an array with key revisions
// key revisions always save the full pad atext
const head = pad.getHeadRevisionNumber();
const keyRevisions = [];
for (var i = 0; i < head; i += 100) {
keyRevisions.push(i);
// try glue everything together
try {
// console.log("check revision ", revNum);
const cs = revision.changeset;
atext = Changeset.applyToAText(cs, atext, pad.pool);
} catch (e) {
console.error(`Bad changeset at revision ${revNum} - ${e.message}`);
continue;
}
// create an array with all revisions
const revisions = [];
for (var i = 0; i <= head; i++) {
revisions.push(i);
}
let atext = Changeset.makeAText('\n');
// run trough all revisions
async.forEachSeries(revisions, (revNum, callback) => {
// console.log('Fetching', revNum)
db.db.get(`pad:${padId}:revs:${revNum}`, (err, revision) => {
if (err) return callback(err);
// check if there is a atext in the keyRevisions
if (~keyRevisions.indexOf(revNum) && (revision === undefined || revision.meta === undefined || revision.meta.atext === undefined)) {
console.error(`No atext in key revision ${revNum}`);
callback();
return;
}
try {
// console.log("check revision ", revNum);
const cs = revision.changeset;
atext = Changeset.applyToAText(cs, atext, pad.pool);
} catch (e) {
console.error(`Bad changeset at revision ${revNum} - ${e.message}`);
callback();
return;
}
if (~keyRevisions.indexOf(revNum)) {
try {
expect(revision.meta.atext.text).to.eql(atext.text);
expect(revision.meta.atext.attribs).to.eql(atext.attribs);
} catch (e) {
console.error(`Atext in key revision ${revNum} doesn't match computed one.`);
console.log(diff.diffChars(atext.text, revision.meta.atext.text).map((op) => { if (!op.added && !op.removed) op.value = op.value.length; return op; }));
// console.error(e)
// console.log('KeyRev. :', revision.meta.atext)
// console.log('Computed:', atext)
callback();
return;
}
}
setImmediate(callback);
});
}, (er) => {
if (pad.atext.text == atext.text) { console.log('ok'); } else {
console.error('Pad AText doesn\'t match computed one! (Computed ', atext.text.length, ', db', pad.atext.text.length, ')');
console.log(diff.diffChars(atext.text, pad.atext.text).map((op) => { if (!op.added && !op.removed) op.value = op.value.length; return op; }));
// check things are working properly
if (~keyRevisions.indexOf(revNum)) {
try {
expect(revision.meta.atext.text).to.eql(atext.text);
expect(revision.meta.atext.attribs).to.eql(atext.attribs);
} catch (e) {
console.error(`Atext in key revision ${revNum} doesn't match computed one.`);
console.log(diff.diffChars(atext.text, revision.meta.atext.text).map((op) => {
if (!op.added && !op.removed) op.value = op.value.length;
return op;
}));
// console.error(e)
// console.log('KeyRev. :', revision.meta.atext)
// console.log('Computed:', atext)
continue;
}
callback(er);
});
}
}
process.exit(0);
} catch (e) {
console.trace(e);
process.exit(1);
// check final text is right...
if (pad.atext.text === atext.text) {
console.log('ok');
} else {
console.error('Pad AText doesn\'t match computed one! (Computed ',
atext.text.length, ', db', pad.atext.text.length, ')');
console.log(diff.diffChars(atext.text, pad.atext.text).map((op) => {
if (!op.added && !op.removed) {
op.value = op.value.length;
return op;
}
}));
}
});

12
bin/createUserSession.js
View File

@ -1,15 +1,19 @@
'use strict';
/*
* A tool for generating a test user session which can be used for debugging configs
* that require sessions.
*/
const m = (f) => `${__dirname}/../${f}`;
// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
process.on('unhandledRejection', (err) => { throw err; });
const fs = require('fs');
const path = require('path');
const querystring = require('querystring');
const request = require(m('src/node_modules/request'));
const settings = require(m('src/node/utils/Settings'));
const supertest = require(m('src/node_modules/supertest'));
const settings = require('ep_etherpad-lite/node/utils/Settings');
const supertest = require('ep_etherpad-lite/node_modules/supertest');
(async () => {
const api = supertest(`http://${settings.ip}:${settings.port}`);

74
bin/deleteAllGroupSessions.js
View File

@ -1,51 +1,47 @@
'use strict';
/*
* A tool for deleting ALL GROUP sessions Etherpad user sessions from the CLI,
* because sometimes a brick is required to fix a face.
*/
const request = require('../src/node_modules/request');
const settings = require(`${__dirname}/../tests/container/loadSettings`).loadSettings();
const supertest = require(`${__dirname}/../src/node_modules/supertest`);
const api = supertest(`http://${settings.ip}:${settings.port}`);
// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
process.on('unhandledRejection', (err) => { throw err; });
const supertest = require('ep_etherpad-lite/node_modules/supertest');
const path = require('path');
const fs = require('fs');
// Set a delete counter which will increment on each delete attempt
// TODO: Check delete is successful before incrementing
let deleteCount = 0;
// get the API Key
const filePath = path.join(__dirname, '../APIKEY.txt');
const apikey = fs.readFileSync(filePath, {encoding: 'utf-8'});
console.log('Deleting all group sessions, please be patient.');
// Set apiVersion to base value, we change this later.
let apiVersion = 1;
let guids;
(async () => {
const settings = require('../tests/container/loadSettings').loadSettings();
const apikey = fs.readFileSync(filePath, {encoding: 'utf-8'});
const api = supertest(`http://${settings.ip}:${settings.port}`);
// Update the apiVersion
api.get('/api/')
.expect((res) => {
apiVersion = res.body.currentVersion;
if (!res.body.currentVersion) throw new Error('No version set in API');
return;
})
.then(() => {
const guri = `/api/${apiVersion}/listAllGroups?apikey=${apikey}`;
api.get(guri)
.then((res) => {
guids = res.body.data.groupIDs;
guids.forEach((groupID) => {
const luri = `/api/${apiVersion}/listSessionsOfGroup?apikey=${apikey}&groupID=${groupID}`;
api.get(luri)
.then((res) => {
if (res.body.data) {
Object.keys(res.body.data).forEach((sessionID) => {
if (sessionID) {
console.log('Deleting', sessionID);
const duri = `/api/${apiVersion}/deleteSession?apikey=${apikey}&sessionID=${sessionID}`;
api.post(duri); // deletes
}
});
} else {
// no session in this group.
}
});
});
});
});
const apiVersionResponse = await api.get('/api/');
const apiVersion = apiVersionResponse.body.currentVersion; // 1.12.5
const groupsResponse = await api.get(`/api/${apiVersion}/listAllGroups?apikey=${apikey}`);
const groups = groupsResponse.body.data.groupIDs; // ['whateverGroupID']
for (const groupID of groups) {
const sessionURI = `/api/${apiVersion}/listSessionsOfGroup?apikey=${apikey}&groupID=${groupID}`;
const sessionsResponse = await api.get(sessionURI);
const sessions = sessionsResponse.body.data;
for (const sessionID of Object.keys(sessions)) {
const deleteURI = `/api/${apiVersion}/deleteSession?apikey=${apikey}&sessionID=${sessionID}`;
await api.post(deleteURI); // delete
deleteCount++;
}
}
console.log(`Deleted ${deleteCount} sessions`);
})();

51
bin/deletePad.js
View File

@ -1,18 +1,21 @@
'use strict';
/*
* A tool for deleting pads from the CLI, because sometimes a brick is required
* to fix a window.
*/
const request = require('../src/node_modules/request');
const settings = require(`${__dirname}/../tests/container/loadSettings`).loadSettings();
const supertest = require(`${__dirname}/../src/node_modules/supertest`);
// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
process.on('unhandledRejection', (err) => { throw err; });
const settings = require('../tests/container/loadSettings').loadSettings();
const supertest = require('ep_etherpad-lite/node_modules/supertest');
const api = supertest(`http://${settings.ip}:${settings.port}`);
const path = require('path');
const fs = require('fs');
if (process.argv.length != 3) {
console.error('Use: node deletePad.js $PADID');
process.exit(1);
}
if (process.argv.length !== 3) throw new Error('Use: node deletePad.js $PADID');
// get the padID
const padId = process.argv[2];
@ -21,28 +24,14 @@ const padId = process.argv[2];
const filePath = path.join(__dirname, '../APIKEY.txt');
const apikey = fs.readFileSync(filePath, {encoding: 'utf-8'});
// Set apiVersion to base value, we change this later.
let apiVersion = 1;
(async () => {
let apiVersion = await api.get('/api/');
apiVersion = apiVersion.body.currentVersion;
if (!apiVersion) throw new Error('No version set in API');
// Update the apiVersion
api.get('/api/')
.expect((res) => {
apiVersion = res.body.currentVersion;
if (!res.body.currentVersion) throw new Error('No version set in API');
return;
})
.end((err, res) => {
// Now we know the latest API version, let's delete pad
const uri = `/api/${apiVersion}/deletePad?apikey=${apikey}&padID=${padId}`;
api.post(uri)
.expect((res) => {
if (res.body.code === 1) {
console.error('Error deleting pad', res.body);
} else {
console.log('Deleted pad', res.body);
}
return;
})
.end(() => {});
});
// end
// Now we know the latest API version, let's delete pad
const uri = `/api/${apiVersion}/deletePad?apikey=${apikey}&padID=${padId}`;
const deleteAttempt = await api.post(uri);
if (deleteAttempt.body.code === 1) throw new Error(`Error deleting pad ${deleteAttempt.body}`);
console.log('Deleted pad', deleteAttempt.body);
})();

35
bin/extractPadData.js
View File

@ -1,34 +1,34 @@
'use strict';
/*
* This is a debug tool. It helps to extract all datas of a pad and move it from
* a productive environment and to a develop environment to reproduce bugs
* there. It outputs a dirtydb file
*/
if (process.argv.length != 3) {
console.error('Use: node extractPadData.js $PADID');
process.exit(1);
}
// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
process.on('unhandledRejection', (err) => { throw err; });
if (process.argv.length !== 3) throw new Error('Use: node extractPadData.js $PADID');
// get the padID
const padId = process.argv[2];
const npm = require('../src/node_modules/npm');
const npm = require('ep_etherpad-lite/node_modules/npm');
npm.load({}, async (er) => {
if (er) {
console.error(`Could not load NPM: ${er}`);
process.exit(1);
}
npm.load({}, async (err) => {
if (err) throw err;
try {
// initialize database
const settings = require('../src/node/utils/Settings');
const db = require('../src/node/db/DB');
require('ep_etherpad-lite/node/utils/Settings');
const db = require('ep_etherpad-lite/node/db/DB');
await db.init();
// load extra modules
const dirtyDB = require('../src/node_modules/dirty');
const padManager = require('../src/node/db/PadManager');
const dirtyDB = require('ep_etherpad-lite/node_modules/dirty');
const padManager = require('ep_etherpad-lite/node/db/PadManager');
const util = require('util');
// initialize output database
@ -67,9 +67,8 @@ npm.load({}, async (er) => {
}
console.log('finished');
process.exit(0);
} catch (er) {
console.error(er);
process.exit(1);
} catch (err) {
console.error(err);
throw err;
}
});

150
bin/importSqlFile.js
View File

@ -1,77 +1,18 @@
'use strict';
// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
process.on('unhandledRejection', (err) => { throw err; });
const startTime = Date.now();
require('ep_etherpad-lite/node_modules/npm').load({}, (er, npm) => {
const fs = require('fs');
const ueberDB = require('ep_etherpad-lite/node_modules/ueberdb2');
const settings = require('ep_etherpad-lite/node/utils/Settings');
const log4js = require('ep_etherpad-lite/node_modules/log4js');
const dbWrapperSettings = {
cache: 0,
writeInterval: 100,
json: false, // data is already json encoded
};
const db = new ueberDB.database(settings.dbType, settings.dbSettings, dbWrapperSettings, log4js.getLogger('ueberDB'));
const sqlFile = process.argv[2];
// stop if the settings file is not set
if (!sqlFile) {
console.error('Use: node importSqlFile.js $SQLFILE');
process.exit(1);
}
log('initializing db');
db.init((err) => {
// there was an error while initializing the database, output it and stop
if (err) {
console.error('ERROR: Problem while initializing the database');
console.error(err.stack ? err.stack : err);
process.exit(1);
} else {
log('done');
log('open output file...');
const lines = fs.readFileSync(sqlFile, 'utf8').split('\n');
const count = lines.length;
let keyNo = 0;
process.stdout.write(`Start importing ${count} keys...\n`);
lines.forEach((l) => {
if (l.substr(0, 27) == 'REPLACE INTO store VALUES (') {
const pos = l.indexOf("', '");
const key = l.substr(28, pos - 28);
let value = l.substr(pos + 3);
value = value.substr(0, value.length - 2);
console.log(`key: ${key} val: ${value}`);
console.log(`unval: ${unescape(value)}`);
db.set(key, unescape(value), null);
keyNo++;
if (keyNo % 1000 == 0) {
process.stdout.write(` ${keyNo}/${count}\n`);
}
}
});
process.stdout.write('\n');
process.stdout.write('done. waiting for db to finish transaction. depended on dbms this may take some time...\n');
db.close(() => {
log(`finished, imported ${keyNo} keys.`);
process.exit(0);
});
}
});
});
function log(str) {
const log = (str) => {
console.log(`${(Date.now() - startTime) / 1000}\t${str}`);
}
};
unescape = function (val) {
const unescape = (val) => {
// value is a string
if (val.substr(0, 1) == "'") {
if (val.substr(0, 1) === "'") {
val = val.substr(0, val.length - 1).substr(1);
return val.replace(/\\[0nrbtZ\\'"]/g, (s) => {
@ -88,16 +29,81 @@ unescape = function (val) {
}
// value is a boolean or NULL
if (val == 'NULL') {
if (val === 'NULL') {
return null;
}
if (val == 'true') {
if (val === 'true') {
return true;
}
if (val == 'false') {
if (val === 'false') {
return false;
}
// value is a number
return val;
};
require('ep_etherpad-lite/node_modules/npm').load({}, (er, npm) => {
const fs = require('fs');
const ueberDB = require('ep_etherpad-lite/node_modules/ueberdb2');
const settings = require('ep_etherpad-lite/node/utils/Settings');
const log4js = require('ep_etherpad-lite/node_modules/log4js');
const dbWrapperSettings = {
cache: 0,
writeInterval: 100,
json: false, // data is already json encoded
};
const db = new ueberDB.database( // eslint-disable-line new-cap
settings.dbType,
settings.dbSettings,
dbWrapperSettings,
log4js.getLogger('ueberDB'));
const sqlFile = process.argv[2];
// stop if the settings file is not set
if (!sqlFile) throw new Error('Use: node importSqlFile.js $SQLFILE');
log('initializing db');
db.init((err) => {
// there was an error while initializing the database, output it and stop
if (err) {
throw err;
} else {
log('done');
log('open output file...');
const lines = fs.readFileSync(sqlFile, 'utf8').split('\n');
const count = lines.length;
let keyNo = 0;
process.stdout.write(`Start importing ${count} keys...\n`);
lines.forEach((l) => {
if (l.substr(0, 27) === 'REPLACE INTO store VALUES (') {
const pos = l.indexOf("', '");
const key = l.substr(28, pos - 28);
let value = l.substr(pos + 3);
value = value.substr(0, value.length - 2);
console.log(`key: ${key} val: ${value}`);
console.log(`unval: ${unescape(value)}`);
db.set(key, unescape(value), null);
keyNo++;
if (keyNo % 1000 === 0) {
process.stdout.write(` ${keyNo}/${count}\n`);
}
}
});
process.stdout.write('\n');
process.stdout.write('done. waiting for db to finish transaction. ' +
'depended on dbms this may take some time..\n');
db.close(() => {
log(`finished, imported ${keyNo} keys.`);
});
}
});
});

69
bin/migrateDirtyDBtoRealDB.js
View File

@ -1,4 +1,12 @@
require('ep_etherpad-lite/node_modules/npm').load({}, (er, npm) => {
'use strict';
// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
process.on('unhandledRejection', (err) => { throw err; });
const util = require('util');
require('ep_etherpad-lite/node_modules/npm').load({}, async (er, npm) => {
process.chdir(`${npm.root}/..`);
// This script requires that you have modified your settings.json file
@ -10,39 +18,42 @@ require('ep_etherpad-lite/node_modules/npm').load({}, (er, npm) => {
const settings = require('ep_etherpad-lite/node/utils/Settings');
let dirty = require('../src/node_modules/dirty');
const ueberDB = require('../src/node_modules/ueberdb2');
const log4js = require('../src/node_modules/log4js');
const dirtyDb = require('ep_etherpad-lite/node_modules/dirty');
const ueberDB = require('ep_etherpad-lite/node_modules/ueberdb2');
const log4js = require('ep_etherpad-lite/node_modules/log4js');
const dbWrapperSettings = {
cache: '0', // The cache slows things down when you're mostly writing.
writeInterval: 0, // Write directly to the database, don't buffer
};
const db = new ueberDB.database(settings.dbType, settings.dbSettings, dbWrapperSettings, log4js.getLogger('ueberDB'));
let i = 0;
let length = 0;
const db = new ueberDB.database( // eslint-disable-line new-cap
settings.dbType,
settings.dbSettings,
dbWrapperSettings,
log4js.getLogger('ueberDB'));
await db.init();
db.init(() => {
console.log('Waiting for dirtyDB to parse its file.');
dirty = dirty('var/dirty.db').on('load', () => {
dirty.forEach(() => {
length++;
});
console.log(`Found ${length} records, processing now.`);
console.log('Waiting for dirtyDB to parse its file.');
const dirty = dirtyDb('var/dirty.db');
const length = await new Promise((resolve) => { dirty.once('load', resolve); });
dirty.forEach(async (key, value) => {
const error = await db.set(key, value);
console.log(`Wrote record ${i}`);
i++;
if (i === length) {
console.log('finished, just clearing up for a bit...');
setTimeout(() => {
process.exit(0);
}, 5000);
}
});
console.log('Please wait for all records to flush to database, then kill this process.');
});
console.log('done?');
console.log(`Found ${length} records, processing now.`);
const p = [];
let numWritten = 0;
dirty.forEach((key, value) => {
let bcb, wcb;
p.push(new Promise((resolve, reject) => {
bcb = (err) => { if (err != null) return reject(err); };
wcb = (err) => {
if (err != null) return reject(err);
if (++numWritten % 100 === 0) console.log(`Wrote record ${numWritten} of ${length}`);
resolve();
};
}));
db.set(key, value, bcb, wcb);
});
await Promise.all(p);
console.log(`Wrote all ${numWritten} records`);
await util.promisify(db.close.bind(db))();
console.log('Finished.');
});

46
bin/plugins/checkPlugin.js
View File

@ -9,16 +9,17 @@
* node bin/plugins/checkPlugin.js ep_whatever autocommit
*/
// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
process.on('unhandledRejection', (err) => { throw err; });
const fs = require('fs');
const childProcess = require('child_process');
// get plugin name & path from user input
const pluginName = process.argv[2];
if (!pluginName) {
console.error('no plugin name specified');
process.exit(1);
}
if (!pluginName) throw new Error('no plugin name specified');
const pluginPath = `node_modules/${pluginName}`;
@ -107,10 +108,7 @@ fs.readdir(pluginPath, (err, rootFiles) => {
files.push(rootFiles[i].toLowerCase());
}
if (files.indexOf('.git') === -1) {
console.error('No .git folder, aborting');
process.exit(1);
}
if (files.indexOf('.git') === -1) throw new Error('No .git folder, aborting');
prepareRepo();
try {
@ -302,8 +300,10 @@ fs.readdir(pluginPath, (err, rootFiles) => {
if (files.indexOf('contributing') === -1 && files.indexOf('contributing.md') === -1) {
console.warn('CONTRIBUTING.md file not found, please create');
if (autoFix) {
console.log('Autofixing missing CONTRIBUTING.md file, please edit the CONTRIBUTING.md file further to include plugin specific details.');
let contributing = fs.readFileSync('bin/plugins/lib/CONTRIBUTING.md', {encoding: 'utf8', flag: 'r'});
console.log('Autofixing missing CONTRIBUTING.md file, please edit the CONTRIBUTING.md ' +
'file further to include plugin specific details.');
let contributing =
fs.readFileSync('bin/plugins/lib/CONTRIBUTING.md', {encoding: 'utf8', flag: 'r'});
contributing = contributing.replace(/\[plugin_name\]/g, pluginName);
fs.writeFileSync(`${pluginPath}/CONTRIBUTING.md`, contributing);
}
@ -311,7 +311,8 @@ fs.readdir(pluginPath, (err, rootFiles) => {
if (files.indexOf('readme') !== -1 && files.indexOf('readme.md') !== -1) {
const readme = fs.readFileSync(`${pluginPath}/${readMeFileName}`, {encoding: 'utf8', flag: 'r'});
const readme =
fs.readFileSync(`${pluginPath}/${readMeFileName}`, {encoding: 'utf8', flag: 'r'});
if (readme.toLowerCase().indexOf('license') === -1) {
console.warn('No license section in README');
if (autoFix) {
@ -335,7 +336,9 @@ fs.readdir(pluginPath, (err, rootFiles) => {
travisConfig = travisConfig.replace(/\[plugin_name\]/g, pluginName);
if (files.indexOf('.travis.yml') === -1) {
console.warn('.travis.yml file not found, please create. .travis.yml is used for automatically CI testing Etherpad. It is useful to know if your plugin breaks another feature for example.');
console.warn('.travis.yml file not found, please create. ' +
'.travis.yml is used for automatically CI testing Etherpad. ' +
'It is useful to know if your plugin breaks another feature for example.');
// TODO: Make it check version of the .travis file to see if it needs an update.
if (autoFix) {
console.log('Autofixing missing .travis.yml file');
@ -345,9 +348,11 @@ fs.readdir(pluginPath, (err, rootFiles) => {
}
if (autoFix) {
// checks the file versioning of .travis and updates it to the latest.
const existingConfig = fs.readFileSync(`${pluginPath}/.travis.yml`, {encoding: 'utf8', flag: 'r'});
const existingConfig =
fs.readFileSync(`${pluginPath}/.travis.yml`, {encoding: 'utf8', flag: 'r'});
const existingConfigLocation = existingConfig.indexOf('##ETHERPAD_TRAVIS_V=');
const existingValue = parseInt(existingConfig.substr(existingConfigLocation + 20, existingConfig.length));
const existingValue =
parseInt(existingConfig.substr(existingConfigLocation + 20, existingConfig.length));
const newConfigLocation = travisConfig.indexOf('##ETHERPAD_TRAVIS_V=');
const newValue = parseInt(travisConfig.substr(newConfigLocation + 20, travisConfig.length));
@ -362,7 +367,8 @@ fs.readdir(pluginPath, (err, rootFiles) => {
}
if (files.indexOf('.gitignore') === -1) {
console.warn(".gitignore file not found, please create. .gitignore files are useful to ensure files aren't incorrectly commited to a repository.");
console.warn('.gitignore file not found, please create. .gitignore files are useful to ' +
"ensure files aren't incorrectly commited to a repository.");
if (autoFix) {
console.log('Autofixing missing .gitignore file');
const gitignore = fs.readFileSync('bin/plugins/lib/gitignore', {encoding: 'utf8', flag: 'r'});
@ -382,12 +388,15 @@ fs.readdir(pluginPath, (err, rootFiles) => {
// if we include templates but don't have translations...
if (files.indexOf('templates') !== -1 && files.indexOf('locales') === -1) {
console.warn('Translations not found, please create. Translation files help with Etherpad accessibility.');
console.warn('Translations not found, please create. ' +
'Translation files help with Etherpad accessibility.');
}
if (files.indexOf('.ep_initialized') !== -1) {
console.warn('.ep_initialized found, please remove. .ep_initialized should never be commited to git and should only exist once the plugin has been executed one time.');
console.warn(
'.ep_initialized found, please remove. .ep_initialized should never be commited to git ' +
'and should only exist once the plugin has been executed one time.');
if (autoFix) {
console.log('Autofixing incorrectly existing .ep_initialized file');
fs.unlinkSync(`${pluginPath}/.ep_initialized`);
@ -395,7 +404,8 @@ fs.readdir(pluginPath, (err, rootFiles) => {
}
if (files.indexOf('npm-debug.log') !== -1) {
console.warn('npm-debug.log found, please remove. npm-debug.log should never be commited to your repository.');
console.warn('npm-debug.log found, please remove. npm-debug.log should never be commited to ' +
'your repository.');
if (autoFix) {
console.log('Autofixing incorrectly existing npm-debug.log file');
fs.unlinkSync(`${pluginPath}/npm-debug.log`);

79
bin/rebuildPad.js
View File

@ -1,43 +1,39 @@
'use strict';
/*
This is a repair tool. It rebuilds an old pad at a new pad location up to a
known "good" revision.
*/
if (process.argv.length != 4 && process.argv.length != 5) {
console.error('Use: node bin/repairPad.js $PADID $REV [$NEWPADID]');
process.exit(1);
// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
process.on('unhandledRejection', (err) => { throw err; });
if (process.argv.length !== 4 && process.argv.length !== 5) {
throw new Error('Use: node bin/repairPad.js $PADID $REV [$NEWPADID]');
}
const npm = require('../src/node_modules/npm');
const async = require('../src/node_modules/async');
const ueberDB = require('../src/node_modules/ueberdb2');
const async = require('ep_etherpad-lite/node_modules/async');
const npm = require('ep_etherpad-lite/node_modules/npm');
const util = require('util');
const padId = process.argv[2];
const newRevHead = process.argv[3];
const newPadId = process.argv[4] || `${padId}-rebuilt`;
let db, oldPad, newPad, settings;
let AuthorManager, ChangeSet, Pad, PadManager;
let db, oldPad, newPad;
let Pad, PadManager;
async.series([
function (callback) {
npm.load({}, (err) => {
if (err) {
console.error(`Could not load NPM: ${err}`);
process.exit(1);
} else {
callback();
}
});
},
function (callback) {
(callback) => npm.load({}, callback),
(callback) => {
// Get a handle into the database
db = require('../src/node/db/DB');
db = require('ep_etherpad-lite/node/db/DB');
db.init(callback);
},
function (callback) {
PadManager = require('../src/node/db/PadManager');
Pad = require('../src/node/db/Pad').Pad;
(callback) => {
Pad = require('ep_etherpad-lite/node/db/Pad').Pad;
PadManager = require('ep_etherpad-lite/node/db/PadManager');
// Get references to the original pad and to a newly created pad
// HACK: This is a standalone script, so we want to write everything
// out to the database immediately. The only problem with this is
@ -46,14 +42,10 @@ async.series([
// Validate the newPadId if specified and that a pad with that ID does
// not already exist to avoid overwriting it.
if (!PadManager.isValidPadId(newPadId)) {
console.error('Cannot create a pad with that id as it is invalid');
process.exit(1);
throw new Error('Cannot create a pad with that id as it is invalid');
}
PadManager.doesPadExists(newPadId, (err, exists) => {
if (exists) {
console.error('Cannot create a pad with that id as it already exists');
process.exit(1);
}
if (exists) throw new Error('Cannot create a pad with that id as it already exists');
});
PadManager.getPad(padId, (err, pad) => {
oldPad = pad;
@ -61,10 +53,10 @@ async.series([
callback();
});
},
function (callback) {
(callback) => {
// Clone all Chat revisions
const chatHead = oldPad.chatHead;
for (var i = 0, curHeadNum = 0; i <= chatHead; i++) {
for (let i = 0, curHeadNum = 0; i <= chatHead; i++) {
db.db.get(`pad:${padId}:chat:${i}`, (err, chat) => {
db.db.set(`pad:${newPadId}:chat:${curHeadNum++}`, chat);
console.log(`Created: Chat Revision: pad:${newPadId}:chat:${curHeadNum}`);
@ -72,10 +64,10 @@ async.series([
}
callback();
},
function (callback) {
(callback) => {
// Rebuild Pad from revisions up to and including the new revision head
AuthorManager = require('../src/node/db/AuthorManager');
Changeset = require('ep_etherpad-lite/static/js/Changeset');
const AuthorManager = require('ep_etherpad-lite/node/db/AuthorManager');
const Changeset = require('ep_etherpad-lite/static/js/Changeset');
// Author attributes are derived from changesets, but there can also be
// non-author attributes with specific mappings that changesets depend on
// and, AFAICT, cannot be recreated any other way
@ -83,7 +75,7 @@ async.series([
for (let curRevNum = 0; curRevNum <= newRevHead; curRevNum++) {
db.db.get(`pad:${padId}:revs:${curRevNum}`, (err, rev) => {
if (rev.meta) {
throw 'The specified revision number could not be found.';
throw new Error('The specified revision number could not be found.');
}
const newRevNum = ++newPad.head;
const newRevId = `pad:${newPad.id}:revs:${newRevNum}`;
@ -91,18 +83,17 @@ async.series([
AuthorManager.addPad(rev.meta.author, newPad.id);
newPad.atext = Changeset.applyToAText(rev.changeset, newPad.atext, newPad.pool);
console.log(`Created: Revision: pad:${newPad.id}:revs:${newRevNum}`);
if (newRevNum == newRevHead) {
if (newRevNum === newRevHead) {
callback();
}
});
}
},
function (callback) {
(callback) => {
// Add saved revisions up to the new revision head
console.log(newPad.head);
const newSavedRevisions = [];
for (const i in oldPad.savedRevisions) {
savedRev = oldPad.savedRevisions[i];
for (const savedRev of oldPad.savedRevisions) {
if (savedRev.revNum <= newRevHead) {
newSavedRevisions.push(savedRev);
console.log(`Added: Saved Revision: ${savedRev.revNum}`);
@ -111,16 +102,14 @@ async.series([
newPad.savedRevisions = newSavedRevisions;
callback();
},
function (callback) {
(callback) => {
// Save the source pad
db.db.set(`pad:${newPadId}`, newPad, (err) => {
console.log(`Created: Source Pad: pad:${newPadId}`);
newPad.saveToDatabase().then(() => callback(), callback);
util.callbackify(newPad.saveToDatabase.bind(newPad))(callback);
});
},
], (err) => {
if (err) { throw err; } else {
console.info('finished');
process.exit(0);
}
if (err) throw err;
console.info('finished');
});

41
bin/release.js
View File

@ -1,8 +1,12 @@
'use strict';
// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
process.on('unhandledRejection', (err) => { throw err; });
const fs = require('fs');
const child_process = require('child_process');
const semver = require('../src/node_modules/semver');
const childProcess = require('child_process');
const semver = require('ep_etherpad-lite/node_modules/semver');
/*
@ -15,7 +19,7 @@ const usage = 'node bin/release.js [patch/minor/major] -- example: "node bin/rel
const release = process.argv[2];
if(!release) {
if (!release) {
console.log(usage);
throw new Error('No release type included');
}
@ -26,14 +30,14 @@ packageJson = JSON.parse(packageJson);
const currentVersion = packageJson.version;
const newVersion = semver.inc(currentVersion, release);
if(!newVersion) {
if (!newVersion) {
console.log(usage);
throw new Error('Unable to generate new version from input');
}
const changelogIncludesVersion = changelog.indexOf(newVersion) !== -1;
if(!changelogIncludesVersion) {
if (!changelogIncludesVersion) {
throw new Error('No changelog record for ', newVersion, ' - please create changelog record');
}
@ -44,24 +48,27 @@ packageJson.version = newVersion;
fs.writeFileSync('src/package.json', JSON.stringify(packageJson, null, 2));
// run npm version `release` where release is patch, minor or major
child_process.execSync('npm install --package-lock-only', {cwd: `src/`});
childProcess.execSync('npm install --package-lock-only', {cwd: 'src/'});
// run npm install --package-lock-only <-- required???
child_process.execSync(`git checkout -b release/${newVersion}`);
child_process.execSync(`git add src/package.json`);
child_process.execSync(`git add src/package-lock.json`);
child_process.execSync(`git commit -m 'bump version'`);
child_process.execSync(`git push origin release/${newVersion}`);
childProcess.execSync(`git checkout -b release/${newVersion}`);
childProcess.execSync('git add src/package.json');
childProcess.execSync('git add src/package-lock.json');
childProcess.execSync('git commit -m "bump version"');
childProcess.execSync(`git push origin release/${newVersion}`);
child_process.execSync(`make docs`);
child_process.execSync(`git clone git@github.com:ether/ether.github.com.git`);
child_process.execSync(`cp -R out/doc/ ether.github.com/doc/v${newVersion}`);
childProcess.execSync('make docs');
childProcess.execSync('git clone git@github.com:ether/ether.github.com.git');
childProcess.execSync(`cp -R out/doc/ ether.github.com/doc/v${newVersion}`);
console.log('Once merged into master please run the following commands');
console.log(`git tag -a ${newVersion} -m ${newVersion} && git push origin master`);
console.log(`cd ether.github.com && git add . && git commit -m '${newVersion} docs'`);
console.log(`Build the windows zip`)
console.log(`Visit https://github.com/ether/etherpad-lite/releases/new and create a new release with 'master' as the target and the version is ${newVersion}. Include the windows zip as an assett`)
console.log('Once the new docs are uploaded then modify the download link on etherpad.org and then pull master onto develop');
console.log('Build the windows zip');
console.log('Visit https://github.com/ether/etherpad-lite/releases/new and create a new release ' +
`with 'master' as the target and the version is ${newVersion}. Include the windows ` +
'zip as an asset');
console.log(`Once the new docs are uploaded then modify the download
link on etherpad.org and then pull master onto develop`);
console.log('Finally go public with an announcement via our comms channels :)');

108
bin/repairPad.js
View File

@ -1,77 +1,59 @@
'use strict';
/*
* This is a repair tool. It extracts all datas of a pad, removes and inserts them again.
*/
// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
process.on('unhandledRejection', (err) => { throw err; });
console.warn('WARNING: This script must not be used while etherpad is running!');
if (process.argv.length != 3) {
console.error('Use: node bin/repairPad.js $PADID');
process.exit(1);
}
if (process.argv.length !== 3) throw new Error('Use: node bin/repairPad.js $PADID');
// get the padID
const padId = process.argv[2];
const npm = require('../src/node_modules/npm');
npm.load({}, async (er) => {
if (er) {
console.error(`Could not load NPM: ${er}`);
process.exit(1);
let valueCount = 0;
const npm = require('ep_etherpad-lite/node_modules/npm');
npm.load({}, async (err) => {
if (err) throw err;
// intialize database
require('ep_etherpad-lite/node/utils/Settings');
const db = require('ep_etherpad-lite/node/db/DB');
await db.init();
// get the pad
const padManager = require('ep_etherpad-lite/node/db/PadManager');
const pad = await padManager.getPad(padId);
// accumulate the required keys
const neededDBValues = [`pad:${padId}`];
// add all authors
neededDBValues.push(...pad.getAllAuthors().map((author) => `globalAuthor:${author}`));
// add all revisions
for (let rev = 0; rev <= pad.head; ++rev) {
neededDBValues.push(`pad:${padId}:revs:${rev}`);
}
try {
// intialize database
const settings = require('../src/node/utils/Settings');
const db = require('../src/node/db/DB');
await db.init();
// get the pad
const padManager = require('../src/node/db/PadManager');
const pad = await padManager.getPad(padId);
// accumulate the required keys
const neededDBValues = [`pad:${padId}`];
// add all authors
neededDBValues.push(...pad.getAllAuthors().map((author) => 'globalAuthor:'));
// add all revisions
for (let rev = 0; rev <= pad.head; ++rev) {
neededDBValues.push(`pad:${padId}:revs:${rev}`);
}
// add all chat values
for (let chat = 0; chat <= pad.chatHead; ++chat) {
neededDBValues.push(`pad:${padId}:chat:${chat}`);
}
//
// NB: this script doesn't actually does what's documented
// since the `value` fields in the following `.forEach`
// block are just the array index numbers
//
// the script therefore craps out now before it can do
// any damage.
//
// See gitlab issue #3545
//
console.info('aborting [gitlab #3545]');
process.exit(1);
// now fetch and reinsert every key
neededDBValues.forEach((key, value) => {
console.log(`Key: ${key}, value: ${value}`);
db.remove(key);
db.set(key, value);
});
console.info('finished');
process.exit(0);
} catch (er) {
if (er.name === 'apierror') {
console.error(er);
} else {
console.trace(er);
}
// add all chat values
for (let chat = 0; chat <= pad.chatHead; ++chat) {
neededDBValues.push(`pad:${padId}:chat:${chat}`);
}
// now fetch and reinsert every key
for (const key of neededDBValues) {
const value = await db.get(key);
// if it isn't a globalAuthor value which we want to ignore..
// console.log(`Key: ${key}, value: ${JSON.stringify(value)}`);
await db.remove(key);
await db.set(key, value);
valueCount++;
}
console.info(`Finished: Replaced ${valueCount} values in the database`);
});