From 48e1d1c23f8c22d9bfcd228229faa7f8aaa34026 Mon Sep 17 00:00:00 2001 From: Richard Hansen Date: Thu, 4 Mar 2021 13:46:42 -0500 Subject: [PATCH 01/51] CSS: Fix class name for outer iframe `` tag * Add the class "pad" to the `` tag in `pad.html` (the outer iframe's parent). * Change the CSS selector that refers to the `` tag in `pad.html` from `html:not(.inner-editor)` to `html.pad`. * Change the class name of the outer iframe's `` tag from "inner-editor" to "outer-editor". * Update CSS rules to use the new class name. --- src/static/css/iframe_editor.css | 2 +- src/static/css/pad/layout.css | 2 +- src/static/js/ace.js | 2 +- src/templates/pad.html | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/static/css/iframe_editor.css b/src/static/css/iframe_editor.css index 2a63c380..d106880b 100644 --- a/src/static/css/iframe_editor.css +++ b/src/static/css/iframe_editor.css @@ -4,7 +4,7 @@ @import url('./lists_and_indents.css'); -html.inner-editor { +html.outer-editor, html.inner-editor { height: auto !important; background-color: transparent !important; } diff --git a/src/static/css/pad/layout.css b/src/static/css/pad/layout.css index e5b79c26..e0bd04b7 100644 --- a/src/static/css/pad/layout.css +++ b/src/static/css/pad/layout.css @@ -4,7 +4,7 @@ html, body { margin: 0; padding: 0; } -html:not(.inner-editor), html:not(.inner-editor) body { +html.pad, html.pad body { overflow: hidden; } body { diff --git a/src/static/js/ace.js b/src/static/js/ace.js index 44556dd0..57ccde13 100644 --- a/src/static/js/ace.js +++ b/src/static/js/ace.js @@ -207,7 +207,7 @@ const Ace2Editor = function () { })();`; const outerHTML = - [doctype, ``]; + [doctype, ``]; pushStyleTagsFor(outerHTML, includedCSS); // bizarrely, in FF2, a file with no "external" dependencies won't finish loading properly diff --git a/src/templates/pad.html b/src/templates/pad.html index 26243806..7bf2346f 100644 --- a/src/templates/pad.html +++ b/src/templates/pad.html @@ -7,7 +7,7 @@ <% e.begin_block("htmlHead"); %> - + <% e.end_block(); %> <%=settings.title%> From 470f40d7db01b68856edf47ac7a5befd50b50340 Mon Sep 17 00:00:00 2001 From: Richard Hansen Date: Thu, 4 Mar 2021 14:37:36 -0500 Subject: [PATCH 02/51] CSS: Use `auto` for iframe body height This change makes no visual difference right now, but will matter (for reasons I don't understand) once we change `ace.js` to build the iframes by constructing elements in JavaScript (vs. writing HTML). --- src/static/css/iframe_editor.css | 1 - src/static/css/pad/layout.css | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/static/css/iframe_editor.css b/src/static/css/iframe_editor.css index d106880b..156eaff6 100644 --- a/src/static/css/iframe_editor.css +++ b/src/static/css/iframe_editor.css @@ -5,7 +5,6 @@ @import url('./lists_and_indents.css'); html.outer-editor, html.inner-editor { - height: auto !important; background-color: transparent !important; } #outerdocbody { diff --git a/src/static/css/pad/layout.css b/src/static/css/pad/layout.css index e0bd04b7..df340831 100644 --- a/src/static/css/pad/layout.css +++ b/src/static/css/pad/layout.css @@ -1,11 +1,12 @@ html, body { width: 100%; - height: 100%; + height: auto; margin: 0; padding: 0; } html.pad, html.pad body { overflow: hidden; + height: 100%; } body { display: flex; From 5546cc5e7be8d085e204857df610bb3395779a49 Mon Sep 17 00:00:00 2001 From: Richard Hansen Date: Thu, 4 Mar 2021 14:54:17 -0500 Subject: [PATCH 03/51] CSS: Delete bogus `` tag Browsers report an error with this tag. Strangely, this tag has existed since Etherpad's very first commit. --- src/static/js/ace.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/static/js/ace.js b/src/static/js/ace.js index 57ccde13..0d273b85 100644 --- a/src/static/js/ace.js +++ b/src/static/js/ace.js @@ -215,7 +215,6 @@ const Ace2Editor = function () { const pluginNames = pluginUtils.clientPluginNames(); outerHTML.push( '', - '', scriptTag(outerScript), '', '', From 60da2373a6eff54769aeadc2e40aa7f7a09cf22a Mon Sep 17 00:00:00 2001 From: Richard Hansen Date: Thu, 4 Mar 2021 15:13:31 -0500 Subject: [PATCH 04/51] CSS: Add comment to `no-skin/pad.css` to silence warning Firefox prints "Style sheet could not be loaded" if the file is empty. --- src/static/skins/no-skin/pad.css | 1 + 1 file changed, 1 insertion(+) diff --git a/src/static/skins/no-skin/pad.css b/src/static/skins/no-skin/pad.css index e69de29b..a9eae81f 100644 --- a/src/static/skins/no-skin/pad.css +++ b/src/static/skins/no-skin/pad.css @@ -0,0 +1 @@ +/* intentionally empty */ From 4ca989a255a9b4eed06b122b9fad97304eb930d7 Mon Sep 17 00:00:00 2001 From: webzwo0i Date: Fri, 5 Mar 2021 08:48:33 +0100 Subject: [PATCH 05/51] sessions: add more endpoints that do not need a session (#4921) * add more endpoints that do not need a session * Update src/node/hooks/express/webaccess.js Co-authored-by: Richard Hansen * Update src/node/hooks/express/webaccess.js Co-authored-by: Richard Hansen Co-authored-by: John McLear Co-authored-by: Richard Hansen --- src/node/hooks/express/webaccess.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/node/hooks/express/webaccess.js b/src/node/hooks/express/webaccess.js index 51d57ae2..5ff957a5 100644 --- a/src/node/hooks/express/webaccess.js +++ b/src/node/hooks/express/webaccess.js @@ -10,12 +10,20 @@ const readOnlyManager = require('../../db/ReadOnlyManager'); hooks.deprecationNotices.authFailure = 'use the authnFailure and authzFailure hooks instead'; const staticPathsRE = new RegExp(`^/(?:${[ - 'api/.*', + 'api(?:/.*)?', 'favicon\\.ico', + 'ep/pad/connection-diagnostic-info', + 'javascript', 'javascripts/.*', + 'jserror/?', 'locales\\.json', + 'locales/.*', + 'rest/.*', 'pluginfw/.*', + 'robots.txt', 'static/.*', + 'stats/?', + 'tests/frontend(?:/.*)?' ].join('|')})$`); exports.normalizeAuthzLevel = (level) => { From 404486069c204cbec840aa802fc6a18dac013418 Mon Sep 17 00:00:00 2001 From: Richard Hansen Date: Thu, 25 Feb 2021 21:11:30 -0500 Subject: [PATCH 06/51] ace: Build the outer and inner iframes programmatically This makes the code easier to read and it silences Chrome's `document.write()` warning: https://developers.google.com/web/updates/2016/08/removing-document-write This is a redo of commit a17f9bf3cfc745a44d0e57b77912e346ffd3ce1c, which was reverted in commit 912f0f195faf19b11a5db928b3846fbb09388004 due to a CSS bug. --- src/static/js/ace.js | 294 ++++++++++++++++++++++++++++--------------- 1 file changed, 192 insertions(+), 102 deletions(-) diff --git a/src/static/js/ace.js b/src/static/js/ace.js index 0d273b85..fd6fca37 100644 --- a/src/static/js/ace.js +++ b/src/static/js/ace.js @@ -28,19 +28,88 @@ const hooks = require('./pluginfw/hooks'); const pluginUtils = require('./pluginfw/shared'); const debugLog = (...args) => {}; -window.debugLog = debugLog; // The inner and outer iframe's locations are about:blank, so relative URLs are relative to that. // Firefox and Chrome seem to do what the developer intends if given a relative URL, but Safari // errors out unless given an absolute URL for a JavaScript-created element. const absUrl = (url) => new URL(url, window.location.href).href; -const scriptTag = - (source) => ``; +const eventFired = async (obj, event, cleanups = [], predicate = () => true) => { + if (typeof cleanups === 'function') { + predicate = cleanups; + cleanups = []; + } + await new Promise((resolve, reject) => { + let cleanup; + const successCb = () => { + if (!predicate()) return; + debugLog(`Ace2Editor.init() ${event} event on`, obj); + cleanup(); + resolve(); + }; + const errorCb = () => { + const err = new Error(`Ace2Editor.init() error event while waiting for ${event} event`); + debugLog(`${err} on object`, obj); + cleanup(); + reject(err); + }; + cleanup = () => { + cleanup = () => {}; + obj.removeEventListener(event, successCb); + obj.removeEventListener('error', errorCb); + }; + cleanups.push(cleanup); + obj.addEventListener(event, successCb); + obj.addEventListener('error', errorCb); + }); +}; + +const pollCondition = async (predicate, cleanups, pollPeriod, timeout) => { + let done = false; + cleanups.push(() => { done = true; }); + // Pause a tick to give the predicate a chance to become true before adding latency. + await new Promise((resolve) => setTimeout(resolve, 0)); + const start = Date.now(); + while (!done && !predicate()) { + if (Date.now() - start > timeout) throw new Error('timeout'); + await new Promise((resolve) => setTimeout(resolve, pollPeriod)); + debugLog('Ace2Editor.init() polling'); + } + if (!done) debugLog('Ace2Editor.init() poll condition became true'); +}; + +// Resolves when the frame's document is ready to be mutated: +// - Firefox seems to replace the frame's contentWindow.document object with a different object +// after the frame is created so we need to wait for the window's load event before continuing. +// - Chrome doesn't need any waiting (not even next tick), but on Windows it never seems to fire +// any events. Eventually the document's readyState becomes 'complete' (even though it never +// fires a readystatechange event), so this function waits for that to happen to avoid returning +// too soon on Firefox. +// - Safari behaves like Chrome. +// I'm not sure how other browsers behave, so this function throws the kitchen sink at the problem. +// Maybe one day we'll find a concise general solution. +const frameReady = async (frame) => { + // Can't do `const doc = frame.contentDocument;` because Firefox seems to asynchronously replace + // the document object after the frame is first created for some reason. ¯\_(ツ)_/¯ + const doc = () => frame.contentDocument; + const cleanups = []; + try { + await Promise.race([ + eventFired(frame, 'load', cleanups), + eventFired(frame.contentWindow, 'load', cleanups), + eventFired(doc(), 'load', cleanups), + eventFired(doc(), 'DOMContentLoaded', cleanups), + eventFired(doc(), 'readystatechange', cleanups, () => doc.readyState === 'complete'), + // If all else fails, poll. + pollCondition(() => doc().readyState === 'complete', cleanups, 10, 5000), + ]); + } finally { + for (const cleanup of cleanups) cleanup(); + } +}; const Ace2Editor = function () { let info = {editor: this}; - window.ace2EditorInfo = info; // Make it accessible to iframes. let loaded = false; let actionsPendingInit = []; @@ -109,16 +178,19 @@ const Ace2Editor = function () { // returns array of {error: , time: +new Date()} this.getUnhandledErrors = () => loaded ? info.ace_getUnhandledErrors() : []; - const pushStyleTagsFor = (buffer, files) => { + const addStyleTagsFor = (doc, files) => { for (const file of files) { - buffer.push(``); + const link = doc.createElement('link'); + link.rel = 'stylesheet'; + link.type = 'text/css'; + link.href = absUrl(encodeURI(file)); + doc.head.appendChild(link); } }; this.destroy = pendingInit(() => { info.ace_dispose(); info.frame.parentNode.removeChild(info.frame); - delete window.ace2EditorInfo; info = null; // prevent IE 6 closure memory leaks }); @@ -135,109 +207,127 @@ const Ace2Editor = function () { `../static/skins/${clientVars.skinName}/pad.css?v=${clientVars.randomVersionString}`, ]; - const doctype = ''; + const skinVariants = clientVars.skinVariants.split(' ').filter((x) => x !== ''); - const iframeHTML = []; - - iframeHTML.push(doctype); - iframeHTML.push(``); - pushStyleTagsFor(iframeHTML, includedCSS); - const requireKernelUrl = - absUrl(`../static/js/require-kernel.js?v=${clientVars.randomVersionString}`); - iframeHTML.push(``); - // Pre-fetch modules to improve load performance. - for (const module of ['ace2_inner', 'ace2_common']) { - const url = absUrl(`../javascripts/lib/ep_etherpad-lite/static/js/${module}.js` + - `?callback=require.define&v=${clientVars.randomVersionString}`); - iframeHTML.push(``); - } - - iframeHTML.push(scriptTag(`(async () => { - parent.parent.debugLog('Ace2Editor.init() inner frame ready'); - const require = window.require; - require.setRootURI(${JSON.stringify(absUrl('../javascripts/src'))}); - require.setLibraryURI(${JSON.stringify(absUrl('../javascripts/lib'))}); - require.setGlobalKeyPath('require'); - - // intentially moved before requiring client_plugins to save a 307 - window.Ace2Inner = require('ep_etherpad-lite/static/js/ace2_inner'); - window.plugins = require('ep_etherpad-lite/static/js/pluginfw/client_plugins'); - window.plugins.adoptPluginsFromAncestorsOf(window); - - window.$ = window.jQuery = require('ep_etherpad-lite/static/js/rjquery').jQuery; - - parent.parent.debugLog('Ace2Editor.init() waiting for plugins'); - await new Promise((resolve, reject) => window.plugins.ensure( - (err) => err != null ? reject(err) : resolve())); - parent.parent.debugLog('Ace2Editor.init() waiting for Ace2Inner.init()'); - const editorInfo = parent.parent.ace2EditorInfo; - await new Promise((resolve, reject) => window.Ace2Inner.init( - editorInfo, (err) => err != null ? reject(err) : resolve())); - parent.parent.debugLog('Ace2Editor.init() Ace2Inner.init() returned'); - editorInfo.onEditorReady(); - })();`)); - - iframeHTML.push(''); - - hooks.callAll('aceInitInnerdocbodyHead', { - iframeHTML, - }); - - iframeHTML.push(' '); - - const outerScript = `(async () => { - await new Promise((resolve) => { window.onload = () => resolve(); }); - parent.debugLog('Ace2Editor.init() outer frame ready'); - window.onload = null; - await new Promise((resolve) => setTimeout(resolve, 0)); - const iframe = document.createElement('iframe'); - iframe.name = 'ace_inner'; - iframe.title = 'pad'; - iframe.scrolling = 'no'; - iframe.frameBorder = 0; - iframe.allowTransparency = true; // for IE - iframe.ace_outerWin = window; - document.body.insertBefore(iframe, document.body.firstChild); - const doc = iframe.contentWindow.document; - doc.open(); - doc.write(${JSON.stringify(iframeHTML.join('\n'))}); - doc.close(); - parent.debugLog('Ace2Editor.init() waiting for inner frame'); - })();`; - - const outerHTML = - [doctype, ``]; - pushStyleTagsFor(outerHTML, includedCSS); - - // bizarrely, in FF2, a file with no "external" dependencies won't finish loading properly - // (throbs busy while typing) - const pluginNames = pluginUtils.clientPluginNames(); - outerHTML.push( - '', - scriptTag(outerScript), - '', - '', - '
', - '
x
', - ''); - - const outerFrame = document.createElement('IFRAME'); + const outerFrame = document.createElement('iframe'); outerFrame.name = 'ace_outer'; outerFrame.frameBorder = 0; // for IE outerFrame.title = 'Ether'; info.frame = outerFrame; document.getElementById(containerId).appendChild(outerFrame); + const outerWindow = outerFrame.contentWindow; - const editorDocument = outerFrame.contentWindow.document; - + // For some unknown reason Firefox replaces outerWindow.document with a new Document object some + // time between running the above code and firing the outerWindow load event. Work around it by + // waiting until the load event fires before mutating the Document object. debugLog('Ace2Editor.init() waiting for outer frame'); - await new Promise((resolve, reject) => { - info.onEditorReady = (err) => err != null ? reject(err) : resolve(); - editorDocument.open(); - editorDocument.write(outerHTML.join('')); - editorDocument.close(); - }); + await frameReady(outerFrame); + debugLog('Ace2Editor.init() outer frame ready'); + + // This must be done after the Window's load event. See above comment. + const outerDocument = outerWindow.document; + + // tag + outerDocument.insertBefore( + outerDocument.implementation.createDocumentType('html', '', ''), outerDocument.firstChild); + outerDocument.documentElement.classList.add('outer-editor', 'outerdoc', ...skinVariants); + + // tag + addStyleTagsFor(outerDocument, includedCSS); + const outerStyle = outerDocument.createElement('style'); + outerStyle.type = 'text/css'; + outerStyle.title = 'dynamicsyntax'; + outerDocument.head.appendChild(outerStyle); + + // tag + outerDocument.body.id = 'outerdocbody'; + outerDocument.body.classList.add('outerdocbody', ...pluginUtils.clientPluginNames()); + const sideDiv = outerDocument.createElement('div'); + sideDiv.id = 'sidediv'; + sideDiv.classList.add('sidediv'); + outerDocument.body.appendChild(sideDiv); + const lineMetricsDiv = outerDocument.createElement('div'); + lineMetricsDiv.id = 'linemetricsdiv'; + lineMetricsDiv.appendChild(outerDocument.createTextNode('x')); + outerDocument.body.appendChild(lineMetricsDiv); + + const innerFrame = outerDocument.createElement('iframe'); + innerFrame.name = 'ace_inner'; + innerFrame.title = 'pad'; + innerFrame.scrolling = 'no'; + innerFrame.frameBorder = 0; + innerFrame.allowTransparency = true; // for IE + innerFrame.ace_outerWin = outerWindow; + outerDocument.body.insertBefore(innerFrame, outerDocument.body.firstChild); + const innerWindow = innerFrame.contentWindow; + + // Wait before mutating the inner document. See above comment recarding outerWindow load. + debugLog('Ace2Editor.init() waiting for inner frame'); + await frameReady(innerFrame); + debugLog('Ace2Editor.init() inner frame ready'); + + // This must be done after the Window's load event. See above comment. + const innerDocument = innerWindow.document; + + // tag + innerDocument.insertBefore( + innerDocument.implementation.createDocumentType('html', '', ''), innerDocument.firstChild); + innerDocument.documentElement.classList.add('inner-editor', ...skinVariants); + + // tag + addStyleTagsFor(innerDocument, includedCSS); + const requireKernel = innerDocument.createElement('script'); + requireKernel.type = 'text/javascript'; + requireKernel.src = + absUrl(`../static/js/require-kernel.js?v=${clientVars.randomVersionString}`); + innerDocument.head.appendChild(requireKernel); + // Pre-fetch modules to improve load performance. + for (const module of ['ace2_inner', 'ace2_common']) { + const script = innerDocument.createElement('script'); + script.type = 'text/javascript'; + script.src = absUrl(`../javascripts/lib/ep_etherpad-lite/static/js/${module}.js` + + `?callback=require.define&v=${clientVars.randomVersionString}`); + innerDocument.head.appendChild(script); + } + const innerStyle = innerDocument.createElement('style'); + innerStyle.type = 'text/css'; + innerStyle.title = 'dynamicsyntax'; + innerDocument.head.appendChild(innerStyle); + const headLines = []; + hooks.callAll('aceInitInnerdocbodyHead', {iframeHTML: headLines}); + const tmp = innerDocument.createElement('div'); + tmp.innerHTML = headLines.join('\n'); + while (tmp.firstChild) innerDocument.head.appendChild(tmp.firstChild); + + // tag + innerDocument.body.id = 'innerdocbody'; + innerDocument.body.classList.add('innerdocbody'); + innerDocument.body.setAttribute('role', 'application'); + innerDocument.body.setAttribute('spellcheck', 'false'); + innerDocument.body.appendChild(innerDocument.createTextNode('\u00A0')); //   + + debugLog('Ace2Editor.init() waiting for require kernel load'); + await eventFired(requireKernel, 'load'); + debugLog('Ace2Editor.init() require kernel loaded'); + const require = innerWindow.require; + require.setRootURI(absUrl('../javascripts/src')); + require.setLibraryURI(absUrl('../javascripts/lib')); + require.setGlobalKeyPath('require'); + + // intentially moved before requiring client_plugins to save a 307 + innerWindow.Ace2Inner = require('ep_etherpad-lite/static/js/ace2_inner'); + innerWindow.plugins = require('ep_etherpad-lite/static/js/pluginfw/client_plugins'); + innerWindow.plugins.adoptPluginsFromAncestorsOf(innerWindow); + + innerWindow.$ = innerWindow.jQuery = require('ep_etherpad-lite/static/js/rjquery').jQuery; + + debugLog('Ace2Editor.init() waiting for plugins'); + await new Promise((resolve, reject) => innerWindow.plugins.ensure( + (err) => err != null ? reject(err) : resolve())); + debugLog('Ace2Editor.init() waiting for Ace2Inner.init()'); + await new Promise((resolve, reject) => innerWindow.Ace2Inner.init( + info, (err) => err != null ? reject(err) : resolve())); + debugLog('Ace2Editor.init() Ace2Inner.init() returned'); loaded = true; doActionsPendingInit(); debugLog('Ace2Editor.init() done'); From 926f0fcefb5a368bdcc3f030475de37bf47adc38 Mon Sep 17 00:00:00 2001 From: Richard Hansen Date: Sat, 6 Mar 2021 16:02:04 -0500 Subject: [PATCH 07/51] CSS: Increase size of contenteditable area --- src/static/css/iframe_editor.css | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/static/css/iframe_editor.css b/src/static/css/iframe_editor.css index 156eaff6..91217441 100644 --- a/src/static/css/iframe_editor.css +++ b/src/static/css/iframe_editor.css @@ -37,6 +37,11 @@ html.outer-editor, html.inner-editor { white-space: normal; word-wrap: break-word; overflow-wrap: break-word; + /* + * Make the contenteditable area at least as big as the screen so that mobile + * users can tap anywhere to bring up their device's keyboard. + */ + min-height: 100vh; } #innerdocbody, #sidediv { From 71dfa7070d0a231a1f0e0cfe9188ea5d446da341 Mon Sep 17 00:00:00 2001 From: Richard Hansen Date: Mon, 8 Mar 2021 16:15:56 -0500 Subject: [PATCH 08/51] deps: Update ueberdb2 to get metrics --- src/package-lock.json | 25 ++++++++++++------------- src/package.json | 2 +- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/package-lock.json b/src/package-lock.json index 14421f86..650201c7 100644 --- a/src/package-lock.json +++ b/src/package-lock.json @@ -15,9 +15,9 @@ } }, "@azure/abort-controller": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-1.0.3.tgz", - "integrity": "sha512-kCibMwqffnwlw3c+e879rCE1Am1I2BfhjOeO54XNA8i/cEuzktnBQbTrzh67XwibHO05YuNgZzSWy9ocVfFAGw==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-1.0.4.tgz", + "integrity": "sha512-lNUmDRVGpanCsiUN3NWxFTdwmdFI53xwhkTFfHDGTYk46ca7Ind3nanJc+U6Zj9Tv+9nTCWRBscWEW1DyKOpTw==", "requires": { "tslib": "^2.0.0" }, @@ -287,9 +287,9 @@ "integrity": "sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w==" }, "@types/node": { - "version": "14.14.31", - "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.31.tgz", - "integrity": "sha512-vFHy/ezP5qI0rFgJ7aQnjDXwAMrG0KqqIH7tQG5PPv3BWBayOPIQNBjVc/P6hhdZfMx51REc6tfDNXHUio893g==" + "version": "14.14.32", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.32.tgz", + "integrity": "sha512-/Ctrftx/zp4m8JOujM5ZhwzlWLx22nbQJiVqz8/zE15gOeEW+uly3FSX4fGFpcfEvFzXcMCJwq9lGVWgyARXhg==" }, "@types/node-fetch": { "version": "2.5.8", @@ -7689,9 +7689,9 @@ "optional": true }, "simple-git": { - "version": "2.35.2", - "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-2.35.2.tgz", - "integrity": "sha512-UjOKsrz92Bx7z00Wla5V6qLSf5X2XSp0sL2gzKw1Bh7iJfDPDaU7gK5avIup0yo1/sMOSUMQer2b9GcnF6nmTQ==", + "version": "2.36.1", + "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-2.36.1.tgz", + "integrity": "sha512-bN18Ea/4IJgqgbZyE9VpVEUkAu9vyP0VWP7acP0CRC1p/N80GGJ0HhIVeFJsm8TdJLBowiJpdLesQuAZ5TFSKw==", "requires": { "@kwsites/file-exists": "^1.1.1", "@kwsites/promise-deferred": "^1.1.1", @@ -8436,13 +8436,12 @@ } }, "ueberdb2": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/ueberdb2/-/ueberdb2-1.3.2.tgz", - "integrity": "sha512-7Ub5jDsIS+qjjsNV7yp1CHXHVe2K9ZUpwaHi9BZf3ai0DxtuHOfMada1wxL6iyEjwYXh/Nsu80iyId51wHFf4A==", + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/ueberdb2/-/ueberdb2-1.4.1.tgz", + "integrity": "sha512-UpzPszU/ZKqkpq2C1OkLmcZ3BkI28YvhcGONgI88UU19Dej1SGytCNbUhXwEtHWszatSc15xBcoIqQ+9Cr05Ig==", "requires": { "async": "^3.2.0", "cassandra-driver": "^4.5.1", - "channels": "0.0.4", "dirty": "^1.1.1", "elasticsearch": "^16.7.1", "mongodb": "^3.6.3", diff --git a/src/package.json b/src/package.json index ada568d9..4c869dc1 100644 --- a/src/package.json +++ b/src/package.json @@ -69,7 +69,7 @@ "threads": "^1.4.0", "tiny-worker": "^2.3.0", "tinycon": "0.6.8", - "ueberdb2": "^1.3.2", + "ueberdb2": "^1.4.1", "underscore": "1.12.0", "unorm": "1.6.0", "wtfnode": "^0.8.4" From fcf43a70899368ee126e376872d38039083d72a5 Mon Sep 17 00:00:00 2001 From: Richard Hansen Date: Mon, 8 Mar 2021 16:30:13 -0500 Subject: [PATCH 09/51] stats: Expose ueberDB metrics --- src/node/db/DB.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/node/db/DB.js b/src/node/db/DB.js index c0993e8e..e1097e90 100644 --- a/src/node/db/DB.js +++ b/src/node/db/DB.js @@ -24,6 +24,7 @@ const ueberDB = require('ueberdb2'); const settings = require('../utils/Settings'); const log4js = require('log4js'); +const stats = require('../stats'); const util = require('util'); // set database settings @@ -48,6 +49,13 @@ exports.init = async () => await new Promise((resolve, reject) => { process.exit(1); } + if (db.metrics != null) { + for (const [metric, value] of Object.entries(db.metrics)) { + if (typeof value !== 'number') continue; + stats.gauge(`ueberdb_${metric}`, () => db.metrics[metric]); + } + } + // everything ok, set up Promise-based methods ['get', 'set', 'findKeys', 'getSub', 'setSub', 'remove'].forEach((fn) => { exports[fn] = util.promisify(db[fn].bind(db)); From 9bc3ac09571b13ac74d818de50dd1a42da377801 Mon Sep 17 00:00:00 2001 From: John McLear Date: Tue, 9 Mar 2021 18:10:49 +0000 Subject: [PATCH 10/51] Include shard.etherpad.com in the README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4bc97228..d3543186 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ Etherpad is extremely flexible providing you the means to modify it to solve wha * [Video Chat](https://video.etherpad.com) - Plugins to enable Video and Audio chat in a pad. * [Collaboration++](https://collab.etherpad.com) - Plugins to improve the really-real time collaboration experience, suitable for busy pads. * [Document Analysis](https://analysis.etherpad.com) - Plugins to improve author and document analysis during and post creation. +* [Scale](https://shard.etherpad.com) - Etherpad running at scale with pad sharding which allows Etherpad to scale to ∞ number of Active Pads with up to ~20,000 edits per second, per pad. # Project Status From d2610284ad223be6fc930de4dbf414f3aac8aa59 Mon Sep 17 00:00:00 2001 From: Richard Hansen Date: Wed, 10 Mar 2021 02:54:18 -0500 Subject: [PATCH 11/51] bin/safeRun.sh: Fix `try: not found` bug This fixes a copy+paste bug introduced in commit 8b28e007842efefccf852a55f783761f86dc6090 (v1.8.8). --- src/bin/safeRun.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bin/safeRun.sh b/src/bin/safeRun.sh index d9efa241..d980b930 100755 --- a/src/bin/safeRun.sh +++ b/src/bin/safeRun.sh @@ -24,8 +24,8 @@ fatal() { error "$@"; exit 1; } LAST_EMAIL_SEND=0 # Move to the Etherpad base directory. -MY_DIR=$(try cd "${0%/*}" && try pwd -P) || exit 1 -try cd "${MY_DIR}/../.." +MY_DIR=$(cd "${0%/*}" && pwd -P) || exit 1 +cd "${MY_DIR}/../.." || exit 1 # Check if a logfile parameter is set LOG="$1" From 9b82d1d37d913fad0d2f230963554ab206629ba2 Mon Sep 17 00:00:00 2001 From: Richard Hansen Date: Wed, 10 Mar 2021 05:12:14 -0500 Subject: [PATCH 12/51] server: Log stats (metrics) on fatal error This might help users troubleshoot rare crashes. --- src/node/server.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/node/server.js b/src/node/server.js index fc62b447..a574116e 100755 --- a/src/node/server.js +++ b/src/node/server.js @@ -46,6 +46,7 @@ const hooks = require('../static/js/pluginfw/hooks'); const pluginDefs = require('../static/js/pluginfw/plugin_defs'); const plugins = require('../static/js/pluginfw/plugins'); const settings = require('./utils/Settings'); +const stats = require('./stats'); const logger = log4js.getLogger('server'); @@ -104,8 +105,6 @@ exports.start = async () => { // Check if Etherpad version is up-to-date UpdateCheck.check(); - // start up stats counting system - const stats = require('./stats'); stats.gauge('memoryUsage', () => process.memoryUsage().rss); stats.gauge('memoryUsageHeap', () => process.memoryUsage().heapUsed); @@ -215,6 +214,7 @@ exports.exit = async (err = null) => { logger.info('Received SIGTERM signal'); err = null; } else if (err != null) { + logger.error(`Metrics at time of fatal error:\n${JSON.stringify(stats.toJSON(), null, 2)}`); logger.error(err.stack || err.toString()); process.exitCode = 1; if (exitCalled) { From 0b9bf4a78ec0890303ae9217131d279232e86f0b Mon Sep 17 00:00:00 2001 From: Richard Hansen Date: Thu, 11 Mar 2021 13:17:47 -0500 Subject: [PATCH 13/51] deps: Update ueberdb2 to get updated metrics --- src/package-lock.json | 24 ++++++++++++------------ src/package.json | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/package-lock.json b/src/package-lock.json index 650201c7..abd5e4b1 100644 --- a/src/package-lock.json +++ b/src/package-lock.json @@ -287,9 +287,9 @@ "integrity": "sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w==" }, "@types/node": { - "version": "14.14.32", - "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.32.tgz", - "integrity": "sha512-/Ctrftx/zp4m8JOujM5ZhwzlWLx22nbQJiVqz8/zE15gOeEW+uly3FSX4fGFpcfEvFzXcMCJwq9lGVWgyARXhg==" + "version": "14.14.33", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.33.tgz", + "integrity": "sha512-oJqcTrgPUF29oUP8AsUqbXGJNuPutsetaa9kTQAQce5Lx5dTYWV02ScBiT/k1BX/Z7pKeqedmvp39Wu4zR7N7g==" }, "@types/node-fetch": { "version": "2.5.8", @@ -7689,9 +7689,9 @@ "optional": true }, "simple-git": { - "version": "2.36.1", - "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-2.36.1.tgz", - "integrity": "sha512-bN18Ea/4IJgqgbZyE9VpVEUkAu9vyP0VWP7acP0CRC1p/N80GGJ0HhIVeFJsm8TdJLBowiJpdLesQuAZ5TFSKw==", + "version": "2.36.2", + "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-2.36.2.tgz", + "integrity": "sha512-orBEf65GfSiQMsYedbJXSiRNnIRvhbeE5rrxZuEimCpWxDZOav0KLy2IEiPi1YJCF+zaC2quiJF8A4TsxI9/tw==", "requires": { "@kwsites/file-exists": "^1.1.1", "@kwsites/promise-deferred": "^1.1.1", @@ -8436,9 +8436,9 @@ } }, "ueberdb2": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/ueberdb2/-/ueberdb2-1.4.1.tgz", - "integrity": "sha512-UpzPszU/ZKqkpq2C1OkLmcZ3BkI28YvhcGONgI88UU19Dej1SGytCNbUhXwEtHWszatSc15xBcoIqQ+9Cr05Ig==", + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/ueberdb2/-/ueberdb2-1.4.2.tgz", + "integrity": "sha512-7glrozqZ4JNebOBRlChBT3ta+x17MHmuoc+1nhCm57F+nvsuVrq4fnkgRKWPNrhUqYwa+6DFkk5a8nJIRtpzfg==", "requires": { "async": "^3.2.0", "cassandra-driver": "^4.5.1", @@ -8773,9 +8773,9 @@ "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==" }, "xmldom": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/xmldom/-/xmldom-0.4.0.tgz", - "integrity": "sha512-2E93k08T30Ugs+34HBSTQLVtpi6mCddaY8uO+pMNk1pqSjV5vElzn4mmh6KLxN3hki8rNcHSYzILoh3TEWORvA==" + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/xmldom/-/xmldom-0.5.0.tgz", + "integrity": "sha512-Foaj5FXVzgn7xFzsKeNIde9g6aFBxTPi37iwsno8QvApmtg7KYrr+OPyRHcJF7dud2a5nGRBXK3n0dL62Gf7PA==" }, "xmlhttprequest-ssl": { "version": "1.5.5", diff --git a/src/package.json b/src/package.json index 4c869dc1..b77cef61 100644 --- a/src/package.json +++ b/src/package.json @@ -69,7 +69,7 @@ "threads": "^1.4.0", "tiny-worker": "^2.3.0", "tinycon": "0.6.8", - "ueberdb2": "^1.4.1", + "ueberdb2": "^1.4.2", "underscore": "1.12.0", "unorm": "1.6.0", "wtfnode": "^0.8.4" From 8e2a21ec84afdf9ba79b214a73a3b3274f96e836 Mon Sep 17 00:00:00 2001 From: webzwo0i Date: Fri, 12 Mar 2021 20:25:14 +0100 Subject: [PATCH 14/51] arrow functions dont have arguments (#4943) --- src/static/js/Changeset.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/static/js/Changeset.js b/src/static/js/Changeset.js index b47979c1..8c3627b0 100644 --- a/src/static/js/Changeset.js +++ b/src/static/js/Changeset.js @@ -50,10 +50,9 @@ exports.error = (msg) => { * @param b {boolean} assertion condition * @param msgParts {string} error to be passed if it fails */ -exports.assert = (b, msgParts) => { +exports.assert = (b, ...msgParts) => { if (!b) { - const msg = Array.prototype.slice.call(arguments, 1).join(''); - exports.error(`Failed assertion: ${msg}`); + exports.error(`Failed assertion: ${msgParts.join('')}`); } }; From 6f591b5c77417eda26abd3c2932751f838588c42 Mon Sep 17 00:00:00 2001 From: webzwo0i Date: Sat, 13 Mar 2021 03:16:22 +0100 Subject: [PATCH 15/51] add class pad to timeslider to fix height issue (#4941) --- src/static/css/pad/layout.css | 2 ++ src/templates/timeslider.html | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/static/css/pad/layout.css b/src/static/css/pad/layout.css index df340831..7f77ca58 100644 --- a/src/static/css/pad/layout.css +++ b/src/static/css/pad/layout.css @@ -4,6 +4,8 @@ html, body { margin: 0; padding: 0; } + +/* used in pad and timeslider */ html.pad, html.pad body { overflow: hidden; height: 100%; diff --git a/src/templates/timeslider.html b/src/templates/timeslider.html index c577f301..b3fd3d00 100644 --- a/src/templates/timeslider.html +++ b/src/templates/timeslider.html @@ -3,7 +3,7 @@ , langs = require("ep_etherpad-lite/node/hooks/i18n").availableLangs %> - + <%=settings.title%> Timeslider