From fc9f2369779a93594b19681eb848cd3f85c2175a Mon Sep 17 00:00:00 2001 From: Richard Hansen Date: Mon, 8 Feb 2021 19:19:49 -0500 Subject: [PATCH] plugins: Use `npm ls` to list the installed plugins This speeds up startup considerably, and we get rid of a lot of buggy code. This works with both npm v6.x and v7.x. --- src/package.json | 1 - src/static/js/pluginfw/plugin_defs.js | 6 +- src/static/js/pluginfw/plugins.js | 45 ++-- src/static/js/pluginfw/read-installed.js | 330 ----------------------- 4 files changed, 27 insertions(+), 355 deletions(-) delete mode 100644 src/static/js/pluginfw/read-installed.js diff --git a/src/package.json b/src/package.json index 42e22975..71e7bd39 100644 --- a/src/package.json +++ b/src/package.json @@ -44,7 +44,6 @@ "express-session": "1.17.1", "find-root": "1.1.0", "formidable": "1.2.1", - "graceful-fs": "4.2.4", "http-errors": "1.8.0", "js-cookie": "^2.2.1", "jsonminify": "0.4.1", diff --git a/src/static/js/pluginfw/plugin_defs.js b/src/static/js/pluginfw/plugin_defs.js index 768d99c3..f7d10879 100644 --- a/src/static/js/pluginfw/plugin_defs.js +++ b/src/static/js/pluginfw/plugin_defs.js @@ -21,6 +21,8 @@ exports.parts = []; // * parts: Each part from the ep.json object is augmented with the following properties: // - plugin: The name of the plugin. // - full_name: Equal to /. -// * package (server-side only): Object containing details about the plugin package (version, -// path). +// * package (server-side only): Object containing details about the plugin package: +// - version +// - path +// - realPath exports.plugins = {}; diff --git a/src/static/js/pluginfw/plugins.js b/src/static/js/pluginfw/plugins.js index ceda3cc4..8323cfc7 100644 --- a/src/static/js/pluginfw/plugins.js +++ b/src/static/js/pluginfw/plugins.js @@ -2,8 +2,8 @@ const fs = require('fs').promises; const hooks = require('./hooks'); -const readInstalled = require('./read-installed.js'); const path = require('path'); +const runNpm = require('../../../node/utils/run_npm'); const tsort = require('./tsort'); const util = require('util'); const settings = require('../../../node/utils/Settings'); @@ -69,28 +69,29 @@ exports.update = async () => { }; exports.getPackages = async () => { - // Load list of installed NPM packages, flatten it to a list, - // and filter out only packages with names that - const dir = settings.root; - const data = await util.promisify(readInstalled)(dir); - - const packages = {}; - const flatten = (deps) => { - for (const [name, dep] of Object.entries(deps)) { - if (name.indexOf(exports.prefix) === 0) { - packages[name] = {...dep}; - // Delete anything that creates loops so that the plugin - // list can be sent as JSON to the web client - delete packages[name].dependencies; - delete packages[name].parent; - } + // Note: Do not pass `--prod` because it does not work if there is no package.json. + const np = runNpm(['ls', '--long', '--json', '--depth=0'], { + stdoutLogger: null, // We want to capture stdout, so don't attempt to log it. + env: { + ...process.env, + // NODE_ENV must be set to development for `npm ls` to show files without a package.json. + NODE_ENV: 'development', + }, + }); + const chunks = []; + await Promise.all([ + (async () => { for await (const chunk of np.stdout) chunks.push(chunk); })(), + np, // Await in parallel to avoid unhandled rejection if np rejects during chunk read. + ]); + const {dependencies = {}} = JSON.parse(Buffer.concat(chunks).toString()); + await Promise.all(Object.entries(dependencies).map(async ([pkg, info]) => { + if (!pkg.startsWith(exports.prefix)) { + delete dependencies[pkg]; + return; } - }; - - const tmp = {}; - tmp[data.name] = data; - flatten(tmp[data.name].dependencies); - return packages; + info.realPath = await fs.realpath(info.path); + })); + return dependencies; }; const loadPlugin = async (packages, pluginName, plugins, parts) => { diff --git a/src/static/js/pluginfw/read-installed.js b/src/static/js/pluginfw/read-installed.js deleted file mode 100644 index f24fe8dc..00000000 --- a/src/static/js/pluginfw/read-installed.js +++ /dev/null @@ -1,330 +0,0 @@ -'use strict'; - -// A copy of npm/lib/utils/read-installed.js -// that is hacked to not cache everything :) - -// Walk through the file-system "database" of installed -// packages, and create a data object related to the -// installed versions of each package. - -/* -This will traverse through all node_modules folders, -resolving the dependencies object to the object corresponding to -the package that meets that dep, or just the version/range if -unmet. - -Assuming that you had this folder structure: - -/path/to -+-- package.json { name = "root" } -`-- node_modules - +-- foo {bar, baz, asdf} - | +-- node_modules - | +-- bar { baz } - | `-- baz - `-- asdf - -where "foo" depends on bar, baz, and asdf, bar depends on baz, -and bar and baz are bundled with foo, whereas "asdf" is at -the higher level (sibling to foo), you'd get this object structure: - -{ -, path: "/path/to" -, parent: null -, dependencies: - { foo : - { version: "1.2.3" - , path: "/path/to/node_modules/foo" - , parent: - , dependencies: - { bar: - { parent: - , path: "/path/to/node_modules/foo/node_modules/bar" - , version: "2.3.4" - , dependencies: { baz: } - } - , baz: { ... } - , asdf: - } - } - , asdf: { ... } - } -} - -Unmet deps are left as strings. -Extraneous deps are marked with extraneous:true -deps that don't meet a requirement are marked with invalid:true - -to READ(packagefolder, parentobj, name, reqver) -obj = read package.json -installed = ./node_modules/* -if parentobj is null, and no package.json - obj = {dependencies:{:"*"}} -deps = Object.keys(obj.dependencies) -obj.path = packagefolder -obj.parent = parentobj -if name, && obj.name !== name, obj.invalid = true -if reqver, && obj.version !satisfies reqver, obj.invalid = true -if !reqver && parentobj, obj.extraneous = true -for each folder in installed - obj.dependencies[folder] = READ(packagefolder+node_modules+folder, - obj, folder, obj.dependencies[folder]) -# walk tree to find unmet deps -for each dep in obj.dependencies not in installed - r = obj.parent - while r - if r.dependencies[dep] - if r.dependencies[dep].verion !satisfies obj.dependencies[dep] - WARN - r.dependencies[dep].invalid = true - obj.dependencies[dep] = r.dependencies[dep] - r = null - else r = r.parent -return obj - - -TODO: -1. Find unmet deps in parent directories, searching as node does up -as far as the left-most node_modules folder. -2. Ignore anything in node_modules that isn't a package folder. - -*/ - -const npm = require('npm/lib/npm.js'); -const fs = require('graceful-fs'); -const path = require('path'); -const semver = require('semver'); -const log = require('log4js').getLogger('pluginfw'); -const util = require('util'); - -let fuSeen = []; -let riSeen = []; -let rpSeen = {}; - -const readJson = (file, callback) => { - fs.readFile(file, (er, buf) => { - if (er) { - callback(er); - return; - } - try { - callback(null, JSON.parse(buf.toString())); - } catch (er) { - callback(er); - } - }); -}; - -const readInstalled = (folder, cb) => { - /* This is where we clear the cache, these three lines are all the - * new code there is */ - fuSeen = []; - rpSeen = {}; - riSeen = []; - - const d = npm.config.get('depth'); - readInstalled_(folder, null, null, null, 0, d, (er, obj) => { - if (er) return cb(er); - // now obj has all the installed things, where they're installed - // figure out the inheritance links, now that the object is built. - resolveInheritance(obj); - cb(null, obj); - }); -}; - -module.exports = readInstalled; - -const readInstalled_ = (folder, parent, name, reqver, depth, maxDepth, cb) => { - let installed, - obj, - real, - link; - let errState = null; - let called = false; - - const next = (er) => { - if (errState) return; - if (er) { - errState = er; - return cb(null, []); - } - if (!installed || !obj || !real || called) return; - called = true; - if (rpSeen[real]) return cb(null, rpSeen[real]); - if (obj === true) { - obj = {dependencies: {}, path: folder}; - for (const i of installed) { - obj.dependencies[i] = '*'; - } - } - if (name && obj.name !== name) obj.invalid = true; - obj.realName = name || obj.name; - obj.dependencies = obj.dependencies || {}; - - // "foo":"http://blah" is always presumed valid - if (reqver && - semver.validRange(reqver) && - !semver.satisfies(obj.version, reqver)) { - obj.invalid = true; - } - - if (parent && - !(name in parent.dependencies) && - !(name in (parent.devDependencies || {}))) { - obj.extraneous = true; - } - obj.path = obj.path || folder; - obj.realPath = real; - obj.link = link; - if (parent && !obj.link) obj.parent = parent; - rpSeen[real] = obj; - obj.depth = depth; - if (depth >= maxDepth) return cb(null, obj); - Promise.all(installed.map(async (pkg) => { - let rv = obj.dependencies[pkg]; - if (!rv && obj.devDependencies) rv = obj.devDependencies[pkg]; - const dir = path.resolve(folder, `node_modules/${pkg}`); - const deps = obj.dependencies[pkg]; - return await util.promisify(readInstalled_)(dir, obj, pkg, deps, depth + 1, maxDepth); - })).then((installedData) => { - for (const dep of installedData) { - obj.dependencies[dep.realName] = dep; - } - - // any strings here are unmet things. however, if it's - // optional, then that's fine, so just delete it. - if (obj.optionalDependencies) { - for (const dep of Object.keys(obj.optionalDependencies)) { - if (typeof obj.dependencies[dep] === 'string') { - delete obj.dependencies[dep]; - } - } - } - return cb(null, obj); - }, (err) => cb(err || new Error(err))); - }; - - fs.readdir(path.resolve(folder, 'node_modules'), (er, i) => { - // error indicates that nothing is installed here - if (er) i = []; - installed = i.filter((f) => f.charAt(0) !== '.'); - next(); - }); - - readJson(path.resolve(folder, 'package.json'), (er, data) => { - obj = copy(data); - - if (!parent) { - obj = obj || true; - er = null; - } - return next(er); - }); - - fs.lstat(folder, (er, st) => { - if (er) { - if (!parent) real = true; - return next(er); - } - fs.realpath(folder, (er, rp) => { - real = rp; - if (st.isSymbolicLink()) link = rp; - next(er); - }); - }); -}; - -// starting from a root object, call findUnmet on each layer of children -const resolveInheritance = (obj) => { - if (typeof obj !== 'object') return; - if (riSeen.indexOf(obj) !== -1) return; - riSeen.push(obj); - if (typeof obj.dependencies !== 'object') { - obj.dependencies = {}; - } - for (const dep of Object.keys(obj.dependencies)) { - findUnmet(obj.dependencies[dep]); - } - for (const dep of Object.keys(obj.dependencies)) { - resolveInheritance(obj.dependencies[dep]); - } -}; - -// find unmet deps by walking up the tree object. -// No I/O -const findUnmet = (obj) => { - if (typeof obj !== 'object') return; - if (fuSeen.indexOf(obj) !== -1) return; - fuSeen.push(obj); - const deps = obj.dependencies = obj.dependencies || {}; - for (const d of Object.keys(deps).filter((d) => typeof deps[d] === 'string')) { - let r = obj.parent; - let found = null; - while (r && !found && typeof deps[d] === 'string') { - // if r is a valid choice, then use that. - found = r.dependencies[d]; - if (!found && r.realName === d) found = r; - - if (!found) { - r = r.link ? null : r.parent; - continue; - } - if (typeof deps[d] === 'string' && - !semver.satisfies(found.version, deps[d])) { - // the bad thing will happen - log.warn(`${obj.path} requires ${d}@'${deps[d]}' but will load\n${found.path},\n` + - `which is version ${found.version}`, 'unmet dependency'); - found.invalid = true; - } - deps[d] = found; - } - } -}; - -const copy = (obj) => { - if (!obj || typeof obj !== 'object') return obj; - if (Array.isArray(obj)) return obj.map(copy); - - const o = {}; - for (const [i, v] of Object.entries(obj)) o[i] = copy(v); - return o; -}; - -if (module === require.main) { - const seen = []; - - const cleanup = (map) => { - if (seen.indexOf(map) !== -1) return; - seen.push(map); - for (const i of Object.keys(map)) { - switch (i) { - case '_id': - case 'path': - case 'extraneous': case 'invalid': - case 'dependencies': case 'name': - continue; - default: delete map[i]; - } - } - for (const dep of Object.values(map.dependencies || {})) { - if (typeof dep === 'object') { - cleanup(dep); - } - } - return map; - }; - - const util = require('util'); - console.error('testing'); - - let called = 0; - npm.load({}, (err) => { - if (err != null) throw err; - readInstalled(process.cwd(), (er, map) => { - console.error(called++); - if (er) return console.error(er.stack || er.message); - cleanup(map); - console.error(util.inspect(map, true, 10, true)); - }); - }); -}