Changeset: New API to simplify attribute processing

This commit is contained in:
Richard Hansen 2021-11-17 16:27:05 -05:00
parent 982d8ad0f2
commit 6cf2055199
6 changed files with 756 additions and 0 deletions

View File

@ -1,3 +1,13 @@
# 1.9.0 (not yet released)
### Notable enhancements
#### For plugin authors
* New APIs for processing attributes: `ep_etherpad-lite/static/js/attributes`
(low-level API) and `ep_etherpad-lite/static/js/AttributeMap` (high-level
API).
# 1.8.15
### Security fixes

View File

@ -54,6 +54,8 @@
, "broadcast_revisions.js"
, "socketio.js"
, "AttributeManager.js"
, "AttributeMap.js"
, "attributes.js"
, "ChangesetUtils.js"
]
, "ace2_inner.js": [
@ -71,6 +73,8 @@
, "linestylefilter.js"
, "domline.js"
, "AttributeManager.js"
, "AttributeMap.js"
, "attributes.js"
, "scroll.js"
, "caretPosition.js"
, "pad_utils.js"

View File

@ -0,0 +1,91 @@
'use strict';
const attributes = require('./attributes');
/**
* A `[key, value]` pair of strings describing a text attribute.
*
* @typedef {[string, string]} Attribute
*/
/**
* A concatenated sequence of zero or more attribute identifiers, each one represented by an
* asterisk followed by a base-36 encoded attribute number.
*
* Examples: '', '*0', '*3*j*z*1q'
*
* @typedef {string} AttributeString
*/
/**
* Convenience class to convert an Op's attribute string to/from a Map of key, value pairs.
*/
class AttributeMap extends Map {
/**
* Converts an attribute string into an AttributeMap.
*
* @param {AttributeString} str - The attribute string to convert into an AttributeMap.
* @param {AttributePool} pool - Attribute pool.
* @returns {AttributeMap}
*/
static fromString(str, pool) {
return new AttributeMap(pool).updateFromString(str);
}
/**
* @param {AttributePool} pool - Attribute pool.
*/
constructor(pool) {
super();
/** @public */
this.pool = pool;
}
/**
* @param {string} k - Attribute name.
* @param {string} v - Attribute value.
* @returns {AttributeMap} `this` (for chaining).
*/
set(k, v) {
k = k == null ? '' : String(k);
v = v == null ? '' : String(v);
this.pool.putAttrib([k, v]);
return super.set(k, v);
}
toString() {
return attributes.attribsToString(attributes.sort([...this]), this.pool);
}
/**
* @param {Iterable<Attribute>} entries - [key, value] pairs to insert into this map.
* @param {boolean} [emptyValueIsDelete] - If true and an entry's value is the empty string, the
* key is removed from this map (if present).
* @returns {AttributeMap} `this` (for chaining).
*/
update(entries, emptyValueIsDelete = false) {
for (let [k, v] of entries) {
k = k == null ? '' : String(k);
v = v == null ? '' : String(v);
if (!v && emptyValueIsDelete) {
this.delete(k);
} else {
this.set(k, v);
}
}
return this;
}
/**
* @param {AttributeString} str - The attribute string identifying the attributes to insert into
* this map.
* @param {boolean} [emptyValueIsDelete] - If true and an entry's value is the empty string, the
* key is removed from this map (if present).
* @returns {AttributeMap} `this` (for chaining).
*/
updateFromString(str, emptyValueIsDelete = false) {
return this.update(attributes.attribsFromString(str, this.pool), emptyValueIsDelete);
}
}
module.exports = AttributeMap;

130
src/static/js/attributes.js Normal file
View File

@ -0,0 +1,130 @@
'use strict';
// Low-level utilities for manipulating attribute strings. For a high-level API, see AttributeMap.
/**
* A `[key, value]` pair of strings describing a text attribute.
*
* @typedef {[string, string]} Attribute
*/
/**
* A concatenated sequence of zero or more attribute identifiers, each one represented by an
* asterisk followed by a base-36 encoded attribute number.
*
* Examples: '', '*0', '*3*j*z*1q'
*
* @typedef {string} AttributeString
*/
/**
* Converts an attribute string into a sequence of attribute identifier numbers.
*
* WARNING: This only works on attribute strings. It does NOT work on serialized operations or
* changesets.
*
* @param {AttributeString} str - Attribute string.
* @yields {number} The attribute numbers (to look up in the associated pool), in the order they
* appear in `str`.
* @returns {Generator<number>}
*/
exports.decodeAttribString = function* (str) {
const re = /\*([0-9a-z]+)|./gy;
let match;
while ((match = re.exec(str)) != null) {
const [m, n] = match;
if (n == null) throw new Error(`invalid character in attribute string: ${m}`);
yield Number.parseInt(n, 36);
}
};
const checkAttribNum = (n) => {
if (typeof n !== 'number') throw new TypeError(`not a number: ${n}`);
if (n < 0) throw new Error(`attribute number is negative: ${n}`);
if (n !== Math.trunc(n)) throw new Error(`attribute number is not an integer: ${n}`);
};
/**
* Inverse of `decodeAttribString`.
*
* @param {Iterable<number>} attribNums - Sequence of attribute numbers.
* @returns {AttributeString}
*/
exports.encodeAttribString = (attribNums) => {
let str = '';
for (const n of attribNums) {
checkAttribNum(n);
str += `*${n.toString(36).toLowerCase()}`;
}
return str;
};
/**
* Converts a sequence of attribute numbers into a sequence of attributes.
*
* @param {Iterable<number>} attribNums - Attribute numbers to look up in the pool.
* @param {AttributePool} pool - Attribute pool.
* @yields {Attribute} The identified attributes, in the same order as `attribNums`.
* @returns {Generator<Attribute>}
*/
exports.attribsFromNums = function* (attribNums, pool) {
for (const n of attribNums) {
checkAttribNum(n);
const attrib = pool.getAttrib(n);
if (attrib == null) throw new Error(`attribute ${n} does not exist in pool`);
yield attrib;
}
};
/**
* Inverse of `attribsFromNums`.
*
* @param {Iterable<Attribute>} attribs - Attributes. Any attributes not already in `pool` are
* inserted into `pool`. No checking is performed to ensure that the attributes are in the
* canonical order and that there are no duplicate keys. (Use an AttributeMap and/or `sort()` if
* required.)
* @param {AttributePool} pool - Attribute pool.
* @yields {number} The attribute number of each attribute in `attribs`, in order.
* @returns {Generator<number>}
*/
exports.attribsToNums = function* (attribs, pool) {
for (const attrib of attribs) yield pool.putAttrib(attrib);
};
/**
* Convenience function that is equivalent to `attribsFromNums(decodeAttribString(str), pool)`.
*
* WARNING: This only works on attribute strings. It does NOT work on serialized operations or
* changesets.
*
* @param {AttributeString} str - Attribute string.
* @param {AttributePool} pool - Attribute pool.
* @yields {Attribute} The attributes identified in `str`, in order.
* @returns {Generator<Attribute>}
*/
exports.attribsFromString = function* (str, pool) {
yield* exports.attribsFromNums(exports.decodeAttribString(str), pool);
};
/**
* Inverse of `attribsFromString`.
*
* @param {Iterable<Attribute>} attribs - Attributes. The attributes to insert into the pool (if
* necessary) and encode. No checking is performed to ensure that the attributes are in the
* canonical order and that there are no duplicate keys. (Use an AttributeMap and/or `sort()` if
* required.)
* @param {AttributePool} pool - Attribute pool.
* @returns {AttributeString}
*/
exports.attribsToString =
(attribs, pool) => exports.encodeAttribString(exports.attribsToNums(attribs, pool));
/**
* Sorts the attributes in canonical order. The order of entries with the same attribute name is
* unspecified.
*
* @param {Attribute[]} attribs - Attributes to sort in place.
* @returns {Attribute[]} `attribs` (for chaining).
*/
exports.sort =
(attribs) => attribs.sort(([keyA], [keyB]) => (keyA > keyB ? 1 : 0) - (keyA < keyB ? 1 : 0));

View File

@ -0,0 +1,178 @@
'use strict';
const AttributeMap = require('../../../static/js/AttributeMap');
const AttributePool = require('../../../static/js/AttributePool');
const attributes = require('../../../static/js/attributes');
describe('AttributeMap', function () {
const attribs = [
['foo', 'bar'],
['baz', 'bif'],
['emptyValue', ''],
];
let pool;
const getPoolSize = () => {
let n = 0;
pool.eachAttrib(() => ++n);
return n;
};
beforeEach(async function () {
pool = new AttributePool();
for (let i = 0; i < attribs.length; ++i) expect(pool.putAttrib(attribs[i])).to.equal(i);
});
it('fromString works', async function () {
const got = AttributeMap.fromString('*0*1*2', pool);
for (const [k, v] of attribs) expect(got.get(k)).to.equal(v);
// Maps iterate in insertion order, so [...got] should be in the same order as attribs.
expect(JSON.stringify([...got])).to.equal(JSON.stringify(attribs));
});
describe('set', function () {
it('stores the value', async function () {
const m = new AttributeMap(pool);
expect(m.size).to.equal(0);
m.set('k', 'v');
expect(m.size).to.equal(1);
expect(m.get('k')).to.equal('v');
});
it('reuses attributes in the pool', async function () {
expect(getPoolSize()).to.equal(attribs.length);
const m = new AttributeMap(pool);
const [k0, v0] = attribs[0];
m.set(k0, v0);
expect(getPoolSize()).to.equal(attribs.length);
expect(m.size).to.equal(1);
expect(m.toString()).to.equal('*0');
});
it('inserts new attributes into the pool', async function () {
const m = new AttributeMap(pool);
expect(getPoolSize()).to.equal(attribs.length);
m.set('k', 'v');
expect(getPoolSize()).to.equal(attribs.length + 1);
expect(JSON.stringify(pool.getAttrib(attribs.length))).to.equal(JSON.stringify(['k', 'v']));
});
describe('coerces key and value to string', function () {
const testCases = [
['object (with toString)', {toString: () => 'obj'}, 'obj'],
['undefined', undefined, ''],
['null', null, ''],
['boolean', true, 'true'],
['number', 1, '1'],
];
for (const [desc, input, want] of testCases) {
describe(desc, function () {
it('key is coerced to string', async function () {
const m = new AttributeMap(pool);
m.set(input, 'value');
expect(m.get(want)).to.equal('value');
});
it('value is coerced to string', async function () {
const m = new AttributeMap(pool);
m.set('key', input);
expect(m.get('key')).to.equal(want);
});
});
}
});
it('returns the map', async function () {
const m = new AttributeMap(pool);
expect(m.set('k', 'v')).to.equal(m);
});
});
describe('toString', function () {
it('sorts attributes', async function () {
const m = new AttributeMap(pool).update(attribs);
const got = [...attributes.attribsFromString(m.toString(), pool)];
const want = attributes.sort([...attribs]);
// Verify that attribs is not already sorted so that this test doesn't accidentally pass.
expect(JSON.stringify(want)).to.not.equal(JSON.stringify(attribs));
expect(JSON.stringify(got)).to.equal(JSON.stringify(want));
});
it('returns all entries', async function () {
const m = new AttributeMap(pool);
expect(m.toString()).to.equal('');
m.set(...attribs[0]);
expect(m.toString()).to.equal('*0');
m.delete(attribs[0][0]);
expect(m.toString()).to.equal('');
m.set(...attribs[1]);
expect(m.toString()).to.equal('*1');
m.set(attribs[1][0], 'new value');
expect(m.toString()).to.equal(attributes.encodeAttribString([attribs.length]));
m.set(...attribs[2]);
expect(m.toString()).to.equal(attributes.attribsToString(
attributes.sort([attribs[2], [attribs[1][0], 'new value']]), pool));
});
});
for (const funcName of ['update', 'updateFromString']) {
const callUpdateFn = (m, ...args) => {
if (funcName === 'updateFromString') {
args[0] = attributes.attribsToString(attributes.sort([...args[0]]), pool);
}
return AttributeMap.prototype[funcName].call(m, ...args);
};
describe(funcName, function () {
it('works', async function () {
const m = new AttributeMap(pool);
m.set(attribs[2][0], 'value to be overwritten');
callUpdateFn(m, attribs);
for (const [k, v] of attribs) expect(m.get(k)).to.equal(v);
expect(m.size).to.equal(attribs.length);
const wantStr = attributes.attribsToString(attributes.sort([...attribs]), pool);
expect(m.toString()).to.equal(wantStr);
callUpdateFn(m, []);
expect(m.toString()).to.equal(wantStr);
});
it('inserts new attributes into the pool', async function () {
const m = new AttributeMap(pool);
callUpdateFn(m, [['k', 'v']]);
expect(m.size).to.equal(1);
expect(m.get('k')).to.equal('v');
expect(getPoolSize()).to.equal(attribs.length + 1);
expect(m.toString()).to.equal(attributes.encodeAttribString([attribs.length]));
});
it('returns the map', async function () {
const m = new AttributeMap(pool);
expect(callUpdateFn(m, [])).to.equal(m);
});
describe('emptyValueIsDelete=false inserts empty values', function () {
for (const emptyVal of ['', null, undefined]) {
it(emptyVal == null ? String(emptyVal) : JSON.stringify(emptyVal), async function () {
const m = new AttributeMap(pool);
m.set('k', 'v');
callUpdateFn(m, [['k', emptyVal]]);
expect(m.size).to.equal(1);
expect(m.toString()).to.equal(attributes.attribsToString([['k', '']], pool));
});
}
});
describe('emptyValueIsDelete=true deletes entries', function () {
for (const emptyVal of ['', null, undefined]) {
it(emptyVal == null ? String(emptyVal) : JSON.stringify(emptyVal), async function () {
const m = new AttributeMap(pool);
m.set('k', 'v');
callUpdateFn(m, [['k', emptyVal]], true);
expect(m.size).to.equal(0);
expect(m.toString()).to.equal('');
});
}
});
});
}
});

View File

@ -0,0 +1,343 @@
'use strict';
const AttributePool = require('../../../static/js/AttributePool');
const attributes = require('../../../static/js/attributes');
describe('attributes', function () {
const attribs = [['foo', 'bar'], ['baz', 'bif']];
let pool;
beforeEach(async function () {
pool = new AttributePool();
for (let i = 0; i < attribs.length; ++i) expect(pool.putAttrib(attribs[i])).to.equal(i);
});
describe('decodeAttribString', function () {
it('is a generator function', async function () {
expect(attributes.decodeAttribString).to.be.a((function* () {}).constructor);
});
describe('rejects invalid attribute strings', function () {
const testCases = ['x', '*0+1', '*A', '*0$', '*', '0', '*-1'];
for (const tc of testCases) {
it(JSON.stringify(tc), async function () {
expect(() => [...attributes.decodeAttribString(tc)])
.to.throwException(/invalid character/);
});
}
});
describe('accepts valid attribute strings', function () {
const testCases = [
['', []],
['*0', [0]],
['*a', [10]],
['*z', [35]],
['*10', [36]],
[
'*0*1*2*3*4*5*6*7*8*9*a*b*c*d*e*f*g*h*i*j*k*l*m*n*o*p*q*r*s*t*u*v*w*x*y*z*10',
[...Array(37).keys()],
],
];
for (const [input, want] of testCases) {
it(`${JSON.stringify(input)} -> ${JSON.stringify(want)}`, async function () {
const got = [...attributes.decodeAttribString(input)];
expect(JSON.stringify(got)).to.equal(JSON.stringify(want));
});
}
});
});
describe('encodeAttribString', function () {
describe('accepts any kind of iterable', function () {
const testCases = [
['generator', (function* () { yield 0; yield 1; })()],
['list', [0, 1]],
['set', new Set([0, 1])],
];
for (const [desc, input] of testCases) {
it(desc, async function () {
expect(attributes.encodeAttribString(input)).to.equal('*0*1');
});
}
});
describe('rejects invalid inputs', function () {
const testCases = [
[null, /.*/], // Different browsers may have different error messages.
[[-1], /is negative/],
[['0'], /not a number/],
[[null], /not a number/],
[[0.5], /not an integer/],
[[{}], /not a number/],
[[true], /not a number/],
];
for (const [input, wantErr] of testCases) {
it(JSON.stringify(input), async function () {
expect(() => attributes.encodeAttribString(input)).to.throwException(wantErr);
});
}
});
describe('accepts valid inputs', function () {
const testCases = [
[[], ''],
[[0], '*0'],
[[10], '*a'],
[[35], '*z'],
[[36], '*10'],
[
[...Array(37).keys()],
'*0*1*2*3*4*5*6*7*8*9*a*b*c*d*e*f*g*h*i*j*k*l*m*n*o*p*q*r*s*t*u*v*w*x*y*z*10',
],
];
for (const [input, want] of testCases) {
it(`${JSON.stringify(input)} -> ${JSON.stringify(want)}`, async function () {
expect(attributes.encodeAttribString(input)).to.equal(want);
});
}
});
});
describe('attribsFromNums', function () {
it('is a generator function', async function () {
expect(attributes.attribsFromNums).to.be.a((function* () {}).constructor);
});
describe('accepts any kind of iterable', function () {
const testCases = [
['generator', (function* () { yield 0; yield 1; })()],
['list', [0, 1]],
['set', new Set([0, 1])],
];
for (const [desc, input] of testCases) {
it(desc, async function () {
const gotAttribs = [...attributes.attribsFromNums(input, pool)];
expect(JSON.stringify(gotAttribs)).to.equal(JSON.stringify(attribs));
});
}
});
describe('rejects invalid inputs', function () {
const testCases = [
[null, /.*/], // Different browsers may have different error messages.
[[-1], /is negative/],
[['0'], /not a number/],
[[null], /not a number/],
[[0.5], /not an integer/],
[[{}], /not a number/],
[[true], /not a number/],
[[9999], /does not exist in pool/],
];
for (const [input, wantErr] of testCases) {
it(JSON.stringify(input), async function () {
expect(() => [...attributes.attribsFromNums(input, pool)]).to.throwException(wantErr);
});
}
});
describe('accepts valid inputs', function () {
const testCases = [
[[], []],
[[0], [attribs[0]]],
[[1], [attribs[1]]],
[[0, 1], [attribs[0], attribs[1]]],
[[1, 0], [attribs[1], attribs[0]]],
];
for (const [input, want] of testCases) {
it(`${JSON.stringify(input)} -> ${JSON.stringify(want)}`, async function () {
const gotAttribs = [...attributes.attribsFromNums(input, pool)];
expect(JSON.stringify(gotAttribs)).to.equal(JSON.stringify(want));
});
}
});
});
describe('attribsToNums', function () {
it('is a generator function', async function () {
expect(attributes.attribsToNums).to.be.a((function* () {}).constructor);
});
describe('accepts any kind of iterable', function () {
const testCases = [
['generator', (function* () { yield attribs[0]; yield attribs[1]; })()],
['list', [attribs[0], attribs[1]]],
['set', new Set([attribs[0], attribs[1]])],
];
for (const [desc, input] of testCases) {
it(desc, async function () {
const gotNums = [...attributes.attribsToNums(input, pool)];
expect(JSON.stringify(gotNums)).to.equal(JSON.stringify([0, 1]));
});
}
});
describe('rejects invalid inputs', function () {
const testCases = [null, [null]];
for (const input of testCases) {
it(JSON.stringify(input), async function () {
expect(() => [...attributes.attribsToNums(input, pool)]).to.throwException();
});
}
});
describe('reuses existing pool entries', function () {
const testCases = [
[[], []],
[[attribs[0]], [0]],
[[attribs[1]], [1]],
[[attribs[0], attribs[1]], [0, 1]],
[[attribs[1], attribs[0]], [1, 0]],
];
for (const [input, want] of testCases) {
it(`${JSON.stringify(input)} -> ${JSON.stringify(want)}`, async function () {
const got = [...attributes.attribsToNums(input, pool)];
expect(JSON.stringify(got)).to.equal(JSON.stringify(want));
});
}
});
describe('inserts new attributes into the pool', function () {
const testCases = [
[[['k', 'v']], [attribs.length]],
[[attribs[0], ['k', 'v']], [0, attribs.length]],
[[['k', 'v'], attribs[0]], [attribs.length, 0]],
];
for (const [input, want] of testCases) {
it(`${JSON.stringify(input)} -> ${JSON.stringify(want)}`, async function () {
const got = [...attributes.attribsToNums(input, pool)];
expect(JSON.stringify(got)).to.equal(JSON.stringify(want));
expect(JSON.stringify(pool.getAttrib(attribs.length)))
.to.equal(JSON.stringify(['k', 'v']));
});
}
});
describe('coerces key and value to string', function () {
const testCases = [
['object (with toString)', {toString: () => 'obj'}, 'obj'],
['undefined', undefined, ''],
['null', null, ''],
['boolean', true, 'true'],
['number', 1, '1'],
];
for (const [desc, inputVal, wantVal] of testCases) {
describe(desc, function () {
for (const [desc, inputAttribs, wantAttribs] of [
['key is coerced to string', [[inputVal, 'value']], [[wantVal, 'value']]],
['value is coerced to string', [['key', inputVal]], [['key', wantVal]]],
]) {
it(desc, async function () {
const gotNums = [...attributes.attribsToNums(inputAttribs, pool)];
// Each attrib in inputAttribs is expected to be new to the pool.
const wantNums = [...Array(attribs.length + 1).keys()].slice(attribs.length);
expect(JSON.stringify(gotNums)).to.equal(JSON.stringify(wantNums));
const gotAttribs = gotNums.map((n) => pool.getAttrib(n));
expect(JSON.stringify(gotAttribs)).to.equal(JSON.stringify(wantAttribs));
});
}
});
}
});
});
describe('attribsFromString', function () {
it('is a generator function', async function () {
expect(attributes.attribsFromString).to.be.a((function* () {}).constructor);
});
describe('rejects invalid attribute strings', function () {
const testCases = [
['x', /invalid character/],
['*0+1', /invalid character/],
['*A', /invalid character/],
['*0$', /invalid character/],
['*', /invalid character/],
['0', /invalid character/],
['*-1', /invalid character/],
['*9999', /does not exist in pool/],
];
for (const [input, wantErr] of testCases) {
it(JSON.stringify(input), async function () {
expect(() => [...attributes.attribsFromString(input, pool)]).to.throwException(wantErr);
});
}
});
describe('accepts valid inputs', function () {
const testCases = [
['', []],
['*0', [attribs[0]]],
['*1', [attribs[1]]],
['*0*1', [attribs[0], attribs[1]]],
['*1*0', [attribs[1], attribs[0]]],
];
for (const [input, want] of testCases) {
it(`${JSON.stringify(input)} -> ${JSON.stringify(want)}`, async function () {
const gotAttribs = [...attributes.attribsFromString(input, pool)];
expect(JSON.stringify(gotAttribs)).to.equal(JSON.stringify(want));
});
}
});
});
describe('attribsToString', function () {
describe('accepts any kind of iterable', function () {
const testCases = [
['generator', (function* () { yield attribs[0]; yield attribs[1]; })()],
['list', [attribs[0], attribs[1]]],
['set', new Set([attribs[0], attribs[1]])],
];
for (const [desc, input] of testCases) {
it(desc, async function () {
const got = attributes.attribsToString(input, pool);
expect(got).to.equal('*0*1');
});
}
});
describe('rejects invalid inputs', function () {
const testCases = [null, [null]];
for (const input of testCases) {
it(JSON.stringify(input), async function () {
expect(() => attributes.attribsToString(input, pool)).to.throwException();
});
}
});
describe('reuses existing pool entries', function () {
const testCases = [
[[], ''],
[[attribs[0]], '*0'],
[[attribs[1]], '*1'],
[[attribs[0], attribs[1]], '*0*1'],
[[attribs[1], attribs[0]], '*1*0'],
];
for (const [input, want] of testCases) {
it(`${JSON.stringify(input)} -> ${JSON.stringify(want)}`, async function () {
const got = attributes.attribsToString(input, pool);
expect(got).to.equal(want);
});
}
});
describe('inserts new attributes into the pool', function () {
const testCases = [
[[['k', 'v']], `*${attribs.length}`],
[[attribs[0], ['k', 'v']], `*0*${attribs.length}`],
[[['k', 'v'], attribs[0]], `*${attribs.length}*0`],
];
for (const [input, want] of testCases) {
it(`${JSON.stringify(input)} -> ${JSON.stringify(want)}`, async function () {
const got = attributes.attribsToString(input, pool);
expect(got).to.equal(want);
expect(JSON.stringify(pool.getAttrib(attribs.length)))
.to.equal(JSON.stringify(['k', 'v']));
});
}
});
});
});