// (C) 2023 Doug Hoyte. MIT license class WrappedBuffer { constructor(buffer) { this._raw = new Uint8Array(buffer) || new Uint8Array(1024); this.length = buffer ? buffer.length : 0; this._expansionRate = 2; } unwrap() { return this._raw.subarray(0, this.length) } get capacity() { return this._raw.byteLength } extend(buf) { const targetSize = buf.length + this.length; if (this.capacity < targetSize) { const oldRaw = this._raw; const newCapacity = Math.max( Math.floor(this.capacity * this._expansionRate), targetSize ); this._raw = new Uint8Array(newCapacity); this._raw.set(oldRaw); } this._raw.set(buf, this.length); this.length += buf.length; return this; } shift() { const first = this._raw[0] this._raw = this._raw.subarray(1) this.length -= 1 return first } splice(n = 1) { const firstSubarray = this._raw.subarray(0, n) this._raw = this._raw.subarray(n) this.length -= n return firstSubarray } } class TrieNode { constructor() { this.children = new Array(256).fill(null); this.value = null; this.isTail = false; } } class HashMap { constructor() { this._root = new TrieNode(); } set(id, value) { let node = this._root; for (let byte of id) { if (!node.children[byte]) { node.children[byte] = new TrieNode(); } node = node.children[byte]; }; node.value = value; node.isTail = true; } get(keyUint8Array) { let node = this._root; for (let element of keyUint8Array) { if (!node.children[element]) { return null; // return null if the key is not in the trie } node = node.children[element]; } return node.value; } *[Symbol.iterator]() { const key = []; function* walk(node) { for (let i = 0; i < 256; i++) { if (!node.children[i]) continue; key.push(i); if (node.children[i].isTail) { yield [new Uint8Array(key), node.children[i].value]; } yield* walk(node.children[i]); key.pop(); } } yield* walk(this._root); } } class Negentropy { constructor(idSize) { if (idSize < 8 || idSize > 32) throw Error("idSize invalid"); this.idSize = idSize; this.items = []; } addItem(timestamp, id) { if (this.sealed) throw Error("already sealed"); if (id.byteLength > 64 || id.byteLength % 2 !== 0) throw Error("bad length for id"); id = id.subarray(0, this.idSize * 2); this.items.push({ timestamp, id }); } seal() { if (this.sealed) throw Error("already sealed"); this.items.sort(itemCompare); this.sealed = true; } _newState() { return { lastTimestampIn: 0, lastTimestampOut: 0, }; } _zeroBound() { return { timestamp: 0, id: new Uint8Array(this.idSize) }; } _maxBound() { return { timestamp: Number.MAX_VALUE, id: new Uint8Array(0) }; } initiate() { if (!this.sealed) throw Error("not sealed"); this.isInitiator = true; let output = new WrappedBuffer(); let state = this._newState(); this.splitRange(0, this.items.length, this._zeroBound(), this._maxBound(), state, output); return output.unwrap(); } reconcile(query) { if (!this.sealed) throw Error("not sealed"); query = new WrappedBuffer(query); let haveIds = [], needIds = []; let output = new WrappedBuffer(); let prevBound = this._zeroBound(); let prevIndex = 0; let state = this._newState(); let skip = false; let doSkip = () => { if (!skip) return; skip = false; this.encodeBound(prevBound, state, output); output.extend(this.encodeVarInt(0)); // mode = Skip }; while (query.length !== 0) { let currBound = this.decodeBound(query, state); let mode = this.decodeVarInt(query); // 0 = Skip, 1 = Fingerprint, 2 = IdList, 3 = IdListResponse let lower = prevIndex; let upper = findUpperBound(this.items, lower, this.items.length, currBound, itemCompare); if (mode === 0) { // Skip skip = true; } else if (mode === 1) { // Fingerprint let theirXorSet = this.getBytes(query, this.idSize); let ourXorSet = new Uint8Array(this.idSize); for (let i = lower; i < upper; ++i) { let item = this.items[i]; for (let j = 0; j < this.idSize; j++) ourXorSet[j] ^= item.id[j]; } let matches = true; for (let i = 0; i < this.idSize; i++) { if (theirXorSet[i] !== ourXorSet[i]) { matches = false; break; } } if (!matches) { doSkip(); this.splitRange(lower, upper, prevBound, currBound, state, output); } else { skip = true; } } else if (mode === 2) { // IdList let numElems = this.decodeVarInt(query); let theirElems = new HashMap(); for (let i = 0; i < numElems; i++) { let id = this.getBytes(query, this.idSize); theirElems.set(id, { offset: i, onBothSides: false }); } let responseHaveIds = []; let responseNeedIndices = []; for (let i = lower; i < upper; i++) { let id = this.items[i].id; let e = theirElems.get(id); if (!e) { // ID exists on our side, but not their side if (this.isInitiator) haveIds.push(id); else responseHaveIds.push(id); } else { // ID exists on both sides e.onBothSides = true; } } for (let [k, v] of theirElems) { if (!v.onBothSides) { // ID exists on their side, but not our side if (this.isInitiator) needIds.push(k); else responseNeedIndices.push(v.offset); } } if (!this.isInitiator) { doSkip(); this.encodeBound(currBound, state, output); output.extend(this.encodeVarInt(3)); // mode = IdListResponse output.extend(this.encodeVarInt(responseHaveIds.length)); for (let id of responseHaveIds) output.extend(id); let bitField = this.encodeBitField(responseNeedIndices); output.extend(this.encodeVarInt(bitField.length)); output.extend(bitField); } else { skip = true; } } else if (mode === 3) { // IdListResponse if (!this.isInitiator) throw Error("unexpected IdListResponse"); skip = true; let numIds = this.decodeVarInt(query); for (let i = 0; i < numIds; i++) { needIds.push(this.getBytes(query, this.idSize)); } let bitFieldSize = this.decodeVarInt(query); let bitField = this.getBytes(query, bitFieldSize); for (let i = lower; i < upper; i++) { if (this.bitFieldLookup(bitField, i - lower)) haveIds.push(this.items[i].id); } } else { throw Error("unexpected mode"); } prevIndex = upper; prevBound = currBound; } return [output.unwrap(), haveIds, needIds]; } splitRange(lower, upper, _lowerBound, upperBound, state, output) { let numElems = upper - lower; let buckets = 16; if (numElems < buckets * 2) { this.encodeBound(upperBound, state, output); output.extend(this.encodeVarInt(2)); // mode = IdList output.extend(this.encodeVarInt(numElems)); for (let it = lower; it < upper; ++it) output.extend(this.items[it].id); } else { let itemsPerBucket = Math.floor(numElems / buckets); let bucketsWithExtra = numElems % buckets; let curr = lower; for (let i = 0; i < buckets; i++) { let ourXorSet = new Uint8Array(this.idSize) for (let bucketEnd = curr + itemsPerBucket + (i < bucketsWithExtra ? 1 : 0); curr != bucketEnd; curr++) { for (let j = 0; j < this.idSize; j++) ourXorSet[j] ^= this.items[curr].id[j]; } if (i === buckets - 1) this.encodeBound(upperBound, state, output); else this.encodeMinimalBound(this.items[curr], this.items[curr - 1], state, output); output.extend(this.encodeVarInt(1)); // mode = Fingerprint output.extend(ourXorSet); } } } // Decoding getByte(buf) { if (buf.length === 0) throw Error("parse ends prematurely"); return buf.shift(); } getBytes(buf, n) { if (buf.length < n) throw Error("parse ends prematurely"); return buf.splice(n); } decodeVarInt(buf) { let res = 0; while (1) { let byte = this.getByte(buf); res = (res << 7) | (byte & 127); if ((byte & 128) === 0) break; } return res; } decodeTimestampIn(encoded, state) { let timestamp = this.decodeVarInt(encoded); timestamp = timestamp === 0 ? Number.MAX_VALUE : timestamp - 1; if (state.lastTimestampIn === Number.MAX_VALUE || timestamp === Number.MAX_VALUE) { state.lastTimestampIn = Number.MAX_VALUE; return Number.MAX_VALUE; } timestamp += state.lastTimestampIn; state.lastTimestampIn = timestamp; return timestamp; } decodeBound(encoded, state) { let timestamp = this.decodeTimestampIn(encoded, state); let len = this.decodeVarInt(encoded); if (len > this.idSize) throw Error("bound key too long"); let id = this.getBytes(encoded, len); return { timestamp, id }; } // Encoding encodeVarInt(n) { if (n === 0) return new Uint8Array([0]); let o = []; while (n !== 0) { o.push(n & 127); n >>>= 7; } o.reverse(); for (let i = 0; i < o.length - 1; i++) o[i] |= 128; return new Uint8Array(o); } encodeTimestampOut(timestamp, state) { if (timestamp === Number.MAX_VALUE) { state.lastTimestampOut = Number.MAX_VALUE; return this.encodeVarInt(0); } let temp = timestamp; timestamp -= state.lastTimestampOut; state.lastTimestampOut = temp; return this.encodeVarInt(timestamp + 1); } encodeBound(key, state, output) { output.extend(this.encodeTimestampOut(key.timestamp, state)); output.extend(this.encodeVarInt(key.id.length)); output.extend(key.id); } encodeMinimalBound(curr, prev, state, output) { output.extend(this.encodeTimestampOut(curr.timestamp, state)); if (curr.timestamp !== prev.timestamp) { output.extend(this.encodeVarInt(0)); } else { let sharedPrefixBytes = 0; for (let i = 0; i < this.idSize; i++) { if (curr.id[i] !== prev.id[i]) break; sharedPrefixBytes++; } output.extend(this.encodeVarInt(sharedPrefixBytes + 1)); output.extend(curr.id.subarray(0, sharedPrefixBytes + 1)); } }; encodeBitField(inds) { if (inds.length === 0) return []; let max = Math.max(...inds); let bitField = new Uint8Array(Math.floor((max + 8) / 8)); for (let ind of inds) bitField[Math.floor(ind / 8)] |= 1 << (ind % 8); return bitField; } bitFieldLookup(bitField, ind) { if (Math.floor((ind + 8) / 8) > bitField.length) return false; return !!(bitField[Math.floor(ind / 8)] & 1 << (ind % 8)); } } /** * @param {Item} a * @param {Item} b * * @returns {number} */ function itemCompare(a, b) { if (a.timestamp === b.timestamp) { return compareUint8Array(a.id, b.id) } return a.timestamp - b.timestamp; } function binarySearch(arr, first, last, cmp) { let count = last - first; while (count > 0) { let it = first; let step = Math.floor(count / 2); it += step; if (cmp(arr[it])) { first = ++it; count -= step + 1; } else { count = step; } } return first; } function findLowerBound(arr, first, last, value, cmp) { return binarySearch(arr, first, last, (a) => cmp(a, value) < 0); } function findUpperBound(arr, first, last, value, cmp) { return binarySearch(arr, first, last, (a) => cmp(value, a) >= 0); } /** * @param {Uint8Array} a * @param {Uint8Array} b */ function compareUint8Array(a, b) { for (let i = 0; i < a.byteLength; i++) { if (a[i] < b[i]) { return -1 } if (a[i] > b[i]) { return 1 } } if (a.byteLength > b.byteLength) { return 1 } if (a.byteLength < b.byteLength) { return -1 } return 0 } module.exports = Negentropy;