negentropy/js/Negentropy.js

// (C) 2023 Doug Hoyte. MIT license

class WrappedBuffer {
    constructor(buffer) {
        this._raw = new Uint8Array(buffer || 256);
        this.length = buffer ? buffer.length : 0;
    }

    unwrap() {
        return this._raw.subarray(0, this.length);
    }

    get capacity() {
        return this._raw.byteLength;
    }

    extend(buf) {
        if (buf._raw) buf = buf.unwrap();
        const targetSize = buf.length + this.length;
        if (this.capacity < targetSize) {
            const oldRaw = this._raw;
            const newCapacity = Math.max(this.capacity * 2, targetSize);
            this._raw = new Uint8Array(newCapacity);
            this._raw.set(oldRaw);
        }

        this._raw.set(buf, this.length);
        this.length += buf.length;
    }

    shift() {
        const first = this._raw[0];
        this._raw = this._raw.subarray(1);
        this.length--;
        return first;
    }

    shiftN(n = 1) {
        const firstSubarray = this._raw.subarray(0, n);
        this._raw = this._raw.subarray(n);
        this.length -= n;
        return firstSubarray;
    }
}


class Negentropy {
    constructor(idSize = 16, frameSizeLimit = 0) {
        if (idSize < 8 || idSize > 32) throw Error("idSize invalid");
        if (frameSizeLimit !== 0 && frameSizeLimit < 4096) throw Error("frameSizeLimit too small");

        this.idSize = idSize;
        this.frameSizeLimit = frameSizeLimit;

        this.addedItems = [];
        this.pendingOutputs = [];

        if (typeof window === 'undefined') { // node.js
            const crypto = require('crypto');
            this.sha256 = async (slice) => new Uint8Array(crypto.createHash('sha256').update(slice).digest());
        } else { // browser
            this.sha256 = async (slice) => new Uint8Array(await crypto.subtle.digest("SHA-256", slice));
        }
    }

    addItem(timestamp, id) {
        if (this.sealed) throw Error("already sealed");
        id = this._loadInputBuffer(id);

        if (id.byteLength > 64 || id.byteLength < this.idSize) throw Error("bad length for id");
        if (id.byteLength > this.idSize) id = id.subarray(0, this.idSize);

        this.addedItems.push([ timestamp, id ]);
    }

    seal() {
        if (this.sealed) throw Error("already sealed");
        this.sealed = true;

        this.addedItems.sort((a, b) => a[0] !== b[0] ? a[0] - b[0] : compareUint8Array(a[1], b[1]));

        if (this.addedItems.length > 1) {
            for (let i = 0; i < this.addedItems.length - 1; i++) {
                if (this.addedItems[i][0] == this.addedItems[i + 1][0] &&
                    compareUint8Array(this.addedItems[i][1], this.addedItems[i + 1][1]) === 0) throw Error("duplicate item inserted");
            }
        }

        this.itemTimestamps = new BigUint64Array(this.addedItems.length);
        this.itemIds = new Uint8Array(this.addedItems.length * this.idSize);

        for (let i = 0; i < this.addedItems.length; i++) {
            let item = this.addedItems[i];
            this.itemTimestamps[i] = BigInt(item[0]);
            this.itemIds.set(item[1], i * this.idSize);
        }

        delete this.addedItems;
    }

    _newState() {
        return {
            lastTimestampIn: 0,
            lastTimestampOut: 0,
        };
    }

    _zeroBound() {
        return { timestamp: 0, id: new Uint8Array(this.idSize) };
    }

    _maxBound() {
        return { timestamp: Number.MAX_VALUE, id: new Uint8Array(0) };
    }

    _loadInputBuffer(inp) {
        if (typeof(inp) === 'string') inp = hexToUint8Array(inp);
        else if (__proto__ !== Uint8Array.prototype) inp = new Uint8Array(inp); // node Buffer?
        return inp;
    }

    numItems() {
        return this.itemTimestamps.length;
    }

    getItemTimestamp(i) {
        return Number(this.itemTimestamps[i]);
    }

    getItemId(i) {
        let offset = i * this.idSize;
        return this.itemIds.subarray(offset, offset + this.idSize);
    }

    getItem(i) {
        return { timestamp: this.getItemTimestamp(i), id: this.getItemId(i), };
    }

    async computeFingerprint(lower, num) {
        let offset = lower * this.idSize;
        let slice = this.itemIds.subarray(offset, offset + (num * this.idSize));
        let output = await this.sha256(slice);
        return output.subarray(0, this.idSize);
    }

    async initiate() {
        if (!this.sealed) throw Error("not sealed");
        this.isInitiator = true;

        await this.splitRange(0, this.numItems(), this._zeroBound(), this._maxBound(), this.pendingOutputs);

        return this.buildOutput();
    }

    async reconcile(query) {
        query = new WrappedBuffer(this._loadInputBuffer(query));
        let haveIds = [], needIds = [];

        if (!this.sealed) throw Error("not sealed");
        this.continuationNeeded = false;

        let prevBound = this._zeroBound();
        let prevIndex = 0;
        let state = this._newState();
        let outputs = [];

        while (query.length !== 0) {
            let currBound = this.decodeBound(query, state);
            let mode = this.decodeVarInt(query); // 0 = Skip, 1 = Fingerprint, 2 = IdList, 3 = deprecated, 4 = Continuation

            let lower = prevIndex;
            let upper = this.findUpperBound(lower, this.numItems(), currBound);

            if (mode === 0) { // Skip
                // Do nothing
            } else if (mode === 1) { // Fingerprint
                let theirFingerprint = this.getBytes(query, this.idSize);
                let ourFingerprint = await this.computeFingerprint(lower, upper - lower);

                if (compareUint8Array(theirFingerprint, ourFingerprint) !== 0) {
                    await this.splitRange(lower, upper, prevBound, currBound, outputs);
                }
            } else if (mode === 2) { // IdList
                let numIds = this.decodeVarInt(query);

                let theirElems = {}; // stringified Uint8Array -> original Uint8Array
                for (let i = 0; i < numIds; i++) {
                    let e = this.getBytes(query, this.idSize);
                    theirElems[e] = e;
                }

                for (let i = lower; i < upper; i++) {
                    let k = this.getItemId(i);

                    if (!theirElems[k]) {
                        // ID exists on our side, but not their side
                        if (this.isInitiator) haveIds.push(this.wantUint8ArrayOutput ? k : uint8ArrayToHex(k));
                    } else {
                        // ID exists on both sides
                        delete theirElems[k];
                    }
                }

                if (this.isInitiator) {
                    for (let v of Object.values(theirElems)) {
                        needIds.push(this.wantUint8ArrayOutput ? v : uint8ArrayToHex(v));
                    }
                } else {
                    let responseHaveIds = [];

                    let it = lower;
                    let didSplit = false;
                    let splitBound = this._zeroBound();

                    let flushIdListOutput = () => {
                        let payload = this.encodeVarInt(2); // mode = IdList

                        payload.extend(this.encodeVarInt(responseHaveIds.length));
                        for (let id of responseHaveIds) payload.extend(id);

                        let nextSplitBound = (it+1) >= upper ? currBound : this.getMinimalBound(this.getItem(it), this.getItem(it+1));

                        outputs.push({
                            start: didSplit ? splitBound : prevBound,
                            end: nextSplitBound,
                            payload: payload,
                        });

                        splitBound = nextSplitBound;
                        didSplit = true;

                        responseHaveIds = [];
                    };

                    for (; it < upper; ++it) {
                        responseHaveIds.push(this.getItemId(it));
                        if (responseHaveIds.length >= 100) flushIdListOutput(); // 100*32 is less than minimum frame size limit of 4k
                    }

                    flushIdListOutput();
                }
            } else if (mode === 3) { // Deprecated
                throw Error("other side is speaking old negentropy protocol");
            } else if (mode === 4) { // Continuation
                this.continuationNeeded = true;
            } else {
                throw Error("unexpected mode");
            }

            prevIndex = upper;
            prevBound = currBound;
        }

        while (outputs.length) {
            this.pendingOutputs.unshift(outputs.pop());
        }

        return [this.buildOutput(), haveIds, needIds];
    }

    async splitRange(lower, upper, lowerBound, upperBound, outputs) {
        let numElems = upper - lower;
        let buckets = 16;

        if (numElems < buckets * 2) {
            let payload = this.encodeVarInt(2); // mode = IdList
            payload.extend(this.encodeVarInt(numElems));
            for (let it = lower; it < upper; ++it) payload.extend(this.getItemId(it));

            outputs.push({
                start: lowerBound,
                end: upperBound,
                payload: payload,
            });
        } else {
            let itemsPerBucket = Math.floor(numElems / buckets);
            let bucketsWithExtra = numElems % buckets;
            let curr = lower;
            let prevBound = this.getItem(curr);

            for (let i = 0; i < buckets; i++) {
                let bucketSize = itemsPerBucket + (i < bucketsWithExtra ? 1 : 0);
                let ourFingerprint = await this.computeFingerprint(curr, bucketSize);
                curr += bucketSize;

                let payload = this.encodeVarInt(1); // mode = Fingerprint
                payload.extend(ourFingerprint);

                outputs.push({
                    start: i === 0 ? lowerBound : prevBound,
                    end: curr === upper ? upperBound : this.getMinimalBound(this.getItem(curr - 1), this.getItem(curr)),
                    payload: payload,
                });

                prevBound = outputs[outputs.length - 1].end;
            }

            outputs[outputs.length - 1].end = upperBound;
        }
    }

    buildOutput() {
        let output = new WrappedBuffer();
        let currBound = this._zeroBound();
        let state = this._newState();

        this.pendingOutputs.sort((a,b) => itemCompare(a.start, b.start));

        while (this.pendingOutputs.length) {
            let o = new WrappedBuffer();

            let p = this.pendingOutputs[0];

            let cmp = itemCompare(p.start, currBound);
            // If bounds are out of order or overlapping, finish and resume next time (shouldn't happen because of sort above)
            if (cmp < 0) break;

            if (cmp !== 0) {
                o.extend(this.encodeBound(p.start, state));
                o.extend(this.encodeVarInt(0)); // mode = Skip
            }

            o.extend(this.encodeBound(p.end, state));
            o.extend(p.payload);

            if (this.frameSizeLimit && output.length + o.length > this.frameSizeLimit - 5) break; // 5 leaves room for Continuation
            output.extend(o);

            currBound = p.end;
            this.pendingOutputs.shift();

        }

        // Server indicates that it has more to send, OR ensure client sends a non-empty message

        if ((!this.isInitiator && this.pendingOutputs.length) || (this.isInitiator && output.length == 0 && this.continuationNeeded)) {
            output.extend(this.encodeBound(this._maxBound(), state));
            output.extend(this.encodeVarInt(4)); // mode = Continue
        }

        let ret = output.unwrap();
        if (!this.wantUint8ArrayOutput) ret = uint8ArrayToHex(ret);
        return ret;
    }

    findUpperBound(first, last, value) {
        let count = last - first;

        while (count > 0) {
            let it = first;
            let step = Math.floor(count / 2);
            it += step;

            if (!(value.timestamp === this.getItemTimestamp(it) ? compareUint8Array(value.id, this.getItemId(it)) < 0 : value.timestamp < this.getItemTimestamp(it))) {
                first = ++it;
                count -= step + 1;
            } else {
                count = step;
            }
        }

        return first;
    }

    // Decoding

    getBytes(buf, n) {
        if (buf.length < n) throw Error("parse ends prematurely");
        return buf.shiftN(n);
    }

    decodeVarInt(buf) {
        let res = 0;

        while (1) {
            if (buf.length === 0) throw Error("parse ends prematurely");
            let byte = buf.shift();
            res = (res << 7) | (byte & 127);
            if ((byte & 128) === 0) break;
        }

        return res;
    }

    decodeTimestampIn(encoded, state) {
        let timestamp = this.decodeVarInt(encoded);
        timestamp = timestamp === 0 ? Number.MAX_VALUE : timestamp - 1;
        if (state.lastTimestampIn === Number.MAX_VALUE || timestamp === Number.MAX_VALUE) {
            state.lastTimestampIn = Number.MAX_VALUE;
            return Number.MAX_VALUE;
        }
        timestamp += state.lastTimestampIn;
        state.lastTimestampIn = timestamp;
        return timestamp;
    }

    decodeBound(encoded, state) {
        let timestamp = this.decodeTimestampIn(encoded, state);
        let len = this.decodeVarInt(encoded);
        if (len > this.idSize) throw Error("bound key too long");
        let id = this.getBytes(encoded, len);
        return { timestamp, id };
    }

    // Encoding

    encodeVarInt(n) {
        if (n === 0) return new WrappedBuffer([0]);

        let o = [];

        while (n !== 0) {
            o.push(n & 127);
            n >>>= 7;
        }

        o.reverse();

        for (let i = 0; i < o.length - 1; i++) o[i] |= 128;

        return new WrappedBuffer(o);
    }

    encodeTimestampOut(timestamp, state) {
        if (timestamp === Number.MAX_VALUE) {
            state.lastTimestampOut = Number.MAX_VALUE;
            return this.encodeVarInt(0);
        }

        let temp = timestamp;
        timestamp -= state.lastTimestampOut;
        state.lastTimestampOut = temp;
        return this.encodeVarInt(timestamp + 1);
    }

    encodeBound(key, state) {
        let output = new WrappedBuffer();

        output.extend(this.encodeTimestampOut(key.timestamp, state));
        output.extend(this.encodeVarInt(key.id.length));
        output.extend(key.id);

        return output;
    }

    getMinimalBound(prev, curr) {
        if (curr.timestamp !== prev.timestamp) {
            return { timestamp: curr.timestamp, id: new Uint8Array(0), };
        } else {
            let sharedPrefixBytes = 0;

            for (let i = 0; i < this.idSize; i++) {
                if (curr.id[i] !== prev.id[i]) break;
                sharedPrefixBytes++;
            }

            return { timestamp: curr.timestamp, id: curr.id.subarray(0, sharedPrefixBytes + 1), };
        }
    };
}


function hexToUint8Array(h) {
    if (h.startsWith('0x')) h = h.substr(2);
    if (h.length % 2 === 1) throw Error("odd length of hex string");
    let arr = new Uint8Array(h.length / 2);
    for (let i = 0; i < arr.length; i++) arr[i] = parseInt(h.substr(i * 2, 2), 16);
    return arr;
}

const uint8ArrayToHexLookupTable = new Array(256);
{
    const hexAlphabet = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'];
    for (let i = 0; i < 256; i++) {
        uint8ArrayToHexLookupTable[i] = hexAlphabet[(i >>> 4) & 0xF] + hexAlphabet[i & 0xF];
    }
}

function uint8ArrayToHex(arr) {
    let out = '';
    for (let i = 0, edx = arr.length; i < edx; i++) {
        out += uint8ArrayToHexLookupTable[arr[i]];
    }
    return out;
}


function compareUint8Array(a, b) {
    for (let i = 0; i < a.byteLength; i++) {
        if (a[i] < b[i]) return -1;
        if (a[i] > b[i]) return 1;
    }

    if (a.byteLength > b.byteLength) return 1;
    if (a.byteLength < b.byteLength) return -1;

    return 0;
}

function itemCompare(a, b) {
    if (a.timestamp === b.timestamp) {
        return compareUint8Array(a.id, b.id);
    }

    return a.timestamp - b.timestamp;
}


module.exports = Negentropy;