Added fast fourier transform recovery

clock_disparity/ghost_test.py

@@ -2,8 +2,8 @@ from networksim import NetworkSimulator
 from ghost_node import Node, NOTARIES, Block, Sig, genesis, SLOT_SIZE
 from distributions import normal_distribution
 
-net = NetworkSimulator(latency=150)
-notaries = [Node(i, net, ts=max(normal_distribution(50, 50)(), 0) * 0.1, sleepy=False) for i in range(NOTARIES)]
+net = NetworkSimulator(latency=45)
+notaries = [Node(i, net, ts=max(normal_distribution(5, 5)(), 0) * 0.1, sleepy=False) for i in range(NOTARIES)]
 net.agents = notaries
 net.generate_peers()
 for i in range(12000):

mimc_stark/merkle_tree.py

@@ -4,13 +4,26 @@ except:
     from pyblake2 import blake2s
 blake = lambda x: blake2s(x).digest()
 
+def permute4(values):
+    o = []
+    ld4 = len(values) // 4
+    for i in range(ld4):
+        o.extend([values[i], values[i + ld4], values[i + ld4 * 2], values[i + ld4 * 3]])
+    return o
+
+def get_index_in_permuted(x, L):
+    ld4 = L // 4
+    return x//ld4 + 4 * (x % ld4)
+
 def merkelize(L):
+    L = permute4(L)
     nodes = [b''] * len(L) + [x.to_bytes(32, 'big') if isinstance(x, int) else x for x in L]
     for i in range(len(L) - 1, 0, -1):
         nodes[i] = blake(nodes[i*2] + nodes[i*2+1])
     return nodes
 
 def mk_branch(tree, index):
+    index = get_index_in_permuted(index, len(tree) // 2)
     index += len(tree) // 2
     o = [tree[index]]
     while index > 1:
@@ -19,6 +32,7 @@ def mk_branch(tree, index):
     return o
 
 def verify_branch(root, index, proof, output_as_int=False):
+    index = get_index_in_permuted(index, 2**len(proof) // 2)
     index += 2**len(proof)
     v = proof[0]
     for p in proof[1:]:
@@ -29,5 +43,3 @@ def verify_branch(root, index, proof, output_as_int=False):
         index //= 2
     assert v == root
     return int.from_bytes(proof[0], 'big') if output_as_int else proof[0]
-
-

mimc_stark/recovery.py

@@ -0,0 +1,106 @@
+from fft import fft, mul_polys
+
+# Calculates modular inverses [1/values[0], 1/values[1] ...]
+def multi_inv(values, modulus):
+    partials = [1]
+    for i in range(len(values)):
+        partials.append(partials[-1] * values[i] % modulus)
+    inv = pow(partials[-1], modulus - 2, modulus)
+    outputs = [0] * len(values)
+    for i in range(len(values), 0, -1):
+        outputs[i-1] = partials[i-1] * inv % modulus
+        inv = inv * values[i-1] % modulus
+    return outputs
+
+# Generates q(x) = poly(k * x)
+def p_of_kx(poly, modulus, k):
+    o = []
+    power_of_k = 1
+    for x in poly:
+        o.append(x * power_of_k % modulus)
+        power_of_k = (power_of_k * k) % modulus
+    return o
+
+# Return (x - root**positions[0]) * (x - root**positions[1]) * ...
+# possibly with a constant factor offset
+def zpoly(positions, modulus, root_of_unity):
+    # If there are not more than 4 positions, use the naive
+    # O(n^2) algorithm as it is faster
+    if len(positions) <= 4:
+        root = [1]
+        for pos in positions:
+            x = pow(root_of_unity, pos, modulus)
+            root.insert(0, 0)
+            for j in range(len(root)-1):
+                root[j] -= root[j+1] * x
+        return [x % modulus for x in root]
+    else:
+        half_order_root_of_unity = pow(root_of_unity, 2, modulus)
+        # Recursively find the zpoly for even indices and odd
+        # indices, operating over a half-size subgroup in each
+        # case
+        left = zpoly([x//2 for x in positions if x%2 == 0],
+                     modulus, half_order_root_of_unity)
+        right = zpoly([x//2 for x in positions if x%2 == 1],
+                     modulus, half_order_root_of_unity)
+        invroot = pow(root_of_unity, modulus - 2, modulus)
+        # Offset the result for the odd indices, and combine
+        # the two
+        o = mul_polys(left, p_of_kx(right, modulus, invroot),
+                         modulus, root_of_unity)
+    # Deal with the special case where mul_polys returns zero
+    # when it should return x ^ (2 ** k) - 1
+    if o == [0] * len(o):
+        return [1] + [0] * (len(o) - 1) + [modulus - 1]
+    else:
+        return o
+
+def erasure_code_recover(vals, modulus, root_of_unity):
+    # Generate the polynomial that is zero at the roots of unity
+    # corresponding to the indices where vals[i] is None
+    import poly_utils
+    z = zpoly([i for i in range(len(vals)) if vals[i] is None],
+              modulus, root_of_unity)
+    zvals = fft(z, modulus, root_of_unity)
+
+    # Pointwise-multiply (vals filling in zero at missing spots) * z
+    # By construction, this equals vals * z
+    vals_with_zeroes = [x or 0 for x in vals]
+    p_times_z_vals = [x*y % modulus for x,y in zip(vals_with_zeroes, zvals)]
+    p_times_z = fft(p_times_z_vals, modulus, root_of_unity, inv=True)
+
+    # Keep choosing k values until the algorithm does not fail
+    # Check only with primitive roots of unity
+    for k in range(2, modulus):
+        if pow(k, (modulus - 1) // 2, modulus) == 1:
+            continue
+        invk = pow(k, modulus - 2, modulus)
+        # Convert p_times_z(x) and z(x) into new polynomials
+        # q1(x) = p_times_z(k*x) and q2(x) = z(k*x)
+        # These are likely to not be 0 at any of the evaluation points.
+        p_times_z_of_kx = [x * pow(k, i, modulus) % modulus
+                           for i, x in enumerate(p_times_z)]
+        p_times_z_of_kx_vals = fft(p_times_z_of_kx, modulus, root_of_unity)
+        z_of_kx = [x * pow(k, i, modulus) for i, x in enumerate(z)]
+        z_of_kx_vals = fft(z_of_kx, modulus, root_of_unity)
+        
+        # Compute q1(x) / q2(x) = p(k*x)
+        inv_z_of_kv_vals = multi_inv(z_of_kx_vals, modulus)
+        p_of_kx_vals = [x*y % modulus for x,y in
+                        zip(p_times_z_of_kx_vals, inv_z_of_kv_vals)]
+        p_of_kx = fft(p_of_kx_vals, modulus, root_of_unity, inv=True)
+
+        # Given q3(x) = p(k*x), recover p(x)
+        p_of_x = [x * pow(invk, i, modulus) % modulus
+                  for i, x in enumerate(p_of_kx)]
+        output = fft(p_of_x, modulus, root_of_unity)
+
+        # Check that the output matches the input
+        success = True
+        for inpd, outd in zip(vals, output):
+            success *= (inpd is None or inpd == outd)
+        if not success:
+            continue
+
+        # Output the evaluations if all good
+        return output

papers/pricing/ethpricing.tex

@@ -297,7 +297,7 @@ However, kth price auctions have a different kind of serious flaw: they are not
 
 A more serious issue is \emph{collusion} between the proposer and some transaction senders. A proposer can potentially collude with low-fee transaction senders (eg. suppose there is a single entity, like an exchange or mining pool, that sends such transactions and can be easily negotiated with) that are sending transactions with some fee $f_{low}$. The proposer can ask them to instead send their transactions with fee $f_{high}$, and refund them $f_{high} - \frac{f_{low}}{2}$. The proposer's revenue is now even higher: the proposer benefits from the increased height of the ``rectangle'' of fee revenue that they would get with the ``dummy transaction'' strategy above, but they would also get a portion of the revenue from transactions that they would otherwise have sacrificed. \footnote{For a more detailed treatment of similar issues, see \cite{li2018} and \cite{rothkopf2007}.}
 
-Hence, both first-price and second-price auctions are unsatisfactory. However, note that these issues are exclusively properties of auctions, and not properties of a fixed-price sale. If being included in the blockchain simply requires paying some $minFee$ (which would be burned, rather than given to the miner, to prevent side-dealing between transaction senders and miners allowing free transactions), then transaction senders have a simple strategy that they can use to set the fee on their transaction. Let $v$ be a user's private valuation for a transaction getting included in the next block. The user would check if $v > minFee$; if it is, they would bid $minFee + \epsilon$ (to provide a slight incentive for the block producer to include the transaction); if $v < minFee$ they would not send the transaction. This is a very simple strategy that does not require knowledge of others' valuations and is optimal for the transaction sender.
+Hence, both first-price and second-price auctions are unsatisfactory, and more complicated fee-setting and fee-sharing schemes also tend not to work, because any scheme where the proposer does not get exactly 100\% of any marginal increment in transaction fees is vulnerable to similar side dealing arrangements. However, note that these issues are exclusively properties of auctions, and not properties of a fixed-price sale. If being included in the blockchain simply requires paying some $minFee$ (which would be burned, rather than given to the miner, to prevent side-dealing between transaction senders and miners allowing free transactions), then transaction senders have a simple strategy that they can use to set the fee on their transaction. Let $v$ be a user's private valuation for a transaction getting included in the next block. The user would check if $v > minFee$; if it is, they would bid $minFee + \epsilon$ (to provide a slight incentive for the block producer to include the transaction); if $v < minFee$ they would not send the transaction. This is a very simple strategy that does not require knowledge of others' valuations and is optimal for the transaction sender.
 
 \section{Improving the Second Best}