Diff estimator tests, and some adjustments to ghost tests

ghost.py

@@ -5,11 +5,11 @@ TRANSIT_TIME = 12
 # Max uncle depth
 UNCLE_DEPTH = 4
 # Uncle block reward (normal block reward = 1)
-UNCLE_REWARD_COEFF = 15/16.
+UNCLE_REWARD_COEFF = 29/32.
 # Reward for including uncles
 NEPHEW_REWARD_COEFF = 1/32.
 # Rounds to test
-ROUNDS = 500000
+ROUNDS = 1000000
 
 import random
 
@@ -17,8 +17,11 @@ all_miners = {}
 
 
 class Miner():
-    def __init__(self, p):
+    def __init__(self, p, backward=0):
+        # Miner hashpower
         self.hashpower = p
+        # Miner mines a few blocks behind the head?
+        self.backward = backward
         self.id = random.randrange(10000000)
         # Set up a few genesis blocks (since the algo is grandpa-dependent,
         # we need two genesis blocks plus some genesis uncles)
@@ -52,6 +55,8 @@ class Miner():
     # Mine a block
     def mine(self):
         HEAD = self.blocks[self.head]
+        for i in range(self.backward):
+            HEAD = self.blocks[HEAD["parent"]]
         H = HEAD
         h = self.blocks[self.blocks[self.head]["parent"]]
         # Select the uncles. The valid set of uncles for a block consists
@@ -60,7 +65,7 @@ class Miner():
         # uncles of those previous blocks
         u = {}
         notu = {}
-        for i in range(UNCLE_DEPTH):
+        for i in range(UNCLE_DEPTH - self.backward):
             for c in self.children.get(h["id"], {}):
                 u[c] = True
             notu[H["id"]] = True
@@ -94,11 +99,26 @@ def cousin_degree(miner, b1, b2):
         t += 1
     return t
 
-# Set hashpower percentages here
+# Set hashpower percentages and strategies
-percentages = [1]*25 + [5, 5, 5, 5, 5, 10, 15, 25]
+# Strategy = how many blocks behind head you mine
+profiles = [
+    # (hashpower, strategy, count)
+    (1, 0, 20),
+    (1, -1, 4),  # cheaters, mine 1/2/4 blocks back to reduce
+    (1, -2, 3),  # chance of being in a two-block fork
+    (1, -4, 3),
+    (5, 4, 1),
+    (10, 1, 1),
+    (15, 1, 1),
+    (25, 1, 1),
+]
+
+total_pct = 0
 miners = []
-for p in percentages:
+for p, b, c in profiles:
-    miners.append(Miner(p))
+    for i in range(c):
+        miners.append(Miner(p, b))
+        total_pct += p
 
 miner_dict = {}
 for m in miners:
@@ -110,7 +130,7 @@ for t in range(ROUNDS):
     if t % 5000 == 0:
         print t
     for m in miners:
-        R = random.randrange(POW_SOLUTION_TIME * sum(percentages))
+        R = random.randrange(POW_SOLUTION_TIME * total_pct)
         if R < m.hashpower and t < ROUNDS - TRANSIT_TIME * 3:
             b = m.mine()
             listen_queue.append([t + TRANSIT_TIME, b])
@@ -149,19 +169,21 @@ for m in miners:
 print "### PRINTING PROFITS ###"
 
 for p in profit:
-    print miner_dict[p].hashpower, profit[p]
+    print miner_dict.get(p, Miner(0)).hashpower, profit.get(p, 0)
 
 print "### PRINTING RESULTS ###"
 
 groupings = {}
 counts = {}
 for p in profit:
-    h = miner_dict[p].hashpower
+    m = miner_dict.get(p, None)
-    counts[h] = counts.get(h, 0) + 1
+    if m:
-    groupings[h] = groupings.get(h, 0) + profit[p]
+        h = str(m.hashpower)+','+str(m.backward)
+        counts[h] = counts.get(h, 0) + 1
+        groupings[h] = groupings.get(h, 0) + profit[p]
 
 for c in counts:
-    print c, groupings[c] / counts[c] / (groupings[1] / counts[1])
+    print c, groupings[c] / counts[c] / (groupings['1,0'] / counts['1,0'])
 
 print " "
 print "Total blocks produced: ", len(all_miners) - UNCLE_DEPTH

stability/csvgen.py

@@ -8,6 +8,9 @@ tests = [
     [fit.diff_estimator, [1, 0, 0.001], [1.2, 10, 1]],
     [fit.diff_estimator, [1, 0, 0.001, 0], [1.2, 10, 1, 5]],
     [fit.ndiff_estimator, [1, 0, 0, 0.001], [1.2, 10, 1, 1]],
+    [fit.tx_diff_estimator, [1, 0, 0.001], [1.2, 10, 1]],
+    [fit.tx_diff_estimator, [1, 0, 0.001, 0, 0], [1.2, 10, 1, 6, 2]],
+    [fit.minimax_fee_estimator, [1, 3], [1.2, 60]],
 ]
 
 vals = [fit.optimize(t, mi, ma, rate=0.4, rounds=12000, tries=10)

stability/fit.py

@@ -2,10 +2,12 @@ import spread
 import math
 import random
 
-o = spread.declutter(spread.load('diff_and_price.csv'))
+o = spread.declutter(spread.load('diff_txs_price.csv'))
 
-diffs = [float(q[2]) for q in o][::-1]
+diffs = [float(q[2]) for q in o]
-prices = [float(q[1]) for q in o][::-1]
+prices = [float(q[1]) for q in o]
+txs = [float(q[3]) for q in o]
+txfees = [float(q[4]) for q in o]
 
 
 def simple_estimator(fac):
@@ -45,6 +47,57 @@ def diff_estimator(fac, dw, mf, exp=1):
     return o
 
 
+def tx_diff_estimator(fac, dw, mf, lin=1, exp=1):
+    fac = (fac - 1) or 0.000001
+    o = [1]
+    initavg = sum([txs[i] for i in range(5)]) / 5.0
+    txavgs = [initavg] * 5
+    for i in range(5, len(txs)):
+        txavgs.append(txavgs[-1] * 0.8 + txs[i] * 0.2)
+
+    derivs = [0] * 14
+    for i in range(14, len(txavgs)):
+        derivs.append(txavgs[i] - txavgs[i - 14])
+    for i in range(0, 14):
+        derivs[i] = derivs[14]
+    vals = [max(txavgs[i] + derivs[i] * dw, txavgs[i] * mf) for i in range(len(txavgs))]
+    for i in range(1, len(txavgs)):
+        growth = (vals[i] * 1.0 / vals[i-1] - 1)
+        if growth > fac:
+            surplus = (growth / fac) - 1
+            o.append(o[-1] * (1 + (surplus * lin * fac) ** exp))
+        elif vals[i] > vals[i-1]:
+            o.append(o[-1])
+        else:
+            surplus = 1 - growth
+            o.append(o[-1] * (1 - (surplus * lin * fac) ** exp))
+        if i and o[-1] < o[-2] * mf:
+            o[-1] = o[-2] * mf
+    return o
+
+
+def minimax_fee_estimator(fac, days):
+    o = [1]
+    initavg = sum([txs[i] for i in range(int(days))]) * 1.0 / days
+    txavgs = [initavg] * int(days)
+    for i in range(int(days), len(txs)):
+        txavgs.append(txavgs[-1] * 1.0 * (days-1) / days + txs[i] * 1.0 / days)
+    initavg2 = sum([txfees[i] for i in range(int(days))]) * 1.0 / days
+    txfeeavgs = [initavg2] * int(days)
+    for i in range(int(days), len(txs)):
+        txfeeavgs.append(txfeeavgs[-1] * 1.0 * (days-1) / days + txfees[i] * 1.0 / days)
+    # Calculate inverse fee, invfee ~= price
+    txavgfees = [t / f for f, t in zip(txfeeavgs, txavgs)]
+    for i in range(1, len(txavgfees)):
+        if txavgfees[i] * 1.0 / txavgfees[i-1] > fac:
+            o.append(o[-1] * txavgfees[i] * 1.0 / txavgfees[i-1] / fac)
+        elif txavgfees[i] > txavgfees[i-1]:
+            o.append(o[-1])
+        else:
+            o.append(o[-1] * txavgfees[i] * 1.0 / txavgfees[i-1])
+    return o
+
+
 def ndiff_estimator(*args):
     fac, dws, mf = args[0], args[1:-1], args[-1]
     o = [1]
@@ -86,6 +139,9 @@ def dual_threshold_estimator(fac1, fac2, dmul):
             o.append(o[-1] * diffs[i] * 1.0 / diffs[i-1])
     return o
 
+infinity = 2.**1023
+infinity *= 2
+
 
 def evaluate_estimates(estimates, crossvalidate=False):
     sz = len(prices) if crossvalidate else 780
@@ -93,8 +149,12 @@ def evaluate_estimates(estimates, crossvalidate=False):
     # compute average
     tot = 0
     for i in range(sz):
+        if estimates[i] == infinity or estimates[i] <= 0:
+            return 10**20
         tot += math.log(prices[i] / estimates[i])
     avg = 2.718281828459 ** (tot * 1.0 / sz)
+    if avg <= 0:
+        return 10**20
     for i in range(1, sz):
         sqdiffsum += math.log(prices[i] / estimates[i] / avg) ** 2
     return sqdiffsum
@@ -102,7 +162,7 @@ def evaluate_estimates(estimates, crossvalidate=False):
 
 # Simulated annealing optimizer
 def optimize(producer, floors, ceilings, rate=0.7, rounds=5000, tries=1):
-    bestvals, besty = None, 999999999999999
+    bestvals, besty = None, 10**21
     for t in range(tries):
         print 'Starting test %d of %d' % (t + 1, tries)
         vals = [f*0.5+c*0.5 for f, c in zip(floors, ceilings)]