From 317032273f67815d548419555957165185b9d767 Mon Sep 17 00:00:00 2001 From: vub Date: Tue, 21 Feb 2017 07:27:32 -0500 Subject: [PATCH] Added a bunch of files --- bintrie.py | 289 ++++++++++++++++++++++++++++++++++++ bintrie_sample.py | 13 ++ casper4/simple_casper.v.py | 295 +++++++++++++++++++++++++++++++++++++ erasure_code/share.pyc | Bin 17489 -> 0 bytes iceage.py | 20 +++ price_analyzer.py | 32 ++++ price_grabber.py | 54 +++++++ random_circuit.py | 20 +++ 8 files changed, 723 insertions(+) create mode 100644 bintrie.py create mode 100644 bintrie_sample.py create mode 100644 casper4/simple_casper.v.py delete mode 100644 erasure_code/share.pyc create mode 100644 iceage.py create mode 100644 price_analyzer.py create mode 100644 price_grabber.py create mode 100644 random_circuit.py diff --git a/bintrie.py b/bintrie.py new file mode 100644 index 0000000..97e474e --- /dev/null +++ b/bintrie.py @@ -0,0 +1,289 @@ +# All nodes are of the form [path1, child1, path2, child2] +# or + +from ethereum import utils +from ethereum.db import EphemDB, ListeningDB +import rlp, sys +import copy + +hashfunc = utils.sha3 + +HASHLEN = 32 + + +# 0100000101010111010000110100100101001001 -> ASCII +def decode_bin(x): + return ''.join([chr(int(x[i:i+8], 2)) for i in range(0, len(x), 8)]) + + +# ASCII -> 0100000101010111010000110100100101001001 +def encode_bin(x): + o = '' + for c in x: + c = ord(c) + p = '' + for i in range(8): + p = str(c % 2) + p + c /= 2 + o += p + return o + + +# Encodes a binary list [0,1,0,1,1,0] of any length into bytes +def encode_bin_path(li): + if li == []: + return '' + b = ''.join([str(x) for x in li]) + b2 = '0' * ((4 - len(b)) % 4) + b + prefix = ['00', '01', '10', '11'][len(b) % 4] + if len(b2) % 8 == 4: + return decode_bin('00' + prefix + b2) + else: + return decode_bin('100000' + prefix + b2) + + +# Decodes bytes into a binary list +def decode_bin_path(p): + if p == '': + return [] + p = encode_bin(p) + if p[0] == '1': + p = p[4:] + assert p[0:2] == '00' + L = ['00', '01', '10', '11'].index(p[2:4]) + p = p[4+((4 - L) % 4):] + return [(1 if x == '1' else 0) for x in p] + + +# Get a node from a database if needed +def dbget(node, db): + if len(node) == HASHLEN: + return rlp.decode(db.get(node)) + return node + + +# Place a node into a database if needed +def dbput(node, db): + r = rlp.encode(node) + if len(r) == HASHLEN or len(r) > HASHLEN * 2: + h = hashfunc(r) + db.put(h, r) + return h + return node + + +# Get a value from a tree +def get(node, db, key): + node = dbget(node, db) + if key == []: + return node[0] + elif len(node) == 1 or len(node) == 0: + return '' + else: + sub = dbget(node[key[0]], db) + if len(sub) == 2: + subpath, subnode = sub + else: + subpath, subnode = '', sub[0] + subpath = decode_bin_path(subpath) + if key[1:len(subpath)+1] != subpath: + return '' + return get(subnode, db, key[len(subpath)+1:]) + + +# Get length of shared prefix of inputs +def get_shared_length(l1, l2): + i = 0 + while i < len(l1) and i < len(l2) and l1[i] == l2[i]: + i += 1 + return i + + +# Replace ['', v] with [v] and compact nodes into hashes +# if needed +def contract_node(n, db): + if len(n[0]) == 2 and n[0][0] == '': + n[0] = [n[0][1]] + if len(n[1]) == 2 and n[1][0] == '': + n[1] = [n[1][1]] + if len(n[0]) != 32: + n[0] = dbput(n[0], db) + if len(n[1]) != 32: + n[1] = dbput(n[1], db) + return dbput(n, db) + + +# Update a trie +def update(node, db, key, val): + node = dbget(node, db) + # Unfortunately this particular design does not allow + # a node to have one child, so at the root for empty + # tries we need to add two dummy children + if node == '': + node = [dbput([encode_bin_path([]), ''], db), + dbput([encode_bin_path([1]), ''], db)] + if key == []: + node = [val] + elif len(node) == 1: + raise Exception("DB must be prefix-free") + else: + assert len(node) == 2, node + sub = dbget(node[key[0]], db) + if len(sub) == 2: + _subpath, subnode = sub + else: + _subpath, subnode = '', sub[0] + subpath = decode_bin_path(_subpath) + sl = get_shared_length(subpath, key[1:]) + if sl == len(subpath): + node[key[0]] = [_subpath, update(subnode, db, key[sl+1:], val)] + else: + subpath_next = subpath[sl] + n = [0, 0] + n[subpath_next] = [encode_bin_path(subpath[sl+1:]), subnode] + n[(1 - subpath_next)] = [encode_bin_path(key[sl+2:]), [val]] + n = contract_node(n, db) + node[key[0]] = dbput([encode_bin_path(subpath[:sl]), n], db) + return contract_node(node, db) + + +# Compression algorithm specialized for merkle proof databases +# The idea is similar to standard compression algorithms, where +# you replace an instance of a repeat with a pointer to the repeat, +# except that here you replace an instance of a hash of a value +# with the pointer of a value. This is useful since merkle branches +# usually include nodes which contain hashes of each other +magic = '\xff\x39' + + +def compress_db(db): + out = [] + values = db.kv.values() + keys = [hashfunc(x) for x in values] + assert len(keys) < 65300 + for v in values: + o = '' + pos = 0 + while pos < len(v): + done = False + if v[pos:pos+2] == magic: + o += magic + magic + done = True + pos += 2 + for i, k in enumerate(keys): + if v[pos:].startswith(k): + o += magic + chr(i // 256) + chr(i % 256) + done = True + pos += len(k) + break + if not done: + o += v[pos] + pos += 1 + out.append(o) + return rlp.encode(out) + + +def decompress_db(ins): + ins = rlp.decode(ins) + vals = [None] * len(ins) + + def decipher(i): + if vals[i] is None: + v = ins[i] + o = '' + pos = 0 + while pos < len(v): + if v[pos:pos+2] == magic: + if v[pos+2:pos+4] == magic: + o += magic + else: + ind = ord(v[pos+2]) * 256 + ord(v[pos+3]) + o += hashfunc(decipher(ind)) + pos += 4 + else: + o += v[pos] + pos += 1 + vals[i] = o + return vals[i] + + for i in range(len(ins)): + decipher(i) + + o = EphemDB() + for v in vals: + o.put(hashfunc(v), v) + return o + + +# Convert a merkle branch directly into RLP (ie. remove +# the hashing indirection). As it turns out, this is a +# really compact way to represent a branch +def compress_branch(db, root): + o = dbget(copy.copy(root), db) + + def evaluate_node(x): + for i in range(len(x)): + if len(x[i]) == HASHLEN and x[i] in db.kv: + x[i] = evaluate_node(dbget(x[i], db)) + elif isinstance(x, list): + x[i] = evaluate_node(x[i]) + return x + + o2 = rlp.encode(evaluate_node(o)) + return o2 + + +def decompress_branch(branch): + branch = rlp.decode(branch) + db = EphemDB() + + def evaluate_node(x): + if isinstance(x, list): + x = [evaluate_node(n) for n in x] + x = dbput(x, db) + return x + evaluate_node(branch) + return db + + +# Test with n nodes and k branch picks +def test(n, m=100): + assert m <= n + db = EphemDB() + x = '' + for i in range(n): + k = hashfunc(str(i)) + v = hashfunc('v'+str(i)) + x = update(x, db, [int(a) for a in encode_bin(rlp.encode(k))], v) + print(x) + print(sum([len(val) for key, val in db.db.items()])) + l1 = ListeningDB(db) + o = 0 + p = 0 + q = 0 + ecks = x + for i in range(m): + x = copy.deepcopy(ecks) + k = hashfunc(str(i)) + v = hashfunc('v'+str(i)) + l2 = ListeningDB(l1) + v2 = get(x, l2, [int(a) for a in encode_bin(rlp.encode(k))]) + assert v == v2 + o += sum([len(val) for key, val in l2.kv.items()]) + cdb = compress_db(l2) + p += len(cdb) + assert decompress_db(cdb).kv == l2.kv + cbr = compress_branch(l2, x) + q += len(cbr) + dbranch = decompress_branch(cbr) + assert v == get(x, dbranch, [int(a) for a in encode_bin(rlp.encode(k))]) + # for k in l2.kv: + # assert k in dbranch.kv + o = { + 'total_db_size': sum([len(val) for key, val in l1.kv.items()]), + 'avg_proof_size': sum([len(val) for key, val in l1.kv.items()]), + 'avg_compressed_proof_size': (p // min(n, m)), + 'avg_branch_size': (q // min(n, m)), + 'compressed_db_size': len(compress_db(l1)) + } + return o diff --git a/bintrie_sample.py b/bintrie_sample.py new file mode 100644 index 0000000..4be7f0a --- /dev/null +++ b/bintrie_sample.py @@ -0,0 +1,13 @@ +import bintrie + +datapoints = [1, 3, 10, 31, 100, 316, 1000, 3162] +o = [] + +for i in range(len(datapoints)): + p = [] + for j in range(i+1): + print 'Running with: %d %d' % (datapoints[i], datapoints[j]) + p.append(bintrie.test(datapoints[i], datapoints[j])['compressed_db_size']) + o.append(p) + +print o diff --git a/casper4/simple_casper.v.py b/casper4/simple_casper.v.py new file mode 100644 index 0000000..f77c605 --- /dev/null +++ b/casper4/simple_casper.v.py @@ -0,0 +1,295 @@ +# Information about validators +validators: { + # Amount of wei the validator holds + deposit: wei_value, + # The epoch the validator is joining + epoch_start: num, + # The epoch the validator is leaving + epoch_end: num, + # The timestamp at which the validator can withdraw + withdrawal_time: timestamp, + # The address which the validator's signatures must verify to (to be later replaced with validation code) + addr: address, + # Addess to withdraw to + withdrawal_addr: address, + # The max epoch at which the validator prepared + max_prepared: num, + # The max epoch at which the validator committed + max_committed: num +}[num] + +# Information for use in processing cryptoeconomic commitments +consensus_messages: { + # Total deposits during this epoch + total_deposits: wei_value, + # How many prepares are there for this hash (hash of message hash + view source) + prepares: wei_value[bytes32], + # Is a commit on the given hash justified? + justified: bool[bytes32], +}[num] # index: epoch + +# ancestry[x][y] = k > 0: x is a kth generation ancestor of y +ancestry: num[bytes32][bytes32] + +# Number of validators +nextValidatorIndex: num + +# Constant that guides the size of validator rewards +interest_rate: decimal(1 / sec) + +# Time between blocks +block_time: timedelta + +# Length of an epoch in blocks +epoch_length: num + +# Withdrawal delay +withdrawal_delay: timedelta + +# Delay after which a message can be slashed due to absence of justification +insufficiency_slash_delay: timedelta + +# Current epoch +current_epoch: num + +def initialize_epoch(epoch: num): + computed_current_epoch = block.number / self.epoch_length + if epoch <= computed_current_epoch and epoch == self.current_epoch + 1: + self.consensus_messages[epoch].total_deposits += self.consensus_messages[self.current_epoch].total_deposits + self.current_epoch = epoch + +def deposit(validation_addr: address, withdrawal_addr: address): + assert self.current_epoch == block.number / self.epoch_length + start_epoch = (self.current_epoch - (self.current_epoch % 50)) + 100 + self.validators[self.nextValidatorIndex] = { + deposit: msg.value, + epoch_start: start_epoch, + epoch_end: 1000000000000000000000000000000, + withdrawal_time: 1000000000000000000000000000000, + addr: validation_addr, + withdrawal_addr: withdrawal_addr, + max_prepared: 0, + max_committed: 0, + } + self.nextValidatorIndex += 1 + self.interest_rate = 0.000001 + self.block_time = 7 + self.epoch_length = 1024 + self.withdrawal_delay = 2500000 + self.insufficiency_slash_delay = self.withdrawal_delay / 2 + self.consensus_messages[start_epoch].total_deposits += msg.value + +def start_withdrawal(index: num, sig: bytes <= 96): + # Signature check + assert len(sig) == 96 + assert ecrecover(sha3("withdraw"), + as_num256(extract32(sig, 0)), + as_num256(extract32(sig, 32)), + as_num256(extract32(sig, 64))) == self.validators[index].addr + # Check that we haven't already withdrawn + assert self.current_epoch == block.number / self.epoch_length + end_epoch = (self.current_epoch - (self.current_epoch % 50)) + 100 + assert self.validators[index].epoch_end > end_epoch + # Set the end epoch + self.validators[index].epoch_end = end_epoch + self.consensus_messages[end_epoch].total_deposits -= self.validators[index].deposit + # Set the withdrawal date + self.validators[index].withdrawal_time = block.timestamp + self.withdrawal_delay + +def withdraw(index: num): + if block.timestamp >= self.validators[index].withdrawal_time: + send(self.validators[index].withdrawal_addr, self.validators[index].deposit) + self.validators[index] = { + deposit: 0, + epoch_start: 0, + epoch_end: 0, + withdrawal_time: 0, + addr: None, + withdrawal_addr: None, + max_prepared: 0, + max_committed: 0, + } + +def prepare(index: num, epoch: num, hash: bytes32, ancestry_hash: bytes32, + epoch_source: num, source_ancestry_hash: bytes32, sig: bytes <= 96): + # Signature check + sighash = sha3(concat("prepare", as_bytes32(epoch), hash, ancestry_hash, as_bytes32(epoch_source), source_ancestry_hash)) + assert len(sig) == 96 + assert ecrecover(sighash, + as_num256(extract32(sig, 0)), + as_num256(extract32(sig, 32)), + as_num256(extract32(sig, 64))) == self.validators[index].addr + # Check that we are in the right epoch + assert self.current_epoch == block.number / self.epoch_length + assert self.current_epoch == epoch + assert self.validators[index].epoch_start <= epoch + assert epoch < self.validators[index].epoch_end + # Check that we have not yet prepared for this epoch + assert self.validators[index].max_prepared == epoch - 1 + # Pay the reward if the blockhash is correct + if True: #~blockhash(epoch * self.epoch_length) == hash: + reward = floor(self.validators[index].deposit * self.interest_rate * self.block_time) + self.validators[index].deposit += reward + self.consensus_messages[self.current_epoch].total_deposits += reward + # Can't prepare for this epoch again + self.validators[index].max_prepared = epoch + # Record that this prepare took place + new_ancestry_hash = sha3(concat(hash, ancestry_hash)) + self.consensus_messages[epoch].prepares[sighash] += self.validators[index].deposit + # If enough prepares with the same epoch_source and hash are made, + # then the hash value is justified for commitment + if self.consensus_messages[epoch].prepares[sighash] >= self.consensus_messages[epoch].total_deposits * 2 / 3 and \ + not self.consensus_messages[epoch].justified[new_ancestry_hash]: + self.consensus_messages[epoch].justified[new_ancestry_hash] = True + # Add a parent-child relation between ancestry hashes to the ancestry table + self.ancestry[ancestry_hash][new_ancestry_hash] = 1 + +def commit(index: num, epoch: num, hash: bytes32, sig: bytes <= 96): + # Signature check + sighash = sha3(concat("commit", as_bytes32(epoch), hash)) + assert len(sig) == 96 + assert ecrecover(sighash, + as_num256(extract32(sig, 0)), + as_num256(extract32(sig, 32)), + as_num256(extract32(sig, 64))) == self.validators[index].addr + # Check that we are in the right epoch + assert self.current_epoch == block.number / self.epoch_length + assert self.current_epoch == epoch + assert self.validators[index].epoch_start <= epoch + assert epoch < self.validators[index].epoch_end + # Check that we have not yet committed for this epoch + assert self.validators[index].max_committed == epoch - 1 + # Pay the reward if the blockhash is correct + if True: #~blockhash(epoch * self.epoch_length) == hash: + reward = floor(self.validators[index].deposit * self.interest_rate * self.block_time) + self.validators[index].deposit += reward + self.consensus_messages[self.current_epoch].total_deposits += reward + # Can't commit for this epoch again + self.validators[index].max_committed = epoch + +# Cannot make two versions of the same message +def intra_epoch_equivocation_slash(index: num, epoch: num, msgtype: bytes <= 16, + args1: bytes <= 250, args2: bytes <= 250, sig1: bytes <= 96, sig2: bytes<= 96): + # Signature check + sighash1 = sha3(concat(msgtype, as_bytes32(epoch), args1)) + sighash2 = sha3(concat(msgtype, as_bytes32(epoch), args2)) + assert ecrecover(sighash1, + as_num256(extract32(sig1, 0)), + as_num256(extract32(sig1, 32)), + as_num256(extract32(sig1, 64))) == self.validators[index].addr + assert ecrecover(sighash2, + as_num256(extract32(sig2, 0)), + as_num256(extract32(sig2, 32)), + as_num256(extract32(sig2, 64))) == self.validators[index].addr + # Check that they're not the same message + assert sighash1 != sighash2 + # Delete the offending validator, and give a 4% "finder's fee" + validator_deposit = self.validators[index].deposit + send(msg.sender, validator_deposit / 25) + self.consensus_messages[self.current_epoch].total_deposits -= (validator_deposit - validator_deposit / 25) + self.validators[index] = { + deposit: 0, + epoch_start: 0, + epoch_end: 0, + withdrawal_time: 0, + addr: None, + withdrawal_addr: None, + max_prepared: 0, + max_committed: 0, + } + +def prepare_commit_inconsistency_slash(index: num, prepare_epoch: num, prepare_hash: bytes32, + prepare_source_epoch: num, prepare_source_ancestry_hash: bytes32, sig1: bytes <= 96, + commit_epoch: num, commit_hash: bytes32, sig2: bytes <= 96): + # Signature check + sighash1 = sha3(concat("prepare", as_bytes32(prepare_epoch), prepare_hash, + as_bytes32(prepare_source_epoch), prepare_source_ancestry_hash)) + sighash2 = sha3(concat("commit", as_bytes32(commit_epoch), commit_hash)) + assert ecrecover(sighash1, + as_num256(extract32(sig1, 0)), + as_num256(extract32(sig1, 32)), + as_num256(extract32(sig1, 64))) == self.validators[index].addr + assert ecrecover(sighash2, + as_num256(extract32(sig2, 0)), + as_num256(extract32(sig2, 32)), + as_num256(extract32(sig2, 64))) == self.validators[index].addr + # Check that they're not the same message + assert sighash1 != sighash2 + # Check that the prepare refers to something older than the commit + assert prepare_source_epoch < commit_epoch + # Check that the prepare is newer than the commit + assert commit_epoch < prepare_epoch + # Delete the offending validator, and give a 4% "finder's fee" + validator_deposit = self.validators[index].deposit + send(msg.sender, validator_deposit / 25) + self.consensus_messages[self.current_epoch].total_deposits -= validator_deposit + self.validators[index] = { + deposit: 0, + epoch_start: 0, + epoch_end: 0, + withdrawal_time: 0, + addr: None, + withdrawal_addr: None, + max_prepared: 0, + max_committed: 0, + } + +def commit_non_justification_slash(index: num, epoch: num, hash: bytes32, sig: bytes <= 96): + # Signature check + sighash = sha3(concat("commit", as_bytes32(epoch), hash)) + assert ecrecover(sighash, + as_num256(extract32(sig, 0)), + as_num256(extract32(sig, 32)), + as_num256(extract32(sig, 64))) == self.validators[index].addr + # Check that the commit is old enough + assert self.current_epoch == block.number / self.epoch_length + assert (epoch - self.current_epoch) * self.block_time > self.insufficiency_slash_delay + assert not self.consensus_messages[epoch].justified[hash] + # Delete the offending validator, and give a 4% "finder's fee" + validator_deposit = self.validators[index].deposit + send(msg.sender, validator_deposit / 25) + self.consensus_messages[self.current_epoch].total_deposits -= validator_deposit + self.validators[index] = { + deposit: 0, + epoch_start: 0, + epoch_end: 0, + withdrawal_time: 0, + addr: None, + withdrawal_addr: None, + max_prepared: 0, + max_committed: 0, + } + +# Fill in the table for which hash is what-degree ancestor of which other hash +def derive_ancestry(top: bytes32, middle: bytes32, bottom: bytes32): + assert self.ancestry[middle][top] + assert self.ancestry[bottom][middle] + self.ancestry[bottom][top] = self.ancestry[bottom][middle] + self.ancestry[middle][top] + +def prepare_non_justification_slash(index: num, epoch: num, hash: bytes32, epoch_source: num, + source_ancestry_hash: bytes32, sig: bytes <= 96): + # Signature check + sighash = sha3(concat("viewchange", as_bytes32(epoch), hash, as_bytes32(epoch_source), source_ancestry_hash)) + assert ecrecover(sighash, + as_num256(extract32(sig, 0)), + as_num256(extract32(sig, 32)), + as_num256(extract32(sig, 64))) == self.validators[index].addr + # Check that the view change is old enough + assert self.current_epoch == block.number / self.epoch_length + assert (epoch - self.current_epoch) * self.block_time > self.insufficiency_slash_delay + # Check that the source ancestry hash had enough prepares + assert not self.consensus_messages[epoch_source].justified[source_ancestry_hash] + # Delete the offending validator, and give a 4% "finder's fee" + validator_deposit = self.validators[index].deposit + send(msg.sender, validator_deposit / 25) + self.consensus_messages[self.current_epoch].total_deposits -= validator_deposit + self.validators[index] = { + deposit: 0, + epoch_start: 0, + epoch_end: 0, + withdrawal_time: 0, + addr: None, + withdrawal_addr: None, + max_prepared: 0, + max_committed: 0, + } diff --git a/erasure_code/share.pyc b/erasure_code/share.pyc deleted file mode 100644 index c060a7f565dcd1ff4903618fa4b60fccfa1784e1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 17489 zcmcIsTWlQHc|NmCkz8IxiWiY9TM}tm7A0A>EUS_X)2?jMc2XxyGD=HZHM3dn49TUI zdz+!OQb|~joY)B(qivA1P180B8Z-#fTiS>ArAYdaHcv%>^rdJ~pa}AmhoUW7^r20^ z@B3#jlt|VFQgP?(KWEOI^Plg(o-?yb|Jz{SKmNvlUM;xfUl+bFa|`MWHemH1$`p=T+lC~-vt9A23#;GV$cQq zMeKLMkcc4{42u|cA-X!?-t^qH1ENPnkBA->Jt}%k^qA;z(c>;Ss7?;LU_!)%3noQO zx*#he>w-h>+94M_pwkCj@SunXU2s^$VHX?`al{2jMI3d(ls?n$&6K+a`XSX%yWp6J zhg|Tmh+{4|F5+PqJR;(_3r>i5#04`VPPpKth#4205^>T6r$wA{!J{HhyWot7M_urk zh%+vDT*PB8I4k0D3i|5IIY_?v6a1SmK>ThWj(!|ha76v6fw*JoghV{)wmi2f)!6jh z=r~m5;Kx#~Ht4oeiJEMt+{9MeH6H7{Z^~_Txy`iO>~dK&;MWB2wW`_L3vK~qfNJR4 z^^~iBimlf?>V{YT1=o1LdfR7;_H;*YP(FTebBh42)0K<{n(Q3ta~de=IF>fwH$AZF zff2BYDUNY^KjyesXLy^3U4XAOY{6f86shbK7vk|YqC!mAkja=?3?usZG=M8xo90Hp<+B`I2ipJP(+7vIF z?uud;d&{+0hROUQnl({v4nUAK$RLQ8$=qWrvd(%~t-s4L%}#kkF88tapfmIWv0W_Z z*TY;cW=$@)UI{izI5mYPC;kwN5f-B?##tO>F^Qr*S5ezst%UL%8buULx7VG@r195{ zqc@W-aNe9V?T0oM(T`x>P?NVEKX>6cp;l>EKOq~1k83Iy1JpkEmUGL>t3G8(5MwLj z3~hg2Ax0yYr`9*3cxow}s@99^;k4q!2P;7V&Ge64@QC2NcXr^(NGXSrkbxzqX`OE8 z11Vf`b)T-jC&w-=JPjsV;-|8iZ#TEKr*HQuL$eaP^IWb_%12Qy*I^EO_;jDrF5!qM z5$C11I~xi=dH{v#KTE-last9Sc#op`52C3lDWUD~dX&3Dba@Arp|VwRG>xMv8Y;*h z9E`nZ4*bLDaR&yd=+S#;VDhKN0B3n)2L|W?g}^Zo_sam|!&oLq+ei>PvjUUzOQ$(uHN@@h5iIuy*oE3SM_>(Ah|z3%li*C zJCXXo8)wItS3hsPdL78@afWLn@dn`qVbVg?(*@grPS?Dx0fT zO1EK*VdKpnEa>!wOA}dyugz{iJ85;ZwTfG) zJ?|D)Ag^B6_*}EI_NcFOR9@C%ZL7~U7TE{^lJaW~Issnh7N6IknI?yQ;t(9PS~AxD zFk!1dyV=KZDt;!=aqWlvENu3=H6W3pT6ncTH{SYPqj5iPfe-<514M3X&^4ZKHtJQs z=t5|m5@TpJ=+He!BrZ@et1>2@LI<@iZ(*}v3=9|pAN2?hX1rZp2a=(=`utmOYrkvM znvL4nEvgajh5)n&lSSX{lku&g-AnC{5)-@)mABqI!_ z9p<34M9SZ`34a=Gw6esqEUd>O^VMot4rDG!8E2@ms;?Hq0@k>6k&VcInAIHOHd10W zu9QPd5-XKh_99DKQ(bvuJyIu?jkvlIs}9b?1`NV-1()eAL~ggeHFdM3yKhCMVjX;#lI)ps{pt+IHxnV`Bq4uvP=}fNB6Mghc~E)L~Tu zJu`<#W5HI9IXt*JdO3VeQk!0p(sVyj<#4sCUUyuXTC+epJZVJ*w`XV%-=D+s|Q?DpbPdS z(Tfu`-VK^ww4vMI$j9LiKzF!&YR*Bd<_e5MJ;`x}Z7!iEnNc$wLim`Qg+N6IrsHX84z9NRJG7^Zt`>8`P}Cp$62kDufYPhI28RmQD4k?n zd6o^gb$1W7 zp>W;-g#JNq9A4(I_lUgBlSl(5p%+=05Ii{6rZ_l=bmNaez#(vTBX592DLEBf=2Dm? zy{VWZK{EIV&+GeK{TqqkDFtO!SzLpRvg4FS<7cT6Nqp8lL6@`m{IKMpu-y;7=yoV= zlSV(x5us&6vRp%8$JA;kq&`<=M$J7bNW->Qsr8m-(R~yu*!-li1Yua^iM*6Ap`h0J z?dGIDB7gsaV6QQw5*3P~Sz^~@&uolvvN78I7s!a~D(`nW55nU=#AJ+JagEVJ z|1@dF=g~ow9DuG)TlCfM2Cs1KS2}BAukR(Gru5pIa!Hqy~4E;=?p9wUY zTO4I7h1d<#T#zo7O?5hjskHJ+B(!A)nU#~m^xbc$pki;k1@2w%*g zQ1=w>tbGg8>ULX5mY@UTti{xsd+zU-&%lji;Mqf3((0$*)&3Pp;b+m8GAne!%<50v zCLZr#^B)TMBk$H_xt$*2gZvOo47#xi0kpKb4?-&xNIZb#JpjQjgX>5(NW4<_P@&F1^mIa70>%yIEsaxRqK!i{^|UPc2}Stx?zbfWwjw zq5=;b!sNAR6v$*O@-W5jZRL_a8+fV;z3e462JZ;sWBFG7yeNe zbS7HUhR8I-IwG=pehxUJjYDp3`H!-hR!+e9(<9+tv-DZHLvujX!srNSzfoN>5bjrRkOhBjyzQbKAIX;d)hH;-< zCs82JS8zn{prATKW#HGKdeD65j=)zkp+F*{YnZR4bI?0}I!>W|lh|}+9&%X@DpBNf zN_mmYpeJM}TIFJtRyS~Jbgk&h`t)iaP2v)Qlh0q_Y6i-g+`3^)XNkO*HVl{JpgwS5B)~Hvj!)jnLaxP2WhA-$qn=|o0ghBw#L-`Hldw_3a zt{blL22#W}D&7taclLWDlG&Qw(c*9TizsJs>|`HA98Vqc#rkVRoQA5kAJlr(JA|xU z>Ps23C1Sp9zGLdlxAnBzvS8>WJtRJJi%+;uLxlw}bPqFo%N_t|Ti$e@VG@`qG4N`K z0dY%b)9#mL7M)9!0%yX5Q`GJ%XOFJl#xuX>h9+v9o}ijXVDq-NZQq09-Ls|q`cja8 zw#&;!qaFa#J7u8d!R^Bbf`WDtr*y?@wX36e;P{1bA{E7 z^7ZIC$0tJ$YJS*z+#sh7wIKgKP5>|=pX988Z8ngQ<#NHnY=dqT@GgL1ZZ-r=)A@gi~FQ@qVJ6ZAh3uO74PB;HR$ye+HfYKfOx z)v+%BDa+W$G~Meh*%IIm+2(jE0T)P7z<$VL8HLhxUy}Ve>mEj-IJjJ_M{%xPDRURc z2TF-u?ICJ2u0+%~$PU9NMLmcR!u^$pI>k;*T4IMCB%X}c1%bhHN5|yxp#x0|x=Lx8 zF*X;mI~rqC0*SPCHO9-37WNvUAxN)U#t1b39}QB%dZ#6<3?^<5$s`lmAgVg-G9JMx zS4g>Swq$~o42h)4P&@UM;r}TyANiYmi)@%99QB3~%MU|9Cv5>r_rpVh=XnrN_aoR9 zz}K2g6t?6w^ca+e01Of+GP0fTh#Y z?R78qZS#jJIKRw-<601`gVHuM^AS8WT8UTvkF&`vo8Z_f^&lch;4AA(#d7!;oEs(2 zWPHernfjf5;@p$xf9Ar)OHbvO3PHHs(&E-c`DR0FmWQ$}@?dtNnPssfsVFAy+-gUD zfYTk=k|+8$x#9KuhkOLJ7{_*<5^MqOSi8<5Di|P}Nqi$01|&iArP;i^Tr3Cgj-~J1 z$5Joum$;tn8s+(B2}qQ($kTLofidQz z8a!|+=1XuMmP>q(sP>`WVHG;zb)l59as+yeD>$NOP+-asC7{7H3>+XZ1^0ljYzj!g zqrx3vLM>l_mYDrAb%gVn_xY?s`xIvGG!2_3C9qM}1m)A@r_Na`KF7BJd>5Y#9KKM<_PD;F!7?6 z!&-ODEneza8$$ZFn6qUw%m^R&Ow0>C0g@GDu0qjcij0WSTz3$Jw$v;2z!yMV)me)9 z2B~Qjwz$Te+_GO`WfcWj#fqSy7+}$078Sd}M3D>S+;XuL{yp)rkT9e@2tKAeWtu>& z$eJK}3&b{v6ds6@R+kWW^2#l zeRqQqgV_dzbc>sMG^ZrCP@aij=P-e^Y0#T$r+plY{6)u607@W=Oc6$c z>3<3}vY)r5AZ9;F1S(M6JZPDtI3ijz8d5*CGMkugY7#CaSrq4E4=Q8};Mp#c>5kG=RY{cQXhx5WF1Ev<$XmQ~lY z7-Jb+9=q)GSpKf53+dS0;%n>`I=}}3FLeyB%^onWp|P2-w-~Uy%MYe8&8Gu;@q$Sq zXgCV>=+(Xlo6j&mto?FZFnt2k%Tf~uQyXzRmzN)K^ilJKUr9<2hlsVVOh^OdDFeOD`y{}3v7V~{Eh z2VcPM-T($q+&Wjm^61nx9^xgjF`9_{(-iedz627ns_bmQ)f1;NZ?TE+_@6^z@#j~0 z`fDr*yCQLMeLcj_OYl#{Hq}>3YUcSHYi|TvV{-Eywg0Pz6(RTEiBYV9iAYedtO8Dw zBGx{V3+I0x#SC!)i&DeT|19g@VL=&SM*-5e)q(UlXvaIEnH47wdrG$|Y?_GA%@%m2 zi!FNGh2|rx|HeY!|1EUwe~JC`k(K}3q__j%{|+dNQ;OK9cq%*mT~au9vovl4bWHC& zr^noe?=z3VOJUQ3xKV9*XoBAa0SKX*$mpZ9O~GMSh%cDwK>hu?jtaKHK0kIaVGB}A znvKT{4ml3oT65RTPYeRr<`(Ca58^!!B`^3@pWS4e0>BY0@nJu#l#f!e!3}f5b3|Aa z;!0wwd{ZRtu7ZJPbO6^*2-@qc%4IlJ+i-k?)R$Q(VSI_y`&iJ#WE${9b<2vF&-g?Z z@II>=XL?!|0?dfOr*y^IFE00jeA{sTs5Tq{sQ|=-_97~? zI-B1tsNE*?HC+1$C)7;adN7$mByUPxs5mX_DB`4L8W7e!iv-B*E(G623j)8eoeTIv zEq}jvVFg-8tC2EcT1ay*P)U>!GpCV5rfXG#k@yT*jEuz$NXA0HWZnQ-qw{}-#WV{F z$LEAh)^G8&?L@g!@V~&~4vIL7g5EG!S*Ci}$}mW_nRh`;n<}lwbf!oaazt+gy)o$> zhw=4z{WykG*jS>t$ZpG({wYp&#E@=Pu2pMH!UT-WyV{b1>xj!THEG7&BA!*+D}BHW zZO!@4EsnDqzLvHJiFl6B%<%pv9&x;g9Pg|t8{P*5_y}RP48faKiD#pI>PLGP*6x}u zcD#0Df{_2>>=d(18lS|e{}mRJhO884&Tm_!qMYP+niuY{P=u3QsBL>DK#UWG3VzU6 z-zdb2WtC3w$6$@!%+#R3kC^x4SnO7o$dSwv0Si#w=F?wOenjlx-2@yUqa1WfQBYU0 z3uHi>YD0^&clS9DdW%;ilG~KE6U4Neg;^ptyoqtm(V;WxIv9Y)H-EqbzAQMjwOp!0 zfO%ZSp)3lHVp|Hyi|wV2w02ucrfr|k5F<2n<~_g8d)l)4B2Oi-mY5W&5q$B+g3L*V zW%-N5rdUR&VwThkO7H64zI|J}w9SaSn-|rNjnWZhh_92ogXqHHiRnQc-P&|d!-)9X zG-bAQK|CZyKG=n}!hu3GT3*jds)<=#p#B3V3J7R->7A;{(Qya|C$slod79}t9#Nw~ zXS}h^MgfcglXb~qP>LKkO>G4N^=mJOf!%DpK*3l03(9^z%Nbjw<_~cX;ZM?LlHJnP zzr@IGU2UaUW~J0ssVj9=^Tugoz=OI{ffarM5i{JsQNpVnCQI)M$`p9?I#%z49ye$3 z*`Wc_TC4z}z(f)+<6uDOuvU3&4$a@zA6J&B=0*h#qVqNcc-)rJqmh3SdtLh7Bwksv ztwp>!fFJWWY0hGw8eomsjn{;*Vub60^y)`Fn9l&dK3sc(;>6D_QNbTw;{O0>zA%JK z_^e>*ir)ylrq_{n;CB=zIJY=UklGmBJ1N5e%h3XuC8W5$sD4vT{D@S9A)egTflFex%=0MOiQ zhmk8Je~QHk7EiHwhQ)I%USu)PLa&tww*pUBS*)>uIVD>bM9?-cxy7bO6+V27^FP7j zE{iu<$OnI#lpa&l{PAmru&&L?vgI`r9iyf%`F)wxS6FJ(hVSGn_e-8O)4iMl<7?X9f-o s3=Q<7+@I;s44}Td8)Cp;4}R*J8R|pXmpPc>zx|*GG6(QIj6a 200000: + newdiff += 2 ** ((i - 200000) // 100000) + diffs.append(newdiff) + times.append(times[-1] + blocktime) + if i % 10000 == 0: + print 'Block %d, approx ETH supply %d, time %r blocktime %.2f' % \ + (i, 60102216 * 1.199 + 5.3 * i, datetime.datetime.utcfromtimestamp(times[-1]).isoformat().replace('T',' '), diffs[-1] / hashpower) + # print int(adjfac * 2048) diff --git a/price_analyzer.py b/price_analyzer.py new file mode 100644 index 0000000..0bcc6b3 --- /dev/null +++ b/price_analyzer.py @@ -0,0 +1,32 @@ +coins = ['BTC', 'ETH', 'LTC', 'XMR', 'XRP', 'DASH', 'MAID', 'XEM', 'DOGE'] +returns = [2.2638834003579813, 8.116901792476218, 1.3165650562863767, 28.082132882509775, 1.085332332146777, 3.133009784746374, 7.253804471524264, 21.965787725641633, 1.4764456923421703] +cov = {'DASH': {'DASH': 0.7644961854506808, 'XEM': 0.12731810810740216, 'DOGE': 0.024594538249552256, 'LTC': -0.017885008134766338, 'MAID': 0.15851962585719734, 'BTC': 0.04633192428471714, 'ETH': 0.06459627506298854, 'XRP': 0.0485013905397046, 'XMR': 0.07136735462548086}, 'XEM': {'DASH': 0.12731810810740216, 'XEM': 2.696964866397394, 'DOGE': 0.2573684776370606, 'LTC': 0.2451590145706355, 'MAID': 0.24638558526778676, 'BTC': 0.3162291738177758, 'ETH': 0.16795613694062173, 'XRP': 0.20794892099014484, 'XMR': 0.24474433942952412}, 'DOGE': {'DASH': 0.024594538249552256, 'XEM': 0.2573684776370606, 'DOGE': 0.9048010148463588, 'LTC': 0.21028685267231811, 'MAID': 0.1969666908833168, 'BTC': 0.18023154281182274, 'ETH': 0.04381871552036941, 'XRP': 0.22505760423798415, 'XMR': 0.10013128092001486}, 'LTC': {'DASH': -0.017885008134766338, 'XEM': 0.2451590145706355, 'DOGE': 0.21028685267231811, 'LTC': 0.3396675230101496, 'MAID': -0.023217502484851917, 'BTC': 0.2188873903143038, 'ETH': 0.0070895267193696165, 'XRP': 0.04038308119562005, 'XMR': 0.10439499530126177}, 'MAID': {'DASH': 0.15851962585719734, 'XEM': 0.24638558526778676, 'DOGE': 0.1969666908833168, 'LTC': -0.023217502484851917, 'MAID': 1.544265885009231, 'BTC': 0.05656467031525303, 'ETH': 0.43029061022937815, 'XRP': 0.060012979312135256, 'XMR': 0.25174166270626736}, 'BTC': {'DASH': 0.04633192428471714, 'XEM': 0.3162291738177758, 'DOGE': 0.18023154281182274, 'LTC': 0.2188873903143038, 'MAID': 0.05656467031525303, 'BTC': 0.25654027047139827, 'ETH': 0.06821288200609678, 'XRP': 0.046123059468558665, 'XMR': 0.1305858044189298}, 'ETH': {'DASH': 0.06459627506298854, 'XEM': 0.16795613694062173, 'DOGE': 0.04381871552036941, 'LTC': 0.0070895267193696165, 'MAID': 0.43029061022937815, 'BTC': 0.06821288200609678, 'ETH': 1.804694928609396, 'XRP': 0.07518882101080702, 'XMR': 0.303144422512906}, 'XRP': {'DASH': 0.0485013905397046, 'XEM': 0.20794892099014484, 'DOGE': 0.22505760423798415, 'LTC': 0.04038308119562005, 'MAID': 0.060012979312135256, 'BTC': 0.046123059468558665, 'ETH': 0.07518882101080702, 'XRP': 0.6588605216541893, 'XMR': 0.14127690730888484}, 'XMR': {'DASH': 0.07136735462548086, 'XEM': 0.24474433942952412, 'DOGE': 0.10013128092001486, 'LTC': 0.10439499530126177, 'MAID': 0.25174166270626736, 'BTC': 0.1305858044189298, 'ETH': 0.303144422512906, 'XRP': 0.14127690730888484, 'XMR': 2.1356359326142704}} + +portfolio = [1.0 / len(coins)] * len(coins) + +def score(dist): + s = 0 + for i1, c1 in enumerate(coins): + # s += dist[i1] * returns[i1] / 20 + if dist[i1] < 0: + s -= 999999 + for i2, c2 in enumerate(coins): + s -= dist[i1] * dist[i2] * cov[c1][c2] + return s + +p = portfolio +orig_score = score(p) + +for i in range(1000): + for j in range(len(coins)): + pup = [(p[k] if k != j else p[k] + 0.01) - 0.01 / len(coins) for k in range(len(p))] + scoreup = score(pup) + pdown = [(p[k] if k != j else p[k] - 0.01) + 0.01 / len(coins) for k in range(len(p))] + scoredown = score(pdown) + if scoreup > orig_score > scoredown: + p = pup + orig_score = score(p) + elif scoredown > orig_score > scoreup: + p = pdown + orig_score = score(p) + print p, orig_score diff --git a/price_grabber.py b/price_grabber.py new file mode 100644 index 0000000..b2d9ad4 --- /dev/null +++ b/price_grabber.py @@ -0,0 +1,54 @@ +import datetime, json, random +try: + from urllib.request import build_opener +except: + from urllib2 import build_opener + +# Makes a request to a given URL (first arg) and optional params (second arg) +def make_request(*args): + opener = build_opener() + opener.addheaders = [('User-agent', + 'Mozilla/5.0'+str(random.randrange(1000000)))] + try: + return opener.open(*args).read().strip() + except Exception as e: + try: + p = e.read().strip() + except: + p = e + raise Exception(p) + +coins = ['BTC', 'ETH', 'LTC', 'XMR', 'XRP', 'DASH', 'MAID', 'XEM', 'DOGE'] +altcoins = coins[1:] + +now = int(datetime.datetime.now().strftime('%s')) +prices = [] +coinstring = ','.join(altcoins) +for t in range(now, now - 86400 * 365, -86400): + btcprice = json.loads(make_request('https://min-api.cryptocompare.com/data/pricehistorical?fsym=BTC&tsyms=USD&ts=%d' % t))["BTC"]["USD"] + cryptoprices = json.loads(make_request('https://min-api.cryptocompare.com/data/pricehistorical?fsym=BTC&tsyms=%s&ts=%d' % (coinstring, t)))["BTC"] + pricelist = {'BTC': btcprice, 'timestamp': t} + for p in altcoins: + pricelist[p] = btcprice * 1.0 / cryptoprices[p] + print pricelist + prices.append(pricelist) + +print 'all prices', prices + +cov = {} +for k1 in coins: + cov[k1] = {} + for k2 in coins: + cov[k1][k2] = 0 +for i in range(len(prices) - 1): + for k1 in coins: + pricechangek1 = prices[i + 1][k1] / prices[i][k1] - 1 + for k2 in coins: + pricechangek2 = prices[i + 1][k2] / prices[i][k2] - 1 + cov[k1][k2] += pricechangek1 * pricechangek2 +print 'raw cov', cov +var = {c: cov[c][c] for c in coins} +for k1 in coins: + for k2 in coins: + cov[k1][k2] /= (var[k1] ** 0.5 * var[k2] ** 0.5) +print 'correlation', cov diff --git a/random_circuit.py b/random_circuit.py new file mode 100644 index 0000000..ce306e4 --- /dev/null +++ b/random_circuit.py @@ -0,0 +1,20 @@ +import random + +modulus = 97 + +def mkrandom(width, length): + o = [] + for i in range(length): + o.append((random.randrange(width), random.randrange(width), + random.randrange(width))) + return o + +def eval(inp, prog): + o = [x for x in inp] + for p in prog: + out, mul1, mul2 = p + o[out] = (o[out] + mul1 * mul2) % modulus + return o + +def mkinp(width): + return [random.randrange(modulus) for i in range(width)]