From 6e96757d103f451079cc3e0f1d47dc9268c8d0d4 Mon Sep 17 00:00:00 2001 From: 0xFugue <119708655+0xFugue@users.noreply.github.com> Date: Thu, 29 Dec 2022 21:34:01 +0530 Subject: [PATCH] typer, json/toml, more network/node type, refactor --- Readme.md | 49 ++------- batch_gen.sh | 47 +++++++++ generate_network.py | 241 ++++++++++++++++++++++++-------------------- 3 files changed, 186 insertions(+), 151 deletions(-) create mode 100755 batch_gen.sh diff --git a/Readme.md b/Readme.md index a1de36e..2e90cc3 100644 --- a/Readme.md +++ b/Readme.md @@ -1,45 +1,12 @@ -This repo contains the scripts to generate different network models for wakukurtosis runs. +This repo contains scripts to generate network models (in JSON) and waku configuration files (in TOMLs) for wakukurtosis runs. -## run_kurtosis_tests.sh -run_kurtosis_tests.sh will run kurtosis on a set of json files in a directory. It requires two arguments. First is a directory containing json files; other file types in the directory are ignored. Second is the github root/prefix of the kurtosis module you run the tests under.
- -> usage: ./run_kurtosis_tests.sh
- -Running this script is somewhat complicated; so follow the following instructions to a dot. You **WILL** require the main.star provided here. The main.star reads a input json and instantiates Waku nodes accordingly. The runs are repeated for each of the input json files under the specified directory. - -#### step 0) - symlink run_kurtosis_tests.sh to the root directory of your kurtosis module.
-#### step 1) - backup the your kurtosis module's own main.star. copy the main.star provided here to the root directory of your kurtosis module.
- !!! WARNING: symlinking the main.star will NOT work !!!
-#### step 3) - put all the json files you want to use in a directory. Call it *Foo*
-#### step 3) - copy the *Foo* directory to the root of your kurtosis module
- !!! WARNING: symlinking the directory will NOT work !!!
-#### step 4) - run this script in the root directory of the kurtosis module. provide the directory (*Foo*) and the github root/prefix of the kurtosis module as arguments to the script
- - -## gen_jsons.sh -gen_jsons.sh can generate given number of Waku networs and outputs them to a directory. Please make sure that the output directory exists; both relative and absolute paths work. The Wakunode parameters are generated at random; edit the MIN and MAX for finer control. The script requires bc & /dev/urandom.
- -> usage: ./gen_jsons.sh <#json files needed>
## generate_network.py -generate_network.py can generate networks with specified number of nodes and topics. the network types currently supported is "configuration_model" and more are on the way. Use with Python3. Comment out the `#draw(fname, H)` line to visualise the generated graph. +generate_network.py generates one network and per-node configuration files. The tool is configurable with specified number of nodes, topics, network types, node types. Use with Python3. Comment out the `#draw(fname, H)` line to visualise the generated graph. -> usage: generate_network [-h] [-o ] [-n <#nodes>] [-t <#topics>] - [-T ]
->>
->> Generates and outputs the Waku network conforming to input parameters ->>
->> optional arguments:
->>   -h, --help show this help message and exit
->>   -o , --output output json filename for the Waku network
->>   -n <#nodes>, --numnodes <#nodes> number of nodes in the Waku network
->>   -t <#topics>, --numtopics <#topics> number of topics in the Waku network
->>   -T , --type network type for the Waku network
->>   -p <#partitions>, --numparts <#partitions> number of partitions in the Waku network
- >>
->>The defaults are: -o "Topology.json"; -n 1; -t 1; -p 1; -T "configuration_model"
+> usage: $./generate_network --help + +## batch_gen.sh +batch_gen.sh can generate given number of Waku networks and outputs them to a directory. Please make sure that the output directory does not exists; both relative and absolute paths work. The Wakunode parameters are generated at random; edit the MIN and MAX for finer control. The script requires bc & /dev/urandom.
+ +> usage: $./batch_gen.sh <#number of networks needed>
diff --git a/batch_gen.sh b/batch_gen.sh new file mode 100755 index 0000000..35e44f9 --- /dev/null +++ b/batch_gen.sh @@ -0,0 +1,47 @@ +#!/bin/sh + +#MAX and MIN for topics and num nodes +MIN=5 +MAX=100 + +#requires bc +getrand(){ + orig=$(od -An -N1 -i /dev/urandom) + range=`echo "$MIN + ($orig % ($MAX - $MIN + 1))" | bc` + RANDOM=$range +} + +getrand1(){ + orig=$(od -An -N1 -i /dev/urandom) + range=`echo "$MIN + ($orig % ($MAX - $MIN + 1))" | bc` + return range + #getrand1 # call the fun and use the return value + #n=$? +} + +if [ "$#" -ne 2 ] || [ $2 -le 0 ] ; then + echo "usage: $0 <#json files needed>" >&2 + exit 1 +fi + +path=$1 +nfiles=$2 +mkdir -p $path + +echo "Ok, will generate $nfiles networks & put them under '$path'." + +nwtype="NEWMANWATTSSTROGATZ" +nodetype="DESKTOP" + + +for i in $(seq $nfiles) +do + getrand + n=$((RANDOM+1)) + getrand + t=$((RANDOM+1)) + dirname="$path/$i/Waku" + mkdir "$path/$i" + echo "Generating ./generate_network.py --dirname $dirname --num-nodes $n --num-topics $t --nw-type $nwtype --node-type $nodetype --num-partitions 1" + $(./generate_network.py --dirname $dirname --num-nodes $n --num-topics $t --nw-type $nwtype --node-type $nodetype --num-partitions 1) +done diff --git a/generate_network.py b/generate_network.py index 67a0896..6c52184 100755 --- a/generate_network.py +++ b/generate_network.py @@ -4,79 +4,68 @@ import matplotlib.pyplot as plt import networkx as nx import random, math import json -import argparse, os, sys +import sys, os +import string +import typer +from enum import Enum + +# Consts +class nw_types(Enum): + configmodel = "CONFIGMODEL" + scalefree = "SCALEFREE" # power law + newmanwattsstrogatz = "NEWMANWATTSSTROGATZ" # mesh, smallworld + barbell = "BARBELL" # partition + balancedtree = "BALANCEDTREE" # committees? + star = "STAR" # spof + +class node_types(Enum): + desktop = "DESKTOP" + mobile = "MOBILE" + +nw_fname = "network_data.json" +prefix = "waku_" + +### I/O related fns ########################################################### # Dump to a json file -def write_json(filename, data_2_dump): - json.dump(data_2_dump, open(filename,'w'), indent=2) +def write_json(dirname, json_dump): + fname = os.path.join(dirname, nw_fname) + json.dump(json_dump, open(fname,'w'), indent=2) -# has trouble with non-integer/non-hashable keys -def read_json(filename): - with open(filename) as f: - jdata = json.load(f) - return nx.node_link_graph(jdata) +def write_toml(dirname, node_name, toml): + fname = os.path.join(dirname, node_name+ ".toml") + f = open(fname, 'w') + f.write(toml) + f.close() # Draw the network and output the image to a file -def draw(fname, H): +def draw(dirname, H): nx.draw(H, pos=nx.kamada_kawai_layout(H), with_labels=True) + fname = os.path.join(dirname, nw_fname) plt.savefig(os.path.splitext(fname)[0] + ".png", format="png") plt.show() -# Initialize parser, set the defaults, and extract the options -def get_options(): - parser = argparse.ArgumentParser( - prog = 'generate_network', - description = '''Generates and outputs - the Waku network conforming to input parameters''', - epilog = '''Defaults: -o "Topology.json"; - -n 1; -t 1; -p 1; -T "configuration_model" - Supported nw types "configuration_model", "scalefree", - "newman_watts_strogatz"''') - parser.add_argument("-o", "--output", - default='Topology.json', dest='fname', - help='output json filename for the Waku network', - type=str, metavar='') - parser.add_argument("-n", "--numnodes", - default=1, dest='num_nodes', - help='number of nodes in the Waku network', - type=int, metavar='<#nodes>') - parser.add_argument("-t", "--numtopics", - default=1, dest='num_topics', - help='number of topics in the Waku network', - type=int, metavar='<#topics>') - parser.add_argument("-T", "--type", - default="configuration_model", dest='nw_type', - help='network type of the Waku network', - type=str, metavar='') - parser.add_argument("-p", "--numparts", - default=1, dest='num_partitions', - help='The number of partitions in the Waku network', - type=int, metavar='<#partitions>') -# parser.add_argument("-e", "--numedges", -# default=1, dest='num_edges', -# help='The number of edges in the Waku network', -# type=int, metavar='#edges>') - return parser.parse_args() +# Has trouble with non-integer/non-hashable keys +def read_json(fname): + with open(fname) as f: + jdata = json.load(f) + return nx.node_link_graph(jdata) -# Generate a random string (UC chars) of len n -def generate_topic_string(n): - rs = "" - for _ in range(n): - r = random.randint(65, 65 + 26 - 1) # generate a random UC char - rs += (chr(r)) # append the char generated - return rs +### topics related fns ########################################################### + +# Generate a random string of upper case chars +def generate_random_string(n): + return "".join(random.choice(string.ascii_uppercase) for _ in range(n)) -# Generate the topics - UC chars prefixed by "topic" +# Generate the topics - topic followed by random UC chars - Eg, topic_XY" def generate_topics(num_topics): - topics = [] - base = 26 - topic_len = int(math.log(num_topics)/math.log(base)) + 1 - topics = {i: f"topic_{generate_topic_string(topic_len)}" for i in range(num_topics)} + topic_len = int(math.log(num_topics)/math.log(26)) + 1 # base is 26 - upper case letters + topics = {i: f"topic_{generate_random_string(topic_len)}" for i in range(num_topics)} return topics @@ -91,12 +80,13 @@ def get_random_sublist(topics): return sublist +### network processing related fns ########################################################### + # Network Types -# https://networkx.org/documentation/stable/reference/generated/networkx.generators.degree_seq.configuration_model.html def generate_config_model(n): #degrees = nx.random_powerlaw_tree_sequence(n, tries=10000) degrees = [random.randint(1, n) for i in range(n)] - if (sum(degrees)) % 2 != 0: # adjust the degree to even + if (sum(degrees)) % 2 != 0: # adjust the degree to be even degrees[-1] += 1 return nx.configuration_model(degrees) # generate the graph @@ -105,79 +95,110 @@ def generate_scalefree_graph(n): return nx.scale_free_graph(n) -# n must be larger than k +# n must be larger than k=D=3 def generate_newman_watts_strogatz_graph(n): - return nx.newman_watts_strogatz_graph(n, 12, 0.5) + return nx.newman_watts_strogatz_graph(n, 3, 0.5) + + +def generate_barbell_graph(n): + return nx.barbell_graph(int(n/2), 1) + + +def generate_balanced_tree(n, fanout=3): + height = int(math.log(n)/math.log(fanout)) + return nx.balanced_tree(fanout, height) + + +def generate_star_graph(n): + return nx.star_graph(n) # Generate the network from nw type -def generate_network(num_nodes, nw_type, prefix): +def generate_network(num_nodes, nw_type): G = nx.empty_graph() - if nw_type == "configuration_model": + if nw_type == nw_types.configmodel: G = generate_config_model(num_nodes) - elif nw_type == "scalefree": + elif nw_type == nw_types.scalefree: G = generate_scalefree_graph(num_nodes) - elif nw_type == "newman_watts_strogatz": + elif nw_type == nw_types.newmanwattsstrogatz: G = generate_newman_watts_strogatz_graph(num_nodes) + elif nw_type == nw_types.barbell: + G = generate_barbell_graph(num_nodes) + elif nw_type == nw_types.balancedtree: + G = generate_balanced_tree(num_nodes) + elif nw_type == nw_types.star: + G = generate_star_graph(num_nodes) else: print(nw_type +": Unsupported network type") sys.exit(1) - H = postprocess_network(G, prefix) - return H + return postprocess_network(G) -# used by generate_dump_data, *ought* to be global to handle partitions -ports_shifted = 0 -def postprocess_network(G, prefix): - G = nx.Graph(G) # prune out parallel/multi edges - G.remove_edges_from(nx.selfloop_edges(G)) # Removing self-loops - # Labeling nodes to match waku containers +# Label the generated network with prefix +def postprocess_network(G): + G = nx.Graph(G) # prune out parallel/multi edges + G.remove_edges_from(nx.selfloop_edges(G)) # remove the self-loops mapping = {i: f"{prefix}{i}" for i in range(len(G))} - return nx.relabel_nodes(G, mapping) + return nx.relabel_nodes(G, mapping) # label the nodes -# Generate dump data from the network and topics -def generate_dump_data(H, topics): - data_to_dump = {} - global ports_shifted - for node in H.nodes: - data_to_dump[node] = {} - data_to_dump[node]["ports-shift"] = ports_shifted - ports_shifted += 1 - data_to_dump[node]["topics"] = get_random_sublist(topics) - data_to_dump[node]["static-nodes"] = [] - for edge in H.edges(node): - data_to_dump[node]["static-nodes"].append(edge[1]) - return data_to_dump +### file format related fns ########################################################### + +#Generate per node toml configs +def generate_toml(topics, node_type=node_types.desktop): + topic_str = " ". join(get_random_sublist(topics)) # topics as a space separated string + if node_type == node_type.desktop: + toml = "rpc-admin = true\nkeep-alive = true\n" + elif node_type == node_type.mobile: + toml = "rpc-admin = true\nkeep-alive = true\n" + else: + print(node_type +": Unsupported node type") + sys.exit(1) + toml += f"topics = \"{topic_str}\"\n" + return toml -def main(): - #extract the CLI arguments and assign params - options = get_options() - fname = options.fname - num_nodes = options.num_nodes - num_topics = options.num_topics - nw_type = options.nw_type - prefix = "waku_" - num_partitions = options.num_partitions - #num_edges = options.num_edges ## need to control num_edges? - - if num_partitions > 1: - print("-p",num_partitions, - "Sorry, we do not yet support partitions") +# Generates network-wide json and per-node toml and writes them +def generate_and_write_files(dirname, num_topics, H): + if not os.path.exists(dirname): + os.mkdir(dirname) + elif not os.path.isfile(dirname) and os.listdir(dirname): + print(dirname +": exists and is not empty") + sys.exit(1) + elif os.path.isfile(dirname): + print(dirname +": exists and is not a directory") sys.exit(1) - # Generate the network and postprocess it - H = generate_network(num_nodes, nw_type, prefix) - # Generate the topics topics = generate_topics(num_topics) - # Generate the dump data - dump_data = generate_dump_data(H, topics) - # Dump the network in a json file - write_json(fname, dump_data) - # Display the graph - draw(fname, H) + json_dump = {} + for node in H.nodes: + write_toml(dirname, node, generate_toml(topics)) # per node toml + json_dump[node] = {} + json_dump[node]["static-nodes"] = [] + for edge in H.edges(node): + json_dump[node]["static-nodes"].append(edge[1]) + write_json(dirname, json_dump) # network wide json + + +### the main ########################################################### +def main( + dirname: str = "Waku", num_nodes: int = 3, num_topics: int = 1, + nw_type: nw_types = "NEWMANWATTSSTROGATZ", + node_type: node_types = "DESKTOP", + num_partitions: int = 1): + + if num_partitions > 1: + print("-p",num_partitions, "Sorry, we do not yet support partitions") + sys.exit(1) + + # Generate the network and do post-process + G = generate_network(num_nodes, nw_type) + postprocess_network(G) + + # Generate file format specific data structs and write the files; optionally, draw the network + generate_and_write_files(dirname, num_topics, G) + draw(dirname, G) if __name__ == "__main__": - main() + typer.run(main)