Changed starlark code to match 0.64.2

This commit is contained in:
Alberto Soutullo 2023-01-19 19:26:34 +01:00
parent 1f714e9670
commit f2d76bed03
9 changed files with 107 additions and 97 deletions

View File

@ -10,28 +10,30 @@ wsl = import_module(system_variables.WSL_MODULE)
nodes = import_module(system_variables.NODE_BUILDERS_MODULE) nodes = import_module(system_variables.NODE_BUILDERS_MODULE)
def run(args): def run(plan, args):
# Load global config file # Load global config file
config_file = args_parser.get_configuration_file_name(args) config_file = args_parser.get_configuration_file_name(plan, args)
config_json = read_file(src=config_file) config_json = read_file(src=config_file)
config = json.decode(config_json) config = json.decode(config_json)
same_toml_configuration = config['same_toml_configuration'] kurtosis_config = config['kurtosis']
wsl_config = config['wsl']
same_toml_configuration = kurtosis_config['same_toml_configuration']
# Load network topology # Load network topology
waku_topology_json = read_file(src=system_variables.TOPOLOGIES_LOCATION + 'network_data.json') waku_topology_json = read_file(src=system_variables.TOPOLOGIES_LOCATION + 'network_data.json')
waku_topology_json = read_file(src="github.com/logos-co/wakurtosis/" + config['topology_path'] + 'network_data.json')
waku_topology = json.decode(waku_topology_json) waku_topology = json.decode(waku_topology_json)
# Set up nodes # Set up nodes
services = nodes.instantiate_services(waku_topology, same_toml_configuration) services = nodes.instantiate_services(plan, waku_topology, same_toml_configuration)
# Set up prometheus + graphana # Set up prometheus + graphana
prometheus_service = prometheus.set_up_prometheus(services) prometheus_service = prometheus.set_up_prometheus(plan, services)
grafana_service = grafana.set_up_graphana(prometheus_service) grafana_service = grafana.set_up_grafana(plan, prometheus_service)
waku.interconnect_waku_nodes(waku_topology, services) waku.interconnect_waku_nodes(plan, waku_topology, services)
# # Setup WSL & Start the Simulation # # Setup WSL & Start the Simulation
wsl_service = wsl.init(waku_services, config['simulation_time'], config['message_rate'], config['min_packet_size'], config['max_packet_size'], config['inter_msg_type'], config['dist_type'], config['emitters_fraction']) wsl_service = wsl.init(plan, services, wsl_config)

View File

@ -1,14 +1,14 @@
# System Imports # System Imports
system_variables = import_module("github.com/logos-co/wakurtosis/src/system_variables.star") system_variables = import_module("github.com/logos-co/wakurtosis/src/system_variables.star")
def get_configuration_file_name(input_args): def get_configuration_file_name(plan, input_args):
# Parse command line argument (config file) # Parse command line argument (config file)
config_file = system_variables.DEFAULT_CONFIG_FILE config_file = system_variables.DEFAULT_CONFIG_FILE
if hasattr(input_args, "config_file"): if hasattr(input_args, "config_file"):
config_file = input_args.config_file config_file = input_args.config_file
print("Got config file: %s" %config_file) plan.print("Got config file: %s" %config_file)
else: else:
print("Got default config file: %s" %config_file) plan.print("Got default config file: %s" %config_file)
return config_file return config_file

View File

@ -2,21 +2,21 @@
system_variables = import_module("github.com/logos-co/wakurtosis/src/system_variables.star") system_variables = import_module("github.com/logos-co/wakurtosis/src/system_variables.star")
def get_toml_configuration_artifact(wakunode_name, same_toml_configuration, artifact_id=""): def get_toml_configuration_artifact(plan, wakunode_name, same_toml_configuration, name):
if same_toml_configuration: if same_toml_configuration:
artifact_id = upload_files( artifact_id = plan.upload_files(
src=system_variables.GENERAL_TOML_CONFIGURATION_PATH, src=system_variables.GENERAL_TOML_CONFIGURATION_PATH,
artifact_id=artifact_id name=name
) )
file_name = system_variables.GENERAL_TOML_CONFIGURATION_NAME file_name = system_variables.GENERAL_TOML_CONFIGURATION_NAME
else: else:
artifact_id = upload_files( artifact_id = plan.upload_files(
src=system_variables.WAKU_CONFIGURATION_FILES_LOCATION + src=system_variables.NODE_CONFIG_FILE_LOCATION +
wakunode_name + wakunode_name +
system_variables.WAKU_CONFIGURATION_FILE_EXTENSION, system_variables.NODE_CONFIGURATION_FILE_EXTENSION,
artifact_id=artifact_id name=name
) )
file_name = wakunode_name + system_variables.WAKU_CONFIGURATION_FILE_EXTENSION file_name = wakunode_name + system_variables.NODE_CONFIGURATION_FILE_EXTENSION
return artifact_id, file_name return artifact_id, file_name
@ -35,6 +35,7 @@ def generate_template_node_targets(services, port_id):
return template_data return template_data
def generate_template_prometheus_url(prometheus_service): def generate_template_prometheus_url(prometheus_service):
prometheus_url = prometheus_service.ip_address + ":" + str( prometheus_url = prometheus_service.ip_address + ":" + str(
prometheus_service.ports[system_variables.PROMETHEUS_PORT_ID].number) prometheus_service.ports[system_variables.PROMETHEUS_PORT_ID].number)
@ -43,18 +44,19 @@ def generate_template_prometheus_url(prometheus_service):
return prometheus_info return prometheus_info
def prepare_artifact_files_grafana(artifact_config_id="", artifact_custom_id="", artifact_dashboard_id=""): def prepare_artifact_files_grafana(plan, artifact_config_name, artifact_customization_name,
config_id = upload_files( artifact_dashboard_name):
config_id = plan.upload_files(
src=system_variables.GRAFANA_CONFIGURATION_PATH, src=system_variables.GRAFANA_CONFIGURATION_PATH,
artifact_id=artifact_config_id name=artifact_config_name
) )
customization_id = upload_files( customization_id = plan.upload_files(
src=system_variables.GRAFANA_CUSTOMIZATION_PATH, src=system_variables.GRAFANA_CUSTOMIZATION_PATH,
artifact_id=artifact_custom_id name=artifact_customization_name
) )
dashboard_id = upload_files( dashboard_id = plan.upload_files(
src=system_variables.GRAFANA_DASHBOARD_PATH, src=system_variables.GRAFANA_DASHBOARD_PATH,
artifact_id=artifact_dashboard_id name=artifact_dashboard_name
) )
return config_id, customization_id, dashboard_id return config_id, customization_id, dashboard_id

View File

@ -6,21 +6,23 @@ files = import_module(system_variables.FILE_HELPERS_MODULE)
templates = import_module(system_variables.TEMPLATES_MODULE) templates = import_module(system_variables.TEMPLATES_MODULE)
def set_up_grafana(prometheus_service): def set_up_grafana(plan, prometheus_service):
config_id, customization_id, dashboard_id = files.prepare_artifact_files_grafana() config_id, customization_id, dashboard_id = files.prepare_artifact_files_grafana(plan,
"grafana_config", "grafana_customization", "grafana_dashboard")
prometheus_data = files.generate_template_prometheus_url(prometheus_service) prometheus_data = files.generate_template_prometheus_url(prometheus_service)
prometheus_template = templates.get_prometheus_template_content_for_grafana() prometheus_template = templates.get_prometheus_template_content_for_grafana()
artifact_id = render_templates( artifact_id = plan.render_templates(
config={ config={
system_variables.CONTAINER_DATASOURCES_FILE_NAME_GRAFANA: struct( system_variables.CONTAINER_DATASOURCES_FILE_NAME_GRAFANA: struct(
template=prometheus_template, template=prometheus_template,
data=prometheus_data, data=prometheus_data,
) )
} },
name="grafana_target"
) )
grafana_service = add_service( grafana_service = plan.add_service(
service_id=system_variables.GRAFANA_SERVICE_ID, service_id=system_variables.GRAFANA_SERVICE_ID,
config=struct( config=struct(
image=system_variables.GRAFANA_IMAGE, image=system_variables.GRAFANA_IMAGE,

View File

@ -6,16 +6,17 @@ files = import_module(system_variables.FILE_HELPERS_MODULE)
templates = import_module(system_variables.TEMPLATES_MODULE) templates = import_module(system_variables.TEMPLATES_MODULE)
def set_up_prometheus(services): def set_up_prometheus(plan, services):
# Create targets.json # Create targets.json
targets_artifact_id = create_prometheus_targets(services) targets_artifact_id = create_prometheus_targets(plan, services)
# Set up prometheus # Set up prometheus
artifact_id = upload_files( artifact_id = plan.upload_files(
src=system_variables.PROMETHEUS_CONFIGURATION_PATH src=system_variables.PROMETHEUS_CONFIGURATION_PATH,
name="prometheus_config"
) )
prometheus_service = add_service( prometheus_service = plan.add_service(
service_id=system_variables.PROMETHEUS_SERVICE_ID, service_id=system_variables.PROMETHEUS_SERVICE_ID,
config=struct( config=struct(
image=system_variables.PROMETHEUS_IMAGE, image=system_variables.PROMETHEUS_IMAGE,
@ -37,20 +38,21 @@ def set_up_prometheus(services):
return prometheus_service return prometheus_service
def create_prometheus_targets(services): def create_prometheus_targets(plan, services):
# get ip and ports of all nodes # get ip and ports of all nodes
template_data = files.generate_template_node_targets(services, template_data = files.generate_template_node_targets(services,
system_variables.PROMETHEUS_PORT_ID) system_variables.PROMETHEUS_PORT_ID)
template = templates.get_prometheus_template() template = templates.get_prometheus_template()
artifact_id = render_templates( artifact_id = plan.render_templates(
config={ config={
system_variables.CONTAINER_TARGETS_FILE_NAME_PROMETHEUS: struct( system_variables.CONTAINER_TARGETS_FILE_NAME_PROMETHEUS: struct(
template=template, template=template,
data=template_data, data=template_data,
) )
} },
name="prometheus_targets"
) )
return artifact_id return artifact_id

View File

@ -6,13 +6,14 @@ waku = import_module(system_variables.WAKU_MODULE)
files = import_module(system_variables.FILE_HELPERS_MODULE) files = import_module(system_variables.FILE_HELPERS_MODULE)
def add_nwaku_service(nwakunode_name, use_general_configuration): def add_nwaku_service(plan, nwakunode_name, use_general_configuration):
artifact_id, configuration_file = files.get_toml_configuration_artifact(nwakunode_name, artifact_id, configuration_file = files.get_toml_configuration_artifact(plan, nwakunode_name,
use_general_configuration) use_general_configuration,
nwakunode_name)
print("Configuration being used file is " + configuration_file) plan.print("Configuration being used file is " + configuration_file)
nwaku_service = add_service( nwaku_service = plan.add_service(
service_id=nwakunode_name, service_id=nwakunode_name,
config=struct( config=struct(
image=system_variables.NWAKU_IMAGE, image=system_variables.NWAKU_IMAGE,
@ -27,11 +28,11 @@ def add_nwaku_service(nwakunode_name, use_general_configuration):
transport_protocol="TCP"), transport_protocol="TCP"),
}, },
files={ files={
system_variables.NODE_CONFIG_FILE_LOCATION: artifact_id system_variables.CONTAINER_NODE_CONFIG_FILE_LOCATION: artifact_id
}, },
entrypoint=system_variables.NWAKU_ENTRYPOINT, entrypoint=system_variables.NWAKU_ENTRYPOINT,
cmd=[ cmd=[
"--config-file=" + system_variables.NODE_CONFIG_FILE_LOCATION + "/" + configuration_file "--config-file=" + system_variables.CONTAINER_NODE_CONFIG_FILE_LOCATION + "/" + configuration_file
] ]
) )
) )
@ -39,14 +40,15 @@ def add_nwaku_service(nwakunode_name, use_general_configuration):
return nwaku_service return nwaku_service
def add_gowaku_service(gowakunode_name, use_general_configuration): def add_gowaku_service(plan, gowakunode_name, use_general_configuration):
artifact_id, configuration_file = files.get_toml_configuration_artifact(gowakunode_name, artifact_id, configuration_file = files.get_toml_configuration_artifact(plan, gowakunode_name,
use_general_configuration) use_general_configuration,
gowakunode_name)
print("Configuration being used file is " + configuration_file) plan.print("Configuration being used file is " + configuration_file)
print("Entrypoint is "+ str(system_variables.GOWAKU_ENTRYPOINT)) plan.print("Entrypoint is "+ str(system_variables.GOWAKU_ENTRYPOINT))
gowaku_service = add_service( gowaku_service = plan.add_service(
service_id=gowakunode_name, service_id=gowakunode_name,
config=struct( config=struct(
image=system_variables.GOWAKU_IMAGE, image=system_variables.GOWAKU_IMAGE,
@ -61,25 +63,23 @@ def add_gowaku_service(gowakunode_name, use_general_configuration):
transport_protocol="TCP"), transport_protocol="TCP"),
}, },
files={ files={
system_variables.NODE_CONFIG_FILE_LOCATION: artifact_id system_variables.CONTAINER_NODE_CONFIG_FILE_LOCATION: artifact_id
}, },
entrypoint=system_variables.GOWAKU_ENTRYPOINT, entrypoint=system_variables.GOWAKU_ENTRYPOINT,
cmd=[ cmd=[
"--config-file=" + system_variables.NODE_CONFIG_FILE_LOCATION + "/" + configuration_file "--config-file=" + system_variables.CONTAINER_NODE_CONFIG_FILE_LOCATION + "/" + configuration_file
] ]
) )
) )
return gowaku_service return gowaku_service
def add_jswaku_service(test, test2):
print("jswaku")
def add_nomos_service(test, test2): def add_nomos_service(plan, test, test2):
print("nomos") plan.print("nomos")
def instantiate_services(network_topology, use_general_configuration): def instantiate_services(plan, network_topology, use_general_configuration):
""" """
As we will need to access for the service information later, the structure is the following: As we will need to access for the service information later, the structure is the following:
@ -111,18 +111,18 @@ def instantiate_services(network_topology, use_general_configuration):
service_builder, information_builder = service_dispatcher[image] service_builder, information_builder = service_dispatcher[image]
service_information = service_builder(service_id, use_general_configuration) service_information = service_builder(plan, service_id, use_general_configuration)
information_builder(services_information, service_id, service_information) information_builder(plan, services_information, service_id, service_information)
return services_information return services_information
def _add_waku_service_information(services_information, new_service_id, service_information): def _add_waku_service_information(plan, services_information, new_service_id, service_information):
new_service_information = {} new_service_information = {}
wakunode_peer_id = waku.get_wakunode_peer_id(new_service_id, system_variables.WAKU_RPC_PORT_ID) wakunode_peer_id = waku.get_wakunode_peer_id(plan, new_service_id, system_variables.WAKU_RPC_PORT_ID)
new_service_information["peer_id"] = wakunode_peer_id new_service_information["peer_id"] = wakunode_peer_id
new_service_information["service_info"] = service_information new_service_information["service_info"] = service_information
@ -133,6 +133,5 @@ def _add_waku_service_information(services_information, new_service_id, service_
service_dispatcher = { service_dispatcher = {
"go-waku": (add_gowaku_service, _add_waku_service_information), "go-waku": (add_gowaku_service, _add_waku_service_information),
"nim-waku": (add_nwaku_service, _add_waku_service_information), "nim-waku": (add_nwaku_service, _add_waku_service_information),
"js-waku": (add_jswaku_service, _add_waku_service_information),
"nomos": (add_nomos_service, "test") "nomos": (add_nomos_service, "test")
} }

View File

@ -8,7 +8,11 @@ WAKU_LIBP2P_PORT_ID = "libp2p"
WAKU_LIBP2P_PORT = 60000 WAKU_LIBP2P_PORT = 60000
WAKU_SETUP_WAIT_TIME = "5" WAKU_SETUP_WAIT_TIME = "5"
NODE_CONFIG_FILE_LOCATION = "/node/configuration_file"
NODE_CONFIG_FILE_LOCATION = "github.com/logos-co/wakurtosis/config/node_config_files/"
CONTAINER_NODE_CONFIG_FILE_LOCATION = "/node/configuration_file/"
NODE_CONFIGURATION_FILE_EXTENSION = ".toml"
NWAKU_ENTRYPOINT = ["/usr/bin/wakunode", "--rpc-address=0.0.0.0", "--metrics-server-address=0.0.0.0"] NWAKU_ENTRYPOINT = ["/usr/bin/wakunode", "--rpc-address=0.0.0.0", "--metrics-server-address=0.0.0.0"]
GOWAKU_ENTRYPOINT = ["/usr/bin/waku", "--rpc-address=0.0.0.0", "--metrics-server-address=0.0.0.0"] GOWAKU_ENTRYPOINT = ["/usr/bin/waku", "--rpc-address=0.0.0.0", "--metrics-server-address=0.0.0.0"]

View File

@ -5,7 +5,7 @@ system_variables = import_module("github.com/logos-co/wakurtosis/src/system_vari
files = import_module(system_variables.FILE_HELPERS_MODULE) files = import_module(system_variables.FILE_HELPERS_MODULE)
def send_json_rpc(service_id, port_id, method, params, extract={}): def send_json_rpc(plan, service_id, port_id, method, params, extract={}):
recipe = struct( recipe = struct(
service_id=service_id, service_id=service_id,
port_id=port_id, port_id=port_id,
@ -16,7 +16,7 @@ def send_json_rpc(service_id, port_id, method, params, extract={}):
extract=extract extract=extract
) )
response = wait(recipe=recipe, response = plan.wait(recipe=recipe,
field="code", field="code",
assertion="==", assertion="==",
target_value=200) target_value=200)
@ -24,10 +24,10 @@ def send_json_rpc(service_id, port_id, method, params, extract={}):
return response return response
def get_wakunode_peer_id(service_id, port_id): def get_wakunode_peer_id(plan, service_id, port_id):
extract = {"peer_id": '.result.listenAddresses | .[0] | split("/") | .[-1]'} extract = {"peer_id": '.result.listenAddresses | .[0] | split("/") | .[-1]'}
response = send_json_rpc(service_id, port_id, response = send_json_rpc(plan, service_id, port_id,
system_variables.GET_WAKU_INFO_METHOD, "", extract) system_variables.GET_WAKU_INFO_METHOD, "", extract)
return response["extract.peer_id"] return response["extract.peer_id"]
@ -47,23 +47,23 @@ def _merge_peer_ids(peer_ids):
return "[" + ",".join(peer_ids) + "]" return "[" + ",".join(peer_ids) + "]"
def connect_wakunode_to_peers(service_id, port_id, peer_ids): def connect_wakunode_to_peers(plan, service_id, port_id, peer_ids):
method = system_variables.CONNECT_TO_PEER_METHOD method = system_variables.CONNECT_TO_PEER_METHOD
params = _merge_peer_ids(peer_ids) params = _merge_peer_ids(peer_ids)
response = send_json_rpc(service_id, port_id, method, params) response = send_json_rpc(plan, service_id, port_id, method, params)
print(response) plan.print(response)
def post_waku_v2_relay_v1_message(service_id, topic): def post_waku_v2_relay_v1_message(plan, service_id, topic):
waku_message = '{"payload": "0x1a2b3c4d5e6f", "timestamp": 1626813243}' waku_message = '{"payload": "0x1a2b3c4d5e6f", "timestamp": 1626813243}'
params = '"' + topic + '"' + ", " + waku_message params = '"' + topic + '"' + ", " + waku_message
response = send_json_rpc(service_id, system_variables.WAKU_RPC_PORT_ID, response = send_json_rpc(service_id, system_variables.WAKU_RPC_PORT_ID,
system_variables.POST_RELAY_MESSAGE, params) system_variables.POST_RELAY_MESSAGE, params)
print(response) plan.print(response)
def get_wakunode_id(service_id, port_id): def get_wakunode_id(service_id, port_id):
@ -75,19 +75,19 @@ def get_wakunode_id(service_id, port_id):
return response["extract.waku_id"] return response["extract.waku_id"]
def make_service_wait(service_id, time): def make_service_wait(plan,service_id, time):
exec_recipe = struct( exec_recipe = struct(
service_id=service_id, service_id=service_id,
command=["sleep", time] command=["sleep", time]
) )
exec(exec_recipe) plan.exec(exec_recipe)
def get_waku_peers(waku_service_id): def get_waku_peers(plan, waku_service_id):
response = send_json_rpc(waku_service_id, system_variables.WAKU_RPC_PORT_ID, response = send_json_rpc(plan, waku_service_id, system_variables.WAKU_RPC_PORT_ID,
system_variables.GET_PEERS_METHOD, "") system_variables.GET_PEERS_METHOD, "")
print(response) plan.print(response)
return response return response
@ -100,13 +100,13 @@ def send_test_messages(topology_information, number_of_messages, time_between_me
post_waku_v2_relay_v1_message(wakunode_name, "test") post_waku_v2_relay_v1_message(wakunode_name, "test")
def interconnect_waku_nodes(topology_information, services): def interconnect_waku_nodes(plan, topology_information, services):
# Interconnect them # Interconnect them
for waku_service_id in topology_information.keys(): for waku_service_id in topology_information.keys():
peers = topology_information[waku_service_id]["static_nodes"] peers = topology_information[waku_service_id]["static_nodes"]
peer_ids = [create_waku_id(services[peer]) for peer in peers] peer_ids = [create_waku_id(services[peer]) for peer in peers]
connect_wakunode_to_peers(waku_service_id, system_variables.WAKU_RPC_PORT_ID, peer_ids) connect_wakunode_to_peers(plan, waku_service_id, system_variables.WAKU_RPC_PORT_ID, peer_ids)

View File

@ -5,26 +5,24 @@ system_variables = import_module("github.com/logos-co/wakurtosis/src/system_vari
files = import_module(system_variables.FILE_HELPERS_MODULE) files = import_module(system_variables.FILE_HELPERS_MODULE)
templates = import_module(system_variables.TEMPLATES_MODULE) templates = import_module(system_variables.TEMPLATES_MODULE)
def create_config(simulation_time, message_rate, min_packet_size, max_packet_size, inter_msg_type, dist_type, emitters_fraction): def create_config(plan, wls_config):
template_data = {"simulation_time": simulation_time, "message_rate" : message_rate, "min_packet_size" : min_packet_size,
"max_packet_size" : max_packet_size, "dist_type" : dist_type, "emitters_fraction" : emitters_fraction, "inter_msg_type" : inter_msg_type}
# Traffic simulation parameters # Traffic simulation parameters
wsl_yml_template = templates.get_wsl_template() wsl_yml_template = templates.get_wsl_template()
artifact_id = render_templates( artifact_id = plan.render_templates(
config={ config={
system_variables.CONTAINER_WSL_CONFIGURATION_FILE_NAME: struct( system_variables.CONTAINER_WSL_CONFIGURATION_FILE_NAME: struct(
template=wsl_yml_template, template=wsl_yml_template,
data=template_data, data=wls_config,
) )
} },
name="wsl_config"
) )
return artifact_id return artifact_id
def create_targets(services): def create_targets(plan, services):
# Get private ip and ports of all nodes # Get private ip and ports of all nodes
template_data = files.generate_template_node_targets(services, system_variables.WAKU_RPC_PORT_ID) template_data = files.generate_template_node_targets(services, system_variables.WAKU_RPC_PORT_ID)
@ -34,26 +32,27 @@ def create_targets(services):
{{.targets}} {{.targets}}
""" """
artifact_id = render_templates( artifact_id = plan.render_templates(
config={ config={
system_variables.CONTAINER_TARGETS_FILE_NAME_WSL: struct( system_variables.CONTAINER_TARGETS_FILE_NAME_WSL: struct(
template=template, template=template,
data=template_data, data=template_data,
) )
} },
name="wsl_targets"
) )
return artifact_id return artifact_id
def init(services, simulation_time, message_rate, min_packet_size, max_packet_size, inter_msg_type, dist_type, emitters_fraction): def init(plan, services, wsl_config):
# Generate simulation config # Generate simulation config
wsl_config = create_config(simulation_time, message_rate, min_packet_size, max_packet_size, inter_msg_type, dist_type, emitters_fraction) wsl_config = create_config(plan, wsl_config)
# Create targets.json # Create targets.json
wsl_targets = create_targets(services) wsl_targets = create_targets(plan, services)
wsl_service = add_service( wsl_service = plan.add_service(
service_id=system_variables.WSL_SERVICE_ID, service_id=system_variables.WSL_SERVICE_ID,
config=struct( config=struct(
image=system_variables.WSL_IMAGE, image=system_variables.WSL_IMAGE,