From f2d76bed03f524ef257c08dde901d8f9cc267cad Mon Sep 17 00:00:00 2001 From: Alberto Soutullo Date: Thu, 19 Jan 2023 19:26:34 +0100 Subject: [PATCH] Changed starlark code to match 0.64.2 --- main.star | 20 +++++++------- src/arguments_parser.star | 6 ++--- src/file_helpers.star | 34 ++++++++++++----------- src/monitoring/grafana.star | 12 +++++---- src/monitoring/prometheus.star | 18 +++++++------ src/node_builders.star | 49 +++++++++++++++++----------------- src/system_variables.star | 6 ++++- src/waku.star | 32 +++++++++++----------- src/wsl.star | 27 +++++++++---------- 9 files changed, 107 insertions(+), 97 deletions(-) diff --git a/main.star b/main.star index e2362de..d834b12 100644 --- a/main.star +++ b/main.star @@ -10,28 +10,30 @@ wsl = import_module(system_variables.WSL_MODULE) nodes = import_module(system_variables.NODE_BUILDERS_MODULE) -def run(args): +def run(plan, args): # Load global config file - config_file = args_parser.get_configuration_file_name(args) + config_file = args_parser.get_configuration_file_name(plan, args) config_json = read_file(src=config_file) config = json.decode(config_json) - same_toml_configuration = config['same_toml_configuration'] + kurtosis_config = config['kurtosis'] + wsl_config = config['wsl'] + + same_toml_configuration = kurtosis_config['same_toml_configuration'] # Load network topology waku_topology_json = read_file(src=system_variables.TOPOLOGIES_LOCATION + 'network_data.json') - waku_topology_json = read_file(src="github.com/logos-co/wakurtosis/" + config['topology_path'] + 'network_data.json') waku_topology = json.decode(waku_topology_json) # Set up nodes - services = nodes.instantiate_services(waku_topology, same_toml_configuration) + services = nodes.instantiate_services(plan, waku_topology, same_toml_configuration) # Set up prometheus + graphana - prometheus_service = prometheus.set_up_prometheus(services) - grafana_service = grafana.set_up_graphana(prometheus_service) + prometheus_service = prometheus.set_up_prometheus(plan, services) + grafana_service = grafana.set_up_grafana(plan, prometheus_service) - waku.interconnect_waku_nodes(waku_topology, services) + waku.interconnect_waku_nodes(plan, waku_topology, services) # # Setup WSL & Start the Simulation - wsl_service = wsl.init(waku_services, config['simulation_time'], config['message_rate'], config['min_packet_size'], config['max_packet_size'], config['inter_msg_type'], config['dist_type'], config['emitters_fraction']) + wsl_service = wsl.init(plan, services, wsl_config) diff --git a/src/arguments_parser.star b/src/arguments_parser.star index e5d86c3..f570d16 100644 --- a/src/arguments_parser.star +++ b/src/arguments_parser.star @@ -1,14 +1,14 @@ # System Imports system_variables = import_module("github.com/logos-co/wakurtosis/src/system_variables.star") -def get_configuration_file_name(input_args): +def get_configuration_file_name(plan, input_args): # Parse command line argument (config file) config_file = system_variables.DEFAULT_CONFIG_FILE if hasattr(input_args, "config_file"): config_file = input_args.config_file - print("Got config file: %s" %config_file) + plan.print("Got config file: %s" %config_file) else: - print("Got default config file: %s" %config_file) + plan.print("Got default config file: %s" %config_file) return config_file diff --git a/src/file_helpers.star b/src/file_helpers.star index be3df51..bdc050c 100644 --- a/src/file_helpers.star +++ b/src/file_helpers.star @@ -2,21 +2,21 @@ system_variables = import_module("github.com/logos-co/wakurtosis/src/system_variables.star") -def get_toml_configuration_artifact(wakunode_name, same_toml_configuration, artifact_id=""): +def get_toml_configuration_artifact(plan, wakunode_name, same_toml_configuration, name): if same_toml_configuration: - artifact_id = upload_files( + artifact_id = plan.upload_files( src=system_variables.GENERAL_TOML_CONFIGURATION_PATH, - artifact_id=artifact_id + name=name ) file_name = system_variables.GENERAL_TOML_CONFIGURATION_NAME else: - artifact_id = upload_files( - src=system_variables.WAKU_CONFIGURATION_FILES_LOCATION + + artifact_id = plan.upload_files( + src=system_variables.NODE_CONFIG_FILE_LOCATION + wakunode_name + - system_variables.WAKU_CONFIGURATION_FILE_EXTENSION, - artifact_id=artifact_id + system_variables.NODE_CONFIGURATION_FILE_EXTENSION, + name=name ) - file_name = wakunode_name + system_variables.WAKU_CONFIGURATION_FILE_EXTENSION + file_name = wakunode_name + system_variables.NODE_CONFIGURATION_FILE_EXTENSION return artifact_id, file_name @@ -35,6 +35,7 @@ def generate_template_node_targets(services, port_id): return template_data + def generate_template_prometheus_url(prometheus_service): prometheus_url = prometheus_service.ip_address + ":" + str( prometheus_service.ports[system_variables.PROMETHEUS_PORT_ID].number) @@ -43,18 +44,19 @@ def generate_template_prometheus_url(prometheus_service): return prometheus_info -def prepare_artifact_files_grafana(artifact_config_id="", artifact_custom_id="", artifact_dashboard_id=""): - config_id = upload_files( +def prepare_artifact_files_grafana(plan, artifact_config_name, artifact_customization_name, + artifact_dashboard_name): + config_id = plan.upload_files( src=system_variables.GRAFANA_CONFIGURATION_PATH, - artifact_id=artifact_config_id + name=artifact_config_name ) - customization_id = upload_files( + customization_id = plan.upload_files( src=system_variables.GRAFANA_CUSTOMIZATION_PATH, - artifact_id=artifact_custom_id + name=artifact_customization_name ) - dashboard_id = upload_files( + dashboard_id = plan.upload_files( src=system_variables.GRAFANA_DASHBOARD_PATH, - artifact_id=artifact_dashboard_id + name=artifact_dashboard_name ) - return config_id, customization_id, dashboard_id \ No newline at end of file + return config_id, customization_id, dashboard_id diff --git a/src/monitoring/grafana.star b/src/monitoring/grafana.star index cfe2a6b..1423962 100644 --- a/src/monitoring/grafana.star +++ b/src/monitoring/grafana.star @@ -6,21 +6,23 @@ files = import_module(system_variables.FILE_HELPERS_MODULE) templates = import_module(system_variables.TEMPLATES_MODULE) -def set_up_grafana(prometheus_service): - config_id, customization_id, dashboard_id = files.prepare_artifact_files_grafana() +def set_up_grafana(plan, prometheus_service): + config_id, customization_id, dashboard_id = files.prepare_artifact_files_grafana(plan, + "grafana_config", "grafana_customization", "grafana_dashboard") prometheus_data = files.generate_template_prometheus_url(prometheus_service) prometheus_template = templates.get_prometheus_template_content_for_grafana() - artifact_id = render_templates( + artifact_id = plan.render_templates( config={ system_variables.CONTAINER_DATASOURCES_FILE_NAME_GRAFANA: struct( template=prometheus_template, data=prometheus_data, ) - } + }, + name="grafana_target" ) - grafana_service = add_service( + grafana_service = plan.add_service( service_id=system_variables.GRAFANA_SERVICE_ID, config=struct( image=system_variables.GRAFANA_IMAGE, diff --git a/src/monitoring/prometheus.star b/src/monitoring/prometheus.star index beb3549..d8d285e 100644 --- a/src/monitoring/prometheus.star +++ b/src/monitoring/prometheus.star @@ -6,16 +6,17 @@ files = import_module(system_variables.FILE_HELPERS_MODULE) templates = import_module(system_variables.TEMPLATES_MODULE) -def set_up_prometheus(services): +def set_up_prometheus(plan, services): # Create targets.json - targets_artifact_id = create_prometheus_targets(services) + targets_artifact_id = create_prometheus_targets(plan, services) # Set up prometheus - artifact_id = upload_files( - src=system_variables.PROMETHEUS_CONFIGURATION_PATH + artifact_id = plan.upload_files( + src=system_variables.PROMETHEUS_CONFIGURATION_PATH, + name="prometheus_config" ) - prometheus_service = add_service( + prometheus_service = plan.add_service( service_id=system_variables.PROMETHEUS_SERVICE_ID, config=struct( image=system_variables.PROMETHEUS_IMAGE, @@ -37,20 +38,21 @@ def set_up_prometheus(services): return prometheus_service -def create_prometheus_targets(services): +def create_prometheus_targets(plan, services): # get ip and ports of all nodes template_data = files.generate_template_node_targets(services, system_variables.PROMETHEUS_PORT_ID) template = templates.get_prometheus_template() - artifact_id = render_templates( + artifact_id = plan.render_templates( config={ system_variables.CONTAINER_TARGETS_FILE_NAME_PROMETHEUS: struct( template=template, data=template_data, ) - } + }, + name="prometheus_targets" ) return artifact_id diff --git a/src/node_builders.star b/src/node_builders.star index 96dc857..f892e73 100644 --- a/src/node_builders.star +++ b/src/node_builders.star @@ -6,13 +6,14 @@ waku = import_module(system_variables.WAKU_MODULE) files = import_module(system_variables.FILE_HELPERS_MODULE) -def add_nwaku_service(nwakunode_name, use_general_configuration): - artifact_id, configuration_file = files.get_toml_configuration_artifact(nwakunode_name, - use_general_configuration) +def add_nwaku_service(plan, nwakunode_name, use_general_configuration): + artifact_id, configuration_file = files.get_toml_configuration_artifact(plan, nwakunode_name, + use_general_configuration, + nwakunode_name) - print("Configuration being used file is " + configuration_file) + plan.print("Configuration being used file is " + configuration_file) - nwaku_service = add_service( + nwaku_service = plan.add_service( service_id=nwakunode_name, config=struct( image=system_variables.NWAKU_IMAGE, @@ -27,11 +28,11 @@ def add_nwaku_service(nwakunode_name, use_general_configuration): transport_protocol="TCP"), }, files={ - system_variables.NODE_CONFIG_FILE_LOCATION: artifact_id + system_variables.CONTAINER_NODE_CONFIG_FILE_LOCATION: artifact_id }, entrypoint=system_variables.NWAKU_ENTRYPOINT, cmd=[ - "--config-file=" + system_variables.NODE_CONFIG_FILE_LOCATION + "/" + configuration_file + "--config-file=" + system_variables.CONTAINER_NODE_CONFIG_FILE_LOCATION + "/" + configuration_file ] ) ) @@ -39,14 +40,15 @@ def add_nwaku_service(nwakunode_name, use_general_configuration): return nwaku_service -def add_gowaku_service(gowakunode_name, use_general_configuration): - artifact_id, configuration_file = files.get_toml_configuration_artifact(gowakunode_name, - use_general_configuration) +def add_gowaku_service(plan, gowakunode_name, use_general_configuration): + artifact_id, configuration_file = files.get_toml_configuration_artifact(plan, gowakunode_name, + use_general_configuration, + gowakunode_name) - print("Configuration being used file is " + configuration_file) - print("Entrypoint is "+ str(system_variables.GOWAKU_ENTRYPOINT)) + plan.print("Configuration being used file is " + configuration_file) + plan.print("Entrypoint is "+ str(system_variables.GOWAKU_ENTRYPOINT)) - gowaku_service = add_service( + gowaku_service = plan.add_service( service_id=gowakunode_name, config=struct( image=system_variables.GOWAKU_IMAGE, @@ -61,25 +63,23 @@ def add_gowaku_service(gowakunode_name, use_general_configuration): transport_protocol="TCP"), }, files={ - system_variables.NODE_CONFIG_FILE_LOCATION: artifact_id + system_variables.CONTAINER_NODE_CONFIG_FILE_LOCATION: artifact_id }, entrypoint=system_variables.GOWAKU_ENTRYPOINT, cmd=[ - "--config-file=" + system_variables.NODE_CONFIG_FILE_LOCATION + "/" + configuration_file + "--config-file=" + system_variables.CONTAINER_NODE_CONFIG_FILE_LOCATION + "/" + configuration_file ] ) ) return gowaku_service -def add_jswaku_service(test, test2): - print("jswaku") -def add_nomos_service(test, test2): - print("nomos") +def add_nomos_service(plan, test, test2): + plan.print("nomos") -def instantiate_services(network_topology, use_general_configuration): +def instantiate_services(plan, network_topology, use_general_configuration): """ As we will need to access for the service information later, the structure is the following: @@ -111,18 +111,18 @@ def instantiate_services(network_topology, use_general_configuration): service_builder, information_builder = service_dispatcher[image] - service_information = service_builder(service_id, use_general_configuration) + service_information = service_builder(plan, service_id, use_general_configuration) - information_builder(services_information, service_id, service_information) + information_builder(plan, services_information, service_id, service_information) return services_information -def _add_waku_service_information(services_information, new_service_id, service_information): +def _add_waku_service_information(plan, services_information, new_service_id, service_information): new_service_information = {} - wakunode_peer_id = waku.get_wakunode_peer_id(new_service_id, system_variables.WAKU_RPC_PORT_ID) + wakunode_peer_id = waku.get_wakunode_peer_id(plan, new_service_id, system_variables.WAKU_RPC_PORT_ID) new_service_information["peer_id"] = wakunode_peer_id new_service_information["service_info"] = service_information @@ -133,6 +133,5 @@ def _add_waku_service_information(services_information, new_service_id, service_ service_dispatcher = { "go-waku": (add_gowaku_service, _add_waku_service_information), "nim-waku": (add_nwaku_service, _add_waku_service_information), - "js-waku": (add_jswaku_service, _add_waku_service_information), "nomos": (add_nomos_service, "test") } diff --git a/src/system_variables.star b/src/system_variables.star index 84af28e..a2c59c3 100644 --- a/src/system_variables.star +++ b/src/system_variables.star @@ -8,7 +8,11 @@ WAKU_LIBP2P_PORT_ID = "libp2p" WAKU_LIBP2P_PORT = 60000 WAKU_SETUP_WAIT_TIME = "5" -NODE_CONFIG_FILE_LOCATION = "/node/configuration_file" + +NODE_CONFIG_FILE_LOCATION = "github.com/logos-co/wakurtosis/config/node_config_files/" + +CONTAINER_NODE_CONFIG_FILE_LOCATION = "/node/configuration_file/" +NODE_CONFIGURATION_FILE_EXTENSION = ".toml" NWAKU_ENTRYPOINT = ["/usr/bin/wakunode", "--rpc-address=0.0.0.0", "--metrics-server-address=0.0.0.0"] GOWAKU_ENTRYPOINT = ["/usr/bin/waku", "--rpc-address=0.0.0.0", "--metrics-server-address=0.0.0.0"] diff --git a/src/waku.star b/src/waku.star index b209496..235c6d3 100644 --- a/src/waku.star +++ b/src/waku.star @@ -5,7 +5,7 @@ system_variables = import_module("github.com/logos-co/wakurtosis/src/system_vari files = import_module(system_variables.FILE_HELPERS_MODULE) -def send_json_rpc(service_id, port_id, method, params, extract={}): +def send_json_rpc(plan, service_id, port_id, method, params, extract={}): recipe = struct( service_id=service_id, port_id=port_id, @@ -16,7 +16,7 @@ def send_json_rpc(service_id, port_id, method, params, extract={}): extract=extract ) - response = wait(recipe=recipe, + response = plan.wait(recipe=recipe, field="code", assertion="==", target_value=200) @@ -24,10 +24,10 @@ def send_json_rpc(service_id, port_id, method, params, extract={}): return response -def get_wakunode_peer_id(service_id, port_id): +def get_wakunode_peer_id(plan, service_id, port_id): extract = {"peer_id": '.result.listenAddresses | .[0] | split("/") | .[-1]'} - response = send_json_rpc(service_id, port_id, + response = send_json_rpc(plan, service_id, port_id, system_variables.GET_WAKU_INFO_METHOD, "", extract) return response["extract.peer_id"] @@ -47,23 +47,23 @@ def _merge_peer_ids(peer_ids): return "[" + ",".join(peer_ids) + "]" -def connect_wakunode_to_peers(service_id, port_id, peer_ids): +def connect_wakunode_to_peers(plan, service_id, port_id, peer_ids): method = system_variables.CONNECT_TO_PEER_METHOD params = _merge_peer_ids(peer_ids) - response = send_json_rpc(service_id, port_id, method, params) + response = send_json_rpc(plan, service_id, port_id, method, params) - print(response) + plan.print(response) -def post_waku_v2_relay_v1_message(service_id, topic): +def post_waku_v2_relay_v1_message(plan, service_id, topic): waku_message = '{"payload": "0x1a2b3c4d5e6f", "timestamp": 1626813243}' params = '"' + topic + '"' + ", " + waku_message response = send_json_rpc(service_id, system_variables.WAKU_RPC_PORT_ID, system_variables.POST_RELAY_MESSAGE, params) - print(response) + plan.print(response) def get_wakunode_id(service_id, port_id): @@ -75,19 +75,19 @@ def get_wakunode_id(service_id, port_id): return response["extract.waku_id"] -def make_service_wait(service_id, time): +def make_service_wait(plan,service_id, time): exec_recipe = struct( service_id=service_id, command=["sleep", time] ) - exec(exec_recipe) + plan.exec(exec_recipe) -def get_waku_peers(waku_service_id): - response = send_json_rpc(waku_service_id, system_variables.WAKU_RPC_PORT_ID, +def get_waku_peers(plan, waku_service_id): + response = send_json_rpc(plan, waku_service_id, system_variables.WAKU_RPC_PORT_ID, system_variables.GET_PEERS_METHOD, "") - print(response) + plan.print(response) return response @@ -100,13 +100,13 @@ def send_test_messages(topology_information, number_of_messages, time_between_me post_waku_v2_relay_v1_message(wakunode_name, "test") -def interconnect_waku_nodes(topology_information, services): +def interconnect_waku_nodes(plan, topology_information, services): # Interconnect them for waku_service_id in topology_information.keys(): peers = topology_information[waku_service_id]["static_nodes"] peer_ids = [create_waku_id(services[peer]) for peer in peers] - connect_wakunode_to_peers(waku_service_id, system_variables.WAKU_RPC_PORT_ID, peer_ids) + connect_wakunode_to_peers(plan, waku_service_id, system_variables.WAKU_RPC_PORT_ID, peer_ids) diff --git a/src/wsl.star b/src/wsl.star index 737f1f9..5684b17 100644 --- a/src/wsl.star +++ b/src/wsl.star @@ -5,26 +5,24 @@ system_variables = import_module("github.com/logos-co/wakurtosis/src/system_vari files = import_module(system_variables.FILE_HELPERS_MODULE) templates = import_module(system_variables.TEMPLATES_MODULE) -def create_config(simulation_time, message_rate, min_packet_size, max_packet_size, inter_msg_type, dist_type, emitters_fraction): +def create_config(plan, wls_config): - template_data = {"simulation_time": simulation_time, "message_rate" : message_rate, "min_packet_size" : min_packet_size, - "max_packet_size" : max_packet_size, "dist_type" : dist_type, "emitters_fraction" : emitters_fraction, "inter_msg_type" : inter_msg_type} - # Traffic simulation parameters wsl_yml_template = templates.get_wsl_template() - artifact_id = render_templates( + artifact_id = plan.render_templates( config={ system_variables.CONTAINER_WSL_CONFIGURATION_FILE_NAME: struct( template=wsl_yml_template, - data=template_data, + data=wls_config, ) - } + }, + name="wsl_config" ) return artifact_id -def create_targets(services): +def create_targets(plan, services): # Get private ip and ports of all nodes template_data = files.generate_template_node_targets(services, system_variables.WAKU_RPC_PORT_ID) @@ -34,26 +32,27 @@ def create_targets(services): {{.targets}} """ - artifact_id = render_templates( + artifact_id = plan.render_templates( config={ system_variables.CONTAINER_TARGETS_FILE_NAME_WSL: struct( template=template, data=template_data, ) - } + }, + name="wsl_targets" ) return artifact_id -def init(services, simulation_time, message_rate, min_packet_size, max_packet_size, inter_msg_type, dist_type, emitters_fraction): +def init(plan, services, wsl_config): # Generate simulation config - wsl_config = create_config(simulation_time, message_rate, min_packet_size, max_packet_size, inter_msg_type, dist_type, emitters_fraction) + wsl_config = create_config(plan, wsl_config) # Create targets.json - wsl_targets = create_targets(services) + wsl_targets = create_targets(plan, services) - wsl_service = add_service( + wsl_service = plan.add_service( service_id=system_variables.WSL_SERVICE_ID, config=struct( image=system_variables.WSL_IMAGE,