vacp2p
/
wakurtosis
mirror of https://github.com/vacp2p/wakurtosis.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Load WSL container in Starlark 

Fixes #13
This commit is contained in:
Jordi Arranz 2022-12-23 16:12:35 +01:00
parent 3a351ea816
commit 247c1db0fd
8 changed files with 154 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

132
main.star
View File

@ -7,6 +7,12 @@ WAKU_LIBP2P_PORT = 60000
WAKU_SETUP_WAIT_TIME = "5"
# WSL Configuration
WSL_IMAGE = "wsl:0.0.1"
WSL_CONFIG_PATH = "/wsl/config"
WSL_TARGETS_PATH = "/wsl/targets"
# WSL_CONFIGURATION_PATH = "github.com/logos-co/wakurtosis/tree/WSL/wsl-module/wsl.yml"
# Prometheus Configuration
PROMETHEUS_IMAGE = "prom/prometheus:latest"
PROMETHEUS_PORT_ID = "prometheus"
@ -206,7 +212,81 @@ def generate_template_data(services):
return template_data
def generate_wsl_targets_template_data(services):
template_data = {}
node_data = []
for wakunode_name in services.keys():
node_data.append(
'"' + services[wakunode_name]["service"].ip_address + ":" + str(services[wakunode_name]["service"].ports[
WAKU_RPC_PORT_ID].number) + '"')
data_as_string = ",".join(node_data)
targets = "[" + data_as_string + "]"
template_data["targets"] = targets
return template_data
def create_wsl_config():
template_data = None
# Traffic simulation parameters
wsl_yml_template = """
general:
debug_level : "DEBUG"
targets_file : "./targets/targets.json"
prng_seed : 0
# Simulation time in seconds
simulation_time : 1000
# Message rate in messages per second
msg_rate : 10
# Packet size in bytes
min_packet_size : 2
max_packet_size : 1024
"""
artifact_id = render_templates(
config={
"wsl.yml": struct(
template=wsl_yml_template,
data=template_data,
)
}
)
return artifact_id
def create_wsl_targets(services):
# Get private ip and ports of all nodes
template_data = generate_wsl_targets_template_data(services)
# Template
template = """
{{.targets}}
"""
artifact_id = render_templates(
config={
"targets.json": struct(
template=template,
data=template_data,
)
}
)
return artifact_id
def create_prometheus_targets(services):
# get ip and ports of all nodes
template_data = generate_template_data(services)
@ -234,8 +314,8 @@ def create_prometheus_targets(services):
return artifact_id
def set_up_prometheus(services):
# Create targets.json
targets_artifact_id = create_prometheus_targets(services)
@ -265,6 +345,34 @@ def set_up_prometheus(services):
return prometheus_service
def set_up_wsl(services):
# Generate simulation config
wsl_config = create_wsl_config()
# Create targets.json
wsl_targets = create_wsl_targets(services)
wsl_service = add_service(
service_id="wsl",
config=struct(
image=WSL_IMAGE,
ports={},
files={
WSL_CONFIG_PATH : wsl_config,
WSL_TARGETS_PATH : wsl_targets,
},
# cmd=["python3", "/wsl/wsl.py",],
cmd=["python3", "wsl.py"]
# cmd=["sleep 1000",],
# entrypoint = ["/bin/bash", "-l", "-c",],
)
)
print('kurtosis service logs -f wakurtosis SERVICE-GUID')
return wsl_service
def set_up_graphana(prometheus_service):
# Set up grafana
@ -326,14 +434,14 @@ def set_up_graphana(prometheus_service):
return grafana_service
def run(args):
waku_topology = read_file(src="github.com/logos-co/wakurtosis/kurtosis-module/starlark/waku_test_topology.json")
waku_topology = read_file(src="github.com/logos-co/wakurtosis/kurtosis-module/starlark/waku_test_topology.json")
same_toml_configuration = args.same_toml_configuration
waku_topology = json.decode(waku_topology)
# decoded = {
# Testing topology
# waku_topology = {
# "waku_0": {
# "ports_shift": 0,
# "topics": "test",
@ -357,16 +465,22 @@ def run(args):
# ]
# }
# }
# same_toml_configuration = waku_topology
# End testing topology
services = instantiate_waku_nodes(waku_topology, same_toml_configuration)
# Set up Waku nodes
waku_services = instantiate_waku_nodes(waku_topology, same_toml_configuration)
# Set up prometheus + graphana
prometheus_service = set_up_prometheus(services)
prometheus_service = set_up_prometheus(waku_services)
set_up_graphana(prometheus_service)
interconnect_waku_nodes(waku_topology, services)
interconnect_waku_nodes(waku_topology, waku_services)
send_test_messages(waku_topology)
# Set up the WSL container & start the simulation
set_up_wsl(waku_services)
ask_connected_nodes(waku_topology)
# send_test_messages(waku_topology)
# ask_connected_nodes(waku_topology)

4
wsl-module/README.md
View File

@ -7,10 +7,8 @@ Kurtosis: https://docs.kurtosis.com/
To build docker image:
`sh ./build.sh`
To run docker image:
`sh ./run.sh`
At the moment the targets.json file is copied to the container during build which means that the container imgage has to be rebuilt whenever the enclave restarts --- and the private IPs of the Waku nodes change. The targets.json is automatically generatted during the build step
Name of the image is wsl:0.0.1
#### Before using this repository make sure that:
For the build step make sure that the Wakurtosis enclave exists and make sure the enclave is running before running the container

3
wsl-module/build.sh
View File

@ -1,4 +1,3 @@
#!/bin/sh
pip freeze > ./requirements.txt
python3 ./make_targets.py
docker image build -t wsl:0.0.1 ./
docker image build --progress=plain -t wsl:0.0.1 ./

8
wsl-module/wsl.yml → wsl-module/config/wsl.yml
View File

@ -2,14 +2,12 @@ general:
debug_level : "DEBUG"
targets_file : "./targets/targets.json"
prng_seed : 0
enclave_name : "wakurtosis"
enclave_dump_path : "enclave.dump"
waku_port : 8545
# Simulation time in seconds
simulation_time : 10
simulation_time : 100
# Message rate in messages per second
msg_rate : 10

5
wsl-module/dockerfile
View File

@ -1,9 +1,6 @@
FROM python:3.11.0
LABEL Maintainer="Daimakaimura"
WORKDIR /wsl
COPY wsl.py ./
COPY wsl.yml ./
COPY wsl.py .
COPY requirements.txt ./
COPY targets.json ./
RUN pip install -r requirements.txt
CMD [ "python3", "./wsl.py"],

93
wsl-module/make_targets.py
View File

@ -1,93 +0,0 @@
#!/usr/bin/env python3
""" Dependencies """
import os, sys, logging, json, argparse
""" Globals """
G_APP_NAME = 'MakeWLSTargets'
G_LOG_LEVEL = 'INFO'
G_LOGGER = None
G_ENCLAVE_DUMP_PATH = 'enclave.dump'
""" Custom logging formatter """
class CustomFormatter(logging.Formatter):
# Set different formats for every logging level
time_name_stamp = "[%(asctime)s.%(msecs)03d] [" + G_APP_NAME + "]"
FORMATS = {
logging.ERROR: time_name_stamp + " ERROR in %(module)s.py %(funcName)s() %(lineno)d - %(msg)s",
logging.WARNING: time_name_stamp + " WARNING - %(msg)s",
logging.CRITICAL: time_name_stamp + " CRITICAL in %(module)s.py %(funcName)s() %(lineno)d - %(msg)s",
logging.INFO: time_name_stamp + " %(msg)s",
logging.DEBUG: time_name_stamp + " %(funcName)s() %(msg)s",
'DEFAULT': time_name_stamp + " %(msg)s",
}
def format(self, record):
log_fmt = self.FORMATS.get(record.levelno, self.FORMATS['DEFAULT'])
formatter = logging.Formatter(log_fmt, '%d-%m-%Y %H:%M:%S')
return formatter.format(record)
def parse_targets(enclave_dump_path, waku_port=8545):
targets = []
G_LOGGER.info('Extracting Waku node addresses from Kurtosus enclance dump in %s' %enclave_dump_path)
for path_obj in os.walk(enclave_dump_path):
if 'waku_' in path_obj[0]:
with open(path_obj[0] + '/spec.json', "r") as read_file:
spec_obj = json.load(read_file)
network_settings = spec_obj['Config']['Labels']
waku_address = network_settings['com.kurtosistech.private-ip']
if len(waku_address) == 0:
G_LOGGER.info('No targets found in %s' %(path_obj[0]))
else:
targets.append('%s:%s' %(waku_address, waku_port))
G_LOGGER.info('Parsed %d Waku nodes' %len(targets))
return targets
def main():
global G_LOGGER
""" Init Logging """
G_LOGGER = logging.getLogger(G_APP_NAME)
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(CustomFormatter())
G_LOGGER.addHandler(handler)
G_LOGGER.info('Started')
""" Parse args """
parser = argparse.ArgumentParser()
parser.add_argument('-e', '--enclave', help='Wakurtosis enclave name', action='store_true', default='wakurtosis')
args = parser.parse_args()
""" Dump enclave info """
# Delete previous dump if any
os.system('rm -rf %s' %G_ENCLAVE_DUMP_PATH)
# Generate new dump
os.system('kurtosis enclave dump %s %s' %(args.enclave, G_ENCLAVE_DUMP_PATH))
""" Parse targets """
targets = parse_targets(G_ENCLAVE_DUMP_PATH)
if len(targets) == 0:
G_LOGGER.error('Cannot find valid targets. Aborting.')
sys.exit(1)
""" Export targets """
with open('targets.json', 'w') as f:
json.dump(targets, f)
G_LOGGER.info('Targets exported to .json')
""" We are done """
G_LOGGER.info('Ended')
if __name__ == "__main__":
main()

3
wsl-module/run.sh
View File

@ -1,3 +0,0 @@
#!/bin/sh
python3 ./make_targets.py
docker run wsl:0.0.1

28
wsl-module/wsl.py
View File

@ -5,7 +5,7 @@ Description: Wakurtosis load simulator
"""
""" Dependencies """
import sys, logging, yaml, json, time, random, os
import sys, logging, yaml, json, time, random, os, argparse
import requests
# from pathlib import Path
# import numpy as np
@ -15,9 +15,8 @@ import requests
""" Globals """
G_APP_NAME = 'WLS'
G_LOG_LEVEL = 'INFO'
G_CONFIG_FILE = './wsl.yml'
G_TARGETS_FILE = './targets.json'
G_LOG_LEVEL = 'DEBUG'
g_DEFAULT_CONFIG_FILE = './config/wsl.yml'
G_LOGGER = None
""" Custom logging formatter """
@ -97,9 +96,12 @@ def send_waku_msg(node_address, topic, payload, nonce=1):
def poisson_interval(rate):
return random.expovariate(rate)
def make_payload(size):
def make_payload(size, rnd=True):
# Size in bytes, supposed to be hexa, 2 hexa digits per byte
if rnd:
payload = ''.join(random.choices('0123456789abcdef', k=int( 2 * size - 1)))
else:
payload = ''.join('00' * int(size))
payload = '0x%s' %payload
@ -150,25 +152,34 @@ def main():
G_LOGGER.info('Started')
""" Parse command line args. """
parser = argparse.ArgumentParser()
parser.add_argument("-cfg", "--config_file", help="Config file", action="store_true", default=g_DEFAULT_CONFIG_FILE)
args = parser.parse_args()
config_file = args.config_file
""" Load config file """
try:
with open(G_CONFIG_FILE, 'r') as f:
with open(config_file, 'r') as f:
config = yaml.safe_load(f)
except Exception as e:
G_LOGGER.error('%s: %s' % (e.__doc__, e))
sys.exit()
G_LOGGER.info('Configuration loaded from %s' %G_CONFIG_FILE)
# Set loglevel from config
G_LOGGER.setLevel(config['general']['debug_level'])
handler.setLevel(config['general']['debug_level'])
G_LOGGER.debug(config)
G_LOGGER.info('Configuration loaded from %s' %config_file)
# Set RPNG seed from config
random.seed(config['general']['prng_seed'])
""" Load targets """
try:
with open(G_TARGETS_FILE, 'r') as read_file:
with open(config['general']['targets_file'], 'r') as read_file:
targets = json.load(read_file)
except Exception as e:
G_LOGGER.error('%s: %s' % (e.__doc__, e))
@ -178,6 +189,7 @@ def main():
G_LOGGER.error('Cannot find valid targets. Aborting.')
sys.exit(1)
G_LOGGER.debug(targets)
G_LOGGER.info('%d targets loaded' %len(targets))
""" Check all nodes are reachable """