logos-execution-zone/lez/wallet/src/multi_client.rs

#![expect(
    clippy::float_arithmetic,
    reason = "One should expect floating point arithmetic in statistic calculations"
)]
#![expect(
    clippy::cast_precision_loss,
    reason = "Operated numbers is not big enough to have precision loss"
)]

use std::{collections::HashMap, path::Path, sync::Arc};

use anyhow::{Context as _, Result};
use lee_core::BlockId;
use sequencer_service_rpc::{RpcClient as _, SequencerClient, SequencerClientBuilder};
use serde::{Deserialize, Serialize};
use tokio::sync::RwLock;
use url::Url;

use crate::config::{MultiSequencerClientConfig, SequencerConnectionData};

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Statistics {
    pub latency_avg: f32,
    pub latency_var: f32,
    pub sample_size: usize,
    pub latest_block_id: BlockId,
    pub errors: u64,
}

#[derive(Debug, Clone)]
pub struct StatisticsUpdate {
    pub latency: f32,
    pub new_latest_block_id: Option<BlockId>,
    pub is_failed: bool,
}

impl Statistics {
    pub fn apply_updates(&mut self, updates: &[StatisticsUpdate]) {
        let CumulativeUpdates {
            failure_count,
            latest_block_id,
            cumulative_latency,
            cumulative_latency_squares,
            additional_sample_size,
        } = CumulativeUpdates::from_metric_updates(updates);

        self.errors = self.errors.saturating_add(failure_count);
        if let Some(latest_block_id) = latest_block_id {
            self.latest_block_id = latest_block_id;
        }

        #[expect(clippy::as_conversions, reason = "int to float conversion is safe")]
        let orig_size_f = self.sample_size as f32;
        #[expect(clippy::as_conversions, reason = "int to float conversion is safe")]
        let mod_size_f = additional_sample_size as f32;

        let latency_avg_old = self.latency_avg;
        let latency_avg_new =
            cumulative_avg(latency_avg_old, cumulative_latency, orig_size_f, mod_size_f);

        let latency_var_new = cumulative_var(
            latency_avg_old,
            latency_avg_new,
            self.latency_var,
            cumulative_latency,
            cumulative_latency_squares,
            orig_size_f,
            mod_size_f,
        );

        self.latency_avg = latency_avg_new;
        self.latency_var = latency_var_new;
        self.sample_size = self.sample_size.saturating_add(additional_sample_size);
    }
}

#[derive(Clone)]
pub struct MultiSequencerClient {
    /// Ordered list of leaders, from best to worst
    pub leader_list: Vec<(SequencerClient, Url)>,
    pub multi_sequencer_client_config: MultiSequencerClientConfig,
    /// Wrapping statistic updates in Arc<RwLock> to not break interfaces too much.
    ///  
    /// It is assumed, that wallet methods can be accesed via immutable reference.
    statistic_updates: Arc<RwLock<Vec<StatisticsUpdate>>>,
}

impl MultiSequencerClient {
    async fn setup(
        conn_data: &[SequencerConnectionData],
        statistics: &mut HashMap<Url, Statistics>,
        multi_sequencer_client_config: MultiSequencerClientConfig,
    ) -> Result<Self> {
        let mut client_list = HashMap::new();

        for SequencerConnectionData {
            sequencer_addr,
            basic_auth,
        } in conn_data
        {
            let sequencer_client = {
                let mut builder = SequencerClientBuilder::default();
                if let Some(basic_auth) = &basic_auth {
                    builder = builder.set_headers(
                        std::iter::once((
                            "Authorization".parse().expect("Header name is valid"),
                            format!("Basic {basic_auth}")
                                .parse()
                                .context("Invalid basic auth format")?,
                        ))
                        .collect(),
                    );
                }
                builder
                    .build(sequencer_addr)
                    .context("Failed to create sequencer client")?
            };

            // If there is statistics for client, actualize it
            if let Some(metric_mut) = statistics.get_mut(sequencer_addr) {
                let metric_updates = actualize_client(&sequencer_client).await;

                log::debug!(
                    "Metered call for {sequencer_addr:?}, metric updates is {metric_updates:?}"
                );

                metric_mut.apply_updates(&[metric_updates]);
            // Otherwise calibrate client data
            } else if let Some(client_statistics) =
                calibrate_client(&sequencer_client, calibration_limit).await
            {
                statistics.insert(sequencer_addr.clone(), client_statistics);
            } else {
                log::warn!("Client {sequencer_addr:?} failed all {calibration_limit} calibration attempts, it may be unhealthy.
                    \n Consider bumping calibration_limit or remove this client altogether");
            }

            client_list.insert(sequencer_addr.clone(), sequencer_client);
        }

        // Dropping client list, for reasons why, see comment in structure definition.

        let leader_list = choose_leaders(
            &client_list,
            metrics,
            multi_sequencer_client_config.distribution_limit,
        )
        .ok_or_else(|| anyhow::anyhow!("Failed to find leader"))?;

        log::info!("Chosen leaders is {leader_list:?}");

        Ok(Self {
            leader_list,
            multi_sequencer_client_config,
        })
    }

    pub async fn new(
        conn_data: &[SequencerConnectionData],
        statistics: &mut HashMap<Url, Statistics>,
        calibration_limit: usize,
    ) -> Result<Self> {
        let (leader_url, leader) = Self::setup(conn_data, statistics, calibration_limit).await?;

        Ok(Self {
            leader,
            leader_url,
            statistic_updates: Arc::new(RwLock::new(vec![])),
        })
    }

    /// Re-choose leader, `statistic_updates` must be empty.
    pub async fn rotate(
        &mut self,
        conn_data: &[SequencerConnectionData],
        statistics: &mut HashMap<Url, Statistics>,
        calibration_limit: usize,
    ) -> Result<()> {
        let (leader_url, leader) = Self::setup(conn_data, statistics, calibration_limit).await?;

        log::info!("Chosen leader is {leader_url:?}");

        self.leader = leader;
        self.leader_url = leader_url;

        Ok(())
    }

    #[must_use]
    pub const fn leader(&self) -> &SequencerClient {
        &self.leader
    }

    #[must_use]
    pub const fn leader_url(&self) -> &Url {
        &self.leader_url
    }

    // Keeping this call abstract, in case if we need to do more than one request
    pub async fn metered_call<R, E, I: AsyncFn(&SequencerClient) -> Result<R, E>>(
        &self,
        call: I,
    ) -> Result<R, E> {
        let (resp, statistics_update) =
            tokio::join!(call(self.leader()), actualize_client(self.leader()));

        log::debug!(
            "Metered call for {:?}, metric updates is {:?}",
            self.leader_url,
            statistics_update
        );

        {
            let mut statistic_updates_guard = self.statistic_updates.write().await;
            statistic_updates_guard.push(statistics_update);
        }

        resp
    }

    /// Update statistics of a leader, clear statist updates log.
    pub async fn update_statistics(&self, leader_statistic: &mut Statistics) {
        let mut statistic_updates = self.statistic_updates.write().await;
        leader_statistic.apply_updates(statistic_updates.as_ref());
        statistic_updates.clear();
    }
}

struct CumulativeUpdates {
    pub failure_count: u64,
    pub latest_block_id: Option<BlockId>,
    /// Necessary for cumulative average calculation.
    pub cumulative_latency: f32,
    /// Necessary for cumulative variance calculation.
    pub cumulative_latency_squares: f32,
    pub additional_sample_size: usize,
}

impl CumulativeUpdates {
    fn from_metric_updates(metric_updates: &[StatisticsUpdate]) -> Self {
        let (failure_count, latest_block_id, cumulative_latency, cumulative_latency_squares) =
            metric_updates
                .iter()
                .fold((0_u64, None, 0_f32, 0_f32), |acc, x| {
                    let StatisticsUpdate {
                        latency,
                        new_latest_block_id,
                        is_failed,
                    } = x;
                    (
                        if *is_failed {
                            acc.0.saturating_add(1)
                        } else {
                            acc.0
                        },
                        match (acc.1, new_latest_block_id) {
                            (None, None) => None,
                            (None, Some(val)) | (Some(val), None) => Some(val),
                            (Some(val_old), Some(val_new)) => Some(std::cmp::max(val_old, val_new)),
                        },
                        if *is_failed { acc.2 } else { acc.2 + latency },
                        if *is_failed {
                            acc.3
                        } else {
                            latency.mul_add(*latency, acc.3)
                        },
                    )
                });

        Self {
            failure_count,
            latest_block_id: latest_block_id.copied(),
            cumulative_latency,
            cumulative_latency_squares,
            additional_sample_size: metric_updates.len().saturating_sub(
                usize::try_from(failure_count).expect("Sample size should fit usize"),
            ),
        }
    }
}

pub fn extract_statistics_from_path(
    path: &Path,
) -> Result<HashMap<Url, Statistics>, anyhow::Error> {
    match std::fs::File::open(path) {
        Ok(file) => {
            let reader = std::io::BufReader::new(file);
            Ok(serde_json::from_reader(reader)?)
        }
        Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
            println!("Statistics not found, choosing empty");
            Ok(HashMap::new())
        }
        Err(err) => Err(err).context("IO error"),
    }
}

/// Measuring `get_last_block_id` as it should be the fastest request on sequencer.
async fn measure_request_duration(client: &SequencerClient) -> (u128, Option<BlockId>) {
    let now = tokio::time::Instant::now();
    let block_id = client.get_last_block_id().await.ok();
    (
        tokio::time::Instant::now().duration_since(now).as_millis(),
        block_id,
    )
}

pub async fn calibrate_client(
    client: &SequencerClient,
    calibration_limit: usize,
) -> Option<Statistics> {
    let mut latencies = vec![];
    let mut latest_block_id = 0;
    let mut errors: u64 = 0;

    // ToDo: Add some DDoS adaptation
    for _ in 0..calibration_limit {
        let (latency, block_id) = measure_request_duration(client).await;

        let Some(block_id) = block_id else {
            errors = errors.saturating_add(1);
            continue;
        };

        latest_block_id = block_id;
        latencies.push(latency);
    }

    // There is no point in guard numbers, exclude client if it fails all requests.
    if latencies.is_empty() {
        return None;
    }

    // Precision loss is fine there
    let sample_size = latencies.len();
    #[expect(clippy::as_conversions, reason = "int to float conversion is safe")]
    let latency_avg = (latencies.iter().sum::<u128>() as f32) / (sample_size as f32);
    #[expect(clippy::as_conversions, reason = "int to float conversion is safe")]
    let latency_var = latencies.iter().fold(0_f32, |acc, x| {
        ((*x as f32) - latency_avg).mul_add((*x as f32) - latency_avg, acc)
    }) / (sample_size as f32);

    Some(Statistics {
        latency_avg,
        latency_var,
        sample_size,
        latest_block_id,
        errors,
    })
}

pub async fn actualize_client(client: &SequencerClient) -> StatisticsUpdate {
    let (latency, block_id) = measure_request_duration(client).await;

    #[expect(clippy::as_conversions, reason = "int to float conversion is safe")]
    let latency = latency as f32;

    StatisticsUpdate {
        latency,
        new_latest_block_id: block_id,
        is_failed: block_id.is_none(),
    }
}

#[must_use]
pub fn choose_leaders(
    client_list: &HashMap<Url, SequencerClient>,
    statistics: &HashMap<Url, Statistics>,
    distribution_limit: usize,
) -> Option<Vec<(SequencerClient, Url)>> {
    // Sort out all unmetered clients
    let mut client_vec: Vec<_> = client_list
        .keys()
        .filter(|item| statistics.contains_key(*item))
        .collect();

    if client_vec.is_empty() {
        return None;
    }

    // Considering the nature of our requests, the latest_block_id is the dominant characteristic
    let max_block_id_addr = client_vec.iter().fold(client_vec[0], |acc, x| {
        let old_latest_block_id = statistics.get(acc).unwrap().latest_block_id;
        let new_latest_block_id = statistics.get(*x).unwrap().latest_block_id;
        if new_latest_block_id > old_latest_block_id {
            *x
        } else {
            acc
        }
    });

    let max_block_id = statistics.get(max_block_id_addr).unwrap().latest_block_id;

    // Sort out all clients running late
    client_vec = client_vec
        .iter()
        .filter_map(|x| {
            let latest_block_id = statistics.get(*x).unwrap().latest_block_id;

            (latest_block_id == max_block_id).then_some(*x)
        })
        .collect();

    // Get the clients with lesser or equal to average error ratio
    #[expect(clippy::as_conversions, reason = "int to float conversion is safe")]
    let avg_err_ratio = client_vec.iter().fold(0_f32, |acc, x| {
        acc + error_ratio(
            statistics.get(*x).unwrap().errors,
            statistics.get(*x).unwrap().sample_size,
        )
    }) / (client_vec.len() as f32);

    client_vec.sort_by(|a, b| {
        let err_ratio_a = error_ratio(
            statistics.get(*a).unwrap().errors,
            statistics.get(*a).unwrap().sample_size,
        );
        let err_ratio_b = error_ratio(
            statistics.get(*b).unwrap().errors,
            statistics.get(*b).unwrap().sample_size,
        );

        err_ratio_a
            .partial_cmp(&err_ratio_b)
            .expect("Ratios must be a valid numbers")
    });

    let mut client_vec = client_vec[..(client_vec
        .iter()
        .position(|item| {
            error_ratio(
                statistics.get(*item).unwrap().errors,
                statistics.get(*item).unwrap().sample_size,
            ) > avg_err_ratio
        })
        .unwrap_or(client_vec.len()))]
        .to_vec();

    // Choose clients with least latency and variance
    client_vec.sort_by(|a, b| {
        let left = statistics.get(*a).unwrap();
        let (left_lat, left_var) = (left.latency_avg, left.latency_var);
        let right = statistics.get(*b).unwrap();
        let (right_lat, right_var) = (right.latency_avg, right.latency_var);

        let right_std = right_var.sqrt();
        let left_std = left_var.sqrt();

        // Client is better if its average is better and variance does not make it worse
        // So basically we want this:
        // [-left_std............right_lat.......left_lat...............+right_std.........
        // +left_std]
        //
        // However one can argue that this:
        //
        // [-left_std...................left_lat........right_lat.........+right_std.......
        // +left_std]
        //
        // is still better, but it is up to discussion
        if (right_lat <= right_lat) && ((right_lat + right_std) < (left_lat + left_std)) {
            std::cmp::Ordering::Less
        } else {
            std::cmp::Ordering::Greater
        }
    });

    Some(
        client_vec
            .iter()
            .take(distribution_limit)
            .map(|addr| {
                let client = client_list
                    .get(*addr)
                    .expect("Missing clients already sorted out");

                (client.clone(), (*addr).clone())
            })
            .collect(),
    )
}

/// Helperfunction to calculate cumulative average.
///
/// Cumulative average calculation is the following problem:
///
/// We want to calculate avarage of a sample of size `N + N_1`
/// where average for `N` is known.
///
/// To do so we need:
/// - old average value
/// - sum_{`i=1}^{N_1}{n_i`}
/// - `N`
/// - `N_1`
fn cumulative_avg(
    latency_avg_old: f32,
    cumulative_latency: f32,
    orig_size_f: f32,
    mod_size_f: f32,
) -> f32 {
    latency_avg_old.mul_add(orig_size_f, cumulative_latency) / (orig_size_f + mod_size_f)
}

/// Helperfunction to calculate cumulative variance.
///
/// Cumulative variance calculation is the following problem:
///
/// We want to calculate variance of a sample of size `N + N_1`
/// where average for `N` is known.
///
/// To do so we need:
/// - old average value
/// - new average value
/// - old variance
/// - sum_{`i=1}^{N_1}{n_i`}
/// - sum_{`i=1}^{N_1}{n_i^2`}
/// - `N`
/// - `N_1`
fn cumulative_var(
    latency_avg_old: f32,
    latency_avg_new: f32,
    latency_var: f32,
    cumulative_latency: f32,
    cumulative_latency_squares: f32,
    orig_size_f: f32,
    mod_size_f: f32,
) -> f32 {
    // The formula was atrocious before.
    // `mul_add` function have less precision loss with drawback of being absolutely unreadable
    ((2_f32 * cumulative_latency).mul_add(
        -latency_avg_new,
        mod_size_f.mul_add(
            latency_avg_new * latency_avg_new,
            latency_var.mul_add(
                orig_size_f,
                (latency_avg_new - latency_avg_old)
                    * (latency_avg_new - latency_avg_old)
                    * orig_size_f,
            ),
        ),
    ) + cumulative_latency_squares)
        / (orig_size_f + mod_size_f)
}

fn error_ratio(errors: u64, size: usize) -> f32 {
    #[expect(clippy::as_conversions, reason = "int to float conversion is safe")]
    let errors_f = errors as f32;
    #[expect(clippy::as_conversions, reason = "int to float conversion is safe")]
    let size_f = size as f32;

    errors_f / (errors_f + size_f)
}

#[cfg(test)]
mod tests {
    use std::collections::HashMap;

    use sequencer_service_rpc::{SequencerClient, SequencerClientBuilder};
    use url::Url;

    use crate::multi_client::{
        CumulativeUpdates, Statistics, StatisticsUpdate, choose_leaders, cumulative_avg,
        cumulative_var,
    };

    fn update_statistics(
        statistics: &mut HashMap<Url, Statistics>,
        leader_url: &Url,
        metric_updates: &[StatisticsUpdate],
    ) -> Result<(), anyhow::Error> {
        let leader_metric = statistics
            .get_mut(leader_url)
            .ok_or_else(|| anyhow::anyhow!("Leader URL is not present in statistics"))?;

        leader_metric.apply_updates(metric_updates);

        Ok(())
    }

    fn client_from_url_unchecked(url: &Url) -> SequencerClient {
        let builder = SequencerClientBuilder::default();
        builder.build(url).unwrap()
    }

    #[test]
    fn cumulative_updates_test() {
        let statistics_updates_vec = vec![
            StatisticsUpdate {
                latency: 100_f32,
                new_latest_block_id: Some(15),
                is_failed: false,
            },
            StatisticsUpdate {
                latency: 115_f32,
                new_latest_block_id: Some(16),
                is_failed: false,
            },
            StatisticsUpdate {
                latency: 50_f32,
                new_latest_block_id: None,
                is_failed: true,
            },
        ];

        let CumulativeUpdates {
            failure_count,
            latest_block_id,
            cumulative_latency,
            cumulative_latency_squares,
            additional_sample_size,
        } = CumulativeUpdates::from_metric_updates(&statistics_updates_vec);

        let epsilon = 0.01_f32;

        let sum_squared_manual = 100_f32.mul_add(100_f32, 115_f32 * 115_f32);

        assert_eq!(additional_sample_size, 2);
        assert_eq!(failure_count, 1);
        assert_eq!(latest_block_id, Some(16));
        assert!((cumulative_latency - 215_f32).abs() < epsilon);
        assert!((cumulative_latency_squares - sum_squared_manual).abs() < epsilon);
    }

    #[test]
    fn cumulative_avg_test() {
        let mut sample = vec![100_f32; 40];

        #[expect(clippy::as_conversions, reason = "int to float conversion is safe")]
        let old_sample_size_f = sample.len() as f32;

        let old_avg = sample.iter().sum::<f32>() / old_sample_size_f;

        let new_samples = vec![
            101_f32, 110_f32, 112_f32, 97_f32, 78_f32, 25_f32, 75_f32, 189_f32, 120_f32, 50_f32,
        ];

        #[expect(clippy::as_conversions, reason = "int to float conversion is safe")]
        let mod_sample_size_f = new_samples.len() as f32;

        let cumulative = new_samples.iter().sum();

        sample.extend_from_slice(&new_samples);

        #[expect(clippy::as_conversions, reason = "int to float conversion is safe")]
        let new_sample_size_f = sample.len() as f32;

        let new_avg_1 = sample.iter().sum::<f32>() / new_sample_size_f;

        let new_avg_2 = cumulative_avg(old_avg, cumulative, old_sample_size_f, mod_sample_size_f);

        let epsilon = 0.01_f32;

        assert!((new_avg_1 - new_avg_2).abs() < epsilon);
    }

    #[test]
    fn cumulative_var_test() {
        let mut sample = vec![100_f32; 40];

        #[expect(clippy::as_conversions, reason = "int to float conversion is safe")]
        let old_sample_size_f = sample.len() as f32;
        let old_avg = sample.iter().sum::<f32>() / old_sample_size_f;

        let old_var = sample
            .iter()
            .fold(0_f32, |acc, x| (x - old_avg).mul_add(x - old_avg, acc))
            / old_sample_size_f;

        let new_samples = vec![
            101_f32, 110_f32, 112_f32, 97_f32, 78_f32, 25_f32, 75_f32, 189_f32, 120_f32, 50_f32,
        ];

        #[expect(clippy::as_conversions, reason = "int to float conversion is safe")]
        let mod_sample_size_f = new_samples.len() as f32;

        let cumulative = new_samples.iter().sum();
        let cumulative_squares = new_samples
            .iter()
            .fold(0_f32, |acc, x| (*x).mul_add(*x, acc));

        let new_avg = cumulative_avg(old_avg, cumulative, old_sample_size_f, mod_sample_size_f);

        sample.extend_from_slice(&new_samples);

        #[expect(clippy::as_conversions, reason = "int to float conversion is safe")]
        let new_var_1 = sample
            .iter()
            .fold(0_f32, |acc, x| (x - new_avg).mul_add(x - new_avg, acc))
            / (sample.len() as f32);

        let new_var_2 = cumulative_var(
            old_avg,
            new_avg,
            old_var,
            cumulative,
            cumulative_squares,
            old_sample_size_f,
            mod_sample_size_f,
        );

        let epsilon = 0.01_f32;

        assert!((new_var_1 - new_var_2).abs() < epsilon);
    }

    #[test]
    fn metric_updates_correctness() {
        let statistics_updates_vec = vec![
            StatisticsUpdate {
                latency: 100_f32,
                new_latest_block_id: Some(105),
                is_failed: false,
            },
            StatisticsUpdate {
                latency: 115_f32,
                new_latest_block_id: Some(106),
                is_failed: false,
            },
            StatisticsUpdate {
                latency: 50_f32,
                new_latest_block_id: None,
                is_failed: true,
            },
        ];

        let addr_leader = Url::parse("https://127.0.0.1:3040").unwrap();

        let leader_statistics = Statistics {
            latency_avg: 100_f32,
            latency_var: 25_f32,
            sample_size: 10,
            latest_block_id: 100,
            errors: 5,
        };

        let cumulative_latency = 100_f32 + 115_f32;
        let cumulative_latency_squares = 100_f32.mul_add(100_f32, 115_f32 * 115_f32);

        let avg_manual = cumulative_avg(
            leader_statistics.latency_avg,
            cumulative_latency,
            10_f32,
            2_f32,
        );
        let var_manual = cumulative_var(
            leader_statistics.latency_avg,
            avg_manual,
            leader_statistics.latency_var,
            cumulative_latency,
            cumulative_latency_squares,
            10_f32,
            2_f32,
        );

        let mut metric_map = HashMap::new();
        metric_map.insert(addr_leader.clone(), leader_statistics);

        update_statistics(&mut metric_map, &addr_leader, &statistics_updates_vec).unwrap();

        let Statistics {
            latency_avg,
            latency_var,
            sample_size,
            latest_block_id,
            errors,
        } = metric_map[&addr_leader];

        let epsilon = 0.01_f32;

        assert_eq!(errors, 6);
        assert_eq!(latest_block_id, 106);
        assert_eq!(sample_size, 12);
        assert!((latency_avg - avg_manual).abs() < epsilon);
        assert!((latency_var - var_manual).abs() < epsilon);
    }

    #[test]
    fn choose_leader_latest_block() {
        let addr_leader = Url::parse("http://127.0.0.1:3040").unwrap();
        let addr_1 = Url::parse("http://127.0.0.1:3041").unwrap();
        let addr_2 = Url::parse("http://127.0.0.1:3042").unwrap();
        let addr_3 = Url::parse("http://127.0.0.1:3043").unwrap();

        let leader = client_from_url_unchecked(&addr_leader);
        let client_1 = client_from_url_unchecked(&addr_1);
        let client_2 = client_from_url_unchecked(&addr_2);
        let client_3 = client_from_url_unchecked(&addr_3);

        let mut client_list = HashMap::new();

        client_list.insert(addr_leader.clone(), leader);
        client_list.insert(addr_1.clone(), client_1);
        client_list.insert(addr_2.clone(), client_2);
        client_list.insert(addr_3.clone(), client_3);

        let mut statistics = HashMap::new();

        statistics.insert(
            addr_3,
            Statistics {
                latency_avg: 100_f32,
                latency_var: 10_f32,
                sample_size: 10,
                latest_block_id: 97,
                errors: 5,
            },
        );

        statistics.insert(
            addr_2,
            Statistics {
                latency_avg: 100_f32,
                latency_var: 10_f32,
                sample_size: 10,
                latest_block_id: 98,
                errors: 5,
            },
        );

        statistics.insert(
            addr_1,
            Statistics {
                latency_avg: 100_f32,
                latency_var: 10_f32,
                sample_size: 10,
                latest_block_id: 99,
                errors: 5,
            },
        );

        statistics.insert(
            addr_leader.clone(),
            Statistics {
                latency_avg: 100_f32,
                latency_var: 10_f32,
                sample_size: 10,
                latest_block_id: 100,
                errors: 5,
            },
        );

        let (leader_url, _) = choose_leader(&client_list, &statistics).unwrap();

        assert_eq!(leader_url, addr_leader);
    }

    #[test]
    fn choose_leader_least_errors() {
        let addr_leader = Url::parse("http://127.0.0.1:3040").unwrap();
        let addr_1 = Url::parse("http://127.0.0.1:3041").unwrap();
        let addr_2 = Url::parse("http://127.0.0.1:3042").unwrap();
        let addr_3 = Url::parse("http://127.0.0.1:3043").unwrap();

        let leader = client_from_url_unchecked(&addr_leader);
        let client_1 = client_from_url_unchecked(&addr_1);
        let client_2 = client_from_url_unchecked(&addr_2);
        let client_3 = client_from_url_unchecked(&addr_3);

        let mut client_list = HashMap::new();

        client_list.insert(addr_leader.clone(), leader);
        client_list.insert(addr_1.clone(), client_1);
        client_list.insert(addr_2.clone(), client_2);
        client_list.insert(addr_3.clone(), client_3);

        let mut statistics = HashMap::new();

        statistics.insert(
            addr_3,
            Statistics {
                latency_avg: 100_f32,
                latency_var: 10_f32,
                sample_size: 10,
                latest_block_id: 100,
                errors: 5,
            },
        );

        statistics.insert(
            addr_2,
            Statistics {
                latency_avg: 100_f32,
                latency_var: 10_f32,
                sample_size: 10,
                latest_block_id: 100,
                errors: 4,
            },
        );

        statistics.insert(
            addr_1,
            Statistics {
                latency_avg: 100_f32,
                latency_var: 10_f32,
                sample_size: 10,
                latest_block_id: 100,
                errors: 3,
            },
        );

        statistics.insert(
            addr_leader.clone(),
            Statistics {
                latency_avg: 100_f32,
                latency_var: 10_f32,
                sample_size: 10,
                latest_block_id: 100,
                errors: 2,
            },
        );

        let (leader_url, _) = choose_leader(&client_list, &statistics).unwrap();

        assert_eq!(leader_url, addr_leader);
    }

    #[test]
    fn choose_leader_simple_latency_check() {
        let addr_leader = Url::parse("http://127.0.0.1:3040").unwrap();
        let addr_1 = Url::parse("http://127.0.0.1:3041").unwrap();
        let addr_2 = Url::parse("http://127.0.0.1:3042").unwrap();
        let addr_3 = Url::parse("http://127.0.0.1:3043").unwrap();

        let leader = client_from_url_unchecked(&addr_leader);
        let client_1 = client_from_url_unchecked(&addr_1);
        let client_2 = client_from_url_unchecked(&addr_2);
        let client_3 = client_from_url_unchecked(&addr_3);

        let mut client_list = HashMap::new();

        client_list.insert(addr_leader.clone(), leader);
        client_list.insert(addr_1.clone(), client_1);
        client_list.insert(addr_2.clone(), client_2);
        client_list.insert(addr_3.clone(), client_3);

        let mut statistics = HashMap::new();

        statistics.insert(
            addr_3,
            Statistics {
                latency_avg: 103_f32,
                latency_var: 10_f32,
                sample_size: 10,
                latest_block_id: 100,
                errors: 5,
            },
        );

        statistics.insert(
            addr_2,
            Statistics {
                latency_avg: 102_f32,
                latency_var: 10_f32,
                sample_size: 10,
                latest_block_id: 100,
                errors: 5,
            },
        );

        statistics.insert(
            addr_1,
            Statistics {
                latency_avg: 101_f32,
                latency_var: 10_f32,
                sample_size: 10,
                latest_block_id: 100,
                errors: 5,
            },
        );

        statistics.insert(
            addr_leader.clone(),
            Statistics {
                latency_avg: 100_f32,
                latency_var: 10_f32,
                sample_size: 10,
                latest_block_id: 100,
                errors: 5,
            },
        );

        let (leader_url, _) = choose_leader(&client_list, &statistics).unwrap();

        assert_eq!(leader_url, addr_leader);
    }

    #[test]
    fn choose_leader_latency_var_check() {
        let addr_leader = Url::parse("http://127.0.0.1:3040").unwrap();
        let addr_1 = Url::parse("http://127.0.0.1:3041").unwrap();
        let addr_2 = Url::parse("http://127.0.0.1:3042").unwrap();
        let addr_3 = Url::parse("http://127.0.0.1:3043").unwrap();

        let leader = client_from_url_unchecked(&addr_leader);
        let client_1 = client_from_url_unchecked(&addr_1);
        let client_2 = client_from_url_unchecked(&addr_2);
        let client_3 = client_from_url_unchecked(&addr_3);

        let mut client_list = HashMap::new();

        client_list.insert(addr_leader.clone(), leader);
        client_list.insert(addr_1.clone(), client_1);
        client_list.insert(addr_2.clone(), client_2);
        client_list.insert(addr_3.clone(), client_3);

        let mut statistics = HashMap::new();

        statistics.insert(
            addr_3,
            Statistics {
                latency_avg: 100_f32,
                latency_var: 13_f32,
                sample_size: 10,
                latest_block_id: 100,
                errors: 5,
            },
        );

        statistics.insert(
            addr_2,
            Statistics {
                latency_avg: 100_f32,
                latency_var: 12_f32,
                sample_size: 10,
                latest_block_id: 100,
                errors: 5,
            },
        );

        statistics.insert(
            addr_1,
            Statistics {
                latency_avg: 100_f32,
                latency_var: 11_f32,
                sample_size: 10,
                latest_block_id: 100,
                errors: 5,
            },
        );

        statistics.insert(
            addr_leader.clone(),
            Statistics {
                latency_avg: 100_f32,
                latency_var: 10_f32,
                sample_size: 10,
                latest_block_id: 100,
                errors: 5,
            },
        );

        let (leader_url, _) = choose_leader(&client_list, &statistics).unwrap();

        assert_eq!(leader_url, addr_leader);
    }
}