2023-06-25 07:53:10 +00:00
using DistTestCore.Codex ;
using DistTestCore ;
using Logging ;
namespace ContinuousTests
{
public class StartupChecker
{
private readonly TestFactory testFactory = new TestFactory ( ) ;
2023-06-29 14:07:49 +00:00
private readonly CodexAccessFactory codexNodeFactory = new CodexAccessFactory ( ) ;
2023-06-25 07:53:10 +00:00
private readonly Configuration config ;
2023-07-07 06:52:53 +00:00
private readonly CancellationToken cancelToken ;
2023-06-25 07:53:10 +00:00
2023-07-07 06:52:53 +00:00
public StartupChecker ( Configuration config , CancellationToken cancelToken )
2023-06-25 07:53:10 +00:00
{
this . config = config ;
2023-07-07 06:52:53 +00:00
this . cancelToken = cancelToken ;
2023-08-30 08:57:20 +00:00
LogReplacements = new List < BaseLogStringReplacement > ( ) ;
2023-06-25 07:53:10 +00:00
}
public void Check ( )
{
2023-07-18 07:47:44 +00:00
var log = new FixtureLog ( new LogConfig ( config . LogPath , false ) , DateTime . UtcNow , "StartupChecks" ) ;
2023-06-25 07:53:10 +00:00
log . Log ( "Starting continuous test run..." ) ;
log . Log ( "Checking configuration..." ) ;
PreflightCheck ( config ) ;
log . Log ( "Contacting Codex nodes..." ) ;
CheckCodexNodes ( log , config ) ;
log . Log ( "All OK." ) ;
}
2023-08-30 08:57:20 +00:00
public List < BaseLogStringReplacement > LogReplacements { get ; }
2023-06-25 07:53:10 +00:00
private void PreflightCheck ( Configuration config )
{
var tests = testFactory . CreateTests ( ) ;
if ( ! tests . Any ( ) )
{
throw new Exception ( "Unable to find any tests." ) ;
}
foreach ( var test in tests )
{
2023-07-07 06:52:53 +00:00
cancelToken . ThrowIfCancellationRequested ( ) ;
2023-06-25 07:53:10 +00:00
var handle = new TestHandle ( test ) ;
handle . GetEarliestMoment ( ) ;
handle . GetLastMoment ( ) ;
}
if ( ! Directory . Exists ( config . LogPath ) )
{
Directory . CreateDirectory ( config . LogPath ) ;
}
2023-06-28 09:01:10 +00:00
var errors = CheckTests ( tests ) ;
2023-06-25 07:53:10 +00:00
if ( errors . Any ( ) )
{
throw new Exception ( "Prerun check failed: " + string . Join ( ", " , errors ) ) ;
}
}
private void CheckCodexNodes ( BaseLog log , Configuration config )
{
2023-06-30 08:08:51 +00:00
var nodes = codexNodeFactory . Create ( config , config . CodexDeployment . CodexContainers , log , new DefaultTimeSet ( ) ) ;
2023-06-25 07:53:10 +00:00
var pass = true ;
foreach ( var n in nodes )
{
2023-07-07 06:52:53 +00:00
cancelToken . ThrowIfCancellationRequested ( ) ;
2023-07-13 07:20:24 +00:00
log . Log ( $"Checking {n.Container.Name} @ '{n.Address.Host}:{n.Address.Port}'..." ) ;
2023-06-25 07:53:10 +00:00
2023-07-13 07:20:24 +00:00
if ( EnsureOnline ( log , n ) )
2023-06-25 07:53:10 +00:00
{
log . Log ( "OK" ) ;
}
else
{
log . Error ( $"No response from '{n.Address.Host}'." ) ;
pass = false ;
}
}
if ( ! pass )
{
throw new Exception ( "Not all codex nodes responded." ) ;
}
}
2023-07-13 07:20:24 +00:00
private bool EnsureOnline ( BaseLog log , CodexAccess n )
2023-06-25 07:53:10 +00:00
{
try
{
var info = n . GetDebugInfo ( ) ;
if ( info = = null | | string . IsNullOrEmpty ( info . id ) ) return false ;
2023-07-13 07:20:24 +00:00
log . Log ( $"Codex version: '{info.codex.version}' revision: '{info.codex.revision}'" ) ;
2023-08-30 08:57:20 +00:00
LogReplacements . Add ( new BaseLogStringReplacement ( info . id , n . GetName ( ) ) ) ;
2023-06-25 07:53:10 +00:00
}
catch
{
return false ;
}
return true ;
}
2023-06-28 09:01:10 +00:00
private List < string > CheckTests ( ContinuousTest [ ] tests )
{
var errors = new List < string > ( ) ;
CheckRequiredNumberOfNodes ( tests , errors ) ;
CheckCustomNamespaceClashes ( tests , errors ) ;
CheckEthereumIndexClashes ( tests , errors ) ;
return errors ;
}
private void CheckEthereumIndexClashes ( ContinuousTest [ ] tests , List < string > errors )
{
var offLimits = config . CodexDeployment . CodexContainers . Length ;
foreach ( var test in tests )
{
if ( test . EthereumAccountIndex ! = - 1 )
{
if ( test . EthereumAccountIndex < = offLimits )
{
errors . Add ( $"Test '{test.Name}' has selected 'EthereumAccountIndex' = {test.EthereumAccountIndex}. All accounts up to and including {offLimits} are being used by the targetted Codex net. Select a different 'EthereumAccountIndex'." ) ;
}
}
}
DuplicatesCheck ( tests , errors ,
considerCondition : t = > t . EthereumAccountIndex ! = - 1 ,
getValue : t = > t . EthereumAccountIndex ,
propertyName : nameof ( ContinuousTest . EthereumAccountIndex ) ) ;
}
private void CheckCustomNamespaceClashes ( ContinuousTest [ ] tests , List < string > errors )
{
DuplicatesCheck ( tests , errors ,
considerCondition : t = > ! string . IsNullOrEmpty ( t . CustomK8sNamespace ) ,
getValue : t = > t . CustomK8sNamespace ,
propertyName : nameof ( ContinuousTest . CustomK8sNamespace ) ) ;
}
private void DuplicatesCheck ( ContinuousTest [ ] tests , List < string > errors , Func < ContinuousTest , bool > considerCondition , Func < ContinuousTest , object > getValue , string propertyName )
{
foreach ( var test in tests )
{
if ( considerCondition ( test ) )
{
var duplicates = tests . Where ( t = > t ! = test & & getValue ( t ) = = getValue ( test ) ) . ToList ( ) ;
if ( duplicates . Any ( ) )
{
duplicates . Add ( test ) ;
errors . Add ( $"Tests '{string.Join(" , ", duplicates.Select(d => d.Name))}' have the same '{propertyName}'. These must be unique." ) ;
return ;
}
}
}
}
private void CheckRequiredNumberOfNodes ( ContinuousTest [ ] tests , List < string > errors )
{
foreach ( var test in tests )
{
2023-08-29 14:08:40 +00:00
if ( test . RequiredNumberOfNodes ! = - 1 )
2023-06-28 09:01:10 +00:00
{
2023-08-29 14:08:40 +00:00
if ( test . RequiredNumberOfNodes < 1 )
{
errors . Add ( $"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Test must require > 0 nodes, or -1 to select all nodes." ) ;
}
else if ( test . RequiredNumberOfNodes > config . CodexDeployment . CodexContainers . Length )
{
errors . Add ( $"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexContainers.Length}" ) ;
}
2023-06-28 09:01:10 +00:00
}
}
}
2023-06-25 07:53:10 +00:00
}
}