test_: unskip test initialize logging (#6229)

* test_: add option to get file from status-backend container

* feat_: automatically create dataDir and logsDir directories

* test_: unskip and update TestInitializeLogging

* fix_: parametrize test

* ci_: use ms precision for func tests docker project_name to

* chore_: rename to extract_data

* fix_: linter

* fix_: set timestamp in python as well

_assets/scripts/run_functional_tests.sh

@@ -24,7 +24,8 @@ mkdir -p "${merged_coverage_reports_path}"
 mkdir -p "${test_results_path}"
 
 all_compose_files="-f ${root_path}/docker-compose.anvil.yml -f ${root_path}/docker-compose.test.status-go.yml"
-project_name="status-go-func-tests-$(date +%s)"
+timestamp=$(python3 -c "import time; print(int(time.time() * 1000))") # Keep in sync with status_backend.py
+project_name="status-go-func-tests-${timestamp}"
 
 export STATUS_BACKEND_URLS=$(eval echo http://${project_name}-status-backend-{1..${STATUS_BACKEND_COUNT}}:3333 | tr ' ' ,)
 

mobile/status.go

@@ -5,7 +5,10 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
+	"path"
 	"runtime"
+	"time"
 	"unsafe"
 
 	"go.uber.org/zap"
@@ -29,6 +32,7 @@ import (
 	"github.com/status-im/status-go/images"
 	"github.com/status-im/status-go/logutils"
 	"github.com/status-im/status-go/logutils/requestlog"
+	"github.com/status-im/status-go/mobile/callog"
 	m_requests "github.com/status-im/status-go/mobile/requests"
 	"github.com/status-im/status-go/multiaccounts"
 	"github.com/status-im/status-go/multiaccounts/accounts"
@@ -49,11 +53,6 @@ import (
 	"github.com/status-im/status-go/services/typeddata"
 	"github.com/status-im/status-go/services/wallet/wallettypes"
 	"github.com/status-im/status-go/signal"
-
-	"path"
-	"time"
-
-	"github.com/status-im/status-go/mobile/callog"
 )
 
 func call(fn any, params ...any) any {
@@ -106,6 +105,11 @@ func initializeApplication(requestJSON string) string {
 	providers.MixpanelAppID = request.MixpanelAppID
 	providers.MixpanelToken = request.MixpanelToken
 
+	err = os.MkdirAll(request.DataDir, 0700)
+	if err != nil {
+		return makeJSONResponse(err)
+	}
+
 	statusBackend.StatusNode().SetMediaServerEnableTLS(request.MediaServerEnableTLS)
 	statusBackend.UpdateRootDataDir(request.DataDir)
 
@@ -153,7 +157,12 @@ func initializeLogging(request *requests.InitializeApplication) error {
 		File:    path.Join(request.LogDir, api.DefaultLogFile),
 	}
 
-	err := logutils.OverrideRootLoggerWithConfig(logSettings)
+	err := os.MkdirAll(request.LogDir, 0700)
+	if err != nil {
+		return err
+	}
+
+	err = logutils.OverrideRootLoggerWithConfig(logSettings)
 	if err != nil {
 		return err
 	}

tests-functional/clients/status_backend.py

@@ -1,16 +1,19 @@
+import io
 import json
 import logging
+import tarfile
+import tempfile
 import time
 import random
 import threading
 import requests
 import docker
+import docker.errors
 import os
 
 from tenacity import retry, stop_after_delay, wait_fixed
 from clients.signals import SignalClient
 from clients.rpc import RpcClient
-from datetime import datetime
 from conftest import option
 from resources.constants import user_1, DEFAULT_DISPLAY_NAME, USER_DIR
 
@@ -19,6 +22,8 @@ NANOSECONDS_PER_SECOND = 1_000_000_000
 
 class StatusBackend(RpcClient, SignalClient):
 
+    container = None
+
     def __init__(self, await_signals=[]):
 
         if option.status_backend_url:
@@ -48,7 +53,7 @@ class StatusBackend(RpcClient, SignalClient):
     def _start_container(self, host_port):
         docker_project_name = option.docker_project_name
 
-        timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
+        timestamp = int(time.time() * 1000)  # Keep in sync with run_functional_tests.sh
         image_name = f"{docker_project_name}-status-backend:latest"
         container_name = f"{docker_project_name}-status-backend-{timestamp}"
 
@@ -153,6 +158,27 @@ class StatusBackend(RpcClient, SignalClient):
         data["StatusProxyStageName"] = "test"
         return data
 
+    def extract_data(self, path: str):
+        if not self.container:
+            return path
+
+        try:
+            stream, _ = self.container.get_archive(path)
+        except docker.errors.NotFound:
+            return None
+
+        temp_dir = tempfile.mkdtemp()
+        tar_bytes = io.BytesIO(b"".join(stream))
+
+        with tarfile.open(fileobj=tar_bytes) as tar:
+            tar.extractall(path=temp_dir)
+            # If the tar contains a single file, return the path to that file
+            # Otherwise it's a directory, just return temp_dir.
+            if len(tar.getmembers()) == 1:
+                return os.path.join(temp_dir, tar.getmembers()[0].name)
+
+        return temp_dir
+
     def create_account_and_login(
         self,
         data_dir=USER_DIR,

tests-functional/tests/test_init_status_app.py

@@ -1,3 +1,4 @@
+from resources.constants import USER_DIR
 from test_cases import StatusBackend
 import pytest
 from clients.signals import SignalType
@@ -41,49 +42,42 @@ class TestInitialiseApp:
         )
 
 
+def assert_file_first_line(path, pattern: str, expected: bool):
+    if not expected:
+        assert path is None
+        return
+    assert os.path.exists(path)
+    with open(path) as file:
+        line = file.readline()
+        line_found = line.find(pattern) >= 0
+        assert line_found == expected
+
+
 @pytest.mark.rpc
-@pytest.mark.skip("waiting for status-backend to be executed on the same host/container")
+@pytest.mark.init
-class TestInitializeLogging:
+@pytest.mark.parametrize("log_enabled,api_logging_enabled", [(True, True), (False, False)])
+def test_check_logs(log_enabled: bool, api_logging_enabled: bool):
+    data_dir = os.path.join(USER_DIR, "data")
+    logs_dir = os.path.join(USER_DIR, "logs")
 
-    @pytest.mark.init
+    backend = StatusBackend()
-    def test_init_logging(self, tmp_path):
+    backend.api_valid_request(
-        self.check_logs(tmp_path, log_enabled=True, api_logging_enabled=True)
+        "InitializeApplication",
+        {
+            "dataDir": str(data_dir),
+            "logDir": str(logs_dir),
+            "logEnabled": log_enabled,
+            "apiLoggingEnabled": api_logging_enabled,
+        },
+    )
 
-    @pytest.mark.init
+    local_geth_log = backend.extract_data(os.path.join(logs_dir, "geth.log"))
-    def test_no_logging(self, tmp_path):
+    local_api_log = backend.extract_data(os.path.join(logs_dir, "api.log"))
-        self.check_logs(tmp_path, log_enabled=False, api_logging_enabled=False)
 
-    def assert_file_first_line(self, path, pattern: str, expected: bool):
+    assert_file_first_line(path=local_geth_log, pattern="logging initialised", expected=log_enabled)
-        assert os.path.exists(path) == expected
-        if not expected:
-            return
-        with open(path) as file:
-            line = file.readline()
-            line_found = line.find(pattern) >= 0
-            assert line_found == expected
 
-    def check_logs(self, path, log_enabled: bool, api_logging_enabled: bool):
+    assert_file_first_line(
-        data_dir = path / "data"
+        path=local_api_log,
-        logs_dir = path / "logs"
+        pattern='"method": "InitializeApplication"',
-
+        expected=api_logging_enabled,
-        data_dir.mkdir()
+    )
-        logs_dir.mkdir()
-
-        backend = StatusBackend()
-        backend.api_valid_request(
-            "InitializeApplication",
-            {
-                "dataDir": str(data_dir),
-                "logDir": str(logs_dir),
-                "logEnabled": log_enabled,
-                "apiLoggingEnabled": api_logging_enabled,
-            },
-        )
-
-        self.assert_file_first_line(logs_dir / "geth.log", pattern="logging initialised", expected=log_enabled)
-
-        self.assert_file_first_line(
-            logs_dir / "api.log",
-            pattern='"method": "InitializeApplication"',
-            expected=api_logging_enabled,
-        )