feat_: add Sentry panic reporting (#6054)

* feat_: report panics to sentry

* test_: sentry options, params and utils

* feat_: toggle sentry with centralized metrics

* test_: sentry init, report and close

* refactor_: rename public api to generic

* docs_: sentry

* fix_: typo in internal/sentry/README.md

Co-authored-by: osmaczko <33099791+osmaczko@users.noreply.github.com>

* fix_: linter

---------

Co-authored-by: osmaczko <33099791+osmaczko@users.noreply.github.com>

.gitignore

@@ -119,6 +119,9 @@ cmd/status-backend/server/endpoints.go
 protocol/messenger_handlers.go
 internal/version/VERSION
 internal/version/GIT_COMMIT
+internal/sentry/SENTRY_CONTEXT_NAME
+internal/sentry/SENTRY_CONTEXT_VERSION
+internal/sentry/SENTRY_PRODUCTION
 
 # Don't ignore generated files in the vendor/ directory
 !vendor/**/*.pb.go

api/geth_backend.go

@@ -35,6 +35,7 @@ import (
 	"github.com/status-im/status-go/eth-node/crypto"
 	"github.com/status-im/status-go/eth-node/types"
 	"github.com/status-im/status-go/images"
+	"github.com/status-im/status-go/internal/sentry"
 	"github.com/status-im/status-go/internal/version"
 	"github.com/status-im/status-go/logutils"
 	"github.com/status-im/status-go/multiaccounts"
@@ -101,6 +102,7 @@ type GethStatusBackend struct {
 	allowAllRPC              bool // used only for tests, disables api method restrictions
 	LocalPairingStateManager *statecontrol.ProcessStateManager
 	centralizedMetrics       *centralizedmetrics.MetricService
+	sentryDSN                string
 
 	logger *zap.Logger
 }
@@ -2119,6 +2121,7 @@ func (b *GethStatusBackend) startNode(config *params.NodeConfig) (err error) {
 	defer func() {
 		if r := recover(); r != nil {
 			err = fmt.Errorf("node crashed on start: %v", err)
+			sentry.RecoverError(err)
 		}
 	}()
 
@@ -2845,3 +2848,25 @@ func (b *GethStatusBackend) getWalletDBPath(keyUID string) (string, error) {
 
 	return filepath.Join(b.rootDataDir, fmt.Sprintf("%s-wallet.db", keyUID)), nil
 }
+
+func (b *GethStatusBackend) SetSentryDSN(dsn string) {
+	b.sentryDSN = dsn
+}
+
+func (b *GethStatusBackend) EnablePanicReporting() error {
+	return sentry.Init(
+		sentry.WithDSN(b.sentryDSN),
+		sentry.WithDefaultContext(),
+	)
+}
+
+func (b *GethStatusBackend) DisablePanicReporting() error {
+	return sentry.Close()
+}
+
+func (b *GethStatusBackend) TogglePanicReporting(enabled bool) error {
+	if enabled {
+		return b.EnablePanicReporting()
+	}
+	return b.DisablePanicReporting()
+}

cmd/status-backend/main.go

@@ -10,12 +10,13 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 
 	"github.com/status-im/status-go/cmd/status-backend/server"
+	"github.com/status-im/status-go/internal/sentry"
 	"github.com/status-im/status-go/internal/version"
 	"github.com/status-im/status-go/logutils"
 )
 
 var (
-	address = flag.String("address", "", "host:port to listen")
+	address = flag.String("address", "127.0.0.1:0", "host:port to listen")
 	logger  = log.New("package", "status-go/cmd/status-backend")
 )
 
@@ -32,6 +33,12 @@ func init() {
 }
 
 func main() {
+	sentry.MustInit(
+		sentry.WithDefaultEnvironmentDSN(),
+		sentry.WithContext("status-backend", version.Version()),
+	)
+	defer sentry.Recover()
+
 	flag.Parse()
 
 	srv := server.NewServer()

common/utils.go

@@ -10,6 +10,7 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/status-im/status-go/eth-node/crypto"
+	"github.com/status-im/status-go/internal/sentry"
 	"github.com/status-im/status-go/logutils"
 	"github.com/status-im/status-go/protocol/identity/alias"
 	"github.com/status-im/status-go/protocol/protobuf"
@@ -89,8 +90,16 @@ func IsNil(i interface{}) bool {
 }
 
 func LogOnPanic() {
-	if err := recover(); err != nil {
+	err := recover()
-		logutils.ZapLogger().Error("panic in goroutine", zap.Any("error", err), zap.Stack("stacktrace"))
+	if err == nil {
+		return
+	}
+
+	logutils.ZapLogger().Error("panic in goroutine",
+		zap.Any("error", err),
+		zap.Stack("stacktrace"))
+
+	sentry.RecoverError(err)
+
 	panic(err)
 }
-}

internal/sentry/README.md

@@ -0,0 +1,110 @@
+# Description
+
+This package encapsulates Sentry integration. So far:
+- only for status-go (including when running as part of desktop and mobile)
+- only for panics (no other error reporting)
+
+Sentry is only enabled for users that **both**:
+1. Opted-in for metrics
+2. Use builds from our release CI
+
+## 🛬 Where
+
+We use self-hosted Sentry: https://sentry.infra.status.im/
+
+## 🕐 When
+
+### Which panics are reported:
+
+- When running inside `status-desktop`/`status-mobile`:
+  - during API calls in `/mobile/status.go`
+  - inside all goroutines
+- When running `status-backend`:
+  - any panic
+
+### Which panics are NOT reported:
+
+- When running inside `status-desktop`/`status-mobile`:
+  - during API calls in `/services/**/api.go` \
+    NOTE: These endpoints are executed through `go-ethereum`'s JSON-RPC server, which internally recovers all panics and doesn't provide any events or option to set an interceptor. 
+    The only way to catch these panics is to replace the JSON-RPC server implementation.
+- When running `status-go` unit tests:
+  - any panic \
+    NOTE: Go internally runs tests in a goroutine. The only way to catch these panics in tests is to manually `defer sentry.Recover()` in each test. This also requires a linter (similar to `lint-panics`) that checks this call is present. \
+    This is not a priority right now, because:
+    1. We have direct access to failed tests logs, which contain the panic stacktrace.
+    2. Panics are not expected to happen. Test must be passing to be able to merge the code. So it's only possible with a flaky test.
+    3. This would only apply to nightly/develop jobs, as we don't gather panic reports from PR-level jobs.
+    
+
+## 📦 What
+
+Full list can be found in `sentry.Event`. 
+
+Notes regarding identity-disclosing properties:
+- `ServerName` - completely removed from all events
+- `Stacktrace`:
+  - No private user paths are exposed, as we only gather reports from CI-built binaries.
+- `TraceID` - so far will be unique for each event
+  >Trace: A collection of spans representing the end-to-end journey of a request through your system that all share the same trace ID.
+
+  More details in [sentry docs](https://docs.sentry.io/product/explore/traces/#key-concepts).
+
+# Configuration
+
+There are 2 main tags to identify the error. The configuration is a bit complicated, but provides full information.
+
+## Parameters
+
+### `Environment`
+
+|                   |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
+|-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Defining question | Where it is running?                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
+| Set time          | - `production` can only be set at build time to prevent users from hacking the environment<br>- All others can be set at runtime, because on CI we sometimes use same build for multiple environments                                                                                                                                                                                                                                                                                  |
+| Expected values   | <table><thead><tr><th>Value</th><th>Description</th></tr></thead><tr><td>`production`</td><td>End user machine</td></tr><tr><td>~~`development`~~</td><td>Developer machine</td></tr><tr><td>~~`ci-pr`~~</td><td>PR-level CI runs</td></tr><tr><td>`ci-main`</td><td>CI runs for stable branch</td></tr><tr><td>`ci-nightly`</td><td>CI nightly jobs on stable branch</td></tr></table>`development` and `ci-pr` are dropped, because we only want to consider panics from stable code |
+
+### `Context`
+
+
+|                   |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
+|-------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Defining question | What is the executable for the library?                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
+| Set time          | Always at build-time                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
+| Expected values   | <table><thead><tr><th>Value</th><th>Running as...</th></tr></thead><tr><td>`status-desktop`</td><td>Library embedded into [status-desktop](https://github.com/status-im/status-desktop)</td></tr><tr><td>`status-mobile`</td><td>Library embedded into [status-mobile](https://github.com/status-im/status-mobile)</td></tr><tr><td>`status-backend`</td><td>Part of `cmd/status-backend`<br>Can be other `cmd/*` as well.</td></tr><tr><td>`matterbridge`</td><td>Part of [Status/Discord bridge app](https://github.com/status-im/matterbridge)</td></tr><tr><td>`status-go-tests`</td><td>Inside status-go tests</td></tr></table> |
+
+
+## Environment variables
+
+To cover these requirements, I added these environment variables:
+
+| Environment variable                              | Provide time                                                                                            | Description                                                                                                                                             |
+|---------------------------------------------------|---------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `SENTRY_DSN`                                      | - At build time with direct call to `sentry.Init`<br>- At runtime with `InitializeApplication` endpoint | [Sentry DSN](https://docs.sentry.io/concepts/key-terms/dsn-explainer/) to be used                                                                       |
+| `SENTRY_CONTEXT_NAME`<br>`SENTRY_CONTEXT_VERSION` | Build time                                                                                              | Execution context of status-go                                                                                                                          |
+| `SENTRY_PRODUCTION`                               | Build time                                                                                              | When `true` or `1`:<br>-Defines if this is a production build<br>-Sets environment to `production`<br>-Has precedence over runtime `SENTRY_ENVIRONMENT` |
+| `SENTRY_ENVIRONMENT`                              | Run time                                                                                                | Sets the environment. Has no effect when `SENTRY_PRODUCTION` is set                                                                                    |
+
+# Client integration
+
+1. Set `SENTRY_CONTEXT_NAME` and `SENTRY_CONTEXT_VERSION` at status-go build time
+2. Provide `sentryDSN` to the `InitializeApplication` call.
+   DSN must be kept private and will be provided by CI. Expect a `STATUS_GO_SENTRY_DSN` environment variable to be provided by CI. 
+
+    <details>
+        <summary>Why can't we consume `STATUS_GO_SENTRY_DSN` directly in status-go build?</summary>
+
+        In theory, we could. But this would require us to mix approaches of getting the env variable to the code.
+        Right now we prefer `go:generate + go:embed` approach (e.g. https://github.com/status-im/status-go/pull/6014), but we can't do it in this case, because we must not write the DSN to a local file, which would be a bit vulnerable. And I don't want to go back to `-ldflags="-X github.com/status-im/status-go/internal/sentry.sentryDSN=$(STATUS_GO_SENTRY_DSN:v%=%)` approach.
+    </details>
+
+# Implementation details
+
+- We recover from panics in here:
+  https://github.com/status-im/status-go/blob/fcedb013166785e7def8710118086f4b650c33b1/common/utils.go#L102 https://github.com/status-im/status-go/blob/fcedb013166785e7def8710118086f4b650c33b1/mobile/callog/status_request_log.go#L69 https://github.com/status-im/status-go/blob/fcedb013166785e7def8710118086f4b650c33b1/cmd/status-backend/main.go#L40
+  This covers all goroutines, because we have a linter to check that all goroutines have `defer common.LogOnPanic`.
+- Sentry is currently initialized in 2 places:
+    - `InitializeApplication` - covers desktop/mobile clients
+      https://github.com/status-im/status-go/blob/fcedb013166785e7def8710118086f4b650c33b1/mobile/status.go#L105-L108
+    - in `status-backend` - covers functional tests:
+      https://github.com/status-im/status-go/blob/fcedb013166785e7def8710118086f4b650c33b1/cmd/status-backend/main.go#L36-L39

internal/sentry/options.go

@@ -0,0 +1,46 @@
+package sentry
+
+import (
+	"os"
+
+	"github.com/getsentry/sentry-go"
+)
+
+const DefaultEnvVarDSN = "SENTRY_DSN_STATUS_GO"
+const DefaultEnvVarEnvironment = "SENTRY_ENVIRONMENT"
+
+type Option func(*sentry.ClientOptions)
+
+func WithDSN(dsn string) Option {
+	return func(o *sentry.ClientOptions) {
+		o.Dsn = dsn
+	}
+}
+
+func WithEnvironmentDSN(name string) Option {
+	return WithDSN(os.Getenv(name))
+}
+
+func WithDefaultEnvironmentDSN() Option {
+	return WithEnvironmentDSN(DefaultEnvVarDSN)
+}
+
+func WithContext(name string, version string) Option {
+	return func(o *sentry.ClientOptions) {
+		if o.Tags == nil {
+			o.Tags = make(map[string]string)
+		}
+		o.Tags["context.name"] = name
+		o.Tags["context.version"] = version
+	}
+}
+
+func WithDefaultContext() Option {
+	return WithContext(DefaultContext(), DefaultContextVersion())
+}
+
+func applyOptions(cfg *sentry.ClientOptions, opts ...Option) {
+	for _, opt := range opts {
+		opt(cfg)
+	}
+}

internal/sentry/options_test.go

@@ -0,0 +1,70 @@
+package sentry
+
+import (
+	"os"
+	"testing"
+
+	"github.com/brianvoe/gofakeit/v6"
+	"github.com/getsentry/sentry-go"
+	"github.com/stretchr/testify/require"
+)
+
+func TestWithDSN(t *testing.T) {
+	t.Parallel()
+
+	dsn := "https://examplePublicKey@o0.ingest.sentry.io/0"
+	option := WithDSN(dsn)
+	cfg := &sentry.ClientOptions{}
+	option(cfg)
+	require.Equal(t, dsn, cfg.Dsn)
+}
+
+func TestWithEnvironmentDSN(t *testing.T) {
+	t.Parallel()
+
+	expectedDSN := gofakeit.LetterN(10)
+	envVar := gofakeit.LetterN(10)
+
+	err := os.Setenv(envVar, expectedDSN)
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		err = os.Unsetenv(envVar)
+		require.NoError(t, err)
+	})
+
+	option := WithEnvironmentDSN(envVar)
+	cfg := &sentry.ClientOptions{}
+	option(cfg)
+	require.Equal(t, expectedDSN, cfg.Dsn)
+}
+
+func TestWithContext(t *testing.T) {
+	t.Parallel()
+
+	name := "test-context"
+	version := "v1.0.0"
+	option := WithContext(name, version)
+	cfg := &sentry.ClientOptions{}
+	option(cfg)
+	require.Equal(t, name, cfg.Tags["context.name"])
+	require.Equal(t, version, cfg.Tags["context.version"])
+}
+
+func TestApplyOptions(t *testing.T) {
+	t.Parallel()
+
+	dsn := gofakeit.LetterN(10)
+	name := gofakeit.LetterN(10)
+	version := gofakeit.LetterN(5)
+
+	options := []Option{
+		WithDSN(dsn),
+		WithContext(name, version),
+	}
+	cfg := &sentry.ClientOptions{}
+	applyOptions(cfg, options...)
+
+	require.Equal(t, dsn, cfg.Dsn)
+	require.Equal(t, name, cfg.Tags["context.name"])
+	require.Equal(t, version, cfg.Tags["context.version"])
+}

internal/sentry/params.go

@@ -0,0 +1,51 @@
+package sentry
+
+import (
+	_ "embed"
+	"os"
+)
+
+//go:generate sh -c "echo $SENTRY_CONTEXT_NAME > SENTRY_CONTEXT_NAME"
+//go:generate sh -c "echo $SENTRY_CONTEXT_VERSION > SENTRY_CONTEXT_VERSION"
+//go:generate sh -c "echo $SENTRY_PRODUCTION > SENTRY_PRODUCTION"
+
+const productionEnvironment = "production"
+
+var (
+	//go:embed SENTRY_CONTEXT_NAME
+	defaultContextName string
+
+	//go:embed SENTRY_CONTEXT_VERSION
+	defaultContextVersion string
+
+	//go:embed SENTRY_PRODUCTION
+	production string
+)
+
+func DefaultContext() string {
+	return defaultContextName
+}
+
+func DefaultContextVersion() string {
+	return defaultContextVersion
+}
+
+func Production() bool {
+	return production == "true" || production == "1"
+}
+
+func Environment() string {
+	return environment(Production(), DefaultEnvVarEnvironment)
+}
+
+func environment(production bool, envvar string) string {
+	if production {
+		return productionEnvironment
+	}
+	env := os.Getenv(envvar)
+	if env == productionEnvironment {
+		// Production environment can only be set during build
+		return ""
+	}
+	return env
+}

internal/sentry/params_test.go

@@ -0,0 +1,48 @@
+package sentry
+
+import (
+	"os"
+	"testing"
+
+	"github.com/brianvoe/gofakeit/v6"
+	"github.com/stretchr/testify/require"
+)
+
+func setEnvVar(t *testing.T, varName, value string) {
+	err := os.Setenv(varName, value)
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		err = os.Unsetenv(varName)
+		require.NoError(t, err)
+	})
+}
+
+func TestEnvironment(t *testing.T) {
+	t.Parallel()
+
+	t.Run("returns production environment when production is true", func(t *testing.T) {
+		varName := gofakeit.LetterN(10)
+		setEnvVar(t, varName, "development")
+
+		// Expect production although the env variable is set to development
+		result := environment(true, varName)
+		require.Equal(t, productionEnvironment, result)
+	})
+
+	t.Run("returns empty string when env is productionEnvironment", func(t *testing.T) {
+		varName := gofakeit.LetterN(10)
+		setEnvVar(t, varName, productionEnvironment)
+
+		result := environment(false, varName)
+		require.Equal(t, "", result)
+	})
+
+	t.Run("returns environment variable when production is false", func(t *testing.T) {
+		varName := gofakeit.LetterN(10)
+		expectedEnv := "development"
+		setEnvVar(t, varName, expectedEnv)
+
+		result := environment(false, varName)
+		require.Equal(t, expectedEnv, result)
+	})
+}

internal/sentry/sentry.go

@@ -1,6 +1,67 @@
 package sentry
 
 import (
-	_ "github.com/brianvoe/gofakeit/v6"
+	"time"
-	_ "github.com/getsentry/sentry-go"
+
+	"github.com/getsentry/sentry-go"
+
+	"github.com/status-im/status-go/internal/version"
 )
+
+func Init(opts ...Option) error {
+	cfg := defaultConfig()
+	applyOptions(cfg, opts...)
+	return sentry.Init(*cfg)
+}
+
+func MustInit(options ...Option) {
+	if err := Init(options...); err != nil {
+		panic(err)
+	}
+}
+
+func Close() error {
+	sentry.Flush(time.Second * 2)
+	// Set DSN to empty string to disable sending events
+	return sentry.Init(sentry.ClientOptions{
+		Dsn: "",
+	})
+}
+
+func Recover() {
+	err := recover()
+	if err == nil {
+		return
+	}
+	RecoverError(err)
+	panic(err)
+}
+
+func RecoverError(err interface{}) {
+	sentry.CurrentHub().Recover(err)
+	sentry.Flush(time.Second * 2)
+}
+
+func defaultConfig() *sentry.ClientOptions {
+	return &sentry.ClientOptions{
+		EnableTracing:  false,
+		Debug:          false,
+		SendDefaultPII: false,
+		Release:        version.Version(),
+		Environment:    Environment(),
+		Tags:           make(map[string]string),
+		BeforeSend:     beforeSend,
+	}
+}
+
+func beforeSend(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
+	event.Modules = nil   // Clear modules as we know all dependencies by commit hash
+	event.ServerName = "" // Clear server name as it might be sensitive
+
+	// Cleanup the stacktrace from last Recover/LogOnPanic frames
+	for _, exception := range event.Exception {
+		trimStacktrace(exception.Stacktrace)
+	}
+
+	return event
+}

internal/sentry/sentry_test.go

@@ -0,0 +1,89 @@
+package sentry
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+	"testing"
+
+	"github.com/brianvoe/gofakeit/v6"
+	"github.com/getsentry/sentry-go"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBeforeSend(t *testing.T) {
+	// Initialize a sample event with a stacktrace
+	event := &sentry.Event{
+		Modules:    map[string]string{"example": "1.0.0"},
+		ServerName: "test-server",
+		Exception: []sentry.Exception{
+			{
+				Stacktrace: &sentry.Stacktrace{
+					Frames: []sentry.Frame{
+						{Module: "github.com/status-im/status-go/other", Function: "OtherFunction"},
+						{Module: "github.com/status-im/status-go/internal/sentry", Function: "Recover"},
+						{Module: "github.com/status-im/status-go/internal/sentry", Function: "RecoverError"},
+					},
+				},
+			},
+		},
+	}
+
+	// Call the beforeSend function
+	result := beforeSend(event, nil)
+
+	// Verify that the stacktrace frames are correctly trimmed
+	require.NotNil(t, result)
+	require.Len(t, result.Exception[0].Stacktrace.Frames, 1)
+	require.Equal(t, "OtherFunction", result.Exception[0].Stacktrace.Frames[0].Function)
+
+	// Verify that Modules and ServerName are empty
+	require.Empty(t, result.Modules)
+	require.Empty(t, result.ServerName)
+}
+
+func TestSentry(t *testing.T) {
+	dsn := fmt.Sprintf("https://%s@sentry.example.com/%d",
+		gofakeit.LetterN(32),
+		gofakeit.Number(0, 1000),
+	)
+	context := gofakeit.LetterN(5)
+	version := gofakeit.LetterN(5)
+
+	var producedEvent *sentry.Event
+	interceptor := func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
+		producedEvent = event
+		return nil
+	}
+
+	err := Init(
+		WithDSN(dsn),
+		WithContext(context, version),
+		func(o *sentry.ClientOptions) {
+			o.BeforeSend = interceptor
+		},
+	)
+	require.NoError(t, err)
+
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+
+	go func() {
+		message := gofakeit.LetterN(5)
+		err := errors.New(message)
+		defer func() {
+			recoveredError := recover().(error)
+			require.NotNil(t, recoveredError)
+			require.ErrorIs(t, err, recoveredError)
+			wg.Done()
+		}()
+		defer Recover()
+		panic(err)
+	}()
+
+	wg.Wait()
+	require.NotNil(t, producedEvent)
+
+	err = Close()
+	require.NoError(t, err)
+}

internal/sentry/stacktrace.go

@@ -0,0 +1,65 @@
+package sentry
+
+import (
+	"slices"
+
+	"github.com/getsentry/sentry-go"
+)
+
+var stacktraceFilters = []struct {
+	Module    string
+	Functions []string
+}{
+	{
+		Module:    "github.com/status-im/status-go/internal/sentry",
+		Functions: []string{"Recover", "RecoverError"},
+	},
+	{
+		Module:    "github.com/status-im/status-go/common",
+		Functions: []string{"LogOnPanic"},
+	},
+	{
+		Module:    "github.com/status-im/status-go/mobile/callog",
+		Functions: []string{"Call.func1"},
+	},
+}
+
+func trimStacktrace(stacktrace *sentry.Stacktrace) {
+	if stacktrace == nil {
+		return
+	}
+
+	if len(stacktrace.Frames) <= 1 {
+		return
+	}
+
+	trim := 0
+
+	// Trim max 2 frames from the end
+	for i := len(stacktrace.Frames) - 1; i >= 0; i-- {
+		if !matchFilter(stacktrace.Frames[i]) {
+			// break as soon as we find a frame that doesn't match
+			break
+		}
+
+		trim++
+		if trim == 2 {
+			break
+		}
+	}
+
+	stacktrace.Frames = stacktrace.Frames[:len(stacktrace.Frames)-trim]
+}
+
+func matchFilter(frame sentry.Frame) bool {
+	for _, filter := range stacktraceFilters {
+		if frame.Module != filter.Module {
+			continue
+		}
+		if !slices.Contains(filter.Functions, frame.Function) {
+			continue
+		}
+		return true
+	}
+	return false
+}

internal/sentry/stacktrace_test.go

@@ -0,0 +1,131 @@
+package sentry
+
+import (
+	"testing"
+
+	"github.com/getsentry/sentry-go"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTrimStacktrace(t *testing.T) {
+	tests := []struct {
+		name       string
+		stacktrace *sentry.Stacktrace
+		expected   []sentry.Frame
+	}{
+		{
+			name:       "nil stacktrace",
+			stacktrace: nil,
+			expected:   nil,
+		},
+		{
+			name: "single frame",
+			stacktrace: &sentry.Stacktrace{
+				Frames: []sentry.Frame{
+					{Module: "github.com/status-im/status-go/internal/sentry", Function: "Recover"},
+				},
+			},
+			expected: []sentry.Frame{
+				{Module: "github.com/status-im/status-go/internal/sentry", Function: "Recover"},
+			},
+		},
+		{
+			name: "multiple frames with matching filters",
+			stacktrace: &sentry.Stacktrace{
+				Frames: []sentry.Frame{
+					{Module: "github.com/status-im/status-go/other", Function: "OtherFunc"},
+					{Module: "github.com/status-im/status-go/mobile/callog", Function: "Call.func1"},
+					{Module: "github.com/status-im/status-go/internal/sentry", Function: "RecoverError"},
+				},
+			},
+			expected: []sentry.Frame{
+				{Module: "github.com/status-im/status-go/other", Function: "OtherFunc"},
+			},
+		},
+		{
+			name: "matching module but not function",
+			stacktrace: &sentry.Stacktrace{
+				Frames: []sentry.Frame{
+					{Module: "github.com/status-im/status-go/other", Function: "OtherFunc"},
+					{Module: "github.com/status-im/status-go/internal/sentry", Function: "Init"},
+				},
+			},
+			expected: []sentry.Frame{
+				{Module: "github.com/status-im/status-go/other", Function: "OtherFunc"},
+				{Module: "github.com/status-im/status-go/internal/sentry", Function: "Init"},
+			},
+		},
+		{
+			name: "multiple frames without matching filters",
+			stacktrace: &sentry.Stacktrace{
+				Frames: []sentry.Frame{
+					{Module: "github.com/status-im/status-go/other", Function: "OtherFunc"},
+					{Module: "github.com/status-im/status-go/other", Function: "OtherFunc2"},
+					{Module: "github.com/status-im/status-go/other", Function: "OtherFunc3"},
+				},
+			},
+			expected: []sentry.Frame{
+				{Module: "github.com/status-im/status-go/other", Function: "OtherFunc"},
+				{Module: "github.com/status-im/status-go/other", Function: "OtherFunc2"},
+				{Module: "github.com/status-im/status-go/other", Function: "OtherFunc3"},
+			},
+		},
+		{
+			name: "matching filters only at the end",
+			stacktrace: &sentry.Stacktrace{
+				Frames: []sentry.Frame{
+					{Module: "github.com/status-im/status-go/internal/sentry", Function: "Recover"},
+					{Module: "github.com/status-im/status-go/other", Function: "OtherFunc"},
+					{Module: "github.com/status-im/status-go/common", Function: "LogOnPanic"},
+				},
+			},
+			expected: []sentry.Frame{
+				{Module: "github.com/status-im/status-go/internal/sentry", Function: "Recover"},
+				{Module: "github.com/status-im/status-go/other", Function: "OtherFunc"},
+			},
+		},
+		{
+			name: "remove at most 2 frames",
+			stacktrace: &sentry.Stacktrace{
+				Frames: []sentry.Frame{
+					{Module: "github.com/status-im/status-go/other", Function: "OtherFunc1"},
+					{Module: "github.com/status-im/status-go/other", Function: "OtherFunc2"},
+					{Module: "github.com/status-im/status-go/other", Function: "OtherFunc3"},
+					{Module: "github.com/status-im/status-go/internal/sentry", Function: "Recover"},
+					{Module: "github.com/status-im/status-go/mobile/callog", Function: "Call.func1"},
+					{Module: "github.com/status-im/status-go/common", Function: "LogOnPanic"},
+				},
+			},
+			expected: []sentry.Frame{
+				{Module: "github.com/status-im/status-go/other", Function: "OtherFunc1"},
+				{Module: "github.com/status-im/status-go/other", Function: "OtherFunc2"},
+				{Module: "github.com/status-im/status-go/other", Function: "OtherFunc3"},
+				{Module: "github.com/status-im/status-go/internal/sentry", Function: "Recover"},
+			},
+		},
+		{
+			name: "break if non-matching frame found",
+			stacktrace: &sentry.Stacktrace{
+				Frames: []sentry.Frame{
+					{Module: "github.com/status-im/status-go/mobile/callog", Function: "Call.func1"},
+					{Module: "github.com/status-im/status-go/internal/sentry", Function: "RecoverError"},
+					{Module: "github.com/status-im/status-go/other", Function: "OtherFunc1"},
+				},
+			},
+			expected: []sentry.Frame{
+				{Module: "github.com/status-im/status-go/mobile/callog", Function: "Call.func1"},
+				{Module: "github.com/status-im/status-go/internal/sentry", Function: "RecoverError"},
+				{Module: "github.com/status-im/status-go/other", Function: "OtherFunc1"},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			trimStacktrace(tt.stacktrace)
+			if tt.stacktrace != nil {
+				require.Equal(t, tt.expected, tt.stacktrace.Frames)
+			}
+		})
+	}
+}

mobile/callog/status_request_log.go

@@ -9,6 +9,8 @@ import (
 	"time"
 
 	"go.uber.org/zap"
+
+	"github.com/status-im/status-go/internal/sentry"
 )
 
 var sensitiveKeys = []string{
@@ -64,6 +66,7 @@ func Call(logger, requestLogger *zap.Logger, fn any, params ...any) any {
 	defer func() {
 		if r := recover(); r != nil {
 			logger.Error("panic found in call", zap.Any("error", r), zap.Stack("stacktrace"))
+			sentry.RecoverError(r)
 			panic(r)
 		}
 	}()

mobile/status.go

@@ -99,6 +99,16 @@ func initializeApplication(requestJSON string) string {
 	if err != nil {
 		return makeJSONResponse(err)
 	}
+
+	// initialize sentry
+	statusBackend.SetSentryDSN(request.SentryDSN)
+	if centralizedMetricsInfo.Enabled {
+		err = statusBackend.EnablePanicReporting()
+		if err != nil {
+			return makeJSONResponse(err)
+		}
+	}
+
 	response := &InitializeApplicationResponse{
 		Accounts:               accs,
 		CentralizedMetricsInfo: centralizedMetricsInfo,
@@ -2202,6 +2212,11 @@ func toggleCentralizedMetrics(requestJSON string) string {
 		return makeJSONResponse(err)
 	}
 
+	err = statusBackend.TogglePanicReporting(request.Enabled)
+	if err != nil {
+		return makeJSONResponse(err)
+	}
+
 	return makeJSONResponse(nil)
 }
 

protocol/requests/initialize_application.go

@@ -14,6 +14,7 @@ type InitializeApplication struct {
 	MixpanelToken string `json:"mixpanelToken"`
 	// MediaServerEnableTLS is optional, if not provided, media server will use TLS by default
 	MediaServerEnableTLS *bool  `json:"mediaServerEnableTLS"`
+	SentryDSN            string `json:"sentryDSN"`
 }
 
 func (i *InitializeApplication) Validate() error {