refactor(communities)_: delegate Community creation in Persistence

In persistence.go, the lack of sufficient knowledge for constructing
fully initialized Community objects required clients to manually call
`initializeCommunity`. This commit addresses the issue by delegating
Community creation to Manager. It also removes queries and logic
duplication.

protocol/communities/community.go

@@ -45,7 +45,6 @@ type Config struct {
 	RequestedToJoinAt                   uint64
 	RequestsToJoin                      []*RequestToJoin
 	MemberIdentity                      *ecdsa.PublicKey
-	SyncedAt                            uint64
 	EventsData                          *EventsData
 	Shard                               *shard.Shard
 	PubsubTopicPrivateKey               *ecdsa.PrivateKey
@@ -2122,7 +2121,6 @@ func (o *Community) CreateDeepCopy() *Community {
 			RequestedToJoinAt:                   o.config.RequestedToJoinAt,
 			RequestsToJoin:                      o.config.RequestsToJoin,
 			MemberIdentity:                      o.config.MemberIdentity,
-			SyncedAt:                            o.config.SyncedAt,
 			EventsData:                          o.config.EventsData,
 		},
 		timesource: o.timesource,

protocol/communities/manager.go

@@ -256,11 +256,11 @@ func NewManager(identity *ecdsa.PrivateKey, installationID string, db *sql.DB, e
 		torrentConfig:               torrentConfig,
 		torrentTasks:                make(map[string]metainfo.Hash),
 		historyArchiveDownloadTasks: make(map[string]*HistoryArchiveDownloadTask),
-		persistence: &Persistence{
+	}
-			logger:     logger,
+
+	manager.persistence = &Persistence{
 		db:                      db,
-			timesource: timesource,
+		recordBundleToCommunity: manager.dbRecordBundleToCommunity,
-		},
 	}
 
 	if managerConfig.accountsManager != nil {
@@ -622,19 +622,7 @@ func (m *Manager) publish(subscription *Subscription) {
 }
 
 func (m *Manager) All() ([]*Community, error) {
-	communities, err := m.persistence.AllCommunities(&m.identity.PublicKey, m.installationID)
+	return m.persistence.AllCommunities(&m.identity.PublicKey)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, c := range communities {
-		err = m.initializeCommunity(c)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return communities, nil
 }
 
 type CommunityShard struct {
@@ -684,71 +672,23 @@ func (m *Manager) GetStoredDescriptionForCommunities(communityIDs []string) (*Kn
 }
 
 func (m *Manager) Joined() ([]*Community, error) {
-	communities, err := m.persistence.JoinedCommunities(&m.identity.PublicKey, m.installationID)
+	return m.persistence.JoinedCommunities(&m.identity.PublicKey)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, c := range communities {
-		err = m.initializeCommunity(c)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return communities, nil
 }
 
 func (m *Manager) Spectated() ([]*Community, error) {
-	communities, err := m.persistence.SpectatedCommunities(&m.identity.PublicKey, m.installationID)
+	return m.persistence.SpectatedCommunities(&m.identity.PublicKey)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, c := range communities {
-		err = m.initializeCommunity(c)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return communities, nil
 }
 
 func (m *Manager) JoinedAndPendingCommunitiesWithRequests() ([]*Community, error) {
-	communities, err := m.persistence.JoinedAndPendingCommunitiesWithRequests(&m.identity.PublicKey, m.installationID)
+	return m.persistence.JoinedAndPendingCommunitiesWithRequests(&m.identity.PublicKey)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, c := range communities {
-		err = m.initializeCommunity(c)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return communities, nil
 }
 
 func (m *Manager) DeletedCommunities() ([]*Community, error) {
-	communities, err := m.persistence.DeletedCommunities(&m.identity.PublicKey, m.installationID)
+	return m.persistence.DeletedCommunities(&m.identity.PublicKey)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, c := range communities {
-		err = m.initializeCommunity(c)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return communities, nil
 }
 
 func (m *Manager) Controlled() ([]*Community, error) {
-	communities, err := m.persistence.CommunitiesWithPrivateKey(&m.identity.PublicKey, m.installationID)
+	communities, err := m.persistence.CommunitiesWithPrivateKey(&m.identity.PublicKey)
 	if err != nil {
 		return nil, err
 	}
@@ -757,10 +697,6 @@ func (m *Manager) Controlled() ([]*Community, error) {
 
 	for _, c := range communities {
 		if c.IsControlNode() {
-			err = m.initializeCommunity(c)
-			if err != nil {
-				return nil, err
-			}
 			controlled = append(controlled, c)
 		}
 	}
@@ -3220,8 +3156,8 @@ func (m *Manager) BanUserFromCommunity(request *requests.BanUserFromCommunity) (
 	return community, nil
 }
 
-// Apply events to raw community
+func (m *Manager) dbRecordBundleToCommunity(r *CommunityRecordBundle) (*Community, error) {
-func (m *Manager) initializeCommunity(community *Community) error {
+	return recordBundleToCommunity(r, &m.identity.PublicKey, m.installationID, m.logger, m.timesource, func(community *Community) error {
 		err := community.updateCommunityDescriptionByEvents()
 		if err != nil {
 			return err
@@ -3240,23 +3176,11 @@ func (m *Manager) initializeCommunity(community *Community) error {
 		HydrateChannelsMembers(community.IDString(), community.config.CommunityDescription)
 
 		return nil
+	})
 }
 
 func (m *Manager) GetByID(id []byte) (*Community, error) {
-	community, err := m.persistence.GetByID(&m.identity.PublicKey, m.installationID, id)
+	return m.persistence.GetByID(&m.identity.PublicKey, id)
-	if err != nil {
-		return nil, err
-	}
-	if community == nil {
-		return nil, nil
-	}
-
-	err = m.initializeCommunity(community)
-	if err != nil {
-		return nil, err
-	}
-
-	return community, nil
 }
 
 func (m *Manager) GetByIDString(idString string) (*Community, error) {

protocol/communities/persistence.go

@@ -12,12 +12,10 @@ import (
 	"time"
 
 	"github.com/golang/protobuf/proto"
-	"go.uber.org/zap"
 
 	"github.com/status-im/status-go/eth-node/crypto"
 	"github.com/status-im/status-go/eth-node/types"
 	"github.com/status-im/status-go/protocol/common"
-	"github.com/status-im/status-go/protocol/common/shard"
 	"github.com/status-im/status-go/protocol/communities/token"
 	"github.com/status-im/status-go/protocol/protobuf"
 	"github.com/status-im/status-go/services/wallet/bigint"
@@ -25,42 +23,168 @@ import (
 
 type Persistence struct {
 	db *sql.DB
-	logger     *zap.Logger
+
-	timesource common.TimeSource
+	recordBundleToCommunity func(*CommunityRecordBundle) (*Community, error)
 }
 
 var ErrOldRequestToJoin = errors.New("old request to join")
 var ErrOldRequestToLeave = errors.New("old request to leave")
 
+type CommunityRecord struct {
+	id           []byte
+	privateKey   []byte
+	controlNode  []byte
+	description  []byte
+	joined       bool
+	verified     bool
+	spectated    bool
+	muted        bool
+	mutedTill    time.Time
+	shardCluster *uint
+	shardIndex   *uint
+}
+
+type EventsRecord struct {
+	id             []byte
+	rawEvents      []byte
+	rawDescription []byte
+}
+
+type RequestToJoinRecord struct {
+	id          []byte
+	publicKey   string
+	clock       int
+	ensName     string
+	chatID      string
+	communityID []byte
+	state       int
+}
+
+type CommunityRecordBundle struct {
+	community      *CommunityRecord
+	events         *EventsRecord
+	requestToJoin  *RequestToJoinRecord
+	installationID *string
+}
+
 const OR = " OR "
 const communitiesBaseQuery = `
-	SELECT c.id, c.private_key, c.control_node, c.description, c.joined, c.spectated, c.verified, c.muted, c.muted_till, r.clock, ae.raw_events, ae.raw_description, c.shard_cluster, c.shard_index, ccn.installation_id
+	SELECT
+		c.id, c.private_key, c.control_node, c.description, c.joined, c.spectated, c.verified, c.muted, c.muted_till, c.shard_cluster, c.shard_index,
+		r.id, r.public_key, r.clock, r.ens_name, r.chat_id, r.state,
+		ae.raw_events, ae.raw_description,
+		ccn.installation_id
 	FROM communities_communities c
 	LEFT JOIN communities_requests_to_join r ON c.id = r.community_id AND r.public_key = ?
 	LEFT JOIN communities_events ae ON c.id = ae.id
 	LEFT JOIN communities_control_node ccn ON c.id = ccn.community_id`
 
+func scanCommunity(scanner func(dest ...any) error) (*CommunityRecordBundle, error) {
+	r := &CommunityRecordBundle{
+		community:      &CommunityRecord{},
+		events:         nil,
+		requestToJoin:  nil,
+		installationID: nil,
+	}
+
+	var mutedTill sql.NullTime
+	var cluster, index sql.NullInt64
+
+	var requestToJoinID []byte
+	var requestToJoinPublicKey, requestToJoinENSName, requestToJoinChatID sql.NullString
+	var requestToJoinClock, requestToJoinState sql.NullInt64
+
+	var events, eventsDescription []byte
+
+	var installationID sql.NullString
+
+	err := scanner(
+		&r.community.id,
+		&r.community.privateKey,
+		&r.community.controlNode,
+		&r.community.description,
+		&r.community.joined,
+		&r.community.spectated,
+		&r.community.verified,
+		&r.community.muted,
+		&mutedTill,
+		&cluster,
+		&index,
+
+		&requestToJoinID,
+		&requestToJoinPublicKey,
+		&requestToJoinClock,
+		&requestToJoinENSName,
+		&requestToJoinChatID,
+		&requestToJoinState,
+
+		&events,
+		&eventsDescription,
+
+		&installationID,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	if mutedTill.Valid {
+		r.community.mutedTill = mutedTill.Time
+	}
+	if cluster.Valid {
+		clusterValue := uint(cluster.Int64)
+		r.community.shardCluster = &clusterValue
+	}
+	if index.Valid {
+		shardIndexValue := uint(index.Int64)
+		r.community.shardIndex = &shardIndexValue
+	}
+
+	if requestToJoinID != nil {
+		r.requestToJoin = &RequestToJoinRecord{
+			id:          requestToJoinID,
+			publicKey:   requestToJoinPublicKey.String,
+			clock:       int(requestToJoinClock.Int64),
+			ensName:     requestToJoinENSName.String,
+			chatID:      requestToJoinChatID.String,
+			communityID: r.community.id,
+			state:       int(requestToJoinState.Int64),
+		}
+	}
+
+	if events != nil {
+		r.events = &EventsRecord{
+			id:             r.community.id,
+			rawEvents:      events,
+			rawDescription: eventsDescription,
+		}
+	}
+
+	if installationID.Valid {
+		r.installationID = &installationID.String
+	}
+
+	return r, nil
+}
+
+func (p *Persistence) saveCommunity(r *CommunityRecord) error {
+	_, err := p.db.Exec(`
+        INSERT INTO communities_communities (
+            id, private_key, control_node, description,
+            joined, spectated, verified, muted,
+            muted_till, shard_cluster, shard_index
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+		r.id, r.privateKey, r.controlNode, r.description,
+		r.joined, r.spectated, r.verified, r.muted,
+		r.mutedTill, r.shardCluster, r.shardIndex)
+	return err
+}
+
 func (p *Persistence) SaveCommunity(community *Community) error {
-	id := community.ID()
+	record, err := communityToRecord(community)
-	privateKey := community.PrivateKey()
-	controlNode := community.ControlNode()
-	wrappedCommunity, err := community.ToProtocolMessageBytes()
 	if err != nil {
 		return err
 	}
-
+	return p.saveCommunity(record)
-	var shardIndex, shardCluster *uint
-	if community.Shard() != nil {
-		index := uint(community.Shard().Index)
-		shardIndex = &index
-		cluster := uint(community.Shard().Cluster)
-		shardCluster = &cluster
-	}
-
-	_, err = p.db.Exec(`
-		INSERT INTO communities_communities (id, private_key, control_node, description, joined, spectated, verified, muted, muted_till, shard_cluster, shard_index) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-		id, crypto.FromECDSA(privateKey), crypto.FromECDSAPub(controlNode), wrappedCommunity, community.config.Joined, community.config.Spectated, community.config.Verified, community.config.Muted, community.config.MuteTill, shardCluster, shardIndex)
-	return err
 }
 
 func (p *Persistence) DeleteCommunityEvents(id types.HexBytes) error {
@@ -68,23 +192,21 @@ func (p *Persistence) DeleteCommunityEvents(id types.HexBytes) error {
 	return err
 }
 
+func (p *Persistence) saveCommunityEvents(r *EventsRecord) error {
+	_, err := p.db.Exec(`
+		INSERT INTO communities_events (
+			id, raw_events, raw_description
+		) VALUES (?, ?, ?);`,
+		r.id, r.rawEvents, r.rawDescription)
+	return err
+}
+
 func (p *Persistence) SaveCommunityEvents(community *Community) error {
-	id := community.ID()
+	record, err := communityToEventsRecord(community)
-
-	if community.config.EventsData == nil {
-		return nil
-	}
-
-	rawEvents, err := communityEventsToJSONEncodedBytes(community.config.EventsData.Events)
 	if err != nil {
 		return err
 	}
-
+	return p.saveCommunityEvents(record)
-	_, err = p.db.Exec(`
-		INSERT INTO communities_events (id, raw_events, raw_description) VALUES (?, ?, ?);`,
-		id, rawEvents, community.config.EventsData.EventsBaseCommunityDescription)
-
-	return err
 }
 
 func (p *Persistence) DeleteCommunity(id types.HexBytes) error {
@@ -129,7 +251,7 @@ func (p *Persistence) ShouldHandleSyncCommunity(community *protobuf.SyncInstalla
 	}
 }
 
-func (p *Persistence) queryCommunities(memberIdentity *ecdsa.PublicKey, installationID string, query string) (response []*Community, err error) {
+func (p *Persistence) queryCommunities(memberIdentity *ecdsa.PublicKey, query string) (response []*Community, err error) {
 	rows, err := p.db.Query(query, common.PubkeyToHex(memberIdentity))
 	if err != nil {
 		return nil, err
@@ -146,37 +268,16 @@ func (p *Persistence) queryCommunities(memberIdentity *ecdsa.PublicKey, installa
 	}()
 
 	for rows.Next() {
-		var publicKeyBytes, privateKeyBytes, controlNodeBytes, descriptionBytes []byte
+		r, err := scanCommunity(rows.Scan)
-		var joined, spectated, verified, muted bool
-		var muteTill sql.NullTime
-		var cluster, index, requestedToJoinAt sql.NullInt64
-		var installationIDStr sql.NullString
-
-		// Community events specific fields
-		var eventsBytes, eventsDescriptionBytes []byte
-		err := rows.Scan(&publicKeyBytes, &privateKeyBytes, &controlNodeBytes, &descriptionBytes, &joined, &spectated, &verified, &muted, &muteTill, &requestedToJoinAt, &eventsBytes, &eventsDescriptionBytes, &cluster, &index, &installationIDStr)
 		if err != nil {
 			return nil, err
 		}
 
-		var clusterValue *uint
+		org, err := p.recordBundleToCommunity(r)
-		if cluster.Valid {
-			v := uint(cluster.Int64)
-			clusterValue = &v
-		}
-
-		var indexValue *uint
-		if index.Valid {
-			v := uint(index.Int64)
-			indexValue = &v
-		}
-
-		isControlDevice := installationIDStr.Valid && installationIDStr.String == installationID
-
-		org, err := p.unmarshalCommunityFromDB(memberIdentity, isControlDevice, publicKeyBytes, privateKeyBytes, controlNodeBytes, descriptionBytes, joined, spectated, verified, muted, muteTill.Time, uint64(requestedToJoinAt.Int64), eventsBytes, eventsDescriptionBytes, clusterValue, indexValue, p.logger)
 		if err != nil {
 			return nil, err
 		}
+
 		response = append(response, org)
 	}
 
@@ -184,21 +285,21 @@ func (p *Persistence) queryCommunities(memberIdentity *ecdsa.PublicKey, installa
 
 }
 
-func (p *Persistence) AllCommunities(memberIdentity *ecdsa.PublicKey, installationID string) ([]*Community, error) {
+func (p *Persistence) AllCommunities(memberIdentity *ecdsa.PublicKey) ([]*Community, error) {
-	return p.queryCommunities(memberIdentity, installationID, communitiesBaseQuery)
+	return p.queryCommunities(memberIdentity, communitiesBaseQuery)
 }
 
-func (p *Persistence) JoinedCommunities(memberIdentity *ecdsa.PublicKey, installationID string) ([]*Community, error) {
+func (p *Persistence) JoinedCommunities(memberIdentity *ecdsa.PublicKey) ([]*Community, error) {
 	query := communitiesBaseQuery + ` WHERE c.joined`
-	return p.queryCommunities(memberIdentity, installationID, query)
+	return p.queryCommunities(memberIdentity, query)
 }
 
-func (p *Persistence) SpectatedCommunities(memberIdentity *ecdsa.PublicKey, installationID string) ([]*Community, error) {
+func (p *Persistence) SpectatedCommunities(memberIdentity *ecdsa.PublicKey) ([]*Community, error) {
 	query := communitiesBaseQuery + ` WHERE c.spectated`
-	return p.queryCommunities(memberIdentity, installationID, query)
+	return p.queryCommunities(memberIdentity, query)
 }
 
-func (p *Persistence) rowsToCommunities(memberIdentity *ecdsa.PublicKey, installationID string, rows *sql.Rows) (comms []*Community, err error) {
+func (p *Persistence) rowsToCommunities(memberIdentity *ecdsa.PublicKey, rows *sql.Rows) (comms []*Community, err error) {
 	defer func() {
 		if err != nil {
 			// Don't shadow original error
@@ -210,218 +311,64 @@ func (p *Persistence) rowsToCommunities(memberIdentity *ecdsa.PublicKey, install
 	}()
 
 	for rows.Next() {
-		var comm *Community
+		r, err := scanCommunity(rows.Scan)
-
-		// Community specific fields
-		var publicKeyBytes, privateKeyBytes, controlNodeBytes, descriptionBytes []byte
-		var joined, spectated, verified, muted bool
-		var muteTill sql.NullTime
-		var cluster, index sql.NullInt64
-		var installationIDStr sql.NullString
-
-		// Request to join specific fields
-		var rtjID, rtjCommunityID []byte
-		var rtjPublicKey, rtjENSName, rtjChatID sql.NullString
-		var rtjClock, rtjState sql.NullInt64
-
-		// Community events specific fields
-		var eventsBytes, eventsDescriptionBytes []byte
-
-		err = rows.Scan(
-			&publicKeyBytes, &privateKeyBytes, &controlNodeBytes, &descriptionBytes, &joined, &spectated, &verified, &muted, &muteTill,
-			&rtjID, &rtjPublicKey, &rtjClock, &rtjENSName, &rtjChatID, &rtjCommunityID, &rtjState, &eventsBytes, &eventsDescriptionBytes, &cluster, &index, &installationIDStr)
 		if err != nil {
 			return nil, err
 		}
 
-		var clusterValue *uint
+		org, err := p.recordBundleToCommunity(r)
-		if cluster.Valid {
-			v := uint(cluster.Int64)
-			clusterValue = &v
-		}
-
-		var indexValue *uint
-		if index.Valid {
-			v := uint(index.Int64)
-			indexValue = &v
-		}
-
-		isControlDevice := installationIDStr.Valid && installationIDStr.String == installationID
-
-		comm, err = p.unmarshalCommunityFromDB(memberIdentity, isControlDevice, publicKeyBytes, privateKeyBytes, controlNodeBytes, descriptionBytes, joined, spectated, verified, muted, muteTill.Time, uint64(rtjClock.Int64), eventsBytes, eventsDescriptionBytes, clusterValue, indexValue, p.logger)
 		if err != nil {
 			return nil, err
 		}
 
-		rtj := unmarshalRequestToJoinFromDB(rtjID, rtjCommunityID, rtjPublicKey, rtjENSName, rtjChatID, rtjClock, rtjState)
+		comms = append(comms, org)
-		if !rtj.Empty() {
-			comm.AddRequestToJoin(rtj)
-		}
-		comms = append(comms, comm)
 	}
 
 	return comms, nil
 }
 
-func (p *Persistence) JoinedAndPendingCommunitiesWithRequests(memberIdentity *ecdsa.PublicKey, installationID string) (comms []*Community, err error) {
+func (p *Persistence) JoinedAndPendingCommunitiesWithRequests(memberIdentity *ecdsa.PublicKey) (comms []*Community, err error) {
-	query := `SELECT
+	query := communitiesBaseQuery + ` WHERE c.Joined OR r.state = ?`
-c.id, c.private_key, c.control_node, c.description, c.joined, c.spectated, c.verified, c.muted, c.muted_till,
-r.id, r.public_key, r.clock, r.ens_name, r.chat_id, r.community_id, r.state, ae.raw_events, ae.raw_description, c.shard_cluster, c.shard_index, ccn.installation_id
-FROM communities_communities c
-LEFT JOIN communities_requests_to_join r ON c.id = r.community_id AND r.public_key = ?
-LEFT JOIN communities_events ae ON c.id = ae.id
-LEFT JOIN communities_control_node ccn ON c.id = ccn.community_id
-WHERE c.Joined OR r.state = ?`
 
 	rows, err := p.db.Query(query, common.PubkeyToHex(memberIdentity), RequestToJoinStatePending)
 	if err != nil {
 		return nil, err
 	}
 
-	return p.rowsToCommunities(memberIdentity, installationID, rows)
+	return p.rowsToCommunities(memberIdentity, rows)
 }
 
-func (p *Persistence) DeletedCommunities(memberIdentity *ecdsa.PublicKey, installationID string) (comms []*Community, err error) {
+func (p *Persistence) DeletedCommunities(memberIdentity *ecdsa.PublicKey) (comms []*Community, err error) {
-	query := `SELECT
+	query := communitiesBaseQuery + ` WHERE NOT c.Joined AND (r.community_id IS NULL or r.state != ?)`
-c.id, c.private_key, c.control_node,  c.description, c.joined, c.spectated, c.verified, c.muted, c.muted_till,
-r.id, r.public_key, r.clock, r.ens_name, r.chat_id, r.community_id, r.state, ae.raw_events, ae.raw_description, c.shard_cluster, c.shard_index, ccn.installation_id
-FROM communities_communities c
-LEFT JOIN communities_requests_to_join r ON c.id = r.community_id AND r.public_key = ?
-LEFT JOIN communities_events ae ON c.id = ae.id
-LEFT JOIN communities_control_node ccn ON c.id = ccn.community_id
-WHERE NOT c.Joined AND (r.community_id IS NULL or r.state != ?)`
 
 	rows, err := p.db.Query(query, common.PubkeyToHex(memberIdentity), RequestToJoinStatePending)
 	if err != nil {
 		return nil, err
 	}
 
-	return p.rowsToCommunities(memberIdentity, installationID, rows)
+	return p.rowsToCommunities(memberIdentity, rows)
 }
 
-func (p *Persistence) CommunitiesWithPrivateKey(memberIdentity *ecdsa.PublicKey, installationID string) ([]*Community, error) {
+func (p *Persistence) CommunitiesWithPrivateKey(memberIdentity *ecdsa.PublicKey) ([]*Community, error) {
 	query := communitiesBaseQuery + ` WHERE c.private_key IS NOT NULL`
-	return p.queryCommunities(memberIdentity, installationID, query)
+	return p.queryCommunities(memberIdentity, query)
 }
 
-func (p *Persistence) GetByID(memberIdentity *ecdsa.PublicKey, installationID string, id []byte) (*Community, error) {
+func (p *Persistence) getByID(id []byte, memberIdentity *ecdsa.PublicKey) (*CommunityRecordBundle, error) {
-	var publicKeyBytes, privateKeyBytes, controlNodeBytes, descriptionBytes []byte
+	r, err := scanCommunity(p.db.QueryRow(communitiesBaseQuery+` WHERE c.id = ?`, common.PubkeyToHex(memberIdentity), id).Scan)
-	var joined bool
+	return r, err
-	var spectated bool
+}
-	var verified bool
-	var muted bool
-	var muteTill sql.NullTime
-	var requestedToJoinAt, cluster, index sql.NullInt64
-	var installationIDStr sql.NullString
 
-	// Community events specific fields
+func (p *Persistence) GetByID(memberIdentity *ecdsa.PublicKey, id []byte) (*Community, error) {
-	var eventsBytes, eventsDescriptionBytes []byte
+	r, err := p.getByID(id, memberIdentity)
-
-	err := p.db.QueryRow(communitiesBaseQuery+` WHERE c.id = ?`, common.PubkeyToHex(memberIdentity), id).Scan(&publicKeyBytes, &privateKeyBytes, &controlNodeBytes, &descriptionBytes, &joined, &spectated, &verified, &muted, &muteTill, &requestedToJoinAt, &eventsBytes, &eventsDescriptionBytes, &cluster, &index, &installationIDStr)
 	if err == sql.ErrNoRows {
 		return nil, nil
-	} else if err != nil {
-		return nil, err
 	}
-
-	var clusterValue *uint
-	if cluster.Valid {
-		v := uint(cluster.Int64)
-		clusterValue = &v
-	}
-
-	var indexValue *uint
-	if index.Valid {
-		v := uint(index.Int64)
-		indexValue = &v
-	}
-
-	isControlDevice := installationIDStr.Valid && installationIDStr.String == installationID
-
-	return p.unmarshalCommunityFromDB(memberIdentity, isControlDevice, publicKeyBytes, privateKeyBytes, controlNodeBytes, descriptionBytes, joined, spectated, verified, muted, muteTill.Time, uint64(requestedToJoinAt.Int64), eventsBytes, eventsDescriptionBytes, clusterValue, indexValue, p.logger)
-}
-
-func (p *Persistence) unmarshalCommunityFromDB(memberIdentity *ecdsa.PublicKey, isControlDevice bool, publicKeyBytes, privateKeyBytes, controlNodeBytes, wrappedCommunity []byte, joined,
-	spectated, verified, muted bool, muteTill time.Time, requestedToJoinAt uint64, eventsBytes []byte,
-	eventsDescriptionBytes []byte, cluster *uint, index *uint, logger *zap.Logger) (*Community, error) {
-
-	var privateKey *ecdsa.PrivateKey
-	var controlNode *ecdsa.PublicKey
-	var err error
-
-	if privateKeyBytes != nil {
-		privateKey, err = crypto.ToECDSA(privateKeyBytes)
-		if err != nil {
-			return nil, err
-		}
-	}
-	if controlNodeBytes != nil {
-		controlNode, err = crypto.UnmarshalPubkey(controlNodeBytes)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	description, err := decodeWrappedCommunityDescription(wrappedCommunity)
 	if err != nil {
 		return nil, err
 	}
 
-	id, err := crypto.DecompressPubkey(publicKeyBytes)
+	return p.recordBundleToCommunity(r)
-	if err != nil {
-		return nil, err
-	}
-
-	eventsData, err := decodeEventsData(eventsBytes, eventsDescriptionBytes)
-	if err != nil {
-		return nil, err
-
-	}
-
-	var s *shard.Shard = nil
-	if cluster != nil && index != nil {
-		s = &shard.Shard{
-			Cluster: uint16(*cluster),
-			Index:   uint16(*index),
-		}
-	}
-
-	config := Config{
-		PrivateKey:                          privateKey,
-		ControlNode:                         controlNode,
-		ControlDevice:                       isControlDevice,
-		CommunityDescription:                description,
-		MemberIdentity:                      memberIdentity,
-		CommunityDescriptionProtocolMessage: wrappedCommunity,
-		Logger:                              logger,
-		ID:                                  id,
-		Verified:                            verified,
-		Muted:                               muted,
-		MuteTill:                            muteTill,
-		RequestedToJoinAt:                   requestedToJoinAt,
-		Joined:                              joined,
-		Spectated:                           spectated,
-		EventsData:                          eventsData,
-		Shard:                               s,
-	}
-	community, err := New(config, p.timesource)
-	if err != nil {
-		return nil, err
-	}
-
-	return community, nil
-}
-
-func unmarshalRequestToJoinFromDB(ID, communityID []byte, publicKey, ensName, chatID sql.NullString, clock, state sql.NullInt64) *RequestToJoin {
-	return &RequestToJoin{
-		ID:          ID,
-		PublicKey:   publicKey.String,
-		Clock:       uint64(clock.Int64),
-		ENSName:     ensName.String,
-		ChatID:      chatID.String,
-		CommunityID: communityID,
-		State:       RequestToJoinState(state.Int64),
-	}
 }
 
 func (p *Persistence) SaveRequestToJoin(request *RequestToJoin) (err error) {
@@ -1315,7 +1262,6 @@ func (p *Persistence) getCommunityTokensInternal(rows *sql.Rows) ([]*token.Commu
 			token.Supply = &bigint.BigInt{Int: supplyBigInt}
 		} else {
 			token.Supply = &bigint.BigInt{Int: big.NewInt(0)}
-			p.logger.Error("can't create bigInt from string")
 		}
 
 		tokens = append(tokens, &token)

protocol/communities/persistence_mapping.go

@@ -0,0 +1,159 @@
+package communities
+
+import (
+	"crypto/ecdsa"
+
+	"go.uber.org/zap"
+
+	"github.com/status-im/status-go/eth-node/crypto"
+	"github.com/status-im/status-go/protocol/common"
+	"github.com/status-im/status-go/protocol/common/shard"
+)
+
+func communityToRecord(community *Community) (*CommunityRecord, error) {
+	wrappedDescription, err := community.ToProtocolMessageBytes()
+	if err != nil {
+		return nil, err
+	}
+
+	var shardIndex, shardCluster *uint
+	if community.Shard() != nil {
+		index := uint(community.Shard().Index)
+		shardIndex = &index
+		cluster := uint(community.Shard().Cluster)
+		shardCluster = &cluster
+	}
+
+	return &CommunityRecord{
+		id:           community.ID(),
+		privateKey:   crypto.FromECDSA(community.PrivateKey()),
+		controlNode:  crypto.FromECDSAPub(community.ControlNode()),
+		description:  wrappedDescription,
+		joined:       community.config.Joined,
+		verified:     community.config.Verified,
+		spectated:    community.config.Spectated,
+		muted:        community.config.Muted,
+		mutedTill:    community.config.MuteTill,
+		shardCluster: shardCluster,
+		shardIndex:   shardIndex,
+	}, nil
+}
+
+func communityToEventsRecord(community *Community) (*EventsRecord, error) {
+	if community.config.EventsData == nil {
+		return nil, nil
+	}
+
+	rawEvents, err := communityEventsToJSONEncodedBytes(community.config.EventsData.Events)
+	if err != nil {
+		return nil, err
+	}
+
+	return &EventsRecord{
+		id:             community.ID(),
+		rawEvents:      rawEvents,
+		rawDescription: community.config.EventsData.EventsBaseCommunityDescription,
+	}, nil
+}
+
+func recordToRequestToJoin(r *RequestToJoinRecord) *RequestToJoin {
+	// FIXME: fill revealed addresses
+	return &RequestToJoin{
+		ID:          r.id,
+		PublicKey:   r.publicKey,
+		Clock:       uint64(r.clock),
+		ENSName:     r.ensName,
+		ChatID:      r.chatID,
+		CommunityID: r.communityID,
+		State:       RequestToJoinState(r.state),
+	}
+}
+
+func recordBundleToCommunity(r *CommunityRecordBundle, memberIdentity *ecdsa.PublicKey, installationID string,
+	logger *zap.Logger, timesource common.TimeSource, initializer func(*Community) error) (*Community, error) {
+	var privateKey *ecdsa.PrivateKey
+	var controlNode *ecdsa.PublicKey
+	var err error
+
+	if r.community.privateKey != nil {
+		privateKey, err = crypto.ToECDSA(r.community.privateKey)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if r.community.controlNode != nil {
+		controlNode, err = crypto.UnmarshalPubkey(r.community.controlNode)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	description, err := decodeWrappedCommunityDescription(r.community.description)
+	if err != nil {
+		return nil, err
+	}
+
+	id, err := crypto.DecompressPubkey(r.community.id)
+	if err != nil {
+		return nil, err
+	}
+
+	var eventsData *EventsData
+	if r.events != nil {
+		eventsData, err = decodeEventsData(r.events.rawEvents, r.events.rawDescription)
+		if err != nil {
+			return nil, err
+
+		}
+	}
+
+	var s *shard.Shard = nil
+	if r.community.shardCluster != nil && r.community.shardIndex != nil {
+		s = &shard.Shard{
+			Cluster: uint16(*r.community.shardCluster),
+			Index:   uint16(*r.community.shardIndex),
+		}
+	}
+
+	isControlDevice := r.installationID != nil && *r.installationID == installationID
+
+	config := Config{
+		PrivateKey:                          privateKey,
+		ControlNode:                         controlNode,
+		ControlDevice:                       isControlDevice,
+		CommunityDescription:                description,
+		MemberIdentity:                      memberIdentity,
+		CommunityDescriptionProtocolMessage: r.community.description,
+		Logger:                              logger,
+		ID:                                  id,
+		Verified:                            r.community.verified,
+		Muted:                               r.community.muted,
+		MuteTill:                            r.community.mutedTill,
+		Joined:                              r.community.joined,
+		Spectated:                           r.community.spectated,
+		EventsData:                          eventsData,
+		Shard:                               s,
+	}
+
+	community, err := New(config, timesource)
+	if err != nil {
+		return nil, err
+	}
+
+	if r.requestToJoin != nil {
+		community.config.RequestedToJoinAt = uint64(r.requestToJoin.clock)
+		requestToJoin := recordToRequestToJoin(r.requestToJoin)
+		if !requestToJoin.Empty() {
+			community.AddRequestToJoin(requestToJoin)
+		}
+	}
+
+	if initializer != nil {
+		err = initializer(community)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return community, nil
+}

protocol/communities/persistence_test.go

@@ -29,6 +29,7 @@ type PersistenceSuite struct {
 	suite.Suite
 
 	db       *Persistence
+	identity *ecdsa.PrivateKey
 }
 
 func (s *PersistenceSuite) SetupTest() {
@@ -40,24 +41,26 @@ func (s *PersistenceSuite) SetupTest() {
 	err = sqlite.Migrate(db)
 	s.Require().NoError(err, "protocol migrate")
 
-	s.db = &Persistence{db: db, timesource: &TimeSourceStub{}}
+	s.identity, err = crypto.GenerateKey()
+	s.Require().NoError(err)
+
+	s.db = &Persistence{db: db, recordBundleToCommunity: func(r *CommunityRecordBundle) (*Community, error) {
+		return recordBundleToCommunity(r, &s.identity.PublicKey, "", nil, &TimeSourceStub{}, nil)
+	}}
 }
 
 func (s *PersistenceSuite) TestSaveCommunity() {
-	id, err := crypto.GenerateKey()
-	s.Require().NoError(err)
-
 	// there is one community inserted by default
-	communities, err := s.db.AllCommunities(&id.PublicKey, "")
+	communities, err := s.db.AllCommunities(&s.identity.PublicKey)
 	s.Require().NoError(err)
 	s.Require().Len(communities, 1)
 
 	community := Community{
 		config: &Config{
-			PrivateKey:           id,
+			PrivateKey:           s.identity,
-			ControlNode:          &id.PublicKey,
+			ControlNode:          &s.identity.PublicKey,
 			ControlDevice:        true,
-			ID:                   &id.PublicKey,
+			ID:                   &s.identity.PublicKey,
 			Joined:               true,
 			Spectated:            true,
 			Verified:             true,
@@ -68,10 +71,10 @@ func (s *PersistenceSuite) TestSaveCommunity() {
 	}
 	s.Require().NoError(s.db.SaveCommunity(&community))
 
-	communities, err = s.db.AllCommunities(&id.PublicKey, "")
+	communities, err = s.db.AllCommunities(&s.identity.PublicKey)
 	s.Require().NoError(err)
 	s.Require().Len(communities, 2)
-	s.Equal(types.HexBytes(crypto.CompressPubkey(&id.PublicKey)), communities[1].ID())
+	s.Equal(types.HexBytes(crypto.CompressPubkey(&s.identity.PublicKey)), communities[1].ID())
 	s.Equal(true, communities[1].Joined())
 	s.Equal(true, communities[1].Spectated())
 	s.Equal(true, communities[1].Verified())
@@ -181,25 +184,20 @@ func (s *PersistenceSuite) TestSetPrivateKey() {
 	s.Zero(rcr.PrivateKey, "private key must be zero value")
 
 	// Set private key
-	pk, err := crypto.GenerateKey()
+	err = s.db.SetPrivateKey(sc.Id, s.identity)
-	s.Require().NoError(err, "crypto.GenerateKey")
-	err = s.db.SetPrivateKey(sc.Id, pk)
 	s.Require().NoError(err, "SetPrivateKey")
 
 	// retrieve row from db again, private key must match the given key
 	rcr, err = s.db.getRawCommunityRow(sc.Id)
 	s.Require().NoError(err, "getRawCommunityRow")
-	s.Equal(crypto.FromECDSA(pk), rcr.PrivateKey, "private key must match given key")
+	s.Equal(crypto.FromECDSA(s.identity), rcr.PrivateKey, "private key must match given key")
 }
 
 func (s *PersistenceSuite) TestJoinedAndPendingCommunitiesWithRequests() {
-	identity, err := crypto.GenerateKey()
-	s.Require().NoError(err, "crypto.GenerateKey shouldn't give any error")
-
 	clock := uint64(time.Now().Unix())
 
 	// Add a new community that we have joined
-	com := s.makeNewCommunity(identity)
+	com := s.makeNewCommunity(s.identity)
 	com.Join()
 	sc, err := com.ToSyncInstallationCommunityProtobuf(clock, nil, nil)
 	s.Require().NoError(err, "Community.ToSyncInstallationCommunityProtobuf shouldn't give any error")
@@ -207,13 +205,13 @@ func (s *PersistenceSuite) TestJoinedAndPendingCommunitiesWithRequests() {
 	s.Require().NoError(err, "saveRawCommunityRow")
 
 	// Add a new community that we have requested to join, but not yet joined
-	com2 := s.makeNewCommunity(identity)
+	com2 := s.makeNewCommunity(s.identity)
 	err = s.db.SaveCommunity(com2)
 	s.Require().NoError(err, "SaveCommunity shouldn't give any error")
 
 	rtj := &RequestToJoin{
 		ID:          types.HexBytes{1, 2, 3, 4, 5, 6, 7, 8},
-		PublicKey:   common.PubkeyToHex(&identity.PublicKey),
+		PublicKey:   common.PubkeyToHex(&s.identity.PublicKey),
 		Clock:       clock,
 		CommunityID: com2.ID(),
 		State:       RequestToJoinStatePending,
@@ -221,7 +219,7 @@ func (s *PersistenceSuite) TestJoinedAndPendingCommunitiesWithRequests() {
 	err = s.db.SaveRequestToJoin(rtj)
 	s.Require().NoError(err, "SaveRequestToJoin shouldn't give any error")
 
-	comms, err := s.db.JoinedAndPendingCommunitiesWithRequests(&identity.PublicKey, "")
+	comms, err := s.db.JoinedAndPendingCommunitiesWithRequests(&s.identity.PublicKey)
 	s.Require().NoError(err, "JoinedAndPendingCommunitiesWithRequests shouldn't give any error")
 	s.Len(comms, 2, "Should have 2 communities")
 
@@ -516,9 +514,6 @@ func (s *PersistenceSuite) TestSaveCheckChannelPermissionResponse() {
 }
 
 func (s *PersistenceSuite) TestGetCommunityRequestsToJoinWithRevealedAddresses() {
-	identity, err := crypto.GenerateKey()
-	s.Require().NoError(err, "crypto.GenerateKey shouldn't give any error")
-
 	clock := uint64(time.Now().Unix())
 	communityID := types.HexBytes{7, 7, 7, 7, 7, 7, 7, 7}
 	revealedAddresses := []string{"address1", "address2", "address3"}
@@ -532,7 +527,7 @@ func (s *PersistenceSuite) TestGetCommunityRequestsToJoinWithRevealedAddresses()
 	// RTJ with 2 revealed Addresses
 	expectedRtj1 := &RequestToJoin{
 		ID:          types.HexBytes{1, 2, 3, 4, 5, 6, 7, 8},
-		PublicKey:   common.PubkeyToHex(&identity.PublicKey),
+		PublicKey:   common.PubkeyToHex(&s.identity.PublicKey),
 		Clock:       clock,
 		CommunityID: communityID,
 		State:       RequestToJoinStateAccepted,
@@ -568,7 +563,7 @@ func (s *PersistenceSuite) TestGetCommunityRequestsToJoinWithRevealedAddresses()
 	signature := []byte("test")
 	expectedRtj2 := &RequestToJoin{
 		ID:          types.HexBytes{8, 7, 6, 5, 4, 3, 2, 1},
-		PublicKey:   common.PubkeyToHex(&identity.PublicKey),
+		PublicKey:   common.PubkeyToHex(&s.identity.PublicKey),
 		Clock:       clock,
 		CommunityID: communityID,
 		State:       RequestToJoinStateAccepted,
@@ -600,7 +595,7 @@ func (s *PersistenceSuite) TestGetCommunityRequestsToJoinWithRevealedAddresses()
 	// RTJ without RevealedAccounts
 	expectedRtjWithoutRevealedAccounts := &RequestToJoin{
 		ID:          types.HexBytes{1, 6, 6, 6, 6, 6, 6, 6},
-		PublicKey:   common.PubkeyToHex(&identity.PublicKey),
+		PublicKey:   common.PubkeyToHex(&s.identity.PublicKey),
 		Clock:       clock,
 		CommunityID: communityID,
 		State:       RequestToJoinStateAccepted,
@@ -617,7 +612,7 @@ func (s *PersistenceSuite) TestGetCommunityRequestsToJoinWithRevealedAddresses()
 	// RTJ with RevealedAccount but with empty Address
 	expectedRtjWithEmptyAddress := &RequestToJoin{
 		ID:          types.HexBytes{2, 6, 6, 6, 6, 6, 6, 6},
-		PublicKey:   common.PubkeyToHex(&identity.PublicKey),
+		PublicKey:   common.PubkeyToHex(&s.identity.PublicKey),
 		Clock:       clock,
 		CommunityID: communityID,
 		State:       RequestToJoinStateAccepted,
@@ -668,18 +663,15 @@ func (s *PersistenceSuite) TestCuratedCommunities() {
 }
 
 func (s *PersistenceSuite) TestGetCommunityRequestToJoinWithRevealedAddresses() {
-	identity, err := crypto.GenerateKey()
-	s.Require().NoError(err, "crypto.GenerateKey shouldn't give any error")
-
 	clock := uint64(time.Now().Unix())
 	communityID := types.HexBytes{7, 7, 7, 7, 7, 7, 7, 7}
 	revealedAddresses := []string{"address1", "address2", "address3"}
 	chainIds := []uint64{1, 2}
-	publicKey := common.PubkeyToHex(&identity.PublicKey)
+	publicKey := common.PubkeyToHex(&s.identity.PublicKey)
 	signature := []byte("test")
 
 	// No data in database
-	_, err = s.db.GetCommunityRequestToJoinWithRevealedAddresses(publicKey, communityID)
+	_, err := s.db.GetCommunityRequestToJoinWithRevealedAddresses(publicKey, communityID)
 	s.Require().ErrorIs(err, sql.ErrNoRows)
 
 	// RTJ with 2 withoutRevealed Addresses