support mention Non-Latin nickname (#3641)

* support mention non-Latin nickname

* use IndexRune instead of Index

* fix mobile issue #15524

* add another test case

protocol/messenger_mention.go

@@ -35,7 +35,7 @@ const (
 var (
 	specialCharsRegex = regexp.MustCompile("[@~\\\\*_\n>`]{1}")
 	endingCharsRegex  = regexp.MustCompile(endingChars)
-	wordRegex         = regexp.MustCompile("^[\\w\\d\\-]*" + endingChars + "|^[\\S]*$")
+	wordRegex         = regexp.MustCompile("^[\\w\\d\\-]*" + endingChars + "|[\\S]+")
 )
 
 type specialCharLocation struct {
@@ -382,7 +382,7 @@ func (m *MentionManager) calculateSuggestionsWithMentionableUsers(chatID string,
 		m.logger.Error("calculateSuggestionsWithMentionableUsers: unknown textOperation", zap.String("chatID", chatID), zap.String("fullText", fullText), zap.Any("state", state))
 	}
 
-	atSignIdx := lastIndexOf(fullText, charAtSign, end)
+	atSignIdx := lastIndexOfAtSign(fullText, end)
 	var suggestions map[string]*MentionableUser
 	if atSignIdx != -1 {
 		searchedText := strings.ToLower(subs(fullText, atSignIdx+1, end))
@@ -939,8 +939,9 @@ func getAtSignIdxs(text string, delta int) []int {
 
 func getAtSignIdxsHelper(text string, delta int, from int, idxs []int) []int {
 	tr := []rune(text)
-	idx := strings.Index(string(tr[from:]), charAtSign)
+	idx := strings.IndexRune(string(tr[from:]), '@')
 	if idx != -1 {
+		idx = utf8.RuneCountInString(text[:idx])
 		idx += from
 		idxs = append(idxs, delta+idx)
 		return getAtSignIdxsHelper(text, delta, idx+1, idxs)
@@ -1213,31 +1214,30 @@ func toInfo(inputSegments []InputSegment) *MentionState {
 	return state
 }
 
-// lastIndexOf returns the index of the last occurrence of substr in s starting from index start.
+// lastIndexOfAtSign returns the index of the last occurrence of substr in s starting from index start.
 // If substr is not present in s, it returns -1.
-func lastIndexOf(s, substr string, start int) int {
+func lastIndexOfAtSign(s string, start int) int {
 	if start < 0 {
 		return -1
 	}
 
 	t := []rune(s)
-	tt := []rune(substr)
 	if start >= len(t) {
 		start = len(t) - 1
 	}
 
 	// Reverse the input strings to find the first occurrence of the reversed substr in the reversed s.
 	reversedS := reverse(t[:start+1])
-	reversedSubstr := reverse(tt)
 
-	idx := strings.Index(reversedS, reversedSubstr)
+	idx := strings.IndexRune(reversedS, '@')
 
 	if idx == -1 {
 		return -1
 	}
 
 	// Calculate the index in the original string.
-	return start - idx - len(tt) + 1
+	idx = utf8.RuneCountInString(reversedS[:idx])
+	return start - idx
 }
 
 // reverse returns the reversed string of input s.

protocol/messenger_mention_test.go

@@ -120,6 +120,13 @@ func TestReplaceMentions(t *testing.T) {
 				LocalNickname: "User Number",
 			},
 		},
+		"0xpk6": {
+			Contact: &Contact{
+				ID:            "0xpk6",
+				LocalNickname: "特别字符",
+				DisplayName:   "特别 字符",
+			},
+		},
 	}
 
 	tests := []struct {
@@ -185,6 +192,10 @@ func TestReplaceMentions(t *testing.T) {
 
 		{"username or nickname of one is a substring of another case 1", "@User Number One @User Number", "@0xpk1 @0xpk5"},
 		{"username or nickname of one is a substring of another case 2", "@User Number @User Number One ", "@0xpk5 @0xpk1 "},
+
+		{"special chars in username case1", "@特别字符", "@0xpk6"},
+		{"special chars in username case2", "@特别字符 ", "@0xpk6 "},
+		{"special chars in username case3", " @特别 字符 ", " @0xpk6 "},
 	}
 
 	for _, tt := range tests {
@@ -474,17 +485,17 @@ func TestSubs(t *testing.T) {
 }
 
 func TestLastIndexOf(t *testing.T) {
-	atSignIdx := lastIndexOf("@", charAtSign, 0)
+	atSignIdx := lastIndexOfAtSign("@", 0)
 	require.Equal(t, 0, atSignIdx)
 
-	atSignIdx = lastIndexOf("@@", charAtSign, 1)
+	atSignIdx = lastIndexOfAtSign("@@", 1)
 	require.Equal(t, 1, atSignIdx)
 
 	//at-sign-idx 0 text @t searched-text t start 2 end 2 new-text
-	atSignIdx = lastIndexOf("@t", charAtSign, 2)
+	atSignIdx = lastIndexOfAtSign("@t", 2)
 	require.Equal(t, 0, atSignIdx)
 
-	atSignIdx = lastIndexOf("at", charAtSign, 3)
+	atSignIdx = lastIndexOfAtSign("at", 3)
 	require.Equal(t, -1, atSignIdx)
 }
 
@@ -833,13 +844,16 @@ func TestMentionSuggestionSpecialChars(t *testing.T) {
 		{"'", 0},
 		{"‘", 0},
 		{"‘@", len(mentionableUserMap)},
+		{"‘@自由人", 1},
 	}
 
-	for _, tc := range testCases {
-		ctx, err := mentionManager.OnChangeText(chatID, tc.inputText)
-		require.NoError(t, err)
-		t.Logf("Input: %+v, MentionState:%+v, InputSegments:%+v\n", tc.inputText, ctx.MentionState, ctx.InputSegments)
-		require.Equal(t, tc.expectedSize, len(ctx.MentionSuggestions))
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i+1), func(t *testing.T) {
+			ctx, err := mentionManager.OnChangeText(chatID, tc.inputText)
+			require.NoError(t, err)
+			t.Logf("Input: %+v, MentionState:%+v, InputSegments:%+v\n", tc.inputText, ctx.MentionState, ctx.InputSegments)
+			require.Equal(t, tc.expectedSize, len(ctx.MentionSuggestions))
+		})
 	}
 }
 
@@ -878,7 +892,7 @@ func TestMentionSuggestionAtSignSpaceCases(t *testing.T) {
 }
 
 func TestSelectMention(t *testing.T) {
-	_, chatID, mentionManager := setupMentionSuggestionTest(t, nil)
+	mentionableUsers, chatID, mentionManager := setupMentionSuggestionTest(t, nil)
 
 	text := "@u2 abc"
 	ctx, err := mentionManager.OnChangeText(chatID, text)
@@ -887,7 +901,7 @@ func TestSelectMention(t *testing.T) {
 
 	ctx, err = mentionManager.OnChangeText(chatID, "@u abc")
 	require.NoError(t, err)
-	require.Equal(t, 3, len(ctx.MentionSuggestions))
+	require.Equal(t, len(mentionableUsers), len(ctx.MentionSuggestions))
 
 	ctx, err = mentionManager.SelectMention(chatID, "@u abc", "u2", "0xpk2")
 	require.NoError(t, err)
@@ -1024,5 +1038,13 @@ func getDefaultMentionableUserMap() map[string]*MentionableUser {
 				EnsName:       "User Number Three",
 			},
 		},
+		"0xpk4": {
+			Contact: &Contact{
+				ID:            "0xpk4",
+				LocalNickname: "自由人",
+				ENSVerified:   true,
+				EnsName:       "User Number Four",
+			},
+		},
 	}
 }