Fix/issue 12651 unfurl status links (#12751)

src/app/modules/main/chat_section/chat_content/input_area/controller.nim

@@ -1,4 +1,4 @@
-import io_interface, chronicles, tables, sequtils, strutils, sugar
+import io_interface, tables, sequtils, strutils, sugar, sets
 
 
 import ../../../../../../app_service/service/settings/service as settings_service
@@ -8,6 +8,7 @@ import ../../../../../../app_service/service/chat/service as chat_service
 import ../../../../../../app_service/service/gif/service as gif_service
 import ../../../../../../app_service/service/gif/dto
 import ../../../../../../app_service/service/message/dto/link_preview
+import ../../../../../../app_service/service/message/dto/urls_unfurling_plan
 import ../../../../../../app_service/service/settings/dto/settings
 import ../../../../../core/eventemitter
 import ../../../../../core/unique_event_emitter
@@ -28,6 +29,8 @@ type
     linkPreviewCache: LinkPreviewCache
     linkPreviewPersistentSetting: UrlUnfurlingMode
     linkPreviewCurrentMessageSetting: UrlUnfurlingMode
+    unfurlRequests: HashSet[string]
+    unfurlingPlan: UrlsUnfurlingPlan
 
 proc newController*(
     delegate: io_interface.AccessInterface,
@@ -55,10 +58,15 @@ proc newController*(
   result.linkPreviewCache = newLinkPreiewCache()
   result.linkPreviewPersistentSetting = settingsService.urlUnfurlingMode()
   result.linkPreviewCurrentMessageSetting = result.linkPreviewPersistentSetting
+  result.unfurlRequests = initHashSet[string]()
+  result.unfurlingPlan = initUrlsUnfurlingPlan()
 
 proc onUnfurlingModeChanged(self: Controller, value: UrlUnfurlingMode)
-proc onUrlsUnfurled(self: Controller, args: LinkPreviewV2DataArgs)
+proc onUrlsUnfurled(self: Controller, args: LinkPreviewDataArgs)
 proc clearLinkPreviewCache*(self: Controller)
+proc asyncUnfurlUrls(self: Controller, urls: seq[string])
+proc asyncUnfurlUnknownUrls(self: Controller, urls: seq[string])
+proc handleUnfurlingPlan*(self: Controller, unfurlNewUrls: bool)
 
 proc delete*(self: Controller) =
   self.events.disconnect()
@@ -93,7 +101,10 @@ proc init*(self: Controller) =
     self.delegate.searchGifsError()
 
   self.events.on(SIGNAL_URLS_UNFURLED) do(e:Args):
-    let args = LinkPreviewV2DataArgs(e)
+    let args = LinkPreviewDataArgs(e)
+    if not self.unfurlRequests.contains(args.requestUuid):
+      return
+    self.unfurlRequests.excl(args.requestUuid)
     self.onUrlsUnfurled(args)
 
   self.events.on(SIGNAL_URL_UNFURLING_MODE_UPDATED) do(e:Args):
@@ -195,7 +206,7 @@ proc isFavorite*(self: Controller, item: GifDto): bool =
 proc getLinkPreviewEnabled*(self: Controller): bool =
   return self.linkPreviewPersistentSetting == UrlUnfurlingMode.Enabled or self.linkPreviewCurrentMessageSetting == UrlUnfurlingMode.Enabled
 
-proc canAskToEnableLinkPreview(self: Controller): bool =
+proc shouldAskToEnableLinkPreview(self: Controller): bool =
   return self.linkPreviewPersistentSetting == UrlUnfurlingMode.AlwaysAsk and self.linkPreviewCurrentMessageSetting == UrlUnfurlingMode.AlwaysAsk
 
 proc setText*(self: Controller, text: string, unfurlNewUrls: bool) =
@@ -204,22 +215,64 @@ proc setText*(self: Controller, text: string, unfurlNewUrls: bool) =
     self.delegate.setUrls(@[])
     return
 
-  let urls = self.messageService.getTextUrls(text)
-  self.delegate.setUrls(urls)
+  self.unfurlingPlan = self.messageService.getTextURLsToUnfurl(text)
+  self.handleUnfurlingPlan(unfurlNewUrls)
 
-  let supportedUrls = urls.filter(x => not x.endsWith(".gif")) # GIFs are currently unfurled by receiver
-  self.delegate.setLinkPreviewUrls(supportedUrls)
-  let newUrls = self.linkPreviewCache.unknownUrls(supportedUrls)
+proc handleUnfurlingPlan*(self: Controller, unfurlNewUrls: bool) =
+  var allUrls = newSeq[string]() # Used for URLs syntax highlighting only
+  var allAllowedUrls = newSeq[string]() # Used for LinkPreviewsModel to keep the urls order
+  var statusAllowedUrls = newSeq[string]()
+  var otherAllowedUrls = newSeq[string]()
+  var askToEnableLinkPreview = false
 
-  let askToEnableLinkPreview = len(newUrls) > 0 and self.canAskToEnableLinkPreview()
+  for metadata in self.unfurlingPlan.urls:
+    allUrls.add(metadata.url)
+
+    if metadata.permission == UrlUnfurlingForbiddenBySettings or
+       metadata.permission == UrlUnfurlingNotSupported:
+        continue
+
+    if metadata.permission == UrlUnfurlingAskUser:
+      if self.linkPreviewCurrentMessageSetting == UrlUnfurlingMode.AlwaysAsk:
+        askToEnableLinkPreview = true
+      else:
+        otherAllowedUrls.add(metadata.url)
+        allAllowedUrls.add(metadata.url)
+      continue
+
+    # Split unfurling into 2 packs, which will be different RPCs.
+    # In most cases we expect status links to ufurl immediately.
+    # In future we could unfurl each link in a separate RPC,
+    # this would give better UX, but might result in worse performance.
+    if metadata.isStatusSharedUrl:
+      statusAllowedUrls.add(metadata.url)
+    else:
+      otherAllowedUrls.add(metadata.url)
+
+    allAllowedUrls.add(metadata.url)
+
+  # Update UI
+  self.delegate.setUrls(allUrls)
+  self.delegate.setLinkPreviewUrls(allAllowedUrls)
   self.delegate.setAskToEnableLinkPreview(askToEnableLinkPreview)
 
   if not unfurlNewUrls:
     return
 
-  if self.getLinkPreviewEnabled() and len(newUrls) > 0:
-    self.messageService.asyncUnfurlUrls(newUrls)
-    self.linkPreviewCache.markAsRequested(newUrls)
+  self.asyncUnfurlUnknownUrls(statusAllowedUrls)
+  self.asyncUnfurlUnknownUrls(otherAllowedUrls)
+
+proc reloadUnfurlingPlan*(self: Controller) =
+  self.setText(self.delegate.getPlainText(), true)
+
+proc asyncUnfurlUrls(self: Controller, urls: seq[string]) =
+  let requestUuid = self.messageService.asyncUnfurlUrls(urls)
+  self.unfurlRequests.incl(requestUuid)
+  self.linkPreviewCache.markAsRequested(urls)
+
+proc asyncUnfurlUnknownUrls(self: Controller, urls: seq[string]) =
+  let newUrls = self.linkPreviewCache.unknownUrls(urls)
+  self.asyncUnfurlUrls(newUrls)
     
 proc linkPreviewsFromCache*(self: Controller, urls: seq[string]): Table[string, LinkPreview] =
   return self.linkPreviewCache.linkPreviews(urls)
@@ -227,24 +280,18 @@ proc linkPreviewsFromCache*(self: Controller, urls: seq[string]): Table[string,
 proc clearLinkPreviewCache*(self: Controller) =
   self.linkPreviewCache.clear()
 
-proc onUrlsUnfurled(self: Controller, args: LinkPreviewV2DataArgs) =
-  if not self.getLinkPreviewEnabled():
-    return
-    
+proc onUrlsUnfurled(self: Controller, args: LinkPreviewDataArgs) =
   let urls = self.linkPreviewCache.add(args.linkPreviews)
   self.delegate.updateLinkPreviewsFromCache(urls)
 
 proc loadLinkPreviews*(self: Controller, urls: seq[string]) =
   if self.getLinkPreviewEnabled():
-    self.messageService.asyncUnfurlUrls(urls)
+    self.asyncUnfurlUrls(urls)
 
 proc setLinkPreviewEnabled*(self: Controller, enabled: bool) =
-  if enabled:
-    discard self.settingsService.saveUrlUnfurlingMode(UrlUnfurlingMode.Enabled)
-    return
-  discard self.settingsService.saveUrlUnfurlingMode(UrlUnfurlingMode.Disabled)
+  let mode = if enabled: UrlUnfurlingMode.Enabled else: UrlUnfurlingMode.Disabled
+  discard self.settingsService.saveUrlUnfurlingMode(mode)
 
 proc onUnfurlingModeChanged(self: Controller, value: UrlUnfurlingMode) =
   self.linkPreviewPersistentSetting = value
-  self.resetLinkPreviews()
-  self.setText(self.delegate.getPlainText(), self.getLinkPreviewEnabled())
+  self.reloadUnfurlingPlan()

src/app/modules/main/chat_section/chat_content/input_area/io_interface.nim

@@ -1,4 +1,4 @@
-import NimQml, tables
+import NimQml, tables, sets
 
 import ../../../../../../app_service/service/gif/dto
 import ../../../../../../app_service/service/message/dto/link_preview
@@ -114,6 +114,9 @@ method clearLinkPreviewCache*(self: AccessInterface) {.base.} =
 method linkPreviewsFromCache*(self: AccessInterface, urls: seq[string]): Table[string, LinkPreview] {.base.} =
   raise newException(ValueError, "No implementation available")
 
+method reloadUnfurlingPlan*(self: AccessInterface) =
+  raise newException(ValueError, "No implementation available")
+
 method loadLinkPreviews*(self: AccessInterface, urls: seq[string]) {.base.} =
   raise newException(ValueError, "No implementation available")
 

src/app/modules/main/chat_section/chat_content/input_area/module.nim

@@ -1,4 +1,4 @@
-import NimQml, tables
+import NimQml, tables, sets
 import io_interface
 import ../io_interface as delegate_interface
 import view, controller
@@ -154,8 +154,8 @@ method addToRecentsGif*(self: Module, item: GifDto) =
 method isFavorite*(self: Module, item: GifDto): bool =
   return self.controller.isFavorite(item)
 
-method setText*(self: Module, text: string, unfurlUrls: bool) =
-  self.controller.setText(text, unfurlUrls)
+method setText*(self: Module, text: string, unfurlNewUrls: bool) =
+  self.controller.setText(text, unfurlNewUrls)
 
 method getPlainText*(self: Module): string =
   return self.view.getPlainText()
@@ -172,6 +172,9 @@ method setLinkPreviewUrls*(self: Module, urls: seq[string]) =
 method linkPreviewsFromCache*(self: Module, urls: seq[string]): Table[string, LinkPreview] =
   return self.controller.linkPreviewsFromCache(urls)
 
+method reloadUnfurlingPlan*(self: Module) =
+  self.controller.reloadUnfurlingPlan()
+
 method loadLinkPreviews*(self: Module, urls: seq[string]) =
   self.controller.loadLinkPreviews(urls)
 

src/app/modules/main/chat_section/chat_content/input_area/view.nim

@@ -58,10 +58,12 @@ QtObject:
       msg: string,
       replyTo: string,
       contentType: int) {.slot.} =
+    # FIXME: Update this when `setText` is async.
     self.delegate.setText(msg, false)
     self.delegate.sendChatMessage(msg, replyTo, contentType, self.linkPreviewModel.getUnfuledLinkPreviews())
 
   proc sendImages*(self: View, imagePathsAndDataJson: string, msg: string, replyTo: string): string {.slot.} =
+    # FIXME: Update this when `setText` is async.
     self.delegate.setText(msg, false)
     self.delegate.sendImages(imagePathsAndDataJson, msg, replyTo, self.linkPreviewModel.getUnfuledLinkPreviews())
 
@@ -232,10 +234,7 @@ QtObject:
 
   proc setLinkPreviewUrls*(self: View, urls: seq[string]) =
     self.linkPreviewModel.setUrls(urls)
-    if(self.delegate.getLinkPreviewEnabled()):
-      self.updateLinkPreviewsFromCache(urls)
-    else:
-      self.linkPreviewModel.removeAllPreviewData()
+    self.updateLinkPreviewsFromCache(urls)
 
   proc clearLinkPreviewCache*(self: View) {.slot.} =
     self.delegate.clearLinkPreviewCache()
@@ -254,10 +253,7 @@ QtObject:
   
   proc setLinkPreviewEnabledForCurrentMessage(self: View, enabled: bool) {.slot.} =
     self.delegate.setLinkPreviewEnabledForThisMessage(enabled)
-    let links = self.linkPreviewModel.getLinks()
-    self.linkPreviewModel.clearItems()
-    self.setLinkPreviewUrls(links)
-    self.loadLinkPreviews(links)
+    self.delegate.reloadUnfurlingPlan()
 
   proc removeLinkPreviewData*(self: View, index: int) {.slot.} =
     self.linkPreviewModel.removePreviewData(index)

src/app/modules/shared_models/link_preview_item.nim

@@ -26,3 +26,8 @@ proc `$`*(self: Item): string =
     immutable: {self.immutable},
     linkPreview: {self.linkPreview},
   )"""
+
+proc markAsImmutable*(self: Item) =
+  self.linkPreview = initLinkPreview(self.linkPreview.url)
+  self.unfurled = false
+  self.immutable = true

src/app/modules/shared_models/link_preview_model.nim

@@ -235,12 +235,13 @@ QtObject:
 
     # Move or insert
     for i in 0 ..< urls.len:
-      let index = self.findUrlIndex(urls[i])
+      let url = urls[i]
+      let index = self.findUrlIndex(url)
       if index >= 0:
         self.moveRow(index, i)
         continue
 
-      let linkPreview = initLinkPreview(urls[i])
+      let linkPreview = initLinkPreview(url)
       var item = Item()
       item.unfurled = false
       item.immutable = false
@@ -266,9 +267,7 @@ QtObject:
     if index < 0 or index >= self.items.len:
       return
 
-    self.items[index].linkPreview = initLinkPreview(self.items[index].linkPreview.url)
-    self.items[index].unfurled = false
-    self.items[index].immutable = true
+    self.items[index].markAsImmutable()
 
     let modelIndex = self.createIndex(index, 0, nil)
     defer: modelIndex.delete
@@ -276,7 +275,13 @@ QtObject:
 
   proc removeAllPreviewData*(self: Model) {.slot.} =
     for i in 0 ..< self.items.len:
-      self.removePreviewData(i)
+      self.items[i].markAsImmutable()
+  
+    let indexStart = self.createIndex(0, 0, nil)
+    let indexEnd = self.createIndex(self.items.len, 0, nil)
+    defer: indexStart.delete
+    defer: indexEnd.delete
+    self.dataChanged(indexStart, indexEnd)
       
   proc getUnfuledLinkPreviews*(self: Model): seq[LinkPreview] =
     result = @[]

src/app_service/service/message/async_tasks.nim

@@ -204,6 +204,7 @@ const asyncGetFirstUnseenMessageIdForTaskArg: Task = proc(argEncoded: string) {.
 type
   AsyncUnfurlUrlsTaskArg = ref object of QObjectTaskArg
     urls*: seq[string]
+    requestUuid*: string
 
 const asyncUnfurlUrlsTask: Task = proc(argEncoded: string) {.gcsafe, nimcall.} =
   let arg = decode[AsyncUnfurlUrlsTaskArg](argEncoded)
@@ -212,7 +213,8 @@ const asyncUnfurlUrlsTask: Task = proc(argEncoded: string) {.gcsafe, nimcall.} =
     let output = %*{
       "error": (if response.error != nil: response.error.message else: ""),
       "response": response.result,
-      "requestedUrls": %*arg.urls
+      "requestedUrls": %*arg.urls,
+      "requestUuid": arg.requestUuid
     }
     arg.finish(output)
   except Exception as e:
@@ -220,7 +222,8 @@ const asyncUnfurlUrlsTask: Task = proc(argEncoded: string) {.gcsafe, nimcall.} =
     let output = %*{
       "error": e.msg,
       "response": "",
-      "requestedUrls": %*arg.urls
+      "requestedUrls": %*arg.urls,
+      "requestUuid": arg.requestUuid
     }
     arg.finish(output)
 

src/app_service/service/message/dto/urls_unfurling_plan.nim

@@ -0,0 +1,71 @@
+import json, strformat, chronicles, Tables
+include ../../../common/json_utils
+
+type 
+  UrlUnfurlingPermission* {.pure.} = enum
+    UrlUnfurlingAllowed = 0
+    UrlUnfurlingAskUser
+    UrlUnfurlingForbiddenBySettings
+    UrlUnfurlingNotSupported
+
+  UrlUnfurlingMetadata* = ref object
+    url*: string
+    permission*: UrlUnfurlingPermission
+    isStatusSharedUrl*: bool
+
+  UrlsUnfurlingPlan* = ref object
+    urls*: seq[UrlUnfurlingMetadata]
+
+proc initUrlsUnfurlingPlan*(): UrlsUnfurlingPlan =
+  result = UrlsUnfurlingPlan()
+  result.urls = newSeq[UrlUnfurlingMetadata]()
+
+proc toUrlUnfurlingPermission*(value: int): UrlUnfurlingPermission =
+  try:
+    return UrlUnfurlingPermission(value)
+  except RangeDefect:
+    return UrlUnfurlingPermission.UrlUnfurlingForbiddenBySettings
+
+proc toUrlUnfurlingMetadata*(jsonObj: JsonNode): UrlUnfurlingMetadata =
+  result = UrlUnfurlingMetadata()
+
+  if jsonObj.kind != JObject:
+    warn "node is not an object", source = "toUrlUnfurlingMetadata"
+    return
+
+  result.url = jsonObj["url"].getStr()
+  result.permission = toUrlUnfurlingPermission(jsonObj["permission"].getInt)
+  result.isStatusSharedUrl = jsonObj["isStatusSharedURL"].getBool()
+
+proc toUrlUnfurlingPlan*(jsonObj: JsonNode): UrlsUnfurlingPlan =
+  result = UrlsUnfurlingPlan()
+  
+  if jsonObj.kind != JObject:
+    warn "node is not an object", source = "toUrlUnfurlingPlan"
+    return
+
+  let urlsSeq = jsonObj["urls"]
+
+  if urlsSeq.kind != JArray:
+    warn "urls is not an array", source = "toUrlUnfurlingPlan"
+    return
+
+  for metadata in urlsSeq:
+    result.urls.add(toUrlUnfurlingMetadata(metadata))
+
+proc `$`*(self: UrlUnfurlingMetadata): string =
+  if self == nil:
+    return "nil"
+  return fmt"""UrlUnfurlingMetadata( permission: {self.permission}, isStatusSharedUrl: {self.isStatusSharedUrl} )"""
+
+proc `$`*(self: UrlsUnfurlingPlan): string =
+  if self == nil:
+    return ""
+
+  var rows = ""
+  for url, metadata in self.urls:
+    rows = fmt"""{rows}
+    url: {url}, metadata: {metadata}"""
+
+  result = fmt"""UrlsUnfurlingPlan({rows}
+  )""" 

src/app_service/service/message/service.nim

@@ -19,6 +19,7 @@ import ./dto/pinned_message_update as pinned_msg_update_dto
 import ./dto/removed_message as removed_msg_dto
 import ./dto/link_preview
 import ./dto/status_link_preview
+import ./dto/urls_unfurling_plan
 import ./message_cursor
 
 import ../../common/message as message_common
@@ -120,8 +121,9 @@ type
     chatId*: string
     message*: MessageDto
 
-  LinkPreviewV2DataArgs* = ref object of Args
+  LinkPreviewDataArgs* = ref object of Args
     linkPreviews*: Table[string, LinkPreview]
+    requestUuid*: string
 
   ReloadMessagesArgs* = ref object of Args
     communityId*: string
@@ -821,8 +823,14 @@ QtObject:
     except Exception as e:
       error "getTextUrls failed", errName = e.name, errDesription = e.msg
 
-  proc onAsyncUnfurlUrlsFinished*(self: Service, response: string) {.slot.}=
+  proc getTextURLsToUnfurl*(self: Service, text: string): UrlsUnfurlingPlan =
+    try:
+      let response = status_go.getTextURLsToUnfurl(text)
+      return toUrlUnfurlingPlan(response.result)
+    except Exception as e:
+      error "getTextURLsToUnfurl failed", errName = e.name, errDesription = e.msg
 
+  proc onAsyncUnfurlUrlsFinished*(self: Service, response: string) {.slot.}=
     let responseObj = response.parseJson
     if responseObj.kind != JObject:
       warn "expected response is not a json object", methodName = "onAsyncUnfurlUrlsFinished"
@@ -858,22 +866,25 @@ QtObject:
       if not linkPreviews.hasKey(url):
         linkPreviews[url] = initLinkPreview(url)
 
-    let args = LinkPreviewV2DataArgs(
-      linkPreviews: linkPreviews
+    let args = LinkPreviewDataArgs(
+      linkPreviews: linkPreviews,
+      requestUuid: responseObj["requestUuid"].getStr
     )
     self.events.emit(SIGNAL_URLS_UNFURLED, args)
 
-
-  proc asyncUnfurlUrls*(self: Service, urls: seq[string]) =
+  proc asyncUnfurlUrls*(self: Service, urls: seq[string]): string =
     if len(urls) == 0:
-      return
+      return ""
+    let uuid = $genUUID()
     let arg = AsyncUnfurlUrlsTaskArg(
       tptr: cast[ByteAddress](asyncUnfurlUrlsTask),
       vptr: cast[ByteAddress](self.vptr),
       slot: "onAsyncUnfurlUrlsFinished",
-      urls: urls
+      urls: urls,
+      requestUuid: uuid,
     )
     self.threadpool.start(arg)
+    return uuid
 
 # See render-inline in status-mobile/src/status_im/ui/screens/chat/message/message.cljs
 proc renderInline(self: Service, parsedText: ParsedText, communityChats: seq[ChatDto]): string =

src/backend/messages.nim

@@ -76,6 +76,10 @@ proc getTextUrls*(text: string): RpcResponse[JsonNode] {.raises: [Exception].} =
   let payload = %*[text]
   result = callPrivateRPC("getTextURLs".prefix, payload)
 
+proc getTextURLsToUnfurl*(text: string): RpcResponse[JsonNode] {.raises: [Exception].} =
+  let payload = %*[text]
+  result = callPrivateRPC("getTextURLsToUnfurl".prefix, payload)
+
 proc unfurlUrls*(urls: seq[string]): RpcResponse[JsonNode] {.raises: [Exception].} =
   let payload = %*[urls]
   result = callPrivateRPC("unfurlURLs".prefix, payload)

ui/imports/shared/controls/chat/ChatInputLinksPreviewArea.qml

@@ -159,7 +159,7 @@ Control {
             sourceModel: root.linkPreviewModel
             filters: [
                 ExpressionFilter {
-                    expression: { return !model.immutable || model.unfurled } // Filter out immutable links that haven't been unfurled yet
+                    expression: !model.immutable || model.unfurled // Filter out immutable links that haven't been unfurled yet
                 }
             ]
         }

vendor/status-go

@@ -1 +1 @@
-Subproject commit 2c55b9c676d71f75dfd07e0973e168c88cc93954
+Subproject commit a51f8aa13c20609c894ea36a9469ce3f57645657