diff --git a/tools/networkmonitor/networkmonitor.nim b/tools/networkmonitor/networkmonitor.nim index cfc11ad53..3910a8ecb 100644 --- a/tools/networkmonitor/networkmonitor.nim +++ b/tools/networkmonitor/networkmonitor.nim @@ -4,17 +4,18 @@ else: {.push raises: [].} import - std/[tables,strutils,times,sequtils,httpclient], + std/[tables,strutils,times,sequtils], chronicles, chronicles/topics_registry, chronos, + chronos/timer as ctime, confutils, eth/keys, eth/p2p/discoveryv5/enr, libp2p/crypto/crypto, metrics, metrics/chronos_httpserver, - presto/[route, server], + presto/[route, server, client], stew/shims/net import @@ -42,7 +43,7 @@ proc setDiscoveredPeersCapabilities( proc setConnectedPeersMetrics(discoveredNodes: seq[Node], node: WakuNode, timeout: chronos.Duration, - client: HttpClient, + restClient: RestClientRef, allPeers: CustomPeersTableRef) {.async.} = let currentTime = $getTime() @@ -81,13 +82,19 @@ proc setConnectedPeersMetrics(discoveredNodes: seq[Node], allPeers[peerId].ip = ip # get more info the peers from its ip address - let location = await ipToLocation(ip, client) - if not location.isOk(): + var location: NodeLocation + try: + # IP-API endpoints are now limited to 45 HTTP requests per minute + # TODO: As network grows, find a better way to now block the whole app + await sleepAsync(1400) + let response = await restClient.ipToLocation(ip) + location = response.data + except: warn "could not get location", ip=ip continue - allPeers[peerId].country = location.get().country - allPeers[peerId].city = location.get().city + allPeers[peerId].country = location.country + allPeers[peerId].city = location.city let peer = toRemotePeerInfo(discNode.record) if not peer.isOk(): @@ -145,11 +152,11 @@ proc setConnectedPeersMetrics(discoveredNodes: seq[Node], # crawls the network discovering peers and trying to connect to them # metrics are processed and exposed proc crawlNetwork(node: WakuNode, + restClient: RestClientRef, conf: NetworkMonitorConf, allPeersRef: CustomPeersTableRef) {.async.} = let crawlInterval = conf.refreshInterval * 1000 - let client = newHttpClient() while true: # discover new random nodes let discoveredNodes = await node.wakuDiscv5.protocol.queryRandom() @@ -163,7 +170,7 @@ proc crawlNetwork(node: WakuNode, # tries to connect to all newly discovered nodes # and populates metrics related to peers we could connect # note random discovered nodes can be already known - await setConnectedPeersMetrics(discoveredNodes, node, conf.timeout, client, allPeersRef) + await setConnectedPeersMetrics(discoveredNodes, node, conf.timeout, restClient, allPeersRef) let totalNodes = flatNodes.len let seenNodes = flatNodes.countIt(it.seen) @@ -281,6 +288,15 @@ when isMainModule: let res = startRestApiServer(conf, allPeersInfo, msgPerContentTopic) if res.isErr(): error "could not start rest api server", err=res.error + quit(1) + + # create a rest client + let clientRest = RestClientRef.new(url="http://ip-api.com", + connectTimeout=ctime.seconds(2)) + if clientRest.isErr(): + error "could not start rest api client", err=res.error + quit(1) + let restClient = clientRest.get() # start waku node let nodeRes = initAndStartNode(conf) @@ -297,6 +313,6 @@ when isMainModule: # spawn the routine that crawls the network # TODO: split into 3 routines (discovery, connections, ip2location) - asyncSpawn crawlNetwork(node, conf, allPeersInfo) + asyncSpawn crawlNetwork(node, restClient, conf, allPeersInfo) runForever() \ No newline at end of file diff --git a/tools/networkmonitor/networkmonitor_utils.nim b/tools/networkmonitor/networkmonitor_utils.nim index fca1ef12a..8e8ac4c88 100644 --- a/tools/networkmonitor/networkmonitor_utils.nim +++ b/tools/networkmonitor/networkmonitor_utils.nim @@ -4,14 +4,16 @@ else: {.push raises: [].} import - std/[json,httpclient], + std/json, + stew/results, + stew/shims/net, chronicles, chronicles/topics_registry, chronos, - stew/results + presto/[client,common] type - NodeLocation = object + NodeLocation* = object country*: string city*: string lat*: string @@ -24,30 +26,28 @@ proc flatten*[T](a: seq[seq[T]]): seq[T] = aFlat &= subseq return aFlat -# using an external api retrieves some data associated with the ip -# TODO: use a cache -# TODO: use nim-presto's HTTP asynchronous client -proc ipToLocation*(ip: string, - client: Httpclient): - Future[Result[NodeLocation, string]] {.async.} = - # naive mechanism to avoid hitting the rate limit - # IP-API endpoints are now limited to 45 HTTP requests per minute - await sleepAsync(1400) - try: - let content = client.getContent("http://ip-api.com/json/" & ip) - let jsonContent = parseJson(content) +proc decodeBytes*(t: typedesc[NodeLocation], value: openArray[byte], + contentType: Opt[ContentTypeData]): RestResult[NodeLocation] = + var res: string + if len(value) > 0: + res = newString(len(value)) + copyMem(addr res[0], unsafeAddr value[0], len(value)) + try: + let jsonContent = parseJson(res) + if $jsonContent["status"].getStr() != "success": + error "query failed", result=jsonContent + return err("query failed") + return ok(NodeLocation( + country: jsonContent["country"].getStr(), + city: jsonContent["city"].getStr(), + lat: $jsonContent["lat"].getFloat(), + long: $jsonContent["lon"].getFloat(), + isp: jsonContent["isp"].getStr() + )) + except: + return err("failed to get the location: " & getCurrentExceptionMsg()) - if $jsonContent["status"].getStr() != "success": - error "query failed", result=jsonContent - return err("query failed: " & $jsonContent) +proc encodeString*(value: string): RestResult[string] = + ok(value) - return ok(NodeLocation( - country: jsonContent["country"].getStr(), - city: jsonContent["city"].getStr(), - lat: jsonContent["lat"].getStr(), - long: jsonContent["lon"].getStr(), - isp: jsonContent["isp"].getStr() - )) - except: - error "failed to get the location for IP", ip=ip, error=getCurrentExceptionMsg() - return err("failed to get the location for IP '" & ip & "':" & getCurrentExceptionMsg()) \ No newline at end of file +proc ipToLocation*(ip: string): RestResponse[NodeLocation] {.rest, endpoint: "json/{ip}", meth: MethodGet.}