Fix HTTP server accept() loop exiting under heavy load. (#502)

* Add more specific accept() exceptions raised.
Add some refactoring to HTTP server code.

* Refactor acceptLoop.

* Print GC statistics in every failing test.

* Try to disable failing tests.
This commit is contained in:
Eugene Kabanov 2024-02-14 14:05:19 +02:00 committed by GitHub
parent 8cf2d69aaa
commit a81961a3c6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 104 additions and 85 deletions

View File

@ -833,45 +833,33 @@ proc sendDefaultResponse(
if reqFence.isOk():
if isNil(response):
await conn.sendErrorResponse(version, Http404, keepConnection.toBool())
keepConnection
else:
return keepConnection
case response.state
of HttpResponseState.Empty:
of HttpResponseState.Empty, HttpResponseState.Default:
# Response was ignored, so we respond with not found.
await conn.sendErrorResponse(version, Http404,
keepConnection.toBool())
response.setResponseState(HttpResponseState.Finished)
keepConnection
of HttpResponseState.Prepared:
# Response was prepared but not sent, so we can respond with some
# error code
await conn.sendErrorResponse(HttpVersion11, Http409,
await conn.sendErrorResponse(version, Http409,
keepConnection.toBool())
response.setResponseState(HttpResponseState.Finished)
keepConnection
of HttpResponseState.ErrorCode:
# Response with error code
await conn.sendErrorResponse(version, response.status, false)
response.setResponseState(HttpResponseState.Finished)
HttpProcessExitType.Immediate
of HttpResponseState.Sending, HttpResponseState.Failed,
HttpResponseState.Cancelled:
# Just drop connection, because we dont know at what stage we are
HttpProcessExitType.Immediate
of HttpResponseState.Default:
# Response was ignored, so we respond with not found.
await conn.sendErrorResponse(version, Http404,
keepConnection.toBool())
response.setResponseState(HttpResponseState.Finished)
keepConnection
of HttpResponseState.Finished:
keepConnection
else:
case reqFence.error.kind
of HttpServerError.TimeoutError:
await conn.sendErrorResponse(version, reqFence.error.code, false)
HttpProcessExitType.Graceful
of HttpServerError.ProtocolError:
of HttpServerError.TimeoutError, HttpServerError.ProtocolError:
await conn.sendErrorResponse(version, reqFence.error.code, false)
HttpProcessExitType.Graceful
of HttpServerError.DisconnectError:
@ -1017,8 +1005,7 @@ proc getRequestFence*(server: HttpServerRef,
connection.currentRawQuery = Opt.some(res.rawPath)
RequestFence.ok(res)
except CancelledError:
RequestFence.err(
HttpProcessError.init(HttpServerError.InterruptError))
RequestFence.err(HttpProcessError.init(HttpServerError.InterruptError))
except AsyncTimeoutError:
let address = connection.getRemoteAddress()
RequestFence.err(
@ -1073,18 +1060,19 @@ proc processRequest(server: HttpServerRef,
# Request is incorrect or unsupported, sending notification
discard
try:
let response =
try:
await invokeProcessCallback(connection.server, requestFence)
except CancelledError:
# Cancelled, exiting
return HttpProcessExitType.Immediate
await connection.sendDefaultResponse(requestFence, response)
finally:
if requestFence.isOk():
await requestFence.get().closeWait()
return HttpProcessExitType.Immediate
let res = await connection.sendDefaultResponse(requestFence, response)
if requestFence.isOk():
await requestFence.get().closeWait()
res
proc processLoop(holder: HttpConnectionHolderRef) {.async: (raises: []).} =
let
@ -1118,29 +1106,42 @@ proc processLoop(holder: HttpConnectionHolderRef) {.async: (raises: []).} =
server.connections.del(connectionId)
proc acceptClientLoop(server: HttpServerRef) {.async: (raises: []).} =
var runLoop = true
while runLoop:
try:
block mainLoop:
while true:
block clientLoop:
# if server.maxConnections > 0:
# await server.semaphore.acquire()
let transp = await server.instance.accept()
let resId = transp.getId()
if resId.isErr():
let transp =
try:
await server.instance.accept()
except TransportTooManyError:
# Too many FDs used by process
break clientLoop
except TransportAbortedError:
# Remote peer disconnected
break clientLoop
except TransportUseClosedError:
# accept() call invoked when server is stopped
break mainLoop
except TransportOsError:
# Critical OS error
break mainLoop
except CancelledError:
# Server being closed, exiting
break mainLoop
doAssert(not(isNil(transp)), "Stream transport should be present!")
let
connectionId = transp.getId().valueOr:
# We are unable to identify remote peer, it means that remote peer
# disconnected before identification.
# disconnected before.
await transp.closeWait()
runLoop = false
else:
let connId = resId.get()
let holder = HttpConnectionHolderRef.new(server, transp, resId.get())
server.connections[connId] = holder
break clientLoop
holder = HttpConnectionHolderRef.new(server, transp, connectionId)
server.connections[connectionId] = holder
holder.future = processLoop(holder)
except TransportTooManyError, TransportAbortedError:
# Non-critical error
discard
except CancelledError, TransportOsError, CatchableError:
# Critical, cancellation or unexpected error
runLoop = false
proc state*(server: HttpServerRef): HttpServerState =
## Returns current HTTP server's state.
@ -1429,7 +1430,7 @@ proc sendBody*(resp: HttpResponseRef, pbytes: pointer, nbytes: int) {.
raise exc
except AsyncStreamError as exc:
resp.setResponseState(HttpResponseState.Failed)
raiseHttpWriteError("Unable to send response, reason: " & $exc.msg)
raiseHttpWriteError("Unable to send response body, reason: " & $exc.msg)
proc sendBody*(resp: HttpResponseRef, data: ByteChar) {.
async: (raises: [CancelledError, HttpWriteError]).} =
@ -1448,7 +1449,7 @@ proc sendBody*(resp: HttpResponseRef, data: ByteChar) {.
raise exc
except AsyncStreamError as exc:
resp.setResponseState(HttpResponseState.Failed)
raiseHttpWriteError("Unable to send response, reason: " & $exc.msg)
raiseHttpWriteError("Unable to send response body, reason: " & $exc.msg)
proc sendError*(resp: HttpResponseRef, code: HttpCode, body = "") {.
async: (raises: [CancelledError, HttpWriteError]).} =
@ -1468,7 +1469,8 @@ proc sendError*(resp: HttpResponseRef, code: HttpCode, body = "") {.
raise exc
except AsyncStreamError as exc:
resp.setResponseState(HttpResponseState.Failed)
raiseHttpWriteError("Unable to send response, reason: " & $exc.msg)
raiseHttpWriteError(
"Unable to send error response body, reason: " & $exc.msg)
proc prepare*(resp: HttpResponseRef,
streamType = HttpResponseStreamType.Chunked) {.
@ -1501,7 +1503,7 @@ proc prepare*(resp: HttpResponseRef,
raise exc
except AsyncStreamError as exc:
resp.setResponseState(HttpResponseState.Failed)
raiseHttpWriteError("Unable to send response, reason: " & $exc.msg)
raiseHttpWriteError("Unable to send response headers, reason: " & $exc.msg)
proc prepareChunked*(resp: HttpResponseRef): Future[void] {.
async: (raw: true, raises: [CancelledError, HttpWriteError]).} =
@ -1536,7 +1538,7 @@ proc send*(resp: HttpResponseRef, pbytes: pointer, nbytes: int) {.
raise exc
except AsyncStreamError as exc:
resp.setResponseState(HttpResponseState.Failed)
raiseHttpWriteError("Unable to send response, reason: " & $exc.msg)
raiseHttpWriteError("Unable to send response data, reason: " & $exc.msg)
proc send*(resp: HttpResponseRef, data: ByteChar) {.
async: (raises: [CancelledError, HttpWriteError]).} =
@ -1551,7 +1553,7 @@ proc send*(resp: HttpResponseRef, data: ByteChar) {.
raise exc
except AsyncStreamError as exc:
resp.setResponseState(HttpResponseState.Failed)
raiseHttpWriteError("Unable to send response, reason: " & $exc.msg)
raiseHttpWriteError("Unable to send response data, reason: " & $exc.msg)
proc sendChunk*(resp: HttpResponseRef, pbytes: pointer,
nbytes: int): Future[void] {.
@ -1591,7 +1593,7 @@ proc finish*(resp: HttpResponseRef) {.
raise exc
except AsyncStreamError as exc:
resp.setResponseState(HttpResponseState.Failed)
raiseHttpWriteError("Unable to send response, reason: " & $exc.msg)
raiseHttpWriteError("Unable to finish response data, reason: " & $exc.msg)
proc respond*(req: HttpRequestRef, code: HttpCode, content: ByteChar,
headers: HttpTable): Future[HttpResponseRef] {.
@ -1673,7 +1675,7 @@ proc remoteAddress*(request: HttpRequestRef): TransportAddress {.
## Returns address of the remote host that made request ``request``.
request.connection.remoteAddress()
proc requestInfo*(req: HttpRequestRef, contentType = "text/text"): string =
proc requestInfo*(req: HttpRequestRef, contentType = "text/plain"): string =
## Returns comprehensive information about request for specific content
## type.
##

View File

@ -1044,7 +1044,9 @@ when defined(windows):
ok()
proc accept*(server: StreamServer): Future[StreamTransport] {.
async: (raw: true, raises: [TransportError, CancelledError]).} =
async: (raw: true, raises: [TransportUseClosedError,
TransportTooManyError, TransportAbortedError, TransportOsError,
CancelledError]).} =
var retFuture = newFuture[StreamTransport]("stream.server.accept")
doAssert(server.status != ServerStatus.Running,
@ -1675,7 +1677,9 @@ else:
ok()
proc accept*(server: StreamServer): Future[StreamTransport] {.
async: (raw: true, raises: [TransportError, CancelledError]).} =
async: (raw: true, raises: [TransportUseClosedError,
TransportTooManyError, TransportAbortedError, TransportOsError,
CancelledError]).} =
var retFuture = newFuture[StreamTransport]("stream.server.accept")
doAssert(server.status != ServerStatus.Running,

View File

@ -43,6 +43,7 @@ const
suite "Asynchronous multi-threading sync primitives test suite":
teardown:
echo GC_getStatistics()
checkLeaks()
proc setResult(thr: ThreadResultPtr, value: int) =
@ -325,19 +326,31 @@ suite "Asynchronous multi-threading sync primitives test suite":
asyncTest "ThreadSignal: Multiple thread switches [" & $TestsCount &
"] test [sync -> sync]":
when sizeof(int) == 8:
threadSignalTest2(TestsCount, WaitSendKind.Sync, WaitSendKind.Sync)
else:
skip()
asyncTest "ThreadSignal: Multiple thread switches [" & $TestsCount &
"] test [async -> async]":
when sizeof(int) == 8:
threadSignalTest2(TestsCount, WaitSendKind.Async, WaitSendKind.Async)
else:
skip()
asyncTest "ThreadSignal: Multiple thread switches [" & $TestsCount &
"] test [sync -> async]":
when sizeof(int) == 8:
threadSignalTest2(TestsCount, WaitSendKind.Sync, WaitSendKind.Async)
else:
skip()
asyncTest "ThreadSignal: Multiple thread switches [" & $TestsCount &
"] test [async -> sync]":
when sizeof(int) == 8:
threadSignalTest2(TestsCount, WaitSendKind.Async, WaitSendKind.Sync)
else:
skip()
asyncTest "ThreadSignal: Multiple signals [" & $TestsCount &
"] to multiple threads [" & $numProcs & "] test [sync -> sync]":