From 73fd1206ab0efc4e4c4c6fc6d5fe93d36c1778b2 Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Mon, 26 Jun 2023 16:28:33 +0300 Subject: [PATCH] Disable HTTP/1.1 pipeline support for HTTP/S server by default. (#410) * Disable HTTP/1.1 pipeline support for HTTP/S server by default. Add tests. * Fix tests according to new behavior. Fix tests to use random assigned OS ports instead of predefined. * Fix flaky tests in testasyncstream. --- chronos/apps/http/httpserver.nim | 23 ++- tests/testasyncstream.nim | 273 ++++++++++++++++--------------- tests/testhttpclient.nim | 221 ++++++++++++------------- tests/testhttpserver.nim | 211 ++++++++++++++++++------ 4 files changed, 432 insertions(+), 296 deletions(-) diff --git a/chronos/apps/http/httpserver.nim b/chronos/apps/http/httpserver.nim index 1da4b44..03aaaf9 100644 --- a/chronos/apps/http/httpserver.nim +++ b/chronos/apps/http/httpserver.nim @@ -25,6 +25,8 @@ type QueryCommaSeparatedArray ## Enable usage of comma as an array item delimiter in url-encoded ## entities (e.g. query string or POST body). + Http11Pipeline + ## Enable HTTP/1.1 pipelining. HttpServerError* {.pure.} = enum TimeoutError, CatchableError, RecoverableError, CriticalError, @@ -198,6 +200,20 @@ proc new*(htype: typedesc[HttpServerRef], ) ok(res) +proc getResponseFlags*(req: HttpRequestRef): set[HttpResponseFlags] = + var defaultFlags: set[HttpResponseFlags] = {} + case req.version + of HttpVersion11: + if HttpServerFlags.Http11Pipeline notin req.connection.server.flags: + return defaultFlags + let header = req.headers.getString(ConnectionHeader, "keep-alive") + if header == "keep-alive": + {HttpResponseFlags.KeepAlive} + else: + defaultFlags + else: + defaultFlags + proc getResponse*(req: HttpRequestRef): HttpResponseRef {.raises: [].} = if req.response.isNone(): var resp = HttpResponseRef( @@ -206,10 +222,7 @@ proc getResponse*(req: HttpRequestRef): HttpResponseRef {.raises: [].} = version: req.version, headersTable: HttpTable.init(), connection: req.connection, - flags: if req.version == HttpVersion11: - {HttpResponseFlags.KeepAlive} - else: - {} + flags: req.getResponseFlags() ) req.response = Opt.some(resp) resp @@ -792,7 +805,7 @@ proc processLoop(server: HttpServerRef, transp: StreamTransport, break else: let request = arg.get() - var keepConn = if request.version == HttpVersion11: true else: false + var keepConn = HttpResponseFlags.KeepAlive in request.getResponseFlags() if lastErrorCode.isNone(): if isNil(resp): # Response was `nil`. diff --git a/tests/testasyncstream.nim b/tests/testasyncstream.nim index 47a6c94..09a0b7e 100644 --- a/tests/testasyncstream.nim +++ b/tests/testasyncstream.nim @@ -145,7 +145,7 @@ proc createBigMessage(message: string, size: int): seq[byte] = suite "AsyncStream test suite": test "AsyncStream(StreamTransport) readExactly() test": - proc testReadExactly(address: TransportAddress): Future[bool] {.async.} = + proc testReadExactly(): Future[bool] {.async.} = proc serveClient(server: StreamServer, transp: StreamTransport) {.async.} = var wstream = newAsyncStreamWriter(transp) @@ -157,9 +157,10 @@ suite "AsyncStream test suite": server.close() var buffer = newSeq[byte](10) - var server = createStreamServer(address, serveClient, {ReuseAddr}) + var server = createStreamServer(initTAddress("127.0.0.1:0"), + serveClient, {ReuseAddr}) server.start() - var transp = await connect(address) + var transp = await connect(server.localAddress()) var rstream = newAsyncStreamReader(transp) await rstream.readExactly(addr buffer[0], 10) check cast[string](buffer) == "0000000000" @@ -171,9 +172,10 @@ suite "AsyncStream test suite": await transp.closeWait() await server.join() result = true - check waitFor(testReadExactly(initTAddress("127.0.0.1:46001"))) == true + check waitFor(testReadExactly()) == true + test "AsyncStream(StreamTransport) readUntil() test": - proc testReadUntil(address: TransportAddress): Future[bool] {.async.} = + proc testReadUntil(): Future[bool] {.async.} = proc serveClient(server: StreamServer, transp: StreamTransport) {.async.} = var wstream = newAsyncStreamWriter(transp) @@ -186,9 +188,10 @@ suite "AsyncStream test suite": var buffer = newSeq[byte](13) var sep = @[byte('N'), byte('N'), byte('z')] - var server = createStreamServer(address, serveClient, {ReuseAddr}) + var server = createStreamServer(initTAddress("127.0.0.1:0"), + serveClient, {ReuseAddr}) server.start() - var transp = await connect(address) + var transp = await connect(server.localAddress()) var rstream = newAsyncStreamReader(transp) var r1 = await rstream.readUntil(addr buffer[0], len(buffer), sep) check: @@ -207,9 +210,10 @@ suite "AsyncStream test suite": await transp.closeWait() await server.join() result = true - check waitFor(testReadUntil(initTAddress("127.0.0.1:46001"))) == true + check waitFor(testReadUntil()) == true + test "AsyncStream(StreamTransport) readLine() test": - proc testReadLine(address: TransportAddress): Future[bool] {.async.} = + proc testReadLine(): Future[bool] {.async.} = proc serveClient(server: StreamServer, transp: StreamTransport) {.async.} = var wstream = newAsyncStreamWriter(transp) @@ -220,9 +224,10 @@ suite "AsyncStream test suite": server.stop() server.close() - var server = createStreamServer(address, serveClient, {ReuseAddr}) + var server = createStreamServer(initTAddress("127.0.0.1:0"), + serveClient, {ReuseAddr}) server.start() - var transp = await connect(address) + var transp = await connect(server.localAddress()) var rstream = newAsyncStreamReader(transp) var r1 = await rstream.readLine() check r1 == "0000000000" @@ -234,9 +239,10 @@ suite "AsyncStream test suite": await transp.closeWait() await server.join() result = true - check waitFor(testReadLine(initTAddress("127.0.0.1:46001"))) == true + check waitFor(testReadLine()) == true + test "AsyncStream(StreamTransport) read() test": - proc testRead(address: TransportAddress): Future[bool] {.async.} = + proc testRead(): Future[bool] {.async.} = proc serveClient(server: StreamServer, transp: StreamTransport) {.async.} = var wstream = newAsyncStreamWriter(transp) @@ -247,9 +253,10 @@ suite "AsyncStream test suite": server.stop() server.close() - var server = createStreamServer(address, serveClient, {ReuseAddr}) + var server = createStreamServer(initTAddress("127.0.0.1:0"), + serveClient, {ReuseAddr}) server.start() - var transp = await connect(address) + var transp = await connect(server.localAddress()) var rstream = newAsyncStreamReader(transp) var buf1 = await rstream.read(10) check cast[string](buf1) == "0000000000" @@ -259,9 +266,10 @@ suite "AsyncStream test suite": await transp.closeWait() await server.join() result = true - check waitFor(testRead(initTAddress("127.0.0.1:46001"))) == true + check waitFor(testRead()) == true + test "AsyncStream(StreamTransport) consume() test": - proc testConsume(address: TransportAddress): Future[bool] {.async.} = + proc testConsume(): Future[bool] {.async.} = proc serveClient(server: StreamServer, transp: StreamTransport) {.async.} = var wstream = newAsyncStreamWriter(transp) @@ -272,9 +280,10 @@ suite "AsyncStream test suite": server.stop() server.close() - var server = createStreamServer(address, serveClient, {ReuseAddr}) + var server = createStreamServer(initTAddress("127.0.0.1:0"), + serveClient, {ReuseAddr}) server.start() - var transp = await connect(address) + var transp = await connect(server.localAddress()) var rstream = newAsyncStreamReader(transp) var res1 = await rstream.consume(10) check: @@ -290,7 +299,8 @@ suite "AsyncStream test suite": await transp.closeWait() await server.join() result = true - check waitFor(testConsume(initTAddress("127.0.0.1:46001"))) == true + check waitFor(testConsume()) == true + test "AsyncStream(StreamTransport) leaks test": check: getTracker("async.stream.reader").isLeaked() == false @@ -299,7 +309,7 @@ suite "AsyncStream test suite": getTracker("stream.transport").isLeaked() == false test "AsyncStream(AsyncStream) readExactly() test": - proc testReadExactly2(address: TransportAddress): Future[bool] {.async.} = + proc testReadExactly2(): Future[bool] {.async.} = proc serveClient(server: StreamServer, transp: StreamTransport) {.async.} = var wstream = newAsyncStreamWriter(transp) @@ -323,9 +333,10 @@ suite "AsyncStream test suite": server.close() var buffer = newSeq[byte](10) - var server = createStreamServer(address, serveClient, {ReuseAddr}) + var server = createStreamServer(initTAddress("127.0.0.1:0"), + serveClient, {ReuseAddr}) server.start() - var transp = await connect(address) + var transp = await connect(server.localAddress()) var rstream = newAsyncStreamReader(transp) var rstream2 = newChunkedStreamReader(rstream) await rstream2.readExactly(addr buffer[0], 10) @@ -347,9 +358,10 @@ suite "AsyncStream test suite": await transp.closeWait() await server.join() result = true - check waitFor(testReadExactly2(initTAddress("127.0.0.1:46001"))) == true + check waitFor(testReadExactly2()) == true + test "AsyncStream(AsyncStream) readUntil() test": - proc testReadUntil2(address: TransportAddress): Future[bool] {.async.} = + proc testReadUntil2(): Future[bool] {.async.} = proc serveClient(server: StreamServer, transp: StreamTransport) {.async.} = var wstream = newAsyncStreamWriter(transp) @@ -373,9 +385,10 @@ suite "AsyncStream test suite": var buffer = newSeq[byte](13) var sep = @[byte('N'), byte('N'), byte('z')] - var server = createStreamServer(address, serveClient, {ReuseAddr}) + var server = createStreamServer(initTAddress("127.0.0.1:0"), + serveClient, {ReuseAddr}) server.start() - var transp = await connect(address) + var transp = await connect(server.localAddress()) var rstream = newAsyncStreamReader(transp) var rstream2 = newChunkedStreamReader(rstream) @@ -404,9 +417,10 @@ suite "AsyncStream test suite": await transp.closeWait() await server.join() result = true - check waitFor(testReadUntil2(initTAddress("127.0.0.1:46001"))) == true + check waitFor(testReadUntil2()) == true + test "AsyncStream(AsyncStream) readLine() test": - proc testReadLine2(address: TransportAddress): Future[bool] {.async.} = + proc testReadLine2(): Future[bool] {.async.} = proc serveClient(server: StreamServer, transp: StreamTransport) {.async.} = var wstream = newAsyncStreamWriter(transp) @@ -425,9 +439,10 @@ suite "AsyncStream test suite": server.stop() server.close() - var server = createStreamServer(address, serveClient, {ReuseAddr}) + var server = createStreamServer(initTAddress("127.0.0.1:0"), + serveClient, {ReuseAddr}) server.start() - var transp = await connect(address) + var transp = await connect(server.localAddress()) var rstream = newAsyncStreamReader(transp) var rstream2 = newChunkedStreamReader(rstream) var r1 = await rstream2.readLine() @@ -449,9 +464,10 @@ suite "AsyncStream test suite": await transp.closeWait() await server.join() result = true - check waitFor(testReadLine2(initTAddress("127.0.0.1:46001"))) == true + check waitFor(testReadLine2()) == true + test "AsyncStream(AsyncStream) read() test": - proc testRead2(address: TransportAddress): Future[bool] {.async.} = + proc testRead2(): Future[bool] {.async.} = proc serveClient(server: StreamServer, transp: StreamTransport) {.async.} = var wstream = newAsyncStreamWriter(transp) @@ -469,9 +485,10 @@ suite "AsyncStream test suite": server.stop() server.close() - var server = createStreamServer(address, serveClient, {ReuseAddr}) + var server = createStreamServer(initTAddress("127.0.0.1:0"), + serveClient, {ReuseAddr}) server.start() - var transp = await connect(address) + var transp = await connect(server.localAddress()) var rstream = newAsyncStreamReader(transp) var rstream2 = newChunkedStreamReader(rstream) var buf1 = await rstream2.read(10) @@ -488,9 +505,10 @@ suite "AsyncStream test suite": await transp.closeWait() await server.join() result = true - check waitFor(testRead2(initTAddress("127.0.0.1:46001"))) == true + check waitFor(testRead2()) == true + test "AsyncStream(AsyncStream) consume() test": - proc testConsume2(address: TransportAddress): Future[bool] {.async.} = + proc testConsume2(): Future[bool] {.async.} = proc serveClient(server: StreamServer, transp: StreamTransport) {.async.} = const @@ -518,9 +536,10 @@ suite "AsyncStream test suite": server.stop() server.close() - var server = createStreamServer(address, serveClient, {ReuseAddr}) + var server = createStreamServer(initTAddress("127.0.0.1:0"), + serveClient, {ReuseAddr}) server.start() - var transp = await connect(address) + var transp = await connect(server.localAddress()) var rstream = newAsyncStreamReader(transp) var rstream2 = newChunkedStreamReader(rstream) @@ -547,9 +566,10 @@ suite "AsyncStream test suite": await transp.closeWait() await server.join() result = true - check waitFor(testConsume2(initTAddress("127.0.0.1:46001"))) == true + check waitFor(testConsume2()) == true + test "AsyncStream(AsyncStream) write(eof) test": - proc testWriteEof(address: TransportAddress): Future[bool] {.async.} = + proc testWriteEof(): Future[bool] {.async.} = let size = 10240 message = createBigMessage("ABCDEFGHIJKLMNOP", size) @@ -578,7 +598,8 @@ suite "AsyncStream test suite": await transp.closeWait() let flags = {ServerFlags.ReuseAddr, ServerFlags.TcpNoDelay} - var server = createStreamServer(address, processClient, flags = flags) + var server = createStreamServer(initTAddress("127.0.0.1:0"), + processClient, flags = flags) server.start() var conn = await connect(server.localAddress()) try: @@ -589,7 +610,8 @@ suite "AsyncStream test suite": await server.closeWait() return true - check waitFor(testWriteEof(initTAddress("127.0.0.1:46001"))) == true + check waitFor(testWriteEof()) == true + test "AsyncStream(AsyncStream) leaks test": check: getTracker("async.stream.reader").isLeaked() == false @@ -624,8 +646,7 @@ suite "ChunkedStream test suite": " in\r\n\r\nchunks.\r\n0;position=4\r\n\r\n", "Wikipedia in\r\n\r\nchunks."], ] - proc checkVector(address: TransportAddress, - inputstr: string): Future[string] {.async.} = + proc checkVector(inputstr: string): Future[string] {.async.} = proc serveClient(server: StreamServer, transp: StreamTransport) {.async.} = var wstream = newAsyncStreamWriter(transp) @@ -637,9 +658,10 @@ suite "ChunkedStream test suite": server.stop() server.close() - var server = createStreamServer(address, serveClient, {ReuseAddr}) + var server = createStreamServer(initTAddress("127.0.0.1:0"), + serveClient, {ReuseAddr}) server.start() - var transp = await connect(address) + var transp = await connect(server.localAddress()) var rstream = newAsyncStreamReader(transp) var rstream2 = newChunkedStreamReader(rstream) var res = await rstream2.read() @@ -650,15 +672,16 @@ suite "ChunkedStream test suite": await server.join() result = ress - proc testVectors(address: TransportAddress): Future[bool] {.async.} = + proc testVectors(): Future[bool] {.async.} = var res = true for i in 0..= 5: let (code, data) = await session.fetch(ha.getUri()) await session.closeWait() @@ -691,26 +703,22 @@ suite "HTTP client testing suite": await server.closeWait() return "redirect-" & $res - proc testBasicAuthorization(): Future[bool] {.async.} = - let session = HttpSessionRef.new({HttpClientFlag.NoVerifyHost}, - maxRedirections = 10) - let url = parseUri("https://guest:guest@jigsaw.w3.org/HTTP/Basic/") - let resp = await session.fetch(url) - await session.closeWait() - if (resp.status == 200) and - ("Your browser made it!" in bytesToString(resp.data)): - return true - else: - echo "RESPONSE STATUS = [", resp.status, "]" - echo "RESPONSE = [", bytesToString(resp.data), "]" - return false + # proc testBasicAuthorization(): Future[bool] {.async.} = + # let session = HttpSessionRef.new({HttpClientFlag.NoVerifyHost}, + # maxRedirections = 10) + # let url = parseUri("https://guest:guest@jigsaw.w3.org/HTTP/Basic/") + # let resp = await session.fetch(url) + # await session.closeWait() + # if (resp.status == 200) and + # ("Your browser made it!" in bytesToString(resp.data)): + # return true + # else: + # echo "RESPONSE STATUS = [", resp.status, "]" + # echo "RESPONSE = [", bytesToString(resp.data), "]" + # return false - proc testConnectionManagement(address: TransportAddress): Future[bool] {. + proc testConnectionManagement(): Future[bool] {. async.} = - let - keepHa = getAddress(address, HttpClientScheme.NonSecure, "/keep") - dropHa = getAddress(address, HttpClientScheme.NonSecure, "/drop") - proc test1( a1: HttpAddress, version: HttpVersion, @@ -772,8 +780,13 @@ suite "HTTP client testing suite": else: return dumbResponse() - var server = createServer(address, process, false) + var server = createServer(initTAddress("127.0.0.1:0"), process, false) server.start() + let address = server.instance.localAddress() + + let + keepHa = getAddress(address, HttpClientScheme.NonSecure, "/keep") + dropHa = getAddress(address, HttpClientScheme.NonSecure, "/drop") try: let @@ -872,11 +885,7 @@ suite "HTTP client testing suite": return true - proc testIdleConnection(address: TransportAddress): Future[bool] {. - async.} = - let - ha = getAddress(address, HttpClientScheme.NonSecure, "/test") - + proc testIdleConnection(): Future[bool] {.async.} = proc test( session: HttpSessionRef, a: HttpAddress @@ -902,11 +911,14 @@ suite "HTTP client testing suite": else: return dumbResponse() - var server = createServer(address, process, false) + var server = createServer(initTAddress("127.0.0.1:0"), process, false) server.start() - let session = HttpSessionRef.new({HttpClientFlag.Http11Pipeline}, - idleTimeout = 1.seconds, - idlePeriod = 200.milliseconds) + let + address = server.instance.localAddress() + ha = getAddress(address, HttpClientScheme.NonSecure, "/test") + session = HttpSessionRef.new({HttpClientFlag.Http11Pipeline}, + idleTimeout = 1.seconds, + idlePeriod = 200.milliseconds) try: var f1 = test(session, ha) var f2 = test(session, ha) @@ -932,12 +944,7 @@ suite "HTTP client testing suite": return true - proc testNoPipeline(address: TransportAddress): Future[bool] {. - async.} = - let - ha = getAddress(address, HttpClientScheme.NonSecure, "/test") - hb = getAddress(address, HttpClientScheme.NonSecure, "/keep-test") - + proc testNoPipeline(): Future[bool] {.async.} = proc test( session: HttpSessionRef, a: HttpAddress @@ -966,10 +973,14 @@ suite "HTTP client testing suite": else: return dumbResponse() - var server = createServer(address, process, false) + var server = createServer(initTAddress("127.0.0.1:0"), process, false) server.start() - let session = HttpSessionRef.new(idleTimeout = 100.seconds, - idlePeriod = 10.milliseconds) + let + address = server.instance.localAddress() + ha = getAddress(address, HttpClientScheme.NonSecure, "/test") + hb = getAddress(address, HttpClientScheme.NonSecure, "/keep-test") + session = HttpSessionRef.new(idleTimeout = 100.seconds, + idlePeriod = 10.milliseconds) try: var f1 = test(session, ha) var f2 = test(session, ha) @@ -1001,8 +1012,7 @@ suite "HTTP client testing suite": return true - proc testServerSentEvents(address: TransportAddress, - secure: bool): Future[bool] {.async.} = + proc testServerSentEvents(secure: bool): Future[bool] {.async.} = const SingleGoodTests = [ ("/test/single/1", "a:b\r\nc: d\re:f\n:comment\r\ng:\n h: j \n\n", @@ -1117,8 +1127,9 @@ suite "HTTP client testing suite": else: return dumbResponse() - var server = createServer(address, process, secure) + var server = createServer(initTAddress("127.0.0.1:0"), process, secure) server.start() + let address = server.instance.localAddress() var session = createSession(secure) @@ -1184,87 +1195,71 @@ suite "HTTP client testing suite": return true test "HTTP all request methods test": - let address = initTAddress("127.0.0.1:30080") - check waitFor(testMethods(address, false)) == 18 + check waitFor(testMethods(false)) == 18 test "HTTP(S) all request methods test": - let address = initTAddress("127.0.0.1:30080") - check waitFor(testMethods(address, true)) == 18 + check waitFor(testMethods(true)) == 18 test "HTTP client response streaming test": - let address = initTAddress("127.0.0.1:30080") - check waitFor(testResponseStreamReadingTest(address, false)) == 8 + check waitFor(testResponseStreamReadingTest(false)) == 8 test "HTTP(S) client response streaming test": - let address = initTAddress("127.0.0.1:30080") - check waitFor(testResponseStreamReadingTest(address, true)) == 8 + check waitFor(testResponseStreamReadingTest(true)) == 8 test "HTTP client (size) request streaming test": - let address = initTAddress("127.0.0.1:30080") - check waitFor(testRequestSizeStreamWritingTest(address, false)) == 2 + check waitFor(testRequestSizeStreamWritingTest(false)) == 2 test "HTTP(S) client (size) request streaming test": - let address = initTAddress("127.0.0.1:30080") - check waitFor(testRequestSizeStreamWritingTest(address, true)) == 2 + check waitFor(testRequestSizeStreamWritingTest(true)) == 2 test "HTTP client (chunked) request streaming test": - let address = initTAddress("127.0.0.1:30080") - check waitFor(testRequestChunkedStreamWritingTest(address, false)) == 2 + check waitFor(testRequestChunkedStreamWritingTest(false)) == 2 test "HTTP(S) client (chunked) request streaming test": - let address = initTAddress("127.0.0.1:30080") - check waitFor(testRequestChunkedStreamWritingTest(address, true)) == 2 + check waitFor(testRequestChunkedStreamWritingTest(true)) == 2 test "HTTP client (size + chunked) url-encoded POST test": - let address = initTAddress("127.0.0.1:30080") - check waitFor(testRequestPostUrlEncodedTest(address, false)) == 2 + check waitFor(testRequestPostUrlEncodedTest(false)) == 2 test "HTTP(S) client (size + chunked) url-encoded POST test": - let address = initTAddress("127.0.0.1:30080") - check waitFor(testRequestPostUrlEncodedTest(address, true)) == 2 + check waitFor(testRequestPostUrlEncodedTest(true)) == 2 test "HTTP client (size + chunked) multipart POST test": - let address = initTAddress("127.0.0.1:30080") - check waitFor(testRequestPostMultipartTest(address, false)) == 2 + check waitFor(testRequestPostMultipartTest(false)) == 2 test "HTTP(S) client (size + chunked) multipart POST test": - let address = initTAddress("127.0.0.1:30080") - check waitFor(testRequestPostMultipartTest(address, true)) == 2 + check waitFor(testRequestPostMultipartTest(true)) == 2 test "HTTP client redirection test": - let address = initTAddress("127.0.0.1:30080") - check waitFor(testRequestRedirectTest(address, false, 5)) == "ok-5-200" + check waitFor(testRequestRedirectTest(false, 5)) == "ok-5-200" test "HTTP(S) client redirection test": - let address = initTAddress("127.0.0.1:30080") - check waitFor(testRequestRedirectTest(address, true, 5)) == "ok-5-200" + check waitFor(testRequestRedirectTest(true, 5)) == "ok-5-200" test "HTTP client maximum redirections test": - let address = initTAddress("127.0.0.1:30080") - check waitFor(testRequestRedirectTest(address, false, 4)) == "redirect-true" + check waitFor(testRequestRedirectTest(false, 4)) == "redirect-true" test "HTTP(S) client maximum redirections test": - let address = initTAddress("127.0.0.1:30080") - check waitFor(testRequestRedirectTest(address, true, 4)) == "redirect-true" + check waitFor(testRequestRedirectTest(true, 4)) == "redirect-true" test "HTTPS basic authorization test": - check waitFor(testBasicAuthorization()) == true + skip() + # This test disabled because remote service is pretty flaky and fails pretty + # often. As soon as more stable service will be found this test should be + # recovered + # check waitFor(testBasicAuthorization()) == true test "HTTP client connection management test": - let address = initTAddress("127.0.0.1:30080") - check waitFor(testConnectionManagement(address)) == true + check waitFor(testConnectionManagement()) == true test "HTTP client idle connection test": - let address = initTAddress("127.0.0.1:30080") - check waitFor(testIdleConnection(address)) == true + check waitFor(testIdleConnection()) == true test "HTTP client no-pipeline test": - let address = initTAddress("127.0.0.1:30080") - check waitFor(testNoPipeline(address)) == true + check waitFor(testNoPipeline()) == true test "HTTP client server-sent events test": - let address = initTAddress("127.0.0.1:30080") - check waitFor(testServerSentEvents(address, false)) == true + check waitFor(testServerSentEvents(false)) == true test "Leaks test": proc getTrackerLeaks(tracker: string): bool = diff --git a/tests/testhttpserver.nim b/tests/testhttpserver.nim index acf8b20..63c92b2 100644 --- a/tests/testhttpserver.nim +++ b/tests/testhttpserver.nim @@ -8,7 +8,8 @@ import std/[strutils, algorithm] import unittest2 import ../chronos, ../chronos/apps/http/httpserver, - ../chronos/apps/http/httpcommon + ../chronos/apps/http/httpcommon, + ../chronos/unittest2/asynctests import stew/base10 {.used.} @@ -17,6 +18,9 @@ suite "HTTP server testing suite": type TooBigTest = enum GetBodyTest, ConsumeBodyTest, PostUrlTest, PostMultipartTest + TestHttpResponse = object + headers: HttpTable + data: string proc httpClient(address: TransportAddress, data: string): Future[string] {.async.} = @@ -33,8 +37,32 @@ suite "HTTP server testing suite": if not(isNil(transp)): await closeWait(transp) - proc testTooBigBodyChunked(address: TransportAddress, - operation: TooBigTest): Future[bool] {.async.} = + proc httpClient2(transp: StreamTransport, + request: string, + length: int): Future[TestHttpResponse] {.async.} = + var buffer = newSeq[byte](4096) + var sep = @[0x0D'u8, 0x0A'u8, 0x0D'u8, 0x0A'u8] + let wres = await transp.write(request) + if wres != len(request): + raise newException(ValueError, "Unable to write full request") + let hres = await transp.readUntil(addr buffer[0], len(buffer), sep) + var hdata = @buffer + hdata.setLen(hres) + zeroMem(addr buffer[0], len(buffer)) + await transp.readExactly(addr buffer[0], length) + let data = bytesToString(buffer.toOpenArray(0, length - 1)) + let headers = + block: + let resp = parseResponse(hdata, false) + if resp.failed(): + raise newException(ValueError, "Unable to decode response headers") + var res = HttpTable.init() + for key, value in resp.headers(hdata): + res.add(key, value) + res + return TestHttpResponse(headers: headers, data: data) + + proc testTooBigBodyChunked(operation: TooBigTest): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. async.} = @@ -59,7 +87,7 @@ suite "HTTP server testing suite": return dumbResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} - let res = HttpServerRef.new(address, process, + let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), process, maxRequestBodySize = 10, socketFlags = socketFlags) if res.isErr(): @@ -67,6 +95,7 @@ suite "HTTP server testing suite": let server = res.get() server.start() + let address = server.instance.localAddress() let request = case operation @@ -97,7 +126,7 @@ suite "HTTP server testing suite": return serverRes and (data.startsWith("HTTP/1.1 413")) test "Request headers timeout test": - proc testTimeout(address: TransportAddress): Future[bool] {.async.} = + proc testTimeout(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. async.} = @@ -110,23 +139,25 @@ suite "HTTP server testing suite": return dumbResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} - let res = HttpServerRef.new(address, process, socketFlags = socketFlags, + let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), + process, socketFlags = socketFlags, httpHeadersTimeout = 100.milliseconds) if res.isErr(): return false let server = res.get() server.start() + let address = server.instance.localAddress() let data = await httpClient(address, "") await server.stop() await server.closeWait() return serverRes and (data.startsWith("HTTP/1.1 408")) - check waitFor(testTimeout(initTAddress("127.0.0.1:30080"))) == true + check waitFor(testTimeout()) == true test "Empty headers test": - proc testEmpty(address: TransportAddress): Future[bool] {.async.} = + proc testEmpty(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. async.} = @@ -139,22 +170,24 @@ suite "HTTP server testing suite": return dumbResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} - let res = HttpServerRef.new(address, process, socketFlags = socketFlags) + let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), + process, socketFlags = socketFlags) if res.isErr(): return false let server = res.get() server.start() + let address = server.instance.localAddress() let data = await httpClient(address, "\r\n\r\n") await server.stop() await server.closeWait() return serverRes and (data.startsWith("HTTP/1.1 400")) - check waitFor(testEmpty(initTAddress("127.0.0.1:30080"))) == true + check waitFor(testEmpty()) == true test "Too big headers test": - proc testTooBig(address: TransportAddress): Future[bool] {.async.} = + proc testTooBig(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. async.} = @@ -167,7 +200,7 @@ suite "HTTP server testing suite": return dumbResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} - let res = HttpServerRef.new(address, process, + let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), process, maxHeadersSize = 10, socketFlags = socketFlags) if res.isErr(): @@ -175,16 +208,17 @@ suite "HTTP server testing suite": let server = res.get() server.start() + let address = server.instance.localAddress() let data = await httpClient(address, "GET / HTTP/1.1\r\n\r\n") await server.stop() await server.closeWait() return serverRes and (data.startsWith("HTTP/1.1 431")) - check waitFor(testTooBig(initTAddress("127.0.0.1:30080"))) == true + check waitFor(testTooBig()) == true test "Too big request body test (content-length)": - proc testTooBigBody(address: TransportAddress): Future[bool] {.async.} = + proc testTooBigBody(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. async.} = @@ -196,7 +230,7 @@ suite "HTTP server testing suite": return dumbResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} - let res = HttpServerRef.new(address, process, + let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), process, maxRequestBodySize = 10, socketFlags = socketFlags) if res.isErr(): @@ -204,6 +238,7 @@ suite "HTTP server testing suite": let server = res.get() server.start() + let address = server.instance.localAddress() let request = "GET / HTTP/1.1\r\nContent-Length: 20\r\n\r\n" let data = await httpClient(address, request) @@ -211,30 +246,26 @@ suite "HTTP server testing suite": await server.closeWait() return serverRes and (data.startsWith("HTTP/1.1 413")) - check waitFor(testTooBigBody(initTAddress("127.0.0.1:30080"))) == true + check waitFor(testTooBigBody()) == true test "Too big request body test (getBody()/chunked encoding)": check: - waitFor(testTooBigBodyChunked(initTAddress("127.0.0.1:30080"), - GetBodyTest)) == true + waitFor(testTooBigBodyChunked(GetBodyTest)) == true test "Too big request body test (consumeBody()/chunked encoding)": check: - waitFor(testTooBigBodyChunked(initTAddress("127.0.0.1:30080"), - ConsumeBodyTest)) == true + waitFor(testTooBigBodyChunked(ConsumeBodyTest)) == true test "Too big request body test (post()/urlencoded/chunked encoding)": check: - waitFor(testTooBigBodyChunked(initTAddress("127.0.0.1:30080"), - PostUrlTest)) == true + waitFor(testTooBigBodyChunked(PostUrlTest)) == true test "Too big request body test (post()/multipart/chunked encoding)": check: - waitFor(testTooBigBodyChunked(initTAddress("127.0.0.1:30080"), - PostMultipartTest)) == true + waitFor(testTooBigBodyChunked(PostMultipartTest)) == true test "Query arguments test": - proc testQuery(address: TransportAddress): Future[bool] {.async.} = + proc testQuery(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. async.} = @@ -252,13 +283,14 @@ suite "HTTP server testing suite": return dumbResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} - let res = HttpServerRef.new(address, process, + let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), process, socketFlags = socketFlags) if res.isErr(): return false let server = res.get() server.start() + let address = server.instance.localAddress() let data1 = await httpClient(address, "GET /?a=1&a=2&b=3&c=4 HTTP/1.0\r\n\r\n") @@ -271,10 +303,10 @@ suite "HTTP server testing suite": (data2.find("TEST_OK:a:П:b:Ц:c:Ю:Ф:Б") >= 0) return r - check waitFor(testQuery(initTAddress("127.0.0.1:30080"))) == true + check waitFor(testQuery()) == true test "Headers test": - proc testHeaders(address: TransportAddress): Future[bool] {.async.} = + proc testHeaders(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. async.} = @@ -292,13 +324,14 @@ suite "HTTP server testing suite": return dumbResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} - let res = HttpServerRef.new(address, process, + let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), process, socketFlags = socketFlags) if res.isErr(): return false let server = res.get() server.start() + let address = server.instance.localAddress() let message = "GET / HTTP/1.0\r\n" & @@ -314,10 +347,10 @@ suite "HTTP server testing suite": await server.closeWait() return serverRes and (data.find(expect) >= 0) - check waitFor(testHeaders(initTAddress("127.0.0.1:30080"))) == true + check waitFor(testHeaders()) == true test "POST arguments (urlencoded/content-length) test": - proc testPostUrl(address: TransportAddress): Future[bool] {.async.} = + proc testPostUrl(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. async.} = @@ -337,13 +370,14 @@ suite "HTTP server testing suite": return dumbResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} - let res = HttpServerRef.new(address, process, + let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), process, socketFlags = socketFlags) if res.isErr(): return false let server = res.get() server.start() + let address = server.instance.localAddress() let message = "POST / HTTP/1.0\r\n" & @@ -357,10 +391,10 @@ suite "HTTP server testing suite": await server.closeWait() return serverRes and (data.find(expect) >= 0) - check waitFor(testPostUrl(initTAddress("127.0.0.1:30080"))) == true + check waitFor(testPostUrl()) == true test "POST arguments (urlencoded/chunked encoding) test": - proc testPostUrl2(address: TransportAddress): Future[bool] {.async.} = + proc testPostUrl2(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. async.} = @@ -380,13 +414,14 @@ suite "HTTP server testing suite": return dumbResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} - let res = HttpServerRef.new(address, process, + let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), process, socketFlags = socketFlags) if res.isErr(): return false let server = res.get() server.start() + let address = server.instance.localAddress() let message = "POST / HTTP/1.0\r\n" & @@ -401,10 +436,10 @@ suite "HTTP server testing suite": await server.closeWait() return serverRes and (data.find(expect) >= 0) - check waitFor(testPostUrl2(initTAddress("127.0.0.1:30080"))) == true + check waitFor(testPostUrl2()) == true test "POST arguments (multipart/content-length) test": - proc testPostMultipart(address: TransportAddress): Future[bool] {.async.} = + proc testPostMultipart(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. async.} = @@ -424,13 +459,14 @@ suite "HTTP server testing suite": return dumbResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} - let res = HttpServerRef.new(address, process, + let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), process, socketFlags = socketFlags) if res.isErr(): return false let server = res.get() server.start() + let address = server.instance.localAddress() let message = "POST / HTTP/1.0\r\n" & @@ -456,10 +492,10 @@ suite "HTTP server testing suite": await server.closeWait() return serverRes and (data.find(expect) >= 0) - check waitFor(testPostMultipart(initTAddress("127.0.0.1:30080"))) == true + check waitFor(testPostMultipart()) == true test "POST arguments (multipart/chunked encoding) test": - proc testPostMultipart2(address: TransportAddress): Future[bool] {.async.} = + proc testPostMultipart2(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. async.} = @@ -479,13 +515,14 @@ suite "HTTP server testing suite": return dumbResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} - let res = HttpServerRef.new(address, process, + let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), process, socketFlags = socketFlags) if res.isErr(): return false let server = res.get() server.start() + let address = server.instance.localAddress() let message = "POST / HTTP/1.0\r\n" & @@ -520,12 +557,12 @@ suite "HTTP server testing suite": await server.closeWait() return serverRes and (data.find(expect) >= 0) - check waitFor(testPostMultipart2(initTAddress("127.0.0.1:30080"))) == true + check waitFor(testPostMultipart2()) == true test "drop() connections test": const ClientsCount = 10 - proc testHTTPdrop(address: TransportAddress): Future[bool] {.async.} = + proc testHTTPdrop(): Future[bool] {.async.} = var eventWait = newAsyncEvent() var eventContinue = newAsyncEvent() var count = 0 @@ -542,7 +579,7 @@ suite "HTTP server testing suite": return dumbResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} - let res = HttpServerRef.new(address, process, + let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), process, socketFlags = socketFlags, maxConnections = 100) if res.isErr(): @@ -550,6 +587,7 @@ suite "HTTP server testing suite": let server = res.get() server.start() + let address = server.instance.localAddress() var clients: seq[Future[string]] let message = "GET / HTTP/1.0\r\nHost: https://127.0.0.1:80\r\n\r\n" @@ -572,7 +610,7 @@ suite "HTTP server testing suite": return false return true - check waitFor(testHTTPdrop(initTAddress("127.0.0.1:30080"))) == true + check waitFor(testHTTPdrop()) == true test "Content-Type multipart boundary test": const AllowedCharacters = { @@ -1190,7 +1228,7 @@ suite "HTTP server testing suite": r6.get() == MediaType.init(req[1][6]) test "SSE server-side events stream test": - proc testPostMultipart2(address: TransportAddress): Future[bool] {.async.} = + proc testPostMultipart2(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. async.} = @@ -1212,13 +1250,14 @@ suite "HTTP server testing suite": return dumbResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} - let res = HttpServerRef.new(address, process, + let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), process, socketFlags = socketFlags) if res.isErr(): return false let server = res.get() server.start() + let address = server.instance.localAddress() let message = "GET / HTTP/1.1\r\n" & @@ -1237,8 +1276,84 @@ suite "HTTP server testing suite": await server.closeWait() return serverRes and (data.find(expect) >= 0) - check waitFor(testPostMultipart2(initTAddress("127.0.0.1:30080"))) == true + check waitFor(testPostMultipart2()) == true + asyncTest "HTTP/1.1 pipeline test": + const TestMessages = [ + ("GET / HTTP/1.0\r\n\r\n", + {HttpServerFlags.Http11Pipeline}, false, "close"), + ("GET / HTTP/1.0\r\nConnection: close\r\n\r\n", + {HttpServerFlags.Http11Pipeline}, false, "close"), + ("GET / HTTP/1.0\r\nConnection: keep-alive\r\n\r\n", + {HttpServerFlags.Http11Pipeline}, false, "close"), + ("GET / HTTP/1.0\r\n\r\n", + {}, false, "close"), + ("GET / HTTP/1.0\r\nConnection: close\r\n\r\n", + {}, false, "close"), + ("GET / HTTP/1.0\r\nConnection: keep-alive\r\n\r\n", + {}, false, "close"), + ("GET / HTTP/1.1\r\n\r\n", + {HttpServerFlags.Http11Pipeline}, true, "keep-alive"), + ("GET / HTTP/1.1\r\nConnection: close\r\n\r\n", + {HttpServerFlags.Http11Pipeline}, false, "close"), + ("GET / HTTP/1.1\r\nConnection: keep-alive\r\n\r\n", + {HttpServerFlags.Http11Pipeline}, true, "keep-alive"), + ("GET / HTTP/1.1\r\n\r\n", + {}, false, "close"), + ("GET / HTTP/1.1\r\nConnection: close\r\n\r\n", + {}, false, "close"), + ("GET / HTTP/1.1\r\nConnection: keep-alive\r\n\r\n", + {}, false, "close") + ] + + proc process(r: RequestFence): Future[HttpResponseRef] {.async.} = + if r.isOk(): + let request = r.get() + return await request.respond(Http200, "TEST_OK", HttpTable.init()) + else: + return dumbResponse() + + for test in TestMessages: + let + socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} + serverFlags = test[1] + res = HttpServerRef.new(initTAddress("127.0.0.1:0"), process, + socketFlags = socketFlags, + serverFlags = serverFlags) + check res.isOk() + + let + server = res.get() + address = server.instance.localAddress() + + server.start() + var transp: StreamTransport + try: + transp = await connect(address) + block: + let response = await transp.httpClient2(test[0], 7) + check: + response.data == "TEST_OK" + response.headers.getString("connection") == test[3] + # We do this sleeping here just because we running both server and + # client in single process, so when we received response from server + # it does not mean that connection has been immediately closed - it + # takes some more calls, so we trying to get this calls happens. + await sleepAsync(50.milliseconds) + let connectionStillAvailable = + try: + let response {.used.} = await transp.httpClient2(test[0], 7) + true + except CatchableError: + false + + check connectionStillAvailable == test[2] + + finally: + if not(isNil(transp)): + await transp.closeWait() + await server.stop() + await server.closeWait() test "Leaks test": check: