Clean up benchmarks and re-enable them (#550)

Limit them to metrics that don't change drastically in CI, remove any concurrency, remove HTTP

Author: adam-fowler

Benchmarks/Benchmarks/HTTP1/HTTP1ChannelBenchmarks.swift

@@ -16,14 +16,26 @@ import Benchmark
 import Hummingbird
 
 let benchmarks = {
+#if CI
+    Benchmark.defaultConfiguration = .init(
+        metrics: [
+            .instructions,
+            .mallocCountTotal,
+        ],
+        warmupIterations: 10
+    )
+    trieRouterBenchmarks()
+    routerBenchmarks()
+#else
     Benchmark.defaultConfiguration = .init(
         metrics: [
             .cpuTotal,
-            .throughput,
+            .instructions,
             .mallocCountTotal,
         ],
         warmupIterations: 10
     )
     trieRouterBenchmarks()
     routerBenchmarks()
+#endif
 }

Benchmarks/Benchmarks/Router/Benchmarks.swift

@@ -16,6 +16,7 @@ import Benchmark
 import HTTPTypes
 import Hummingbird
 import HummingbirdCore
+import HummingbirdRouter
 import Logging
 import NIOCore
 import NIOEmbedded
@@ -34,95 +35,141 @@ struct BasicBenchmarkContext: RequestContext {
 }
 
 public struct BenchmarkRequestContextSource: RequestContextSource {
-    public let allocator = ByteBufferAllocator()
     public let logger = Logger(label: "Benchmark")
 }
 
 /// Writes ByteBuffers to AsyncChannel outbound writer
 struct BenchmarkBodyWriter: Sendable, ResponseBodyWriter {
+    func finish(_ trailingHeaders: HTTPFields?) async throws {}
     func write(_: ByteBuffer) async throws {}
 }
 
+/// Implementation of a basic request context that supports everything the Hummingbird library needs
+struct BasicRouterBenchmarkContext: RouterRequestContext {
+    typealias Source = BenchmarkRequestContextSource
+
+    var coreContext: CoreRequestContextStorage
+    var routerContext: RouterBuilderContext
+
+    init(source: Source) {
+        self.coreContext = CoreRequestContextStorage(source: source)
+        self.routerContext = .init()
+    }
+}
+
 typealias ByteBufferWriter = (ByteBuffer) async throws -> Void
 extension Benchmark {
     @discardableResult
-    convenience init?<Context: RequestContext>(
-        name: String,
-        context: Context.Type = BasicBenchmarkContext.self,
+    convenience init?<ResponderBuilder: HTTPResponderBuilder>(
+        _ name: String,
         configuration: Benchmark.Configuration = Benchmark.defaultConfiguration,
         request: HTTPRequest,
-        writeBody: @escaping @Sendable (ByteBufferWriter) async throws -> Void = { _ in },
-        setupRouter: @escaping @Sendable (Router<Context>) async throws -> Void
-    ) where Context.Source == BenchmarkRequestContextSource {
-        let router = Router(context: Context.self)
+        writeBody: (@Sendable (ByteBufferWriter) async throws -> Void)? = nil,
+        createRouter: @escaping @Sendable () async throws -> ResponderBuilder
+    ) where ResponderBuilder.Responder.Context: RequestContext, ResponderBuilder.Responder.Context.Source == BenchmarkRequestContextSource {
         self.init(name, configuration: configuration) { benchmark in
-            let responder = router.buildResponder()
-            benchmark.startMeasurement()
+            let responder = try await createRouter().buildResponder()
+
+            if let writeBody {
+                let context = ResponderBuilder.Responder.Context(source: BenchmarkRequestContextSource())
 
-            for _ in benchmark.scaledIterations {
-                for _ in 0..<50 {
-                    try await withThrowingTaskGroup(of: Void.self) { group in
-                        let context = Context(source: BenchmarkRequestContextSource())
+                benchmark.startMeasurement()
+
+                for _ in benchmark.scaledIterations {
+                    for _ in 0..<50 {
                         let (requestBody, source) = RequestBody.makeStream()
                         let Request = Request(head: request, body: requestBody)
-                        group.addTask {
-                            let response = try await responder.respond(to: Request, context: context)
-                            _ = try await response.body.write(BenchmarkBodyWriter())
-                        }
                         try await writeBody(source.yield)
                         source.finish()
+                        let response = try await responder.respond(to: Request, context: context)
+                        _ = try await response.body.write(BenchmarkBodyWriter())
+                    }
+                }
+            } else {
+                let context = ResponderBuilder.Responder.Context(source: BenchmarkRequestContextSource())
+                let (requestBody, source) = RequestBody.makeStream()
+                let Request = Request(head: request, body: requestBody)
+                source.finish()
+
+                benchmark.startMeasurement()
+
+                for _ in benchmark.scaledIterations {
+                    for _ in 0..<50 {
+                        let response = try await responder.respond(to: Request, context: context)
+                        _ = try await response.body.write(BenchmarkBodyWriter())
                     }
                 }
             }
-        } setup: {
-            try await setupRouter(router)
-            Self.blackHole(MultiThreadedEventLoopGroup.singleton.any())
         }
     }
 }
 
+struct EmptyMiddleware<Context>: RouterMiddleware {
+    func handle(_ request: Request, context: Context, next: (Request, Context) async throws -> Response) async throws -> Response {
+        return try await next(request, context)
+    }
+}
+
+
 extension HTTPField.Name {
     static let test = Self("Test")!
 }
 
 func routerBenchmarks() {
     let buffer = ByteBufferAllocator().buffer(repeating: 0xFF, count: 10000)
     Benchmark(
-        name: "Router:GET",
+        "Router:GET",
         configuration: .init(warmupIterations: 10),
         request: .init(method: .get, scheme: "http", authority: "localhost", path: "/")
-    ) { router in
+    ) {
+        let router = Router(context: BasicBenchmarkContext.self)
         router.get { _, _ in
             buffer
         }
+        return router
     }
 
     Benchmark(
-        name: "Router:PUT",
+        "Router:Parameters",
+        configuration: .init(warmupIterations: 10),
+        request: .init(method: .get, scheme: "http", authority: "localhost", path: "/testthis")
+    ) {
+        let router = Router(context: BasicBenchmarkContext.self)
+        router.get("{test}") { _, context in
+            try context.parameters.require("test")
+        }
+        return router
+    }
+
+    Benchmark(
+        "Router:PUT",
         configuration: .init(warmupIterations: 10),
         request: .init(method: .put, scheme: "http", authority: "localhost", path: "/")
     ) { write in
         try await write(buffer)
         try await write(buffer)
         try await write(buffer)
         try await write(buffer)
-    } setupRouter: { router in
+    } createRouter: {
+        let router = Router(context: BasicBenchmarkContext.self)
         router.put { request, _ in
             let body = try await request.body.collect(upTo: .max)
             return body.readableBytes.description
         }
+        return router
     }
 
     Benchmark(
-        name: "Router:Echo",
+        "Router:Echo",
         configuration: .init(warmupIterations: 10),
         request: .init(method: .post, scheme: "http", authority: "localhost", path: "/")
     ) { write in
         try await write(buffer)
         try await write(buffer)
         try await write(buffer)
         try await write(buffer)
-    } setupRouter: { router in
+    } createRouter: { 
+        let router = Router(context: BasicBenchmarkContext.self)
         router.post { request, _ in
             Response(status: .ok, headers: [:], body: .init { writer in
                 for try await buffer in request.body {
@@ -131,24 +178,39 @@ func routerBenchmarks() {
                 try await writer.finish(nil)
             })
         }
+        return router
     }
 
     Benchmark(
-        name: "Middleware Chain",
+        "Router:Middleware",
         configuration: .init(warmupIterations: 10),
         request: .init(method: .get, scheme: "http", authority: "localhost", path: "/")
-    ) { router in
-        struct EmptyMiddleware<Context>: RouterMiddleware {
-            func handle(_ request: Request, context: Context, next: (Request, Context) async throws -> Response) async throws -> Response {
-                return try await next(request, context)
-            }
-        }
+    ) { 
+        let router = Router(context: BasicBenchmarkContext.self)
         router.middlewares.add(EmptyMiddleware())
         router.middlewares.add(EmptyMiddleware())
         router.middlewares.add(EmptyMiddleware())
         router.middlewares.add(EmptyMiddleware())
         router.get { _, _ in
             HTTPResponse.Status.ok
         }
+        return router
+    }
+
+    Benchmark(
+        "RouterBuilder:Middleware",
+        configuration: .init(warmupIterations: 10),
+        request: .init(method: .get, scheme: "http", authority: "localhost", path: "/")
+    ) {
+        let router = RouterBuilder(context: BasicRouterBenchmarkContext.self) {
+            EmptyMiddleware()
+            EmptyMiddleware()
+            EmptyMiddleware()
+            EmptyMiddleware()
+            Get { _,_ -> HTTPResponse.Status in
+                .ok
+            }
+        }
+        return router
     }
 }

Benchmarks/Benchmarks/Router/RouterBenchmarks.swift

@@ -11,24 +11,13 @@ let package = Package(
         .package(path: "../../hummingbird"),
     ],
     targets: [
-        // HTTP1 benchmarks
-        .executableTarget(
-            name: "HTTP1",
-            dependencies: [
-                .product(name: "Benchmark", package: "package-benchmark"),
-                .product(name: "Hummingbird", package: "hummingbird"),
-            ],
-            path: "Benchmarks/HTTP1",
-            plugins: [
-                .plugin(name: "BenchmarkPlugin", package: "package-benchmark"),
-            ]
-        ),
         // Router benchmarks
         .executableTarget(
             name: "Router",
             dependencies: [
                 .product(name: "Benchmark", package: "package-benchmark"),
                 .product(name: "Hummingbird", package: "hummingbird"),
+                .product(name: "HummingbirdRouter", package: "hummingbird"),
             ],
             path: "Benchmarks/Router",
             plugins: [