Repository: akka/akka-http

Issue #1724: java.lang.IllegalStateException: Cannot open connection when slot still has an open connection
Created: 2018-01-01T07:14:59Z
Build.sbt
lazy val akkaHttpVersion = "10.1.0-RC1"
lazy val akkaVersion    = "2.5.8"
Code -
val QueueSize = 100
  //A create http client pool to core server
  val poolClientFlow = Http().cachedHostConnectionPool[Promise[HttpResponse]](ip, port)

  val queue =
    Source.queue[(HttpRequest, Promise[HttpResponse])](QueueSize, OverflowStrategy.dropNew)
      .via(poolClientFlow)
      .toMat(Sink.foreach({
        case ((Success(resp), p)) => p.success(resp)
        case ((Failure(e), p)) => p.failure(e)
      }))(Keep.left)
      .run()

  def queueRequest(request: HttpRequest): Future[HttpResponse] = {
    val responsePromise = Promise[HttpResponse]()
    queue.offer(request -> responsePromise).flatMap {
      case QueueOfferResult.Enqueued => responsePromise.future
      case QueueOfferResult.Dropped => Future.failed(new RuntimeException("Queue overflowed. Try again later."))
      case QueueOfferResult.Failure(ex) => Future.failed(ex)
      case QueueOfferResult.QueueClosed => Future.failed(new RuntimeException("Queue was closed (pool shut down) while running the request. Try again later."))
    }
  }
Settings
host-connection-pool {
   # The maximum number of parallel connections that a connection pool to a
    # single host endpoint is allowed to establish. Must be greater than zero.
    max-connections = 200

    # The minimum number of parallel connections that a pool should keep alive ("hot").
    # If the number of connections is falling below the given threshold, new ones are being spawned.
    # You can use this setting to build a hot pool of "always on" connections.
    # Default is 0, meaning there might be no active connection at given moment.
    # Keep in mind that `min-connections` should be smaller than `max-connections` or equal
    min-connections = 50

    # The maximum number of times failed requests are attempted again,
    # (if the request can be safely retried) before giving up and returning an error.
    # Set to zero to completely disable request retries.
    max-retries = 5
  
   # The maximum number of open requests accepted into the pool across all
    # materializations of any of its client flows.
    # Protects against (accidentally) overloading a single pool with too many client flow materializations.
    # Note that with N concurrent materializations the max number of open request in the pool
    # will never exceed N * max-connections * pipelining-limit.
    # Must be a power of 2 and > 0!
    max-open-requests =128
    
    
    client {
  	 # The time period within which the TCP connecting process must be completed.
    connecting-timeout = 10s
     # The time after which an idle connection will be automatically closed.
    # Set to `infinite` to completely disable idle timeouts.
    idle-timeout = 60s
    
    # Socket options to set for the listening socket. If a setting is left
    # undefined, it will use whatever the default on the system is.
    socket-options {
      so-receive-buffer-size = undefined
      so-send-buffer-size = undefined
      so-reuse-address = undefined
      so-traffic-class = undefined
      tcp-keep-alive = undefined
      tcp-oob-inline = undefined
      tcp-no-delay = undefined
    }
    
   }

When using Host Connection Pool mechanism to connect to a http server , this exceptions are happening when multiple connections are happening with in a second
2018-01-01 12:42:53.153 [default-akka.actor.default-dispatcher-42] ERROR a.h.impl.engine.client.PoolGateway -  [35 (Unconnected)] Slot execution failed. That's probably a bug. Please file a bug at https://github.com/akka/akka-http/issues. Slot is restarted. 
java.lang.IllegalStateException: Cannot open connection when slot still has an open connection
	at akka.http.impl.engine.client.pool.NewHostConnectionPool$HostConnectionPoolStage$$anon$1$Slot.openConnection(NewHostConnectionPool.scala:285)
	at akka.http.impl.engine.client.pool.SlotState$Unconnected$.onPreConnect(SlotState.scala:151)
	at akka.http.impl.engine.client.pool.NewHostConnectionPool$HostConnectionPoolStage$$anon$1$Slot.$anonfun$updateState$6(NewHostConnectionPool.scala:212)
	at akka.http.impl.engine.client.pool.NewHostConnectionPool$HostConnectionPoolStage$$anon$1$Slot.runOneTransition$1(NewHostConnectionPool.scala:175)
	at akka.http.impl.engine.client.pool.NewHostConnectionPool$HostConnectionPoolStage$$anon$1$Slot.loop$1(NewHostConnectionPool.scala:239)
	at akka.http.impl.engine.client.pool.NewHostConnectionPool$HostConnectionPoolStage$$anon$1$Slot.updateState(NewHostConnectionPool.scala:248)
	at akka.http.impl.engine.client.pool.NewHostConnectionPool$HostConnectionPoolStage$$anon$1$Slot.onConnectionFailed(NewHostConnectionPool.scala:166)
	at akka.http.impl.engine.client.pool.NewHostConnectionPool$HostConnectionPoolStage$$anon$1$SlotConnection.$anonfun$onUpstreamFailure$1(NewHostConnectionPool.scala:395)
	at akka.http.impl.engine.client.pool.NewHostConnectionPool$HostConnectionPoolStage$$anon$1$SlotConnection.$anonfun$onUpstreamFailure$1$adapted(NewHostConnectionPool.scala:393)
	at akka.http.impl.engine.client.pool.NewHostConnectionPool$HostConnectionPoolStage$$anon$1$SlotConnection.withSlot(NewHostConnectionPool.scala:328)
	at akka.http.impl.engine.client.pool.NewHostConnectionPool$HostConnectionPoolStage$$anon$1$SlotConnection.onUpstreamFailure(NewHostConnectionPool.scala:393)
	at akka.stream.stage.GraphStageLogic$SubSinkInlet.$anonfun$_sink$1(GraphStage.scala:1266)
	at akka.stream.stage.GraphStageLogic$SubSinkInlet.$anonfun$_sink$1$adapted(GraphStage.scala:1255)
	at akka.stream.impl.fusing.GraphInterpreter.runAsyncInput(GraphInterpreter.scala:447)
	at akka.stream.impl.fusing.GraphInterpreterShell$AsyncInput.execute(ActorGraphInterpreter.scala:464)
	at akka.stream.impl.fusing.GraphInterpreterShell.processEvent(ActorGraphInterpreter.scala:559)
	at akka.stream.impl.fusing.ActorGraphInterpreter.akka$stream$impl$fusing$ActorGraphInterpreter$$processEvent(ActorGraphInterpreter.scala:741)
	at akka.stream.impl.fusing.ActorGraphInterpreter.akka$stream$impl$fusing$ActorGraphInterpreter$$shortCircuitBatch(ActorGraphInterpreter.scala:731)
	at akka.stream.impl.fusing.ActorGraphInterpreter$$anonfun$receive$1.applyOrElse(ActorGraphInterpreter.scala:757)
	at akka.actor.Actor.aroundReceive(Actor.scala:517)
	at akka.actor.Actor.aroundReceive$(Actor.scala:515)
	at akka.stream.impl.fusing.ActorGraphInterpreter.aroundReceive(ActorGraphInterpreter.scala:666)
	at akka.actor.ActorCell.receiveMessage(ActorCell.scala:527)
	at akka.actor.ActorCell.invoke(ActorCell.scala:496)
	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:257)
	at akka.dispatch.Mailbox.run(Mailbox.scala:224)
	at akka.dispatch.Mailbox.exec(Mailbox.scala:234)
	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)

Duplicate issue #1809: Http Client - Slot execution failed
Created: 2018-01-23T16:39:00Z
Using 10.1.0-M1, I get the error: "Slot execution failed. That's probably a bug." At scale, this seems to happen relatively frequently. It causes the future to never be fulfilled.
The full debug log:
{"@timestamp":"2018-01-23T16:31:31.873+00:00","@version":1,"message":"[0 (Idle)] State change [WaitingForEndOfResponseEntity] -> [Idle]","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-108","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-60","akkaTimestamp":"16:31:31.873UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:31.874+00:00","@version":1,"message":"[0 (Idle)] Slot became idle... Trying to pull","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-108","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-60","akkaTimestamp":"16:31:31.873UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:32.863+00:00","@version":1,"message":"[0 (Idle)] Dispatching request","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-62","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-70","akkaTimestamp":"16:31:32.863UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:32.863+00:00","@version":1,"message":"[0 (WaitingForResponse)] State change [WaitingForEndOfRequestEntity] -> [WaitingForResponse]","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-62","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-70","akkaTimestamp":"16:31:32.863UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:32.863+00:00","@version":1,"message":"[0 (WaitingForResponse)] State change [Idle] -> [WaitingForResponse]","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-62","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-70","akkaTimestamp":"16:31:32.863UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:32.866+00:00","@version":1,"message":"[0 (WaitingForResponse)] Received response","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-107","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-62","akkaTimestamp":"16:31:32.866UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:32.866+00:00","@version":1,"message":"[0 (WaitingForResponseDispatch)] State change [WaitingForResponse] -> [WaitingForResponseDispatch]","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-107","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-62","akkaTimestamp":"16:31:32.866UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:32.867+00:00","@version":1,"message":"[0 (WaitingForResponseEntitySubscription)] State change [WaitingForResponseDispatch] -> [WaitingForResponseEntitySubscription]","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-107","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-62","akkaTimestamp":"16:31:32.866UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:32.867+00:00","@version":1,"message":"[0 (WaitingForEndOfResponseEntity)] State change [WaitingForResponseEntitySubscription] -> [WaitingForEndOfResponseEntity]","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-107","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-62","akkaTimestamp":"16:31:32.866UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:32.867+00:00","@version":1,"message":"[0 (Idle)] State change [WaitingForEndOfResponseEntity] -> [Idle]","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-107","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-62","akkaTimestamp":"16:31:32.866UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:32.867+00:00","@version":1,"message":"[0 (Idle)] Slot became idle... Trying to pull","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-107","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-62","akkaTimestamp":"16:31:32.866UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:35.623+00:00","@version":1,"message":"[0 (Idle)] Dispatching request","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-107","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-78","akkaTimestamp":"16:31:35.623UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:35.623+00:00","@version":1,"message":"[0 (WaitingForResponse)] State change [WaitingForEndOfRequestEntity] -> [WaitingForResponse]","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-107","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-78","akkaTimestamp":"16:31:35.623UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:35.623+00:00","@version":1,"message":"[0 (WaitingForResponse)] State change [Idle] -> [WaitingForResponse]","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-107","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-78","akkaTimestamp":"16:31:35.623UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:35.625+00:00","@version":1,"message":"[0 (WaitingForResponse)] Received response","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-78","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-107","akkaTimestamp":"16:31:35.624UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:35.625+00:00","@version":1,"message":"[0 (WaitingForResponseDispatch)] State change [WaitingForResponse] -> [WaitingForResponseDispatch]","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-80","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-107","akkaTimestamp":"16:31:35.625UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:35.625+00:00","@version":1,"message":"[0 (WaitingForResponseEntitySubscription)] State change [WaitingForResponseDispatch] -> [WaitingForResponseEntitySubscription]","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-80","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-107","akkaTimestamp":"16:31:35.625UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:35.625+00:00","@version":1,"message":"[0 (WaitingForEndOfResponseEntity)] State change [WaitingForResponseEntitySubscription] -> [WaitingForEndOfResponseEntity]","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-80","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-107","akkaTimestamp":"16:31:35.625UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:35.625+00:00","@version":1,"message":"[0 (Idle)] State change [WaitingForEndOfResponseEntity] -> [Idle]","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-80","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-107","akkaTimestamp":"16:31:35.625UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:35.625+00:00","@version":1,"message":"[0 (Idle)] Slot became idle... Trying to pull","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-80","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-107","akkaTimestamp":"16:31:35.625UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:36.408+00:00","@version":1,"message":"[0 (Idle)] Dispatching request","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-114","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-103","akkaTimestamp":"16:31:36.408UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:36.408+00:00","@version":1,"message":"[0 (WaitingForResponse)] State change [WaitingForEndOfRequestEntity] -> [WaitingForResponse]","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-114","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-103","akkaTimestamp":"16:31:36.408UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:36.408+00:00","@version":1,"message":"[0 (WaitingForResponse)] State change [Idle] -> [WaitingForResponse]","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-114","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-103","akkaTimestamp":"16:31:36.408UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:36.408+00:00","@version":1,"message":"[1 (Idle)] Dispatching request","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-80","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-103","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego","akkaTimestamp":"16:31:36.408UTC"}
{"@timestamp":"2018-01-23T16:31:36.408+00:00","@version":1,"message":"[1 (WaitingForResponse)] State change [WaitingForEndOfRequestEntity] -> [WaitingForResponse]","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-80","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-103","akkaTimestamp":"16:31:36.408UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:36.408+00:00","@version":1,"message":"[1 (WaitingForResponse)] State change [Idle] -> [WaitingForResponse]","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-80","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-103","akkaTimestamp":"16:31:36.408UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:36.410+00:00","@version":1,"message":"[2 (Idle)] Dispatching request","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-80","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-103","akkaTimestamp":"16:31:36.410UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:36.410+00:00","@version":1,"message":"[2 (WaitingForResponse)] State change [WaitingForEndOfRequestEntity] -> [WaitingForResponse]","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-80","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-103","akkaTimestamp":"16:31:36.410UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:36.410+00:00","@version":1,"message":"[2 (WaitingForResponse)] State change [Idle] -> [WaitingForResponse]","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-80","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-103","akkaTimestamp":"16:31:36.410UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:36.410+00:00","@version":1,"message":"[3 (Unconnected)] Dispatching request","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-114","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-56","akkaTimestamp":"16:31:36.410UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:36.413+00:00","@version":1,"message":"[3 (Unconnected)] Slot execution failed. That's probably a bug. Please file a bug at https://github.com/akka/akka-http/issues. Slot is restarted.","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-80","level":"ERROR","level_value":40000,"stack_trace":"java.lang.IllegalStateException: Cannot open connection when slot still has an open connection\n\tat akka.http.impl.engine.client.pool.NewHostConnectionPool$HostConnectionPoolStage$$anon$1$Slot.openConnection(NewHostConnectionPool.scala:285)\n\tat akka.http.impl.engine.client.pool.SlotState$Unconnected$.onNewRequest(SlotState.scala:156)\n\tat akka.http.impl.engine.client.pool.NewHostConnectionPool$HostConnectionPoolStage$$anon$1$Slot$$anonfun$dispatchRequest$1.apply(NewHostConnectionPool.scala:147)\n\tat akka.http.impl.engine.client.pool.NewHostConnectionPool$HostConnectionPoolStage$$anon$1$Slot$$anonfun$dispatchRequest$1.apply(NewHostConnectionPool.scala:147)\n\tat akka.http.impl.engine.client.pool.NewHostConnectionPool$HostConnectionPoolStage$$anon$1$Slot.runOneTransition$1(NewHostConnectionPool.scala:175)\n\tat akka.http.impl.engine.client.pool.NewHostConnectionPool$HostConnectionPoolStage$$anon$1$Slot.loop$1(NewHostConnectionPool.scala:239)\n\tat akka.http.impl.engine.client.pool.NewHostConnectionPool$HostConnectionPoolStage$$anon$1$Slot.updateState(NewHostConnectionPool.scala:248)\n\tat akka.http.impl.engine.client.pool.NewHostConnectionPool$HostConnectionPoolStage$$anon$1$Slot.dispatchRequest(NewHostConnectionPool.scala:147)\n\tat akka.http.impl.engine.client.pool.NewHostConnectionPool$HostConnectionPoolStage$$anon$1.dispatchRequest(NewHostConnectionPool.scala:114)\n\tat akka.http.impl.engine.client.pool.NewHostConnectionPool$HostConnectionPoolStage$$anon$1.onPush(NewHostConnectionPool.scala:81)\n\tat akka.stream.impl.fusing.GraphInterpreter.processPush(GraphInterpreter.scala:499)\n\tat akka.stream.impl.fusing.GraphInterpreter.processEvent(GraphInterpreter.scala:462)\n\tat akka.stream.impl.fusing.GraphInterpreter.execute(GraphInterpreter.scala:368)\n\tat akka.stream.impl.fusing.GraphInterpreterShell.runBatch(ActorGraphInterpreter.scala:571)\n\tat akka.stream.impl.fusing.ActorGraphInterpreter$SimpleBoundaryEvent$class.execute(ActorGraphInterpreter.scala:42)\n\tat akka.stream.impl.fusing.ActorGraphInterpreter$BatchingActorInputBoundary$OnNext.execute(ActorGraphInterpreter.scala:75)\n\tat akka.stream.impl.fusing.GraphInterpreterShell.processEvent(ActorGraphInterpreter.scala:546)\n\tat akka.stream.impl.fusing.ActorGraphInterpreter.akka$stream$impl$fusing$ActorGraphInterpreter$$processEvent(ActorGraphInterpreter.scala:725)\n\tat akka.stream.impl.fusing.ActorGraphInterpreter$$anonfun$receive$1.applyOrElse(ActorGraphInterpreter.scala:740)\n\tat akka.actor.Actor$class.aroundReceive(Actor.scala:513)\n\tat akka.stream.impl.fusing.ActorGraphInterpreter.aroundReceive(ActorGraphInterpreter.scala:650)\n\tat akka.actor.ActorCell.receiveMessage(ActorCell.scala:519)\n\tat akka.actor.ActorCell.invoke(ActorCell.scala:488)\n\tat akka.dispatch.Mailbox.processMailbox(Mailbox.scala:257)\n\tat akka.dispatch.Mailbox.run(Mailbox.scala:224)\n\tat akka.dispatch.Mailbox.exec(Mailbox.scala:234)\n\tat akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)\n\tat akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)\n\tat akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)\n\tat akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)\n","sourceThread":"lego-akka.actor.default-dispatcher-56","akkaTimestamp":"16:31:36.410UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:36.413+00:00","@version":1,"message":"[3 (Unconnected)] Establishing connection","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-80","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-56","akkaTimestamp":"16:31:36.410UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}
{"@timestamp":"2018-01-23T16:31:36.413+00:00","@version":1,"message":"[3 (PreConnecting)] State change [Unconnected] -> [PreConnecting]","logger_name":"akka.http.impl.engine.client.PoolGateway","thread_name":"lego-akka.actor.default-dispatcher-80","level":"DEBUG","level_value":10000,"sourceThread":"lego-akka.actor.default-dispatcher-56","akkaTimestamp":"16:31:36.412UTC","akkaSource":"lego/Pool(shared->http://localhost:9995)","sourceActorSystem":"lego"}

Closing comment (2018-01-23T16:43:23Z): That's probably a duplicate of #1724. Can you try version "10.1.0-RC1+64-3a7a288f" from this maven repository https://dl.bintray.com/akka/maven/?