Changelog 3.1 version

3.1.1

released 25th February 2025

Client

HttpCache: Cache isn't updated when Vary header for 304 response matches but not equal to Vary for 200 response

Logging: messages are printed per line with OkHttp logger format

Race condition when writing to a buffer leads to NPE inside CIOReaderKt.readFrom

WebSocket and SSE don't respect connection timeout set in the HttpTimeout plugin

https://youtrack.jetbrains.com/issue/KTOR-8206

Hi, I am using ktor websocket client. Http client is configured with connection timeout

During experiments I noticed that connection timeout is not honored.

I believe that the problem is in the plugin - it exits early for all websocket requests.

Http client:

private val client = HttpClient(Java) {
        install(WebSockets) { pingInterval = PING_INTERVAL.inWholeMilliseconds }
        install(HttpTimeout) {
            connectTimeoutMillis = 3000
        }
    }

    public companion object Plugin :
        HttpClientPlugin<HttpTimeoutCapabilityConfiguration, HttpTimeout>,
        HttpClientEngineCapability<HttpTimeoutCapabilityConfiguration> {
...
        @OptIn(InternalAPI::class)
        override fun install(plugin: HttpTimeout, scope: HttpClient) {
            scope.plugin(HttpSend).intercept { request ->
                val isWebSocket = request.url.protocol.isWebsocket()
                if (isWebSocket || request.body is ClientUpgradeContent) return@intercept execute(request)

                var configuration = request.getCapabilityOrNull(HttpTimeout)
                if (configuration == null && plugin.hasNotNullTimeouts()) {
                    configuration = HttpTimeoutCapabilityConfiguration()
                    request.setCapability(HttpTimeout, configuration)
                }

                configuration?.apply {
                    connectTimeoutMillis = connectTimeoutMillis ?: plugin.connectTimeoutMillis
                    socketTimeoutMillis = socketTimeoutMillis ?: plugin.socketTimeoutMillis
                    requestTimeoutMillis = requestTimeoutMillis ?: plugin.requestTimeoutMillis

                    val requestTimeout = requestTimeoutMillis ?: plugin.requestTimeoutMillis
                    if (requestTimeout == null || requestTimeout == INFINITE_TIMEOUT_MS) return@apply

                    val executionContext = request.executionContext
                    val killer = scope.launch {
                        delay(requestTimeout)
                        val cause = HttpRequestTimeoutException(request)
                        LOGGER.trace("Request timeout: ${request.url}")
                        executionContext.cancel(cause.message!!, cause)
                    }

                    request.executionContext.invokeOnCompletion {
                        killer.cancel()
                    }
                }
                execute(request)
            }
        }
    }

ArrayIndexOutOfBounds kotlinx-io

JS/WASM fails with "IllegalStateException: Content-Length mismatch" on requesting gzipped content

Core

formData: implementation of copying Source is broken

TLS client: IOException while writing to a closed TLS socket since 3.0.0

https://youtrack.jetbrains.com/issue/KTOR-7860

If you close a TCP TLS socket while you are writing a buffer, the app crash with the following error:

FATAL EXCEPTION: DefaultDispatcher-worker-9

                                                                                                    PID: 29107

                                                                                                    java.io.IOException: Channel is closed for write

                                                                                                     at io.ktor.utils.io.ByteChannel.getWriteBuffer(ByteChannel.kt:49)

                                                                                                     at io.ktor.utils.io.ByteWriteChannelOperationsKt.writeByte(ByteWriteChannelOperations.kt:18)

                                                                                                     at io.ktor.network.tls.RenderKt.writeRecord(Render.kt:18)

                                                                                                     at io.ktor.network.tls.TLSClientHandshake$output$1.invokeSuspend(TLSClientHandshake.kt:119)

                                                                                                     at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:33)

                                                                                                     at kotlinx.coroutines.DispatchedTask.run(DispatchedTask.kt:101)

                                                                                                     at kotlinx.coroutines.scheduling.CoroutineScheduler.runSafely(CoroutineScheduler.kt:589)

                                                                                                     at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.executeTask(CoroutineScheduler.kt:832)

                                                                                                     at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.runWorker(CoroutineScheduler.kt:720)

                                                                                                     at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.run(CoroutineScheduler.kt:707)

                                                                                                     Suppressed: kotlinx.coroutines.internal.DiagnosticCoroutineContextException: [CoroutineName(cio-tls-encoder), ActorCoroutine{Cancelling}@cf8623c, Dispatchers.Default]

I'm only able to reproduce this error using version 3.0.1 with TCP TLS socket. If you use the same code version 3.0.1 without TLS or version 2.3.13 with and without TLS, all is working fine.

OOM on SourceByteReadChannel for large input

Docs

Docs: Docker example includes unnecessary copy statements in the Dockerfile

Gradle Plugin

Support enabling development mode from the CLI

Server

Exception thrown in onCallRespond makes the client wait for response indefinitely

Resources: a / route isn't resolved when there is a sibling `staticResources`

Other

NPE in readBuffer

https://youtrack.jetbrains.com/issue/KTOR-8086

Encountered when Grazie team was migrating to Ktor 2 to 3. Occurs when there are many concurrent requests.

https://jetbrains.slack.com/archives/C07U498LLUR/p1737356811807179

java.lang.NullPointerException
  at kotlinx.io.Buffer.write(Buffer.kt:452)
  at kotlinx.io.Buffer.readAtMostTo(Buffer.kt:319)
  at kotlinx.io.Buffer.transferFrom(Buffer.kt:486)
  at io.ktor.utils.io.ByteReadChannelOperationsKt.readBuffer(ByteReadChannelOperations.kt:84)
  at io.ktor.utils.io.ByteReadChannelOperationsKt.toByteArray(ByteReadChannelOperations.kt:38)
  at io.ktor.client.plugins.DefaultTransformKt$defaultTransformers$2.invokeSuspend(DefaultTransform.kt:81)
  at io.ktor.client.plugins.DefaultTransformKt$defaultTransformers$2.invoke(DefaultTransform.kt)
  at io.ktor.client.plugins.DefaultTransformKt$defaultTransformers$2.invoke(DefaultTransform.kt)
  at io.ktor.util.pipeline.DebugPipelineContext.proceedLoop(DebugPipelineContext.kt:79)
  at io.ktor.util.pipeline.DebugPipelineContext.proceed(DebugPipelineContext.kt:57)
  at io.ktor.client.HttpClient$4.invokeSuspend(HttpClient.kt:1379)
  at io.ktor.client.HttpClient$4.invoke(HttpClient.kt)
  at io.ktor.client.HttpClient$4.invoke(HttpClient.kt)
  at io.ktor.util.pipeline.DebugPipelineContext.proceedLoop(DebugPipelineContext.kt:79)
  at io.ktor.util.pipeline.DebugPipelineContext.proceed(DebugPipelineContext.kt:57)
  at io.ktor.client.plugins.ReceiveError$install$1.invokeSuspend(HttpCallValidator.kt:149)
  at io.ktor.client.plugins.ReceiveError$install$1.invoke(HttpCallValidator.kt)
  at io.ktor.client.plugins.ReceiveError$install$1.invoke(HttpCallValidator.kt)
  at io.ktor.util.pipeline.DebugPipelineContext.proceedLoop(DebugPipelineContext.kt:79)
  at io.ktor.util.pipeline.DebugPipelineContext.proceed(DebugPipelineContext.kt:57)
  at io.ktor.util.pipeline.DebugPipelineContext.execute$ktor_utils(DebugPipelineContext.kt:63)
  at io.ktor.util.pipeline.Pipeline.execute(Pipeline.kt:86)
  at io.ktor.client.call.HttpClientCall.bodyNullable(HttpClientCall.kt:87)
  at ai.grazie.client.ktor.GrazieKtorHTTPClient.toResponse(GrazieKtorHTTPClient.kt:342)
  at ai.grazie.client.ktor.GrazieKtorHTTPClient.access$toResponse(GrazieKtorHTTPClient.kt:31)
  at ai.grazie.client.ktor.GrazieKtorHTTPClient$send$2.invoke(GrazieKtorHTTPClient.kt:62)
  at ai.grazie.client.ktor.GrazieKtorHTTPClient$send$2.invoke(GrazieKtorHTTPClient.kt:62)
  at ai.grazie.client.ktor.GrazieKtorHTTPClient$sendAndWaitBody$2.invokeSuspend(GrazieKtorHTTPClient.kt:172)
  at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:33)
  at kotlinx.coroutines.DispatchedTask.run(DispatchedTask.kt:101)
  at kotlinx.coroutines.EventLoopImplBase.processNextEvent(EventLoop.common.kt:263)
  at kotlinx.coroutines.BlockingCoroutine.joinBlocking(Builders.kt:95)
  at kotlinx.coroutines.BuildersKt__BuildersKt.runBlocking(Builders.kt:69)
  at kotlinx.coroutines.BuildersKt.runBlocking(Unknown Source)
  at kotlinx.coroutines.BuildersKt__BuildersKt.runBlocking$default(Builders.kt:47)
  at kotlinx.coroutines.BuildersKt.runBlocking$default(Unknown Source)

3.1.0

released 12th February 2025

Client

Add reconnection in ClientSSESession

OutOfMemoryError when sending a large binary file through ByteReadChannel converted from InputStream

https://youtrack.jetbrains.com/issue/KTOR-6185

Im trying to upload a 5G video and i see a crash on android device:
this is a crash:
Fatal Exception: java.lang.OutOfMemoryError: Failed to allocate a 48 byte allocation with 4278960 free bytes and 4178KB until OOM, target footprint 536870912, growth limit 536870912; giving up on allocation because <1% of heap free after GC.
       at java.nio.ByteBuffer.wrap(ByteBuffer.java:322)
       at io.ktor.utils.io.bits.MemoryFactoryJvmKt.useMemory$default(MemoryFactoryJvm.kt:20)
       at io.ktor.utils.io.bits.MemoryJvmKt.copyTo-SG11BkQ(MemoryJvm.kt:191)
       at io.ktor.utils.io.core.BufferPrimitivesJvmKt.writeFully(BufferPrimitivesJvm.kt:22)
       at io.ktor.utils.io.ByteBufferChannel.readAsMuchAsPossible(ByteBufferChannel.kt:516)
       at io.ktor.utils.io.ByteBufferChannel.readAsMuchAsPossible$default(ByteBufferChannel.kt:499)
       at io.ktor.utils.io.ByteBufferChannel.readRemainingSuspend(ByteBufferChannel.kt:2046)
       at io.ktor.utils.io.ByteBufferChannel.access$getWriteOp(ByteBufferChannel.kt:23)
       at io.ktor.utils.io.ByteBufferChannel.access$readRemainingSuspend(ByteBufferChannel.kt:23)
       at io.ktor.utils.io.ByteBufferChannel$readRemainingSuspend$1.invokeSuspend(ByteBufferChannel.kt:13)
       at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:33)
       at kotlinx.coroutines.internal.DispatchedContinuation.resumeWith(DispatchedContinuation.kt:202)
       at io.ktor.utils.io.internal.CancellableReusableContinuation.resumeWith(CancellableReusableContinuation.kt:93)
       at io.ktor.utils.io.ByteBufferChannel.resumeReadOp(ByteBufferChannel.kt:2059)
       at io.ktor.utils.io.ByteBufferChannel.flushImpl(ByteBufferChannel.kt:185)
       at io.ktor.utils.io.ByteBufferChannel.flush(ByteBufferChannel.kt:195)
       at io.ktor.utils.io.ByteBufferChannel.writing(ByteBufferChannel.kt:446)
       at io.ktor.utils.io.ByteBufferChannel.writeAsMuchAsPossible(ByteBufferChannel.kt:1347)
       at io.ktor.utils.io.ByteBufferChannel.writeSuspend(ByteBufferChannel.kt:1438)
       at io.ktor.utils.io.ByteBufferChannel.access$getWriteOp(ByteBufferChannel.kt:23)
       at io.ktor.utils.io.ByteBufferChannel.access$writeSuspend(ByteBufferChannel.kt:23)
       at io.ktor.utils.io.ByteBufferChannel.access$getWriteOp(ByteBufferChannel.kt:23)
       at io.ktor.utils.io.ByteBufferChannel.access$writeSuspend(ByteBufferChannel.kt:23)
       at io.ktor.utils.io.ByteBufferChannel$writeSuspend$1.invokeSuspend(ByteBufferChannel.kt:13)
       at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:33)
       at kotlinx.coroutines.DispatchedTask.run(DispatchedTask.kt:106)
       at kotlinx.coroutines.scheduling.CoroutineScheduler.runSafely(CoroutineScheduler.kt:584)
       at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.executeTask(CoroutineScheduler.kt:793)
       at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.runWorker(CoroutineScheduler.kt:697)
       at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.run(CoroutineScheduler.kt:684)

This is my code:

val call = client.patch(url) {
                headers {
                    append("Upload-Offset", offset)
                    append("Tus-Resumable", "1.0.0")
                    append("Content-Type", "application/offset+octet-stream")
                    if (sizeBytes != null)
                        append("Content-Length", (sizeBytes - offset.toLong()).toString())
                }
                setBody(byteReadChannel)
                onUpload { bytesSentTotal, size -\>
                    trySendBlocking(
                        UploadStatus(
                            bytesSentTotal + offset.toLong(),
                            sizeBytes ?: size,
                            null
                        )
                    )
                }
            }.call

..........................................................................................................................

File(entry.deviceMedia.data).inputStream()
                            .toByteReadChannelWithOffset(offset = offset.toInt())

..........................................................................................................................

@OptIn(DelicateCoroutinesApi::class)
@Suppress("BlockingMethodInNonBlockingContext")
@JvmName("toByteReadChannelWithArrayPool")
fun InputStream.toByteReadChannelWithOffset(
    context: CoroutineContext = Dispatchers.IO,
    pool: ObjectPool\<ByteArray\> = ByteArrayPool,
    offset: Int = 0
): ByteReadChannel = GlobalScope.writer(context, autoFlush = true) {
    val buffer = pool.borrow()
    try {
        this@toByteReadChannelWithOffset.skip(offset.toLong())
        while (true) {
            val readCount = read(buffer, 0, buffer.size)
            if (readCount \< 0) break
            if (readCount == 0) continue

            channel.writeFully(buffer, 0, readCount)
        }
    } catch (cause: Throwable) {
        channel.close(cause)
    } finally {
        pool.recycle(buffer)
        close()
    }
}.channel

Auth: BasicAuthProvider caches credentials until process death

Support WebSockets in Curl engine

Darwin: Ambiguous DarwinHttpRequestException for SSL Pinning failure

"AbortError: BodyStreamBuffer was aborted" error when canceling parent job

Java, Js, Darwin: Response header Sec-WebSocket-Protocol is missing

Auth: Make re-auth/refresh status codes configurable

Logging: HTTP method is logged with the class name

HttpRequestRetry: race condition for isClosedForRead leads to EOFException: Channel is already closed

https://youtrack.jetbrains.com/issue/KTOR-8051

In rare cases, this line of code can throw a Channel already closed exception. I could only reproduce this regression on 3.0.2 or newer

java.io.EOFException: Channel is already closed
	at io.ktor.utils.io.ByteReadChannelOperationsKt.readFully(ByteReadChannelOperations.kt:464)
	at io.ktor.utils.io.ByteReadChannelOperationsKt.readFully$default(ByteReadChannelOperations.kt:462)
	at io.ktor.client.statement.ReadersKt.readBytes(Readers.kt:15)
	at io.ktor.client.plugins.HttpRequestRetryKt$HttpRequestRetry$2$1.invokeSuspend(HttpRequestRetry.kt:300)
	at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:33)
	at kotlinx.coroutines.DispatchedTaskKt.resume(DispatchedTask.kt:165)
	at kotlinx.coroutines.DispatchedTaskKt.dispatch(DispatchedTask.kt:154)
	at kotlinx.coroutines.CancellableContinuationImpl.dispatchResume(CancellableContinuationImpl.kt:470)
	at kotlinx.coroutines.CancellableContinuationImpl.resumeImpl$kotlinx_coroutines_core(CancellableContinuationImpl.kt:504)
	at kotlinx.coroutines.CancellableContinuationImpl.resumeImpl$kotlinx_coroutines_core$default(CancellableContinuationImpl.kt:493)
	at kotlinx.coroutines.CancellableContinuationImpl.resumeWith(CancellableContinuationImpl.kt:359)
	at kotlinx.coroutines.ResumeAwaitOnCompletion.invoke(JobSupport.kt:1557)
	at kotlinx.coroutines.JobSupport.notifyCompletion(JobSupport.kt:1625)
	at kotlinx.coroutines.JobSupport.completeStateFinalization(JobSupport.kt:316)
	at kotlinx.coroutines.JobSupport.finalizeFinishingState(JobSupport.kt:233)
	at kotlinx.coroutines.JobSupport.tryMakeCompletingSlowPath(JobSupport.kt:946)
	at kotlinx.coroutines.JobSupport.tryMakeCompleting(JobSupport.kt:894)
	at kotlinx.coroutines.JobSupport.makeCompletingOnce$kotlinx_coroutines_core(JobSupport.kt:859)
	at kotlinx.coroutines.AbstractCoroutine.resumeWith(AbstractCoroutine.kt:98)
	at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:46)
	at kotlinx.coroutines.DispatchedTask.run(DispatchedTask.kt:101)
	at kotlinx.coroutines.internal.LimitedDispatcher$Worker.run(LimitedDispatcher.kt:113)
	at kotlinx.coroutines.scheduling.TaskImpl.run(Tasks.kt:89)
	at kotlinx.coroutines.scheduling.CoroutineScheduler.runSafely(CoroutineScheduler.kt:589)
	at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.executeTask(CoroutineScheduler.kt:823)
	at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.runWorker(CoroutineScheduler.kt:720)
	at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.run(CoroutineScheduler.kt:707)

The raw stream and the response stream are not in agreement on if the steam is actually closed:
Snapsot taken from line ByteReadChannelOperations.kt:464
{width=70%}

Snapshot from same breakpoint but taken at a higher stack frame at line HttpRequestRetry.kt:304
{width=70%}

You can reproduce it by running this minimal reproduction example:

KtorFlakyRetry.zip

Curl: Error linking curl in linkDebugExecutableLinuxX64 on macOS

https://youtrack.jetbrains.com/issue/KTOR-6361

I have a KMP project using the curl ktor client and when compiling the project on linux, everything links ok.

When compiling the project on macOS, though, I get the folllowing log:

> Task :gameplay-app:linkDebugExecutableLinuxX64 FAILED
e: /Users/brunojcm/.konan/dependencies/apple-llvm-20200714-macos-x64-essentials/bin/ld.lld invocation reported errors

The /Users/brunojcm/.konan/dependencies/apple-llvm-20200714-macos-x64-essentials/bin/ld.lld command returned non-zero exit code: 1.
output:
ld.lld: error: undefined symbol: curl_global_init
>>> referenced by out
>>>               /private/var/folders/0b/kkc0ysjs6m357dwbhcj2n_n00000gn/T/konan_temp17233035539013036940/result.o:(libcurl_curl_global_init_wrapper23)
>>> did you mean: _curl_global_init
>>> defined in: /usr/local/opt/curl/lib/libcurl.a

ld.lld: error: undefined symbol: curl_slist_append
>>> referenced by out
>>>               /private/var/folders/0b/kkc0ysjs6m357dwbhcj2n_n00000gn/T/konan_temp17233035539013036940/result.o:(libcurl_curl_slist_append_wrapper27)
>>> did you mean: _curl_slist_append
>>> defined in: /usr/local/opt/curl/lib/libcurl.a

ld.lld: error: undefined symbol: curl_slist_free_all
>>> referenced by out
>>>               /private/var/folders/0b/kkc0ysjs6m357dwbhcj2n_n00000gn/T/konan_temp17233035539013036940/result.o:(libcurl_curl_slist_free_all_wrapper28)

ld.lld: error: undefined symbol: curl_easy_strerror
>>> referenced by out
>>>               /private/var/folders/0b/kkc0ysjs6m357dwbhcj2n_n00000gn/T/konan_temp17233035539013036940/result.o:(libcurl_curl_easy_strerror_wrapper33)

ld.lld: error: undefined symbol: curl_easy_pause
>>> referenced by out
>>>               /private/var/folders/0b/kkc0ysjs6m357dwbhcj2n_n00000gn/T/konan_temp17233035539013036940/result.o:(libcurl_curl_easy_pause_wrapper35)

ld.lld: error: undefined symbol: curl_easy_init
>>> referenced by out
>>>               /private/var/folders/0b/kkc0ysjs6m357dwbhcj2n_n00000gn/T/konan_temp17233035539013036940/result.o:(libcurl_curl_easy_init_wrapper36)

ld.lld: error: undefined symbol: curl_easy_cleanup
>>> referenced by out
>>>               /private/var/folders/0b/kkc0ysjs6m357dwbhcj2n_n00000gn/T/konan_temp17233035539013036940/result.o:(libcurl_curl_easy_cleanup_wrapper38)

ld.lld: error: undefined symbol: curl_multi_init
>>> referenced by out
>>>               /private/var/folders/0b/kkc0ysjs6m357dwbhcj2n_n00000gn/T/konan_temp17233035539013036940/result.o:(libcurl_curl_multi_init_wrapper44)

ld.lld: error: undefined symbol: curl_multi_add_handle
>>> referenced by out
>>>               /private/var/folders/0b/kkc0ysjs6m357dwbhcj2n_n00000gn/T/konan_temp17233035539013036940/result.o:(libcurl_curl_multi_add_handle_wrapper45)

ld.lld: error: undefined symbol: curl_multi_remove_handle
>>> referenced by out
>>>               /private/var/folders/0b/kkc0ysjs6m357dwbhcj2n_n00000gn/T/konan_temp17233035539013036940/result.o:(libcurl_curl_multi_remove_handle_wrapper46)

ld.lld: error: undefined symbol: curl_multi_poll
>>> referenced by out
>>>               /private/var/folders/0b/kkc0ysjs6m357dwbhcj2n_n00000gn/T/konan_temp17233035539013036940/result.o:(libcurl_curl_multi_poll_wrapper49)

ld.lld: error: undefined symbol: curl_multi_wakeup
>>> referenced by out
>>>               /private/var/folders/0b/kkc0ysjs6m357dwbhcj2n_n00000gn/T/konan_temp17233035539013036940/result.o:(libcurl_curl_multi_wakeup_wrapper50)

ld.lld: error: undefined symbol: curl_multi_perform
>>> referenced by out
>>>               /private/var/folders/0b/kkc0ysjs6m357dwbhcj2n_n00000gn/T/konan_temp17233035539013036940/result.o:(libcurl_curl_multi_perform_wrapper51)

ld.lld: error: undefined symbol: curl_multi_cleanup
>>> referenced by out
>>>               /private/var/folders/0b/kkc0ysjs6m357dwbhcj2n_n00000gn/T/konan_temp17233035539013036940/result.o:(libcurl_curl_multi_cleanup_wrapper52)

ld.lld: error: undefined symbol: curl_multi_info_read
>>> referenced by out
>>>               /private/var/folders/0b/kkc0ysjs6m357dwbhcj2n_n00000gn/T/konan_temp17233035539013036940/result.o:(libcurl_curl_multi_info_read_wrapper53)

ld.lld: error: undefined symbol: curl_multi_strerror
>>> referenced by out
>>>               /private/var/folders/0b/kkc0ysjs6m357dwbhcj2n_n00000gn/T/konan_temp17233035539013036940/result.o:(libcurl_curl_multi_strerror_wrapper54)

ld.lld: error: undefined symbol: curl_easy_setopt
>>> referenced by out
>>>               /private/var/folders/0b/kkc0ysjs6m357dwbhcj2n_n00000gn/T/konan_temp17233035539013036940/result.o:(knifunptr_libcurl39_curl_easy_setopt)

ld.lld: error: undefined symbol: curl_easy_getinfo
>>> referenced by out
>>>               /private/var/folders/0b/kkc0ysjs6m357dwbhcj2n_n00000gn/T/konan_temp17233035539013036940/result.o:(knifunptr_libcurl42_curl_easy_getinfo)

FAILURE: Build failed with an exception.

* What went wrong:
Execution failed for task ':gameplay-app:linkDebugExecutableLinuxX64'.
> Compilation finished with errors

Is this an issue with Ktor?

Apache5 client: Upgrade HttpClient to 5.4

Access to the configuration options of a HttpClient plugin to tweak or wrap them with additional logic

https://youtrack.jetbrains.com/issue/KTOR-7213

The use case is passing a preconfigured HttpClient instance to the SDK client factory to construct an SDK client instance for some server (for instance, Space SDK client for accessing Space HTTP API). In Space SDK, we use Ktor HttpClient to make requests to Space HTTP API and allow SDK users to customize the underlying HttpClient according to their needs:

public fun ktorClientForSpace(block: HttpClientConfig<*>.() -> Unit = {}): HttpClient = HttpClient {
    block()
    configureKtorClientForSpace()
}

This seems to be a good idea, because the SDK client is basically just a wrapper around HttpClient and we’d better rely on Ktor possibilities for different customization options rather than hide them completely behind SDK API.

At the same time, SDK handles request authentication and specifically, refreshing access tokens when they expire and retrying the request with the newly obtained token. This refresh & retry does not rely on the HttpRequestRetry plugin. But it turns out this token refresh doesn’t work well with the HttpRequestRetry plugin configured by the SDK user. When the retry intervals are long enough, the access token expires earlier than the retry count is exceeded. SDK then performs access token refresh and retries the request transparently, but the request with the new token is seen by the HttpRequestRetry plugin as a completely new one, so the retry count is reset. The result is an indefinite loop of retries with intermittent access token updates.

To fix this issue, I considered using HttpRequestRetry plugin for token refresh as well. But the problem is, in SDK code I cannot neither tweak nor access the plugin configuration specified by the caller, because HttpRequestRetry.Configuration is write-only due to visibility restrictions of its members:

public fun HttpClientConfig<*>.configureKtorClientForSpace(configureRetry: HttpRequestRetry.Configuration.() -> Unit = {}) {
    val retryWithCustomConfig = HttpRequestRetry.prepare(configureRetry)
    install(HttpRequestRetry) {
        retryIf {
            // DOESN'T WORK - no access to retryWithCustomConfig.retryIf 
            // retryWithCustomConfig.retryIf || authentication token has expired
        }
        modifyRequest {
            // DOESN'T WORK - no access to retryWithCustomConfig.modifyRequest 
            // if authentication token has expired, refresh the token first
            // then apply retryWithCustomConfig.modifyRequest
        }
    }
}

The only idea I currently have is to re-implement some basic subset of HttpRequestRetry in Space SDK code itself and strongly advise against using the HttpRequestRetry plugin for the underlying HttpClient. But this is not a good solution because of the two reasons. We'll need to mirror part of the HttpRequestRetry plugin API in the Space SDK client API. We also have no means to ensure that the caller doesn't still install the HttpRequestRetry plugin into the client — we can only put a warning into the Javadoc or SDK documentation, but we cannot do anything at compile or run time to prevent that.

Logging: Format log like OkHttp client does

Support static linking for curl on all platforms

Support ARM target in Ktor client with Kotlin/Native and Curl

Windows: undefined symbols in linker when ktor-client-curl is used

https://youtrack.jetbrains.com/issue/KTOR-4307

I've just added ktor-client-curl version 2.0.1 to my Windows Kotlin/Native target.
However, it fails to compile on GitHub Actions, with errors like these:

> Task :client:linkDebugTestMingwX64
w: duplicate library name: org.jetbrains.kotlinx:atomicfu
w: duplicate library name: org.jetbrains.kotlinx:atomicfu-cinterop-interop
e: C:\Users\runneradmin\.konan\dependencies\llvm-11.1.0-windows-x64-essentials/bin/clang++ invocation reported errors
The C:\Users\runneradmin\.konan\dependencies\llvm-11.1.0-windows-x64-essentials/bin/clang++ command returned non-zero exit code: 1.
output:
lld-link: error: undefined symbol: __mingw_init_ehandler
>>> referenced by E:/mingwbuild/mingw-w64-crt-git/src/mingw-w64/mingw-w64-crt/crt\crtexe.c:288
>>>               C:\Users\runneradmin\.konan\dependencies\msys2-mingw-w64-x86_64-1\x86_64-w64-mingw32\lib\crt2.o:(__tmainCRTStartup)
lld-link: error: undefined symbol: __security_init_cookie
>>> referenced by E:/mingwbuild/mingw-w64-crt-git/src/mingw-w64/mingw-w64-crt/crt\crtexe.c:194
>>>               C:\Users\runneradmin\.konan\dependencies\msys2-mingw-w64-x86_64-1\x86_64-w64-mingw32\lib\crt2.o:(.l_startw)
>>> referenced by E:/mingwbuild/mingw-w64-crt-git/src/mingw-w64/mingw-w64-crt/crt\crtexe.c:222
>>>               C:\Users\runneradmin\.konan\dependencies\msys2-mingw-w64-x86_64-1\x86_64-w64-mingw32\lib\crt2.o:(.l_start)
lld-link: error: undefined symbol: mingw_app_type
>>> referenced by C:\Users\runneradmin\.konan\dependencies\msys2-mingw-w64-x86_64-1\x86_64-w64-mingw32\lib\crt2.o:(.refptr.mingw_app_type)
lld-link: error: undefined symbol: mingw_initcharmax
>>> referenced by C:\Users\runneradmin\.konan\dependencies\msys2-mingw-w64-x86_64-1\x86_64-w64-mingw32\lib\crt2.o:(.refptr.mingw_initcharmax)
lld-link: error: undefined symbol: mingw_initltssuo_force
>>> referenced by C:\Users\runneradmin\.konan\dependencies\msys2-mingw-w64-x86_64-1\x86_64-w64-mingw32\lib\crt2.o:(.refptr.mingw_initltssuo_force)
lld-link: error: undefined symbol: mingw_initltsdyn_force
>>> referenced by C:\Users\runneradmin\.konan\dependencies\msys2-mingw-w64-x86_64-1\x86_64-w64-mingw32\lib\crt2.o:(.refptr.mingw_initltsdyn_force)
lld-link: error: undefined symbol: mingw_initltsdrot_force
>>> referenced by C:\Users\runneradmin\.konan\dependencies\msys2-mingw-w64-x86_64-1\x86_64-w64-mingw32\lib\crt2.o:(.refptr.mingw_initltsdrot_force)
clang++: error: linker command failed with exit code 1 (use -v to see invocation)
> Task :client:linkDebugTestMingwX64 FAILED

I've confirmed that libcurl is installed (eg. https://github.com/batect/docker-client/runs/6404281016?check_suite_focus=true#step:12:24).

SaveBodyPlugin: UninitializedPropertyAccessException when reading response body within receivePipeline

https://youtrack.jetbrains.com/issue/KTOR-7952

Plugin:

import io.ktor.client.HttpClient
import io.ktor.client.engine.cio.CIO
import io.ktor.client.plugins.HttpClientPlugin
import io.ktor.client.plugins.contentnegotiation.ContentNegotiation
import io.ktor.client.request.HttpRequestPipeline
import io.ktor.client.request.get
import io.ktor.client.statement.HttpReceivePipeline
import io.ktor.client.statement.HttpResponse
import io.ktor.client.statement.bodyAsText
import io.ktor.serialization.kotlinx.json.json
import io.ktor.util.AttributeKey
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.launch
import kotlinx.serialization.json.Json
import org.slf4j.LoggerFactory

suspend fun main() = coroutineScope {
    val logger = LoggerFactory.getLogger("repro")
    val client = HttpClient(CIO) {
        install(MyRequestLogger) {
            saveFunction = {
                launch(Dispatchers.Default) {
                    logger.info("Received response in plugin: {}", it.bodyAsText())
                }
            }
        }
        install(ContentNegotiation) {
            json(json = Json {
                ignoreUnknownKeys = true
            })
        }
    }

    while (true) {
        client.get("https://jsonplaceholder.typicode.com/todos/1")
    }
}

class MyRequestLogger(
    private val config: Config,
) {
    class Config {
        var saveFunction: suspend (HttpResponse) -> Unit = {}
    }

    companion object Plugin : HttpClientPlugin<Config, MyRequestLogger> {
        override val key = AttributeKey<MyRequestLogger>("HttpRequestLogger")

        override fun prepare(block: Config.() -> Unit): MyRequestLogger {
            val config = Config().apply(block)
            return MyRequestLogger(config)
        }

        override fun install(plugin: MyRequestLogger, scope: HttpClient) {
            scope.requestPipeline.intercept(HttpRequestPipeline.Before) {
                proceed()
            }

            scope.receivePipeline.intercept(HttpReceivePipeline.After) { response ->
                plugin.config.saveFunction(response)
                proceedWith(response)
            }
        }
    }
}

Exception:

Exception in thread "main" java.io.IOException: lateinit property writerJob has not been initialized
	at io.ktor.utils.io.CloseToken.getCause(CloseToken.kt:37)
	at io.ktor.utils.io.ByteChannel.cancel(ByteChannel.kt:136)
	at io.ktor.utils.io.ByteWriteChannelOperationsKt$writer$job$1.invokeSuspend(ByteWriteChannelOperations.kt:150)
	at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith$$$capture(ContinuationImpl.kt:33)
	at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt)
	at --- Async.Stack.Trace --- (captured by IntelliJ IDEA debugger)
	at kotlinx.coroutines.debug.internal.DebugProbesImpl$CoroutineOwner.<init>(DebugProbesImpl.kt:531)
	at kotlinx.coroutines.debug.internal.DebugProbesImpl.createOwner(DebugProbesImpl.kt:510)
	at kotlinx.coroutines.debug.internal.DebugProbesImpl.probeCoroutineCreated$kotlinx_coroutines_core(DebugProbesImpl.kt:497)
	at kotlin.coroutines.jvm.internal.DebugProbesKt.probeCoroutineCreated(DebugProbes.kt:7)
	at kotlin.coroutines.intrinsics.IntrinsicsKt__IntrinsicsJvmKt.createCoroutineUnintercepted(IntrinsicsJvm.kt:161)
	at kotlinx.coroutines.intrinsics.CancellableKt.startCoroutineCancellable(Cancellable.kt:26)
	at kotlinx.coroutines.CoroutineStart.invoke(CoroutineStart.kt:358)
	at kotlinx.coroutines.AbstractCoroutine.start(AbstractCoroutine.kt:124)
	at kotlinx.coroutines.BuildersKt__Builders_commonKt.launch(Builders.common.kt:52)
	at kotlinx.coroutines.BuildersKt.launch(Unknown Source)
	at kotlinx.coroutines.BuildersKt__Builders_commonKt.launch$default(Builders.common.kt:43)
	at kotlinx.coroutines.BuildersKt.launch$default(Unknown Source)
	at io.ktor.utils.io.ByteWriteChannelOperationsKt.writer(ByteWriteChannelOperations.kt:139)
	at io.ktor.utils.io.ByteWriteChannelOperationsKt.writer(ByteWriteChannelOperations.kt:131)
	at io.ktor.utils.io.ByteWriteChannelOperationsKt.writer$default(ByteWriteChannelOperations.kt:126)
	at io.ktor.client.plugins.internal.ByteChannelReplay.replay(ByteChannelReplay.kt:32)
	at io.ktor.client.plugins.DoubleReceivePluginKt$SaveBodyPlugin$2$1.invokeSuspend$lambda$0(DoubleReceivePlugin.kt:70)
	at io.ktor.client.plugins.observer.DelegatedResponse.getRawContent(DelegatedCall.kt:78)
	at io.ktor.client.call.HttpClientCall.getResponseContent$suspendImpl(HttpClientCall.kt:67)
	at io.ktor.client.call.HttpClientCall.getResponseContent(HttpClientCall.kt)
	at io.ktor.client.call.HttpClientCall.bodyNullable(HttpClientCall.kt:84)
	at io.ktor.client.statement.HttpResponseKt.bodyAsText(HttpResponse.kt:123)
	at io.ktor.client.statement.HttpResponseKt.bodyAsText$default(HttpResponse.kt:102)
	at io.heapy.kotbot.KotbotReproKt$main$2$client$1$1$1$1.invokeSuspend(KotbotRepro.kt:26)
	at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith$$$capture(ContinuationImpl.kt:33)
	at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt)
	at --- Async.Stack.Trace --- (captured by IntelliJ IDEA debugger)
	at kotlinx.coroutines.debug.internal.DebugProbesImpl$CoroutineOwner.<init>(DebugProbesImpl.kt:531)
	at kotlinx.coroutines.debug.internal.DebugProbesImpl.createOwner(DebugProbesImpl.kt:510)
	at kotlinx.coroutines.debug.internal.DebugProbesImpl.probeCoroutineCreated$kotlinx_coroutines_core(DebugProbesImpl.kt:497)
	at kotlin.coroutines.jvm.internal.DebugProbesKt.probeCoroutineCreated(DebugProbes.kt:7)
	at kotlin.coroutines.intrinsics.IntrinsicsKt__IntrinsicsJvmKt.createCoroutineUnintercepted(IntrinsicsJvm.kt:161)
	at kotlinx.coroutines.intrinsics.CancellableKt.startCoroutineCancellable(Cancellable.kt:26)
	at kotlinx.coroutines.CoroutineStart.invoke(CoroutineStart.kt:358)
	at kotlinx.coroutines.AbstractCoroutine.start(AbstractCoroutine.kt:124)
	at kotlinx.coroutines.BuildersKt__Builders_commonKt.launch(Builders.common.kt:52)
	at kotlinx.coroutines.BuildersKt.launch(Unknown Source)
	at kotlinx.coroutines.BuildersKt__Builders_commonKt.launch$default(Builders.common.kt:43)
	at kotlinx.coroutines.BuildersKt.launch$default(Unknown Source)
	at io.heapy.kotbot.KotbotReproKt$main$2$client$1$1$1.invokeSuspend(KotbotRepro.kt:25)
	at io.heapy.kotbot.KotbotReproKt$main$2$client$1$1$1.invoke(KotbotRepro.kt)
	at io.heapy.kotbot.KotbotReproKt$main$2$client$1$1$1.invoke(KotbotRepro.kt)
	at io.heapy.kotbot.MyRequestLogger$Plugin$install$2.invokeSuspend(KotbotRepro.kt:63)
	at io.heapy.kotbot.MyRequestLogger$Plugin$install$2.invoke(KotbotRepro.kt)
	at io.heapy.kotbot.MyRequestLogger$Plugin$install$2.invoke(KotbotRepro.kt)
	at io.ktor.util.pipeline.DebugPipelineContext.proceedLoop(DebugPipelineContext.kt:79)
	at io.ktor.util.pipeline.DebugPipelineContext.proceed(DebugPipelineContext.kt:57)
	at io.ktor.util.pipeline.DebugPipelineContext.proceedWith(DebugPipelineContext.kt:42)
	at io.ktor.client.plugins.DoubleReceivePluginKt$SaveBodyPlugin$2$1.invokeSuspend(DoubleReceivePlugin.kt:72)
	at io.ktor.client.plugins.DoubleReceivePluginKt$SaveBodyPlugin$2$1.invoke(DoubleReceivePlugin.kt)
	at io.ktor.client.plugins.DoubleReceivePluginKt$SaveBodyPlugin$2$1.invoke(DoubleReceivePlugin.kt)
	at io.ktor.util.pipeline.DebugPipelineContext.proceedLoop(DebugPipelineContext.kt:79)
	at io.ktor.util.pipeline.DebugPipelineContext.proceed(DebugPipelineContext.kt:57)
	at io.ktor.util.pipeline.DebugPipelineContext.execute$ktor_utils(DebugPipelineContext.kt:63)
	at io.ktor.util.pipeline.Pipeline.execute(Pipeline.kt:86)
	at io.ktor.client.HttpClient$2.invokeSuspend(HttpClient.kt:1345)
	at io.ktor.client.HttpClient$2.invoke(HttpClient.kt)
	at io.ktor.client.HttpClient$2.invoke(HttpClient.kt)
	at io.ktor.util.pipeline.DebugPipelineContext.proceedLoop(DebugPipelineContext.kt:79)
	at io.ktor.util.pipeline.DebugPipelineContext.proceed(DebugPipelineContext.kt:57)
	at io.ktor.util.pipeline.DebugPipelineContext.proceedWith(DebugPipelineContext.kt:42)
	at io.ktor.client.engine.HttpClientEngine$install$1.invokeSuspend(HttpClientEngine.kt:82)
	at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith$$$capture(ContinuationImpl.kt:33)
	at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt)
	at kotlinx.coroutines.DispatchedTaskKt.resume(DispatchedTask.kt:165)
	at kotlinx.coroutines.DispatchedTaskKt.dispatch(DispatchedTask.kt:154)
	at kotlinx.coroutines.CancellableContinuationImpl.dispatchResume(CancellableContinuationImpl.kt:470)
	at kotlinx.coroutines.CancellableContinuationImpl.resumeImpl$kotlinx_coroutines_core(CancellableContinuationImpl.kt:504)
	at kotlinx.coroutines.CancellableContinuationImpl.resumeImpl$kotlinx_coroutines_core$default(CancellableContinuationImpl.kt:493)
	at kotlinx.coroutines.CancellableContinuationImpl.resumeWith(CancellableContinuationImpl.kt:359)
	at kotlinx.coroutines.ResumeAwaitOnCompletion.invoke(JobSupport.kt:1557)
	at kotlinx.coroutines.JobSupport.notifyCompletion(JobSupport.kt:1625)
	at kotlinx.coroutines.JobSupport.completeStateFinalization(JobSupport.kt:316)
	at kotlinx.coroutines.JobSupport.finalizeFinishingState(JobSupport.kt:233)
	at kotlinx.coroutines.JobSupport.tryMakeCompletingSlowPath(JobSupport.kt:946)
	at kotlinx.coroutines.JobSupport.tryMakeCompleting(JobSupport.kt:894)
	at kotlinx.coroutines.JobSupport.makeCompletingOnce$kotlinx_coroutines_core(JobSupport.kt:859)
	at kotlinx.coroutines.AbstractCoroutine.resumeWith(AbstractCoroutine.kt:98)
	at kotlinx.coroutines.debug.internal.DebugProbesImpl$CoroutineOwner.resumeWith(DebugProbesImpl.kt:545)
	at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith$$$capture(ContinuationImpl.kt:46)
	at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt)
	at kotlinx.coroutines.UndispatchedCoroutine.afterResume(CoroutineContext.kt:266)
	at kotlinx.coroutines.AbstractCoroutine.resumeWith(AbstractCoroutine.kt:100)
	at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith$$$capture(ContinuationImpl.kt:46)
	at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt)
	at kotlinx.coroutines.DispatchedTask.run(DispatchedTask.kt:101)
	at kotlinx.coroutines.internal.LimitedDispatcher$Worker.run(LimitedDispatcher.kt:113)
	at kotlinx.coroutines.scheduling.TaskImpl.run(Tasks.kt:89)
	at kotlinx.coroutines.scheduling.CoroutineScheduler.runSafely(CoroutineScheduler.kt:589)
	at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.executeTask(CoroutineScheduler.kt:823)
	at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.runWorker(CoroutineScheduler.kt:720)
	at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.run(CoroutineScheduler.kt:707)
Caused by: java.io.IOException: lateinit property writerJob has not been initialized
	at io.ktor.utils.io.CloseToken.<init>(CloseToken.kt:27)
	at io.ktor.utils.io.ByteChannel.cancel(ByteChannel.kt:134)
	... 96 more
Caused by: kotlin.UninitializedPropertyAccessException: lateinit property writerJob has not been initialized
	at io.ktor.client.plugins.internal.ByteChannelReplay$CopyFromSourceTask.getWriterJob(ByteChannelReplay.kt:47)
	at io.ktor.client.plugins.internal.ByteChannelReplay$CopyFromSourceTask.awaitImpatiently(ByteChannelReplay.kt:84)
	at io.ktor.client.plugins.internal.ByteChannelReplay$replay$1.invokeSuspend(ByteChannelReplay.kt:33)
	at io.ktor.client.plugins.internal.ByteChannelReplay$replay$1.invoke(ByteChannelReplay.kt)
	at io.ktor.client.plugins.internal.ByteChannelReplay$replay$1.invoke(ByteChannelReplay.kt)
	at io.ktor.utils.io.ByteWriteChannelOperationsKt$writer$job$1.invokeSuspend(ByteWriteChannelOperations.kt:142)
	... 95 more

ContentNegotiation client plugin: no way to opt out of Accept on a per-request basis

https://youtrack.jetbrains.com/issue/KTOR-7722

The Ktor ContentNegotatiation client plugin always adds the accept header for all registered content types.

This makes it difficult to opt out on a per-request basis. For example, lets say a client has json installed. The server has an endpoint which can return either application/pdf and application/json, depending on the Accept header. We want to use the client to obtain application/pdf for one particular request.

Out of the box this appears to be impossible to control deterministically because application/json is always added to the Accept header by the ContentNegotiation plugin, with no q parameter, which means implicitly q=1.0. Therefore specifying accept(ContentType.Application.Pdf) in the RequestBuilder ends up in an accept header like this:

Accept: application/json,application/pdf

and now the server is free to send a JSON response.

In addition, it is not possible to override the automatically added value with accept(ContentType.Application.Json).withParameter("q", "0.8") either, because the logic checks for an exact match, and so ends up sending:

Accept: application/json,application/json; q=0.8,application/pdf

Possible Solutions

I would recommend that the ContentNegotiation plugin add the registered content types with a lower q setting e.g. 0.8 by default (though this could be made configurable). That way if the user does not specify anything for accept explicitly, the value will apply, however if the user does specify the accept explicitly, that value will be preferred by the server over the automatically set accept value.

Alternatively, a request attribute could be used to opt out of the automatically added Accept header for particular requests, perhaps by exposing a new RequestBuilder extension called acceptOnly , which if specified by the user would add the user's Accept specification, and opt out of the registered specifications.

In addition to the above, the gate for adding the registered content type should match the content type ignoring parameters, therefore if the user explicitly adds the registered content type with a lower q value or other parameters, that change is not ignored by the ContentNegotiation plugin.

Workaround

Perhaps there is an easier workaround, but I ended up creating a plugin to modify the request headers:

import io.ktor.client.plugins.api.*
import io.ktor.client.request.*
import io.ktor.util.*

val RequestModOperationKey = AttributeKey<(HttpRequestBuilder, Any) -> Unit>("request-mod-operation-key")

val RequestModPlugin = createClientPlugin("RequestModPlugin") {
  on(SendingRequest) { request, content ->
    request.attributes.getOrNull(RequestModOperationKey)?.invoke(request, content)
  }
}

fun HttpRequestBuilder.modRequest(block: (HttpRequestBuilder, Any) -> Unit) {
  setAttributes {
    put(RequestModOperationKey, block)
  }
}

and used like this:

httpClient.request {
    // workaround ContentNegotiation plugin always inserting the Accept application/json header
    // at implicit q=1.0 -- so the server returns JSON instead of PDF
    modRequest { request, _ ->
      request.headers.remove(HttpHeaders.Accept)
      request.accept(ContentType.Application.Pdf)
    }
}

Support receiving multipart data with Ktor client

Client CIO engine support for wasm-js and js

Core

Write readable name of the application to the logs

Add operator contains to ContentType objects

IOException: Fail to select descriptor for ACCEPT

Fail to parse url: file:/path/to/file.txt

Add media type for Yaml

Support conversion between byte channel interfaces and kotlinx-io primitives

Uncaught cannot write to a channel errors from ws-pinger

https://youtrack.jetbrains.com/issue/KTOR-8008

We migrated a server (websockets) app from Ktor 2 to Ktor 3, and since then we've been getting exceptions like the following logged rather frequently:

java.io.IOException: Cannot write to a channel
	at io.ktor.utils.io.CloseToken.getCause(CloseToken.kt:37)
	at io.ktor.utils.io.ByteChannel.cancel(ByteChannel.kt:135)
	at io.ktor.server.netty.cio.NettyHttpResponsePipeline.respondWithFailure(NettyHttpResponsePipeline.kt:106)
	at io.ktor.server.netty.cio.NettyHttpResponsePipeline.respondWithBodyAndTrailerMessage(NettyHttpResponsePipeline.kt:253)
	at io.ktor.server.netty.cio.NettyHttpResponsePipeline.access$respondWithBodyAndTrailerMessage(NettyHttpResponsePipeline.kt:24)
	at io.ktor.server.netty.cio.NettyHttpResponsePipeline$respondWithBodyAndTrailerMessage$1.invokeSuspend(NettyHttpResponsePipeline.kt)
	at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:33)
	at kotlinx.coroutines.DispatchedTask.run(DispatchedTask.kt:99)
	at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173)
	at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166)
	at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	at java.base/java.lang.Thread.run(Thread.java:840)
	Suppressed: kotlinx.coroutines.internal.DiagnosticCoroutineContextException: [CoroutineName(ws-pinger), StandaloneCoroutine{Cancelling}@3b00c365, io.ktor.server.netty.EventLoopGroupProxy@7d87fde9]
Caused by: java.io.IOException: Cannot write to a channel
	at io.ktor.utils.io.CloseToken.<init>(CloseToken.kt:27)
	at io.ktor.utils.io.ByteChannel.cancel(ByteChannel.kt:133)
	... 14 more
Caused by: io.ktor.util.cio.ChannelWriteException: Cannot write to a channel
	at io.ktor.server.netty.cio.NettyHttpResponsePipeline.respondWithFailure(NettyHttpResponsePipeline.kt:100)
	... 13 more
Caused by: io.netty.channel.StacklessClosedChannelException
	at io.netty.channel.AbstractChannel.close(ChannelPromise)(Unknown Source)

This is skipping all of our existing error handlers, and getting logged in an uncaught exception handler registered with the threads. The exception mentions ws-pinger, so it seems like this is an issue with the Ktor websocket pinger. Is this a bug? Is there someway to catch and gracefully handle this error in the application code?

ktor-server-core: Test files are part of the distribution code

ByteWriteChannel is missing writeFloat()/readFloat()

UrlBuilder: Support telephone scheme

UDPSocketBuilder missing `bind` overload with `hostName` and `port`

Make Url class @Serializable and JVM Serializable

Support NodeJs target for ktor-network

Improve parsing of supported media types (MIME types)

Migrate to kotlin.AutoCloseable

Docs

Migrate Docker Compose to V2

cURL Engine: Update documentation on how to install libcurl for different Linux distributions

Curl: Document how to install libcurl on Ubuntu 22.04

https://youtrack.jetbrains.com/issue/KTOR-5894

Hi, I am trying to get a Kotlin native client running on Ubuntu 22.04 and am running into errors with the libcurl bindings.

I have confirmed that the code works as expected on MacOS, so I'm fairly confident that this is either an issue with the library, or with my local environment. If the latter, a more descriptive exception would be appreciated, because as far as I can tell, all the necessary dependencies have been installed on my end. Thanks!

Additionally, I tried disabling the cache as suggested, to no effect.

8:31:35 AM: Executing 'runDebugExecutableNative'...

> Task :compileKotlinNative UP-TO-DATE

> Task :linkDebugExecutableNative FAILED
2 actionable tasks: 1 executed, 1 up-to-date
e: /home/ryan/.konan/dependencies/x86_64-unknown-linux-gnu-gcc-8.3.0-glibc-2.19-kernel-4.9-2/x86_64-unknown-linux-gnu/bin/ld.gold invocation reported errors
Please try to disable compiler caches and rerun the build. To disable compiler caches, add the following line to the gradle.properties file in the project's root directory:
    
    kotlin.native.cacheKind.linuxX64=none
    
Also, consider filing an issue with full Gradle log here: https://kotl.in/issue
The /home/ryan/.konan/dependencies/x86_64-unknown-linux-gnu-gcc-8.3.0-glibc-2.19-kernel-4.9-2/x86_64-unknown-linux-gnu/bin/ld.gold command returned non-zero exit code: 1.
output:
/home/ryan/.konan/dependencies/x86_64-unknown-linux-gnu-gcc-8.3.0-glibc-2.19-kernel-4.9-2/x86_64-unknown-linux-gnu/bin/ld.gold: error: cannot find -lcurl
/home/ryan/.konan/kotlin-native-prebuilt-linux-x86_64-1.8.20/klib/cache/linux_x64-gSTATIC/io.ktor:ktor-client-curl-cinterop-libcurl/unspecified/69d1614c6d0a41ecbd731874ee49c865c1c8bac290758825bc4d66cfffd1d903/io.ktor:ktor-client-curl-cinterop-libcurl-cache/bin/libio.ktor:ktor-client-curl-cinterop-libcurl-cache.a(result.o):out:function libcurl_curl_global_init_wrapper23: error: undefined reference to 'curl_global_init'
/home/ryan/.konan/kotlin-native-prebuilt-linux-x86_64-1.8.20/klib/cache/linux_x64-gSTATIC/io.ktor:ktor-client-curl-cinterop-libcurl/unspecified/69d1614c6d0a41ecbd731874ee49c865c1c8bac290758825bc4d66cfffd1d903/io.ktor:ktor-client-curl-cinterop-libcurl-cache/bin/libio.ktor:ktor-client-curl-cinterop-libcurl-cache.a(result.o):out:function libcurl_curl_slist_append_wrapper27: error: undefined reference to 'curl_slist_append'
/home/ryan/.konan/kotlin-native-prebuilt-linux-x86_64-1.8.20/klib/cache/linux_x64-gSTATIC/io.ktor:ktor-client-curl-cinterop-libcurl/unspecified/69d1614c6d0a41ecbd731874ee49c865c1c8bac290758825bc4d66cfffd1d903/io.ktor:ktor-client-curl-cinterop-libcurl-cache/bin/libio.ktor:ktor-client-curl-cinterop-libcurl-cache.a(result.o):out:function libcurl_curl_slist_free_all_wrapper28: error: undefined reference to 'curl_slist_free_all'
/home/ryan/.konan/kotlin-native-prebuilt-linux-x86_64-1.8.20/klib/cache/linux_x64-gSTATIC/io.ktor:ktor-client-curl-cinterop-libcurl/unspecified/69d1614c6d0a41ecbd731874ee49c865c1c8bac290758825bc4d66cfffd1d903/io.ktor:ktor-client-curl-cinterop-libcurl-cache/bin/libio.ktor:ktor-client-curl-cinterop-libcurl-cache.a(result.o):out:function libcurl_curl_easy_strerror_wrapper33: error: undefined reference to 'curl_easy_strerror'
/home/ryan/.konan/kotlin-native-prebuilt-linux-x86_64-1.8.20/klib/cache/linux_x64-gSTATIC/io.ktor:ktor-client-curl-cinterop-libcurl/unspecified/69d1614c6d0a41ecbd731874ee49c865c1c8bac290758825bc4d66cfffd1d903/io.ktor:ktor-client-curl-cinterop-libcurl-cache/bin/libio.ktor:ktor-client-curl-cinterop-libcurl-cache.a(result.o):out:function libcurl_curl_easy_pause_wrapper35: error: undefined reference to 'curl_easy_pause'
/home/ryan/.konan/kotlin-native-prebuilt-linux-x86_64-1.8.20/klib/cache/linux_x64-gSTATIC/io.ktor:ktor-client-curl-cinterop-libcurl/unspecified/69d1614c6d0a41ecbd731874ee49c865c1c8bac290758825bc4d66cfffd1d903/io.ktor:ktor-client-curl-cinterop-libcurl-cache/bin/libio.ktor:ktor-client-curl-cinterop-libcurl-cache.a(result.o):out:function libcurl_curl_easy_init_wrapper36: error: undefined reference to 'curl_easy_init'
/home/ryan/.konan/kotlin-native-prebuilt-linux-x86_64-1.8.20/klib/cache/linux_x64-gSTATIC/io.ktor:ktor-client-curl-cinterop-libcurl/unspecified/69d1614c6d0a41ecbd731874ee49c865c1c8bac290758825bc4d66cfffd1d903/io.ktor:ktor-client-curl-cinterop-libcurl-cache/bin/libio.ktor:ktor-client-curl-cinterop-libcurl-cache.a(result.o):out:function libcurl_curl_easy_cleanup_wrapper38: error: undefined reference to 'curl_easy_cleanup'
/home/ryan/.konan/kotlin-native-prebuilt-linux-x86_64-1.8.20/klib/cache/linux_x64-gSTATIC/io.ktor:ktor-client-curl-cinterop-libcurl/unspecified/69d1614c6d0a41ecbd731874ee49c865c1c8bac290758825bc4d66cfffd1d903/io.ktor:ktor-client-curl-cinterop-libcurl-cache/bin/libio.ktor:ktor-client-curl-cinterop-libcurl-cache.a(result.o):out:function libcurl_curl_multi_init_wrapper44: error: undefined reference to 'curl_multi_init'
/home/ryan/.konan/kotlin-native-prebuilt-linux-x86_64-1.8.20/klib/cache/linux_x64-gSTATIC/io.ktor:ktor-client-curl-cinterop-libcurl/unspecified/69d1614c6d0a41ecbd731874ee49c865c1c8bac290758825bc4d66cfffd1d903/io.ktor:ktor-client-curl-cinterop-libcurl-cache/bin/libio.ktor:ktor-client-curl-cinterop-libcurl-cache.a(result.o):out:function libcurl_curl_multi_add_handle_wrapper45: error: undefined reference to 'curl_multi_add_handle'
/home/ryan/.konan/kotlin-native-prebuilt-linux-x86_64-1.8.20/klib/cache/linux_x64-gSTATIC/io.ktor:ktor-client-curl-cinterop-libcurl/unspecified/69d1614c6d0a41ecbd731874ee49c865c1c8bac290758825bc4d66cfffd1d903/io.ktor:ktor-client-curl-cinterop-libcurl-cache/bin/libio.ktor:ktor-client-curl-cinterop-libcurl-cache.a(result.o):out:function libcurl_curl_multi_remove_handle_wrapper46: error: undefined reference to 'curl_multi_remove_handle'
/home/ryan/.konan/kotlin-native-prebuilt-linux-x86_64-1.8.20/klib/cache/linux_x64-gSTATIC/io.ktor:ktor-client-curl-cinterop-libcurl/unspecified/69d1614c6d0a41ecbd731874ee49c865c1c8bac290758825bc4d66cfffd1d903/io.ktor:ktor-client-curl-cinterop-libcurl-cache/bin/libio.ktor:ktor-client-curl-cinterop-libcurl-cache.a(result.o):out:function libcurl_curl_multi_poll_wrapper49: error: undefined reference to 'curl_multi_poll'
/home/ryan/.konan/kotlin-native-prebuilt-linux-x86_64-1.8.20/klib/cache/linux_x64-gSTATIC/io.ktor:ktor-client-curl-cinterop-libcurl/unspecified/69d1614c6d0a41ecbd731874ee49c865c1c8bac290758825bc4d66cfffd1d903/io.ktor:ktor-client-curl-cinterop-libcurl-cache/bin/libio.ktor:ktor-client-curl-cinterop-libcurl-cache.a(result.o):out:function libcurl_curl_multi_wakeup_wrapper50: error: undefined reference to 'curl_multi_wakeup'
/home/ryan/.konan/kotlin-native-prebuilt-linux-x86_64-1.8.20/klib/cache/linux_x64-gSTATIC/io.ktor:ktor-client-curl-cinterop-libcurl/unspecified/69d1614c6d0a41ecbd731874ee49c865c1c8bac290758825bc4d66cfffd1d903/io.ktor:ktor-client-curl-cinterop-libcurl-cache/bin/libio.ktor:ktor-client-curl-cinterop-libcurl-cache.a(result.o):out:function libcurl_curl_multi_perform_wrapper51: error: undefined reference to 'curl_multi_perform'
/home/ryan/.konan/kotlin-native-prebuilt-linux-x86_64-1.8.20/klib/cache/linux_x64-gSTATIC/io.ktor:ktor-client-curl-cinterop-libcurl/unspecified/69d1614c6d0a41ecbd731874ee49c865c1c8bac290758825bc4d66cfffd1d903/io.ktor:ktor-client-curl-cinterop-libcurl-cache/bin/libio.ktor:ktor-client-curl-cinterop-libcurl-cache.a(result.o):out:function libcurl_curl_multi_cleanup_wrapper52: error: undefined reference to 'curl_multi_cleanup'
/home/ryan/.konan/kotlin-native-prebuilt-linux-x86_64-1.8.20/klib/cache/linux_x64-gSTATIC/io.ktor:ktor-client-curl-cinterop-libcurl/unspecified/69d1614c6d0a41ecbd731874ee49c865c1c8bac290758825bc4d66cfffd1d903/io.ktor:ktor-client-curl-cinterop-libcurl-cache/bin/libio.ktor:ktor-client-curl-cinterop-libcurl-cache.a(result.o):out:function libcurl_curl_multi_info_read_wrapper53: error: undefined reference to 'curl_multi_info_read'
/home/ryan/.konan/kotlin-native-prebuilt-linux-x86_64-1.8.20/klib/cache/linux_x64-gSTATIC/io.ktor:ktor-client-curl-cinterop-libcurl/unspecified/69d1614c6d0a41ecbd731874ee49c865c1c8bac290758825bc4d66cfffd1d903/io.ktor:ktor-client-curl-cinterop-libcurl-cache/bin/libio.ktor:ktor-client-curl-cinterop-libcurl-cache.a(result.o):out:function libcurl_curl_multi_strerror_wrapper54: error: undefined reference to 'curl_multi_strerror'
/home/ryan/.konan/kotlin-native-prebuilt-linux-x86_64-1.8.20/klib/cache/linux_x64-gSTATIC/io.ktor:ktor-client-curl-cinterop-libcurl/unspecified/69d1614c6d0a41ecbd731874ee49c865c1c8bac290758825bc4d66cfffd1d903/io.ktor:ktor-client-curl-cinterop-libcurl-cache/bin/libio.ktor:ktor-client-curl-cinterop-libcurl-cache.a(result.o):out:knifunptr_libcurl39_curl_easy_setopt: error: undefined reference to 'curl_easy_setopt'
/home/ryan/.konan/kotlin-native-prebuilt-linux-x86_64-1.8.20/klib/cache/linux_x64-gSTATIC/io.ktor:ktor-client-curl-cinterop-libcurl/unspecified/69d1614c6d0a41ecbd731874ee49c865c1c8bac290758825bc4d66cfffd1d903/io.ktor:ktor-client-curl-cinterop-libcurl-cache/bin/libio.ktor:ktor-client-curl-cinterop-libcurl-cache.a(result.o):out:knifunptr_libcurl42_curl_easy_getinfo: error: undefined reference to 'curl_easy_getinfo'

FAILURE: Build failed with an exception.

* What went wrong:
Execution failed for task ':linkDebugExecutableNative'.
> Compilation finished with errors

* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Run with --scan to get full insights.

* Get more help at https://help.gradle.org

BUILD FAILED in 1s
8:31:37 AM: Execution finished 'runDebugExecutableNative'.

Infrastructure

ktor-client-curl artifacts aren’t published after EAP 1146

Update to Kotlin 2.1.0

Network

EXC_GUARD in SelectorHelper.selectionLoop

https://youtrack.jetbrains.com/issue/KTOR-8132

Some users of my app get the following crash on iOS:

Crashed: Thread
EXC_GUARD 0x08fd4dbfade2dead

0  libsystem_kernel.dylib         0x1d7c close + 8
1  RemoteGamepad                  0x133f0f8 kfun:io.ktor.network.selector.SelectorHelper.selectionLoop#internal + 247 (SelectUtilsNix.kt:247)
2  RemoteGamepad                  0x1340abc kfun:io.ktor.network.selector.SelectorHelper.SelectorHelper$start$job$2.invoke#internal + 54 (SelectUtilsNix.kt:54)
3  RemoteGamepad                  0x109eb8 kfun:kotlin.coroutines.intrinsics.createCoroutineUnintercepted$$inlined$createCoroutineFromSuspendFunction$2.invokeSuspend#internal + 4365459128 (IntrinsicsNative.kt:4365459128)
4  RemoteGamepad                  0x107ad8 kfun:kotlin.coroutines.native.internal.BaseContinuationImpl#resumeWith(kotlin.Result<kotlin.Any?>){} + 50 (ContinuationImpl.kt:50)
5  RemoteGamepad                  0x262ad4 kfun:kotlinx.coroutines.DispatchedTask#run(){} + 26 (Continuation.kt:26)
6  RemoteGamepad                  0x2825c8 kfun:kotlinx.coroutines.MultiWorkerDispatcher.$workerRunLoop$lambda$2COROUTINE$0.invokeSuspend#internal + 12 (Runnable.kt:12)
7  RemoteGamepad                  0x282fa8 kfun:kotlinx.coroutines.MultiWorkerDispatcher.MultiWorkerDispatcher$workerRunLoop$2.invoke#internal + 123 (MultithreadedDispatchers.kt:123)
8  RemoteGamepad                  0x109eb8 kfun:kotlin.coroutines.intrinsics.createCoroutineUnintercepted$$inlined$createCoroutineFromSuspendFunction$2.invokeSuspend#internal + 4365459128 (IntrinsicsNative.kt:4365459128)
9  RemoteGamepad                  0x107ad8 kfun:kotlin.coroutines.native.internal.BaseContinuationImpl#resumeWith(kotlin.Result<kotlin.Any?>){} + 50 (ContinuationImpl.kt:50)
10 RemoteGamepad                  0x262ad4 kfun:kotlinx.coroutines.DispatchedTask#run(){} + 26 (Continuation.kt:26)
11 RemoteGamepad                  0x21044c kfun:kotlinx.coroutines.EventLoopImplBase#processNextEvent(){}kotlin.Long + 15 (ObjectiveCUtils.kt:15)
12 RemoteGamepad                  0x279f28 kfun:kotlinx.coroutines#runBlocking(kotlin.coroutines.CoroutineContext;kotlin.coroutines.SuspendFunction1<kotlinx.coroutines.CoroutineScope,0:0>){0§<kotlin.Any?>}0:0 + 49 (EventLoop.common.kt:49)
13 RemoteGamepad                  0x283298 kfun:kotlinx.coroutines.MultiWorkerDispatcher.MultiWorkerDispatcher$1$$inlined$apply$2.$<bridge-DNN>invoke(){}#internal + 123 (MultithreadedDispatchers.kt:123)
14 RemoteGamepad                  0x1e281a4 Worker::processQueueElement(bool) + 46868
15 RemoteGamepad                  0x1e27804 (anonymous namespace)::workerRoutine(void*) + 44404
16 libsystem_pthread.dylib        0x17d0 _pthread_start + 136
17 libsystem_pthread.dylib        0x1480 thread_start + 8

From a quick Google search it seems EXC_GUARD is thrown when closing a socket which Ktor does not own (see close in stack trace). How does this happen?

https://stackoverflow.com/questions/32429431/exc-guard-exception

This issue could maybe be related to https://youtrack.jetbrains.com/issue/KTOR-7299/IOException-Fail-to-select-descriptor-for-ACCEPT

Unable to close socket with open read/write channels on Native

Socket.accept doesn't throw an exception on closing a socket on Native

Server

Ktor Server on Android: java.nio.file.ClosedWatchServiceException

CallLogging: Unhelpful log output "Application started: ..."

CIO Server Engine fails for requests with more than 64 headers

Jetty idleTimeout not working

SessionStorage.read() is called for non-authenticated routes and static assets

https://youtrack.jetbrains.com/issue/KTOR-7194

Hello,

I have user authentication configured with a custom session storage that is set up to read and write from an external database. I am running into an issue where a simple GET call to an un-authenticated route causes 10+ external calls to be made from SessionStorage.read() to retrieve the web session. This is especially a problem for web pages that serve many static assets from the server, since each call to /assets tries to retrieve the web session unnecessarily. It seems like if a session cookie exists, ktor will try to read it even if authenticate() is not used for the route.

Authentication Configuration

fun Application.configureSecurity() {

    install(Sessions) {
        cookie<UserSessionPrincipal>("user_session", DatabaseSessionStorage()) {
            cookie.secure = KTOR_ENV == "prod"
            cookie.maxAgeInSeconds = 2629746 // 1 Month
            serializer = KotlinxSessionSerializer(Json)
        }
    }

    install(Authentication) {
        session<UserSessionPrincipal>("auth-session") {
            validate { session ->
                if (Clock.System.now() > session.expireTimestampUtc) {
                    sessions.clear<UserSessionPrincipal>()
                    throw UnauthorizedException()
                }

                return@validate session
            }
            challenge { throw UnauthorizedException() }
        }
    }
}

Session Storage

// client is a REST client that sends requests to an external service
private class DatabaseSessionStorage : SessionStorage {

    override suspend fun write(id: String, value: String) {
        val session = Json.decodeFromString<UserSessionPrincipal>(value)
        val request = WebSessionPostRequest().JsonBuilder()
            .sessionId(id)
            .expireTimestampUtc(session.expireTimestampUtc)
            .build()
        client.upsertWebSessionForUser(session.userId, request)
    }

    override suspend fun read(id: String): String  {
        val session = try {
            client.getWebSessionBySessionId(id)
        } catch (e: Exception) {
            throw NoSuchElementException("Session with id $id not found")
        }

        return Json.encodeToString(
            UserSessionPrincipal(session.userId, session.email, session.isEmailVerified, session.expireTimestampUtc)
        )
    }

    override suspend fun invalidate(id: String) {
        client.deleteWebSession(id)
    }
}

Routing

fun Application.configureRouting() {
    routing {
        get("/test") {
            call.respond(FreeMarkerContent("index.ftl", null))
        }
        staticResources("/assets", "assets")
        authenticate("auth-form") {
            // Routes responsible for setting web session from a login page
        }
        authenticate("auth-session") {
            // Routes requiring session authentication
        }
    }
}

A call to /test with a session cookie set will then result in 10+ calls to the external API to try and retrieve the session details, first for the original call to the un-authenticated route, then a call to each of the requested static assets located in /assets from the web page.

Please let me know if this is just a mis-configuration on my end or if a bug fix is warranted. Either way it is very unexpected behavior that a simple GET request for a web page will result in the server trying to retrieve web session details when they are not required or explicitly asked for.

Thanks!

Read mutipart upload regression from 2.x to 3.x

https://youtrack.jetbrains.com/issue/KTOR-8082

We read multipart upload channel by chunks with this code, which worked perfrectly in 2.x:

IoByteBufferPool.useInstance { buffer ->        
        val bufferCapacity = buffer.capacity().toLong()
        while (writtenBytes < limit) {
            val bufferLimit = minOf(limit - writtenBytes, bufferCapacity).toInt()
            buffer.clear()
            buffer.limit(bufferLimit)
            val readCount = this.read(buffer)
            if (readCount < 0) break
            if (readCount == 0) {
                continue
            }

            buffer.flip()
            while (buffer.hasRemaining()) {
                writtenBytes += channel.write(buffer)
            }
        }
}

3.x start failing with exception:

kotlinx.coroutines.JobCancellationException Parent job is Cancelling
Caused by: java.io.IOException Limit of 52428800 bytes exceeded while scanning for "
--------------------------b3ed4acffac321e2"
	at io.ktor.utils.io.ByteReadChannelOperationsKt.readUntil(ByteReadChannelOperations.kt:549)
	at io.ktor.utils.io.ByteReadChannelOperationsKt$readUntil$1.invokeSuspend(ByteReadChannelOperations.kt)
	at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:33)
	at kotlinx.coroutines.DispatchedTaskKt.resume(DispatchedTask.kt:221)
	at kotlinx.coroutines.DispatchedTaskKt.resumeUnconfined(DispatchedTask.kt:177)
	at kotlinx.coroutines.DispatchedTaskKt.dispatch(DispatchedTask.kt:149)
	at kotlinx.coroutines.CancellableContinuationImpl.dispatchResume(CancellableContinuationImpl.kt:470)
	at kotlinx.coroutines.CancellableContinuationImpl.resumeImpl$kotlinx_coroutines_core(CancellableContinuationImpl.kt:504)
	at kotlinx.coroutines.CancellableContinuationImpl.resumeImpl$kotlinx_coroutines_core$default(CancellableContinuationImpl.kt:493)
	at kotlinx.coroutines.CancellableContinuationImpl.resumeWith(CancellableContinuationImpl.kt:359)
	at io.ktor.utils.io.ByteChannel$Slot$Task$DefaultImpls.resume(ByteChannel.kt:233)
	at io.ktor.utils.io.ByteChannel$Slot$Write.resume(ByteChannel.kt:253)
	at io.ktor.utils.io.ByteChannel.moveFlushToReadBuffer(ByteChannel.kt:330)
	at io.ktor.utils.io.ByteChannel.getReadBuffer(ByteChannel.kt:41)
	at io.ktor.utils.io.jvm.javaio.BlockingKt$toInputStream$1.read(Blocking.kt:30)
	at java.base/java.nio.channels.Channels$ReadableByteChannelImpl.read(Unknown Source)

where 52428800 is exactly the limit in the snippet

DoubleReceive: NullPointerException caused by race condition

https://youtrack.jetbrains.com/issue/KTOR-8045

We are experiencing random test failures in our CI pipeline with the following error:

java.lang.NullPointerException
	at kotlinx.io.Buffer.recycleHead$kotlinx_io_core(Buffer.kt:585)
	at kotlinx.io.Buffer.skip(Buffer.kt:295)
	at io.ktor.utils.io.core.ByteReadPacketKt.discard(ByteReadPacket.kt:64)
	at io.ktor.utils.io.core.ByteReadPacketKt.discard$default(ByteReadPacket.kt:60)
	at io.ktor.server.plugins.doublereceive.MemoryCache$dispose$1.invokeSuspend(ByteArrayCache.kt:56)
	at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:33)
	at kotlinx.coroutines.DispatchedTask.run(DispatchedTask.kt:100)
	at kotlinx.coroutines.scheduling.CoroutineScheduler.runSafely(CoroutineScheduler.kt:586)
	at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.executeTask(CoroutineScheduler.kt:829)
	at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.runWorker(CoroutineScheduler.kt:717)
	at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.run(CoroutineScheduler.kt:704)

After analyzing the issue, we identified that the behavior of MemoryCache.dispose was modified in PR #4231, which introduced the following change:

GlobalScope.launch {
    reader.discard()
    fullBody?.discard()
}

The issue appears to be a race condition caused by the concurrent execution of a newly launched coroutine. Upon further investigation, we found that the dispose function is being invoked twice in our error scenarios.

The DoubleReceive plugin registers a ResponseSent hook that calls the dispose method. The ResponseSent hook is triggered every time call.response is invoked (as detailed in this StackOverflow post). In our case, the StatusPage plugin makes a second call during error handling, leading to multiple invocations of dispose.

We created a minimal example with a unit test to reproduce the issue (project attached).

    @RepeatedTest(1000)
    fun `my test`() = testApplication {
        application {
            install(DoubleReceive) { cacheRawRequest = true }

            install(StatusPages) {
                status(HttpStatusCode.BadRequest) { call, status ->
                    call.respondText(text = "400: Bad Request", status = status)
                }
            }
            routing {
                post("/") {
                    val request = call.receiveText()
                    call.respond(HttpStatusCode.BadRequest, request)
                }
            }
        }

        val client = createClient {  }

        client.post("/") {
            setBody("Hello World")
        }.also {
            assertEquals(HttpStatusCode.BadRequest, it.status)
        }
    }

Uncaught ClosedWatchServiceException exception thrown by finalizer when closing the server

https://youtrack.jetbrains.com/issue/KTOR-7963

Subject:
Error: ClosedWatchServiceException When Stopping Embedded Server

Description:
I am encountering an issue with the embedded server in my project. When the server instance is stopped and the garbage collector is triggered, the following error occurs:

Uncaught exception thrown by finalizer
java.nio.file.ClosedWatchServiceException

at sun.nio.fs.AbstractPoller.invoke(AbstractPoller.java:216)

at sun.nio.fs.AbstractPoller.close(AbstractPoller.java:144)

at sun.nio.fs.LinuxWatchService$Poller.finalize(LinuxWatchService.java:321)

at java.lang.Daemons$FinalizerDaemon.doFinalize(Daemons.java:370)

at java.lang.Daemons$FinalizerDaemon.processReference(Daemons.java:350)

at java.lang.Daemons$FinalizerDaemon.runInternal(Daemons.java:322)

at java.lang.Daemons$Daemon.run(Daemons.java:131)

at java.lang.Thread.run(Thread.java:1012)

Steps to Reproduce:

Start the provided project.
On the main screen, click the button labeled "Open Second Activity." Wait for a second.
Click the button labeled "Go Back." Wait for a few seconds.
Observe the exception in the logs.

Notes:

If the exception does not appear on the first attempt, repeat steps 2–3 several times until the error occurs.
The number of exceptions increases with the number of repetitions.
To reproduce the issue faster, I manually trigger the garbage collector during testing. However, the bug can still occur without manual triggering.

Without manually triggering the garbage collector, you may need to repeat steps 2–3 several times.

Attachments:

A minimal reproducible project is provided to help debug the issue.
A video demonstration of the issue is included, showcasing how the error occurs

Request for Assistance:
I’m unsure if this behavior is due to a mistake in how I’m managing the embeddedServeror if it is a bug in the implementation. I would greatly appreciate your help in understanding this issue. If there’s a better approach to handle the WatchService lifecycle during server shutdown or to avoid the ClosedWatchServiceException, please let me know.

If this is a bug, I’d be grateful for any guidance on how it might be resolved. Thank you for taking the time to review my issue and for any assistance you can provide!

Support CIO server on WasmJS and JS targets

Swagger: Add deepLinking configuration

Micrometer: Add UptimeMetrics to standard meterBinders

MicrometerMetrics: Do not write unknown HTTP method names to metrics

Add heartbeat to SSE

Swagger UI: Missing Favicon while browsing the UI

`receiveMultipart` throws IllegalStateException instead of UnsupportedMediaTypeException

https://youtrack.jetbrains.com/issue/KTOR-7470

Given this routing

fun Route.myRouting() {
    post<MyResource> { request ->
       call.receiveMultipart()
    }
}

Action

A request is sent which does not have a Content-Type header

Expected

UnsupportedMediaException is thrown, 415 code is returned

Actual

IllegalStateException is thrown, 500 code is returned.

java.lang.IllegalStateException: Content-Type header is required for multipart processing
	at io.ktor.server.engine.DefaultTransformJvmKt.multiPartData(DefaultTransformJvm.kt:37)
	at io.ktor.server.engine.DefaultTransformJvmKt.defaultPlatformTransformations(DefaultTransformJvm.kt:29)
	at io.ktor.server.engine.DefaultTransformKt$installDefaultTransformations$2.invokeSuspend(DefaultTransform.kt:69)
	at io.ktor.server.engine.DefaultTransformKt$installDefaultTransformations$2.invoke(DefaultTransform.kt)
	at io.ktor.server.engine.DefaultTransformKt$installDefaultTransformations$2.invoke(DefaultTransform.kt)
	at io.ktor.util.pipeline.DebugPipelineContext.proceedLoop(DebugPipelineContext.kt:80)
	at io.ktor.util.pipeline.DebugPipelineContext.proceed(DebugPipelineContext.kt:57)
	at io.ktor.util.pipeline.DebugPipelineContext.execute$ktor_utils(DebugPipelineContext.kt:63)
	at io.ktor.util.pipeline.Pipeline.execute(Pipeline.kt:77)
	at io.ktor.server.request.ApplicationReceiveFunctionsKt.receiveNullable(ApplicationReceiveFunctions.kt:103)
...
	at io.ktor.server.resources.RoutingKt$handle$2.invokeSuspend(Routing.kt:265)
	at io.ktor.server.resources.RoutingKt$handle$2.invoke(Routing.kt)
	at io.ktor.server.resources.RoutingKt$handle$2.invoke(Routing.kt)
	at io.ktor.server.routing.Route$buildPipeline$1$1.invokeSuspend(Route.kt:116)
	at io.ktor.server.routing.Route$buildPipeline$1$1.invoke(Route.kt)
	at io.ktor.server.routing.Route$buildPipeline$1$1.invoke(Route.kt)
	at io.ktor.util.pipeline.DebugPipelineContext.proceedLoop(DebugPipelineContext.kt:80)
	at io.ktor.util.pipeline.DebugPipelineContext.proceed(DebugPipelineContext.kt:57)
	at io.ktor.util.pipeline.DebugPipelineContext.execute$ktor_utils(DebugPipelineContext.kt:63)
	at io.ktor.util.pipeline.Pipeline.execute(Pipeline.kt:77)
	at io.ktor.server.routing.Routing$executeResult$$inlined$execute$1.invokeSuspend(Pipeline.kt:478)
	at io.ktor.server.routing.Routing$executeResult$$inlined$execute$1.invoke(Pipeline.kt)
	at io.ktor.server.routing.Routing$executeResult$$inlined$execute$1.invoke(Pipeline.kt)
	at io.ktor.util.debug.ContextUtilsKt$initContextInDebugMode$2.invokeSuspend(ContextUtils.kt:20)
	at io.ktor.util.debug.ContextUtilsKt$initContextInDebugMode$2.invoke(ContextUtils.kt)
	at io.ktor.util.debug.ContextUtilsKt$initContextInDebugMode$2.invoke(ContextUtils.kt)
	at kotlinx.coroutines.intrinsics.UndispatchedKt.startUndispatchedOrReturn(Undispatched.kt:78)
	at kotlinx.coroutines.BuildersKt__Builders_commonKt.withContext(Builders.common.kt:167)
	at kotlinx.coroutines.BuildersKt.withContext(Unknown Source)
	at io.ktor.util.debug.ContextUtilsKt.initContextInDebugMode(ContextUtils.kt:20)
	at io.ktor.server.routing.Routing.executeResult(Routing.kt:190)
	at io.ktor.server.routing.Routing.interceptor(Routing.kt:64)
	at io.ktor.server.routing.Routing$Plugin$install$1.invokeSuspend(Routing.kt:140)
	at io.ktor.server.routing.Routing$Plugin$install$1.invoke(Routing.kt)
	at io.ktor.server.routing.Routing$Plugin$install$1.invoke(Routing.kt)
	at io.ktor.util.pipeline.DebugPipelineContext.proceedLoop(DebugPipelineContext.kt:80)
	at io.ktor.util.pipeline.DebugPipelineContext.proceed(DebugPipelineContext.kt:57)
	at kotlinx.coroutines.intrinsics.UndispatchedKt.startUndispatchedOrReturn(Undispatched.kt:78)
	at kotlinx.coroutines.BuildersKt__Builders_commonKt.withContext(Builders.common.kt:167)
	at kotlinx.coroutines.BuildersKt.withContext(Unknown Source)
	at com.deviceinsight.mosaix.tenantcontext.TenantContextKt$TenantContextPlugin$2$1.invokeSuspend(TenantContext.kt:31)
	at com.deviceinsight.mosaix.tenantcontext.TenantContextKt$TenantContextPlugin$2$1.invoke(TenantContext.kt)
	at com.deviceinsight.mosaix.tenantcontext.TenantContextKt$TenantContextPlugin$2$1.invoke(TenantContext.kt)
	at io.ktor.util.pipeline.DebugPipelineContext.proceedLoop(DebugPipelineContext.kt:80)
	at io.ktor.util.pipeline.DebugPipelineContext.proceed(DebugPipelineContext.kt:57)
	at io.ktor.server.engine.BaseApplicationEngineKt$installDefaultTransformationChecker$1.invokeSuspend(BaseApplicationEngine.kt:124)
	at io.ktor.server.engine.BaseApplicationEngineKt$installDefaultTransformationChecker$1.invoke(BaseApplicationEngine.kt)
	at io.ktor.server.engine.BaseApplicationEngineKt$installDefaultTransformationChecker$1.invoke(BaseApplicationEngine.kt)
	at io.ktor.util.pipeline.DebugPipelineContext.proceedLoop(DebugPipelineContext.kt:80)
	at io.ktor.util.pipeline.DebugPipelineContext.proceed(DebugPipelineContext.kt:57)
	at io.ktor.server.application.hooks.CallFailed$install$1$1.invokeSuspend(CommonHooks.kt:45)
	at io.ktor.server.application.hooks.CallFailed$install$1$1.invoke(CommonHooks.kt)
	at io.ktor.server.application.hooks.CallFailed$install$1$1.invoke(CommonHooks.kt)
	at kotlinx.coroutines.intrinsics.UndispatchedKt.startUndispatchedOrReturn(Undispatched.kt:78)
	at kotlinx.coroutines.CoroutineScopeKt.coroutineScope(CoroutineScope.kt:264)
	at io.ktor.server.application.hooks.CallFailed$install$1.invokeSuspend(CommonHooks.kt:44)
	at io.ktor.server.application.hooks.CallFailed$install$1.invoke(CommonHooks.kt)
	at io.ktor.server.application.hooks.CallFailed$install$1.invoke(CommonHooks.kt)
	at io.ktor.util.pipeline.DebugPipelineContext.proceedLoop(DebugPipelineContext.kt:80)
	at io.ktor.util.pipeline.DebugPipelineContext.proceed(DebugPipelineContext.kt:57)
	at io.ktor.server.application.hooks.CallFailed$install$1$1.invokeSuspend(CommonHooks.kt:45)
	at io.ktor.server.application.hooks.CallFailed$install$1$1.invoke(CommonHooks.kt)
	at io.ktor.server.application.hooks.CallFailed$install$1$1.invoke(CommonHooks.kt)
	at kotlinx.coroutines.intrinsics.UndispatchedKt.startUndispatchedOrReturn(Undispatched.kt:78)
	at kotlinx.coroutines.CoroutineScopeKt.coroutineScope(CoroutineScope.kt:264)
	at io.ktor.server.application.hooks.CallFailed$install$1.invokeSuspend(CommonHooks.kt:44)
	at io.ktor.server.application.hooks.CallFailed$install$1.invoke(CommonHooks.kt)
	at io.ktor.server.application.hooks.CallFailed$install$1.invoke(CommonHooks.kt)
	at io.ktor.util.pipeline.DebugPipelineContext.proceedLoop(DebugPipelineContext.kt:80)
	at io.ktor.util.pipeline.DebugPipelineContext.proceed(DebugPipelineContext.kt:57)
	at io.ktor.util.pipeline.DebugPipelineContext.execute$ktor_utils(DebugPipelineContext.kt:63)
	at io.ktor.util.pipeline.Pipeline.execute(Pipeline.kt:77)
	at io.ktor.server.engine.DefaultEnginePipelineKt$defaultEnginePipeline$1$invokeSuspend$$inlined$execute$1.invokeSuspend(Pipeline.kt:478)
	at io.ktor.server.engine.DefaultEnginePipelineKt$defaultEnginePipeline$1$invokeSuspend$$inlined$execute$1.invoke(Pipeline.kt)
	at io.ktor.server.engine.DefaultEnginePipelineKt$defaultEnginePipeline$1$invokeSuspend$$inlined$execute$1.invoke(Pipeline.kt)
	at io.ktor.util.debug.ContextUtilsKt$initContextInDebugMode$2.invokeSuspend(ContextUtils.kt:20)
	at io.ktor.util.debug.ContextUtilsKt$initContextInDebugMode$2.invoke(ContextUtils.kt)
	at io.ktor.util.debug.ContextUtilsKt$initContextInDebugMode$2.invoke(ContextUtils.kt)
	at kotlinx.coroutines.intrinsics.UndispatchedKt.startUndispatchedOrReturn(Undispatched.kt:78)
	at kotlinx.coroutines.BuildersKt__Builders_commonKt.withContext(Builders.common.kt:167)
	at kotlinx.coroutines.BuildersKt.withContext(Unknown Source)
	at io.ktor.util.debug.ContextUtilsKt.initContextInDebugMode(ContextUtils.kt:20)
	at io.ktor.server.engine.DefaultEnginePipelineKt$defaultEnginePipeline$1.invokeSuspend(DefaultEnginePipeline.kt:123)
	at io.ktor.server.engine.DefaultEnginePipelineKt$defaultEnginePipeline$1.invoke(DefaultEnginePipeline.kt)
	at io.ktor.server.engine.DefaultEnginePipelineKt$defaultEnginePipeline$1.invoke(DefaultEnginePipeline.kt)
	at io.ktor.util.pipeline.DebugPipelineContext.proceedLoop(DebugPipelineContext.kt:80)
	at io.ktor.util.pipeline.DebugPipelineContext.proceed(DebugPipelineContext.kt:57)
	at io.ktor.util.pipeline.DebugPipelineContext.execute$ktor_utils(DebugPipelineContext.kt:63)
	at io.ktor.util.pipeline.Pipeline.execute(Pipeline.kt:77)
	at io.ktor.server.netty.NettyApplicationCallHandler$handleRequest$1$invokeSuspend$$inlined$execute$1.invokeSuspend(Pipeline.kt:478)
	at io.ktor.server.netty.NettyApplicationCallHandler$handleRequest$1$invokeSuspend$$inlined$execute$1.invoke(Pipeline.kt)
	at io.ktor.server.netty.NettyApplicationCallHandler$handleRequest$1$invokeSuspend$$inlined$execute$1.invoke(Pipeline.kt)
	at io.ktor.util.debug.ContextUtilsKt$initContextInDebugMode$2.invokeSuspend(ContextUtils.kt:20)
	at io.ktor.util.debug.ContextUtilsKt$initContextInDebugMode$2.invoke(ContextUtils.kt)
	at io.ktor.util.debug.ContextUtilsKt$initContextInDebugMode$2.invoke(ContextUtils.kt)
	at kotlinx.coroutines.intrinsics.UndispatchedKt.startUndispatchedOrReturn(Undispatched.kt:78)
	at kotlinx.coroutines.BuildersKt__Builders_commonKt.withContext(Builders.common.kt:167)
	at kotlinx.coroutines.BuildersKt.withContext(Unknown Source)
	at io.ktor.util.debug.ContextUtilsKt.initContextInDebugMode(ContextUtils.kt:20)
	at io.ktor.server.netty.NettyApplicationCallHandler$handleRequest$1.invokeSuspend(NettyApplicationCallHandler.kt:140)
	at io.ktor.server.netty.NettyApplicationCallHandler$handleRequest$1.invoke(NettyApplicationCallHandler.kt)
	at io.ktor.server.netty.NettyApplicationCallHandler$handleRequest$1.invoke(NettyApplicationCallHandler.kt)
	at kotlinx.coroutines.intrinsics.UndispatchedKt.startCoroutineUndispatched(Undispatched.kt:44)
	at kotlinx.coroutines.CoroutineStart.invoke(CoroutineStart.kt:112)
	at kotlinx.coroutines.AbstractCoroutine.start(AbstractCoroutine.kt:126)
	at kotlinx.coroutines.BuildersKt__Builders_commonKt.launch(Builders.common.kt:56)
	at kotlinx.coroutines.BuildersKt.launch(Unknown Source)
	at io.ktor.server.netty.NettyApplicationCallHandler.handleRequest(NettyApplicationCallHandler.kt:41)
	at io.ktor.server.netty.NettyApplicationCallHandler.channelRead(NettyApplicationCallHandler.kt:33)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.access$600(AbstractChannelHandlerContext.java:61)
	at io.netty.channel.AbstractChannelHandlerContext$7.run(AbstractChannelHandlerContext.java:425)
	at io.netty.util.concurrent.AbstractEventExecutor.runTask$$$capture(AbstractEventExecutor.java:173)
	at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java)
	at io.netty.util.concurrent.AbstractEventExecutor.safeExecute$$$capture(AbstractEventExecutor.java:166)
	at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java)
	at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
	at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:413)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at io.ktor.server.netty.EventLoopGroupProxy$Companion.create$lambda$1$lambda$0(NettyApplicationEngine.kt:296)
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	at java.base/java.lang.Thread.run(Thread.java:1583)