From f78227b12dc5029e6b19e3e6d787683a168e3cae Mon Sep 17 00:00:00 2001 From: Natsu Kagami Date: Wed, 29 Nov 2023 15:03:22 +0100 Subject: [PATCH 1/4] Make AbstractCurlBackend async friendly --- .../client4/curl/AbstractCurlBackend.scala | 68 ++++++++++++++----- .../sttp/client4/curl/CurlBackends.scala | 4 +- 2 files changed, 53 insertions(+), 19 deletions(-) diff --git a/core/src/main/scalanative/sttp/client4/curl/AbstractCurlBackend.scala b/core/src/main/scalanative/sttp/client4/curl/AbstractCurlBackend.scala index 47a557e39a..7418a1df8f 100644 --- a/core/src/main/scalanative/sttp/client4/curl/AbstractCurlBackend.scala +++ b/core/src/main/scalanative/sttp/client4/curl/AbstractCurlBackend.scala @@ -25,35 +25,59 @@ import scala.scalanative.unsigned._ abstract class AbstractCurlBackend[F[_]](_monad: MonadError[F], verbose: Boolean) extends GenericBackend[F, Any] { override implicit def monad: MonadError[F] = _monad + /** Given a [[CurlHandle]], perform the request and return a [[CurlCode]]. */ + protected def performCurl(c: CurlHandle): F[CurlCode] + + /** Same as [[performCurl]], but also checks and throws runtime exceptions on bad [[CurlCode]]s. */ + final def perform(c: CurlHandle) = performCurl(c).flatMap(lift) + type R = Any with Effect[F] override def close(): F[Unit] = monad.unit(()) - private var headers: CurlList = _ - private var multiPartHeaders: Seq[CurlList] = Seq() + /** A request-specific context, with allocated zones and headers. */ + private class Context() { + val zone: Zone = Zone.open() + var headers: CurlList = _ + var multiPartHeaders: Seq[CurlList] = Seq() + + def close() = { + zone.close() + if (headers.ptr != null) headers.ptr.free() + multiPartHeaders.foreach(_.ptr.free()) + } + } + + private object Context { + def apply[T](body: Context => F[T]): F[T] = { + implicit val ctx = new Context() + body(ctx).ensure(monad.unit(ctx.close())) + } + } override def send[T](request: GenericRequest[T, R]): F[Response[T]] = adjustExceptions(request) { - unsafe.Zone { implicit z => + def perform(implicit ctx: Context): F[Response[T]] = { + implicit val z = ctx.zone val curl = CurlApi.init if (verbose) { curl.option(Verbose, parameter = true) } if (request.tags.nonEmpty) { - monad.error(new UnsupportedOperationException("Tags are not supported")) + return monad.error(new UnsupportedOperationException("Tags are not supported")) } val reqHeaders = request.headers if (reqHeaders.nonEmpty) { reqHeaders.find(_.name == "Accept-Encoding").foreach(h => curl.option(AcceptEncoding, h.value)) request.body match { case _: MultipartBody[_] => - headers = transformHeaders( + ctx.headers = transformHeaders( reqHeaders :+ Header.contentType(MediaType.MultipartFormData) ) case _ => - headers = transformHeaders(reqHeaders) + ctx.headers = transformHeaders(reqHeaders) } - curl.option(HttpHeader, headers.ptr) + curl.option(HttpHeader, ctx.headers.ptr) } val spaces = responseSpace @@ -62,6 +86,8 @@ abstract class AbstractCurlBackend[F[_]](_monad: MonadError[F], verbose: Boolean case None => handleBase(request, curl, spaces) } } + + Context(ctx => perform(ctx)) } private def adjustExceptions[T](request: GenericRequest[_, _])(t: => F[T]): F[T] = @@ -70,8 +96,9 @@ abstract class AbstractCurlBackend[F[_]](_monad: MonadError[F], verbose: Boolean ) private def handleBase[T](request: GenericRequest[T, R], curl: CurlHandle, spaces: CurlSpaces)(implicit - z: unsafe.Zone + ctx: Context ) = { + implicit val z = ctx.zone curl.option(WriteFunction, AbstractCurlBackend.wdFunc) curl.option(WriteData, spaces.bodyResp) curl.option(TimeoutMs, request.options.readTimeout.toMillis) @@ -79,13 +106,11 @@ abstract class AbstractCurlBackend[F[_]](_monad: MonadError[F], verbose: Boolean curl.option(Url, request.uri.toString) setMethod(curl, request.method) setRequestBody(curl, request.body) - monad.flatMap(lift(curl.perform)) { _ => + monad.flatMap(perform(curl)) { _ => curl.info(ResponseCode, spaces.httpCode) val responseBody = fromCString((!spaces.bodyResp)._1) val responseHeaders_ = parseHeaders(fromCString((!spaces.headersResp)._1)) val httpCode = StatusCode((!spaces.httpCode).toInt) - if (headers.ptr != null) headers.ptr.free() - multiPartHeaders.foreach(_.ptr.free()) free((!spaces.bodyResp)._1) free((!spaces.headersResp)._1) free(spaces.bodyResp.asInstanceOf[Ptr[CSignedChar]]) @@ -112,19 +137,18 @@ abstract class AbstractCurlBackend[F[_]](_monad: MonadError[F], verbose: Boolean } private def handleFile[T](request: GenericRequest[T, R], curl: CurlHandle, file: SttpFile, spaces: CurlSpaces)( - implicit z: unsafe.Zone + implicit ctx: Context ) = { + implicit val z = ctx.zone val outputPath = file.toPath.toString val outputFilePtr: Ptr[FILE] = fopen(toCString(outputPath), toCString("wb")) curl.option(WriteData, outputFilePtr) curl.option(Url, request.uri.toString) setMethod(curl, request.method) setRequestBody(curl, request.body) - monad.flatMap(lift(curl.perform)) { _ => + monad.flatMap(perform(curl)) { _ => curl.info(ResponseCode, spaces.httpCode) val httpCode = StatusCode((!spaces.httpCode).toInt) - if (headers.ptr != null) headers.ptr.free() - multiPartHeaders.foreach(_.ptr.free()) free(spaces.httpCode.asInstanceOf[Ptr[CSignedChar]]) fclose(outputFilePtr) curl.cleanup() @@ -159,7 +183,10 @@ abstract class AbstractCurlBackend[F[_]](_monad: MonadError[F], verbose: Boolean lift(m) } - private def setRequestBody(curl: CurlHandle, body: GenericRequestBody[R])(implicit zone: Zone): F[CurlCode] = + private def setRequestBody(curl: CurlHandle, body: GenericRequestBody[R])(implicit + ctx: Context + ): F[CurlCode] = { + implicit val z = ctx.zone body match { // todo: assign to monad object case b: BasicBodyPart => val str = basicBodyToString(b) @@ -178,7 +205,7 @@ abstract class AbstractCurlBackend[F[_]](_monad: MonadError[F], verbose: Boolean if (otherHeaders.nonEmpty) { val curlList = transformHeaders(otherHeaders) part.withHeaders(curlList.ptr) - multiPartHeaders = multiPartHeaders :+ curlList + ctx.multiPartHeaders = ctx.multiPartHeaders :+ curlList } } lift(curl.option(Mimepost, mime)) @@ -187,6 +214,7 @@ abstract class AbstractCurlBackend[F[_]](_monad: MonadError[F], verbose: Boolean case NoBody => monad.unit(CurlCode.Ok) } + } private def basicBodyToString(body: BodyPart[_]): String = body match { @@ -269,6 +297,12 @@ abstract class AbstractCurlBackend[F[_]](_monad: MonadError[F], verbose: Boolean } } +/** Curl backends that performs the curl operation with a simple `curl_easy_perform`. */ +abstract class AbstractSyncCurlBackend[F[_]](_monad: MonadError[F], verbose: Boolean) + extends AbstractCurlBackend[F](_monad, verbose) { + override def performCurl(c: CurlHandle): F[CurlCode.CurlCode] = monad.unit(c.perform) +} + object AbstractCurlBackend { val wdFunc: CFuncPtr4[Ptr[Byte], CSize, CSize, Ptr[CurlFetch], CSize] = { (ptr: Ptr[CChar], size: CSize, nmemb: CSize, data: Ptr[CurlFetch]) => diff --git a/core/src/main/scalanative/sttp/client4/curl/CurlBackends.scala b/core/src/main/scalanative/sttp/client4/curl/CurlBackends.scala index a976c1f00c..2c0524f192 100644 --- a/core/src/main/scalanative/sttp/client4/curl/CurlBackends.scala +++ b/core/src/main/scalanative/sttp/client4/curl/CurlBackends.scala @@ -9,13 +9,13 @@ import scala.util.Try // Curl supports redirects, but it doesn't store the history, so using FollowRedirectsBackend is more convenient -private class CurlBackend(verbose: Boolean) extends AbstractCurlBackend(IdMonad, verbose) with SyncBackend {} +private class CurlBackend(verbose: Boolean) extends AbstractSyncCurlBackend(IdMonad, verbose) with SyncBackend {} object CurlBackend { def apply(verbose: Boolean = false): SyncBackend = FollowRedirectsBackend(new CurlBackend(verbose)) } -private class CurlTryBackend(verbose: Boolean) extends AbstractCurlBackend(TryMonad, verbose) with Backend[Try] {} +private class CurlTryBackend(verbose: Boolean) extends AbstractSyncCurlBackend(TryMonad, verbose) with Backend[Try] {} object CurlTryBackend { def apply(verbose: Boolean = false): Backend[Try] = FollowRedirectsBackend(new CurlTryBackend(verbose)) From db44c67bec92b437d2123501e99028ea4d067948 Mon Sep 17 00:00:00 2001 From: Natsu Kagami Date: Wed, 29 Nov 2023 22:51:39 +0100 Subject: [PATCH 2/4] Make `perform` private --- .../scalanative/sttp/client4/curl/AbstractCurlBackend.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scalanative/sttp/client4/curl/AbstractCurlBackend.scala b/core/src/main/scalanative/sttp/client4/curl/AbstractCurlBackend.scala index 7418a1df8f..f1f52359c2 100644 --- a/core/src/main/scalanative/sttp/client4/curl/AbstractCurlBackend.scala +++ b/core/src/main/scalanative/sttp/client4/curl/AbstractCurlBackend.scala @@ -29,7 +29,7 @@ abstract class AbstractCurlBackend[F[_]](_monad: MonadError[F], verbose: Boolean protected def performCurl(c: CurlHandle): F[CurlCode] /** Same as [[performCurl]], but also checks and throws runtime exceptions on bad [[CurlCode]]s. */ - final def perform(c: CurlHandle) = performCurl(c).flatMap(lift) + private final def perform(c: CurlHandle) = performCurl(c).flatMap(lift) type R = Any with Effect[F] From b0d68aa793c99dfad994f271e27cd81ac0ba7ddb Mon Sep 17 00:00:00 2001 From: Natsu Kagami Date: Wed, 29 Nov 2023 23:07:56 +0100 Subject: [PATCH 3/4] Hide header lists behind a buffer and expose `transformHeaders` from Context --- .../client4/curl/AbstractCurlBackend.scala | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/core/src/main/scalanative/sttp/client4/curl/AbstractCurlBackend.scala b/core/src/main/scalanative/sttp/client4/curl/AbstractCurlBackend.scala index f1f52359c2..22ddb019d7 100644 --- a/core/src/main/scalanative/sttp/client4/curl/AbstractCurlBackend.scala +++ b/core/src/main/scalanative/sttp/client4/curl/AbstractCurlBackend.scala @@ -14,6 +14,7 @@ import sttp.monad.MonadError import sttp.monad.syntax._ import scala.collection.immutable.Seq +import scala.collection.mutable.ArrayBuffer import scala.io.Source import scala.scalanative.libc.stdio.{fclose, fopen, FILE} import scala.scalanative.libc.stdlib._ @@ -37,14 +38,23 @@ abstract class AbstractCurlBackend[F[_]](_monad: MonadError[F], verbose: Boolean /** A request-specific context, with allocated zones and headers. */ private class Context() { - val zone: Zone = Zone.open() - var headers: CurlList = _ - var multiPartHeaders: Seq[CurlList] = Seq() + implicit val zone: Zone = Zone.open() + private val headers = ArrayBuffer[CurlList]() + + /** Create a new Headers list that gets cleaned up when the context is destroyed. */ + def transformHeaders(reqHeaders: Iterable[Header]): CurlList = { + val h = reqHeaders + .map(header => s"${header.name}: ${header.value}") + .foldLeft(new CurlList(null)) { case (acc, h) => + new CurlList(acc.ptr.append(h)) + } + headers += h + h + } def close() = { zone.close() - if (headers.ptr != null) headers.ptr.free() - multiPartHeaders.foreach(_.ptr.free()) + headers.foreach(l => if (l.ptr != null) l.ptr.free()) } } @@ -69,15 +79,15 @@ abstract class AbstractCurlBackend[F[_]](_monad: MonadError[F], verbose: Boolean val reqHeaders = request.headers if (reqHeaders.nonEmpty) { reqHeaders.find(_.name == "Accept-Encoding").foreach(h => curl.option(AcceptEncoding, h.value)) - request.body match { + val headers = request.body match { case _: MultipartBody[_] => - ctx.headers = transformHeaders( + ctx.transformHeaders( reqHeaders :+ Header.contentType(MediaType.MultipartFormData) ) case _ => - ctx.headers = transformHeaders(reqHeaders) + ctx.transformHeaders(reqHeaders) } - curl.option(HttpHeader, ctx.headers.ptr) + curl.option(HttpHeader, headers.ptr) } val spaces = responseSpace @@ -203,9 +213,8 @@ abstract class AbstractCurlBackend[F[_]](_monad: MonadError[F], verbose: Boolean val otherHeaders = headers.filterNot(_.is(HeaderNames.ContentType)) if (otherHeaders.nonEmpty) { - val curlList = transformHeaders(otherHeaders) + val curlList = ctx.transformHeaders(otherHeaders) part.withHeaders(curlList.ptr) - ctx.multiPartHeaders = ctx.multiPartHeaders :+ curlList } } lift(curl.option(Mimepost, mime)) @@ -281,13 +290,6 @@ abstract class AbstractCurlBackend[F[_]](_monad: MonadError[F], verbose: Boolean override protected def cleanupWhenGotWebSocket(response: Nothing, e: GotAWebSocketException): F[Unit] = response } - private def transformHeaders(reqHeaders: Iterable[Header])(implicit z: Zone): CurlList = - reqHeaders - .map(header => s"${header.name}: ${header.value}") - .foldLeft(new CurlList(null)) { case (acc, h) => - new CurlList(acc.ptr.append(h)) - } - private def toByteArray(str: String): F[Array[Byte]] = monad.unit(str.getBytes) private def lift(code: CurlCode): F[CurlCode] = From 6f99ee55fa2ffa8dee685846f127ab860ec1f2f7 Mon Sep 17 00:00:00 2001 From: Natsu Kagami Date: Wed, 29 Nov 2023 23:09:34 +0100 Subject: [PATCH 4/4] Change `Context.apply` => `Context.evaluateUsing` --- .../scalanative/sttp/client4/curl/AbstractCurlBackend.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/main/scalanative/sttp/client4/curl/AbstractCurlBackend.scala b/core/src/main/scalanative/sttp/client4/curl/AbstractCurlBackend.scala index 22ddb019d7..337f9a2661 100644 --- a/core/src/main/scalanative/sttp/client4/curl/AbstractCurlBackend.scala +++ b/core/src/main/scalanative/sttp/client4/curl/AbstractCurlBackend.scala @@ -59,7 +59,9 @@ abstract class AbstractCurlBackend[F[_]](_monad: MonadError[F], verbose: Boolean } private object Context { - def apply[T](body: Context => F[T]): F[T] = { + + /** Create a new context and evaluates the body with it. Closes the context at the end. */ + def evaluateUsing[T](body: Context => F[T]): F[T] = { implicit val ctx = new Context() body(ctx).ensure(monad.unit(ctx.close())) } @@ -97,7 +99,7 @@ abstract class AbstractCurlBackend[F[_]](_monad: MonadError[F], verbose: Boolean } } - Context(ctx => perform(ctx)) + Context.evaluateUsing(ctx => perform(ctx)) } private def adjustExceptions[T](request: GenericRequest[_, _])(t: => F[T]): F[T] =