From 60475411928eaf608a310d4b650528c54c8b4489 Mon Sep 17 00:00:00 2001 From: Lanqing Huang Date: Wed, 8 Jan 2025 13:29:38 +0800 Subject: [PATCH] Add scalafmt (#179) Nothing changes besides codes been formatted. I copied `.scalafmt` from mill and did very little tunings for consistency and readability. My local test results ```sh $ ./mill -i __.publishArtifacts + __.test [418/420] requests[2.13.15].test.test [418] [420/420] requests[3.3.4].test.test [420] [417/420] requests[2.12.20].test.test [417] [420/420] ==================================================================== __.publishArtifacts + __.test ======================================================================= 1s ``` My local binary compatibility check results ``` $ ./mill -i __.mimaReportBinaryIssues [build.mill-64/68] compile [build.mill-64] [info] compiling 1 Scala source to /home/lqhuang/Git/requests-scala/out/mill-build/compile.dest/classes ... [build.mill-64] [info] done compiling [211/212] requests.jvm[2.12.20].mimaReportBinaryIssues [210/212] requests.jvm[2.13.15].mimaReportBinaryIssues [212/212] requests.jvm[3.3.4].mimaReportBinaryIssues [211] Scanning binary compatibility in /home/lqhuang/Git/requests-scala/out/requests/jvm/2.12.20/compile.dest/classes ... [210] Scanning binary compatibility in /home/lqhuang/Git/requests-scala/out/requests/jvm/2.13.15/compile.dest/classes ... [212] Scanning binary compatibility in /home/lqhuang/Git/requests-scala/out/requests/jvm/3.3.4/compile.dest/classes ... [210] Binary compatibility check passed [211] Binary compatibility check passed [212] Binary compatibility check passed [212/212] ============================== __.mimaReportBinaryIssues ============================== 8s ``` --- .gitignore | 4 +- .scalafmt.conf | 12 + build.mill | 3 +- requests/src/requests/Exceptions.scala | 27 +- requests/src/requests/Model.scala | 252 ++++++------ requests/src/requests/Requester.scala | 371 ++++++++++-------- requests/src/requests/Session.scala | 76 ++-- requests/src/requests/StatusMessages.scala | 2 +- requests/src/requests/Util.scala | 27 +- requests/src/requests/package.scala | 2 +- requests/test/resources/badssl.com-client.md | 4 +- .../src-2/requests/Scala2RequestTests.scala | 48 +-- requests/test/src/requests/FileUtils.scala | 8 +- .../test/src/requests/HttpbinTestSuite.scala | 6 +- requests/test/src/requests/ModelTests.scala | 29 +- requests/test/src/requests/RequestTests.scala | 193 ++++----- requests/test/src/requests/ServerUtils.scala | 13 +- 17 files changed, 595 insertions(+), 482 deletions(-) create mode 100644 .scalafmt.conf diff --git a/.gitignore b/.gitignore index 8d3ee55..a687f79 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,13 @@ target/ *.iml -.idea +.idea/ +.vscode/ .settings .classpath .project .cache .sbtserver +.scala-build/ project/.sbtserver tags nohup.out diff --git a/.scalafmt.conf b/.scalafmt.conf new file mode 100644 index 0000000..2cc4941 --- /dev/null +++ b/.scalafmt.conf @@ -0,0 +1,12 @@ +version = "3.8.3" + +maxColumn = 100 +runner.dialect = scala213 + +newlines.beforeCurlyLambdaParams = multilineWithCaseOnly +rewrite.trailingCommas.style = always + +assumeStandardLibraryStripMargin = true +docstrings.style = Asterisk + +project.git = true diff --git a/build.mill b/build.mill index 0e3820c..544f08e 100644 --- a/build.mill +++ b/build.mill @@ -1,7 +1,8 @@ package build import mill._ -import mill.scalalib._ +import scalalib._ +import scalanativelib._ import mill.scalalib.publish.{Developer, License, PomSettings, VersionControl} import $ivy.`de.tototec::de.tobiasroeser.mill.vcs.version::0.4.1` diff --git a/requests/src/requests/Exceptions.scala b/requests/src/requests/Exceptions.scala index 943d727..f578d95 100644 --- a/requests/src/requests/Exceptions.scala +++ b/requests/src/requests/Exceptions.scala @@ -1,18 +1,29 @@ package requests // base class for all custom exceptions thrown by requests. -class RequestsException(val message: String, val cause: Option[Throwable] = None) extends Exception(message, cause.getOrElse(null)) +class RequestsException( + val message: String, + val cause: Option[Throwable] = None, +) extends Exception(message, cause.getOrElse(null)) -class TimeoutException(val url: String, val readTimeout: Int, val connectTimeout: Int) -extends RequestsException(s"Request to $url timed out. (readTimeout: $readTimeout, connectTimout: $connectTimeout)") +class TimeoutException( + val url: String, + val readTimeout: Int, + val connectTimeout: Int, +) extends RequestsException( + s"Request to $url timed out. (readTimeout: $readTimeout, connectTimout: $connectTimeout)", + ) class UnknownHostException(val url: String, val host: String) -extends RequestsException(s"Unknown host $host in url $url") + extends RequestsException(s"Unknown host $host in url $url") class InvalidCertException(val url: String, cause: Throwable) -extends RequestsException(s"Unable to validate SSL certificates for $url", Some(cause)) + extends RequestsException( + s"Unable to validate SSL certificates for $url", + Some(cause), + ) class RequestFailedException(val response: Response) -extends RequestsException( - s"Request to ${response.url} failed with status code ${response.statusCode}\n${response.text()}" -) + extends RequestsException( + s"Request to ${response.url} failed with status code ${response.statusCode}\n${response.text()}", + ) diff --git a/requests/src/requests/Model.scala b/requests/src/requests/Model.scala index 6ba7135..2a98243 100644 --- a/requests/src/requests/Model.scala +++ b/requests/src/requests/Model.scala @@ -12,107 +12,105 @@ import java.util.zip.{DeflaterOutputStream, GZIPOutputStream} import javax.net.ssl.SSLContext /** - * Mechanisms for compressing the upload stream; supports Gzip and Deflate - * by default - */ -trait Compress{ + * Mechanisms for compressing the upload stream; supports Gzip and Deflate by default + */ +trait Compress { def headers: Seq[(String, String)] def wrap(x: OutputStream): OutputStream } -object Compress{ - object Gzip extends Compress{ - def headers = Seq( - "Content-Encoding" -> "gzip" - ) +object Compress { + object Gzip extends Compress { + def headers = Seq("Content-Encoding" -> "gzip") def wrap(x: OutputStream) = new GZIPOutputStream(x) } - object Deflate extends Compress{ - def headers = Seq( - "Content-Encoding" -> "deflate" - ) + object Deflate extends Compress { + def headers = Seq("Content-Encoding" -> "deflate") def wrap(x: OutputStream) = new DeflaterOutputStream(x) } - object None extends Compress{ + object None extends Compress { def headers = Nil def wrap(x: OutputStream) = x } } /** - * The equivalent of configuring a [[Requester.apply]] or [[Requester.stream]] - * call, but without invoking it. Useful if you want to further customize it - * and make the call later via the overloads of `apply`/`stream` that take a - * [[Request]]. - */ -case class Request(url: String, - auth: RequestAuth = RequestAuth.Empty, - params: Iterable[(String, String)] = Nil, - headers: Iterable[(String, String)] = Nil, - readTimeout: Int = 0, - connectTimeout: Int = 0, - proxy: (String, Int) = null, - cert: Cert = null, - sslContext: SSLContext = null, - cookies: Map[String, HttpCookie] = Map(), - cookieValues: Map[String, String] = Map(), - maxRedirects: Int = 5, - verifySslCerts: Boolean = true, - autoDecompress: Boolean = true, - compress: Compress = Compress.None, - keepAlive: Boolean = true, - check: Boolean = true) + * The equivalent of configuring a [[Requester.apply]] or [[Requester.stream]] call, but without + * invoking it. Useful if you want to further customize it and make the call later via the overloads + * of `apply`/`stream` that take a [[Request]]. + */ +case class Request( + url: String, + auth: RequestAuth = RequestAuth.Empty, + params: Iterable[(String, String)] = Nil, + headers: Iterable[(String, String)] = Nil, + readTimeout: Int = 0, + connectTimeout: Int = 0, + proxy: (String, Int) = null, + cert: Cert = null, + sslContext: SSLContext = null, + cookies: Map[String, HttpCookie] = Map(), + cookieValues: Map[String, String] = Map(), + maxRedirects: Int = 5, + verifySslCerts: Boolean = true, + autoDecompress: Boolean = true, + compress: Compress = Compress.None, + keepAlive: Boolean = true, + check: Boolean = true, +) /** - * Represents the different things you can upload in the body of a HTTP - * request. By default, allows form-encoded key-value pairs, arrays of bytes, - * strings, files, and InputStreams. These types can be passed directly to - * the `data` parameter of [[Requester.apply]] and will be wrapped automatically - * by the implicit constructors. - */ -trait RequestBlob{ + * Represents the different things you can upload in the body of a HTTP request. By default, allows + * form-encoded key-value pairs, arrays of bytes, strings, files, and InputStreams. These types can + * be passed directly to the `data` parameter of [[Requester.apply]] and will be wrapped + * automatically by the implicit constructors. + */ +trait RequestBlob { def headers: Seq[(String, String)] = Nil def write(out: java.io.OutputStream): Unit } -object RequestBlob{ - object EmptyRequestBlob extends RequestBlob{ +object RequestBlob { + object EmptyRequestBlob extends RequestBlob { def write(out: java.io.OutputStream): Unit = () override def headers = Seq("Content-Length" -> "0") } - implicit class ByteSourceRequestBlob[T](x: T)(implicit f: T => geny.Writable) extends RequestBlob{ + implicit class ByteSourceRequestBlob[T](x: T)(implicit f: T => geny.Writable) + extends RequestBlob { private[this] val s = f(x) override def headers = super.headers ++ - s.httpContentType.map("Content-Type" -> _) ++ - s.contentLength.map("Content-Length" -> _.toString) + s.httpContentType.map("Content-Type" -> _) ++ + s.contentLength.map("Content-Length" -> _.toString) def write(out: java.io.OutputStream) = s.writeBytesTo(out) } - implicit class FileRequestBlob(x: java.io.File) extends RequestBlob{ + + implicit class FileRequestBlob(x: java.io.File) extends RequestBlob { override def headers = super.headers ++ Seq( "Content-Type" -> "application/octet-stream", - "Content-Length" -> x.length().toString + "Content-Length" -> x.length().toString, ) def write(out: java.io.OutputStream) = Util.transferTo(new FileInputStream(x), out) } - implicit class NioFileRequestBlob(x: java.nio.file.Path) extends RequestBlob{ + + implicit class NioFileRequestBlob(x: java.nio.file.Path) extends RequestBlob { override def headers = super.headers ++ Seq( "Content-Type" -> "application/octet-stream", - "Content-Length" -> java.nio.file.Files.size(x).toString + "Content-Length" -> java.nio.file.Files.size(x).toString, ) - def write(out: java.io.OutputStream) = Util.transferTo(java.nio.file.Files.newInputStream(x), out) + def write(out: java.io.OutputStream) = + Util.transferTo(java.nio.file.Files.newInputStream(x), out) } - implicit class FormEncodedRequestBlob(val x: Iterable[(String, String)]) extends RequestBlob{ + implicit class FormEncodedRequestBlob(val x: Iterable[(String, String)]) extends RequestBlob { val serialized = Util.urlEncode(x).getBytes - override def headers = super.headers ++ Seq( - "Content-Type" -> "application/x-www-form-urlencoded" - ) + override def headers = + super.headers ++ Seq("Content-Type" -> "application/x-www-form-urlencoded") def write(out: java.io.OutputStream) = { out.write(serialized) } } - implicit class MultipartFormRequestBlob(val parts: Iterable[MultiItem]) extends RequestBlob{ + implicit class MultipartFormRequestBlob(val parts: Iterable[MultiItem]) extends RequestBlob { val boundary = UUID.randomUUID().toString val crlf = "\r\n" val pref = "--" @@ -122,23 +120,28 @@ object RequestBlob{ // encode params up front for the length calculation - val partBytes = parts.map(p => (p.name.getBytes(), if (p.filename == null) Array[Byte]() else p.filename.getBytes(), p)) - - override def headers = Seq( - "Content-Type" -> s"multipart/form-data; boundary=$boundary" + val partBytes = parts.map(p => + ( + p.name.getBytes(), + if (p.filename == null) Array[Byte]() else p.filename.getBytes(), + p, + ), ) + + override def headers = Seq("Content-Type" -> s"multipart/form-data; boundary=$boundary") def write(out: java.io.OutputStream) = { def writeBytes(s: String): Unit = out.write(s.getBytes()) partBytes.foreach { - case(name, filename, part) => + case (name, filename, part) => writeBytes(pref + boundary + crlf) - part.data.headers.foreach { case (headerName, headerValue) => - writeBytes(s"$headerName: $headerValue$crlf") + part.data.headers.foreach { + case (headerName, headerValue) => + writeBytes(s"$headerName: $headerValue$crlf") } writeBytes(ContentDisposition) out.write(name) - if (filename.nonEmpty){ + if (filename.nonEmpty) { writeBytes(filenameSnippet) out.write(filename) } @@ -156,46 +159,50 @@ object RequestBlob{ } case class MultiPart(items: MultiItem*) extends RequestBlob.MultipartFormRequestBlob(items) -case class MultiItem(name: String, - data: RequestBlob, - filename: String = null) +case class MultiItem(name: String, data: RequestBlob, filename: String = null) /** - * Wraps the array of bytes returned in the body of a HTTP response - */ -class ResponseBlob(val bytes: Array[Byte]){ + * Wraps the array of bytes returned in the body of a HTTP response + */ +class ResponseBlob(val bytes: Array[Byte]) { override def toString = s"ResponseBlob(${bytes.length} bytes)" def text = new String(bytes) override def hashCode() = java.util.Arrays.hashCode(bytes) - override def equals(obj: scala.Any) = obj match{ + override def equals(obj: scala.Any) = obj match { case r: ResponseBlob => java.util.Arrays.equals(bytes, r.bytes) - case _ => false + case _ => false } } - /** * Represents a HTTP response * - * @param url the URL that the original request was made to - * @param statusCode the status code of the response - * @param statusMessage a string that describes the status code. - * This is not the reason phrase sent by the server, - * but a string describing [[statusCode]], as hardcoded in this library - * @param headers the raw headers the server sent back with the response - * @param data the response body; may contain HTML, JSON, or binary or textual data - * @param history the response of any redirects that were performed before - * arriving at the current response + * @param url + * the URL that the original request was made to + * @param statusCode + * the status code of the response + * @param statusMessage + * a string that describes the status code. This is not the reason phrase sent by the server, but + * a string describing [[statusCode]], as hardcoded in this library + * @param headers + * the raw headers the server sent back with the response + * @param data + * the response body; may contain HTML, JSON, or binary or textual data + * @param history + * the response of any redirects that were performed before arriving at the current response */ -case class Response(url: String, - statusCode: Int, - @deprecated("Value is inferred from `statusCode`", "0.9.0") - statusMessage: String, - data: geny.Bytes, - headers: Map[String, Seq[String]], - history: Option[Response]) extends geny.ByteData with geny.Readable{ +case class Response( + url: String, + statusCode: Int, + @deprecated("Value is inferred from `statusCode`", "0.9.0") + statusMessage: String, + data: geny.Bytes, + headers: Map[String, Seq[String]], + history: Option[Response], +) extends geny.ByteData + with geny.Readable { def bytes = data.array @@ -203,15 +210,16 @@ case class Response(url: String, def contents = data.array /** - * Returns the cookies set by this response, and by any redirects that lead up to it - */ - val cookies: Map[String, HttpCookie] = history.toSeq.flatMap(_.cookies).toMap ++ headers - .get("set-cookie") - .iterator - .flatten - .flatMap(java.net.HttpCookie.parse(_).asScala) - .map(x => x.getName -> x) - .toMap + * Returns the cookies set by this response, and by any redirects that lead up to it + */ + val cookies: Map[String, HttpCookie] = + history.toSeq.flatMap(_.cookies).toMap ++ headers + .get("set-cookie") + .iterator + .flatten + .flatMap(java.net.HttpCookie.parse(_).asScala) + .map(x => x.getName -> x) + .toMap def contentType = headers.get("content-type").flatMap(_.headOption) @@ -229,35 +237,45 @@ case class Response(url: String, override def contentLength: Option[Long] = Some(data.array.length) } -case class StreamHeaders(url: String, - statusCode: Int, - @deprecated("Value is inferred from `statusCode`", "0.9.0") - statusMessage: String, - headers: Map[String, Seq[String]], - history: Option[Response]){ +case class StreamHeaders( + url: String, + statusCode: Int, + @deprecated("Value is inferred from `statusCode`", "0.9.0") + statusMessage: String, + headers: Map[String, Seq[String]], + history: Option[Response], +) { def is2xx = statusCode.toString.charAt(0) == '2' def is3xx = statusCode.toString.charAt(0) == '3' def is4xx = statusCode.toString.charAt(0) == '4' def is5xx = statusCode.toString.charAt(0) == '5' } + /** - * Different ways you can authorize a HTTP request; by default, HTTP Basic - * auth and Proxy auth are supported - */ -trait RequestAuth{ + * Different ways you can authorize a HTTP request; by default, HTTP Basic auth and Proxy auth are + * supported + */ +trait RequestAuth { def header: Option[String] } -object RequestAuth{ - object Empty extends RequestAuth{ +object RequestAuth { + object Empty extends RequestAuth { def header = None } + implicit def implicitBasic(x: (String, String)): Basic = new Basic(x._1, x._2) - class Basic(username: String, password: String) extends RequestAuth{ - def header = Some("Basic " + java.util.Base64.getEncoder.encodeToString((username + ":" + password).getBytes())) + + class Basic(username: String, password: String) extends RequestAuth { + def header = Some( + "Basic " + java.util.Base64.getEncoder.encodeToString((username + ":" + password).getBytes()), + ) } - case class Proxy(username: String, password: String) extends RequestAuth{ - def header = Some("Proxy-Authorization " + java.util.Base64.getEncoder.encodeToString((username + ":" + password).getBytes())) + case class Proxy(username: String, password: String) extends RequestAuth { + def header = Some( + "Proxy-Authorization " + java.util.Base64.getEncoder + .encodeToString((username + ":" + password).getBytes()), + ) } case class Bearer(token: String) extends RequestAuth { def header = Some(s"Bearer $token") @@ -265,7 +283,7 @@ object RequestAuth{ } sealed trait Cert -object Cert{ +object Cert { implicit def implicitP12(path: String): P12 = P12(path, None) implicit def implicitP12(x: (String, String)): P12 = P12(x._1, Some(x._2)) case class P12(p12: String, pwd: Option[String] = None) extends Cert diff --git a/requests/src/requests/Requester.scala b/requests/src/requests/Requester.scala index b207feb..a778788 100644 --- a/requests/src/requests/Requester.scala +++ b/requests/src/requests/Requester.scala @@ -16,8 +16,7 @@ import scala.concurrent.{ExecutionException, Future} import javax.net.ssl.SSLContext - -trait BaseSession{ +trait BaseSession { def headers: Map[String, String] def cookies: mutable.Map[String, HttpCookie] def readTimeout: Int @@ -45,14 +44,15 @@ trait BaseSession{ def send(method: String) = Requester(method, this) } -object BaseSession{ +object BaseSession { val defaultHeaders = Map( "User-Agent" -> "requests-scala", "Accept-Encoding" -> "gzip, deflate", - "Accept" -> "*/*" + "Accept" -> "*/*", ) } -object Requester{ + +object Requester { val officialHttpMethods = Set("GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE") private lazy val methodField: java.lang.reflect.Field = { val m = classOf[HttpURLConnection].getDeclaredField("method") @@ -60,55 +60,68 @@ object Requester{ m } } -case class Requester(verb: String, - sess: BaseSession){ + +case class Requester(verb: String, sess: BaseSession) { private val upperCaseVerb = verb.toUpperCase /** - * Makes a single HTTP request, and returns a [[Response]] object. Requires - * all uploaded request `data` to be provided up-front, and aggregates all - * downloaded response `data` before returning it in the response. If you - * need streaming access to the upload and download, use the [[Requester.stream]] - * function instead. - * - * @param url The URL to which you want to make this HTTP request - * @param auth HTTP authentication you want to use with this request; defaults to none - * @param params URL params to pass to this request, for `GET`s and `DELETE`s - * @param headers Custom headers to use, in addition to the defaults - * @param data Body data to pass to this request, for POSTs and PUTs. Can be a - * Map[String, String] of form data, bulk data as a String or Array[Byte], - * or MultiPart form data. - * @param readTimeout How many milliseconds to wait for data to be read before timing out - * @param connectTimeout How many milliseconds to wait for a connection before timing out - * @param proxy Host and port of a proxy you want to use - * @param cert Client certificate configuration - * @param sslContext Client sslContext configuration - * @param cookies Custom cookies to send up with this request - * @param maxRedirects How many redirects to automatically resolve; defaults to 5. - * You can also set it to 0 to prevent Requests from resolving - * redirects for you - * @param verifySslCerts Set this to false to ignore problems with SSL certificates - * @param check Throw an exception on a 4xx or 5xx response code. Defaults to `true` - */ - def apply(url: String, - auth: RequestAuth = sess.auth, - params: Iterable[(String, String)] = Nil, - headers: Iterable[(String, String)] = Nil, - data: RequestBlob = RequestBlob.EmptyRequestBlob, - readTimeout: Int = sess.readTimeout, - connectTimeout: Int = sess.connectTimeout, - proxy: (String, Int) = sess.proxy, - cert: Cert = sess.cert, - sslContext: SSLContext = sess.sslContext, - cookies: Map[String, HttpCookie] = Map(), - cookieValues: Map[String, String] = Map(), - maxRedirects: Int = sess.maxRedirects, - verifySslCerts: Boolean = sess.verifySslCerts, - autoDecompress: Boolean = sess.autoDecompress, - compress: Compress = sess.compress, - keepAlive: Boolean = true, - check: Boolean = sess.check, - chunkedUpload: Boolean = sess.chunkedUpload): Response = { + * Makes a single HTTP request, and returns a [[Response]] object. Requires all uploaded request + * `data` to be provided up-front, and aggregates all downloaded response `data` before returning + * it in the response. If you need streaming access to the upload and download, use the + * [[Requester.stream]] function instead. + * + * @param url + * The URL to which you want to make this HTTP request + * @param auth + * HTTP authentication you want to use with this request; defaults to none + * @param params + * URL params to pass to this request, for `GET`s and `DELETE`s + * @param headers + * Custom headers to use, in addition to the defaults + * @param data + * Body data to pass to this request, for POSTs and PUTs. Can be a Map[String, String] of form + * data, bulk data as a String or Array[Byte], or MultiPart form data. + * @param readTimeout + * How many milliseconds to wait for data to be read before timing out + * @param connectTimeout + * How many milliseconds to wait for a connection before timing out + * @param proxy + * Host and port of a proxy you want to use + * @param cert + * Client certificate configuration + * @param sslContext + * Client sslContext configuration + * @param cookies + * Custom cookies to send up with this request + * @param maxRedirects + * How many redirects to automatically resolve; defaults to 5. You can also set it to 0 to + * prevent Requests from resolving redirects for you + * @param verifySslCerts + * Set this to false to ignore problems with SSL certificates + * @param check + * Throw an exception on a 4xx or 5xx response code. Defaults to `true` + */ + def apply( + url: String, + auth: RequestAuth = sess.auth, + params: Iterable[(String, String)] = Nil, + headers: Iterable[(String, String)] = Nil, + data: RequestBlob = RequestBlob.EmptyRequestBlob, + readTimeout: Int = sess.readTimeout, + connectTimeout: Int = sess.connectTimeout, + proxy: (String, Int) = sess.proxy, + cert: Cert = sess.cert, + sslContext: SSLContext = sess.sslContext, + cookies: Map[String, HttpCookie] = Map(), + cookieValues: Map[String, String] = Map(), + maxRedirects: Int = sess.maxRedirects, + verifySslCerts: Boolean = sess.verifySslCerts, + autoDecompress: Boolean = sess.autoDecompress, + compress: Compress = sess.compress, + keepAlive: Boolean = true, + check: Boolean = sess.check, + chunkedUpload: Boolean = sess.chunkedUpload, + ): Response = { val out = new ByteArrayOutputStream() var streamHeaders: StreamHeaders = null @@ -134,7 +147,7 @@ case class Requester(verb: String, keepAlive = keepAlive, check = check, chunkedUpload = chunkedUpload, - onHeadersReceived = sh => streamHeaders = sh + onHeadersReceived = sh => streamHeaders = sh, ) w.writeBytesTo(out) @@ -145,50 +158,49 @@ case class Requester(verb: String, statusMessage = streamHeaders.statusMessage, data = new geny.Bytes(out.toByteArray), headers = streamHeaders.headers, - history = streamHeaders.history + history = streamHeaders.history, ) } /** - * Performs a streaming HTTP request. Most of the parameters are the same as - * [[apply]], except that the `data` parameter is missing, and no [[Response]] - * object is returned. Instead, the caller gets access via three callbacks - * (described below). This provides a lower-level API than [[Requester.apply]], - * allowing the caller fine-grained access to the upload/download streams - * so they can direct them where-ever necessary without first aggregating all - * the data into memory. - * - * @param onHeadersReceived the second callback to be called, this provides - * access to the response's status code, status - * message, headers, and any previous re-direct - * responses. Returns a boolean, where `false` can - * be used to - * - * @return a `Writable` that can be used to write the output data to any - * destination - */ - def stream(url: String, - auth: RequestAuth = sess.auth, - params: Iterable[(String, String)] = Nil, - blobHeaders: Iterable[(String, String)] = Nil, - headers: Iterable[(String, String)] = Nil, - data: RequestBlob = RequestBlob.EmptyRequestBlob, - readTimeout: Int = sess.readTimeout, - connectTimeout: Int = sess.connectTimeout, - proxy: (String, Int) = sess.proxy, - cert: Cert = sess.cert, - sslContext: SSLContext = sess.sslContext, - cookies: Map[String, HttpCookie] = Map(), - cookieValues: Map[String, String] = Map(), - maxRedirects: Int = sess.maxRedirects, - verifySslCerts: Boolean = sess.verifySslCerts, - autoDecompress: Boolean = sess.autoDecompress, - compress: Compress = sess.compress, - keepAlive: Boolean = true, - check: Boolean = true, - chunkedUpload: Boolean = false, - redirectedFrom: Option[Response] = None, - onHeadersReceived: StreamHeaders => Unit = null): geny.Readable = new geny.Readable { + * Performs a streaming HTTP request. Most of the parameters are the same as [[apply]], except + * that the `data` parameter is missing, and no [[Response]] object is returned. Instead, the + * caller gets access via three callbacks (described below). This provides a lower-level API than + * [[Requester.apply]], allowing the caller fine-grained access to the upload/download streams so + * they can direct them where-ever necessary without first aggregating all the data into memory. + * + * @param onHeadersReceived + * the second callback to be called, this provides access to the response's status code, status + * message, headers, and any previous re-direct responses. Returns a boolean, where `false` can + * be used to + * + * @return + * a `Writable` that can be used to write the output data to any destination + */ + def stream( + url: String, + auth: RequestAuth = sess.auth, + params: Iterable[(String, String)] = Nil, + blobHeaders: Iterable[(String, String)] = Nil, + headers: Iterable[(String, String)] = Nil, + data: RequestBlob = RequestBlob.EmptyRequestBlob, + readTimeout: Int = sess.readTimeout, + connectTimeout: Int = sess.connectTimeout, + proxy: (String, Int) = sess.proxy, + cert: Cert = sess.cert, + sslContext: SSLContext = sess.sslContext, + cookies: Map[String, HttpCookie] = Map(), + cookieValues: Map[String, String] = Map(), + maxRedirects: Int = sess.maxRedirects, + verifySslCerts: Boolean = sess.verifySslCerts, + autoDecompress: Boolean = sess.autoDecompress, + compress: Compress = sess.compress, + keepAlive: Boolean = true, + check: Boolean = true, + chunkedUpload: Boolean = false, + redirectedFrom: Option[Response] = None, + onHeadersReceived: StreamHeaders => Unit = null, + ): geny.Readable = new geny.Readable { def readBytesThrough[T](f: java.io.InputStream => T): T = { val url0 = new java.net.URL(url) @@ -215,7 +227,7 @@ case class Requester(verb: String, else if (!verifySslCerts) Util.noVerifySSLContext else - SSLContext.getDefault + SSLContext.getDefault, ) .connectTimeout(Duration.ofMillis(connectTimeout)) .build() @@ -229,7 +241,8 @@ case class Requester(verb: String, val allCookies = sessionCookieValues ++ cookieValues - val (contentLengthHeader, otherBlobHeaders) = blobHeaders.partition(_._1.equalsIgnoreCase("Content-Length")) + val (contentLengthHeader, otherBlobHeaders) = + blobHeaders.partition(_._1.equalsIgnoreCase("Content-Length")) val allHeaders = otherBlobHeaders ++ @@ -238,16 +251,20 @@ case class Requester(verb: String, compress.headers ++ auth.header.map("Authorization" -> _) ++ (if (allCookies.isEmpty) None - else Some("Cookie" -> allCookies - .map { case (k, v) => s"""$k="$v"""" } - .mkString("; ") - )) + else + Some( + "Cookie" -> allCookies + .map { case (k, v) => s"""$k="$v"""" } + .mkString("; "), + )) val lastOfEachHeader = allHeaders.foldLeft(ListMap.empty[String, (String, String)]) { case (acc, (k, v)) => acc.updated(k.toLowerCase, k -> v) } - val headersKeyValueAlternating = lastOfEachHeader.values.toList.flatMap { case (k, v) => Seq(k, v) } + val headersKeyValueAlternating = lastOfEachHeader.values.toList.flatMap { + case (k, v) => Seq(k, v) + } val requestBodyInputStream = new PipedInputStream() val requestBodyOutputStream = new PipedOutputStream(requestBodyInputStream) @@ -258,23 +275,27 @@ case class Requester(verb: String, }) val requestBuilder = - HttpRequest.newBuilder() + HttpRequest + .newBuilder() .uri(url1.toURI) .timeout(Duration.ofMillis(readTimeout)) .headers(headersKeyValueAlternating: _*) - .method(upperCaseVerb, + .method( + upperCaseVerb, (contentLengthHeader.headOption.map(_._2), compress) match { - case (Some("0"), _) => HttpRequest.BodyPublishers.noBody() - case (Some(n), Compress.None) => HttpRequest.BodyPublishers.fromPublisher(bodyPublisher, n.toInt) - case _ => bodyPublisher - } + case (Some("0"), _) => HttpRequest.BodyPublishers.noBody() + case (Some(n), Compress.None) => + HttpRequest.BodyPublishers.fromPublisher(bodyPublisher, n.toInt) + case _ => bodyPublisher + }, ) - val fut = httpClient.sendAsync(requestBuilder.build(), HttpResponse.BodyHandlers.ofInputStream()) + val fut = httpClient.sendAsync( + requestBuilder.build(), + HttpResponse.BodyHandlers.ofInputStream(), + ) - usingOutputStream(compress.wrap(requestBodyOutputStream)) { os => - data.write(os) - } + usingOutputStream(compress.wrap(requestBodyOutputStream)) { os => data.write(os) } val response = try @@ -282,27 +303,36 @@ case class Requester(verb: String, catch { case e: ExecutionException => throw e.getCause match { - case e: javax.net.ssl.SSLHandshakeException => new InvalidCertException(url, e) + case e: javax.net.ssl.SSLHandshakeException => new InvalidCertException(url, e) case _: HttpConnectTimeoutException | _: HttpTimeoutException => new TimeoutException(url, readTimeout, connectTimeout) - case e: java.net.UnknownHostException => - new UnknownHostException(url, e.getMessage) - case e: java.net.ConnectException => - new UnknownHostException(url, e.getMessage) - case e => - new RequestsException(e.getMessage, Some(e)) + case e: java.net.UnknownHostException => new UnknownHostException(url, e.getMessage) + case e: java.net.ConnectException => new UnknownHostException(url, e.getMessage) + case e => new RequestsException(e.getMessage, Some(e)) } } val responseCode = response.statusCode() val headerFields = - response.headers().map.asScala + response + .headers() + .map + .asScala .filter(_._1 != null) - .map { case (k, v) => (k.toLowerCase(), v.asScala.toList) }.toMap - - val deGzip = autoDecompress && headerFields.get("content-encoding").toSeq.flatten.exists(_.contains("gzip")) + .map { case (k, v) => (k.toLowerCase(), v.asScala.toList) } + .toMap + + val deGzip = autoDecompress && headerFields + .get("content-encoding") + .toSeq + .flatten + .exists(_.contains("gzip")) val deDeflate = - autoDecompress && headerFields.get("content-encoding").toSeq.flatten.exists(_.contains("deflate")) + autoDecompress && headerFields + .get("content-encoding") + .toSeq + .flatten + .exists(_.contains("deflate")) def persistCookies() = { if (sess.persistCookies) { headerFields @@ -314,9 +344,11 @@ case class Requester(verb: String, } } - if (responseCode.toString.startsWith("3") && - responseCode.toString != "304" && - maxRedirects > 0) { + if ( + responseCode.toString.startsWith("3") && + responseCode.toString != "304" && + maxRedirects > 0 + ) { val out = new ByteArrayOutputStream() Util.transferTo(response.body, out) val bytes = out.toByteArray @@ -327,7 +359,7 @@ case class Requester(verb: String, statusMessage = StatusMessages.byStatusCode.getOrElse(responseCode, ""), data = new geny.Bytes(bytes), headers = headerFields, - history = redirectedFrom + history = redirectedFrom, ) persistCookies() val newUrl = current.headers("location").head @@ -353,7 +385,7 @@ case class Requester(verb: String, check = check, chunkedUpload = chunkedUpload, redirectedFrom = Some(current), - onHeadersReceived = onHeadersReceived + onHeadersReceived = onHeadersReceived, ).readBytesThrough(f) } else { persistCookies() @@ -362,7 +394,7 @@ case class Requester(verb: String, statusCode = responseCode, statusMessage = StatusMessages.byStatusCode.getOrElse(responseCode, ""), headers = headerFields, - history = redirectedFrom + history = redirectedFrom, ) if (onHeadersReceived != null) onHeadersReceived(streamHeaders) @@ -374,17 +406,20 @@ case class Requester(verb: String, // https://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html section 9.4 if (upperCaseVerb == "HEAD") f(new ByteArrayInputStream(Array())) else if (stream != null) { - try f( - if (deGzip) new GZIPInputStream(stream) - else if (deDeflate) new InflaterInputStream(stream) - else stream - ) finally if (!keepAlive) stream.close() + try + f( + if (deGzip) new GZIPInputStream(stream) + else if (deDeflate) new InflaterInputStream(stream) + else stream, + ) + finally if (!keepAlive) stream.close() } else { f(new ByteArrayInputStream(Array())) } } - if (streamHeaders.statusCode == 304 || streamHeaders.is2xx || !check) processWrappedStream(f) + if (streamHeaders.statusCode == 304 || streamHeaders.is2xx || !check) + processWrappedStream(f) else { val errorOutput = new ByteArrayOutputStream() processWrappedStream(geny.Internal.transfer(_, errorOutput)) @@ -395,49 +430,55 @@ case class Requester(verb: String, statusMessage = streamHeaders.statusMessage, data = new geny.Bytes(errorOutput.toByteArray), headers = streamHeaders.headers, - history = streamHeaders.history - ) + history = streamHeaders.history, + ), ) } } } } - private def usingOutputStream[T](os: OutputStream)(fn: OutputStream => T): Unit = - try fn(os) finally os.close() + private def usingOutputStream[T](os: OutputStream)( + fn: OutputStream => T, + ): Unit = + try fn(os) + finally os.close() /** - * Overload of [[Requester.apply]] that takes a [[Request]] object as configuration - */ - def apply(r: Request, data: RequestBlob, chunkedUpload: Boolean): Response = apply( - r.url, - r.auth, - r.params, - r.headers, - data, - r.readTimeout, - r.connectTimeout, - r.proxy, - r.cert, - r.sslContext, - r.cookies, - r.cookieValues, - r.maxRedirects, - r.verifySslCerts, - r.autoDecompress, - r.compress, - r.keepAlive, - r.check, - chunkedUpload - ) + * Overload of [[Requester.apply]] that takes a [[Request]] object as configuration + */ + def apply(r: Request, data: RequestBlob, chunkedUpload: Boolean): Response = + apply( + r.url, + r.auth, + r.params, + r.headers, + data, + r.readTimeout, + r.connectTimeout, + r.proxy, + r.cert, + r.sslContext, + r.cookies, + r.cookieValues, + r.maxRedirects, + r.verifySslCerts, + r.autoDecompress, + r.compress, + r.keepAlive, + r.check, + chunkedUpload, + ) /** * Overload of [[Requester.stream]] that takes a [[Request]] object as configuration */ - def stream(r: Request, - data: RequestBlob, - chunkedUpload: Boolean, - onHeadersReceived: StreamHeaders => Unit): geny.Writable = + def stream( + r: Request, + data: RequestBlob, + chunkedUpload: Boolean, + onHeadersReceived: StreamHeaders => Unit, + ): geny.Writable = stream( url = r.url, auth = r.auth, @@ -460,6 +501,6 @@ case class Requester(verb: String, check = r.check, chunkedUpload = chunkedUpload, redirectedFrom = None, - onHeadersReceived = onHeadersReceived + onHeadersReceived = onHeadersReceived, ) } diff --git a/requests/src/requests/Session.scala b/requests/src/requests/Session.scala index e1b4633..29d5aad 100644 --- a/requests/src/requests/Session.scala +++ b/requests/src/requests/Session.scala @@ -6,39 +6,45 @@ import javax.net.ssl.SSLContext import scala.collection.mutable /** - * A long-lived session; this can be used to automatically persist cookies - * from one request to the next, or to set default configuration that will - * be shared between requests. These configuration flags can all be - * over-ridden by the parameters on [[Requester.apply]] or [[Requester.stream]] - * - * @param auth HTTP authentication you want to use with this request; defaults to none - * @param headers Custom headers to use, in addition to the defaults - * @param readTimeout How long to wait for data to be read before timing out - * @param connectTimeout How long to wait for a connection before timing out - * @param proxy Host and port of a proxy you want to use - * @param cookies Custom cookies to send up with this request - * @param maxRedirects How many redirects to automatically resolve; defaults to 5. - * You can also set it to 0 to prevent Requests from resolving - * redirects for you - * @param verifySslCerts Set this to false to ignore problems with SSL certificates - */ -case class Session(headers: Map[String, String] = BaseSession.defaultHeaders, - cookieValues: Map[String, String] = Map(), - cookies: mutable.Map[String, HttpCookie] = mutable.LinkedHashMap.empty[String, HttpCookie], - auth: RequestAuth = RequestAuth.Empty, - proxy: (String, Int) = null, - cert: Cert = null, - sslContext: SSLContext = null, - persistCookies: Boolean = true, - maxRedirects: Int = 5, - readTimeout: Int = 10 * 1000, - connectTimeout: Int = 10 * 1000, - verifySslCerts: Boolean = true, - autoDecompress: Boolean = true, - compress: Compress = Compress.None, - chunkedUpload: Boolean = false, - check: Boolean = true) - extends BaseSession{ - - for((k, v) <- cookieValues) cookies(k) = new HttpCookie(k, v) + * A long-lived session; this can be used to automatically persist cookies from one request to the + * next, or to set default configuration that will be shared between requests. These configuration + * flags can all be over-ridden by the parameters on [[Requester.apply]] or [[Requester.stream]] + * + * @param auth + * HTTP authentication you want to use with this request; defaults to none + * @param headers + * Custom headers to use, in addition to the defaults + * @param readTimeout + * How long to wait for data to be read before timing out + * @param connectTimeout + * How long to wait for a connection before timing out + * @param proxy + * Host and port of a proxy you want to use + * @param cookies + * Custom cookies to send up with this request + * @param maxRedirects + * How many redirects to automatically resolve; defaults to 5. You can also set it to 0 to prevent + * Requests from resolving redirects for you + * @param verifySslCerts + * Set this to false to ignore problems with SSL certificates + */ +case class Session( + headers: Map[String, String] = BaseSession.defaultHeaders, + cookieValues: Map[String, String] = Map(), + cookies: mutable.Map[String, HttpCookie] = mutable.LinkedHashMap.empty[String, HttpCookie], + auth: RequestAuth = RequestAuth.Empty, + proxy: (String, Int) = null, + cert: Cert = null, + sslContext: SSLContext = null, + persistCookies: Boolean = true, + maxRedirects: Int = 5, + readTimeout: Int = 10 * 1000, + connectTimeout: Int = 10 * 1000, + verifySslCerts: Boolean = true, + autoDecompress: Boolean = true, + compress: Compress = Compress.None, + chunkedUpload: Boolean = false, + check: Boolean = true, +) extends BaseSession { + for ((k, v) <- cookieValues) cookies(k) = new HttpCookie(k, v) } diff --git a/requests/src/requests/StatusMessages.scala b/requests/src/requests/StatusMessages.scala index 56930cb..bc5fb30 100644 --- a/requests/src/requests/StatusMessages.scala +++ b/requests/src/requests/StatusMessages.scala @@ -90,6 +90,6 @@ object StatusMessages { 527 -> "Railgun Error", 530 -> "Site is Frozen", 598 -> "Network Read Timeout Error", - 599 -> "Network Connect Timeout Error" + 599 -> "Network Connect Timeout Error", ) } diff --git a/requests/src/requests/Util.scala b/requests/src/requests/Util.scala index 3d6c425..5b11a0d 100644 --- a/requests/src/requests/Util.scala +++ b/requests/src/requests/Util.scala @@ -7,11 +7,13 @@ import java.security.cert.X509Certificate import javax.net.ssl.{KeyManagerFactory, SSLContext, TrustManager, X509TrustManager} object Util { - def transferTo(is: InputStream, - os: OutputStream, - bufferSize: Int = 8 * 1024) = { + def transferTo( + is: InputStream, + os: OutputStream, + bufferSize: Int = 8 * 1024, + ) = { val buffer = new Array[Byte](bufferSize) - while ( { + while ({ is.read(buffer) match { case -1 => false case n => @@ -22,8 +24,9 @@ object Util { } def urlEncode(x: Iterable[(String, String)]) = { - x.map{case (k, v) => URLEncoder.encode(k, "UTF-8") + "=" + URLEncoder.encode(v, "UTF-8")} - .mkString("&") + x.map { + case (k, v) => URLEncoder.encode(k, "UTF-8") + "=" + URLEncoder.encode(v, "UTF-8") + }.mkString("&") } private[requests] val noVerifySSLContext = { @@ -39,9 +42,11 @@ object Util { private[requests] val noVerifySocketFactory = noVerifySSLContext.getSocketFactory - private[requests] def clientCertSSLContext(cert: Cert, verifySslCerts: Boolean) = cert match { + private[requests] def clientCertSSLContext( + cert: Cert, + verifySslCerts: Boolean, + ) = cert match { case Cert.P12(path, password) => - val pass = password.map(_.toCharArray).getOrElse(Array.emptyCharArray) val keyManagers = { @@ -61,8 +66,10 @@ object Util { } @deprecated("No longer used", "0.9.0") - private[requests] def clientCertSocketFactory(cert: Cert, verifySslCerts: Boolean) = - clientCertSSLContext(cert, verifySslCerts).getSocketFactory + private[requests] def clientCertSocketFactory( + cert: Cert, + verifySslCerts: Boolean, + ) = clientCertSSLContext(cert, verifySslCerts).getSocketFactory private lazy val trustAllCerts = Array[TrustManager](new X509TrustManager() { def getAcceptedIssuers = new Array[X509Certificate](0) diff --git a/requests/src/requests/package.scala b/requests/src/requests/package.scala index 655f5b0..0a15a4e 100644 --- a/requests/src/requests/package.scala +++ b/requests/src/requests/package.scala @@ -34,4 +34,4 @@ package object requests extends _root_.requests.BaseSession { def chunkedUpload: Boolean = false def check: Boolean = true -} \ No newline at end of file +} diff --git a/requests/test/resources/badssl.com-client.md b/requests/test/resources/badssl.com-client.md index 16fda8e..20bebcf 100644 --- a/requests/test/resources/badssl.com-client.md +++ b/requests/test/resources/badssl.com-client.md @@ -31,4 +31,6 @@ And then answer ` ` for no password. Remove temporary files. -```rm badssl.com.ca-cert.ca badssl.com.crt badssl.com.private.key badssl.com.private-nopass.key temp.pem``` \ No newline at end of file +``` +rm badssl.com.ca-cert.ca badssl.com.crt badssl.com.private.key badssl.com.private-nopass.key temp.pem +``` diff --git a/requests/test/src-2/requests/Scala2RequestTests.scala b/requests/test/src-2/requests/Scala2RequestTests.scala index 24f77d5..e97bc73 100644 --- a/requests/test/src-2/requests/Scala2RequestTests.scala +++ b/requests/test/src-2/requests/Scala2RequestTests.scala @@ -4,40 +4,44 @@ import utest._ import ujson._ object Scala2RequestTests extends HttpbinTestSuite { - val tests = Tests{ - - test("params"){ - - test("post"){ - for(chunkedUpload <- Seq(true, false)) { - val res1 = requests.post( - s"http://$localHttpbin/post", - data = Map("hello" -> "world", "foo" -> "baz"), - chunkedUpload = chunkedUpload - ).text() + val tests = Tests { + test("params") { + test("post") { + for (chunkedUpload <- Seq(true, false)) { + val res1 = requests + .post( + s"http://$localHttpbin/post", + data = Map("hello" -> "world", "foo" -> "baz"), + chunkedUpload = chunkedUpload, + ) + .text() assert(read(res1).obj("form") == Obj("foo" -> "baz", "hello" -> "world")) } } test("put") { for (chunkedUpload <- Seq(true, false)) { - val res1 = requests.put( - s"http://$localHttpbin/put", - data = Map("hello" -> "world", "foo" -> "baz"), - chunkedUpload = chunkedUpload - ).text() + val res1 = requests + .put( + s"http://$localHttpbin/put", + data = Map("hello" -> "world", "foo" -> "baz"), + chunkedUpload = chunkedUpload, + ) + .text() assert(read(res1).obj("form") == Obj("foo" -> "baz", "hello" -> "world")) } } - test("send"){ + test("send") { requests.send("get")(s"http://$localHttpbin/get?hello=world&foo=baz") - val res1 = requests.send("put")( - s"http://$localHttpbin/put", - data = Map("hello" -> "world", "foo" -> "baz"), - chunkedUpload = true - ).text + val res1 = requests + .send("put")( + s"http://$localHttpbin/put", + data = Map("hello" -> "world", "foo" -> "baz"), + chunkedUpload = true, + ) + .text assert(read(res1).obj("form") == Obj("foo" -> "baz", "hello" -> "world")) } diff --git a/requests/test/src/requests/FileUtils.scala b/requests/test/src/requests/FileUtils.scala index e2a8d8b..95ddc4b 100644 --- a/requests/test/src/requests/FileUtils.scala +++ b/requests/test/src/requests/FileUtils.scala @@ -7,11 +7,15 @@ import javax.net.ssl.{KeyManagerFactory, SSLContext} object FileUtils { - def createSslContext(keyStorePath: String, keyStorePassword: String): SSLContext = { + def createSslContext( + keyStorePath: String, + keyStorePassword: String, + ): SSLContext = { val stream: InputStream = new FileInputStream(keyStorePath) val sslContext = SSLContext.getInstance("TLS") - val keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm) + val keyManagerFactory = + KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm) val keyStore = KeyStore.getInstance("PKCS12") keyStore.load(stream, keyStorePassword.toCharArray) keyManagerFactory.init(keyStore, keyStorePassword.toCharArray) diff --git a/requests/test/src/requests/HttpbinTestSuite.scala b/requests/test/src/requests/HttpbinTestSuite.scala index e97c4bf..d5d4566 100644 --- a/requests/test/src/requests/HttpbinTestSuite.scala +++ b/requests/test/src/requests/HttpbinTestSuite.scala @@ -5,15 +5,15 @@ import org.testcontainers.containers.wait.strategy.Wait import utest._ abstract class HttpbinTestSuite extends TestSuite { - private val containerDef = GenericContainer.Def( "kennethreitz/httpbin", exposedPorts = Seq(80), - waitStrategy = Wait.forHttp("/") + waitStrategy = Wait.forHttp("/"), ) private val container = containerDef.start() - val localHttpbin: String = s"${container.containerIpAddress}:${container.mappedPort(80)}" + val localHttpbin: String = + s"${container.containerIpAddress}:${container.mappedPort(80)}" override def utestAfterAll(): Unit = { container.stop() diff --git a/requests/test/src/requests/ModelTests.scala b/requests/test/src/requests/ModelTests.scala index 4a4982d..dae218c 100644 --- a/requests/test/src/requests/ModelTests.scala +++ b/requests/test/src/requests/ModelTests.scala @@ -6,7 +6,7 @@ import java.nio.file.{FileSystems, Path} import utest._ -object ModelTests extends TestSuite{ +object ModelTests extends TestSuite { val tests = Tests { test("multipart file uploads should contain application/octet-stream content type") { val path = getClass.getResource("/license.zip").getPath @@ -14,32 +14,19 @@ object ModelTests extends TestSuite{ val nioPath = FileSystems.getDefault.getPath(path) val fileKey = "fileKey" val fileName = "fileName" - - val javaFileMultipart = MultiPart( - MultiItem( - fileKey, - file, - fileName - ) - ) - - val nioPathMultipart = MultiPart( - MultiItem( - fileKey, - nioPath, - fileName - ) - ) - + + val javaFileMultipart = MultiPart(MultiItem(fileKey, file, fileName)) + val nioPathMultipart = MultiPart(MultiItem(fileKey, nioPath, fileName)) + val javaFileOutputStream = new ByteArrayOutputStream() val nioPathOutputStream = new ByteArrayOutputStream() - + javaFileMultipart.write(javaFileOutputStream) nioPathMultipart.write(nioPathOutputStream) - + val javaFileString = new String(javaFileOutputStream.toByteArray) val nioPathString = new String(nioPathOutputStream.toByteArray) - + assert(javaFileString.contains("Content-Type: application/octet-stream")) assert(nioPathString.contains("Content-Type: application/octet-stream")) } diff --git a/requests/test/src/requests/RequestTests.scala b/requests/test/src/requests/RequestTests.scala index cae8151..59d63b8 100644 --- a/requests/test/src/requests/RequestTests.scala +++ b/requests/test/src/requests/RequestTests.scala @@ -5,80 +5,81 @@ import ujson._ object RequestTests extends HttpbinTestSuite { - val tests = Tests{ - test("matchingMethodWorks"){ - val requesters = Seq( - requests.delete, - requests.get, - requests.post, - requests.put - ) + val tests = Tests { + test("matchingMethodWorks") { + val requesters = Seq(requests.delete, requests.get, requests.post, requests.put) - for(baseUrl <- Seq(s"http://$localHttpbin", "https://httpbin.org")){ - for(r <- requesters){ - for(r2 <- requesters){ + for (baseUrl <- Seq(s"http://$localHttpbin", "https://httpbin.org")) { + for (r <- requesters) { + for (r2 <- requesters) { val res = r(s"$baseUrl/${r2.verb.toLowerCase()}", check = false) if (r.verb == r2.verb) assert(res.statusCode == 200) else assert(res.statusCode == 405) - if (r.verb == r2.verb){ + if (r.verb == r2.verb) { val res = r(s"$baseUrl/${r2.verb.toLowerCase()}") assert(res.statusCode == 200) - }else intercept[RequestFailedException]{ - r(s"$baseUrl/${r2.verb.toLowerCase()}") - } + } else + intercept[RequestFailedException] { + r(s"$baseUrl/${r2.verb.toLowerCase()}") + } } } } } - test("params"){ - test("get"){ + test("params") { + test("get") { // All in URL - val res1 = requests.get(s"http://$localHttpbin/get?hello=world&foo=baz").text() + val res1 = + requests.get(s"http://$localHttpbin/get?hello=world&foo=baz").text() assert(read(res1).obj("args") == Obj("foo" -> "baz", "hello" -> "world")) // All in params val res2 = requests.get( s"http://$localHttpbin/get", - params = Map("hello" -> "world", "foo" -> "baz") + params = Map("hello" -> "world", "foo" -> "baz"), ) assert(read(res2).obj("args") == Obj("foo" -> "baz", "hello" -> "world")) // Mixed URL and params - val res3 = requests.get( - s"http://$localHttpbin/get?hello=world", - params = Map("foo" -> "baz") - ).text() + val res3 = requests + .get( + s"http://$localHttpbin/get?hello=world", + params = Map("foo" -> "baz"), + ) + .text() assert(read(res3).obj("args") == Obj("foo" -> "baz", "hello" -> "world")) // Needs escaping val res4 = requests.get( s"http://$localHttpbin/get?hello=world", - params = Map("++-- lol" -> " !@#$%") + params = Map("++-- lol" -> " !@#$%"), ) assert(read(res4).obj("args") == Obj("++-- lol" -> " !@#$%", "hello" -> "world")) } } - test("multipart"){ - for(chunkedUpload <- Seq(true, false)) { - val response = requests.post( - s"http://$localHttpbin/post", - data = MultiPart( - MultiItem("file1", "Hello!".getBytes, "foo.txt"), - MultiItem("file2", "Goodbye!") - ), - chunkedUpload = chunkedUpload - ).text() + test("multipart") { + for (chunkedUpload <- Seq(true, false)) { + val response = requests + .post( + s"http://$localHttpbin/post", + data = MultiPart( + MultiItem("file1", "Hello!".getBytes, "foo.txt"), + MultiItem("file2", "Goodbye!"), + ), + chunkedUpload = chunkedUpload, + ) + .text() assert(read(response).obj("files") == Obj("file1" -> "Hello!")) assert(read(response).obj("form") == Obj("file2" -> "Goodbye!")) } } - test("cookies"){ - test("session"){ + test("cookies") { + test("session") { val s = requests.Session(cookieValues = Map("hello" -> "world")) val res1 = s.get(s"http://$localHttpbin/cookies").text().trim assert(read(res1) == Obj("cookies" -> Obj("hello" -> "world"))) @@ -86,25 +87,27 @@ object RequestTests extends HttpbinTestSuite { val res2 = s.get(s"http://$localHttpbin/cookies").text().trim assert(read(res2) == Obj("cookies" -> Obj("freeform" -> "test", "hello" -> "world"))) } - test("raw"){ + test("raw") { val res1 = requests.get(s"http://$localHttpbin/cookies").text().trim assert(read(res1) == Obj("cookies" -> Obj())) requests.get(s"http://$localHttpbin/cookies/set?freeform=test") val res2 = requests.get(s"http://$localHttpbin/cookies").text().trim assert(read(res2) == Obj("cookies" -> Obj())) } - test("space"){ + test("space") { val s = requests.Session(cookieValues = Map("hello" -> "hello, world")) val res1 = s.get(s"http://$localHttpbin/cookies").text().trim assert(read(res1) == Obj("cookies" -> Obj("hello" -> "hello, world"))) s.get(s"http://$localHttpbin/cookies/set?freeform=test+test") val res2 = s.get(s"http://$localHttpbin/cookies").text().trim - assert(read(res2) == Obj("cookies" -> Obj("freeform" -> "test test", "hello" -> "hello, world"))) + assert( + read(res2) == Obj("cookies" -> Obj("freeform" -> "test test", "hello" -> "hello, world")), + ) } } - test("redirects"){ - test("max"){ + test("redirects") { + test("max") { val res1 = requests.get(s"http://$localHttpbin/absolute-redirect/4") assert(res1.statusCode == 200) val res2 = requests.get(s"http://$localHttpbin/absolute-redirect/5") @@ -114,7 +117,7 @@ object RequestTests extends HttpbinTestSuite { val res4 = requests.get(s"http://$localHttpbin/absolute-redirect/6", maxRedirects = 10) assert(res4.statusCode == 200) } - test("maxRelative"){ + test("maxRelative") { val res1 = requests.get(s"http://$localHttpbin/relative-redirect/4") assert(res1.statusCode == 200) val res2 = requests.get(s"http://$localHttpbin/relative-redirect/5") @@ -126,19 +129,19 @@ object RequestTests extends HttpbinTestSuite { } } - test("test_reproduction"){ + test("test_reproduction") { requests.get(s"http://$localHttpbin/status/304").text() - } - test("streaming"){ + + test("streaming") { val res1 = requests.get(s"http://$localHttpbin/stream/5").text() assert(res1.linesIterator.length == 5) val res2 = requests.get(s"http://$localHttpbin/stream/52").text() assert(res2.linesIterator.length == 52) } - test("timeouts"){ - test("read"){ + test("timeouts") { + test("read") { intercept[TimeoutException] { requests.get(s"http://$localHttpbin/delay/1", readTimeout = 10) } @@ -147,7 +150,7 @@ object RequestTests extends HttpbinTestSuite { requests.get(s"http://$localHttpbin/delay/3", readTimeout = 2000) } } - test("connect"){ + test("connect") { intercept[TimeoutException] { // use remote httpbin.org so it needs more time to connect requests.get(s"https://httpbin.org/delay/1", connectTimeout = 1) @@ -155,20 +158,20 @@ object RequestTests extends HttpbinTestSuite { } } - test("failures"){ - intercept[UnknownHostException]{ + test("failures") { + intercept[UnknownHostException] { requests.get("https://doesnt-exist-at-all.com/") } - intercept[InvalidCertException]{ + intercept[InvalidCertException] { requests.get("https://expired.badssl.com/") } requests.get("https://doesnt-exist.com/", verifySslCerts = false) - intercept[java.net.MalformedURLException]{ + intercept[java.net.MalformedURLException] { requests.get("://doesnt-exist.com/") } } - test("decompress"){ + test("decompress") { val res1 = requests.get(s"http://$localHttpbin/gzip") assert(read(res1.text()).obj("headers").obj("Host").str == localHttpbin) @@ -181,94 +184,104 @@ object RequestTests extends HttpbinTestSuite { val res4 = requests.get(s"http://$localHttpbin/deflate", autoDecompress = false) assert(res4.bytes.length < res2.bytes.length) - (res1.bytes.length, res2.bytes.length, res3.bytes.length, res4.bytes.length) + ( + res1.bytes.length, + res2.bytes.length, + res3.bytes.length, + res4.bytes.length, + ) } - test("compression"){ + test("compression") { val res1 = requests.post( s"http://$localHttpbin/post", compress = requests.Compress.None, - data = new RequestBlob.ByteSourceRequestBlob("Hello World") + data = new RequestBlob.ByteSourceRequestBlob("Hello World"), ) assert(res1.text().contains(""""Hello World"""")) val res2 = requests.post( s"http://$localHttpbin/post", compress = requests.Compress.Gzip, - data = new RequestBlob.ByteSourceRequestBlob("I am cow") + data = new RequestBlob.ByteSourceRequestBlob("I am cow"), + ) + assert( + read(new String(res2.bytes))("data").toString + .contains("data:application/octet-stream;base64,H4sIAAAAAA"), ) - assert(read(new String(res2.bytes))("data").toString.contains("data:application/octet-stream;base64,H4sIAAAAAA")) val res3 = requests.post( s"http://$localHttpbin/post", compress = requests.Compress.Deflate, - data = new RequestBlob.ByteSourceRequestBlob("Hear me moo") + data = new RequestBlob.ByteSourceRequestBlob("Hear me moo"), ) - assert(read(new String(res3.bytes))("data").toString == - """"data:application/octet-stream;base64,eJzzSE0sUshNVcjNzwcAFokD3g=="""") - } + assert( + read(new String(res3.bytes))( + "data", + ).toString == """"data:application/octet-stream;base64,eJzzSE0sUshNVcjNzwcAFokD3g=="""", + ) + } - test("headers"){ - test("default"){ + test("headers") { + test("default") { val res = requests.get(s"http://$localHttpbin/headers").text() val hs = read(res)("headers").obj assert(hs("User-Agent").str == "requests-scala") assert(hs("Accept-Encoding").str == "gzip, deflate") assert(hs("Accept").str == "*/*") - test("hasNoCookie"){ + test("hasNoCookie") { assert(!hs.contains("Cookie")) } } } - test("clientCertificate"){ + test("clientCertificate") { val base = sys.env("MILL_TEST_RESOURCE_DIR") val url = "https://client.badssl.com" - val instruction = "https://github.com/lihaoyi/requests-scala/blob/master/requests/test/resources/badssl.com-client.md" - val certificateExpiredMessage = s"WARNING: Certificate may have expired and needs to be updated. Please check: $instruction and/or file issue" - test("passwordProtected"){ + val instruction = + "https://github.com/lihaoyi/requests-scala/blob/master/requests/test/resources/badssl.com-client.md" + val certificateExpiredMessage = + s"WARNING: Certificate may have expired and needs to be updated. Please check: $instruction and/or file issue" + test("passwordProtected") { val res = requests.get( url, cert = Cert.implicitP12(s"$base/badssl.com-client.p12", "badssl.com"), - check = false + check = false, ) if (res.statusCode == 400) println(certificateExpiredMessage) else assert(res.statusCode == 200) } - test("noPassword"){ + test("noPassword") { val res = requests.get( "https://client.badssl.com", cert = Cert.implicitP12(s"$base/badssl.com-client-nopass.p12"), - check = false + check = false, ) if (res.statusCode == 400) println(certificateExpiredMessage) else assert(res.statusCode == 200) } - test("sslContext"){ + test("sslContext") { val res = requests.get( "https://client.badssl.com", sslContext = FileUtils.createSslContext(s"$base/badssl.com-client.p12", "badssl.com"), - check = false + check = false, ) if (res.statusCode == 400) println(certificateExpiredMessage) else assert(res.statusCode == 200) } - test("noCert"){ - val res = requests.get( - "https://client.badssl.com", - check = false - ) + test("noCert") { + val res = requests.get("https://client.badssl.com", check = false) assert(res.statusCode == 400) } } - test("selfSignedCertificate"){ + test("selfSignedCertificate") { val res = requests.get( "https://self-signed.badssl.com", verifySslCerts = false, @@ -276,7 +289,7 @@ object RequestTests extends HttpbinTestSuite { assert(res.statusCode == 200) } - test("gzipError"){ + test("gzipError") { val response = requests.head("https://api.github.com/users/lihaoyi") assert(response.statusCode == 200) assert(response.data.array.isEmpty) @@ -285,9 +298,9 @@ object RequestTests extends HttpbinTestSuite { } /** - * Compress with each compression mode and call server. Server expands - * and passes it back so we can compare - */ + * Compress with each compression mode and call server. Server expands and passes it back so we + * can compare + */ test("compressionData") { import requests.Compress._ val str = "I am deflater mouse" @@ -297,7 +310,7 @@ object RequestTests extends HttpbinTestSuite { requests.post( s"http://localhost:$port/echo", compress = c, - data = new RequestBlob.ByteSourceRequestBlob(str) + data = new RequestBlob.ByteSourceRequestBlob(str), ) assert(str == response.data.toString) } @@ -307,9 +320,13 @@ object RequestTests extends HttpbinTestSuite { // Ensure when duplicate headers are passed to requests, we only pass the last one // to the server. This preserves the 0.8.x behavior, and can always be overriden // by passing a comma-separated list of headers instead - test("duplicateHeaders"){ - val res = requests.get(s"http://$localHttpbin/get", headers = Seq("x-y" -> "a", "x-y" -> "b")) - assert(ujson.read(res)("headers")("X-Y") == Str("b")) // make sure it's not "a,b" + test("duplicateHeaders") { + val res = requests.get( + s"http://$localHttpbin/get", + headers = Seq("x-y" -> "a", "x-y" -> "b"), + ) + // make sure it's not "a,b" + assert(ujson.read(res)("headers")("X-Y") == Str("b")) } } } diff --git a/requests/test/src/requests/ServerUtils.scala b/requests/test/src/requests/ServerUtils.scala index 81b0ea0..341ce83 100644 --- a/requests/test/src/requests/ServerUtils.scala +++ b/requests/test/src/requests/ServerUtils.scala @@ -16,7 +16,8 @@ object ServerUtils { } private class EchoServer extends HttpHandler { - private val server: HttpServer = HttpServer.create(new InetSocketAddress(0), 0) + private val server: HttpServer = + HttpServer.create(new InetSocketAddress(0), 0) server.createContext("/echo", this) server.setExecutor(null); // default executor server.start() @@ -43,10 +44,11 @@ object ServerUtils { } } - /** Stream uncompresser - * @param c - * Compression mode - */ + /** + * Stream uncompresser + * @param c + * Compression mode + */ private class Plumper(c: Compress) { private def wrap(is: InputStream): InputStream = @@ -77,5 +79,4 @@ object ServerUtils { sb.toString() } } - }