Skip to content

Commit

Permalink
Build: use sbt-tpolecat plugin for stricter compilation
Browse files Browse the repository at this point in the history
  • Loading branch information
alexklibisz committed Nov 19, 2023
1 parent e0e1f0d commit eb2be42
Show file tree
Hide file tree
Showing 15 changed files with 237 additions and 182 deletions.
26 changes: 17 additions & 9 deletions build.sbt
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import ElasticsearchPluginPlugin.autoImport._
import ElasticsearchPluginPlugin.autoImport.*
import org.typelevel.scalacoptions.*

Global / scalaVersion := "2.13.12"

Expand All @@ -8,14 +9,20 @@ lazy val Elastic4sVersion = "8.11.0"
lazy val ElastiknnVersion = IO.read(file("version")).strip()
lazy val LuceneVersion = "9.8.0"

lazy val ScalacOptions = List("-Xfatal-warnings", "-Ywarn-unused:imports")
lazy val TestSettings = Seq(
Test / parallelExecution := false,
Test / logBuffered := false,
Test / testOptions += Tests.Argument("-oD"),
libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.17" % Test
)

lazy val TpolecatSettings = Seq(
Test / tpolecatExcludeOptions ++= Set(
ScalacOptions.warnNonUnitStatement,
ScalacOptions.warnNumericWiden
)
)

lazy val `elastiknn-root` = project
.in(file("."))
.settings(
Expand All @@ -40,7 +47,7 @@ lazy val `elastiknn-api4s` = project
"org.elasticsearch" % "elasticsearch-x-content" % ElasticsearchVersion,
"io.circe" %% "circe-parser" % CirceVersion % Test
),
scalacOptions ++= ScalacOptions,
TpolecatSettings,
TestSettings
)

Expand All @@ -53,7 +60,7 @@ lazy val `elastiknn-client-elastic4s` = project
libraryDependencies ++= Seq(
"com.sksamuel.elastic4s" %% "elastic4s-client-esjava" % Elastic4sVersion
),
scalacOptions ++= ScalacOptions,
TpolecatSettings,
TestSettings
)

Expand All @@ -67,7 +74,7 @@ lazy val `elastiknn-lucene` = project
"org.apache.lucene" % "lucene-core" % LuceneVersion,
"org.apache.lucene" % "lucene-analysis-common" % LuceneVersion % Test
),
scalacOptions ++= ScalacOptions,
TpolecatSettings,
TestSettings
)

Expand All @@ -85,7 +92,7 @@ lazy val `elastiknn-models` = project
"--add-exports",
"java.base/jdk.internal.vm.annotation=ALL-UNNAMED"
),
scalacOptions ++= ScalacOptions,
TpolecatSettings,
TestSettings
)

Expand All @@ -94,7 +101,8 @@ lazy val `elastiknn-models-benchmarks` = project
.dependsOn(`elastiknn-models`, `elastiknn-api4s`)
.enablePlugins(JmhPlugin)
.settings(
Jmh / javaOptions ++= Seq("--add-modules", "jdk.incubator.vector")
Jmh / javaOptions ++= Seq("--add-modules", "jdk.incubator.vector"),
TpolecatSettings
)

lazy val `elastiknn-plugin` = project
Expand Down Expand Up @@ -124,14 +132,14 @@ lazy val `elastiknn-plugin` = project
"ch.qos.logback" % "logback-classic" % "1.4.11" % Test,
"com.klibisz.futil" %% "futil" % "0.1.2" % Test
),
scalacOptions ++= ScalacOptions,
TpolecatSettings,
TestSettings
)

lazy val `elastiknn-plugin-integration-tests` = project
.in(file("elastiknn-plugin-integration-tests"))
.dependsOn(`elastiknn-plugin` % "test->test")
.settings(
scalacOptions ++= ScalacOptions,
TpolecatSettings,
TestSettings
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package com.klibisz.elastiknn.api

sealed trait Mapping {
def dims: Int
}

object Mapping {
final case class SparseBool(dims: Int) extends Mapping

final case class JaccardLsh(dims: Int, L: Int, k: Int) extends Mapping

final case class HammingLsh(dims: Int, L: Int, k: Int) extends Mapping

final case class DenseFloat(dims: Int) extends Mapping

final case class CosineLsh(dims: Int, L: Int, k: Int) extends Mapping

final case class L2Lsh(dims: Int, L: Int, k: Int, w: Int) extends Mapping

final case class PermutationLsh(dims: Int, k: Int, repeating: Boolean) extends Mapping
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package com.klibisz.elastiknn.api

sealed trait NearestNeighborsQuery {
def field: String

def vec: Vec

def similarity: Similarity

def withVec(v: Vec): NearestNeighborsQuery
}

object NearestNeighborsQuery {
sealed trait ApproximateQuery extends NearestNeighborsQuery {
def candidates: Int

def withCandidates(candidates: Int): ApproximateQuery
}

final case class Exact(field: String, similarity: Similarity, vec: Vec = Vec.Empty()) extends NearestNeighborsQuery {
override def withVec(v: Vec): NearestNeighborsQuery = copy(vec = v)
}

final case class CosineLsh(field: String, candidates: Int, vec: Vec = Vec.Empty()) extends ApproximateQuery {
override def withVec(v: Vec): NearestNeighborsQuery = copy(vec = v)

override def withCandidates(candidates: Int): ApproximateQuery = copy(candidates = candidates)

override def similarity: Similarity = Similarity.Cosine
}

final case class HammingLsh(field: String, candidates: Int, vec: Vec = Vec.Empty()) extends ApproximateQuery {
override def withVec(v: Vec): NearestNeighborsQuery = copy(vec = v)

override def withCandidates(candidates: Int): ApproximateQuery = copy(candidates = candidates)

override def similarity: Similarity = Similarity.Hamming
}

final case class JaccardLsh(field: String, candidates: Int, vec: Vec = Vec.Empty()) extends ApproximateQuery {
override def withVec(v: Vec): NearestNeighborsQuery = copy(vec = v)

override def withCandidates(candidates: Int): ApproximateQuery = copy(candidates = candidates)

override def similarity: Similarity = Similarity.Jaccard
}

final case class L2Lsh(field: String, candidates: Int, probes: Int = 0, vec: Vec = Vec.Empty()) extends ApproximateQuery {
override def withVec(v: Vec): NearestNeighborsQuery = copy(vec = v)

override def withCandidates(candidates: Int): ApproximateQuery = copy(candidates = candidates)

override def similarity: Similarity = Similarity.L2
}

final case class PermutationLsh(field: String, similarity: Similarity, candidates: Int, vec: Vec = Vec.Empty())
extends ApproximateQuery {
override def withVec(v: Vec): NearestNeighborsQuery = copy(vec = v)

override def withCandidates(candidates: Int): ApproximateQuery = copy(candidates = candidates)
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package com.klibisz.elastiknn.api

sealed trait Similarity

object Similarity {
case object Cosine extends Similarity

case object Hamming extends Similarity

case object Jaccard extends Similarity

case object L1 extends Similarity

case object L2 extends Similarity

val values: Seq[Similarity] = Vector(Cosine, Jaccard, Hamming, L1, L2)
}
89 changes: 89 additions & 0 deletions elastiknn-api4s/src/main/scala/com/klibisz/elastiknn/api/Vec.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
package com.klibisz.elastiknn.api

import scala.annotation.tailrec
import scala.util.Random

sealed trait Vec

object Vec {

sealed trait KnownDims {
this: Vec =>
def dims: Int
}

final case class SparseBool(trueIndices: Array[Int], totalIndices: Int) extends Vec with KnownDims {
def sorted(): SparseBool = copy(trueIndices.sorted)

def isSorted: Boolean = {
@tailrec
def check(i: Int): Boolean =
if (i == trueIndices.length) true
else if (trueIndices(i) < trueIndices(i - 1)) false
else check(i + 1)

check(1)
}

override def equals(other: Any): Boolean = other match {
case other: SparseBool => (trueIndices sameElements other.trueIndices) && totalIndices == other.totalIndices
case _ => false
}

override def toString: String = s"SparseBool(${trueIndices.take(3).mkString(",")},...,${trueIndices.length}/$totalIndices)"

def dims: Int = totalIndices
}

object SparseBool {

def random(totalIndices: Int, bias: Double = 0.5)(implicit rng: Random): SparseBool = {
var trueIndices = Array.empty[Int]
(0 until totalIndices).foreach(i => if (rng.nextDouble() <= bias) trueIndices :+= i else ())
SparseBool(trueIndices, totalIndices)
}

def randoms(totalIndices: Int, n: Int, bias: Double = 0.5)(implicit rng: Random): Vector[SparseBool] =
(0 until n).map(_ => random(totalIndices, bias)).toVector
}

final case class DenseFloat(values: Array[Float]) extends Vec with KnownDims {
override def equals(other: Any): Boolean = other match {
case other: DenseFloat => other.values sameElements values
case _ => false
}

override def toString: String = s"DenseFloat(${values.take(3).map(n => f"$n%.2f").mkString(",")},...,${values.length})"

def dot(other: DenseFloat): Float = {
var (i, dp) = (0, 0f)
while (i < other.values.length) {
dp += (other.values(i) * values(i))
i += 1
}
dp
}

override def dims: Int = values.length
}

object DenseFloat {
def apply(values: Float*): DenseFloat = DenseFloat(values.toArray)

def random(length: Int, unit: Boolean = false, scale: Int = 1)(implicit rng: Random): DenseFloat = {
val v = DenseFloat((0 until length).toArray.map(_ => rng.nextGaussian().toFloat * scale))
if (unit) {
val norm = math.sqrt(v.values.map(x => x * x).sum.toDouble).toFloat
DenseFloat(v.values.map(_ / norm))
} else v
}

def randoms(length: Int, n: Int, unit: Boolean = false, scale: Int = 1)(implicit rng: Random): Vector[DenseFloat] =
(0 until n).map(_ => random(length, unit, scale)).toVector
}

final case class Indexed(index: String, id: String, field: String) extends Vec

final case class Empty() extends Vec

}
Loading

0 comments on commit eb2be42

Please sign in to comment.