diff --git a/build.sbt b/build.sbt index 3f9b8a063c..c9d8ee252b 100644 --- a/build.sbt +++ b/build.sbt @@ -102,7 +102,8 @@ lazy val commonLib = project("common-lib").settings( "com.gu" %% "scanamo" % "1.0.0-M8", // Necessary to have a mix of play library versions due to scala-java8-compat incompatibility "com.typesafe.play" %% "play-ahc-ws" % "2.8.9", - "org.yaml" % "snakeyaml" % "1.31" + "org.yaml" % "snakeyaml" % "1.31", + "org.testcontainers" % "elasticsearch" % "1.19.2" % Test ), dependencyOverrides += "org.apache.thrift" % "libthrift" % "0.13.0" ) @@ -136,30 +137,31 @@ lazy val kahuna = playProject("kahuna", 9005).settings( lazy val leases = playProject("leases", 9012) -lazy val mediaApi = playProject("media-api", 9001).settings( - libraryDependencies ++= Seq( - "org.apache.commons" % "commons-email" % "1.5", - "org.parboiled" %% "parboiled" % "2.1.5", - "org.http4s" %% "http4s-core" % "0.23.17", - "com.softwaremill.quicklens" %% "quicklens" % "1.4.11", - "com.whisk" %% "docker-testkit-scalatest" % "0.9.8" % Test, - "com.whisk" %% "docker-testkit-impl-spotify" % "0.9.8" % Test +lazy val mediaApi = playProject("media-api", 9001) + .dependsOn(commonLib % "compile;test->test") + .settings( + libraryDependencies ++= Seq( + "org.apache.commons" % "commons-email" % "1.5", + "org.parboiled" %% "parboiled" % "2.1.5", + "org.http4s" %% "http4s-core" % "0.23.17", + "com.softwaremill.quicklens" %% "quicklens" % "1.4.11", + ) ) -) lazy val metadataEditor = playProject("metadata-editor", 9007) -lazy val thrall = playProject("thrall", 9002).settings( - pipelineStages := Seq(digest, gzip), - libraryDependencies ++= Seq( - "org.codehaus.groovy" % "groovy-json" % "3.0.7", - "com.yakaz.elasticsearch.plugins" % "elasticsearch-action-updatebyquery" % "2.2.0", - "com.amazonaws" % "amazon-kinesis-client" % "1.8.10", - "com.whisk" %% "docker-testkit-scalatest" % "0.9.8" % Test, - "com.whisk" %% "docker-testkit-impl-spotify" % "0.9.8" % Test, - "com.google.protobuf" % "protobuf-java" % "3.19.6" +lazy val thrall = playProject("thrall", 9002) + .dependsOn(commonLib % "compile;test->test") + .settings( + pipelineStages := Seq(digest, gzip), + libraryDependencies ++= Seq( + "org.codehaus.groovy" % "groovy-json" % "3.0.7", + "com.yakaz.elasticsearch.plugins" % "elasticsearch-action-updatebyquery" % "2.2.0", + "com.amazonaws" % "amazon-kinesis-client" % "1.8.10", + "org.testcontainers" % "elasticsearch" % "1.19.2" % Test, + "com.google.protobuf" % "protobuf-java" % "3.19.6" + ) ) -) lazy val usage = playProject("usage", 9009).settings( libraryDependencies ++= Seq( diff --git a/common-lib/src/test/scala/com/gu/mediaservice/testlib/ElasticSearchDockerBase.scala b/common-lib/src/test/scala/com/gu/mediaservice/testlib/ElasticSearchDockerBase.scala new file mode 100644 index 0000000000..e9120c4054 --- /dev/null +++ b/common-lib/src/test/scala/com/gu/mediaservice/testlib/ElasticSearchDockerBase.scala @@ -0,0 +1,46 @@ +package com.gu.mediaservice.testlib + +import org.scalatest.{BeforeAndAfterAll, Suite} +import org.testcontainers.containers.wait.strategy.Wait +import org.testcontainers.elasticsearch.ElasticsearchContainer + +import scala.util.Properties +import scala.concurrent.duration._ +import scala.compat.java8.DurationConverters._ +import scala.jdk.CollectionConverters._ + + +trait ElasticSearchDockerBase extends BeforeAndAfterAll { + self: Suite => + val useEsDocker = Properties.envOrElse("USE_DOCKER_FOR_TESTS", "true").toBoolean + + val esContainer: Option[ElasticsearchContainer] = if (useEsDocker) { + { + val container = new ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch:7.16.2") + .withExposedPorts(9200) + .withAccessToHost(true) + .withEnv(Map( + "cluster.name" -> "media-service", + "xpack.security.enabled" -> "false", + "discovery.type" -> "single-node", + "network.host" -> "0.0.0.0" + ).asJava) + .waitingFor(Wait.forHttp("/") + .forPort(9200) + .forStatusCode(200) + .withStartupTimeout(180.seconds.toJava) + ) + container.start() + Some(container) + } + } else None + + val esPort = esContainer.map(_.getMappedPort(9200)).getOrElse(9200) + val esTestUrl = Properties.envOrElse("ES6_TEST_URL", s"http://localhost:$esPort") + + override protected def afterAll(): Unit = { + super.afterAll() + + esContainer foreach { _.stop() } + } +} diff --git a/media-api/test/lib/elasticsearch/ElasticSearchTest.scala b/media-api/test/lib/elasticsearch/ElasticSearchTest.scala index ee3176e219..a40776f126 100644 --- a/media-api/test/lib/elasticsearch/ElasticSearchTest.scala +++ b/media-api/test/lib/elasticsearch/ElasticSearchTest.scala @@ -11,7 +11,6 @@ import com.gu.mediaservice.model.leases.DenySyndicationLease import com.gu.mediaservice.model.usage.PublishedUsageStatus import com.sksamuel.elastic4s.ElasticDsl import com.sksamuel.elastic4s.ElasticDsl._ -import com.whisk.docker.{DockerContainer, DockerReadyChecker} import lib.querysyntax._ import lib.{MediaApiConfig, MediaApiMetrics} import org.joda.time.DateTime @@ -25,6 +24,7 @@ import play.api.mvc.Security.AuthenticatedRequest import scala.concurrent.duration._ import scala.concurrent.{Await, Future} +import scala.concurrent.ExecutionContext.Implicits.global class ElasticSearchTest extends ElasticSearchTestBase with Eventually with ElasticSearchExecutions with MockitoSugar { @@ -47,20 +47,12 @@ class ElasticSearchTest extends ElasticSearchTestBase with Eventually with Elast current = "Images_Current", migration = "Images_Migration" ), - url = es6TestUrl, + url = esTestUrl, shards = 1, replicas = 0 ) - - def esContainer = if (useEsDocker) Some(DockerContainer("docker.elastic.co/elasticsearch/elasticsearch:7.16.2") - .withPorts(9200 -> Some(9200)) - .withEnv("cluster.name=media-service", "xpack.security.enabled=false", "discovery.type=single-node", "network.host=0.0.0.0") - .withReadyChecker( - DockerReadyChecker.HttpResponseCode(9200, "/", Some("0.0.0.0")).within(10.minutes).looped(40, 1250.millis) - ) - ) else None private lazy val ES = new ElasticSearch(mediaApiConfig, mediaApiMetrics, elasticConfig, () => List.empty, mock[Scheduler]) lazy val client = ES.client diff --git a/media-api/test/lib/elasticsearch/ElasticSearchTestBase.scala b/media-api/test/lib/elasticsearch/ElasticSearchTestBase.scala index 30d4180c68..466c604a65 100644 --- a/media-api/test/lib/elasticsearch/ElasticSearchTestBase.scala +++ b/media-api/test/lib/elasticsearch/ElasticSearchTestBase.scala @@ -1,39 +1,23 @@ package lib.elasticsearch -import java.util.UUID -import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} import com.gu.mediaservice.model._ -import com.whisk.docker.impl.spotify.DockerKitSpotify -import com.whisk.docker.scalatest.DockerTestKit -import com.whisk.docker.{DockerContainer, DockerKit} +import com.gu.mediaservice.testlib.ElasticSearchDockerBase import org.joda.time.DateTime import org.scalatest.concurrent.PatienceConfiguration.{Interval, Timeout} import org.scalatest.concurrent.ScalaFutures import org.scalatest.funspec.AnyFunSpec -import org.scalatest.time.{Milliseconds, Seconds, Span} -import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers +import org.scalatest.time.{Milliseconds, Seconds, Span} import play.api.libs.json.JsString -import scala.concurrent.duration._ -import scala.util.Properties +import java.util.UUID -trait ElasticSearchTestBase extends AnyFunSpec with BeforeAndAfterAll with Matchers with ScalaFutures with Fixtures with DockerKit with DockerTestKit with DockerKitSpotify with ConditionFixtures { +trait ElasticSearchTestBase extends AnyFunSpec with ElasticSearchDockerBase with Matchers with ScalaFutures with Fixtures with ConditionFixtures { val interval = Interval(Span(100, Milliseconds)) val timeout = Timeout(Span(10, Seconds)) - val useEsDocker = Properties.envOrElse("USE_DOCKER_FOR_TESTS", "true").toBoolean - val es6TestUrl = Properties.envOrElse("ES6_TEST_URL", "http://localhost:9200") - - def esContainer: Option[DockerContainer] - - final override def dockerContainers: List[DockerContainer] = - esContainer.toList ++ super.dockerContainers - - final override val StartContainersTimeout = 1.minute - lazy val images = Seq( createImage("getty-image-1", Agency("Getty Images")), createImage("getty-image-2", Agency("Getty Images")), diff --git a/thrall/test/lib/elasticsearch/ElasticSearchTest.scala b/thrall/test/lib/elasticsearch/ElasticSearchTest.scala index 0416fb5e57..f06fd6f182 100644 --- a/thrall/test/lib/elasticsearch/ElasticSearchTest.scala +++ b/thrall/test/lib/elasticsearch/ElasticSearchTest.scala @@ -1,18 +1,16 @@ package lib.elasticsearch -import java.util.UUID import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} import com.gu.mediaservice.model import com.gu.mediaservice.model._ import com.gu.mediaservice.model.leases.{LeasesByMedia, MediaLease} import com.gu.mediaservice.model.usage.{PublishedUsageStatus, SyndicatedUsageStatus} -import com.gu.mediaservice.model.usage.Usage -import com.sksamuel.elastic4s.ElasticDsl import com.sksamuel.elastic4s.ElasticDsl._ -import com.sksamuel.elastic4s.http._ import org.joda.time.{DateTime, DateTimeZone} -import play.api.libs.json.{JsDefined, JsLookupResult, JsObject, JsString, Json} +import play.api.libs.json.{JsString, Json} +import java.util.UUID +import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.{Await, Future} class ElasticSearchTest extends ElasticSearchTestBase { diff --git a/thrall/test/lib/elasticsearch/ElasticSearchTestBase.scala b/thrall/test/lib/elasticsearch/ElasticSearchTestBase.scala index 2ea69fc60d..08a4aa2014 100644 --- a/thrall/test/lib/elasticsearch/ElasticSearchTestBase.scala +++ b/thrall/test/lib/elasticsearch/ElasticSearchTestBase.scala @@ -3,29 +3,23 @@ package lib.elasticsearch import akka.actor.Scheduler import com.gu.mediaservice.lib.elasticsearch.{ElasticSearchAliases, ElasticSearchConfig} import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} +import com.gu.mediaservice.testlib.ElasticSearchDockerBase import com.sksamuel.elastic4s.ElasticDsl import com.sksamuel.elastic4s.ElasticDsl._ -import com.whisk.docker.impl.spotify.DockerKitSpotify -import com.whisk.docker.scalatest.DockerTestKit -import com.whisk.docker.{DockerContainer, DockerKit, DockerReadyChecker} import helpers.Fixtures import org.joda.time.DateTime +import org.scalatest.BeforeAndAfterEach import org.scalatest.concurrent.{Eventually, ScalaFutures} import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers -import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} import org.scalatestplus.mockito.MockitoSugar import play.api.libs.json.{JsDefined, JsLookupResult, Json} import scala.concurrent.Await -import scala.concurrent.ExecutionContext.global +import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration._ -import scala.util.Properties -trait ElasticSearchTestBase extends AnyFreeSpec with Matchers with Fixtures with BeforeAndAfterAll with BeforeAndAfterEach with Eventually with ScalaFutures with DockerKit with DockerTestKit with DockerKitSpotify with MockitoSugar { - - val useEsDocker = Properties.envOrElse("USE_DOCKER_FOR_TESTS", "true").toBoolean - val esTestUrl = Properties.envOrElse("ES6_TEST_URL", "http://localhost:9200") +trait ElasticSearchTestBase extends AnyFreeSpec with Matchers with Fixtures with ElasticSearchDockerBase with BeforeAndAfterEach with Eventually with ScalaFutures with MockitoSugar { val oneHundredMilliseconds = Duration(100, MILLISECONDS) val fiveSeconds = Duration(5, SECONDS) @@ -45,14 +39,6 @@ trait ElasticSearchTestBase extends AnyFreeSpec with Matchers with Fixtures with replicas = 0 ) - val esContainer = if (useEsDocker) Some(DockerContainer("docker.elastic.co/elasticsearch/elasticsearch:7.16.2") - .withPorts(9200 -> Some(9200)) - .withEnv("cluster.name=media-service", "xpack.security.enabled=false", "discovery.type=single-node", "network.host=0.0.0.0") - .withReadyChecker( - DockerReadyChecker.HttpResponseCode(9200, "/", Some("0.0.0.0")).within(10.minutes).looped(40, 1250.millis) - ) - ) else None - lazy val ES = new ElasticSearch(elasticSearchConfig, None, mock[Scheduler]) override def beforeAll { @@ -82,13 +68,9 @@ trait ElasticSearchTestBase extends AnyFreeSpec with Matchers with Fixtures with override def afterAll: Unit = { super.afterAll() - } - - final override def dockerContainers: List[DockerContainer] = - esContainer.toList ++ super.dockerContainers - - final override val StartContainersTimeout = 1.minute + esContainer foreach { _.stop() } + } def reloadedImage(id: String) = { implicit val logMarker: LogMarker = MarkerMap() diff --git a/thrall/test/lib/elasticsearch/GoodToGoCheckTest.scala b/thrall/test/lib/elasticsearch/GoodToGoCheckTest.scala index 420e5aaba1..2ef2ab0051 100644 --- a/thrall/test/lib/elasticsearch/GoodToGoCheckTest.scala +++ b/thrall/test/lib/elasticsearch/GoodToGoCheckTest.scala @@ -2,11 +2,10 @@ package lib.elasticsearch import com.gu.mediaservice.lib.elasticsearch.MappingTest import com.gu.mediaservice.lib.logging.LogMarker -import com.sksamuel.elastic4s.requests.count.CountResponse import com.sksamuel.elastic4s.ElasticDsl._ -import com.sksamuel.elastic4s.Response import org.joda.time.{DateTime, DateTimeZone, Period} +import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future class GoodToGoCheckTest extends ElasticSearchTestBase { diff --git a/thrall/test/lib/elasticsearch/ImageModelTest.scala b/thrall/test/lib/elasticsearch/ImageModelTest.scala index 5db3583b3f..9489b4ab0f 100644 --- a/thrall/test/lib/elasticsearch/ImageModelTest.scala +++ b/thrall/test/lib/elasticsearch/ImageModelTest.scala @@ -4,7 +4,8 @@ import com.gu.mediaservice.lib.elasticsearch.MappingTest import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} import scala.collection.TraversableLike -import scala.concurrent.{Await, Future} +import scala.concurrent.Await +import scala.concurrent.ExecutionContext.Implicits.global class ImageModelTest extends ElasticSearchTestBase { implicit val logMarker: LogMarker = MarkerMap()