Skip to content

use testcontainers directly instead of docker-testkit #4241

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
andrew-nowak merged 2 commits into from
Jun 20, 2024
Merged

use testcontainers directly instead of docker-testkit #4241

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
afc5fe0
use testcontainers directly instead of docker-testkit
andrew-nowak Jan 26, 2024
302b96f
share common elasticsearch docker base into common-lib new testlib pa…
andrew-nowak Jun 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions build.sbt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,8 +142,7 @@ lazy val mediaApi = playProject("media-api", 9001).settings(
"org.parboiled" %% "parboiled" % "2.1.5",
"org.http4s" %% "http4s-core" % "0.23.17",
"com.softwaremill.quicklens" %% "quicklens" % "1.4.11",
"com.whisk" %% "docker-testkit-scalatest" % "0.9.8" % Test,
"com.whisk" %% "docker-testkit-impl-spotify" % "0.9.8" % Test
"org.testcontainers" % "elasticsearch" % "1.19.2" % Test
)
)

Expand All @@ -155,8 +154,7 @@ lazy val thrall = playProject("thrall", 9002).settings(
"org.codehaus.groovy" % "groovy-json" % "3.0.7",
"com.yakaz.elasticsearch.plugins" % "elasticsearch-action-updatebyquery" % "2.2.0",
"com.amazonaws" % "amazon-kinesis-client" % "1.8.10",
"com.whisk" %% "docker-testkit-scalatest" % "0.9.8" % Test,
"com.whisk" %% "docker-testkit-impl-spotify" % "0.9.8" % Test,
"org.testcontainers" % "elasticsearch" % "1.19.2" % Test,
"com.google.protobuf" % "protobuf-java" % "3.19.6"
)
)
Expand Down
14 changes: 5 additions & 9 deletions media-api/test/lib/elasticsearch/ElasticSearchTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,20 +11,24 @@ import com.gu.mediaservice.model.leases.DenySyndicationLease
import com.gu.mediaservice.model.usage.PublishedUsageStatus
import com.sksamuel.elastic4s.ElasticDsl
import com.sksamuel.elastic4s.ElasticDsl._
import com.whisk.docker.{DockerContainer, DockerReadyChecker}
import lib.querysyntax._
import lib.{MediaApiConfig, MediaApiMetrics}
import org.joda.time.DateTime
import org.scalatest.concurrent.Eventually
import org.scalatestplus.mockito.MockitoSugar
import org.testcontainers.containers.wait.strategy.Wait
import org.testcontainers.elasticsearch.ElasticsearchContainer
import play.api.Configuration
import play.api.inject.ApplicationLifecycle
import play.api.libs.json.{JsString, Json}
import play.api.mvc.AnyContent
import play.api.mvc.Security.AuthenticatedRequest

import scala.compat.java8.DurationConverters._
import scala.concurrent.duration._
import scala.concurrent.{Await, Future}
import scala.concurrent.ExecutionContext.Implicits.global
import scala.jdk.CollectionConverters._

class ElasticSearchTest extends ElasticSearchTestBase with Eventually with ElasticSearchExecutions with MockitoSugar {

Expand Down Expand Up @@ -53,14 +57,6 @@ class ElasticSearchTest extends ElasticSearchTestBase with Eventually with Elast
)



def esContainer = if (useEsDocker) Some(DockerContainer("docker.elastic.co/elasticsearch/elasticsearch:7.16.2")
.withPorts(9200 -> Some(9200))
.withEnv("cluster.name=media-service", "xpack.security.enabled=false", "discovery.type=single-node", "network.host=0.0.0.0")
.withReadyChecker(
DockerReadyChecker.HttpResponseCode(9200, "/", Some("0.0.0.0")).within(10.minutes).looped(40, 1250.millis)
)
) else None
private lazy val ES = new ElasticSearch(mediaApiConfig, mediaApiMetrics, elasticConfig, () => List.empty, mock[Scheduler])
lazy val client = ES.client

Expand Down
49 changes: 36 additions & 13 deletions media-api/test/lib/elasticsearch/ElasticSearchTestBase.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,38 +1,61 @@
package lib.elasticsearch

import java.util.UUID
import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap}
import com.gu.mediaservice.model._
import com.whisk.docker.impl.spotify.DockerKitSpotify
import com.whisk.docker.scalatest.DockerTestKit
import com.whisk.docker.{DockerContainer, DockerKit}
import org.joda.time.DateTime
import org.scalatest.concurrent.PatienceConfiguration.{Interval, Timeout}
import org.scalatest.concurrent.ScalaFutures
import org.scalatest.funspec.AnyFunSpec
import org.scalatest.time.{Milliseconds, Seconds, Span}
import org.scalatest.BeforeAndAfterAll
import org.scalatest.matchers.should.Matchers
import org.testcontainers.containers.wait.strategy.Wait
import org.testcontainers.elasticsearch.ElasticsearchContainer
import play.api.libs.json.JsString

import scala.concurrent.duration._
import java.util.concurrent.atomic.AtomicReference
import scala.util.Properties
import scala.concurrent.duration._
import scala.compat.java8.DurationConverters._
import scala.jdk.CollectionConverters._

trait ElasticSearchTestBase extends AnyFunSpec with BeforeAndAfterAll with Matchers with ScalaFutures with Fixtures with DockerKit with DockerTestKit with DockerKitSpotify with ConditionFixtures {

trait ElasticSearchTestBase extends AnyFunSpec with BeforeAndAfterAll with Matchers with ScalaFutures with Fixtures with ConditionFixtures {

val interval = Interval(Span(100, Milliseconds))
val timeout = Timeout(Span(10, Seconds))

val useEsDocker = Properties.envOrElse("USE_DOCKER_FOR_TESTS", "true").toBoolean
val es6TestUrl = Properties.envOrElse("ES6_TEST_URL", "http://localhost:9200")

def esContainer: Option[DockerContainer]

final override def dockerContainers: List[DockerContainer] =
esContainer.toList ++ super.dockerContainers

final override val StartContainersTimeout = 1.minute
val esContainer: Option[ElasticsearchContainer] = if (useEsDocker) {
{
val container = new ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch:7.16.2")
.withExposedPorts(9200)
.withAccessToHost(true)
.withEnv(Map(
"cluster.name" -> "media-service",
"xpack.security.enabled" -> "false",
"discovery.type" -> "single-node",
"network.host" -> "0.0.0.0"
).asJava)
.waitingFor(Wait.forHttp("/")
.forPort(9200)
.forStatusCode(200)
.withStartupTimeout(180.seconds.toJava)
)
container.start()
Some(container)
}
} else None

val esPort = esContainer.map(_.getMappedPort(9200)).getOrElse(9200)
val es6TestUrl = Properties.envOrElse("ES6_TEST_URL", s"http://localhost:$esPort")

override protected def afterAll(): Unit = {
super.afterAll()

esContainer foreach { _.stop() }
}
Copy link
Copy Markdown
Contributor

@twrichards twrichards Jun 20, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

any chance we could DRY this out and have this and thrall equivalent extend something in common? (appreciate it's not rule of 3, but seems like a lot of code with various config options therein which presents scope for divergence).


lazy val images = Seq(
createImage("getty-image-1", Agency("Getty Images")),
Expand Down
8 changes: 3 additions & 5 deletions thrall/test/lib/elasticsearch/ElasticSearchTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,16 @@
package lib.elasticsearch

import java.util.UUID
import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap}
import com.gu.mediaservice.model
import com.gu.mediaservice.model._
import com.gu.mediaservice.model.leases.{LeasesByMedia, MediaLease}
import com.gu.mediaservice.model.usage.{PublishedUsageStatus, SyndicatedUsageStatus}
import com.gu.mediaservice.model.usage.Usage
import com.sksamuel.elastic4s.ElasticDsl
import com.sksamuel.elastic4s.ElasticDsl._
import com.sksamuel.elastic4s.http._
import org.joda.time.{DateTime, DateTimeZone}
import play.api.libs.json.{JsDefined, JsLookupResult, JsObject, JsString, Json}
import play.api.libs.json.{JsString, Json}

import java.util.UUID
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.{Await, Future}

class ElasticSearchTest extends ElasticSearchTestBase {
Expand Down
52 changes: 33 additions & 19 deletions thrall/test/lib/elasticsearch/ElasticSearchTestBase.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,27 +5,27 @@ import com.gu.mediaservice.lib.elasticsearch.{ElasticSearchAliases, ElasticSearc
import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap}
import com.sksamuel.elastic4s.ElasticDsl
import com.sksamuel.elastic4s.ElasticDsl._
import com.whisk.docker.impl.spotify.DockerKitSpotify
import com.whisk.docker.scalatest.DockerTestKit
import com.whisk.docker.{DockerContainer, DockerKit, DockerReadyChecker}
import helpers.Fixtures
import org.joda.time.DateTime
import org.scalatest.concurrent.{Eventually, ScalaFutures}
import org.scalatest.freespec.AnyFreeSpec
import org.scalatest.matchers.should.Matchers
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
import org.scalatestplus.mockito.MockitoSugar
import org.testcontainers.containers.wait.strategy.Wait
import org.testcontainers.elasticsearch.ElasticsearchContainer
import play.api.libs.json.{JsDefined, JsLookupResult, Json}

import scala.compat.java8.DurationConverters.FiniteDurationops
import scala.concurrent.Await
import scala.concurrent.ExecutionContext.global
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration._
import scala.jdk.CollectionConverters.mapAsJavaMapConverter
import scala.util.Properties

trait ElasticSearchTestBase extends AnyFreeSpec with Matchers with Fixtures with BeforeAndAfterAll with BeforeAndAfterEach with Eventually with ScalaFutures with DockerKit with DockerTestKit with DockerKitSpotify with MockitoSugar {
trait ElasticSearchTestBase extends AnyFreeSpec with Matchers with Fixtures with BeforeAndAfterAll with BeforeAndAfterEach with Eventually with ScalaFutures with MockitoSugar {

val useEsDocker = Properties.envOrElse("USE_DOCKER_FOR_TESTS", "true").toBoolean
val esTestUrl = Properties.envOrElse("ES6_TEST_URL", "http://localhost:9200")

val oneHundredMilliseconds = Duration(100, MILLISECONDS)
val fiveSeconds = Duration(5, SECONDS)
Expand All @@ -35,6 +35,30 @@ trait ElasticSearchTestBase extends AnyFreeSpec with Matchers with Fixtures with

override implicit val patienceConfig: PatienceConfig = PatienceConfig(tenSeconds, oneHundredMilliseconds)

val esContainer: Option[ElasticsearchContainer] = if (useEsDocker) {
{
val container = new ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch:7.16.2")
.withExposedPorts(9200)
.withAccessToHost(true)
.withEnv(Map(
"cluster.name" -> "media-service",
"xpack.security.enabled" -> "false",
"discovery.type" -> "single-node",
"network.host" -> "0.0.0.0"
).asJava)
.waitingFor(Wait.forHttp("/")
.forPort(9200)
.forStatusCode(200)
.withStartupTimeout(180.seconds.toJava)
)
container.start()
Some(container)
}
} else None

val esPort = esContainer.map(_.getMappedPort(9200)).getOrElse(9200)
val esTestUrl = Properties.envOrElse("ES6_TEST_URL", s"http://localhost:$esPort")

val elasticSearchConfig = ElasticSearchConfig(
aliases = ElasticSearchAliases(
current = "Images_Current",
Expand All @@ -45,13 +69,7 @@ trait ElasticSearchTestBase extends AnyFreeSpec with Matchers with Fixtures with
replicas = 0
)

val esContainer = if (useEsDocker) Some(DockerContainer("docker.elastic.co/elasticsearch/elasticsearch:7.16.2")
.withPorts(9200 -> Some(9200))
.withEnv("cluster.name=media-service", "xpack.security.enabled=false", "discovery.type=single-node", "network.host=0.0.0.0")
.withReadyChecker(
DockerReadyChecker.HttpResponseCode(9200, "/", Some("0.0.0.0")).within(10.minutes).looped(40, 1250.millis)
)
) else None


lazy val ES = new ElasticSearch(elasticSearchConfig, None, mock[Scheduler])

Expand Down Expand Up @@ -82,13 +100,9 @@ trait ElasticSearchTestBase extends AnyFreeSpec with Matchers with Fixtures with

override def afterAll: Unit = {
super.afterAll()
}

final override def dockerContainers: List[DockerContainer] =
esContainer.toList ++ super.dockerContainers

final override val StartContainersTimeout = 1.minute

esContainer foreach { _.stop() }
}

def reloadedImage(id: String) = {
implicit val logMarker: LogMarker = MarkerMap()
Expand Down
3 changes: 1 addition & 2 deletions thrall/test/lib/elasticsearch/GoodToGoCheckTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,10 @@ package lib.elasticsearch

import com.gu.mediaservice.lib.elasticsearch.MappingTest
import com.gu.mediaservice.lib.logging.LogMarker
import com.sksamuel.elastic4s.requests.count.CountResponse
import com.sksamuel.elastic4s.ElasticDsl._
import com.sksamuel.elastic4s.Response
import org.joda.time.{DateTime, DateTimeZone, Period}

import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.Future

class GoodToGoCheckTest extends ElasticSearchTestBase {
Expand Down
3 changes: 2 additions & 1 deletion thrall/test/lib/elasticsearch/ImageModelTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ import com.gu.mediaservice.lib.elasticsearch.MappingTest
import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap}

import scala.collection.TraversableLike
import scala.concurrent.{Await, Future}
import scala.concurrent.Await
import scala.concurrent.ExecutionContext.Implicits.global

class ImageModelTest extends ElasticSearchTestBase {
implicit val logMarker: LogMarker = MarkerMap()
Expand Down