WIP S3 SDK upgrade

auth/app/auth/AuthController.scala

@@ -8,9 +8,10 @@ import com.gu.mediaservice.lib.auth.provider.AuthenticationProviders
 import com.gu.mediaservice.lib.auth.{Authentication, Authorisation, Internal}
 import com.gu.mediaservice.lib.guardian.auth.PandaAuthenticationProvider
 import play.api.libs.json.Json
-import play.api.mvc.{BaseController, ControllerComponents, Result}
+import play.api.mvc.{Action, AnyContent, BaseController, ControllerComponents, Result}
 
 import java.net.URI
+import java.time.Instant
 import java.util.Date
 import scala.concurrent.{ExecutionContext, Future}
 import scala.util.Try
@@ -32,12 +33,16 @@ class AuthController(auth: Authentication, providers: AuthenticationProviders, v
     respond(indexData, indexLinks)
   }
 
-  def cookieMonster = auth { request =>
+  // This allows us to force a reset on a users panda cookie for debug purposes
+  def cookieMonster: Action[AnyContent] = auth { request =>
     providers.userProvider match {
-      case panda: PandaAuthenticationProvider =>{
-        val cookieBatter = panda.readAuthenticatedUser(request).map(user => panda.generateCookie(user.copy(expires = new Date().getTime)))
-        cookieBatter.fold(respond("Me want cookie."))(cookie => respond("Cookies are a sometimes food.").withCookies(cookie))
-      }
+      case panda: PandaAuthenticationProvider =>
+        val cookieBatter = panda.readAuthenticatedUser(request).map(user => panda.generateCookie(user.copy(expires = Instant.now())))
+
+        cookieBatter match {
+          case Some(cookie) => respond("Cookies are a sometimes food.").withCookies(cookie)
+          case None => respond("Me want cookie.")
+        }
       case _ => respond("Me want cookie.")
     }
   }

build.sbt

@@ -90,10 +90,10 @@ val maybeBBCLib: Option[sbt.ProjectReference] = if(bbcBuildProcess) Some(bbcProj
 
 lazy val commonLib = project("common-lib").settings(
   libraryDependencies ++= Seq(
-    "com.gu" %% "editorial-permissions-client" % "4.0.0",
-    "com.gu" %% "pan-domain-auth-play_3-0" % "9.0.0",
+    "com.gu" %% "editorial-permissions-client" % "6.0.3",
+    "com.gu" %% "pan-domain-auth-play_3-0" % "19.0.0",
     "com.amazonaws" % "aws-java-sdk-iam" % awsSdkVersion,
-    "com.amazonaws" % "aws-java-sdk-s3" % awsSdkVersion,
+    "software.amazon.awssdk" % "s3" % awsSdkV2Version,
     "com.amazonaws" % "aws-java-sdk-ec2" % awsSdkVersion,
     "com.amazonaws" % "aws-java-sdk-sqs" % awsSdkVersion,
     "com.amazonaws" % "aws-java-sdk-sns" % awsSdkVersion,

common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala

@@ -1,14 +1,13 @@
 package com.gu.mediaservice.lib
 
-import org.apache.pekko.actor.{Cancellable, Scheduler}
 import com.gu.mediaservice.lib.aws.S3
 import com.gu.mediaservice.lib.config.CommonConfig
 import com.gu.mediaservice.lib.logging.GridLogging
+import org.apache.pekko.actor.{Cancellable, Scheduler}
 import org.joda.time.DateTime
 
-import java.util.concurrent.atomic.AtomicReference
 import java.io.InputStream
-import scala.jdk.CollectionConverters._
+import java.util.concurrent.atomic.AtomicReference
 import scala.concurrent.ExecutionContext
 import scala.concurrent.duration._
 import scala.util.control.NonFatal
@@ -25,15 +24,14 @@ abstract class BaseStore[TStoreKey, TStoreVal](bucket: String, config: CommonCon
   protected def getS3Object(key: String): Option[String] = s3.getObjectAsString(bucket, key)
 
   protected def getLatestS3Stream: Option[InputStream] = {
-    val objects = s3.client
-      .listObjects(bucket).getObjectSummaries.asScala
-      .filterNot(_.getKey == "AMAZON_SES_SETUP_NOTIFICATION")
+    val objects = s3.listObjects(bucket)
+      .filterNot(_.key() == "AMAZON_SES_SETUP_NOTIFICATION")
 
     if (objects.nonEmpty) {
-      val obj = objects.maxBy(_.getLastModified)
-      logger.info(s"Latest key ${obj.getKey} in bucket $bucket")
+      val obj = objects.maxBy(_.lastModified())
+      logger.info(s"Latest key ${obj.key()} in bucket $bucket")
 
-      val stream = s3.client.getObject(bucket, obj.getKey).getObjectContent
+      val stream = s3.getObject(bucket, obj.key())
       Some(stream)
     } else {
       logger.error(s"Bucket $bucket is empty")

common-lib/src/main/scala/com/gu/mediaservice/lib/DateTimeUtils.scala

@@ -1,10 +1,8 @@
 package com.gu.mediaservice.lib
 
 import java.time.format.DateTimeFormatter
-import java.time.{Instant, LocalDateTime, ZoneId, ZonedDateTime}
-import org.joda.time.DateTime
-
 import java.time.temporal.ChronoUnit
+import java.time.{Instant, OffsetDateTime, ZoneId, ZonedDateTime}
 import scala.concurrent.duration.{DurationLong, FiniteDuration}
 import scala.util.Try
 
@@ -17,8 +15,21 @@ object DateTimeUtils {
 
   def toString(instant: Instant): String = instant.atZone(EuropeLondonZone).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)
 
-  // TODO move this to a LocalDateTime
-  def fromValueOrNow(value: Option[String]): DateTime = Try{new DateTime(value.get)}.getOrElse(DateTime.now)
+  private def parseAsInstant(raw: String): Option[Instant] =
+    Try { Instant.parse(raw) }.toOption
+
+  private def parseAsOffset(raw: String): Option[OffsetDateTime] =
+    Try { OffsetDateTime.parse(raw) }.toOption
+
+  private def parseAsZoned(raw: String): Option[ZonedDateTime] =
+    Try { ZonedDateTime.parse(raw) }.toOption
+
+  private def fromValue(value: String): Option[Instant] =
+    parseAsZoned(value).map(_.toInstant) orElse parseAsOffset(value).map(_.toInstant) orElse parseAsInstant(value)
+
+  // TODO move this to a LocalDateTime?
+  def fromValueOrNow(value: Option[String]): Instant =
+    value.flatMap(fromValue).getOrElse(Instant.now)
 
   def timeUntilNextInterval(interval: FiniteDuration, now: ZonedDateTime = DateTimeUtils.now()): FiniteDuration = {
     val nowRoundedDownToTheHour = now.truncatedTo(ChronoUnit.HOURS)

common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala

@@ -1,14 +1,13 @@
 package com.gu.mediaservice.lib
 
-import com.amazonaws.services.s3.model.{DeleteObjectsRequest, MultiObjectDeleteException}
-
-import java.io.File
-import com.gu.mediaservice.lib.config.CommonConfig
 import com.gu.mediaservice.lib.aws.S3Object
+import com.gu.mediaservice.lib.config.CommonConfig
 import com.gu.mediaservice.lib.logging.LogMarker
-import com.gu.mediaservice.model.{MimeType, Png}
-import org.joda.time.DateTime
+import com.gu.mediaservice.model.MimeType
+import software.amazon.awssdk.services.s3.model._
 
+import java.io.File
+import java.time.Instant
 import scala.concurrent.Future
 import scala.jdk.CollectionConverters._
 
@@ -48,20 +47,20 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config
   private def bulkDelete(bucket: String, keys: List[String]): Future[Map[String, Boolean]] = keys match {
     case Nil => Future.successful(Map.empty)
     case _ => Future {
-      try {
-        client.deleteObjects(
-          new DeleteObjectsRequest(bucket).withKeys(keys: _*)
-        )
+      val objectsToDelete = keys.map(key => ObjectIdentifier.builder().key(key).build()).asJava
+      val resp = client.deleteObjects(
+        DeleteObjectsRequest.builder().bucket(bucket).delete(Delete.builder().objects(objectsToDelete).build()).build()
+      )
+      if (resp.hasErrors) {
+        val errorKeys = resp.errors().asScala.map(_.key()).toSet
+        logger.warn(s"Partial failure when deleting images from $bucket: ${resp.errors()}")
+        keys.map { key =>
+          key -> !errorKeys.contains(key)
+        }.toMap
+      } else {
         keys.map { key =>
           key -> true
         }.toMap
-      } catch {
-        case partialFailure: MultiObjectDeleteException =>
-          logger.warn(s"Partial failure when deleting images from $bucket: ${partialFailure.getMessage} ${partialFailure.getErrors}")
-          val errorKeys = partialFailure.getErrors.asScala.map(_.getKey).toSet
-          keys.map { key =>
-            key -> !errorKeys.contains(key)
-          }.toMap
       }
     }
   }
@@ -73,8 +72,14 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config
   def deletePNG(id: String)(implicit logMarker: LogMarker): Future[Unit] = deleteImage(imageBucket, optimisedPngKeyFromId(id))
   def deletePNGs(ids: Set[String]) = bulkDelete(imageBucket, ids.map(optimisedPngKeyFromId).toList)
 
-  def doesOriginalExist(id: String): Boolean =
-    client.doesObjectExist(imageBucket, fileKeyFromId(id))
+  def doesOriginalExist(id: String): Boolean = {
+    try {
+      client.headObject(HeadObjectRequest.builder().bucket(imageBucket).key(fileKeyFromId(id)).build())
+      true
+    } catch {
+      case _: NoSuchKeyException => false
+    }
+  }
 }
 
 sealed trait ImageWrapper {
@@ -95,7 +100,7 @@ sealed trait StorableImage extends ImageWrapper {
 }
 
 case class StorableThumbImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty) extends StorableImage
-case class StorableOriginalImage(id: String, file: File, mimeType: MimeType, lastModified: DateTime, meta: Map[String, String] = Map.empty) extends StorableImage {
+case class StorableOriginalImage(id: String, file: File, mimeType: MimeType, lastModified: Instant, meta: Map[String, String] = Map.empty) extends StorableImage {
   override def toProjectedS3Object(thumbBucket: String): S3Object = S3Object(
     thumbBucket,
     ImageIngestOperations.fileKeyFromId(id),

common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala

@@ -4,11 +4,11 @@ import com.gu.mediaservice.lib.aws.S3
 import com.gu.mediaservice.lib.config.CommonConfig
 import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker}
 import com.gu.mediaservice.model.MimeType
-import org.slf4j.LoggerFactory
+import software.amazon.awssdk.services.s3.model.{DeleteObjectRequest, HeadObjectRequest, ListObjectsRequest}
 
 import java.io.File
-import scala.jdk.CollectionConverters._
 import scala.concurrent.Future
+import scala.jdk.CollectionConverters._
 
 // TODO: If deleteObject fails - we should be catching the errors here to avoid them bubbling to the application
 class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage with GridLogging {
@@ -28,19 +28,19 @@ class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage
   }
 
   def deleteImage(bucket: String, id: String)(implicit logMarker: LogMarker) = Future {
-    client.deleteObject(bucket, id)
+    client.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(id).build())
     logger.info(logMarker, s"Deleted image $id from bucket $bucket")
   }
 
   def deleteVersionedImage(bucket: String, id: String)(implicit logMarker: LogMarker) = Future {
-    val objectVersion = client.getObjectMetadata(bucket, id).getVersionId
-    client.deleteVersion(bucket, id, objectVersion)
+    val objectVersion = client.headObject(HeadObjectRequest.builder().bucket(bucket).key(id).build()).versionId()
+    client.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(id).versionId(objectVersion).build())
     logger.info(logMarker, s"Deleted image $id from bucket $bucket (version: $objectVersion)")
   }
 
   def deleteFolder(bucket: String, id: String)(implicit logMarker: LogMarker) = Future {
-		val files = client.listObjects(bucket, id).getObjectSummaries.asScala
-		files.foreach(file => client.deleteObject(bucket, file.getKey))
+		val files = client.listObjects(ListObjectsRequest.builder().bucket(bucket).prefix(s"$id/").build()).contents().asScala
+		files.foreach(file => client.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(file.key()).build()))
 		logger.info(logMarker, s"Deleting images in folder $id from bucket $bucket")
 	}
 

common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala

@@ -2,6 +2,7 @@ package com.gu.mediaservice.lib.auth
 
 import com.gu.mediaservice.lib.BaseStore
 import com.gu.mediaservice.lib.config.CommonConfig
+import software.amazon.awssdk.services.s3.model.ListObjectsRequest
 
 import scala.jdk.CollectionConverters._
 import scala.concurrent.ExecutionContext
@@ -11,14 +12,13 @@ class KeyStore(bucket: String, config: CommonConfig)(implicit ec: ExecutionConte
 
   def lookupIdentity(key: String): Option[ApiAccessor] = store.get().get(key)
 
-  def findKey(prefix: String): Option[String] = s3.syncFindKey(bucket, prefix)
-
   def update(): Unit = {
     store.set(fetchAll)
   }
 
   private def fetchAll: Map[String, ApiAccessor] = {
-    val keys = s3.client.listObjects(bucket).getObjectSummaries.asScala.map(_.getKey)
+    val listObjects = s3.client.listObjects(ListObjectsRequest.builder().bucket(bucket).build())
+    val keys = listObjects.contents().asScala.map(_.key())
     keys.flatMap(k => getS3Object(k).map(k -> ApiAccessor(_))).toMap
   }
 }