diff --git a/auth/app/auth/AuthController.scala b/auth/app/auth/AuthController.scala index 73c180991a..1e10989e55 100644 --- a/auth/app/auth/AuthController.scala +++ b/auth/app/auth/AuthController.scala @@ -8,9 +8,10 @@ import com.gu.mediaservice.lib.auth.provider.AuthenticationProviders import com.gu.mediaservice.lib.auth.{Authentication, Authorisation, Internal} import com.gu.mediaservice.lib.guardian.auth.PandaAuthenticationProvider import play.api.libs.json.Json -import play.api.mvc.{BaseController, ControllerComponents, Result} +import play.api.mvc.{Action, AnyContent, BaseController, ControllerComponents, Result} import java.net.URI +import java.time.Instant import java.util.Date import scala.concurrent.{ExecutionContext, Future} import scala.util.Try @@ -32,12 +33,16 @@ class AuthController(auth: Authentication, providers: AuthenticationProviders, v respond(indexData, indexLinks) } - def cookieMonster = auth { request => + // This allows us to force a reset on a users panda cookie for debug purposes + def cookieMonster: Action[AnyContent] = auth { request => providers.userProvider match { - case panda: PandaAuthenticationProvider =>{ - val cookieBatter = panda.readAuthenticatedUser(request).map(user => panda.generateCookie(user.copy(expires = new Date().getTime))) - cookieBatter.fold(respond("Me want cookie."))(cookie => respond("Cookies are a sometimes food.").withCookies(cookie)) - } + case panda: PandaAuthenticationProvider => + val cookieBatter = panda.readAuthenticatedUser(request).map(user => panda.generateCookie(user.copy(expires = Instant.now()))) + + cookieBatter match { + case Some(cookie) => respond("Cookies are a sometimes food.").withCookies(cookie) + case None => respond("Me want cookie.") + } case _ => respond("Me want cookie.") } } diff --git a/build.sbt b/build.sbt index 625f3a80dd..d8e8d1da86 100644 --- a/build.sbt +++ b/build.sbt @@ -90,10 +90,10 @@ val maybeBBCLib: Option[sbt.ProjectReference] = if(bbcBuildProcess) Some(bbcProj lazy val commonLib = project("common-lib").settings( libraryDependencies ++= Seq( - "com.gu" %% "editorial-permissions-client" % "4.0.0", - "com.gu" %% "pan-domain-auth-play_3-0" % "9.0.0", + "com.gu" %% "editorial-permissions-client" % "6.0.3", + "com.gu" %% "pan-domain-auth-play_3-0" % "19.0.0", "com.amazonaws" % "aws-java-sdk-iam" % awsSdkVersion, - "com.amazonaws" % "aws-java-sdk-s3" % awsSdkVersion, + "software.amazon.awssdk" % "s3" % awsSdkV2Version, "com.amazonaws" % "aws-java-sdk-ec2" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-sqs" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-sns" % awsSdkVersion, diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala index 0aeb698ccc..fb4bf63e5e 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala @@ -1,14 +1,13 @@ package com.gu.mediaservice.lib -import org.apache.pekko.actor.{Cancellable, Scheduler} import com.gu.mediaservice.lib.aws.S3 import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.GridLogging +import org.apache.pekko.actor.{Cancellable, Scheduler} import org.joda.time.DateTime -import java.util.concurrent.atomic.AtomicReference import java.io.InputStream -import scala.jdk.CollectionConverters._ +import java.util.concurrent.atomic.AtomicReference import scala.concurrent.ExecutionContext import scala.concurrent.duration._ import scala.util.control.NonFatal @@ -25,15 +24,14 @@ abstract class BaseStore[TStoreKey, TStoreVal](bucket: String, config: CommonCon protected def getS3Object(key: String): Option[String] = s3.getObjectAsString(bucket, key) protected def getLatestS3Stream: Option[InputStream] = { - val objects = s3.client - .listObjects(bucket).getObjectSummaries.asScala - .filterNot(_.getKey == "AMAZON_SES_SETUP_NOTIFICATION") + val objects = s3.listObjects(bucket) + .filterNot(_.key() == "AMAZON_SES_SETUP_NOTIFICATION") if (objects.nonEmpty) { - val obj = objects.maxBy(_.getLastModified) - logger.info(s"Latest key ${obj.getKey} in bucket $bucket") + val obj = objects.maxBy(_.lastModified()) + logger.info(s"Latest key ${obj.key()} in bucket $bucket") - val stream = s3.client.getObject(bucket, obj.getKey).getObjectContent + val stream = s3.getObject(bucket, obj.key()) Some(stream) } else { logger.error(s"Bucket $bucket is empty") diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/DateTimeUtils.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/DateTimeUtils.scala index e96094bca1..e276c5d522 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/DateTimeUtils.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/DateTimeUtils.scala @@ -1,10 +1,8 @@ package com.gu.mediaservice.lib import java.time.format.DateTimeFormatter -import java.time.{Instant, LocalDateTime, ZoneId, ZonedDateTime} -import org.joda.time.DateTime - import java.time.temporal.ChronoUnit +import java.time.{Instant, OffsetDateTime, ZoneId, ZonedDateTime} import scala.concurrent.duration.{DurationLong, FiniteDuration} import scala.util.Try @@ -17,8 +15,21 @@ object DateTimeUtils { def toString(instant: Instant): String = instant.atZone(EuropeLondonZone).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME) - // TODO move this to a LocalDateTime - def fromValueOrNow(value: Option[String]): DateTime = Try{new DateTime(value.get)}.getOrElse(DateTime.now) + private def parseAsInstant(raw: String): Option[Instant] = + Try { Instant.parse(raw) }.toOption + + private def parseAsOffset(raw: String): Option[OffsetDateTime] = + Try { OffsetDateTime.parse(raw) }.toOption + + private def parseAsZoned(raw: String): Option[ZonedDateTime] = + Try { ZonedDateTime.parse(raw) }.toOption + + private def fromValue(value: String): Option[Instant] = + parseAsZoned(value).map(_.toInstant) orElse parseAsOffset(value).map(_.toInstant) orElse parseAsInstant(value) + + // TODO move this to a LocalDateTime? + def fromValueOrNow(value: Option[String]): Instant = + value.flatMap(fromValue).getOrElse(Instant.now) def timeUntilNextInterval(interval: FiniteDuration, now: ZonedDateTime = DateTimeUtils.now()): FiniteDuration = { val nowRoundedDownToTheHour = now.truncatedTo(ChronoUnit.HOURS) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala index 0fdd57bb67..1338e2f0ff 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala @@ -1,14 +1,13 @@ package com.gu.mediaservice.lib -import com.amazonaws.services.s3.model.{DeleteObjectsRequest, MultiObjectDeleteException} - -import java.io.File -import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.aws.S3Object +import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.LogMarker -import com.gu.mediaservice.model.{MimeType, Png} -import org.joda.time.DateTime +import com.gu.mediaservice.model.MimeType +import software.amazon.awssdk.services.s3.model._ +import java.io.File +import java.time.Instant import scala.concurrent.Future import scala.jdk.CollectionConverters._ @@ -48,20 +47,20 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config private def bulkDelete(bucket: String, keys: List[String]): Future[Map[String, Boolean]] = keys match { case Nil => Future.successful(Map.empty) case _ => Future { - try { - client.deleteObjects( - new DeleteObjectsRequest(bucket).withKeys(keys: _*) - ) + val objectsToDelete = keys.map(key => ObjectIdentifier.builder().key(key).build()).asJava + val resp = client.deleteObjects( + DeleteObjectsRequest.builder().bucket(bucket).delete(Delete.builder().objects(objectsToDelete).build()).build() + ) + if (resp.hasErrors) { + val errorKeys = resp.errors().asScala.map(_.key()).toSet + logger.warn(s"Partial failure when deleting images from $bucket: ${resp.errors()}") + keys.map { key => + key -> !errorKeys.contains(key) + }.toMap + } else { keys.map { key => key -> true }.toMap - } catch { - case partialFailure: MultiObjectDeleteException => - logger.warn(s"Partial failure when deleting images from $bucket: ${partialFailure.getMessage} ${partialFailure.getErrors}") - val errorKeys = partialFailure.getErrors.asScala.map(_.getKey).toSet - keys.map { key => - key -> !errorKeys.contains(key) - }.toMap } } } @@ -73,8 +72,14 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config def deletePNG(id: String)(implicit logMarker: LogMarker): Future[Unit] = deleteImage(imageBucket, optimisedPngKeyFromId(id)) def deletePNGs(ids: Set[String]) = bulkDelete(imageBucket, ids.map(optimisedPngKeyFromId).toList) - def doesOriginalExist(id: String): Boolean = - client.doesObjectExist(imageBucket, fileKeyFromId(id)) + def doesOriginalExist(id: String): Boolean = { + try { + client.headObject(HeadObjectRequest.builder().bucket(imageBucket).key(fileKeyFromId(id)).build()) + true + } catch { + case _: NoSuchKeyException => false + } + } } sealed trait ImageWrapper { @@ -95,7 +100,7 @@ sealed trait StorableImage extends ImageWrapper { } case class StorableThumbImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty) extends StorableImage -case class StorableOriginalImage(id: String, file: File, mimeType: MimeType, lastModified: DateTime, meta: Map[String, String] = Map.empty) extends StorableImage { +case class StorableOriginalImage(id: String, file: File, mimeType: MimeType, lastModified: Instant, meta: Map[String, String] = Map.empty) extends StorableImage { override def toProjectedS3Object(thumbBucket: String): S3Object = S3Object( thumbBucket, ImageIngestOperations.fileKeyFromId(id), diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala index 7e164f876b..db45fd19f1 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala @@ -4,11 +4,11 @@ import com.gu.mediaservice.lib.aws.S3 import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} import com.gu.mediaservice.model.MimeType -import org.slf4j.LoggerFactory +import software.amazon.awssdk.services.s3.model.{DeleteObjectRequest, HeadObjectRequest, ListObjectsRequest} import java.io.File -import scala.jdk.CollectionConverters._ import scala.concurrent.Future +import scala.jdk.CollectionConverters._ // TODO: If deleteObject fails - we should be catching the errors here to avoid them bubbling to the application class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage with GridLogging { @@ -28,19 +28,19 @@ class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage } def deleteImage(bucket: String, id: String)(implicit logMarker: LogMarker) = Future { - client.deleteObject(bucket, id) + client.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(id).build()) logger.info(logMarker, s"Deleted image $id from bucket $bucket") } def deleteVersionedImage(bucket: String, id: String)(implicit logMarker: LogMarker) = Future { - val objectVersion = client.getObjectMetadata(bucket, id).getVersionId - client.deleteVersion(bucket, id, objectVersion) + val objectVersion = client.headObject(HeadObjectRequest.builder().bucket(bucket).key(id).build()).versionId() + client.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(id).versionId(objectVersion).build()) logger.info(logMarker, s"Deleted image $id from bucket $bucket (version: $objectVersion)") } def deleteFolder(bucket: String, id: String)(implicit logMarker: LogMarker) = Future { - val files = client.listObjects(bucket, id).getObjectSummaries.asScala - files.foreach(file => client.deleteObject(bucket, file.getKey)) + val files = client.listObjects(ListObjectsRequest.builder().bucket(bucket).prefix(s"$id/").build()).contents().asScala + files.foreach(file => client.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(file.key()).build())) logger.info(logMarker, s"Deleting images in folder $id from bucket $bucket") } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala index d3c90bc739..1f0131baff 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala @@ -2,6 +2,7 @@ package com.gu.mediaservice.lib.auth import com.gu.mediaservice.lib.BaseStore import com.gu.mediaservice.lib.config.CommonConfig +import software.amazon.awssdk.services.s3.model.ListObjectsRequest import scala.jdk.CollectionConverters._ import scala.concurrent.ExecutionContext @@ -11,14 +12,13 @@ class KeyStore(bucket: String, config: CommonConfig)(implicit ec: ExecutionConte def lookupIdentity(key: String): Option[ApiAccessor] = store.get().get(key) - def findKey(prefix: String): Option[String] = s3.syncFindKey(bucket, prefix) - def update(): Unit = { store.set(fetchAll) } private def fetchAll: Map[String, ApiAccessor] = { - val keys = s3.client.listObjects(bucket).getObjectSummaries.asScala.map(_.getKey) + val listObjects = s3.client.listObjects(ListObjectsRequest.builder().bucket(bucket).build()) + val keys = listObjects.contents().asScala.map(_.key()) keys.flatMap(k => getS3Object(k).map(k -> ApiAccessor(_))).toMap } } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index dc58e38f8a..5453e36c4a 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -1,18 +1,25 @@ package com.gu.mediaservice.lib.aws -import com.amazonaws.services.s3.model._ -import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder, model} import com.amazonaws.util.IOUtils -import com.amazonaws.{AmazonServiceException, ClientConfiguration} import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch} import com.gu.mediaservice.model._ -import org.joda.time.{DateTime, Duration} +import software.amazon.awssdk.core.ResponseInputStream +import software.amazon.awssdk.core.sync.RequestBody +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.{S3Object => S3ObjectSummary, _} +import software.amazon.awssdk.services.s3.presigner.S3Presigner +import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest import java.io.File import java.net.URI -import scala.jdk.CollectionConverters._ +import java.time.Instant +import scala.concurrent.duration.DurationInt import scala.concurrent.{ExecutionContext, Future} +import scala.jdk.CollectionConverters._ +import scala.jdk.DurationConverters.ScalaDurationOps +import scala.language.implicitConversions case class S3Object(uri: URI, size: Long, metadata: S3Metadata) @@ -25,7 +32,7 @@ object S3Object { def apply(bucket: String, key: String, size: Long, metadata: S3Metadata): S3Object = apply(objectUrl(bucket, key), size, metadata) - def apply(bucket: String, key: String, file: File, mimeType: Option[MimeType], lastModified: Option[DateTime], + def apply(bucket: String, key: String, file: File, mimeType: Option[MimeType], lastModified: Option[Instant], meta: Map[String, String] = Map.empty, cacheControl: Option[String] = None): S3Object = { S3Object( bucket, @@ -46,68 +53,64 @@ object S3Object { case class S3Metadata(userMetadata: Map[String, String], objectMetadata: S3ObjectMetadata) object S3Metadata { - def apply(meta: ObjectMetadata): S3Metadata = { + def fromHeadObjectResponse(hor: HeadObjectResponse): S3Metadata = { S3Metadata( - meta.getUserMetadata.asScala.toMap, + hor.metadata().asScala.toMap, S3ObjectMetadata( - contentType = Option(meta.getContentType).filterNot(_.toLowerCase == "application/octet-stream").map(MimeType.apply), - cacheControl = Option(meta.getCacheControl), - lastModified = Option(meta.getLastModified).map(new DateTime(_)) + contentType = Option(hor.contentType()).filterNot(_.toLowerCase == "application/octet-stream").map(MimeType.apply), + cacheControl = Option(hor.cacheControl()), + lastModified = Option(hor.lastModified()) ) ) } } -case class S3ObjectMetadata(contentType: Option[MimeType], cacheControl: Option[String], lastModified: Option[DateTime]) +case class S3ObjectMetadata(contentType: Option[MimeType], cacheControl: Option[String], lastModified: Option[Instant]) -class S3(config: CommonConfig) extends GridLogging with ContentDisposition with RoundedExpiration { +class S3(config: CommonConfig) extends GridLogging with ContentDisposition with RoundedExpiration with S3Ops { type Bucket = String type Key = String type UserMetadata = Map[String, String] - lazy val client: AmazonS3 = S3Ops.buildS3Client(config) + lazy val client: S3Client = S3Ops.buildS3Client(config) + lazy val presigner: S3Presigner = S3Presigner.builder().s3Client(client).build() - def signUrl(bucket: Bucket, url: URI, image: Image, expiration: DateTime = cachableExpiration(), imageType: ImageFileType = Source): String = { + def signUrl(bucket: Bucket, url: URI, image: Image, imageType: ImageFileType = Source): String = { // get path and remove leading `/` val key: Key = url.getPath.drop(1) val contentDisposition = getContentDisposition(image, imageType, config.shortenDownloadFilename) - val headers = new ResponseHeaderOverrides().withContentDisposition(contentDisposition) + val objReq = GetObjectRequest.builder().bucket(bucket).key(key).responseContentDisposition(contentDisposition).build() + val requestt = GetObjectPresignRequest.builder().getObjectRequest(objReq).signatureDuration(10.minutes.toJava).build() - val request = new GeneratePresignedUrlRequest(bucket, key).withExpiration(expiration.toDate).withResponseHeaders(headers) - client.generatePresignedUrl(request).toExternalForm + presigner.presignGetObject(requestt).url().toExternalForm } - def getObject(bucket: Bucket, url: URI): model.S3Object = { + def getObject(bucket: Bucket, url: URI): ResponseInputStream[GetObjectResponse] = { // get path and remove leading `/` val key: Key = url.getPath.drop(1) - client.getObject(new GetObjectRequest(bucket, key)) + client.getObject(GetObjectRequest.builder().bucket(bucket).key(key).build()) } def getObjectAsString(bucket: Bucket, key: String): Option[String] = { - val content = client.getObject(new GetObjectRequest(bucket, key)) - val stream = content.getObjectContent + + val content = client.getObject(GetObjectRequest.builder().bucket(bucket).key(key).build()) try { - Some(IOUtils.toString(stream).trim) + Some(IOUtils.toString(content).trim) } catch { - case e: AmazonServiceException if e.getErrorCode == "NoSuchKey" => + case _: NoSuchKeyException => logger.warn(s"Cannot find key: $key in bucket: $bucket") None } finally { - stream.close() + content.close() } } def store(bucket: Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None) (implicit ex: ExecutionContext, logMarker: LogMarker): Future[S3Object] = Future { - val metadata = new ObjectMetadata - mimeType.foreach(m => metadata.setContentType(m.name)) - cacheControl.foreach(metadata.setCacheControl) - metadata.setUserMetadata(meta.asJava) - val fileMarkers = Map( "bucket" -> bucket, "fileName" -> id, @@ -115,26 +118,32 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with ) val markers = logMarker ++ fileMarkers - val req = new PutObjectRequest(bucket, id, file).withMetadata(metadata) + val req = { + val builder = PutObjectRequest.builder().bucket(bucket).key(id).metadata(meta.asJava) + mimeType.foreach(mime => builder.contentType(mime.name)) + cacheControl.foreach(builder.cacheControl) + builder.build() + } + Stopwatch(s"S3 client.putObject ($req)"){ - client.putObject(req) + client.putObject(req, RequestBody.fromFile(file)) // once we've completed the PUT read back to ensure that we are returning reality - val metadata = client.getObjectMetadata(bucket, id) - S3Object(bucket, id, metadata.getContentLength, S3Metadata(metadata)) + val response = client.headObject(HeadObjectRequest.builder().bucket(bucket).key(id).build()) + S3Object(bucket, id, response.contentLength(), S3Metadata.fromHeadObjectResponse(response)) }(markers) } def storeIfNotPresent(bucket: Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None) (implicit ex: ExecutionContext, logMarker: LogMarker): Future[S3Object] = { - Future{ - Some(client.getObjectMetadata(bucket, id)) + Future { + Some(client.headObject(HeadObjectRequest.builder().bucket(bucket).key(id).build())) }.recover { // translate this exception into the object not existing - case as3e:AmazonS3Exception if as3e.getStatusCode == 404 => None + case as3e: S3Exception if as3e.statusCode() == 404 => None }.flatMap { case Some(objectMetadata) => logger.info(logMarker, s"Skipping storing of S3 file $id as key is already present in bucket $bucket") - Future.successful(S3Object(bucket, id, objectMetadata.getContentLength, S3Metadata(objectMetadata))) + Future.successful(S3Object(bucket, id, objectMetadata.contentLength(), S3Metadata.fromHeadObjectResponse(objectMetadata))) case None => store(bucket, id, file, mimeType, meta, cacheControl) } @@ -143,30 +152,69 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with def list(bucket: Bucket, prefixDir: String) (implicit ex: ExecutionContext): Future[List[S3Object]] = Future { - val req = new ListObjectsRequest().withBucketName(bucket).withPrefix(s"$prefixDir/") + val req = ListObjectsRequest.builder().bucket(bucket).prefix(s"$prefixDir/").build() val listing = client.listObjects(req) - val summaries = listing.getObjectSummaries.asScala - summaries.map(summary => (summary.getKey, summary)).foldLeft(List[S3Object]()) { + val summaries = listing.contents().asScala + summaries.map(summary => (summary.key(), summary)).foldLeft(List[S3Object]()) { case (memo: List[S3Object], (key: String, summary: S3ObjectSummary)) => - S3Object(bucket, key, summary.getSize, getMetadata(bucket, key)) :: memo + S3Object(bucket, key, summary.size(), getMetadata(bucket, key)) :: memo } } def getMetadata(bucket: Bucket, key: Key): S3Metadata = { - val meta = client.getObjectMetadata(bucket, key) - S3Metadata(meta) + val resp = client.headObject(HeadObjectRequest.builder().bucket(bucket).key(key).build()) + S3Metadata.fromHeadObjectResponse(resp) } def getUserMetadata(bucket: Bucket, key: Key): Map[Bucket, Bucket] = - client.getObjectMetadata(bucket, key).getUserMetadata.asScala.toMap + client.headObject(HeadObjectRequest.builder().bucket(bucket).key(key).build()).metadata().asScala.toMap +} + +trait S3Ops { + val client: S3Client + + def doesObjectExist(bucket: String, key: String): Boolean = + try { + client.headObject(HeadObjectRequest.builder().bucket(bucket).key(key).build()) + true + } catch { + case _: NoSuchKeyException => false + } - def syncFindKey(bucket: Bucket, prefixName: String): Option[Key] = { - val req = new ListObjectsRequest().withBucketName(bucket).withPrefix(s"$prefixName-") - val listing = client.listObjects(req) - val summaries = listing.getObjectSummaries.asScala - summaries.headOption.map(_.getKey) + def getObject(bucket: String, key: String): ResponseInputStream[GetObjectResponse] = { + client.getObject(GetObjectRequest.builder().bucket(bucket).key(key).build()) } + def copyObject(fromBucket: String, fromKey: String, toBucket: String, toKey: String): CopyObjectResponse = { + client.copyObject(CopyObjectRequest.builder() + .sourceBucket(fromBucket).sourceKey(fromKey) + .destinationBucket(toBucket).destinationKey(toKey) + .build() + ) + } + + def listObjects(bucket: String): Seq[S3ObjectSummary] = { + client.listObjectsV2( + ListObjectsV2Request.builder().bucket(bucket).build() + ).contents().asScala.toList + } + + def listObjects(bucket: String, prefix: String): Seq[S3ObjectSummary] = { + client.listObjectsV2( + ListObjectsV2Request.builder().bucket(bucket).prefix(prefix).build() + ).contents().asScala.toList + } + + def deleteObject(bucket: String, key: String): DeleteObjectResponse = { + client.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(key).build()) + } + + def putObject(bucket: String, key: String, contents: String): PutObjectResponse = { + client.putObject( + PutObjectRequest.builder().bucket(bucket).key(key).build(), + RequestBody.fromString(contents) + ) + } } object S3Ops { @@ -174,16 +222,25 @@ object S3Ops { // TODO: Make this region aware - i.e. RegionUtils.getRegion(region).getServiceEndpoint(AmazonS3.ENDPOINT_PREFIX) val s3Endpoint = "s3.amazonaws.com" - def buildS3Client(config: CommonConfig, localstackAware: Boolean = true, maybeRegionOverride: Option[String] = None): AmazonS3 = { + def apply(_client: S3Client): S3Ops = { + new S3Ops { + override val client: S3Client = _client + } + } + def buildS3Client( + config: CommonConfig, + localstackAware: Boolean = true, + maybeRegionOverride: Option[Region] = None + ): S3Client = { val builder = config.awsLocalEndpoint match { case Some(_) if config.isDev => // TODO revise closer to the time of deprecation https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/ // `withPathStyleAccessEnabled` for localstack // see https://github.com/localstack/localstack/issues/1512 - AmazonS3ClientBuilder.standard().withPathStyleAccessEnabled(true) - case _ => AmazonS3ClientBuilder.standard() + S3Client.builder().forcePathStyle(true) + case _ => S3Client.builder() } - config.withAWSCredentials(builder, localstackAware, maybeRegionOverride).build() + config.withAWSCredentialsV2(builder, localstackAware, maybeRegionOverride).build() } } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/formatting/package.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/formatting/package.scala index 2d69d15575..f16f93c2b9 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/formatting/package.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/formatting/package.scala @@ -3,6 +3,7 @@ package com.gu.mediaservice.lib import org.joda.time.DateTime import org.joda.time.format.{DateTimeFormatter, DateTimeFormatterBuilder, ISODateTimeFormat} +import java.time.Instant import scala.concurrent.duration.Duration import scala.util.Try @@ -26,6 +27,9 @@ package object formatting { def printDateTime(date: DateTime): String = date.toString() def printOptDateTime(date: Option[DateTime]): Option[String] = date.map(printDateTime) + def printInstant(instant: Instant): String = instant.toString + def printOptInstant(instant: Option[Instant]): Option[String] = instant.map(printInstant) + // Only use this on dates that have been confidently written using printDateTime def unsafeParseDateTime(string: String): DateTime = dateTimeFormat.parseDateTime(string) diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/DateTimeUtilsTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/DateTimeUtilsTest.scala index 2cce5b07d7..384533df4a 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/DateTimeUtilsTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/DateTimeUtilsTest.scala @@ -1,6 +1,5 @@ package com.gu.mediaservice.lib -import org.joda.time.DateTime import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should.Matchers @@ -11,20 +10,17 @@ import scala.concurrent.duration.{DurationInt, DurationLong, FiniteDuration} class DateTimeUtilsTest extends AnyFunSpec with Matchers { it ("should convert a string to a DateTime") { - val dateString = "2020-01-01T12:34:56.000Z" + val dateString = "2020-01-01T12:34:56Z" val actual = DateTimeUtils.fromValueOrNow(Some(dateString)) - actual shouldBe a[DateTime] actual.toString shouldBe dateString } it ("should handle an invalid date string input and return a DateTime") { - val actual = DateTimeUtils.fromValueOrNow(Some("nonsense")) - actual shouldBe a[DateTime] + noException should be thrownBy DateTimeUtils.fromValueOrNow(Some("nonsense")) } it ("should return a date with no input") { - val actual = DateTimeUtils.fromValueOrNow(None) - actual shouldBe a[DateTime] + noException should be thrownBy DateTimeUtils.fromValueOrNow(None) } it ("should return the time until the next instance of the interval relative to the hour"){ diff --git a/image-loader/app/controllers/ImageLoaderController.scala b/image-loader/app/controllers/ImageLoaderController.scala index ab44b57a88..d56019ba03 100644 --- a/image-loader/app/controllers/ImageLoaderController.scala +++ b/image-loader/app/controllers/ImageLoaderController.scala @@ -1,9 +1,5 @@ package controllers -import org.apache.pekko.Done -import org.apache.pekko.stream.Materializer -import org.apache.pekko.stream.scaladsl.Source -import com.amazonaws.services.s3.AmazonS3 import com.amazonaws.services.sqs.model.{Message => SQSMessage} import com.amazonaws.util.IOUtils import com.drew.imaging.ImageProcessingException @@ -14,24 +10,29 @@ import com.gu.mediaservice.lib.argo.model.Link import com.gu.mediaservice.lib.auth.Authentication.OnBehalfOfPrincipal import com.gu.mediaservice.lib.auth._ import com.gu.mediaservice.lib.aws.{S3Ops, SimpleSqsMessageConsumer, SqsHelpers} -import com.gu.mediaservice.lib.formatting.printDateTime +import com.gu.mediaservice.lib.formatting.printInstant import com.gu.mediaservice.lib.logging.{FALLBACK, LogMarker, MarkerMap} import com.gu.mediaservice.lib.play.RequestLoggingFilter import com.gu.mediaservice.lib.{DateTimeUtils, ImageIngestOperations, ImageStorageProps} import com.gu.mediaservice.model.{UnsupportedMimeTypeException, UploadInfo} -import org.scanamo.{ConditionNotMet, ScanamoError} import lib.FailureResponse.Response +import lib._ import lib.imaging.{MimeTypeDetection, NoSuchImageExistsInS3, UserImageLoaderException} import lib.storage.{ImageLoaderStore, S3FileDoesNotExistException} -import lib._ import model.upload.UploadRequest import model.{Projector, QuarantineUploader, S3FileExtractedMetadata, S3IngestObject, StatusType, UploadStatus, UploadStatusRecord, UploadStatusUri, Uploader} +import org.apache.pekko.Done +import org.apache.pekko.stream.Materializer +import org.apache.pekko.stream.scaladsl.Source +import org.scanamo.{ConditionNotMet, ScanamoError} import play.api.data.Form import play.api.data.Forms._ import play.api.inject.ApplicationLifecycle import play.api.libs.json.Json import play.api.mvc._ +import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.cloudwatch.model.Dimension +import software.amazon.awssdk.services.s3.model.{GetObjectRequest, HeadObjectRequest, NoSuchKeyException} import java.io.{File, FileOutputStream} import java.net.URI @@ -159,7 +160,7 @@ class ImageLoaderController(auth: Authentication, val approximateReceiveCount = getApproximateReceiveCount(sqsMessage) - if(config.maybeUploadLimitInBytes.exists(_ < s3IngestObject.contentLength)){ + if (config.maybeUploadLimitInBytes.exists(_ < s3IngestObject.contentLength)){ val errorMessage = s"File size exceeds the maximum allowed size (${config.maybeUploadLimitInBytes.get / 1_000_000}MB). Moving to fail bucket." logger.warn(logMarker, errorMessage) store.moveObjectToFailedBucket(s3IngestObject.key) @@ -170,8 +171,7 @@ class ImageLoaderController(auth: Authentication, } metrics.failedIngestsFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions) Future.unit - } - else if (approximateReceiveCount > 2) { + } else if (approximateReceiveCount > 2) { metrics.abandonedMessagesFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions) val errorMessage = s"File processing has been attempted $approximateReceiveCount times. Moving to fail bucket." logger.warn(logMarker, errorMessage) @@ -183,10 +183,11 @@ class ImageLoaderController(auth: Authentication, } Future.unit } else { - attemptToProcessIngestedFile(s3IngestObject, isUiUpload)(logMarker) map { digestedFile => + attemptToProcessIngestedFile(s3IngestObject, isUiUpload)(logMarker) flatMap { digestedFile => metrics.successfulIngestsFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions) logger.info(logMarker, s"Successfully processed image ${digestedFile.file.getName}") store.deleteObjectFromIngestBucket(s3IngestObject.key) + Future.unit } recover { case _: UnsupportedMimeTypeException => metrics.failedIngestsFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions) @@ -295,7 +296,7 @@ class ImageLoaderController(auth: Authentication, val uploadStatus = if(config.maybeQuarantineBucket.isDefined) StatusType.Pending else StatusType.Completed val uploadExpiry = Instant.now.getEpochSecond + config.uploadStatusExpiry.toSeconds - val record = UploadStatusRecord(req.body.digest, filename, uploadedByToRecord, printDateTime(uploadTimeToRecord), identifiers, uploadStatus, None, uploadExpiry) + val record = UploadStatusRecord(req.body.digest, filename, uploadedByToRecord, printInstant(uploadTimeToRecord), identifiers, uploadStatus, None, uploadExpiry) val result = for { uploadRequest <- uploader.loadFile( req.body, @@ -558,7 +559,7 @@ class ImageLoaderController(auth: Authentication, } } - lazy val replicaS3: AmazonS3 = S3Ops.buildS3Client(config, maybeRegionOverride = Some("us-west-1")) + private lazy val replicaS3 = S3Ops.buildS3Client(config, maybeRegionOverride = Some(Region.US_WEST_1)) private case class RestoreFromReplicaForm(imageId: String) def restoreFromReplica: Action[AnyContent] = AuthenticatedAndAuthorised.async { implicit request => @@ -575,20 +576,28 @@ class ImageLoaderController(auth: Authentication, "requestId" -> RequestLoggingFilter.getRequestId(request) ) + def doesObjectExist(bucket: String, key: String) = + try { + replicaS3.headObject(HeadObjectRequest.builder().bucket(bucket).key(key).build()) + true + } catch { + case _: NoSuchKeyException => false + } + Future { config.maybeImageReplicaBucket match { case _ if store.doesOriginalExist(imageId) => Future.successful(Conflict("Image already exists in main bucket")) case None => Future.successful(NotImplemented("No replica bucket configured")) - case Some(replicaBucket) if replicaS3.doesObjectExist(replicaBucket, fileKeyFromId(imageId)) => + case Some(replicaBucket) if doesObjectExist(replicaBucket, fileKeyFromId(imageId)) => val s3Key = fileKeyFromId(imageId) logger.info(logMarker, s"Restoring image $imageId from replica bucket $replicaBucket (key: $s3Key)") - val replicaObject = replicaS3.getObject(replicaBucket, s3Key) - val metadata = S3FileExtractedMetadata(replicaObject.getObjectMetadata) - val stream = replicaObject.getObjectContent + val replicaObject = replicaS3.getObject(GetObjectRequest.builder().bucket(replicaBucket).key(s3Key).build()) + val metadata = S3FileExtractedMetadata(replicaObject.response()) + val stream = replicaObject val tempFile = createTempFile(s"restoringReplica-$imageId") val fos = new FileOutputStream(tempFile) try { diff --git a/image-loader/app/lib/ImageLoaderStore.scala b/image-loader/app/lib/ImageLoaderStore.scala index fefb71177c..6823d4fe31 100644 --- a/image-loader/app/lib/ImageLoaderStore.scala +++ b/image-loader/app/lib/ImageLoaderStore.scala @@ -1,32 +1,33 @@ package lib.storage -import com.amazonaws.HttpMethod -import com.amazonaws.services.s3.model.{AmazonS3Exception, GeneratePresignedUrlRequest, S3Object} import lib.ImageLoaderConfig import com.gu.mediaservice.lib import com.gu.mediaservice.lib.logging.LogMarker +import software.amazon.awssdk.core.ResponseInputStream +import software.amazon.awssdk.services.s3.model.{DeleteObjectResponse, GetObjectResponse, NoSuchKeyException, PutObjectRequest} +import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest import java.io.File -import java.time.ZonedDateTime -import java.util.Date +import java.time.{Duration, Instant, ZonedDateTime} +import scala.jdk.CollectionConverters._ class S3FileDoesNotExistException extends Exception() class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperations(config.imageBucket, config.thumbnailBucket, config) { - private def handleNotFound[T](key: String)(doWork: => T)(loggingIfNotFound: => Unit): T = { + private def handleNotFound[T](doWork: => T)(loggingIfNotFound: => Unit): T = { try { doWork } catch { - case e: AmazonS3Exception if e.getStatusCode == 404 || e.getStatusCode == 403 => + case _: NoSuchKeyException => loggingIfNotFound throw new S3FileDoesNotExistException case other: Throwable => throw other } } - def getS3Object(key: String)(implicit logMarker: LogMarker): S3Object = handleNotFound(key) { - client.getObject(config.maybeIngestBucket.get, key) + def getS3Object(key: String)(implicit logMarker: LogMarker): ResponseInputStream[GetObjectResponse] = handleNotFound { + getObject(config.maybeIngestBucket.get, key) } { logger.error(logMarker, s"Attempted to read $key from ingest bucket, but it does not exist.") } @@ -41,29 +42,34 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati ) } - def generatePreSignedUploadUrl(filename: String, expiration: ZonedDateTime, uploadedBy: String, mediaId: String): String = { - val request = new GeneratePresignedUrlRequest( - config.maybeBucketForUIUploads.get, // bucket - s"$uploadedBy/$filename", // key - ) - .withMethod(HttpMethod.PUT) - .withExpiration(Date.from(expiration.toInstant)); + def generatePreSignedUploadUrl( + filename: String, + expiration: ZonedDateTime, + uploadedBy: String, + mediaId: String + ): String = { + val putObjectRequest = PutObjectRequest.builder() + .bucket(config.maybeBucketForUIUploads.get) + .key(s"$uploadedBy/$filename") + .metadata(Map("media-id" -> mediaId).asJava) + .build() + val presignRequest = PutObjectPresignRequest.builder() + .putObjectRequest(putObjectRequest) + .signatureDuration(Duration.between(Instant.now, expiration)) + .build() - // sent by the client in manager.js - request.putCustomRequestHeader("x-amz-meta-media-id", mediaId) - - client.generatePresignedUrl(request).toString + presigner.presignPutObject(presignRequest).url().toExternalForm } - def moveObjectToFailedBucket(key: String)(implicit logMarker: LogMarker) = handleNotFound(key){ - client.copyObject(config.maybeIngestBucket.get, key, config.maybeFailBucket.get, key) + def moveObjectToFailedBucket(key: String)(implicit logMarker: LogMarker): DeleteObjectResponse = handleNotFound { + copyObject(config.maybeIngestBucket.get, key, config.maybeFailBucket.get, key) deleteObjectFromIngestBucket(key) } { logger.warn(logMarker, s"Attempted to copy $key from ingest bucket to fail bucket, but it does not exist.") } - def deleteObjectFromIngestBucket(key: String)(implicit logMarker: LogMarker) = handleNotFound(key) { - client.deleteObject(config.maybeIngestBucket.get,key) + def deleteObjectFromIngestBucket(key: String)(implicit logMarker: LogMarker): DeleteObjectResponse = handleNotFound { + deleteObject(config.maybeIngestBucket.get,key) } { logger.warn(logMarker, s"Attempted to delete $key from ingest bucket, but it does not exist.") } diff --git a/image-loader/app/model/Projector.scala b/image-loader/app/model/Projector.scala index 20b222382a..5ebe10a53f 100644 --- a/image-loader/app/model/Projector.scala +++ b/image-loader/app/model/Projector.scala @@ -1,54 +1,51 @@ package model -import java.io.{File, FileOutputStream} -import com.amazonaws.services.s3.AmazonS3 -import com.gu.mediaservice.{GridClient, ImageDataMerger} -import com.gu.mediaservice.lib.auth.Authentication -import com.amazonaws.services.s3.model.{GetObjectRequest, ObjectMetadata, S3Object => AwsS3Object} import com.gu.mediaservice.lib.ImageIngestOperations.{fileKeyFromId, optimisedPngKeyFromId} -import com.gu.mediaservice.lib.{ImageIngestOperations, ImageStorageProps, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage} -import com.gu.mediaservice.lib.aws.{Embedder, EmbedderMessage, S3Ops} +import com.gu.mediaservice.lib.auth.Authentication +import com.gu.mediaservice.lib.aws.{Embedder, S3Ops} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch} import com.gu.mediaservice.lib.net.URI +import com.gu.mediaservice.lib._ import com.gu.mediaservice.model.{Image, MimeType, UploadInfo} +import com.gu.mediaservice.{GridClient, ImageDataMerger} import lib.imaging.{MimeTypeDetection, NoSuchImageExistsInS3} import lib.{DigestedFile, ImageLoaderConfig} import model.upload.UploadRequest import org.apache.commons.io.IOUtils -import org.joda.time.{DateTime, DateTimeZone} -import play.api.libs.ws.WSRequest -import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse +import _root_.play.api.libs.ws.WSRequest +import software.amazon.awssdk.services.s3.model.GetObjectResponse -import java.nio.file.Path -import scala.jdk.CollectionConverters._ +import java.io.{File, FileOutputStream, InputStream} +import java.time.{Instant, OffsetDateTime} import scala.concurrent.duration.Duration import scala.concurrent.{Await, ExecutionContext, Future} +import scala.jdk.CollectionConverters._ object Projector { import Uploader.toImageUploadOpsCfg def apply(config: ImageLoaderConfig, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, maybeEmbedder: Option[Embedder])(implicit ec: ExecutionContext): Projector - = new Projector(toImageUploadOpsCfg(config), S3Ops.buildS3Client(config), imageOps, processor, auth, maybeEmbedder) + = new Projector(toImageUploadOpsCfg(config), S3Ops(S3Ops.buildS3Client(config)), imageOps, processor, auth, maybeEmbedder) } case class S3FileExtractedMetadata( uploadedBy: String, - uploadTime: DateTime, + uploadTime: Instant, uploadFileName: Option[String], identifiers: Map[String, String] ) object S3FileExtractedMetadata { - def apply(s3ObjectMetadata: ObjectMetadata): S3FileExtractedMetadata = { - val lastModified = new DateTime(s3ObjectMetadata.getLastModified) - val userMetadata = s3ObjectMetadata.getUserMetadata.asScala.toMap + def apply(s3ObjectMetadata: GetObjectResponse): S3FileExtractedMetadata = { + val lastModified = s3ObjectMetadata.lastModified() + val userMetadata = s3ObjectMetadata.metadata().asScala.toMap apply(lastModified, userMetadata) } - def apply(lastModified: DateTime, userMetadata: Map[String, String]): S3FileExtractedMetadata = { + def apply(lastModified: Instant, userMetadata: Map[String, String]): S3FileExtractedMetadata = { val fileUserMetadata = userMetadata.map { case (key, value) => // Fix up the contents of the metadata. ( @@ -61,7 +58,7 @@ object S3FileExtractedMetadata { } val uploadedBy = fileUserMetadata.getOrElse(ImageStorageProps.uploadedByMetadataKey, "re-ingester") - val uploadedTimeRaw = fileUserMetadata.get(ImageStorageProps.uploadTimeMetadataKey).map(new DateTime(_).withZone(DateTimeZone.UTC)) + val uploadedTimeRaw = fileUserMetadata.get(ImageStorageProps.uploadTimeMetadataKey).map(OffsetDateTime.parse).map(_.toInstant) val uploadTime = uploadedTimeRaw.getOrElse(lastModified) val identifiers = fileUserMetadata.filter{ case (key, _) => key.startsWith(ImageStorageProps.identifierMetadataKeyPrefix) @@ -81,7 +78,7 @@ object S3FileExtractedMetadata { } class Projector(config: ImageUploadOpsCfg, - s3: AmazonS3, + s3: S3Ops, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, @@ -104,7 +101,7 @@ class Projector(config: ImageUploadOpsCfg, try { val digestedFile = getSrcFileDigestForProjection(s3Source, imageId, tempFile) - val extractedS3Meta = S3FileExtractedMetadata(s3Source.getObjectMetadata) + val extractedS3Meta = S3FileExtractedMetadata(s3Source.response()) val finalImageFuture = projectImage(digestedFile, extractedS3Meta, gridClient, onBehalfOfFn) val finalImage = Await.result(finalImageFuture, Duration.Inf) @@ -116,10 +113,10 @@ class Projector(config: ImageUploadOpsCfg, } } - private def getSrcFileDigestForProjection(s3Src: AwsS3Object, imageId: String, tempFile: File) = { + private def getSrcFileDigestForProjection(s3Src: InputStream, imageId: String, tempFile: File) = { val fos = new FileOutputStream(tempFile) try { - IOUtils.copy(s3Src.getObjectContent, fos) + IOUtils.copy(s3Src, fos) DigestedFile(tempFile, imageId) } finally { fos.close() @@ -159,11 +156,11 @@ class Projector(config: ImageUploadOpsCfg, class ImageUploadProjectionOps(config: ImageUploadOpsCfg, imageOps: ImageOperations, processor: ImageProcessor, - s3: AmazonS3, + s3: S3Ops, maybeEmbedder: Option[Embedder], ) extends GridLogging { - import Uploader.{fromUploadRequestShared, toMetaMap} + import Uploader.fromUploadRequestShared def projectImageFromUploadRequest(uploadRequest: UploadRequest) @@ -216,10 +213,10 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg, logger.warn(logMarker, s"image did not exist in bucket $bucket at key $key") Future.successful(None) // falls back to creating from original file case true => - val obj = s3.getObject(new GetObjectRequest(bucket, key)) + val obj = s3.getObject(bucket, key) val fos = new FileOutputStream(outFile) try { - IOUtils.copy(obj.getObjectContent, fos) + IOUtils.copy(obj, fos) } finally { fos.close() obj.close() diff --git a/image-loader/app/model/S3IngestObject.scala b/image-loader/app/model/S3IngestObject.scala index 7621ac2067..d59a3c10fb 100644 --- a/image-loader/app/model/S3IngestObject.scala +++ b/image-loader/app/model/S3IngestObject.scala @@ -4,6 +4,7 @@ import com.gu.mediaservice.lib.ImageStorageProps import com.gu.mediaservice.lib.logging.LogMarker import lib.storage.ImageLoaderStore +import java.time.Instant import scala.jdk.CollectionConverters._ case class S3IngestObject ( @@ -11,7 +12,7 @@ case class S3IngestObject ( uploadedBy: String, filename:String, maybeMediaIdFromUiUpload: Option[String], - uploadTime: java.util.Date, + uploadTime: Instant, contentLength: Long, getInputStream: () => java.io.InputStream, identifiers: Map[String, String] = Map.empty @@ -24,17 +25,18 @@ object S3IngestObject { val keyParts = key.split("/") val s3Object = store.getS3Object(key) - val metadata = s3Object.getObjectMetadata + val response = s3Object.response() + val metadata = response.metadata().asScala S3IngestObject( key, uploadedBy = keyParts.head, filename = keyParts.last, - maybeMediaIdFromUiUpload = metadata.getUserMetadata.asScala.get("media-id"), // set by the client in upload in manager.js - uploadTime = metadata.getLastModified, - contentLength = metadata.getContentLength, - getInputStream = () => s3Object.getObjectContent, - identifiers = metadata.getUserMetadata.asScala.collect{ + maybeMediaIdFromUiUpload = metadata.get("media-id"), // set by the client in upload in manager.js + uploadTime = response.lastModified(), + contentLength = response.contentLength(), + getInputStream = () => s3Object, + identifiers = metadata.collect { case (key, value) if key.startsWith(ImageStorageProps.identifierMetadataKeyPrefix) => key.stripPrefix(ImageStorageProps.identifierMetadataKeyPrefix) -> value }.toMap diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala index aaa811e52c..f9c299c774 100644 --- a/image-loader/app/model/Uploader.scala +++ b/image-loader/app/model/Uploader.scala @@ -1,15 +1,10 @@ package model -import com.gu.mediaservice.{GridClient, ImageDataMerger} import com.gu.mediaservice.lib.Files.createTempFile import com.gu.mediaservice.lib.ImageIngestOperations.fileKeyFromId - -import java.io.File -import java.nio.file.{Files, Path} import com.gu.mediaservice.lib.argo.ArgoHelpers import com.gu.mediaservice.lib.auth.Authentication -import com.gu.mediaservice.lib.{BrowserViewableImage, ImageStorageProps, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage} -import com.gu.mediaservice.lib.aws.{Embedder, EmbedderMessage, S3Object, S3Vectors, UpdateMessage} +import com.gu.mediaservice.lib.aws.{Embedder, EmbedderMessage, S3Object, UpdateMessage} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.formatting._ import com.gu.mediaservice.lib.imaging.ImageOperations @@ -17,19 +12,22 @@ import com.gu.mediaservice.lib.imaging.ImageOperations.{optimisedMimeType, thumb import com.gu.mediaservice.lib.logging._ import com.gu.mediaservice.lib.metadata.{FileMetadataHelper, ImageMetadataConverter} import com.gu.mediaservice.lib.net.URI +import com.gu.mediaservice.lib._ import com.gu.mediaservice.model._ import com.gu.mediaservice.syntax.MessageSubjects -import lib.{DigestedFile, ImageLoaderConfig, Notifications} +import com.gu.mediaservice.{GridClient, ImageDataMerger} import lib.imaging.{FileMetadataReader, MimeTypeDetection} import lib.storage.ImageLoaderStore +import lib.{DigestedFile, ImageLoaderConfig, Notifications} import model.Uploader.{fromUploadRequestShared, toImageUploadOpsCfg} import model.upload.{OptimiseOps, OptimiseWithPngQuant, UploadRequest} import org.joda.time.DateTime -import play.api.libs.json.{JsObject, Json} -import play.api.libs.ws.WSRequest -import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse +import _root_.play.api.libs.json.Json +import _root_.play.api.libs.ws.WSRequest -import scala.collection.compat._ +import java.io.File +import java.nio.file.Files +import java.time.Instant import scala.concurrent.{ExecutionContext, Future} case class ImageUpload(uploadRequest: UploadRequest, image: Image) @@ -40,24 +38,24 @@ case object ImageUpload { fileMetadata: FileMetadata, metadata: ImageMetadata): Image = { val usageRights = NoRights Image( - uploadRequest.imageId, - uploadRequest.uploadTime, - uploadRequest.uploadedBy, - None, - Some(uploadRequest.uploadTime), - uploadRequest.identifiers, - uploadRequest.uploadInfo, - source, - Some(thumbnail), - png, - fileMetadata, - None, - metadata, - metadata, - usageRights, - usageRights, - List(), - List(), + id = uploadRequest.imageId, + uploadTime = new DateTime(uploadRequest.uploadTime.toEpochMilli), + uploadedBy = uploadRequest.uploadedBy, + softDeletedMetadata = None, + lastModified = Some(new DateTime(uploadRequest.uploadTime.toEpochMilli)), + identifiers = uploadRequest.identifiers, + uploadInfo = uploadRequest.uploadInfo, + source = source, + thumbnail = Some(thumbnail), + optimisedPng = png, + fileMetadata = fileMetadata, + userMetadata = None, + metadata = metadata, + originalMetadata = metadata, + usageRights = usageRights, + originalUsageRights = usageRights, + exports = List(), + usages = List(), // ImageEmbedding will be written by lambda later embedding = None ) @@ -176,7 +174,7 @@ object Uploader extends GridLogging { colourModel <- colourModelFuture } yield { val fullFileMetadata = fileMetadata.copy(colourModel = colourModel) - val metadata = ImageMetadataConverter.fromFileMetadata(fullFileMetadata, s3Source.metadata.objectMetadata.lastModified) + val metadata = ImageMetadataConverter.fromFileMetadata(fullFileMetadata, s3Source.metadata.objectMetadata.lastModified.map(lm => new DateTime(lm.toEpochMilli))) val sourceAsset = Asset.fromS3Object(s3Source, sourceDimensions, sourceOrientationMetadata) val thumbAsset = Asset.fromS3Object(s3Thumb, thumbDimensions) @@ -238,7 +236,7 @@ object Uploader extends GridLogging { def toMetaMap(uploadRequest: UploadRequest): Map[String, String] = { val baseMeta = Map( ImageStorageProps.uploadedByMetadataKey -> uploadRequest.uploadedBy, - ImageStorageProps.uploadTimeMetadataKey -> printDateTime(uploadRequest.uploadTime) + ImageStorageProps.uploadTimeMetadataKey -> printInstant(uploadRequest.uploadTime) ) ++ uploadRequest.identifiersMeta ++ uploadRequest.uploadInfo.filename.map(ImageStorageProps.filenameMetadataKey -> _) @@ -320,7 +318,7 @@ object Uploader extends GridLogging { } def patchUploadRequestWithS3Metadata(request: UploadRequest, s3Object: S3Object): UploadRequest = { - val metadata = S3FileExtractedMetadata(s3Object.metadata.objectMetadata.lastModified.getOrElse(new DateTime), s3Object.metadata.userMetadata) + val metadata = S3FileExtractedMetadata(s3Object.metadata.objectMetadata.lastModified.getOrElse(Instant.now), s3Object.metadata.userMetadata) request.copy( uploadTime = metadata.uploadTime, uploadedBy = metadata.uploadedBy, @@ -403,7 +401,7 @@ class Uploader( def loadFile(digestedFile: DigestedFile, uploadedBy: String, identifiers: Map[String, String], - uploadTime: DateTime, + uploadTime: Instant, filename: Option[String]) (implicit ec:ExecutionContext, logMarker: LogMarker): Future[UploadRequest] = Future { @@ -470,7 +468,7 @@ class Uploader( val mediaId = imageUpload.image.id logger.info(logMarker, s"Copying $mediaId to lower environment queue bucket $lowerEnvironmentQueueBucket") try { - store.client.copyObject( + store.copyObject( config.imageBucket, fileKeyFromId(mediaId), lowerEnvironmentQueueBucket, s"${uploadRequest.uploadedBy}/${uploadRequest.uploadInfo.filename.getOrElse(mediaId)}" ) diff --git a/image-loader/app/model/upload/UploadRequest.scala b/image-loader/app/model/upload/UploadRequest.scala index cf5fb3848c..f4b8c92040 100644 --- a/image-loader/app/model/upload/UploadRequest.scala +++ b/image-loader/app/model/upload/UploadRequest.scala @@ -1,21 +1,16 @@ package model.upload import com.gu.mediaservice.lib.ImageStorageProps - -import java.io.File -import java.util.UUID import com.gu.mediaservice.model.{MimeType, UploadInfo} -import net.logstash.logback.marker.{LogstashMarker, Markers} -import org.joda.time.format.ISODateTimeFormat -import org.joda.time.{DateTime, DateTimeZone} -import scala.jdk.CollectionConverters._ +import java.io.File +import java.time.Instant case class UploadRequest( imageId: String, tempFile: File, mimeType: Option[MimeType], - uploadTime: DateTime, + uploadTime: Instant, uploadedBy: String, identifiers: Map[String, String], uploadInfo: UploadInfo, diff --git a/image-loader/test/scala/model/ImageUploadTest.scala b/image-loader/test/scala/model/ImageUploadTest.scala index 2430a911a2..a379269317 100644 --- a/image-loader/test/scala/model/ImageUploadTest.scala +++ b/image-loader/test/scala/model/ImageUploadTest.scala @@ -1,26 +1,24 @@ package model -import java.io.File -import java.net.URI -import java.util.UUID import com.drew.imaging.ImageProcessingException -import com.gu.mediaservice.lib.{StorableImage, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage} -import com.gu.mediaservice.lib.aws.{EmbedderMessage, S3Metadata, S3Object, S3ObjectMetadata, S3Ops} +import com.gu.mediaservice.lib.aws.{S3Metadata, S3Object, S3ObjectMetadata} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.LogMarker -import com.gu.mediaservice.model.{FileMetadata, Jpeg, MimeType, Png, Tiff, UploadInfo} +import com.gu.mediaservice.lib.{StorableImage, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage} +import com.gu.mediaservice.model._ import lib.imaging.MimeTypeDetection import model.upload.{OptimiseWithPngQuant, UploadRequest} -import org.joda.time.DateTime import org.scalatest.Assertion import org.scalatest.funsuite.AsyncFunSuite import org.scalatest.matchers.should.Matchers import org.scalatestplus.mockito.MockitoSugar -import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse import test.lib.ResourceHelpers -import java.nio.file.Path +import java.io.File +import java.net.URI +import java.time.Instant +import java.util.UUID import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success} @@ -77,7 +75,7 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar { imageId = randomId, tempFile = tempFile, mimeType = MimeTypeDetection.guessMimeType(tempFile).toOption, - uploadTime = DateTime.now(), + uploadTime = Instant.now(), uploadedBy = "uploadedBy", identifiers = Map(), uploadInfo = ul diff --git a/image-loader/test/scala/model/ProjectorTest.scala b/image-loader/test/scala/model/ProjectorTest.scala index eccec5b65f..6393d55ca2 100644 --- a/image-loader/test/scala/model/ProjectorTest.scala +++ b/image-loader/test/scala/model/ProjectorTest.scala @@ -1,12 +1,8 @@ package model -import java.io.File -import java.net.URI -import java.util.{Date, UUID} -import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.s3.model.ObjectMetadata import com.gu.mediaservice.GridClient import com.gu.mediaservice.lib.auth.Authentication +import com.gu.mediaservice.lib.aws.S3Ops import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} @@ -14,7 +10,6 @@ import com.gu.mediaservice.model._ import com.gu.mediaservice.model.leases.LeasesByMedia import lib.DigestedFile import org.joda.time.{DateTime, DateTimeZone} -import org.mockito.ArgumentMatchers.any import org.mockito.Mockito.{times, verify, when} import org.scalatest.concurrent.ScalaFutures import org.scalatest.freespec.AnyFreeSpec @@ -22,13 +17,16 @@ import org.scalatest.matchers.should.Matchers import org.scalatest.time.{Millis, Span} import org.scalatestplus.mockito.MockitoSugar import play.api.libs.json.{JsArray, JsString} -import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.GetObjectResponse import test.lib.ResourceHelpers -import java.nio.file.Path +import java.io.File +import java.net.URI +import java.time.Instant import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future import scala.jdk.CollectionConverters._ -import scala.concurrent.{ExecutionContext, Future} class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with MockitoSugar { @@ -44,9 +42,10 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc private val maybeEmbedder = None - private val s3 = mock[AmazonS3] + private val s3 = mock[S3Client] + private val auth = mock[Authentication] - private val projector = new Projector(config, s3, imageOperations, ImageProcessor.identity, auth, maybeEmbedder) + private val projector = new Projector(config, S3Ops(s3), imageOperations, ImageProcessor.identity, auth, maybeEmbedder) // FIXME temporary ignored as test is not executable in CI/CD machine // because graphic lib files like srgb.icc, cmyk.icc are in root directory instead of resources @@ -57,7 +56,7 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc val id = "id123" val fileDigest = DigestedFile(testFile, id) val uploadedBy = "test" - val uploadTime = new DateTime("2020-01-24T17:36:08.456Z").withZone(DateTimeZone.UTC) + val uploadTime = Instant.parse("2020-01-24T17:36:08.456Z") val uploadFileName = Some("getty.jpg") // expected @@ -234,37 +233,39 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc "S3FileExtractedMetadata" - { "should extract URL encoded metadata" in { - val s3Metadata = new ObjectMetadata() - s3Metadata.setLastModified(new Date(1613388118000L)) - s3Metadata.setUserMetadata(Map( - "file-name" -> "This%20photo%20was%20taken%20in%20%C5%81%C3%B3d%C5%BA.jpg", - "uploaded-by" -> "s%C3%A9b.cevey%40theguardian.co.uk", - "upload-time" -> "2021-02-01T12%3A52%3A34%2B09%3A00", - "identifier!picdarurn" -> "12*543%5E25" - ).asJava) + val res = GetObjectResponse.builder() + .lastModified(Instant.ofEpochMilli(1613388118000L)) + .metadata(Map( + "file-name" -> "This%20photo%20was%20taken%20in%20%C5%81%C3%B3d%C5%BA.jpg", + "uploaded-by" -> "s%C3%A9b.cevey%40theguardian.co.uk", + "upload-time" -> "2021-02-01T12%3A52%3A34%2B09%3A00", + "identifier!picdarurn" -> "12*543%5E25" + ).asJava) + .build() - val result = S3FileExtractedMetadata(s3Metadata) + val result = S3FileExtractedMetadata(res) result.uploadFileName shouldBe Some("This photo was taken in Łódź.jpg") result.uploadedBy shouldBe "séb.cevey@theguardian.co.uk" - result.uploadTime.toString shouldBe "2021-02-01T03:52:34.000Z" + result.uploadTime.toString shouldBe "2021-02-01T03:52:34Z" result.identifiers.size shouldBe 1 result.identifiers.get("picdarurn") shouldBe Some("12*543^25") } "should remap headers with underscores to dashes" in { - val s3Metadata = new ObjectMetadata() - s3Metadata.setLastModified(new Date(1613388118000L)) - s3Metadata.setUserMetadata(Map( - "file_name" -> "filename.jpg", - "uploaded_by" -> "user", - "upload_time" -> "2021-02-01T12%3A52%3A34%2B09%3A00", - "identifier!picdarurn" -> "12*543" - ).asJava) + val res = GetObjectResponse.builder() + .lastModified(Instant.ofEpochMilli(1613388118000L)) + .metadata(Map( + "file_name" -> "filename.jpg", + "uploaded_by" -> "user", + "upload_time" -> "2021-02-01T12%3A52%3A34%2B09%3A00", + "identifier!picdarurn" -> "12*543" + ).asJava) + .build() - val result = S3FileExtractedMetadata(s3Metadata) + val result = S3FileExtractedMetadata(res) result.uploadFileName shouldBe Some("filename.jpg") result.uploadedBy shouldBe "user" - result.uploadTime.toString shouldBe "2021-02-01T03:52:34.000Z" + result.uploadTime.toString shouldBe "2021-02-01T03:52:34Z" result.identifiers.size shouldBe 1 result.identifiers.get("picdarurn") shouldBe Some("12*543") } @@ -272,14 +273,15 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc "should correctly read in non URL encoded values" in { // we have plenty of values in S3 that are not URL encoded // and we must be able to read them correctly - val s3Metadata = new ObjectMetadata() - s3Metadata.setLastModified(new Date(1613388118000L)) - s3Metadata.setUserMetadata(Map( - "uploaded_by" -> "user", - "upload_time" -> "2019-12-11T01:12:10.427Z", - ).asJava) + val res = GetObjectResponse.builder() + .lastModified(Instant.ofEpochMilli(1613388118000L)) + .metadata(Map( + "uploaded_by" -> "user", + "upload_time" -> "2019-12-11T01:12:10.427Z", + ).asJava) + .build() - val result = S3FileExtractedMetadata(s3Metadata) + val result = S3FileExtractedMetadata(res) result.uploadedBy shouldBe "user" result.uploadTime.toString shouldBe "2019-12-11T01:12:10.427Z" } diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index 0d98fa21c5..1f0ae58837 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -299,7 +299,7 @@ class MediaApi( export <- source.exports.find(_.id.contains(exportId)) asset <- export.assets.find(_.dimensions.exists(_.width == width)) s3Object <- Try(s3Client.getObject(config.imgPublishingBucket, asset.file)).toOption - file = StreamConverters.fromInputStream(() => s3Object.getObjectContent) + file = StreamConverters.fromInputStream(() => s3Object) entity = HttpEntity.Streamed(file, asset.size, asset.mimeType.map(_.name)) result = Result(ResponseHeader(OK), entity).withHeaders("Content-Disposition" -> getContentDisposition(source, export, asset, config.shortenDownloadFilename)) } yield { @@ -422,7 +422,7 @@ class MediaApi( logger.info(logMarker, s"Download original image: $id from user: ${Authentication.getIdentity(request.user)}") mediaApiMetrics.incrementImageDownload(apiKey, mediaApiMetrics.OriginalDownloadType) val s3Object = s3Client.getObject(config.imageBucket, image.source.file) - val file = StreamConverters.fromInputStream(() => s3Object.getObjectContent) + val file = StreamConverters.fromInputStream(() => s3Object) val entity = HttpEntity.Streamed(file, image.source.size, image.source.mimeType.map(_.name)) if(config.recordDownloadAsUsage) { diff --git a/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PandaAuthenticationProvider.scala b/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PandaAuthenticationProvider.scala index 126127df98..08bc20de3a 100644 --- a/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PandaAuthenticationProvider.scala +++ b/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PandaAuthenticationProvider.scala @@ -2,7 +2,7 @@ package com.gu.mediaservice.lib.guardian.auth import com.gu.mediaservice.lib.argo.ArgoHelpers import com.gu.mediaservice.lib.argo.model.Link -import com.gu.mediaservice.lib.auth.Authentication.{UserPrincipal, Principal} +import com.gu.mediaservice.lib.auth.Authentication.{Principal, UserPrincipal} import com.gu.mediaservice.lib.auth.provider.AuthenticationProvider.RedirectUri import com.gu.mediaservice.lib.auth.provider._ import com.gu.mediaservice.lib.aws.S3Ops @@ -17,7 +17,6 @@ import play.api.libs.typedmap.{TypedEntry, TypedKey, TypedMap} import play.api.libs.ws.{DefaultWSCookie, WSClient, WSRequest} import play.api.mvc.{ControllerComponents, Cookie, RequestHeader, Result} -import scala.concurrent.duration.{Duration, HOURS} import scala.concurrent.{ExecutionContext, Future} import scala.util.Try @@ -34,8 +33,6 @@ class PandaAuthenticationProvider( override def wsClient: WSClient = resources.wsClient override def controllerComponents: ControllerComponents = resources.controllerComponents - override def apiGracePeriod: Long = Duration(24, HOURS).toMillis - val loginLinks = List( Link("login", resources.commonConfig.services.loginUriTemplate) ) diff --git a/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PermissionsAuthorisationProvider.scala b/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PermissionsAuthorisationProvider.scala index c374d5b611..22a254c568 100644 --- a/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PermissionsAuthorisationProvider.scala +++ b/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PermissionsAuthorisationProvider.scala @@ -25,7 +25,7 @@ class PermissionsAuthorisationProvider(configuration: Configuration, resources: provider case _ => val permissionsStage = if(config.isProd) { "PROD" } else { "CODE" } - PermissionsProvider(PermissionsConfig(permissionsStage, config.awsRegion, config.awsCredentials, permissionsBucket)) + PermissionsProvider(PermissionsConfig(permissionsStage, config.awsRegion, config.awsCredentialsV2, permissionsBucket)) } override def initialise(): Unit = { diff --git a/rest-lib/src/test/scala/com/gu/mediaservice/lib/guardian/auth/PandaAuthenticationProviderTest.scala b/rest-lib/src/test/scala/com/gu/mediaservice/lib/guardian/auth/PandaAuthenticationProviderTest.scala index 985aac29b1..29ecf50d50 100644 --- a/rest-lib/src/test/scala/com/gu/mediaservice/lib/guardian/auth/PandaAuthenticationProviderTest.scala +++ b/rest-lib/src/test/scala/com/gu/mediaservice/lib/guardian/auth/PandaAuthenticationProviderTest.scala @@ -10,7 +10,7 @@ class PandaAuthenticationProviderTest extends AnyFunSuite with Matchers { import com.gu.mediaservice.lib.guardian.auth.PandaAuthenticationProvider.validateUser val user: AuthenticatedUser = AuthenticatedUser(User("Barry", "Chuckle", "barry.chuckle@guardian.co.uk", None), - "media-service", Set("media-service"), Instant.now().plusSeconds(100).toEpochMilli, multiFactor = true) + "media-service", Set("media-service"), Instant.now().plusSeconds(100), multiFactor = true) test("user fails email domain validation") { validateUser(user, "chucklevision.biz", None) must be(false) diff --git a/thrall/app/controllers/ReaperController.scala b/thrall/app/controllers/ReaperController.scala index 42e454d025..7179698881 100644 --- a/thrall/app/controllers/ReaperController.scala +++ b/thrall/app/controllers/ReaperController.scala @@ -1,7 +1,5 @@ package controllers -import org.apache.pekko.actor.Scheduler -import com.gu.mediaservice.lib.{DateTimeUtils, ImageIngestOperations} import com.gu.mediaservice.lib.auth.Permissions.DeleteImage import com.gu.mediaservice.lib.auth.{Authentication, Authorisation, BaseControllerWithLoginRedirects} import com.gu.mediaservice.lib.aws.S3Vectors @@ -9,17 +7,20 @@ import com.gu.mediaservice.lib.config.Services import com.gu.mediaservice.lib.elasticsearch.ReapableEligibility import com.gu.mediaservice.lib.logging.{GridLogging, MarkerMap} import com.gu.mediaservice.lib.metadata.SoftDeletedMetadataTable +import com.gu.mediaservice.lib.{DateTimeUtils, ImageIngestOperations} import com.gu.mediaservice.model.{ImageStatusRecord, SoftDeletedMetadata} -import lib.{BatchDeletionIds, ThrallConfig, ThrallMetrics, ThrallStore} import lib.elasticsearch.ElasticSearch +import lib.{BatchDeletionIds, ThrallConfig, ThrallMetrics, ThrallStore} +import org.apache.pekko.actor.Scheduler import org.joda.time.{DateTime, DateTimeZone} import play.api.libs.json.{JsValue, Json} import play.api.mvc.{Action, AnyContent, ControllerComponents} import scalaz.NonEmptyList +import java.time.format.DateTimeFormatter +import java.time.{ZoneId, ZonedDateTime} import scala.concurrent.duration.DurationInt import scala.concurrent.{ExecutionContext, Future} -import scala.jdk.CollectionConverters._ import scala.language.postfixOps import scala.util.control.NonFatal import scala.util.{Failure, Success} @@ -66,7 +67,7 @@ class ReaperController( interval = INTERVAL, ){ () => try { - if (store.client.doesObjectExist(reaperBucket, CONTROL_FILE_NAME)) { + if (store.doesObjectExist(reaperBucket, CONTROL_FILE_NAME)) { logger.info("Reaper is paused") es.countTotalSoftReapable(isReapable).map(metrics.softReapable.increment(Nil, _)) es.countTotalHardReapable(isReapable, config.hardReapImagesAge).map(metrics.hardReapable.increment(Nil, _)) @@ -102,14 +103,15 @@ class ReaperController( } } - private def s3DirNameFromDate(date: DateTime) = date.toString("YYYY-MM-dd") + private lazy val dateOnlyFormatter = DateTimeFormatter.ISO_LOCAL_DATE + private def s3DirNameFromDate(date: ZonedDateTime) = date.format(dateOnlyFormatter) private def persistedBatchDeleteOperation(deleteType: String)(doBatchDelete: => Future[JsValue]) = config.maybeReaperBucket match { case None => Future.failed(new Exception("Reaper bucket not configured")) case Some(reaperBucket) => doBatchDelete.map { json => - val now = DateTime.now(DateTimeZone.UTC) - val key = s"$deleteType/${s3DirNameFromDate(now)}/$deleteType-${now.toString()}.json" - store.client.putObject(reaperBucket, key, json.toString()) + val now = ZonedDateTime.now(ZoneId.of("UTC")) + val key = s"$deleteType/${s3DirNameFromDate(now)}/$deleteType-${now.toString}.json" + store.putObject(reaperBucket, key, json.toString()) json } } @@ -181,23 +183,24 @@ class ReaperController( }).map(Json.toJson(_)) } def index = withLoginRedirect { - val now = DateTime.now(DateTimeZone.UTC) + val now = ZonedDateTime.now(ZoneId.of("UTC")) + (config.maybeReaperBucket, config.maybeReaperCountPerRun) match { case (None, _) => NotImplemented("'s3.reaper.bucket' not configured in thrall.conf") case (_, None) => NotImplemented("'reaper.countPerRun' not configured in thrall.conf") case (Some(reaperBucket), Some(countOfImagesToReap)) => - val isPaused = store.client.doesObjectExist(reaperBucket, CONTROL_FILE_NAME) + val isPaused = store.doesObjectExist(reaperBucket, CONTROL_FILE_NAME) val recentRecords = List(now, now.minusDays(1), now.minusDays(2)).flatMap { day => val s3DirName = s3DirNameFromDate(day) - store.client.listObjects(reaperBucket, s"soft/$s3DirName/").getObjectSummaries.asScala.toList ++ - store.client.listObjects(reaperBucket, s"hard/$s3DirName/").getObjectSummaries.asScala.toList + store.listObjects(reaperBucket, s"soft/$s3DirName/") ++ + store.listObjects(reaperBucket, s"hard/$s3DirName/") } val recentRecordKeys = recentRecords - .filter(_.getLastModified after now.minusHours(48).toDate) - .sortBy(_.getLastModified) + .filter(_.lastModified() isAfter now.minusHours(48).toInstant) + .sortBy(_.lastModified()) .reverse - .map(_.getKey) + .map(_.key()) Ok(views.html.reaper(isPaused, INTERVAL.toString(), countOfImagesToReap, recentRecordKeys)) }} @@ -205,22 +208,23 @@ class ReaperController( def reaperRecord(key: String) = auth { config.maybeReaperBucket match { case None => NotImplemented("Reaper bucket not configured") case Some(reaperBucket) => - Ok( - store.client.getObjectAsString(reaperBucket, key) - ).as(JSON) + store.getObjectAsString(reaperBucket, key) match { + case Some(contents) => Ok(contents).as(JSON) + case None => NotFound(s"No record found for $key") + } }} def pauseReaper = auth { config.maybeReaperBucket match { case None => NotImplemented("Reaper bucket not configured") case Some(reaperBucket) => - store.client.putObject(reaperBucket, CONTROL_FILE_NAME, "") + store.putObject(reaperBucket, CONTROL_FILE_NAME, "") Redirect(routes.ReaperController.index) }} def resumeReaper = auth { config.maybeReaperBucket match { case None => NotImplemented("Reaper bucket not configured") case Some(reaperBucket) => - store.client.deleteObject(reaperBucket, CONTROL_FILE_NAME) + store.deleteObject(reaperBucket, CONTROL_FILE_NAME) Redirect(routes.ReaperController.index) }} diff --git a/thrall/app/lib/SyncChecker.scala b/thrall/app/lib/SyncChecker.scala index 959057ce58..b47218aad8 100644 --- a/thrall/app/lib/SyncChecker.scala +++ b/thrall/app/lib/SyncChecker.scala @@ -1,18 +1,17 @@ package lib +import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, MarkerMap} +import lib.elasticsearch.ElasticSearch import org.apache.pekko.Done import org.apache.pekko.actor.ActorSystem -import org.apache.pekko.stream.{Materializer, RestartSettings} import org.apache.pekko.stream.scaladsl.{RestartSource, Source} -import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.s3.model.ListObjectsV2Request -import com.gu.mediaservice.lib.elasticsearch.InProgress -import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, MarkerMap} -import lib.elasticsearch.ElasticSearch +import org.apache.pekko.stream.{Materializer, RestartSettings} +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request -import scala.jdk.CollectionConverters._ import scala.concurrent.duration.DurationInt -import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future} +import scala.concurrent.{ExecutionContext, Future} +import scala.jdk.CollectionConverters.CollectionHasAsScala import scala.util.{Failure, Success} sealed trait SyncCheckJob {} @@ -20,7 +19,7 @@ case class Prefix(prefix: String) extends SyncCheckJob case object Other extends SyncCheckJob class SyncChecker( - s3: AmazonS3, + s3: S3Client, es: ElasticSearch, imageBucketName: String, actorSystem: ActorSystem @@ -39,12 +38,12 @@ class SyncChecker( private def paginatedListObjects(request: ListObjectsV2Request, maybeContinuationToken: Option[String] = None): Future[Seq[String]] = { Future { val fullRequest = maybeContinuationToken match { - case Some(token) => request.withContinuationToken(token) + case Some(token) => request.toBuilder.continuationToken(token).build() case None => request } val result = s3.listObjectsV2(fullRequest) - val keys = result.getObjectSummaries.asScala.toSeq.map(_.getKey.split("/").last) - val nextContinuationToken = Option(result.getNextContinuationToken) + val keys = result.contents().asScala.toSeq.map(_.key().split("/").last) + val nextContinuationToken = Option(result.nextContinuationToken()) keys -> nextContinuationToken } flatMap { case (keys, None) => Future.successful(keys) @@ -54,15 +53,17 @@ class SyncChecker( } private def listFilesWithPrefix(prefix: String): Future[Seq[String]] = { - val request = new ListObjectsV2Request() - .withBucketName(imageBucketName) - .withPrefix(prefix) + val request = ListObjectsV2Request.builder() + .bucket(imageBucketName) + .prefix(prefix) + .build() paginatedListObjects(request) } private def listFilesAtRoot(): Future[Seq[String]] = { - val request = new ListObjectsV2Request() - .withBucketName(imageBucketName) - .withDelimiter("/") + val request = ListObjectsV2Request.builder() + .bucket(imageBucketName) + .delimiter("/") + .build() paginatedListObjects(request) }