Add format auto-detection for zero-config S3 ingestion

data-prepper-plugins/format-detection/build.gradle

@@ -0,0 +1,24 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+plugins {
+    id 'java'
+}
+
+dependencies {
+    implementation 'org.apache.tika:tika-core:2.9.2'
+    implementation 'org.xerial.snappy:snappy-java:1.1.10.8'
+    testImplementation 'org.junit.jupiter:junit-jupiter:5.10.2'
+    testImplementation 'org.assertj:assertj-core:3.25.3'
+    testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
+}
+
+test {
+    useJUnitPlatform()
+}

data-prepper-plugins/format-detection/src/main/java/org/opensearch/dataprepper/plugins/formatdetection/Confidence.java

@@ -0,0 +1,19 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.dataprepper.plugins.formatdetection;
+
+/**
+ * Confidence level of a format detection result.
+ */
+public enum Confidence {
+    HIGH,
+    MEDIUM,
+    LOW
+}

data-prepper-plugins/format-detection/src/main/java/org/opensearch/dataprepper/plugins/formatdetection/DetectedCompression.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.dataprepper.plugins.formatdetection;
+
+/**
+ * Compression types detectable from magic bytes.
+ */
+public enum DetectedCompression {
+    GZIP,
+    ZSTD,
+    SNAPPY,
+    NONE
+}

data-prepper-plugins/format-detection/src/main/java/org/opensearch/dataprepper/plugins/formatdetection/DetectedFormat.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.dataprepper.plugins.formatdetection;
+
+/**
+ * Data formats that can be detected from S3 object bytes.
+ */
+public enum DetectedFormat {
+    PARQUET,
+    AVRO,
+    ORC,
+    PDF,
+    IMAGE,
+    JSON,
+    NDJSON,
+    XML,
+    CSV,
+    TSV,
+    TEXT,
+    UNKNOWN
+}

data-prepper-plugins/format-detection/src/main/java/org/opensearch/dataprepper/plugins/formatdetection/FormatDetectionResult.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.dataprepper.plugins.formatdetection;
+
+/**
+ * Result of detecting the format and compression of a data sample.
+ */
+public class FormatDetectionResult {
+    private final DetectedCompression compression;
+    private final DetectedFormat format;
+    private final Confidence confidence;
+
+    public FormatDetectionResult(final DetectedCompression compression,
+                                 final DetectedFormat format,
+                                 final Confidence confidence) {
+        this.compression = compression;
+        this.format = format;
+        this.confidence = confidence;
+    }
+
+    public DetectedCompression getCompression() {
+        return compression;
+    }
+
+    public DetectedFormat getFormat() {
+        return format;
+    }
+
+    public Confidence getConfidence() {
+        return confidence;
+    }
+
+    @Override
+    public String toString() {
+        return "FormatDetectionResult{compression=" + compression +
+                ", format=" + format +
+                ", confidence=" + confidence + "}";
+    }
+}