diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java b/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java deleted file mode 100644 index 1a888610a9e..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java +++ /dev/null @@ -1,570 +0,0 @@ -package edu.harvard.iq.dataverse.export; - -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.Embargo; -import edu.harvard.iq.dataverse.FileMetadata; - -import edu.harvard.iq.dataverse.dataaccess.DataAccess; -import static edu.harvard.iq.dataverse.dataaccess.DataAccess.getStorageIO; -import edu.harvard.iq.dataverse.dataaccess.DataAccessOption; -import edu.harvard.iq.dataverse.dataaccess.StorageIO; -import io.gdcc.spi.export.ExportException; -import io.gdcc.spi.export.Exporter; -import io.gdcc.spi.export.XMLExporter; -import edu.harvard.iq.dataverse.settings.JvmSettings; -import edu.harvard.iq.dataverse.util.BundleUtil; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.net.URL; -import java.net.URLClassLoader; -import java.nio.channels.Channel; -import java.nio.channels.Channels; -import java.nio.channels.WritableByteChannel; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.sql.Timestamp; -import java.time.LocalDate; -import java.time.ZoneId; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.ServiceConfigurationError; -import java.util.ServiceLoader; -import java.util.Set; -import java.util.logging.Level; -import java.util.logging.Logger; -import jakarta.ws.rs.core.MediaType; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.FileInputStream; - -import org.apache.commons.io.IOUtils; - -/** - * - * @author skraffmi - */ -public class ExportService { - - private static ExportService service; - private ServiceLoader loader; - private Map exporterMap = new HashMap<>(); - - private static final Logger logger = Logger.getLogger(ExportService.class.getCanonicalName()); - - private ExportService() { - /* - * Step 1 - find the EXPORTERS dir and add all jar files there to a class loader - */ - List jarUrls = new ArrayList<>(); - Optional exportPathSetting = JvmSettings.EXPORTERS_DIRECTORY.lookupOptional(String.class); - if (exportPathSetting.isPresent()) { - Path exporterDir = Paths.get(exportPathSetting.get()); - // Get all JAR files from the configured directory - try (DirectoryStream stream = Files.newDirectoryStream(exporterDir, "*.jar")) { - // Using the foreach loop here to enable catching the URI/URL exceptions - for (Path path : stream) { - logger.log(Level.FINE, "Adding {0}", path.toUri().toURL()); - // This is the syntax required to indicate a jar file from which classes should - // be loaded (versus a class file). - jarUrls.add(new URL("jar:" + path.toUri().toURL() + "!/")); - } - } catch (IOException e) { - logger.warning("Problem accessing external Exporters: " + e.getLocalizedMessage()); - } - } - URLClassLoader cl = URLClassLoader.newInstance(jarUrls.toArray(new URL[0]), this.getClass().getClassLoader()); - - /* - * Step 2 - load all Exporters that can be found, using the jars as additional - * sources - */ - loader = ServiceLoader.load(Exporter.class, cl); - /* - * Step 3 - Fill exporterMap with providerName as the key, allow external - * exporters to replace internal ones for the same providerName. FWIW: From the - * logging it appears that ServiceLoader returns classes in ~ alphabetical order - * rather than by class loader, so internal classes handling a given - * providerName may be processed before or after external ones. - */ - loader.forEach(exp -> { - String formatName = exp.getFormatName(); - // If no entry for this providerName yet or if it is an external exporter - if (!exporterMap.containsKey(formatName) || exp.getClass().getClassLoader().equals(cl)) { - exporterMap.put(formatName, exp); - } - logger.log(Level.FINE, "SL: " + exp.getFormatName() + " from " + exp.getClass().getCanonicalName() - + " and classloader: " + exp.getClass().getClassLoader().getClass().getCanonicalName()); - }); - } - - public static synchronized ExportService getInstance() { - if (service == null) { - service = new ExportService(); - } - return service; - } - - public List getExportersLabels() { - List retList = new ArrayList<>(); - - exporterMap.values().forEach(exp -> { - String[] temp = new String[2]; - temp[0] = exp.getDisplayName(BundleUtil.getCurrentLocale()); - temp[1] = exp.getFormatName(); - retList.add(temp); - }); - return retList; - } - - public InputStream getExport(DatasetVersion datasetVersion, String formatName) throws ExportException, IOException { - - Dataset dataset = datasetVersion.getDataset(); - InputStream exportInputStream = null; - - if (datasetVersion.isDraft()) { - // For drafts we create the export on the fly rather than caching. - Exporter exporter = exporterMap.get(formatName); - if (exporter != null) { - try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { - // getPrerequisiteFormatName logic copied from exportFormat() - if (exporter.getPrerequisiteFormatName().isPresent()) { - String prereqFormatName = exporter.getPrerequisiteFormatName().get(); - try (InputStream preReqStream = getExport(datasetVersion, prereqFormatName)) { - InternalExportDataProvider dataProvider = new InternalExportDataProvider(datasetVersion, preReqStream); - exporter.exportDataset(dataProvider, outputStream); - } catch (IOException ioe) { - throw new ExportException("Could not get prerequisite " + prereqFormatName + " to create " + formatName + " export for dataset " + dataset.getId(), ioe); - } - } else { - InternalExportDataProvider dataProvider = new InternalExportDataProvider(datasetVersion); - exporter.exportDataset(dataProvider, outputStream); - } - return new ByteArrayInputStream(outputStream.toByteArray()); - } - } - } else { - // for non-drafts (published versions) we try to locate an already existing, cached export - exportInputStream = getCachedExportFormat(dataset, formatName); - } - - // The DDI export is limited for restricted and actively embargoed files (no - // data/file description sections).and when an embargo ends, we need to refresh - // this export. - boolean clearCachedExport = false; - if (formatName.equals(DDIExporter.PROVIDER_NAME) && (exportInputStream != null)) { - // We want ddi and there was a cached version - LocalDate exportLocalDate = null; - Date lastExportDate = dataset.getLastExportTime(); - // if lastExportDate == null, assume it's not set because were exporting for the - // first time now (e.g. during publish) and therefore no changes are needed - if (lastExportDate != null) { - exportLocalDate = lastExportDate.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); - logger.fine("Last export date: " + exportLocalDate.toString()); - // Track which embargoes we've already checked - Set embargoIds = new HashSet(); - // Check for all files in the latest released version - for (FileMetadata fm : dataset.getLatestVersionForCopy().getFileMetadatas()) { - // ToDo? This loop is necessary because we have not stored the date when the - // next embargo in this datasetversion will end. If we knew that (another - // dataset/datasetversion column), we could make - // one check that nextembargoEnd exists and is after the last export and before - // now versus scanning through files until we potentially find such an embargo. - Embargo e = fm.getDataFile().getEmbargo(); - if (e != null) { - logger.fine("Datafile: " + fm.getDataFile().getId()); - logger.fine("Embargo end date: " + e.getFormattedDateAvailable()); - } - if (e != null && !embargoIds.contains(e.getId()) && e.getDateAvailable().isAfter(exportLocalDate) - && e.getDateAvailable().isBefore(LocalDate.now())) { - logger.fine("Request that the ddi export be cleared."); - // The file has been embargoed and the embargo ended after the last export and - // before the current date, so we need to remove the cached DDI export and make - // it refresh - clearCachedExport = true; - break; - } else if (e != null) { - logger.fine("adding embargo to checked list: " + e.getId()); - embargoIds.add(e.getId()); - } - } - } - if (clearCachedExport) { - try { - exportInputStream.close(); - clearCachedExport(dataset, formatName); - } catch (Exception ex) { - logger.warning("Failure deleting DDI export format for dataset id: " + dataset.getId() - + " after embargo expiration: " + ex.getLocalizedMessage()); - } finally { - exportInputStream = null; - } - } - } - - if (exportInputStream != null) { - return exportInputStream; - } - - // if it doesn't exist, we'll try to run the export: - exportFormat(dataset, formatName); - - // and then try again: - exportInputStream = getCachedExportFormat(dataset, formatName); - - if (exportInputStream != null) { - return exportInputStream; - } - - // if there is no cached export still - we have to give up and throw - // an exception! - throw new ExportException("Failed to export the dataset as " + formatName); - - } - - public String getLatestPublishedAsString(Dataset dataset, String formatName) { - if (dataset == null) { - return null; - } - DatasetVersion releasedVersion = dataset.getReleasedVersion(); - if (releasedVersion == null) { - return null; - } - InputStream inputStream = null; - InputStreamReader inp = null; - try { - inputStream = getExport(releasedVersion, formatName); - if (inputStream != null) { - inp = new InputStreamReader(inputStream, "UTF8"); - BufferedReader br = new BufferedReader(inp); - StringBuilder sb = new StringBuilder(); - String line; - while ((line = br.readLine()) != null) { - sb.append(line); - sb.append('\n'); - } - br.close(); - inp.close(); - inputStream.close(); - return sb.toString(); - } - } catch (IOException ex) { - logger.log(Level.FINE, ex.getMessage(), ex); - return null; - } finally { - IOUtils.closeQuietly(inp); - IOUtils.closeQuietly(inputStream); - } - return null; - - } - - // A convenience wrapper method; the actual implementation has been moved - // into exportFormats() below. - public void exportAllFormats(Dataset dataset) throws ExportException { - exportFormats(dataset, List.of()); - } - - /** - * This method is added to supplement the classic exportAllFormats() in order - * to allow the metadata export APIs to selectively re-export only the formats - * specified. This is to finally allow an instance admin to avoid running - * a complete, from-scratch reexport when only _some_, or just one of them - * actually needs to be refreshed. On a large instance this can waste a - * significant amount of time and CPU cycles. (new as of 6.12) - * This method calls the cacheExport() method for every valid/supported - * format name supplied, or for every Exporter available, if an empty List - * is passed. - * Only the latest published version is used for exports. - * exportAllFormats() above is now a convenience wrapper, with the - * implementation moved here. - * - * @param dataset - * @param formatNames - * @throws ExportException - */ - public void exportFormats(Dataset dataset, List formatNames) throws ExportException { - if (dataset == null) { - throw new ExportException("exportFormats called with null Dataset"); - } - - if (formatNames == null) { - throw new ExportException("exportFormats called with null formatNames (use an empty List for \"all\""); - } - - try { - clearCachedFormats(dataset, formatNames); - } catch (IOException ex) { - Logger.getLogger(ExportService.class.getName()).log(Level.SEVERE, null, ex); - } - - try { - DatasetVersion releasedVersion = dataset.getReleasedVersion(); - if (releasedVersion == null) { - throw new ExportException("No released version for dataset " + dataset.getGlobalId().toString()); - } - InternalExportDataProvider dataProvider = new InternalExportDataProvider(releasedVersion); - - for (Exporter e : exporterMap.values()) { - String formatName = e.getFormatName(); - if (formatNames.isEmpty() || formatNames.contains(formatName)) { - if (e.getPrerequisiteFormatName().isPresent()) { - String prereqFormatName = e.getPrerequisiteFormatName().get(); - try (InputStream preReqStream = getExport(dataset.getReleasedVersion(), prereqFormatName)) { - dataProvider.setPrerequisiteInputStream(preReqStream); - cacheExport(dataset, dataProvider, formatName, e); - dataProvider.setPrerequisiteInputStream(null); - } catch (IOException ioe) { - throw new ExportException("Could not get prerequisite " + e.getPrerequisiteFormatName() + " to create " + formatName + "export for dataset " + dataset.getId(), ioe); - } - } else { - cacheExport(dataset, dataProvider, formatName, e); - } - } - } - // Finally, if we have been able to successfully export in all available - // formats, we'll increment the "last exported" time stamp: - if (formatNames.isEmpty()) { - dataset.setLastExportTime(new Timestamp(new Date().getTime())); - } - - } catch (ServiceConfigurationError serviceError) { - throw new ExportException("Service configuration error during export. " + serviceError.getMessage()); - } catch (RuntimeException e) { - logger.log(Level.FINE, e.getMessage(), e); - throw new ExportException( - "Unknown runtime exception exporting metadata. " + (e.getMessage() == null ? "" : e.getMessage())); - } - } - - // A convenience wrapper method - public void clearAllCachedFormats(Dataset dataset) throws IOException { - clearCachedFormats(dataset, List.of()); - dataset.setLastExportTime(null); - } - - public void clearCachedFormats(Dataset dataset, List formatNames) throws IOException { - if (dataset == null) { - throw new ExportException("cleareCachedFormats called with null Dataset"); - } - - if (formatNames == null) { - throw new ExportException("clearCachedFormats called with null formatNames (use an empty List for \"all\""); - } - - for (Exporter e : exporterMap.values()) { - String formatName = e.getFormatName(); - if (formatNames.isEmpty() || formatNames.contains(formatName)) { - try { - clearCachedExport(dataset, formatName); - } catch (IOException ex) { - // not fatal - } - } - } - } - - // This method finds the exporter for the format requested, - // then produces the dataset metadata as a JsonObject, then calls - // the "cacheExport()" method that will save the produced output - // in a file in the dataset directory. - public void exportFormat(Dataset dataset, String formatName) throws ExportException { - try { - - Exporter e = exporterMap.get(formatName); - if (e != null) { - DatasetVersion releasedVersion = dataset.getReleasedVersion(); - if (releasedVersion == null) { - throw new ExportException( - "No published version found during export. " + dataset.getGlobalId().toString()); - } - if(e.getPrerequisiteFormatName().isPresent()) { - String prereqFormatName = e.getPrerequisiteFormatName().get(); - try (InputStream preReqStream = getExport(releasedVersion, prereqFormatName)) { - InternalExportDataProvider dataProvider = new InternalExportDataProvider(releasedVersion, preReqStream); - cacheExport(dataset, dataProvider, formatName, e); - } catch (IOException ioe) { - throw new ExportException ("Could not get prerequisite " + e.getPrerequisiteFormatName() + " to create " + formatName + "export for dataset " + dataset.getId(), ioe); - } - } else { - InternalExportDataProvider dataProvider = new InternalExportDataProvider(releasedVersion); - cacheExport(dataset, dataProvider, formatName, e); - } - // As with exportAll, we should update the lastexporttime for the dataset - dataset.setLastExportTime(new Timestamp(new Date().getTime())); - } else { - throw new ExportException("Exporter not found"); - } - } catch (IllegalStateException e) { - // IllegalStateException can potentially mean very different, and - // unexpected things. An exporter attempting to get a single primitive - // value from a fieldDTO that is in fact a Multiple and contains a - // json vector (this has happened, for example, when the code in the - // DDI exporter was not updated following a metadata fieldtype change), - // will result in IllegalStateException. - throw new ExportException("IllegalStateException caught when exporting " + formatName + " for dataset " - + dataset.getGlobalId().toString() - + "; may or may not be due to a mismatch between an exporter code and a metadata block update. " - + e.getMessage()); - } - - } - - public Exporter getExporter(String formatName) throws ExportException { - Exporter e = exporterMap.get(formatName); - if (e != null) { - return e; - } - throw new ExportException("No such Exporter: " + formatName); - } - - // This method runs the selected metadata exporter, caching the output - // in a file in the dataset directory / container based on its DOI: - private void cacheExport(Dataset dataset, InternalExportDataProvider dataProvider, String format, Exporter exporter) - throws ExportException { - - OutputStream outputStream = null; - try { - boolean tempFileUsed = false; - File tempFile = null; - StorageIO storageIO = null; - - // With some storage drivers, we can open a WritableChannel, or OutputStream - // to directly write the generated metadata export that we want to cache; - // Some drivers (like Swift) do not support that, and will give us an - // "operation not supported" exception. If that's the case, we'll have - // to save the output into a temp file, and then copy it over to the - // permanent storage using the IO "save" command: - try { - storageIO = DataAccess.getStorageIO(dataset); - Channel outputChannel = storageIO.openAuxChannel("export_" + format + ".cached", - DataAccessOption.WRITE_ACCESS); - outputStream = Channels.newOutputStream((WritableByteChannel) outputChannel); - } catch (IOException ioex) { - // A common case = an IOException in openAuxChannel which is not supported by S3 - // stores for WRITE_ACCESS - tempFileUsed = true; - tempFile = File.createTempFile("tempFileToExport", ".tmp"); - outputStream = new FileOutputStream(tempFile); - } - - try { - // Write the metadata export file to the outputStream, which may be the final - // location or a temp file - exporter.exportDataset(dataProvider, outputStream); - outputStream.flush(); - outputStream.close(); - if (tempFileUsed) { - logger.fine("Saving export_" + format + ".cached aux file from temp file: " - + Paths.get(tempFile.getAbsolutePath())); - storageIO.savePathAsAux(Paths.get(tempFile.getAbsolutePath()), "export_" + format + ".cached"); - boolean tempFileDeleted = tempFile.delete(); - logger.fine("tempFileDeleted: " + tempFileDeleted); - } - } catch (ExportException exex) { - /* - * This exception is from the particular exporter and may not affect other - * exporters (versus other exceptions in this method which are from the basic - * mechanism to create a file) So we'll catch it here and report so that loops - * over other exporters can continue. Todo: Might be better to create a new - * exception subtype and send it upward, but the callers currently just log and - * ignore beyond terminating any loop over exporters. - */ - logger.warning("Exception thrown while creating export_" + format + ".cached : " + exex.getMessage()); - } catch (IOException ioex) { - throw new ExportException("IO Exception thrown exporting as " + "export_" + format + ".cached"); - } - - } catch (IOException ioex) { - // This catches any problem creating a local temp file in the catch clause above - throw new ExportException("IO Exception thrown before exporting as " + "export_" + format + ".cached"); - } finally { - IOUtils.closeQuietly(outputStream); - } - - } - - private void clearCachedExport(Dataset dataset, String format) throws IOException { - try { - StorageIO storageIO = getStorageIO(dataset); - storageIO.deleteAuxObject("export_" + format + ".cached"); - - } catch (IOException ex) { - throw new IOException("IO Exception caught deleting export_" + format + ".cached"); - } - } - - // This method checks if the metadata has already been exported in this - // format and cached on disk. If it has, it'll open the file and retun - // the file input stream. If not, it'll return null. - private InputStream getCachedExportFormat(Dataset dataset, String formatName) throws ExportException, IOException { - - StorageIO dataAccess = null; - - try { - dataAccess = DataAccess.getStorageIO(dataset); - } catch (IOException ioex) { - throw new IOException("IO Exception thrown exporting as " + "export_" + formatName + ".cached", ioex); - } - - InputStream cachedExportInputStream = null; - - try { - cachedExportInputStream = dataAccess.getAuxFileAsInputStream("export_" + formatName + ".cached"); - return cachedExportInputStream; - } catch (IOException ioex) { - throw new IOException("IO Exception thrown exporting as " + "export_" + formatName + ".cached", ioex); - } - - } - - /* - * The below method, getCachedExportSize(), is not currently used. An exercise - * for the reader could be to refactor it if it's needed to be compatible with - * storage drivers other than local filesystem. Files.exists() would need to be - * discarded. -- L.A. 4.8 - */ -// public Long getCachedExportSize(Dataset dataset, String formatName) { -// try { -// if (dataset.getFileSystemDirectory() != null) { -// Path cachedMetadataFilePath = Paths.get(dataset.getFileSystemDirectory().toString(), "export_" + formatName + ".cached"); -// if (Files.exists(cachedMetadataFilePath)) { -// return cachedMetadataFilePath.toFile().length(); -// } -// } -// } catch (Exception ioex) { -// // don't do anything - we'll just return null -// } -// -// return null; -// } - public Boolean isXMLFormat(String provider) { - Exporter e = exporterMap.get(provider); - if (e != null) { - return e instanceof XMLExporter; - } - return null; - } - - public String getMediaType(String provider) { - Exporter e = exporterMap.get(provider); - if (e != null) { - return e.getMediaType(); - } - return MediaType.TEXT_PLAIN; - } - -} diff --git a/src/main/java/edu/harvard/iq/dataverse/export/service/ExportCache.java b/src/main/java/edu/harvard/iq/dataverse/export/service/ExportCache.java new file mode 100644 index 00000000000..d4c2f2fe28e --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/export/service/ExportCache.java @@ -0,0 +1,46 @@ +package edu.harvard.iq.dataverse.export.service; + +import edu.harvard.iq.dataverse.Dataset; +import io.gdcc.spi.export.ExportException; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Optional; + +/** + * Storage abstraction for cached metadata exports. Implementations own all + * knowledge about where and under which names cached exports live; the export + * pipeline only ever deals in {@link ExportCacheKey}s and streams. + */ +public sealed interface ExportCache permits StorageIOCache { + + /** + * Looks up a cached export. + * @return the cached export stream, or empty if none is cached. Note: the caller is responsible for closing the stream. + * @throws IOException on actual storage failures (not on a cache miss) + */ + Optional read(ExportCacheKey key) throws IOException; + + /** + * Produces and stores an export. The {@code writer} callback receives the output stream to write to. + * Any implementations guarantee that a partially written export is never made visible under the cache key + * (i.e., a failed write leaves either the previous entry or no entry). + */ + void write(ExportCacheKey key, ExportStreamWriter writer) throws ExportException, IOException; + + /** Removes a cached export. Absence of the entry is not an error. */ + void evict(ExportCacheKey key) throws IOException; + + /** + * Removes all cached exports for a dataset, across all versions and formats, including legacy (pre-versioning) entries. + * Intended for publish/deaccession hooks and the admin "reexport" API. + */ + void evictAll(Dataset dataset) throws IOException; + + /** Callback that renders an export into the store-provided stream. */ + @FunctionalInterface + interface ExportStreamWriter { + void writeTo(OutputStream out) throws ExportException, IOException; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/export/service/ExportCacheInvalidator.java b/src/main/java/edu/harvard/iq/dataverse/export/service/ExportCacheInvalidator.java new file mode 100644 index 00000000000..5208098285e --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/export/service/ExportCacheInvalidator.java @@ -0,0 +1,23 @@ +package edu.harvard.iq.dataverse.export.service; + +import java.util.List; + +/** + * Represents an abstraction for determining whether a cached export needs to be invalidated and regenerated. + *

+ * This sealed interface is intended to enforce a controlled hierarchy of classes that implement the cache + * invalidation logic, ensuring behavior consistency across different implementations. If necessary, the contract + * may be altered to allow more dynamic discovery of invalidators. + */ +public sealed interface ExportCacheInvalidator permits FileEmbargoExpiryInvalidator { + + /** + * A collection of {@link ExportCacheInvalidator} instances. + * This list is intended to centralize all invalidation mechanisms for export cache entries. + * Any new implementations must be added here in addition to the "permits" on the interface seal. + */ + List invalidators = List.of(new FileEmbargoExpiryInvalidator()); + + /** Should a cached export for this key be discarded and regenerated? */ + boolean isStale(ExportCacheKey key); +} diff --git a/src/main/java/edu/harvard/iq/dataverse/export/service/ExportCacheKey.java b/src/main/java/edu/harvard/iq/dataverse/export/service/ExportCacheKey.java new file mode 100644 index 00000000000..be1068f1a53 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/export/service/ExportCacheKey.java @@ -0,0 +1,48 @@ +package edu.harvard.iq.dataverse.export.service; + +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetVersion; + +import java.util.Objects; + +/** + * This record encapsulates information related to the dataset, the version of the dataset, + * and the format name used for the export, enabling precise identification + * of cache entries for export operations. + */ +public record ExportCacheKey(Dataset dataset, DatasetVersion version, String formatName) { + + /** + * Constructs an ExportCacheKey instance with the specified dataset, dataset version, and format name. + * @param dataset the dataset associated with this cache key; must not be null + * @param version the dataset version associated with this cache key; must not be null + * @param formatName the format name used for export operations; must not be null or blank + * @throws NullPointerException if the dataset, version, or formatName is null + * @throws IllegalArgumentException if the formatName is blank or empty + */ + public ExportCacheKey(Dataset dataset, DatasetVersion version, String formatName) { + this.dataset = Objects.requireNonNull(dataset); + this.version = Objects.requireNonNull(version); + if (Objects.requireNonNull(formatName).isBlank()) { + throw new IllegalArgumentException("formatName must not be blank or empty"); + } + this.formatName = formatName; + } + + /** + * Convenience wrapper to create a cache key fro ma version and format alone. + * Note: the entity object must have a reference to the dataset present! + * @param version the dataset version + * @param formatName the target format + * @throws NullPointerException if either version, the dataset in the version or the format are null + * @throws IllegalArgumentException if the format name is blank or empty + */ + public ExportCacheKey(DatasetVersion version, String formatName) { + this(Objects.requireNonNull(version).getDataset(), version, formatName); + } + + /** The one canonical, version-qualified aux tag. */ + public String auxTag() { + return "export_" + formatName + "_" + version.getFriendlyVersionNumber() + ".cached"; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/export/service/ExportServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/export/service/ExportServiceBean.java new file mode 100644 index 00000000000..9886c2614aa --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/export/service/ExportServiceBean.java @@ -0,0 +1,297 @@ +package edu.harvard.iq.dataverse.export.service; + +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetVersion; +import io.gdcc.spi.export.ExportException; +import io.gdcc.spi.export.Exporter; +import io.gdcc.spi.export.XMLExporter; +import jakarta.ejb.EJB; +import jakarta.ejb.Stateless; +import jakarta.ws.rs.core.MediaType; +import org.apache.commons.io.IOUtils; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.sql.Timestamp; +import java.util.Date; +import java.util.List; +import java.util.ServiceConfigurationError; +import java.util.logging.Level; +import java.util.logging.Logger; + +@Stateless +public class ExportServiceBean { + + private static final Logger logger = Logger.getLogger(ExportServiceBean.class.getCanonicalName()); + + @EJB + ExporterRegistryBean exporterRegistry; + + public InputStream getExport(DatasetVersion datasetVersion, String formatName) throws ExportException, IOException { + + Dataset dataset = datasetVersion.getDataset(); + InputStream exportInputStream = null; + + if (datasetVersion.isDraft()) { + // For drafts we create the export on the fly rather than caching. + Exporter exporter = exporterMap.get(formatName); + if (exporter != null) { + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + // getPrerequisiteFormatName logic copied from exportFormat() + if (exporter.getPrerequisiteFormatName().isPresent()) { + String prereqFormatName = exporter.getPrerequisiteFormatName().get(); + try (InputStream preReqStream = getExport(datasetVersion, prereqFormatName)) { + InternalExportDataProvider dataProvider = new InternalExportDataProvider(datasetVersion, preReqStream); + exporter.exportDataset(dataProvider, outputStream); + } catch (IOException ioe) { + throw new ExportException("Could not get prerequisite " + prereqFormatName + " to create " + formatName + " export for dataset " + dataset.getId(), ioe); + } + } else { + InternalExportDataProvider dataProvider = new InternalExportDataProvider(datasetVersion); + exporter.exportDataset(dataProvider, outputStream); + } + return new ByteArrayInputStream(outputStream.toByteArray()); + } + } + } else { + // for non-drafts (published versions) we try to locate an already existing, cached export + exportInputStream = getCachedExportFormat(dataset, formatName); + } + + if (exportInputStream != null) { + return exportInputStream; + } + + // if it doesn't exist, we'll try to run the export: + exportFormat(dataset, formatName); + + // and then try again: + exportInputStream = getCachedExportFormat(dataset, formatName); + + if (exportInputStream != null) { + return exportInputStream; + } + + // if there is no cached export still - we have to give up and throw + // an exception! + throw new ExportException("Failed to export the dataset as " + formatName); + + } + + public String getLatestPublishedAsString(Dataset dataset, String formatName) { + if (dataset == null) { + return null; + } + DatasetVersion releasedVersion = dataset.getReleasedVersion(); + if (releasedVersion == null) { + return null; + } + InputStream inputStream = null; + InputStreamReader inp = null; + try { + inputStream = getExport(releasedVersion, formatName); + if (inputStream != null) { + inp = new InputStreamReader(inputStream, "UTF8"); + BufferedReader br = new BufferedReader(inp); + StringBuilder sb = new StringBuilder(); + String line; + while ((line = br.readLine()) != null) { + sb.append(line); + sb.append('\n'); + } + br.close(); + inp.close(); + inputStream.close(); + return sb.toString(); + } + } catch (IOException ex) { + logger.log(Level.FINE, ex.getMessage(), ex); + return null; + } finally { + IOUtils.closeQuietly(inp); + IOUtils.closeQuietly(inputStream); + } + return null; + + } + + // A convenience wrapper method; the actual implementation has been moved + // into exportFormats() below. + public void exportAllFormats(Dataset dataset) throws ExportException { + exportFormats(dataset, List.of()); + } + + /** + * This method is added to supplement the classic exportAllFormats() in order + * to allow the metadata export APIs to selectively re-export only the formats + * specified. This is to finally allow an instance admin to avoid running + * a complete, from-scratch reexport when only _some_, or just one of them + * actually needs to be refreshed. On a large instance this can waste a + * significant amount of time and CPU cycles. (new as of 6.12) + * This method calls the cacheExport() method for every valid/supported + * format name supplied, or for every Exporter available, if an empty List + * is passed. + * Only the latest published version is used for exports. + * exportAllFormats() above is now a convenience wrapper, with the + * implementation moved here. + * + * @param dataset + * @param formatNames + * @throws ExportException + */ + public void exportFormats(Dataset dataset, List formatNames) throws ExportException { + if (dataset == null) { + throw new ExportException("exportFormats called with null Dataset"); + } + + if (formatNames == null) { + throw new ExportException("exportFormats called with null formatNames (use an empty List for \"all\""); + } + + try { + clearCachedFormats(dataset, formatNames); + } catch (IOException ex) { + Logger.getLogger(ExportService.class.getName()).log(Level.SEVERE, null, ex); + } + + try { + DatasetVersion releasedVersion = dataset.getReleasedVersion(); + if (releasedVersion == null) { + throw new ExportException("No released version for dataset " + dataset.getGlobalId().toString()); + } + InternalExportDataProvider dataProvider = new InternalExportDataProvider(releasedVersion); + + for (Exporter e : exporterMap.values()) { + String formatName = e.getFormatName(); + if (formatNames.isEmpty() || formatNames.contains(formatName)) { + if (e.getPrerequisiteFormatName().isPresent()) { + String prereqFormatName = e.getPrerequisiteFormatName().get(); + try (InputStream preReqStream = getExport(dataset.getReleasedVersion(), prereqFormatName)) { + dataProvider.setPrerequisiteInputStream(preReqStream); + cacheExport(dataset, dataProvider, formatName, e); + dataProvider.setPrerequisiteInputStream(null); + } catch (IOException ioe) { + throw new ExportException("Could not get prerequisite " + e.getPrerequisiteFormatName() + " to create " + formatName + "export for dataset " + dataset.getId(), ioe); + } + } else { + cacheExport(dataset, dataProvider, formatName, e); + } + } + } + // Finally, if we have been able to successfully export in all available + // formats, we'll increment the "last exported" time stamp: + if (formatNames.isEmpty()) { + dataset.setLastExportTime(new Timestamp(new Date().getTime())); + } + + } catch (ServiceConfigurationError serviceError) { + throw new ExportException("Service configuration error during export. " + serviceError.getMessage()); + } catch (RuntimeException e) { + logger.log(Level.FINE, e.getMessage(), e); + throw new ExportException( + "Unknown runtime exception exporting metadata. " + (e.getMessage() == null ? "" : e.getMessage())); + } + } + + // A convenience wrapper method + public void clearAllCachedFormats(Dataset dataset) throws IOException { + clearCachedFormats(dataset, List.of()); + dataset.setLastExportTime(null); + } + + public void clearCachedFormats(Dataset dataset, List formatNames) throws IOException { + if (dataset == null) { + throw new ExportException("cleareCachedFormats called with null Dataset"); + } + + if (formatNames == null) { + throw new ExportException("clearCachedFormats called with null formatNames (use an empty List for \"all\""); + } + + for (Exporter e : exporterMap.values()) { + String formatName = e.getFormatName(); + if (formatNames.isEmpty() || formatNames.contains(formatName)) { + try { + clearCachedExport(dataset, formatName); + } catch (IOException ex) { + // not fatal + } + } + } + } + + // This method finds the exporter for the format requested, + // then produces the dataset metadata as a JsonObject, then calls + // the "cacheExport()" method that will save the produced output + // in a file in the dataset directory. + public void exportFormat(Dataset dataset, String formatName) throws ExportException { + try { + + Exporter e = exporterMap.get(formatName); + if (e != null) { + DatasetVersion releasedVersion = dataset.getReleasedVersion(); + if (releasedVersion == null) { + throw new ExportException( + "No published version found during export. " + dataset.getGlobalId().toString()); + } + if(e.getPrerequisiteFormatName().isPresent()) { + String prereqFormatName = e.getPrerequisiteFormatName().get(); + try (InputStream preReqStream = getExport(releasedVersion, prereqFormatName)) { + InternalExportDataProvider dataProvider = new InternalExportDataProvider(releasedVersion, preReqStream); + cacheExport(dataset, dataProvider, formatName, e); + } catch (IOException ioe) { + throw new ExportException ("Could not get prerequisite " + e.getPrerequisiteFormatName() + " to create " + formatName + "export for dataset " + dataset.getId(), ioe); + } + } else { + InternalExportDataProvider dataProvider = new InternalExportDataProvider(releasedVersion); + cacheExport(dataset, dataProvider, formatName, e); + } + // As with exportAll, we should update the lastexporttime for the dataset + dataset.setLastExportTime(new Timestamp(new Date().getTime())); + } else { + throw new ExportException("Exporter not found"); + } + } catch (IllegalStateException e) { + // IllegalStateException can potentially mean very different, and + // unexpected things. An exporter attempting to get a single primitive + // value from a fieldDTO that is in fact a Multiple and contains a + // json vector (this has happened, for example, when the code in the + // DDI exporter was not updated following a metadata fieldtype change), + // will result in IllegalStateException. + throw new ExportException("IllegalStateException caught when exporting " + formatName + " for dataset " + + dataset.getGlobalId().toString() + + "; may or may not be due to a mismatch between an exporter code and a metadata block update. " + + e.getMessage()); + } + + } + + public Exporter getExporter(String formatName) throws ExportException { + Exporter e = exporterMap.get(formatName); + if (e != null) { + return e; + } + throw new ExportException("No such Exporter: " + formatName); + } + + public Boolean isXMLFormat(String provider) { + Exporter e = exporterMap.get(provider); + if (e != null) { + return e instanceof XMLExporter; + } + return null; + } + + public String getMediaType(String provider) { + Exporter e = exporterMap.get(provider); + if (e != null) { + return e.getMediaType(); + } + return MediaType.TEXT_PLAIN; + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/export/service/ExporterRegistryBean.java b/src/main/java/edu/harvard/iq/dataverse/export/service/ExporterRegistryBean.java new file mode 100644 index 00000000000..66eac0039af --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/export/service/ExporterRegistryBean.java @@ -0,0 +1,176 @@ +package edu.harvard.iq.dataverse.export.service; + +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.BundleUtil; +import io.gdcc.spi.export.Exporter; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import jakarta.ejb.Lock; +import jakarta.ejb.LockType; +import jakarta.ejb.Singleton; +import jakarta.ejb.Startup; + +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.ServiceLoader; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * ExporterRegistry is responsible for managing the registration, retrieval, and lifecycle of {@code Exporter}s. + * It dynamically loads exporters from external JAR files and provides access to those exporters via their format names. + *

+ * This class is designed as a Jakarta EJB Singleton and is initialized at application startup. + * It uses a non-modifiable {@link Map} internally to store exporters under their format name, ensuring the state of + * the map is always consistent and thread-safe. + *

+ * Key responsibilities: + *

    + *
  • Locates and loads exporter JAR files from a specified directory.
    • + *
  • Use {@code ServiceLoader} to discover and register {@code Exporter} implementations dynamically.
    • + *
  • Allows external exporters to replace internal ones for the same format name.
    • + *
  • Provides thread-safe access to registered exporters and their metadata.
    • + *
+ * @implNote

Note on Concurrency: EJB singletons use container-managed concurrency by default, where every business + * method implicitly runs under an exclusive {@code @Lock(LockType.WRITE)}, meaning only one caller at + * a time may use the bean. Since this registry is populated once in and is effectively immutable afterwards, + * that exclusivity is unnecessary.

+ *

The class-level {@code @Lock(LockType.READ)} instead allows any number of callers to read from the + * registry concurrently, avoiding an application-wide bottleneck on exporter lookups. If a method that + * mutates the registry is ever added (e.g. a reload operation), it must be annotated with + * {@code @Lock(LockType.WRITE)} to regain exclusive access for that method.

+ */ +@Singleton +@Startup +@Lock(LockType.READ) +public class ExporterRegistryBean { + + /** + * Represents a set of labels associated with an exporter. + */ + public record Labels( + String localizedDisplayName, + String formatName + ) {} + + private static final Logger logger = Logger.getLogger(ExporterRegistryBean.class.getCanonicalName()); + + // When the class is initialized, the exporter map is an empty, non-modifiable map (key = format name). + // Once the exporters have been located and loaded, the map is replaced, fully loaded, still unmodifiable. + // No half-initialized state is exposable this way. Future optimizations may use @Lock on it, too, for example, + // when implementing a reload mechanism. + private Map exporters = Map.of(); + // Caching the classloader used to load plugin JAR files, keeping it open, will allow reuse for reloads + // or loading more resources from plugin JARs. May be dropped later if not necessary. + private URLClassLoader exporterClassLoader; + + /** + * Retrieves an exporter associated with the specified format name. + * + * @param formatName the name of the format for which to retrieve the exporter + * @return an {@code Optional} containing the exporter if found, or + * an empty {@code Optional} if no exporter is associated with the given format name + */ + public Optional get(String formatName) { + return Optional.ofNullable(exporters.get(formatName)); + } + + /** + * Retrieves a list of all registered exporters in the system. + * @return an unmodifiable list of {@link Exporter} instances representing all the exporters currently available + */ + public List getAll() { + return List.copyOf(exporters.values()); + } + + /** + * Retrieves a list of {@link Labels} representing the exporters registered in the system. + * @return a list of {@code Labels} objects + */ + public List getLabels() { + return exporters.values().stream() + .map(exporter -> new Labels( + exporter.getDisplayName(BundleUtil.getCurrentLocale()), + exporter.getFormatName())) + .toList(); + } + + @PostConstruct + private void initialize() { + /* + * Step 1 - find the EXPORTERS dir and add all jar files there to a class loader + */ + List jarUrls = new ArrayList<>(); + Optional exportPathSetting = JvmSettings.EXPORTERS_DIRECTORY.lookupOptional(String.class); + if (exportPathSetting.isPresent()) { + Path exporterDir = Paths.get(exportPathSetting.get()); + // Get all JAR files from the configured directory + try (DirectoryStream stream = Files.newDirectoryStream(exporterDir, "*.jar")) { + // Using the foreach loop here to enable catching the URI/URL exceptions + for (Path path : stream) { + logger.log(Level.FINE, "Adding {0}", path.toUri().toURL()); + // This is the syntax required to indicate a jar file from which classes should + // be loaded (versus a class file). + jarUrls.add(new URL("jar:" + path.toUri().toURL() + "!/")); + } + } catch (IOException e) { + logger.warning("Problem accessing external Exporters: " + e.getLocalizedMessage()); + } + } + this.exporterClassLoader = URLClassLoader.newInstance(jarUrls.toArray(new URL[0]), this.getClass().getClassLoader()); + + /* + * Step 2 - load all Exporters that can be found, using the jars as additional sources + */ + ServiceLoader loader = ServiceLoader.load(Exporter.class, this.exporterClassLoader); + + /* + * Step 3 - Fill exporterMap with providerName as the key, allow external + * exporters to replace internal ones for the same providerName. FWIW: From the + * logging it appears that ServiceLoader returns classes in ~ alphabetical order + * rather than by class loader, so internal classes handling a given + * providerName may be processed before or after external ones. + */ + Map loadedExporters = new HashMap<>(); + loader.forEach(exp -> { + String formatName = exp.getFormatName(); + // If no entry for this providerName yet or if it is an external exporter + if (!exporters.containsKey(formatName) || exp.getClass().getClassLoader().equals(this.exporterClassLoader)) { + loadedExporters.put(formatName, exp); + } + logger.log( + Level.FINE, + "SL: {0} from {1} and classloader: {2}", + new Object[]{ + formatName, + exp.getClass().getCanonicalName(), + exp.getClass().getClassLoader().getClass().getCanonicalName() + }); + }); + this.exporters = loadedExporters; + + } + + @PreDestroy + private void tearDown() { + if (exporterClassLoader == null) { + return; + } + + try { + exporterClassLoader.close(); + } catch (IOException e) { + logger.log(Level.WARNING, "Could not close exporter classloader", e); + } + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/export/service/FileEmbargoExpiryInvalidator.java b/src/main/java/edu/harvard/iq/dataverse/export/service/FileEmbargoExpiryInvalidator.java new file mode 100644 index 00000000000..e9367f2b3ad --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/export/service/FileEmbargoExpiryInvalidator.java @@ -0,0 +1,68 @@ +package edu.harvard.iq.dataverse.export.service; + +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.Embargo; +import edu.harvard.iq.dataverse.FileMetadata; + +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.Date; +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Logger; + +/** + * The {@code FileEmbargoExpiryInvalidator} class implements the {@link ExportCacheInvalidator} interface to determine + * whether a cached export should be invalidated due to the expiration of an embargo on any file within a dataset. + * This invalidation ensures that stale cached exports do not persist beyond the embargo period. + *

+ * Note: This code was originally a part of {@code ExportService}, written mostly by qqmyers. + * Back there it was targeting DDI format only, but with pluggable exports, any format may export file metadata. + */ +public final class FileEmbargoExpiryInvalidator implements ExportCacheInvalidator { + + private static final Logger logger = Logger.getLogger(FileEmbargoExpiryInvalidator.class.getCanonicalName()); + + @Override + public boolean isStale(ExportCacheKey key) { + return isStaleDueToExpiredEmbargo(key.dataset()); + } + + /** + * Checks whether a cached export has been rendered stale because an embargo + * on one of the dataset's files ended after the last export ran. + */ + private boolean isStaleDueToExpiredEmbargo(Dataset dataset) { + Date lastExportDate = dataset.getLastExportTime(); + // if lastExportDate == null, assume it's not set because we're exporting for the + // first time now (e.g. during publish) and therefore no changes are needed + if (lastExportDate == null) { + return false; + } + LocalDate exportLocalDate = lastExportDate.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); + logger.fine("Last export date: " + exportLocalDate); + // Track which embargoes we've already checked + Set embargoIds = new HashSet<>(); + // Check for all files in the latest released version + for (FileMetadata fm : dataset.getLatestVersionForCopy().getFileMetadatas()) { + // ToDo? This loop is necessary because we have not stored the date when the + // next embargo in this datasetversion will end. If we knew that (another + // dataset/datasetversion column), we could make one check that nextembargoEnd + // exists and is after the last export and before now versus scanning through + // files until we potentially find such an embargo. + Embargo e = fm.getDataFile().getEmbargo(); + if (e == null || embargoIds.contains(e.getId())) { + continue; + } + logger.fine("Datafile: " + fm.getDataFile().getId() + ", embargo end date: " + e.getFormattedDateAvailable()); + if (e.getDateAvailable().isAfter(exportLocalDate) && e.getDateAvailable().isBefore(LocalDate.now(ZoneId.systemDefault()))) { + // The embargo ended after the last export and before the current date, + // so the cached export needs to be refreshed. + logger.fine("Request that the cached export be cleared."); + return true; + } + embargoIds.add(e.getId()); + } + return false; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java b/src/main/java/edu/harvard/iq/dataverse/export/service/InternalExportDataProvider.java similarity index 99% rename from src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java rename to src/main/java/edu/harvard/iq/dataverse/export/service/InternalExportDataProvider.java index 4a04aea41e3..9093577ca71 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/service/InternalExportDataProvider.java @@ -1,4 +1,4 @@ -package edu.harvard.iq.dataverse.export; +package edu.harvard.iq.dataverse.export.service; import java.io.InputStream; import java.util.Optional; diff --git a/src/main/java/edu/harvard/iq/dataverse/export/service/StorageIOCache.java b/src/main/java/edu/harvard/iq/dataverse/export/service/StorageIOCache.java new file mode 100644 index 00000000000..7ca5aaa33dc --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/export/service/StorageIOCache.java @@ -0,0 +1,147 @@ +package edu.harvard.iq.dataverse.export.service; + +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.dataaccess.DataAccess; +import edu.harvard.iq.dataverse.dataaccess.StorageIO; +import io.gdcc.spi.export.ExportException; +import jakarta.enterprise.context.ApplicationScoped; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * {@link ExportCache} backed by Dataverse's {@link StorageIO} layer, storing exports as auxiliary objects alongside the dataset. + *

+ * Naming Schema: The canonical "aux tag" is version-qualified ({@code export__.cached}, + * see {@link ExportCacheKey#auxTag()}) and is the only name ever written. + *

+ * The legacy, unqualified name ({@code export_.cached}) predates version qualification and only ever described + * the latest released version. It is therefore consulted as a read fallback exclusively for that version. + * It will be deleted alongside the canonical name on eviction, so a stale legacy entry can never resurrect an invalidated export. + *

+ * Write Atomicity: Exports are always rendered to a local temp file first. + * Then it gets persisted via {@link StorageIO#savePathAsAux(Path, String)} as an auxiliary dataset file. + *

+ * Note: This class replaces the former {@code ExportService.cacheExport()} method, mostly written by qqmyers. + * Instead of its "try openAuxChannel, fall back to temp file for S3/Swift" branching, there now is one code path for all drivers. + * Readers can never observe a half-written export under the cache key. The cost is one extra local write per export, + * which is negligible next to export generation itself. + *

+ * Note 2: This class is an application scoped CDI bean (single instance). The cache itself is stateless, + * and every operation operates on their own {@code StorageIO}. But: if we add a write lock later on to avoid race + * conditions during writes, we will require an instance wide single map to store these locks, which CDI gives us for free. + * In addition, one might use a Hazelcast-backed map to acquire multi-instance wide locks! + * And lastly, making this an injectable CDI bean makes mocking it in tests very easy. + */ +@ApplicationScoped +public final class StorageIOCache implements ExportCache { + + private static final Logger logger = Logger.getLogger(StorageIOCache.class.getCanonicalName()); + + private static final String TAG_PREFIX = "export_"; + private static final String TAG_SUFFIX = ".cached"; + + /** + * Reads an input stream associated with the given export cache key. + * + * @param key the export cache key containing dataset, format, and versioning information. + * @return an {@code Optional} containing the input stream if available, otherwise an empty {@code Optional}. + * @throws IOException if an I/O error occurs while attempting to read the data. + */ + @Override + public Optional read(ExportCacheKey key) throws IOException { + StorageIO storage = storageFor(key.dataset()); + return tryRead(storage, key.auxTag()); + } + + /** + * Writes the export cache data to a temporary file and ensures it is properly persisted to the dataset's storage. + * Handles file cleanup to maintain system integrity. + * @param key The {@code ExportCacheKey} representing the metadata export about to be cached. + * @param writer The {@code ExportStreamWriter} functional interface implementation responsible for writing data + * to the output stream. This wraps the underlying exporter, writing the actual data format. + * @throws ExportException If an error occurs during the export process. + * @throws IOException If an I/O error occurs while creating, writing, or managing the temporary file. + */ + @Override + public void write(ExportCacheKey key, ExportStreamWriter writer) throws ExportException, IOException { + Path tempFile = Files.createTempFile("dataverse-export-", ".tmp"); + try { + try (OutputStream out = new BufferedOutputStream(Files.newOutputStream(tempFile))) { + writer.writeTo(out); + } + // Persist to storage only after the metadata export has been fully and successfully rendered. + // A failure above leaves the cache untouched. + storageFor(key.dataset()).savePathAsAux(tempFile, key.auxTag()); + logger.log(Level.FINE, key.version() + ": Cached export written: {0}", key.auxTag()); + } finally { + try { + Files.deleteIfExists(tempFile); + } catch (IOException e) { + // Warn, but do not fail if the temp file could not be deleted. (The main operation was a success) + logger.log(Level.WARNING, e, () -> key.version() + ": could not delete export temp file " + tempFile); + } + } + } + + @Override + public void evict(ExportCacheKey key) throws IOException { + deleteQuietly(storageFor(key.dataset()), key.auxTag()); + } + + @Override + public void evictAll(Dataset dataset) throws IOException { + StorageIO storage = storageFor(dataset); + List auxTags = storage.listAuxObjects(); + for (String tag : auxTags) { + if (tag.startsWith(TAG_PREFIX) && tag.endsWith(TAG_SUFFIX)) { + deleteQuietly(storage, tag); + } + } + } + + private static Optional tryRead(StorageIO storage, String auxTag) { + // Distinguish "not cached" (normal, frequent) from actual failures: only read if the aux object exists. + try { + if (!storage.isAuxObjectCached(auxTag)) { + return Optional.empty(); + } + } catch (IOException e) { + // Treat as a "cache miss" so the pipeline regenerates rather than failing over a cache IO issue. + logger.log(Level.FINE, e, () -> "Existence check failed for " + auxTag); + return Optional.empty(); + } + try { + return Optional.of(storage.getAuxFileAsInputStream(auxTag)); + } catch (IOException e) { + // Exists-then-vanished race, or a genuine storage problem. + // Treated as a "cache miss" so the pipeline regenerates rather than failing over a cache IO issue. + logger.log(Level.WARNING, e, () -> "Could not open cached export " + auxTag); + return Optional.empty(); + } + } + + private static void deleteQuietly(StorageIO storage, String auxTag) { + try { + storage.deleteAuxObject(auxTag); + } catch (IOException e) { + // Absence is the common case here and not an error. + // Real failures are logged but non-fatal, as the entry will be overwritten or ignored on the next pipeline run. + logger.log(Level.FINE, e, () -> "Could not delete aux object " + auxTag); + } + } + + // Extracted to static method to avoid repeating it in multiple places, allowing substituion + // and extension to a StorageProvider functional interface (which is mockable on its own). + private static StorageIO storageFor(Dataset dataset) throws IOException { + return DataAccess.getStorageIO(dataset); + } +} diff --git a/src/test/java/edu/harvard/iq/dataverse/export/HugeDatasetExportPerformanceIT.java b/src/test/java/edu/harvard/iq/dataverse/export/service/HugeDatasetExportPerformanceIT.java similarity index 98% rename from src/test/java/edu/harvard/iq/dataverse/export/HugeDatasetExportPerformanceIT.java rename to src/test/java/edu/harvard/iq/dataverse/export/service/HugeDatasetExportPerformanceIT.java index 63bf826167d..afd340a6613 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/HugeDatasetExportPerformanceIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/service/HugeDatasetExportPerformanceIT.java @@ -1,4 +1,4 @@ -package edu.harvard.iq.dataverse.export; +package edu.harvard.iq.dataverse.export.service; import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.Dataset; diff --git a/src/test/java/edu/harvard/iq/dataverse/export/InternalExportProviderTest.java b/src/test/java/edu/harvard/iq/dataverse/export/service/InternalExportProviderTest.java similarity index 97% rename from src/test/java/edu/harvard/iq/dataverse/export/InternalExportProviderTest.java rename to src/test/java/edu/harvard/iq/dataverse/export/service/InternalExportProviderTest.java index c072788735e..d794f626602 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/InternalExportProviderTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/service/InternalExportProviderTest.java @@ -1,4 +1,4 @@ -package edu.harvard.iq.dataverse.export; +package edu.harvard.iq.dataverse.export.service; import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.DataTable; diff --git a/src/test/java/edu/harvard/iq/dataverse/export/TabularDataExportIT.java b/src/test/java/edu/harvard/iq/dataverse/export/service/TabularDataExportIT.java similarity index 99% rename from src/test/java/edu/harvard/iq/dataverse/export/TabularDataExportIT.java rename to src/test/java/edu/harvard/iq/dataverse/export/service/TabularDataExportIT.java index b28f7b52376..4f4d235e5a0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/TabularDataExportIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/service/TabularDataExportIT.java @@ -1,4 +1,4 @@ -package edu.harvard.iq.dataverse.export; +package edu.harvard.iq.dataverse.export.service; import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.Dataset;