From 34860317277a06256726a3f90160f6eea282c8a4 Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Tue, 31 Mar 2026 20:32:36 -0400 Subject: [PATCH 1/3] Add support for heap dump analysis --- .../io/cryostat/reports/HeapDumpFormData.java | 35 +++++ .../reports/PresignedHeapDumpFormData.java | 36 +++++ .../java/io/cryostat/reports/Producers.java | 9 ++ .../io/cryostat/reports/ReportResource.java | 126 ++++++++++++++++++ 4 files changed, 206 insertions(+) create mode 100644 src/main/java/io/cryostat/reports/HeapDumpFormData.java create mode 100644 src/main/java/io/cryostat/reports/PresignedHeapDumpFormData.java diff --git a/src/main/java/io/cryostat/reports/HeapDumpFormData.java b/src/main/java/io/cryostat/reports/HeapDumpFormData.java new file mode 100644 index 0000000..c8e06bb --- /dev/null +++ b/src/main/java/io/cryostat/reports/HeapDumpFormData.java @@ -0,0 +1,35 @@ +/* + * Copyright The Cryostat Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cryostat.reports; + +import jakarta.ws.rs.core.MediaType; +import org.jboss.resteasy.reactive.PartType; +import org.jboss.resteasy.reactive.RestForm; +import org.jboss.resteasy.reactive.multipart.FileUpload; + +public class HeapDumpFormData { + @RestForm + @PartType(MediaType.APPLICATION_OCTET_STREAM) + public FileUpload file; + + @RestForm + @PartType(MediaType.TEXT_PLAIN) + public String heapDumpId; + + @RestForm + @PartType(MediaType.TEXT_PLAIN) + public String jvmId; +} diff --git a/src/main/java/io/cryostat/reports/PresignedHeapDumpFormData.java b/src/main/java/io/cryostat/reports/PresignedHeapDumpFormData.java new file mode 100644 index 0000000..b669386 --- /dev/null +++ b/src/main/java/io/cryostat/reports/PresignedHeapDumpFormData.java @@ -0,0 +1,36 @@ +/* + * Copyright The Cryostat Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cryostat.reports; + +import java.net.URI; + +import jakarta.ws.rs.core.MediaType; +import org.jboss.resteasy.reactive.PartType; +import org.jboss.resteasy.reactive.RestForm; + +public class PresignedHeapDumpFormData { + @RestForm + @PartType(MediaType.TEXT_PLAIN) + public URI uri; + + @RestForm + @PartType(MediaType.TEXT_PLAIN) + public String heapDumpId; + + @RestForm + @PartType(MediaType.TEXT_PLAIN) + public String jvmId; +} diff --git a/src/main/java/io/cryostat/reports/Producers.java b/src/main/java/io/cryostat/reports/Producers.java index 78bf8a4..b93dab0 100644 --- a/src/main/java/io/cryostat/reports/Producers.java +++ b/src/main/java/io/cryostat/reports/Producers.java @@ -18,6 +18,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ForkJoinPool; +import io.cryostat.core.diagnostic.InterruptibleHeapDumpReportGenerator; import io.cryostat.core.reports.InterruptibleReportGenerator; import io.cryostat.core.util.RuleFilterParser; import io.cryostat.libcryostat.sys.FileSystem; @@ -40,6 +41,14 @@ InterruptibleReportGenerator produceReportGenerator() { singleThread ? Executors.newSingleThreadExecutor() : ForkJoinPool.commonPool()); } + @Produces + // RequestScoped so that each individual report generation request has its own interruptible + // generator with an independent task queueing thread which dispatches to the shared common pool + @RequestScoped + InterruptibleHeapDumpReportGenerator produceHeapDumpReportGenerator() { + return new InterruptibleHeapDumpReportGenerator(); + } + @Produces @ApplicationScoped RuleFilterParser produceRuleFilterParser() { diff --git a/src/main/java/io/cryostat/reports/ReportResource.java b/src/main/java/io/cryostat/reports/ReportResource.java index 7df22ad..472a3ac 100644 --- a/src/main/java/io/cryostat/reports/ReportResource.java +++ b/src/main/java/io/cryostat/reports/ReportResource.java @@ -43,6 +43,8 @@ import javax.net.ssl.TrustManagerFactory; import javax.net.ssl.X509TrustManager; +import io.cryostat.core.diagnostic.HeapDumpAnalysis; +import io.cryostat.core.diagnostic.InterruptibleHeapDumpReportGenerator; import io.cryostat.core.reports.InterruptibleReportGenerator; import io.cryostat.core.reports.InterruptibleReportGenerator.AnalysisResult; import io.cryostat.core.util.RuleFilterParser; @@ -107,6 +109,7 @@ public class ReportResource { Optional storageCertPath; @Inject InterruptibleReportGenerator generator; + @Inject InterruptibleHeapDumpReportGenerator heapDumpGenerator; @Inject RuleFilterParser rfp; @Inject FileSystem fs; @Inject ObjectMapper mapper; @@ -255,6 +258,129 @@ public String getReport(RoutingContext ctx, @BeanParam RecordingFormData form) } } + + @Blocking + @Path("heapdump/remote_report") + @Produces(MediaType.APPLICATION_JSON) + @Consumes(MediaType.MULTIPART_FORM_DATA) + @POST + public String getHeapDumpReportFromPresigned(RoutingContext ctx, @BeanParam PresignedHeapDumpFormData form) + throws IOException, URISyntaxException { + // TODO queue these requests so we don't overload ourselves, in particular by reading + // multiple Heap Dump files into memory at once for analysis. We should process these serially + // from the queue. If we are getting overloaded then our response time to each subsequent + // request will continue to grow unbounded, so at some point we should stop accepting + // requests when the queue is too long. + // Since this is a @Blocking method that runs on a worker thread pool, can we implement this + // serial queueing behaviour by simply synchronizing on a shared singleton resource ex. the + // generator instance? + // A better long-term solution would be to use a shared messaging queue between Cryostat and + // the report generators, so that Cryostat can put up a URL for a presigned recording to be + // processed and a free report generator can claim that work item and then post back the + // report response + long timeout = TimeUnit.MILLISECONDS.toNanos(Long.parseLong(timeoutMs)); + long start = System.nanoTime(); + + logger.debugv("Attempting to download presigned heap dump from {0}", form.uri); + HttpURLConnection httpConn = (HttpURLConnection) form.uri.toURL().openConnection(); + httpConn.setRequestMethod("GET"); + if (httpConn instanceof HttpsURLConnection) { + HttpsURLConnection httpsConn = (HttpsURLConnection) httpConn; + if (storageTlsIgnore) { + try { + httpsConn.setSSLSocketFactory( + ignoreSslContext(storageTlsVersion).getSocketFactory()); + } catch (Exception e) { + logger.error(e); + throw new InternalServerErrorException(e); + } + } else if (storageCaPath.isPresent() || storageCertPath.isPresent()) { + if (!(storageCaPath.isPresent() && storageCertPath.isPresent())) { + Exception e = + new IllegalStateException( + String.format( + "%s and %s must be both set or both unset", + "cryostat.storage.tls.ca.path", + "cryostat.storage.tls.cert.path")); + logger.error(e); + throw new InternalServerErrorException(e); + } + try { + httpsConn.setSSLSocketFactory( + trustSslCertContext( + storageTlsVersion, + storageCaPath.get(), + storageCertPath.get()) + .getSocketFactory()); + } catch (Exception e) { + logger.error(e); + throw new InternalServerErrorException(e); + } + } + if (!storageHostnameVerify) { + httpsConn.setHostnameVerifier((hostname, session) -> true); + } + } + if (storageAuthMethod.isPresent() && storageAuth.isPresent()) { + httpConn.setRequestProperty( + "Authorization", + String.format("%s %s", storageAuthMethod.get(), storageAuth.get())); + } + + assertContentLength(httpConn.getContentLengthLong()); + try (var stream = httpConn.getInputStream()) { + Future evalFuture = null; + + evalFuture = heapDumpGenerator.generateInterruptibly(form.jvmId, form.heapDumpId, stream); + long elapsed = System.nanoTime() - start; + ctxHelper(ctx, evalFuture); + return mapper.writeValueAsString( + evalFuture.get(timeout - elapsed, TimeUnit.NANOSECONDS)); + } catch (ExecutionException | InterruptedException e) { + logger.error(e); + throw new InternalServerErrorException(e); + } catch (TimeoutException e) { + logger.error(e); + throw new ServerErrorException(Response.Status.GATEWAY_TIMEOUT, e); + } catch (Exception e) { + logger.error(e); + throw e; + } finally { + httpConn.disconnect(); + } + } + + @Blocking + @Path("heapdump/report") + @Produces(MediaType.APPLICATION_JSON) + @Consumes(MediaType.MULTIPART_FORM_DATA) + @POST + public String getHeapDumpReport(RoutingContext ctx, @BeanParam HeapDumpFormData form) + throws IOException { + FileUpload upload = form.file; + + Pair> uploadResult = handleUpload(upload); + java.nio.file.Path file = uploadResult.getLeft(); + long timeout = TimeUnit.MILLISECONDS.toNanos(Long.parseLong(timeoutMs)); + long start = uploadResult.getRight().getLeft(); + long elapsed = uploadResult.getRight().getRight(); + + Future evalFuture = null; + + try (var stream = fs.newInputStream(file)) { + evalFuture = heapDumpGenerator.generateInterruptibly(form.jvmId, form.heapDumpId, stream); + ctxHelper(ctx, evalFuture); + return mapper.writeValueAsString( + evalFuture.get(timeout - elapsed, TimeUnit.NANOSECONDS)); + } catch (ExecutionException | InterruptedException e) { + throw new InternalServerErrorException(e); + } catch (TimeoutException e) { + throw new ServerErrorException(Response.Status.GATEWAY_TIMEOUT, e); + } finally { + cleanupHelper(evalFuture, file, upload.fileName(), start); + } + } + private void assertContentLength(long length) { if (memoryFactor <= 0) { return; From a6a2dc6f298a3a84a1ae307eabc2c4c3f5a44acc Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Tue, 9 Jun 2026 14:46:48 -0400 Subject: [PATCH 2/3] Bulkhead annotation for fault tolerance, refactor --- pom.xml | 4 + .../io/cryostat/reports/ReportResource.java | 108 ++++++------------ 2 files changed, 38 insertions(+), 74 deletions(-) diff --git a/pom.xml b/pom.xml index 9a46da9..ddefc2e 100644 --- a/pom.xml +++ b/pom.xml @@ -61,6 +61,10 @@ io.quarkus quarkus-rest-jackson + + io.quarkus + quarkus-smallrye-fault-tolerance + io.quarkus quarkus-container-image-docker diff --git a/src/main/java/io/cryostat/reports/ReportResource.java b/src/main/java/io/cryostat/reports/ReportResource.java index 472a3ac..94123bb 100644 --- a/src/main/java/io/cryostat/reports/ReportResource.java +++ b/src/main/java/io/cryostat/reports/ReportResource.java @@ -70,6 +70,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.eclipse.microprofile.faulttolerance.Bulkhead; import org.jboss.logging.Logger; import org.jboss.resteasy.reactive.multipart.FileUpload; import org.openjdk.jmc.common.io.IOToolkit; @@ -130,6 +131,7 @@ void onStart(@Observes StartupEvent ev) { @Produces(MediaType.TEXT_PLAIN) public void healthCheck() {} + @Bulkhead @Blocking @Path("remote_report") @Produces(MediaType.APPLICATION_JSON) @@ -154,52 +156,7 @@ public String getReportFromPresigned(RoutingContext ctx, @BeanParam PresignedFor logger.debugv("Attempting to download presigned recording from {0}", form.uri); HttpURLConnection httpConn = (HttpURLConnection) form.uri.toURL().openConnection(); - httpConn.setRequestMethod("GET"); - if (httpConn instanceof HttpsURLConnection) { - HttpsURLConnection httpsConn = (HttpsURLConnection) httpConn; - if (storageTlsIgnore) { - try { - httpsConn.setSSLSocketFactory( - ignoreSslContext(storageTlsVersion).getSocketFactory()); - } catch (Exception e) { - logger.error(e); - throw new InternalServerErrorException(e); - } - } else if (storageCaPath.isPresent() || storageCertPath.isPresent()) { - if (!(storageCaPath.isPresent() && storageCertPath.isPresent())) { - Exception e = - new IllegalStateException( - String.format( - "%s and %s must be both set or both unset", - "cryostat.storage.tls.ca.path", - "cryostat.storage.tls.cert.path")); - logger.error(e); - throw new InternalServerErrorException(e); - } - try { - httpsConn.setSSLSocketFactory( - trustSslCertContext( - storageTlsVersion, - storageCaPath.get(), - storageCertPath.get()) - .getSocketFactory()); - } catch (Exception e) { - logger.error(e); - throw new InternalServerErrorException(e); - } - } - if (!storageHostnameVerify) { - httpsConn.setHostnameVerifier((hostname, session) -> true); - } - } - if (storageAuthMethod.isPresent() && storageAuth.isPresent()) { - httpConn.setRequestProperty( - "Authorization", - String.format("%s %s", storageAuthMethod.get(), storageAuth.get())); - } - - assertContentLength(httpConn.getContentLengthLong()); - try (var stream = httpConn.getInputStream()) { + try (var stream = getPresignedObjectStream(httpConn)) { Predicate predicate = rfp.parse(form.filter); Future> evalMapFuture = null; @@ -224,6 +181,7 @@ public String getReportFromPresigned(RoutingContext ctx, @BeanParam PresignedFor } @Blocking + @Bulkhead @Path("report") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.MULTIPART_FORM_DATA) @@ -258,31 +216,7 @@ public String getReport(RoutingContext ctx, @BeanParam RecordingFormData form) } } - - @Blocking - @Path("heapdump/remote_report") - @Produces(MediaType.APPLICATION_JSON) - @Consumes(MediaType.MULTIPART_FORM_DATA) - @POST - public String getHeapDumpReportFromPresigned(RoutingContext ctx, @BeanParam PresignedHeapDumpFormData form) - throws IOException, URISyntaxException { - // TODO queue these requests so we don't overload ourselves, in particular by reading - // multiple Heap Dump files into memory at once for analysis. We should process these serially - // from the queue. If we are getting overloaded then our response time to each subsequent - // request will continue to grow unbounded, so at some point we should stop accepting - // requests when the queue is too long. - // Since this is a @Blocking method that runs on a worker thread pool, can we implement this - // serial queueing behaviour by simply synchronizing on a shared singleton resource ex. the - // generator instance? - // A better long-term solution would be to use a shared messaging queue between Cryostat and - // the report generators, so that Cryostat can put up a URL for a presigned recording to be - // processed and a free report generator can claim that work item and then post back the - // report response - long timeout = TimeUnit.MILLISECONDS.toNanos(Long.parseLong(timeoutMs)); - long start = System.nanoTime(); - - logger.debugv("Attempting to download presigned heap dump from {0}", form.uri); - HttpURLConnection httpConn = (HttpURLConnection) form.uri.toURL().openConnection(); + private InputStream getPresignedObjectStream(HttpURLConnection httpConn) { httpConn.setRequestMethod("GET"); if (httpConn instanceof HttpsURLConnection) { HttpsURLConnection httpsConn = (HttpsURLConnection) httpConn; @@ -321,16 +255,41 @@ public String getHeapDumpReportFromPresigned(RoutingContext ctx, @BeanParam Pres httpsConn.setHostnameVerifier((hostname, session) -> true); } } + logger.debugv("Attempting to download presigned heap dump from {0}", form.uri); if (storageAuthMethod.isPresent() && storageAuth.isPresent()) { httpConn.setRequestProperty( "Authorization", String.format("%s %s", storageAuthMethod.get(), storageAuth.get())); } - assertContentLength(httpConn.getContentLengthLong()); - try (var stream = httpConn.getInputStream()) { - Future evalFuture = null; + return httpConn.getInputStream(); + } + @Blocking + @Bulkhead + @Path("heapdump/remote_report") + @Produces(MediaType.APPLICATION_JSON) + @Consumes(MediaType.MULTIPART_FORM_DATA) + @POST + public String getHeapDumpReportFromPresigned(RoutingContext ctx, @BeanParam PresignedHeapDumpFormData form) + throws IOException, URISyntaxException { + // TODO queue these requests so we don't overload ourselves, in particular by reading + // multiple Heap Dump files into memory at once for analysis. We should process these serially + // from the queue. If we are getting overloaded then our response time to each subsequent + // request will continue to grow unbounded, so at some point we should stop accepting + // requests when the queue is too long. + // Since this is a @Blocking method that runs on a worker thread pool, can we implement this + // serial queueing behaviour by simply synchronizing on a shared singleton resource ex. the + // generator instance? + // A better long-term solution would be to use a shared messaging queue between Cryostat and + // the report generators, so that Cryostat can put up a URL for a presigned recording to be + // processed and a free report generator can claim that work item and then post back the + // report response + long timeout = TimeUnit.MILLISECONDS.toNanos(Long.parseLong(timeoutMs)); + long start = System.nanoTime(); + HttpURLConnection httpConn = (HttpURLConnection) form.uri.toURL().openConnection(); + try (var stream = getPresignedObjectStream(httpConn)) { + Future evalFuture = null; evalFuture = heapDumpGenerator.generateInterruptibly(form.jvmId, form.heapDumpId, stream); long elapsed = System.nanoTime() - start; ctxHelper(ctx, evalFuture); @@ -351,6 +310,7 @@ public String getHeapDumpReportFromPresigned(RoutingContext ctx, @BeanParam Pres } @Blocking + @Bulkhead @Path("heapdump/report") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.MULTIPART_FORM_DATA) From 50822701b65a7c0ab1fe71084c33f58cda02f382 Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Tue, 9 Jun 2026 16:17:43 -0400 Subject: [PATCH 3/3] Adjust after core refactor --- src/main/java/io/cryostat/reports/Producers.java | 9 ++++++--- .../java/io/cryostat/reports/ReportResource.java | 15 +++++++++------ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/main/java/io/cryostat/reports/Producers.java b/src/main/java/io/cryostat/reports/Producers.java index b93dab0..0a9baaa 100644 --- a/src/main/java/io/cryostat/reports/Producers.java +++ b/src/main/java/io/cryostat/reports/Producers.java @@ -18,7 +18,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ForkJoinPool; -import io.cryostat.core.diagnostic.InterruptibleHeapDumpReportGenerator; +import io.cryostat.core.diagnostic.HeapDumpReportGenerator; import io.cryostat.core.reports.InterruptibleReportGenerator; import io.cryostat.core.util.RuleFilterParser; import io.cryostat.libcryostat.sys.FileSystem; @@ -45,8 +45,11 @@ InterruptibleReportGenerator produceReportGenerator() { // RequestScoped so that each individual report generation request has its own interruptible // generator with an independent task queueing thread which dispatches to the shared common pool @RequestScoped - InterruptibleHeapDumpReportGenerator produceHeapDumpReportGenerator() { - return new InterruptibleHeapDumpReportGenerator(); + HeapDumpReportGenerator produceHeapDumpReportGenerator() { + boolean singleThread = + Runtime.getRuntime().availableProcessors() < 2 + || Boolean.getBoolean(ReportResource.SINGLETHREAD_PROPERTY); + return new HeapDumpReportGenerator(singleThread ? Executors.newSingleThreadExecutor() : ForkJoinPool.commonPool()); } @Produces diff --git a/src/main/java/io/cryostat/reports/ReportResource.java b/src/main/java/io/cryostat/reports/ReportResource.java index 94123bb..70bb006 100644 --- a/src/main/java/io/cryostat/reports/ReportResource.java +++ b/src/main/java/io/cryostat/reports/ReportResource.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; +import java.net.ProtocolException; import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.StandardCopyOption; @@ -44,7 +45,7 @@ import javax.net.ssl.X509TrustManager; import io.cryostat.core.diagnostic.HeapDumpAnalysis; -import io.cryostat.core.diagnostic.InterruptibleHeapDumpReportGenerator; +import io.cryostat.core.diagnostic.HeapDumpReportGenerator; import io.cryostat.core.reports.InterruptibleReportGenerator; import io.cryostat.core.reports.InterruptibleReportGenerator.AnalysisResult; import io.cryostat.core.util.RuleFilterParser; @@ -88,6 +89,9 @@ public class ReportResource { @ConfigProperty(name = "io.cryostat.reports.timeout", defaultValue = "29000") String timeoutMs; + @ConfigProperty(name = "io.cryostat.reports.heap-dump-memory-limit", defaultValue = "0") + int heapDumpMemoryLimit; + @ConfigProperty(name = "cryostat.storage.auth-method") Optional storageAuthMethod; @@ -110,7 +114,7 @@ public class ReportResource { Optional storageCertPath; @Inject InterruptibleReportGenerator generator; - @Inject InterruptibleHeapDumpReportGenerator heapDumpGenerator; + @Inject HeapDumpReportGenerator heapDumpGenerator; @Inject RuleFilterParser rfp; @Inject FileSystem fs; @Inject ObjectMapper mapper; @@ -216,7 +220,7 @@ public String getReport(RoutingContext ctx, @BeanParam RecordingFormData form) } } - private InputStream getPresignedObjectStream(HttpURLConnection httpConn) { + private InputStream getPresignedObjectStream(HttpURLConnection httpConn) throws IOException, ProtocolException { httpConn.setRequestMethod("GET"); if (httpConn instanceof HttpsURLConnection) { HttpsURLConnection httpsConn = (HttpsURLConnection) httpConn; @@ -255,7 +259,6 @@ private InputStream getPresignedObjectStream(HttpURLConnection httpConn) { httpsConn.setHostnameVerifier((hostname, session) -> true); } } - logger.debugv("Attempting to download presigned heap dump from {0}", form.uri); if (storageAuthMethod.isPresent() && storageAuth.isPresent()) { httpConn.setRequestProperty( "Authorization", @@ -290,7 +293,7 @@ public String getHeapDumpReportFromPresigned(RoutingContext ctx, @BeanParam Pres HttpURLConnection httpConn = (HttpURLConnection) form.uri.toURL().openConnection(); try (var stream = getPresignedObjectStream(httpConn)) { Future evalFuture = null; - evalFuture = heapDumpGenerator.generateInterruptibly(form.jvmId, form.heapDumpId, stream); + evalFuture = heapDumpGenerator.generate(stream, heapDumpMemoryLimit); long elapsed = System.nanoTime() - start; ctxHelper(ctx, evalFuture); return mapper.writeValueAsString( @@ -328,7 +331,7 @@ public String getHeapDumpReport(RoutingContext ctx, @BeanParam HeapDumpFormData Future evalFuture = null; try (var stream = fs.newInputStream(file)) { - evalFuture = heapDumpGenerator.generateInterruptibly(form.jvmId, form.heapDumpId, stream); + evalFuture = heapDumpGenerator.generate(stream, heapDumpMemoryLimit); ctxHelper(ctx, evalFuture); return mapper.writeValueAsString( evalFuture.get(timeout - elapsed, TimeUnit.NANOSECONDS));