Skip to content
Draft
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions src/main/java/io/cryostat/reports/HeapDumpFormData.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/*
* Copyright The Cryostat Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.cryostat.reports;

import jakarta.ws.rs.core.MediaType;
import org.jboss.resteasy.reactive.PartType;
import org.jboss.resteasy.reactive.RestForm;
import org.jboss.resteasy.reactive.multipart.FileUpload;

public class HeapDumpFormData {
@RestForm
@PartType(MediaType.APPLICATION_OCTET_STREAM)
public FileUpload file;

@RestForm
@PartType(MediaType.TEXT_PLAIN)
public String heapDumpId;

@RestForm
@PartType(MediaType.TEXT_PLAIN)
public String jvmId;
}
36 changes: 36 additions & 0 deletions src/main/java/io/cryostat/reports/PresignedHeapDumpFormData.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*
* Copyright The Cryostat Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.cryostat.reports;

import java.net.URI;

import jakarta.ws.rs.core.MediaType;
import org.jboss.resteasy.reactive.PartType;
import org.jboss.resteasy.reactive.RestForm;

public class PresignedHeapDumpFormData {
@RestForm
@PartType(MediaType.TEXT_PLAIN)
public URI uri;

@RestForm
@PartType(MediaType.TEXT_PLAIN)
public String heapDumpId;

@RestForm
@PartType(MediaType.TEXT_PLAIN)
public String jvmId;
}
9 changes: 9 additions & 0 deletions src/main/java/io/cryostat/reports/Producers.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import java.util.concurrent.Executors;
import java.util.concurrent.ForkJoinPool;

import io.cryostat.core.diagnostic.InterruptibleHeapDumpReportGenerator;
import io.cryostat.core.reports.InterruptibleReportGenerator;
import io.cryostat.core.util.RuleFilterParser;
import io.cryostat.libcryostat.sys.FileSystem;
Expand All @@ -40,6 +41,14 @@ InterruptibleReportGenerator produceReportGenerator() {
singleThread ? Executors.newSingleThreadExecutor() : ForkJoinPool.commonPool());
}

@Produces
// RequestScoped so that each individual report generation request has its own interruptible
// generator with an independent task queueing thread which dispatches to the shared common pool
@RequestScoped
InterruptibleHeapDumpReportGenerator produceHeapDumpReportGenerator() {
return new InterruptibleHeapDumpReportGenerator();
}

@Produces
@ApplicationScoped
RuleFilterParser produceRuleFilterParser() {
Expand Down
126 changes: 126 additions & 0 deletions src/main/java/io/cryostat/reports/ReportResource.java
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.X509TrustManager;

import io.cryostat.core.diagnostic.HeapDumpAnalysis;
import io.cryostat.core.diagnostic.InterruptibleHeapDumpReportGenerator;
import io.cryostat.core.reports.InterruptibleReportGenerator;
import io.cryostat.core.reports.InterruptibleReportGenerator.AnalysisResult;
import io.cryostat.core.util.RuleFilterParser;
Expand Down Expand Up @@ -107,6 +109,7 @@ public class ReportResource {
Optional<java.nio.file.Path> storageCertPath;

@Inject InterruptibleReportGenerator generator;
@Inject InterruptibleHeapDumpReportGenerator heapDumpGenerator;
@Inject RuleFilterParser rfp;
@Inject FileSystem fs;
@Inject ObjectMapper mapper;
Expand Down Expand Up @@ -255,6 +258,129 @@ public String getReport(RoutingContext ctx, @BeanParam RecordingFormData form)
}
}


@Blocking
@Path("heapdump/remote_report")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.MULTIPART_FORM_DATA)
@POST
public String getHeapDumpReportFromPresigned(RoutingContext ctx, @BeanParam PresignedHeapDumpFormData form)
throws IOException, URISyntaxException {
// TODO queue these requests so we don't overload ourselves, in particular by reading

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The resource contention issues here are even more potentially egregious with heap dumps than for JFR analysis, so it's probably worth trying to solve this or at least mitigate the potential negative effects now.

https://quarkus.io/guides/smallrye-fault-tolerance

A Bulkhead might be a good starting point.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've added a Bulkhead to the report methods for now, I'll have to do some more research into the smallrye fault tolerance system and look at what we've done with other cryostat endpoints to if we should add anything else.

// multiple Heap Dump files into memory at once for analysis. We should process these serially
// from the queue. If we are getting overloaded then our response time to each subsequent
// request will continue to grow unbounded, so at some point we should stop accepting
// requests when the queue is too long.
// Since this is a @Blocking method that runs on a worker thread pool, can we implement this
// serial queueing behaviour by simply synchronizing on a shared singleton resource ex. the
// generator instance?
// A better long-term solution would be to use a shared messaging queue between Cryostat and
// the report generators, so that Cryostat can put up a URL for a presigned recording to be
// processed and a free report generator can claim that work item and then post back the
// report response
long timeout = TimeUnit.MILLISECONDS.toNanos(Long.parseLong(timeoutMs));
long start = System.nanoTime();

logger.debugv("Attempting to download presigned heap dump from {0}", form.uri);
HttpURLConnection httpConn = (HttpURLConnection) form.uri.toURL().openConnection();
httpConn.setRequestMethod("GET");
if (httpConn instanceof HttpsURLConnection) {
Comment thread
Josh-Matsuoka marked this conversation as resolved.
HttpsURLConnection httpsConn = (HttpsURLConnection) httpConn;
if (storageTlsIgnore) {
try {
httpsConn.setSSLSocketFactory(
ignoreSslContext(storageTlsVersion).getSocketFactory());
} catch (Exception e) {
logger.error(e);
throw new InternalServerErrorException(e);
}
} else if (storageCaPath.isPresent() || storageCertPath.isPresent()) {
if (!(storageCaPath.isPresent() && storageCertPath.isPresent())) {
Exception e =
new IllegalStateException(
String.format(
"%s and %s must be both set or both unset",
"cryostat.storage.tls.ca.path",
"cryostat.storage.tls.cert.path"));
logger.error(e);
throw new InternalServerErrorException(e);
}
try {
httpsConn.setSSLSocketFactory(
trustSslCertContext(
storageTlsVersion,
storageCaPath.get(),
storageCertPath.get())
.getSocketFactory());
} catch (Exception e) {
logger.error(e);
throw new InternalServerErrorException(e);
}
}
if (!storageHostnameVerify) {
httpsConn.setHostnameVerifier((hostname, session) -> true);
}
}
if (storageAuthMethod.isPresent() && storageAuth.isPresent()) {
httpConn.setRequestProperty(
"Authorization",
String.format("%s %s", storageAuthMethod.get(), storageAuth.get()));
}

assertContentLength(httpConn.getContentLengthLong());
try (var stream = httpConn.getInputStream()) {
Future<HeapDumpAnalysis> evalFuture = null;

evalFuture = heapDumpGenerator.generateInterruptibly(form.jvmId, form.heapDumpId, stream);
long elapsed = System.nanoTime() - start;
ctxHelper(ctx, evalFuture);
return mapper.writeValueAsString(
evalFuture.get(timeout - elapsed, TimeUnit.NANOSECONDS));
} catch (ExecutionException | InterruptedException e) {
logger.error(e);
throw new InternalServerErrorException(e);
} catch (TimeoutException e) {
logger.error(e);
throw new ServerErrorException(Response.Status.GATEWAY_TIMEOUT, e);
} catch (Exception e) {
logger.error(e);
throw e;
} finally {
httpConn.disconnect();
}
}

@Blocking
@Path("heapdump/report")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.MULTIPART_FORM_DATA)
@POST
public String getHeapDumpReport(RoutingContext ctx, @BeanParam HeapDumpFormData form)
throws IOException {
FileUpload upload = form.file;

Pair<java.nio.file.Path, Pair<Long, Long>> uploadResult = handleUpload(upload);
java.nio.file.Path file = uploadResult.getLeft();
long timeout = TimeUnit.MILLISECONDS.toNanos(Long.parseLong(timeoutMs));
long start = uploadResult.getRight().getLeft();
long elapsed = uploadResult.getRight().getRight();

Future<HeapDumpAnalysis> evalFuture = null;

try (var stream = fs.newInputStream(file)) {
evalFuture = heapDumpGenerator.generateInterruptibly(form.jvmId, form.heapDumpId, stream);
ctxHelper(ctx, evalFuture);
return mapper.writeValueAsString(
evalFuture.get(timeout - elapsed, TimeUnit.NANOSECONDS));
} catch (ExecutionException | InterruptedException e) {
throw new InternalServerErrorException(e);
} catch (TimeoutException e) {
throw new ServerErrorException(Response.Status.GATEWAY_TIMEOUT, e);
} finally {
cleanupHelper(evalFuture, file, upload.fileName(), start);
}
}

private void assertContentLength(long length) {
if (memoryFactor <= 0) {
return;
Expand Down
Loading