diff --git a/cryostat-core/src/main/java/io/cryostat/core/diagnostic/HeapDumpAnalysis.java b/cryostat-core/src/main/java/io/cryostat/core/diagnostic/HeapDumpAnalysis.java new file mode 100644 index 00000000..8fd58b7f --- /dev/null +++ b/cryostat-core/src/main/java/io/cryostat/core/diagnostic/HeapDumpAnalysis.java @@ -0,0 +1,321 @@ +/* + * Copyright The Cryostat Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cryostat.core.diagnostic; + +import static java.util.Collections.unmodifiableList; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import org.openjdk.jmc.joverflow.batch.BatchProblemRecorder; +import org.openjdk.jmc.joverflow.batch.DetailedStats; +import org.openjdk.jmc.joverflow.batch.ReferencedObjCluster.Collections; +import org.openjdk.jmc.joverflow.batch.ReferencedObjCluster.DupArrays; +import org.openjdk.jmc.joverflow.batch.ReferencedObjCluster.DupStrings; +import org.openjdk.jmc.joverflow.batch.ReferencedObjCluster.HighSizeObjects; +import org.openjdk.jmc.joverflow.batch.ReferencedObjCluster.WeakHashMaps; +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.Snapshot; +import org.openjdk.jmc.joverflow.heap.parser.DumpCorruptedException; +import org.openjdk.jmc.joverflow.heap.parser.HeapDumpReader; +import org.openjdk.jmc.joverflow.heap.parser.HprofParsingCancelledException; +import org.openjdk.jmc.joverflow.heap.parser.ReadBuffer; +import org.openjdk.jmc.joverflow.stats.ObjectHistogram; +import org.openjdk.jmc.joverflow.stats.ObjectHistogram.ProblemFieldsEntry; +import org.openjdk.jmc.joverflow.stats.StandardStatsCalculator; +import org.openjdk.jmc.joverflow.support.HeapStats; +import org.openjdk.jmc.joverflow.util.ObjectToIntMap.Entry; +import org.openjdk.jmc.joverflow.util.VerboseOutputCollector; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +public class HeapDumpAnalysis { + + // Passed into the constructor + private int readBufferMemoryLimit; + + // Reference Chains + private List> collectionClusters; + private List> duplicateArrayClusters; + private List> duplicateStringClusters; + private List> highSizeObjectClusters; + private List> weakHashMapClusters; + + // Class Histogram + private List objectHistogram; + private HistogramStats histogramStats; + + // Fundamental Stats + private FundamentalStats fundamentalStats; + + // Problem Fields (null) + private List nullProblemFields; + // Problem Fields (nearly null) + private List nearNullProblemFields; + // Problem Fields (null) + private List fullBytesFields; + // Problem Fields (nearly null) + private List highBytesFields; + + // Classloader Stats + private List classLoaderInstanceStats; + private List classLoaderClassStats; + + // String Stats + private CompressibleStringStats compressibleStringStats; + private DuplicateStringStats duplicateStringStats; + + private HeapStats heapStats; + private DetailedStats detailedStats; + + public HeapDumpAnalysis(int readBufferLimit) { + readBufferMemoryLimit = readBufferLimit; + classLoaderInstanceStats = new ArrayList(); + classLoaderClassStats = new ArrayList(); + } + + @SuppressFBWarnings( + value = "NP_UNWRITTEN_PUBLIC_OR_PROTECTED_FIELD", + justification = "key in ObjectToIntMap is written when entry map is generated") + public void analyze(InputStream heapDumpStream) + throws IOException, DumpCorruptedException, HprofParsingCancelledException { + Path tmpFile = Files.createTempFile("", ".hprof"); + // Copy the heap dump from storage to a temporary file for analysis + Files.copy(heapDumpStream, tmpFile); + VerboseOutputCollector vc = new VerboseOutputCollector(); + HeapDumpReader reader = + HeapDumpReader.createReader( + new ReadBuffer.CachedReadBufferFactory( + tmpFile.toString(), calculateReadBufMemory()), + 0, + vc); + Snapshot snapshot = reader.read(); + try { + // Parse the heap dump using the JOVerflow libraries + BatchProblemRecorder recorder = new BatchProblemRecorder(); + StandardStatsCalculator ssc = new StandardStatsCalculator(snapshot, recorder, true); + heapStats = ssc.calculate(); + + // TODO: Should this be configurable? + int minOvhdToReport = (int) heapStats.totalObjSize / 1000; + detailedStats = recorder.getDetailedStats(minOvhdToReport); + + // Reference Chains + collectionClusters = detailedStats.collectionClusters; + duplicateArrayClusters = detailedStats.dupArrayClusters; + duplicateStringClusters = detailedStats.dupStringClusters; + highSizeObjectClusters = detailedStats.highSizeObjClusters; + weakHashMapClusters = detailedStats.weakHashMapClusters; + + // Object Histogram + // 0 lists the full histogram + objectHistogram = heapStats.objHisto.getListSortedByInclusiveSize(0); + // Fields that are null/zero/non-existent + nullProblemFields = heapStats.objHisto.getListSortedByNullFieldsOvhd(1.0f); + nearNullProblemFields = heapStats.objHisto.getListSortedByNullFieldsOvhd(0.9f); + // Fields with unused high bytes (100th, 90th percentile) + fullBytesFields = heapStats.objHisto.getListSortedByUnusedHiByteFieldsOvhd(1.0f); + highBytesFields = heapStats.objHisto.getListSortedByUnusedHiByteFieldsOvhd(0.9f); + histogramStats = + new HistogramStats( + heapStats.nClasses, + heapStats.nObjects, + heapStats.objHisto.calculateNumSmallInstClasses()[0], + heapStats.objHisto.calculateNumSmallInstClasses()[1]); + + // Fundamental Stats + fundamentalStats = + new FundamentalStats( + heapStats.ptrSize, + heapStats.usingNarrowPointers, + heapStats.objHeaderSize, + heapStats.objAlignment, + heapStats.nObjects, + heapStats.nInstances, + heapStats.nObjectArrays, + heapStats.nValueArrays, + heapStats.totalObjSize, + heapStats.totalInstSize, + heapStats.totalObjArraySize, + heapStats.totalValueArraySize); + + // ClassLoader Stats + for (Entry e : + heapStats.classloaderStats.getCLInstToNumLoadedClasses().getEntries()) { + classLoaderInstanceStats.add(new AggregateValue(e.key.valueAsString(), e.value)); + } + for (Entry e : + heapStats.classloaderStats.getClClazzToNumLoadedClasses().getEntries()) { + classLoaderClassStats.add(new AggregateValue(e.key.valueAsString(), e.value)); + } + + // Compressible String Stats + compressibleStringStats = + new CompressibleStringStats( + heapStats.compressibleStringStats.nTotalStrings, + heapStats.compressibleStringStats.totalUsedBackingArrayBytes, + heapStats.compressibleStringStats.nCompressedStrings, + heapStats.compressibleStringStats.compressedBackingArrayBytes, + heapStats.compressibleStringStats.nAsciiCharBackedStrings, + heapStats.compressibleStringStats.asciiCharBackingArrayBytes); + + // Duplicate String stats + duplicateStringStats = + new DuplicateStringStats( + heapStats.dupStringStats.nStrings, + heapStats.dupStringStats.nUniqueStringValues, + heapStats.dupStringStats.nUniqueDupStringValues, + heapStats.dupStringStats.dupStringsOverhead); + + } catch (DumpCorruptedException | HprofParsingCancelledException e) { + // Rethrow, the caller will deal with it + throw e; + } finally { + // Clean up the temporary file and reset the memory buffer + Files.deleteIfExists(tmpFile); + snapshot.discard(); + snapshot.resetReadBuffer( + new ReadBuffer.CachedReadBufferFactory(tmpFile.toString(), 25 * 1024 * 1024)); + } + } + + @SuppressFBWarnings("DM_GC") + // Taken from JMC's model generation for the JOverflow UI. + // Dynamically determine read buffer size, we should have this be configurable + private int calculateReadBufMemory() { + if (readBufferMemoryLimit == 0) { + System.gc(); + Runtime runtime = Runtime.getRuntime(); + long availableMemory = + runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory(); + return (int) Math.min(1000 * 1024 * 1024, availableMemory / 3); + } else { + return readBufferMemoryLimit; + } + } + + public List> getCollectionClusters() { + return unmodifiableList(collectionClusters); + } + + public List> getDuplicateArrayClusters() { + return unmodifiableList(duplicateArrayClusters); + } + + public List> getDuplicateStringClusters() { + return unmodifiableList(duplicateStringClusters); + } + + public List> getHighSizeObjectClusters() { + return unmodifiableList(highSizeObjectClusters); + } + + public List> getWeakHashMapClusters() { + return unmodifiableList(weakHashMapClusters); + } + + public List getObjectHistogram() { + return unmodifiableList(objectHistogram); + } + + public HistogramStats getHistogramStats() { + return histogramStats; + } + + public FundamentalStats getFundamentalStats() { + return fundamentalStats; + } + + public List getNullProblemFields() { + return unmodifiableList(nullProblemFields); + } + + public List getNearNullProblemFields() { + return unmodifiableList(nearNullProblemFields); + } + + public List getFullBytesFields() { + return unmodifiableList(fullBytesFields); + } + + public List getHighBytesFields() { + return unmodifiableList(highBytesFields); + } + + public List getClassLoaderInstanceStats() { + return unmodifiableList(classLoaderInstanceStats); + } + + public List getClassLoaderClassStats() { + return unmodifiableList(classLoaderClassStats); + } + + public CompressibleStringStats getCompressibleStringStats() { + return compressibleStringStats; + } + + public DuplicateStringStats getDuplicateStringStats() { + return duplicateStringStats; + } + + public HeapStats getHeapStats() { + return heapStats; + } + + public DetailedStats getDetailedStats() { + return detailedStats; + } + + public record FundamentalStats( + int pointerSize, + boolean narrowPointers, + int objectHeaderSize, + int objectHeaderAlignment, + int numObjects, + int objectInstances, + int objectArrays, + int primitiveArrays, + long objectSize, + long instanceSize, + long objArraySize, + long primitiveSize) {} + ; + + public record CompressibleStringStats( + int stringObjects, + long backingArrayBytes, + int compressedStrings, + long compressedStringBytes, + int asciiStrings, + long asciiStringBytes) {} + ; + + public record HistogramStats( + int totalClasses, int totalObjects, int zeroInstances, int singleInstances) {} + ; + + public record DuplicateStringStats( + int totalStrings, int uniqueStrings, int duplicateStrings, long overhead) {} + ; + + public record AggregateValue(String value, long count) {} + ; +} diff --git a/cryostat-core/src/main/java/io/cryostat/core/diagnostic/HeapDumpReportGenerator.java b/cryostat-core/src/main/java/io/cryostat/core/diagnostic/HeapDumpReportGenerator.java new file mode 100644 index 00000000..d529655b --- /dev/null +++ b/cryostat-core/src/main/java/io/cryostat/core/diagnostic/HeapDumpReportGenerator.java @@ -0,0 +1,38 @@ +/* + * Copyright The Cryostat Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cryostat.core.diagnostic; + +import java.io.InputStream; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; + +public class HeapDumpReportGenerator { + + private final ExecutorService executor; + + public HeapDumpReportGenerator(ExecutorService exec) { + executor = exec; + } + + public Future generate(InputStream inputStream, int readBufferLimit) { + return executor.submit( + () -> { + HeapDumpAnalysis analysisResult = new HeapDumpAnalysis(readBufferLimit); + analysisResult.analyze(inputStream); + return analysisResult; + }); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/batch/BatchProblemRecorder.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/batch/BatchProblemRecorder.java new file mode 100644 index 00000000..201f718d --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/batch/BatchProblemRecorder.java @@ -0,0 +1,697 @@ +/* + * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.batch; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.openjdk.jmc.joverflow.descriptors.CollectionInstanceDescriptor; +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaLazyReadObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaValueArray; +import org.openjdk.jmc.joverflow.heap.model.Snapshot; +import org.openjdk.jmc.joverflow.support.ClassAndOvhdComboList; +import org.openjdk.jmc.joverflow.support.ClassAndSizeComboList; +import org.openjdk.jmc.joverflow.support.Constants.ProblemKind; +import org.openjdk.jmc.joverflow.support.HeapStats; +import org.openjdk.jmc.joverflow.support.PrimitiveArrayWrapper; +import org.openjdk.jmc.joverflow.support.ProblemRecorder; +import org.openjdk.jmc.joverflow.support.RefChainElement; +import org.openjdk.jmc.joverflow.support.RefChainElementImpl; +import org.openjdk.jmc.joverflow.util.ObjectToIntMap; +import org.openjdk.jmc.joverflow.util.SmallSet; + +/** + * Implementaiton of ProblemRecorder that is used by the command-line (batch) JOverflow tool. It + * collects information about clusters of problematic objects in the form that is compact enough and + * suitable for printing in the batch report. However, it might not be flexible enough to manipulate + * with in an interactive tool. + */ +public class BatchProblemRecorder implements ProblemRecorder { + + private static final int HIGH_SIZE = 1; + + private IdentityHashMap refererToHSCluster = new IdentityHashMap<>(128); + private IdentityHashMap refererToColCluster = new IdentityHashMap<>(128); + private IdentityHashMap refererToDSCluster = new IdentityHashMap<>(256); + private IdentityHashMap refererToDACluster = new IdentityHashMap<>(32); + private IdentityHashMap refererToWMCluster = new IdentityHashMap<>(4); + + private JavaHeapObject lastObj; + + @Override + public void initialize(Snapshot snapshot, HeapStats hs) { + // Mark classes for which we are going to record ref chains, because we know + // or suspect that their instances' impl-inclusive size is high (>= 2%) + long minOvhdForHSClasses = hs.totalObjSize / 50; + for (JavaClass clazz : snapshot.getClasses()) { + if (clazz.isCollection() || clazz.getTotalShallowInstanceSize() >= minOvhdForHSClasses) { + // We don't know total impl-inclusive size for any collection yet, so + // we will record a ref chain to each of them, and then will filter + // out those whose total size is too small + clazz.setFlag(HIGH_SIZE); + } + } + } + + @Override + public void recordProblematicCollection( + JavaLazyReadObject col, CollectionInstanceDescriptor colDesc, ProblemKind ovhdKind, int ovhd, + RefChainElement referer) { + CollectionCluster colCluster = refererToColCluster.get(referer); + if (colCluster == null) { + colCluster = new CollectionCluster(); + refererToColCluster.put(referer, colCluster); + } + JavaClass colClazz = col.getClazz(); + if (ovhdKind == ProblemKind.SMALL || ovhdKind == ProblemKind.SPARSE_SMALL + || ovhdKind == ProblemKind.SPARSE_LARGE) { + colCluster.addCollectionInstanceWithNumEls(colClazz, ovhdKind, ovhd, colDesc.getNumElements()); + } else { + colCluster.addCollectionInstance(colClazz, ovhdKind, ovhd); + } + + if (col != lastObj && colClazz.flagIsSet(HIGH_SIZE)) { + recordHighSizeObject(col, referer, colDesc.getImplSize()); + } + } + + @Override + public void recordGoodCollection( + JavaLazyReadObject col, CollectionInstanceDescriptor colDesc, RefChainElement referer) { + CollectionCluster colCluster = refererToColCluster.get(referer); + if (colCluster == null) { + colCluster = new CollectionCluster(); + refererToColCluster.put(referer, colCluster); + } + colCluster.addGoodCollection(); + + if (col != lastObj && col.getClazz().flagIsSet(HIGH_SIZE)) { + recordHighSizeObject(col, referer, colDesc.getImplSize()); + } + } + + @Override + public void recordDuplicateString( + JavaObject stringObj, String stringValue, int implInclusiveSize, int ovhd, boolean hasDupBackingCharArray, + RefChainElement referer) { + DupStringCluster dsCluster = refererToDSCluster.get(referer); + if (dsCluster == null) { + dsCluster = new DupStringCluster(); + refererToDSCluster.put(referer, dsCluster); + } + dsCluster.addDupString(stringValue, ovhd, hasDupBackingCharArray); + + if (stringObj != lastObj && stringObj.getClazz().flagIsSet(HIGH_SIZE)) { + recordHighSizeObject(stringObj, referer, implInclusiveSize); + } + } + + @Override + public void recordNonDuplicateString(JavaObject stringObj, int implInclusiveSize, RefChainElement referer) { + DupStringCluster dsCluster = refererToDSCluster.get(referer); + if (dsCluster == null) { + dsCluster = new DupStringCluster(); + refererToDSCluster.put(referer, dsCluster); + } + dsCluster.addNonDupString(); + + if (stringObj != lastObj && stringObj.getClazz().flagIsSet(HIGH_SIZE)) { + recordHighSizeObject(stringObj, referer, implInclusiveSize); + } + } + + @Override + public void recordDuplicateArray(JavaValueArray ar, int ovhd, RefChainElement referer) { + DupArrayCluster daCluster = refererToDACluster.get(referer); + if (daCluster == null) { + daCluster = new DupArrayCluster(); + refererToDACluster.put(referer, daCluster); + } + daCluster.addDupArray(ar, ovhd); + + if (ar != lastObj && ar.getClazz().flagIsSet(HIGH_SIZE)) { + recordHighSizeObject(ar, referer, ar.getSize()); + } + } + + @Override + public void recordNonDuplicateArray(JavaValueArray ar, RefChainElement referer) { + DupArrayCluster daCluster = refererToDACluster.get(referer); + if (daCluster == null) { + daCluster = new DupArrayCluster(); + refererToDACluster.put(referer, daCluster); + } + daCluster.addNonDupArray(); + + if (ar != lastObj && ar.getClazz().flagIsSet(HIGH_SIZE)) { + recordHighSizeObject(ar, referer, ar.getSize()); + } + } + + @Override + public void recordWeakHashMapWithBackRefs( + JavaObject col, CollectionInstanceDescriptor colDesc, int ovhd, String valueTypeAndFieldSample, + RefChainElement referer) { + WeakHashMapCluster wmCluster = refererToWMCluster.get(referer); + if (wmCluster == null) { + wmCluster = new WeakHashMapCluster(); + refererToWMCluster.put(referer, wmCluster); + } + wmCluster.addWeakHashMap(col.getClazz().getHumanFriendlyName(), ovhd, valueTypeAndFieldSample); + + if (col != lastObj && col.getClazz().flagIsSet(HIGH_SIZE)) { + recordHighSizeObject(col, referer, colDesc.getImplSize()); + } + } + + @Override + public boolean shouldRecordGoodInstance(JavaObject obj) { + return (obj != lastObj && obj.getClazz().flagIsSet(HIGH_SIZE)); + } + + @Override + public void recordGoodInstance(JavaObject obj, RefChainElement referer) { + recordHighSizeObject(obj, referer, obj.getSize()); + } + + private void recordHighSizeObject(JavaHeapObject obj, RefChainElement referer, int size) { + HighSizeObjCluster cluster = refererToHSCluster.get(referer); + if (cluster == null) { + cluster = new HighSizeObjCluster(); + refererToHSCluster.put(referer, cluster); + } + cluster.addInstance(obj.getClazz(), size); + lastObj = obj; + } + + @SuppressWarnings("unchecked") + public DetailedStats getDetailedStats(int minOvhd) { + List> clustersWithFullRefChains = getProblematicDataClustersWithFullRefChains( + minOvhd); + List> clustersWithNearestField = getProblematicDataClustersWithNearestField( + minOvhd); + + List> highSizeObjClusters = new ArrayList<>(2); + highSizeObjClusters.add((List) clustersWithFullRefChains.get(4)); + highSizeObjClusters.add((List) clustersWithNearestField.get(4)); + List> collectionClusters = new ArrayList<>(2); + collectionClusters.add((List) clustersWithFullRefChains.get(0)); + collectionClusters.add((List) clustersWithNearestField.get(0)); + List> dupStringClusters = new ArrayList<>(2); + dupStringClusters.add((List) clustersWithFullRefChains.get(1)); + dupStringClusters.add((List) clustersWithNearestField.get(1)); + List> dupArrayClusters = new ArrayList<>(2); + dupArrayClusters.add((List) clustersWithFullRefChains.get(2)); + dupArrayClusters.add((List) clustersWithNearestField.get(2)); + List> weakHashMapClusters = new ArrayList<>(2); + weakHashMapClusters.add((List) clustersWithFullRefChains.get(3)); + weakHashMapClusters.add((List) clustersWithNearestField.get(3)); + + return new DetailedStats(minOvhd, highSizeObjClusters, collectionClusters, weakHashMapClusters, + dupStringClusters, dupArrayClusters); + } + + private List> getProblematicDataClustersWithFullRefChains(int minOvhd) { + ArrayList hsClusters = new ArrayList<>(64); + ArrayList colClusters = new ArrayList<>(64); + ArrayList dsClusters = new ArrayList<>(128); + ArrayList daClusters = new ArrayList<>(64); + ArrayList wmClusters = new ArrayList<>(4); + + generateFullRefChainClusters(refererToHSCluster, hsClusters, minOvhd * 5); + generateFullRefChainClusters(refererToColCluster, colClusters, minOvhd); + generateFullRefChainClusters(refererToDSCluster, dsClusters, minOvhd); + generateFullRefChainClusters(refererToDACluster, daClusters, minOvhd); + generateFullRefChainClusters(refererToWMCluster, wmClusters, minOvhd); + + List> result = new ArrayList<>(4); + result.add(colClusters); + result.add(dsClusters); + result.add(daClusters); + result.add(wmClusters); + result.add(hsClusters); + return result; + } + + private void generateFullRefChainClusters( + IdentityHashMap refererToCluster, ArrayList clusterList, + int minOvhd) { + Set> colEntries = refererToCluster.entrySet(); + for (Map.Entry entry : colEntries) { + RefChainElement referer = entry.getKey(); + T cluster = entry.getValue(); + if (cluster.getTotalOverhead() < minOvhd) { + continue; + } + clusterList.add(cluster.getFinalCluster(referer)); + } + + clusterList.sort(ReferencedObjCluster.DEFAULT_COMPARATOR); + } + + private List> getProblematicDataClustersWithNearestField(int minOvhd) { + ArrayList hsClusters = new ArrayList<>(64); + ArrayList colClusters = new ArrayList<>(64); + ArrayList dsClusters = new ArrayList<>(128); + ArrayList daClusters = new ArrayList<>(64); + ArrayList wmClusters = new ArrayList<>(4); + + generateFieldClusters(refererToHSCluster, hsClusters, minOvhd * 5); + generateFieldClusters(refererToColCluster, colClusters, minOvhd); + generateFieldClusters(refererToDSCluster, dsClusters, minOvhd); + generateFieldClusters(refererToDACluster, daClusters, minOvhd); + generateFieldClusters(refererToWMCluster, wmClusters, minOvhd); + + List> result = new ArrayList<>(4); + result.add(colClusters); + result.add(dsClusters); + result.add(daClusters); + result.add(wmClusters); + result.add(hsClusters); + return result; + } + + @SuppressWarnings("unchecked") // This is only for the (T) entry.getValue().createCopy() line + private void generateFieldClusters( + IdentityHashMap refererToCluster, ArrayList clusterList, + int minOvhd) { + HashMap fieldToCluster = new HashMap<>(); + + Set> allClusters = refererToCluster.entrySet(); + for (Map.Entry entry : allClusters) { + RefChainElement referer = entry.getKey(); + if (referer instanceof RefChainElementImpl.GCRoot) { + continue; + } + + // Find the nearest field referencing this collection cluster. If there are + // any intermediate collections or arrays between this cluster and the field, + // they become a part of the "extended field reference". + ArrayList fieldDescBuf = new ArrayList<>(4); + while (referer != null && !(referer instanceof RefChainElementImpl.GCRoot)) { + if (referer instanceof RefChainElementImpl.AbstractField) { + // Continue if this field belongs to one of the classes that are usually + // non-informative on their own, like UnmodifiableCollections etc. + RefChainElementImpl.AbstractField fieldDesc = (RefChainElementImpl.AbstractField) referer; + String clazzName = fieldDesc.getJavaClass().getName(); + if (!(clazzName.startsWith("java.util.Collections$") || clazzName.startsWith("java.lang.ref.") + || clazzName.equals("java.util.BitSet"))) { + break; + } + } + fieldDescBuf.add(0, referer); + referer = referer.getReferer(); + } + // Reached a GC root, but haven't found a field + if (referer == null || referer instanceof RefChainElementImpl.GCRoot) { + continue; + } + + // Finally, got to a field + fieldDescBuf.add(0, referer); + ExtendedField fieldReferer = new ExtendedField(fieldDescBuf); + + T cluster = fieldToCluster.get(fieldReferer); + if (cluster == null) { + cluster = (T) entry.getValue().createCopy(fieldReferer); + fieldToCluster.put(fieldReferer, cluster); + } else { + cluster.addCluster(entry.getValue()); + } + } + + Set> fieldClusters = fieldToCluster.entrySet(); + for (Map.Entry entry : fieldClusters) { + T cluster = entry.getValue(); + if (cluster.getTotalOverhead() < minOvhd) { + continue; + } + + RefChainElement referer = entry.getKey().toReferenceChain(); + clusterList.add(cluster.getFinalCluster(referer)); + } + + clusterList.sort(ReferencedObjCluster.DEFAULT_COMPARATOR); + } + + private abstract static class AbstractClusterNode { + + abstract int getTotalOverhead(); + + /** + * Creates a cluster, performing a deep copy of all the information from the given original + * cluster, except for the parent, which is set anew as a Node with the given descriptor. + * Used when creating the alternative view, where collection clusters are aggregated just by + * the nearest data field. + */ + abstract AbstractClusterNode createCopy(RefChainElement parentDesc); + + /** Adds all the information from the other cluster to this one. */ + abstract void addCluster(AbstractClusterNode other); + + /** + * Generates and returns the "final" cluster object, that contains finalized data about the + * specific kind of overhead, for consumption by the clients of this code. + */ + abstract ReferencedObjCluster getFinalCluster(RefChainElement referer); + + // Use Comparator instead of implementing Comparable if sorting is needed +// @Override +// public int compareTo(AbstractClusterNode other) { +// int totalOverhead = getTotalOverhead(); +// int otherTotalOverhead = other.getTotalOverhead(); +// if (totalOverhead < otherTotalOverhead) { +// return 1; +// } else if (totalOverhead > otherTotalOverhead) { +// return -1; +// } else { +// return 0; +// } +// } + + // Debugging + @SuppressWarnings("unused") + void printNode(String indent) { + System.out.println(indent + this.toString()); + } + } + + /** + * A leaf node that contains info about all duplicated strings reachable via the given reference + * chain. + */ + private static class DupStringCluster extends AbstractClusterNode { + + private int totalOverhead; + private int numDupBackingCharArrays; + private int numNonDupStrings; + + /** Maps a string value to the number of instances of that string */ + private final ObjectToIntMap strings; + + private DupStringCluster(ObjectToIntMap strings) { + this.strings = strings; + } + + DupStringCluster() { + this(new ObjectToIntMap(5)); + } + + @Override + int getTotalOverhead() { + return totalOverhead; + } + + void addDupString(String string, int overhead, boolean hasDupBackingCharArray) { + strings.putOneOrIncrement(string); + totalOverhead += overhead; + if (hasDupBackingCharArray) { + numDupBackingCharArrays++; + } + } + + void addNonDupString() { + numNonDupStrings++; + } + + @Override + DupStringCluster createCopy(RefChainElement parentDesc) { + DupStringCluster copy = new DupStringCluster(strings.clone()); + copy.totalOverhead = totalOverhead; + copy.numDupBackingCharArrays = numDupBackingCharArrays; + copy.numNonDupStrings = numNonDupStrings; + return copy; + } + + @Override + void addCluster(AbstractClusterNode other) { + DupStringCluster otherCluster = (DupStringCluster) other; + ObjectToIntMap otherStrings = otherCluster.strings; + ObjectToIntMap.Entry entries[] = otherStrings.getEntries(); + for (ObjectToIntMap.Entry entry : entries) { + strings.putOrIncrementBy(entry.key, entry.value); + } + totalOverhead += otherCluster.totalOverhead; + numDupBackingCharArrays += otherCluster.numDupBackingCharArrays; + numNonDupStrings += otherCluster.numNonDupStrings; + } + + @Override + ReferencedObjCluster getFinalCluster(RefChainElement referer) { + return new ReferencedObjCluster.DupStrings(referer, totalOverhead, numDupBackingCharArrays, + numNonDupStrings, strings.getEntriesSortedByValueThenKey()); + } + } + + /** + * A leaf node that contains info about all duplicated arrays reachable via the given reference + * chain. + */ + private static class DupArrayCluster extends AbstractClusterNode { + + private int totalOverhead; + private int numNonDupArrays; + + /** Maps a unique array value (contents) to the number of instances of that array */ + private final ObjectToIntMap arrays; + + private DupArrayCluster(ObjectToIntMap arrays) { + this.arrays = arrays; + } + + DupArrayCluster() { + this(new ObjectToIntMap(5)); + } + + @Override + int getTotalOverhead() { + return totalOverhead; + } + + void addDupArray(JavaValueArray ar, int overhead) { + PrimitiveArrayWrapper arWrapper = new PrimitiveArrayWrapper(ar); + arrays.putOneOrIncrement(arWrapper); + totalOverhead += overhead; + } + + void addNonDupArray() { + numNonDupArrays++; + } + + @Override + DupArrayCluster createCopy(RefChainElement parentDesc) { + DupArrayCluster copy = new DupArrayCluster(arrays.clone()); + copy.totalOverhead = totalOverhead; + copy.numNonDupArrays = numNonDupArrays; + return copy; + } + + @Override + void addCluster(AbstractClusterNode other) { + DupArrayCluster otherCluster = (DupArrayCluster) other; + ObjectToIntMap otherStrings = otherCluster.arrays; + ObjectToIntMap.Entry entries[] = otherStrings.getEntries(); + for (ObjectToIntMap.Entry entry : entries) { + arrays.putOrIncrementBy(entry.key, entry.value); + } + totalOverhead += otherCluster.totalOverhead; + numNonDupArrays += otherCluster.numNonDupArrays; + } + + @Override + ReferencedObjCluster getFinalCluster(RefChainElement referer) { + return new ReferencedObjCluster.DupArrays(referer, totalOverhead, numNonDupArrays, + arrays.getEntriesSortedByValueThenKey()); + } + } + + /** + * A leaf node that contains info about all problematic collections reachable via the given + * reference chain. Note that this kind of node cannot have children, so if some collection + * happens to be a problematic one, but also references other collections, there will be two + * nodes for it - an ordinary Node and a CollectionCluster. + */ + private static class CollectionCluster extends AbstractClusterNode { + + ClassAndOvhdComboList entries; + private int numGoodCollections; + + CollectionCluster() { + entries = new ClassAndOvhdComboList(); + } + + void addCollectionInstance(JavaClass colClass, ProblemKind ovhdKind, int ovhd) { + entries.addCollectionInfo(colClass, ovhdKind, ovhd, 1); + } + + void addCollectionInstanceWithNumEls(JavaClass colClass, ProblemKind ovhdKind, int ovhd, int numElements) { + entries.addCollectionInfoWithNumEls(colClass, ovhdKind, ovhd, 1, numElements, numElements); + } + + void addGoodCollection() { + numGoodCollections++; + } + + @Override + int getTotalOverhead() { + return entries.getTotalOverhead(); + } + + @Override + CollectionCluster createCopy(RefChainElement parentDesc) { + CollectionCluster copy = new CollectionCluster(); + copy.entries = entries.clone(); + copy.numGoodCollections = numGoodCollections; + return copy; + } + + @Override + void addCluster(AbstractClusterNode other) { + CollectionCluster otherCluster = (CollectionCluster) other; + entries.merge(otherCluster.entries); + numGoodCollections += otherCluster.numGoodCollections; + } + + @Override + ReferencedObjCluster getFinalCluster(RefChainElement referer) { + return new ReferencedObjCluster.Collections(referer, entries.getFinalList(), entries.getTotalOverhead(), + numGoodCollections); + } + } + + /** + * A leaf node that contains info about all problematic WeakHashMaps (those that have references + * from values back to keys) reachable via the given reference chain. Note that this kind of + * node cannot have children, so if some WeakHashMap happens to be a problematic one, but also + * have references to other collections, there will be two nodes for it - an ordinary Node and a + * WeakHashMapCluster. + */ + private static class WeakHashMapCluster extends AbstractClusterNode { + + private final SmallSet colClasses; + private final SmallSet valueTypeAndFieldSamples; + private int numInstances, totalOverhead; + + WeakHashMapCluster() { + colClasses = new SmallSet<>(); + valueTypeAndFieldSamples = new SmallSet<>(); + } + + private WeakHashMapCluster(WeakHashMapCluster from) { + totalOverhead = from.totalOverhead; + numInstances = from.numInstances; + colClasses = new SmallSet<>(from.colClasses); + valueTypeAndFieldSamples = new SmallSet<>(from.valueTypeAndFieldSamples); + } + + void addWeakHashMap(String colClass, int ovhd, String valueTypeAndFieldSample) { + totalOverhead += ovhd; + numInstances++; + colClasses.add(colClass); + valueTypeAndFieldSamples.add(valueTypeAndFieldSample); + } + + @Override + int getTotalOverhead() { + return totalOverhead; + } + + @Override + WeakHashMapCluster createCopy(RefChainElement parentDesc) { + return new WeakHashMapCluster(this); + } + + @Override + void addCluster(AbstractClusterNode other) { + WeakHashMapCluster otherCluster = (WeakHashMapCluster) other; + totalOverhead += otherCluster.totalOverhead; + numInstances += otherCluster.numInstances; + colClasses.addAll(otherCluster.colClasses); + valueTypeAndFieldSamples.addAll(otherCluster.valueTypeAndFieldSamples); + } + + @Override + ReferencedObjCluster getFinalCluster(RefChainElement referer) { + return new ReferencedObjCluster.WeakHashMaps(referer, numInstances, totalOverhead, colClasses, + valueTypeAndFieldSamples); + } + } + + /** + * A leaf node that contains info about all objects of certain classes, for which we know or + * expect the total size to be high, reachable via the given reference chain. + */ + private static class HighSizeObjCluster extends AbstractClusterNode { + + ClassAndSizeComboList entries; + + HighSizeObjCluster() { + entries = new ClassAndSizeComboList(); + } + + void addInstance(JavaClass colClass, int implInclusiveSize) { + entries.addInstanceInfo(colClass, implInclusiveSize, 1); + } + + @Override + int getTotalOverhead() { + return entries.getTotalSize(); + } + + @Override + HighSizeObjCluster createCopy(RefChainElement parentDesc) { + HighSizeObjCluster copy = new HighSizeObjCluster(); + copy.entries = entries.clone(); + return copy; + } + + @Override + void addCluster(AbstractClusterNode other) { + HighSizeObjCluster otherCluster = (HighSizeObjCluster) other; + entries.merge(otherCluster.entries); + } + + @Override + ReferencedObjCluster getFinalCluster(RefChainElement referer) { + return new ReferencedObjCluster.HighSizeObjects(referer, entries.getFinalList(), entries.getTotalSize()); + } + } + +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/batch/DetailedStats.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/batch/DetailedStats.java new file mode 100644 index 00000000..82dbea78 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/batch/DetailedStats.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.batch; + +import java.util.List; + +/** + * Container for detailed stats about problematic objects, in the form that is used by the + * command-line (batch) JOverflow tool. + */ +public class DetailedStats { + + /** Minimum amount of memory overhead for a problem, object etc. that we report */ + public final int minOvhdToReport; + + /** + * Reverse reference chains and nearest data fields pointing at clusters of objects with high + * memory consumption. + */ + public final List> highSizeObjClusters; + + /** + * Reverse reference chains and nearest data fields pointing at clusters of problematic + * Collections. + */ + public final List> collectionClusters; + + /** + * Reverse reference chains and nearest data fields pointing at clusters of WeakHashMaps that + * have values pointing back to keys. + */ + public final List> weakHashMapClusters; + + /** + * Reverse reference chains and nearest data fields pointing at clusters of duplicated strings. + * See ReferenceChain.getReports() for details. + */ + public final List> dupStringClusters; + + /** + * Reverse reference chains and nearest data fields pointing at clusters of duplicated arrays. + * See ReferenceChain.getReports() for details. + */ + public final List> dupArrayClusters; + + public DetailedStats(int minOvhdToReport, List> hsClusters, + List> colClusters, + List> wmClusters, + List> dsClusters, + List> dupArrayClusters) { + this.minOvhdToReport = minOvhdToReport; + this.highSizeObjClusters = hsClusters; + this.collectionClusters = colClusters; + this.weakHashMapClusters = wmClusters; + this.dupStringClusters = dsClusters; + this.dupArrayClusters = dupArrayClusters; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/batch/ExtendedField.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/batch/ExtendedField.java new file mode 100644 index 00000000..d96ba62e --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/batch/ExtendedField.java @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.batch; + +import java.util.List; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.support.RefChainElement; +import org.openjdk.jmc.joverflow.support.RefChainElementImpl; + +/** + * Extended field element. Used internally to temporarily represent a field referencing a cluster of + * problematic objects. The field can reference it directly, or via one or more intermediate + * compound objects (collection, array, etc.) So this class provides an API of RefChainElement, but + * internally is more like a reference chain, and is eventually converted into one. + */ +public class ExtendedField implements RefChainElement { + + private final List fieldAndExt; + private final String valueAsString; + + @Override + public JavaClass getJavaClass() { + return fieldAndExt.get(0).getJavaClass(); + } + + @Override + public RefChainElement getReferer() { + return null; + } + + ExtendedField(List fieldAndExt) { + this.fieldAndExt = fieldAndExt; + this.valueAsString = valueAsString(); + } + + public RefChainElement toReferenceChain() { + RefChainElement referer = null; + for (RefChainElement curElement : fieldAndExt) { + RefChainElement newElement; + if (curElement instanceof RefChainElementImpl.InstanceFieldOrLinkedList) { + RefChainElementImpl.InstanceFieldOrLinkedList field = (RefChainElementImpl.InstanceFieldOrLinkedList) curElement; + newElement = RefChainElementImpl.createInstanceFieldOrLinkedListElementInFinalForm(field.getJavaClass(), + field.getFieldIdx(), referer, field.isInstanceField()); + } else if (curElement instanceof RefChainElementImpl.StaticField) { + RefChainElementImpl.StaticField field = (RefChainElementImpl.StaticField) curElement; + newElement = RefChainElementImpl.createStaticFieldElementInFinalForm(field.getJavaClass(), + field.getFieldIdx(), referer); + } else if (curElement instanceof RefChainElementImpl.Collection) { + RefChainElementImpl.Collection col = (RefChainElementImpl.Collection) curElement; + newElement = RefChainElementImpl.createCompoundCollectionElementInFinalForm(col.getJavaClass(), + referer); + } else if (curElement instanceof RefChainElementImpl.Array) { + RefChainElementImpl.Array ar = (RefChainElementImpl.Array) curElement; + newElement = RefChainElementImpl.createCompoundArrayElementInFinalForm(ar.getJavaClass(), referer); + } else { + throw new RuntimeException("Unsupported ref chain element type: " + curElement.getClass()); + } + referer = newElement; + } + return referer; + } + + @Override + public String toString() { + return valueAsString; + } + + private String valueAsString() { + if (fieldAndExt.size() == 1) { + return fieldAndExt.get(0).toString(); + } + + StringBuilder sb = new StringBuilder(32); + for (RefChainElement ext : fieldAndExt) { + sb.append(ext.toString()); + sb.append("-->"); + } + return sb.toString(); + } + + @Override + public boolean equals(Object otherObj) { + if (!(otherObj instanceof ExtendedField)) { + return false; + } + + ExtendedField other = (org.openjdk.jmc.joverflow.batch.ExtendedField) otherObj; + return (this.valueAsString.equals(other.valueAsString)); + } + + @Override + public int hashCode() { + return valueAsString.hashCode(); + } + + @Override + public boolean shallowEquals(Object other) { + return equals(other); + } + + @Override + public int shallowHashCode() { + return hashCode(); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/batch/FormattedOutputBuffer.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/batch/FormattedOutputBuffer.java new file mode 100644 index 00000000..eea52644 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/batch/FormattedOutputBuffer.java @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.batch; + +import org.openjdk.jmc.joverflow.util.MemNumFormatter; + +/** + * An output buffer that accumulates formatted text. It also has some intelligence for checking + * certain user-specified values (e.g. the overhead caused by some particular memory problem) + * against the previously specified threshold. If such a "critical issue" is detected, information + * about it is stored separately, and eventually is placed on top of the output. + */ +final class FormattedOutputBuffer { + + private final StringBuilder sb = new StringBuilder(32000); + private final StringBuilder csb = new StringBuilder(1000); + + private final MemNumFormatter nf; + private final long totalHeapSize; + + private String currentSection, currentIssue; + private double currentCriticalLevelInPercent; + private boolean criticalIssuesInCurrentSectionFound; + + FormattedOutputBuffer(long totalHeapSize) { + this.totalHeapSize = totalHeapSize; + nf = new MemNumFormatter(totalHeapSize); + } + + void print(String s) { + sb.append(s); + } + + void println(String s) { + sb.append(s); + sb.append('\n'); + } + + void println() { + sb.append('\n'); + } + + void format(String format, Object ... args) { + String result = String.format(format, args); + sb.append(result); + } + + String k(long num) { + return nf.getNumInKAndPercent(num); + } + + void startSection(String sectionName, String issueName, double criticalLevelInPercent) { + currentSection = sectionName; + currentIssue = issueName; + currentCriticalLevelInPercent = criticalLevelInPercent; + sb.append('\n'); + sb.append(sectionName); + sb.append("\n"); + criticalIssuesInCurrentSectionFound = false; + } + + void startSection(String sectionName) { + startSection(sectionName, "", 0.0); + } + + void criticalCheck(long ovhd, String issueQualifier) { + if (((double) ovhd) / totalHeapSize * 100 <= currentCriticalLevelInPercent) { + return; + } + + if (!criticalIssuesInCurrentSectionFound) { + csb.append("\n Section "); + csb.append(currentSection); + csb.append(" (overhead > "); + csb.append((int) currentCriticalLevelInPercent); + csb.append("% reported):\n"); + criticalIssuesInCurrentSectionFound = true; + } + csb.append(" "); + csb.append(currentIssue); + csb.append(" "); + csb.append(issueQualifier); + csb.append(" "); + csb.append(nf.getNumInKAndPercent(ovhd)); + csb.append('\n'); + } + + MemNumFormatter getMemNumFormatter() { + return nf; + } + + String getOutput() { + if (csb.length() > 0) { + csb.insert(0, "0. IMPORTANT ISSUES OVERVIEW:\n"); + csb.append('\n'); + sb.insert(0, csb); + } + + return sb.toString(); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/batch/ReferencedObjCluster.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/batch/ReferencedObjCluster.java new file mode 100644 index 00000000..99dd04e2 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/batch/ReferencedObjCluster.java @@ -0,0 +1,531 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.batch; + +import java.util.Comparator; +import java.util.List; + +import org.openjdk.jmc.joverflow.support.ClassAndOvhdCombo; +import org.openjdk.jmc.joverflow.support.ClassAndSizeCombo; +import org.openjdk.jmc.joverflow.support.PrimitiveArrayWrapper; +import org.openjdk.jmc.joverflow.support.RefChainElement; +import org.openjdk.jmc.joverflow.support.ReferenceChain; +import org.openjdk.jmc.joverflow.util.ClassUtils; +import org.openjdk.jmc.joverflow.util.MemNumFormatter; +import org.openjdk.jmc.joverflow.util.MiscUtils; +import org.openjdk.jmc.joverflow.util.ObjectToIntMap; +import org.openjdk.jmc.joverflow.util.SmallSet; + +/** + * Represents a cluster of objects, i.e. all objects reachable via the same reference chain, that + * have something in common (like, they all are known collections with some kind(s) of overhead, or + * duplicated strings). One can mentally visualize a cluster of, say, 100 empty HashMaps, reachable + * from a GC root via a chain of references as something like: + *

+ * GC root1 -> A.b -> {ArrayList} -> C.d -> 100 empty HashMaps, 1000 bytes overhead, 2 + * non-empty HashMaps + *

+ * Note that the reference chain may be full, as above, or, for convenience, the tool may provide + * another "view" for problematic objects, where clusters are aggregated by the nearest data field. + * In that case, the above reference chain will be reduced to just C.d. + *

+ * The abstract ReferencedObjCluster class itself contains only the functionality that is common for + * clusters of objects with all kinds of problems (e.g. inefficient collections vs. duplicate + * strings). Its concrete subclasses provide details specific for each problem kind. + *

+ * Note that when information is aggregated by class, we don't distinguish class versions. In other + * words, two classes with same name but different loaders are treated as the same class. + *

+ * Note also that this class implements compareTo() but has no implementation of equals(). In other + * words, it's currently not guaranteed that compareTo() returns zero if and only if equals() + * returns true. However, that matters if instances of a class are used in classes like + * PriorityQueue, which is highly unlikely for this class and its subclasses. + */ +public abstract class ReferencedObjCluster { + + private final RefChainElement referer; + private final int totalOverhead; + + ReferencedObjCluster(RefChainElement referer, int totalOverhead) { + this.referer = referer; + this.totalOverhead = totalOverhead; + } + + /** + * Returns the total memory overhead, in bytes, imposed by all problematic objects in this + * cluster. + */ + public int getTotalOverhead() { + return totalOverhead; + } + + /** + * Returns the number of problematic objects in this cluster. Note that some objects reachable + * via the same reference chain, and therefore located in the same cluster, may be "good". Their + * number is returned by a separate, subclass-specific method. + */ + public abstract int getNumBadObjects(); + + /** Returns the reference chain leading to all objects in this cluster. */ + public RefChainElement getReferer() { + return referer; + } + + /** + * Returns a simple brief string representation for this cluster and its reference chain. + */ + public abstract String clusterAsString(MemNumFormatter nf); + + public static final Comparator DEFAULT_COMPARATOR = new Comparator() { + + @Override + public int compare(ReferencedObjCluster o1, ReferencedObjCluster o2) { + int ovhdDiff = o2.getTotalOverhead() - o1.getTotalOverhead(); + if (ovhdDiff != 0) { + return ovhdDiff; + } + + // Perform some more checks to make order stable for clusters with same overhead. + // For such clusters, the number of bad objects is usually same, but let's + // check it first. + int badObjNumDiff = o2.getNumBadObjects() - o1.getNumBadObjects(); + if (badObjNumDiff != 0) { + return badObjNumDiff; + } + + // Perform the most expensive check if nothing else works + String thisRefChain = ReferenceChain.toStringInReverseOrder(o1.referer, 100); + String otherRefChain = ReferenceChain.toStringInReverseOrder(o2.referer, 100); + return thisRefChain.compareTo(otherRefChain); + } + }; + + /** + * A cluster of known collections and object arrays that are problematic, i.e. each suffers from + * some problem like emptiness or sparseness, and thus has overhead value associated with it. + * The details are aggregated up to the "collection class : overhead kind : overhead value and + * number of instances for this class/kind" level. So an example information in this cluster may + * look like "HashMap : 100 of SPARSE_SMALL overhead 2000 bytes, 200 of EMPTY overhead 3000 + * bytes; ConcurrentHashMap : 10 of EMPTY overhead 1500 bytes" + */ + public static class Collections extends ReferencedObjCluster { + + private final List classAndOvhdList; + private final int numGoodCollections; + + public Collections(RefChainElement referer, List classAndOvhdList, int totalOverhead, + int numGoodCollections) { + super(referer, totalOverhead); + this.classAndOvhdList = classAndOvhdList; + this.numGoodCollections = numGoodCollections; + } + + /** + * Returns the detailed breakdown of problematic collections in this cluster. Many clusters + * contain collections of just one type (say HashMap) with one kind of problem (say empty). + * However, some clusters may represent things like "1000 empty ArrayLists with overhead of + * 10,000 bytes; 500 sparse ArrayLists with overhead of 5000 bytes; 1000 small HashMaps with + * overhead of 40,000 bytes" etc. That's why we return a list here. + */ + public List getList() { + return classAndOvhdList; + } + + @Override + public int getNumBadObjects() { + int numObjects = 0; + for (ClassAndOvhdCombo combo : classAndOvhdList) { + numObjects += combo.getNumInstances(); + } + return numObjects; + } + + /** + * Returns the number of collections that don't have any problems, but are reachable via the + * same reference chain as the problematic objects in this cluster. + */ + public int getNumGoodCollections() { + return numGoodCollections; + } + + @Override + public String clusterAsString(MemNumFormatter nf) { + StringBuilder buf = new StringBuilder(48); + + buf.append(nf.getNumInKAndPercent(getTotalOverhead())).append(":"); + + String prevCollectionClassName = null; + for (ClassAndOvhdCombo entry : classAndOvhdList) { + if (!entry.getClazz().getName().equals(prevCollectionClassName)) { + buf.append(' ').append(entry.getClazz().getHumanFriendlyName()).append(": "); + } else { + buf.append(", "); + } + prevCollectionClassName = entry.getClazz().getName(); + + buf.append(entry.getNumInstances()).append(" of "); + buf.append(entry.getProblemKind().name()).append(' '); + buf.append(nf.getNumInKAndPercent(entry.getOverhead())); + } + + if (numGoodCollections > 0) { + buf.append(", ").append(numGoodCollections).append(" good collections"); + } + + return buf.toString(); + } + } + + /** + * A cluster of duplicate strings. Most of the time such clusters contain more than one string + * value; the details are aggregated up to the "string value : number of instances" level. So + * example information in this cluster may look like "1000 of "Foo", 500 of "Bar" ... and 3000 + * more strings, of which 50 are unique". + */ + public static class DupStrings extends ReferencedObjCluster { + + // We now don't print long strings fully, but just in case, want to be able to + private static final boolean PRINT_LONG_STRINGS_FULLY = false; + // We now don't print all strings, but just in case, want to be able to + private static final boolean PRINT_ALL_STRINGS = false; + + private final int numDupBackingCharArrays; + private final int numNonDupStrings; + private final ObjectToIntMap.Entry entries[]; + + public DupStrings(RefChainElement referer, int totalOverhead, int numDupBackingCharArrays, int numNonDupStrings, + ObjectToIntMap.Entry entries[]) { + super(referer, totalOverhead); + this.numDupBackingCharArrays = numDupBackingCharArrays; + this.numNonDupStrings = numNonDupStrings; + this.entries = entries; + } + + /** + * Returns the breakdown of duplicate strings in this cluster: string value and the number + * of String instances with this value. + */ + public ObjectToIntMap.Entry[] getEntries() { + return entries; + } + + @Override + public int getNumBadObjects() { + int result = 0; + for (int i = 0; i < entries.length; i++) { + result += entries[i].value; + } + return result; + } + + /** + * Returns the total number of backing char arrays for duplicate strings in this cluster. + * This number can be smaller than the number of strings if some strings share the same + * backing array. + */ + public int getNumDupBackingCharArrays() { + return numDupBackingCharArrays; + } + + /** + * Returns the number of strings that are not duplicated, but are reachable via the same + * reference chain as the problematic strings in this cluster. + */ + public int getNumNonDupStrings() { + return numNonDupStrings; + } + + @Override + public String clusterAsString(MemNumFormatter nf) { + int nUniqueStrings = entries.length; + int nAllStrings = getNumBadObjects(); + + StringBuilder buf = new StringBuilder(64); + buf.append(nf.getNumInKAndPercent(getTotalOverhead())); + buf.append(' ').append(nAllStrings).append(" dup strings ("); + buf.append(nUniqueStrings).append(" unique)"); + buf.append(", ").append(numDupBackingCharArrays).append(" dup backing arrays"); + if (numNonDupStrings > 0) { + buf.append(", ").append(numNonDupStrings).append(" nondup strings"); + } + buf.append(":\n"); + + String s = null; + int len = 0; + int count = 0; + + for (ObjectToIntMap.Entry entry : entries) { + if (s != null) { // Avoid very long lines + buf.append(", "); + len += s.length() + 2; + if (len > 70) { + buf.append('\n'); + len = 0; + } + } + s = entry.key; + int maxLen = PRINT_LONG_STRINGS_FULLY ? 0 : 80; + s = MiscUtils.removeEndLinesAndAddQuotes(s, maxLen); + buf.append(entry.value).append(" of ").append(s); + + count++; + if (!PRINT_ALL_STRINGS && count >= 10) { + int nRemainingStringGroups = entries.length - count; + int nTotalRemainingStrings = 0; + for (int i = count; i < entries.length; i++) { + nTotalRemainingStrings += entries[i].value; + } + if (nTotalRemainingStrings > 0) { + buf.append("\n... and "); + buf.append(nTotalRemainingStrings); + buf.append(" more strings, of which "); + buf.append(nRemainingStringGroups); + buf.append(" are unique"); + } + break; + } + } + + return buf.toString(); + } + } + + /** + * A cluster of duplicated primitive arrays. It may contain more than one array value; the + * details are aggregated up to the "array type/value : number of instances" level. So example + * information in this cluster may look like "100 of byte[]{0x1F, 0x2A, 0x33}, 50 of char[]{abc} + * ... and 30 more arrays, of which 12 are unique". + */ + public static class DupArrays extends ReferencedObjCluster { + + private final int numNonDupArrays; + private final ObjectToIntMap.Entry entries[]; + + public DupArrays(RefChainElement referer, int totalOverhead, int numNonDupArrays, + ObjectToIntMap.Entry entries[]) { + super(referer, totalOverhead); + this.numNonDupArrays = numNonDupArrays; + this.entries = entries; + } + + /** + * Returns the breakdown of duplicate arrays in this cluster: array value (contents) and the + * number of array objects with this value. + */ + public ObjectToIntMap.Entry[] getEntries() { + return entries; + } + + @Override + public int getNumBadObjects() { + int result = 0; + for (ObjectToIntMap.Entry entry : entries) { + result += entry.value; + } + return result; + } + + /** + * Returns the number of arrays that are not duplicated, but are reachable via the same + * reference chain as the problematic arrays in this cluster. + */ + public int getNumNonDupArrays() { + return numNonDupArrays; + } + + @Override + public String clusterAsString(MemNumFormatter nf) { + int nUniqueArrays = entries.length; + int nAllArrays = getNumBadObjects(); + + StringBuilder buf = new StringBuilder(64); + buf.append(nf.getNumInKAndPercent(getTotalOverhead())); + buf.append(' ').append(nAllArrays).append(" dup arrays ("); + buf.append(nUniqueArrays).append(" unique)"); + if (numNonDupArrays > 0) { + buf.append(", ").append(numNonDupArrays).append(" nondup arrays"); + } + buf.append(":\n"); + + String s = null; + int len = 0; + int count = 0; + + for (ObjectToIntMap.Entry entry : entries) { + if (s != null) { // Avoid very long lines + buf.append(", "); + len += s.length() + 2; + if (len > 70) { + buf.append('\n'); + len = 0; + } + } + s = entry.key.getArray().valueAsString(); + + buf.append(entry.value).append(" of ").append(s); + + count++; + if (count >= 10) { + int nRemainingStringGroups = entries.length - count; + int nTotalRemainingStrings = 0; + for (int i = count; i < entries.length; i++) { + nTotalRemainingStrings += entries[i].value; + } + if (nTotalRemainingStrings > 0) { + buf.append("\n... and "); + buf.append(nTotalRemainingStrings); + buf.append(" more arrays, of which "); + buf.append(nRemainingStringGroups); + buf.append(" are unique"); + } + break; + } + } + + return buf.toString(); + } + } + + /** + * A cluster of objects for which we know or expect the total size to be high. The details are + * aggregated up to the "class : total size and number of instances for this class" level. + */ + public static class HighSizeObjects extends ReferencedObjCluster { + + private final List classAndSizeList; + + public HighSizeObjects(RefChainElement referer, List classAndSizeList, int totalSize) { + super(referer, totalSize); + this.classAndSizeList = classAndSizeList; + } + + /** + * Returns the detailed breakdown of classes in this cluster. + */ + public List getList() { + return classAndSizeList; + } + + @Override + public int getNumBadObjects() { + int numObjects = 0; + for (ClassAndSizeCombo combo : classAndSizeList) { + numObjects += combo.getNumInstances(); + } + return numObjects; + } + + @Override + public String clusterAsString(MemNumFormatter nf) { + StringBuilder buf = new StringBuilder(48); + + buf.append(nf.getNumInKAndPercent(getTotalOverhead())).append(":"); + + boolean first = true; + for (ClassAndSizeCombo entry : classAndSizeList) { + if (!first) { + buf.append(','); + } + first = false; + + buf.append(' ').append(entry.getClazz().getHumanFriendlyName()).append(": "); + + buf.append(entry.getNumInstances()).append(" instances "); + buf.append(nf.getNumInKAndPercent(entry.getSizeOrOvhd())); + } + + return buf.toString(); + } + } + + /** + * A cluster of instances of WeakHashMap (and/or its subclasses), that are problematic, because + * there are references back from values to keys. The details are aggregated to the level of + * "all weak collection classes in this cluster : a set of ValueType.field samples". So example + * information in this cluster may look like "3 WeakHashMap instances, have back refs from + * instances of Foo.bar, Boo.baz". + */ + public static class WeakHashMaps extends ReferencedObjCluster { + + private final int numInstances; + private final SmallSet colClasses; + private final SmallSet valueTypeAndFieldSamples; + + public WeakHashMaps(RefChainElement referer, int numInstances, int totalOverhead, SmallSet colClasses, + SmallSet valueTypeAndFieldSamples) { + super(referer, totalOverhead); + this.numInstances = numInstances; + this.colClasses = colClasses; + this.valueTypeAndFieldSamples = valueTypeAndFieldSamples; + } + + /** Returns names of classes of weak hashmap objects in this cluster. */ + public String[] getClasses() { + return colClasses.getElements(String[].class); + } + + /** + * Returns an array of Strings representing all class/field pairs for objects that weak + * hashmaps in this cluster contain as values, that have references back to keys. For + * example, may return ["Foo.bar", "Boo.baz"] meaning that the problematic weak hashmaps in + * this cluster contain objects of classes Foo and Bar as values, and that fields Foo.bar + * and Boo.baz may point back to keys. + */ + public String[] getBackRefs() { + return valueTypeAndFieldSamples.getElements(String[].class); + } + + @Override + public int getNumBadObjects() { + return numInstances; + } + + @Override + public String clusterAsString(MemNumFormatter nf) { + StringBuilder buf = new StringBuilder(64); + String[] classes = colClasses.getElements(String[].class); + for (int i = 0; i < classes.length; i++) { + classes[i] = ClassUtils.getShortNameForPopularClass(classes[i]); + } + String[] valueTypesAndFields = valueTypeAndFieldSamples.getElements(String[].class); + + buf.append(nf.getNumInKAndPercent(getTotalOverhead())).append(' '); + buf.append(numInstances).append(" of "); + buf.append(MiscUtils.asCommaSeparatedList(classes)).append(' '); + buf.append("have back refs from: "); + buf.append(MiscUtils.asCommaSeparatedList(valueTypesAndFields)); + + return buf.toString(); + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/batch/ReportFormatter.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/batch/ReportFormatter.java new file mode 100644 index 00000000..76aab124 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/batch/ReportFormatter.java @@ -0,0 +1,687 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.batch; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +import org.openjdk.jmc.joverflow.descriptors.CollectionClassDescriptor; +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.stats.ClassloaderStats; +import org.openjdk.jmc.joverflow.stats.LengthHistogram; +import org.openjdk.jmc.joverflow.stats.ObjectHistogram; +import org.openjdk.jmc.joverflow.support.CompressibleStringStats; +import org.openjdk.jmc.joverflow.support.Constants; +import org.openjdk.jmc.joverflow.support.DupArrayStats; +import org.openjdk.jmc.joverflow.support.DupStringStats; +import org.openjdk.jmc.joverflow.support.HeapStats; +import org.openjdk.jmc.joverflow.support.NumberEncodingStringStats; +import org.openjdk.jmc.joverflow.support.RefChainElement; +import org.openjdk.jmc.joverflow.support.RefChainElementImpl; +import org.openjdk.jmc.joverflow.support.ReferenceChain; +import org.openjdk.jmc.joverflow.support.ShortArrayStats; +import org.openjdk.jmc.joverflow.util.MiscUtils; +import org.openjdk.jmc.joverflow.util.ObjectToIntMap; + +/** + * Functionality for generating text report from results of batch analysis (HeapStats and + * DetailedStats objects). This is currently used by command-line JOverflow tool. + */ +public class ReportFormatter { + + private static final String DASH_SEPARATOR = "\n---------------------------------------------------"; + + private final HeapStats hs; + private final DetailedStats ds; + private final FormattedOutputBuffer b; + + public ReportFormatter(HeapStats hs, DetailedStats ds) { + this.hs = hs; + this.ds = ds; + b = new FormattedOutputBuffer(hs.totalObjSize); + } + + public HeapStats getHeapStats() { + return hs; + } + + public DetailedStats getDetailedStats() { + return ds; + } + + public String getReport( + boolean printFullObjectHistogram, int printedRefChainDepth, String[] refChainStopperClassPrefixes) { + formatOverallStats(); + formatDetailedStats(printFullObjectHistogram, printedRefChainDepth, refChainStopperClassPrefixes); + return b.getOutput(); + } + + private void formatOverallStats() { + b.println(DASH_SEPARATOR); + b.println("1. OVERALL STATS:\n"); + + b.println("1.0 Fundamentals\n"); + + b.format("Pointer size: %d bytes %s\n", hs.ptrSize, hs.usingNarrowPointers ? "(narrow in 64-bit mode)" : ""); + b.format("Object header size: %d bytes\n", hs.objHeaderSize); + b.format("Object alignment: %d bytes\n", hs.objAlignment); + + b.format("\nTotal num of objects: %,d\n", hs.nObjects); + b.format("Instances: %,d, object arrays: %,d, primitive arrays: %,d\n", hs.nInstances, hs.nObjectArrays, + hs.nValueArrays); + b.format("Total size of all objects: %s\n", b.k(hs.totalObjSize)); + b.format("Instances: %s, object arrays: %s, primitive arrays: %s\n", b.k(hs.totalInstSize), + b.k(hs.totalObjArraySize), b.k(hs.totalValueArraySize)); + + b.print("\nMinimum reported overhead: "); + if (ds.minOvhdToReport < 1024) { + b.println(ds.minOvhdToReport + " bytes"); + } else { + b.println((ds.minOvhdToReport / 1024) + "K"); + } + + b.println("\n1.1 Assorted raw stats; no distinction between collections and standalone arrays\n"); + + b.format("Total size of object headers: %s\n", b.k(hs.ovhdObjHeaders)); + b.format("Num and size of all *$Entry instances: %,d, %s\n", hs.nEntryInstances, b.k(hs.entryClassSize)); + + ShortArrayStats soa = hs.shortObjArrayStats; + b.format("\nNum and overhead of length 0 obj arrays: %,d %s\n", soa.n0LenObjs, b.k(soa.ovhd0LenObjs)); + b.format("Num and overhead of length 1 obj arrays: %,d %s\n", soa.n1LenObjs, b.k(soa.ovhd1LenObjs)); + b.format("Num and overhead of length 2..4 obj arrays: %,d %s\n", soa.n4LenObjs, b.k(soa.ovhd4LenObjs)); + b.format("Num and overhead of length 5..8 obj arrays: %,d %s\n", soa.n8LenObjs, b.k(soa.ovhd8LenObjs)); + + ShortArrayStats spa = hs.shortPrimitiveArrayStats; + b.format("\nNum and overhead of length 0 primitive arrays: %,d %s\n", spa.n0LenObjs, b.k(spa.ovhd0LenObjs)); + b.format("Num and overhead of length 1 primitive arrays: %,d %s\n", spa.n1LenObjs, b.k(spa.ovhd1LenObjs)); + b.format("Num and overhead of length 2..4 primitive arrays: %,d %s\n", spa.n4LenObjs, b.k(spa.ovhd4LenObjs)); + b.format("Num and overhead of length 5..8 primitive arrays: %,d %s\n", spa.n8LenObjs, b.k(spa.ovhd8LenObjs)); + + ShortArrayStats sstrs = hs.shortStringStats; + b.format("\nNum and overhead of length 0 Strings: %,d %s\n", sstrs.n0LenObjs, b.k(sstrs.ovhd0LenObjs)); + b.format("Num and overhead of length 1 Strings: %,d %s\n", sstrs.n1LenObjs, b.k(sstrs.ovhd1LenObjs)); + b.format("Num and overhead of length 2..4 Strings: %,d %s\n", sstrs.n4LenObjs, b.k(sstrs.ovhd4LenObjs)); + b.format("Num and overhead of length 5..8 Strings: %,d %s\n", sstrs.n8LenObjs, b.k(sstrs.ovhd8LenObjs)); + + NumberEncodingStringStats nesStats = hs.numberEncodingStringStats; + b.format("\nNum and overhead of Strings that encode int numbers: %,d %s\n", nesStats.nStringsEncodingInts, + b.k(nesStats.stringsEncodingIntsOvhd)); + + b.format("\nNum of boxed Numbers: %,d\n", hs.nBoxedNumbers); + b.format("Overhead of boxed Numbers: %s\n", b.k(hs.ovhdBoxedNumbers)); + + b.format("\nNum of wrapped Unmodifiable* collection classes:\n"); + for (ObjectToIntMap.Entry entry : hs.unmodifiableClasses) { + b.format("%20s : %,d\n", entry.key, entry.value); + } + // FIXME: We probably can't get anything useful out of these collections. Remove calculation of this? +// b.format("\nNum of wrapped Synchronized* collection classes:\n"); +// for (ObjectToIntMap.Entry entry : hs.synchronizedClasses) { +// b.format("%20s : %4d\n", entry.key, entry.value); +// } + + b.println("\n1.2 Stats on the JVM that produced the dump (from System.getProperties()):\n"); + + HashMap systemProps = hs.systemProperties; + if (systemProps != null) { + for (Map.Entry entry : systemProps.entrySet()) { + String key = entry.getKey(); + if (key.startsWith("\"java.runtime.") || key.startsWith("\"java.vm.") + || key.startsWith("\"java.specification.") || key.startsWith("\"os.")) { + b.println(key + " = " + entry.getValue()); + } + } + } else { + b.println(" *** Could not be found ***"); + } + + b.println("\n1.3 Stats on classloaders\n"); + + ClassloaderStats clStats = hs.classloaderStats; + ObjectToIntMap clInstToNumLoadedClasses = clStats.getCLInstToNumLoadedClasses(); + ObjectToIntMap clClazzToNumLoadedClasses = clStats.getClClazzToNumLoadedClasses(); + int numClTypesWithLoadedClasses = 0; + for (ObjectToIntMap.Entry entry : clClazzToNumLoadedClasses.getEntries()) { + if (entry.value > 0) { + numClTypesWithLoadedClasses++; + } + } + int numClInstancesWithOneLoaded = 0; + HashSet clClassesWithOneLoaded = new HashSet<>(); + for (ObjectToIntMap.Entry entry : clInstToNumLoadedClasses.getEntries()) { + if (entry.value == 1) { + numClInstancesWithOneLoaded++; + clClassesWithOneLoaded.add(entry.key.getClazz()); + } + } + + b.format("Num of classes extending java.lang.ClassLoader: %,d\n", clClazzToNumLoadedClasses.size()); + b.format("Classloader instances with loaded classes: num: %,d types: %,d\n", clInstToNumLoadedClasses.size(), + numClTypesWithLoadedClasses); + b.format("Classloader instances with only one loaded class: num: %,d types: %,d\n", numClInstancesWithOneLoaded, + clClassesWithOneLoaded.size()); + + b.println("\n1.4 Stats on compressible strings\n"); + + CompressibleStringStats cs = hs.compressibleStringStats; + b.format("Total num of String objects: %,d\n", cs.nTotalStrings); + b.format("Total used bytes in backing arrays: %s\n", b.k(cs.totalUsedBackingArrayBytes)); + int percent = (int) (((double) cs.nCompressedStrings) * 100 / cs.nTotalStrings); + b.format("Num of Strings with backing byte[] arrays: %,d (%d%% of all Strings)\n", cs.nCompressedStrings, + percent); + b.format("Total used bytes in backing byte[] arrays: %s\n", b.k(cs.compressedBackingArrayBytes)); + percent = (int) (((double) cs.nAsciiCharBackedStrings) * 100 / cs.nTotalStrings); + b.format("Num of Strings backed by ASCII char[] arrays: %,d (%d%% of all Strings)\n", + cs.nAsciiCharBackedStrings, percent); + b.format("Total used bytes in backing ASCII char[] arrays: %s\n", b.k(cs.asciiCharBackingArrayBytes)); + } + + private void formatDetailedStats( + boolean printFullObjectHistogram, int printedRefChainDepth, String[] refChainStopperClassPrefixes) { + b.println(DASH_SEPARATOR); + b.startSection("2. CLASS AND OBJECT INFORMATION:", "High memory consumption by instances of", 10.0); + + List objHistogram = hs.objHisto + .getListSortedByInclusiveSize(printFullObjectHistogram ? 0 : ds.minOvhdToReport); + + b.format("\nTotal classes: %,d Total objects: %,d\n", hs.nClasses, hs.nObjects); + int[] smallInstClasses = hs.objHisto.calculateNumSmallInstClasses(); + b.format("Classes with no instances: %,d Classes with 1 instance: %,d\n", smallInstClasses[0], + smallInstClasses[1]); + b.println("\nObject histogram for top memory consumers"); + b.println(" #instances Shallow size Impl-inclusive size Class name"); + b.println("---------------------------------------------------------------"); + for (ObjectHistogram.Entry entry : objHistogram) { + b.format("%,10d %16s %16s %s\n", entry.getNumInstances(), b.k(entry.getTotalShallowSize()), + b.k(entry.getTotalInclusiveSize()), entry.getClazz().getHumanFriendlyNameWithLoaderIfNeeded()); + b.criticalCheck(entry.getTotalInclusiveSize(), entry.getClazz().getHumanFriendlyNameWithLoaderIfNeeded()); + } + + b.println(DASH_SEPARATOR); + b.startSection("3. NUMBER, SIZE AND NEAREST FIELDS FOR HIGH MEMORY CONSUMERS:"); + + b.println(); + List hsFields = ds.highSizeObjClusters.get(1); + for (ReferencedObjCluster.HighSizeObjects c : hsFields) { + RefChainElement classAndField = c.getReferer(); + String classAndFieldStr = ReferenceChain.toStringInStraightOrder(classAndField); + + String fieldDefiningClass = getFieldDefiningClassFromFieldRefChain( + ReferenceChain.getRootElement(classAndField)); + if (fieldDefiningClass != null) { + classAndFieldStr += " (defined in " + fieldDefiningClass + ")"; + } + + b.print(" "); + b.print(classAndFieldStr); + b.println(" -->"); + b.println(c.clusterAsString(b.getMemNumFormatter())); + } + + b.println(DASH_SEPARATOR); + b.startSection("4. NUMBER, SIZE AND REF CHAINS FOR TOP MEMORY CONSUMERS:"); + + b.println(); + List hsReverseChains = ds.highSizeObjClusters.get(0); + for (ReferencedObjCluster c : hsReverseChains) { + b.println(c.clusterAsString(b.getMemNumFormatter())); + b.print(" "); + b.println(ReferenceChain.toStringInReverseOrder(c.getReferer(), printedRefChainDepth, + refChainStopperClassPrefixes)); + } + + b.println(DASH_SEPARATOR); + b.startSection("5. PROBLEMATIC COLLECTIONS:", "High overhead due to problematic collections of kind", 2.0); + b.format("\nTotal collections: %,d\n", hs.numCols); + + b.format("\nEmpty unused collections total number: %,10d, ovhd: %14s\n", hs.numEmptyUnusedCols, + b.k(hs.emptyUnusedColsOverhead)); + b.criticalCheck(hs.emptyUnusedColsOverhead, "empty unused"); + printClassesWithProblemKind(Constants.ProblemKind.EMPTY_UNUSED, false, false); + b.format("\nEmpty used collections total number: %,10d, ovhd: %14s\n", hs.numEmptyUsedCols, + b.k(hs.emptyUsedColsOverhead)); + b.criticalCheck(hs.emptyUsedColsOverhead, "empty used"); + printClassesWithProblemKind(Constants.ProblemKind.EMPTY_USED, false, false); + b.format("\nEmpty collections total number: %,10d, ovhd: %14s\n", hs.numEmptyCols, + b.k(hs.emptyColsOverhead)); + b.criticalCheck(hs.emptyColsOverhead, "empty"); + printClassesWithProblemKind(Constants.ProblemKind.EMPTY, false, false); + b.format("\nSmall sparse collections total number: %,10d, ovhd: %14s\n", hs.numSparseSmallCols, + b.k(hs.sparseSmallColsOverhead)); + b.criticalCheck(hs.sparseSmallColsOverhead, "small sparse"); + printClassesWithProblemKind(Constants.ProblemKind.SPARSE_SMALL, false, false); + b.format("\nLarge sparse collections total number: %,10d, ovhd: %14s\n", hs.numSparseLargeCols, + b.k(hs.sparseLargeColsOverhead)); + b.criticalCheck(hs.sparseLargeColsOverhead, "large sparse"); + printClassesWithProblemKind(Constants.ProblemKind.SPARSE_LARGE, false, false); + b.format("\nBoxed Number collections total number: %,10d, ovhd: %14s\n", hs.numBoxedNumberCols, + b.k(hs.boxedNumberColsOverhead)); + b.criticalCheck(hs.boxedNumberColsOverhead, "boxed number"); + printClassesWithProblemKind(Constants.ProblemKind.BOXED, false, false); + b.format("\nVertical bar collections total number: %,10d, ovhd: %14s\n", hs.numBarCols, + b.k(hs.barColsOverhead)); + b.criticalCheck(hs.barColsOverhead, "vertical bar"); + printClassesWithProblemKind(Constants.ProblemKind.BAR, false, false); + b.format("\nSmall collections total number: %,10d, ovhd: %14s\n", hs.numSmallCols, + b.k(hs.smallColsOverhead)); + b.criticalCheck(hs.smallColsOverhead, "small"); + printClassesWithProblemKind(Constants.ProblemKind.SMALL, false, false); + + b.println(DASH_SEPARATOR); + b.startSection("6. PROBLEMATIC STANDALONE OBJECT ARRAYS:", + "High overhead due to problematic object arrays of kind", 2.0); + b.format("\nTotal standalone obj arrays: %,d\n", hs.numObjArrays); + + b.format("\nLength 0 object arrays number: %,10d, ovhd: %14s\n", hs.numLengthZeroObjArrays, + b.k(hs.lengthZeroObjArraysOverhead)); + b.criticalCheck(hs.lengthZeroObjArraysOverhead, "length 0"); + printClassesWithProblemKind(Constants.ProblemKind.LENGTH_ZERO, true, false); + b.format("\nLength 1 object arrays number: %,10d, ovhd: %14s\n", hs.numLengthOneObjArrays, + b.k(hs.lengthOneObjArraysOverhead)); + b.criticalCheck(hs.lengthOneObjArraysOverhead, "length 1"); + printClassesWithProblemKind(Constants.ProblemKind.LENGTH_ONE, true, false); + b.format("\nEmpty object arrays number: %,10d, ovhd: %14s\n", hs.numEmptyObjArrays, + b.k(hs.emptyObjArraysOverhead)); + b.criticalCheck(hs.emptyObjArraysOverhead, "empty"); + printClassesWithProblemKind(Constants.ProblemKind.EMPTY, true, false); + b.format("\nSparse object arrays number: %,10d, ovhd: %14s\n", hs.numSparseObjArrays, + b.k(hs.sparseObjArraysOverhead)); + b.criticalCheck(hs.sparseObjArraysOverhead, "sparse"); + printClassesWithProblemKind(Constants.ProblemKind.SPARSE_ARRAY, true, false); + b.format("\nBoxed Number object arrays number: %,10d, ovhd: %14s\n", hs.numBoxedNumberArrays, + b.k(hs.boxedNumberArraysOverhead)); + b.criticalCheck(hs.boxedNumberArraysOverhead, "boxed"); + printClassesWithProblemKind(Constants.ProblemKind.BOXED, true, false); + b.format("\nVertical bar object arrays number: %,10d, ovhd: %14s\n", hs.numBarObjArrays, + b.k(hs.barObjArraysOverhead)); + b.criticalCheck(hs.barObjArraysOverhead, "vertical bar"); + printClassesWithProblemKind(Constants.ProblemKind.BAR, true, false); + + b.println(DASH_SEPARATOR); + b.startSection("7. PROBLEMATIC STANDALONE PRIMITIVE ARRAYS:", + "High overhead due to problematic primitive arrays of kind", 2.0); + b.format("\nTotal standalone primitive arrays: %,d\n", hs.numValueArrays); + + b.format("\nLength 0 primitive arrays number: %,8d, ovhd: %14s\n", hs.numLengthZeroValueArrays, + b.k(hs.lengthZeroValueArraysOverhead)); + b.criticalCheck(hs.lengthZeroValueArraysOverhead, "length 0"); + printClassesWithProblemKind(Constants.ProblemKind.LENGTH_ZERO, true, true); + b.format("\nLength 1 primitive arrays number: %,8d, ovhd: %14s\n", hs.numLengthOneValueArrays, + b.k(hs.lengthOneValueArraysOverhead)); + b.criticalCheck(hs.lengthOneValueArraysOverhead, "length 1"); + printClassesWithProblemKind(Constants.ProblemKind.LENGTH_ONE, true, true); + b.format("\nEmpty primitive arrays number: %,8d, ovhd: %14s\n", hs.numEmptyValueArrays, + b.k(hs.emptyValueArraysOverhead)); + b.criticalCheck(hs.emptyValueArraysOverhead, "empty"); + printClassesWithProblemKind(Constants.ProblemKind.EMPTY, true, true); + b.format("\nLong zero-tail primitive arrays number: %,8d, ovhd: %14s\n", hs.numLZTValueArrays, + b.k(hs.lztValueArraysOverhead)); + b.criticalCheck(hs.lztValueArraysOverhead, "long zero-tail (LZT)"); + printClassesWithProblemKind(Constants.ProblemKind.LZT, true, true); + b.format("\nUnused high bytes primitive arrays number:%,8d, ovhd: %14s\n", hs.numUnusedHiBytesValueArrays, + b.k(hs.unusedHiBytesValueArraysOverhead)); + b.criticalCheck(hs.unusedHiBytesValueArraysOverhead, "unused high bytes"); + printClassesWithProblemKind(Constants.ProblemKind.UNUSED_HI_BYTES, true, true); + + b.println(DASH_SEPARATOR); + b.startSection("8. NUMBER, OVERHEAD AND NEAREST FIELDS FOR PROBLEMATIC COLLECTIONS AND ARRAYS:"); + + b.println(); + List colFields = ds.collectionClusters.get(1); + for (ReferencedObjCluster.Collections c : colFields) { + if (c.getTotalOverhead() < ds.minOvhdToReport / 4) { + break; + } + RefChainElement classAndField = c.getReferer(); + String classAndFieldStr = ReferenceChain.toStringInStraightOrder(classAndField); + + String fieldDefiningClass = getFieldDefiningClassFromFieldRefChain( + ReferenceChain.getRootElement(classAndField)); + if (fieldDefiningClass != null) { + classAndFieldStr += " (defined in " + fieldDefiningClass + ")"; + } + + b.print(" "); + b.print(classAndFieldStr); + b.println(" -->"); + b.println(c.clusterAsString(b.getMemNumFormatter())); + } + + b.println(DASH_SEPARATOR); + b.startSection("9. NUMBER, OVERHEAD AND REF CHAINS FOR PROBLEMATIC COLLECTIONS AND ARRAYS:"); + + b.println(); + List colReverseChains = ds.collectionClusters.get(0); + for (ReferencedObjCluster c : colReverseChains) { + // For individual clusters, we set a smaller overhead threshold + if (c.getTotalOverhead() < ds.minOvhdToReport / 4) { + break; + } + b.println(c.clusterAsString(b.getMemNumFormatter())); + b.print(" "); + b.println(ReferenceChain.toStringInReverseOrder(c.getReferer(), printedRefChainDepth, + refChainStopperClassPrefixes)); + } + + b.println(DASH_SEPARATOR); + b.startSection("10. DUPLICATE STRING STATS:", "High overhead due to duplicate strings", 5.0); + + DupStringStats dss = hs.dupStringStats; + b.format("\nTotal strings: %,d Unique strings: %,d Duplicate values: %,d Overhead: %s\n", dss.nStrings, + dss.nUniqueStringValues, dss.nUniqueDupStringValues, b.k(dss.dupStringsOverhead)); + b.criticalCheck(dss.dupStringsOverhead, ""); + + b.format("Top duplicate Strings:\n"); + b.format(" Ovhd Num char[]s Num objs Max arr len Value\n"); + for (DupStringStats.Entry entry : dss.dupStrings) { + if (entry.overhead < ds.minOvhdToReport) { + break; + } + + b.format("%14s %6d %6d %6d ", b.k(entry.overhead), entry.nBackingArrays, + entry.nStringInstances, entry.maxArrayLen); + + String string = MiscUtils.removeEndLinesAndAddQuotes(entry.string, 100); + b.println(string); + } + + b.println(DASH_SEPARATOR); + b.startSection("11. NUMBER, OVERHEAD AND NEAREST FIELDS FOR DUPLICATE STRINGS:"); + + b.println(); + List rsFields = ds.dupStringClusters.get(1); + for (ReferencedObjCluster c : rsFields) { + if (c.getTotalOverhead() < ds.minOvhdToReport / 4) { + break; + } + RefChainElement classAndField = c.getReferer(); + String classAndFieldStr = ReferenceChain.toStringInStraightOrder(classAndField); + + String fieldDefiningClass = getFieldDefiningClassFromFieldRefChain( + ReferenceChain.getRootElement(classAndField)); + if (fieldDefiningClass != null) { + classAndFieldStr += " (defined in " + fieldDefiningClass + ")"; + } + + b.print(" "); + b.print(classAndFieldStr); + b.println(" -->"); + b.println(c.clusterAsString(b.getMemNumFormatter())); + } + + b.println(DASH_SEPARATOR); + b.startSection("12. NUMBER, OVERHEAD AND REF CHAINS FOR DUPLICATE STRINGS:"); + + b.println(); + List rsReverseChains = ds.dupStringClusters.get(0); + for (ReferencedObjCluster c : rsReverseChains) { + // For individual clusters, we set a smaller overhead threshold + if (c.getTotalOverhead() < ds.minOvhdToReport / 4) { + break; + } + b.println(c.clusterAsString(b.getMemNumFormatter())); + b.print(" "); + b.println(ReferenceChain.toStringInReverseOrder(c.getReferer(), printedRefChainDepth, + refChainStopperClassPrefixes)); + } + + b.println(DASH_SEPARATOR); + b.startSection("13. DUPLICATE PRIMITIVE ARRAY STATS:", "High overhead due to duplicate primitive arrays", 5.0); + + DupArrayStats das = hs.dupArrayStats; + b.format("\nTotal primitive arrays: %,d Unique arrays: %,d Duplicate values: %,d Overhead: %s\n", das.nArrays, + das.nUniqueArrays, das.nDifferentDupArrayValues, b.k(das.dupArraysOverhead)); + b.criticalCheck(das.dupArraysOverhead, ""); + + b.format("Top duplicate primitive arrays:\n"); + b.format(" Ovhd Num objs Array len Value\n"); + for (DupArrayStats.Entry entry : das.dupArrays) { + if (entry.overhead < ds.minOvhdToReport) { + break; + } + + b.format("%14s %6d %6d ", b.k(entry.overhead), entry.nArrayInstances, + entry.firstArray.getLength()); + + String arrayAsString = entry.firstArray.valueAsString(); + b.println(arrayAsString); + } + + b.println(DASH_SEPARATOR); + b.startSection("14. NUMBER, OVERHEAD AND NEAREST FIELDS FOR DUPLICATE PRIMITIVE ARRAYS:"); + + b.println(); + List daFields = ds.dupArrayClusters.get(1); + for (ReferencedObjCluster c : daFields) { + if (c.getTotalOverhead() < ds.minOvhdToReport / 4) { + break; + } + RefChainElement classAndField = c.getReferer(); + String classAndFieldStr = ReferenceChain.toStringInStraightOrder(classAndField); + + String fieldDefiningClass = getFieldDefiningClassFromFieldRefChain( + ReferenceChain.getRootElement(classAndField)); + if (fieldDefiningClass != null) { + classAndFieldStr += " (defined in " + fieldDefiningClass + ")"; + } + + b.print(" "); + b.print(classAndFieldStr); + b.println(" -->"); + b.println(c.clusterAsString(b.getMemNumFormatter())); + } + + b.println(DASH_SEPARATOR); + b.startSection("15. NUMBER, OVERHEAD AND REF CHAINS FOR DUPLICATE PRIMITIVE ARRAYS:"); + + b.println(); + List daReverseChains = ds.dupArrayClusters.get(0); + for (ReferencedObjCluster c : daReverseChains) { + // For individual clusters, we set a smaller overhead threshold + if (c.getTotalOverhead() < ds.minOvhdToReport / 4) { + break; + } + b.println(c.clusterAsString(b.getMemNumFormatter())); + b.print(" "); + b.println(ReferenceChain.toStringInReverseOrder(c.getReferer(), printedRefChainDepth, + refChainStopperClassPrefixes)); + } + + b.println(DASH_SEPARATOR); + b.startSection( + "16. WEAK HASHMAPS WITH REFS FROM VALUES TO KEYS\n" + + " (conservative estimate; deep object size not calculated):", + "Found WeakHashMaps with references from values to keys, minimum overhead", 0.01); + + b.format("\n Ovhd Num collections Type\n"); + Constants.ProblemKind weakKind = Constants.ProblemKind.WEAK_MAP_WITH_BACK_REFS; + for (CollectionClassDescriptor cd : hs.overheadsByClass) { + int numProblematicCollections = cd.getNumProblematicCollections(weakKind); + if (numProblematicCollections == 0) { + continue; + } + + String className = cd.getClazz().getHumanFriendlyName(); + int ovhd = cd.getProblematicCollectionsOverhead(weakKind); + b.criticalCheck(ovhd, ""); + b.format("%14s %,8d %s\n", b.k(ovhd), numProblematicCollections, className); + } + + b.println(); + List wmReverseChains = ds.weakHashMapClusters.get(0); + for (ReferencedObjCluster c : wmReverseChains) { + // Here we print data for any overhead, just in case + b.println(c.clusterAsString(b.getMemNumFormatter())); + b.print(" "); + b.println(ReferenceChain.toStringInReverseOrder(c.getReferer(), printedRefChainDepth, + refChainStopperClassPrefixes)); + } + + b.println(); + List wmFields = ds.weakHashMapClusters.get(1); + for (ReferencedObjCluster c : wmFields) { + // Here we print data for any overhead, just in case + b.print(" "); + b.print(ReferenceChain.toStringInStraightOrder(c.getReferer())); + b.println(" -->"); + b.println(c.clusterAsString(b.getMemNumFormatter())); + } + b.println(); + + b.println(DASH_SEPARATOR); + b.startSection("17. DATA FIELDS ALWAYS OR ALMOST ALWAYS NULL/ZERO, OR NO FIELDS:", + "High overhead due to fields that are null/zero/non-existent in", 1.0); + + printProblemFieldsHistogram(hs.objHisto, true, 1.0f, ds.minOvhdToReport); + printProblemFieldsHistogram(hs.objHisto, true, 0.9f, ds.minOvhdToReport); + + b.println(DASH_SEPARATOR); + b.startSection("18. PRIMITIVE DATA FIELDS WITH UNUSED HIGH BYTES:", + "High overhead due to primitive fields with unused high bytes", 1.0); + + printProblemFieldsHistogram(hs.objHisto, false, 1.0f, ds.minOvhdToReport); + printProblemFieldsHistogram(hs.objHisto, false, 0.9f, ds.minOvhdToReport); + + b.println(DASH_SEPARATOR); + b.startSection("19. STRING LENGTH STATISTICS:", "High memory consumption by Strings of length", 10.0); + + List strLenHisto = hs.stringLengthHistogram + .getPrunedAndSortedEntries(ds.minOvhdToReport); + + b.println("\n Length Count Size"); + b.println("----------------------------------------------"); + for (LengthHistogram.Entry entry : strLenHisto) { + int strLen = entry.getLength(); + String formattedLen = strLen >= 0 ? String.format("%,d", strLen) : "other"; + b.format(" %9s %,10d %16s\n", formattedLen, entry.getCount(), b.k(entry.getSize())); + b.criticalCheck(entry.getSize(), formattedLen); + } + } + + private void printClassesWithProblemKind( + Constants.ProblemKind problemKind, boolean selectArrays, boolean selectPrimitiveArrays) { + for (CollectionClassDescriptor cd : hs.overheadsByClass) { + JavaClass clazz = cd.getClazz(); + if (clazz.isArray()) { + if (!selectArrays) { + continue; + } + if (clazz.isAnyDimPrimitiveArray()) { + if (!selectPrimitiveArrays) { + continue; + } + } else { // An object array + if (selectPrimitiveArrays) { + continue; + } + } + } else { // Not an array + if (selectArrays) { + continue; + } + } + + int ovhd = cd.getProblematicCollectionsOverhead(problemKind); + if (ovhd < ds.minOvhdToReport) { + continue; + } + b.format("%39s %,8d %14s\n", clazz.getHumanFriendlyName(), + cd.getNumProblematicCollections(problemKind), b.k(ovhd)); + } + } + + /** + * If nullFields parameter is true, prints the info on null/zero fields. Otherwise, prints the + * info on primitive fields underutilizing high bytes. + */ + private void printProblemFieldsHistogram( + ObjectHistogram objHisto, boolean nullFields, float percentile, int minOverheadToReport) { + List problemClasses = nullFields + ? objHisto.getListSortedByNullFieldsOvhd(percentile) + : objHisto.getListSortedByUnusedHiByteFieldsOvhd(percentile); + int nClasses = problemClasses.size(); + int nObjects = 0; + long totalOverhead = 0; + + for (ObjectHistogram.ProblemFieldsEntry probClazz : problemClasses) { + nObjects += probClazz.getNumInstances(); + totalOverhead += probClazz.getAllProblemFieldsOvhd(); + } + int percentileAsInt = (int) (percentile * 100); + + String subIssue = String.format("%d%% of instances", percentileAsInt); + if (nullFields) { + b.format("\nClasses with some fields null/zero in %s, or no fields: %,d\n", subIssue, nClasses); + b.format("Objects: %,d Overhead: %s\n", nObjects, b.k(totalOverhead)); + b.criticalCheck(totalOverhead, subIssue); + b.println("Null fields ovhd #instances Class name / Null fields"); + } else { + b.format("\nClasses with primitive fields that don't use high byte(s) in %s: %,d\n", subIssue, nClasses); + b.format("Objects: %,d Overhead: %s\n", nObjects, b.k(totalOverhead)); + b.criticalCheck(totalOverhead, subIssue); + b.println("Bad fields ovhd #instances Class name / Null fields"); + } + + b.println("----------------------------------------------------------------"); + for (ObjectHistogram.ProblemFieldsEntry nfClass : problemClasses) { + if (nfClass.getAllProblemFieldsOvhd() < minOverheadToReport) { + break; + } + b.format("%15s %,10d %38s\n", b.k(nfClass.getAllProblemFieldsOvhd()), nfClass.getNumInstances(), + nfClass.getClazz().getHumanFriendlyNameWithLoaderIfNeeded()); + b.print(" "); + b.println(nfClass.getFieldsAsString()); + } + } + + private static String getFieldDefiningClassFromFieldRefChain(RefChainElement desc) { + if (!(desc instanceof RefChainElementImpl.AbstractField)) { + return null; + } + + RefChainElementImpl.AbstractField fieldDesc = (RefChainElementImpl.AbstractField) desc; + JavaClass clazz = fieldDesc.getJavaClass(); + int fieldIdx = fieldDesc.getFieldIdx(); + + JavaClass defClazz = clazz.getDeclaringClassForField(fieldIdx); + if (defClazz == clazz || defClazz == null) { + return null; + } else { + return defClazz.getName(); + } + } + +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/AbstractArrayBasedCollectionDescriptor.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/AbstractArrayBasedCollectionDescriptor.java new file mode 100644 index 00000000..25677475 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/AbstractArrayBasedCollectionDescriptor.java @@ -0,0 +1,290 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.descriptors; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObjectArray; +import org.openjdk.jmc.joverflow.heap.model.JavaThing; + +/** + * Base descriptor for Collection classes that keep their elements in an Object array, and this + * array is not always fully utilized (in other words, collection's capacity may exceed its size). + * Some of these classes also have a 'size' or similar field, while some don't. There are also other + * differences in data representation (e.g. for maps the array can points to workload objects + * directly or via intermediate "Entry" objects). Hence this descriptor has different concrete + * subclasses. + */ +abstract class AbstractArrayBasedCollectionDescriptor extends AbstractCollectionDescriptor + implements CollectionInstanceDescriptor.CapacityDifferentFromSize { + protected final Factory factory; + + AbstractArrayBasedCollectionDescriptor(JavaObject col, Factory factory) { + super(col); + this.factory = factory; + } + + @Override + public int getSparsenessOverhead(int ptrSize) { + int capacity, numNonEmptySlots; + + if (factory.classDesc.isMap()) { + + // We count the number of non-empty slots instead of simply calling + // getNumElements(), since in collections like HashMap, which features + // chains of Entries, we can get a sparseness problem (too many null slots) + // even when the number of elements is more than half the capacity. + JavaObjectArray els = getElementsArray(); + if (els == null) { // Can happen if elements is unresolved + return -1; + } + + capacity = els.getLength(); + numNonEmptySlots = 0; + JavaHeapObject[] elements = els.getElements(); + for (JavaHeapObject element : elements) { + if (element != null) { + numNonEmptySlots++; + } + } + } else { + capacity = getCapacity(); + numNonEmptySlots = getNumElements(); + } + + if (numNonEmptySlots < capacity / 2) { + return (capacity - numNonEmptySlots) * ptrSize; + } else { + return -1; + } + } + + @Override + public int getDefaultCapacity() { + return factory.initialCapacity; + } + + @Override + public int getCapacity() { + JavaObjectArray els = getElementsArray(); + if (els == null) { // Can happen if elements is unresolved + return 0; + } + + return els.getLength(); + } + + @Override + public void iterateList(ListIteratorCallback cb) { + JavaObjectArray elsArray = getElementsArray(); + if (elsArray == null) { + return; // Can happen if elements is unresolved + } + if (!cb.scanImplementationObject(elsArray)) { + return; + } + + // Iterate over all of the array elements, not just the number that size() + // (getNumElements()) gives us. This is in case there are some references + // there for whatever reason (inconsistency or something), and we don't want + // the corresponding objects to end up being recorded as not reachable from + // any GC root. + JavaHeapObject[] elements = elsArray.getElements(); + for (JavaHeapObject element : elements) { + if (element == null) { + continue; + } + if (!cb.scanListElement(element)) { + break; + } + } + } + + @Override + public void iterateMap(MapIteratorCallback cb) { + JavaObjectArray entriesArray = getElementsArray(); + if (entriesArray == null) { + return; // Can happen if entries array is unresolved + } + if (!cb.scanImplementationObject(entriesArray)) { + return; + } + + int numElements = getNumElements(); + if (numElements == 0) { + return; + } + + JavaHeapObject[] entries = entriesArray.getElements(); + JavaThing[] entryFields = null; + int keyFieldIdx = -1, valueFieldIdx = -1, nextFieldIdx = -1; + + outerLoop: for (JavaHeapObject entryThing : entries) { + if (entryThing == null || !(entryThing instanceof JavaObject)) { + continue; + } + JavaObject entry = (JavaObject) entryThing; + + while (entry != null) { + if (!cb.scanImplementationObject(entry)) { + break; // We get this in BFS if entry already seen + } + if (keyFieldIdx == -1) { + keyFieldIdx = factory.getKeyFieldIdx(entry); + valueFieldIdx = factory.getValueFieldIdx(entry); + nextFieldIdx = factory.getEntryNextFieldIdx(entry); + } + entryFields = entry.getFields(entryFields); + JavaThing keyThing = entryFields[keyFieldIdx]; + JavaThing valueThing = entryFields[valueFieldIdx]; + JavaHeapObject key = null, value = null; + if (keyThing instanceof JavaHeapObject) { + key = (JavaHeapObject) keyThing; + } + if (valueThing instanceof JavaHeapObject) { + value = (JavaHeapObject) valueThing; + } + if (!cb.scanMapEntry(key, value)) { + break outerLoop; + } + + JavaObject prevEntry = entry; + if (!(entryFields[nextFieldIdx] instanceof JavaObject)) { + break; // Unresolved object + } + entry = (JavaObject) entryFields[nextFieldIdx]; + if (entry == prevEntry) { + break; + } + } + } + } + + @Override + AbstractCollectionDescriptor.Factory getFactory() { + return factory; + } + + public JavaObjectArray getElementsArray() { + JavaThing els = fields[factory.elsArrayFieldIdx]; // May be null or UnresolvedObject + return els instanceof JavaObjectArray ? (JavaObjectArray) els : null; + } + + /** + * Returns the sum of shallow sizes of this collection object and its elements array. Marks both + * the head collection object and the array with + * {@link org.openjdk.jmc.joverflow.heap.model.JavaLazyReadObject#setVisitedAsCollectionImpl()} + */ + protected int getDirectImplSize() { + col.setVisitedAsCollectionImpl(); + int colSize = col.getSize(); + JavaObjectArray els = getElementsArray(); + if (els == null) { + return colSize; + } else { + els.setVisitedAsCollectionImpl(); + return colSize + els.getSize(); + } + } + + /** + * Returns additional memory size for $Entry objects used by most maps. It is very important + * that this method also marks these $Entry objects as setVisitedAsCollectionImpl(), so that + * their size is not taken into account when calculating implementation-inlcusive size for + * classes in DetailedStatsCalculator.handleInstance(). + */ + protected int getMapEntriesImplSize() { + int numEls = getNumElements(); + if (numEls == 0) { + return 0; + } + + JavaObjectArray entriesArray = getElementsArray(); + if (entriesArray == null) { + return 0; // Unresolved + } + + JavaHeapObject[] entries = entriesArray.getElements(); + int nextFieldIdx = factory.getEntryNextFieldIdx(entries); + if (nextFieldIdx == -1) { // Array is empty - may happen with concurrent updates?? + return 0; + } + JavaThing[] entryFields = null; // Reusable fields, to reduce GC pressure + + int size = 0; + for (JavaHeapObject entryThing : entries) { + if (entryThing == null || !(entryThing instanceof JavaObject)) { + continue; // Unresolved object + } + JavaObject entry = (JavaObject) entryThing; + + // It turns out that sometimes HashMaps etc. may be corrupted due to concurrent + // updates by multiple threads. As a result, their entries form a cycle. + // Thus we check each entry here if it has already been visited. + while (entry != null && !entry.isVisitedAsCollectionImpl()) { + entry.setVisitedAsCollectionImpl(); + size += entry.getSize(); + entryFields = entry.getFields(entryFields); + JavaObject prevEntry = entry; + if (!(entryFields[nextFieldIdx] instanceof JavaObject)) { + break; // Unresolved object + } + entry = (JavaObject) entryFields[nextFieldIdx]; + if (entry == prevEntry) { + break; + } + } + } + + return size; + } + + static abstract class Factory extends AbstractCollectionDescriptor.Factory { + private final int elsArrayFieldIdx; + private final int initialCapacity; + + Factory(JavaClass clazz, boolean isMap, String elsArrayFieldName, int initialCapacity, JavaClass[] implClasses, + String[] parentColClassNames) { + super(clazz, isMap, implClasses, parentColClassNames, false, new String[] {elsArrayFieldName}); + elsArrayFieldIdx = clazz.getInstanceFieldIndex(elsArrayFieldName); + this.initialCapacity = initialCapacity; + } + + Factory(JavaClass clazz, AbstractArrayBasedCollectionDescriptor.Factory superclassFactory) { + super(clazz, superclassFactory); + this.elsArrayFieldIdx = superclassFactory.elsArrayFieldIdx; + this.initialCapacity = superclassFactory.initialCapacity; + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/AbstractCollectionDescriptor.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/AbstractCollectionDescriptor.java new file mode 100644 index 00000000..fe32d9f5 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/AbstractCollectionDescriptor.java @@ -0,0 +1,403 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.descriptors; + +import org.openjdk.jmc.joverflow.heap.model.ImplInclusiveSizeCalculator; +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaField; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaInt; +import org.openjdk.jmc.joverflow.heap.model.JavaLong; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaThing; +import org.openjdk.jmc.joverflow.util.IntArrayList; + +/** + * Implementation of common class-level functionality for all descriptors. + */ +abstract class AbstractCollectionDescriptor implements CollectionInstanceDescriptor { + protected final JavaObject col; + protected final JavaThing[] fields; + private int implInclusiveSize = -1; + + AbstractCollectionDescriptor(JavaObject col) { + this.col = col; + fields = col.getFields(); + } + + @Override + public CollectionClassDescriptor getClassDescriptor() { + return getFactory().classDesc; + } + + @Override + public int getImplSize() { + if (implInclusiveSize == -1) { + implInclusiveSize = doGetImplSize(); + } + return implInclusiveSize; + } + + /** + * Actually calculates the implementation-inclusive size for this collection. See + * {@link #getImplSize()}. Since this calculation may be expensive, this method is called only + * once by {@link #getImplSize()}, and the result is cached. + */ + protected abstract int doGetImplSize(); + + abstract Factory getFactory(); + + @Override + public long getModCount() { + Factory factory = getFactory(); + int modCountFieldIdx = factory.modCountFieldIdx; + if (modCountFieldIdx == -1) { + throw new RuntimeException("There is no modCount field in class " + factory.classDesc.getClassName()); + } + JavaThing modCountField = fields[modCountFieldIdx]; + if (modCountField instanceof JavaLong) { + return ((JavaLong) modCountField).getValue(); + } else { + return ((JavaInt) modCountField).getValue(); + } + } + + @Override + public final JavaHeapObject getSampleElement() { + class ListSampler implements ListIteratorCallback { + JavaHeapObject sampleElement; + private int counter; + + @Override + public boolean scanListElement(JavaHeapObject element) { + counter++; + if (element == null) { + return (counter < 3); + } + if (counter == 1) { + sampleElement = element; + } else { + if (sampleElement != null && element.getClazz() == sampleElement.getClazz()) { + sampleElement = element; + } else { + sampleElement = null; + return false; + } + } + return (counter < 3); + } + + @Override + public boolean scanImplementationObject(JavaHeapObject implObj) { + return true; + } + } + + ListSampler listSampler = new ListSampler(); + iterateList(listSampler); + return listSampler.sampleElement; + } + + @Override + public final JavaHeapObject[] getSampleKeyAndValue() { + class MapSampler implements MapIteratorCallback { + JavaHeapObject sampleKey, sampleValue; + private int counter; + + @Override + public boolean scanMapEntry(JavaHeapObject key, JavaHeapObject value) { + counter++; + if (counter == 1) { + sampleKey = key; + sampleValue = value; + } else { + if (key != null) { + if (sampleKey != null && key.getClazz() == sampleKey.getClazz()) { + sampleKey = key; + } else { + sampleKey = null; + } + } + if (value != null) { + if (sampleValue != null && value.getClazz() == sampleValue.getClazz()) { + sampleValue = value; + } else { + sampleValue = null; + } + } + } + return (counter < 3); + } + + @Override + public boolean scanImplementationObject(JavaHeapObject implObj) { + return true; + } + } + + MapSampler cb = new MapSampler(); + iterateMap(cb); + return new JavaHeapObject[] {cb.sampleKey, cb.sampleValue}; + } + + @Override + public final boolean hasExtraObjFields() { + return getFactory().knownAndPrimitiveFieldIndices != null; + } + + @Override + public final void filterExtraObjFields(JavaThing[] fields) { + int[] knownAndPrimitiveFieldIndices = getFactory().knownAndPrimitiveFieldIndices; + for (int idx : knownAndPrimitiveFieldIndices) { + fields[idx] = null; + } + } + + /** + * An instance of Factory is created for every known collection or standalone array type. Its + * role is to generate a new CollectionInstanceDescriptor, with all the internals initialized + * properly, for each JavaObject (that should be of this Factory's associated type) given to it. + *

+ * The abstract Factory class provides data and functionality common to all factories. Its + * concrete subclasses are associated with concrete subclasses of AbstractCollectionDescriptor. + */ + abstract static class Factory implements ImplInclusiveSizeCalculator { + + protected final CollectionClassDescriptor classDesc; + private String keyFieldName = "key"; + private String valueFieldName = "value"; + private int keyFieldIdx = -1, valueFieldIdx = -1; + private int modCountFieldIdx = -1; + private int entryNextFieldIdx = -1; + protected final int[] knownAndPrimitiveFieldIndices; + + /** + * Creates and returns an instance of CollectionInstanceDescriptor for a JavaObject that + * represents a collection or array of type associated with this Factory. + */ + abstract CollectionInstanceDescriptor get(JavaObject col); + + CollectionClassDescriptor getClassDescriptor() { + return classDesc; + } + + /** + * Creates a factory, that in turn would create an instance of CollectionInstanceDescriptor + * for any JavaObject that has the same class as clazz below. Most parameters are + * self-explanatory. The knownObjFieldNames parameter is needed to properly scan collections + * in breadth-first mode. In that mode we need to take special measures for JOverflow to + * properly follow references from collections to objects such as HashMap.KeySet etc., that + * we consider separate from "collection implementation" fields such as HashMap.table. For + * more information, see {@link CollectionInstanceDescriptor#hasExtraObjFields()}) and + * {@link #getKnownAndPrimitiveFieldIndices(JavaClass, String[])}. + */ + Factory(JavaClass clazz, boolean isMap, JavaClass[] implClasses, String[] parentColClassNames, + boolean hasOtherCollectionInImpl, String[] knownObjFieldNames) { + boolean canDetermineModCount = setModCountFieldIdx(clazz); + knownAndPrimitiveFieldIndices = knownObjFieldNames != null + ? getKnownAndPrimitiveFieldIndices(clazz, knownObjFieldNames) : null; + classDesc = new CollectionClassDescriptor(clazz, isMap, canDetermineModCount, implClasses, + parentColClassNames, hasOtherCollectionInImpl); + clazz.setImplInclusiveSizeCalculator(this); + } + + /** + * Creates a new Factory, where all properties are the same as in superClassFactory except + * for the class, which is supplied via clazz. It is intended to be used for generating + * factories of descriptors for subclasses of known collection classes, where the + * superclass' collection implementation is reused. Should not be called directly - only by + * implementations of cloneForSubclass(JavaClass) below. + */ + Factory(JavaClass clazz, Factory superclassFactory) { + classDesc = superclassFactory.classDesc.cloneForSubclass(clazz); + this.keyFieldName = superclassFactory.keyFieldName; + this.keyFieldIdx = superclassFactory.keyFieldIdx; + this.valueFieldIdx = superclassFactory.valueFieldIdx; + this.modCountFieldIdx = superclassFactory.modCountFieldIdx; + this.knownAndPrimitiveFieldIndices = superclassFactory.knownAndPrimitiveFieldIndices; + clazz.setImplInclusiveSizeCalculator(this); + } + + /** + * Calls the above clone constructor (that should be properly re-implemented in each + * subclass of Factory). + */ + abstract Factory cloneForSubclass(JavaClass clazz); + + @Override + public int calculateImplInclusiveSize(JavaObject javaObj) { + CollectionInstanceDescriptor colDesc = get(javaObj); + return colDesc.getImplSize(); + } + + protected int getEntryNextFieldIdx(JavaThing[] entries) { + if (entryNextFieldIdx == -1) { + for (JavaThing entry : entries) { + if (entry != null && entry instanceof JavaObject) { + entryNextFieldIdx = getEntryNextFieldIdx((JavaObject) entry); + break; + } + } + } + + if (entryNextFieldIdx == -1) { + // Couldn't find any proper elements in this entries array. + // Perhaps it's really empty (and size field is not null due to e.g. + // concurrent updates). Or perhaps the dump is corrupted a bit, and + // none of the elements is a resolved object. + return -1; + } + + return entryNextFieldIdx; + } + + protected int getEntryNextFieldIdx(JavaObject entry) { + if (entryNextFieldIdx == -1) { + JavaClass entryClazz = entry.getClazz(); + // We have to be very careful with the 'next' field, because in some classes, + // e.g. WeakHashMap$Entry, a superclass may have its own field with the same + // name. Here we rely on the fact that fields in getFields() are ordered from + // the superclass to subclass. + JavaField fieldDescs[] = entryClazz.getFieldsForInstance(); + for (int i = fieldDescs.length - 1; i >= 0; i--) { + if ("next".equals(fieldDescs[i].getName())) { + entryNextFieldIdx = i; + break; + } + } + + } + return entryNextFieldIdx; + } + + /** + * Sets the non-standard name for the field that in most maps is called "key" (in + * WhateverMap$Entry class) + */ + void setMapKeyFieldName(String keyFieldName) { + this.keyFieldName = keyFieldName; + } + + /** + * Sets the non-standard name for the field that in most maps is called "value" (in + * WhateverMap$Entry class) + */ + void setValueFieldName(String valueFieldName) { + this.valueFieldName = valueFieldName; + } + + protected int getKeyFieldIdx(JavaObject entry) { + if (keyFieldIdx == -1) { + keyFieldIdx = entry.getClazz().getInstanceFieldIndex(keyFieldName); + } + return keyFieldIdx; + } + + protected int getValueFieldIdx(JavaObject entry) { + if (valueFieldIdx == -1) { + valueFieldIdx = entry.getClazz().getInstanceFieldIndex(valueFieldName); + } + return valueFieldIdx; + } + + protected boolean setModCountFieldIdx(JavaClass clazz) { + modCountFieldIdx = clazz.getInstanceFieldIndexOrMinusOne("modCount"); + if (modCountFieldIdx != -1) { + JavaField modCountField = clazz.getFieldForInstance(modCountFieldIdx); + if (modCountField != null) { + char modCountFieldType = modCountField.getTypeId(); + if (modCountFieldType != 'J' && modCountFieldType != 'I') { + // modCount is neither long or int - probably something wrong + modCountFieldIdx = -1; + } + } else { + modCountFieldIdx = -1; // Paranoia + } + } + return modCountFieldIdx != -1; + } + } + + /** + * Returns a list of field indices for fields that are either: - "known" to us in advance in + * this collection class (that is, used for standard metrics calculation, e.g. HashMap.table) - + * primitive fields - "banned" fields, per JavaClass.getBannedFieldIndices() + *

+ * The common thing for all these fields is that when they are filtered out, what remains is + * "extra" object fields (see {@link CollectionInstanceDescriptor#hasExtraObjFields()}). + * However, if it turns out that all of this class's fields fall into one of the above + * categories, i.e. there will be no "extra" fields, null is returned. + */ + private static int[] getKnownAndPrimitiveFieldIndices(JavaClass clazz, String[] knownObjFieldNames) { + JavaField[] fields = clazz.getFieldsForInstance(); + int[] bannedFieldIndices = clazz.getBannedFieldIndices(); + // Set banned fields to null + if (bannedFieldIndices != null) { + for (int bannedFieldIdx : bannedFieldIndices) { + fields[bannedFieldIdx] = null; + } + } + // Set known fields to null + for (String knownFieldName : knownObjFieldNames) { + // Go from fields of this class to superclass fields, since any name in + // knownObjFieldNames is more likely to refer to a field defined in this + // class rather than in some superclass. This is especially important in + // a situation when both subclass and superclass have fields with the same + // name. + for (int i = fields.length - 1; i >= 0; i--) { + if (fields[i] != null && fields[i].getName().equals(knownFieldName)) { + fields[i] = null; + break; + } + } + } + // Set primitive fields to null + for (int i = 0; i < fields.length; i++) { + if (fields[i] != null && !fields[i].isReference()) { + fields[i] = null; + } + } + + IntArrayList ia = new IntArrayList(fields.length); + for (int i = 0; i < fields.length; i++) { + if (fields[i] == null) { + ia.add(i); + } + } + + if (ia.size() == fields.length) { + return null; + } + return ia.toArray(); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/AbstractLinkedCollectionDescriptor.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/AbstractLinkedCollectionDescriptor.java new file mode 100644 index 00000000..b3e05d69 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/AbstractLinkedCollectionDescriptor.java @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.descriptors; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaInt; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.support.Constants; +import org.openjdk.jmc.joverflow.util.ClassUtils; + +/** + * Base descriptor for collection classes that keep their elements in a linked structure, such as + * LinkedList, TreeMap, ConcurrentLinkedQueue etc. Such classes may or may not have the 'size' + * field, so this descriptor supports different methods of determining collection size. + */ +public abstract class AbstractLinkedCollectionDescriptor extends AbstractCollectionDescriptor implements Constants { + protected final Factory factory; + + protected AbstractLinkedCollectionDescriptor(JavaObject col, Factory factory) { + super(col); + this.factory = factory; + } + + @Override + public int getNumElements() { + if (factory.sizeFieldIdx != -1) { + return ((JavaInt) fields[factory.sizeFieldIdx]).getValue(); + } else { + return getSizeByCountingElements(); + } + } + + @Override + AbstractCollectionDescriptor.Factory getFactory() { + return factory; + } + + /** + * Returns the size of the described collection determined by counting its elements. The + * implementation may, of course, cache the value and return it on subsequent invocations. + */ + protected abstract int getSizeByCountingElements(); + + static abstract class Factory extends AbstractCollectionDescriptor.Factory { + protected final int sizeFieldIdx, rootFieldIdx; + private final String elementFieldName; + private int elementFieldIdx = -1; + + /** + * Note that sizeFieldName parameter may be null, which means that the described class does + * not have 'size' field. In that case, the descriptor subclass should provide the + * implementation of {@link AbstractLinkedCollectionDescriptor#getSizeByCountingElements()} + * method. + */ + Factory(JavaClass clazz, boolean isMap, String sizeFieldName, String rootFieldName, String elementFieldName, + JavaClass[] implClasses) { + super(clazz, isMap, implClasses, null, false, new String[] {rootFieldName}); + sizeFieldIdx = sizeFieldName != null ? clazz.getInstanceFieldIndex(sizeFieldName) : -1; + rootFieldIdx = clazz.getInstanceFieldIndex(rootFieldName); + this.elementFieldName = elementFieldName; + } + + protected Factory(JavaClass clazz, AbstractCollectionDescriptor.Factory superclassFactory) { + super(clazz, superclassFactory); + this.sizeFieldIdx = ((Factory) superclassFactory).sizeFieldIdx; + this.rootFieldIdx = ((Factory) superclassFactory).rootFieldIdx; + this.elementFieldName = ((Factory) superclassFactory).elementFieldName; + } + + protected int getElementFieldIdx(JavaObject entry) { + if (elementFieldIdx == -1) { + JavaClass entryClass = entry.getClazz(); + String elFieldName = ClassUtils.getExactFieldName(elementFieldName, entryClass); + elementFieldIdx = entry.getClazz().getInstanceFieldIndex(elFieldName); + } + return elementFieldIdx; + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/ArrayBasedCollectionDescriptor.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/ArrayBasedCollectionDescriptor.java new file mode 100644 index 00000000..3e8eef35 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/ArrayBasedCollectionDescriptor.java @@ -0,0 +1,380 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.descriptors; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaInt; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObjectArray; +import org.openjdk.jmc.joverflow.heap.model.JavaThing; + +/** + * Descriptor for a bunch of simple array-based Collection classes that are implemented in the same + * way: they have an int field for size and an object array field for elements. These elements + * contain either objects that are the actually collection's "payload" (for e.g. ArrayList and + * Vector), or intermediate $Entry objects (for e.g. HashMap and Hashtable). Note that this + * descriptor's code currently assumes that for lists the array always points directly to the + * payload, whereas for maps it always points to entries. Thus it does not support a map with + * payload contained in elements array: there is currently a separate IdentityHashMapDescriptor for + * it. ConcurrentHashMap, that has intermediate Segment objects, also uses a separate + * ConcurrentHashMapDescriptor. + */ +public class ArrayBasedCollectionDescriptor extends AbstractArrayBasedCollectionDescriptor { + + /** + * Tag whether a HashMap is the new version. Specifically, for JDK8, in which the HashMap + * implementation has changed. + */ + private final boolean jdk8HashMap; + + // Since this class sometimes needs to do different things with its + // superclasses, it needs a specific factory to hide its superclass. + final Factory factory; + + ArrayBasedCollectionDescriptor(JavaObject col, Factory factory) { + this(col, factory, false); + } + + /** + * This constructor can be used when it is explicitly told whether the implementation of HashMap + * is a new one or not. + * + * @param col + * @param factory + * @param jdk8HashMap + */ + ArrayBasedCollectionDescriptor(JavaObject col, Factory factory, boolean jdk8HashMap) { + super(col, factory); + this.factory = factory; + this.jdk8HashMap = jdk8HashMap; + } + + @Override + public int getNumElements() { + return ((JavaInt) fields[factory.sizeFieldIdx]).getValue(); + } + + @Override + public int doGetImplSize() { + int implSize = getDirectImplSize(); + if (!factory.getClassDescriptor().isMap()) { + return implSize; + } + + return implSize + getMapEntriesImplSize(); + } + + /** + * Overrides superclass' method. + */ + @Override + public int getMapEntriesImplSize() { + return jdk8HashMap ? getMapBinsImplSize() : super.getMapEntriesImplSize(); + } + + /** + * Override superclass. For JDK8, HashMap implementation has changed, specifically when entries' + * number increase, at some point, a bucket in a HashMap could transform into a Tree from linked + * list. In that case, the way to traverse the map should be changed. + */ + @Override + public void iterateMap(MapIteratorCallback cb) { + if (jdk8HashMap) { + iterateMapEntryOrTree(cb); + } else { + super.iterateMap(cb); + } + } + + /** + * This helper method traverses a tree as a bucket inside a HashMap and calculates the + * implementation size of the objects in the tree + */ + private int preOrderTraverseTree(JavaObject root) { + if (root == null) { + return 0; + } + + int size = root.getSize(); + + JavaThing leftField = root.getField(factory.leftFieldName); + if (!(leftField instanceof JavaObject)) { + // Probably corrupted heap + return 0; + } + size += preOrderTraverseTree((JavaObject) leftField); + + JavaThing rightField = root.getField(factory.rightFieldName); + if (!(rightField instanceof JavaObject)) { + // Probably corrupted heap + return 0; + } + size += preOrderTraverseTree((JavaObject) rightField); + + return size; + } + + private void preOrderTraverseTree(JavaObject root, MapIteratorCallback cb) { + if (root == null) { + return; + } + + JavaThing entryOrTreeNode = root.getField("entry"); + if (entryOrTreeNode == null || !(entryOrTreeNode instanceof JavaObject)) { + return; + } + + JavaThing keyThing = ((JavaObject) entryOrTreeNode) + .getField(factory.getKeyFieldIdx((JavaObject) entryOrTreeNode)); + JavaHeapObject key = (keyThing instanceof JavaHeapObject) ? (JavaHeapObject) keyThing : null; + + JavaThing valueThing = ((JavaObject) entryOrTreeNode) + .getField(factory.getValueFieldIdx((JavaObject) entryOrTreeNode)); + JavaHeapObject value = (valueThing instanceof JavaHeapObject) ? (JavaHeapObject) valueThing : null; + + if (!cb.scanMapEntry(key, value)) { + return; + } + + if (root.getField(factory.leftFieldName) instanceof JavaObject) { + preOrderTraverseTree((JavaObject) root.getField(factory.leftFieldName), cb); + } + + if (root.getField(factory.rightFieldName) instanceof JavaObject) { + preOrderTraverseTree((JavaObject) root.getField(factory.rightFieldName), cb); + } + } + + /** + * This helper method gets the implementation size of the elements inside a HashMap, of which + * each element should be either an Entry or a TreeBin + */ + private int getMapBinsImplSize() { + int numEls = getNumElements(); + if (numEls == 0) { + return 0; + } + + JavaObjectArray binsArray = getElementsArray(); + if (binsArray == null) { + // Unresolved + return 0; + } + + JavaHeapObject[] bins = binsArray.getElements(); + + // Reusable fields, to reduce GC pressure + JavaThing[] binsFields = null; + + int size = 0; + for (JavaHeapObject binThing : bins) { + if (binThing == null || !(binThing instanceof JavaObject)) { + continue; // Unresolved object + } + + JavaObject bin = (JavaObject) binThing; + // It turns out that sometimes HashMaps etc. may be corrupted due to + // concurrent updates by multiple threads. As a result, their entries form a cycle. + // Thus we check each entry here if it has already been visited. + + // TODO: if (bin.getClazz().getName().equals("java.lang.Object")) { + if (bin.getField("root") == null) { + int nextFieldIdx = factory.getEntryNextFieldIdx(bin); + if (nextFieldIdx == -1) { + // corrupted heap + return 0; + } + + while (bin != null && !bin.isVisitedAsCollectionImpl()) { + bin.setVisitedAsCollectionImpl(); + size += bin.getSize(); + + binsFields = bin.getFields(binsFields); + if (!(binsFields[nextFieldIdx] instanceof JavaObject)) { + // Unresolved object + break; + } + + JavaObject prevBin = bin; + bin = (JavaObject) binsFields[nextFieldIdx]; + if (bin == prevBin) { + break; + } + } + } else { + // The bin is of TreeBin + JavaThing treeRoot = bin.getField("root"); + if (treeRoot == null || !(treeRoot instanceof JavaObject)) { + // Unresolved object + continue; + } + // Traverse the tree to calculate size + size += preOrderTraverseTree((JavaObject) treeRoot); + } + } + + return size; + } + + /** + * This method is created for the new version of HashMap(s) (include and after JDK8), in which a + * bin of the HashMap backing array could be either a list of entry or a tree. This method won't + * be called if the heap dump is based on a JDK whose HashMap backing array is an array of + * Entry instead of an array of Object. + */ + private void iterateMapEntryOrTree(MapIteratorCallback cb) { + JavaObjectArray entriesArray = getElementsArray(); + if (entriesArray == null) { + return; // Can happen if entries array is unresolved + } + + if (!cb.scanImplementationObject(entriesArray)) { + return; + } + + int numElements = getNumElements(); + if (numElements == 0) { + return; + } + + JavaHeapObject[] entries = entriesArray.getElements(); + JavaThing[] entryFields = null; + + int keyFieldIdx = -1, valueFieldIdx = -1, nextFieldIdx = -1; + + outerLoop: for (JavaHeapObject entryThing : entries) { + if (entryThing == null || !(entryThing instanceof JavaObject)) { + continue; + } + + JavaObject entry = (JavaObject) entryThing; + + if (keyFieldIdx < 0) { + keyFieldIdx = factory.getKeyFieldIdx(entry); + valueFieldIdx = factory.getValueFieldIdx(entry); + nextFieldIdx = factory.getEntryNextFieldIdx(entry); + } + + if (entry.getField("root") == null) { + while (entry != null) { + if (!cb.scanImplementationObject(entry)) { + // We get this in BFS if entry already seen + break; + } + entryFields = entry.getFields(entryFields); + + JavaThing keyThing = entryFields[keyFieldIdx]; + JavaHeapObject key = (keyThing instanceof JavaHeapObject) ? (JavaHeapObject) keyThing : null; + + JavaThing valueThing = entryFields[valueFieldIdx]; + JavaHeapObject value = (valueThing instanceof JavaHeapObject) ? (JavaHeapObject) valueThing : null; + + if (!cb.scanMapEntry(key, value)) { + break outerLoop; + } + + if (!(entryFields[nextFieldIdx] instanceof JavaObject)) { + break; // Unresolved object + } + JavaObject prevEntry = entry; + entry = (JavaObject) entryFields[nextFieldIdx]; + if (entry == prevEntry) { + break; + } + } + } else { + if (!cb.scanImplementationObject(entry)) { + break; + } + JavaThing root = entry.getField("root"); + if (!(root instanceof JavaObject)) { + continue; + } + // Traverse the tree to scan + preOrderTraverseTree((JavaObject) root, cb); + } + } + } + + static class Factory extends AbstractArrayBasedCollectionDescriptor.Factory { + + private final int sizeFieldIdx; + + /** + * This flag is used for letting the factory know whether the map it describes has a new + * implementation (see details in JDK8) or not. + */ + private final boolean jdk8HashMap; + + /** + * Name for the "left" child and "right" child when a bin inside a HashMap is a tree. In + * each TreeNode, there is field name like those. + */ + private final String leftFieldName; + private final String rightFieldName; + + Factory(JavaClass clazz, boolean isMap, String sizeFieldName, String elsArrayFieldName, int initialCapacity, + JavaClass[] implClasses, String[] parentColClassNames) { + this(clazz, isMap, sizeFieldName, elsArrayFieldName, initialCapacity, implClasses, parentColClassNames, + false); + } + + Factory(JavaClass clazz, boolean isMap, String sizeFieldName, String elsArrayFieldName, int initialCapacity, + JavaClass[] implClasses, String[] parentColClassNames, boolean jdk8HashMap) { + super(clazz, isMap, elsArrayFieldName, initialCapacity, implClasses, parentColClassNames); + sizeFieldIdx = clazz.getInstanceFieldIndex(sizeFieldName); + this.jdk8HashMap = jdk8HashMap; + leftFieldName = jdk8HashMap ? "left" : null; + rightFieldName = jdk8HashMap ? "right" : null; + } + + Factory(JavaClass clazz, AbstractArrayBasedCollectionDescriptor.Factory superclassFactory) { + super(clazz, superclassFactory); + this.sizeFieldIdx = ((Factory) superclassFactory).sizeFieldIdx; + this.jdk8HashMap = ((Factory) superclassFactory).jdk8HashMap; + this.leftFieldName = ((Factory) superclassFactory).leftFieldName; + this.rightFieldName = ((Factory) superclassFactory).rightFieldName; + } + + @Override + AbstractCollectionDescriptor.Factory cloneForSubclass(JavaClass clazz) { + return new Factory(clazz, this); + } + + @Override + ArrayBasedCollectionDescriptor get(JavaObject col) { + return new ArrayBasedCollectionDescriptor(col, this, jdk8HashMap); + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/ArrayDequeDescriptor.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/ArrayDequeDescriptor.java new file mode 100644 index 00000000..ac4deaff --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/ArrayDequeDescriptor.java @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.descriptors; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaInt; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObjectArray; + +/** + * Descriptor for java.util.ArrayDeque. + */ +public class ArrayDequeDescriptor extends AbstractArrayBasedCollectionDescriptor { + private int numElements = -1; // Cached number of elements + + ArrayDequeDescriptor(JavaObject col, Factory factory) { + super(col, factory); + } + + @Override + public int getNumElements() { + if (numElements != -1) { + return numElements; + } + JavaObjectArray elsArray = getElementsArray(); + if (elsArray == null) { + // Unresolved in corrupted heap dump + numElements = 0; + return numElements; + } + + Factory f = (Factory) factory; + int tail = ((JavaInt) fields[f.tailFieldIdx]).getValue(); + int head = ((JavaInt) fields[f.headFieldIdx]).getValue(); + numElements = (tail - head) & (elsArray.getLength() - 1); + return numElements; + } + + @Override + public int doGetImplSize() { + return getDirectImplSize(); + } + + static class Factory extends AbstractArrayBasedCollectionDescriptor.Factory { + + private final int headFieldIdx, tailFieldIdx; + + Factory(JavaClass clazz) { + super(clazz, false, "elements", 16, new JavaClass[] {}, null); + headFieldIdx = clazz.getInstanceFieldIndex("head"); + tailFieldIdx = clazz.getInstanceFieldIndex("tail"); + } + + private Factory(JavaClass clazz, AbstractArrayBasedCollectionDescriptor.Factory superclassFactory) { + super(clazz, superclassFactory); + this.headFieldIdx = ((Factory) superclassFactory).headFieldIdx; + this.tailFieldIdx = ((Factory) superclassFactory).tailFieldIdx; + } + + @Override + AbstractCollectionDescriptor.Factory cloneForSubclass(JavaClass clazz) { + return new Factory(clazz, this); + } + + @Override + CollectionInstanceDescriptor get(JavaObject col) { + return new ArrayDequeDescriptor(col, this); + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/CollectionClassDescriptor.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/CollectionClassDescriptor.java new file mode 100644 index 00000000..bf4f2e7b --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/CollectionClassDescriptor.java @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.descriptors; + +import org.openjdk.jmc.joverflow.heap.model.CollectionClassProperties; +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.support.Constants; +import org.openjdk.jmc.joverflow.support.Constants.ProblemKind; + +/** + * Provides information associated with a single Collection class. + */ +public class CollectionClassDescriptor implements CollectionClassProperties { + private final JavaClass clazz; + private final boolean isMap; + private final boolean canDetermineModCount; + private final boolean hasOtherCollectionInImpl; + private final String[] implClassNames; + private final String[] parentColClassNames; + + private static final int NUM_ANTIPATTERNS = ProblemKind.values().length; + + private int nProblematicCols[] = new int[NUM_ANTIPATTERNS]; + private int problematicColsOverhead[] = new int[NUM_ANTIPATTERNS]; + + CollectionClassDescriptor(JavaClass clazz, boolean isMap, boolean canDetermineModCount, JavaClass[] implClasses, + String[] parentColClassNames, boolean hasOtherCollectionInImpl) { + this(clazz, isMap, canDetermineModCount, getImplClassNames(implClasses), parentColClassNames, + hasOtherCollectionInImpl); + } + + /** + * Returns a new CollectionClassDescriptor, where all properties are the same as in this + * descriptor, except for the class name. It is intended to be used for generating class + * descriptors for subclasses of known collection classes, where the superclass' collection + * implementation is reused. + */ + CollectionClassDescriptor cloneForSubclass(JavaClass subClazz) { + return new CollectionClassDescriptor(subClazz, isMap, canDetermineModCount, implClassNames, parentColClassNames, + hasOtherCollectionInImpl); + } + + private CollectionClassDescriptor(JavaClass clazz, boolean isMap, boolean canDetermineModCount, + String[] implClassNames, String[] parentColClassNames, boolean hasOtherCollectionInImpl) { + this.clazz = clazz; + this.isMap = isMap; + this.canDetermineModCount = canDetermineModCount; + this.implClassNames = implClassNames; + this.parentColClassNames = parentColClassNames; + this.hasOtherCollectionInImpl = hasOtherCollectionInImpl; + if (!clazz.isArray()) { // Not a pseudo-collection descriptor for array type + clazz.setCollectionClassProperties(this); + } + } + + /** Returns JavaClass for this descriptor */ + public JavaClass getClazz() { + return clazz; + } + + /** Returns class name for this descriptor */ + public String getClassName() { + return clazz.getName(); + } + + @Override + public boolean isMap() { + return isMap; + } + + @Override + public boolean hasOtherCollectionInImpl() { + return hasOtherCollectionInImpl; + } + + /** + * Returns true if for this collection class the information on whether it has been used + * (modified) is available (typically through the 'modCount' data field). + */ + public boolean canDetermineModCount() { + return canDetermineModCount; + } + + /** + * Returns true if the given class belongs to the implementation of this collection. For + * example, if this instance describes java.util.HashMap, returns true for HashMap$Entry. + */ + public boolean isImplClassName(String className) { + if (implClassNames == null) { + return false; + } + + for (String implClassName : implClassNames) { + if (className.equals(implClassName)) { + return true; + } + } + + return false; + } + + /** + * Returns true if this collection's class is in implementation of the given class. For example, + * should return true if this instance describes java.util.HashMap and className is + * java.util.HashSet. + */ + public boolean isInImplementationOf(String className) { + if (parentColClassNames == null) { + return false; + } + + for (String parentColClassName : parentColClassNames) { + if (className.equals(parentColClassName)) { + return true; + } + } + + return false; + } + + /** + * Returns true if this descriptor represents a primitive array type. + */ + public boolean isPrimitiveArray() { + return clazz.isAnyDimPrimitiveArray(); + } + + // Functionality for keeping track of overhead stats of all instances of the + // associated collection class. + + public void addProblematicCollection(ProblemKind kind, int ovhd) { + int kindIdx = kind.ordinal(); + nProblematicCols[kindIdx]++; + problematicColsOverhead[kindIdx] += ovhd; + } + + public int getNumProblematicCollections(ProblemKind kind) { + return nProblematicCols[kind.ordinal()]; + } + + public int getProblematicCollectionsOverhead(Constants.ProblemKind kind) { + return problematicColsOverhead[kind.ordinal()]; + } + + public long getTotalOverhead() { + long totalOvhd = 0; + for (Constants.ProblemKind kind : ProblemKind.values()) { + totalOvhd += getProblematicCollectionsOverhead(kind); + } + return totalOvhd; + } + + @Override + public String toString() { + return clazz.getName(); + } + + private static String[] getImplClassNames(JavaClass[] implClasses) { + String[] implClassNames = new String[implClasses.length]; + for (int i = 0; i < implClasses.length; i++) { + implClassNames[i] = implClasses[i].getName(); + } + return implClassNames; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/CollectionDescriptors.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/CollectionDescriptors.java new file mode 100644 index 00000000..9f5170d1 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/CollectionDescriptors.java @@ -0,0 +1,453 @@ +/* + * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.descriptors; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.logging.Logger; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaLazyReadObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObjectArray; +import org.openjdk.jmc.joverflow.heap.model.JavaValueArray; +import org.openjdk.jmc.joverflow.heap.model.Snapshot; +import org.openjdk.jmc.joverflow.support.Constants; +import org.openjdk.jmc.joverflow.util.ClassUtils; + +/** + */ +public class CollectionDescriptors implements Constants { + private final static Logger LOGGER = Logger.getLogger("org.openjdk.jmc.joverflow.descriptors"); //$NON-NLS-1$ + + private static final String[] EMPTY_STRS = new String[] {}; + private static final JavaClass[] EMPTY_CLZ = new JavaClass[] {}; + + private final HashMap colDescs; + private final Snapshot snapshot; + + // Just for a reference, below is a (likely incomplete) list of collection + // classes or collection implementation classes that reference an array + // of collection elements (or intermediate Entry objects) directly: + // HASH_MAP, LINKED_HASH_MAP, HASHTABLE, ARRAY_LIST, VECTOR, + // CONCURRENT_HASH_MAP_SEGMENT, WEAK_HASH_MAP, STACK, ARRAY_BLOCKING_QUEUE, + // ARRAY_DEQUE, ATTRIBUTE_LIST, IDENTITY_HASH_MAP, PROPERTIES + + /** + * This field is to record whether HashMap(s) on the given heap dump is new or not. + * Specifically, new HashMap has backing array of type Object[] instead of HashMap$Entry[] as + * old version (JDK7 and some early JDK8 internal builds). Also there are some changes on + * HashMap fields. Please see implementation of HashMap in JDK8 for details. + *

+ * This change also brings up changes in LinkedHashMap, HashSet, LinkedHashSet, + * ConcurrentHashMap, etc. + */ + private final boolean JDK8_HASHMAP; + + private boolean isJdk8HashMap() { + JavaClass clazz = snapshot.getClassForName(HASH_MAP); + return (clazz.getStaticField("ALTERNATIVE_HASHING_THRESHOLD_DEFAULT") == null); + } + + /** + * Creates a CollectionDescriptor factory for each known collection class that's present in the + * given snapshot. Then walks all the classes in the snapshot, and creates a factory for + * subclasses of the above known collections. + */ + public CollectionDescriptors(Snapshot snapshot) { + this.snapshot = snapshot; + setBannedFields(snapshot); + colDescs = new HashMap<>(); + JDK8_HASHMAP = isJdk8HashMap(); + initDescFactories(); + createDescFactoriesForSubclasses(); + } + + public CollectionInstanceDescriptor getDescriptor(JavaObject col) { + AbstractCollectionDescriptor.Factory factory = colDescs.get(col.getClazz().getName()); + if (factory == null) { + return null; + } + return factory.get(col); + } + + public CollectionClassDescriptor getClassDescriptor(JavaObject col) { + return getClassDescriptor(col.getClazz().getName()); + } + + public CollectionClassDescriptor getClassDescriptor(String clazzName) { + AbstractCollectionDescriptor.Factory factory = colDescs.get(clazzName); + if (factory == null) { + return null; + } + return factory.getClassDescriptor(); + } + + public CollectionClassDescriptor getStandaloneArrayDescriptor(JavaLazyReadObject ar) { + if (!(ar instanceof JavaObjectArray || ar instanceof JavaValueArray)) { + throw new IllegalArgumentException("Argument of non-array type " + ar.getClass()); + } + + String clazzName = ar.getClazz().getName(); + StandaloneArrayDescFactory factory = (StandaloneArrayDescFactory) colDescs.get(clazzName); + if (factory == null) { + factory = new StandaloneArrayDescFactory(ar.getClazz()); + colDescs.put(clazzName, factory); + } + return factory.getClassDescriptor(); + } + + public ArrayList getOverheadsByClass() { + ArrayList result = new ArrayList<>(); + for (AbstractCollectionDescriptor.Factory factory : colDescs.values()) { + result.add(factory.getClassDescriptor()); + } + + result.sort(new Comparator() { + @Override + public int compare(CollectionClassDescriptor d1, CollectionClassDescriptor d2) { + long o1 = d1.getTotalOverhead(); + long o2 = d2.getTotalOverhead(); + if (o1 > o2) { + return -1; + } else if (o1 < o2) { + return 1; + } else { + return 0; + } + } + }); + + while (true) { + long lastElOvhd = result.get(result.size() - 1).getTotalOverhead(); + if (lastElOvhd == 0) { + result.remove(result.size() - 1); + } else { + break; + } + } + return result; + } + + public Snapshot getSnapshot() { + return snapshot; + } + + private void initDescFactories() { + JavaClass clazz; + + colDescs.put(HASH_MAP, newHashMapDescFactory()); + colDescs.put(LINKED_HASH_MAP, newLinkedHashMapDescFactory()); + clazz = snapshot.getClassForName(HASH_SET); + if (clazz != null) { + colDescs.put(HASH_SET, hashSetDescriptorFactory(clazz)); + } + clazz = snapshot.getClassForName(LINKED_HASH_SET); + if (clazz != null) { + colDescs.put(LINKED_HASH_SET, linkedHashSetDescriptorFactory(clazz)); + } + + addArrayBasedDescFactory(ARRAY_LIST, false, "size", "elementData|array", 10, EMPTY_STRS); + addArrayBasedDescFactory(VECTOR, false, "elementCount", "elementData", 10, EMPTY_STRS); + addArrayBasedDescFactory(STACK, false, "elementCount", "elementData", 10, EMPTY_STRS); + + addArrayBasedDescFactory(HASHTABLE, true, "count|size", "table", 11, + new String[] {HASHTABLE_ENTRY, ClassUtils.arrayOf(HASHTABLE_ENTRY)}); + + // java.util.Properties just extends java.util.Hashtable + addArrayBasedDescFactory(PROPERTIES, true, "count|size", "table", 11, + new String[] {HASHTABLE_ENTRY, ClassUtils.arrayOf(HASHTABLE_ENTRY)}); + + clazz = snapshot.getClassForName(CONCURRENT_HASH_MAP); + if (clazz != null) { + putConcurrentHashMap(clazz); + } + + clazz = snapshot.getClassForName(WEAK_HASH_MAP); + if (clazz != null) { + colDescs.put(WEAK_HASH_MAP, new WeakHashMapDescriptor.Factory(clazz, + getClassesForNames(new String[] {WEAK_HASH_MAP_ENTRY, ClassUtils.arrayOf(WEAK_HASH_MAP_ENTRY)}))); + } + + String[] entryClass; + clazz = snapshot.getClassForName(TREE_MAP); + if (clazz != null) { + // In JRockit jrockit_160_22_D1.1.1-3 there is for some reason TreeMap$Node instead + // of TreeMap$Entry. Maybe it's too old? + if (snapshot.getClassForName(TREE_MAP_ENTRY) != null) { + entryClass = new String[] {TREE_MAP_ENTRY}; + } else if (snapshot.getClassForName(TREE_MAP_NODE) != null) { + entryClass = new String[] {TREE_MAP_NODE}; + } else { + entryClass = new String[0]; + } + colDescs.put(TREE_MAP, new TreeMapDescriptor.Factory(clazz, getClassesForNames(entryClass))); + } + + // In JDK7-u2(?), they renamed LinkedList$Entry to $Node and made other changes. + // Furthermore, in Android, 'header' is called 'voidLink', and 'element' is 'data' + boolean isNewLinkedList = snapshot.getClassForName(LINKED_LIST_NODE) != null; + addLinkedListDescFactory(LINKED_LIST, "size", "header|first|voidLink", "element|item|data", + new String[] {isNewLinkedList ? LINKED_LIST_NODE : LINKED_LIST_ENTRY}); + + clazz = snapshot.getClassForName(IDENTITY_HASH_MAP); + if (clazz != null) { + colDescs.put(IDENTITY_HASH_MAP, new IdentityHashMapDescriptor.Factory(clazz)); + } + + addArrayBasedDescFactory(ARRAY_BLOCKING_QUEUE, false, "count", "items", 0, EMPTY_STRS); + + clazz = snapshot.getClassForName(ARRAY_DEQUE); + if (clazz != null) { + colDescs.put(ARRAY_DEQUE, new ArrayDequeDescriptor.Factory(clazz)); + } + + addArrayBasedDescFactory(ATTRIBUTE_LIST, false, "size", "elementData", 10, EMPTY_STRS); + + addLinkedListDescFactory(CONCURRENT_LINKED_QUEUE, null, "head", "item", + new String[] {CONCURRENT_LINKED_QUEUE_NODE}); + + if (snapshot.getClassForName(COPY_ON_WRITE_ARRAY_LIST) != null) { + colDescs.put(COPY_ON_WRITE_ARRAY_LIST, newCopyOnWriteArrayListFactory()); + } + + clazz = snapshot.getClassForName(COPY_ON_WRITE_ARRAY_SET); + if (clazz != null) { + colDescs.put(COPY_ON_WRITE_ARRAY_SET, + new CopyOnWriteArraySetDescriptor.Factory(clazz, newCopyOnWriteArrayListFactory())); + } + + addArrayBasedDescFactory(PRIORITY_QUEUE, false, "size", "queue", 11, EMPTY_STRS); + + // TODO: TreeSet + // TODO: LinkedBlockingQueue, LinkedBlockingDequeue + // TODO: PriorityBlockingQueue, DelayQueue, SynchronousQueue + // TODO: BeanContextServicesSupport, BeanContextSupport? + // TODO: ConcurrentSkipListMap, ConcurrentSkipListSet + } + + private ArrayBasedCollectionDescriptor.Factory addArrayBasedDescFactory( + String className, boolean isMap, String sizeFieldName, String elsArrayFieldName, int defaultInitialCapacity, + String[] implClassNames) { + JavaClass clazz = snapshot.getClassForName(className); + if (clazz == null) { + return null; + } + + sizeFieldName = ClassUtils.getExactFieldName(sizeFieldName, clazz); + elsArrayFieldName = ClassUtils.getExactFieldName(elsArrayFieldName, clazz); + + ArrayBasedCollectionDescriptor.Factory factory = new ArrayBasedCollectionDescriptor.Factory(clazz, isMap, + sizeFieldName, elsArrayFieldName, defaultInitialCapacity, getClassesForNames(implClassNames), null); + colDescs.put(className, factory); + return factory; + } + + private void addLinkedListDescFactory( + String className, String sizeFieldName, String rootFieldName, String elementFieldName, + String[] implClassNames) { + JavaClass clazz = snapshot.getClassForName(className); + if (clazz == null) { + return; + } + + rootFieldName = ClassUtils.getExactFieldName(rootFieldName, clazz); + + colDescs.put(className, new LinkedCollectionDescriptor.Factory(clazz, sizeFieldName, rootFieldName, + elementFieldName, getClassesForNames(implClassNames))); + } + + private ArrayBasedCollectionDescriptor.Factory newHashMapDescFactory() { + JavaClass clazz = snapshot.getClassForName(HASH_MAP); + if (clazz == null) { + return null; + } + String implClassName = JDK8_HASHMAP ? JAVA_LANG_OBJECT : HASH_MAP_ENTRY; + return new ArrayBasedCollectionDescriptor.Factory(clazz, true, "size", "table", 16, + getClassesForNames(new String[] {implClassName, ClassUtils.arrayOf(implClassName)}), + new String[] {HASH_SET}, JDK8_HASHMAP); + } + + private ArrayBasedCollectionDescriptor.Factory newLinkedHashMapDescFactory() { + JavaClass clazz = snapshot.getClassForName(LINKED_HASH_MAP); + if (clazz == null) { + return null; + } + String implClassName = JDK8_HASHMAP ? JAVA_LANG_OBJECT : HASH_MAP_ENTRY; + return new LinkedHashMapDescriptor.Factory(clazz, + getClassesForNames(new String[] {LINKED_HASH_MAP_ENTRY, ClassUtils.arrayOf(implClassName)}), + JDK8_HASHMAP); + } + + private HashSetDescriptor.Factory hashSetDescriptorFactory(JavaClass clazz) { + String implClassName = JDK8_HASHMAP ? JAVA_LANG_OBJECT : HASH_MAP_ENTRY; + return new HashSetDescriptor.Factory(clazz, + getClassesForNames(new String[] {HASH_MAP, implClassName, ClassUtils.arrayOf(implClassName)}), + newHashMapDescFactory()); + } + + private HashSetDescriptor.Factory linkedHashSetDescriptorFactory(JavaClass clazz) { + String implClassName = JDK8_HASHMAP ? JAVA_LANG_OBJECT : HASH_MAP_ENTRY; + return new HashSetDescriptor.Factory(clazz, + getClassesForNames( + new String[] {LINKED_HASH_MAP, LINKED_HASH_MAP_ENTRY, ClassUtils.arrayOf(implClassName)}), + newLinkedHashMapDescFactory()); + } + + private void putConcurrentHashMap(JavaClass clazz) { + // IMPORTANT: in some internal early builds of JDK8, ConcurrentHashMap still + // not changed while HashMap changed. In those cases, JOverflow thinks it is a new + // version, but ConcurrentHashMap descriptor won't work. The following code has fixed + // this problem temporarily, but it will probably need to be fixed in a better way + // in the future. Also it may need to be revised corresponding to the updates of + // Java in the future. + JavaClass chmNodeClazz = snapshot.getClassForName(CONCURRENT_HASH_MAP_NODE); + if (!JDK8_HASHMAP || chmNodeClazz == null) { + // not a new version + JavaClass chmSegmentClazz = snapshot.getClassForName(CONCURRENT_HASH_MAP_SEGMENT); + colDescs.put(CONCURRENT_HASH_MAP, + new ConcurrentHashMapDescriptor.Factory(clazz, chmSegmentClazz, + getClassesForNames(new String[] {CONCURRENT_HASH_MAP_SEGMENT, CONCURRENT_HASH_MAP_ENTRY, + ClassUtils.arrayOf(CONCURRENT_HASH_MAP_SEGMENT), + ClassUtils.arrayOf(CONCURRENT_HASH_MAP_ENTRY)}))); + } else { + colDescs.put(CONCURRENT_HASH_MAP, + new ConcurrentHashMapDescriptorForJdk8.Factory(clazz, chmNodeClazz, getClassesForNames( + new String[] {CONCURRENT_HASH_MAP_NODE, ClassUtils.arrayOf(CONCURRENT_HASH_MAP_NODE)}))); + } + } + + private FullyUtilizedArrayListDescriptor.Factory newCopyOnWriteArrayListFactory() { + JavaClass clazz = snapshot.getClassForName(COPY_ON_WRITE_ARRAY_LIST); + return new FullyUtilizedArrayListDescriptor.Factory(clazz, "array", EMPTY_CLZ, + new String[] {COPY_ON_WRITE_ARRAY_SET}); + } + + private void createDescFactoriesForSubclasses() { + JavaClass[] classes = snapshot.getClasses(); + for (JavaClass clazz : classes) { + String className = clazz.getName(); + if (colDescs.containsKey(className)) { + continue; + } + + JavaClass superclazz = clazz.getSuperclass(); + while (superclazz != null) { + String superName = superclazz.getName(); + AbstractCollectionDescriptor.Factory superclassFactory = colDescs.get(superName); + if (superclassFactory != null) { + colDescs.put(className, superclassFactory.cloneForSubclass(clazz)); + break; + } + superclazz = superclazz.getSuperclass(); + } + } + } + + /** + * Marks indices of fields in collections or their implementation detail classes, that should + * not be scanned in a normal fashion during detailed heap scanning, because data reachable from + * them is also reachable in a more appropriate (shorter and cleaner) way. For example, for + * LinkedHashMap we shouldn't scan its linked list (which starts at LinkedHashMap.header field), + * since all the elements in it are available from the 'table' array. + */ + private static void setBannedFields(Snapshot snapshot) { + JavaClass.setFieldBanned(snapshot.getClassForName(LINKED_HASH_MAP), "header"); + JavaClass.setFieldBanned(snapshot.getClassForName(LINKED_HASH_MAP_ENTRY), "before"); + JavaClass.setFieldBanned(snapshot.getClassForName(LINKED_HASH_MAP_ENTRY), "after"); + JavaClass.setFieldBanned(snapshot.getClassForName(LINKED_LIST), "last"); + JavaClass.setFieldBanned(snapshot.getClassForName(CONCURRENT_LINKED_QUEUE), "tail"); + // Important: this should find and ban the 'next' field that belongs to the + // java.lang.ref.Reference superclass of WeakHashMap$Entry - NOT the 'next' + // field defined in WeakHashMap$Entry itself. + JavaClass.setFieldBanned(snapshot.getClassForName(WEAK_HASH_MAP_ENTRY), "next"); + } + + private JavaClass[] getClassesForNames(String[] classNames) { + ArrayList clazzes = new ArrayList<>(classNames.length); + for (int i = 0; i < classNames.length; i++) { + JavaClass clazz = snapshot.getClassForName(classNames[i]); + if (clazz == null) { + /* + * TODO: this works at the moment, but doesn't look like the most elegant way to + * address the problem with possibly different implementation class names in + * different JDK versions/implementations (the alternative names below are for + * Android). The most radical solution would be to just declare all e.g. HashMap$* + * classes as implementation of HashMap, but unfortunately something like + * HashMap$Values does not seem to be "implementation details" in exactly the same + * sense as HashMap$Entry. + */ + if (classNames[i].equals("java.util.HashMap$Entry")) { + clazz = snapshot.getClassForName("java.util.HashMap$HashMapEntry"); + } else if (classNames[i].equals("[Ljava.util.HashMap$Entry;")) { + clazz = snapshot.getClassForName("java.util.HashMap$Entry[]"); + if (clazz == null) { + clazz = snapshot.getClassForName("java.util.HashMap$HashMapEntry[]"); + } + } else if (classNames[i].equals("java.util.Hashtable$Entry")) { + clazz = snapshot.getClassForName("java.util.Hashtable$HashtableEntry"); + } else if (classNames[i].equals("[Ljava.util.Hashtable$Entry;")) { + clazz = snapshot.getClassForName("java.util.Hashtable$Entry[]"); + if (clazz == null) { + clazz = snapshot.getClassForName("java.util.Hashtable$HashtableEntry[]"); + } + } + if (classNames[i].equals("java.util.LinkedHashMap$Entry")) { + clazz = snapshot.getClassForName("java.util.LinkedHashMap$LinkedEntry"); + } else if (classNames[i].equals("[Ljava.util.LinkedHashMap$Entry;")) { + clazz = snapshot.getClassForName("java.util.LinkedHashMap$LinkedEntry[]"); + } else if (classNames[i].equals("[Ljava.util.concurrent.ConcurrentHashMap$HashEntry;")) { + clazz = snapshot.getClassForName("java.util.concurrent.ConcurrentHashMap$HashEntry[]"); + } else if (classNames[i].equals("[Ljava.util.concurrent.ConcurrentHashMap$Segment;")) { + clazz = snapshot.getClassForName("java.util.concurrent.ConcurrentHashMap$Segment[]"); + } else if (classNames[i].equals("[Ljava.util.WeakHashMap$Entry;")) { + clazz = snapshot.getClassForName("java.util.WeakHashMap$Entry[]"); + } else if (classNames[i].equals("java.util.LinkedList$Entry")) { + clazz = snapshot.getClassForName("java.util.LinkedList$Link"); + if (clazz == null) { + clazz = snapshot.getClassForName("java.util.LinkedList$ListItr"); + } + } + if (clazz == null) { + LOGGER.severe("CollectionDescriptors: Class " + classNames[i] + " not found"); + } + } + if (clazz != null) { + clazzes.add(clazz); + } + } + + return clazzes.toArray(new JavaClass[clazzes.size()]); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/CollectionInstanceDescriptor.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/CollectionInstanceDescriptor.java new file mode 100644 index 00000000..d2bf5e0b --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/CollectionInstanceDescriptor.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.descriptors; + +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaThing; + +/** + * Provides information associated with an individual instance of some Collection class. + */ +public interface CollectionInstanceDescriptor { + /** Returns the descriptor for this collection's class */ + CollectionClassDescriptor getClassDescriptor(); + + /** Returns the number of elements in the associated collection */ + int getNumElements(); + + /** + * Returns the size, in bytes, of this collection instance's implementation. That's the size of + * the collection's main object plus all of the objects implementing this collection that the + * main one points to, such as HashMap$Entry, etc. Obviously the size of objects contained by + * this collection is not factored in by this method. + *

+ * A very important side-effect of this method is that it marks all of the collection + * implementation objects, such as HashMap$Entry instances, with + * JavaHeapObject.setVisitedAsCollectionImpl(). + *

+ * Since this method can be called repeatedly on the same descriptor instance, it should either + * be very fast or cache its result upon the first invocation. + */ + int getImplSize(); + + /** + * If the underlying collection object is a list, iterates all its elements, calling + * cb.scanListElement() for each element, and cb.scanImplementationObject() for each internal + * implementation object, such as ArrayList.elementData[] array. + */ + void iterateList(ListIteratorCallback cb); + + /** + * If the underlying collection object is a map, iterates all its elements, calling + * cb.scanMapEntry() for each key-value pair, and cb.scanImplementationObject() for each + * internal implementation object, such as HashMap.entries[] array and HashMap$Entry objects. + */ + void iterateMap(MapIteratorCallback cb); + + /** + * If this descriptor represents a list, returns one of its elements. The goal of this method is + * to return the most likely type for the list's elements. + *

+ * To obtain the result, several elements are taken randomly, and their types are compared. If + * all types are the same, one of the elements is returned. Otherwise, i.e. if types are not the + * same or the list is empty, null is returned. + */ + JavaHeapObject getSampleElement(); + + /** + * If this descriptor represents a map, returns one of its key-value pairs. The goal of this + * method is to return the most likely types for the keys and values in this map. + *

+ * To obtain the result, several map entries are taken randomly, and their types are compared. + * If all types within keys or values are the same, one of the entries is returned. Otherwise, + * i.e. if types are not the same or the map is empty, null is returned for key, value or both. + */ + JavaHeapObject[] getSampleKeyAndValue(); + + /** + * If CollectionClassDescriptor.canDetermineModCount() returns true, this method returns the + * number of times this collection has been modified (usually stored in the 'modCount' field of + * the collection class). + */ + long getModCount(); + + /** + * Returns true if this collection's class has "extra" object fields - that is, fields not + * directly responsible for its implementation and not otherwise handled when processing + * collections of this class. Such fields can be well-known, e.g. WeakHashMap.queue, which has + * always been there, but is not used to calculate impl-inclusive size or other collection + * metrics. Or they can be something we generally don't know about in advance, such as various + * optimizations that can be introduced in existing collection classes. An example is + * HashMap.frontCache, that may or may not be there depending on which version of HashMap is + * used. The information on extra fields is needed to properly handle them during breadth-first + * scan, which by default handles only known parts of collection classes and ignores other + * fields. + */ + boolean hasExtraObjFields(); + + /** + * Given an array of this collection object's fields that has been already initialized by + * JavaObject.getFields(), nulls out all of the elements in it except those corresponding to + * "extra" object fields, as explained in {@link #hasExtraObjFields()}. + */ + void filterExtraObjFields(JavaThing[] fields); + + /** + * Provides additional methods for instances of Collection classes that are array-based + * internally, and where capacity can exceed the number of elements in the collection (size). + */ + public interface CapacityDifferentFromSize { + + /** + * If this collection instance is sparse, returns the overhead, in bytes, due to sparseness. + * Otherwise, returns -1. + *

+ * A simple array-based collection is typically considered sparse if the number of elements + * in it is less than half its full capacity; the overhead would be the total size of the + * unused pointers in the array. + */ + int getSparsenessOverhead(int ptrSize); + + /** + * Returns the default capacity for instances of this Collection class. For example, for + * HashMap it's 16. + */ + int getDefaultCapacity(); + + /** + * Returns the current capacity of this instance. It is greater or equal to the number of + * elements in the colleciton. + */ + int getCapacity(); + } + + /** + * Used for iterating elements of the collection that is a list, see + * {@link CollectionInstanceDescriptor#iterateList(ListIteratorCallback)}. + */ + public interface ListIteratorCallback { + + /** + * Called from inside iterateList() for each collection workload element. If the + * implementation returns true, it means "continue", false tells iterateList() that + * iterating should stop. + */ + boolean scanListElement(JavaHeapObject element); + + /** + * Called from inside iterateList() for each collection implementation object, e.g. + * ArrayList.elements array. If the implementation of this callback returns true, it means + * "continue". False signals iterateMap() that this object has already been seen, and thus + * iteration should either stop completely or skip some objects to ensure that this object + * is not visited again, i.e. endless looping is avoided. + */ + boolean scanImplementationObject(JavaHeapObject implObj); + } + + /** + * Used for iterating elements of the collection that is a map, see + * {@link CollectionInstanceDescriptor#iterateMap(MapIteratorCallback)}. + */ + public interface MapIteratorCallback { + + /** + * Called from inside iterateMap() for each key-value pair in the collection. Note that + * existing implementations of iterateMap() work such that if one or both of objects are + * unresolved (usually that happens because the heap dump is corrupted), null is used + * instead of such an object. If the implementation of this callback returns true, it means + * "continue", false tells iterateMap() that iterating should stop. + */ + boolean scanMapEntry(JavaHeapObject key, JavaHeapObject value); + + /** + * Called from inside iterateMap() for each collection implementation object, e.g. + * HashMap.table array or HashMap$Entry object. If the implementation of this callback + * returns true, it means "continue". False signals iterateMap() that this object has + * already been seen, and thus iteration should either stop completely or skip some objects + * to ensure that this object is not visited again, i.e. endless looping is avoided. + */ + boolean scanImplementationObject(JavaHeapObject implObj); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/ConcurrentHashMapDescriptor.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/ConcurrentHashMapDescriptor.java new file mode 100644 index 00000000..b106bc73 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/ConcurrentHashMapDescriptor.java @@ -0,0 +1,420 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.descriptors; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaInt; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObjectArray; +import org.openjdk.jmc.joverflow.heap.model.JavaThing; +import org.openjdk.jmc.joverflow.heap.model.UnresolvedObject; +import org.openjdk.jmc.joverflow.support.Constants; + +/** + * Descriptor for instances of ConcurrentHashMap. + */ +public class ConcurrentHashMapDescriptor extends AbstractCollectionDescriptor + implements CollectionInstanceDescriptor.CapacityDifferentFromSize, Constants { + private final Factory factory; + private int cachedNumElements = -1; + private int cachedTotalCapacity = -1; + + private ConcurrentHashMapDescriptor(JavaObject col, Factory factory) { + super(col); + this.factory = factory; + } + + @Override + public int getNumElements() { + if (cachedNumElements != -1) { + return cachedNumElements; + } + + int result = 0; + JavaThing segmentField = fields[factory.segmentsFieldIdx]; + if (segmentField == null || !(segmentField instanceof JavaObjectArray)) { + // Likely unresolved object in a corrupted heap dump + return 0; + } + JavaObjectArray segments = (JavaObjectArray) segmentField; + JavaHeapObject[] segs = segments.getElements(); + for (JavaThing segThing : segs) { + // Can be null in JDK7/8, where individual Segments are created lazily. + if (segThing == null || !(segThing instanceof JavaObject)) { + continue; + } + JavaObject seg = (JavaObject) segThing; + + result += getNumElementsInSegment(seg); + } + + cachedNumElements = result; + return result; + } + + private int getNumElementsInSegment(JavaObject seg) { + return ((JavaInt) seg.getField(factory.segLengthFieldIdx)).getValue(); + } + + @Override + public int doGetImplSize() { + col.setVisitedAsCollectionImpl(); + int result = col.getSize(); + // TODO: shall we also look at views here, like keySet etc.? + JavaThing segmentField = fields[factory.segmentsFieldIdx]; + if (segmentField == null || !(segmentField instanceof JavaObjectArray)) { + return result; // Likely unresolved object in a corrupted heap dump + } + + JavaObjectArray segments = (JavaObjectArray) segmentField; + segments.setVisitedAsCollectionImpl(); + result += segments.getSize(); + JavaHeapObject[] segs = segments.getElements(); + for (JavaThing segThing : segs) { + // Can be null in JDK7/8, where individual Segments are created lazily. + if (segThing == null || !(segThing instanceof JavaObject)) { + continue; + } + JavaObject seg = (JavaObject) segThing; + seg.setVisitedAsCollectionImpl(); + result += seg.getSize(); + // CHM$Segment extends ReentrantLock, which has one pointer field, 'Sync sync' + JavaThing segSyncField = seg.getField(factory.segSyncFieldIdx); + if (!(segSyncField == null || segSyncField instanceof UnresolvedObject)) { + JavaObject segSync = (JavaObject) segSyncField; + segSync.setVisitedAsCollectionImpl(); + result += segSync.getSize(); + } + JavaThing segTableThing = seg.getField(factory.segTableFieldIdx); + if (segTableThing == null || !(segTableThing instanceof JavaObjectArray)) { + // Unresolved + continue; + } + JavaObjectArray segTable = (JavaObjectArray) segTableThing; + segTable.setVisitedAsCollectionImpl(); + result += segTable.getSize(); + int nElsInSeg = getNumElementsInSegment(seg); + if (nElsInSeg == 0) { + continue; + } + + result += getSegmentEntriesSize(segTable); + } + + return result; + } + + private int getSegmentEntriesSize(JavaObjectArray entriesArray) { + JavaHeapObject[] entries = entriesArray.getElements(); + int nextFieldIdx = factory.getEntryNextFieldIdx(entries); + + // Reusable fields, to reduce GC pressure + JavaThing[] entryFields = null; + + int size = 0; + for (JavaHeapObject entryThing : entries) { + if (entryThing == null || !(entryThing instanceof JavaObject)) { + // Unresolved or inconsistent + continue; + } + JavaObject entry = (JavaObject) entryThing; + while (true) { + entry.setVisitedAsCollectionImpl(); + size += entry.getSize(); + entryFields = entry.getFields(entryFields); + JavaObject prevEntry = entry; + JavaThing entryThing1 = entryFields[nextFieldIdx]; + if (entryThing1 == null || !(entryThing1 instanceof JavaObject)) { + break; + } + entry = (JavaObject) entryThing1; + if (entry == prevEntry) { + break; + } + } + } + + return size; + } + + @Override + public int getSparsenessOverhead(int ptrSize) { + int totalEls = 0; + int totalCapacity = 0; + int emptySegOverhead = 0; + JavaThing segsThing = fields[factory.segmentsFieldIdx]; + if (segsThing == null || !(segsThing instanceof JavaObjectArray)) { + return 0; + } + JavaObjectArray segments = (JavaObjectArray) segsThing; + JavaHeapObject[] segs = segments.getElements(); + for (JavaHeapObject segThing : segs) { + // Can be null in JDK7/8, where individual Segments are created lazily. + if (segThing == null || !(segThing instanceof JavaObject)) { + continue; + } + JavaObject seg = (JavaObject) segThing; + JavaThing segTableField = seg.getField(factory.segTableFieldIdx); + if (segTableField == null || segTableField instanceof UnresolvedObject) { + continue; + } + int nElsInSeg = getNumElementsInSegment(seg); + totalEls += nElsInSeg; + JavaObjectArray segTable = (JavaObjectArray) segTableField; + totalCapacity += segTable.getLength(); + if (nElsInSeg == 0) { + emptySegOverhead += seg.getSize() + segTable.getSize(); + } + } + + this.cachedTotalCapacity = totalCapacity; + + if (totalEls >= totalCapacity / 2) { + return -1; + } + return (totalCapacity - totalEls) * ptrSize + emptySegOverhead; + } + + @Override + public int getDefaultCapacity() { + return 16 * 16; + } + + @Override + public int getCapacity() { + if (cachedTotalCapacity != -1) { + return cachedTotalCapacity; + } + + int totalCapacity = 0; + JavaThing segsThing = fields[factory.segmentsFieldIdx]; + if (segsThing == null || !(segsThing instanceof JavaObjectArray)) { + cachedTotalCapacity = 0; + return cachedTotalCapacity; + } + JavaObjectArray segments = (JavaObjectArray) segsThing; + JavaHeapObject[] segs = segments.getElements(); + for (JavaHeapObject segThing : segs) { + // Can be null in JDK7/8, where individual Segments are created lazily. + if (segThing == null || !(segThing instanceof JavaObject)) { + continue; + } + JavaThing segTableField = ((JavaObject) segThing).getField(factory.segTableFieldIdx); + if (segTableField == null || segTableField instanceof UnresolvedObject) { + continue; + } + JavaObjectArray segTable = (JavaObjectArray) segTableField; + totalCapacity += segTable.getLength(); + } + + this.cachedTotalCapacity = totalCapacity; + return totalCapacity; + } + + @Override + public void iterateList(ListIteratorCallback cb) { + throw new UnsupportedOperationException(); + } + + @Override + public void iterateMap(MapIteratorCallback cb) { + JavaThing segsThing = fields[factory.segmentsFieldIdx]; + if (segsThing == null || !(segsThing instanceof JavaObjectArray)) { + return; + } + + JavaObjectArray segments = (JavaObjectArray) segsThing; + if (!cb.scanImplementationObject(segments)) { + return; + } + JavaHeapObject[] segs = segments.getElements(); + for (JavaHeapObject seg : segs) { + if (seg == null) { + continue; + } + cb.scanImplementationObject(seg); + } + + int numElements = getNumElements(); + if (numElements == 0) { + return; + } + + int keyFieldIdx = -1, valueFieldIdx = -1; + + for (JavaHeapObject segThing : segs) { + // Can be null in JDK7/8, where individual Segments are created lazily. + if (segThing == null || !(segThing instanceof JavaObject)) { + continue; + } + JavaObject seg = (JavaObject) segThing; + + JavaThing segTableField = seg.getField(factory.segTableFieldIdx); + if (segTableField == null || segTableField instanceof UnresolvedObject) { + continue; + } + JavaObjectArray segTable = (JavaObjectArray) segTableField; + if (!cb.scanImplementationObject(segTable)) { + continue; + } + + int nElsInSeg = getNumElementsInSegment(seg); + if (nElsInSeg == 0) { + continue; + } + + JavaHeapObject[] table = segTable.getElements(); + int nextFieldIdx = factory.getEntryNextFieldIdx(table); + JavaThing[] entryFields = null; + + outerLoop: for (JavaHeapObject entryThing : table) { + if (entryThing == null || !(entryThing instanceof JavaObject)) { + continue; + } + JavaObject entry = (JavaObject) entryThing; + + while (true) { + if (!cb.scanImplementationObject(entry)) { + break; + } + + if (keyFieldIdx == -1) { + keyFieldIdx = factory.getKeyFieldIdx(entry); + valueFieldIdx = factory.getValueFieldIdx(entry); + } + entryFields = entry.getFields(entryFields); + JavaThing keyThing = entryFields[keyFieldIdx]; + JavaThing valueThing = entryFields[valueFieldIdx]; + + JavaHeapObject key = null, value = null; + if (keyThing instanceof JavaHeapObject) { + key = (JavaHeapObject) keyThing; + } + if (valueThing instanceof JavaHeapObject) { + value = (JavaHeapObject) valueThing; + } + if (!cb.scanMapEntry(key, value)) { + break outerLoop; + } + + JavaObject prevEntry = entry; + JavaThing entryThing1 = entryFields[nextFieldIdx]; + if (entryThing1 == null || !(entryThing1 instanceof JavaObject)) { + break; + } + entry = (JavaObject) entryThing1; + if (entry == prevEntry) { + break; + } + } + } + } + } + + @Override + public long getModCount() { + JavaThing segsThing = fields[factory.segmentsFieldIdx]; + if (segsThing == null || !(segsThing instanceof JavaObjectArray)) { + return 0; + } + JavaObjectArray segments = (JavaObjectArray) segsThing; + JavaHeapObject[] segs = segments.getElements(); + int result = 0; + + for (JavaHeapObject segThing : segs) { + // Can be null in JDK7/8, where individual Segments are created lazily. + if (segThing == null || !(segThing instanceof JavaObject)) { + continue; + } + JavaObject seg = (JavaObject) segThing; + result += ((JavaInt) seg.getField(factory.segModCountFieldIdx)).getValue(); + } + return result; + } + + @Override + AbstractCollectionDescriptor.Factory getFactory() { + return factory; + } + + static class Factory extends AbstractCollectionDescriptor.Factory { + + private static final String SEGMENTS_FIELD_NAME = "segments"; + + /* + * TODO: this all gets problematic when fields are added/changed in CHM and CHM$Segment... + * Probably need to use "self-reflection" more actively, at least when it comes to fields + * like 'sync', that are only relevant for determining deep CHM size. + */ + private final int segmentsFieldIdx, segLengthFieldIdx, segTableFieldIdx, segSyncFieldIdx; + private final int segModCountFieldIdx; + + Factory(JavaClass mapClazz, JavaClass segmentClazz, JavaClass[] allImplClasses) { + super(mapClazz, true, allImplClasses, null, false, new String[] {SEGMENTS_FIELD_NAME}); + + segmentsFieldIdx = mapClazz.getInstanceFieldIndex(SEGMENTS_FIELD_NAME); + segLengthFieldIdx = segmentClazz.getInstanceFieldIndex("count"); + segTableFieldIdx = segmentClazz.getInstanceFieldIndex("table"); + segSyncFieldIdx = segmentClazz.getInstanceFieldIndex("sync"); + segModCountFieldIdx = segmentClazz.getInstanceFieldIndex("modCount"); + } + + private Factory(JavaClass clazz, AbstractCollectionDescriptor.Factory superclassFactory) { + super(clazz, superclassFactory); + + this.segmentsFieldIdx = ((Factory) superclassFactory).segmentsFieldIdx; + this.segLengthFieldIdx = ((Factory) superclassFactory).segLengthFieldIdx; + this.segTableFieldIdx = ((Factory) superclassFactory).segTableFieldIdx; + this.segSyncFieldIdx = ((Factory) superclassFactory).segSyncFieldIdx; + this.segModCountFieldIdx = ((Factory) superclassFactory).segModCountFieldIdx; + } + + @Override + AbstractCollectionDescriptor.Factory cloneForSubclass(JavaClass clazz) { + return new Factory(clazz, this); + } + + @Override + CollectionInstanceDescriptor get(JavaObject col) { + return new ConcurrentHashMapDescriptor(col, this); + } + + @Override + protected boolean setModCountFieldIdx(JavaClass clazz) { + // We know that modCount for HashSet can be found, though in non-standard way. + // See ConcurrentHashMapDescriptor.getModCount(). + return true; + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/ConcurrentHashMapDescriptorForJdk8.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/ConcurrentHashMapDescriptorForJdk8.java new file mode 100644 index 00000000..fdc762ed --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/ConcurrentHashMapDescriptorForJdk8.java @@ -0,0 +1,372 @@ +/* + * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.descriptors; + +import java.util.logging.Logger; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaLazyReadObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObjectArray; +import org.openjdk.jmc.joverflow.heap.model.JavaThing; +import org.openjdk.jmc.joverflow.heap.model.UnresolvedObject; +import org.openjdk.jmc.joverflow.support.Constants; + +// @SuppressWarnings("unused") +public class ConcurrentHashMapDescriptorForJdk8 extends AbstractCollectionDescriptor + implements CollectionInstanceDescriptor.CapacityDifferentFromSize, Constants { + private final static Logger LOGGER = Logger.getLogger("org.openjdk.jmc.joverflow.descriptors"); //$NON-NLS-1$ + + private final Factory factory; + private int cachedNumElements = -1; + private int cachedTotalCapacity = -1; + + private ConcurrentHashMapDescriptorForJdk8(JavaObject col, Factory factory) { + super(col); + this.factory = factory; + } + + /** + * Assume: table is not null + */ + private int getNumElementsForOneTable(JavaObjectArray table) { + int result = 0; + JavaHeapObject[] tableElems = table.getElements(); + for (JavaThing tabThing : tableElems) { + // Can be null in JDK7/8, where individual Nodes are created lazily. + if (tabThing == null || !(tabThing instanceof JavaObject)) { + continue; + } + result += 1; + } + return result; + } + + @Override + public int getNumElements() { + if (cachedNumElements != -1) { + return cachedNumElements; + } + // IMPORTANT: The following part is added because in JDK8, the + // ConcurrentHashMap implementation has changed a lot, e.g. there + // are two tables in it, of which the second is a field called + // "nextTable". Thus for the sake of safety, we also need to calculate the + // sizes of the second table, although it is probably not inflated yet. + int result = 0; + int[] index = {factory.tableFieldIdx, factory.nextTableFieldIdx}; + for (int i = 0; i < 2; i++) { + JavaThing tableField = fields[index[i]]; + if (tableField != null && tableField instanceof JavaObjectArray) { + result += getNumElementsForOneTable((JavaObjectArray) tableField); + } + } + cachedNumElements = result; + return result; + } + + @Override + public void iterateList(ListIteratorCallback cb) { + throw new UnsupportedOperationException(); + } + + /** + * Helper method. Iterate a table inside a ConcurrentHashMap. + * + * @param table + * @param cb + */ + private void iterateOneTable(JavaHeapObject[] table, MapIteratorCallback cb) { + int keyFieldIdx = factory.nodeKeyFieldIdx; + int valFieldIdx = factory.nodeValFieldIdx; + int nextFieldIdx = factory.nodeNextFieldIdx; + JavaThing[] nodeFields = null; + + outerLoop: for (JavaHeapObject nodeThing : table) { + if (nodeThing == null || !(nodeThing instanceof JavaObject)) { + continue; + } + JavaObject node = (JavaObject) nodeThing; + + while (true) { + if (!cb.scanImplementationObject(node)) { + break; + } + + // array may be resized if node switches from + // Node to TreeBin or vice versa + nodeFields = node.getFields(nodeFields); + + JavaHeapObject key = null; + JavaThing keyThing = nodeFields[keyFieldIdx]; + if (keyThing instanceof JavaHeapObject) { + key = (JavaHeapObject) keyThing; + } + + JavaHeapObject val = null; + JavaThing valThing = nodeFields[valFieldIdx]; + if (valThing instanceof JavaHeapObject) { + val = (JavaHeapObject) valThing; + } + + if (!cb.scanMapEntry(key, val)) { + break outerLoop; + } + + JavaObject prevNode = node; + JavaThing nextNode = nodeFields[nextFieldIdx]; + if (nextNode == null || !(nextNode instanceof JavaObject)) { + break; + } + node = (JavaObject) nextNode; + if (node == prevNode) { + break; + } + } + } + } + + /** + * In new versions of ConcurrentHashMap(from JDK8), there are two tables there. This method + * overrides its superclass. It iterates one table first, and iterates the second one. + */ + @Override + public void iterateMap(MapIteratorCallback cb) { + int[] index = {factory.tableFieldIdx, factory.nextTableFieldIdx}; + for (int i = 0; i < 2; i++) { + JavaThing tableThing = fields[index[i]]; + JavaObjectArray table; + if (tableThing != null && tableThing instanceof JavaObjectArray) { + table = (JavaObjectArray) tableThing; + if (cb.scanImplementationObject(table)) { + JavaHeapObject[] tableElems = table.getElements(); + int numElements = getNumElements(); + if (numElements != 0) { + iterateOneTable(tableElems, cb); + } + } + } + } + } + + /** + * Overrides superclass method. Get the capacity of each of the two tables inside the + * ConcurrentHashMap and add them. This might not be revised in the future, according to how the + * ConcurrentHashMap works + */ + @Override + public int getCapacity() { + if (cachedTotalCapacity != -1) { + return cachedTotalCapacity; + } + + int totalCapacity = 0; + int[] index = {factory.tableFieldIdx, factory.nextTableFieldIdx}; + for (int i = 0; i < 2; i++) { + JavaThing tableThing = fields[index[i]]; + if (tableThing != null && tableThing instanceof JavaObjectArray) { + JavaObjectArray table = (JavaObjectArray) tableThing; + totalCapacity += table.getLength(); + } + } + + cachedTotalCapacity = totalCapacity; + return totalCapacity; + } + + /** + * Helper method, used for iterating over one table inside the ConcurrentHashMap. + */ + private int getSizeOfOneTable(JavaObjectArray table) { + int result = 0; + for (JavaThing nodeThing : table.getElements()) { + // Can be null in JDK7/8, where individual Segments are created lazily. + if (nodeThing == null || !(nodeThing instanceof JavaObject)) { + continue; + } + JavaObject node = (JavaObject) nodeThing; + while (true) { + node.setVisitedAsCollectionImpl(); + result += node.getSize(); + JavaThing nodeKeyField = node.getField(factory.nodeKeyFieldIdx); + if (!(nodeKeyField == null || nodeKeyField instanceof UnresolvedObject)) { + if (nodeKeyField instanceof JavaLazyReadObject) { + JavaLazyReadObject key = (JavaLazyReadObject) nodeKeyField; + key.setVisitedAsCollectionImpl(); + result += key.getSize(); + } else if (nodeKeyField instanceof JavaClass) { + JavaClass key = (JavaClass) nodeKeyField; + result += key.getSize(); + } else { + // I don't think it can be anything else + LOGGER.severe("Unexpected nodeKeyField: " + nodeKeyField.getClass().getName()); + } + } + JavaThing nodeValField = node.getField(factory.nodeValFieldIdx); + if (!(nodeValField == null || nodeValField instanceof UnresolvedObject)) { + if (nodeValField instanceof JavaLazyReadObject) { + JavaLazyReadObject val = (JavaLazyReadObject) nodeValField; + val.setVisitedAsCollectionImpl(); + result += val.getSize(); + } else if (nodeValField instanceof JavaClass) { + JavaClass val = (JavaClass) nodeValField; + result += val.getSize(); + } else { + LOGGER.severe("Unexpected nodeValField: " + nodeValField.getClass().getName()); + } + } + JavaThing nextNodeThing = node.getField(factory.nodeNextFieldIdx); + if (nextNodeThing == null || !(nextNodeThing instanceof JavaObject)) { + // Unresolved + break; + } + JavaObject prev = node; + node = (JavaObject) nextNodeThing; + if (node == prev) { + break; + } + } + } + return result; + } + + @Override + protected int doGetImplSize() { + // TODO: shall we also look at views here, like keySet etc.? + col.setVisitedAsCollectionImpl(); + int result = col.getSize(); + int[] index = {factory.tableFieldIdx, factory.nextTableFieldIdx}; + for (int i = 0; i < 2; i++) { + JavaThing tableThing = fields[index[i]]; + if (tableThing == null || !(tableThing instanceof JavaObjectArray)) { + // Likely unresolved object in a corrupted heap dump + return result; + } + JavaObjectArray table = (JavaObjectArray) tableThing; + table.setVisitedAsCollectionImpl(); + result += table.getSize(); + result += getSizeOfOneTable(table); + } + + return result; + } + + @Override + Factory getFactory() { + return factory; + } + + /** + * Override superclass. Calculate total length of two tables, and total elements number in two + * tables. + */ + // FIXME: the way of seeing whether it is sparseness might not be accurate because there are two tables. + @Override + public int getSparsenessOverhead(int ptrSize) { + int totalEls = 0; + int totalCapacity = 0; + int emptyTableOverhead = 0; + int[] index = {factory.tableFieldIdx, factory.nextTableFieldIdx}; + for (int i = 0; i < 2; i++) { + JavaThing tableThing = fields[index[i]]; + if (tableThing != null && !(tableThing instanceof JavaObjectArray)) { + JavaObjectArray table = (JavaObjectArray) tableThing; + int nElsInTab = getNumElementsForOneTable(table); + totalEls += nElsInTab; + totalCapacity += table.getLength(); + if (nElsInTab == 0) { + emptyTableOverhead += table.getSize(); + } + } + } + + cachedTotalCapacity = totalCapacity; + + if (totalEls >= totalCapacity / 2) { + return -1; + } + return (totalCapacity - totalEls) * ptrSize + emptyTableOverhead; + } + + @Override + public int getDefaultCapacity() { + return 16; + } + + @Override + public long getModCount() { + return 0; + } + + static class Factory extends AbstractCollectionDescriptor.Factory { + private static final String TABLE_NAME = "table"; + private static final String NEXT_TABLE_NAME = "nextTable"; + private final int tableFieldIdx, nextTableFieldIdx, nodeKeyFieldIdx, nodeValFieldIdx, nodeNextFieldIdx; + + Factory(JavaClass mapClazz, JavaClass nodeClazz, JavaClass[] allImplClasses) { + super(mapClazz, true, allImplClasses, null, false, new String[] {TABLE_NAME, NEXT_TABLE_NAME}); + + tableFieldIdx = mapClazz.getInstanceFieldIndex(TABLE_NAME); + nextTableFieldIdx = mapClazz.getInstanceFieldIndex(NEXT_TABLE_NAME); + nodeKeyFieldIdx = nodeClazz.getInstanceFieldIndex("key"); + nodeValFieldIdx = nodeClazz.getInstanceFieldIndex("val"); + nodeNextFieldIdx = nodeClazz.getInstanceFieldIndex("next"); + + } + + private Factory(JavaClass clazz, AbstractCollectionDescriptor.Factory superclassFactory) { + super(clazz, superclassFactory); + tableFieldIdx = ((Factory) superclassFactory).tableFieldIdx; + nextTableFieldIdx = ((Factory) superclassFactory).nextTableFieldIdx; + nodeKeyFieldIdx = ((Factory) superclassFactory).nodeKeyFieldIdx; + nodeValFieldIdx = ((Factory) superclassFactory).nodeValFieldIdx; + nodeNextFieldIdx = ((Factory) superclassFactory).nodeNextFieldIdx; + } + + @Override + CollectionInstanceDescriptor get(JavaObject col) { + return new ConcurrentHashMapDescriptorForJdk8(col, this); + } + + @Override + org.openjdk.jmc.joverflow.descriptors.AbstractCollectionDescriptor.Factory cloneForSubclass(JavaClass clazz) { + return new Factory(clazz, this); + } + + // FIXME: this is a problem in new version of ConcurrentHashMap because it doesn't have modCount field any more + @Override + protected boolean setModCountFieldIdx(JavaClass clazz) { + return false; + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/CopyOnWriteArraySetDescriptor.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/CopyOnWriteArraySetDescriptor.java new file mode 100644 index 00000000..e5f755b8 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/CopyOnWriteArraySetDescriptor.java @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.descriptors; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaThing; + +/** + * Descriptor fo instances of java.util.concurrent.CopyOnWriteArraySet. + */ +public class CopyOnWriteArraySetDescriptor extends AbstractCollectionDescriptor { + private final Factory factory; + private final FullyUtilizedArrayListDescriptor listDesc; + + private CopyOnWriteArraySetDescriptor(JavaObject col, Factory factory) { + super(col); + this.factory = factory; + JavaThing alField = fields[factory.alFieldIdx]; + if (alField == null || !(alField instanceof JavaObject)) { + // Unresolved object, or collection caught in inconsistent state + listDesc = null; + return; + } + + JavaObject list = (JavaObject) alField; + listDesc = (FullyUtilizedArrayListDescriptor) factory.cowaListDescFactory.get(list); + } + + @Override + protected int doGetImplSize() { + if (listDesc == null) { + return col.getSize(); + } + listDesc.col.setVisitedAsCollectionImpl(); + return col.getSize() + ((listDesc != null) ? listDesc.getImplSize() : 0); + } + + @Override + Factory getFactory() { + return factory; + } + + @Override + public int getNumElements() { + return listDesc != null ? listDesc.getNumElements() : 0; + } + + @Override + public void iterateList(final ListIteratorCallback cb) { + if (listDesc == null) { + return; + } + listDesc.iterateList(new ListIteratorCallback() { + @Override + public boolean scanListElement(JavaHeapObject element) { + return cb.scanListElement(element); + } + + @Override + public boolean scanImplementationObject(JavaHeapObject implObj) { + return cb.scanImplementationObject(implObj); + } + }); + } + + @Override + public void iterateMap(MapIteratorCallback cb) { + throw new UnsupportedOperationException(); + } + + static class Factory extends AbstractCollectionDescriptor.Factory { + private static final String AL_FIELD_NAME = "al"; + + private final int alFieldIdx; + private final FullyUtilizedArrayListDescriptor.Factory cowaListDescFactory; + + Factory(JavaClass cowaSetClazz, FullyUtilizedArrayListDescriptor.Factory cowaListDescFactory) { + // Same as with HashSet, we deliberately treat CopyOnWriteArrayList al field + // as an "extra" field that should be followed during breadth-first scan. In that + // way, we scan COWASet -> COWAList, and eventually whatever extra fields there + // may be in that "dependent" HashMap. To make the code treat COWASet.al as extra + // field, we don't put it below in knownFieldNames parameter, despite the fact + // that this field is quite well-known. + super(cowaSetClazz, false, new JavaClass[] {}, null, true, new String[] {}); + this.cowaListDescFactory = cowaListDescFactory; + this.alFieldIdx = cowaSetClazz.getInstanceFieldIndex(AL_FIELD_NAME); + } + + private Factory(JavaClass clazz, AbstractCollectionDescriptor.Factory superclassFactory) { + super(clazz, superclassFactory); + this.cowaListDescFactory = ((Factory) superclassFactory).cowaListDescFactory; + this.alFieldIdx = ((Factory) superclassFactory).alFieldIdx; + } + + @Override + AbstractCollectionDescriptor.Factory cloneForSubclass(JavaClass clazz) { + return new Factory(clazz, this); + } + + @Override + CollectionInstanceDescriptor get(JavaObject col) { + return new CopyOnWriteArraySetDescriptor(col, this); + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/FullyUtilizedArrayListDescriptor.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/FullyUtilizedArrayListDescriptor.java new file mode 100644 index 00000000..f914a0e2 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/FullyUtilizedArrayListDescriptor.java @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.descriptors; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObjectArray; +import org.openjdk.jmc.joverflow.heap.model.JavaThing; + +/** + * Descriptor for List Collection classes that keep their elements in an object array, and this + * array is fully utilized (collection's capacity is always the same as its size). + */ +class FullyUtilizedArrayListDescriptor extends AbstractCollectionDescriptor { + protected final Factory factory; + + FullyUtilizedArrayListDescriptor(JavaObject col, Factory factory) { + super(col); + this.factory = factory; + } + + @Override + public int getNumElements() { + JavaObjectArray elsArray = getElementsArray(); + if (elsArray == null) { + // Can happen if elements is unresolved + return 0; + } + return elsArray.getLength(); + } + + @Override + public void iterateList(ListIteratorCallback cb) { + JavaObjectArray elsArray = getElementsArray(); + if (elsArray == null) { + // Can happen if elements is unresolved + return; + } + if (!cb.scanImplementationObject(elsArray)) { + return; + } + + JavaHeapObject[] elements = elsArray.getElements(); + for (JavaHeapObject element : elements) { + if (element == null) { + continue; + } + if (!cb.scanListElement(element)) { + break; + } + } + } + + @Override + public void iterateMap(MapIteratorCallback cb) { + throw new UnsupportedOperationException(); + } + + @Override + protected int doGetImplSize() { + return getDirectImplSize(); + } + + @Override + AbstractCollectionDescriptor.Factory getFactory() { + return factory; + } + + public JavaObjectArray getElementsArray() { + JavaThing els = fields[factory.elsArrayFieldIdx]; // May be null or UnresolvedObject + return els instanceof JavaObjectArray ? (JavaObjectArray) els : null; + } + + /** + * Returns the sum of shallow sizes of this collection object and its elements array. + */ + protected int getDirectImplSize() { + col.setVisitedAsCollectionImpl(); + int colSize = col.getSize(); + JavaObjectArray els = getElementsArray(); + if (els == null) { + return colSize; + } else { + els.setVisitedAsCollectionImpl(); + return colSize + els.getSize(); + } + } + + static class Factory extends AbstractCollectionDescriptor.Factory { + + private final int elsArrayFieldIdx; + + Factory(JavaClass clazz, String elsArrayFieldName, JavaClass[] implClasses, String[] parentColClassNames) { + super(clazz, false, implClasses, parentColClassNames, false, new String[] {elsArrayFieldName}); + elsArrayFieldIdx = clazz.getInstanceFieldIndex(elsArrayFieldName); + } + + Factory(JavaClass clazz, FullyUtilizedArrayListDescriptor.Factory superclassFactory) { + super(clazz, superclassFactory); + this.elsArrayFieldIdx = superclassFactory.elsArrayFieldIdx; + } + + @Override + CollectionInstanceDescriptor get(JavaObject col) { + return new FullyUtilizedArrayListDescriptor(col, this); + } + + @Override + AbstractCollectionDescriptor.Factory cloneForSubclass(JavaClass clazz) { + return new Factory(clazz, this); + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/HashSetDescriptor.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/HashSetDescriptor.java new file mode 100644 index 00000000..1024eddc --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/HashSetDescriptor.java @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.descriptors; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaThing; +import org.openjdk.jmc.joverflow.support.Constants; +import org.openjdk.jmc.joverflow.util.ClassUtils; + +/** + * Descriptor for instances of HashSet and LinkedHashSet. + */ +public class HashSetDescriptor extends AbstractCollectionDescriptor + implements CollectionInstanceDescriptor.CapacityDifferentFromSize, Constants { + private final Factory factory; + private final ArrayBasedCollectionDescriptor mapDesc; + + private HashSetDescriptor(JavaObject col, Factory factory) { + super(col); + this.factory = factory; + JavaThing mapField = fields[factory.mapFieldIdx]; + if (mapField == null || !(mapField instanceof JavaObject)) { + // Unresolved object, or collection caught in inconsistent state + mapDesc = null; + return; + } + + JavaObject map = (JavaObject) mapField; + mapDesc = factory.hashMapDescFactory.get(map); + } + + @Override + public int getNumElements() { + return (mapDesc != null) ? mapDesc.getNumElements() : 0; + } + + @Override + public void iterateList(final ListIteratorCallback cb) { + if (mapDesc == null) { + return; + } + mapDesc.iterateMap(new MapIteratorCallback() { + @Override + public boolean scanMapEntry(JavaHeapObject key, JavaHeapObject value) { + return cb.scanListElement(key); + } + + @Override + public boolean scanImplementationObject(JavaHeapObject implObj) { + return cb.scanImplementationObject(implObj); + } + }); + } + + @Override + public void iterateMap(MapIteratorCallback cb) { + throw new UnsupportedOperationException(); + } + + @Override + public int doGetImplSize() { + if (mapDesc == null) { + return col.getSize(); + } + mapDesc.col.setVisitedAsCollectionImpl(); + return col.getSize() + ((mapDesc != null) ? mapDesc.getImplSize() : 0); + } + + @Override + public int getSparsenessOverhead(int ptrSize) { + if (mapDesc == null) { + return 0; + } + return mapDesc.getSparsenessOverhead(ptrSize); + } + + @Override + public int getDefaultCapacity() { + return mapDesc != null ? mapDesc.getDefaultCapacity() : 16; + } + + @Override + public int getCapacity() { + return mapDesc != null ? mapDesc.getCapacity() : 0; + } + + @Override + public long getModCount() { + return mapDesc != null ? mapDesc.getModCount() : 0; + } + + @Override + AbstractCollectionDescriptor.Factory getFactory() { + return factory; + } + + static class Factory extends AbstractCollectionDescriptor.Factory { + // Possible names of fields for the backing HashMap in different impls of HashSet + private static final String MAP_FIELD_NAMES = "map|backingMap"; + + private final int mapFieldIdx; + private final ArrayBasedCollectionDescriptor.Factory hashMapDescFactory; + + Factory(JavaClass hashSetClazz, JavaClass[] implClasses, + ArrayBasedCollectionDescriptor.Factory hashMapDescFactory) { + // We deliberately treat HashSet.map as an "extra" field that should be followed + // during breadth-first scan. In that way, we scan HashSet->HashMap, and eventually + // fields such as HashMap.keySet in that "dependent" HashMap. To make the code + // treat (Linked)HashSet.map as extra field, we don't put it below in knownFieldNames + // parameter, despite the fact that this field is quite well-known. + super(hashSetClazz, false, implClasses, null, true, new String[] {}); + this.hashMapDescFactory = hashMapDescFactory; + String mapFieldName = ClassUtils.getExactFieldName(MAP_FIELD_NAMES, hashSetClazz); + mapFieldIdx = hashSetClazz.getInstanceFieldIndex(mapFieldName); + } + + private Factory(JavaClass clazz, AbstractCollectionDescriptor.Factory superclassFactory) { + super(clazz, superclassFactory); + this.hashMapDescFactory = ((Factory) superclassFactory).hashMapDescFactory; + this.mapFieldIdx = ((Factory) superclassFactory).mapFieldIdx; + } + + @Override + AbstractCollectionDescriptor.Factory cloneForSubclass(JavaClass clazz) { + return new Factory(clazz, this); + } + + @Override + CollectionInstanceDescriptor get(JavaObject col) { + return new HashSetDescriptor(col, this); + } + + @Override + protected boolean setModCountFieldIdx(JavaClass clazz) { + // We know that modCount for HashSet can be found, though in non-standard way. + // See HashSetDescriptor.getModCount(). + return true; + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/IdentityHashMapDescriptor.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/IdentityHashMapDescriptor.java new file mode 100644 index 00000000..9f98b545 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/IdentityHashMapDescriptor.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.descriptors; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObjectArray; +import org.openjdk.jmc.joverflow.util.ClassUtils; + +/** + * Descriptor for IdentityHashMap. Unlike other maps, IdentityHashMap is implemented as a single + * array of contained objects, thus it needs a special implementation of content sampling. + */ +public class IdentityHashMapDescriptor extends ArrayBasedCollectionDescriptor { + + private IdentityHashMapDescriptor(JavaObject col, Factory factory) { + super(col, factory); + } + + @Override + public int getImplSize() { + return getDirectImplSize(); + } + + @Override + public void iterateMap(MapIteratorCallback cb) { + JavaObjectArray elsArray = getElementsArray(); + if (elsArray == null) { + // Unresolved in corrupted heap dump + return; + } + if (!cb.scanImplementationObject(elsArray)) { + return; + } + + int numElements = getNumElements(); + if (numElements == 0) { + return; + } + + JavaHeapObject[] elements = elsArray.getElements(); + + for (int idx = 0; idx < elements.length; idx++) { + JavaHeapObject key = elements[idx++]; + JavaHeapObject value = elements[idx]; + if (key == null && value == null) { + continue; + } + if (!cb.scanMapEntry(key, value)) { + break; + } + } + } + + static class Factory extends ArrayBasedCollectionDescriptor.Factory { + + Factory(JavaClass clazz) { + super(clazz, true, "size", ClassUtils.getExactFieldName("table|elementData", clazz), 32, new JavaClass[] {}, + null); + } + + Factory(JavaClass clazz, AbstractArrayBasedCollectionDescriptor.Factory superclassFactory) { + super(clazz, superclassFactory); + } + + @Override + AbstractCollectionDescriptor.Factory cloneForSubclass(JavaClass clazz) { + return new Factory(clazz, this); + } + + @Override + ArrayBasedCollectionDescriptor get(JavaObject col) { + return new IdentityHashMapDescriptor(col, this); + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/LinkedCollectionDescriptor.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/LinkedCollectionDescriptor.java new file mode 100644 index 00000000..e0827ab2 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/LinkedCollectionDescriptor.java @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.descriptors; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaThing; + +/** + * Descriptor for linked-list style data structures, such as java.util.LinkedList. + */ +public class LinkedCollectionDescriptor extends AbstractLinkedCollectionDescriptor { + private int cachedSize = -1; + + private LinkedCollectionDescriptor(JavaObject col, Factory factory) { + super(col, factory); + } + + @Override + public int doGetImplSize() { + int result = col.getSize(); + + JavaThing rootField = col.getField(factory.rootFieldIdx); + if (rootField == null || !(rootField instanceof JavaObject)) { + return result; + } + + JavaObject entry = (JavaObject) rootField; + int nextFieldIdx = factory.getEntryNextFieldIdx(entry); + long rootEntryObjOfsInFile = entry.getObjOfsInFile(); + + // Reusable fields, to reduce GC pressure + JavaThing[] entryFields = null; + + while (!entry.isVisitedAsCollectionImpl()) { + // Just in case there is a corruption somewhere + entry.setVisitedAsCollectionImpl(); + result += entry.getSize(); + + entryFields = entry.getFields(entryFields); + JavaObject prevEntry = entry; + JavaThing entryThing = entryFields[nextFieldIdx]; + if (entryThing == null || !(entryThing instanceof JavaObject)) { + break; + } + entry = (JavaObject) entryThing; + if (entry.getObjOfsInFile() == prevEntry.getObjOfsInFile() + || entry.getObjOfsInFile() == rootEntryObjOfsInFile) { + break; + } + } + + return result; + } + + @Override + public void iterateList(ListIteratorCallback cb) { + int numElements = getNumElements(); + if (numElements == 0) { + return; + } + + JavaThing rootField = col.getField(factory.rootFieldIdx); + if (rootField == null || !(rootField instanceof JavaObject)) { + return; + } + + JavaObject entry = (JavaObject) rootField; + int nextFieldIdx = factory.getEntryNextFieldIdx(entry); + int elementFieldIdx = factory.getElementFieldIdx(entry); + long rootEntryObjOfsInFile = entry.getObjOfsInFile(); + // Reusable fields, to reduce GC pressure + JavaThing[] entryFields = null; + + while (true) { + if (!cb.scanImplementationObject(entry)) { + break; + } + entryFields = entry.getFields(entryFields); + JavaThing payloadThing = entryFields[elementFieldIdx]; + if (payloadThing != null && (payloadThing instanceof JavaHeapObject)) { + JavaHeapObject payload = (JavaHeapObject) payloadThing; + if (!cb.scanListElement(payload)) { + break; + } + } + + JavaObject prevEntry = entry; + JavaThing entryThing = entryFields[nextFieldIdx]; + if (entryThing == null || !(entryThing instanceof JavaObject)) { + break; + } + entry = (JavaObject) entryThing; + if (entry.getObjOfsInFile() == prevEntry.getObjOfsInFile() + || entry.getObjOfsInFile() == rootEntryObjOfsInFile) { + break; + } + } + } + + // This method is used when the descriptor is used for a collection, e.g. + // ConcurrentLinkedQueue, that does not provide a 'size' field. + @Override + protected int getSizeByCountingElements() { + if (cachedSize != -1) { + return cachedSize; + } + + JavaThing rootField = col.getField(factory.rootFieldIdx); + if (rootField == null || !(rootField instanceof JavaObject)) { + cachedSize = 0; + return cachedSize; + } + + int result = 0; + JavaObject entry = (JavaObject) rootField; + int nextFieldIdx = factory.getEntryNextFieldIdx(entry); + long rootEntryObjOfsInFile = entry.getObjOfsInFile(); + // Reusable fields, to reduce GC pressure + JavaThing[] entryFields = null; + + while (true) { + result++; + entryFields = entry.getFields(entryFields); + + JavaObject prevEntry = entry; + JavaThing entryThing = entryFields[nextFieldIdx]; + if (entryThing == null || !(entryThing instanceof JavaObject)) { + break; + } + entry = (JavaObject) entryThing; + if (entry.getObjOfsInFile() == prevEntry.getObjOfsInFile() + || entry.getObjOfsInFile() == rootEntryObjOfsInFile) { + break; + } + } + + cachedSize = result; + return result; + } + + @Override + public void iterateMap(MapIteratorCallback cb) { + throw new UnsupportedOperationException(); + } + + static class Factory extends AbstractLinkedCollectionDescriptor.Factory { + + Factory(JavaClass clazz, String sizeFieldName, String rootFieldName, String elementFieldName, + JavaClass[] implClasses) { + super(clazz, false, sizeFieldName, rootFieldName, elementFieldName, implClasses); + } + + private Factory(JavaClass clazz, AbstractCollectionDescriptor.Factory superclassFactory) { + super(clazz, superclassFactory); + } + + @Override + AbstractCollectionDescriptor.Factory cloneForSubclass(JavaClass clazz) { + return new Factory(clazz, this); + } + + @Override + CollectionInstanceDescriptor get(JavaObject col) { + return new LinkedCollectionDescriptor(col, this); + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/LinkedHashMapDescriptor.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/LinkedHashMapDescriptor.java new file mode 100644 index 00000000..c71cc770 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/LinkedHashMapDescriptor.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.descriptors; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaThing; +import org.openjdk.jmc.joverflow.support.Constants; + +/** + * Descriptor for LinkedHashMap. The main reason it exists is that in addition to the stuff standard + * for all array-based maps, each LinkedHashMap contains the non-null 'Entry header' field, which + * does not contain any workload, but is used just as an implementation convenience to manage the + * linked list. + */ +public class LinkedHashMapDescriptor extends ArrayBasedCollectionDescriptor { + + private LinkedHashMapDescriptor(JavaObject col, Factory factory) { + super(col, factory); + } + + @Override + public int getImplSize() { + int result = super.getImplSize(); + int idx = ((Factory) factory).headerFieldIdx; + JavaThing headerThing = idx != -1 ? col.getField(idx) : null; + if (headerThing == null || !(headerThing instanceof JavaObject)) { + // Heap dump partly corrupted, or object caught in inconsistent state + return result; + } + JavaObject header = (JavaObject) headerThing; + header.setVisitedAsCollectionImpl(); + result += header.getSize(); + return result; + } + + static class Factory extends ArrayBasedCollectionDescriptor.Factory { + private final int headerFieldIdx; + + Factory(JavaClass clazz, JavaClass[] implClasses, boolean jdk8HashMap) { + super(clazz, true, "size", "table", 16, implClasses, new String[] {Constants.LINKED_HASH_SET}, jdk8HashMap); + headerFieldIdx = jdk8HashMap ? clazz.getInstanceFieldIndexOrMinusOne("head") + : clazz.getInstanceFieldIndexOrMinusOne("header"); + } + + Factory(JavaClass clazz, AbstractArrayBasedCollectionDescriptor.Factory superclassFactory) { + super(clazz, superclassFactory); + this.headerFieldIdx = ((Factory) superclassFactory).headerFieldIdx; + } + + @Override + AbstractCollectionDescriptor.Factory cloneForSubclass(JavaClass clazz) { + return new Factory(clazz, this); + } + + @Override + ArrayBasedCollectionDescriptor get(JavaObject col) { + return new LinkedHashMapDescriptor(col, this); + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/StandaloneArrayDescFactory.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/StandaloneArrayDescFactory.java new file mode 100644 index 00000000..0d0eb1f7 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/StandaloneArrayDescFactory.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.descriptors; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; + +/** + * We use CollectionClassDescriptors for standalone arrays just to unify gathering stats. In effect, + * this factory is used to create a separate CollectionClassDescriptor for each array type, e.g. an + * array of java.lang.String instances and so on. + */ +public class StandaloneArrayDescFactory extends AbstractCollectionDescriptor.Factory { + private static JavaClass[] EMPTY_JAVACLASS_ARRAY = new JavaClass[] {}; + + StandaloneArrayDescFactory(JavaClass clazz) { + super(clazz, false, EMPTY_JAVACLASS_ARRAY, null, false, null); + } + + @Override + AbstractCollectionDescriptor.Factory cloneForSubclass(JavaClass clazz) { + throw new UnsupportedOperationException(); + } + + @Override + CollectionInstanceDescriptor get(JavaObject col) { + throw new UnsupportedOperationException(); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/TreeMapDescriptor.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/TreeMapDescriptor.java new file mode 100644 index 00000000..20effb44 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/TreeMapDescriptor.java @@ -0,0 +1,269 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.descriptors; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObjectArray; +import org.openjdk.jmc.joverflow.heap.model.JavaThing; +import org.openjdk.jmc.joverflow.heap.model.UnresolvedObject; + +/** + * Descriptor for java.util.TreeMap. It is somewhat complicated due to the fact that JRockit seems + * to use its own version of this class, where each node has Object keys[], values[] rather than + * Object key, value. + */ +public class TreeMapDescriptor extends AbstractLinkedCollectionDescriptor { + private static final JavaHeapObject[] EMPTY_OBJ_ARRAY = new JavaHeapObject[0]; + + private int keyFieldIdx, valueFieldIdx, leftFieldIdx, rightFieldIdx; + + private TreeMapDescriptor(JavaObject col, Factory factory) { + super(col, factory); + } + + @Override + public int doGetImplSize() { + col.setVisitedAsCollectionImpl(); + int result = col.getSize(); + + JavaThing rootThing = col.getField(factory.rootFieldIdx); + if (rootThing == null || rootThing instanceof UnresolvedObject) { + return result; + } + JavaObject rootEntry = (JavaObject) rootThing; + + Factory thisFactory = (Factory) factory; + leftFieldIdx = thisFactory.getLeftFieldIdx(rootEntry); + rightFieldIdx = thisFactory.getRightFieldIdx(rootEntry); + keyFieldIdx = thisFactory.getKeyFieldIdx(rootEntry); + valueFieldIdx = thisFactory.getValueFieldIdx(rootEntry); + + result += getDeepEntrySize(rootThing, null); + return result; + } + + private int getDeepEntrySize(JavaThing entryThing, JavaThing[] entryFields) { + if (entryThing == null || entryThing instanceof UnresolvedObject) { + return 0; + } + + JavaObject entry = (JavaObject) entryThing; + if (entry.isVisitedAsCollectionImpl()) { + // Just in case there is a corruption + return 0; + } + entry.setVisitedAsCollectionImpl(); + + // Reuse fields to reduce GC pressure + entryFields = entry.getFields(entryFields); + + // It looks like in some heap dumps, an instance of TreeMap$Node may be + // reachable from *outside* its encapsulating TreeMap. So far I observed it + // once (with default2011-10-25_11_27_00.hprof), and the ref chain looked + // like something really outside a normal TreeMap. If such a node is reached + // first, it ends up with impl-inclusive size updated for its own class, + // so if we subsequently update impl-inclusive size for TreeMap here, the + // sum total of inclusive sizes for all classess will end up being greater + // than the sum total of shallow sizes. So here we take measures to fix + // the sizes properly + int entrySize = entry.getSize(); + if (entry.isVisited() && entry.getClazz().getSnapshot().isCalculatingStats()) { + entry.getClazz().updateInclusiveInstanceSize(-entrySize); + } + int result = entrySize; + if (((Factory) factory).isJRockitVersion) { + JavaObjectArray keys = (JavaObjectArray) entryFields[keyFieldIdx]; + keys.setVisitedAsCollectionImpl(); + result += keys.getSize(); + JavaObjectArray values = (JavaObjectArray) entryFields[valueFieldIdx]; + values.setVisitedAsCollectionImpl(); + result += values.getSize(); + } + + JavaThing leftThing = entryFields[leftFieldIdx]; + JavaThing rightThing = entryFields[rightFieldIdx]; + result += getDeepEntrySize(leftThing, entryFields); + result += getDeepEntrySize(rightThing, entryFields); + return result; + } + + @Override + public void iterateList(ListIteratorCallback cb) { + throw new UnsupportedOperationException(); + } + + @Override + public void iterateMap(MapIteratorCallback cb) { + JavaThing rootThing = col.getField(factory.rootFieldIdx); + if (rootThing == null || !(rootThing instanceof JavaObject)) { + return; + } + JavaObject rootEntry = (JavaObject) rootThing; + + Factory thisFactory = (Factory) factory; + keyFieldIdx = thisFactory.getKeyFieldIdx(rootEntry); + valueFieldIdx = thisFactory.getValueFieldIdx(rootEntry); + leftFieldIdx = thisFactory.getLeftFieldIdx(rootEntry); + rightFieldIdx = thisFactory.getRightFieldIdx(rootEntry); + + scanEntry(rootEntry, null, cb); + } + + private void scanEntry(JavaThing entryThing, JavaThing[] entryFields, MapIteratorCallback cb) { + if (entryThing == null || entryThing instanceof UnresolvedObject) { + return; + } + JavaObject entry = (JavaObject) entryThing; + if (!cb.scanImplementationObject(entry)) { + return; + } + + entryFields = entry.getFields(entryFields); + JavaThing keyThing = entryFields[keyFieldIdx]; + JavaThing valueThing = entryFields[valueFieldIdx]; + JavaHeapObject key = null, value = null; + + if (((Factory) factory).isJRockitVersion) { + JavaObjectArray keysArr = (keyThing != null && keyThing instanceof JavaObjectArray) + ? (JavaObjectArray) keyThing : null; + JavaObjectArray valuesArr = (valueThing != null && valueThing instanceof JavaObjectArray) + ? (JavaObjectArray) valueThing : null; + JavaHeapObject[] keys = EMPTY_OBJ_ARRAY; + if (keysArr != null) { + if (!cb.scanImplementationObject(keysArr)) { + return; + } + keys = keysArr.getElements(); + } + JavaHeapObject[] values = EMPTY_OBJ_ARRAY; + if (valuesArr != null) { + if (!cb.scanImplementationObject(valuesArr)) { + return; + } + values = valuesArr.getElements(); + } + int maxLen = (keys.length > values.length) ? keys.length : values.length; + + for (int i = 0; i < maxLen; i++) { + key = (i < keys.length) ? keys[i] : null; + value = (i < values.length) ? values[i] : null; + if (!cb.scanMapEntry(key, value)) { + return; + } + } + + } else { + if (keyThing != null && keyThing instanceof JavaHeapObject) { + key = (JavaHeapObject) keyThing; + } + if (valueThing != null && valueThing instanceof JavaHeapObject) { + value = (JavaHeapObject) valueThing; + } + + if (!cb.scanMapEntry(key, value)) { + return; + } + } + + JavaThing leftThing = entryFields[leftFieldIdx]; + JavaThing rightThing = entryFields[rightFieldIdx]; + scanEntry(leftThing, entryFields, cb); + scanEntry(rightThing, entryFields, cb); + } + + @Override + protected int getSizeByCountingElements() { + throw new UnsupportedOperationException("Should never be called"); + } + + static class Factory extends AbstractLinkedCollectionDescriptor.Factory { + + private int leftFieldIdx = -1, rightFieldIdx = -1; + private boolean isJRockitVersion, keyFieldIdxInitialized; + + Factory(JavaClass clazz, JavaClass[] implClasses) { + super(clazz, true, "size", "root", "value", implClasses); + } + + private Factory(JavaClass clazz, AbstractCollectionDescriptor.Factory superclassFactory) { + super(clazz, superclassFactory); + } + + @Override + AbstractCollectionDescriptor.Factory cloneForSubclass(JavaClass clazz) { + return new Factory(clazz, this); + } + + @Override + CollectionInstanceDescriptor get(JavaObject col) { + return new TreeMapDescriptor(col, this); + } + + @Override + protected int getKeyFieldIdx(JavaObject entry) { + if (keyFieldIdxInitialized) { + return super.getKeyFieldIdx(entry); + } + + keyFieldIdxInitialized = true; + + // Check if this is a JRockit version, that has Object keys[] instead of Object key + if (entry.getField("key") != null) { + // Normal version, do the usual thing + isJRockitVersion = false; + return super.getKeyFieldIdx(entry); + } + + isJRockitVersion = true; + super.setMapKeyFieldName("keys"); + super.setValueFieldName("values"); + return super.getKeyFieldIdx(entry); + } + + protected int getLeftFieldIdx(JavaObject entry) { + if (leftFieldIdx == -1) { + leftFieldIdx = entry.getClazz().getInstanceFieldIndex("left"); + } + return leftFieldIdx; + } + + protected int getRightFieldIdx(JavaObject entry) { + if (rightFieldIdx == -1) { + rightFieldIdx = entry.getClazz().getInstanceFieldIndex("right"); + } + return rightFieldIdx; + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/WeakHashMapDescriptor.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/WeakHashMapDescriptor.java new file mode 100644 index 00000000..7419dc7a --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/descriptors/WeakHashMapDescriptor.java @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.descriptors; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaField; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObjectArray; +import org.openjdk.jmc.joverflow.heap.model.JavaThing; +import org.openjdk.jmc.joverflow.support.Constants; +import org.openjdk.jmc.joverflow.util.ClassUtils; + +/** + * So far this class exists solely to provide the getKeysAndValues() method, that we don't want + * (because we don't need yet) to implement in other descriptors. Note that even if we define that + * method in CollectionInstanceDescriptor, we will still need to have this class, because its keys + * are kept in different objects/fields than in say normal HashMap. + */ +public class WeakHashMapDescriptor extends ArrayBasedCollectionDescriptor implements Constants { + private static final JavaHeapObject[] EMPTY_ARRAY = new JavaHeapObject[0]; + + private WeakHashMapDescriptor(JavaObject col, Factory factory) { + super(col, factory); + } + + public JavaHeapObject[][] getKeysAndValues() { + int size = getNumElements(); + if (size <= 0) { + // "< 0" is weird, but I've seen at least one heap dump with this value. + // Could that be some concurrency-related inconsistency? + return new JavaHeapObject[][] {EMPTY_ARRAY, EMPTY_ARRAY}; + } + + Factory f = (Factory) factory; + + JavaObjectArray entriesArray = getElementsArray(); + if (entriesArray == null) { + return new JavaHeapObject[][] {EMPTY_ARRAY, EMPTY_ARRAY}; + } + JavaHeapObject[] entries = entriesArray.getElements(); + JavaThing[] entryFields = null; // Reusable fields, to reduce GC pressure + + JavaHeapObject keys[] = new JavaHeapObject[size]; + JavaHeapObject values[] = new JavaHeapObject[size]; + + int entryIdx = 0; + outerLoop: for (JavaHeapObject entryThing : entries) { + if (entryThing == null || !(entryThing instanceof JavaObject)) { + continue; + } + JavaObject entry = (JavaObject) entryThing; + while (true) { + entryFields = entry.getFields(entryFields); + JavaThing keyThing = entryFields[f.referentFieldIdx]; + JavaThing valueThing = entryFields[f.valueFieldIdx]; + // Instanceof checks below are protection against unresolved objects + if (keyThing != null && keyThing instanceof JavaHeapObject && valueThing instanceof JavaHeapObject) { + keys[entryIdx] = (JavaHeapObject) keyThing; + values[entryIdx] = (JavaHeapObject) valueThing; + entryIdx++; + // There is a small chance that the table is caught in an inconsistent state, + // with more entries than getSize() tells. Just ignore the excessive entry. + if (entryIdx == keys.length) { + break outerLoop; + } + } + JavaObject prevEntry = entry; + JavaThing entryThing1 = entryFields[f.nextFieldIdx]; + if (entryThing1 == null || !(entryThing1 instanceof JavaObject)) { + break; + } + entry = (JavaObject) entryThing1; + if (entry == prevEntry) { + throw new RuntimeException("Problem in data: WeakHashMap$Entry.next points to itself?"); + } + } + } + + return new JavaHeapObject[][] {keys, values}; + } + + static class Factory extends ArrayBasedCollectionDescriptor.Factory { + + private final int referentFieldIdx, valueFieldIdx, nextFieldIdx; + + Factory(JavaClass clazz, JavaClass[] implClasses) { + super(clazz, true, ClassUtils.getExactFieldName("size|elementCount", clazz), + ClassUtils.getExactFieldName("table|elementData", clazz), 16, implClasses, null); + // In WeakHashMap.Entry, we don't have the standard 'key' field, because + // it extends WeakReference and uses its 'referent' field as a key. + setMapKeyFieldName("referent"); + + JavaClass entryClass = null; + for (JavaClass implClass : implClasses) { + if (implClass.getName().equals(WEAK_HASH_MAP_ENTRY)) { + entryClass = implClass; + break; + } + } + if (entryClass == null) { + throw new RuntimeException("Could not find class " + WEAK_HASH_MAP_ENTRY); + } + + referentFieldIdx = entryClass.getInstanceFieldIndex("referent"); + valueFieldIdx = entryClass.getInstanceFieldIndex("value"); + + int localNextFieldIdx = -1; + // We have to be very careful with the 'next' field, because both WeakHashMap$Entry + // has its own declared 'next' field, AND its superclass java.lang.Reference + // has its own field with the same name! Here we depend on the fact that fields + // in getFields() are ordered from superclass to subclass + JavaField fieldDescs[] = entryClass.getFieldsForInstance(); + for (int i = fieldDescs.length - 1; i >= 0; i--) { + if ("next".equals(fieldDescs[i].getName())) { + localNextFieldIdx = i; + break; + } + } + int nDefinedFields = entryClass.getDefinedFields().length; + if (localNextFieldIdx == -1 || localNextFieldIdx < fieldDescs.length - nDefinedFields) { + throw new RuntimeException("Could not find field next in " + WEAK_HASH_MAP_ENTRY); + } + + nextFieldIdx = localNextFieldIdx; + } + + private Factory(JavaClass clazz, AbstractArrayBasedCollectionDescriptor.Factory superclassFactory) { + super(clazz, superclassFactory); + Factory f = (Factory) superclassFactory; + this.referentFieldIdx = f.referentFieldIdx; + this.valueFieldIdx = f.valueFieldIdx; + this.nextFieldIdx = f.nextFieldIdx; + } + + @Override + AbstractCollectionDescriptor.Factory cloneForSubclass(JavaClass clazz) { + return new Factory(clazz, this); + } + + @Override + ArrayBasedCollectionDescriptor get(JavaObject col) { + return new WeakHashMapDescriptor(col, this); + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/AbstractJavaHeapObjectVisitor.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/AbstractJavaHeapObjectVisitor.java new file mode 100644 index 00000000..cab41e23 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/AbstractJavaHeapObjectVisitor.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +/* + * The Original Code is HAT. The Initial Developer of the Original Code is Bill Foote, with + * contributions from others at JavaSoft/Sun. + */ + +package org.openjdk.jmc.joverflow.heap.model; + +/** + * A visitor for a JavaThing. @see JavaObject#visitReferencedObjects() + */ +abstract public class AbstractJavaHeapObjectVisitor implements JavaHeapObjectVisitor { + @Override + abstract public void visit(JavaHeapObject other); + + /** + * Should the given field be excluded from the set of things visited? + * + * @return true if it should. + */ + @Override + public boolean exclude(JavaClass clazz, JavaField f) { + return false; + } + + /** + * @return true iff exclude might ever return true + */ + @Override + public boolean mightExclude() { + return false; + } + +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/ArrayTypeCodes.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/ArrayTypeCodes.java new file mode 100644 index 00000000..67af13ad --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/ArrayTypeCodes.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +/* + * The Original Code is HAT. The Initial Developer of the Original Code is Bill Foote, with + * contributions from others at JavaSoft/Sun. + */ + +package org.openjdk.jmc.joverflow.heap.model; + +/** + * Primitive array type codes as defined by VM specification. + */ +public interface ArrayTypeCodes { + // Typecodes for array elements. + // Refer to newarray instruction in VM Spec. + public static final int T_BOOLEAN = 4; + public static final int T_CHAR = 5; + public static final int T_FLOAT = 6; + public static final int T_DOUBLE = 7; + public static final int T_BYTE = 8; + public static final int T_SHORT = 9; + public static final int T_INT = 10; + public static final int T_LONG = 11; +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/CollectionClassProperties.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/CollectionClassProperties.java new file mode 100644 index 00000000..e1c23da7 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/CollectionClassProperties.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +/** + * Provides functionality for querying certain properties of a class that represents a Java + * collection. + */ +public interface CollectionClassProperties { + + /** + * Returns true if this collection class is a map, i.e. it contains two parallel sets of + * objects. If false, the collection is a list of some kind and collects a single set of + * objects. + */ + public boolean isMap(); + + /** + * Returns true if this collection class uses another first-class collection in its + * implementation. For example, HashSet uses HashMap, and so this method returns true for + * HashSet. + */ + public boolean hasOtherCollectionInImpl(); +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/HeapStringReader.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/HeapStringReader.java new file mode 100644 index 00000000..e204dca7 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/HeapStringReader.java @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +/** + * Encapsulates functionality for reading Strings from the heap dump. For better performance, keeps + * and reuses a single internal byte and char array. + */ +public class HeapStringReader implements ImplInclusiveSizeCalculator { + // Indices of fields within a String instance + private final int stringValueIdx, stringOffsetIdx, stringCountIdx; + + private byte[] byteBuf = new byte[200]; + private char[] charBuf = new char[100]; + + private JavaThing[] fields; + private JavaValueArray valueArray; + + HeapStringReader(Snapshot snapshot) { + JavaClass stringClass = snapshot.getClassForName("java.lang.String"); + stringValueIdx = stringClass.getInstanceFieldIndex("value"); + stringOffsetIdx = stringClass.getInstanceFieldIndexOrMinusOne("offset"); + stringCountIdx = stringClass.getInstanceFieldIndexOrMinusOne("count"); + stringClass.setImplInclusiveSizeCalculator(this); + } + + public String readString(JavaObject strObj) { + fields = strObj.getFields(fields); + JavaThing stringValueField = fields[stringValueIdx]; + // Looks like the problem below may occur in heap dumps containing minor + // corruption/incompleteness (some pointers cannot be resolved). HotSpot + // occasionally produces such dumps for unknown reason. + if (stringValueField == null || !(stringValueField instanceof JavaValueArray)) { + return null; + } + + valueArray = (JavaValueArray) stringValueField; + int offset = 0; + int count = valueArray.getLength(); + if (stringOffsetIdx != -1) { // Old-fashioned String implementation with offset/count + offset = ((JavaInt) fields[stringOffsetIdx]).getValue(); + count = ((JavaInt) fields[stringCountIdx]).getValue(); + } + readCharElements(offset, count); + + return new String(charBuf, 0, count); + } + + /** + * Returns JavaValueArray that was obtained internally in the last call to readString(). + */ + public JavaValueArray getLastReadBackingArray() { + return valueArray; + } + + public JavaValueArray getCharArrayForString(JavaObject strObj) { + fields = strObj.getFields(fields); + JavaThing stringValueField = fields[stringValueIdx]; + if (stringValueField == null || !(stringValueField instanceof JavaValueArray)) { + return null; + } + + return (JavaValueArray) stringValueField; + } + + private void readCharElements(int offset, int count) { + if (charBuf.length < count) { + charBuf = new char[count]; + } + + boolean isCompressedString = valueArray.getClazz().isByteArray(); + + int len = valueArray.getLength(); + if (isCompressedString) { + if (byteBuf.length < len) { + byteBuf = new byte[len]; + } + } else { + if (byteBuf.length < len * 2) { + byteBuf = new byte[len * 2]; + } + } + + int byteBufLen = valueArray.readValue(byteBuf); + if (byteBufLen > byteBuf.length) { + throw new RuntimeException("Unexpected length: " + byteBufLen); + } + + if (isCompressedString) { + int byteIdx = offset; + for (int charIdx = 0; charIdx < count; charIdx++) { + charBuf[charIdx] = (char) (byteBuf[byteIdx++] & 0xff); + } + } else { + int byteIdx = offset * 2; + for (int charIdx = 0; charIdx < count; charIdx++) { + int b1 = (byteBuf[byteIdx++] & 0xff); + int b2 = (byteBuf[byteIdx++] & 0xff); + charBuf[charIdx] = (char) ((b1 << 8) + b2); + } + } + } + + @Override + public int calculateImplInclusiveSize(JavaObject javaObj) { + int result = javaObj.getSize(); + JavaValueArray charArray = getCharArrayForString(javaObj); + if (charArray != null) { + result += charArray.getSize(); + } + return result; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/ImplInclusiveSizeCalculator.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/ImplInclusiveSizeCalculator.java new file mode 100644 index 00000000..1d979d67 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/ImplInclusiveSizeCalculator.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +/** + * Defines a method to calculate implementation-inclusive size for the given object, assuming that + * it can be done for an object of that type. See {@link JavaThing#getImplInclusiveSize()} + */ +public interface ImplInclusiveSizeCalculator { + + /** + * Calculates implementation-inclusive size for the given object. See + * {@link JavaThing#getImplInclusiveSize()}. + */ + public int calculateImplInclusiveSize(JavaObject javaObj); + +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaBoolean.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaBoolean.java new file mode 100644 index 00000000..93db15c9 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaBoolean.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +import org.openjdk.jmc.joverflow.util.StringInterner; + +/** + * Represents the value of a boolean field in an instance. + */ +public class JavaBoolean extends JavaValue { + private boolean value; + + public JavaBoolean(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + void setValue(boolean value) { + this.value = value; + } + + @Override + public boolean isZero() { + return value == false; + } + + @Override + public boolean isFloatingPointNumber() { + return false; + } + + @Override + public int getSize() { + return 1; + } + + @Override + public String valueAsString() { + return StringInterner.internString(Boolean.toString(value)); + } + + @Override + public String toString() { + return valueAsString(); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaByte.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaByte.java new file mode 100644 index 00000000..85ba4dc0 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaByte.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +import org.openjdk.jmc.joverflow.util.StringInterner; + +/** + * Represents the value of a byte field in an instance. + */ +public class JavaByte extends JavaValue { + private byte value; + + public JavaByte(byte value) { + this.value = value; + } + + public byte getValue() { + return value; + } + + void setValue(byte value) { + this.value = value; + } + + @Override + public boolean isZero() { + return value == 0; + } + + @Override + public boolean isFloatingPointNumber() { + return false; + } + + @Override + public int getSize() { + return 1; + } + + @Override + public String valueAsString() { + return StringInterner.internString("0x" + Integer.toString((value) & 0xff, 16)); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaChar.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaChar.java new file mode 100644 index 00000000..32109e10 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaChar.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +import org.openjdk.jmc.joverflow.util.StringInterner; + +/** + * Represents the value of a char field in an instance. + */ +public class JavaChar extends JavaValue { + private char value; + + public JavaChar(char value) { + this.value = value; + } + + public char getValue() { + return value; + } + + void setValue(char value) { + this.value = value; + } + + @Override + public boolean isZero() { + return value == 0; + } + + @Override + public boolean isFloatingPointNumber() { + return false; + } + + @Override + public int getSize() { + return 2; + } + + @Override + public String valueAsString() { + return StringInterner.internString(Character.toString(value)); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaClass.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaClass.java new file mode 100644 index 00000000..348872c0 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaClass.java @@ -0,0 +1,941 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +import java.util.ArrayList; + +import org.openjdk.jmc.joverflow.heap.parser.ReadBuffer; +import org.openjdk.jmc.joverflow.util.ClassUtils; +import org.openjdk.jmc.joverflow.util.IntArrayList; +import org.openjdk.jmc.joverflow.util.LongToObjectMap; +import org.openjdk.jmc.joverflow.util.StringInterner; + +/** + * Represents a Java class. Information for classes representing instance types is contained in the + * heap dump. Classes for array types may or may not be present in the heap dump; if some array + * class is not present, it's synthesized by the tool. If there are multiple versions of some class, + * they are chained together + */ +public class JavaClass extends JavaHeapObject { + private static final ArrayList EMPTY_CLASS_LIST = new ArrayList<>(0); + + public static final JavaField[] NO_FIELDS = new JavaField[0]; + public static final JavaThing[] NO_VALUES = new JavaThing[0]; + + /** See {@link JavaHeapObject#isVisited()} */ + private static final int VISITED_MASK = 1 << 31; + + /** Object id for this class */ + private final long id; + /** Index in the class list maintained in Snapshot */ + private int classListIdx; + /** Class name, in dotted format */ + private final String name; + /** Human-friendly name, initialized lazily */ + private String humanFriendlyName; + /** For array classes, it's the number of array dimensions; otherwize 0 */ + private final byte numArrayDimensions; + + // These are JavaObjectRef before resolve + private JavaThing superclass; + private JavaThing loader; + private JavaThing signers; + private JavaThing protectionDomain; + + /** Instance field descriptors */ + private JavaField[] fields; + /** Static field descriptors */ + private JavaField[] staticFields; + /** Static field values */ + private JavaThing[] staticValues; + + /** Subclasses of this class. Trimmed to size or set to singleton empty list if needed */ + private ArrayList subclasses = new ArrayList<>(); + + /** A snapshot that this class and its instances belong to. Set on resolve. */ + private Snapshot snapshot; + + /** + * The next version of this class, or null if this class has only one version. Versions are + * classes with same name but different classloaders. In order to easily distinguish + * multi-version classes from single-version ones, nextVersion is not null even for the last + * class in the version chain - in that case it points to the class itself. + */ + private JavaClass nextVersion; + + /** Version number for this class, starting from 0 */ + private int versionNumber; + + /** Size of instance data fields, in bytes, in the .hprof file */ + private final int fieldsSizeInFile; + + /** Size of an instance, including VM overhead */ + private int instanceSize; + + /** Number of instances of this class; updated during the first (overall stats) pass */ + private int numInstances; + + /** + * Shallow size of all instances of this class. Used only for arrays, since for objects it can + * be economically calculated as instanceSize * numInstances + */ + private long totalShallowInstanceSize; + + /** + * Inclusive size of all instances of this class. Inclusive size is greater than shallow size + * (instanceSize above) for known Collections and Strings, is smaller than shallow size for + * (standalone) char[] arrays, etc. + */ + private long totalInclusiveInstanceSize; + + /** + * If non-null, can be used to calculate implementation-inclusive size for an instance of this + * class. See {@link org.openjdk.jmc.joverflow.heap.model.JavaThing#getImplInclusiveSize()}. + */ + private ImplInclusiveSizeCalculator implInclusiveSizeCalculator; + + /** Total number of fields, including inherited ones */ + private int totalNumFields; + + /** True if this class has any reference fields, false otherwise */ + private boolean hasRefFields; + + /** Can be used for associating an arbitrary object with this JavaClass */ + private Object attachment; + + /** Can be used for storing any information, typically a group of booleans */ + private int flags; + + /** + * Indices of "banned" fields, that should not be scanned. See + * {@link org.openjdk.jmc.joverflow.descriptors.CollectionDescriptors#setBannedFields(Snapshot)} + */ + private IntArrayList bannedFieldIndices; + + /** Non-null if this class is a known collection class */ + private CollectionClassProperties colProperties; + + /** + * Non-zero only for boxed number objects, e.g. java.lang.Integer. Denotes the size of the + * wrapped number in bytes, e.g. 4 for int or 2 for char. + */ + private final byte boxedNumberSize; + + /** "Visited" etc. bits */ + private int tags; + + /** Constructor for classes coming from the heap dump */ + public JavaClass(long id, String name, long superclassId, long loaderId, long signersId, long protDomainId, + JavaField[] fields, JavaField[] staticFields, JavaThing[] staticValues, int fieldsSizeInFile, + int instanceSize) { + this.id = id; + this.name = name; + this.superclass = new JavaObjectRef(superclassId); + this.loader = new JavaObjectRef(loaderId); + this.signers = new JavaObjectRef(signersId); + this.protectionDomain = new JavaObjectRef(protDomainId); + this.fields = fields; + this.staticFields = staticFields; + this.staticValues = staticValues; + this.fieldsSizeInFile = fieldsSizeInFile; + this.instanceSize = instanceSize; + + if (name.startsWith("java.lang.")) { + if (name.equals("java.lang.Integer") || name.equals("java.lang.Float")) { + boxedNumberSize = 4; + } else if (name.equals("java.lang.Long") || name.equals("java.lang.Double")) { + boxedNumberSize = 8; + } else if (name.equals("java.lang.Short") || name.equals("java.lang.Character")) { + boxedNumberSize = 2; + } else if (name.equals("java.lang.Byte") || name.equals("java.lang.Boolean")) { + boxedNumberSize = 1; + } else { + boxedNumberSize = 0; + } + } else { + boxedNumberSize = 0; + } + + int numDimensions = 0; + while (name.charAt(numDimensions) == '[') { + numDimensions++; + } + numArrayDimensions = (byte) numDimensions; + } + + /** Constructor for synthesized classes */ + public JavaClass(String name, long superclassId, long loaderId, long signersId, long protDomainId, + JavaField[] fields, JavaField[] staticFields, JavaThing[] staticValues, int fieldsSizeInFile, + int instanceSize) { + this(-1L, name, superclassId, loaderId, signersId, protDomainId, fields, staticFields, staticValues, + fieldsSizeInFile, instanceSize); + } + + @Override + public final JavaClass getClazz() { + return snapshot.getJavaLangClass(); + } + + @Override + public final int getGlobalObjectIndex() { + return -classListIdx; + } + + void setClassListIdx(int classListIdx) { + this.classListIdx = classListIdx; + } + + /** + * Returns an index of this class in the internal list of classes. This can be used as a unique + * "lightweight" ID for a class. + */ + public int getClassListIdx() { + return classListIdx; + } + + /** Adds the given JavaClass to the chain of versions for this class. */ + public void addNextVersion(JavaClass cls) { + JavaClass curClass = this; + while (curClass.nextVersion != null && curClass.nextVersion != curClass) { + curClass = curClass.nextVersion; + } + curClass.nextVersion = cls; + cls.nextVersion = cls; // Signals that a class with this name has multiple versions + cls.versionNumber = curClass.versionNumber + 1; + } + + public JavaClass getNextVersion() { + return nextVersion == this ? null : nextVersion; + } + + public boolean hasMultipleVersions() { + return nextVersion != null; + } + + /** + * Returns the size of an instance of this class in memory of the JVM that produced the heap + * dump. The size is our best guess. Returned value includes object header size (8 bytes on + * 32-bit JVM, different values on 64-bit JVM), and object alignment (usually at 8-byte + * boundaries). For array types the result is undefined. + */ + public int getInstanceSize() { + return instanceSize; + } + + /** Returns the size in bytes of instance data fields as stored in the .hprof file */ + public int getFieldsSizeInFile() { + return fieldsSizeInFile; + } + + void updateInstanceSize(int newInstanceSize) { + this.instanceSize = newInstanceSize; + } + + /** + * If this class represents a boxed number object, e.g. java.lang.Integer, returns the size in + * bytes of the wrapped number, e.g. 4 for int. + */ + public int getBoxedNumberSize() { + return boxedNumberSize; + } + + public final int getHprofPointerSize() { + return snapshot.getHprofPointerSize(); + } + + public final int getPointerSize() { + return snapshot.getPointerSize(); + } + + public final int getObjectHeaderSize() { + return snapshot.getObjectHeaderSize(); + } + + public final int getArrayHeaderSize() { + return snapshot.getArrayHeaderSize(); + } + + public final int getObjectAlignment() { + return snapshot.getObjectAlignment(); + } + + void resolve(Snapshot snapshot, ArrayList roots) { + if (this.snapshot != null) { + return; + } + this.snapshot = snapshot; + if (!subclasses.isEmpty()) { + subclasses.trimToSize(); + } else { + subclasses = EMPTY_CLASS_LIST; + } + + JavaField[] allFields = getFieldsForInstance(); + for (JavaField field : allFields) { + if (field.isReference()) { + hasRefFields = true; + break; + } + } + + // Dereference via a special method to avoid creating multiple JavaObjects + // for the same classloader ID. + loader = snapshot.dereferenceClassLoader(((JavaObjectRef) loader).getId(), this); + signers = snapshot.dereferenceField(((JavaObjectRef) signers).getId(), null); + protectionDomain = snapshot.dereferenceField(((JavaObjectRef) protectionDomain).getId(), null); + + if (signers != null || protectionDomain != null) { + int len = staticFields.length; + staticValues[len - 2] = signers; + staticValues[len - 1] = protectionDomain; + } + + for (int i = 0; i < staticFields.length; i++) { + JavaField field = staticFields[i]; + JavaThing value = staticValues[i]; + if (value == null) { + continue; + } + if (value instanceof JavaObjectRef) { + long id = ((JavaObjectRef) value).getId(); + value = snapshot.dereferenceField(id, field); + + if (value != null) { + if (value.isHeapAllocated() && getLoader() == null) { + // Static fields are only roots if they are in classes + // loaded by the root classloader. + String s = "Static reference from " + getName() + "." + field.getName(); + roots.add(new Root(id, readId(), Root.JAVA_STATIC, s)); + } + } + staticValues[i] = value; + } + } + } + + /** Resolve the superclass of this class, recursively */ + void resolveSuperclass(LongToObjectMap classIdToJavaClass) { + if (superclass == null) { + // We must be java.lang.Object or a synthetic class, so we have no superclass. + hasRefFields = false; + return; + } + + totalNumFields = fields.length; + if (superclass instanceof JavaObjectRef) { + superclass = classIdToJavaClass.get(((JavaObjectRef) superclass).getId()); + } + if (superclass != null) { + JavaClass sc = (JavaClass) superclass; + sc.resolveSuperclass(classIdToJavaClass); + totalNumFields += sc.totalNumFields; + ((JavaClass) superclass).addSubclass(this); + } + } + + /** + * Returns true if this class has any reference-type fields, defined either in it or in some + * superclass. + */ + public boolean hasReferenceFields() { + return hasRefFields; + } + + public void setAttachment(Object attachment) { + this.attachment = attachment; + } + + public Object getAttachment() { + return attachment; + } + + public void setFlag(int flag) { + flags |= flag; + } + + public boolean flagIsSet(int flag) { + return (flags & flag) != 0; + } + + public boolean isString() { + return snapshot.getJavaLangStringClass() == this; + } + + public boolean isCharArray() { + return snapshot.getCharArrayClass() == this; + } + + public boolean isByteArray() { + return snapshot.getByteArrayClass() == this; + } + + public void incNumInstances() { + numInstances++; + } + + public int getNumInstances() { + return numInstances; + } + + /** + * Updates the total shallow size of all instances of this class. Used only for arrays - for + * objects, we can calculate it by multiplying object size by the number of instances. + */ + public void updateShallowInstanceSize(int size) { + totalShallowInstanceSize += size; + } + + public long getTotalShallowInstanceSize() { + if (isArray()) { + return totalShallowInstanceSize; + } else { + return ((long) getInstanceSize()) * getNumInstances(); + } + } + + /** + * Updates inclusive size of all instances of this class. Inclusive size is greater than shallow + * size for known Collections and Strings, and is smaller than shallow size for (standalone) + * char[] arrays, etc. + */ + public void updateInclusiveInstanceSize(int size) { + totalInclusiveInstanceSize += size; + } + + public long getTotalInclusiveInstanceSize() { + return totalInclusiveInstanceSize; + } + + /** + * Sets indices of "banned" fields in this class, if any. See + * {@link org.openjdk.jmc.joverflow.descriptors.CollectionDescriptors#setBannedFields(Snapshot)} + */ + public void addBannedField(int bannedFieldIndex) { + if (bannedFieldIndices == null) { + bannedFieldIndices = new IntArrayList(2); + } + this.bannedFieldIndices.add(bannedFieldIndex); + for (JavaClass subClass : getSubclasses()) { + subClass.addBannedField(bannedFieldIndex); + } + } + + public static void setFieldBanned(JavaClass clazz, String fieldName) { + if (clazz != null) { + int fieldIdx = clazz.getInstanceFieldIndexOrMinusOne(fieldName); + if (fieldIdx != -1) { + clazz.addBannedField(fieldIdx); + } + } + } + + /** + * Returns indices of "banned" fields, or null if there are no such fields. See + * {@link org.openjdk.jmc.joverflow.descriptors.CollectionDescriptors#setBannedFields(Snapshot)} + */ + public int[] getBannedFieldIndices() { + return bannedFieldIndices == null ? null : bannedFieldIndices.internalArray(); + } + + public boolean isCollection() { + return colProperties != null; + } + + public void setCollectionClassProperties(CollectionClassProperties colProperties) { + this.colProperties = colProperties; + } + + public CollectionClassProperties getCollectionClassProperties() { + return colProperties; + } + + public boolean isCollectionWithOtherCollectionInImpl() { + return (colProperties != null && colProperties.hasOtherCollectionInImpl()); + } + + public boolean isClassLoader() { + return snapshot.getJavaLangClassLoaderClass().isAssignableFrom(this); + } + + public boolean isSameOrHierarchicallyRelated(JavaClass other) { + if (this == other) { + return true; + } + + JavaThing superClass = this.superclass; + while (superClass != null && superClass != other) { + superClass = ((JavaClass) superClass).superclass; + } + if (superClass == other) { + return true; + } + + superClass = other.superclass; + while (superClass != null && superClass != this) { + superClass = ((JavaClass) superClass).superclass; + } + if (superClass == this) { + return true; + } + + return false; + } + + /** + * Returns a JavaField object for the field with the specified number, that should be declared + * in this class. + */ + public JavaField getDeclaredField(int i) { + if (i < 0 || i >= fields.length) { + throw new Error("No field for index " + i + " in class " + name); + } + return fields[i]; + } + + /** + * Get the total number of fields that are part of an instance of this class. That is, include + * superclasses. + */ + public int getNumFieldsForInstance() { + return totalNumFields; + } + + /** + * Get a JavaField object for the field with the specified index. Returned field may be declared + * in this class or in one of its superclasses. + */ + public JavaField getFieldForInstance(int idx) { + if (superclass != null) { + JavaClass sc = (JavaClass) superclass; + if (idx < sc.totalNumFields) { + return sc.getFieldForInstance(idx); + } + idx -= sc.totalNumFields; + } + return getDeclaredField(idx); + } + + /** + * Given the field index, returns this class or a class from this class's superclass chain, that + * declares that field. The field index is a number that could be passed into + * {@link #getFieldForInstance(int)}. + */ + public JavaClass getDeclaringClassForField(int idx) { + if (idx >= totalNumFields) { + return null; + } + + if (superclass != null) { + JavaClass sc = (JavaClass) superclass; + if (idx < sc.totalNumFields) { + return sc.getDeclaringClassForField(idx); + } + } + return this; + } + + /** + * Given the field name, returns this class or a class from this class's superclass chain, that + * declares this field. The field can be both instance and static. + */ + public JavaClass getDeclaringClassForField(String name) { + for (JavaField field : fields) { + if (field.getName().equals(name)) { + return this; + } + } + for (JavaField field : staticFields) { + if (field.getName().equals(name)) { + return this; + } + } + + if (superclass != null) { + return ((JavaClass) superclass).getDeclaringClassForField(name); + } else { + return null; + } + } + + /** + * Returns true if for the two classes fieldIdx represents logically the same field. This will + * be true if clz1 == clz2, or they are hierarchically related, or have a common superclass. + */ + public static boolean isSameField(JavaClass clz1, JavaClass clz2, int fieldIdx) { + if (clz1 == clz2) { + return true; + } + if (clz1.getNumFieldsForInstance() <= fieldIdx || clz2.getNumFieldsForInstance() <= fieldIdx) { + return false; + } + JavaClass superClz1 = clz1.getDeclaringClassForField(fieldIdx); + JavaClass superClz2 = clz2.getDeclaringClassForField(fieldIdx); + return (superClz1 == superClz2); + } + + @Override + public long readId() { + return id; + } + + public String getName() { + return name; + } + + /** + * Same as + * {@link org.openjdk.jmc.joverflow.util.ClassUtils#getShortNameForPopularClass(String)}, but + * also: - makes names for anonymous classes, like MyFooClass$6, more informative, by adding + * "(SuperClassName)" to them; - For array classes, returns a human-friendly name, such as + * "boolean[]" instead of "[B" or "Object[]" instead of "[Ljava.lang.Object;" + */ + public String getHumanFriendlyName() { + if (humanFriendlyName == null) { + String className = getName(); + StringBuilder resultBuf = new StringBuilder(className.length() + 10); + if (isArray()) { + int numDims = getNumArrayDimensions(); + if (isAnyDimPrimitiveArray()) { + resultBuf.append(JavaValueArray.getElementTypeName(className.charAt(className.length() - 1))); + } else { + resultBuf.append(ClassUtils + .getShortNameForPopularClass(className.substring(numDims + 1, className.length() - 1))); + } + for (int i = 0; i < numDims; i++) { + resultBuf.append("[]"); + } + } else { + resultBuf.append(ClassUtils.getShortNameForPopularClass(className)); + // Now deal with anonymous inner classes - their names by themselves are pretty useless + int dollarIdx = resultBuf.indexOf("$"); + if (dollarIdx != -1 && dollarIdx != resultBuf.length() - 1) { + int nextCharAfterDollar = resultBuf.charAt(dollarIdx + 1); + if (Character.isDigit(nextCharAfterDollar)) { + // Anonymous class + resultBuf.append(" (extends "); + resultBuf.append(ClassUtils.getShortNameForPopularClass(getSuperclass().getName())); + resultBuf.append(')'); + } + } + } + + humanFriendlyName = StringInterner.internString(resultBuf.toString()); + } + return humanFriendlyName; + } + + /** + * Same as {@link #getHumanFriendlyName()}, but additionally, for classes with multiple + * versions, appends the loader id in the end of the returned string. + */ + public String getHumanFriendlyNameWithLoaderIfNeeded() { + String name = getHumanFriendlyName(); + if (hasMultipleVersions()) { + name += " loader " + (loader != null ? loader.valueAsString() : "null"); + } + return name; + } + + public boolean isArray() { + return numArrayDimensions > 0; + } + + /** + * Returns true if this class represents a single-dimension primitive array. + */ + boolean isSingleDimPrimitiveArray() { + return isArray() && name.length() == 2; + } + + /** + * Returns true if this class represents an array with primitive elements and any number of + * dimensions. Note that multi-dimensional arrays are technically not primitive. However, this + * method is needed to distinguish arrays that are presented differently in print, thus any + * array with primitive ulimate elements is "primitive" for its purposes. + */ + public boolean isAnyDimPrimitiveArray() { + return isArray() && name.charAt(name.length() - 2) == '['; + } + + public int getNumArrayDimensions() { + return numArrayDimensions; + } + + public ArrayList getSubclasses() { + return subclasses; + } + + /** This can only safely be called after resolve() */ + public JavaClass getSuperclass() { + return (JavaClass) superclass; + } + + /** + * This can only safely be called after resolve(). May return an UnresolvedObject, thus the + * return type is JavaThing rather than JavaObject. + */ + public JavaThing getLoader() { + return loader; + } + + /** This can only safely be called after resolve() */ + public boolean isBootstrap() { + return loader == null; + } + + /** This can only safely be called after resolve() */ + public JavaThing getSigners() { + return signers; + } + + /** This can only safely be called after resolve() */ + public JavaThing getProtectionDomain() { + return protectionDomain; + } + + /** Returns the fields defined in this class */ + public JavaField[] getDefinedFields() { + return fields; + } + + /** + * Returns all the fields in an instance of this class, including superclass fields. Fields in + * the returned array are ordered "naturally", i.e. from the topmost superclass down. + */ + public JavaField[] getFieldsForInstance() { + ArrayList v = new ArrayList<>(totalNumFields); + addFields(v); + if (v.isEmpty()) { + return NO_FIELDS; + } else { + return v.toArray(new JavaField[v.size()]); + } + } + + /** + * For the given field name, returns the index of the field in the instance (that is, in the + * array returned by JavaClass.getDefinedFields()). If there is no field with the given name, + * throws a RuntimeException. + */ + public int getInstanceFieldIndex(String fieldName) { + int result = getInstanceFieldIndexOrMinusOne(fieldName); + if (result == -1) { + throw new RuntimeException(ClassUtils.getMessageForMissingField(this, fieldName)); + } + return result; + } + + /** + * For the given field name, returns the index of the field in the instance (that is, in the + * array returned by JavaObject.getDefinedFields()). If there is no field with the given name, + * returns -1. Search in the fields is done from the topmost superclass to subclasses. + */ + public int getInstanceFieldIndexOrMinusOne(String fieldName) { + JavaField fieldDescs[] = getFieldsForInstance(); + for (int i = 0; i < fieldDescs.length; i++) { + if (fieldName.equals(fieldDescs[i].getName())) { + return i; + } + } + return -1; + } + + public JavaField[] getStaticFields() { + return staticFields; + } + + public JavaThing[] getStaticValues() { + return staticValues; + } + + /** + * Returns value of the static field with the given name. If the field with the given name is + * not found, returns null. + */ + public JavaThing getStaticField(String name) { + for (int i = 0; i < staticFields.length; i++) { + if (staticFields[i].getName().equals(name)) { + return staticValues[i]; + } + } + return null; + } + + // Use Comparator instead of implementing Comparable if sorting is needed +// @Override +// public int compareTo(JavaThing other) { +// if (other instanceof JavaClass) { +// return name.compareTo(((JavaClass) other).name); +// } +// return super.compareTo(other); +// } + + /** + * Returns true iff a variable of this type is assignable from an instance of other. In other + * words, if this class is the same or a superclass of other. + */ + public boolean isAssignableFrom(JavaClass other) { + if (this == other) { + return true; + } else if (other == null) { + return false; + } else { + return isAssignableFrom((JavaClass) other.superclass); + // Trivial tail recursion: I have faith in javac. + } + } + + public boolean isOrSubclassOf(String name) { + JavaClass clazz = this; + while (clazz != null) { + if (clazz.getName().equals(name)) { + return true; + } + clazz = clazz.getSuperclass(); + } + return false; + } + + @Override + /** Note: size and impl-inclusive size for Java class itself are pretty useless */ + public int getSize() { + JavaClass cl = snapshot.getJavaLangClass(); + if (cl == null) { + return 0; + } else { + return cl.getInstanceSize(); + } + } + + @Override + /** Note: size and impl-inclusive size for Java class itself are pretty useless */ + public final int getImplInclusiveSize() { + return getSize(); + } + + public void setImplInclusiveSizeCalculator(ImplInclusiveSizeCalculator implCalc) { + this.implInclusiveSizeCalculator = implCalc; + } + + public ImplInclusiveSizeCalculator getImplInclusiveSizeCalculator() { + return implInclusiveSizeCalculator; + } + + @Override + public String valueAsString() { + return StringInterner + .internString("class " + name + (hasMultipleVersions() ? " loader " + loader.valueAsString() : "")); + } + + public final Snapshot getSnapshot() { + return snapshot; + } + + @Override + public boolean isVisited() { + return (tags & VISITED_MASK) != 0; + } + + @Override + public void setVisited() { + tags |= VISITED_MASK; + } + + @Override + public boolean setVisitedIfNot() { + if (isVisited()) { + return false; + } + setVisited(); + return true; + } + + @Override + public String toString() { + return valueAsString(); + } + + @Override + public void visitReferencedObjects(JavaHeapObjectVisitor v) { + super.visitReferencedObjects(v); + JavaHeapObject sc = getSuperclass(); + if (sc != null) { + v.visit(getSuperclass()); + } + + JavaThing other; + other = getLoader(); + if (other instanceof JavaHeapObject) { + v.visit((JavaHeapObject) other); + } + other = getSigners(); + if (other instanceof JavaHeapObject) { + v.visit((JavaHeapObject) other); + } + other = getProtectionDomain(); + if (other instanceof JavaHeapObject) { + v.visit((JavaHeapObject) other); + } + + for (int i = 0; i < staticFields.length; i++) { + JavaField f = staticFields[i]; + if (!v.exclude(this, f) && f.isReference()) { + other = staticValues[i]; + if (other instanceof JavaHeapObject) { + v.visit((JavaHeapObject) other); + } + } + } + } + + // package-privates below this point + + final ReadBuffer getReadBuffer() { + return snapshot.getReadBuffer(); + } + + // Internals only below this point + + private void addFields(ArrayList v) { + if (superclass != null) { + ((JavaClass) superclass).addFields(v); + } + for (JavaField field : fields) { + v.add(field); + } + } + + private void addSubclass(JavaClass sub) { + subclasses.add(sub); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaDouble.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaDouble.java new file mode 100644 index 00000000..acd72cb5 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaDouble.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +import org.openjdk.jmc.joverflow.util.StringInterner; + +/** + * Represents the value of a double field in an instance. + */ +public class JavaDouble extends JavaValue { + private double value; + + public JavaDouble(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + void setValue(double value) { + this.value = value; + } + + @Override + public boolean isZero() { + // The doubleToLongBits() call below is a workaround, since a simple + // == 0.0 check may crash the VM (at least JDK 1.6.0_29 on Mac) if + // value is NaN. + return (Double.doubleToLongBits(value) & 0x7FFFFFFFFFFFFFFFL) == 0; + } + + @Override + public boolean isFloatingPointNumber() { + return true; + } + + @Override + public int getSize() { + return 8; + } + + @Override + public String valueAsString() { + return StringInterner.internString(Double.toString(value)); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaField.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaField.java new file mode 100644 index 00000000..2df1b03a --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaField.java @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +/** + * Data field descriptor. Provides the field name, but does not distinguish between different object + * types - a deficiency that comes from HPROF heap dump format. + */ +public class JavaField { + private final String name; + private final char typeId; // 'I', 'B' etc. for primitive types, 'L' for classes and '[' for arrays + private final byte sizeInInstance; + + private static final JavaField[] STATIC_QUAZI_FIELDS = new JavaField[3]; + static { + STATIC_QUAZI_FIELDS[0] = newInstance("", '[', 0); + STATIC_QUAZI_FIELDS[1] = newInstance("", 'L', 0); + } + + public static JavaField newInstance(String name, char typeId, int pointerSize) { + byte sizeInInstance = 0; + if (typeId == 'L' || typeId == '[') { + sizeInInstance = (byte) pointerSize; + } else { + switch (typeId) { + case 'I': + case 'F': + sizeInInstance = 4; + break; + case 'Z': + case 'B': + sizeInInstance = 1; + break; + case 'S': + case 'C': + sizeInInstance = 2; + break; + case 'J': + case 'D': + sizeInInstance = 8; + break; + } + } + + return new JavaField(name, typeId, sizeInInstance); + } + + private JavaField(String name, char typeId, byte sizeInInstance) { + this.name = name; + this.typeId = typeId; + this.sizeInInstance = sizeInInstance; + } + + public boolean isReference() { + return (typeId == '[' || typeId == 'L'); + } + + public String getName() { + return name; + } + + public char getTypeId() { + return typeId; + } + + public int getSizeInInstance() { + return sizeInInstance; + } + + /** + * Adds two "quazi-fields" for data that exists in any Class object in addition to the normal + * static fields. This is done to enable our heap scanner to follow these references. + */ + public static void addStaticQuaziFields(JavaField[] staticFields) { + System.arraycopy(STATIC_QUAZI_FIELDS, 0, staticFields, staticFields.length - 2, 2); + } + + // Debugging + + @Override + public String toString() { + return Character.toString(typeId) + ' ' + name + " sizeInInstance = " + sizeInInstance; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaFloat.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaFloat.java new file mode 100644 index 00000000..b9ab0027 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaFloat.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +import org.openjdk.jmc.joverflow.util.StringInterner; + +/** + * Represents the value of a float field in an instance. + */ +public class JavaFloat extends JavaValue { + private float value; + + public JavaFloat(float value) { + this.value = value; + } + + public float getValue() { + return value; + } + + void setValue(float value) { + this.value = value; + } + + @Override + public boolean isZero() { + return value == 0.0f; + } + + @Override + public boolean isFloatingPointNumber() { + return true; + } + + @Override + public int getSize() { + return 4; + } + + @Override + public String valueAsString() { + return StringInterner.internString(Float.toString(value)); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaHeapObject.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaHeapObject.java new file mode 100644 index 00000000..3897889d --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaHeapObject.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +import org.openjdk.jmc.joverflow.util.MiscUtils; +import org.openjdk.jmc.joverflow.util.StringInterner; + +/** + * Represents an object that's allocated out of the Java heap. It can be a JavaClass, a + * JavaObjectArray, a JavaValueArray or a JavaObject. + */ +public abstract class JavaHeapObject extends JavaThing { + + /** + * Returns true if this object has been visited during detailed analysis. Uses the tags field in + * the object. After the object is marked visited, it cannot be scanned again. Depth-first and + * breadth-first heap scan algorithms fundamentally depend on this. + *

+ * Long ago this info was kept in a separate 'IdentitySetOfObjects visited'. Replacing it with a + * bit directly in the objects resulted in about 15% improvement in both performance and memory + * usage. + */ + public abstract boolean isVisited(); + + /** @see #isVisited() */ + public abstract void setVisited(); + + /** + * Sets this object's "visited" tag. Returns true if it has not been set before, and false if + * this object has already been visited. + */ + public abstract boolean setVisitedIfNot(); + + public abstract JavaClass getClazz(); + + /** + * Returns the object's global index. This index is not equal to the object id returned by + * {@link #readId()}. Each JavaLazyReadObject (representing a Java instance, object array or + * primitive array) has a unique index that is > 0. Each JavaClass (that represents a Java + * class) has a unique index that's <= 0. The value returned for JavaClass is an index into + * the internal class list, and thus increments by one. The value returned for a JavaHeapObject + * is a position in the internal compact table, and increments by 3 or 4. In contrast, the long + * object id normally increments by comparatively large numbers. + */ + public abstract int getGlobalObjectIndex(); + + /** + * Returns the object's heap ID (which is the value of that's object address in the machine + * memory at the time when the heap dump was taken; don't confuse it with the "internal id" + * returned by {@link JavaLazyReadObject#getInternalId()} - the latter is the object's offset in + * heap dump file. Note: for non-class objects, this is done by reading the ID from the mmapped + * file! + */ + public abstract long readId(); + + @Override + public String idAsString() { + return StringInterner.internString(getClazz().getHumanFriendlyName() + '@' + MiscUtils.toHex(readId())); + } + + @Override + public String toString() { + return idAsString(); + } + + /** + * Tell the visitor about all of the objects we refer to. + */ + public void visitReferencedObjects(JavaHeapObjectVisitor v) { + v.visit(getClazz()); + } + + @Override + public boolean isHeapAllocated() { + return true; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaHeapObjectVisitor.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaHeapObjectVisitor.java new file mode 100644 index 00000000..4fccb468 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaHeapObjectVisitor.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +/* + * The Original Code is HAT. The Initial Developer of the Original Code is Bill Foote, with + * contributions from others at JavaSoft/Sun. + */ + +package org.openjdk.jmc.joverflow.heap.model; + +/** + * A visitor for a JavaThing. @see JavaObject#visitReferencedObjects() + */ +public interface JavaHeapObjectVisitor { + public void visit(JavaHeapObject other); + + /** + * Should the given field be excluded from the set of things visited? + * + * @return true if it should. + */ + public boolean exclude(JavaClass clazz, JavaField f); + + /** + * @return true iff exclude might ever return true + */ + public boolean mightExclude(); +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaInt.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaInt.java new file mode 100644 index 00000000..d0795092 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaInt.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +import org.openjdk.jmc.joverflow.util.StringInterner; + +/** + * Represents the value of an int field in an instance. + */ +public class JavaInt extends JavaValue { + private int value; + + public JavaInt(int value) { + this.value = value; + } + + public int getValue() { + return value; + } + + void setValue(int value) { + this.value = value; + } + + @Override + public boolean isZero() { + return value == 0; + } + + @Override + public boolean isFloatingPointNumber() { + return false; + } + + @Override + public int getSize() { + return 4; + } + + @Override + public String valueAsString() { + return StringInterner.internString(Integer.toString(value)); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaLazyReadObject.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaLazyReadObject.java new file mode 100644 index 00000000..40e805ba --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaLazyReadObject.java @@ -0,0 +1,282 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +import java.io.IOException; + +import org.openjdk.jmc.joverflow.heap.parser.DumpCorruptedException; +import org.openjdk.jmc.joverflow.heap.parser.ReadBuffer; + +/* + * Base class for lazily read Java heap objects (JavaObject, JavaObjectArray and JavaValueArray). + * Contains a long offset into the heap dump, at which the contents of this object are located. Also + * contains a pointer back into an array in JavaObjectTable where the data from which this object + * gets created is contained. + */ +public abstract class JavaLazyReadObject extends JavaHeapObject { + + /** See {@link JavaHeapObject#isVisited()} */ + private static final int VISITED_MASK = 1 << 31; + + /** See {@link #isVisitedAsCollectionImpl()} */ + private static final int VISITED_COLLECTION_IMPL_MASK = 1 << 30; + + /** See {@link #isVisitedAsOther()} */ + private static final int VISITED_OTHER = 1 << 29; + + /** + * Mask that denotes the number of bits available for assigning an "internal id" to the object. + * These are 32 bits minus the above bits (various "visited" etc.) + */ + protected static final int INTERNAL_ID_MASK = 0x1FFFFFFF; + + /** JavaClass for this object */ + protected final JavaClass clazz; + + /** File offset from which this object data starts */ + private final long objOfsInFile; + + /** Data chunk in JavaObjectTable where data for this object is stored */ + private final int[] dataChunk; + + /** Offset in dataChunk at which the data for this object is stored */ + private final int startPosInChunk; + + private final int globalObjectIndex; + + protected JavaLazyReadObject(JavaClass clazz, long objOfsInFile, int[] dataChunk, int startPosInChunk, + int globalObjectIndex) { + this.clazz = clazz; + this.objOfsInFile = objOfsInFile; + this.dataChunk = dataChunk; + this.startPosInChunk = startPosInChunk; + this.globalObjectIndex = globalObjectIndex; + } + + @Override + public final JavaClass getClazz() { + return clazz; + } + + /** + * Returns the starting offset of this object in the heap dump file. This is internal + * information needed to read object contents, but it can also be used as an object's unique + * identifier. + */ + public final long getObjOfsInFile() { + return objOfsInFile; + } + + @Override + public final int getGlobalObjectIndex() { + return globalObjectIndex; + } + + /** + * Reads this object's content from mmapped file and returns it as byte array + */ + public final byte[] getValue() { + try { + return readValue(); + } catch (IOException ex) { + throw new DumpCorruptedException.Runtime( + "lazy read failed at offset " + objOfsInFile + " with exception " + ex); + } + } + + @Override + public final long readId() { + return readId(clazz.getReadBuffer(), clazz.getHprofPointerSize()); + } + + final long readId(ReadBuffer readBuf, int hprofPointerSize) { + try { + if (hprofPointerSize == 4) { + return (readBuf.getInt(objOfsInFile)) & Snapshot.SMALL_ID_MASK; + } else { + return readBuf.getLong(objOfsInFile); + } + } catch (IOException ex) { + throw new DumpCorruptedException.Runtime( + "lazy read failed at offset " + objOfsInFile + " with exception " + ex); + } + } + + /** + * Returns true if this object has been visited during detailed analysis. Uses the tags field in + * the object. After the object is marked visited, it cannot be scanned again. + */ + @Override + public boolean isVisited() { + int tagsPos = startPosInChunk + 2; + return (dataChunk[tagsPos] & VISITED_MASK) != 0; + } + + /** @see #isVisited() */ + @Override + public void setVisited() { + int tagsPos = startPosInChunk + 2; + dataChunk[tagsPos] |= VISITED_MASK; + } + + /** + * Sets this object's "visited" tag. Returns true if it has not been set before, and false if + * this object has already been visited. + */ + @Override + public boolean setVisitedIfNot() { + if (isVisited()) { + return false; + } + setVisited(); + return true; + } + + static boolean isVisited(int tagWord) { + return (tagWord & VISITED_MASK) != 0; + } + + /** + * Returns true if this object has been visited as a part of collection implementation, when + * calculating its size. It signals that the object is not a standalone one. In some situations + * this is crucially important, for example when we sort char[] arrays into those that back + * Strings and those that are standalone (independent). + */ + public boolean isVisitedAsCollectionImpl() { + int tagsPos = startPosInChunk + 2; + return (dataChunk[tagsPos] & VISITED_COLLECTION_IMPL_MASK) != 0; + } + + /** @see #isVisitedAsCollectionImpl() */ + public void setVisitedAsCollectionImpl() { + int tagsPos = startPosInChunk + 2; + dataChunk[tagsPos] |= VISITED_COLLECTION_IMPL_MASK; + } + + /** + * Returns true if this object has been visited as part of some secondary operation - not the + * general visit (isVisited()), and not a visit as a part of known collection + * (isVisitedAsCollectionImpl()). Parts of the tool with non-overlapping functionality are free + * to use this for their own purposes, for example to avoid counting the same boxed Number + * referenced from an Object[] array multiple times. + */ + public boolean isVisitedAsOther() { + int tagsPos = startPosInChunk + 2; + return (dataChunk[tagsPos] & VISITED_OTHER) != 0; + } + + /** @see #isVisitedAsOther() */ + public void setVisitedAsOther() { + int tagsPos = startPosInChunk + 2; + dataChunk[tagsPos] |= VISITED_OTHER; + } + + /** + * Returns the internal id for this object, that should been previously set by + * {@link #setInternalId(int)}. So far these ids are used to handle duplicate Strings and + * arrays. Separate String or array objects with the same logical value are assigned the same + * id. Each String or array with a different logical value has a different id. + */ + public int getInternalId() { + int tagsPos = startPosInChunk + 2; + return dataChunk[tagsPos] & INTERNAL_ID_MASK; + } + + /** See {@link #getInternalId()} */ + public void setInternalId(int id) { + int tagsPos = startPosInChunk + 2; + dataChunk[tagsPos] |= id; + } + + protected abstract byte[] readValue() throws IOException; + + /** Reads object ID from given index from given byte array */ + protected final long objectIdAt(int index, byte[] data) { + int idSize = getClazz().getHprofPointerSize(); + if (idSize == 4) { + return (intAt(index, data)) & Snapshot.SMALL_ID_MASK; + } else { + return longAt(index, data); + } + } + + // utility methods to read primitive types from byte array + + protected static byte byteAt(int index, byte[] value) { + return value[index]; + } + + protected static boolean booleanAt(int index, byte[] value) { + return (value[index] & 0xff) != 0; + } + + protected static char charAt(int index, byte[] value) { + int b1 = (value[index++] & 0xff); + int b2 = (value[index] & 0xff); + return (char) ((b1 << 8) + b2); + } + + protected static short shortAt(int index, byte[] value) { + int b1 = (value[index++] & 0xff); + int b2 = (value[index] & 0xff); + return (short) ((b1 << 8) + b2); + } + + protected static int intAt(int index, byte[] value) { + int b1 = (value[index++] & 0xff); + int b2 = (value[index++] & 0xff); + int b3 = (value[index++] & 0xff); + int b4 = (value[index] & 0xff); + return ((b1 << 24) + (b2 << 16) + (b3 << 8) + b4); + } + + protected static long longAt(int index, byte[] value) { + long val = 0; + for (int j = 0; j < 8; j++) { + val = val << 8; + int b = (value[index++]) & 0xff; + val |= b; + } + return val; + } + + protected static float floatAt(int index, byte[] value) { + int val = intAt(index, value); + return Float.intBitsToFloat(val); + } + + protected static double doubleAt(int index, byte[] value) { + long val = longAt(index, value); + return Double.longBitsToDouble(val); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaLong.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaLong.java new file mode 100644 index 00000000..63dd18ae --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaLong.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +import org.openjdk.jmc.joverflow.util.StringInterner; + +/** + * Represents the value of a long field in an instance. + */ +public class JavaLong extends JavaValue { + private long value; + + public JavaLong(long value) { + this.value = value; + } + + public long getValue() { + return value; + } + + void setValue(long value) { + this.value = value; + } + + @Override + public boolean isZero() { + return value == 0L; + } + + @Override + public boolean isFloatingPointNumber() { + return false; + } + + @Override + public int getSize() { + return 8; + } + + @Override + public String valueAsString() { + return StringInterner.internString(Long.toString(value)); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaObject.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaObject.java new file mode 100644 index 00000000..f93f998e --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaObject.java @@ -0,0 +1,363 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +import java.io.IOException; + +import org.openjdk.jmc.joverflow.heap.parser.ReadBuffer; +import org.openjdk.jmc.joverflow.util.MiscUtils; +import org.openjdk.jmc.joverflow.util.StringInterner; + +/** + * Represents a Java instance. + */ +public class JavaObject extends JavaLazyReadObject { + + public JavaObject(JavaClass clazz, long objOfsInFile, int[] dataChunk, int startPosInChunk, int globalObjectIndex) { + super(clazz, objOfsInFile, dataChunk, startPosInChunk, globalObjectIndex); + } + + /** + * Returns the total size of this object in the heap. That is a sum of object's data (workload) + * size plus the size of the object header. + */ + @Override + public final int getSize() { + return getClazz().getInstanceSize(); + } + + @Override + public final int getImplInclusiveSize() { + ImplInclusiveSizeCalculator implSizeCalc = getClazz().getImplInclusiveSizeCalculator(); + if (implSizeCalc != null) { + return implSizeCalc.calculateImplInclusiveSize(this); + } else { + return getSize(); + } + } + + /** + * Are we the same type as other? We are iff our clazz is the same type as other's. + */ + @Override + public boolean isSameTypeAs(JavaThing other) { + if (!(other instanceof JavaObject)) { + return false; + } + JavaObject oo = (JavaObject) other; + return getClazz().equals(oo.getClazz()); + } + + /** Returns all of the fields of this instance */ + public JavaThing[] getFields() { + return parseFields(getValue(), true, null); + } + + /** + * Puts all of the fields of this instance into the supplied array, and returns it. The array + * should either be one previously returned by {@link #getFields()}, or be null, in which case a + * new array is created. When the supplied array is reused, its elements which represent + * primitive fields are reused as well. + */ + public JavaThing[] getFields(JavaThing[] fields) { + return parseFields(getValue(), true, fields); + } + + /** + * Returns an array with as many slots as the number of fields in this instance, where primitive + * fields may or may not be initialized, depending on the parameter value. If setPrimitiveFields + * == false, nulls are returned for primitive fields. This is a performance optimization for the + * case when the caller needs only references. + */ + public JavaThing[] getFields(boolean setPrimitiveFields) { + return parseFields(getValue(), setPrimitiveFields, null); + } + + /** + * Returns the value of field of given name, or null if a field with this name doesn't exist. + */ + public JavaThing getField(String name) { + JavaThing[] flds = getFields(); + JavaField[] instFields = getClazz().getFieldsForInstance(); + for (int i = 0; i < instFields.length; i++) { + if (instFields[i].getName().equals(name)) { + return flds[i]; + } + } + return null; + } + + /** + * Returns the field with the specified index in the array that getFields() produces. + */ + public JavaThing getField(int idx) { + // Note: I tried to reuse an array here, attached to JavaClass.FieldStats, + // but that didn't speedup things - actually, slowed them down a little. + JavaThing[] fields = getFields(); + return fields[idx]; + } + + // Use Comparator instead of implementing Comparable if sorting is needed +// @Override +// public int compareTo(JavaThing other) { +// if (other instanceof JavaObject) { +// JavaObject oo = (JavaObject) other; +// return getClazz().getName().compareTo(oo.getClazz().getName()); +// } +// return super.compareTo(other); +// } + + @Override + public void visitReferencedObjects(JavaHeapObjectVisitor v) { + JavaThing[] flds = getFields(); + for (int i = 0; i < flds.length; i++) { + if (flds[i] != null) { + if (v.mightExclude() + && v.exclude(getClazz().getDeclaringClassForField(i), getClazz().getFieldForInstance(i))) { + // skip it + } else if (flds[i] instanceof JavaHeapObject) { + v.visit((JavaHeapObject) flds[i]); + } + } + } + } + + /** + * Describe the reference that this thing has to target. This will only be called if target is + * in the array returned by getChildrenForRootset. + */ + public String describeReferenceTo(JavaHeapObject target) { + JavaThing[] flds = getFields(); + for (int i = 0; i < flds.length; i++) { + if (flds[i] == target) { + JavaField f = getClazz().getFieldForInstance(i); + return "." + f.getName(); + } + } + throw new IllegalArgumentException(this + " does not refer to " + target); + } + + @Override + public String valueAsString() { + if (getClazz().isString()) { + String s = getClazz().getSnapshot().getStringReader().readString(this); + if (s != null) { + return StringInterner.internString(MiscUtils.removeEndLinesAndAddQuotes(s, 0)); + } else { + // This is actually more likely an unresolved string in a heap dump with + // minor corruption/incompleteness. They are occasionally produced by + // HotSpot for unknown reason. + return "null"; + } + } else { + // TODO: eventually may want to e.g. list field values here + return idAsString(); + } + } + + @Override + public String toString() { + return valueAsString(); + } + + // Internals only below this point + + @Override + protected final byte[] readValue() throws IOException { + JavaClass clazz = getClazz(); + int length = clazz.getFieldsSizeInFile(); + if (length == 0) { + return Snapshot.EMPTY_BYTE_ARRAY; + } + int idSize = clazz.getHprofPointerSize(); + ReadBuffer buf = clazz.getReadBuffer(); + // Skip this object's id, class id, stack trace and object length + long offset = getObjOfsInFile() + 2 * idSize + 8; + byte[] res = new byte[length]; + buf.get(offset, res); + return res; + } + + private JavaThing[] parseFields(byte[] data, boolean setPrimitiveFields, JavaThing[] fieldValues) { + JavaClass cl = getClazz(); + int target = cl.getNumFieldsForInstance(); + boolean reusingFieldArray = false; + if (fieldValues == null || fieldValues.length != target) { + fieldValues = new JavaThing[target]; + } else { + reusingFieldArray = true; + } + Snapshot snapshot = cl.getSnapshot(); + int idSize = snapshot.getHprofPointerSize(); + int fieldNo = 0; + + // In the dump file, the fields are stored in this order: + // fields of most derived class (immediate class) are stored + // first and then the super class and so on. In this object, + // fields are stored in the reverse ("natural") order. i.e., + // fields of most super class are stored first. + + // target variable is used to compensate for the fact that + // the dump file starts field values from the leaf working + // upwards in the inheritance hierarchy, whereas JavaObject + // starts with the top of the inheritance hierarchy and works down. + JavaField[] fields = cl.getDefinedFields(); + target -= fields.length; + JavaClass currClass = cl; + int index = 0; + + for (int i = 0; i < fieldValues.length; i++, fieldNo++) { + while (fieldNo >= fields.length) { + currClass = currClass.getSuperclass(); + fields = currClass.getDefinedFields(); + fieldNo = 0; + target -= fields.length; + } + + JavaField f = fields[fieldNo]; + int fieldValueIdx = target + fieldNo; + char sig = f.getTypeId(); + if (sig == 'L' || sig == '[') { + long id = objectIdAt(index, data); + index += idSize; + fieldValues[fieldValueIdx] = snapshot.dereferenceField(id, f); + } else if (setPrimitiveFields) { + switch (sig) { + case 'Z': { + byte value = byteAt(index, data); + if (reusingFieldArray) { + ((JavaBoolean) fieldValues[fieldValueIdx]).setValue(value != 0); + } else { + fieldValues[fieldValueIdx] = new JavaBoolean(value != 0); + } + index++; + break; + } + case 'B': { + byte value = byteAt(index, data); + if (reusingFieldArray) { + ((JavaByte) fieldValues[fieldValueIdx]).setValue(value); + } else { + fieldValues[fieldValueIdx] = new JavaByte(value); + } + index++; + break; + } + case 'S': { + short value = shortAt(index, data); + if (reusingFieldArray) { + ((JavaShort) fieldValues[fieldValueIdx]).setValue(value); + } else { + fieldValues[fieldValueIdx] = new JavaShort(value); + } + index += 2; + break; + } + case 'C': { + char value = charAt(index, data); + if (reusingFieldArray) { + ((JavaChar) fieldValues[fieldValueIdx]).setValue(value); + } else { + fieldValues[fieldValueIdx] = new JavaChar(value); + } + index += 2; + break; + } + case 'I': { + int value = intAt(index, data); + if (reusingFieldArray) { + ((JavaInt) fieldValues[fieldValueIdx]).setValue(value); + } else { + fieldValues[fieldValueIdx] = new JavaInt(value); + } + index += 4; + break; + } + case 'J': { + long value = longAt(index, data); + if (reusingFieldArray) { + ((JavaLong) fieldValues[fieldValueIdx]).setValue(value); + } else { + fieldValues[fieldValueIdx] = new JavaLong(value); + } + index += 8; + break; + } + case 'F': { + float value = floatAt(index, data); + if (reusingFieldArray) { + ((JavaFloat) fieldValues[fieldValueIdx]).setValue(value); + } else { + fieldValues[fieldValueIdx] = new JavaFloat(value); + } + index += 4; + break; + } + case 'D': { + double value = doubleAt(index, data); + if (reusingFieldArray) { + ((JavaDouble) fieldValues[fieldValueIdx]).setValue(value); + } else { + fieldValues[fieldValueIdx] = new JavaDouble(value); + } + index += 8; + break; + } + default: + throw new RuntimeException("invalid signature: " + sig); + } + } else { + // setPrimitiveFields == false + switch (sig) { + case 'I': + case 'F': + index += 4; + break; + case 'Z': + case 'B': + index++; + break; + case 'S': + case 'C': + index += 2; + break; + case 'J': + case 'D': + index += 8; + break; + } + } + } + return fieldValues; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaObjectArray.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaObjectArray.java new file mode 100644 index 00000000..b67f8d2f --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaObjectArray.java @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +import java.io.IOException; +import java.nio.BufferUnderflowException; + +import org.openjdk.jmc.joverflow.heap.parser.DumpCorruptedException; +import org.openjdk.jmc.joverflow.heap.parser.ReadBuffer; +import org.openjdk.jmc.joverflow.util.MiscUtils; +import org.openjdk.jmc.joverflow.util.StringInterner; + +/** + * Represents a Java object array. + */ +public class JavaObjectArray extends JavaLazyReadObject { + private final int length; + + public JavaObjectArray(JavaClass clazz, long objOfsInFile, int length, int[] dataChunk, int startPosInChunk, + int globalObjectIndex) { + super(clazz, objOfsInFile, dataChunk, startPosInChunk, globalObjectIndex); + this.length = length; + } + + /** + * Returns the total size of this object in the heap. That is the sum of object's data + * (workload) size plus all the VM overhead: size of the object header plus the size of array + * length field. + */ + @Override + public final int getSize() { + JavaClass clz = getClazz(); + return MiscUtils.getAlignedObjectSize(clz.getPointerSize() * length + clz.getArrayHeaderSize(), + clz.getObjectAlignment()); + } + + @Override + public final int getImplInclusiveSize() { + return getSize(); + } + + @Override + public String valueAsString() { + return valueAsString(false); + } + + public String valueAsString(boolean bigLimit) { + StringBuilder result; + JavaHeapObject[] elements = getElements(); + int limit = bigLimit ? 1000 : 32; + result = new StringBuilder(limit * 16); + String humanFriendlyName = getClazz().getHumanFriendlyName(); + result.append(humanFriendlyName); + int arrayStartPos = humanFriendlyName.indexOf('['); + if (arrayStartPos != -1) { + result.insert(arrayStartPos + 1, elements.length); + } + result.append('{'); + int num = 0; + + for (JavaHeapObject element : elements) { + if (num > 0) { + result.append(", "); + } + if (num >= limit || (!bigLimit && result.length() > 74)) { + result.append(" ..."); + break; + } + num++; + + result.append(element != null ? element.valueAsString() : "null"); + } + result.append('}'); + return StringInterner.internString(result.toString()); + } + + public JavaHeapObject[] getElements() { + Snapshot snapshot = getClazz().getSnapshot(); + byte[] data = getValue(); + final int idSize = snapshot.getHprofPointerSize(); + JavaHeapObject[] elements = new JavaHeapObject[length]; + int index = 0; + for (int i = 0; i < elements.length; i++) { + long id = objectIdAt(index, data); + index += idSize; + elements[i] = snapshot.getObjectForId(id); + } + return elements; + } + + // Use Comparator instead of implementing Comparable if sorting is needed +// @Override +// public int compareTo(JavaThing other) { +// if (other instanceof JavaObjectArray) { +// return 0; +// } +// return super.compareTo(other); +// } + + public int getLength() { + return length; + } + + @Override + public void visitReferencedObjects(JavaHeapObjectVisitor v) { + super.visitReferencedObjects(v); + JavaHeapObject[] elements = getElements(); + for (JavaHeapObject element : elements) { + if (element != null) { + v.visit(element); + } + } + } + + @Override + protected final byte[] readValue() throws IOException { + if (length == 0) { + return Snapshot.EMPTY_BYTE_ARRAY; + } + ReadBuffer buf = clazz.getReadBuffer(); + int idSize = clazz.getHprofPointerSize(); + // Skip object ID, stack trace serial number (int), array length (int), + // and array class ID + long offset = getObjOfsInFile() + 2 * idSize + 8; + byte[] res = new byte[length * idSize]; + try { + buf.get(offset, res); + } catch (BufferUnderflowException ex) { + throw new DumpCorruptedException.Runtime( + "object array size for array ID " + MiscUtils.toHex(readId()) + " at offset " + getObjOfsInFile() + + " is " + res.length + " bytes, which exceeds heap dump file size"); + } + return res; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaObjectRef.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaObjectRef.java new file mode 100644 index 00000000..4ef49441 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaObjectRef.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +import org.openjdk.jmc.joverflow.util.MiscUtils; +import org.openjdk.jmc.joverflow.util.StringInterner; + +/** + * A forward reference to an object. This is an intermediate representation for a JavaThing, when we + * have the thing's ID, but we might not have read the thing yet. + */ +public class JavaObjectRef extends JavaThing { + private long id; + + public JavaObjectRef(long id) { + this.id = id; + } + + public long getId() { + return id; + } + + @Override + public boolean isHeapAllocated() { + return true; + } + + @Override + public int getSize() { + return 0; + } + + @Override + public final int getImplInclusiveSize() { + return getSize(); + } + + @Override + public String idAsString() { + return valueAsString(); + } + + @Override + public String valueAsString() { + return StringInterner.internString("Unresolved object " + MiscUtils.toHex(id)); + } + + @Override + public String toString() { + return valueAsString(); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaObjectTable.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaObjectTable.java new file mode 100644 index 00000000..384c3891 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaObjectTable.java @@ -0,0 +1,327 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +import java.util.AbstractCollection; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Contains the base information about all instances and arrays (but not classes) of the heap dump + * in the compact table form. Provides a method for creating an instance of JavaLazyReadObject that + * in effect contains the same information as the table, but in a more manageable form. Also + * provides a method to iterate over all objects in the table. + *

+ * The internal table is organized, conceptually, as an array of ints. However, in reality it + * consists of multiple 1MB "chunks", to facilitate building this table incrementally and to avoid + * issues with GC that a very big array could create. Each heap object corresponds to 3 (for + * instances) or 4 (for arrays) ints in the array. The first two ints contain the object's offset in + * the dump, its class index in a separate class table, and a bit specifying whether this object is + * an array, all squeezed collectively into 64 bits. The third int is the "tag word", where various + * setVisited() etc. methods of {@link JavaLazyReadObject} can set bits as they need. Finally, for + * arrays the fourth int contains the array's length. + *

+ * Since we currently use ints to index objects, and objects on average take 3.5 slots in the table, + * the maximum number of objects that this table can accomodate is 2^31 / 3.5 ~= 613 million. + */ +class JavaObjectTable { + + private static final int CHUNK_MAGNITUDE = 20; // Corresponds to 1024*1024 + private static final int CHUNK_SIZE = 1 << CHUNK_MAGNITUDE; + + private static final int POS_IN_CHUNK_MASK = CHUNK_SIZE - 1; + + private static final long LONG_LOW_WORD_MASK = 0x0FFFFFFFFL; + + // The following variables control how the high int of the two ints (that + // collectively contain obj offset, class index and "is array" bit) is used. + // The maximum number of classes is 2^(32 - classIdxShift). + // The maximum object offset is 2^(32 + classIdxShift - 1). + // For example, if we have a 32GB (2^35) file, classIdxShift = 4. Then the + // maximum number of classes this table can accomodate is 2^28 = 268435456. + private final int classIdxShift, arrayMask, objOfsHighWordMask; + + private final int[][] objects; + private final JavaClass[] classes; + + private final int numObjs, lastObjEndPos; + + private JavaObjectTable(int[][] objects, JavaClass[] classes, int numObjs, int lastObjEndPos, int classIdxShift, + int arrayMask) { + this.objects = objects; + this.classes = classes; + this.numObjs = numObjs; + this.lastObjEndPos = lastObjEndPos; + this.classIdxShift = classIdxShift; + this.arrayMask = arrayMask; + this.objOfsHighWordMask = arrayMask - 1; + } + + JavaLazyReadObject getObject(int objPosInTable) { + int chunkIdx = objPosInTable >> CHUNK_MAGNITUDE; + int[] chunk = objects[chunkIdx]; + int posInCurChunk = objPosInTable & POS_IN_CHUNK_MASK; + int startPosInCurChunk = posInCurChunk; + int classAndOfsWord1 = chunk[posInCurChunk++]; + int classAndOfsWord2 = chunk[posInCurChunk++]; + long objOfsInFile = ((classAndOfsWord2) & LONG_LOW_WORD_MASK) + | (((long) (classAndOfsWord1 & objOfsHighWordMask)) << 32); + int classIdx = classAndOfsWord1 >>> classIdxShift; + JavaClass clazz = classes[classIdx]; + boolean isArray = (classAndOfsWord1 & arrayMask) != 0; + if (isArray) { + int length = chunk[posInCurChunk + 1]; + if (clazz.isSingleDimPrimitiveArray()) { + return new JavaValueArray(clazz, objOfsInFile, length, chunk, startPosInCurChunk, objPosInTable); + } else { + return new JavaObjectArray(clazz, objOfsInFile, length, chunk, startPosInCurChunk, objPosInTable); + } + } else { + return new JavaObject(clazz, objOfsInFile, chunk, startPosInCurChunk, objPosInTable); + } + } + + int size() { + return numObjs; + } + + Collection getObjects() { + return new AbstractCollection() { + @Override + public Iterator iterator() { + return new Iterator() { + private int curObjPos = 1; // 1 is important; see Builder.posInCurChunk + private int curChunk = 0; + private int curChunkEndPos = CHUNK_SIZE - 1; + + @Override + public boolean hasNext() { + return curObjPos < lastObjEndPos; + } + + @Override + public JavaLazyReadObject next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + JavaLazyReadObject result = getObject(curObjPos); + if (result instanceof JavaObject) { + curObjPos += 3; + } else { + curObjPos += 4; + } + if (curObjPos > curChunkEndPos - 3) { + curChunk++; + curObjPos = curChunk * CHUNK_SIZE; + curChunkEndPos = curObjPos + CHUNK_SIZE - 1; + } + return result; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + + @Override + public int size() { + return numObjs; + } + }; + } + + Collection getUnvisitedObjects() { + + class UnvisitedObjIterator implements Iterator { + private int curChunk = 0; + private int curChunkEndPos = CHUNK_SIZE - 1; + private int curObjPos = 1; // 1 is important; see Builder.posInCurChunk + + UnvisitedObjIterator() { + moveToNextUnvisitedObjectIfNeeded(); + } + + @Override + public boolean hasNext() { + return curObjPos < lastObjEndPos; + } + + @Override + public JavaLazyReadObject next() { + JavaLazyReadObject result = getObject(curObjPos); + if (result instanceof JavaObject) { + curObjPos += 3; + } else { + curObjPos += 4; + } + moveToNextUnvisitedObjectIfNeeded(); + return result; + } + + private void moveToNextUnvisitedObjectIfNeeded() { + while (curObjPos < lastObjEndPos) { + if (curObjPos > curChunkEndPos - 3) { + curChunk++; + curObjPos = curChunk * CHUNK_SIZE; + curChunkEndPos = curObjPos + CHUNK_SIZE - 1; + } + if (curObjIsVisited()) { + if (curObjIsArray()) { + curObjPos += 4; + } else { + curObjPos += 3; + } + } else { + break; + } + } + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + private boolean curObjIsVisited() { + return JavaLazyReadObject.isVisited(objects[curChunk][(curObjPos & POS_IN_CHUNK_MASK) + 2]); + } + + private boolean curObjIsArray() { + int firstWord = objects[curChunk][curObjPos & POS_IN_CHUNK_MASK]; + return (firstWord & arrayMask) != 0; + } + } + + return new AbstractCollection() { + @Override + public Iterator iterator() { + return new UnvisitedObjIterator(); + } + + @Override + public int size() { + throw new UnsupportedOperationException(); + } + }; + } + + static class Builder { + + private final int classIdxShift; + private final int arrayMask; + + private final ArrayList chunksAsList; + + private int curChunkIdx = -1; + private int[] curChunk; + private int posInCurChunk; // Set to 1 for the very first object; see constructor + + private int numObjs; + + Builder(long hprofFileSize) { + int nObjOfsBits = 0; + while (hprofFileSize > 0) { + nObjOfsBits++; + hprofFileSize >>= 1; + } + if (nObjOfsBits > 32) { + classIdxShift = nObjOfsBits - 31; + } else { + classIdxShift = 1; + } + arrayMask = 1 << (classIdxShift - 1); + + chunksAsList = new ArrayList<>(); + addChunk(); + // It is important that the very first object is located at position 1: + // this guarantees that the implementation of getObjectGlobalIndex() in + // JavaLazyReadObject returns only non-zero positive values, as required. + posInCurChunk = 1; + } + + JavaObjectTable buildJavaObjectTable(JavaClass[] classes) { + int[][] objects = chunksAsList.toArray(new int[chunksAsList.size()][]); + int lastObjEndPos = curChunkIdx * CHUNK_SIZE + posInCurChunk; + return new JavaObjectTable(objects, classes, numObjs, lastObjEndPos, classIdxShift, arrayMask); + } + + int addJavaObject(int classIdx, long objOfsInFile) { + if (posInCurChunk > CHUNK_SIZE - 4) { + addChunk(); + } + int curAbsPos = curChunkIdx * CHUNK_SIZE + posInCurChunk; + + addClassAndOfs(classIdx, objOfsInFile, false); + posInCurChunk++; // Tags word + return curAbsPos; + } + + int addJavaArray(int classIdx, long objOfsInFile, int length) { + if (posInCurChunk > CHUNK_SIZE - 4) { + addChunk(); + } + int curAbsPos = curChunkIdx * CHUNK_SIZE + posInCurChunk; + + addClassAndOfs(classIdx, objOfsInFile, true); + posInCurChunk++; // Tags word + addInt(length); + return curAbsPos; + } + + int getNumObjects() { + return numObjs; + } + + private void addClassAndOfs(int classIdx, long objOfsInFile, boolean isArray) { + numObjs++; + curChunk[posInCurChunk++] = ((int) (objOfsInFile >> 32)) | (classIdx << classIdxShift) + | (isArray ? arrayMask : 0); + curChunk[posInCurChunk++] = (int) objOfsInFile; + } + + private void addInt(int intNum) { + curChunk[posInCurChunk++] = intNum; + } + + private void addChunk() { + curChunkIdx++; + curChunk = new int[CHUNK_SIZE]; + chunksAsList.add(curChunk); + posInCurChunk = 0; + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaShort.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaShort.java new file mode 100644 index 00000000..bf9ef7e5 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaShort.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +import org.openjdk.jmc.joverflow.util.StringInterner; + +/** + * Represents the value of a short field in an instance. + */ +public class JavaShort extends JavaValue { + private short value; + + public JavaShort(short value) { + this.value = value; + } + + public short getValue() { + return value; + } + + void setValue(short value) { + this.value = value; + } + + @Override + public boolean isZero() { + return value == 0; + } + + @Override + public boolean isFloatingPointNumber() { + return false; + } + + @Override + public int getSize() { + return 2; + } + + @Override + public String valueAsString() { + return StringInterner.internString(Short.toString(value)); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaThing.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaThing.java new file mode 100644 index 00000000..3c68bbe0 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaThing.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +/** + * Represents a java "Thing". A thing is anything that can be the value of a field. This includes + * JavaHeapObject, JavaObjectRef, and JavaValue. + */ +public abstract class JavaThing { + + protected JavaThing() { + } + + /** + * Are we the same type as other? + * + * @see JavaObject#isSameTypeAs(JavaThing) + */ + public boolean isSameTypeAs(JavaThing other) { + return getClass() == other.getClass(); + } + + /** + * Returns true iff this represents a heap-allocated object + */ + abstract public boolean isHeapAllocated(); + + /** + * Returns the size of this object, in bytes, including VM overhead. For primitive types, + * returns the size of the value in memory, e.g. 1 for boolean, 4 for int, etc. + */ + abstract public int getSize(); + + /** + * Returns implementation-inclusive size for this object. Currently, this size is different + * (higher than) getSize() value for known Collections and Strings. For collections, it is the + * size of the object itself plus all of its internal implementation objects, such as + * HashMap$Entry - but not the size of the "workload", i.e. collection elements. For Strings, + * it's the size of the object itself plus the size of its char[] array. + *

+ * Note that unlike {@link JavaClass#getTotalInclusiveInstanceSize()} this calculates only the + * "local" implementation-inclusive size for the given object. Thus, for example, it will return + * the same size as getSize() when called for a char[] array that belongs to some String. That + * is in contrast with the total inclusive size in JavaClass, which is 0 for all char[] arrays + * that belong to Strings. + *

+ * Also note that this calculation may be relatively expensive, as it may require traversing all + * of the implementation objects for a given collection. + */ + public abstract int getImplInclusiveSize(); + + /** + * Returns a string that uniquely identifies this thing. For objects, it's typically a + * combination of class name and numeric object ID (its memory address). For primitive values, + * it's the value itself. + */ + abstract public String idAsString(); + + /** + * Returns a human-readable string representation of the value of this thing + */ + abstract public String valueAsString(); + + // Use Comparator instead of implementing Comparable if sorting is needed +// /** +// * Compare our string representation to other's +// * +// * @see java.lang.String#compareTo(String) +// */ +// public int compareTo(JavaThing other) { +// return toString().compareTo(other.toString()); +// } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaValue.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaValue.java new file mode 100644 index 00000000..661628e9 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaValue.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +/** + * Abstract base class for all value types (ints, longs, floats, etc.) + */ +public abstract class JavaValue extends JavaThing { + /** + * Returns true if the contained value is zero (more accurately, a type-specific default + * initialization value) + */ + public abstract boolean isZero(); + + public abstract boolean isFloatingPointNumber(); + + @Override + public boolean isHeapAllocated() { + return false; + } + + @Override + public final int getImplInclusiveSize() { + return getSize(); + } + + @Override + public String idAsString() { + return valueAsString(); + } + + @Override + public String toString() { + return valueAsString(); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaValueArray.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaValueArray.java new file mode 100644 index 00000000..48f7e8a2 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/JavaValueArray.java @@ -0,0 +1,475 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +import java.io.IOException; +import java.nio.BufferUnderflowException; + +import org.openjdk.jmc.joverflow.heap.parser.DumpCorruptedException; +import org.openjdk.jmc.joverflow.heap.parser.ReadBuffer; +import org.openjdk.jmc.joverflow.util.MiscUtils; +import org.openjdk.jmc.joverflow.util.StringInterner; + +/** + * A primitive array, that is, an array of ints, boolean, floats etc. + */ +public class JavaValueArray extends JavaLazyReadObject implements ArrayTypeCodes { + private final int length; + + public JavaValueArray(JavaClass clazz, long objOfsInFile, int length, int[] dataChunk, int startPosInChunk, + int globalObjectIndex) { + super(clazz, objOfsInFile, dataChunk, startPosInChunk, globalObjectIndex); + this.length = length; + } + + /** + * Returns the total size of this object in the heap. That is a sum of object's data (workload) + * size plus the size of the object header, plus the size of array length field, adjusted with + * MiscUtils.getAlignedObjectSize(). + */ + @Override + public final int getSize() { + JavaClass clz = getClazz(); + return MiscUtils.getAlignedObjectSize(elementSize(getElementType()) * length + clz.getArrayHeaderSize(), + clz.getObjectAlignment()); + } + + @Override + public final int getImplInclusiveSize() { + return getSize(); + } + + public final char getElementType() { + String clazzName = getClazz().getName(); + return clazzName.charAt(clazzName.length() - 1); + } + + public final int getElementSize() { + return elementSize(getElementType()); + } + + @Override + protected final byte[] readValue() throws IOException { + if (length == 0) { + return Snapshot.EMPTY_BYTE_ARRAY; + } + ReadBuffer buf = clazz.getReadBuffer(); + int idSize = clazz.getHprofPointerSize(); + // Skip this object's ID, stack trace serial number (int), array length (int) + // and element type (byte) + long offset = getObjOfsInFile() + idSize + 9; + int sizeInBytes = length * elementSize(getElementType()); + byte[] res = new byte[sizeInBytes]; + try { + buf.get(offset, res); + } catch (BufferUnderflowException ex) { + throw new DumpCorruptedException.Runtime("primitive array size for " + getClazz().getHumanFriendlyName() + + " array ID " + MiscUtils.toHex(readId()) + " at offset " + getObjOfsInFile() + " is " + length + + " bytes, which exceeds heap dump file size"); + } + return res; + } + + /** + * Optimized version of readValue() that reads bytes into the provided array and returns the + * number of bytes read. If the provided array is shorter than needed, its contents are not + * changed. Thus the caller should always compare the returned result with array length, and if + * the result is larger, allocate an array of sufficient size and repeat the attempt. + */ + public final int readValue(byte[] value) { + if (length == 0) { + return 0; + } + try { + ReadBuffer buf = clazz.getReadBuffer(); + int idSize = clazz.getHprofPointerSize(); + // Skip this object's ID, stack trace serial number (int), array length (int) + // and element type (byte) + long offset = getObjOfsInFile() + idSize + 9; + int sizeInBytes = length * elementSize(getElementType()); + if (sizeInBytes > value.length) { + return sizeInBytes; + } + + buf.get(offset, value, sizeInBytes); + return sizeInBytes; + } catch (IOException ex) { + throw new DumpCorruptedException.Runtime("exception caught: " + ex); + } catch (BufferUnderflowException ex) { + throw new DumpCorruptedException.Runtime("primitive array size for " + getClazz().getHumanFriendlyName() + + " array ID " + MiscUtils.toHex(readId()) + " at offset " + getObjOfsInFile() + " is " + length + + " bytes, which exceeds heap dump file size"); + } + } + + @Override + public void visitReferencedObjects(JavaHeapObjectVisitor v) { + super.visitReferencedObjects(v); + } + + public int getLength() { + return length; + } + + public Object getElements() { + final int len = getLength(); + final char et = getElementType(); + byte[] data = getValue(); + int index = 0; + switch (et) { + case 'Z': { + boolean[] res = new boolean[len]; + for (int i = 0; i < len; i++) { + res[i] = booleanAt(index, data); + index++; + } + return res; + } + case 'B': { + byte[] res = new byte[len]; + for (int i = 0; i < len; i++) { + res[i] = byteAt(index, data); + index++; + } + return res; + } + case 'C': { + char[] res = new char[len]; + for (int i = 0; i < len; i++) { + res[i] = charAt(index, data); + index += 2; + } + return res; + } + case 'S': { + short[] res = new short[len]; + for (int i = 0; i < len; i++) { + res[i] = shortAt(index, data); + index += 2; + } + return res; + } + case 'I': { + int[] res = new int[len]; + for (int i = 0; i < len; i++) { + res[i] = intAt(index, data); + index += 4; + } + return res; + } + case 'J': { + long[] res = new long[len]; + for (int i = 0; i < len; i++) { + res[i] = longAt(index, data); + index += 8; + } + return res; + } + case 'F': { + float[] res = new float[len]; + for (int i = 0; i < len; i++) { + res[i] = floatAt(index, data); + index += 4; + } + return res; + } + case 'D': { + double[] res = new double[len]; + for (int i = 0; i < len; i++) { + res[i] = doubleAt(index, data); + index += 8; + } + return res; + } + default: { + throw new RuntimeException("unknown primitive type?"); + } + } + } + + private void checkIndex(int index) { + if (index < 0 || index >= getLength()) { + throw new ArrayIndexOutOfBoundsException(index); + } + } + + private void requireType(char type) { + if (getElementType() != type) { + throw new RuntimeException("not of type : " + type); + } + } + + public boolean getBooleanAt(int index) { + checkIndex(index); + requireType('Z'); + return booleanAt(index, getValue()); + } + + public byte getByteAt(int index) { + checkIndex(index); + requireType('B'); + return byteAt(index, getValue()); + } + + public char getCharAt(int index) { + checkIndex(index); + requireType('C'); + return charAt(index << 1, getValue()); + } + + public short getShortAt(int index) { + checkIndex(index); + requireType('S'); + return shortAt(index << 1, getValue()); + } + + public int getIntAt(int index) { + checkIndex(index); + requireType('I'); + return intAt(index << 2, getValue()); + } + + public long getLongAt(int index) { + checkIndex(index); + requireType('J'); + return longAt(index << 3, getValue()); + } + + public float getFloatAt(int index) { + checkIndex(index); + requireType('F'); + return floatAt(index << 2, getValue()); + } + + public double getDoubleAt(int index) { + checkIndex(index); + requireType('D'); + return doubleAt(index << 3, getValue()); + } + + @Override + public String valueAsString() { + return valueAsString(false); + } + + public String valueAsString(boolean bigLimit) { + StringBuilder result; + byte[] value = getValue(); + char elementSignature = getElementType(); + int elSize = elementSize(elementSignature); + int limit = bigLimit ? 1000 : (elementSignature == 'C') ? 32 : 10; + result = new StringBuilder(limit * 8); + result.append(getElementTypeName(elementSignature)).append('['); + result.append(value.length / elSize).append(']'); + for (int i = 1; i < getClazz().getNumArrayDimensions(); i++) { + result.append("[]"); + } + result.append('{'); + int num = 0; + + for (int i = 0; i < value.length;) { + if (num > 0 && elementSignature != 'C') { + result.append(", "); + } + if (num >= limit || result.length() > 74) { + result.append(" ..."); + break; + } + num++; + + switch (elementSignature) { + case 'C': + result.append(charAsString(charAt(i, value))); + break; + case 'Z': + result.append(booleanAsString(booleanAt(i, value))); + break; + case 'B': + result.append(byteAsString(byteAt(i, value))); + break; + case 'S': + result.append(shortAt(i, value)); + break; + case 'I': + result.append(intAt(i, value)); + break; + case 'J': // long + result.append(longAt(i, value)); + break; + case 'F': + result.append(floatAt(i, value)); + break; + case 'D': // double + result.append(doubleAt(i, value)); + break; + default: { + throw new RuntimeException("unknown primitive type?"); + } + } + + i += elSize; + } + + result.append('}'); + return StringInterner.internString(result.toString()); + } + + /** For an array element signature such as 'C', returns human-readable name ("char") */ + public static String getElementTypeName(char sig) { + switch (sig) { + case 'B': + return "byte"; + case 'Z': + return "boolean"; + case 'C': + return "char"; + case 'S': + return "short"; + case 'I': + return "int"; + case 'F': + return "float"; + case 'J': + return "long"; + case 'D': + return "double"; + default: + return null; + } + } + + public String[] getValuesAsStrings() { + final int len = getLength(); + final char et = getElementType(); + byte[] data = getValue(); + String[] res = new String[len]; + int index = 0; + switch (et) { + case 'Z': { + for (int i = 0; i < len; i++) { + res[i] = StringInterner.internString(booleanAsString(booleanAt(index, data))); + index++; + } + return res; + } + case 'B': { + for (int i = 0; i < len; i++) { + res[i] = StringInterner.internString(byteAsString(byteAt(index, data))); + index++; + } + return res; + } + case 'C': { + for (int i = 0; i < len; i++) { + res[i] = StringInterner.internString(charAsString(charAt(index, data))); + index += 2; + } + return res; + } + case 'S': { + for (int i = 0; i < len; i++) { + res[i] = StringInterner.internString(String.valueOf(shortAt(index, data))); + index += 2; + } + return res; + } + case 'I': { + for (int i = 0; i < len; i++) { + res[i] = StringInterner.internString(String.valueOf(intAt(index, data))); + index += 4; + } + return res; + } + case 'J': { + for (int i = 0; i < len; i++) { + res[i] = StringInterner.internString(String.valueOf(longAt(index, data))); + index += 8; + } + return res; + } + case 'F': { + for (int i = 0; i < len; i++) { + res[i] = StringInterner.internString(String.valueOf(floatAt(index, data))); + index += 4; + } + return res; + } + case 'D': { + for (int i = 0; i < len; i++) { + res[i] = StringInterner.internString(String.valueOf(doubleAt(index, data))); + index += 8; + } + return res; + } + default: { + throw new RuntimeException("unknown primitive type?"); + } + } + } + + private static int elementSize(char sig) { + switch (sig) { + case 'B': + case 'Z': + return 1; + case 'C': + case 'S': + return 2; + case 'I': + case 'F': + return 4; + case 'J': + case 'D': + return 8; + default: + throw new RuntimeException("invalid array element type: " + sig); + } + } + + private static String charAsString(char val) { + if (val > 32) { + return String.valueOf(val); + } else { + return "\\0x" + Integer.toString(val, 16); + } + } + + private static String booleanAsString(boolean val) { + return val ? "true" : "false"; + } + + private static String byteAsString(byte val) { + int intVal = 0xFF & val; + return "0x" + Integer.toString(intVal, 16); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/Root.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/Root.java new file mode 100644 index 00000000..026bfb5e --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/Root.java @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +/** + * Represents a member of the rootset, that is, one of the objects that the GC starts from when + * marking reachable objects. + *

+ * Note that this class implements compareTo() but has no implementation of equals(). In other + * words, it's currently not guaranteed that compareTo() returns zero if and only if equals() + * returns true. However, that only matters if instances of a class are used in classes like + * PriorityQueue, which is highly unlikely for this class and its subclasses. + */ +public class Root implements Comparable { + + private long id; // ID of the JavaThing we refer to + private long refererId; // Thread or Class responsible for this, or 0 + private int type; + private JavaHeapObject referer = null; + private StackTrace stackTrace = null; + + // Values for type. Higher values are more interesting -- see getType(). + // See also getTypeName() + public final static int INVALID_TYPE = 0; + public final static int UNKNOWN = 1; + public final static int SYSTEM_CLASS = 2; + + public final static int JNI_LOCAL = 3; + public final static int JNI_GLOBAL = 4; + public final static int THREAD_BLOCK = 5; + public final static int BUSY_MONITOR = 6; + public final static int JAVA_LOCAL = 7; + public final static int NATIVE_STACK = 8; + public final static int JAVA_STATIC = 9; + + private static final String UNKNOWN_ROOT_STR = "Unknown GC root"; + + public static final Root UNKNOWN_ROOT = new Root(0, 0, UNKNOWN, "") { + @Override + public String getIdString() { + return UNKNOWN_ROOT_STR; + } + }; + + public Root(long id, long refererId, int type, String description) { + this(id, refererId, type, description, null); + } + + public Root(long id, long refererId, int type, String description, StackTrace stackTrace) { + this.id = id; + this.refererId = refererId; + this.type = type; + this.stackTrace = stackTrace; + } + + public long getId() { + return id; + } + + public String getIdString() { + return getTypeName() + '@' + getId(); + } + + public boolean isUnknownRoot() { + return this == UNKNOWN_ROOT; + } + + @Override + public String toString() { + return getIdString(); + } + + /** + * Return type. We guarantee that more interesting roots will have a type that is numerically + * higher. + */ + public int getType() { + return type; + } + + public String getTypeName() { + switch (type) { + case INVALID_TYPE: + return "Invalid (?!?)"; + case UNKNOWN: + return "Unknown"; + case SYSTEM_CLASS: + return "System Class"; + case JNI_LOCAL: + return "JNI Local"; + case JNI_GLOBAL: + return "JNI Global"; + case THREAD_BLOCK: + return "Thread Block"; + case BUSY_MONITOR: + return "Busy Monitor"; + case JAVA_LOCAL: + return "Java Local"; + case NATIVE_STACK: + return "Native Stack (possibly Java local)"; + case JAVA_STATIC: + return "Java Static"; + default: + return "??"; + } + } + + /** + * Get the object that's responsible for this root, if there is one. This will be null, a Thread + * object, or a Class object. + */ + public JavaHeapObject getReferer() { + return referer; + } + + /** + * @return the stack trace responsible for this root, or null if there is none. + */ + public StackTrace getStackTrace() { + return stackTrace; + } + + void resolve(Snapshot ss) { + if (refererId != 0) { + referer = ss.getObjectForId(refererId); + } + if (stackTrace != null) { + stackTrace.resolve(ss); + } + } + + /** + * Helps to sort Roots in the order of more interesting to less interesting. More interesting + * roots have higher type values. + */ + @Override + public int compareTo(Root other) { + // A root with higher type value should come first. So if this Root + // should come first, a negative value should be returned below. + // That's what will happen if this.type > other.type. + if (other.type != this.type) { + return other.type - this.type; + } else { + // Later we may differentiate roots within the same type + return 0; + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/Snapshot.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/Snapshot.java new file mode 100644 index 00000000..9f84a1e6 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/Snapshot.java @@ -0,0 +1,929 @@ +/* + * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +import java.io.IOException; +import java.lang.ref.SoftReference; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; + +import org.openjdk.jmc.joverflow.heap.parser.ReadBuffer; +import org.openjdk.jmc.joverflow.support.Constants; +import org.openjdk.jmc.joverflow.util.IntToIntMap; +import org.openjdk.jmc.joverflow.util.LongToIntMap; +import org.openjdk.jmc.joverflow.util.LongToObjectMap; +import org.openjdk.jmc.joverflow.util.MiscUtils; +import org.openjdk.jmc.joverflow.util.NumberToIntMap; +import org.openjdk.jmc.joverflow.util.VerboseOutputCollector; + +/** + * Represents all of the contents of the heap dump. + */ +public class Snapshot { + public static final long SMALL_ID_MASK = 0x0FFFFFFFFL; + public static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + + /** + * Size of object pointers used in this heap dump. May be different from the actual pointer size + * in memory of the JVM that generated this heap dump, because of narrow references in 64-bit + * mode. That is, for the 64-bit JVM hprofPointerSize is always 8, but in the real JVM the + * pointer size is usually 4, unless the heap is really big (> ~26GB) + */ + private final int hprofPointerSize; + + /** Object pointer size in the JVM that generated this heap dump */ + private final int pointerSize; + + /** Object header size in the JVM that generated this heap dump */ + private final int objHeaderSize; + + /** Memory object alignment granularity in the JVM that generated this dump */ + private final int objAlignment; + + /** If true, narrow (32-bit) pointers are used in 64-bit mode */ + private final boolean usingNarrowPointers; + + /** All GC roots in this heap dump */ + private final ArrayList roots; + + private final JavaObjectTable objectTable; + + private final NumberToIntMap objIdToPosInObjectTable; + + private final JavaClass[] classes; + + /** + * Maps a Java class name to the respective JavaClass object. Classes with same name but + * different ids (loaders) are chained, thus classes.size() won't return the exact number of + * classes in this snapshot + */ + private final HashMap classNameToJavaClass; + + /** Maps an object ID to the respective JavaClass object */ + private final LongToObjectMap classIdToJavaClass; + + /** Maps a classloader object ID to the respective JavaHeapObject */ + private final LongToObjectMap classLoaders; + + /** + * Rough total size of all instances and arrays (but not classes) in the heap dump. The size of + * an object is the size of its data in the heap dump, which is not guaranteed to be the same as + * in memory, plus header size (which is objHeaderSize above). No adjustment for object + * alignment is made. + */ + private final long roughTotalObjectSize; + + /** Heap dump memory buffer used for lazy reads of JavaHeapObject contents */ + private ReadBuffer readBuf; + + private HeapStringReader stringReader; + + /** + * If this is true, warnings are not recorded for objects that fail to resolve. It's set to true + * when unexpected EOF is detected when reading a HPROF file, and therefore some objects are + * expected to be missing and references to them will be unresolved. + */ + private final boolean unresolvedObjectsOk; + + /** Soft cache of finalizeable objects - lazily initialized */ + private SoftReference> finalizablesCache; + + /** java.lang.ref.Reference class */ + private JavaClass weakReferenceClass; + /** Index of 'referent' field in java.lang.ref.Reference class */ + private int referentFieldIndex; + + /** Several popular classes, used for fast recognition */ + private JavaClass javaLangClass, javaLangString, javaLangClassLoader, charArrayClass, byteArrayClass; + + /** + * This is set to true when global object stats is being calculated, to signal that e.g. heap + * inspection operations should not be performed, or that some assumptions valid after stats + * calculation is done, are not yet true. + */ + private volatile boolean calculatingStats; + + private final VerboseOutputCollector vc; + + private Snapshot(int hprofPointerSize, int pointerSize, int objHeaderSize, int objAlignment, + boolean usingNarrowPointers, long roughTotalObjectSize, ArrayList roots, JavaObjectTable objectTable, + NumberToIntMap objIdToPosInObjectTable, JavaClass[] classes, + HashMap classNameToJavaClass, LongToObjectMap classIdToJavaClass, + ReadBuffer readBuf, VerboseOutputCollector vc, boolean unresolvedObjectsOk) { + this.hprofPointerSize = hprofPointerSize; + this.pointerSize = pointerSize; + this.objHeaderSize = objHeaderSize; + this.objAlignment = objAlignment; + this.usingNarrowPointers = usingNarrowPointers; + this.roughTotalObjectSize = roughTotalObjectSize; + + this.roots = roots; + this.objectTable = objectTable; + this.objIdToPosInObjectTable = objIdToPosInObjectTable; + + this.classes = classes; + this.classNameToJavaClass = classNameToJavaClass; + this.classIdToJavaClass = classIdToJavaClass; + classLoaders = new LongToObjectMap<>(50, false); + + this.readBuf = readBuf; + this.vc = vc; + this.unresolvedObjectsOk = unresolvedObjectsOk; + + javaLangClass = getClassForName("java.lang.Class"); + javaLangString = getClassForName("java.lang.String"); + javaLangClassLoader = getClassForName("java.lang.ClassLoader"); + charArrayClass = getClassForName("[C"); + byteArrayClass = getClassForName("[B"); + weakReferenceClass = getClassForName("java.lang.ref.Reference"); + + // The code below should be called in the very end of constructor. + // Note that it exposes this Snapshot instance, which is technically still + // partially constructed, to methods of JavaClass. While not a good style, + // it works here, and makes things overall a bit simpler than putting code + // below into a separate finishInitialization() method. + for (JavaClass clazz : classes) { + // The call below results in callbacks to dereferenceField() and dereferenceClassLoader() + clazz.resolve(this, roots); + } + + this.stringReader = new HeapStringReader(this); + + JavaField[] fields = weakReferenceClass.getFieldsForInstance(); + for (int i = 0; i < fields.length; i++) { + if ("referent".equals(fields[i].getName())) { + referentFieldIndex = i; + break; + } + } + } + + /** + * Returns the heap object with the specified id. If an object with this id does not exist (may + * happen with corrupted heap dumps), returns an instance of a fake class for this id. + */ + public JavaHeapObject getObjectForId(long id) { + if (id == 0) { + return null; + } + + int objPosInTable = objIdToPosInObjectTable.get(id); + if (objPosInTable >= 0) { + return objectTable.getObject(objPosInTable); + } else { + return classIdToJavaClass.get(id); + } + } + + /** + * Returns the heap object with the specified global index. Each JavaHeapObject in the heap dump + * has a unique index, that is returned by {@link JavaHeapObject#getGlobalObjectIndex()}. The + * index is not equal to the object id. + */ + public JavaHeapObject getObjectAtGlobalIndex(int globalIndex) { + if (globalIndex > 0) { + return objectTable.getObject(globalIndex); + } else { + return classes[-globalIndex]; + } + } + + public JavaClass getClassForName(String name) { + return classNameToJavaClass.get(name); + } + + public JavaClass getClassForId(long id) { + return classIdToJavaClass.get(id); + } + + public List getRoots() { + return roots; + } + + public Collection getObjects() { + return objectTable.getObjects(); + } + + public Collection getUnvisitedObjects() { + return objectTable.getUnvisitedObjects(); + } + + /** + * Returns an enumeration of all classes in this snapshot, including multiple versions of class + * with the same name. + */ + public JavaClass[] getClasses() { + return classes; + } + + public Collection getClassLoaders() { + return classLoaders.values(); + } + + /** + * Returns the total number of all objects in the heap dump. This includes instances and arrays, + * but not classes. + */ + public int getNumObjects() { + return objIdToPosInObjectTable.size(); + } + + public int getNumClasses() { + return classes.length; + } + + public ReadBuffer getReadBuffer() { + return readBuf; + } + + /** + * Allows the user to replace the ReadBuffer instance used by this Snapshot, for example to + * reduce memory usage once no more intensive random-access operations are performed. The + * provided factory should use exactly the same heap dump file. + */ + public void resetReadBuffer(ReadBuffer.Factory bufFactory) { + try { + readBuf = bufFactory.create(null); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + /** + * Should be called by the user before removing references to this Snapshot instance, i.e. + * before it gets GCed. Frees any external resources that it may have used internally. + */ + public void discard() { + readBuf.close(); + } + + /** + * Called during class resolution, to convert static field ids into objects, and when parsing + * instance fields, to obtain JavaHeapObjects for them. Note that if called multiple times for + * the same id, it will produce multiple separate JavaHeapObjects. For static fields, we + * consider this tolerable; for other fields, the user should be careful not store all these + * objects permanently if performance is a concern. + */ + JavaThing dereferenceField(long objId, JavaField field) { + if (field != null && !field.isReference()) { + // If this happens, we must be a field that represents an int. + // (This only happens with .bod-style files) + return new JavaLong(objId); + } + if (objId == 0) { + return null; + } + + JavaThing result = getObjectForId(objId); + + if (result == null) { + if (!unresolvedObjectsOk && vc != null) { + vc.addWarning("Failed to resolve object", "id = " + MiscUtils.toHex(objId) + (field != null + ? ("for field " + field.getName() + " (signature " + field.getTypeId() + ")") : "")); + } + result = new UnresolvedObject(objId); + } + return result; + } + + /** + * Called during class resolution, to convert a classloader ID into JavaObject. This is used to + * avoid creating multiple JavaObjects for the same classloader ID, and also to add each + * classloader to GC roots. The latter, though it's not clear whether it's a theoretically + * correct thing to do, definitely helps to reduce the number of objects not reachable from any + * known GC root. + *

+ * Returns a JavaObject for loader, or an UnresolvedObject instance if there is no object with + * the given ID. + * + * @return a JavaObject for loader, or an UnresolvedObject instance if there is no object with + * the given ID. + */ + JavaThing dereferenceClassLoader(long objId, JavaClass clazz) { + if (objId == 0) { + return null; + } + JavaObject loader = classLoaders.get(objId); + if (loader != null) { + return loader; + } + + loader = (JavaObject) getObjectForId(objId); + if (loader == null) { + if (!unresolvedObjectsOk && vc != null) { + vc.addWarning("Failed to resolve classloader", + "id = " + MiscUtils.toHex(objId) + " for class " + clazz.getHumanFriendlyName()); + } + return new UnresolvedObject(objId); + } + + classLoaders.put(objId, loader); + + // Make each classloader a GC root. This considerably reduces the number of + // objects that would otherwise be found not attached to any GC root. + String s = "Classloader of " + clazz.getName(); + roots.add(new Root(objId, clazz.readId(), Root.JAVA_STATIC, s)); + return loader; + } + + public synchronized Iterator getFinalizerObjects() { + ArrayList obj; + if (finalizablesCache != null && (obj = finalizablesCache.get()) != null) { + return obj.iterator(); + } + + JavaClass clazz = getClassForName("java.lang.ref.Finalizer"); + JavaObject queue = (JavaObject) clazz.getStaticField("queue"); + JavaThing tmp = queue.getField("head"); + ArrayList finalizables = new ArrayList<>(); + if (tmp != null) { + JavaObject head = (JavaObject) tmp; + while (true) { + JavaHeapObject referent = (JavaHeapObject) head.getField("referent"); + JavaThing next = head.getField("next"); + if (next == null || next.equals(head)) { + break; + } + head = (JavaObject) next; + finalizables.add(referent); + } + } + finalizablesCache = new SoftReference<>(finalizables); + return finalizables.iterator(); + } + + /** + * Returns the rough size of all objects (instances and arrays, but not classes), calculated + * while reading the heap dump. + */ + public long getRoughTotalObjectSize() { + return roughTotalObjectSize; + } + + public int getHprofPointerSize() { + return hprofPointerSize; + } + + /** + * Returns the pointer size as it should have been in the JVM that generated this heap dump. + */ + public int getPointerSize() { + return pointerSize; + } + + /** + * Returns the object header size as it should have been in the JVM that generated this heap + * dump. + */ + public int getObjectHeaderSize() { + return objHeaderSize; + } + + /** + * Returns the array header size as it should have been in the JVM that generated this heap + * dump. + */ + public int getArrayHeaderSize() { + return objHeaderSize + 4; + } + + /** + * Returns the object alignment as it should have been in the JVM that generated this heap dump. + */ + public int getObjectAlignment() { + return objAlignment; + } + + /** Returns true if narrow (32-bit) pointers are used in 64-bit mode */ + public boolean usingNarrowPointers() { + return usingNarrowPointers; + } + + public HeapStringReader getStringReader() { + return stringReader; + } + + public VerboseOutputCollector getVerboseOutputCollector() { + return vc; + } + + JavaClass getJavaLangClass() { + return javaLangClass; + } + + JavaClass getJavaLangStringClass() { + return javaLangString; + } + + JavaClass getJavaLangClassLoaderClass() { + return javaLangClassLoader; + } + + JavaClass getCharArrayClass() { + return charArrayClass; + } + + JavaClass getByteArrayClass() { + return byteArrayClass; + } + + public JavaClass getWeakReferenceClass() { + return weakReferenceClass; + } + + public int getReferentFieldIndex() { + return referentFieldIndex; + } + + /** Returns true if global stats is currently being calculated. */ + public boolean isCalculatingStats() { + return calculatingStats; + } + + public void setCalculatingStats(boolean value) { + calculatingStats = value; + } + + public static class Builder { + + /** + * From sampling some .hprof files, average approximate object size determined as (file_size + * / num_objects) is 70. We use it to set initial size of the main table mapping object IDs + * to object info. + */ + private static final int EXPECTED_OBJ_SIZE_IN_FILE = 70; + + private int hprofPointerSize; + + /** Object pointer size in the JVM that generated this heap dump */ + private int pointerSize; + + private int objHeaderSize; + private int objAlignment; + private boolean usingNarrowPointers; + + private static ObjTableSizePolicy objTableSizePolicy; + + private final ArrayList roots = new ArrayList<>(); + + private final JavaObjectTable.Builder objTableBuilder; + + private final NumberToIntMap objIdToPosInObjectTable; + + /** + * List of all JavaClass objects. However, if the given class hasn't been read by the time + * its ID is read, it gets temporarily replaced with a Long representing that class's ID. + */ + private final ArrayList classList; + + /** + * Maps class ID to JavaClass. However, if the given class hasn't been read by the time its + * ID is read, it gets temporarily replaced with an Integer representing that class's index + * in classList. + */ + private final LongToObjectMap classIdToJavaClass; + + private final HashMap classNameToJavaClass; + + private boolean unresolvedObjectsOk; + + private final VerboseOutputCollector vc; + + private long roughTotalObjectSize; + + /** + * Constructs a Snapshot.Builder instance with the specified settings. If + * explicitPointerSize > 0, it's specified by the user and should be used instead of the + * value we half-read/half-guess from the hprof file. + */ + public Builder(long hprofFileSize, int hprofIdentifierSize, int explicitPointerSize, + VerboseOutputCollector vc) { + this.vc = vc; + + this.hprofPointerSize = hprofIdentifierSize; + /* + * See https://bugs.openjdk.java.net/browse/JDK-7145625. In 64-bit mode, we assume that + * if heap size is less than ~26GB, narrow (32-bit) pointers are used, otherwise wide + * (full 64-bit) pointers are used. + */ + if (explicitPointerSize > 0) { + pointerSize = explicitPointerSize; + } else { + if (hprofIdentifierSize == 4) { + pointerSize = Constants.POINTER_SIZE_IN_32BIT_MODE; + } else { + if (hprofFileSize < 26L * 1024 * 1024 * 1024) { + pointerSize = Constants.NARROW_POINTER_SIZE_IN_64BIT_MODE; + usingNarrowPointers = true; + } else { + pointerSize = Constants.WIDE_POINTER_SIZE_IN_64BIT_MODE; + } + } + } + // Assume HotSpot object header for now. On JRockit, it is always 8. + // If HprofReader comes across any JRockit-specific class, it will + // request a change by calling updateObjectHeaderSize() below. + objHeaderSize = hprofIdentifierSize == 4 ? Constants.STANDARD_32BIT_OBJ_HEADER_SIZE + : pointerSize == Constants.NARROW_POINTER_SIZE_IN_64BIT_MODE + ? Constants.HOTSPOT_64BIT_NARROW_REF_OBJ_HEADER_SIZE + : Constants.HOTSPOT_64BIT_WIDE_REF_OBJ_HEADER_SIZE; + // See the comments to the constant below. In principle, its value may vary, + // but in practice it's unlikely. + objAlignment = Constants.DEFAULT_OBJECT_ALIGNMENT_IN_MEMORY; + + // Set the approximate size for objIdToPosInObjectTable to avoid excessive rehashing + int objTableSize = objTableSizePolicy != null ? objTableSizePolicy.getInitialObjTableSize(hprofFileSize) + : (int) (hprofFileSize / EXPECTED_OBJ_SIZE_IN_FILE); + if (pointerSize == 4) { + objIdToPosInObjectTable = new IntToIntMap(objTableSize); + } else { + objIdToPosInObjectTable = new LongToIntMap(objTableSize); + } + + classList = new ArrayList<>(objTableSize / 2000); + objTableBuilder = new JavaObjectTable.Builder(hprofFileSize); + + classIdToJavaClass = new LongToObjectMap<>(objTableSize / 2000, false); + classNameToJavaClass = new HashMap<>(objTableSize / 2000); + } + + /** + * Sets custom ObjTableSizePolicy, that will be used to determine initial object table size. + * By default, it's set as file_size / EXPECTED_OBJ_SIZE_IN_FILE. + */ + public static void setObjTableSizePolicy(ObjTableSizePolicy policy) { + objTableSizePolicy = policy; + } + + /** + * Perform potentially memory-consuming operations once all objects are read. This should be + * called before buildSnapshot(), i.e. before a ReadBuffer, that may take quite some memory, + * is allocated. + */ + public void onFinishReadObjects() { + objIdToPosInObjectTable.adjustCapacityIfNeeded(); + } + + @SuppressWarnings("unchecked") + public Snapshot buildSnapshot(ReadBuffer readBuf) { + checkForMissingJavaClasses(); + + JavaClass[] classes = classList.toArray(new JavaClass[classList.size()]); + JavaObjectTable objectTable = objTableBuilder.buildJavaObjectTable(classes); + resolveSuperclasses(classes); + recheckPointerSize(objectTable, readBuf); + roots.sort(null); // More interesting roots will be scanned first + + Snapshot snapshot = new Snapshot(hprofPointerSize, pointerSize, objHeaderSize, objAlignment, + usingNarrowPointers, roughTotalObjectSize, roots, objectTable, objIdToPosInObjectTable, classes, + classNameToJavaClass, (LongToObjectMap) (LongToObjectMap) classIdToJavaClass, readBuf, + vc, unresolvedObjectsOk); + return snapshot; + } + + public void addJavaObject(long id, long classID, long objOfsInFile, int objDataSize) { + int classIdx = getClassIdxForClassID(classID); + int objPosInTable = objTableBuilder.addJavaObject(classIdx, objOfsInFile); + objIdToPosInObjectTable.put(id, objPosInTable); + roughTotalObjectSize += objDataSize + objHeaderSize; + } + + public void addJavaObjectArray(long id, long classID, long objOfsInFile, int length, int objDataSize) { + int classIdx = getClassIdxForClassID(classID); + int objPosInTable = objTableBuilder.addJavaArray(classIdx, objOfsInFile, length); + objIdToPosInObjectTable.put(id, objPosInTable); + roughTotalObjectSize += objDataSize + objHeaderSize + 4; + } + + public void addJavaValueArray( + long id, char primitiveSignature, long objOfsInFile, int length, int objDataSize) { + JavaClass clazz = getPrimitiveArrayClass(primitiveSignature); + int classIdx = clazz.getClassListIdx(); + int objPosInTable = objTableBuilder.addJavaArray(classIdx, objOfsInFile, length); + objIdToPosInObjectTable.put(id, objPosInTable); + roughTotalObjectSize += objDataSize + objHeaderSize + 4; + } + + public void addRoot(Root r) { + roots.add(r); + } + + public void addClass(JavaClass clazz) { + Object classIdxOrNull = classIdToJavaClass.get(clazz.readId()); + if (classIdxOrNull == null) { // Class not seen before + int classIdx = classList.size(); + clazz.setClassListIdx(classIdx); + classList.add(clazz); + } else { + int classIdx = (Integer) classIdxOrNull; + clazz.setClassListIdx(classIdx); + classList.set(classIdx, clazz); + } + addToClassMaps(clazz); + recheckObjectHeaderSize(clazz); + } + + public int getPointerSize() { + return pointerSize; + } + + public int getNumAllObjects() { + return objTableBuilder.getNumObjects(); + } + + public int getNumClasses() { + return classList.size(); + } + + public void setUnresolvedObjectsOk(boolean v) { + unresolvedObjectsOk = v; + } + + public int getInMemoryInstanceSize(int instanceFieldsSize) { + // Add object header size + int result = instanceFieldsSize + objHeaderSize; + // Take into account object alignment + return MiscUtils.getAlignedObjectSize(result, objAlignment); + } + + private int getClassIdxForClassID(long classID) { + Object classOrIdx = classIdToJavaClass.get(classID); + if (classOrIdx == null) { + int classIdx = classList.size(); + classList.add(classID); + classIdToJavaClass.put(classID, classIdx); + return classIdx; + } else if (classOrIdx instanceof JavaClass) { + return ((JavaClass) classOrIdx).getClassListIdx(); + } else { + return (Integer) classOrIdx; + } + } + + /** + * Creates fake JavaClass objects for "orphan" classes (those for which we read class ID but + * not the actual class object). + */ + private void checkForMissingJavaClasses() { + for (int i = 0; i < classList.size(); i++) { + Object clazzOrId = classList.get(i); + if (clazzOrId instanceof Long) { + // JavaClass object for this ID hasn't been read from dump. Create a fake class + long classId = (Long) clazzOrId; + if (!unresolvedObjectsOk) { + if (vc != null) { + vc.addWarning("Failed to resolve object", "No JavaClass found for class ID = " + classId); + } + } + // TODO: we don't specify the correct instance size for this class. But does it matter? + JavaClass clazz = createFakeClass(classId, 0); + clazz.setClassListIdx(i); + classList.set(i, clazz); + addToClassMaps(clazz); + } + } + } + + private void addToClassMaps(JavaClass clazz) { + classIdToJavaClass.put(clazz.readId(), clazz); + JavaClass existingClass = classNameToJavaClass.get(clazz.getName()); + if (existingClass != null) { + existingClass.addNextVersion(clazz); + } else { + classNameToJavaClass.put(clazz.getName(), clazz); + } + } + + /** + * Returns a JavaClass representing an array with specified element type, or creates one and + * adds to the relevant data structures. + */ + JavaClass getPrimitiveArrayClass(char elementSignature) { + String className = "[" + elementSignature; + JavaClass clazz = classNameToJavaClass.get(className); + if (clazz == null) { + clazz = new JavaClass(className, 0, 0, 0, 0, JavaClass.NO_FIELDS, JavaClass.NO_FIELDS, + JavaClass.NO_VALUES, 0, 0); + int classIdx = classList.size(); + clazz.setClassListIdx(classIdx); + classList.add(clazz); + classNameToJavaClass.put(className, clazz); + } + return clazz; + } + + /** + * Creates and adds a fake class object with the specified ID and instance size. This is + * called when for some JavaObject there is no class object with the specified ID. In that + * case, there is nothing we can do but construct an artificial class so that the given + * object looks normal. + */ + private JavaClass createFakeClass(long classID, int instSize) { + // Create a fake class name based on ID. + String name = "unknown-class<@" + MiscUtils.toHex(classID) + ">"; + + // Create fake fields convering the given instance size. + // Create as many as int type fields and for the left over + // size create byte type fields. + int numInts = instSize / 4; + int numBytes = instSize % 4; + JavaField[] fields = new JavaField[numInts + numBytes]; + int i; + for (i = 0; i < numInts; i++) { + fields[i] = JavaField.newInstance("unknown-field-" + i, 'I', pointerSize); + } + for (i = 0; i < numBytes; i++) { + fields[i + numInts] = JavaField.newInstance("unknown-field-" + i + numInts, 'B', pointerSize); + } + + // Create fake instance class + return new JavaClass(classID, name, 0, 0, 0, 0, fields, JavaClass.NO_FIELDS, JavaClass.NO_VALUES, instSize, + getInMemoryInstanceSize(instSize)); + } + + /** + * Set the actual superclass for each class instead of the forward reference. We do this + * separately from resolveClass() calls, because we need superclasses for getFields() to + * work properly, which in turn is needed by recheckPointerSize() below. + */ + @SuppressWarnings("unchecked") + private void resolveSuperclasses(JavaClass[] classes) { + for (JavaClass clazz : classes) { + clazz.resolveSuperclass((LongToObjectMap) (LongToObjectMap) classIdToJavaClass); + } + } + + /** + * Using various guessing methods, attempt to make a more accurate estimate of the object + * header size in the JVM that generated this heap dump. + */ + private void recheckObjectHeaderSize(JavaClass c) { + if (c.getName().startsWith("jrockit.vm.") && objHeaderSize != Constants.JROCKIT_OBJ_HEADER_SIZE) { + // On JRockit, object header size is always 8 bytes. Unfortunately, there + // is no way to tell that the heap dump is generated by JRockit except by + // this kind of guessing. + updateObjectHeaderSize(Constants.JROCKIT_OBJ_HEADER_SIZE); + } + } + + /** + * Since there is no explicit info on object header size in HPROF file, we have to guess, + * and later may need to call this method to correct our guess. + */ + private void updateObjectHeaderSize(int objHeaderSize) { + this.objHeaderSize = objHeaderSize; + // Update instance size for all classes that have already been registered + for (Object clazzOrID : classList) { + if (!(clazzOrID instanceof JavaClass)) { + continue; + } + JavaClass clazz = (JavaClass) clazzOrID; + clazz.updateInstanceSize(getInMemoryInstanceSize(clazz.getFieldsSizeInFile())); + } + } + + /** + * Using various guessing methods, attempt to make a more accurate estimate of the pointer + * size in the JVM that generated this heap dump. This is only relevant for 64-bit heap + * dumps. + */ + private void recheckPointerSize(JavaObjectTable objectTable, ReadBuffer readBuf) { + if (hprofPointerSize == 4) { + return; // 32-bit mode, nothing to check + } + + Collection allObjects = objectTable.getObjects(); + JavaLazyReadObject prevObj = null; + long prevObjId = 0; + int nCheckedObjs = 0; + for (JavaLazyReadObject obj : allObjects) { + if (prevObj == null) { + prevObj = obj; + prevObjId = prevObj.readId(readBuf, hprofPointerSize); + continue; + } + // "Object id" is actually the object's address in the JVM memory + long objId = obj.readId(readBuf, hprofPointerSize); + + if (prevObj instanceof JavaObject && obj instanceof JavaObject) { + if (++nCheckedObjs > 1000000) { + break; // Put an upper bound on time of this operation + } + + long prevObjSize = objId - prevObjId; + if (prevObjSize > 12 && prevObjSize <= 40) { + if (verifyObjSize((JavaObject) prevObj, (int) prevObjSize)) { + break; + } + } + } + + prevObj = obj; + prevObjId = objId; + } + } + + /** + * For the given object and its size calculated from addresses of consecutive objs in the + * heap dump, attempts to compare that size with the size calculated based on object's + * fields and current pointerSize. Returns true if it is able to confirm the pointer size + * one or another way, and false if no definitive conclusions can be made. + */ + private boolean verifyObjSize(JavaObject obj, int objSize) { + JavaClass clazz = obj.getClazz(); + JavaField[] fields = clazz.getFieldsForInstance(); + int nPointers = 0, nInts = 0; + for (JavaField field : fields) { + if (field.isReference()) { + nPointers++; + } else if (field.getTypeId() == 'I') { + nInts++; + } else { + return false; // We don't know exact size of other field types in the JVM + } + } + if (nPointers < 2) { + return false; // Not enough information + } + int expectedObjSize = MiscUtils.getAlignedObjectSize(objHeaderSize + pointerSize * nPointers + 4 * nInts, + objAlignment); + + if (expectedObjSize == objSize) { + return true; // Current pointerSize is correct + } + + int altPointerSize = pointerSize == Constants.NARROW_POINTER_SIZE_IN_64BIT_MODE + ? Constants.WIDE_POINTER_SIZE_IN_64BIT_MODE : Constants.NARROW_POINTER_SIZE_IN_64BIT_MODE; + int altObjHeaderSize = altPointerSize == Constants.NARROW_POINTER_SIZE_IN_64BIT_MODE + ? Constants.HOTSPOT_64BIT_NARROW_REF_OBJ_HEADER_SIZE + : Constants.HOTSPOT_64BIT_WIDE_REF_OBJ_HEADER_SIZE; + int newExpectedObjSize = MiscUtils + .getAlignedObjectSize(altObjHeaderSize + altPointerSize * nPointers + 4 * nInts, objAlignment); + if (newExpectedObjSize == objSize) { + // Looks like the other pointer size is the correct one +// System.err.println("!!! For obj of class " + clazz.getName() + " nPointers = " + nPointers +// + ", nInts = " + nInts + ", expectedObjSize with current ptr size of " + pointerSize + " is " +// + expectedObjSize + ", actual objSize = " + objSize); +// System.err.println("!!! Expected size with alternative ptr size of " + altPointerSize + " is " +// + newExpectedObjSize); + pointerSize = altPointerSize; + objHeaderSize = altObjHeaderSize; + + /* + * TODO: Should we recalculate instance size for all classes here? + * + * The problem is, we don't know for sure the size of byte, char etc. fields in + * instances. But it also may be the case that these sizes are equally imprecise in + * the original 'fieldsSize' value in the heap dump. + */ + } + return true; + } + } + + /** + * An instance of this class can be created and passed to + * Snapshot.Builder.setObjTableSizePolicy() to customize the way that the initial object table + * size is calculated. + */ + public interface ObjTableSizePolicy { + + /** + * Given the .hprof file size, returns the initial size of object table. + */ + int getInitialObjTableSize(long hprofFileSize); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/StackFrame.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/StackFrame.java new file mode 100644 index 00000000..c1bf2868 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/StackFrame.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +/* + * The Original Code is HAT. The Initial Developer of the Original Code is Bill Foote, with + * contributions from others at JavaSoft/Sun. + */ + +package org.openjdk.jmc.joverflow.heap.model; + +/** + * Represents a stack frame. + */ +public class StackFrame { + // Values for the lineNumber data member. These are the same + // as the values used in the JDK 1.2 heap dump file. + public final static int LINE_NUMBER_UNKNOWN = -1; + public final static int LINE_NUMBER_COMPILED = -2; + public final static int LINE_NUMBER_NATIVE = -3; + + private String methodName; + private String methodSignature; + private String className; + private String sourceFileName; + private int lineNumber; + + public StackFrame(String methodName, String methodSignature, String className, String sourceFileName, + int lineNumber) { + this.methodName = methodName; + this.methodSignature = methodSignature; + this.className = className; + this.sourceFileName = sourceFileName; + this.lineNumber = lineNumber; + } + + public void resolve(Snapshot snapshot) { + } + + public String getMethodName() { + return methodName; + } + + public String getMethodSignature() { + return methodSignature; + } + + public String getClassName() { + return className; + } + + public String getSourceFileName() { + return sourceFileName; + } + + public String getLineNumber() { + switch (lineNumber) { + case LINE_NUMBER_UNKNOWN: + return "(unknown)"; + case LINE_NUMBER_COMPILED: + return "(compiled method)"; + case LINE_NUMBER_NATIVE: + return "(native method)"; + default: + return Integer.toString(lineNumber, 10); + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/StackTrace.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/StackTrace.java new file mode 100644 index 00000000..f9b107df --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/StackTrace.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +/* + * The Original Code is HAT. The Initial Developer of the Original Code is Bill Foote, with + * contributions from others at JavaSoft/Sun. + */ + +package org.openjdk.jmc.joverflow.heap.model; + +/** + * Represents a stack trace, that is, an ordered collection of stack frames. + */ +public class StackTrace { + private StackFrame[] frames; + + public StackTrace(StackFrame[] frames) { + this.frames = frames; + } + + /** + * @param depth + * The minimum reasonable depth is 1. + * @return a (possibly new) StackTrace that is limited to depth. + */ + public StackTrace traceForDepth(int depth) { + if (depth >= frames.length) { + return this; + } else { + StackFrame[] f = new StackFrame[depth]; + System.arraycopy(frames, 0, f, 0, depth); + return new StackTrace(f); + } + } + + public void resolve(Snapshot snapshot) { + for (int i = 0; i < frames.length; i++) { + frames[i].resolve(snapshot); + } + } + + public StackFrame[] getFrames() { + return frames; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/UnresolvedObject.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/UnresolvedObject.java new file mode 100644 index 00000000..7655497e --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/model/UnresolvedObject.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.model; + +import org.openjdk.jmc.joverflow.util.MiscUtils; +import org.openjdk.jmc.joverflow.util.StringInterner; + +/** + * Represents a pseudo-object hanging from the object reference that cannot be resolved. That should + * not happen in a well-formed hprof file, but can be observed if e.g. the file has been truncated. + */ +public class UnresolvedObject extends JavaValue { + private final long objId; + + public UnresolvedObject(long objId) { + this.objId = objId; + } + + @Override + public boolean isZero() { + return false; + } + + @Override + public boolean isFloatingPointNumber() { + return false; + } + + @Override + public int getSize() { + return 0; + } + + @Override + public String valueAsString() { + return StringInterner.internString("Unresolved object " + MiscUtils.toHex(objId)); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/ByteArrayReadBuffer.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/ByteArrayReadBuffer.java new file mode 100644 index 00000000..e89498c1 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/ByteArrayReadBuffer.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.parser; + +import java.io.IOException; + +/** + * Implementation of ReadBuffer that uses a simple byte[] backing array. + */ +public class ByteArrayReadBuffer extends ReadBuffer { + private byte[] ar; + + ByteArrayReadBuffer(byte[] array) { + this.ar = array; + } + + @Override + public void get(long longPos, byte[] buf) { + System.arraycopy(ar, (int) longPos, buf, 0, buf.length); + } + + @Override + public void get(long longPos, byte[] buf, int num) { + System.arraycopy(ar, (int) longPos, buf, 0, num); + } + + @Override + public int getInt(long longPos) { + int pos = (int) longPos; + return getInt(pos); + } + + private int getInt(int pos) { + return ((ar[pos] & 0xFF) << 24) | ((ar[pos + 1] & 0xFF) << 16) | ((ar[pos + 2] & 0xFF) << 8) + | (ar[pos + 3] & 0xFF); + } + + @Override + public long getLong(long longPos) throws IOException { + int pos = (int) longPos; + return (((long) getInt(pos)) << 32) | ((getInt(pos + 4)) & 0xFFFFFFFFL); + } + + @Override + public void close() { + // Nothing to do + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/CachedReadBuffer.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/CachedReadBuffer.java new file mode 100644 index 00000000..d7b83a3b --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/CachedReadBuffer.java @@ -0,0 +1,409 @@ +/* + * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.parser; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.logging.Logger; + +/** + * The implementation of ReadBuffer that uses in-heap, pure Java LRU disk cache. This cache consists + * of a number of fixed-size pages; based on experiments, 512KB seems the most optimal size for a + * page. The file itself is divided into 512KB regions with fixed borders. Any page can map to any + * region. Once all pages are mapped and a read from an unmapped region is requested, the least + * recently used page is selected for swapping. We keep the linked list of pages from most recently + * used to least recently used at all times, and update it each time we read from a page that is not + * the same as the page used on the previous call. + *

+ * The need for this cache arises due to the fact that ReadBuffer impl-n that uses mmap() + * (MappedReadBuffer and MappedReadMultiBuffer) takes memory outside the JVM heap, and we can't + * control how much it takes. It looks like at times that amount is pretty high, maybe even higher + * than .hprof file size (which is rather surprising). It's inconvenietn to use a tool that needs an + * unknown amount of memory in addition to the -Xmx setting, and/or is slow when that amount is + * short, though the JVM heap fits into the machine's RAM. It also looks like a specialized cache, + * which takes advantage of knowledge of JOverflow heap dump access patterns, may work better than + * the standard one-size-fits-all mmap paging algorithm. Specifically, we know that once JOverflow + * has read some object, it will not return to it any time soon, or at all. If the object is big + * enough to occupy a whole page, we can immediately discard that page, i.e. make it the first one + * to swap out. + */ +public class CachedReadBuffer extends ReadBuffer { + private final static Logger LOGGER = Logger.getLogger("org.openjdk.jmc.joverflow.heap.parser"); //$NON-NLS-1$ + private static final int PAGE_SIZE_MAGNITUDE = 19; // 512KB, seems optimal from experiments + private static final int PAGE_SIZE = 1 << PAGE_SIZE_MAGNITUDE; + private static final int PAGE_START_MASK = ~(PAGE_SIZE - 1); + + private final int numPagesInPool; + private final FileReadBuffer frb; + private final long fileSize; + private final byte[] buffer; + private final Page[] pageIdxInFileToPage; + private Page mostRecentlyUsed, leastRecentlyUsed; + + private byte tmpBuf[] = new byte[8]; + + // Performance optimizations + private final int[] numBytesReadFromFilePage; + private int pass = 1; + + // Debugging + private static final boolean DEBUG = false; // Perform debug checks + private static final boolean DEBUG_PERF = false; // Track performance + private volatile long numReads, numPageSwaps, lastReadPos; + private volatile int numChanges; + + /** + * Returns a new instance of LRUFileCache for the given file. The size of the cache is set based + * on the file size and the maximum/free heap size. In essence, we want to set it as big as + * possible, but do not exceed 60% of the available heap, so that the tool has enough memory for + * the data it generates later, and memory allocation has enough room to operate without calling + * GC too frequently. + */ + static CachedReadBuffer createInstance(RandomAccessFile file, int preferredSize) throws IOException { + long fileSize = file.length(); + long memForCache = (preferredSize <= 0) ? determineCacheSizeFromFreeMem(fileSize) : preferredSize; + // No need to have a cache larger than the file length + if (memForCache > fileSize) { + memForCache = fileSize; + } + int numPages = (int) ((memForCache + PAGE_SIZE - 1) / PAGE_SIZE); + return new CachedReadBuffer(file, numPages); + } + + private CachedReadBuffer(RandomAccessFile file, int numPgsInPool) throws IOException { + this.numPagesInPool = numPgsInPool; + Page[] pagePool = new Page[numPgsInPool]; + buffer = new byte[numPgsInPool * PAGE_SIZE]; + for (int i = 0; i < numPgsInPool; i++) { + pagePool[i] = new Page(i); + } + frb = new FileReadBuffer(file); + fileSize = file.length(); + int numPagesInFile = (int) ((fileSize >> PAGE_SIZE_MAGNITUDE) + 1); + pageIdxInFileToPage = new Page[numPagesInFile]; + numBytesReadFromFilePage = new int[numPagesInFile]; + + prereadPages(pagePool); + + long memForCache = (long) numPgsInPool * PAGE_SIZE; + LOGGER.fine("\nDisk cache size set to " + (memForCache >> 20) + "MB"); + + if (DEBUG_PERF) { + // Track how well the cached pages are utilized... + LOGGER.fine("CRB: numPagesInPool = " + numPgsInPool); + numPageSwaps = 1; // To avoid accidental division by zero + Thread observer = new Thread() { + @Override + public void run() { + while (true) { + LOGGER.fine("CRB: swps = " + numPageSwaps + ", swps/pg = " + (numPageSwaps / numPagesInPool) + + ", relpos = " + (lastReadPos * 100 / fileSize) + "%" + ", reads/swps = " + + (numReads / numPageSwaps) + ", numChanges = " + numChanges); + try { + Thread.sleep(500); + } catch (InterruptedException ex) { + } + } + } + }; + observer.setDaemon(true); + observer.start(); + } + } + + @Override + public void get(long pos, byte[] buf) throws IOException { + get(pos, buf, buf.length); + } + + @SuppressWarnings("unused") // For unused lines inside this method, due to DEBUG_PERF + @Override + public void get(long pos, byte[] buf, int numBytesToRead) throws IOException { + if (DEBUG_PERF) { + lastReadPos = pos; + numReads++; + } + int numBytesLeft = numBytesToRead; + int posInBuf = 0; + while (numBytesLeft > 0) { + int pageIdxInFile = (int) (pos >> PAGE_SIZE_MAGNITUDE); + Page page = pageIdxInFileToPage[pageIdxInFile]; + + if (page == null) { + page = leastRecentlyUsed; + if (pass == 2) { + // This improves performance a bit by reducing the number of swaps by 3% or so. + // We try to find a page which is both close to LRU and from which enough + // bytes have been read to reasonably expect that no more will be read in the + // future. That it works depends on the fact that JOverflow, during both + // of its heap scans (overall and detailed), generally does not read the + // same byte in the heap dump more than once. The exception are char[] arrays + // shared by several String instances, but that doesn't happen very often. + Page candidate = page; + int threshold = PAGE_SIZE * 4 / 5; + for (int i = 0; i < numPagesInPool / 8; i++) { + int candidateBytes = numBytesReadFromFilePage[candidate.pageIdxInFile]; + if (candidateBytes > threshold) { + page = candidate; + break; + } else { + candidate = candidate.next; + if (candidate == null) { + break; + } + } + } + if (DEBUG_PERF && page != leastRecentlyUsed) { + numChanges++; + } + } + pageIdxInFileToPage[page.pageIdxInFile] = null; + page.fill(pos & PAGE_START_MASK, pageIdxInFile); + numPageSwaps++; + pageIdxInFileToPage[pageIdxInFile] = page; + } + + // Update the linked list + if (page != mostRecentlyUsed) { + if (DEBUG) { + checkListConsistency(page); + } + // We use not the same page as on previous call. It becomes most-recently used. + Page oldPreviousPage = page.previous; + Page oldNextPage = page.next; + page.previous = mostRecentlyUsed; + // Deal with pointers in/to old Page instance that has just been reused + if (oldPreviousPage != null) { + oldPreviousPage.next = oldNextPage; + } + if (oldNextPage != null) { + oldNextPage.previous = oldPreviousPage; + } + + mostRecentlyUsed.next = page; + mostRecentlyUsed = page; + if (page == leastRecentlyUsed) { + leastRecentlyUsed = leastRecentlyUsed.next; + } + page.next = null; + + if (DEBUG) { + checkListConsistency(page); + } + } + + int startPosInPage = (int) (pos - page.startPosInFile); + int numBytesToEndOfPage = (PAGE_SIZE - startPosInPage); + int numBytesToCopy = (numBytesLeft < numBytesToEndOfPage) ? numBytesLeft : numBytesToEndOfPage; + System.arraycopy(buffer, page.startPosInBuffer + startPosInPage, buf, posInBuf, numBytesToCopy); + numBytesReadFromFilePage[pageIdxInFile] += numBytesToCopy; + numBytesLeft -= numBytesToCopy; + if (numBytesLeft > 0) { + posInBuf += numBytesToCopy; + pos += numBytesToCopy; + } + + if (numBytesToCopy == PAGE_SIZE) { + // If we read the whole page, we will not get back to it any time soon. + // Thus it's the best candidate for reuse, and we declare it LRU. + // Note that previously this page was declared MRU. + Page oldPreviousPage = page.previous; + page.previous = null; + page.next = leastRecentlyUsed; + leastRecentlyUsed = page; + mostRecentlyUsed = oldPreviousPage; + mostRecentlyUsed.next = null; + if (DEBUG) { + checkListConsistency(page); + } + } + } + } + + @Override + public int getInt(long pos) throws IOException { + get(pos, tmpBuf, 4); + return ((tmpBuf[0] & 0xFF) << 24) | ((tmpBuf[1] & 0xFF) << 16) | ((tmpBuf[2] & 0xFF) << 8) | (tmpBuf[3] & 0xFF); + } + + @Override + public long getLong(long pos) throws IOException { + get(pos, tmpBuf, 8); + int word1 = ((tmpBuf[0] & 0xFF) << 24) | ((tmpBuf[1] & 0xFF) << 16) | ((tmpBuf[2] & 0xFF) << 8) + | (tmpBuf[3] & 0xFF); + int word2 = ((tmpBuf[4] & 0xFF) << 24) | ((tmpBuf[5] & 0xFF) << 16) | ((tmpBuf[6] & 0xFF) << 8) + | (tmpBuf[7] & 0xFF); + return (((long) word1) << 32) | ((word2) & 0xFFFFFFFFL); + } + + @Override + public void close() { + frb.close(); + } + + /** + * This method should be called between object scanning passes to make the page eviction + * optimization work. A pass is a period when the contents of all or most objects are read. + */ + public void incrementPass() { + pass++; + } + + private static long determineCacheSizeFromFreeMem(long fileSize) { + Runtime runtime = Runtime.getRuntime(); + + // Perform a GC to know for sure how much used/free memory we have + System.gc(); + long freeMem = runtime.freeMemory(); + long totalMem = runtime.totalMemory(); + long maxMem = runtime.maxMemory(); + long usedMem = totalMem - freeMem; + + // Reserve memory for the cache so that 42% of max heap size remains available + // after that + long maxMemForWholeApp = maxMem * 58 / 100; + long minMemForWholeApp = totalMem * 58 / 100; + long maxMemForCache = maxMemForWholeApp - usedMem; + long minMemForCache = minMemForWholeApp - usedMem; + + long memForCache = maxMemForCache; + if (memForCache <= 0) { + // We are severely memory-constrained. It might make sense to throw an + // exception? For now, let's see how things are going to work if we deal + // with this silently. + // Set minimum sensible size for cache + memForCache = fileSize / 12; + } + // In practice, it looks like after cache size of ~ fileSize/4 there is + // no significant performance growth. However, reserving a large cache may + // cause the GC to fire more frequently - at least if the resulting amount + // of used memory is higher than totalMem, but not high enough to force the + // JVM to grow the heap to maxMem size. + // But obviously, there is no reason to reserve a very small cache if enough + // memory is available even within the current totalMem. Furthermore, with too + // small a cache we may get a serious CPU under-use due to disk I/O happening + // all the time - so increased GC activity won't be a problem compared to that. + long maxCacheFromFileSize = fileSize / 4; + + if (memForCache > maxCacheFromFileSize) { + memForCache = maxCacheFromFileSize; + if (memForCache < minMemForCache) { + memForCache = minMemForCache; + } + } + + // Finally, since we use a single byte[] array for the cache, we cannot + // exceed its maximum size + if (memForCache > Integer.MAX_VALUE) { + memForCache = Integer.MAX_VALUE; + } + + return memForCache; + } + + private void prereadPages(Page[] pagePool) throws IOException { + int bytesToRead = (int) Math.min(buffer.length, fileSize); // buffer.length <= fileSize + frb.get(0, buffer, bytesToRead); + int numPages = pagePool.length; // buffer.length == numPages * PAGE_SIZE + for (int i = 0; i < numPages; i++) { + Page page = pagePool[i]; + pageIdxInFileToPage[i] = page; + page.startPosInFile = (long) i * PAGE_SIZE; + page.pageIdxInFile = i; + if (i > 0) { + page.previous = pagePool[i - 1]; + } + if (i < numPages - 1) { + page.next = pagePool[i + 1]; + } + } + leastRecentlyUsed = pagePool[0]; + mostRecentlyUsed = pagePool[numPages - 1]; + } + + // Debugging methods. Activate calls to checkListConsistency() if you make + // any changes or optimizations in linked list management, to verify that + // it didn't cause any problems. + private void checkListConsistency(Page page) { + assertTrue(page.previous != null || page == leastRecentlyUsed, "page.previous is null", page); + assertTrue(page.next != null || page == mostRecentlyUsed, "page.next is null", page); + assertTrue(leastRecentlyUsed != mostRecentlyUsed, "mru == lru", page); + int listSize = 1; + page = leastRecentlyUsed; + while (page != mostRecentlyUsed) { + page = page.next; + listSize++; + } + assertTrue(listSize == numPagesInPool, "listSize = " + listSize, page); + } + + private void assertTrue(boolean v, String errorKind, Page page) { + if (!v) { + throw new Error(reportPageListError(errorKind, page)); + } + } + + private String reportPageListError(String errorKind, Page page) { + return "In-heap file cache internal error: " + errorKind + '\n' + "page = " + page + '\n' + "LRU = " + + leastRecentlyUsed + "; MRU = " + mostRecentlyUsed + '\n' + ", numPagesInPool = " + numPagesInPool; + } + + private class Page { + private final int startPosInBuffer; + private long startPosInFile; + private int pageIdxInFile; + Page next, previous; + + private final int idxInPagePool; // Debugging + + Page(int idxInPagePool) { + startPosInBuffer = idxInPagePool * PAGE_SIZE; + this.idxInPagePool = idxInPagePool; + } + + void fill(long startPosInFile, int pageIdxInFile) throws IOException { + this.pageIdxInFile = pageIdxInFile; + this.startPosInFile = startPosInFile; + long numBytesToEnd = fileSize - startPosInFile; + int numBytesToRead = (numBytesToEnd < PAGE_SIZE) ? (int) numBytesToEnd : PAGE_SIZE; + frb.get(startPosInFile, buffer, startPosInBuffer, numBytesToRead); + } + + @Override + public String toString() { + return "Page " + idxInPagePool + ", idxInFile = " + pageIdxInFile + ", previous = " + + (previous != null ? previous.idxInPagePool : "null") + ", next = " + + (next != null ? next.idxInPagePool : "null"); + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/DumpCorruptedException.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/DumpCorruptedException.java new file mode 100644 index 00000000..f5f0e63b --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/DumpCorruptedException.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.parser; + +/** + * This exception is thrown when the tool detects that a heap dump is corrupted. + */ +public class DumpCorruptedException extends Exception { + static final long serialVersionUID = 1000001L; // To make Eclipse compiler happy + + public DumpCorruptedException(String message) { + super(message); + } + + /** + * A RuntimeException used to wrap an instance of DumpCorruptedException, to avoid polluting + * code with 'throws DumpCorruptedException" statements. + */ + public static class Runtime extends RuntimeException { + static final long serialVersionUID = 1000002L; // To make Eclipse compiler happy + + public Runtime(String message) { + super(new DumpCorruptedException(message)); + } + + @Override + public DumpCorruptedException getCause() { + return (DumpCorruptedException) super.getCause(); + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/FileReadBuffer.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/FileReadBuffer.java new file mode 100644 index 00000000..af209206 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/FileReadBuffer.java @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.parser; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.logging.Logger; + +/** + * Implementation of ReadBuffer using a RandomAccessFile + */ +class FileReadBuffer extends ReadBuffer { + private final static Logger LOGGER = Logger.getLogger("org.openjdk.jmc.joverflow.heap.parser"); //$NON-NLS-1$ + + /** underlying file to read */ + private RandomAccessFile file; + + FileReadBuffer(RandomAccessFile file) { + this.file = file; + } + + private void seek(long pos) throws IOException { + file.seek(pos); + } + + @Override + public void get(long pos, byte[] buf) throws IOException { + seek(pos); + int remainingBytes = buf.length; + do { + int bytesRead = file.read(buf); + remainingBytes -= bytesRead; + } while (remainingBytes > 0); + } + + @Override + public void get(long pos, byte[] buf, int num) throws IOException { + seek(pos); + int remainingBytes = num; + do { + int bytesRead = file.read(buf, 0, num); + remainingBytes -= bytesRead; + } while (remainingBytes > 0); + } + + public void get(long pos, byte[] buf, int startPosInBuf, int num) throws IOException { + seek(pos); + int remainingBytes = num; + do { + int bytesRead = file.read(buf, startPosInBuf, num); + remainingBytes -= bytesRead; + } while (remainingBytes > 0); + } + + @Override + public int getInt(long pos) throws IOException { + seek(pos); + return file.readInt(); + } + + @Override + public long getLong(long pos) throws IOException { + seek(pos); + return file.readLong(); + } + + @Override + public void close() { + try { + file.close(); + } catch (IOException ex) { + LOGGER.severe("Failed to close file " + file + ": " + ex); + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/HeapDumpReader.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/HeapDumpReader.java new file mode 100644 index 00000000..8983cda2 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/HeapDumpReader.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.parser; + +import org.openjdk.jmc.joverflow.heap.model.Snapshot; +import org.openjdk.jmc.joverflow.util.VerboseOutputCollector; + +/** + * Abstract base class for reading object dump files, with a static method that returns a concrete + * implementation. One of the reasons that we don't simply provide a static method for reading a + * snapshot, is that one can invoke HeapDumpReader.read(), and then call getProgressPercentage() + * method on the same object periodically, to get an estimate of the reading progress. Dump reading + * can be also cancelled via cancelReading() call. + */ +public abstract class HeapDumpReader { + /** Read a snapshot from a data input stream. */ + abstract public Snapshot read() throws DumpCorruptedException, HprofParsingCancelledException; + + abstract public int getProgressPercentage(); + + abstract public void cancelReading(); + + /** + * Create a reader for a heap dump. An instance of ReadBuffer.Factory passed to this method + * defines the source of the dump (e.g. a file vs. a byte array), and any additional information + * about the dump. + */ + public static HeapDumpReader createReader( + ReadBuffer.Factory bufFactory, int explicitPointerSize, VerboseOutputCollector vc) + throws DumpCorruptedException { + return new HprofReader(bufFactory, false, explicitPointerSize, vc); + } + +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/HprofParsingCancelledException.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/HprofParsingCancelledException.java new file mode 100644 index 00000000..a028064a --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/HprofParsingCancelledException.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.parser; + +/** + * This exception is thrown by heap dump reading/processing code when the user explicitly cancels + * the operation. + */ +public class HprofParsingCancelledException extends Exception { + static final long serialVersionUID = 1000003L; // To make Eclipse compiler happy + + /** + * A RuntimeException used to wrap an instance of HprofParsingCancelledException, to avoid + * polluting code with 'throws HprofParsingCancelledException" statements. + */ + public static class Runtime extends RuntimeException { + static final long serialVersionUID = 10000004L; // To make Eclipse compiler happy + + public Runtime() { + super(new HprofParsingCancelledException()); + } + + @Override + public HprofParsingCancelledException getCause() { + return (HprofParsingCancelledException) super.getCause(); + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/HprofReader.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/HprofReader.java new file mode 100644 index 00000000..4d249bb4 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/HprofReader.java @@ -0,0 +1,1042 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.parser; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.EOFException; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.HashMap; + +import org.openjdk.jmc.joverflow.heap.model.ArrayTypeCodes; +import org.openjdk.jmc.joverflow.heap.model.JavaBoolean; +import org.openjdk.jmc.joverflow.heap.model.JavaByte; +import org.openjdk.jmc.joverflow.heap.model.JavaChar; +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaDouble; +import org.openjdk.jmc.joverflow.heap.model.JavaField; +import org.openjdk.jmc.joverflow.heap.model.JavaFloat; +import org.openjdk.jmc.joverflow.heap.model.JavaInt; +import org.openjdk.jmc.joverflow.heap.model.JavaLong; +import org.openjdk.jmc.joverflow.heap.model.JavaObjectRef; +import org.openjdk.jmc.joverflow.heap.model.JavaShort; +import org.openjdk.jmc.joverflow.heap.model.JavaThing; +import org.openjdk.jmc.joverflow.heap.model.Root; +import org.openjdk.jmc.joverflow.heap.model.Snapshot; +import org.openjdk.jmc.joverflow.heap.model.StackFrame; +import org.openjdk.jmc.joverflow.heap.model.StackTrace; +import org.openjdk.jmc.joverflow.util.FileUtils; +import org.openjdk.jmc.joverflow.util.LongToObjectMap; +import org.openjdk.jmc.joverflow.util.MiscUtils; +import org.openjdk.jmc.joverflow.util.VerboseOutputCollector; + +/** + * Functionality for reading a hprof file. + */ +class HprofReader extends HeapDumpReader /* imports */ implements ArrayTypeCodes { + final static int MAGIC_NUMBER = 0x4a415641; + // That's "JAVA", the first part of "JAVA PROFILE ..." + private final static String[] VERSIONS = {" PROFILE 1.0\0", " PROFILE 1.0.1\0", " PROFILE 1.0.2\0",}; + + // The following version numbers are indices into VERSIONS. The instance + // data member version is set to one of these, and it drives decisions when + // reading the file. + // + // Version 1.0.1 added HPROF_GC_PRIM_ARRAY_DUMP, which requires no + // version-sensitive parsing. + // + // Version 1.0.1 changed the type of a constant pool entry from a signature + // to a typecode. + // + // Version 1.0.2 added HPROF_HEAP_DUMP_SEGMENT and HPROF_HEAP_DUMP_END + // to allow a large heap to be dumped as a sequence of heap dump segments. + // + // The HPROF agent in J2SE 1.2 through to 5.0 generate a version 1.0.1 + // file. In Java SE 6.0 the version is either 1.0.1 or 1.0.2 depending on + // the size of the heap (normally it will be 1.0.1 but for multi-GB + // heaps the heap dump will not fit in a HPROF_HEAP_DUMP record so the + // dump is generated as version 1.0.2). + @SuppressWarnings("unused") + private final static int VERSION_JDK12BETA3 = 0; + private final static int VERSION_JDK12BETA4 = 1; + private final static int VERSION_JDK6 = 2; + + // Record types + static final int HPROF_UTF8 = 0x01; + static final int HPROF_LOAD_CLASS = 0x02; + static final int HPROF_UNLOAD_CLASS = 0x03; + static final int HPROF_FRAME = 0x04; + static final int HPROF_TRACE = 0x05; + static final int HPROF_ALLOC_SITES = 0x06; + static final int HPROF_HEAP_SUMMARY = 0x07; + + static final int HPROF_START_THREAD = 0x0a; + static final int HPROF_END_THREAD = 0x0b; + + static final int HPROF_HEAP_DUMP = 0x0c; + + static final int HPROF_CPU_SAMPLES = 0x0d; + static final int HPROF_CONTROL_SETTINGS = 0x0e; + static final int HPROF_LOCKSTATS_WAIT_TIME = 0x10; + static final int HPROF_LOCKSTATS_HOLD_TIME = 0x11; + + static final int HPROF_GC_ROOT_UNKNOWN = 0xff; + static final int HPROF_GC_ROOT_JNI_GLOBAL = 0x01; + static final int HPROF_GC_ROOT_JNI_LOCAL = 0x02; + static final int HPROF_GC_ROOT_JAVA_FRAME = 0x03; + static final int HPROF_GC_ROOT_NATIVE_STACK = 0x04; + static final int HPROF_GC_ROOT_STICKY_CLASS = 0x05; + static final int HPROF_GC_ROOT_THREAD_BLOCK = 0x06; + static final int HPROF_GC_ROOT_MONITOR_USED = 0x07; + static final int HPROF_GC_ROOT_THREAD_OBJ = 0x08; + + static final int HPROF_GC_CLASS_DUMP = 0x20; + static final int HPROF_GC_INSTANCE_DUMP = 0x21; + static final int HPROF_GC_OBJ_ARRAY_DUMP = 0x22; + static final int HPROF_GC_PRIM_ARRAY_DUMP = 0x23; + + static final int HPROF_HEAP_DUMP_SEGMENT = 0x1c; + static final int HPROF_HEAP_DUMP_END = 0x2c; + + private final static int T_CLASS = 2; + + private final ReadBuffer.Factory bufFactory; + + private final File hprofFile; // Non-null if we use a real disk file + private final byte[] fileImageBytes; // Non-null if we use a byte[] array with file image + private PositionDataInputStream in; + private final long fileSize; + + private int version; // The version of .hprof being read + + private int dumpsToSkip; + + private int identifierSize; // Size, in bytes, of identifiers in HPROF file. + private final LongToObjectMap names; + + // HashMap, used to map the thread sequence number + // (aka "serial number") to the thread object ID for + // HPROF_GC_ROOT_THREAD_OBJ. ThreadObject is a trivial inner class, + // at the end of this file. + private HashMap threadObjects; + + /** Maps class object ID to class name (in dotted format) */ + private LongToObjectMap classNameFromObjectID; + + // Maps stack frame ID to StackFrame. + // Null if we are not tracking call stacks + private HashMap stackFrames; + + // HashMap maps stack frame ID to StackTrace + // Null if we are not tracking call stacks + private HashMap stackTraces; + + // Maps class serial # to class object ID + // Null if we are not tracking call stacks + private HashMap classNameFromSerialNo; + + private Snapshot.Builder snpBuilder; + + // Maximum size of a mapped byte buffer used for lazy reads of object contents. + // If the heap dump file is longer than this, we need to use more than one BB for it. + private static final int MAX_BB_SIZE = Integer.MAX_VALUE; + + // True if heap dump is longer than 2GB - in that case we'll have to create multiple + // mapped byte buffers to perform random reads efficiently + private final boolean longFile; + + private final ArrayList mappedBBEndOfs; + private long currentBBMaxOfs; + private long prevObjStartOfs; + + // If > 0, use this instead of the value that we half-read/half-guess from the snapshot + private final int explicitPointerSize; + + // Diagnostics and progress tracking + private final VerboseOutputCollector vc; + private volatile boolean cancelled; + + HprofReader(ReadBuffer.Factory bufFactory, boolean callStack, int explicitPointerSize, VerboseOutputCollector vc) + throws DumpCorruptedException { + this.bufFactory = bufFactory; + String fileName = bufFactory.getFileName(); + int dumpNumber = 1; + if (fileName != null) { + int pos = fileName.lastIndexOf('#'); + if (pos > -1) { + String num = fileName.substring(pos + 1, fileName.length()); + try { + dumpNumber = Integer.parseInt(num, 10); + } catch (NumberFormatException ex) { + String msg = "in file name \"" + fileName + "\", a dump number was " + + "expected after the :, but \"" + num + "\" was found instead."; + throw new DumpCorruptedException(msg); + } + fileName = fileName.substring(0, pos); + } + } + + fileImageBytes = bufFactory.getFileImageBytes(); + if (fileImageBytes == null) { // .hprof file will be read from disk + try { + hprofFile = FileUtils.checkFileExistsAndReadable(fileName, false); + this.fileSize = hprofFile.length(); + if (fileSize == 0) { + throw new DumpCorruptedException("file size is 0"); + } + } catch (IOException ex) { + throw new DumpCorruptedException(ex.getMessage()); + } + } else { // We have the .hprof file bytes in Java heap - typically in tests + hprofFile = null; + this.fileSize = fileImageBytes.length; + } + + this.vc = vc; + + this.dumpsToSkip = dumpNumber - 1; + this.explicitPointerSize = explicitPointerSize; + names = new LongToObjectMap<>((int) (fileSize / 100000), false); + threadObjects = new HashMap<>(43); + classNameFromObjectID = new LongToObjectMap<>(1000, false); + if (callStack) { + stackFrames = new HashMap<>(43); + stackTraces = new HashMap<>(43); + classNameFromSerialNo = new HashMap<>(); + } + + longFile = fileSize > MAX_BB_SIZE; + if (longFile) { + mappedBBEndOfs = new ArrayList<>(); + currentBBMaxOfs = MAX_BB_SIZE - 1; + } else { + mappedBBEndOfs = null; + } + } + + @Override + public Snapshot read() throws DumpCorruptedException, HprofParsingCancelledException { + String dumpCorruptedExMsg = ""; + ReadBuffer readBuf = null; + try { + if (hprofFile != null) { + in = new PositionDataInputStream(new BufferedInputStream(new FileInputStream(hprofFile))); + } else { + in = new PositionDataInputStream(new ByteArrayInputStream(fileImageBytes)); + } + + doRead(); + + // Some very simple/obvious sanity checks + if (snpBuilder.getNumAllObjects() == 0) { + throw new DumpCorruptedException("did not read any objects"); + } + if (snpBuilder.getNumClasses() == 0) { + throw new DumpCorruptedException("did not read any classes"); + } + + snpBuilder.onFinishReadObjects(); + + long[] mappedBBEndOfsArray = null; + if (mappedBBEndOfs != null) { + mappedBBEndOfsArray = new long[mappedBBEndOfs.size() + 1]; + for (int i = 0; i < mappedBBEndOfs.size(); i++) { + mappedBBEndOfsArray[i] = mappedBBEndOfs.get(i); + } + mappedBBEndOfsArray[mappedBBEndOfsArray.length - 1] = fileSize - 1; + } + readBuf = bufFactory.create(mappedBBEndOfsArray); + } catch (IOException ex) { + dumpCorruptedExMsg = "caught exception " + ex + ". Details:\n"; + StringWriter exWriterBuf = new StringWriter(200); + ex.printStackTrace(new PrintWriter(exWriterBuf)); + dumpCorruptedExMsg += exWriterBuf.toString(); + } finally { + if (in != null) { + try { + in.close(); + } catch (IOException ex) { + dumpCorruptedExMsg += "\nAlso, could not close the stream properly: caught exception " + ex; + if (readBuf != null) { + readBuf.close(); + } + } + } + } + + if (dumpCorruptedExMsg.length() > 0) { + throw new DumpCorruptedException(dumpCorruptedExMsg); + } + + try { + return snpBuilder.buildSnapshot(readBuf); + } catch (RuntimeException ex) { + if (readBuf != null) { + readBuf.close(); + } + throw ex; + } catch (Error er) { // Most likely an OOM + if (readBuf != null) { + readBuf.close(); + } + throw er; + } + } + + @Override + public synchronized int getProgressPercentage() { + if (in != null) { + return (int) (in.position() * 100 / fileSize); + } else { + return 0; + } + } + + @Override + public void cancelReading() { + cancelled = true; + } + + private void doRead() throws DumpCorruptedException, IOException, HprofParsingCancelledException { + int magicNumber = in.readInt(); + if (magicNumber != MAGIC_NUMBER) { + throw new DumpCorruptedException("unrecognized magic number: " + magicNumber); + } + + version = readVersionHeader(); + identifierSize = in.readInt(); + if (identifierSize != 4 && identifierSize != 8) { + throw new DumpCorruptedException("unsupported format: " + "specifies pointer size of " + identifierSize + + ". JOverflow supports only size 4 and 8."); + } + + snpBuilder = new Snapshot.Builder(fileSize, identifierSize, explicitPointerSize, vc); + + skipBytes(8); // long creationDate = in.readLong(); +// System.out.println("Dump file created " + (new Date(creationDate))); + + while (true) { + int type; + try { + type = in.readUnsignedByte(); + } catch (EOFException ignored) { + break; + } + in.readInt(); // Timestamp of this record + // Length of record: readInt() will return negative value for record length >2GB. So store 32bit value in long to keep it unsigned. + long length = in.readInt() & 0xffffffffL; +// System.out.println("Read record type " + type + ", length " + length + " at position " + toHex(currPos)); + if (length < 0) { + throw new DumpCorruptedException( + "bad record length of " + length + " at byte " + (in.position() - 4) + " of file."); + } + switch (type) { + case HPROF_UTF8: { + long id = readID(); + byte[] chars = new byte[(int) length - identifierSize]; + in.readFully(chars); + names.put(id, new String(chars)); + break; + } + + case HPROF_LOAD_CLASS: { + int serialNo = in.readInt(); // Not used + long classID = readID(); + in.readInt(); // int stackTraceSerialNo, unused + long classNameID = readID(); + String nm = getNameFromID(classNameID).replace('/', '.'); + classNameFromObjectID.put(classID, nm); + if (classNameFromSerialNo != null) { + classNameFromSerialNo.put(serialNo, nm); + } + break; + } + + case HPROF_HEAP_DUMP: { + if (dumpsToSkip <= 0) { + try { + vc.debug("Sub-dump of length " + length + " starts at position " + in.position()); + readHeapDump(length); + } catch (EOFException exp) { + handleEOF(exp); + } +// System.out.println("Finished processing instances in heap dump."); + return; + } else { + dumpsToSkip--; + skipBytes(length); + } + break; + } + + case HPROF_HEAP_DUMP_END: { + if (version >= VERSION_JDK6) { + if (dumpsToSkip <= 0) { + skipBytes(length); // should be no-op + return; + } else { + // skip this dump (of the end record for a sequence of dump segments) + dumpsToSkip--; + } + } else { + // HPROF_HEAP_DUMP_END only recognized in >= 1.0.2 + vc.addWarning("Ignoring unrecognized record type", Integer.toString(type)); + } + skipBytes(length); // should be no-op + break; + } + + case HPROF_HEAP_DUMP_SEGMENT: { + if (version >= VERSION_JDK6) { + if (dumpsToSkip <= 0) { + try { + vc.debug("Segment of length " + length + " starts at position " + in.position()); + // read the dump segment + readHeapDump(length); + } catch (EOFException exp) { + handleEOF(exp); + } + } else { + // all segments comprising the heap dump will be skipped + skipBytes(length); + } + } else { + // HPROF_HEAP_DUMP_SEGMENT only recognized in >= 1.0.2 + vc.addWarning("Ignoring unrecognized record type", Integer.toString(type)); + skipBytes(length); + } + break; + } + + case HPROF_FRAME: { + if (stackFrames == null) { + skipBytes(length); + } else { + long id = readID(); + String methodName = getNameFromID(readID()); + String methodSig = getNameFromID(readID()); + String sourceFile = getNameFromID(readID()); + int classSer = in.readInt(); + String className = classNameFromSerialNo.get(Integer.valueOf(classSer)); + int lineNumber = in.readInt(); + if (lineNumber < StackFrame.LINE_NUMBER_NATIVE) { + vc.addWarning("Weird stack frame line number", Integer.toString(lineNumber)); + lineNumber = StackFrame.LINE_NUMBER_UNKNOWN; + } + stackFrames.put(id, new StackFrame(methodName, methodSig, className, sourceFile, lineNumber)); + } + break; + } + + case HPROF_TRACE: { + if (stackTraces == null) { + skipBytes(length); + } else { + int serialNo = in.readInt(); + in.readInt(); // int threadSeq, not used + StackFrame[] frames = new StackFrame[in.readInt()]; + for (int i = 0; i < frames.length; i++) { + long fid = readID(); + frames[i] = stackFrames.get(Long.valueOf(fid)); + if (frames[i] == null) { + throw new DumpCorruptedException("stack frame " + toHex(fid) + " not found"); + } + } + stackTraces.put(serialNo, new StackTrace(frames)); + } + break; + } + + case HPROF_HEAP_SUMMARY: + case HPROF_UNLOAD_CLASS: + case HPROF_ALLOC_SITES: + case HPROF_START_THREAD: + case HPROF_END_THREAD: + case HPROF_CPU_SAMPLES: + case HPROF_CONTROL_SETTINGS: + case HPROF_LOCKSTATS_WAIT_TIME: + case HPROF_LOCKSTATS_HOLD_TIME: { + // Ignore these record types + skipBytes(length); + break; + } + + default: { + skipBytes(length); + vc.addWarning("Ignoring unrecognized record type", Integer.toString(type)); + } + } + } + } + + private void skipBytes(long length) throws IOException, DumpCorruptedException { + long remainingBytes = length; + do { + int skippedBytes = in.skipBytes((int) length); + remainingBytes -= skippedBytes; + if (remainingBytes > 0) { + if (in.position() >= fileSize) { + throw new DumpCorruptedException("Reached end of file while trying to skip " + length + " bytes"); + + } + } + } while (remainingBytes > 0); + } + + private int readVersionHeader() throws IOException, DumpCorruptedException { + int candidatesLeft = VERSIONS.length; + boolean[] matched = new boolean[VERSIONS.length]; + for (int i = 0; i < candidatesLeft; i++) { + matched[i] = true; + } + + int pos = 0; + while (candidatesLeft > 0) { + char c = (char) in.readByte(); + for (int i = 0; i < VERSIONS.length; i++) { + if (matched[i]) { + if (c != VERSIONS[i].charAt(pos)) { // Not matched + matched[i] = false; + --candidatesLeft; + } else if (pos == VERSIONS[i].length() - 1) { // Full match + vc.debug("Hprof file version: " + VERSIONS[i]); + return i; + } + } + } + ++pos; + } + throw new DumpCorruptedException("version string not recognized at byte " + (pos + 3)); + } + + private void readHeapDump(long dumpLength) + throws DumpCorruptedException, IOException, HprofParsingCancelledException { + long startPos = in.position(); + long endPos = startPos + dumpLength; + // "Chunks" below are used to check for cancellation periodically + int curChunk = (int) (in.position() >> 19); // Check every 512K + + long id, pos; + while ((pos = in.position()) < endPos) { + int recordType = in.readUnsignedByte(); + + int newCurChunk = (int) (pos >> 19); + if (newCurChunk > curChunk) { + curChunk = newCurChunk; + checkForCancellation(); + } + + switch (recordType) { + case HPROF_GC_INSTANCE_DUMP: { + readInstance(); + break; + } + case HPROF_GC_OBJ_ARRAY_DUMP: { + readArray(false); + break; + } + case HPROF_GC_PRIM_ARRAY_DUMP: { + readArray(true); + break; + } + + case HPROF_GC_ROOT_UNKNOWN: { + id = readID(); + snpBuilder.addRoot(new Root(id, 0, Root.UNKNOWN, "")); + break; + } + case HPROF_GC_ROOT_THREAD_OBJ: { + id = readID(); + int threadSeq = in.readInt(); + int stackSeq = in.readInt(); + threadObjects.put(threadSeq, new ThreadObject(id, stackSeq)); + break; + } + case HPROF_GC_ROOT_JNI_GLOBAL: { + id = readID(); + readID(); // long globalRefId, ignored for now + snpBuilder.addRoot(new Root(id, 0, Root.JNI_GLOBAL, "")); + break; + } + case HPROF_GC_ROOT_JNI_LOCAL: { + id = readID(); + int threadSeq = in.readInt(); + int depth = in.readInt(); + ThreadObject to = getThreadObjectFromSequence(threadSeq); + StackTrace st = getStackTraceFromSerial(to.stackSeq); + if (st != null) { + st = st.traceForDepth(depth + 1); + } + snpBuilder.addRoot(new Root(id, to.threadId, Root.JNI_LOCAL, "", st)); + break; + } + case HPROF_GC_ROOT_JAVA_FRAME: { + id = readID(); + int threadSeq = in.readInt(); + int depth = in.readInt(); + ThreadObject to = getThreadObjectFromSequence(threadSeq); + StackTrace st = getStackTraceFromSerial(to.stackSeq); + if (st != null) { + st = st.traceForDepth(depth + 1); + } + snpBuilder.addRoot(new Root(id, to.threadId, Root.JAVA_LOCAL, "", st)); + break; + } + case HPROF_GC_ROOT_NATIVE_STACK: { + id = readID(); + int threadSeq = in.readInt(); + ThreadObject to = getThreadObjectFromSequence(threadSeq); + StackTrace st = getStackTraceFromSerial(to.stackSeq); + snpBuilder.addRoot(new Root(id, to.threadId, Root.NATIVE_STACK, "", st)); + break; + } + case HPROF_GC_ROOT_STICKY_CLASS: { + id = readID(); + snpBuilder.addRoot(new Root(id, 0, Root.SYSTEM_CLASS, "")); + break; + } + case HPROF_GC_ROOT_THREAD_BLOCK: { + id = readID(); + int threadSeq = in.readInt(); + ThreadObject to = getThreadObjectFromSequence(threadSeq); + StackTrace st = getStackTraceFromSerial(to.stackSeq); + snpBuilder.addRoot(new Root(id, to.threadId, Root.THREAD_BLOCK, "", st)); + break; + } + case HPROF_GC_ROOT_MONITOR_USED: { + id = readID(); + snpBuilder.addRoot(new Root(id, 0, Root.BUSY_MONITOR, "")); + break; + } + case HPROF_GC_CLASS_DUMP: { + readClass(); + break; + } + default: { + throw new DumpCorruptedException("unrecognized heap dump sub-record type: " + recordType + + ". Technical info: position = " + pos + ", bytes left = " + (endPos - pos)); + } + } + } + + if (pos != endPos) { + vc.addWarning("Error reading heap dump or heap dump segment", + "Byte count is " + pos + " instead of " + endPos + ". Difference is " + (endPos - pos)); + skipBytes(endPos - pos); + } + } + + private long readID() throws IOException { + return (identifierSize == 4) ? (Snapshot.SMALL_ID_MASK & in.readInt()) : in.readLong(); + } + + /** + * Read a java value. If result is non-null, it's expected to be an array of one element. We use + * it to fake multiple return values. Returns the number of bytes read. + */ + private int readValue(JavaThing[] resultArr) throws DumpCorruptedException, IOException { + byte type = in.readByte(); + return 1 + readValueForType(type, resultArr); + } + + private int readValueForType(byte type, JavaThing[] resultArr) throws DumpCorruptedException, IOException { + if (version >= VERSION_JDK12BETA4) { + type = signatureFromTypeId(type); + } + return readValueForTypeSignature(type, resultArr); + } + + private int readValueForTypeSignature(byte type, JavaThing[] resultArr) throws DumpCorruptedException, IOException { + switch (type) { + case '[': + case 'L': { + long id = readID(); + if (resultArr != null) { + resultArr[0] = new JavaObjectRef(id); + } + return identifierSize; + } + case 'Z': { + int b = in.readByte(); + if (b != 0 && b != 1) { + vc.addWarning("Illegal boolean value read", Integer.toString(b)); + } + if (resultArr != null) { + resultArr[0] = new JavaBoolean(b != 0); + } + return 1; + } + case 'B': { + byte b = in.readByte(); + if (resultArr != null) { + resultArr[0] = new JavaByte(b); + } + return 1; + } + case 'S': { + short s = in.readShort(); + if (resultArr != null) { + resultArr[0] = new JavaShort(s); + } + return 2; + } + case 'C': { + char ch = in.readChar(); + if (resultArr != null) { + resultArr[0] = new JavaChar(ch); + } + return 2; + } + case 'I': { + int val = in.readInt(); + if (resultArr != null) { + resultArr[0] = new JavaInt(val); + } + return 4; + } + case 'J': { + long val = in.readLong(); + if (resultArr != null) { + resultArr[0] = new JavaLong(val); + } + return 8; + } + case 'F': { + float val = in.readFloat(); + if (resultArr != null) { + resultArr[0] = new JavaFloat(val); + } + return 4; + } + case 'D': { + double val = in.readDouble(); + if (resultArr != null) { + resultArr[0] = new JavaDouble(val); + } + return 8; + } + default: { + throw new DumpCorruptedException("Bad value signature: " + type); + } + } + } + + private ThreadObject getThreadObjectFromSequence(int threadSeq) throws DumpCorruptedException, IOException { + ThreadObject to = threadObjects.get(Integer.valueOf(threadSeq)); + if (to == null) { + throw new DumpCorruptedException("thread " + threadSeq + " not found for JNI local ref"); + } + return to; + } + + private String getNameFromID(long id) throws IOException { + if (id == 0L) { + return ""; + } + String result = names.get(id); + if (result == null) { + vc.addWarning("name not found", "at " + toHex(id)); + return "unresolved name " + toHex(id); + } + return result; + } + + private StackTrace getStackTraceFromSerial(int ser) throws IOException { + if (stackTraces == null) { + return null; + } + StackTrace result = stackTraces.get(Integer.valueOf(ser)); + if (result == null) { + vc.addWarning("Stack trace not found", "for serial # " + ser); + } + return result; + } + + /** Handles a HPROF_GC_CLASS_DUMP. Returns the number of bytes read. */ + private int readClass() throws DumpCorruptedException, IOException { + long id = readID(); + skipBytes(4); // StackTrace stackTrace = getStackTraceFromSerial(in.readInt()); + long superId = readID(); + long classLoaderId = readID(); + long signersId = readID(); + long protDomainId = readID(); + readID(); // long reserved1, unused + readID(); // long reserved2, unused + int fieldsSize = in.readInt(); + int bytesRead = 7 * identifierSize + 8; + + int numConstPoolEntries = in.readUnsignedShort(); + bytesRead += 2; + for (int i = 0; i < numConstPoolEntries; i++) { + in.readUnsignedShort(); // int index, unused + bytesRead += 2; + bytesRead += readValue(null); // We ignore the values + } + + int numStatics = in.readUnsignedShort(); + bytesRead += 2; + // We may need additional quasi-fields for signers and protection domain + int numQuasiFields = (signersId != 0 || protDomainId != 0) ? 2 : 0; + int nAllStatics = numStatics + numQuasiFields; + JavaField[] staticFields = nAllStatics > 0 ? new JavaField[nAllStatics] : JavaClass.NO_FIELDS; + JavaThing[] staticValues = nAllStatics > 0 ? new JavaThing[nAllStatics] : JavaClass.NO_VALUES; + if (numStatics > 0) { + JavaThing[] valueBin = new JavaThing[1]; + for (int i = 0; i < numStatics; i++) { + long nameId = readID(); + bytesRead += identifierSize; + byte type = in.readByte(); + bytesRead++; + bytesRead += readValueForType(type, valueBin); + String fieldName = getNameFromID(nameId); + if (version >= VERSION_JDK12BETA4) { + type = signatureFromTypeId(type); + } + staticFields[i] = JavaField.newInstance(fieldName, (char) type, snpBuilder.getPointerSize()); + staticValues[i] = valueBin[0]; + } + } + if (numQuasiFields > 0) { + JavaField.addStaticQuaziFields(staticFields); + } + + int numFields = in.readUnsignedShort(); + bytesRead += 2; + JavaField[] fields = numFields > 0 ? new JavaField[numFields] : JavaClass.NO_FIELDS; + for (int i = 0; i < numFields; i++) { + long nameId = readID(); + bytesRead += identifierSize; + byte type = in.readByte(); + bytesRead++; + String fieldName = getNameFromID(nameId); + if (version >= VERSION_JDK12BETA4) { + type = signatureFromTypeId(type); + } + fields[i] = JavaField.newInstance(fieldName, (char) type, snpBuilder.getPointerSize()); + } + + String name = classNameFromObjectID.get(id); + if (name == null) { + vc.addWarning("Class name not found", "for " + toHex(id)); + name = "unknown-name@" + toHex(id); + } + + JavaClass c = new JavaClass(id, name, superId, classLoaderId, signersId, protDomainId, fields, staticFields, + staticValues, fieldsSize, snpBuilder.getInMemoryInstanceSize(fieldsSize)); + snpBuilder.addClass(c); + + return bytesRead; + } + + private String toHex(long addr) { + return MiscUtils.toHex(addr); + } + + /** + * Handles a HPROF_GC_INSTANCE_DUMP Return number of bytes read + */ + private int readInstance() throws DumpCorruptedException, IOException { + long objOfsInFile = in.position(); + long id = readID(); + skipBytes(4); // StackTrace stackTrace = getStackTraceFromSerial(in.readInt()); + long classID = readID(); + int objDataSize = in.readInt(); + int bytesRead = (2 * identifierSize) + 8 + objDataSize; + skipBytes(objDataSize); + snpBuilder.addJavaObject(id, classID, objOfsInFile, objDataSize); + if (longFile) { + handlePossibleBBBorder(objOfsInFile); + } + return bytesRead; + } + + /** + * Handles a HPROF_GC_OBJ_ARRAY_DUMP or HPROF_GC_PRIM_ARRAY_DUMP. Returns number of bytes read. + */ + private int readArray(boolean isPrimitive) throws DumpCorruptedException, IOException { + long objOfsInFile = in.position(); + long id = readID(); + skipBytes(4); // StackTrace stackTrace = getStackTraceFromSerial(in.readInt()); + int num = in.readInt(); + int bytesRead = identifierSize + 8; + long arrayClassID; + if (isPrimitive) { + arrayClassID = in.readByte(); + bytesRead++; + } else { + arrayClassID = readID(); + bytesRead += identifierSize; + } + + // Check for primitive arrays: + char primitiveSignature = 0x00; + int elSize = 0; + if (isPrimitive || version < VERSION_JDK12BETA4) { + switch ((int) arrayClassID) { + case T_BOOLEAN: { + primitiveSignature = 'Z'; + elSize = 1; + break; + } + case T_CHAR: { + primitiveSignature = 'C'; + elSize = 2; + break; + } + case T_FLOAT: { + primitiveSignature = 'F'; + elSize = 4; + break; + } + case T_DOUBLE: { + primitiveSignature = 'D'; + elSize = 8; + break; + } + case T_BYTE: { + primitiveSignature = 'B'; + elSize = 1; + break; + } + case T_SHORT: { + primitiveSignature = 'S'; + elSize = 2; + break; + } + case T_INT: { + primitiveSignature = 'I'; + elSize = 4; + break; + } + case T_LONG: { + primitiveSignature = 'J'; + elSize = 8; + break; + } + } + if (version >= VERSION_JDK12BETA4 && primitiveSignature == 0x00) { + throw new DumpCorruptedException("unrecognized typecode: " + arrayClassID); + } + } + + int dataSize = isPrimitive ? elSize * num : identifierSize * num; + if (in.position() + dataSize > fileSize) { + throw new DumpCorruptedException((isPrimitive ? "Primitive" : "Object") + " array at position " + + in.position() + " is " + dataSize + " bytes long, that does not fit into the dump file"); + } + + bytesRead += dataSize; + skipBytes(dataSize); + + if (isPrimitive) { + snpBuilder.addJavaValueArray(id, primitiveSignature, objOfsInFile, num, dataSize); + } else { + snpBuilder.addJavaObjectArray(id, arrayClassID, objOfsInFile, num, dataSize); + } + if (longFile) { + handlePossibleBBBorder(objOfsInFile); + } + + return bytesRead; + } + + private byte signatureFromTypeId(byte typeId) throws DumpCorruptedException, IOException { + switch (typeId) { + case T_CLASS: + return (byte) 'L'; + case T_BOOLEAN: + return (byte) 'Z'; + case T_CHAR: + return (byte) 'C'; + case T_FLOAT: + return (byte) 'F'; + case T_DOUBLE: + return (byte) 'D'; + case T_BYTE: + return (byte) 'B'; + case T_SHORT: + return (byte) 'S'; + case T_INT: + return (byte) 'I'; + case T_LONG: + return (byte) 'J'; + default: + throw new DumpCorruptedException("invalid type id of " + typeId); + } + } + + private void handlePossibleBBBorder(long thisObjStartOfs) { + if (thisObjStartOfs >= currentBBMaxOfs) { + if (prevObjStartOfs > 0) { + // Normal case + mappedBBEndOfs.add(prevObjStartOfs - 1); + } else { + // Seems to happen only in tests, when maxBBSize is small + mappedBBEndOfs.add(Long.valueOf(MAX_BB_SIZE)); + } + currentBBMaxOfs = mappedBBEndOfs.get(mappedBBEndOfs.size() - 1) + MAX_BB_SIZE; + } + prevObjStartOfs = thisObjStartOfs; + } + + private void handleEOF(EOFException exp) { + vc.addWarning("Unexpected EOF", "Will miss information"); + // we have EOF, we have to tolerate missing references + snpBuilder.setUnresolvedObjectsOk(true); + } + + private void checkForCancellation() throws HprofParsingCancelledException { + if (cancelled) { + throw new HprofParsingCancelledException(); + } + } + + /** + * A trivial data-holder class for HPROF_GC_ROOT_THREAD_OBJ. + */ + private static class ThreadObject { + + long threadId; + int stackSeq; + + ThreadObject(long threadId, int stackSeq) { + this.threadId = threadId; + this.stackSeq = stackSeq; + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/MappedReadBuffer.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/MappedReadBuffer.java new file mode 100644 index 00000000..5b28db5a --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/MappedReadBuffer.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.parser; + +import java.io.IOException; +import java.nio.MappedByteBuffer; + +/** + * Implementation of ReadBuffer using memory-mapped file buffer. + */ +class MappedReadBuffer extends ReadBuffer { + + private final MappedByteBuffer buf; + + MappedReadBuffer(MappedByteBuffer buf) { + this.buf = buf; + } + + @Override + public synchronized void get(long pos, byte[] res) throws IOException { + seek(pos); + buf.get(res); + } + + @Override + public synchronized void get(long pos, byte[] res, int num) throws IOException { + seek(pos); + buf.get(res, 0, num); + } + + @Override + public synchronized int getInt(long pos) throws IOException { + seek(pos); + return buf.getInt(); + } + + @Override + public synchronized long getLong(long pos) throws IOException { + seek(pos); + return buf.getLong(); + } + + private void seek(long pos) throws IOException { + assert pos <= Integer.MAX_VALUE : "position overflow"; + buf.position((int) pos); + } + + @Override + public void close() { + // Nothing to do + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/MappedReadMultiBuffer.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/MappedReadMultiBuffer.java new file mode 100644 index 00000000..19c6f574 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/MappedReadMultiBuffer.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.parser; + +import java.io.IOException; +import java.nio.MappedByteBuffer; + +/** + * The implementation of ReadBuffer that supports files of size larger than Integer.MAX_VALUE, i.e. + * larger than a single MappedByteBuffer can hold. It is done by opening multiple MappedByteBuffers + * for the file, and choosing the right one every time we need to read something from the file. + */ +class MappedReadMultiBuffer extends ReadBuffer { + private final MappedByteBuffer bufs[]; + private final long mappedBBEndOfs[]; + private final int maxBufSize; + + MappedReadMultiBuffer(MappedByteBuffer bufs[], long mappedBBEndOfs[], int maxBufSize) { + this.bufs = bufs; + this.mappedBBEndOfs = mappedBBEndOfs; + this.maxBufSize = maxBufSize; + } + + private MappedByteBuffer seek(long pos) throws IOException { + int bufIdx = (int) (pos / maxBufSize); + while (pos > mappedBBEndOfs[bufIdx]) { + bufIdx++; + } + MappedByteBuffer buf = bufs[bufIdx]; + if (bufIdx > 0) { + buf.position((int) (pos - mappedBBEndOfs[bufIdx - 1] - 1)); + } else { + buf.position((int) pos); + } + return buf; + } + + @Override + public void get(long pos, byte[] res) throws IOException { + MappedByteBuffer buf = seek(pos); + buf.get(res); + } + + @Override + public void get(long pos, byte[] res, int num) throws IOException { + MappedByteBuffer buf = seek(pos); + buf.get(res, 0, num); + } + + @Override + public int getInt(long pos) throws IOException { + MappedByteBuffer buf = seek(pos); + return buf.getInt(); + } + + @Override + public long getLong(long pos) throws IOException { + MappedByteBuffer buf = seek(pos); + return buf.getLong(); + } + + @Override + public void close() { + // Nothing to do + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/PositionDataInputStream.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/PositionDataInputStream.java new file mode 100644 index 00000000..f6f42430 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/PositionDataInputStream.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.parser; + +import java.io.DataInputStream; +import java.io.InputStream; + +/** + * A DataInputStream that keeps track of total bytes read (in effect 'position' in stream) so far. + */ +public class PositionDataInputStream extends DataInputStream { + + public PositionDataInputStream(InputStream in) { + super(in instanceof PositionInputStream ? in : new PositionInputStream(in)); + } + + public long position() { + return ((PositionInputStream) in).position(); + } + + @Override + public boolean markSupported() { + return false; + } + + @Override + public void mark(int readLimit) { + throw new UnsupportedOperationException("mark"); + } + + @Override + public void reset() { + throw new UnsupportedOperationException("reset"); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/PositionInputStream.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/PositionInputStream.java new file mode 100644 index 00000000..e97fa776 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/PositionInputStream.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.parser; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; + +/** + * InputStream that keeps track of total bytes read (in effect 'position' in stream) from the input + * stream. + */ +public class PositionInputStream extends FilterInputStream { + private long position = 0L; + + public PositionInputStream(InputStream in) { + super(in); + } + + @Override + public int read() throws IOException { + int res = super.read(); + if (res != -1) { + position++; + } + return res; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int res = super.read(b, off, len); + if (res != -1) { + position += res; + } + return res; + } + + @Override + public long skip(long n) throws IOException { + long res = super.skip(n); + position += res; + return res; + } + + @Override + public boolean markSupported() { + return false; + } + + @Override + public void mark(int readLimit) { + throw new UnsupportedOperationException("mark"); + } + + @Override + public void reset() { + throw new UnsupportedOperationException("reset"); + } + + public long position() { + return position; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/ReadBuffer.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/ReadBuffer.java new file mode 100644 index 00000000..f9659ade --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/heap/parser/ReadBuffer.java @@ -0,0 +1,214 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.heap.parser; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; + +import org.openjdk.jmc.common.io.IOToolkit; + +/** + * Abstract superclass for positionable read only buffer classes. A concrete implementation may use + * a mmapped file, a random-access file, a backing array in JVM memory, etc. + */ +public abstract class ReadBuffer { + + /** + * An instance of a concrete subclass of this class serves three purposes: + *

    + *
  • Encapsulates information about the dump source (the API supports a file vs. a byte[] + * array, the latter supposed to be used in tests)
    • + *
  • Provides a create() method that allows for delayed creation of a ReadBuffer instance of a + * type associated with a Factory type (e.g. a CachedReadBufferFactory creates an instance of + * CachedReadBuffer)
    • + *
  • Contains additional information specified at construction time, for example preferred + * size for the CachedReadBuffer to be created.
    • + *
+ */ + public static abstract class Factory { + + abstract String getFileName(); + + abstract byte[] getFileImageBytes(); + + /** + * This method is called internally by HprofReader after the heap dump file has been parsed, + * and we are about to start random access operations with it. supplementalInfo can be any + * object; what it is exactly is an internal contract between HprofReader and Factory + * subclasses. + */ + abstract public ReadBuffer create(Object supplementalInfo) throws IOException; + } + + /** + * This factory creates an instance of CachedReadBuffer, which is our own HPROF file cache + * implementation that uses memory only in the JVM heap. + */ + public static class CachedReadBufferFactory extends Factory { + private final String fileName; + private final int preferredCacheSize; + + /** + * If preferredSize is greater than zero, it will be used as a disk cache size without + * further checks. Otherwise, an appropriate cache size will be calculated based on the + * available JVM heap size and other factors. + */ + public CachedReadBufferFactory(String fileName, int preferredCacheSize) { + this.fileName = fileName; + this.preferredCacheSize = preferredCacheSize; + } + + @Override + String getFileName() { + return fileName; + } + + @Override + byte[] getFileImageBytes() { + return null; + } + + @Override + public ReadBuffer create(Object supplementalInfo) throws IOException { + RandomAccessFile file = new RandomAccessFile(fileName, "r"); + try { + return CachedReadBuffer.createInstance(file, preferredCacheSize); + } catch (IOException e) { + IOToolkit.closeSilently(file); + throw e; + } + } + } + + /** + * This factory creates an instance of MappedBuffer or MappedReadMultiBuffer, which uses mmap() + * internally, which uses memory outside the JVM heap. + *

+ * Note that if the file given to this object is over 2GB, an instance of MappedReadMultiBuffer + * is created, which critically depends on data in a long[] array passed to it via the create() + * method of this factory. This array specifies the borders of 2GB-or-so "segments" within the + * HPROF file, and it can only be generated by HprofReader. + */ + public static class MmappedBufferFactory extends Factory { + private final String fileName; + + @Override + String getFileName() { + return fileName; + } + + @Override + byte[] getFileImageBytes() { + return null; + } + + public MmappedBufferFactory(String fileName) { + this.fileName = fileName; + } + + @Override + public ReadBuffer create(Object supplementalInfo) throws IOException { + long mappedBBEndOfs[] = (long[]) supplementalInfo; + int maxSingleMappedBufSize = Integer.MAX_VALUE; + RandomAccessFile file = new RandomAccessFile(fileName, "r"); + FileChannel ch = file.getChannel(); + try { + long size = ch.size(); + + if (size <= maxSingleMappedBufSize) { + // Use a single backing MappedByteBuffer + MappedByteBuffer buf = ch.map(FileChannel.MapMode.READ_ONLY, 0, size); + return new MappedReadBuffer(buf); + } else if (mappedBBEndOfs != null) { + // Use multiple backing MappedByteBuffers + // Actually, it looks like the internal implementation of MappedByteBuffer supports + // long file size and offsets. However, there is no public API for it... + MappedByteBuffer[] bufs = new MappedByteBuffer[mappedBBEndOfs.length]; + long startOfs = 0; + for (int i = 0; i < mappedBBEndOfs.length; i++) { + bufs[i] = ch.map(FileChannel.MapMode.READ_ONLY, startOfs, mappedBBEndOfs[i] - startOfs + 1); + startOfs = mappedBBEndOfs[i] + 1; + } + ch.close(); + file.close(); + return new MappedReadMultiBuffer(bufs, mappedBBEndOfs, maxSingleMappedBufSize); + } + } finally { + IOToolkit.closeSilently(ch); + IOToolkit.closeSilently(file); + } + return new FileReadBuffer(file); + } + } + + /** + * This factory creates an instance of ByteArrayBuffer, which uses file contents that have + * already been read directly into memory. + */ + public static class ByteArrayBufferFactory extends Factory { + private final byte[] fileImageBytes; + + public ByteArrayBufferFactory(byte[] fileImageBytes) { + this.fileImageBytes = fileImageBytes; + } + + @Override + String getFileName() { + return null; + } + + @Override + byte[] getFileImageBytes() { + return fileImageBytes; + } + + @Override + public ReadBuffer create(Object supplementalInfo) { + return new ByteArrayReadBuffer(fileImageBytes); + } + } + + // Read methods, that return only byte array and numeric primitive types. + + public abstract void get(long pos, byte[] buf) throws IOException; + + public abstract void get(long pos, byte[] buf, int num) throws IOException; + + public abstract int getInt(long pos) throws IOException; + + public abstract long getLong(long pos) throws IOException; + + public abstract void close(); +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/BarArrayHandler.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/BarArrayHandler.java new file mode 100644 index 00000000..2276ef43 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/BarArrayHandler.java @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import org.openjdk.jmc.joverflow.descriptors.ArrayBasedCollectionDescriptor; +import org.openjdk.jmc.joverflow.descriptors.CollectionDescriptors; +import org.openjdk.jmc.joverflow.descriptors.CollectionInstanceDescriptor; +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObjectArray; +import org.openjdk.jmc.joverflow.heap.model.JavaValueArray; +import org.openjdk.jmc.joverflow.heap.model.Snapshot; +import org.openjdk.jmc.joverflow.support.Constants; +import org.openjdk.jmc.joverflow.util.MiscUtils; + +/** + * Detects "vertical bar"-shaped multi-dimensional arrays and ArrayLists/Vectors: those where the + * "vertical" (outer) dimension is considerably bigger than the "horizonal" one, i.e. which consist + * of a large number of short sub-arrays. Since each sub-array has its own header plus a pointer to + * it from the outer array, "vertical bar" arrays can create significant overhead + */ +class BarArrayHandler { + private final JavaHeapObject[] elements; + private final int outerDim; + private final CollectionDescriptors colDescriptors; + + /** + * Returns an instance of BarArrayHandler if the given collection is an ArrayList, Vector or a + * subclass of one of these, and its size is greater than one. Otherwise, returns null. + */ + static BarArrayHandler createInstance(CollectionInstanceDescriptor colDesc, CollectionDescriptors colDescriptors) { + JavaClass clazz = colDesc.getClassDescriptor().getClazz(); + if (!isArrayListOrVector(clazz)) { + return null; + } + int numEls = colDesc.getNumElements(); + if (numEls <= 1) { + return null; + } + ArrayBasedCollectionDescriptor arrayColDesc = (ArrayBasedCollectionDescriptor) colDesc; + JavaObjectArray elsArray = arrayColDesc.getElementsArray(); + if (elsArray == null) { + return null; // Unresolved + } + JavaHeapObject[] elements = elsArray.getElements(); + + return new BarArrayHandler(elements, numEls, colDescriptors); + } + + /** + * Returns an instance of BarArrayHandler for the given standalone array, if its size is greater + * than one. + */ + static BarArrayHandler createInstance(JavaHeapObject[] elements, CollectionDescriptors colDescriptors) { + if (elements.length <= 1) { + return null; + } + return new BarArrayHandler(elements, elements.length, colDescriptors); + } + + private BarArrayHandler(JavaHeapObject[] elements, int numEls, CollectionDescriptors colDescriptors) { + this.elements = elements; + this.outerDim = numEls; + this.colDescriptors = colDescriptors; + } + + /** + * Checks if the array or collection associated with this object has a "vertical bar" shape. If + * so, returns the overhead; otherwise returns 0. + */ + int calculateOverhead() { + Snapshot snapshot = colDescriptors.getSnapshot(); + int ptrSize = snapshot.getPointerSize(); + int arrHeaderSize = snapshot.getArrayHeaderSize(); + + int maxInnerDim = 0, minInnerDim = Integer.MAX_VALUE; + @SuppressWarnings("unused") + int totalSubArraysLen = 0, numNonNullEls = 0; + JavaClass subArrayClazz = null; + + // Current two-dimensional array's overhead due to array header and object + // alignment for each sub-array + int oldInnerArraysOvhd = 0; + int elementSize = 0; + int subCollectionObjSize = 0; + + for (int i = 0; i < outerDim; i++) { + JavaHeapObject line = elements[i]; + if (line == null) { + continue; + } + + JavaClass clazz = line.getClazz(); + if (subArrayClazz != null) { + // Make sure all elements of the top array have the same type + if (clazz != subArrayClazz) { + return 0; + } + } else { + subArrayClazz = clazz; + if (!clazz.isArray()) { + subCollectionObjSize = clazz.getInstanceSize(); + } + } + + int subArrayLen; + + if (line instanceof JavaObjectArray) { + JavaObjectArray objSubArray = (JavaObjectArray) line; + subArrayLen = objSubArray.getLength(); + elementSize = ptrSize; + } else if (line instanceof JavaValueArray) { + JavaValueArray valueSubArray = (JavaValueArray) line; + subArrayLen = valueSubArray.getLength(); + if (elementSize == 0) { + elementSize = valueSubArray.getElementSize(); + } + } else { + if (!isArrayListOrVector(clazz)) { + return 0; + } + CollectionInstanceDescriptor subColDesc = colDescriptors.getDescriptor((JavaObject) line); + subArrayLen = subColDesc.getNumElements(); + elementSize = ptrSize; + } + + numNonNullEls++; + totalSubArraysLen += subArrayLen; + if (subArrayLen > maxInnerDim) { + maxInnerDim = subArrayLen; + } + if (subArrayLen < minInnerDim) { + minInnerDim = subArrayLen; + } + + int unalignedLineSize = subArrayLen * elementSize + arrHeaderSize; + int alignedLineSize = MiscUtils.getAlignedObjectSize(unalignedLineSize, snapshot.getObjectAlignment()); + oldInnerArraysOvhd += arrHeaderSize + (alignedLineSize - unalignedLineSize); + } + + // If all subarrays are null, this kind of overhead is undefined + if (numNonNullEls == 0) { + return 0; + } + // Check if this is actually not a "vertical-bar" shaped array + if (maxInnerDim >= outerDim) { + return 0; + } + + int outerPtrOvhd = (outerDim - maxInnerDim) * ptrSize; + int innerColOvhd = (outerDim - maxInnerDim) * subCollectionObjSize; + + // Calculate inner arrays overhead in the "new" array, where we swap + // dimensions, so that there are maxInnerDim lines each with outerDim elements + int unalignedLineSize = outerDim * elementSize + arrHeaderSize; + int alignedLineSize = MiscUtils.getAlignedObjectSize(unalignedLineSize, snapshot.getObjectAlignment()); + int newInnerArraysOvhd = (arrHeaderSize + alignedLineSize - unalignedLineSize) * maxInnerDim; + + int innerArraysOvhd = oldInnerArraysOvhd - newInnerArraysOvhd; + + int ovhd = outerPtrOvhd + innerColOvhd + innerArraysOvhd; + return ovhd; + } + + private static boolean isArrayListOrVector(JavaClass clazz) { + return clazz.isOrSubclassOf(Constants.ARRAY_LIST) || clazz.isOrSubclassOf(Constants.VECTOR); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/BreadthFirstHeapScanner.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/BreadthFirstHeapScanner.java new file mode 100644 index 00000000..4221e450 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/BreadthFirstHeapScanner.java @@ -0,0 +1,607 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; + +import org.openjdk.jmc.joverflow.descriptors.CollectionInstanceDescriptor; +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObjectArray; +import org.openjdk.jmc.joverflow.heap.model.JavaThing; +import org.openjdk.jmc.joverflow.heap.model.JavaValueArray; +import org.openjdk.jmc.joverflow.heap.model.Snapshot; +import org.openjdk.jmc.joverflow.heap.parser.HprofParsingCancelledException; +import org.openjdk.jmc.joverflow.stats.InterimRefChainTree.ParentType; +import org.openjdk.jmc.joverflow.support.ProblemRecorder; +import org.openjdk.jmc.joverflow.support.RefChainElement; +import org.openjdk.jmc.joverflow.util.IntArrayList; + +/** + * The heap scanner impl-n that scans the heap in breadth-first order starting from GC roots. The + * subtree for the first root is scanned fully, then the subtree for the second one, etc. + *

+ * The implementation is heavily optimized for performance. The main goal is to scan objects in each + * "frontier" in the strictly ascending or descending order, to minimize page swapping in the disk + * cache. For that, the frontier is sorted. Sorting is made complicated by the fact that the + * frontier is not a simple array of JavaHeapObjects, but consists of group of objects, where each + * group is referenced by the same "parent" object (instance, class, array or collection). Thus we + * have to sort objects within each group, sort groups by the first object, and then use an + * equivalent of merging a large number of sorted arrays to fetch objects in a globally sorted order + * from the frontier. + *

+ * One additional optimization that affects performance by at least 10 per cent is alternating + * sorting direction. It results in alternating forward and backward passes over the heap dump. This + * results in considerable reduction in page swapping, since when we stop and then go backward + * (rather than start from the other end of the dump again) we read from pages that are already in + * memory. + */ +public class BreadthFirstHeapScanner extends HeapScaner { + private final ProblemChecker objHandler; + private final InterimRefChainTree refChain; + + private final ArrayList nextFrontier; + private ObjGroup[] curFrontier; + private int curFrontierEndIdx, curFrontierStartIdx; + private final ArrayList fieldBuf; + private final IntArrayList elementBuf; + private final FieldObjComparator fieldObjComparator = new FieldObjComparator(); + private final ObjGroupComparator objGroupComparator = new ObjGroupComparator(); + private boolean sortingDirection; + + BreadthFirstHeapScanner(Snapshot snapshot, ProblemChecker objHandler, ProblemRecorder problemRecorder) { + super(snapshot, new InterimRefChainTree(problemRecorder)); + this.refChain = (InterimRefChainTree) getRefChain(); + this.objHandler = objHandler; + nextFrontier = new ArrayList<>(1000); + fieldBuf = new ArrayList<>(100); + elementBuf = new IntArrayList(1000); + } + + @Override + protected void scanObjectsFromRootObj(JavaHeapObject obj) { + while (obj != null) { + if (obj.setVisitedIfNot()) { + currentProcessedObjNo++; + if (cancelled) { + throw new HprofParsingCancelledException.Runtime(); + } + + JavaClass clazz = obj.getClazz(); + + if (clazz.isString()) { + objHandler.handleString((JavaObject) obj); + } else { + if (obj instanceof JavaObject) { + JavaObject javaObj = (JavaObject) obj; + JavaThing[] fields = javaObj.getFields(); + + CollectionInstanceDescriptor colDesc = objHandler.handleInstance(javaObj, fields); + + if (fields.length > 0 && obj.getClazz().hasReferenceFields()) { + // Nullify various fields like those for auxiliary linked list in LinkedHashMap, + // so that scanObjectFields does not have problems with long lists, etc. + int[] bannedFieldIndices = clazz.getBannedFieldIndices(); + if (bannedFieldIndices != null) { + for (int bannedFieldIdx : bannedFieldIndices) { + fields[bannedFieldIdx] = null; + } + } + + if (colDesc != null) { + // obj is a collection. Add its elements ("payload") to the nextFrontier, + // and handle its internal implementation objects immediately. + if (colDesc.getClassDescriptor().isMap()) { + colDesc.iterateMap(new CollectionInstanceDescriptor.MapIteratorCallback() { + @Override + public boolean scanMapEntry(JavaHeapObject key, JavaHeapObject value) { + pushCollectionElement(key); + pushCollectionElement(value); + return true; + } + + @Override + public boolean scanImplementationObject(JavaHeapObject implObj) { + if (implObj.setVisitedIfNot()) { + currentProcessedObjNo++; + if (implObj instanceof JavaObject) { + JavaObject implJavaObj = (JavaObject) implObj; + objHandler.handleInstance(implJavaObj, implJavaObj.getFields()); + } else { + // We set elements param to null to avoid overhead. They won't be + // used by handler anyway, because array should be marked as + // visitedAsCollectionImpl() by this time. + objHandler.handleObjectArray((JavaObjectArray) implObj, null); + } + return true; + } else { + // Probably corrupted data - we shouldn't see a visited object + return false; + } + } + }); + } else { + colDesc.iterateList(new CollectionInstanceDescriptor.ListIteratorCallback() { + @Override + public boolean scanListElement(JavaHeapObject element) { + pushCollectionElement(element); + return true; + } + + @Override + public boolean scanImplementationObject(JavaHeapObject implObj) { + if (implObj.setVisitedIfNot()) { + currentProcessedObjNo++; + if (implObj instanceof JavaObject) { + JavaObject implJavaObj = (JavaObject) implObj; + objHandler.handleInstance(implJavaObj, implJavaObj.getFields()); + } else { + objHandler.handleObjectArray((JavaObjectArray) implObj, null); + } + return true; + } else { + // Probably corrupted data - we shouldn't see a visited object + return false; + } + } + }); + } + finishCollectionPush(obj); + + if (colDesc.hasExtraObjFields()) { + colDesc.filterExtraObjFields(fields); + pushFields(obj, fields, ParentType.INSTANCE); + } + + } else { + // An ordinary, non-collection object + pushFields(obj, fields, ParentType.INSTANCE); + } + } + } else if (obj instanceof JavaClass) { + JavaThing[] staticFields = ((JavaClass) obj).getStaticValues(); + + if (staticFields.length > 0) { + pushFields(obj, staticFields, ParentType.CLAZZ); + } + } else if (obj instanceof JavaObjectArray) { + JavaObjectArray objArray = (JavaObjectArray) obj; + JavaHeapObject[] elements = objArray.getElements(); + objHandler.handleObjectArray(objArray, elements); + + if (elements.length > 0) { + pushArrayElements(obj, elements); + } + } else { + objHandler.handleValueArray((JavaValueArray) obj); + } + } + } + + // Determine the next object to scan + obj = getNextObjFromFrontier(); + } + + curFrontier = null; + } + + private JavaHeapObject getNextObjFromFrontier() { + JavaHeapObject result; + int objIdxInCurParent; + ObjGroup curObjGroup; + + while (true) { + if (curFrontier == null || curFrontierEndIdx < curFrontierStartIdx) { + curFrontier = getNewFrontier(); + if (curFrontier == null) { + return null; + } + curFrontierEndIdx = curFrontier.length - 1; + curFrontierStartIdx = 0; + } + + curObjGroup = curFrontier[curFrontierStartIdx]; + objIdxInCurParent = curObjGroup.getCurChildLogicalIndex(); + result = curObjGroup.getCurChild(snapshot); + curObjGroup.curChildIdx++; + + // Re-sort groups within current frontier if necessary + if (curObjGroup.hasSingleChild() || curObjGroup.curChildIdx >= curObjGroup.getChildArrayLen()) { + // Just move to the next object group within this frontier + curFrontier[curFrontierStartIdx] = null; // Help the GC + curFrontierStartIdx++; + } else { + // If the next element in the current group is not in the right order wrt. the + // first element in the next group, re-sort groups within current frontier to + // get the smallest element first + int nextObjGlobalIdx = curObjGroup.getCurChildGlobalIndex(); + // nextObjGlobalIdx > 0 below means the next object is not a class + if (nextObjGlobalIdx > 0 && (curFrontierEndIdx - curFrontierStartIdx > 0) + && ordered(curFrontier[curFrontierStartIdx + 1].getCurChildGlobalIndex(), nextObjGlobalIdx)) { + int i = findPosInCurFrontierToInsert(nextObjGlobalIdx); + ObjGroup tmp = curFrontier[curFrontierStartIdx]; + if (i == curFrontierStartIdx + 2) { // No need to call arraycopy() for one element + curFrontier[curFrontierStartIdx] = curFrontier[curFrontierStartIdx + 1]; + } else { + System.arraycopy(curFrontier, curFrontierStartIdx + 1, curFrontier, curFrontierStartIdx, + i - 1 - curFrontierStartIdx); + } + curFrontier[i - 1] = tmp; + } + } + + if (result == null) { + continue; + } + if (!result.isVisited()) { + break; + } + } + + refChain.setCurParent(curObjGroup.parentObj, curObjGroup.type, curObjGroup.refererToParent); + refChain.setCurIndexInParent(objIdxInCurParent); + return result; + } + + private ObjGroup[] getNewFrontier() { + if (nextFrontier.isEmpty()) { + return null; + } + + ObjGroup[] frontier = nextFrontier.toArray(new ObjGroup[nextFrontier.size()]); + nextFrontier.clear(); + + Arrays.sort(frontier, objGroupComparator); + sortingDirection = !sortingDirection; + return frontier; + } + + /** + * Returns the index of element *before which* the element with global index elGlobalIdx should + * be inserted. + */ + private int findPosInCurFrontierToInsert(int elGlobalIdx) { + int left = curFrontierStartIdx; + int right = curFrontierEndIdx; + if (ordered(curFrontier[right].getCurChildGlobalIndex(), elGlobalIdx)) { + return right + 1; + } + + int i; + while (true) { + i = (left + right) >>> 1; // "(left + right) / 2" but without risk of overflow + if (ordered(curFrontier[i].getCurChildGlobalIndex(), elGlobalIdx)) { + left = i; + if (!ordered(curFrontier[i + 1].getCurChildGlobalIndex(), elGlobalIdx)) { + break; + } + } else { + right = i; + } + } + return i; + } + + private void pushFields(JavaHeapObject obj, JavaThing[] fields, ParentType parentType) { + for (int i = 0; i < fields.length; i++) { + JavaThing field = fields[i]; + if (field != null && (field instanceof JavaHeapObject) && !((JavaHeapObject) field).isVisited()) { + fieldBuf.add(new FieldObj((JavaHeapObject) field, i)); + } + } + if (fieldBuf.isEmpty()) { + return; + } + + ObjGroup nextGroup; + if (fieldBuf.size() > 1) { + FieldObj[] foEls = fieldBuf.toArray(new FieldObj[fieldBuf.size()]); + Arrays.sort(foEls, fieldObjComparator); + nextGroup = new ObjGroupFields(refChain.getLastRefChainElement(), parentType, obj, foEls); + } else { + FieldObj fo = fieldBuf.get(0); + nextGroup = new ObjGroupFields(refChain.getLastRefChainElement(), parentType, obj, fo); + } + nextFrontier.add(nextGroup); + fieldBuf.clear(); + } + + private void pushArrayElements(JavaHeapObject obj, JavaHeapObject[] elements) { + for (JavaHeapObject element : elements) { + if (element != null && !element.isVisited()) { + elementBuf.add(element.getGlobalObjectIndex()); + } + } + if (elementBuf.isEmpty()) { + return; + } + + ObjGroup nextGroup; + if (elementBuf.size() > 1) { + int[] els = elementBuf.toArray(); + sortGlobalObjectIndexesInOrder(els); + nextGroup = new ObjGroupCol(refChain.getLastRefChainElement(), ParentType.ARRAY, obj, els); + } else { + nextGroup = new ObjGroupCol(refChain.getLastRefChainElement(), ParentType.ARRAY, obj, elementBuf.get(0)); + } + nextFrontier.add(nextGroup); + elementBuf.clear(); + } + + private void pushCollectionElement(JavaHeapObject element) { + if (element != null && !element.isVisited()) { + elementBuf.add(element.getGlobalObjectIndex()); + } + } + + private void finishCollectionPush(JavaHeapObject obj) { + if (elementBuf.isEmpty()) { + return; + } + + ObjGroup nextGroup; + if (elementBuf.size() > 1) { + int[] els = elementBuf.toArray(); + sortGlobalObjectIndexesInOrder(els); + nextGroup = new ObjGroupCol(refChain.getLastRefChainElement(), ParentType.COLLECTION, obj, els); + } else { + nextGroup = new ObjGroupCol(refChain.getLastRefChainElement(), ParentType.COLLECTION, obj, + elementBuf.get(0)); + } + nextFrontier.add(nextGroup); + elementBuf.clear(); + } + + /** + * Returns true if i1 and i2 are ordered according to sortingDirection. Note that here we use + * (!sortingDirection), as opposed to "direct" sortingDirection in JavaHeapObjComparator, + * FieldObjComparator and objGroupComparator. That's because when this method is called, + * sortingDirection, which is used for the next frontier, is different than for objects in the + * current frontier. + */ + private boolean ordered(int i1, int i2) { + if (!sortingDirection) { + return i1 < i2; + } else { + return i1 > i2; + } + } + + /** + * Sorts global object elements in els according to the current sortingDirection. Same is done + * in {@link FieldObjComparator} and {@link ObjGroupComparator}. + */ + private void sortGlobalObjectIndexesInOrder(int[] els) { + Arrays.sort(els); + if (!sortingDirection) { // Flip the order + int len = els.length; + for (int i = 0; i < len / 2; i++) { + int tmp = els[i]; + int otherIdx = len - i - 1; + els[i] = els[otherIdx]; + els[otherIdx] = tmp; + } + } + } + + /** + * Represents a parent object and a group of its children. Children, which are JavaHeapObjects + * (possibly wrapped into FieldObj) are sorted by their global object index. + */ + private static abstract class ObjGroup { + final RefChainElement refererToParent; + final ParentType type; + final JavaHeapObject parentObj; + protected int curChildIdx; + + ObjGroup(RefChainElement refererToParent, InterimRefChainTree.ParentType type, JavaHeapObject parentObj) { + this.refererToParent = refererToParent; + this.type = type; + this.parentObj = parentObj; + } + + abstract JavaHeapObject getCurChild(Snapshot snapshot); + + abstract int getCurChildLogicalIndex(); + + abstract int getCurChildGlobalIndex(); + + abstract boolean hasSingleChild(); + + abstract int getChildArrayLen(); + + @Override + public String toString() { + return type + ", refererToParent = " + refererToParent; + } + } + + /** Objects that are fields of a parent instance or Clazz */ + private static class ObjGroupFields extends ObjGroup { + + // Either FieldObj[] for multiple children, or FieldObj for a single child + private final Object oneOrMoreChildren; + + ObjGroupFields(RefChainElement refererToParent, InterimRefChainTree.ParentType type, JavaHeapObject parentObj, + Object oneOrMoreChildren) { + super(refererToParent, type, parentObj); + this.oneOrMoreChildren = oneOrMoreChildren; + } + + @Override + JavaHeapObject getCurChild(Snapshot snapshot) { + if (oneOrMoreChildren instanceof FieldObj[]) { + return snapshot.getObjectAtGlobalIndex(((FieldObj[]) oneOrMoreChildren)[curChildIdx].objGlobalIdx); + } else { + return snapshot.getObjectAtGlobalIndex(((FieldObj) oneOrMoreChildren).objGlobalIdx); + } + } + + @Override + int getCurChildLogicalIndex() { + if (oneOrMoreChildren instanceof FieldObj[]) { + return ((FieldObj[]) oneOrMoreChildren)[curChildIdx].objFieldIdx; + } else { + return ((FieldObj) oneOrMoreChildren).objFieldIdx; + } + } + + @Override + int getCurChildGlobalIndex() { + if (oneOrMoreChildren instanceof FieldObj[]) { + FieldObj[] children = (FieldObj[]) oneOrMoreChildren; + return children[curChildIdx].objGlobalIdx; + } else { + return ((FieldObj) oneOrMoreChildren).objGlobalIdx; + } + } + + @Override + boolean hasSingleChild() { + return oneOrMoreChildren instanceof FieldObj; + } + + @Override + int getChildArrayLen() { + return ((FieldObj[]) oneOrMoreChildren).length; + } + } + + /** Objects that are elements of a parent collection or array */ + private static class ObjGroupCol extends ObjGroup { + + private final int[] childrenGlobalIndexes; + private final int singleChildGlobalIndex; + + ObjGroupCol(RefChainElement refererToParent, InterimRefChainTree.ParentType type, JavaHeapObject parentObj, + int[] childrenGlobalIndexes) { + super(refererToParent, type, parentObj); + this.childrenGlobalIndexes = childrenGlobalIndexes; + this.singleChildGlobalIndex = Integer.MIN_VALUE; + } + + ObjGroupCol(RefChainElement refererToParent, InterimRefChainTree.ParentType type, JavaHeapObject parentObj, + int singleChildGlobalIndex) { + super(refererToParent, type, parentObj); + this.childrenGlobalIndexes = null; + this.singleChildGlobalIndex = singleChildGlobalIndex; + } + + @Override + JavaHeapObject getCurChild(Snapshot snapshot) { + if (childrenGlobalIndexes != null) { + return snapshot.getObjectAtGlobalIndex(childrenGlobalIndexes[curChildIdx]); + } else { + if (curChildIdx != 0) { + throw new RuntimeException(); + } + return snapshot.getObjectAtGlobalIndex(singleChildGlobalIndex); + } + } + + @Override + int getCurChildLogicalIndex() { + return curChildIdx; + } + + @Override + int getCurChildGlobalIndex() { + if (childrenGlobalIndexes != null) { + return childrenGlobalIndexes[curChildIdx]; + } else { + return singleChildGlobalIndex; + } + } + + @Override + boolean hasSingleChild() { + return childrenGlobalIndexes == null; + } + + @Override + int getChildArrayLen() { + return childrenGlobalIndexes.length; + } + } + + /** + * A wrapper around JavaHeapObject that also encapsulate a field index for that object under its + * instance/Clazz parent. We have to use it to enable sorting of fields without losing logical + * field indices. + */ + private static class FieldObj { + final int objGlobalIdx, objFieldIdx; + + FieldObj(JavaHeapObject obj, int objFieldIdx) { + this.objGlobalIdx = obj.getGlobalObjectIndex(); + this.objFieldIdx = objFieldIdx; + } + } + + // A very important property of the comparator classes below is that they + // compare global indices of two JavaHeapObjects depending on the current + // "sorting direction". This is done to make our algorithm traverse a heap + // dump first in forward direction, then backward, then forward again, etc. + // This, in turn, allows the page buffer to reuse the same pages when the + // "direction turn" is made, and ultimately to reduce page ins/page outs + // quite considerably. Same is done in sortGlobalObjectIndexesInOrder() method. + + private class FieldObjComparator implements Comparator { + @Override + public int compare(FieldObj o1, FieldObj o2) { + int idx1 = o1.objGlobalIdx; + int idx2 = o2.objGlobalIdx; + if (sortingDirection) { + return idx1 - idx2; + } else { + return idx2 - idx1; + } + } + } + + private class ObjGroupComparator implements Comparator { + @Override + public int compare(ObjGroup o1, ObjGroup o2) { + int firstObjIdx1 = o1.getCurChildGlobalIndex(); + int firstObjIdx2 = o2.getCurChildGlobalIndex(); + if (sortingDirection) { + return firstObjIdx1 - firstObjIdx2; + } else { + return firstObjIdx2 - firstObjIdx1; + } + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/ClassloaderStats.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/ClassloaderStats.java new file mode 100644 index 00000000..59dcbe21 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/ClassloaderStats.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaThing; +import org.openjdk.jmc.joverflow.heap.model.Snapshot; +import org.openjdk.jmc.joverflow.util.ObjectToIntMap; + +/** + * Container for statistics on class loaders. + */ +public class ClassloaderStats { + private final ObjectToIntMap clInstToNumLoadedClasses; + private final ObjectToIntMap clClazzToNumLoadedClasses; + + ClassloaderStats(Snapshot snapshot) { + clInstToNumLoadedClasses = new ObjectToIntMap<>(10); + clClazzToNumLoadedClasses = new ObjectToIntMap<>(10); + + JavaClass[] classes = snapshot.getClasses(); + + // First, find all classes that extend java.lang.ClassLoader. + // We want to know about all such classes, including those with zero loaded classes. + for (JavaClass clazz : classes) { + if (clazz.isOrSubclassOf("java.lang.ClassLoader")) { + clClazzToNumLoadedClasses.put(clazz, 0); + } + } + + for (JavaClass clazz : classes) { + JavaThing loaderThing = clazz.getLoader(); + if (!(loaderThing instanceof JavaObject)) { + continue; + } + JavaObject loader = (JavaObject) loaderThing; + clInstToNumLoadedClasses.putOneOrIncrement(loader); + JavaClass loaderClazz = loader.getClazz(); + clClazzToNumLoadedClasses.putOneOrIncrement(loaderClazz); + } + } + + public ObjectToIntMap getCLInstToNumLoadedClasses() { + return clInstToNumLoadedClasses; + } + + public ObjectToIntMap getClClazzToNumLoadedClasses() { + return clClazzToNumLoadedClasses; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/DataFieldStats.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/DataFieldStats.java new file mode 100644 index 00000000..8cf0594d --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/DataFieldStats.java @@ -0,0 +1,263 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import org.openjdk.jmc.joverflow.heap.model.JavaChar; +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaField; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaInt; +import org.openjdk.jmc.joverflow.heap.model.JavaLong; +import org.openjdk.jmc.joverflow.heap.model.JavaShort; +import org.openjdk.jmc.joverflow.heap.model.JavaThing; +import org.openjdk.jmc.joverflow.heap.model.JavaValue; +import org.openjdk.jmc.joverflow.util.IntArrayList; +import org.openjdk.jmc.joverflow.util.LongArrayList; + +/** + * An instance of this class is created for a JavaClass that describes instances (i.e. not arrays). + * It is used to keep track of field statistics, for example to determine fields that are always or + * almost always null/zero, etc. + */ +public class DataFieldStats { + /** Signals that the given class has no problematic (e.g. always-null) fields */ + public static final int[] NO_REQUESTED_FIELDS = new int[] {}; + + /** Signals that the given class has no data fields at all, like java.lang.Object */ + public static final int[] CLASS_HAS_NO_FIELDS = new int[] {}; + + private static final DataFieldStats EMPTY_STATS = new DataFieldStats(new JavaField[] {}); + + private final JavaField[] allFields; + + // These arrays have the same size as allFields, i.e. each element maps to a single field + private final int[] numInstancesWhereThisFieldIsNotNull; + private final int[] numInstancesWhereThisFieldUnderutilizesHiBytes; + private final byte[] minUnusedBytesForThisField; + + private int numInstancesWithAllNullFields; + + static DataFieldStats newInstance(JavaClass clazz) { + if (clazz.isArray()) { + return EMPTY_STATS; + } else { + JavaField[] allInstanceFields = clazz.getFieldsForInstance(); + return new DataFieldStats(allInstanceFields); + } + } + + private DataFieldStats(JavaField[] allFields) { + this.allFields = allFields; + numInstancesWhereThisFieldIsNotNull = new int[allFields.length]; + numInstancesWhereThisFieldUnderutilizesHiBytes = new int[allFields.length]; + minUnusedBytesForThisField = new byte[allFields.length]; + for (int i = 0; i < minUnusedBytesForThisField.length; i++) { + minUnusedBytesForThisField[i] = Byte.MAX_VALUE; + } + } + + /** + * Should be called for each instance of the given class, passing values of fields of this + * instance. + */ + void handleFields(JavaThing[] fields) { + boolean someNonNullFieldFound = false; + for (int fieldIdx = 0; fieldIdx < fields.length; fieldIdx++) { + JavaThing field = fields[fieldIdx]; + if (field != null) { + if (field instanceof JavaHeapObject) { + numInstancesWhereThisFieldIsNotNull[fieldIdx]++; + someNonNullFieldFound = true; + } else { + JavaValue value = (JavaValue) field; + if (value.isZero()) { + int valueSize = value.getSize(); + if (!value.isFloatingPointNumber() && valueSize > 1) { + numInstancesWhereThisFieldUnderutilizesHiBytes[fieldIdx]++; + byte numUnusedBytes = (byte) valueSize; + if (numUnusedBytes < minUnusedBytesForThisField[fieldIdx]) { + minUnusedBytesForThisField[fieldIdx] = numUnusedBytes; + } + } + } else { + numInstancesWhereThisFieldIsNotNull[fieldIdx]++; + someNonNullFieldFound = true; + if (!value.isFloatingPointNumber() && value.getSize() > 1) { + byte numUnusedBytes = getNumUnusedBytes(value); + if (numUnusedBytes > 0) { + numInstancesWhereThisFieldUnderutilizesHiBytes[fieldIdx]++; + if (numUnusedBytes < minUnusedBytesForThisField[fieldIdx]) { + minUnusedBytesForThisField[fieldIdx] = numUnusedBytes; + } + } + } + } + } + } + } + + if (!someNonNullFieldFound) { + numInstancesWithAllNullFields++; + } + } + + /** + * Returns the set of fields of this class (field indices within all instance fields defined in + * this class) which are null/zero in all its instances (if maxNonNullFieldInstances == 0), or + * in at most maxNonNullFieldInstances instances of this class. A zero-size array + * CLASS_HAS_NO_FIELDS is returned if there are no fields at all in this class. Another + * zero-size array NO_REQUESTED_FIELDS is returned if there are no problematic fields. + */ + public int[] getPercentileEmptyFields(int maxNonNullFieldInstances) { + if (allFields.length == 0) { + return CLASS_HAS_NO_FIELDS; + } + + IntArrayList fieldIdxs = new IntArrayList(allFields.length); + for (int i = 0; i < numInstancesWhereThisFieldIsNotNull.length; i++) { + int nNonNulls = numInstancesWhereThisFieldIsNotNull[i]; + if (nNonNulls >= 0 && nNonNulls <= maxNonNullFieldInstances) { + fieldIdxs.add(i); + } + } + + if (fieldIdxs.size() == 0) { + return NO_REQUESTED_FIELDS; + } else { + return fieldIdxs.toArray(); + } + } + + public int getNumInstancesWithFieldNotNull(int fieldIdx) { + return numInstancesWhereThisFieldIsNotNull[fieldIdx]; + } + + public int getNumInstancesWithAllNullFields() { + return numInstancesWithAllNullFields; + } + + /** + * Returns the set of fields of this class (field indices within all instance fields defined in + * this class) which are char, short, int or long and where some number of high bytes are not + * used in at least minBadInstances instances. Returns null if no such fields are found in this + * class. + */ + public UnderutilizedFields getUnusedHiBytesFields(int minBadInstances) { + if (allFields.length == 0) { + return null; + } + + IntArrayList fieldIdxs = new IntArrayList(allFields.length); + LongArrayList fieldOvhd = new LongArrayList(allFields.length); + for (int i = 0; i < numInstancesWhereThisFieldUnderutilizesHiBytes.length; i++) { + int nBadInstances = numInstancesWhereThisFieldUnderutilizesHiBytes[i]; + if (nBadInstances >= minBadInstances) { + int minUnusedBytes = minUnusedBytesForThisField[i]; + // Check whether *all* bytes in the given field are unused. If so, + // it means that this field is always 0. Such fields are reported + // separately, by getPercentileEmptyFields(), i.e. treated as + // "always null/zero". We don't report them here, to avoid counting + // the same overhead twice. + if (minUnusedBytes == allFields[i].getSizeInInstance()) { + continue; + } + + fieldIdxs.add(i); + fieldOvhd.add(((long) minUnusedBytes) * nBadInstances); + } + } + + if (fieldIdxs.size() == 0) { + return null; + } else { + return new UnderutilizedFields(fieldIdxs.toArray(), fieldOvhd.toArray()); + } + } + + public int getNumFields() { + return allFields.length; + } + + private byte getNumUnusedBytes(JavaValue value) { + if (value instanceof JavaInt) { + JavaInt intValue = (JavaInt) value; + int val = intValue.getValue(); + if (val >= Byte.MIN_VALUE && val <= Byte.MAX_VALUE) { + return 3; + } else if (val >= Short.MIN_VALUE && val <= Short.MAX_VALUE) { + return 2; + } else { + return 0; + } + } else if (value instanceof JavaChar) { + JavaChar charValue = (JavaChar) value; + char val = charValue.getValue(); + if (val <= 0xFF) { + return 1; + } else { + return 0; + } + } else if (value instanceof JavaShort) { + JavaShort shortValue = (JavaShort) value; + short val = shortValue.getValue(); + if (val >= Byte.MIN_VALUE && val <= Byte.MAX_VALUE) { + return 1; + } else { + return 0; + } + } else if (value instanceof JavaLong) { + JavaLong longValue = (JavaLong) value; + long val = longValue.getValue(); + if (val >= Byte.MIN_VALUE && val <= Byte.MAX_VALUE) { + return 7; + } else if (val >= Short.MIN_VALUE && val <= Short.MAX_VALUE) { + return 6; + } else if (val >= Integer.MIN_VALUE && val <= Integer.MAX_VALUE) { + return 4; + } else { + return 0; + } + } + return 0; + } + + public static class UnderutilizedFields { + public final int[] fieldIndices; + public final long[] unusedBytesOvhd; + + private UnderutilizedFields(int[] fieldIndices, long[] unusedBytesOvhd) { + this.fieldIndices = fieldIndices; + this.unusedBytesOvhd = unusedBytesOvhd; + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/DepthFirstHeapScaner.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/DepthFirstHeapScaner.java new file mode 100644 index 00000000..951c7e16 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/DepthFirstHeapScaner.java @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import org.openjdk.jmc.joverflow.descriptors.CollectionDescriptors; +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaLazyReadObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObjectArray; +import org.openjdk.jmc.joverflow.heap.model.JavaThing; +import org.openjdk.jmc.joverflow.heap.model.JavaValueArray; +import org.openjdk.jmc.joverflow.heap.model.Snapshot; +import org.openjdk.jmc.joverflow.heap.parser.HprofParsingCancelledException; +import org.openjdk.jmc.joverflow.support.ProblemRecorder; +import org.openjdk.jmc.joverflow.util.FastStack; + +/** + * The heap scanner impl-n that scans a heap dump in depth-first order starting from GC roots. It + * takes an instance of ProblemChecker, that should implement methods that check objects given to it + * for problems and other interesting properties. + */ +class DepthFirstHeapScaner extends HeapScaner { + + private final ProblemChecker objHandler; + private final FastStack fieldsOrArrayElsStack; + private final InterimRefChainStack refChain; + + // When this is true, getNextObjToScan() tries to find the next object to + // scan such that it's located in the heap dump close to the previously + // scanned object. That may reduce the number of page swaps in CachedReadBuffer + // by 3-5%, thus reducing the number of disk reads and ultimately time. + // However, for smaller heap dumps this may result in a significant increase + // in CPU cycles with no benefit. We probably need to turn this option on + // adaptively, when it becomes clear that page swapping takes too much time. + private boolean optimizeForLocality = false; + + DepthFirstHeapScaner(Snapshot snapshot, ProblemChecker objHandler, ProblemRecorder problemRecorder, + CollectionDescriptors colDescriptors) { + super(snapshot, new InterimRefChainStack(problemRecorder, colDescriptors)); + this.refChain = (InterimRefChainStack) getRefChain(); + this.objHandler = objHandler; + fieldsOrArrayElsStack = new FastStack<>(256); + } + + /** + * Scans all of the objects reachable from obj in depth-first order. We don't use recursion here + * (which is easier to write and understand), because for long reference chains, such as those + * found in linked lists, deep recursion can lead to StackOverflowError. + */ + @Override + protected void scanObjectsFromRootObj(JavaHeapObject obj) { + while (obj != null) { + if (obj.setVisitedIfNot()) { + currentProcessedObjNo++; + if (cancelled) { + throw new HprofParsingCancelledException.Runtime(); + } + + JavaClass clazz = obj.getClazz(); + refChain.push(obj); + if (clazz.isString()) { + objHandler.handleString((JavaObject) obj); + refChain.pop(); + } else { + if (obj instanceof JavaObject) { + JavaObject javaObj = (JavaObject) obj; + // Note that in principle the call below could be replaced with the variant + // with false boolean arg, which will initialize only reference fields. This + // would speed up field parsing in JavaObject and field scanning below, but + // may cause problems in other parts of the system, that expect JavaObjects + // to have all fields set properly - like checking for all-zero fields. + JavaThing[] fields = javaObj.getFields(); + + objHandler.handleInstance(javaObj, fields); + + // Nullify various fields like those for auxiliary linked list in LinkedHashMap, + // so that scanObjectFields does not have problems with long lists, etc. + int[] bannedFieldIndices = clazz.getBannedFieldIndices(); + if (bannedFieldIndices != null) { + for (int bannedFieldIdx : bannedFieldIndices) { + fields[bannedFieldIdx] = null; + } + } + + refChain.pushIndexContainer(new TwoHandIndexContainer()); + fieldsOrArrayElsStack.push(fields); + } else if (obj instanceof JavaClass) { + JavaThing[] staticFields = ((JavaClass) obj).getStaticValues(); + refChain.pushIndexContainer(new TwoHandIndexContainer()); + fieldsOrArrayElsStack.push(staticFields); + } else if (obj instanceof JavaObjectArray) { + JavaObjectArray objArray = (JavaObjectArray) obj; + JavaHeapObject[] elements = objArray.getElements(); + objHandler.handleObjectArray(objArray, elements); + + refChain.pushIndexContainer(new TwoHandIndexContainer()); + fieldsOrArrayElsStack.push(elements); + } else { + objHandler.handleValueArray((JavaValueArray) obj); + refChain.pop(); + } + } + } + + // Now determine the next object to scan + obj = getNextObjToScan(obj); + } + } + + /** + * Most of the complexity in this method is due to experimental functionality where we try to + * find next object to scan that is close enough to the current object inside the hep dump, in + * order to minimize our disk cache misses. + */ + private JavaHeapObject getNextObjToScan(JavaHeapObject oldObj) { + long oldObjOfsInFile = -1; + JavaHeapObject obj = null; + + while (!fieldsOrArrayElsStack.isEmpty()) { + JavaThing[] fieldsOrElements = fieldsOrArrayElsStack.peek(); + TwoHandIndexContainer curIdxContainer = (TwoHandIndexContainer) refChain.getCurrentIndexContainer(); + int nextIdx = curIdxContainer.incrementAndGetBase(); + while (nextIdx < fieldsOrElements.length) { + JavaThing objThing = fieldsOrElements[nextIdx]; + // Ignore null, primitive and already visited fields + if (objThing != null && objThing instanceof JavaHeapObject) { + obj = (JavaHeapObject) objThing; + if (!obj.isVisited()) { + if (optimizeForLocality && (obj instanceof JavaLazyReadObject)) { + oldObjOfsInFile = (oldObj instanceof JavaLazyReadObject) + ? ((JavaLazyReadObject) oldObj).getObjOfsInFile() : -1; + if (oldObjOfsInFile != -1) { + curIdxContainer.setBase(nextIdx - 1); + break; // We'll attempt to look for a closer located object + } + } + + curIdxContainer.setBase(nextIdx); + curIdxContainer.set(nextIdx); + return obj; + } else { + obj = null; + } + } + nextIdx++; + } + + if (obj == null) { // Fields or elements of the top object exhausted + fieldsOrArrayElsStack.pop(); + refChain.pop2(); // Pop the index holder and object + continue; + } + + // Look through the next few elements to see if any of them is closer to old obj + long curObjOfsInFile = ((JavaLazyReadObject) obj).getObjOfsInFile(); + long minDistance = Math.abs(curObjOfsInFile - oldObjOfsInFile); + int bestIdx = nextIdx; + int nCheckedFields = 0; + nextIdx++; + while (nextIdx < fieldsOrElements.length && nCheckedFields < 8) { + JavaThing objThing = fieldsOrElements[nextIdx]; + if (objThing != null && objThing instanceof JavaHeapObject) { + obj = (JavaHeapObject) objThing; + if (!obj.isVisited()) { + if (!(obj instanceof JavaLazyReadObject)) { + curIdxContainer.set(nextIdx); + return obj; + } + nCheckedFields++; + curObjOfsInFile = ((JavaLazyReadObject) obj).getObjOfsInFile(); + long distance = Math.abs(curObjOfsInFile - oldObjOfsInFile); + if (distance < minDistance) { + bestIdx = nextIdx; + minDistance = distance; + } + } + } + nextIdx++; + } + + curIdxContainer.set(bestIdx); + return (JavaHeapObject) fieldsOrElements[bestIdx]; + } + + return obj; + } + +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/DetailedDupStringStatsCalculator.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/DetailedDupStringStatsCalculator.java new file mode 100644 index 00000000..f57a1620 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/DetailedDupStringStatsCalculator.java @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import java.util.Collection; + +import org.openjdk.jmc.joverflow.descriptors.CollectionDescriptors; +import org.openjdk.jmc.joverflow.descriptors.CollectionInstanceDescriptor; +import org.openjdk.jmc.joverflow.heap.model.HeapStringReader; +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaLazyReadObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObjectArray; +import org.openjdk.jmc.joverflow.heap.model.JavaThing; +import org.openjdk.jmc.joverflow.heap.model.JavaValueArray; +import org.openjdk.jmc.joverflow.heap.model.Snapshot; +import org.openjdk.jmc.joverflow.heap.parser.HprofParsingCancelledException; +import org.openjdk.jmc.joverflow.support.DupStringStats; +import org.openjdk.jmc.joverflow.support.ProblemRecorder; + +/** + * Calculates stats for all duplicated strings in the heap, without exception - unlike the standard + * DetailedStatsCalculator that calculates all sorts of metrics, but takes into account at most 100 + * (we may change the number) top duplicated strings. Still, after calculating aggregated data, this + * calculator returns information only about the string clusters with the overhead exceeding the + * specified minimum. + */ +class DetailedDupStringStatsCalculator implements ProblemChecker { + private final Snapshot snapshot; + private final InterimRefChain refChain; + private final DepthFirstHeapScaner scaner; + + private final HeapStringReader stringReader; + private DupStringHandler dupStringHandler; + + public DetailedDupStringStatsCalculator(Snapshot snapshot, ProblemRecorder recorder) { + this.snapshot = snapshot; + CollectionDescriptors colDescriptors = new CollectionDescriptors(snapshot); + scaner = new DepthFirstHeapScaner(snapshot, this, recorder, colDescriptors); + refChain = scaner.getRefChain(); + + stringReader = snapshot.getStringReader(); + } + + public DupStringStats calculate() throws HprofParsingCancelledException { + // Iterate over the heap dump sequentially, and find all duplicated strings + DupStringStats dss = findDupStrings(); + + // Using info from the previous step, collect aggregated info about clusters + // of duplicated strings + findRefsToDupStrings(dss); + return dss; + } + + /** + * Iterates the heap dump sequentially, and finds all the duplicated strings in it. + */ + private DupStringStats findDupStrings() { + Collection allObjects = snapshot.getObjects(); + StringStatsCollector stringDupMap = new StringStatsCollector(snapshot); + + for (JavaLazyReadObject obj : allObjects) { + JavaClass clazz = obj.getClazz(); + if (!clazz.isString()) { + continue; + } + + stringDupMap.add((JavaObject) obj); + } + + return stringDupMap.getDuplicationStats(); + } + + /** + * Walks the heap dumps depth-first, and collects the information about clusters of duplicated + * strings. + */ + private void findRefsToDupStrings(DupStringStats dss) throws HprofParsingCancelledException { + dupStringHandler = new DupStringHandler(stringReader, dss.dupStrings, refChain, dss.stringInstShallowSize); + + scaner.analyzeViaRoots(); + } + + @Override + public CollectionInstanceDescriptor handleInstance(JavaObject obj, JavaThing[] fields) { + return null; + } + + @Override + public void handleObjectArray(JavaObjectArray array, JavaHeapObject[] elements) { + } + + @Override + public void handleValueArray(JavaValueArray array) { + } + + @Override + public void handleString(JavaObject strObj) { + boolean isDuplicated = dupStringHandler.handleString(strObj); + + JavaValueArray backingCharArray = dupStringHandler.getLastReadBackingArray(); + if (backingCharArray != null) { + backingCharArray.setVisited(); + } + + if (!isDuplicated) { + // Normal, non-duplicated string. Record it, so that eventually for fields + // pointing at duplicated strings we also know how many random strings they + // also point to. + refChain.recordCurrentRefChainForNonDupString(strObj, strObj.getSize() + backingCharArray.getSize()); + } + } + + public int getProgressPercentage() { + return scaner.getProgressPercentage(); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/DetailedStatsCalculator.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/DetailedStatsCalculator.java new file mode 100644 index 00000000..bcb417d5 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/DetailedStatsCalculator.java @@ -0,0 +1,663 @@ +/* + * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import java.util.ArrayList; +import java.util.HashMap; + +import org.openjdk.jmc.joverflow.descriptors.CollectionClassDescriptor; +import org.openjdk.jmc.joverflow.descriptors.CollectionDescriptors; +import org.openjdk.jmc.joverflow.descriptors.CollectionInstanceDescriptor; +import org.openjdk.jmc.joverflow.heap.model.HeapStringReader; +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaLazyReadObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObjectArray; +import org.openjdk.jmc.joverflow.heap.model.JavaThing; +import org.openjdk.jmc.joverflow.heap.model.JavaValueArray; +import org.openjdk.jmc.joverflow.heap.model.Snapshot; +import org.openjdk.jmc.joverflow.heap.parser.CachedReadBuffer; +import org.openjdk.jmc.joverflow.heap.parser.HprofParsingCancelledException; +import org.openjdk.jmc.joverflow.heap.parser.ReadBuffer; +import org.openjdk.jmc.joverflow.support.Constants; +import org.openjdk.jmc.joverflow.support.DupStringStats; +import org.openjdk.jmc.joverflow.support.HeapStats; +import org.openjdk.jmc.joverflow.support.ProblemRecorder; + +/** + * This class handles heap dump objects that are given to it by the instance of HeapScaner (that is + * created in its constructor). The HeapScaner scans the heap from GC roots, keeping the reference + * chain from a GC root to the current object at all times. This class analyzes each scanned object + * for various kinds of problems, and records problem type/overhead if anything found via the + * supplied instance of {@link org.openjdk.jmc.joverflow.support.ProblemRecorder}. See the latter + * for more information on problems and object kinds that they can occur on. + */ +class DetailedStatsCalculator implements ProblemChecker, Constants { + private final Snapshot snapshot; + private final HeapScaner scaner; + private final ProblemRecorder problemRecorder; + private final InterimRefChain refChain; + private final CollectionDescriptors colDescriptors; + + private final HeapStats heapStats; + + private final int ptrSize, objHeaderSize, arrayHeaderSize; + + private int numCols; + @SuppressWarnings("unused") + private long totalColImplSize; // May use in future + private int numEmptyUnusedCols, numEmptyUsedCols, numEmptyCols, numSmallCols; + private int numSparseSmallCols, numSparseLargeCols, numBoxedNumberCols, numBarCols; + private long emptyUsedColsOvhd, emptyUnusedColsOvhd, emptyColsOvhd, smallColsOvhd; + private long sparseSmallColsOvhd, sparseLargeColsOvhd, boxedNumberColsOvhd, barColsOvhd; + + private int numObjArrays; + @SuppressWarnings("unused") + private long totalObjArraysShallowSize; // May use in future + private int numLengthZeroObjArrays, numLengthOneObjArrays, numEmptyObjArrays; + private int numSparseArrays, numBoxedNumberArrays, numBarObjArrays; + private long lengthZeroObjArraysOvhd, lengthOneObjArraysOvhd, emptyObjArraysOvhd; + private long sparseObjArraysOvhd, boxNumObjArraysOvhd, barObjArraysOvhd; + + private int numValueArrays; + private int numLengthZeroValueArrays, numLengthOneValueArrays, numEmptyValueArrays; + private int numLZTValueArrays, numUnusedHiBytesValueArrays; + private long lengthZeroValueArraysOvhd, lengthOneValueArraysOvhd, emptyValueArraysOvhd; + private long lztValueArraysOvhd, unusedHiBytesArraysOvhd; + + // Handling duplicate Strings + private final HeapStringReader stringReader; + private final int stringInstShallowSize; + private final DupStringHandler dupStringHandler; + private final DupArrayHandler dupArrayHandler; + + public DetailedStatsCalculator(Snapshot snapshot, HeapStats heapStats, ProblemRecorder problemRecorder, + boolean useBreadthFirstScan) { + this.snapshot = snapshot; + this.problemRecorder = problemRecorder; + colDescriptors = new CollectionDescriptors(snapshot); + scaner = useBreadthFirstScan ? new BreadthFirstHeapScanner(snapshot, this, problemRecorder) + : new DepthFirstHeapScaner(snapshot, this, problemRecorder, colDescriptors); + refChain = scaner.getRefChain(); + + this.heapStats = heapStats; + ptrSize = snapshot.getPointerSize(); + objHeaderSize = snapshot.getObjectHeaderSize(); + arrayHeaderSize = objHeaderSize + 4; + + DupStringStats dupStringStats = heapStats.dupStringStats; + stringReader = snapshot.getStringReader(); + stringInstShallowSize = dupStringStats.stringInstShallowSize; + dupStringHandler = new DupStringHandler(stringReader, dupStringStats.dupStrings, refChain, + stringInstShallowSize); + + dupArrayHandler = new DupArrayHandler(heapStats.dupArrayStats.dupArrays, refChain); + + for (JavaClass clazz : snapshot.getClasses()) { + if (clazz.isArray()) { + continue; + } + clazz.setAttachment(DataFieldStats.newInstance(clazz)); + } + } + + /** + * Invokes methods of HeapScaner, which results in callbacks into this class, that perform + * detailed stats calculations. In the end, updates the instance of HeapStats passed to the + * constructor. + */ + public void calculate() throws HprofParsingCancelledException { + scaner.analyzeViaRoots(); + scaner.analyzeViaAllObjectsEnum(); + scaner.done(); + + // Collect the contents of java.lang.System.props table (this is the one returned by System.getProperties()) + HashMap systemProps = SystemPropertiesReader.readProperties(snapshot, colDescriptors); + + // IMPORTANT: should do this so that optimizations in CachedReadBuffer do not + // cause problems if objects are read from the dump again and repeatedly, for + // example by the GUI JOverflow tool. + ReadBuffer readBuf = snapshot.getReadBuffer(); + if (readBuf instanceof CachedReadBuffer) { + ((CachedReadBuffer) readBuf).incrementPass(); + } + + ArrayList overheadsByClass = colDescriptors.getOverheadsByClass(); + ObjectHistogram objHisto = new ObjectHistogram(snapshot); + + heapStats.setObjectHistogram(objHisto) + .setCollectionNumberStats(numCols, numEmptyUnusedCols, numEmptyUsedCols, numEmptyCols, numSmallCols, + numSparseSmallCols, numSparseLargeCols, numBoxedNumberCols, numBarCols) + .setCollectionOverhead(emptyUnusedColsOvhd, emptyUsedColsOvhd, emptyColsOvhd, smallColsOvhd, + sparseSmallColsOvhd, sparseLargeColsOvhd, boxedNumberColsOvhd, barColsOvhd) + .setObjArrayNumberStats(numObjArrays, numLengthZeroObjArrays, numLengthOneObjArrays, numEmptyObjArrays, + numSparseArrays, numBoxedNumberArrays, numBarObjArrays) + .setObjArrayOverhead(lengthZeroObjArraysOvhd, lengthOneObjArraysOvhd, emptyObjArraysOvhd, + sparseObjArraysOvhd, boxNumObjArraysOvhd, barObjArraysOvhd) + .setCollectionOverheadByClass(overheadsByClass) + .setValueArrayNumberStats(numValueArrays, numLengthZeroValueArrays, numLengthOneValueArrays, + numEmptyValueArrays, numLZTValueArrays, numUnusedHiBytesValueArrays) + .setValueArrayOverhead(lengthZeroValueArraysOvhd, lengthOneValueArraysOvhd, emptyValueArraysOvhd, + lztValueArraysOvhd, unusedHiBytesArraysOvhd) + .setSystemProperties(systemProps); + } + + @Override + public CollectionInstanceDescriptor handleInstance(JavaObject obj, JavaThing[] fields) { + JavaClass clazz = obj.getClazz(); + DataFieldStats fieldStats = (DataFieldStats) clazz.getAttachment(); + fieldStats.handleFields(fields); + + if (obj.isVisitedAsCollectionImpl()) { + return null; + } + + if (clazz.isCollection()) { + return handleCollection(obj); + } else { + clazz.updateInclusiveInstanceSize(clazz.getInstanceSize()); + if (problemRecorder.shouldRecordGoodInstance(obj)) { + refChain.recordCurrentRefChainForGoodInstance(obj); + } + return null; + } + } + + /** + * For an object that is a known collection, checks if it has any problems. If a problem is + * found, it's recorded along with the current reference chain from GC root. Also records the + * implementation-inclusive size of this collection in its JavaClass, unless this object happens + * to be a part of implementation of another collection (like HashMap in HashSet). When + * impl-inclusive size is determined in {@link CollectionInstanceDescriptor#getImplSize()}, all + * the collection impl-n objects, e.g. HashMap$Entry, are marked with + * {@link JavaLazyReadObject#setVisitedAsCollectionImpl()}. This is important, since such + * objects are half-ignored later by handleInstance() method above. + */ + private CollectionInstanceDescriptor handleCollection(JavaObject col) { + CollectionInstanceDescriptor colDesc = colDescriptors.getDescriptor(col); + CollectionClassDescriptor classDesc = colDesc.getClassDescriptor(); + + // Check if this collection is in the implementation (via encapsulation) of + // another one. For example, an instance of java.util.HashMap is encapsulated + // by an instance of java.util.HashSet. In this case, the current collection + // should not be inspected for overhead on its own. + JavaObject potentialParentCol = refChain.getPointingJavaObject(); + if (potentialParentCol != null) { + if (classDesc.isInImplementationOf(potentialParentCol.getClazz().getName())) { + return null; + } + } + + numCols++; + // Get impl-inclusive size and mark collection implementation objects + int implSize = colDesc.getImplSize(); + + col.getClazz().updateInclusiveInstanceSize(implSize); + totalColImplSize += implSize; + + // Check if this collection is empty. A collection with this problem cannot + // have other problems. + int nEls = colDesc.getNumElements(); + if (nEls == 0) { + ProblemKind problemKind; + if (colDesc.getClassDescriptor().canDetermineModCount()) { + if (colDesc.getModCount() != 0) { + problemKind = ProblemKind.EMPTY_USED; + numEmptyUsedCols++; + emptyUsedColsOvhd += implSize; + } else { + problemKind = ProblemKind.EMPTY_UNUSED; + numEmptyUnusedCols++; + emptyUnusedColsOvhd += implSize; + } + } else { + problemKind = ProblemKind.EMPTY; + numEmptyCols++; + emptyColsOvhd += implSize; + } + classDesc.addProblematicCollection(problemKind, implSize); + refChain.recordCurrentRefChainForColCluster(col, colDesc, problemKind, implSize); + return colDesc; + } + + int ovhd; + boolean goodCollection = true; + + // Check if this collection is sparse + if (colDesc instanceof CollectionInstanceDescriptor.CapacityDifferentFromSize) { + CollectionInstanceDescriptor.CapacityDifferentFromSize arColDesc = (CollectionInstanceDescriptor.CapacityDifferentFromSize) colDesc; + ovhd = arColDesc.getSparsenessOverhead(ptrSize); + if (ovhd > 0) { + goodCollection = false; + ProblemKind problemKind; + if (arColDesc.getCapacity() <= arColDesc.getDefaultCapacity()) { + problemKind = ProblemKind.SPARSE_SMALL; + numSparseSmallCols++; + sparseSmallColsOvhd += ovhd; + } else { + problemKind = ProblemKind.SPARSE_LARGE; + numSparseLargeCols++; + sparseLargeColsOvhd += ovhd; + } + classDesc.addProblematicCollection(problemKind, ovhd); + refChain.recordCurrentRefChainForColCluster(col, colDesc, problemKind, ovhd); + } + } + + if (nEls <= SMALL_COL_MAX_SIZE) { + goodCollection = false; + // Calculate overhead as a number of bytes we save if we replace this data + // structure with an array of objects (or two, for maps). The array's own + // overhead is its header. The formula below is still not ideal, because the + // user likely won't be able to keep the exact-size array for each small + // collection - instead, they would probably have to use arrays of the same + // (highest) fixed size for all collections created at the same place in the code. + int multiplier = colDesc.getClassDescriptor().isMap() ? 2 : 1; + ovhd = colDesc.getImplSize() - multiplier * (nEls * ptrSize + arrayHeaderSize); + + numSmallCols++; + smallColsOvhd += ovhd; + classDesc.addProblematicCollection(ProblemKind.SMALL, ovhd); + refChain.recordCurrentRefChainForColCluster(col, colDesc, ProblemKind.SMALL, ovhd); + } + + // Check if this collection contains boxed numbers. + /* + * TODO: Our calculations for boxed arrays are much more precise, since they take into + * account possible multiple pointers to the same boxed object. To implement the same for a + * collection, we need to iterate all its elements, which may be time-consuming... + */ + ovhd = 0; + if (classDesc.isMap()) { + JavaHeapObject[] entryObjs = colDesc.getSampleKeyAndValue(); + int totalObjSize = 0, totalBoxedSize = 0, numPtrs = 0; + for (JavaHeapObject keyOrValue : entryObjs) { + if (keyOrValue == null) { + continue; + } + int boxedNumSize = keyOrValue.getClazz().getBoxedNumberSize(); + if (boxedNumSize > 0) { + totalBoxedSize += boxedNumSize; + totalObjSize += keyOrValue.getSize(); + numPtrs++; + } + } + if (totalBoxedSize > 0) { + // Take into account what happens if we replace this with an array of numbers, + // with a normal array header size. + ovhd = colDesc.getImplSize() + (totalObjSize + ptrSize * numPtrs - totalBoxedSize) * nEls + - arrayHeaderSize * numPtrs; + } + } else { + JavaHeapObject obj = colDesc.getSampleElement(); + if (obj != null) { + int boxedNumSize = obj.getClazz().getBoxedNumberSize(); + if (boxedNumSize > 0) { + ovhd = colDesc.getImplSize() + (obj.getSize() + ptrSize - boxedNumSize) * nEls - arrayHeaderSize; + } + } + } + + if (ovhd > 0) { + goodCollection = false; + numBoxedNumberCols++; + boxedNumberColsOvhd += ovhd; + classDesc.addProblematicCollection(ProblemKind.BOXED, ovhd); + refChain.recordCurrentRefChainForColCluster(col, colDesc, ProblemKind.BOXED, ovhd); + } + + // Check if this is a WeakHashMap or its subclass, in which elements have hard + // references back to keys. + WeakMapHandler wmHandler = WeakMapHandler.createInstance(colDesc); + if (wmHandler != null) { + WeakMapHandler.Result result = wmHandler.calculateOverhead(); + if (result != null) { +// numBadWeakCols++; +// badWeakColsOverhead += ovhd; + goodCollection = false; + classDesc.addProblematicCollection(ProblemKind.WEAK_MAP_WITH_BACK_REFS, result.overhead); + refChain.recordCurrentRefChainForWeakHashMapWithBackRefs(col, colDesc, result.overhead, + result.valueTypeAndFieldSample); + } + } + + BarArrayHandler barHandler = BarArrayHandler.createInstance(colDesc, colDescriptors); + if (barHandler != null) { + ovhd = barHandler.calculateOverhead(); + if (ovhd > 0) { + goodCollection = false; + numBarCols++; + barColsOvhd += ovhd; + classDesc.addProblematicCollection(ProblemKind.BAR, ovhd); + refChain.recordCurrentRefChainForColCluster(col, colDesc, ProblemKind.BAR, ovhd); + } + } + + if (goodCollection) { // No defects found for this collection + refChain.recordCurrentRefChainForGoodCollection(col, colDesc); + } + + return colDesc; + } + + @Override + public void handleObjectArray(JavaObjectArray objArray, JavaHeapObject[] elements) { + if (objArray.isVisitedAsCollectionImpl()) { + return; + } + + numObjArrays++; + int arraySize = objArray.getSize(); + totalObjArraysShallowSize += arraySize; + objArray.getClazz().updateInclusiveInstanceSize(arraySize); + + boolean goodArray = true; + + if (elements.length == 0) { +// goodArray = false; + CollectionClassDescriptor classDesc = colDescriptors.getStandaloneArrayDescriptor(objArray); + numLengthZeroObjArrays++; + int ovhd = arraySize; + lengthZeroObjArraysOvhd += ovhd; + classDesc.addProblematicCollection(ProblemKind.LENGTH_ZERO, ovhd); + refChain.recordCurrentRefChainForColCluster(objArray, new ArrayObjDescriptor(classDesc, 0, arraySize), + ProblemKind.LENGTH_ZERO, ovhd); + return; + } + + if (elements.length == 1) { + goodArray = false; + CollectionClassDescriptor classDesc = colDescriptors.getStandaloneArrayDescriptor(objArray); + numLengthOneObjArrays++; + int ovhd = arraySize; + lengthOneObjArraysOvhd += ovhd; + classDesc.addProblematicCollection(ProblemKind.LENGTH_ONE, ovhd); + refChain.recordCurrentRefChainForColCluster(objArray, new ArrayObjDescriptor(classDesc, 0, arraySize), + ProblemKind.LENGTH_ONE, ovhd); + } + + int nNullEntries = 0; + boolean boxedNumsPresent = false; + int totalBoxedNumOvhd = 0; + for (JavaHeapObject element : elements) { + if (element != null) { + int primitiveNumSize = element.getClazz().getBoxedNumberSize(); + if (primitiveNumSize > 0) { + boxedNumsPresent = true; + // Below is how much memory we would save (or maybe lose) if we replace a + // pointer with the primitive type array slot + totalBoxedNumOvhd += (ptrSize - primitiveNumSize); + JavaLazyReadObject elementObj = (JavaLazyReadObject) element; + // If the same Number object is referenced from two places, don't count it twice + if (!elementObj.isVisitedAsOther()) { + elementObj.setVisitedAsOther(); + totalBoxedNumOvhd += element.getSize(); // Savings from getting rid of boxed Number + } + } + } else { + nNullEntries++; + } + } + + CollectionClassDescriptor classDesc = colDescriptors.getStandaloneArrayDescriptor(objArray); + ArrayObjDescriptor arrayDesc = new ArrayObjDescriptor(classDesc, elements.length, arraySize); + + if (nNullEntries > elements.length / 2) { + // Empty or sparse object array dangling from something other than a known collection + goodArray = false; + if (nNullEntries == elements.length) { + numEmptyObjArrays++; + int ovhd = objArray.getSize(); + emptyObjArraysOvhd += ovhd; + classDesc.addProblematicCollection(ProblemKind.EMPTY, ovhd); + refChain.recordCurrentRefChainForColCluster(objArray, arrayDesc, ProblemKind.EMPTY, ovhd); + } else { + numSparseArrays++; + int ovhd = nNullEntries * ptrSize; + sparseObjArraysOvhd += ovhd; + classDesc.addProblematicCollection(ProblemKind.SPARSE_ARRAY, ovhd); + refChain.recordCurrentRefChainForColCluster(objArray, arrayDesc, ProblemKind.SPARSE_ARRAY, ovhd); + } + } + + if (boxedNumsPresent) { + numBoxedNumberArrays++; + // In extreme cases, the overhead of boxed numbers can actually be negative. For example, + // with 4-byte pointers, if we have 20 elements of Double[] array pointing at a single + // Double object, we will use 4*20 + 16 = 96 bytes. However, a double[] array of the same + // size would use 8*20 = 160 bytes. Thus we count all boxed arrays for consistency above, + // but we add up the overhead and store the details only for those where overhead is real. + if (totalBoxedNumOvhd > 0) { + goodArray = false; + boxNumObjArraysOvhd += totalBoxedNumOvhd; + classDesc.addProblematicCollection(ProblemKind.BOXED, totalBoxedNumOvhd); + refChain.recordCurrentRefChainForColCluster(objArray, arrayDesc, ProblemKind.BOXED, totalBoxedNumOvhd); + } + } + + BarArrayHandler barHandler = BarArrayHandler.createInstance(elements, colDescriptors); + if (barHandler != null) { + int ovhd = barHandler.calculateOverhead(); + if (ovhd > 0) { + goodArray = false; + numBarObjArrays++; + barObjArraysOvhd += ovhd; + classDesc.addProblematicCollection(ProblemKind.BAR, ovhd); + refChain.recordCurrentRefChainForColCluster(objArray, arrayDesc, ProblemKind.BAR, ovhd); + } + } + + if (goodArray) { // No defects found for this array + refChain.recordCurrentRefChainForGoodCollection(objArray, arrayDesc); + } + } + + @Override + public void handleValueArray(JavaValueArray valueArray) { + if (valueArray.isVisitedAsCollectionImpl()) { + return; + } + + numValueArrays++; + valueArray.getClazz().updateInclusiveInstanceSize(valueArray.getSize()); + boolean goodArray = true; + + byte[] data = valueArray.getValue(); + int elSize = valueArray.getElementSize(); + int numElements = data.length / elSize; + CollectionClassDescriptor classDesc = colDescriptors.getStandaloneArrayDescriptor(valueArray); + ArrayObjDescriptor arrayDesc = new ArrayObjDescriptor(classDesc, numElements, valueArray.getSize()); + char elementType = valueArray.getElementType(); + + PrimitiveArrayHandler pah = PrimitiveArrayHandler.createInstance(data, elSize, + elementType == 'F' || elementType == 'D'); + + if (pah.isLength0()) { +// goodArray = false; + numLengthZeroValueArrays++; + int ovhd = valueArray.getSize(); + lengthZeroValueArraysOvhd += ovhd; + classDesc.addProblematicCollection(ProblemKind.LENGTH_ZERO, ovhd); + refChain.recordCurrentRefChainForColCluster(valueArray, arrayDesc, ProblemKind.LENGTH_ZERO, ovhd); + return; + } + + if (pah.isLength1()) { + goodArray = false; + numLengthOneValueArrays++; + int ovhd = valueArray.getSize() + ptrSize - elSize; + lengthOneValueArraysOvhd += ovhd; + classDesc.addProblematicCollection(ProblemKind.LENGTH_ONE, ovhd); + refChain.recordCurrentRefChainForColCluster(valueArray, arrayDesc, ProblemKind.LENGTH_ONE, ovhd); + } + if (pah.isEmpty()) { + goodArray = false; + numEmptyValueArrays++; + int ovhd = valueArray.getSize(); + emptyValueArraysOvhd += ovhd; + classDesc.addProblematicCollection(ProblemKind.EMPTY, ovhd); + refChain.recordCurrentRefChainForColCluster(valueArray, arrayDesc, ProblemKind.EMPTY, ovhd); + } + int ovhd = pah.getLztOverhead(); + if (ovhd > 0) { + goodArray = false; + numLZTValueArrays++; + lztValueArraysOvhd += ovhd; + classDesc.addProblematicCollection(ProblemKind.LZT, ovhd); + refChain.recordCurrentRefChainForColCluster(valueArray, arrayDesc, ProblemKind.LZT, ovhd); + } + ovhd = pah.getUnusedHighBytesOvhd(); + if (ovhd > 0) { + goodArray = false; + numUnusedHiBytesValueArrays++; + unusedHiBytesArraysOvhd += ovhd; + classDesc.addProblematicCollection(ProblemKind.UNUSED_HI_BYTES, ovhd); + refChain.recordCurrentRefChainForColCluster(valueArray, arrayDesc, ProblemKind.UNUSED_HI_BYTES, ovhd); + } + + if (goodArray) { + refChain.recordCurrentRefChainForGoodCollection(valueArray, arrayDesc); + } + + dupArrayHandler.handleArray(valueArray); + } + + /** + * Checks the given String for duplication. Additionally, calculates the inclusive size of this + * String, that is, the size of the object itself plus the size of its char[] array, unless it + * has already been seen before (i.e. this char[] is utilized by more than one String). + */ + @Override + public void handleString(JavaObject strObj) { + JavaClass stringClazz = strObj.getClazz(); + stringClazz.updateInclusiveInstanceSize(stringInstShallowSize); + + boolean duplicated = dupStringHandler.handleString(strObj); + + int implInclusiveSize = stringInstShallowSize; + JavaValueArray backingCharArray = duplicated ? dupStringHandler.getLastReadBackingArray() + : stringReader.getCharArrayForString(strObj); + if (backingCharArray != null) { // Not sure why we can get null here - truncated heap dumps? + if (!backingCharArray.isVisited()) { + int backingCharArraySize = backingCharArray.getSize(); + stringClazz.updateInclusiveInstanceSize(backingCharArraySize); + backingCharArray.setVisited(); + implInclusiveSize += backingCharArraySize; + scaner.incrementCurrentProcessedObjNo(); + } + } + + if (!duplicated) { + // Normal, non-duplicated string. Record it, so that eventually for fields + // pointing at duplicated strings we also know how many "good" strings they + // also point to. + refChain.recordCurrentRefChainForNonDupString(strObj, implInclusiveSize); + } + } + + public int getProgressPercentage() { + return scaner.getProgressPercentage(); + } + + public void cancelCalculation() { + scaner.cancelCalculation(); + } + + /** + * A collection instance descriptor that's instantiated and used for any object array. + */ + private static class ArrayObjDescriptor implements CollectionInstanceDescriptor { + + private static final String NOT_SUPPORTED = "is not supported for arrays"; + + private final CollectionClassDescriptor classDesc; + private final int numElements, arraySize; + + ArrayObjDescriptor(CollectionClassDescriptor classDesc, int numElements, int arraySize) { + this.classDesc = classDesc; + this.numElements = numElements; + this.arraySize = arraySize; + } + + @Override + public CollectionClassDescriptor getClassDescriptor() { + return classDesc; + } + + @Override + public int getNumElements() { + return numElements; + } + + @Override + public int getImplSize() { + return arraySize; + } + + @Override + public void iterateList(ListIteratorCallback cb) { + throw new UnsupportedOperationException(); + } + + @Override + public void iterateMap(MapIteratorCallback cb) { + throw new UnsupportedOperationException(); + } + + @Override + public JavaHeapObject getSampleElement() { + throw new UnsupportedOperationException("Getting sample element " + NOT_SUPPORTED); + } + + @Override + public JavaHeapObject[] getSampleKeyAndValue() { + throw new UnsupportedOperationException("Getting sample key/value " + NOT_SUPPORTED); + } + + @Override + public long getModCount() { + throw new UnsupportedOperationException("Getting modCount " + NOT_SUPPORTED); + } + + @Override + public boolean hasExtraObjFields() { + return false; + } + + @Override + public void filterExtraObjFields(JavaThing[] fields) { + throw new UnsupportedOperationException("Filtering extra obj fields " + NOT_SUPPORTED); + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/DupArrayHandler.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/DupArrayHandler.java new file mode 100644 index 00000000..159f3503 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/DupArrayHandler.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import java.util.List; + +import org.openjdk.jmc.joverflow.heap.model.JavaValueArray; +import org.openjdk.jmc.joverflow.support.DupArrayStats; +import org.openjdk.jmc.joverflow.util.ValueWitIntIdMap; + +/** + * A utility class for handling duplicated arrays during detailed analysis. + */ +public class DupArrayHandler { + private final ValueWitIntIdMap dupArrays; + private final InterimRefChain refChain; + + DupArrayHandler(List dupArrayList, InterimRefChain refChain) { + dupArrays = new ValueWitIntIdMap<>(dupArrayList.size()); + for (DupArrayStats.Entry entry : dupArrayList) { + dupArrays.put(entry); + } + this.refChain = refChain; + } + + boolean handleArray(JavaValueArray array) { + int internalId = array.getInternalId(); + DupArrayStats.Entry ae = dupArrays.get(internalId); + + boolean isDuplicate = (ae != null); + + if (isDuplicate) { + int ovhd = ae.getOvhdForNextArrayCopy(); + refChain.recordCurrentRefChainForDupArray(array, ovhd); + } else { + refChain.recordCurrentRefChainForNonDupArray(array); + } + + return isDuplicate; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/DupStringHandler.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/DupStringHandler.java new file mode 100644 index 00000000..1275ee4c --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/DupStringHandler.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import java.util.List; + +import org.openjdk.jmc.joverflow.heap.model.HeapStringReader; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaValueArray; +import org.openjdk.jmc.joverflow.support.DupStringStats; +import org.openjdk.jmc.joverflow.util.ValueWitIntIdMap; + +/** + * A utility class for handling duplicated String instances during detailed analysis. + */ +public class DupStringHandler { + private final HeapStringReader stringReader; + private final ValueWitIntIdMap dupStrings; + private final InterimRefChain refChain; + private final int stringInstShallowSize; + + private JavaValueArray backingCharArray; // Backing char[] array read by last handleString() call + + DupStringHandler(HeapStringReader stringReader, List dupStringList, InterimRefChain refChain, + int stringInstShallowSize) { + this.stringReader = stringReader; + dupStrings = new ValueWitIntIdMap<>(dupStringList.size()); + for (DupStringStats.Entry entry : dupStringList) { + dupStrings.put(entry); + } + this.refChain = refChain; + this.stringInstShallowSize = stringInstShallowSize; + } + + /** + * Analyzes the given string for duplication. If the string is duplicated, calculates its + * overhead and records the reference chain for it. Returns true if the string is duplicated, + * false otherwise. + *

+ * IMPORTANT: it does not read the backing char array of this string if it's not redundant, and + * if it's read, does not mark it as visited. Thus the caller may check its original status, and + * then MUST set it to visited so that further overhead calculations are done correctly. + */ + boolean handleString(JavaObject strObj) { + int internalId = strObj.getInternalId(); + DupStringStats.Entry se = dupStrings.get(internalId); + + if (se == null) { // Non-duplicate string + return false; + } + + backingCharArray = stringReader.getCharArrayForString(strObj); + if (backingCharArray == null) { + // Paranoid check + // Probably unresolved pointer in a corrupted heap dump + return false; + } + + int implInclusiveSize = stringInstShallowSize; + int ovhd = se.getOvhdForNextStringCopy(); + boolean hasDupBackingCharArray = false; + if (!backingCharArray.isVisited()) { + implInclusiveSize += backingCharArray.getSize(); + hasDupBackingCharArray = true; + } + + refChain.recordCurrentRefChainForDupString(strObj, se.string, implInclusiveSize, ovhd, hasDupBackingCharArray); + return true; + } + + /** + * Returns the backing char[] array read by the last handleString() call. + */ + JavaValueArray getLastReadBackingArray() { + return backingCharArray; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/HeapScaner.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/HeapScaner.java new file mode 100644 index 00000000..7369e1ae --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/HeapScaner.java @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import java.util.List; +import java.util.logging.Logger; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaLazyReadObject; +import org.openjdk.jmc.joverflow.heap.model.JavaValueArray; +import org.openjdk.jmc.joverflow.heap.model.Root; +import org.openjdk.jmc.joverflow.heap.model.Snapshot; +import org.openjdk.jmc.joverflow.heap.parser.HprofParsingCancelledException; + +/** + * Base functionality that's common for all heap scaners. + */ +abstract class HeapScaner { + private final static Logger LOGGER = Logger.getLogger("org.openjdk.jmc.joverflow.stats"); //$NON-NLS-1$ + + protected final Snapshot snapshot; + private final InterimRefChain refChain; + + // Reporting progress + protected int currentProcessedObjNo; + private final int nTotalObjects; + protected boolean cancelled; + + // Debugging + private static final boolean REPORT_UNVISITED = false; + + protected HeapScaner(Snapshot snapshot, InterimRefChain refChain) { + this.snapshot = snapshot; + this.refChain = refChain; + nTotalObjects = snapshot.getNumObjects(); + } + + protected abstract void scanObjectsFromRootObj(JavaHeapObject rootObj); + + /** + * Analyzes the heap for anti-patterns starting from the set of heap roots. This has the + * advantage that when a problematic data structure is found, we have a trace to it from a root + * readily available. + */ + protected void analyzeViaRoots() throws HprofParsingCancelledException { + List roots = snapshot.getRoots(); + + for (Root root : roots) { + refChain.setCurrentRoot(root); + JavaHeapObject rootObj = snapshot.getObjectForId(root.getId()); + if (rootObj == null) { + continue; + } + if (rootObj instanceof JavaValueArray) { + continue; + } + scanObjectsFromRootObj(rootObj); + } + + refChain.setCurrentRoot(null); + } + + /** + * Analyzes the heap by scanning all of the (not yet scanned) objects in it. We have this method + * because it seems that some live objects may not always be reachable from the root set. I am + * not sure why is that. + */ + protected void analyzeViaAllObjectsEnum() throws HprofParsingCancelledException { + int nObjsBefore = currentProcessedObjNo; + refChain.setCurrentRoot(Root.UNKNOWN_ROOT); + if (cancelled) { + throw new HprofParsingCancelledException(); + } + + // First, scan all collections that contain other collections in their + // implementation, to avoid counting the latter as first-class collections + for (JavaLazyReadObject javaHeapObj : snapshot.getUnvisitedObjects()) { + if (javaHeapObj.getClazz().isCollectionWithOtherCollectionInImpl()) { + refChain.resetCurrentRoot(); + scanObjectsFromRootObj(javaHeapObj); + } + } + if (cancelled) { + throw new HprofParsingCancelledException(); + } + + // Next, scan all collections, to avoid counting as standalone e.g. Object[] + // arrays that belong to collections. Same with Strings. + for (JavaHeapObject javaHeapObj : snapshot.getUnvisitedObjects()) { + JavaClass objClazz = javaHeapObj.getClazz(); + if (objClazz.isCollection() || objClazz.isString()) { + refChain.resetCurrentRoot(); + scanObjectsFromRootObj(javaHeapObj); + } + } + if (cancelled) { + throw new HprofParsingCancelledException(); + } + + // Finally, scan all remaining objects + for (JavaHeapObject javaHeapObj : snapshot.getUnvisitedObjects()) { + refChain.resetCurrentRoot(); + scanObjectsFromRootObj(javaHeapObj); + } + if (cancelled) { + throw new HprofParsingCancelledException(); + } + + if (REPORT_UNVISITED) { + LOGGER.info("\rDebug info:"); + LOGGER.info("Objects unreachable from GC roots: " + (currentProcessedObjNo - nObjsBefore)); + } + } + + /** Should be necessarily called after all objects have been scanned */ + protected void done() { + refChain.convertRefChainElementsToFinalRepresentation(); + } + + protected InterimRefChain getRefChain() { + return refChain; + } + + /** Used for providing user an estimate of progress made. */ + public synchronized int getProgressPercentage() { + return (int) (((long) currentProcessedObjNo) * 100 / nTotalObjects); + } + + public synchronized void cancelCalculation() { + cancelled = true; + } + + void incrementCurrentProcessedObjNo() { + currentProcessedObjNo++; + } + +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/InterimRefChain.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/InterimRefChain.java new file mode 100644 index 00000000..d91d7b1c --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/InterimRefChain.java @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import java.util.ArrayList; + +import org.openjdk.jmc.joverflow.descriptors.CollectionInstanceDescriptor; +import org.openjdk.jmc.joverflow.heap.model.JavaLazyReadObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaValueArray; +import org.openjdk.jmc.joverflow.heap.model.Root; +import org.openjdk.jmc.joverflow.support.Constants; +import org.openjdk.jmc.joverflow.support.ProblemRecorder; +import org.openjdk.jmc.joverflow.support.RefChainElement; +import org.openjdk.jmc.joverflow.support.RefChainElementImpl; + +/** + */ +public abstract class InterimRefChain { + private final ProblemRecorder problemRecorder; + + protected RefChainElementImpl.GCRoot curRootRefChainElement; + protected final ArrayList rootElements; + protected boolean newGCRoot; + protected RefChainElement curCondensedRefChainElement; + + public InterimRefChain(ProblemRecorder problemRecorder) { + this.problemRecorder = problemRecorder; + rootElements = new ArrayList<>(); + } + + void setCurrentRoot(Root currentRoot) { + if (curRootRefChainElement != null) { + // Converting the previous scanned subtree to final format now removes unneeded objects early + curRootRefChainElement.switchTreeToFinalFormat(); + } + + if (currentRoot == null) { + currentRoot = Root.UNKNOWN_ROOT; + } + curRootRefChainElement = new RefChainElementImpl.GCRoot(currentRoot); + rootElements.add(curRootRefChainElement); + resetCurrentRoot(); + } + + void resetCurrentRoot() { + newGCRoot = true; + curCondensedRefChainElement = curRootRefChainElement; + onCurrentRootReset(); + } + + /** + * Called in the end of both setCurrentRoot() and resetCurrentRoot() methods. Should perform + * operations specific to the given implementation of InterimRefChain. + */ + protected abstract void onCurrentRootReset(); + + /** + * For the object that is in the end of the current reference chain, returns the object that + * points to it. Returns null if this object is referenced directly by the GC root, or if the + * pointer is from an array rather than an object field. + */ + protected abstract JavaObject getPointingJavaObject(); + + /** + * This operation is for internal use only (in the recordXXX() methods below). In + * InterimRefChainStack it's quite expensive. + */ + protected abstract RefChainElement getLastRefChainElement(); + + protected void convertRefChainElementsToFinalRepresentation() { + // Convert the last scanned subtree to final format + curRootRefChainElement.switchTreeToFinalFormat(); + } + + /** + * Records permanently the snapshot of the current reference chain, associating it with the + * collection (or standalone array) object at the end of the chain, and the specified kind and + * value of overhead. + */ + void recordCurrentRefChainForColCluster( + JavaLazyReadObject col, CollectionInstanceDescriptor colDesc, Constants.ProblemKind ovhdKind, int ovhd) { + RefChainElement referer = getLastRefChainElement(); + problemRecorder.recordProblematicCollection(col, colDesc, ovhdKind, ovhd, referer); + } + + /** + * Records permanently the snapshot of the current reference chain, associating it with a good, + * no-problem collection object at the end of the chain. This is used to determine which + * problematic collection clusters also include (many) "normal", good collections. + */ + void recordCurrentRefChainForGoodCollection(JavaLazyReadObject col, CollectionInstanceDescriptor colDesc) { + RefChainElement referer = getLastRefChainElement(); + problemRecorder.recordGoodCollection(col, colDesc, referer); + } + + /** + * Records permanently the snapshot of the current reference chain, associating it with the + * duplicated string object at the end of the chain, the specified overhead value, and whether + * or not there is a duplicated backing char array for this string. + */ + void recordCurrentRefChainForDupString( + JavaObject stringObj, String s, int implInclusiveSize, int ovhd, boolean hasDupBackingCharArray) { + RefChainElement referer = getLastRefChainElement(); + problemRecorder.recordDuplicateString(stringObj, s, implInclusiveSize, ovhd, hasDupBackingCharArray, referer); + } + + /** + * Records permanently the snapshot of the current reference chain, associating it with the + * non-duplicated string object at the end of the chain. This is used in detailed analysis for + * duplicated strings, to determine which duplicated string clusters also include (many) + * "normal", nonduplicated strings. + */ + void recordCurrentRefChainForNonDupString(JavaObject stringObj, int implInclusiveSize) { + RefChainElement referer = getLastRefChainElement(); + problemRecorder.recordNonDuplicateString(stringObj, implInclusiveSize, referer); + } + + /** + * Records permanently the snapshot of the current reference chain, associating it with the + * duplicated array object at the end of the chain and the specified overhead value. + */ + void recordCurrentRefChainForDupArray(JavaValueArray ar, int ovhd) { + RefChainElement referer = getLastRefChainElement(); + problemRecorder.recordDuplicateArray(ar, ovhd, referer); + } + + /** + * Records permanently the snapshot of the current reference chain, associating it with the + * non-duplicated array object at the end of the chain. This is used in detailed analysis for + * duplicated arrays, to determine which duplicated array clusters also include (many) "normal", + * nonduplicated arrays. + */ + void recordCurrentRefChainForNonDupArray(JavaValueArray ar) { + RefChainElement referer = getLastRefChainElement(); + problemRecorder.recordNonDuplicateArray(ar, referer); + } + + /** + * Same as previous methods, but for an instance of WeakHashMaps that has hard references back + * from values to keys. + */ + void recordCurrentRefChainForWeakHashMapWithBackRefs( + JavaObject col, CollectionInstanceDescriptor colDesc, int ovhd, String valueTypeAndSample) { + RefChainElement referer = getLastRefChainElement(); + problemRecorder.recordWeakHashMapWithBackRefs(col, colDesc, ovhd, valueTypeAndSample, referer); + } + + /** + * Records permanently the snapshot of the current reference chain, associating it with the good + * instance at the end of the chain. + */ + void recordCurrentRefChainForGoodInstance(JavaObject obj) { + RefChainElement referer = getLastRefChainElement(); + problemRecorder.recordGoodInstance(obj, referer); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/InterimRefChainStack.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/InterimRefChainStack.java new file mode 100644 index 00000000..4cc4ff3c --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/InterimRefChainStack.java @@ -0,0 +1,554 @@ +/* + * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import java.util.ArrayList; +import java.util.logging.Logger; + +import org.openjdk.jmc.joverflow.descriptors.CollectionClassDescriptor; +import org.openjdk.jmc.joverflow.descriptors.CollectionDescriptors; +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaField; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObjectArray; +import org.openjdk.jmc.joverflow.support.Constants; +import org.openjdk.jmc.joverflow.support.ProblemRecorder; +import org.openjdk.jmc.joverflow.support.RefChainElement; +import org.openjdk.jmc.joverflow.support.RefChainElementImpl; +import org.openjdk.jmc.joverflow.util.FastStack; +import org.openjdk.jmc.joverflow.util.IndexContainer; + +/** + * Functionality for maintaining the internal reference chain leading from the current root to the + * currently scanned object, and recording its snapshot when we come across an interesting object. + * This reference chain representation is internal and interim; the permanent ones, that are + * accumulated while JOverflow is running, are eventually returned by it as ReferenceChain objects. + */ +class InterimRefChainStack extends InterimRefChain { + + private final CollectionDescriptors colDescriptors; + + // Contains alternating objects representing a JavaHeapObject and an + // integer index (of field or array element) in the form of IndexContainer. + private final FastStack refChain = new FastStack<>(512); + private final ArrayList condensedRefChain; + + // Management of conversion from the concrete to condensed (abstract) + // reference chain + private int numCommonRefChainEls; + private int lastRecordedRefTreeIdx; + private int[] refChainToRefTreeIdx = new int[16000]; + + InterimRefChainStack(ProblemRecorder problemRecorder, CollectionDescriptors colDescriptors) { + super(problemRecorder); + this.colDescriptors = colDescriptors; + condensedRefChain = new ArrayList<>(256); + } + + @Override + protected void onCurrentRootReset() { + condensedRefChain.clear(); + condensedRefChain.add(curCondensedRefChainElement); + } + + void push(JavaHeapObject javaHeapObj) { + refChain.push(javaHeapObj); + } + + void pushIndexContainer(IndexContainer idx) { + // Note that I tried to use a pool of reusable IndexContainers instead of creating a new one + // here each time, but looks like it didn't improve performance, and might even have made it + // slightly worse. + refChain.push(idx); + } + + void setCurrentFieldOrArrayIndex(int idx) { + ((IndexContainer) refChain.peek()).set(idx); + } + + IndexContainer getCurrentIndexContainer() { + return (IndexContainer) refChain.peek(); + } + + void pop() { + refChain.pop(); + + if (refChain.size() < numCommonRefChainEls) { + numCommonRefChainEls = refChain.size(); + } + } + + void pop2() { + refChain.pop(); + refChain.pop(); + + if (refChain.size() < numCommonRefChainEls) { + numCommonRefChainEls = refChain.size(); + } + } + + @Override + protected JavaObject getPointingJavaObject() { + if (refChain.size() < 2) { + return null; + } + + Object obj = refChain.get(refChain.size() - 3); + if (!(obj instanceof JavaObject)) { + return null; + } + return (JavaObject) obj; + } + + int size() { + return refChain.size(); + } + + JavaHeapObject peekJavaHeapObject(int idxFromBack) { + return (JavaHeapObject) refChain.get(refChain.size() - 1 - idxFromBack); + } + + int peekIndex(int idxFromBack) { + return ((IndexContainer) refChain.get(refChain.size() - 1 - idxFromBack)).get(); + } + + /** + * Takes the current concrete reference chain and, using information about the previous recorded + * ref chain for the same root, generates the elements of the condensed reference chain. The + * elements will hang off either a new GC root, or will be attached to some existing + * RefChainElement (adding an incremental condensed reference chain). In the condensed ref + * chain, multiple objects representing implementation details of known collections and + * easy-to-discover linked lists are collapsed into special individual RefChainElements. Returns + * the last (leaf) element of the condensed ref chain. + */ + @Override + protected RefChainElement getLastRefChainElement() { + if (refChainToRefTreeIdx.length < refChain.size()) { + int[] old = refChainToRefTreeIdx; + refChainToRefTreeIdx = new int[refChain.size() * 5 / 4]; + System.arraycopy(old, 0, refChainToRefTreeIdx, 0, old.length); + } + int chainSizeMinusOne = refChain.size() - 1; + RefChainElement curRefChainElement = curCondensedRefChainElement; + + if (newGCRoot || numCommonRefChainEls == 0) { // Build non-incremental condensed ref chain + newGCRoot = false; + int refTreeElementIdx = -1; // Proper initial value given loop code below + for (int i = 0; i < chainSizeMinusOne; i++) { + refTreeElementIdx++; + CollapsedObj clpsObj = collapseLinkedListImpl(i, curRefChainElement); + if (clpsObj == null) { + clpsObj = collapseCollectionImpl(i, curRefChainElement); + } + + if (clpsObj != null) { // We went through a collection or linked list implementation + curRefChainElement = clpsObj.desc; + for (int j = i; j <= clpsObj.endIdx; j++) { + refChainToRefTreeIdx[j] = refTreeElementIdx; + } + i = clpsObj.endIdx; + } else { // index i points at a regular object in the reference chain + refChainToRefTreeIdx[i] = refChainToRefTreeIdx[i + 1] = refTreeElementIdx; + JavaHeapObject javaHeapObj = (JavaHeapObject) refChain.get(i++); + int fieldOrArrIdx = ((IndexContainer) refChain.get(i)).get(); + curRefChainElement = getLinkDesc(javaHeapObj, fieldOrArrIdx, curRefChainElement); + } + condensedRefChain.add(curRefChainElement); + } + lastRecordedRefTreeIdx = refTreeElementIdx; + numCommonRefChainEls = chainSizeMinusOne; + curCondensedRefChainElement = curRefChainElement; + return curRefChainElement; + } + + // Otherwise, need to build an incremental condensedRefChain. + int lastCommonRefChainIdx = numCommonRefChainEls - 1; + if ((lastCommonRefChainIdx & 1) == 1) { + // Need lastCommonRefChainIdx to point at an object, not at field after it. + // For that, the number should be even. + lastCommonRefChainIdx--; + } + + // Incremental compressed ref chain cannot be constructed properly if + // lastCommonRefChainIdx points into the middle of compressible data structure + lastCommonRefChainIdx = skipBackPotentialCollectionImplOrLinkedList(lastCommonRefChainIdx); + + // This gets lastCommonRefChainIdx above a compressible data structure + // or another object that will be the start of the incremental chain + lastCommonRefChainIdx -= 2; + // This points at the object that starts the incremental chain + // +2 is essentially a performance optimization; don't use it when in doubt. + int startIdx = lastCommonRefChainIdx == 0 ? 0 : lastCommonRefChainIdx + 2; + + // If startIdx == 0, we have to attach the new chain directly to the GC root. + // That does not allow us to use normal indexing machinery below. + int nStepsBack = startIdx > 0 ? lastRecordedRefTreeIdx - refChainToRefTreeIdx[lastCommonRefChainIdx] + : lastRecordedRefTreeIdx + 1; + int refTreeElementIdx = startIdx > 0 ? refChainToRefTreeIdx[lastCommonRefChainIdx] : -1; + + int lastIndex = condensedRefChain.size() - 1; + for (int i = 0; i < nStepsBack; i++, lastIndex--) { + condensedRefChain.remove(lastIndex); + } + curRefChainElement = condensedRefChain.get(lastIndex); + + for (int i = startIdx; i < chainSizeMinusOne; i++) { + refTreeElementIdx++; + CollapsedObj clpsObj = collapseLinkedListImpl(i, curRefChainElement); + if (clpsObj == null) { + clpsObj = collapseCollectionImpl(i, curRefChainElement); + } + + boolean stitchingLinkedListParts = false; + if (clpsObj != null) { // We went through a collection or linked list implementation + if (curRefChainElement == clpsObj.desc) { + // May happen for a linked list - we have two entities representing compressed + // parts of the same list. We need to, in effect, merge them together. + refTreeElementIdx--; + stitchingLinkedListParts = true; + } else { + curRefChainElement = clpsObj.desc; + } + for (int j = i; j <= clpsObj.endIdx; j++) { + refChainToRefTreeIdx[j] = refTreeElementIdx; + } + i = clpsObj.endIdx; + } else { // index i points at a regular object in the reference chain + refChainToRefTreeIdx[i] = refChainToRefTreeIdx[i + 1] = refTreeElementIdx; + JavaHeapObject javaHeapObj = (JavaHeapObject) refChain.get(i++); + int idx = ((IndexContainer) refChain.get(i)).get(); + curRefChainElement = getLinkDesc(javaHeapObj, idx, curRefChainElement); + } + if (!stitchingLinkedListParts) { + condensedRefChain.add(curRefChainElement); + } + } + + lastRecordedRefTreeIdx = refTreeElementIdx; + numCommonRefChainEls = chainSizeMinusOne; + curCondensedRefChainElement = curRefChainElement; + return curRefChainElement; + } + + /** + * Starting from the reference chain element at the specified index, skips back as far as needed + * so that the resulting object is not an array or an instance of an inner class. This is a + * rough way to skip chain elements that are potentially in the implementation of some + * collection. Also takes some measures if we happen to be inside a linked list. + */ + private int skipBackPotentialCollectionImplOrLinkedList(int idx) { + int llStepsBack = 0; + while (true) { + if (idx <= 2) { + return idx; + } + + IndexContainer idx1 = (IndexContainer) refChain.get(idx + 1); + IndexContainer idx0 = (IndexContainer) refChain.get(idx - 1); + JavaHeapObject obj1 = (JavaHeapObject) refChain.get(idx); + JavaHeapObject obj0 = (JavaHeapObject) refChain.get(idx - 2); + if ((idx0.get() == idx1.get()) && (obj1.getClazz() == obj0.getClazz())) { + // We are in a linked list. Walk back a little, so that the new + // part of LL is not too short, and the old and the new parts can + // later be "stitched" together. + idx -= 2; + llStepsBack++; + if (llStepsBack > 2) { + return idx; + } + } else { + String className = obj1.getClazz().getName(); + if (className.charAt(0) == '[' || className.contains("$")) { + idx -= 2; + } else { + CollectionClassDescriptor colDesc = colDescriptors.getClassDescriptor(className); + if (colDesc != null && colDesc.isInImplementationOf(obj0.getClazz().getName())) { + idx -= 2; + } else { + return idx; + } + } + } + } + } + + private CollapsedObj collapseCollectionImpl(int startIdx, RefChainElement referer) { + JavaHeapObject javaHeapObj = (JavaHeapObject) refChain.get(startIdx); + if (!(javaHeapObj instanceof JavaObject)) { + return null; + } + + JavaObject javaObj = (JavaObject) javaHeapObj; + CollectionClassDescriptor colDesc = colDescriptors.getClassDescriptor(javaObj); + if (colDesc == null) { + return null; + } + + int refChainIdx = startIdx + 2; + if (refChainIdx >= refChain.size()) { + return null; + } + JavaHeapObject obj = (JavaHeapObject) refChain.get(refChainIdx); + String objClassName = obj.getClazz().getName(); + if (!(colDesc.isImplClassName(objClassName) || objClassName.equals(Constants.OBJECT_ARRAY))) { + return null; + } + + // Collections never point to their elements directly. + // We skip the field index (e.g. HashMap.table), the object that the field points at + // (e.g. array of HashMap$Entry objects), the field or array index following that object + // (e.g. index into array of entries), and look at the next object + int refChainSize = refChain.size(); + refChainIdx = startIdx + 4; + if (refChainIdx >= refChainSize) { + // Can currently happen for e.g. ConcurrentHashMap in JDK7, where Segments are allocated lazily + // So we can have CHM -> idx of 'segments' field -> Segment[] -> null + return new CollapsedObj(javaObj.getClazz(), refChainSize - 1, referer); + } + + obj = (JavaHeapObject) refChain.get(refChainIdx); + objClassName = obj.getClazz().getName(); + + while (colDesc.isImplClassName(objClassName)) { + refChainIdx += 2; + if (refChainIdx >= refChainSize) { + break; + } + objClassName = ((JavaHeapObject) refChain.get(refChainIdx)).getClazz().getName(); + } + + return new CollapsedObj(javaObj.getClazz(), refChainIdx - 1, referer); + } + + private CollapsedObj collapseLinkedListImpl(int startIdx, RefChainElement referer) { + JavaHeapObject javaHeapObj = (JavaHeapObject) refChain.get(startIdx); + if (!(javaHeapObj instanceof JavaObject)) { + return null; + } + + int refChainLastIdx = refChain.size() - 1; + if (startIdx + 3 >= refChainLastIdx) { + return null; + } + + int refChainIdx = startIdx; + int fieldIdx = ((IndexContainer) refChain.get(startIdx + 1)).get(); + JavaClass elementClass = javaHeapObj.getClazz(); + JavaClass llDefiningClass = elementClass.getDeclaringClassForField(fieldIdx); + + while (true) { + int fieldIdx1 = ((IndexContainer) refChain.get(refChainIdx + 3)).get(); + if (fieldIdx1 != fieldIdx) { + break; + } + javaHeapObj = (JavaHeapObject) refChain.get(refChainIdx + 2); + JavaClass elementClass1 = javaHeapObj.getClazz(); + if (elementClass1.getDeclaringClassForField(fieldIdx) != llDefiningClass) { + break; + } + if (elementClass != null && elementClass1 != elementClass) { + elementClass = null; + } + + refChainIdx += 2; + if (refChainIdx + 3 > refChainLastIdx) { + break; + } + } + + if (refChainIdx == startIdx) { + return null; + } + + JavaClass llClazz = elementClass != null ? elementClass : llDefiningClass; + + // Check if we actually look at the same linked list that's already been there + if (referer instanceof RefChainElementImpl.InstanceFieldOrLinkedList) { + RefChainElementImpl.InstanceFieldOrLinkedList otherList = (RefChainElementImpl.InstanceFieldOrLinkedList) referer; + if (!otherList.isInstanceField() && otherList.getJavaClass() == llClazz + && otherList.getFieldIdx() == fieldIdx) { + return new CollapsedObj(refChainIdx + 1, referer); + } + } else if (referer instanceof RefChainElementImpl.Collection) { + // This is intended to help stitch together things like java.util.LinkedList, + // which we first recognize as a collection, and then, when we see it from the + // middle, as a noncategorized linked list. + RefChainElementImpl.Collection otherCol = (RefChainElementImpl.Collection) referer; + CollectionClassDescriptor colDesc = colDescriptors.getClassDescriptor(otherCol.getJavaClass().getName()); + if (colDesc != null && colDesc.isImplClassName(llClazz.getName())) { + // Let's now take a look at the last element - maybe it also belongs to this collection + // (though not technically to the linked list), like LinkedList$Entry.element + JavaHeapObject lastObj = (JavaHeapObject) refChain.get(refChainIdx + 2); + if (colDesc.isImplClassName(lastObj.getClazz().getName())) { + refChainIdx += 2; + } + int endIdx = refChainIdx + 1 < refChain.size() ? refChainIdx + 1 : refChainIdx; + return new CollapsedObj(endIdx, referer); + } + } + + return new CollapsedObj(llClazz, fieldIdx, refChainIdx + 1, referer); + } + + private String getFullLinkDesc(JavaHeapObject javaHeapObj, int idx) { + if (javaHeapObj instanceof JavaObject) { + JavaObject javaObj = (JavaObject) javaHeapObj; + return javaObj.getClazz().getName() + '.' + getFieldDesc(javaObj, idx); + } else if (javaHeapObj instanceof JavaClass) { + JavaClass clazz = (JavaClass) javaHeapObj; + return clazz.getName() + ':' + getStaticFieldDesc(clazz, idx); + } else if (javaHeapObj instanceof JavaObjectArray) { + JavaObjectArray arrayObj = (JavaObjectArray) javaHeapObj; + return arrayObj.getClazz().getName() + '[' + idx + ']'; + } else { + return null; + } + } + + private RefChainElement getLinkDesc(JavaHeapObject javaHeapObj, int idx, RefChainElement referer) { + if (javaHeapObj instanceof JavaObject) { + return RefChainElementImpl.getInstanceFieldElement(javaHeapObj.getClazz(), idx, referer); + } else if (javaHeapObj instanceof JavaClass) { + JavaClass clazz = (JavaClass) javaHeapObj; + return RefChainElementImpl.getStaticFieldElement(clazz, idx, referer); + } else if (javaHeapObj instanceof JavaObjectArray) { + return RefChainElementImpl.getCompoundArrayElement(javaHeapObj.getClazz(), referer); + } else { + throw new RuntimeException("JavaHeapObject of wrong type is supplied: " + javaHeapObj); + } + } + + private String getFieldDesc(JavaObject javaObj, int idx) { + JavaClass clazz = javaObj.getClazz(); + return clazz.getFieldForInstance(idx).getName(); + } + + private String getStaticFieldDesc(JavaClass clazz, int idx) { + JavaField[] statics = clazz.getStaticFields(); + return statics[idx].getName(); + } + + /** + * Used for passing back information about a collapsed series of reference chain elements - + * those that belong to a collection implementation or to the linked list. + */ + private static class CollapsedObj { + RefChainElement desc; + int endIdx; // The index of the last element in this object + + CollapsedObj(JavaClass clazz, int endIdx, RefChainElement referer) { + this.desc = RefChainElementImpl.getCompoundCollectionElement(clazz, referer); + this.endIdx = endIdx; + } + + CollapsedObj(JavaClass clazz, int fieldIdx, int endIdx, RefChainElement referer) { + this.desc = RefChainElementImpl.getCompoundLinkedListElement(clazz, fieldIdx, referer); + this.endIdx = endIdx; + } + + CollapsedObj(int endIdx, RefChainElement desc) { + this.desc = desc; + this.endIdx = endIdx; + } + } + + // Debugging + + /** Debugging: print all the elements of the current reference chain as-is. */ + public void printFullRefChain() { + Logger logger = Logger.getLogger("org.openjdk.jmc.joverflow.stats"); //$NON-NLS-1$ + StringBuilder sb = new StringBuilder(); + sb.append(curRootRefChainElement.getRoot().getTypeName()); + sb.append(" "); + + int chainSizeMinusOne = refChain.size() - 1; + for (int i = 0; i < chainSizeMinusOne; i++) { + JavaHeapObject javaHeapObj = (JavaHeapObject) refChain.get(i++); + int idx = ((IndexContainer) refChain.get(i)).get(); + sb.append("-->" + getFullLinkDesc(javaHeapObj, idx)); + } + logger.fine(sb.toString()); + } + + /** + * Debugging: returns the current full reference chain, condensing elements that are + * implementation details of known collection classes. + */ + StringBuilder getPrintableCondensedRefChain() { + StringBuilder result = new StringBuilder(160); + result.append(curRootRefChainElement.getRoot().getTypeName()).append('@') + .append(curRootRefChainElement.getRoot().getId()); + result.append(" "); + + int chainSizeMinusOne = refChain.size() - 1; + int startIdx = chainSizeMinusOne - 2 * 8; + if (startIdx <= 0) { + startIdx = 0; + } else { + // Avoid starting from e.g. HashMap$Entry + while (((JavaHeapObject) refChain.get(startIdx)).getClazz().getName().indexOf('$') > 0) { + startIdx -= 2; + if (startIdx == 0) { + break; + } + } + if (startIdx > 0) { + result.append("-->..."); + } + } + + for (int i = startIdx; i < chainSizeMinusOne; i++) { + CollapsedObj clpsObj = collapseLinkedListImpl(i, null); + if (clpsObj == null) { + clpsObj = collapseCollectionImpl(i, null); + } + + if (clpsObj != null) { + result.append(clpsObj.desc.toString()); + i = clpsObj.endIdx; + } else { + JavaHeapObject javaHeapObj = (JavaHeapObject) refChain.get(i++); + int idx = ((IndexContainer) refChain.get(i)).get(); + result.append("-->"); + result.append(getLinkDesc(javaHeapObj, idx, null)); + } + } + + JavaHeapObject lastObj = (JavaHeapObject) refChain.get(chainSizeMinusOne); + String shortClassName = lastObj.getClazz().getHumanFriendlyName(); + result.append("->>").append(shortClassName); + return result; + } + +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/InterimRefChainTree.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/InterimRefChainTree.java new file mode 100644 index 00000000..84310d95 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/InterimRefChainTree.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.support.ProblemRecorder; +import org.openjdk.jmc.joverflow.support.RefChainElement; +import org.openjdk.jmc.joverflow.support.RefChainElementImpl; + +/** + */ +class InterimRefChainTree extends InterimRefChain { + + enum ParentType { + INSTANCE, COLLECTION, ARRAY, CLAZZ + } + + private JavaHeapObject curParent; + private ParentType curParentType; + private int curIndexInParent; + + InterimRefChainTree(ProblemRecorder problemRecorder) { + super(problemRecorder); + } + + @Override + protected void onCurrentRootReset() { + curParent = null; + } + + @Override + protected JavaObject getPointingJavaObject() { + if (!(curParent instanceof JavaObject)) { + return null; + } + + return (JavaObject) curParent; + } + + @Override + protected RefChainElement getLastRefChainElement() { + if (curParent == null) { + return curCondensedRefChainElement; + } + + JavaClass curParentClazz = curParent.getClazz(); + switch (curParentType) { + case INSTANCE: + if (curCondensedRefChainElement instanceof RefChainElementImpl.InstanceFieldOrLinkedList) { + RefChainElementImpl.InstanceFieldOrLinkedList crc = (RefChainElementImpl.InstanceFieldOrLinkedList) curCondensedRefChainElement; + if (crc.getFieldIdx() == curIndexInParent + && curParentClazz.isSameOrHierarchicallyRelated(crc.getJavaClass())) { + crc.switchToLinkedList(); + return curCondensedRefChainElement; + } + } + + return RefChainElementImpl.getInstanceFieldElement(curParentClazz, curIndexInParent, + curCondensedRefChainElement); + case COLLECTION: + return RefChainElementImpl.getCompoundCollectionElement(curParentClazz, curCondensedRefChainElement); + case ARRAY: + return RefChainElementImpl.getCompoundArrayElement(curParentClazz, curCondensedRefChainElement); + case CLAZZ: + return RefChainElementImpl.getStaticFieldElement((JavaClass) curParent, curIndexInParent, + curCondensedRefChainElement); + } + return null; + } + + void setCurParent(JavaHeapObject curParent, ParentType curParentType, RefChainElement referer) { + this.curParent = curParent; + this.curParentType = curParentType; + curIndexInParent = -1; + curCondensedRefChainElement = referer; + } + + void setCurIndexInParent(int index) { + curIndexInParent = index; + } + + void incCurIndexInParent() { + curIndexInParent++; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/LengthHistogram.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/LengthHistogram.java new file mode 100644 index 00000000..115ac47a --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/LengthHistogram.java @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +import org.openjdk.jmc.joverflow.util.IntToObjectMap; + +/** + * A histogram that groups objects (for example, strings or arrays) by their length, and for each + * length keeps the count and total size of the respective objects. + */ +public class LengthHistogram { + private final ArrayList values; + + private LengthHistogram(ArrayList values) { + this.values = values; + } + + /** + * Get a list of entries, sorted in ascending order by length, and "pruned" so that all entries + * with size < sizeThreshold are combined into a single entry with length == + * Entry.SPECIAL_VALUE. + */ + public List getPrunedAndSortedEntries(int sizeThreshold) { + ArrayList result = new ArrayList<>(values.size() / 2); + Entry entryForOthers = new Entry(Entry.SPECIAL_VALUE); + for (Entry entry : values) { + if (entry.getSize() >= sizeThreshold) { + result.add(entry); + } else { + entryForOthers.addEntry(entry); + } + } + result.add(entryForOthers); + + result.sort(SPECIAL_LENGTH_COMPARATOR); + return result; + } + + private static final Comparator SPECIAL_LENGTH_COMPARATOR = new Comparator() { + @Override + public int compare(Entry o1, Entry o2) { + if (o1.length == Entry.SPECIAL_VALUE) { + return 1; + } else if (o2.length == Entry.SPECIAL_VALUE) { + return -1; + } else { + return o1.length - o2.length; + } + } + }; + + public static class Builder { + private final IntToObjectMap lenToEntry; + + public Builder(int capacity) { + lenToEntry = new IntToObjectMap<>(capacity, false); + } + + /** Add an object with the given length and size */ + public void addInstance(int length, int size) { + Entry entry = lenToEntry.get(length); + if (entry == null) { + entry = new Entry(length); + lenToEntry.put(length, entry); + } + entry.addInstance(size); + } + + public LengthHistogram build() { + ArrayList result = new ArrayList<>(lenToEntry.size()); + result.addAll(lenToEntry.values()); + return new LengthHistogram(result); + } + + public int size() { + return lenToEntry.size(); + } + } + + public static class Entry { + public static final int SPECIAL_VALUE = -1; + + private final int length; + private int count; + private long size; + + private Entry(int length) { + this.length = length; + } + + private void addInstance(int instSize) { + count++; + size += instSize; + } + + private void addEntry(Entry other) { + count += other.count; + size += other.size; + } + + public int getLength() { + return length; + } + + public int getCount() { + return count; + } + + public long getSize() { + return size; + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/LongLivedStringClustersCalculator.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/LongLivedStringClustersCalculator.java new file mode 100644 index 00000000..a961c9a2 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/LongLivedStringClustersCalculator.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.logging.Logger; + +import org.openjdk.jmc.joverflow.batch.BatchProblemRecorder; +import org.openjdk.jmc.joverflow.batch.DetailedStats; +import org.openjdk.jmc.joverflow.batch.ReferencedObjCluster; +import org.openjdk.jmc.joverflow.heap.model.Snapshot; +import org.openjdk.jmc.joverflow.heap.parser.HprofParsingCancelledException; +import org.openjdk.jmc.joverflow.support.DupStringStats; +import org.openjdk.jmc.joverflow.support.ReferenceChain; + +/** + * Looks at several (currently only 3 is really supported) heap dumps, and determines which + * duplicated string clusters are most likely long-lived. So far this is based just on the fact that + * the same cluster shows up in two out of three dumps. + */ +public class LongLivedStringClustersCalculator { + private static final Logger LOGGER = Logger.getLogger("org.openjdk.jmc.joverflow.stats"); //$NON-NLS-1$ + + private final int numDumps; + private final List dupStringsToFields[]; + + private volatile DetailedDupStringStatsCalculator ssc; + private int curDumpNum; + + @SuppressWarnings("unchecked") + public LongLivedStringClustersCalculator(int numDumps) { + this.numDumps = numDumps; + dupStringsToFields = new ArrayList[numDumps]; + } + + public DupStringStats update(Snapshot snapshot) throws HprofParsingCancelledException { + long totalObjSize = snapshot.getRoughTotalObjectSize(); + int minOverhead = (int) (totalObjSize / 1000); + BatchProblemRecorder recorder = new BatchProblemRecorder(); + + ssc = new DetailedDupStringStatsCalculator(snapshot, recorder); + DupStringStats dss = ssc.calculate(); + + DetailedStats ds = recorder.getDetailedStats(minOverhead); + dupStringsToFields[curDumpNum] = ds.dupStringClusters.get(1); + + curDumpNum++; + ssc = null; // Help the GC + return dss; + } + + public void calculate() { + int minNumRepeats = numDumps / 2 + 1; + + LinkedHashSet longLivedFields = new LinkedHashSet<>(); + + for (int i = 0; i < numDumps - minNumRepeats + 1; i++) { + // Get each cluster from the current dump, and check if it's present in the + // next (at least) minNumRepeats - 1 dumps + List clusters1 = dupStringsToFields[i]; + + for (ReferencedObjCluster cluster1 : clusters1) { + String classAndField1 = ReferenceChain.toStringInStraightOrder(cluster1.getReferer()); + + for (int j = i + 1; j < numDumps; j++) { + List clusters2 = dupStringsToFields[j]; + + for (ReferencedObjCluster cluster2 : clusters2) { + // TODO: the fact that there is just a single check below implies that there are just 3 dumps in total + // FIXME: The code below is most likely wrong; it has been changed just to compile after we changed many things in ReferenceChain etc. + if (cluster2.getReferer().toString().equals(classAndField1)) { + + longLivedFields.add(classAndField1); + } + } + } + } + } + + LOGGER.info("\nLONG-LIVED FIELDS:"); + for (String longLivedField : longLivedFields) { + LOGGER.info(longLivedField); + } + } + + public int getProgressPercentage() { + DetailedDupStringStatsCalculator sscCopy = ssc; + if (sscCopy != null) { + return sscCopy.getProgressPercentage(); + } else { + return 100; + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/ObjectHistogram.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/ObjectHistogram.java new file mode 100644 index 00000000..c1201413 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/ObjectHistogram.java @@ -0,0 +1,483 @@ +/* + * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaField; +import org.openjdk.jmc.joverflow.heap.model.Snapshot; +import org.openjdk.jmc.joverflow.util.IntArrayList; + +/** + * Generates several kinds of object histograms: + *

+ * 1. The general histogram, where for each class we give the number of its instances, their total + * shallow size, and their total implementation-inclusive size. Implementation-inclusive size is + * greater than shallow size for: - known collections, where it's calculated as a sum of sizes of + * the collection instance and all objects that are used as the implementation of this collection. + * For example, for HashMap that would be HashMap$Entry[] array and total size of all Entries. + * Objects that are contained in the collection are not counted. - Strings, where we calculate the + * sum of the String instance size and the size of char[] array that it references, unless that + * array has already been claimed by some other String. + *

+ * 2. The histogram of classes with no or only one instance. + *

+ * 3. The histogram of classes with some fields null or zero in all or most of the instances of this + * class. + *

+ * 4. The histogram of classes where some multi-byte primitive fields (char, short, int or long) + * don't utilize the high byte(s) in all or most of the instances of this class. + *

+ * Note that currently an instance of this class generates the histograms lazily, thus it keeps a + * reference to the whole Snapshot instance. We may want to avoid that in the future. + */ +public class ObjectHistogram { + + private final Snapshot snapshot; + + ObjectHistogram(Snapshot snapshot) { + this.snapshot = snapshot; + } + + /** + * Returns the list of entries sorted by implementation-inclusive size of instances of the + * respective class. For classes that have same values for the above, sorting is done + * alphabetically. + */ + public List getListSortedByInclusiveSize(int minInclusiveSize) { + JavaClass[] classes = snapshot.getClasses(); + ArrayList result = new ArrayList<>(minInclusiveSize == 0 ? classes.length : classes.length / 100); + for (JavaClass clazz : classes) { + if (clazz.getTotalInclusiveInstanceSize() >= minInclusiveSize) { + result.add(new Entry(clazz)); + } + } + + result.sort(new Comparator() { + @Override + public int compare(Entry o1, Entry o2) { + long diff = (o2.getTotalInclusiveSize() - o1.getTotalInclusiveSize()); + if (diff > 0) { + return 1; + } else if (diff < 0) { + return -1; + } else { + return o1.getClazz().getName().compareTo(o2.getClazz().getName()); + } + } + }); + return result; + } + + /** + * Returns the number of classes with no instances and the number of classes with exactly one + * instance, as a two-element array. + */ + public int[] calculateNumSmallInstClasses() { + int numZeroInstObjects = 0; + int numOneInstObjects = 0; + JavaClass[] classes = snapshot.getClasses(); + for (JavaClass clazz : classes) { + int numInst = clazz.getNumInstances(); + switch (numInst) { + case 0: + numZeroInstObjects++; + break; + case 1: + numOneInstObjects++; + break; + } + } + return new int[] {numZeroInstObjects, numOneInstObjects}; + } + + /** + * Returns all classes with instances such that some data fields are null in either all the + * instances (if percentile == 1.0), or in the given percentile of instances. For example, if + * percentile == 0.9f, it would return classes with some fields that are null in 90% or more of + * their instances. + *

+ * If just some fields are null/zero in instances of the given class, their overhead is + * calculated as sizeof(all null fields) * num_problematic_instances. If all fields are + * null/zero (or class has no fields at all, like java.lang.Object), the overhead is calculated + * as sizeof(whole instance) * num_problematic_instances. + *

+ * Note that we treat multiple class versions (classes with the same name but different loaders) + * as distinct classes here. + */ + public List getListSortedByNullFieldsOvhd(float percentile) { + ArrayList result = findClassesWithNullFields(percentile); + checkForSuperclassesDefiningProblemFields(result); + + result.sort(PROBLEM_OVERHEAD_COMPARATOR); + return result; + } + + /** + * Returns all classes with instances that have some multi-byte primitive fields (char, short, + * int or long), and some of these fields don't use their high byte(s) in either all the + * instances (if percentile == 1.0), or in the given precentile of instances. For example, if + * percentile == 0.9f, it would return classes with some fields underutilized in 90% or more of + * their instances. + *

+ * Note that we treat multiple class versions (classes with the same name but different loaders) + * as different classes here. + */ + public List getListSortedByUnusedHiByteFieldsOvhd(float percentile) { + ArrayList result = findClassesWithUnusedHiByteFields(percentile); + checkForSuperclassesDefiningProblemFields(result); + + result.sort(PROBLEM_OVERHEAD_COMPARATOR); + return result; + } + + private static final Comparator PROBLEM_OVERHEAD_COMPARATOR = new Comparator() { + @Override + public int compare(ProblemFieldsEntry o1, ProblemFieldsEntry o2) { + long diff = (o2.allProblemFieldsOvhd - o1.allProblemFieldsOvhd); + if (diff > 0) { + return 1; + } else if (diff < 0) { + return -1; + } else { + return 0; + } + } + }; + + private ArrayList findClassesWithNullFields(float percentile) { + JavaClass[] classes = snapshot.getClasses(); + ArrayList result = new ArrayList<>(classes.length / 100); + + for (JavaClass clazz : classes) { + if (clazz.isString()) { + continue; // Strings are handled specially in DetailedStatsCalculator + } + if (clazz.getNumInstances() == 0) { + continue; + } + + DataFieldStats stats = (DataFieldStats) clazz.getAttachment(); + if (stats == null) { + continue; + } + + int numClazzInstances = clazz.getNumInstances(); + int maxNonNullFieldInstances = percentile == 1.0f ? 0 : (int) (numClazzInstances * (1 - percentile)); + + int[] emptyFields = stats.getPercentileEmptyFields(maxNonNullFieldInstances); + if (emptyFields == DataFieldStats.NO_REQUESTED_FIELDS) { + continue; + } + + ProblemFieldsEntry.Status status = stats.getNumFields() == 0 ? ProblemFieldsEntry.Status.NO_FIELDS + : stats.getNumFields() == emptyFields.length ? ProblemFieldsEntry.Status.ALL_FIELDS_EMPTY + : ProblemFieldsEntry.Status.SOME_FIELDS_EMPTY; + + String[] emptyFieldNames = new String[emptyFields.length]; + JavaClass[] emptyFieldDeclaringClasses = new JavaClass[emptyFields.length]; + long[] perFieldOvhd = new long[emptyFields.length]; + long allFieldsOvhd = 0; + + for (int i = 0; i < emptyFields.length; i++) { + int fieldIdx = emptyFields[i]; + JavaField field = clazz.getFieldForInstance(fieldIdx); + emptyFieldNames[i] = field.getName(); + emptyFieldDeclaringClasses[i] = clazz.getDeclaringClassForField(fieldIdx); + perFieldOvhd[i] = ((long) field.getSizeInInstance()) + * (numClazzInstances - stats.getNumInstancesWithFieldNotNull(fieldIdx)); + allFieldsOvhd += perFieldOvhd[i]; + } + + // If all fields are null in all instances (or instances have no fields, + // like java.lang.Object), we define the overhead as the whole instance size, + // including object header etc. + if ((percentile == 1.0 && status == ProblemFieldsEntry.Status.ALL_FIELDS_EMPTY) + || stats.getNumFields() == 0) { + allFieldsOvhd = ((long) clazz.getInstanceSize()) * numClazzInstances; + } + + ProblemFieldsEntry entry = new ProblemFieldsEntry(clazz, emptyFieldNames, emptyFieldDeclaringClasses, + perFieldOvhd, allFieldsOvhd, status); + result.add(entry); + } + return result; + } + + private ArrayList findClassesWithUnusedHiByteFields(float percentile) { + JavaClass[] classes = snapshot.getClasses(); + ArrayList result = new ArrayList<>(classes.length / 100); + + for (JavaClass clazz : classes) { + if (clazz.isString()) { + continue; // Strings are handled specially in DetailedStatsCalculator + } + if (clazz.getNumInstances() == 0) { + continue; + } + + DataFieldStats stats = (DataFieldStats) clazz.getAttachment(); + if (stats == null) { + continue; + } + + int numClazzInstances = clazz.getNumInstances(); + int minBadInstances = percentile == 1.0f ? numClazzInstances : (int) (numClazzInstances * percentile); + + DataFieldStats.UnderutilizedFields problemFields = stats.getUnusedHiBytesFields(minBadInstances); + if (problemFields == null) { + continue; + } + + int nProblemFields = problemFields.fieldIndices.length; + String[] problemFieldNames = new String[nProblemFields]; + JavaClass[] emptyFieldDeclaringClasses = new JavaClass[nProblemFields]; + long[] perFieldOvhd = new long[nProblemFields]; + long allFieldsOvhd = 0; + + for (int i = 0; i < nProblemFields; i++) { + int fieldIdx = problemFields.fieldIndices[i]; + JavaField field = clazz.getFieldForInstance(fieldIdx); + problemFieldNames[i] = field.getName(); + emptyFieldDeclaringClasses[i] = clazz.getDeclaringClassForField(fieldIdx); + perFieldOvhd[i] = problemFields.unusedBytesOvhd[i]; + allFieldsOvhd += perFieldOvhd[i]; + } + + ProblemFieldsEntry entry = new ProblemFieldsEntry(clazz, problemFieldNames, emptyFieldDeclaringClasses, + perFieldOvhd, allFieldsOvhd, ProblemFieldsEntry.Status.SOME_FIELDS_UNUSED_HI_BYTES); + result.add(entry); + } + return result; + } + + /** + * Checks entries for classes such that X.foo and Y.foo are always-null fields, and foo is + * defined in class Z that's superclass for X and Y, and there are no other subclasses of Z (or, + * to put it in a different way, all subclasses of Z have field foo always null). + */ + private void checkForSuperclassesDefiningProblemFields(ArrayList entries) { + HashMap classToEntry = new HashMap<>(); + for (ProblemFieldsEntry entry : entries) { + classToEntry.put(entry.getClazz().getName(), entry); + } + + HashMap classToFields = new HashMap<>(); +// HashSet superclasses = new HashSet<>(); + for (ProblemFieldsEntry entry : entries) { + String entryClassName = entry.getClazz().getName(); + JavaClass[] declaringClasses = entry.problemFieldDeclaringClasses; + for (int i = 0; i < declaringClasses.length; i++) { + if (entryClassName.equals(declaringClasses[i].getName())) { + continue; + } + + // For the i-th field, declaring class is not the same as entryClass + String fieldName = entry.problemFieldNames[i]; + JavaClass superClazz = entry.problemFieldDeclaringClasses[i]; + String superName = superClazz.getName(); + + // Check whether this field is null in the superclass (or superclass has no instances) + if (superClazz.getNumInstances() > 0) { + ProblemFieldsEntry superclassEntry = classToEntry.get(superName); + if (superclassEntry != null) { + if (superclassEntry.containsField(fieldName, superClazz) == -1) { + continue; + } + } + } + + // Check whether all subclazzes have the given field always null + ArrayList subclazzes = superClazz.getSubclasses(); + int numSubclazzesWithNullField = 0; + for (JavaClass subclazz : subclazzes) { + String subclazzName = subclazz.getName(); + ProblemFieldsEntry entry1 = classToEntry.get(subclazzName); + if (entry1 != null) { + if (entry1.containsField(fieldName, superClazz) != -1) { + numSubclazzesWithNullField++; + break; + } + } + } + if (numSubclazzesWithNullField != subclazzes.size()) { + continue; + } + + // Record the information that this field should go to the superclass + IntArrayList fields = classToFields.get(entryClassName); + if (fields == null) { + fields = new IntArrayList(8); + classToFields.put(entryClassName, fields); + } + fields.add(i); +// superclasses.add(superName); + } + } + } + + /** + * This entry is created for every class, and provides aggregated basic characteristics of all + * its instances. + */ + public static class Entry { + private final JavaClass clazz; + + private Entry(JavaClass clazz) { + this.clazz = clazz; + } + + public JavaClass getClazz() { + return clazz; + } + + public int getNumInstances() { + return clazz.getNumInstances(); + } + + public long getTotalInclusiveSize() { + return clazz.getTotalInclusiveInstanceSize(); + } + + public long getTotalShallowSize() { + return clazz.getTotalShallowInstanceSize(); + } + } + + /** + * Describes a class where some or all fields are problematic (null/zero, have high bytes + * unused, ...) in all or most instances. Also, describes a class with no fields at all: if + * there are many instances of such a class, it's probably a sign of something rather + * sub-optimal. + */ + public static class ProblemFieldsEntry { + + public enum Status { + SOME_FIELDS_EMPTY, ALL_FIELDS_EMPTY, NO_FIELDS, SOME_FIELDS_UNUSED_HI_BYTES + } + + private final JavaClass clazz; + private final int numInstances; + private final String[] problemFieldNames; + private final JavaClass[] problemFieldDeclaringClasses; + private final long[] perFieldOvhd; + private final long allProblemFieldsOvhd; + private final Status status; + + private ProblemFieldsEntry(JavaClass clazz, String[] problemFieldNames, + JavaClass[] problemFieldDeclaringClasses, long[] perFieldOvhd, long totalOverhead, Status status) { + this.clazz = clazz; + this.numInstances = clazz.getNumInstances(); + this.problemFieldNames = problemFieldNames; + this.problemFieldDeclaringClasses = problemFieldDeclaringClasses; + this.perFieldOvhd = perFieldOvhd; + this.allProblemFieldsOvhd = totalOverhead; + this.status = status; + } + + public JavaClass getClazz() { + return clazz; + } + + public int getNumInstances() { + return numInstances; + } + + public long getAllProblemFieldsOvhd() { + return allProblemFieldsOvhd; + } + + public String[] getProblemFieldNames() { + return problemFieldNames; + } + + public JavaClass[] getProblemFieldDeclaringClasses() { + return problemFieldDeclaringClasses; + } + + public long[] getPerFieldOvhd() { + return perFieldOvhd; + } + + public Status getStatus() { + return status; + } + + public String getFieldsAsString() { + if (status == Status.ALL_FIELDS_EMPTY) { + return "all"; + } else if (status == Status.NO_FIELDS) { + return "fieldless class"; + } + + StringBuilder result = new StringBuilder(problemFieldNames.length * 16); + JavaClass declaringClass = null, prevDeclaringClass = null; + for (int i = 0; i < problemFieldNames.length; i++) { + declaringClass = problemFieldDeclaringClasses[i]; + if (prevDeclaringClass == null) { + prevDeclaringClass = declaringClass; + } + if (prevDeclaringClass != clazz && prevDeclaringClass != declaringClass) { + result.append(" (defined in ").append(prevDeclaringClass.getHumanFriendlyName()).append(")"); + } + if (i > 0) { + if (declaringClass == prevDeclaringClass) { + result.append(", "); + } else { + result.append("; "); + } + } + prevDeclaringClass = declaringClass; + result.append(problemFieldNames[i]); + } + if (declaringClass != clazz && declaringClass != null) { + result.append(" (defined in ").append(declaringClass.getHumanFriendlyName()).append(")"); + } + + return result.toString(); + } + + public int containsField(String fieldName, JavaClass declaringClazz) { + for (int j = 0; j < problemFieldNames.length; j++) { + if (problemFieldNames[j].equals(fieldName) && problemFieldDeclaringClasses[j] == declaringClazz) { + return j; + } + } + return -1; + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/OverallStatsCalculator.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/OverallStatsCalculator.java new file mode 100644 index 00000000..3ea5c31b --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/OverallStatsCalculator.java @@ -0,0 +1,267 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import java.util.Collection; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaLazyReadObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObjectArray; +import org.openjdk.jmc.joverflow.heap.model.JavaValueArray; +import org.openjdk.jmc.joverflow.heap.model.Snapshot; +import org.openjdk.jmc.joverflow.heap.parser.CachedReadBuffer; +import org.openjdk.jmc.joverflow.heap.parser.HprofParsingCancelledException; +import org.openjdk.jmc.joverflow.heap.parser.ReadBuffer; +import org.openjdk.jmc.joverflow.support.Constants; +import org.openjdk.jmc.joverflow.support.DupArrayStats; +import org.openjdk.jmc.joverflow.support.HeapStats; +import org.openjdk.jmc.joverflow.support.ShortArrayStats; +import org.openjdk.jmc.joverflow.util.ObjectToIntMap; + +/** + * Functionality for calculating overall, generally high-level stats about objects in the heap. + */ +class OverallStatsCalculator implements Constants { + private final Snapshot snapshot; + private final int ptrSize; + + private int nObjs, nObjs2ndPass; + private volatile boolean cancelled; + + public OverallStatsCalculator(Snapshot snapshot) { + this.snapshot = snapshot; + ptrSize = snapshot.getPointerSize(); + } + + /** + * Calculates and returns the overall object stats. + * + * @throws org.openjdk.jmc.joverflow.heap.parser.DumpCorruptedException.Runtime + * if reading some object from heap dump uncovers that the file is corrupted + */ + public HeapStats calculate() throws HprofParsingCancelledException { + int objHeaderSize = snapshot.getObjectHeaderSize(); + long arrHeaderSize = snapshot.getArrayHeaderSize(); // Made long to get long result when multiplying by int + + StringStatsCollector stringStatsCollector = new StringStatsCollector(snapshot); + PrimitiveArrayDuplicationMap arrayDupMap = new PrimitiveArrayDuplicationMap(snapshot); + + Collection allObjects = snapshot.getObjects(); + + nObjs = 0; + int nInstances = 0, nObjectArrays = 0; + long totalObjectSize = 0, totalInstSize = 0, totalObjArraySize = 0; + int nEntryInstances = 0; + long entryClassSize = 0; + int n0LenObjArrays = 0, n1ObjArrays = 0, n4ObjArrays = 0, n8ObjArrays = 0; + int n0LenValArrays = 0, n1LenValArrays = 0, n4LenValArrays = 0, n8LenValArrays = 0; + int lenZeroObjArraySize = 0, lenOneObjArraySize = 0; + int nBoxedNumbers = 0; + long ovhdBoxedNumbers = 0; + + int curChunk = 0; + + ObjectToIntMap unmodifiableClassInstanceCount = new ObjectToIntMap<>(15); + ObjectToIntMap synchronizedClassInstanceCount = new ObjectToIntMap<>(15); + + for (JavaLazyReadObject obj : allObjects) { + nObjs++; + + JavaClass clazz = obj.getClazz(); + clazz.incNumInstances(); + String clazzName = clazz.getName(); + int objSize = obj.getSize(); + totalObjectSize += objSize; + + if (obj instanceof JavaObject) { + nInstances++; + totalInstSize += objSize; + + int primitiveNumSize; + + if (clazzName.endsWith("$Entry")) { + nEntryInstances++; + entryClassSize += objSize; + } else if (clazzName.startsWith("java.util.Collections$Unmodifiable")) { + unmodifiableClassInstanceCount.putOneOrIncrement(clazzName); + } else if (clazzName.startsWith("java.util.Collections$Synchronized")) { + synchronizedClassInstanceCount.putOneOrIncrement(clazzName); + } else if (clazz.isString()) { + stringStatsCollector.add((JavaObject) obj); + } else if ((primitiveNumSize = clazz.getBoxedNumberSize()) != 0) { + nBoxedNumbers++; + ovhdBoxedNumbers += objSize - primitiveNumSize + ptrSize; + } + + } else if (obj instanceof JavaObjectArray) { + nObjectArrays++; + totalObjArraySize += objSize; + clazz.updateShallowInstanceSize(objSize); + JavaObjectArray objArray = (JavaObjectArray) obj; + int length = objArray.getLength(); + + if (length == 0) { + n0LenObjArrays++; + if (lenZeroObjArraySize == 0) { + lenZeroObjArraySize = objArray.getSize(); + } + } else if (length == 1) { + n1ObjArrays++; + if (lenOneObjArraySize == 0) { + lenOneObjArraySize = objArray.getSize(); + } + } else if (length <= 4) { + n4ObjArrays++; + } else if (length <= 8) { + n8ObjArrays++; + } + } else if (obj instanceof JavaValueArray) { + clazz.updateShallowInstanceSize(objSize); + JavaValueArray valArray = (JavaValueArray) obj; + int length = valArray.getLength(); + if (length == 0) { +// System.out.println("Zero-length val array: " + obj + " , size = " + objSize); + n0LenValArrays++; + } else if (length == 1) { + n1LenValArrays++; + } else if (length <= 4) { + n4LenValArrays++; + } else if (length <= 8) { + n8LenValArrays++; + } + + // Performance optimization: scan as many primitive arrays as possible on + // the first pass, although there is a separate 2nd pass for them. However, + // reading more objects from disk now improves cache locality. + if (!(clazz.isCharArray() || clazz.isByteArray())) { + // This array, because of its type, is guaranteed to not belong to a String + arrayDupMap.add(valArray); + } + } + +// System.out.println(obj + " , size = " + objSize); + + int newCurChunk = nObjs >> 17; // Check every 128K objects + if (newCurChunk > curChunk) { + curChunk = newCurChunk; + if (cancelled) { + throw new HprofParsingCancelledException(); + } + } + } + + long ovhdObjectHeaders = nObjs * objHeaderSize; + + ObjectToIntMap.Entry[] unmodifiableClasses = unmodifiableClassInstanceCount + .getEntriesSortedByValueThenKey(); + ObjectToIntMap.Entry[] synchronizedClasses = synchronizedClassInstanceCount + .getEntriesSortedByValueThenKey(); + + // Do one more pass, this time to uncover duplicated primitive arrays. + // We could not do it on the previous pass, because there we generally + // unable to distinguish standalone char[] arrays from those that are + // backing Strings. + curChunk = 0; + + for (JavaLazyReadObject obj : allObjects) { + nObjs2ndPass++; // This is pure progress tracking + if (!(obj instanceof JavaValueArray)) { + continue; + } + JavaClass clazz = obj.getClazz(); + // Ignore if not a char[] or byte[] array - should have been scanned on previous pass + if (!(clazz.isCharArray() || clazz.isByteArray())) { + continue; + } + // Ignore if it's a char[] array for some String + if (obj.isVisitedAsCollectionImpl()) { + continue; + } + + JavaValueArray valArray = (JavaValueArray) obj; + arrayDupMap.add(valArray); + + int newCurChunk = nObjs2ndPass >> 17; // Check every 128K objects + if (newCurChunk > curChunk) { + curChunk = newCurChunk; + if (cancelled) { + throw new HprofParsingCancelledException(); + } + } + } + + arrayDupMap.calculateFinalStats(); + DupArrayStats dupArrayStats = new DupArrayStats(arrayDupMap.getNumArrays(), arrayDupMap.getNumUniqueArrays(), + arrayDupMap.getNumDifferentDupArrayValues(), arrayDupMap.getDupArrays(), + arrayDupMap.getDupArraysOverhead()); + + // IMPORTANT: should do this for optimizations in CachedReadBuffer to work! + ReadBuffer readBuf = snapshot.getReadBuffer(); + if (readBuf instanceof CachedReadBuffer) { + ((CachedReadBuffer) readBuf).incrementPass(); + } + + ClassloaderStats clStats = new ClassloaderStats(snapshot); + + return new HeapStats() + .setGeneralStats(ptrSize, objHeaderSize, snapshot.getObjectAlignment(), snapshot.usingNarrowPointers(), + snapshot.getNumClasses(), nObjs, nInstances, nObjectArrays, totalObjectSize, totalInstSize, + totalObjArraySize) + .setObjOverheadStats(ovhdObjectHeaders, nEntryInstances, entryClassSize).setClassloaderStats(clStats) + .setShortObjArrayStats(new ShortArrayStats(n0LenObjArrays, lenZeroObjArraySize * n0LenObjArrays, + n1ObjArrays, lenOneObjArraySize * n1ObjArrays, n4ObjArrays, arrHeaderSize * n4ObjArrays, + n8ObjArrays, arrHeaderSize * n8ObjArrays)) + // TODO: need a better way to calculate overhead for short primitive arrays, at least of size 0 and 1 + // Currently it's likely inconsistent with what is reported by detailed analysis + .setShortPrimitiveArrayStats(new ShortArrayStats(n0LenValArrays, arrHeaderSize * n0LenValArrays, + n1LenValArrays, arrHeaderSize * n1LenValArrays, n4LenValArrays, arrHeaderSize * n4LenValArrays, + n8LenValArrays, arrHeaderSize * n8LenValArrays)) + .setShortStringStats(stringStatsCollector.getShortStringStats()) + .setBoxedNumberStats(nBoxedNumbers, ovhdBoxedNumbers) + .setWrappedCollectionStats(unmodifiableClasses, synchronizedClasses) + .setDupStringStats(stringStatsCollector.getDuplicationStats()) + .setCompressibleStringStats(stringStatsCollector.getCompressibleStringStats()) + .setNumberEncodingStringStats(stringStatsCollector.getNumberEncodingStringStats()) + .setStringLengthHistogram(stringStatsCollector.getLengthHistogram()).setDupArrayStats(dupArrayStats); + } + + /** Used for progress reporting */ + public synchronized int getProgressPercentage() { + return (int) (((long) ((nObjs * 3 + nObjs2ndPass) / 4) * 100 / snapshot.getNumObjects())); + } + + public void cancelCalculation() { + cancelled = true; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/PrimitiveArrayDuplicationMap.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/PrimitiveArrayDuplicationMap.java new file mode 100644 index 00000000..dbca1883 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/PrimitiveArrayDuplicationMap.java @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; + +import org.openjdk.jmc.joverflow.heap.model.JavaValueArray; +import org.openjdk.jmc.joverflow.heap.model.Snapshot; +import org.openjdk.jmc.joverflow.support.DupArrayStats; +import org.openjdk.jmc.joverflow.util.ValueWitIntIdMap; +import org.openjdk.jmc.joverflow.util.ValueWithIntId; + +/** + * Functionality for analyzing standalone primitive arrays for duplication. + *

+ * Two primitive arrays are duplicate if they have the same type, length and contents. A (unique) + * value of the array is its contents. + *

+ * Works similar to StringDuplicationMap; however to compare arrays we first use a simple hash code, + * and in case of a match, call Arrays.compare(). + */ +class PrimitiveArrayDuplicationMap { + private final ValueWitIntIdMap table; + + private int nTotalArrays, nUniqueArrays; + private int currentId = 1; + + private int numDifferentDupArrayValues; + private ArrayList dupArrays; + private long dupArraysOvhd; + + PrimitiveArrayDuplicationMap(Snapshot snapshot) { + int capacity = (snapshot.getNumObjects() / 10); + table = new ValueWitIntIdMap<>(capacity); + } + + void add(JavaValueArray array) { + nTotalArrays++; + + byte[] bytes = array.getValue(); + int checksum = checksum(bytes); + + InternalEntry entry = table.get(checksum); + if (entry == null) { // No possible entry for this array + entry = new InternalEntry(checksum, array, currentId++); + table.put(entry); + } else { // There is an entry, but it may or may not match + InternalEntry prevEntry = null; + while (entry != null) { + if (entry.firstArray.getClazz() == array.getClazz() && entry.firstArray.getLength() == array.getLength() + && Arrays.equals(entry.firstArray.getValue(), bytes)) { + break; // Found matching entry + } else { + prevEntry = entry; + entry = entry.next; + } + } + if (entry == null) { // No matching entry found + entry = new InternalEntry(checksum, array, currentId++); + prevEntry.next = entry; + } + } + + entry.nArrayInst++; + array.setInternalId(entry.uniqueId); + } + + /** + * Must be called after all arrays are processed via {@link #add(JavaValueArray)} call, to + * calculate the stats returned by methods below. + */ + void calculateFinalStats() { + ArrayList result = new ArrayList<>(table.size() / 20); + for (InternalEntry entry : table.values()) { + while (entry != null) { + nUniqueArrays++; + if (entry.nArrayInst > 1) { + int overhead = entry.firstArray.getSize() * (entry.nArrayInst - 1); + dupArraysOvhd += overhead; + + result.add(new DupArrayStats.Entry(entry.firstArray, entry.uniqueId, entry.nArrayInst, overhead)); + } + + entry = entry.next; + } + } + + result.sort(new Comparator() { + @Override + public int compare(DupArrayStats.Entry e1, DupArrayStats.Entry e2) { + if (e1.overhead > e2.overhead) { + return -1; + } else if (e1.overhead < e2.overhead) { + return 1; + } else { + return 0; + } + } + }); + + dupArrays = result; + numDifferentDupArrayValues = result.size(); + } + + int getNumArrays() { + return nTotalArrays; + } + + int getNumUniqueArrays() { + return nUniqueArrays; + } + + int getNumDifferentDupArrayValues() { + return numDifferentDupArrayValues; + } + + long getDupArraysOverhead() { + return dupArraysOvhd; + } + + ArrayList getDupArrays() { + return dupArrays; + } + + private static int checksum(byte[] bytes) { + if (bytes.length == 0) { + return 0; + } + + int h = 0; + for (byte b : bytes) { + h = 31 * h + b; + } + return h; + } + + /** + * Represents a unique array value (by value we mean contents). We use entries of this kind + * internally, until the final results are calculated. + */ + private static class InternalEntry implements ValueWithIntId { + // Checksum of this array's contents + final int checksum; + // The first array with this value + final JavaValueArray firstArray; + // A unique id for this array + final int uniqueId; + // Number of instances of identical arrays + int nArrayInst; + // checksum collision list + InternalEntry next; + + InternalEntry(int checksum, JavaValueArray firstArray, int uniqueId) { + this.checksum = checksum; + this.firstArray = firstArray; + this.uniqueId = uniqueId; + } + + @Override + public int getId() { + return checksum; + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/PrimitiveArrayHandler.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/PrimitiveArrayHandler.java new file mode 100644 index 00000000..ffc145fc --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/PrimitiveArrayHandler.java @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +/** + * Detects all the problems that can occur with primitive arrays: length 0, length 1, empty + * (contains only zeros), long zero-tail (more than half consecutive elements, from the back, equal + * to zero), and unused high bytes (high bytes in each element are unused, e.g. all elements in an + * int[] array would fit into shorts). + */ +public class PrimitiveArrayHandler { + private final byte[] data; + private final int elSize; + private final boolean isFloatOrDouble; + private boolean length0, length1, empty; + private int lztOvhd = -1, unusedHighBytesOvhd = -1; + + static PrimitiveArrayHandler createInstance(byte[] data, int elSize, boolean isFloatOrDouble) { + return new PrimitiveArrayHandler(data, elSize, isFloatOrDouble); + } + + /** Returns true if the underlying array has length 0 */ + public boolean isLength0() { + return length0; + } + + /** Returns true if the underlying array has length 1 */ + public boolean isLength1() { + return length1; + } + + /** Returns true if the underlying array is empty, i.e. all its elements are 0 */ + public boolean isEmpty() { + return empty; + } + + /** + * Returns a positive overhead value if the underlying array is "long zero-tail", i.e. half of + * more of its elements from the back are all zeros. + */ + public int getLztOverhead() { + return lztOvhd; + } + + /** + * Returns a positive overhead value if in the underlying array (that should not be a byte[], + * float[] or double[]), all elements don't utilize the upper byte (for short[] and char[]), the + * upper two or three bytes (for int[]), or the upper four, six or seven bytes (for long[]). + */ + public int getUnusedHighBytesOvhd() { + return unusedHighBytesOvhd; + } + + private PrimitiveArrayHandler(byte[] data, int elSize, boolean isFloatOrDouble) { + this.data = data; + this.elSize = elSize; + this.isFloatOrDouble = isFloatOrDouble; + checkForProblems(); + } + + private void checkForProblems() { + int numElements = data.length / elSize; + + if (data.length == 0) { + length0 = true; + return; // Cannot have any other problems + } + + if (numElements == 1) { + length1 = true; + } + + switch (elSize) { + case 1: + checkForElSize1(); + break; + case 2: + checkForElSize2(); + break; + case 4: + checkForElSize4(); + break; + case 8: + checkForElSize8(); + break; + default: + throw new IllegalArgumentException("Unsupported elSize: " + elSize); + } + } + + private void checkForElSize1() { + int i; + for (i = data.length - 1; i >= 0; i--) { + if (data[i] != 0) { + break; + } + } + if (i < 0) { + empty = true; + } else if (i < data.length / 2) { + lztOvhd = data.length - i + 1; + } + } + + private void checkForElSize2() { + int zeroElementMinIndex = data.length; + boolean highByteUnusedForAll = true, zeroTailSoFar = true; + for (int i = data.length - 2; i >= 0; i -= 2) { + int b1 = (data[i] & 0xff); + int b2 = (data[i + 1] & 0xff); + short value = (short) ((b1 << 8) + b2); + + if (value == 0 && zeroTailSoFar) { + zeroElementMinIndex = i; + } else { + zeroTailSoFar = false; + highByteUnusedForAll = (value >= Byte.MIN_VALUE && value <= Byte.MAX_VALUE); + // Stop as early as possible + if (!highByteUnusedForAll) { + break; + } + } + } + + if (zeroElementMinIndex == 0) { + empty = true; + } else { + if (zeroElementMinIndex < data.length / 2) { + lztOvhd = data.length - zeroElementMinIndex; + } + if (highByteUnusedForAll) { + unusedHighBytesOvhd = data.length / 2; + } + } + } + + private void checkForElSize4() { + int zeroElementMinIndex = data.length; + // For float[] and double[] arrays, checking for unused high bytes doesn't work + boolean threeBytesUnusedForAll = !isFloatOrDouble, twoBytesUnusedForAll = !isFloatOrDouble; + boolean zeroTailSoFar = true; + for (int i = data.length - 4; i >= 0; i -= 4) { + int b1 = (data[i] & 0xFF); + int b2 = (data[i + 1] & 0xFF); + int b3 = (data[i + 2] & 0xFF); + int b4 = (data[i + 3] & 0xFF); + int value = (b1 << 24) + (b2 << 16) + (b3 << 8) + b4; + + if (value == 0 && zeroTailSoFar) { + zeroElementMinIndex = i; + } else { + zeroTailSoFar = false; + threeBytesUnusedForAll &= (value >= Byte.MIN_VALUE && value <= Byte.MAX_VALUE); + twoBytesUnusedForAll &= (value >= Short.MIN_VALUE && value <= Short.MAX_VALUE); + // Stop as early as possible + if (!threeBytesUnusedForAll && !twoBytesUnusedForAll) { + break; + } + } + } + + if (zeroElementMinIndex == 0) { + empty = true; + } else { + if (zeroElementMinIndex < data.length / 2) { + lztOvhd = data.length - zeroElementMinIndex; + } + if (threeBytesUnusedForAll) { + unusedHighBytesOvhd = data.length / 4 * 3; + } else if (twoBytesUnusedForAll) { + unusedHighBytesOvhd = data.length / 2; + } + } + } + + private void checkForElSize8() { + int zeroElementMinIndex = data.length; + // For float[] and double[] arrays checking for unused high bytes doesn't work + boolean sevenBytesUnusedForAll = !isFloatOrDouble, sixBytesUnusedForAll = !isFloatOrDouble; + boolean fourBytesUnusedForAll = !isFloatOrDouble, zeroTailSoFar = true; + for (int i = data.length - 8; i >= 0; i -= 8) { + long value = 0; + for (int j = i; j < i + 8; j++) { + value <<= 8; + int b = (data[j]) & 0xFF; + value |= b; + } + + if (value == 0 && zeroTailSoFar) { + zeroElementMinIndex = i; + } else { + zeroTailSoFar = false; + sevenBytesUnusedForAll &= (value >= Byte.MIN_VALUE && value <= Byte.MAX_VALUE); + sixBytesUnusedForAll &= (value >= Short.MIN_VALUE && value <= Short.MAX_VALUE); + fourBytesUnusedForAll &= (value >= Integer.MIN_VALUE && value <= Integer.MAX_VALUE); + // Stop as early as possible + if (!sevenBytesUnusedForAll && !sixBytesUnusedForAll && !fourBytesUnusedForAll) { + break; + } + } + } + + if (zeroElementMinIndex == 0) { + empty = true; + } else { + if (zeroElementMinIndex < data.length / 2) { + lztOvhd = data.length - zeroElementMinIndex; + } + if (sevenBytesUnusedForAll) { + unusedHighBytesOvhd = data.length / 8 * 7; + } else if (sixBytesUnusedForAll) { + unusedHighBytesOvhd = data.length / 8 * 6; + } else if (fourBytesUnusedForAll) { + unusedHighBytesOvhd = data.length / 2; + } + } + } + +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/ProblemChecker.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/ProblemChecker.java new file mode 100644 index 00000000..1a3df70b --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/ProblemChecker.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import org.openjdk.jmc.joverflow.descriptors.CollectionInstanceDescriptor; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObjectArray; +import org.openjdk.jmc.joverflow.heap.model.JavaThing; +import org.openjdk.jmc.joverflow.heap.model.JavaValueArray; + +/** + * This interface contains methods that should be implemented by a class that checks instances and + * arrays for problems as they are given to it by {@link HeapScaner} implementation. + */ +interface ProblemChecker { + + /** + * Called before individual fields of the given instance are scanned. If the given object is a + * known collection, returns an instance of CollectionInstanceDescriptor for it. Otherwise, + * returns null + */ + CollectionInstanceDescriptor handleInstance(JavaObject obj, JavaThing[] fields); + + /** Called before individual object elements are scanned */ + void handleObjectArray(JavaObjectArray array, JavaHeapObject[] elements); + + void handleValueArray(JavaValueArray array); + + void handleString(JavaObject string); + +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/StandardStatsCalculator.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/StandardStatsCalculator.java new file mode 100644 index 00000000..073ed3e8 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/StandardStatsCalculator.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import org.openjdk.jmc.joverflow.heap.model.Snapshot; +import org.openjdk.jmc.joverflow.heap.parser.DumpCorruptedException; +import org.openjdk.jmc.joverflow.heap.parser.HprofParsingCancelledException; +import org.openjdk.jmc.joverflow.support.HeapStats; +import org.openjdk.jmc.joverflow.support.ProblemRecorder; + +/** + * Generates the standard heap dump statistics, by calling OverallStatsCalculator and then + * DetailedStatsCalculator. Provides an additional method to query progress percentage. + */ +public class StandardStatsCalculator { + + private final Snapshot snapshot; + private final ProblemRecorder problemRecorder; + private final boolean useBreadthFirst; + + private OverallStatsCalculator osc; + private DetailedStatsCalculator dsc; + private volatile int stage; + + public StandardStatsCalculator(Snapshot snapshot, ProblemRecorder problemRecorder, boolean useBreadthFirst) { + this.snapshot = snapshot; + this.problemRecorder = problemRecorder; + this.useBreadthFirst = useBreadthFirst; + } + + public HeapStats calculate() throws DumpCorruptedException, HprofParsingCancelledException { + snapshot.setCalculatingStats(true); + try { + osc = new OverallStatsCalculator(snapshot); + + stage = 1; + HeapStats hs = osc.calculate(); + osc = null; // Help the GC + + problemRecorder.initialize(snapshot, hs); + + dsc = new DetailedStatsCalculator(snapshot, hs, problemRecorder, useBreadthFirst); + stage = 2; + dsc.calculate(); + + return hs; + } catch (DumpCorruptedException.Runtime ex) { + throw ex.getCause(); + } finally { + snapshot.setCalculatingStats(false); + } + } + + public synchronized int getProgressPercentage() { + // Below we assume that calculating detailed stats takes approximately + // twice as much time as calculating overall stats - hence the number 33. + switch (stage) { + case 0: + return 0; + case 1: { + OverallStatsCalculator localOsc = osc; + if (localOsc == null) { + // Finished overall stats, but haven't started detailed yet + return 33; + } else { + return localOsc.getProgressPercentage() / 3; + } + } + case 2: + return 33 + dsc.getProgressPercentage() * 2 / 3; + } + + return 0; + } + + public synchronized void cancelCalculation() { + OverallStatsCalculator localOsc = osc; + if (localOsc != null) { + osc.cancelCalculation(); + } + + DetailedStatsCalculator localDsc = dsc; + if (localDsc != null) { + localDsc.cancelCalculation(); + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/StringStatsCollector.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/StringStatsCollector.java new file mode 100644 index 00000000..577f9b45 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/StringStatsCollector.java @@ -0,0 +1,333 @@ +/* + * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; + +import org.openjdk.jmc.joverflow.heap.model.HeapStringReader; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaValueArray; +import org.openjdk.jmc.joverflow.heap.model.Snapshot; +import org.openjdk.jmc.joverflow.support.CompressibleStringStats; +import org.openjdk.jmc.joverflow.support.DupStringStats; +import org.openjdk.jmc.joverflow.support.NumberEncodingStringStats; +import org.openjdk.jmc.joverflow.support.ShortArrayStats; + +/** + * Functionality for analyzing Strings for duplication. + *

+ * For JDK versions prior to JDK7u?, there are two ways in which two (or more) java.lang.String + * objects can be duplicate: instances with same value can either point to the same backing char + * array, or point to different char arrays. In the first case the overhead is the shallow size of + * the second and subsequent String objects with the same value. In the second case it's as above + * plus the size of the second and subsequent char arrays. + *

+ * This class maintains a hash map that maps a unique string value to the record containing the + * number of separate String objects with this value, number of backing char arrays, etc. When all + * the strings are processed by add() calls, getDuplicationStats() calculates and returns the final + * results. + */ +class StringStatsCollector { + + private final HeapStringReader stringReader; + + private int stringInstShallowSize; // Shallow size of a String instance + + // Duplicate strings management + private int nTotalStrings, numBackingCharArrays; + private final HashMap table; + private int currentId = 1; + + // Short strings management + private int nZeroLengthStrings, n1Strings, n4Strings, n8Strings; + + // Potentially compressible string management + private int nCompressedStrings, nAsciiCharBackedStrings; + private long usedBackingArrayBytes, compressedBackingArrayBytes, asciiCharBackingArrayBytes; + + // Management of strings that encode numbers + private int nStringsEncodingInts; + private long stringsEncodingIntsOvhd; + + // String length histogram builder + private LengthHistogram.Builder lenHistoBuilder; + + StringStatsCollector(Snapshot snapshot) { + stringReader = snapshot.getStringReader(); + + // It looks like generally about 1/4..1/3 of objects are Strings + int capacity = (snapshot.getNumObjects() / 4); + table = new HashMap<>(capacity); + lenHistoBuilder = new LengthHistogram.Builder(capacity); + } + + /** + * Add the information for the given string heap object to the table and analyze it for + * duplication. Regardless of whether this string is duplicated or not, returns the String equal + * to the value of the analyzed string object. + */ + String add(JavaObject strObj) { + nTotalStrings++; + if (stringInstShallowSize == 0) { + stringInstShallowSize = strObj.getSize(); + } + + String strVal = stringReader.readString(strObj); + if (strVal == null) { + return null; + } + + InternalEntry entry = table.get(strVal); + if (entry == null) { + entry = new InternalEntry(currentId++); + table.put(strVal, entry); + } + strObj.setInternalId(entry.uniqueId); + + entry.nStringInst++; + + // Check if its backing array has been seen before + JavaValueArray backingArray = stringReader.getLastReadBackingArray(); + boolean arrayNotSeenBefore = !backingArray.isVisitedAsCollectionImpl(); + int backingArraySize = 0; // Will remain zero if backing array already seen + if (arrayNotSeenBefore) { + backingArray.setVisitedAsCollectionImpl(); + numBackingCharArrays++; + entry.nBackingArrs++; + backingArraySize = backingArray.getSize(); + entry.totalBackingArrSize += backingArraySize; + int arrayLen = backingArray.getLength(); + if (arrayLen > entry.maxArrayLen) { + entry.maxArrayLen = arrayLen; + } + } + + // Check for potentially compressible strings + int strLen = strVal.length(); + if (backingArray.getClazz().isByteArray()) { + nCompressedStrings++; + if (arrayNotSeenBefore) { + usedBackingArrayBytes += strLen; + compressedBackingArrayBytes += strLen; + } + } else { + if (arrayNotSeenBefore) { + usedBackingArrayBytes += strLen * 2; + } + boolean compressible = true; + for (int i = 0; i < strVal.length(); i++) { + if (strVal.charAt(i) > 255) { + compressible = false; + break; + } + } + if (compressible) { + nAsciiCharBackedStrings++; + if (arrayNotSeenBefore) { + asciiCharBackingArrayBytes += strLen * 2; + } + } + } + + checkForShortString(strVal); + + if (isEncodedIntNumber(strVal)) { + nStringsEncodingInts++; + // Below we rely on the fact that backingArraySize is 0 if the backing + // array has already been seen before + stringsEncodingIntsOvhd += stringInstShallowSize + backingArraySize - 4; + } + + lenHistoBuilder.addInstance(strLen, stringInstShallowSize + backingArraySize); + + return strVal; + } + + public DupStringStats getDuplicationStats() { + ArrayList dupStringList = new ArrayList<>(table.size() / 20); + long dupStringsOvhd = 0; + for (Map.Entry tableEntry : table.entrySet()) { + InternalEntry ie = tableEntry.getValue(); + // If there is just one instance of this String, there is no redundancy + if (ie == null || ie.nStringInst == 1) { + tableEntry.setValue(null); // Help the GC + continue; + } + + int overhead = (ie.nStringInst - 1) * stringInstShallowSize; + if (ie.nBackingArrs > 1) { + // If nBackingArrs == 0, it means that we have several String instances with the + // same value, all pointing to the same char[] array (or bunch of arrays) already + // claimed by other String(s). + // If nBackingArrs == 1, we have several String instances with the same value, + // all pointing at the same array. + // In both cases, the overhead is only caused by redundant String instances - + // there are no redundant backing char[] arrays. + // Need (long) below to avoid overflow + overhead += (int) (((long) ie.totalBackingArrSize) * (ie.nBackingArrs - 1) / ie.nBackingArrs); + } + + dupStringList.add(new DupStringStats.Entry(tableEntry.getKey(), ie.uniqueId, ie.nStringInst, + ie.nBackingArrs, ie.maxArrayLen, overhead)); + dupStringsOvhd += overhead; + } + + dupStringList.sort(new Comparator() { + @Override + public int compare(DupStringStats.Entry e1, DupStringStats.Entry e2) { + return e2.overhead - e1.overhead; + } + }); + + return new DupStringStats(stringInstShallowSize, nTotalStrings, table.size(), numBackingCharArrays, + dupStringList, dupStringsOvhd); + } + + public ShortArrayStats getShortStringStats() { + long strShallowSize = stringInstShallowSize; // Made long to correctly calculate long values + return new ShortArrayStats(nZeroLengthStrings, strShallowSize * nZeroLengthStrings, n1Strings, + strShallowSize * n1Strings, n4Strings, strShallowSize * n4Strings, n8Strings, + strShallowSize * n8Strings); + } + + public CompressibleStringStats getCompressibleStringStats() { + return new CompressibleStringStats(nTotalStrings, numBackingCharArrays, usedBackingArrayBytes, + nCompressedStrings, compressedBackingArrayBytes, nAsciiCharBackedStrings, asciiCharBackingArrayBytes); + } + + public NumberEncodingStringStats getNumberEncodingStringStats() { + return new NumberEncodingStringStats(nStringsEncodingInts, stringsEncodingIntsOvhd); + } + + public LengthHistogram getLengthHistogram() { + return lenHistoBuilder.build(); + } + + private void checkForShortString(String string) { + int strLen = string.length(); + if (strLen == 0) { + nZeroLengthStrings++; + } else if (strLen == 1) { + n1Strings++; + } else if (strLen <= 4) { + n4Strings++; + } else if (strLen <= 8) { + n8Strings++; + } + } + + /** + * Checks whether the given string encodes an integer number with radix 10. The code is adapted + * from Integer.parseInt(). We didnt' want to use that method directly because it throws an + * exception if the string is wrong, which can be very costly. It also doesn't make sense for us + * to use a radix other than 10 - and calculations that use non-custom radix likely incur + * additional overhead. + */ + private boolean isEncodedIntNumber(String s) { + if (s.isEmpty()) { + return false; + } + + int result = 0; +// boolean negative = false; + int i = 0, len = s.length(); + int limit = -Integer.MAX_VALUE; + + char firstChar = s.charAt(0); + if (firstChar < '0') { // Possible leading "+" or "-" + if (firstChar == '-') { +// negative = true; + limit = Integer.MIN_VALUE; + } else if (firstChar != '+') { + return false; + } + + if (len == 1) { + return false; // Cannot have lone "+" or "-" + } + i++; + } + + int multmin = limit / 10; + while (i < len) { + // Accumulating negatively avoids surprises near MAX_VALUE + int digit = getDigitFromChar(s.charAt(i++)); + if (digit < 0) { + return false; + } + if (result < multmin) { + return false; + } + result *= 10; + if (result < limit + digit) { + return false; + } + result -= digit; + } + + // Real result: negative ? result : -result; + return true; + } + + private static int getDigitFromChar(char c) { + if (c >= '0' && c <= '9') { + return (c - '0'); + } else { + return -1; + } + } + + /** + * Represents a unique string value. We use entries of this kind internally, until the final + * results are calculated. + */ + private static class InternalEntry { + // A unique id for this string + final int uniqueId; + // Number of instances of this string + int nStringInst; + // Number of backing char arrays for this string + int nBackingArrs; + // Total size, in bytes, of all backing arrays for this string. + int totalBackingArrSize; + // Max backing array length, in chars, for this string + private int maxArrayLen; + + InternalEntry(int uniqueId) { + this.uniqueId = uniqueId; + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/SystemPropertiesReader.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/SystemPropertiesReader.java new file mode 100644 index 00000000..76ede2bd --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/SystemPropertiesReader.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import java.util.HashMap; + +import org.openjdk.jmc.joverflow.descriptors.CollectionDescriptors; +import org.openjdk.jmc.joverflow.descriptors.CollectionInstanceDescriptor; +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaThing; +import org.openjdk.jmc.joverflow.heap.model.Snapshot; + +/** + * Reads the contents of the system properties table, normally stored in the Properties props field + * of java.lang.System class. + */ +class SystemPropertiesReader { + + static HashMap readProperties(Snapshot snapshot, CollectionDescriptors colDesriptors) { + JavaClass systemClazz = snapshot.getClassForName("java.lang.System"); + JavaThing propsThing = systemClazz.getStaticField("props"); + if (propsThing == null) { + propsThing = findFieldOfTypeProperties(systemClazz); + if (propsThing == null) { + return null; + } + } + if (!(propsThing instanceof JavaObject)) { + return null; // Unresolved + } + + CollectionInstanceDescriptor colDesc = colDesriptors.getDescriptor((JavaObject) propsThing); + if (colDesc == null || !colDesc.getClassDescriptor().isMap()) { + return null; // Unknown type? + } + + final HashMap result = new HashMap<>(32); + + colDesc.iterateMap(new CollectionInstanceDescriptor.MapIteratorCallback() { + @Override + public boolean scanMapEntry(JavaHeapObject key, JavaHeapObject value) { + String keyStr = key != null ? key.valueAsString() : "Unresolved object"; + String valueStr = value != null ? value.valueAsString() : "Unresolved object"; + result.put(keyStr, valueStr); + return true; + } + + @Override + public boolean scanImplementationObject(JavaHeapObject implObj) { + return true; + } + }); + + return result; + } + + private static JavaThing findFieldOfTypeProperties(JavaClass clazz) { + JavaThing[] staticFields = clazz.getStaticValues(); + for (JavaThing field : staticFields) { + if (!(field instanceof JavaObject)) { + continue; + } + String className = ((JavaObject) field).getClazz().getName(); + if (className.equals("java.lang.Properties")) { + return field; + } + } + return null; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/TwoHandIndexContainer.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/TwoHandIndexContainer.java new file mode 100644 index 00000000..6a5747c6 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/TwoHandIndexContainer.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import org.openjdk.jmc.joverflow.util.IndexContainer; + +/** + * An IndexContainer which provides an additional number, called base, so that two numbers become an + * equivalent of two "hands" of the clock. + */ +public class TwoHandIndexContainer extends IndexContainer { + private int baseIndex = -1; + + public void setBase(int index) { + this.baseIndex = index; + } + + public int getBase() { + return baseIndex; + } + + public int incrementAndGetBase() { + baseIndex++; + return baseIndex; + } + + @Override + public String toString() { + return "index = " + get() + ", base = " + baseIndex; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/WeakMapHandler.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/WeakMapHandler.java new file mode 100644 index 00000000..57cce37f --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/stats/WeakMapHandler.java @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.stats; + +import org.openjdk.jmc.joverflow.descriptors.CollectionInstanceDescriptor; +import org.openjdk.jmc.joverflow.descriptors.WeakHashMapDescriptor; +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaHeapObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObjectArray; +import org.openjdk.jmc.joverflow.heap.model.JavaThing; +import org.openjdk.jmc.joverflow.support.Constants; +import org.openjdk.jmc.joverflow.util.SimpleIdentitySet; + +/** + * Inspects an instance of WeakHashMap or its subclass for a special problem: elements that point + * back at keys in the same map. That breaks the weakness property and prevents entries where keys + * are not referenced from anywhere (except their own elements) from being GCed. + */ +class WeakMapHandler { + private final WeakHashMapDescriptor colDesc; + + /** + * Returns an instance of WeakMapHandler if the given collection is a WeakHashMap or its + * subclass. Otherwise, returns null. + */ + static WeakMapHandler createInstance(CollectionInstanceDescriptor colDesc) { + JavaClass clazz = colDesc.getClassDescriptor().getClazz(); + if (!clazz.isOrSubclassOf(Constants.WEAK_HASH_MAP)) { + return null; + } + if (colDesc.getNumElements() == 0) { + return null; + } + + return new WeakMapHandler(colDesc); + } + + private WeakMapHandler(CollectionInstanceDescriptor colDesc) { + this.colDesc = (WeakHashMapDescriptor) colDesc; + } + + /** + * Checks if WeakHashMap associated with this object has the problem. If so, returns a + * combination of the overhead for problematic entries, and a sample value type/field, like + * "Foo.bar", for an object that references a key in this table. + */ + Result calculateOverhead() { + JavaHeapObject keysAndValues[][] = colDesc.getKeysAndValues(); + if (keysAndValues[0].length == 0) { + return null; + } + + JavaHeapObject[] keys = keysAndValues[0]; + SimpleIdentitySet keySet = new SimpleIdentitySet<>(keys.length); + for (JavaHeapObject key : keys) { + if (key != null) { + keySet.add(key); + } + } + // Important: this operation can only be performed if the empty check above is done! + // Let's comment it out. Safety is more important than a little saved memory + // keysAndValues[0] = null; // Help the GC + JavaHeapObject[] values = keysAndValues[1]; + + int ovhd = 0; + String valueTypeAndFieldSample = null; + for (JavaHeapObject value : values) { + if (value instanceof JavaObject) { + JavaObject valueObj = (JavaObject) value; + // A weak reference back to key is ok + if (valueObj.getClazz().isOrSubclassOf(Constants.WEAK_REFERENCE)) { + break; + } + + JavaThing[] fields = valueObj.getFields(false); + for (int i = 0; i < fields.length; i++) { + JavaThing fieldThing = fields[i]; + if (fieldThing == null || !(fieldThing instanceof JavaHeapObject)) { + continue; + } + JavaHeapObject field = (JavaHeapObject) fieldThing; + if (keySet.contains(field)) { + ovhd += field.getSize() + valueObj.getSize(); + if (valueTypeAndFieldSample == null) { + valueTypeAndFieldSample = getStringForValueAndField(valueObj, i); + } + break; + } + } + } else if (value instanceof JavaObjectArray) { + JavaHeapObject[] elements = ((JavaObjectArray) value).getElements(); + for (JavaHeapObject element : elements) { + if (element == null) { + continue; + } + // A weak reference back to key is ok + if (element.getClazz().isOrSubclassOf(Constants.WEAK_REFERENCE)) { + continue; + } + + if (keySet.contains(element)) { + ovhd += element.getSize() + value.getSize(); + if (valueTypeAndFieldSample == null) { + valueTypeAndFieldSample = value.getClazz().getName(); + } + break; + } + } + } + } + + if (ovhd > 0) { + return new Result(ovhd, valueTypeAndFieldSample); + } else { + return null; + } + } + + private static String getStringForValueAndField(JavaObject value, int fieldIdx) { + JavaClass clazz = value.getClazz(); + return clazz.getName() + '.' + clazz.getFieldForInstance(fieldIdx).getName(); + } + + static class Result { + final int overhead; + final String valueTypeAndFieldSample; + + Result(int overhead, String valueTypeAndFieldSample) { + this.overhead = overhead; + this.valueTypeAndFieldSample = valueTypeAndFieldSample; + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/ClassAndOvhdCombo.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/ClassAndOvhdCombo.java new file mode 100644 index 00000000..7de82b4f --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/ClassAndOvhdCombo.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.support; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.support.Constants.ProblemKind; + +/** + * Represents a bunch of problematic collections, all of which have the same class and problem, e.g. + * empty HashMaps. Other information available is the number of collections and their total + * overhead. So example information in this instance may look like "HashMap: 100 EMPTY instances, + * overhead 1000 bytes". + */ +public class ClassAndOvhdCombo extends ClassAndSizeCombo { + + private final Constants.ProblemKind problemKind; + + ClassAndOvhdCombo(JavaClass clazz, ProblemKind problemKind, int numInstances, int ovhd) { + super(clazz, numInstances, ovhd); + this.problemKind = problemKind; + } + + public Constants.ProblemKind getProblemKind() { + return problemKind; + } + + public int getOverhead() { + return getSizeOrOvhd(); + } + + @Override + public ClassAndOvhdCombo clone() { + return (ClassAndOvhdCombo) super.clone(); + } + + /** + * Instances of this class are created only for certain kinds of problematic collections: + * currently that's SMALL, SPARSE_SMALL and SPARSE_LARGE. It contains additional aggregated + * information about the number of elements of the said collections. + */ + public static class Extended extends ClassAndOvhdCombo { + private long totalNumElements; + private int maxNumElements; + + Extended(JavaClass collectionClazz, ProblemKind problemKind, int numInstances, int ovhd, + long numElementsInCollection, int maxNumElements) { + super(collectionClazz, problemKind, numInstances, ovhd); + totalNumElements = numElementsInCollection; + this.maxNumElements = maxNumElements; + } + + void addInstances(int nInstances, int ovhd, long totalNumElements, int maxNumElements) { + super.addInstances(nInstances, ovhd); + this.totalNumElements += totalNumElements; + if (maxNumElements > this.maxNumElements) { + this.maxNumElements = maxNumElements; + } + } + + public long getTotalNumElements() { + return totalNumElements; + } + + public float getAverageNumElements() { + return (float) (((double) totalNumElements) / getNumInstances()); + } + + public int getMaxNumElements() { + return maxNumElements; + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/ClassAndOvhdComboList.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/ClassAndOvhdComboList.java new file mode 100644 index 00000000..0cbddd07 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/ClassAndOvhdComboList.java @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.support; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.support.Constants.ProblemKind; + +/** + * A temporary list of instances of {@link ClassAndOvhdCombo}, used during data collection and + * aggregation. + *

+ * When we aggregate entries that have the same class/field etc., we don't distinguish between class + * versions, i.e. we consider two classes with the same name but different classloaders as a single + * class. This fixed policy may need to be made flexible at some point. + */ +public class ClassAndOvhdComboList implements Cloneable { + private LinkedList list; + private int totalOverhead; + + public List getFinalList() { + ArrayList result = new ArrayList<>(list.size()); + result.addAll(list); + if (result.size() > 1) { + result.sort(null); + } + return result; + } + + /** Returns the total overhead of all entries in this list */ + public int getTotalOverhead() { + return totalOverhead; + } + + public void addCollectionInfo(JavaClass colClass, ProblemKind ovhdKind, int ovhd, int nInstances) { + totalOverhead += ovhd; + if (list == null) { + list = new LinkedList<>(); + addNewEntry(colClass, ovhdKind, ovhd, nInstances); + return; + } + + addInfo(colClass, ovhdKind, ovhd, nInstances); + } + + public void addCollectionInfoWithNumEls( + JavaClass colClass, ProblemKind ovhdKind, int ovhd, int nInstances, long numElementsInCollection, + int maxNumElements) { + totalOverhead += ovhd; + if (list == null) { + list = new LinkedList<>(); + addNewEntryWithNumEls(colClass, ovhdKind, ovhd, nInstances, numElementsInCollection, maxNumElements); + return; + } + + addInfoWithNumEls(colClass, ovhdKind, ovhd, nInstances, numElementsInCollection, maxNumElements); + } + + public void merge(ClassAndOvhdComboList other) { + LinkedList otherList = other.list; + if (otherList == null) { + return; // Can happen if that list only contains good collections + } + for (ClassAndOvhdCombo entry : otherList) { + if (entry instanceof ClassAndOvhdCombo.Extended) { + ClassAndOvhdCombo.Extended extEntry = (ClassAndOvhdCombo.Extended) entry; + addCollectionInfoWithNumEls(entry.getClazz(), entry.getProblemKind(), entry.getOverhead(), + entry.getNumInstances(), extEntry.getTotalNumElements(), extEntry.getMaxNumElements()); + } else { + addCollectionInfo(entry.getClazz(), entry.getProblemKind(), entry.getOverhead(), + entry.getNumInstances()); + } + } + } + + @Override + public ClassAndOvhdComboList clone() { + ClassAndOvhdComboList result = new ClassAndOvhdComboList(); + if (list != null) { // Can happen if this cluster only contains good collections + result.list = new LinkedList<>(); + for (ClassAndOvhdCombo entry : list) { + result.list.add(entry.clone()); + } + } + result.totalOverhead = totalOverhead; + return result; + } + + private void addInfo(JavaClass colClass, ProblemKind ovhdKind, int ovhd, int nInstances) { + for (ClassAndOvhdCombo entry : list) { + if (entry.getClazz().getName().equals(colClass.getName()) && entry.getProblemKind() == ovhdKind) { + entry.addInstances(nInstances, ovhd); + return; + } + } + + addNewEntry(colClass, ovhdKind, ovhd, nInstances); + } + + private void addNewEntry(JavaClass colClass, ProblemKind ovhdKind, int ovhd, int nInstances) { + list.add(new ClassAndOvhdCombo(colClass, ovhdKind, nInstances, ovhd)); + } + + private void addInfoWithNumEls( + JavaClass colClass, ProblemKind ovhdKind, int ovhd, int nInstances, long numElementsInCollection, + int maxNumElements) { + for (ClassAndOvhdCombo entry : list) { + if (entry.getClazz().getName().equals(colClass.getName()) && entry.getProblemKind() == ovhdKind) { + ((ClassAndOvhdCombo.Extended) entry).addInstances(nInstances, ovhd, numElementsInCollection, + maxNumElements); + return; + } + } + + addNewEntryWithNumEls(colClass, ovhdKind, ovhd, nInstances, numElementsInCollection, maxNumElements); + } + + private void addNewEntryWithNumEls( + JavaClass colClass, ProblemKind ovhdKind, int ovhd, int nInstances, long numElementsInCollection, + int maxNumElements) { + list.add(new ClassAndOvhdCombo.Extended(colClass, ovhdKind, nInstances, ovhd, numElementsInCollection, + maxNumElements)); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/ClassAndSizeCombo.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/ClassAndSizeCombo.java new file mode 100644 index 00000000..329e9091 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/ClassAndSizeCombo.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.support; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; + +/** + * Represents a bunch of objects, all of which belong to the same class. + *

+ * Note that this class implements compareTo() but has no implementation of equals(). In other + * words, it's currently not guaranteed that compareTo() returns zero if and only if equals() + * returns true. However, that only matters if instances of a class are used in classes like + * PriorityQueue, which is highly unlikely for this class and its subclasses. + */ +public class ClassAndSizeCombo implements Cloneable, Comparable { + private final JavaClass clazz; + private int numInstances; + private int sizeOrOvhd; + + ClassAndSizeCombo(JavaClass clazz, int numInstances, int sizeOrOvhd) { + this.clazz = clazz; + this.numInstances = numInstances; + this.sizeOrOvhd = sizeOrOvhd; + } + + public JavaClass getClazz() { + return clazz; + } + + public int getNumInstances() { + return numInstances; + } + + public int getSizeOrOvhd() { + return sizeOrOvhd; + } + + // Methods used only when collecting and aggregating data + + void addInstances(int nInstances, int ovhd) { + this.numInstances += nInstances; + this.sizeOrOvhd += ovhd; + } + + @Override + public ClassAndSizeCombo clone() { + try { + return (ClassAndSizeCombo) super.clone(); + } catch (CloneNotSupportedException ex) { + throw new RuntimeException(ex); + } + } + + @Override + public int compareTo(ClassAndSizeCombo other) { + return other.getSizeOrOvhd() - this.getSizeOrOvhd(); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/ClassAndSizeComboList.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/ClassAndSizeComboList.java new file mode 100644 index 00000000..dcc0773a --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/ClassAndSizeComboList.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.support; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; + +/** + * A temporary list of instances of {@link ClassAndSizeCombo}, used during data collection and + * aggregation. + */ +public class ClassAndSizeComboList implements Cloneable { + + private LinkedList list; + private int totalSize; + + public ClassAndSizeComboList() { + list = new LinkedList<>(); + } + + public List getFinalList() { + ArrayList result = new ArrayList<>(list.size()); + result.addAll(list); + if (result.size() > 1) { + result.sort(null); + } + return result; + } + + public int getTotalSize() { + return totalSize; + } + + public void addInstanceInfo(JavaClass clazz, int size, int nInstances) { + totalSize += size; + + for (ClassAndSizeCombo entry : list) { + if (entry.getClazz() == clazz) { + entry.addInstances(nInstances, size); + return; + } + } + + list.add(new ClassAndSizeCombo(clazz, nInstances, size)); + } + + public void merge(ClassAndSizeComboList other) { + LinkedList otherList = other.list; + for (ClassAndSizeCombo entry : otherList) { + addInstanceInfo(entry.getClazz(), entry.getSizeOrOvhd(), entry.getNumInstances()); + } + } + + @Override + public ClassAndSizeComboList clone() { + ClassAndSizeComboList result = new ClassAndSizeComboList(); + for (ClassAndSizeCombo entry : list) { + result.list.add(entry.clone()); + } + result.totalSize = totalSize; + return result; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/CompressibleStringStats.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/CompressibleStringStats.java new file mode 100644 index 00000000..d05c214b --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/CompressibleStringStats.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.support; + +/** + * Container for statistics on Strings that are, or can be, "compressed" - that is, their backing + * char[] array can be replaced with byte[], because all the characters in it are ASCII. + *

+ * In reality, there is a subtle issue when a heap dump from JDK version earlier than JDK7u6 is + * used, where several String objects can point to the same backing array and furthermore can use + * different or overlapping parts of it. In this situation there may be no good way to calculate the + * number of chars (bytes) backing certain String objects. Currently when making calculations we + * look only at the chars (bytes) that logically belong to the string (i.e. we may look only at a + * part of the backing array). Furthermore, when calculating all totals, we add up the numbers of + * chars (bytes) equal to String.length(), and we add zero if the given backing array has already + * been seen before, from another String object. As a result, the number of "used backing array + * bytes" may end up considerably smaller than the total number of bytes in all arrays backing + * Strings. + */ +public class CompressibleStringStats { + /** Total number of String objects */ + public final int nTotalStrings; + /** Total number of backing char[] arrays */ + public final int nBackingCharArrays; + + /** Used bytes in backing arrays, calculated as explained in class-level comments */ + public final long totalUsedBackingArrayBytes; + /** Number of String objects backed by byte[] arrays */ + public final int nCompressedStrings; + /** Number of used bytes in backing byte[] arrays */ + public final long compressedBackingArrayBytes; + /** Number of String objects backed by char[] arrays with ASCII chars only */ + public final int nAsciiCharBackedStrings; + /** Number of used in backing char[] arrays with ASCII chars only */ + public final long asciiCharBackingArrayBytes; + + public CompressibleStringStats(int nTotalStrings, int nBackingCharArrays, long totalUsedBackingArrayBytes, + int nCompressedStrings, long compressedBackingArrayBytes, int nAsciiCharBackedStrings, + long asciiCharBackingArrayBytes) { + this.nTotalStrings = nTotalStrings; + this.nBackingCharArrays = nBackingCharArrays; + this.totalUsedBackingArrayBytes = totalUsedBackingArrayBytes; + this.nCompressedStrings = nCompressedStrings; + this.compressedBackingArrayBytes = compressedBackingArrayBytes; + this.nAsciiCharBackedStrings = nAsciiCharBackedStrings; + this.asciiCharBackingArrayBytes = asciiCharBackingArrayBytes; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/Constants.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/Constants.java new file mode 100644 index 00000000..dc226c7c --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/Constants.java @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.support; + +/** + */ +public interface Constants { + // Collection classes + String HASH_MAP = "java.util.HashMap"; + String LINKED_HASH_MAP = "java.util.LinkedHashMap"; + String HASH_SET = "java.util.HashSet"; + String LINKED_HASH_SET = "java.util.LinkedHashSet"; + String HASHTABLE = "java.util.Hashtable"; + String ARRAY_LIST = "java.util.ArrayList"; + String VECTOR = "java.util.Vector"; + String CONCURRENT_HASH_MAP = "java.util.concurrent.ConcurrentHashMap"; + String WEAK_HASH_MAP = "java.util.WeakHashMap"; + String STACK = "java.util.StaNonFinalck"; + String TREE_MAP = "java.util.TreeMap"; + String LINKED_LIST = "java.util.LinkedList"; + String IDENTITY_HASH_MAP = "java.util.IdentityHashMap"; + String ARRAY_BLOCKING_QUEUE = "java.util.concurrent.ArrayBlockingQueue"; + String ARRAY_DEQUE = "java.util.ArrayDeque"; + String PROPERTIES = "java.util.Properties"; + String ATTRIBUTE_LIST = "javax.management.AttributeList"; + String CONCURRENT_LINKED_QUEUE = "java.util.concurrent.ConcurrentLinkedQueue"; + String COPY_ON_WRITE_ARRAY_LIST = "java.util.concurrent.CopyOnWriteArrayList"; + String COPY_ON_WRITE_ARRAY_SET = "java.util.concurrent.CopyOnWriteArraySet"; + String PRIORITY_QUEUE = "java.util.PriorityQueue"; + + // Collection implementation detail classes + String HASH_MAP_ENTRY = "java.util.HashMap$Entry"; + String LINKED_HASH_MAP_ENTRY = "java.util.LinkedHashMap$Entry"; + String HASHTABLE_ENTRY = "java.util.Hashtable$Entry"; + String CONCURRENT_HASH_MAP_SEGMENT = "java.util.concurrent.ConcurrentHashMap$Segment"; + String CONCURRENT_HASH_MAP_ENTRY = "java.util.concurrent.ConcurrentHashMap$HashEntry"; + String CONCURRENT_HASH_MAP_NODE = "java.util.concurrent.ConcurrentHashMap$Node"; + String WEAK_HASH_MAP_ENTRY = "java.util.WeakHashMap$Entry"; + String TREE_MAP_ENTRY = "java.util.TreeMap$Entry"; + String TREE_MAP_NODE = "java.util.TreeMap$Node"; // Probably in older JDK versions + String LINKED_LIST_ENTRY = "java.util.LinkedList$Entry"; // Before JDK7-update-something (2?) + String LINKED_LIST_NODE = "java.util.LinkedList$Node"; // After JDK7-update-something (2?) + String CONCURRENT_LINKED_QUEUE_NODE = "java.util.concurrent.ConcurrentLinkedQueue$Node"; + + // Other useful classes + String JAVA_LANG_OBJECT = "java.lang.Object"; + String JAVA_LANG_STRING = "java.lang.String"; + String WEAK_REFERENCE = "java.lang.ref.WeakReference"; + String OBJECT_ARRAY = "[Ljava.lang.Object;"; + + String CHAR_ARRAY = "[C"; + + // JVM in-memory pointer sizes, in bytes, in different modes + static final int POINTER_SIZE_IN_32BIT_MODE = 4; + static final int NARROW_POINTER_SIZE_IN_64BIT_MODE = 4; + static final int WIDE_POINTER_SIZE_IN_64BIT_MODE = 8; + + // JVM in-memory object header sizes, in bytes, for various JVMs + static final int STANDARD_32BIT_OBJ_HEADER_SIZE = 8; + static final int HOTSPOT_64BIT_NARROW_REF_OBJ_HEADER_SIZE = 12; + static final int HOTSPOT_64BIT_WIDE_REF_OBJ_HEADER_SIZE = 16; + static final int JROCKIT_OBJ_HEADER_SIZE = 8; + + /** + * We assume that objects in memory are aligned at the byte granularity below. The value may be + * different if compressed references are used. However, it should be rare in practice - it + * requires an explicit JVM otpion -XX:ObjectAlignmentInBytes=, and it makes sense to use it + * only to enable narrow pointers for heap sizes larger than ~32GB. + */ + static final int DEFAULT_OBJECT_ALIGNMENT_IN_MEMORY = 8; + + /** + * All problems that we recognize for collections and standalone arrays. Some problems can occur + * in both collections and arrays; others are specific to only one kind of data structures. + */ + static enum ProblemKind { + /** + * Collection or standalone array that contains no elements, for which we cannot determine + * whether it has been used (i.e. whether it previously contained elements that have been + * deleted subsequently). For arrays that's always the case; for collections it should not + * be the case unless some collection doesn't have the 'modCount' field. + */ + EMPTY, + /** + * Empty collection that has never contained any elements (because its modCount == 0) + */ + EMPTY_UNUSED, + /** + * Empty collection that previously contained some elements (because its modCount != 0) + */ + EMPTY_USED, + /** + * Array-based collection of default or smaller capacity, has less than half slots occupied + */ + SPARSE_SMALL, + /** + * Array-based collection of larger than default capacity, has less than half slots occupied + */ + SPARSE_LARGE, + /** + * Standalone array has less than half slots occupied + */ + SPARSE_ARRAY, + /** + * Collection or standalone array contains boxed numbers + */ + BOXED, + /** + * Standalone array of length 0 + */ + LENGTH_ZERO, + /** + * Standalone array of length 1 + */ + LENGTH_ONE, + /** + * Standalone primitive array of types short[], char[], int[] or long[] where some of the + * high bytes are unused in each element + */ + UNUSED_HI_BYTES, + /** + * A WeakHashMap or subclass, where elements point back to keys + */ + WEAK_MAP_WITH_BACK_REFS, + /** + * An array of subarrays, where the outer dimension is bigger than inner + */ + BAR, + /** + * Long zero-tail - a primitive array that ends with a series of zeros that is half the + * array's length or longer + */ + LZT, + /** + * Small collections, with impl overhead too big compared to workload. They can, in + * principle, be replaced with arrays (or pairs of arrays for maps) + */ + SMALL + } + +// FIXME: Unused arrays that in either case should not be exposed. Consider if they are useful in any way or if they can be removed. +// /** +// * Kinds of problems that collections can have. Note that sparseness and "vertical bar" shape +// * are only defined for array-based collections. +// */ +// static ProblemKind[] COL_SPECIFIC_PROBLEM_KINDS = new ProblemKind[] {ProblemKind.EMPTY_UNUSED, +// ProblemKind.EMPTY_USED, ProblemKind.SPARSE_SMALL, ProblemKind.SPARSE_LARGE, ProblemKind.BOXED, +// ProblemKind.BAR, ProblemKind.SMALL, ProblemKind.EMPTY}; +// +// /** Kinds of problems that standalone object arrays can have */ +// static ProblemKind[] OBJ_ARRAY_SPECIFIC_PROBLEM_KINDS = new ProblemKind[] {ProblemKind.LENGTH_ZERO, +// ProblemKind.LENGTH_ONE, ProblemKind.EMPTY, ProblemKind.SPARSE_ARRAY, ProblemKind.BOXED, ProblemKind.BAR}; +// +// /** Kinds of problems that standalone primitive arrays can have */ +// static ProblemKind[] VALUE_ARRAY_SPECIFIC_PROBLEM_KINDS = new ProblemKind[] {ProblemKind.LENGTH_ZERO, +// ProblemKind.LENGTH_ONE, ProblemKind.EMPTY, ProblemKind.LZT}; + + /** Maximum size for small collections, see ProblemKind.SMALL */ + int SMALL_COL_MAX_SIZE = 4; + +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/DupArrayStats.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/DupArrayStats.java new file mode 100644 index 00000000..8d452f69 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/DupArrayStats.java @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.support; + +import java.util.ArrayList; + +import org.openjdk.jmc.joverflow.heap.model.JavaValueArray; +import org.openjdk.jmc.joverflow.util.ValueWithIntId; + +/** + * Container for stats on duplicated primitive arrays. + */ +public class DupArrayStats { + /** Total number of standalone primitive array objects */ + public final int nArrays; + + /** + * Number of unique array values. Some of the arrays with these values can be duplicated, some + * not. + */ + public final int nUniqueArrays; + + /** + * Number of different values of duplicated arrays. In other words, this is the number of unique + * array values, for each of which more than one separate array object exists. Since for some + * unique array values only one array object exists, nDifferentDupArrayValues is always less + * than nUniqueArrays. + */ + public final int nDifferentDupArrayValues; + + /** Detailed information about individual duplicated arrays */ + public final ArrayList dupArrays; + + /** Overhead due to all duplicated arrays */ + public final long dupArraysOverhead; + + /** Sets the overall duplicated string stats */ + public DupArrayStats(int nArrays, int nUniqueArrays, int nDifferentDupArrayValues, ArrayList dupArrays, + long dupArraysOverhead) { + this.nArrays = nArrays; + this.nUniqueArrays = nUniqueArrays; + this.nDifferentDupArrayValues = nDifferentDupArrayValues; + this.dupArrays = dupArrays; + this.dupArraysOverhead = dupArraysOverhead; + } + + /** + * Represents a unique array value (unique contents), for which multiple array objects exist. + */ + public static class Entry implements ValueWithIntId { + public final JavaValueArray firstArray; + public final int internalId; + public final int nArrayInstances; + public final int overhead; + + // These are used to calculate overhead properly for each individual + // duplicate array instance detected during detailed analysis + private int runningRemainingOvhd; + private int runningRemainingNInstances; + + public Entry(JavaValueArray firstArray, int internalId, int nArrayInstances, int overhead) { + this.firstArray = firstArray; + this.internalId = internalId; + this.nArrayInstances = nArrayInstances; + this.overhead = overhead; + + this.runningRemainingNInstances = nArrayInstances; + this.runningRemainingOvhd = overhead; + } + + @Override + public int getId() { + return internalId; + } + + /** + * Returns the overhead for the next copy of the array in this Entry. We aim to report the + * total overhead in this entry that's evenly distributed between all copies of this array. + * This is consistent with the fact that the overhead can be fully eliminated only when + * all places where this array is attached to a long-lived data structure + * are somehow patched to get rid of duplication. So ideally, this method should return the + * same value each time it's called. However, in practice that value would almost always be + * a non-integer number, like 8.43. Reporting e.g. floor(), ceil() or round() of this number + * is not good, because the sum of the resulting numbers won't match the total overhead. So + * instead this method returns a series of numbers, which in the above example will be 8s + * and 9s, such that in the end their sum is equal to the total overhead. + */ + public int getOvhdForNextArrayCopy() { + int result = runningRemainingOvhd / runningRemainingNInstances; + runningRemainingOvhd -= result; + runningRemainingNInstances--; + return result; + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/DupStringStats.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/DupStringStats.java new file mode 100644 index 00000000..6b6d3b56 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/DupStringStats.java @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.support; + +import java.util.ArrayList; + +import org.openjdk.jmc.joverflow.util.ValueWithIntId; + +/** + * Container various duplicate string stats, calculated by + * {@link org.openjdk.jmc.joverflow.stats.StringStatsCollector}. + *

+ * In particular, getDupStrings() returns the list of duplicate strings sorted by overhead. The + * overhead is defined as specified in {@link org.openjdk.jmc.joverflow.stats.StringStatsCollector}, + * but in reality we perform a bit more crude estimate, since duplicate strings may reference + * sub-ranges of backing char[] arrays, and we don't keep length of each backing char array. The + * estimate is conservative, i.e. the returned overhead should not be greater than the real one. We + * also take care of the fact that some shallowly duplicate Strings with different values may point + * to the same backing arrays. For example, a pair of objects with values "a", "b" both point to + * 'abc' array, and another pair "a", "b" points to 'abd'. In this case, the size of both 'abc' and + * 'abd' will count towards overhead, but only once for each (i.e. 3 + 3 chars, not 3 + 3 + 3 + 3). + */ +public class DupStringStats { + /** Shallow size of java.lang.String instance in the analyzed heap */ + public final int stringInstShallowSize; + + /** Total number of instances of java.lang.String */ + public final int nStrings; + + /** + * Number of unique string values. Some of the string with these values can be duplicated, some + * not. + */ + public final int nUniqueStringValues; + + /** + * Number of different values of duplicated strings. In other words, this is the number of + * unique string values, for each of which more than one java.lang.String object exists. Since + * for some unique string values only one java.lang.String instance exists, + * nDifferentDupArrayValues is always less than nUniqueStringValues. + */ + public final int nUniqueDupStringValues; + + /** Number of backing char arrays for java.lang.Strings */ + public final int nBackingCharArrays; + + /** Detailed information about individual duplicate strings */ + public final ArrayList dupStrings; + + /** Overhead due to all duplicated strings */ + public final long dupStringsOverhead; + + public DupStringStats(int stringInstShallowSize, int nStrings, int nUniqueStringValues, int nBackingCharArrays, + ArrayList dupStrings, long dupStringsOverhead) { + this.stringInstShallowSize = stringInstShallowSize; + this.nStrings = nStrings; + this.nUniqueStringValues = nUniqueStringValues; + this.nUniqueDupStringValues = dupStrings.size(); + this.nBackingCharArrays = nBackingCharArrays; + this.dupStrings = dupStrings; + this.dupStringsOverhead = dupStringsOverhead; + } + + /** + * Represents a unique string value for which multiple String instances exist. They are backed + * by one or more char[] arrays. + */ + public static class Entry implements ValueWithIntId { + /** + * String value (result of String.toString()) + */ + public final String string; + /** + * Internal id of all javaObjs equal to this string + */ + public final int internalId; + /** + * Num of java.lang.String instances with this value + */ + public final int nStringInstances; + /** + * Number of backing char[] arrays for them + */ + public final int nBackingArrays; + /** + * Max length for these arrays, in chars + */ + public final int maxArrayLen; + /** + * Overhead - how much space we would save if we get rid of all instances but one and all + * char arrays but one. + */ + public final int overhead; + + // These are used to calculate overhead properly for each duplicate string + // detected during detailed analysis + private int runningRemainingOvhd; + private int runningRemainingNInstances; + + public Entry(String string, int internalId, int nStringInstances, int nBackingArrays, int maxArrayLen, + int overhead) { + this.string = string; + this.internalId = internalId; + this.nStringInstances = nStringInstances; + this.nBackingArrays = nBackingArrays; + this.maxArrayLen = maxArrayLen; + this.overhead = overhead; + + this.runningRemainingNInstances = nStringInstances; + this.runningRemainingOvhd = overhead; + } + + @Override + public int getId() { + return internalId; + } + + /** + * Returns the overhead for the next copy of the string in this Entry. We aim to report the + * total overhead in this entry that's evenly distributed between all copies of this string. + * This is consistent with the fact that the overhead can be fully eliminated only when + * *all* places where this string is attached to a long-lived data structure are patched + * with an intern() call. So ideally, this method should return the same value each time + * it's called. However, in practice that value would almost always be a non-integer number, + * like 8.43. Reporting e.g. floor(), ceil() or round() of this number is not good, because + * the sum of the resulting numbers won't match the total overhead. So instead this method + * returns a series of numbers, which in the above example will be 8s and 9s, such that in + * the end their sum is equal to the total overhead. + */ + public int getOvhdForNextStringCopy() { + int result = runningRemainingOvhd / runningRemainingNInstances; + runningRemainingOvhd -= result; + runningRemainingNInstances--; + return result; + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/HeapStats.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/HeapStats.java new file mode 100644 index 00000000..cb9655f4 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/HeapStats.java @@ -0,0 +1,342 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.support; + +import java.util.ArrayList; +import java.util.HashMap; + +import org.openjdk.jmc.joverflow.descriptors.CollectionClassDescriptor; +import org.openjdk.jmc.joverflow.stats.ClassloaderStats; +import org.openjdk.jmc.joverflow.stats.LengthHistogram; +import org.openjdk.jmc.joverflow.stats.ObjectHistogram; +import org.openjdk.jmc.joverflow.util.ObjectToIntMap; + +/** + * Container for heap statistics. Most of the contained information in it falls in two categories: + * "overall stats" (higher level information) and 'detailed stats" (lower level, more detailed + * information). They are filled out by OverallStatsCalculator and DetailedStatsCalculator + * respectively. This class also references two separate objects, DupStringStats and DupArrayStats, + * that themselves are filled out in two stages in the same "calculator" classes. + */ +public class HeapStats { + // ---------------------------- Overall stats ------------------------------- + /** Pointer size */ + public int ptrSize; + /** Object header size */ + public int objHeaderSize; + /** Object alignment in memory */ + public int objAlignment; + /** True if compressed pointers are used in 64-bit mode */ + public boolean usingNarrowPointers; + /** Total number of classes */ + public int nClasses; + + /** Total number of objects */ + public int nObjects; + /** Total number of instances */ + public int nInstances; + /** Total number of object arrays */ + public int nObjectArrays; + /** Total number of value arrays */ + public int nValueArrays; + + /** Total size of objects */ + public long totalObjSize; + /** Total instance size */ + public long totalInstSize; + /** Total object array size */ + public long totalObjArraySize; + /** Total value array size */ + public long totalValueArraySize; + + /** Overhead of VM object headers */ + public long ovhdObjHeaders; + /** Number of instances of *$Entry classes */ + public int nEntryInstances; + /** Overhead of *$Entry classes */ + public long entryClassSize; + + public ClassloaderStats classloaderStats; + + public ShortArrayStats shortStringStats; + + public ShortArrayStats shortPrimitiveArrayStats; + + public ShortArrayStats shortObjArrayStats; + + /** Number of boxed Numbers (e.g. Integer or Long) */ + public int nBoxedNumbers; + /** Overhead of boxed Numbers */ + public long ovhdBoxedNumbers; + + // Statistics for wrapped Unmodifiable* and Synchronized* collection classes + public ObjectToIntMap.Entry[] unmodifiableClasses; + // Not used when printing reports in ReportFormatter. Can perhaps be removed entirely. +// public ObjectToIntMap.Entry[] synchronizedClasses; + + // --------------------------- Detailed stats ------------------------------- + + /** Total number of Collections */ + public int numCols; // + /** Number of empty, small, sparse, boxed-number and bar Collections */ + public int numEmptyUnusedCols, numEmptyUsedCols, numEmptyCols, numSmallCols, numSparseSmallCols, numSparseLargeCols, + numBoxedNumberCols, numBarCols; + /** Overhead caused by the above abnormal Collections */ + public long emptyUnusedColsOverhead, emptyUsedColsOverhead, emptyColsOverhead, smallColsOverhead, + sparseSmallColsOverhead, sparseLargeColsOverhead, boxedNumberColsOverhead, barColsOverhead; + + /** Total number of standalone object arrays */ + public int numObjArrays; + /** Number of length 0, length 1, empty, sparse, boxed-number and bar object arrays */ + public int numLengthZeroObjArrays, numLengthOneObjArrays, numEmptyObjArrays, numSparseObjArrays; + public int numBoxedNumberArrays, numBarObjArrays; + /** Overhead caused by the above abnormal arrays */ + public long lengthZeroObjArraysOverhead, lengthOneObjArraysOverhead, emptyObjArraysOverhead; + public long sparseObjArraysOverhead, boxedNumberArraysOverhead, barObjArraysOverhead; + + /** Total number of standalone primitive arrays */ + public int numValueArrays; + /** + * Number of length 0, length 1, empty, long-zero-tail and unused-high-bytes primitive arrays + */ + public int numLengthZeroValueArrays, numLengthOneValueArrays; + public int numEmptyValueArrays, numLZTValueArrays, numUnusedHiBytesValueArrays; + /** Overhead caused by the above abnormal arrays */ + public long lengthZeroValueArraysOverhead, lengthOneValueArraysOverhead; + public long emptyValueArraysOverhead, lztValueArraysOverhead, unusedHiBytesValueArraysOverhead; + + /** Overhead stats (empty, sparse, etc) by Collection class */ + public ArrayList overheadsByClass; + + /** Object histogram containing various stats on memory consumption by objects */ + public ObjectHistogram objHisto; + + /** Stats for duplicate strings, both overall and detailed */ + public DupStringStats dupStringStats; + + /** Stats for compressible strings (those that use only ASCII characters) */ + public CompressibleStringStats compressibleStringStats; + + /** Stats for strings that encode numbers, e.g. ints */ + public NumberEncodingStringStats numberEncodingStringStats; + + /** Stats on length of Strings */ + public LengthHistogram stringLengthHistogram; + + /** Stats for duplicated arrays, both overall and detailed */ + public DupArrayStats dupArrayStats; + + /** Contents of java.lang.System.props table, or null if unresolved etc. */ + public HashMap systemProperties; + + // ----------------------------Setting overall stats ------------------------ + + public HeapStats setGeneralStats( + int ptrSize, int objHeaderSize, int objAlignment, boolean usingNarrowPointers, int nClasses, int nObjects, + int nInstances, int nObjectArrays, long totalObjSize, long totalInstSize, long totalObjArraySize) { + this.ptrSize = ptrSize; + this.objHeaderSize = objHeaderSize; + this.objAlignment = objAlignment; + this.usingNarrowPointers = usingNarrowPointers; + this.nClasses = nClasses; + this.nObjects = nObjects; + this.nInstances = nInstances; + this.nObjectArrays = nObjectArrays; + this.nValueArrays = nObjects - nInstances - nObjectArrays; + this.totalObjSize = totalObjSize; + this.totalInstSize = totalInstSize; + this.totalObjArraySize = totalObjArraySize; + this.totalValueArraySize = totalObjSize - totalInstSize - totalObjArraySize; + return this; + } + + public HeapStats setObjOverheadStats(long ovhdObjHeaders, int nEntryInstances, long entryClassSize) { + this.ovhdObjHeaders = ovhdObjHeaders; + this.nEntryInstances = nEntryInstances; + this.entryClassSize = entryClassSize; + return this; + } + + public HeapStats setClassloaderStats(ClassloaderStats clStats) { + this.classloaderStats = clStats; + return this; + } + + public HeapStats setShortObjArrayStats(ShortArrayStats shortArrayStats) { + this.shortObjArrayStats = shortArrayStats; + return this; + } + + public HeapStats setShortPrimitiveArrayStats(ShortArrayStats shortArrayStats) { + this.shortPrimitiveArrayStats = shortArrayStats; + return this; + } + + public HeapStats setShortStringStats(ShortArrayStats shortStringStats) { + this.shortStringStats = shortStringStats; + return this; + } + + public HeapStats setBoxedNumberStats(int nBoxedNumbers, long ovhdBoxedNumbers) { + this.nBoxedNumbers = nBoxedNumbers; + this.ovhdBoxedNumbers = ovhdBoxedNumbers; + return this; + } + + public HeapStats setWrappedCollectionStats( + ObjectToIntMap.Entry[] unmodifiableClasses, ObjectToIntMap.Entry[] synchronizedClasses) { + this.unmodifiableClasses = unmodifiableClasses; + // Not used when printing reports in ReportFormatter. Can perhaps be removed entirely including method argument. +// this.synchronizedClasses = synchronizedClasses; + return this; + } + + public HeapStats setDupStringStats(DupStringStats dupStringStats) { + this.dupStringStats = dupStringStats; + return this; + } + + public HeapStats setCompressibleStringStats(CompressibleStringStats compressibleStringStats) { + this.compressibleStringStats = compressibleStringStats; + return this; + } + + public HeapStats setNumberEncodingStringStats(NumberEncodingStringStats nesStats) { + this.numberEncodingStringStats = nesStats; + return this; + } + + public HeapStats setStringLengthHistogram(LengthHistogram lenHisto) { + this.stringLengthHistogram = lenHisto; + return this; + } + + public HeapStats setDupArrayStats(DupArrayStats dupArrayStats) { + this.dupArrayStats = dupArrayStats; + return this; + } + + // --------------------------- Setting detailed stats ----------------------- + + public HeapStats setObjectHistogram(ObjectHistogram objHisto) { + this.objHisto = objHisto; + return this; + } + + public HeapStats setCollectionNumberStats( + int numCols, int numEmptyUnusedCols, int numEmptyUsedCols, int numEmptyCols, int numSmallCols, + int numSparseSmallCols, int numSparseLargeCols, int numBoxedNumberCols, int numBarCols) { + this.numCols = numCols; + this.numEmptyUnusedCols = numEmptyUnusedCols; + this.numEmptyUsedCols = numEmptyUsedCols; + this.numEmptyCols = numEmptyCols; + this.numSmallCols = numSmallCols; + this.numSparseSmallCols = numSparseSmallCols; + this.numSparseLargeCols = numSparseLargeCols; + this.numBoxedNumberCols = numBoxedNumberCols; + this.numBarCols = numBarCols; + return this; + } + + public HeapStats setCollectionOverhead( + long emptyUnusedColsOvhd, long emptyUsedColsOvhd, long emptyColsOvhd, long smallColsOvhd, + long sparseSmallColsOvhd, long sparseLargeColsOvhd, long boxedNumberColsOvhd, long barColsOvhd) { + this.emptyUnusedColsOverhead = emptyUnusedColsOvhd; + this.emptyUsedColsOverhead = emptyUsedColsOvhd; + this.emptyColsOverhead = emptyColsOvhd; + this.smallColsOverhead = smallColsOvhd; + this.sparseSmallColsOverhead = sparseSmallColsOvhd; + this.sparseLargeColsOverhead = sparseLargeColsOvhd; + this.boxedNumberColsOverhead = boxedNumberColsOvhd; + this.barColsOverhead = barColsOvhd; + return this; + } + + public HeapStats setObjArrayNumberStats( + int numObjArrays, int numLengthZeroObjArrays, int numLengthOneObjArrays, int numEmptyObjArrays, + int numSparseObjArrays, int numBoxedNumberArrays, int numBarObjArrays) { + this.numObjArrays = numObjArrays; + this.numLengthZeroObjArrays = numLengthZeroObjArrays; + this.numLengthOneObjArrays = numLengthOneObjArrays; + this.numEmptyObjArrays = numEmptyObjArrays; + this.numSparseObjArrays = numSparseObjArrays; + this.numBoxedNumberArrays = numBoxedNumberArrays; + this.numBarObjArrays = numBarObjArrays; + return this; + } + + public HeapStats setObjArrayOverhead( + long lengthZeroObjArraysOvhd, long lengthOneObjArraysOvhd, long emptyArraysOvhd, long sparseArraysOvhd, + long boxedNumberArraysOvhd, long barArraysOvhd) { + this.lengthZeroObjArraysOverhead = lengthZeroObjArraysOvhd; + this.lengthOneObjArraysOverhead = lengthOneObjArraysOvhd; + this.emptyObjArraysOverhead = emptyArraysOvhd; + this.sparseObjArraysOverhead = sparseArraysOvhd; + this.boxedNumberArraysOverhead = boxedNumberArraysOvhd; + this.barObjArraysOverhead = barArraysOvhd; + return this; + } + + public HeapStats setCollectionOverheadByClass(ArrayList overheadsByClass) { + this.overheadsByClass = overheadsByClass; + return this; + } + + public HeapStats setValueArrayNumberStats( + int numValueArrays, int numLengthZeroValueArrays, int numLengthOneValueArrays, int numEmptyValueArrays, + int numLZTValueArrays, int numUnusedHiBytesValueArrays) { + this.numValueArrays = numValueArrays; + this.numLengthZeroValueArrays = numLengthZeroValueArrays; + this.numLengthOneValueArrays = numLengthOneValueArrays; + this.numEmptyValueArrays = numEmptyValueArrays; + this.numLZTValueArrays = numLZTValueArrays; + this.numUnusedHiBytesValueArrays = numUnusedHiBytesValueArrays; + return this; + } + + public HeapStats setValueArrayOverhead( + long lengthZeroValueArraysOvhd, long lengthOneValueArraysOvhd, long emptyValueArraysOvhd, + long lztValueArraysOvhd, long unusedHiBytesValueArraysOvhd) { + this.lengthZeroValueArraysOverhead = lengthZeroValueArraysOvhd; + this.lengthOneValueArraysOverhead = lengthOneValueArraysOvhd; + this.emptyValueArraysOverhead = emptyValueArraysOvhd; + this.lztValueArraysOverhead = lztValueArraysOvhd; + this.unusedHiBytesValueArraysOverhead = unusedHiBytesValueArraysOvhd; + return this; + } + + public HeapStats setSystemProperties(HashMap systemProps) { + this.systemProperties = systemProps; + return this; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/NumberEncodingStringStats.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/NumberEncodingStringStats.java new file mode 100644 index 00000000..d79c16c2 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/NumberEncodingStringStats.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.support; + +/** + * Container for basic statistics about strings encoding numbers. + */ +public class NumberEncodingStringStats { + public final int nStringsEncodingInts; + public final long stringsEncodingIntsOvhd; + + public NumberEncodingStringStats(int nStringsEncodingInts, long stringsEncodingIntsOvhd) { + this.nStringsEncodingInts = nStringsEncodingInts; + this.stringsEncodingIntsOvhd = stringsEncodingIntsOvhd; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/PrimitiveArrayWrapper.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/PrimitiveArrayWrapper.java new file mode 100644 index 00000000..d202c654 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/PrimitiveArrayWrapper.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.support; + +import org.openjdk.jmc.joverflow.heap.model.JavaValueArray; + +/** + * A wrapper for primitive arrays (JavaValueArray instances), that provides fast equals() and + * hashCode() methods, allowing objects of this class to be used in maps etc. However, these methods + * use JavaValueArray.getInternalId(), and thus work only if this method returns a unique value for + * unique arrays. + */ +public class PrimitiveArrayWrapper implements Comparable { + private final JavaValueArray array; + + public PrimitiveArrayWrapper(JavaValueArray array) { + this.array = array; + } + + public JavaValueArray getArray() { + return array; + } + + @Override + public boolean equals(Object otherObj) { + if (otherObj == null || !(otherObj instanceof PrimitiveArrayWrapper)) { + return false; + } + PrimitiveArrayWrapper other = (PrimitiveArrayWrapper) otherObj; + return other.array.getInternalId() == array.getInternalId(); + } + + @Override + public int hashCode() { + return array.getInternalId(); + } + + /** + * Does not do any meaningful contents comparison; exists just to ensure stable sorting order + * when other properties, such as overhead, is the same for two arrays + */ + @Override + public int compareTo(PrimitiveArrayWrapper other) { + return other.array.getInternalId() - this.array.getInternalId(); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/ProblemRecorder.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/ProblemRecorder.java new file mode 100644 index 00000000..5b8aaf68 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/ProblemRecorder.java @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.support; + +import org.openjdk.jmc.joverflow.descriptors.CollectionInstanceDescriptor; +import org.openjdk.jmc.joverflow.heap.model.JavaLazyReadObject; +import org.openjdk.jmc.joverflow.heap.model.JavaObject; +import org.openjdk.jmc.joverflow.heap.model.JavaValueArray; +import org.openjdk.jmc.joverflow.heap.model.Snapshot; + +/** + * JOverflow core heap scanning code invokes methods of this interface when it comes across + * problematic objects such as empty collections or duplicate strings. The implementation is + * supposed to record these problems, likely in an aggregated form, and then provide the resulting + * data to the user. There is an additional method that can be called by the heap scanner to report + * a Java instance that does not have any problems. But since reporting non-problematic objects + * requires additional time and memory and may not be needed in every scenario, that method is + * called by the scanner only when it is configured to do so. + */ +public interface ProblemRecorder { + + /** + * Implementation of this method may initialize some internal data using information from the + * passed objects. + */ + public void initialize(Snapshot snapshot, HeapStats hs); + + /** + * Reports a problematic collection or object array class with the specified problem and + * overhead of specified kind, and the reference chain leading to that object from some GC root. + * An instance of CollectionClassDescriptor that is also passed to the method allows its + * implementation to find more information about the problematic collection, if needed, e.g. its + * implementation-inclusive size, number of elements, etc. + */ + public void recordProblematicCollection( + JavaLazyReadObject col, CollectionInstanceDescriptor colDesc, Constants.ProblemKind ovhdKind, int ovhd, + RefChainElement referer); + + /** + * Reports a good collection or object array, that does not have any problems, and a reference + * chain leading to it. + */ + public void recordGoodCollection( + JavaLazyReadObject col, CollectionInstanceDescriptor colDesc, RefChainElement referer); + + /** + * Reports a duplicate string with the associated overhead and reference chain leading to it + * from some GC root. If hasDupBackingCharArray is true, the backing char array is duplicated; + * otherwise there are two or more String objects pointing at the same backing char array. + */ + public void recordDuplicateString( + JavaObject strObj, String stringValue, int implInclusiveSize, int ovhd, boolean hasDupBackingCharArray, + RefChainElement referer); + + /** + * Reports a good string, that does not have any duplicates, and a reference chain leading to + * it. + */ + public void recordNonDuplicateString(JavaObject strObj, int implInclusiveSize, RefChainElement referer); + + /** + * Reports a duplicate primitive array with the associated overhead and reference chain leading + * to it from some GC root. + */ + public void recordDuplicateArray(JavaValueArray ar, int ovhd, RefChainElement referer); + + /** + * Reports a good primitive array, that does not have any duplicates, and a reference chain + * leading to it from some GC root. + */ + public void recordNonDuplicateArray(JavaValueArray ar, RefChainElement referer); + + /** + * Reports a problematic instance of WeakHashMap or its subclass, that incurs the specified + * minimum overhead due to references from values pointing back to keys. + */ + public void recordWeakHashMapWithBackRefs( + JavaObject col, CollectionInstanceDescriptor colDesc, int ovhd, String valueTypeAndFieldSample, + RefChainElement referer); + + /** + * If this method returns true for the given object, + * {@link #recordGoodInstance(JavaObject, RefChainElement)} will be called for it next. + */ + public boolean shouldRecordGoodInstance(JavaObject obj); + + /** + * Reports a good Java instance, that does not have any problems, with the reference chain + * leading to it from some GC root. Will be called by the heap scanner only if previously + * {@link #shouldRecordGoodInstance(JavaObject)} returned true for it. + *

+ * NOTE: currently objects that are good in principle, but belong to the implementation of some + * collection, such as HashMap$Entry, are not reported here. That is done in part to keep + * uniform our view of the heap, as it is generated by the core heap scanner, where + * implementation details of collections are not exposed in any way. + */ + public void recordGoodInstance(JavaObject obj, RefChainElement referer); +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/RefChainElement.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/RefChainElement.java new file mode 100644 index 00000000..3989bf90 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/RefChainElement.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.support; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; + +/** + * Represents an immutable node in a reference chain leading to a problematic object cluster. + */ +public interface RefChainElement { + + /** + * Returns a JavaClass for the main part of this reference chain element. For example, for a + * class/field element Foo.bar it returns Foo. For a compound element {HashMap} it returns + * java.util.HashMap, etc. + */ + public JavaClass getJavaClass(); + + /** + * Returns an element that refers to this one, or null if there is none, which only happens if + * this element is a GC root. + */ + public RefChainElement getReferer(); + + /** + * Returns true if contents of this object are the same as the other one's. However, parent + * RefChainElements of the two objects aren't compared. + */ + public boolean shallowEquals(Object other); + + /** + * Returns the hash code for this object, that does not take into account its parent + * RefChainElement. + */ + public int shallowHashCode(); +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/RefChainElementImpl.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/RefChainElementImpl.java new file mode 100644 index 00000000..566b6631 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/RefChainElementImpl.java @@ -0,0 +1,659 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.support; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaField; +import org.openjdk.jmc.joverflow.heap.model.Root; +import org.openjdk.jmc.joverflow.util.FastStack; +import org.openjdk.jmc.joverflow.util.IndexContainer; +import org.openjdk.jmc.joverflow.util.StringInterner; + +/** + * Concrete classes implementing RefChainElement that represent various cases. A ref chain element + * is usually a "Clazz.field" combination, but for convenience we also provide compound and array + * nodes. Compound nodes, in turn can represent collections with collapsed implementation details + * (printed as e.g. "{HashMap}"), or collapsed custom linked lists (printed as e.g. + * "{MyList.next}"). Array nodes represent aggregated references from arrays (printed as e.g. + * "Object[]"). + *

+ * When we aggregate nodes that have the same class/field etc., we don't distinguish between class + * versions, i.e. we consider two classes with the same name but different classloaders as a single + * class. This fixed policy may need to be made flexible at some point + */ +public class RefChainElementImpl { + + private static final int MAX_STATIC_CHILDREN_LIST_SIZE = 100; + + /** Creates or returns an existing element for instance data field */ + public static InstanceFieldOrLinkedList getInstanceFieldElement( + JavaClass clazz, int fieldIdx, RefChainElement refererElement) { + ElementWithChildren referer = (ElementWithChildren) refererElement; + ElementWithChildren[] children = (ElementWithChildren[]) referer.refererOrChildren; + int clazzIdx = clazz.getClassListIdx(); + String clazzName = clazz.getName(); + for (ElementWithChildren child : children) { + if (child instanceof InstanceFieldOrLinkedList) { + InstanceFieldOrLinkedList ifChild = (InstanceFieldOrLinkedList) child; + // Aggregation by class name, which treats different class versions + // (classes with same name but different loaders) as one class. + if (ifChild.isInstanceField() && ifChild.getFieldIdx() == fieldIdx + && (ifChild.getJavaClass().getClassListIdx() == clazzIdx + || ifChild.getJavaClass().getName().equals(clazzName))) { + return ifChild; + } + } + } + + InstanceFieldOrLinkedList result = new InstanceFieldOrLinkedList(clazz, fieldIdx, true); + referer.addChild(result); + return result; + } + + /** Creates or returns an existing element for static data field */ + public static StaticField getStaticFieldElement( + JavaClass clazz, int staticFieldIdx, RefChainElement refererElement) { + ElementWithChildren referer = (ElementWithChildren) refererElement; + ElementWithChildren[] children = (ElementWithChildren[]) referer.refererOrChildren; + int clazzIdx = clazz.getClassListIdx(); + String clazzName = clazz.getName(); + + // In applications with extremely large number of classes, data structures like + // a Vector of classes, where each class references some problematic object via + // a static field, we may end up with very long arrays of children. For example, + // we may have a {Vector} with a few thousand children like SomeClazz:someStaticField. + // Sequential lookup in such arrays can bring JOverflow to a crawl. Thus after a + // certain threshold we switch to a hashtable representation of children. Since we + // don't carry hashtable size, we have to take special measures during lookup and + // element adding. + int nChildren = children.length; + if (nChildren <= MAX_STATIC_CHILDREN_LIST_SIZE) { + // Unusual loop below is a performance optimization implemented after profiling. + // In general, an element that we need is likely to have been added more recently, + // so backward search is likely to hit it faster. + for (int i = children.length - 1; i >= 0; i--) { + ElementWithChildren child = children[i]; + if (child instanceof StaticField) { + StaticField sfChild = (StaticField) child; + // Aggregation by class name, no version distinguishing yet. + // Looks like order of sub-statements below is important performance-wise: + // cheaper checks should be done first. + if (sfChild.getFieldIdx() == staticFieldIdx && (sfChild.getJavaClass().getClassListIdx() == clazzIdx + || sfChild.getJavaClass().getName().equals(clazzName))) { + return sfChild; + } + } + } + } else { // Perform hash lookup + // hash below should match AbstractElement. and AbstractField.shallowHashCode()! + int hash = (clazz.getName().hashCode() << 4) + staticFieldIdx; + int pos = (hash & 0x7FFFFFFF) % nChildren; + int nIters = 0; + while (children[pos] != null && nIters++ < nChildren) { + ElementWithChildren child = children[pos]; + if (child instanceof StaticField) { + StaticField sfChild = (StaticField) child; + if (sfChild.getFieldIdx() == staticFieldIdx && (sfChild.getJavaClass().getClassListIdx() == clazzIdx + || sfChild.getJavaClass().getName().equals(clazzName))) { + return sfChild; + } + } + pos = (pos + 1) % nChildren; + } + } + + StaticField result = new StaticField(clazz, staticFieldIdx); + + if (nChildren <= MAX_STATIC_CHILDREN_LIST_SIZE) { + referer.addChild(result); + if (((ElementWithChildren[]) referer.refererOrChildren).length > 100) { + referer.convertChildrenToHashFormat(); + } + } else { + referer.addChildToHashChildren(result); + } + + return result; + } + + /** Creates or returns an existing element for a compound collection */ + public static Collection getCompoundCollectionElement(JavaClass clazz, RefChainElement refererElement) { + ElementWithChildren referer = (ElementWithChildren) refererElement; + ElementWithChildren[] children = (ElementWithChildren[]) referer.refererOrChildren; + int clazzIdx = clazz.getClassListIdx(); + String clazzName = clazz.getName(); + for (ElementWithChildren child : children) { + if (child instanceof Collection && + // Aggregation by class name, no version distinguishing yet + (child.getJavaClass().getClassListIdx() == clazzIdx + || child.getJavaClass().getName().equals(clazzName))) { + return (Collection) child; + } + } + Collection result = new Collection(clazz); + referer.addChild(result); + return result; + } + + /** Creates or returns an existing element for compound linked list */ + public static InstanceFieldOrLinkedList getCompoundLinkedListElement( + JavaClass clazz, int fieldIdx, RefChainElement refererElement) { + ElementWithChildren referer = (ElementWithChildren) refererElement; + ElementWithChildren[] children = (ElementWithChildren[]) referer.refererOrChildren; + int clazzIdx = clazz.getClassListIdx(); + for (ElementWithChildren child : children) { + if (child instanceof InstanceFieldOrLinkedList) { + InstanceFieldOrLinkedList llChild = (InstanceFieldOrLinkedList) child; + // Aggregation by class name, no version distinguishing yet + if (!llChild.isInstanceField() + && (llChild.getJavaClass().getClassListIdx() == clazzIdx + || llChild.getJavaClass().getName().equals(clazz.getName())) + && llChild.getFieldIdx() == fieldIdx) { + return llChild; + } + } + } + + InstanceFieldOrLinkedList result = new InstanceFieldOrLinkedList(clazz, fieldIdx, false); + referer.addChild(result); + return result; + } + + /** Creates or returns an existing element for a compound array */ + public static Array getCompoundArrayElement(JavaClass clazz, RefChainElement refererElement) { + ElementWithChildren referer = (ElementWithChildren) refererElement; + ElementWithChildren[] children = (ElementWithChildren[]) referer.refererOrChildren; + for (ElementWithChildren child : children) { + // Aggregation by class name, no version distinguishing yet + if (child instanceof Array && child.getJavaClass().getName().equals(clazz.getName())) { + return (Array) child; + } + } + + Array result = new Array(clazz); + referer.addChild(result); + return result; + } + + // The following methods are currently used only from org.openjdk.jmc.joverflow.batch.ExtendedField + // to create RefChainElements that are already in final form. + + /** Creates an element for instance data field in final form */ + public static InstanceFieldOrLinkedList createInstanceFieldOrLinkedListElementInFinalForm( + JavaClass clazz, int fieldIdx, RefChainElement refererElement, boolean isInstanceField) { + InstanceFieldOrLinkedList result = new InstanceFieldOrLinkedList(clazz, fieldIdx, isInstanceField); + result.setReferer(refererElement); + return result; + } + + /** Creates an element for static data field in final form */ + public static StaticField createStaticFieldElementInFinalForm( + JavaClass clazz, int staticFieldIdx, RefChainElement refererElement) { + StaticField result = new StaticField(clazz, staticFieldIdx); + result.setReferer(refererElement); + return result; + } + + /** Creates an element for a compound collection in final form */ + public static Collection createCompoundCollectionElementInFinalForm( + JavaClass clazz, RefChainElement refererElement) { + Collection result = new Collection(clazz); + result.setReferer(refererElement); + return result; + } + + /** Creates an element for a compound array in final form */ + public static Array createCompoundArrayElementInFinalForm(JavaClass clazz, RefChainElement refererElement) { + Array result = new Array(clazz); + result.setReferer(refererElement); + return result; + } + + /** + * Common abstract superclass of elements that contain Clazz and field. See concrete subclasses + * {@link org.openjdk.jmc.joverflow.support.RefChainElementImpl.InstanceFieldOrLinkedList} and + * {@link org.openjdk.jmc.joverflow.support.RefChainElementImpl.StaticField}. + */ + public static abstract class AbstractField extends AbstractElement { + protected final char fieldIdx; // char instead of int to save memory + + /** + * Returns the index of the field within the instance fields or static fields of the + * corresponding JavaClass. + */ + public int getFieldIdx() { + return fieldIdx; + } + + protected AbstractField(JavaClass clazz, int fieldIdx) { + super(clazz); + this.fieldIdx = (char) fieldIdx; + } + + @Override + public boolean shallowEquals(Object otherObj) { + if (!super.shallowEquals(otherObj)) { + return false; + } + return this.fieldIdx == ((AbstractField) otherObj).fieldIdx; + } + + @Override + public int shallowHashCode() { + return (super.shallowHashCode() << 4) + fieldIdx; + } + } + + /** + * Denotes either an instance field element of reference chain or a custom linked list (a + * repeating chain of class-field elements in aggregated form), e.g. Foo.bar, where bar is an + * instance (non-static) field defined in Foo, but possibly declared in some superclass of Foo. + */ + public static class InstanceFieldOrLinkedList extends AbstractField { + + private boolean isInstanceField; + + private InstanceFieldOrLinkedList(JavaClass clazz, int fieldIdx, boolean isInstanceField) { + super(clazz, fieldIdx); + this.isInstanceField = isInstanceField; + } + + public boolean isInstanceField() { + return isInstanceField; + } + + /** Returns the name of this field, e.g. "bar" for Foo.bar */ + public String getFieldName() { + return getJavaClass().getFieldForInstance(fieldIdx).getName(); + } + + /** + * Returns the class that declares this field, which may be either the same as returned by + * getJavaClass(), or one of its superclasses. + */ + public JavaClass getFieldDeclaringClass() { + return getJavaClass().getDeclaringClassForField(fieldIdx); + } + + public void switchToLinkedList() { + isInstanceField = false; + cachedToStringValue = null; // For debugging + } + + @Override + public String toString() { + if (cachedToStringValue == null) { + String clsName = getJavaClass().getHumanFriendlyName(); + if (isInstanceField) { + cachedToStringValue = StringInterner.internString(clsName + '.' + getFieldName()); + } else { + cachedToStringValue = StringInterner.internString( + '{' + clsName + '.' + getJavaClass().getFieldForInstance(fieldIdx).getName() + '}'); + } + } + return cachedToStringValue; + } + + @Override + public boolean shallowEquals(Object otherObj) { + if (!super.shallowEquals(otherObj)) { + return false; + } + return this.isInstanceField == ((InstanceFieldOrLinkedList) otherObj).isInstanceField; + } + + @Override + public int shallowHashCode() { + return (super.shallowHashCode() << 1) + (isInstanceField ? 1 : 0); + } + } + + /** + * Static field element of reference chain, e.g. Foo.baz, where baz is a static field declared + * in Foo. + */ + public static class StaticField extends AbstractField { + + private StaticField(JavaClass clazz, int fieldIdx) { + super(clazz, fieldIdx); + } + + @Override + public String toString() { + if (cachedToStringValue == null) { + String clsName = getJavaClass().getHumanFriendlyName(); + JavaField[] statics = getJavaClass().getStaticFields(); + cachedToStringValue = StringInterner.internString(clsName + ':' + statics[fieldIdx].getName()); + } + return cachedToStringValue; + } + } + + /** Compound element of reference chain representing a known collection */ + public static class Collection extends AbstractElement { + + private Collection(JavaClass clazz) { + super(clazz); + } + + @Override + public String toString() { + if (cachedToStringValue == null) { + String clsName = getJavaClass().getHumanFriendlyName(); + cachedToStringValue = StringInterner.internString('{' + clsName + '}'); + } + return cachedToStringValue; + } + } + + /** Compound element of reference chain representing an array */ + public static class Array extends AbstractElement { + + private Array(JavaClass clazz) { + super(clazz); + } + + @Override + public String toString() { + if (cachedToStringValue == null) { + cachedToStringValue = StringInterner.internString(getJavaClass().getHumanFriendlyName()); + } + return cachedToStringValue; + } + } + + /** + * Intermediate abstract class supporting references to children, which are needed to build + * properly aggregated reference chains during data collection. When the full graph is built, + * switchToFinalFormat() should be called for all root elements, to convert all the elements + * into the final, more economical form, where we only store the reference to the parent element + * (referer). + */ + private static abstract class ElementWithChildren implements RefChainElement { + + protected static final int INIT_CHILDREN_SIZE = 2; + + /** + * During data collection, stores an ElementWithChildren[] array. After it's done and + * switchToFinalFormat() is called for all root elements, points at the parent + * ElementWithChildren instance. + */ + protected Object refererOrChildren = new ElementWithChildren[INIT_CHILDREN_SIZE]; + + void addChild(ElementWithChildren child) { + ElementWithChildren[] children = (ElementWithChildren[]) refererOrChildren; + int curArraySize = children.length; + if (children[curArraySize - 1] != null) { + ElementWithChildren[] oldChildren = children; + children = new ElementWithChildren[curArraySize + 4]; + System.arraycopy(oldChildren, 0, children, 0, curArraySize); + children[curArraySize] = child; + refererOrChildren = children; + } else { + int idx = curArraySize - 1; + while (idx >= 0 && children[idx] == null) { + idx--; + } + children[idx + 1] = child; + } + } + + void setReferer(RefChainElement referer) { + this.refererOrChildren = referer; + } + + /** + * Converts the internal representation from the intermediate "element-children" format into + * the final "element-parent" format. We use a loop and internal stack to avoid the (easier + * to write and understand) recursion, which can lead to StackOverflowError. + */ + static void switchSubtreeToFinalFormat(ElementWithChildren rootElement) { + FastStack stack = new FastStack<>(80); + stack.push(rootElement); + stack.push(rootElement.refererOrChildren); + stack.push(new IndexContainer()); + + while (!stack.isEmpty()) { + ElementWithChildren parent = (ElementWithChildren) stack.peek(2); + ElementWithChildren[] children = (ElementWithChildren[]) stack.peek(1); + IndexContainer index = (IndexContainer) stack.peek(); + int nextIdx = index.incrementAndGet(); + while (nextIdx < children.length && children[nextIdx] == null) { + nextIdx = index.incrementAndGet(); + } + if (nextIdx < children.length) { + ElementWithChildren child = children[nextIdx]; + ElementWithChildren[] childsChildren = (ElementWithChildren[]) child.refererOrChildren; + if (childsChildren[0] != null || childsChildren.length > INIT_CHILDREN_SIZE) { + // Optimization to avoid pushing arrays that are guaranteed to be empty + stack.push(child); + stack.push(child.refererOrChildren); + stack.push(new IndexContainer()); + } + child.refererOrChildren = parent; + } else { + stack.pop(3); + } + } + } + + void convertChildrenToHashFormat() { + ElementWithChildren[] oldChildren = (ElementWithChildren[]) refererOrChildren; + int capacity = (oldChildren.length * 2) | 1; + createTable(oldChildren, capacity); + } + + void addChildToHashChildren(ElementWithChildren child) { + ElementWithChildren[] children = (ElementWithChildren[]) refererOrChildren; + int capacity = children.length; + int pos = (child.shallowHashCode() & 0x7FFFFFFF) % capacity; + // We don't store table size, so use "bad table behavior" as a signal that + // it needs rehashing + int seqLen = 0; + while (children[pos] != null) { + seqLen++; + if (seqLen > capacity / 8) { + createTable(children, (capacity * 3 / 2) | 1); + addChildToHashChildren(child); + return; + } + pos = (pos + 1) % capacity; + } + children[pos] = child; + } + + void createTable(ElementWithChildren[] oldChildren, int capacity) { + ElementWithChildren[] children = new ElementWithChildren[capacity]; + for (ElementWithChildren child : oldChildren) { + if (child == null) { + continue; + } + int pos = (child.shallowHashCode() & 0x7FFFFFFF) % capacity; + while (children[pos] != null) { + pos = (pos + 1) % capacity; + } + children[pos] = child; + } + refererOrChildren = children; + } + + @Override + public abstract boolean equals(Object other); + + @Override + public abstract int hashCode(); + } + + private static abstract class AbstractElement extends ElementWithChildren { + + private final JavaClass clazz; + protected String cachedToStringValue; + + @Override + public JavaClass getJavaClass() { + return clazz; + } + + @Override + public RefChainElement getReferer() { + return (RefChainElement) refererOrChildren; + } + + private AbstractElement(JavaClass clazz) { + this.clazz = clazz; + } + + @Override + public boolean equals(Object other) { + if (!shallowEquals(other)) { + return false; + } + return referersEqual((AbstractElement) other); + } + + @Override + public int hashCode() { + return (referersHashCode() << 4) + shallowHashCode(); + } + + /** Should be overridden in subclass that defines additional data fields */ + @Override + public boolean shallowEquals(Object otherObj) { + if (otherObj == this) { + return true; + } + if (otherObj == null) { + return false; + } + if (this.getClass() != otherObj.getClass()) { + return false; + } + return (clazz.getClassListIdx() == ((AbstractElement) otherObj).clazz.getClassListIdx() + || clazz.getName().equals(((AbstractElement) otherObj).clazz.getName())); + } + + private boolean referersEqual(AbstractElement other) { + RefChainElement thisReferer = this.getReferer(); + RefChainElement otherReferer = other.getReferer(); + if (thisReferer == otherReferer) { + // Probably won't happen, but just in case + return true; + } + if (thisReferer == null || otherReferer == null) { + return false; + } + return thisReferer.equals(otherReferer); + } + + /** Should be overridden in subclass that defines additional data fields */ + @Override + public int shallowHashCode() { + return clazz.getName().hashCode(); + } + + private int referersHashCode() { + RefChainElement referer = getReferer(); + if (referer == null) { + return 0; + } + return referer.hashCode(); + } + } + + public static class GCRoot extends ElementWithChildren { + + private final Root root; + + @Override + public JavaClass getJavaClass() { + return null; + } + + @Override + public RefChainElement getReferer() { + return null; + } + + public Root getRoot() { + return root; + } + + public GCRoot(Root root) { + this.root = root; + } + + /** + * This method should be called for each GCRoot instance after heap scanning is over, i.e. + * the ref chain graph is complete, but before making any calls to + * RefChainElement.getReferer(). It walks the ref chains hanging off this root all the way + * down, and converts the internal format of ref chain elements from "element-children" into + * the final "element-parent" format. + */ + public void switchTreeToFinalFormat() { +// switchToFinalFormat(null); + switchSubtreeToFinalFormat(this); + } + + @Override + public String toString() { + return StringInterner.internString(root.getIdString()); + } + + @Override + public boolean equals(Object otherObj) { + if (otherObj == this) { + return true; + } + if (!(otherObj instanceof GCRoot)) { + return false; + } + GCRoot other = (GCRoot) otherObj; + return this.root == other.root; + } + + @Override + public int hashCode() { + return root.hashCode(); + } + + @Override + public boolean shallowEquals(Object other) { + return equals(other); + } + + @Override + public int shallowHashCode() { + return hashCode(); + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/ReferenceChain.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/ReferenceChain.java new file mode 100644 index 00000000..78a0f965 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/ReferenceChain.java @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.support; + +import java.util.ArrayList; +import java.util.List; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; + +/** + * Convenience static methods for manipulations with a reference chain going up from a + * RefChainElement instance. + */ +public class ReferenceChain { + private static final String[] EMPTY_STRING_ARRAY = new String[0]; + + /** + * Returns the full reference chain in straight order for the given RefChainElement. + */ + public static List getChain(RefChainElement referer) { + ArrayList reverseResult = new ArrayList<>(); + do { + reverseResult.add(referer); + referer = referer.getReferer(); + } while (referer != null); + if (reverseResult.size() == 1) { + return reverseResult; + } + ArrayList result = new ArrayList<>(reverseResult.size()); + for (int i = reverseResult.size() - 1; i >= 0; i--) { + result.add(reverseResult.get(i)); + } + return result; + } + + public static RefChainElement getRootElement(RefChainElement referer) { + RefChainElement parent = referer.getReferer(); + while (parent != null) { + referer = parent; + parent = referer.getReferer(); + } + return referer; + } + + public static String toStringInReverseOrder(RefChainElement referer, int maxChainDepth) { + return toStringInReverseOrder(referer, maxChainDepth, EMPTY_STRING_ARRAY); + } + + public static String toStringInReverseOrder( + RefChainElement referer, int maxChainDepth, String[] stopperClassPrefixes) { + StringBuilder sb = new StringBuilder(200); + + // First, check if we have one of the "stopper" classes in the chain + int endIdx; + RefChainElement curElement = referer; + for (endIdx = 0; curElement != null; endIdx++) { + JavaClass clazz = curElement.getJavaClass(); + if (clazz != null) { // clazz is null for root + if (startsWithOneOf(clazz.getName(), stopperClassPrefixes)) { + break; + } + } + curElement = curElement.getReferer(); + } + + if (curElement == null) { + // No stopper classes found + endIdx = maxChainDepth; + } else { + // So that the stopper is actually included + endIdx++; + } + + curElement = referer; + for (int i = 0; curElement != null && i < endIdx; i++) { + if (curElement.getReferer() != null) { + sb.append("<--"); + } else { + sb.append("<<-"); + } + sb.append(curElement.toString()); + curElement = curElement.getReferer(); + } + + if (curElement != null) { + // We haven't reached the root + sb.append("<--..."); + RefChainElement parent = curElement.getReferer(); + while (parent != null) { + curElement = parent; + parent = curElement.getReferer(); + } + sb.append("<<-"); + sb.append(((RefChainElementImpl.GCRoot) curElement).getRoot().getIdString()); + } + + return sb.toString(); + } + + public static String toStringInStraightOrder(RefChainElement referer) { + StringBuilder sb = new StringBuilder(80); + List chain = getChain(referer); + + int startIdx = 0; + if (chain.get(startIdx) instanceof RefChainElementImpl.GCRoot) { + sb.append(((RefChainElementImpl.GCRoot) chain.get(startIdx)).getRoot().getIdString()); + sb.append("->>"); + startIdx++; + } + + for (int i = startIdx; i < chain.size() - 1; i++) { + sb.append(chain.get(i).toString()); + sb.append("-->"); + } + sb.append(chain.get(chain.size() - 1).toString()); + + return sb.toString(); + } + + private static boolean startsWithOneOf(String str, String[] prefixes) { + if (prefixes.length == 0) { + return false; + } + + for (String prefix : prefixes) { + if (str.startsWith(prefix)) { + return true; + } + } + return false; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/ShortArrayStats.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/ShortArrayStats.java new file mode 100644 index 00000000..5475ad7a --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/support/ShortArrayStats.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.support; + +/** + * Container for basic statistics about short arrays or strings. + */ +public class ShortArrayStats { + /** Number of zero-length objects */ + public final int n0LenObjs; + /** Overhead of zero-length objects */ + public final long ovhd0LenObjs; + /** Number of 1-slot objects */ + public final int n1LenObjs; + /** Overhead of 1-slot objects */ + public final long ovhd1LenObjs; + /** Number of 1..4-slot objects */ + public final int n4LenObjs; + /** Overhead of 1..4-slot objects */ + public final long ovhd4LenObjs; + /** Number of 4..8-slot objects */ + public final int n8LenObjs; + /** Overhead of 4..8-slot objects */ + public final long ovhd8LenObjs; + + public ShortArrayStats(int n0LenObjs, long ovhd0LenObjs, int n1LenObjs, long ovhd1LenObjs, int n4LenObjs, + long ovhd4LenObjs, int n8LenObjs, long ovhd8LenObjs) { + this.n0LenObjs = n0LenObjs; + this.ovhd0LenObjs = ovhd0LenObjs; + this.n1LenObjs = n1LenObjs; + this.ovhd1LenObjs = ovhd1LenObjs; + this.n4LenObjs = n4LenObjs; + this.ovhd4LenObjs = ovhd4LenObjs; + this.n8LenObjs = n8LenObjs; + this.ovhd8LenObjs = ovhd8LenObjs; + } + +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/ClassUtils.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/ClassUtils.java new file mode 100644 index 00000000..9bc56b2b --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/ClassUtils.java @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +import java.util.HashMap; + +import org.openjdk.jmc.joverflow.heap.model.JavaClass; +import org.openjdk.jmc.joverflow.heap.model.JavaField; +import org.openjdk.jmc.joverflow.support.Constants; + +/** + * Various utility methods related to analyzed Java classes/objects. + */ +public class ClassUtils implements Constants { + + private static final HashMap POPULAR_CLASS_SHORT_NAMES = new HashMap<>(); + private static final String[] POPULAR_PACKAGES = new String[] {"java.lang.", "java.util.", "java.util.concurrent.", + "java.lang.ref."}; + + /** + * Some fields may be called differently in different implementations of Java libraries. For + * example, ArrayList.elementData in the standard JDK vs. ArrayList.array in Android. Given a + * string like "elementData|array", this method returns the name of the field that exists in the + * given class in this heap dump. + */ + public static String getExactFieldName(String oneOrMoreFieldNames, JavaClass clazz) { + int splitIdx = oneOrMoreFieldNames.indexOf('|'); + if (splitIdx != -1) { + int startIdx = 0; + while (true) { + String fieldName = oneOrMoreFieldNames.substring(startIdx, splitIdx); + if (clazz.getDeclaringClassForField(fieldName) != null) { + return fieldName; + } + startIdx = splitIdx + 1; + if (startIdx >= oneOrMoreFieldNames.length()) { + break; + } + splitIdx = oneOrMoreFieldNames.indexOf('|', startIdx); + if (splitIdx == -1) { + splitIdx = oneOrMoreFieldNames.length(); + } + } + throw new RuntimeException(ClassUtils.getMessageForMissingField(clazz, oneOrMoreFieldNames)); + } else { + if (clazz.getDeclaringClassForField(oneOrMoreFieldNames) == null) { + throw new RuntimeException(ClassUtils.getMessageForMissingField(clazz, oneOrMoreFieldNames)); + } + return oneOrMoreFieldNames; + } + } + + public static String getMessageForMissingField(JavaClass clazz, String fieldName) { + JavaField fieldDescs[] = clazz.getFieldsForInstance(); + StringBuilder msg = new StringBuilder(); + msg.append(clazz.getName()).append(": field ").append(fieldName).append(" not found.\n"); + msg.append("Existing fields:\n"); + for (JavaField fieldDesc : fieldDescs) { + msg.append(fieldDesc.getTypeId()).append(' ').append(fieldDesc.getName()).append('\n'); + } + return msg.toString(); + } + + /** + * If the given class name is "popular" (it belongs to one of POPULAR_PACKAGES above), returns + * the short class name. Otherwise, returns the class name unchanged. + */ + public static String getShortNameForPopularClass(String className) { + String shortName = POPULAR_CLASS_SHORT_NAMES.get(className); + if (shortName != null) { + return shortName; + } + + for (String pkg : POPULAR_PACKAGES) { + if (!className.startsWith(pkg)) { + continue; + } + int lastDotIdx = className.lastIndexOf('.'); + if (lastDotIdx != pkg.length() - 1) { + continue; + } + shortName = className.substring(lastDotIdx + 1); + POPULAR_CLASS_SHORT_NAMES.put(className, shortName); + return shortName; + } + + return className; + } + + public static boolean isAnonymousInnerClass(String className) { + if (!Character.isDigit(className.charAt(className.length() - 1))) { + return false; + } + int dollarIdx = className.lastIndexOf('$'); + if (dollarIdx == -1) { + return false; + } + return (Character.isDigit(className.charAt(dollarIdx + 1))); + } + + public static String arrayOf(String className) { + return "[L" + className + ';'; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/FastStack.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/FastStack.java new file mode 100644 index 00000000..c01c236e --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/FastStack.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +import java.util.ArrayList; + +/** + * An implementation of stack that's faster than java.util.Stack, since it's based on ArrayList, and + * thus its methods are not synchronized. + */ +public class FastStack extends ArrayList { + private static final long serialVersionUID = -5206255737294693344L; + + public FastStack() { + super(); + } + + public FastStack(int initialCapacity) { + super(initialCapacity); + } + + public E pop() { + return remove(size() - 1); + } + + public void push(E e) { + add(e); + } + + public E peek() { + return get(size() - 1); + } + + public E peek(int idxFromBack) { + return get(size() - 1 - idxFromBack); + } + + public void pop(int numElsToPop) { + super.removeRange(size() - numElsToPop, size()); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/FileUtils.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/FileUtils.java new file mode 100644 index 00000000..88634e64 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/FileUtils.java @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; + +/** + * Simple file-related utilities. + */ +public class FileUtils { + + public static ArrayList readTextFile(String fileName) throws IOException { + return readTextFile(new File(fileName)); + } + + public static ArrayList readTextFile(File file) throws IOException { + FileReader reader = new FileReader(file); + BufferedReader br = new BufferedReader(reader); + + ArrayList lines = new ArrayList<>(); + String s; + try { + while ((s = br.readLine()) != null) { + lines.add(s); + } + } finally { + br.close(); + } + + return lines; + } + + public static byte[] readBytesFromFile(String fileName) throws IOException { + return readBytesFromFile(new File(fileName)); + } + + public static byte[] readBytesFromFile(File file) throws IOException { + long longSize = file.length(); + if (longSize > Integer.MAX_VALUE) { + throw new IOException( + "File length is " + longSize + ". Cannot read files longer than " + Integer.MAX_VALUE); + } + int size = (int) longSize; + byte[] result = new byte[size]; + + BufferedInputStream in = new BufferedInputStream(new FileInputStream(file)); + try { + int readBytes = 0; + while (readBytes < size) { + readBytes += in.read(result, readBytes, size - readBytes); + } + } finally { + in.close(); + } + + return result; + } + + public static void writeTextToFile(File file, List lines) throws IOException { + PrintStream out = new PrintStream(new BufferedOutputStream(new FileOutputStream(file))); + for (String line : lines) { + out.println(line); + } + out.close(); + } + + public static void writeBytesToFile(File file, byte[] bytes) throws IOException { + try (BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(file))) { + out.write(bytes); + } + } + + public static File fileExistsAndReadableOrExit(String fileName) { + try { + return checkFileExistsAndReadable(fileName, false); + } catch (IOException ex) { + System.err.println(ex.getMessage()); + System.exit(-1); + } + return null; // Never reached; just makes the compiler happy + } + + public static File dirExistsAndReadableOrExit(String dirName) { + try { + return checkFileExistsAndReadable(dirName, true); + } catch (IOException ex) { + System.err.println(ex.getMessage()); + System.exit(-1); + } + return null; // Never reached; just makes the compiler happy + } + + public static File checkFileExistsAndReadable(String fileName, boolean isDirectory) throws IOException { + File file = new File(fileName); + if (!file.exists()) { + throw new IOException("File " + fileName + " does not exist"); + } + + if (isDirectory) { + if (!file.isDirectory()) { + throw new IOException(fileName + " is not a directory"); + } + } else if (!file.isFile()) { + throw new IOException("File " + fileName + " is not a normal file"); + } + + if (!file.canRead()) { + throw new IOException("File " + fileName + " cannot be read"); + } + return file; + } + + public static File dirExistsAndWritableOrExit(String dirName) { + File dir = new File(dirName); + if (!dir.exists()) { + System.err.println("Directory " + dirName + " does not exist"); + System.exit(-1); + } + + if (!dir.isDirectory()) { + System.err.println("File " + dirName + " is not a directory"); + System.exit(-1); + } + + if (!dir.canWrite()) { + System.err.println("Directory " + dirName + " is not writable"); + System.exit(-1); + } + + return dir; + } + + public static File fileWritableOrExit(String fileName) { + File f = new File(fileName); + + if (f.exists() && !f.isFile()) { + System.err.println("File " + fileName + " is not a normal file"); + System.exit(-1); + } + + if (f.exists() && !f.canWrite()) { + System.err.println("File " + fileName + " is not writable"); + System.exit(-1); + } + + try { + FileOutputStream fo = new FileOutputStream(f); + fo.close(); + } catch (IOException ex) { + System.err.println("Cannot write to file " + fileName + ": " + ex.getMessage()); + System.exit(-1); + } + + return f; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/IndexContainer.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/IndexContainer.java new file mode 100644 index 00000000..53e00617 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/IndexContainer.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +/** + * A wrapper for int number, that supports changing the number. + */ +public class IndexContainer { + + private int index = -1; + + public void set(int index) { + this.index = index; + } + + public int get() { + return index; + } + + public int incrementAndGet() { + index++; + return index; + } + + @Override + public String toString() { + return Integer.toString(index); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/IntArrayList.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/IntArrayList.java new file mode 100644 index 00000000..a05f7d79 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/IntArrayList.java @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * A simple growable array of ints. + */ +public class IntArrayList implements Iterable { + private static final int[] EMPTY_ARRAY = new int[0]; + + private int[] array; + private int size; + + public IntArrayList(int capacity) { + array = new int[capacity]; + } + + public int get(int idx) { + return array[idx]; + } + + public int size() { + return size; + } + + public void add(int value) { + if (size == array.length) { + int[] oldArray = array; + array = new int[oldArray.length * 2]; + System.arraycopy(oldArray, 0, array, 0, oldArray.length); + } + array[size++] = value; + } + + public boolean contains(int v) { + for (int i = 0; i < size; i++) { + if (array[i] == v) { + return true; + } + } + return false; + } + + /** + * Returns the internal array, with size unchanged. Useful for quick inspection of contents, if + * we don't want to store them long-term. + */ + public int[] internalArray() { + return array; + } + + /** Returns a copy of the contents of the internal array, with exact size. */ + public int[] toArray() { + if (size == 0) { + return EMPTY_ARRAY; + } + + int[] result = new int[size]; + System.arraycopy(array, 0, result, 0, size); + return result; + } + + public boolean isEmpty() { + return size == 0; + } + + public void clear() { + size = 0; + } + + @Override + public IntArrayListIterator iterator() { + return new IntArrayListIterator(this); + } + + /** Iterator for iterating over an IntArrayList */ + public static class IntArrayListIterator implements Iterator { + private final IntArrayList list; + private int index; + + public IntArrayListIterator(IntArrayList setList) { + this.list = setList; + index = 0; + } + + @Override + public boolean hasNext() { + return index >= 0 && index < list.size(); + } + + @Override + public Integer next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return list.get(index++); + } + + /** Unsupported on IntArrayList */ + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/IntToIntMap.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/IntToIntMap.java new file mode 100644 index 00000000..c8a588b6 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/IntToIntMap.java @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +/** + * A simple, low-memory-overhead hash map whose main usage is mapping object IDs to int numbers. The + * map does not support negative values, and returns -1 to signal that the value for the given key + * does not exist. + *

+ * Object IDs passed to this map as keys are long numbers that always have only 32 meaningful (i.e. + * changing) bits. They may represent either real 32-bit pointers, or "narrow" pointers used by the + * JVM in 64-bit mode. In the latter case, these bits may either only the lower 32-bit word of the + * long number, or 29 upper bits in the low word and the lower 3 bits in the high word. Note that + * more than 3 bits in the high word may be actually set, but only the 3 lower bits there may + * change. + *

+ * In the case of narrow pointers in 64-bit mode we have IDs that always end with 8, but may be + * longer than 8 hexadecimal places, like 0x6e07b0808 and 0x7e07b07b8. To deal with this case + * properly, we internally convert a long ID into an int by ORing its int words. Furthermore, we + * take only the lower 3 bits from the high word. + */ +public class IntToIntMap extends NumberToIntMap { + private int[] keys; + + /** + * If useOnlyLowWord is true, it means that we have case (3) described in the class-level + * javadoc. Otherwise, it's case (2). Case (1) is handled correctly regardless of the value of + * this parameter. + */ + public IntToIntMap(int expectedMaxSize) { + super(expectedMaxSize); + } + + @Override + public void put(long key, int value) { + int intKey = longKeyToIntKey(key); + doPut(intKey, value); + } + + private void doPut(int intKey, int value) { + int idx = hash(intKey); + while (values[idx] != -1) { + if (keys[idx] == intKey) { + throwCollisionException(intKey); + } + idx = nextKeyIndex(idx); + } + + keys[idx] = intKey; + values[idx] = value; + + finishPut(); + } + + @Override + public int get(long key) { + int intKey = longKeyToIntKey(key); + int idx = hash(intKey); + while (values[idx] != -1) { + if (keys[idx] == intKey) { + return values[idx]; + } + idx = nextKeyIndex(idx); + } + return -1; + } + + private int longKeyToIntKey(long key) { + // Preserve all of the meaningful 32 bits in the long object id + return ((int) key) | (((int) (key >> 32)) & 7); + } + + private int hash(int h) { + /* + * Looks like it helps a bit to do some hash randomization similar to what's done in + * java.util.HashMap.hash(). But apparently one has to be very careful with this - doing + * this wrong will very easily cause a negative effect. + */ +// h ^= (h >>> 7) ^ (h >>> 4); + h &= 0x7FFFFFFF; + return h % capacity; + } + + @Override + protected void rehash(int newCapacity) { + long time = System.currentTimeMillis(); + int[] oldKeys = keys; + int[] oldValues = values; + + capacity = newCapacity; + createTable(); + + for (int i = 0; i < oldKeys.length; i++) { + if (oldValues[i] != -1) { + doPut(oldKeys[i], oldValues[i]); + } + } + rehashTime += System.currentTimeMillis() - time; + } + + @Override + protected void createTable() { + threshold = capacity / 4 * 3; + size = 0; + keys = new int[capacity]; + values = new int[capacity]; + for (int i = 0; i < capacity; i++) { + values[i] = -1; + } + } + + /* + * Looks like moving this code into a separate method improves performance a bit, maybe because + * inlining is made easier in the caller. + */ + private void throwCollisionException(int intKey) { + throw new RuntimeException("Collision for intKey = " + Integer.toHexString(intKey) + + ". Verify that IDs have 32 meaningful bits and/or that useOnlyLowWord was " + "set correctly."); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/IntToObjectMap.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/IntToObjectMap.java new file mode 100644 index 00000000..5c2c709f --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/IntToObjectMap.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +/** + * A simple, low-memory-overhead hash map that maps an int number to an object. If "linked" + * parameter is set to true at creation time, the objects get linked internally, so that the order + * of iteration over values becomes the same as the order in which elements have been added to the + * table. + */ +public class IntToObjectMap extends NumberToObjectMap { + private int[] keys; + + public IntToObjectMap(int initialCapacity, boolean linked) { + super(initialCapacity, linked); + } + + @Override + public void put(long key, V value) { + int intKey = (int) key; + int idx = hash(intKey) % capacity; + while (values[idx] != null) { + if (keys[idx] == intKey) { + values[idx] = value; + return; + } + idx = (idx + 1) % capacity; + } + + keys[idx] = intKey; + values[idx] = value; + + finishPut(idx); + } + + @Override + public V get(long key) { + int intKey = (int) key; + int idx = hash(intKey) % capacity; + while (values[idx] != null) { + if (keys[idx] == intKey) { + return values[idx]; + } + idx = (idx + 1) % capacity; + } + return null; + } + + private int hash(int v) { + return v & 0x7FFFFFFF; + } + + @Override + protected void rehash() { + long time = System.currentTimeMillis(); + int[] oldKeys = keys; + V[] oldValues = values; + int[] oldNextElement = nextElement; + + capacity = (capacity * 3 / 2) | 1; + createTable(); + + if (linked) { + int idx = firstElementIdx; + while (idx != -1) { + put(oldKeys[idx], oldValues[idx]); + idx = oldNextElement[idx]; + } + } else { + for (int i = 0; i < oldKeys.length; i++) { + if (oldValues[i] != null) { + put(oldKeys[i], oldValues[i]); + } + } + } + rehashTime += System.currentTimeMillis() - time; + numRehashes++; + } + + @Override + @SuppressWarnings("unchecked") // Just for the (V[]) cast + protected void createTable() { + threshold = capacity / 4 * 3; + size = 0; + keys = new int[capacity]; + values = (V[]) (new Object[capacity]); + if (linked) { + nextElement = new int[capacity]; + prevAddedElementIdx = -1; + } + } + +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/LongArrayList.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/LongArrayList.java new file mode 100644 index 00000000..055670b9 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/LongArrayList.java @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * A simple growable array of longs. + */ +public class LongArrayList implements Iterable { + private static final long[] EMPTY_ARRAY = new long[0]; + + private long[] array; + private int size; + + public LongArrayList(int capacity) { + array = new long[capacity]; + } + + public long get(int idx) { + return array[idx]; + } + + public int size() { + return size; + } + + public void add(long value) { + if (size == array.length) { + long[] oldArray = array; + array = new long[oldArray.length * 2]; + System.arraycopy(oldArray, 0, array, 0, oldArray.length); + } + array[size++] = value; + } + + public boolean contains(long v) { + for (int i = 0; i < size; i++) { + if (array[i] == v) { + return true; + } + } + return false; + } + + /** + * Returns the internal array, with size unchanged. Useful for quick inspection of contents, if + * we don't want to store them long-term. + */ + public long[] internalArray() { + return array; + } + + /** Returns a copy of the contents of the internal array, with exact size. */ + public long[] toArray() { + if (size == 0) { + return EMPTY_ARRAY; + } + + long[] result = new long[size]; + System.arraycopy(array, 0, result, 0, size); + return result; + } + + public boolean isEmpty() { + return size == 0; + } + + public void clear() { + size = 0; + } + + @Override + public LongArrayListIterator iterator() { + return new LongArrayListIterator(this); + } + + /** Iterator for iterating over an IntArrayList */ + public static class LongArrayListIterator implements Iterator { + private final LongArrayList list; + private int index; + + public LongArrayListIterator(LongArrayList list) { + this.list = list; + } + + @Override + public boolean hasNext() { + return index >= 0 && index < list.size(); + } + + @Override + public Long next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return list.get(index++); + } + + /** Unsupported on LongArrayList */ + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/LongToIntMap.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/LongToIntMap.java new file mode 100644 index 00000000..dc7ad4f9 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/LongToIntMap.java @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +/** + * A simple, low-memory-overhead hash map that maps a long number to an int number. Does not support + * negative values, and returns -1 result to signal that the value for the given key does not exist. + */ +public class LongToIntMap extends NumberToIntMap { + private long[] keys; + + public LongToIntMap(int expectedMaxSize) { + super(expectedMaxSize); + } + + @Override + public void put(long key, int value) { + int idx = hash(key); + while (values[idx] != -1) { + if (keys[idx] == key) { + values[idx] = value; + return; + } + idx = nextKeyIndex(idx); + } + + keys[idx] = key; + values[idx] = value; + + finishPut(); + } + + @Override + public int get(long key) { + int idx = hash(key); + while (values[idx] != -1) { + if (keys[idx] == key) { + return values[idx]; + } + idx = nextKeyIndex(idx); + } + return -1; + } + + private int hash(long v) { + int h = ((int) (v ^ (v >>> 32))); + // Looks like it helps a bit to do some hash randomization similar to what's + // done in java.util.HashMap.hash(). But apparently one has to be very + // careful with this - doing this wrong may easily cause a negative effect. + h ^= (h >>> 7) ^ (h >>> 4); + h &= 0x7FFFFFFF; + return h % capacity; + } + + @Override + protected void rehash(int newCapacity) { + long time = System.currentTimeMillis(); + long[] oldKeys = keys; + int[] oldValues = values; + + capacity = newCapacity; + createTable(); + + for (int i = 0; i < oldKeys.length; i++) { + if (oldValues[i] != -1) { + put(oldKeys[i], oldValues[i]); + } + } + rehashTime += System.currentTimeMillis() - time; + } + + @Override + protected void createTable() { + threshold = capacity / 4 * 3; + size = 0; + keys = new long[capacity]; + values = new int[capacity]; + for (int i = 0; i < capacity; i++) { + values[i] = -1; + } + } + +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/LongToObjectMap.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/LongToObjectMap.java new file mode 100644 index 00000000..68f87080 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/LongToObjectMap.java @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +/** + * A simple, low-memory-overhead hash map that maps a long number to an object. If "linked" + * parameter is set to true at creation time, the objects get linked internally, so that the order + * of iteration over values becomes the same as the order in which elements have been added to the + * table. + */ +public class LongToObjectMap extends NumberToObjectMap { + private long[] keys; + + public LongToObjectMap(int initialCapacity, boolean linked) { + super(initialCapacity, linked); + } + + @Override + public void put(long key, V value) { + int idx = hash(key) % capacity; + while (values[idx] != null) { + if (keys[idx] == key) { + values[idx] = value; + return; + } + idx = (idx + 1) % capacity; + } + + keys[idx] = key; + values[idx] = value; + + finishPut(idx); + } + + @Override + public V get(long key) { + int idx = hash(key) % capacity; + while (values[idx] != null) { + if (keys[idx] == key) { + return values[idx]; + } + idx = (idx + 1) % capacity; + } + return null; + } + + private int hash(long v) { + return ((int) (v ^ (v >>> 32))) & 0x7FFFFFFF; + } + + @Override + protected void rehash() { + long time = System.currentTimeMillis(); + long[] oldKeys = keys; + V[] oldValues = values; + int[] oldNextElement = nextElement; + + capacity = (capacity * 3 / 2) | 1; + createTable(); + + if (linked) { + int idx = firstElementIdx; + while (idx != -1) { + put(oldKeys[idx], oldValues[idx]); + idx = oldNextElement[idx]; + } + } else { + for (int i = 0; i < oldKeys.length; i++) { + if (oldValues[i] != null) { + put(oldKeys[i], oldValues[i]); + } + } + } + rehashTime += System.currentTimeMillis() - time; + numRehashes++; + } + + @Override + @SuppressWarnings("unchecked") // Just for the (V[]) cast + protected void createTable() { + threshold = capacity / 4 * 3; + size = 0; + keys = new long[capacity]; + values = (V[]) (new Object[capacity]); + if (linked) { + nextElement = new int[capacity]; + prevAddedElementIdx = -1; + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/MemNumFormatter.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/MemNumFormatter.java new file mode 100644 index 00000000..b1bd9526 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/MemNumFormatter.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +/** + * Functionality for pretty formatting of memory-related numbers. + */ +public class MemNumFormatter { + private final long totalSize; + + public MemNumFormatter(long totalSize) { + this.totalSize = totalSize; + } + + public String getNumInKAndPercent(long num) { + double pct = ((double) num * 100) / totalSize; + String fmt = (pct >= 10) ? "%,dK (%.1f%%)" : "%,dK (%.1f%%)"; + return String.format(fmt, num / 1024, pct); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/MiscUtils.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/MiscUtils.java new file mode 100644 index 00000000..ced45e52 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/MiscUtils.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +/** + * Miscellaneous utility methods. + */ +public class MiscUtils { + + /** + * Returns object size that's padded to the specified alignment granularity (which is usually 8 + * bytes). + */ + public static int getAlignedObjectSize(int size, int objAlignment) { + if ((size & (objAlignment - 1)) == 0) { + return size; + } else { + return (size & (~(objAlignment - 1))) + objAlignment; + } + } + + public static String toHex(long addr) { + return "0x" + Long.toHexString(addr); + } + + /** + * For a given string, replaces all occurrences of chars 10 and 13 with "\\n" and "\\r" string + * literals, and adds quotes before and after the string. Additionally, if maxLen is greater + * than zero, truncates the string to that length if it's longer, adding " ...[length xyz]" in + * the end + */ + public static String removeEndLinesAndAddQuotes(String s, int maxLen) { + if (maxLen > 0 && s.length() > maxLen) { // Don't print very long strings fully + s = s.substring(0, maxLen - 16) + " ...[length " + s.length() + ']'; + } + + if (s.indexOf('\n') == -1 && s.indexOf('\r') == -1) { + return "\"" + s + "\""; + } + + char[] dst = new char[s.length() * 2 + 2]; + dst[0] = '"'; + int dstIdx = 1; + int len = s.length(); + + for (int i = 0; i < len; i++) { + char c = s.charAt(i); + if (c == '\n') { + dst[dstIdx++] = '\\'; + dst[dstIdx++] = 'n'; + } else if (c == '\r') { + dst[dstIdx++] = '\\'; + dst[dstIdx++] = 'r'; + } else { + dst[dstIdx++] = c; + } + } + + dst[dstIdx++] = '"'; + + return new String(dst, 0, dstIdx); + } + + public static String asCommaSeparatedList(String[] strings) { + if (strings.length == 1) { + return strings[0]; + } + + StringBuilder sb = new StringBuilder(strings.length * 10); + sb.append(strings[0]); + for (int i = 1; i < strings.length; i++) { + sb.append(", "); + sb.append(strings[i]); + } + return sb.toString(); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/NumberToIntMap.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/NumberToIntMap.java new file mode 100644 index 00000000..758d4bcf --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/NumberToIntMap.java @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +/** + * Superclass for IntToIntMap and LongToIntMap, with their common functionality. + *

+ * Note that I've tried to replace the simple implementation where the size of the map can be just + * any odd number, with a supposedly faster one, similar to java.util.HashMap, where the size is a + * power of two. However, that resulted in no noticeable performance improvement, but rather + * significant memory growth. E.g. for CRMSoa.hprof used memory with power-of-two map was 731M vs + * 623M for the simple table. It might be that in the power-of-two we indeed have fewer hash + * collisions - but it's very likely that for tables of this size increased memory usage causes + * slowdown that well offsets any improvements. And anyway, smaller memory usage is a higher + * priority for us than a few per cent speedup. + */ +public abstract class NumberToIntMap { + + protected int[] values; + + protected int size, capacity, threshold; + + protected long rehashTime; // Debugging + + protected NumberToIntMap(int expectedMaxSize) { + if (expectedMaxSize < 11) { + expectedMaxSize = 11; // Protect ourselves from stupidly small capacity + } + capacity = (4 * expectedMaxSize) / 3; + createTable(); + } + + public abstract void put(long key, int value); + + public abstract int get(long key); + + public int size() { + return size; + } + + public int capacity() { + return capacity; + } + + /** + * This method may be called once it's known that no more elements are going to be added to the + * table. It checks whether the current capacity is appropriate compared to size (no more than + * 10% larger or smaller than size * 4 / 3. If not, capacity is adjusted and table is rehashed. + */ + public void adjustCapacityIfNeeded() { + int optimalCapacity = (4 * size / 3 + 10) | 1; + int avgCapacityValue = (optimalCapacity + capacity) / 2; + if (((double) Math.abs(optimalCapacity - capacity)) / avgCapacityValue > 0.1) { + rehash(optimalCapacity); + } + } + + protected void finishPut() { + size++; + if (size > threshold) { + rehash((capacity * 3 / 2) | 1); + } + } + + protected final int nextKeyIndex(int idx) { + // It looks like avoiding '%' operation here, using 'if' instead, + // improves performance noticeably. + int nextIdx = idx + 1; + return (nextIdx < capacity ? nextIdx : 0); + } + + protected abstract void rehash(int newCapacity); + + protected abstract void createTable(); + + public long getRehashTimeMillis() { + return rehashTime; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/NumberToObjectMap.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/NumberToObjectMap.java new file mode 100644 index 00000000..8c91e266 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/NumberToObjectMap.java @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +import java.util.AbstractCollection; +import java.util.Collection; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Superclass for IntToObjectMap and LongToObjectMap, with their common functionality. + */ +public abstract class NumberToObjectMap { + + protected V[] values; + + protected int size, capacity, threshold; + + // Element linking support + protected final boolean linked; + protected int[] nextElement; + protected int firstElementIdx, prevAddedElementIdx; + + private volatile Collection valuesCollectionView = null; + + // Debugging + protected long rehashTime, numRehashes; + + protected NumberToObjectMap(int initialCapacity, boolean linked) { + if (initialCapacity < 11) { + initialCapacity = 11; // Protect ourselves from stupidly small capacity + } + capacity = initialCapacity | 1; + this.linked = linked; + createTable(); + } + + public abstract void put(long key, V value); + + public abstract V get(long key); + + public int size() { + return size; + } + + public int capacity() { + return capacity; + } + + public Collection values() { + Collection vs = valuesCollectionView; + return (vs != null ? vs : (valuesCollectionView = new Values())); + } + + protected void finishPut(int idx) { + if (linked) { + if (prevAddedElementIdx == -1) { + firstElementIdx = idx; + } else { + nextElement[prevAddedElementIdx] = idx; + nextElement[idx] = -1; + } + prevAddedElementIdx = idx; + } + + size++; + if (size > threshold) { + rehash(); + } + } + + protected abstract void rehash(); + + protected abstract void createTable(); + + public long getRehashTimeMillis() { + return rehashTime; + } + + public long getNumRehashes() { + return numRehashes; + } + + private final class Values extends AbstractCollection { + + @Override + public Iterator iterator() { + return linked ? new LinkedValueIterator() : new ValueIterator(); + } + + @Override + public int size() { + return size; + } + + @Override + public boolean contains(Object o) { + throw new UnsupportedOperationException(); + } + + @Override + public void clear() { + throw new UnsupportedOperationException(); + } + } + + private final class ValueIterator implements Iterator { + + private V next; // Next entry to return + private int index; // Current slot + + ValueIterator() { + if (size > 0) { // Advance to first entry + V[] t = values; + while (index < t.length && (next = t[index]) == null) { + index++; + } + } + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public V next() { + V v = next; + if (v == null) { + throw new NoSuchElementException(); + } + + next = null; + index++; + V[] t = values; + while (index < t.length && (next = t[index]) == null) { + index++; + } + + return v; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + } + + private final class LinkedValueIterator implements Iterator { + + private int index; // Current slot + + LinkedValueIterator() { + index = firstElementIdx; + } + + @Override + public boolean hasNext() { + return index != -1; + } + + @Override + public V next() { + if (index == -1) { + throw new NoSuchElementException(); + } + + V v = values[index]; + index = nextElement[index]; + + return v; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + } + +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/ObjectToIntMap.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/ObjectToIntMap.java new file mode 100644 index 00000000..f3031be8 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/ObjectToIntMap.java @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +import java.util.Arrays; +import java.util.Comparator; + +/** + * A simple, low-memory-overhead hash map that maps an Object to a single int value. It is intended + * to be used to count the number of instances or some other metrics associated with a given object. + */ +public class ObjectToIntMap implements Cloneable { + private K[] keys; + private int[] values; + private int size, capacity, threshold; + private long rehashTime; + + public ObjectToIntMap(int initialCapacity) { + capacity = initialCapacity | 1; + createTable(); + } + + public void put(K key, int value) { + int idx = hash(key) % capacity; + while (keys[idx] != null) { + if (keys[idx].equals(key)) { + values[idx] = value; + return; + } + idx = (idx + 1) % capacity; + } + + keys[idx] = key; + values[idx] = value; + size++; + if (size > threshold) { + rehash(); + } + } + + /** + * Returns the value for the given key. If this key is not present in the map, returns -1. + */ + public int get(K key) { + int idx = hash(key) % capacity; + while (keys[idx] != null) { + if (keys[idx].equals(key)) { + return values[idx]; + } + idx = (idx + 1) % capacity; + } + return -1; + } + + /** + * If the given key is not present in this table, this is equivalent to calling put(key, 1). + * Otherwise, takes the existing value for key and increments it by one. + */ + public void putOneOrIncrement(K key) { + int idx = hash(key) % capacity; + while (keys[idx] != null) { + if (keys[idx].equals(key)) { + values[idx]++; + return; + } + idx = (idx + 1) % capacity; + } + + keys[idx] = key; + values[idx] = 1; + size++; + if (size > threshold) { + rehash(); + } + } + + /** + * If the given key is not present in this table, this is equivalent to calling put(key, value). + * Otherwise, takes the existing value for key and increments it by the input value. + */ + public void putOrIncrementBy(K key, int value) { + int idx = hash(key) % capacity; + while (keys[idx] != null) { + if (keys[idx].equals(key)) { + values[idx] += value; + return; + } + idx = (idx + 1) % capacity; + } + + keys[idx] = key; + values[idx] = value; + size++; + if (size > threshold) { + rehash(); + } + } + + @SuppressWarnings("unchecked") + public Entry[] getEntries() { + Entry[] result = new Entry[size]; + int entryIdx = 0; + for (int i = 0; i < capacity; i++) { + if (keys[i] != null) { + result[entryIdx++] = new Entry<>(keys[i], values[i]); + } + } + + return result; + } + + public Entry[] getEntriesSortedByValueThenKey() { + Entry[] result = getEntries(); + + Arrays.sort(result, new Comparator>() { + @Override + @SuppressWarnings("unchecked") + public int compare(Entry e1, Entry e2) { + if (e1.value > e2.value) { + return -1; + } else if (e1.value < e2.value) { + return 1; + } else { + // Values are the same - try to compare keys to ensure order is stable + if (e1.key instanceof Comparable) { + return ((Comparable) e1.key).compareTo(e2.key); + } else { + return 0; + } + } + } + }); + + return result; + } + + public static class Entry { + public K key; + public int value; + + private Entry(K key, int value) { + this.key = key; + this.value = value; + } + } + + public int size() { + return size; + } + + private int hash(K key) { + return key.hashCode() & Integer.MAX_VALUE; + } + + private void rehash() { + long time = System.currentTimeMillis(); + K[] oldKeys = keys; + int[] oldValues = values; + + capacity = (capacity * 3 / 2) | 1; + createTable(); + + for (int i = 0; i < oldKeys.length; i++) { + if (oldKeys[i] != null) { + put(oldKeys[i], oldValues[i]); + } + } + rehashTime += System.currentTimeMillis() - time; + } + + @SuppressWarnings("unchecked") // For the (K[]) cast below + private void createTable() { + threshold = capacity / 4 * 3; + size = 0; + keys = (K[]) (new Object[capacity]); + values = new int[capacity]; + } + + public long getRehashTimeMillis() { + return rehashTime; + } + + @Override + @SuppressWarnings("unchecked") + public ObjectToIntMap clone() { + try { + return (ObjectToIntMap) super.clone(); + } catch (CloneNotSupportedException ex) { + throw new RuntimeException(ex); + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/Pair.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/Pair.java new file mode 100644 index 00000000..d0e6b3b7 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/Pair.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +/** + * Simple class intended to be used primarily for returning two values from a method, or to + * represent a tuple in a list of tuples. + */ +public class Pair { + private final V1 v1; + private final V2 v2; + + public Pair(V1 v1, V2 v2) { + this.v1 = v1; + this.v2 = v2; + } + + public V1 getV1() { + return v1; + } + + public V2 getV2() { + return v2; + } + + @Override + public boolean equals(Object otherObj) { + if (otherObj == null) { + return false; + } + if (!(otherObj instanceof Pair)) { + return false; + } + + @SuppressWarnings("rawtypes") + Pair other = (Pair) otherObj; + return this.v1.equals(other.v1) && this.v2.equals(other.v2); + } + + @Override + public int hashCode() { + return (v1.hashCode() << 16) + v2.hashCode(); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/ProgressMeter.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/ProgressMeter.java new file mode 100644 index 00000000..892dedb3 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/ProgressMeter.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +/** + * A simple progress meter/reporter, that periodically queries and reports the progress. A subclass + * should implement queryPercentage() and reportProgress() methods, that should return the + * percentage progress (0..100) for the current operation, and print or otherwise report it. + */ +public abstract class ProgressMeter extends Thread { + private volatile boolean stopped; + + public ProgressMeter() { + super("JOverflow Progress Meter thread"); + setDaemon(true); + } + + @Override + public void run() { + while (!stopped) { + int progressPerecentage = queryPercentage(); + reportProgress(progressPerecentage); + + try { + Thread.sleep(1000); + } catch (InterruptedException ex) { + } + } + } + + public void stopReporting() { + stopped = true; + } + + public abstract int queryPercentage(); + + public abstract void reportProgress(int progressPerecentage); +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/SetOfLongs.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/SetOfLongs.java new file mode 100644 index 00000000..1e8e4fb5 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/SetOfLongs.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +/** + * A Set containing elements of type long. The implementation is simple and doesn't support element + * removal. + */ +public class SetOfLongs { + private long[] table; + private boolean[] occupied; + private int size, capacity, threshold; + + public SetOfLongs(int initialCapacity) { + capacity = initialCapacity | 1; + createTable(); + } + + /** + * Adds a value to this set. If the value is already there, does nothing and returns false; + * otherwise returns true. + */ + public boolean add(long v) { + int idx = hash(v) % capacity; + while (occupied[idx]) { + if (table[idx] == v) { + return false; + } + idx = (idx + 1) % capacity; + } + + table[idx] = v; + occupied[idx] = true; + size++; + if (size > threshold) { + rehash(); + } + return true; + } + + public boolean contains(long v) { + int idx = hash(v) % capacity; + while (occupied[idx]) { + if (table[idx] == v) { + return true; + } + idx = (idx + 1) % capacity; + } + return false; + } + + public int size() { + return size; + } + + private int hash(long v) { + return ((int) (v ^ (v >>> 32))) & 0x7FFFFFFF; + } + + private void rehash() { + long[] oldTable = table; + boolean[] oldOccupied = occupied; + + capacity = (capacity * 3 / 2) | 1; + createTable(); + + for (int i = 0; i < oldTable.length; i++) { + if (oldOccupied[i]) { + add(oldTable[i]); + } + } + } + + private void createTable() { + threshold = capacity / 4 * 3; + size = 0; + table = new long[capacity]; + occupied = new boolean[capacity]; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/SimpleIdentitySet.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/SimpleIdentitySet.java new file mode 100644 index 00000000..d066b83d --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/SimpleIdentitySet.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +/** + * A Set of objects. The primary reason to use this is to avoid memory overhead of + * java.util.HashSet. The implementation is simple, identity-based and doesn't support object + * removal. + */ +public class SimpleIdentitySet { + private V[] table; + private int size, threshold; + private long rehashTime; + + public SimpleIdentitySet(int expectedObjNum) { + int capacity = (expectedObjNum * 4 / 3 + 3) | 1; + createTable(capacity); + } + + /** + * Adds an instance to this set. If the instance is already there, does nothing and returns + * false; otherwise returns true. + */ + public boolean add(V v) { + int idx = hash(v.hashCode()) % table.length; + V vt; + while ((vt = table[idx]) != null) { + if (vt == v) { + return false; + } + idx = (idx + 1) % table.length; + } + + table[idx] = v; + size++; + if (size > threshold) { + rehash(); + } + return true; + } + + public boolean contains(V v) { + int idx = hash(v.hashCode()) % table.length; + V vt; + while ((vt = table[idx]) != null) { + if (vt == v) { + return true; + } + idx = (idx + 1) % table.length; + } + return false; + } + + public int size() { + return size; + } + + public long getRehashTimeMillis() { + return rehashTime; + } + + private static int hash(int h) { + return h & Integer.MAX_VALUE; + } + + private void rehash() { + long time = System.currentTimeMillis(); + V[] oldTable = table; + + int capacity = (table.length * 3 / 2) | 1; + createTable(capacity); + + for (V v : oldTable) { + if (v != null) { + add(v); + } + } + rehashTime += System.currentTimeMillis() - time; + } + + @SuppressWarnings("unchecked") // Just for the (V[]) cast below + private void createTable(int capacity) { + threshold = capacity / 4 * 3; + size = 0; + table = (V[]) (new Object[capacity]); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/SmallSet.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/SmallSet.java new file mode 100644 index 00000000..7b78b89a --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/SmallSet.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +import java.util.Arrays; + +/** + * A set of objects that's expected to be small, perhaps 4 elements at most. The implementation is + * optimized for size. Removal and null elements are not supported. + */ +public class SmallSet { + Object elements[]; + + /** Creates a new set, with the default capacity of 2 */ + public SmallSet() { + elements = new Object[2]; + } + + /** Creates a new set, copying into it all the elements of the provided set */ + public SmallSet(SmallSet other) { + elements = new Object[other.elements.length]; + System.arraycopy(other.elements, 0, elements, 0, other.elements.length); + } + + /** + * Adds an element to the set, if it's not present. Returns true if there was no element with + * the same value previously in this set; false otherwise. + */ + public boolean add(V v) { + for (int i = 0; i < elements.length; i++) { + if (elements[i] == null) { + elements[i] = v; + return true; + } else if (elements[i].equals(v)) { + return false; + } + } + + Object oldElements[] = elements; + elements = new Object[oldElements.length + 2]; + System.arraycopy(oldElements, 0, elements, 0, oldElements.length); + elements[oldElements.length] = v; + return true; + } + + /** Adds all the elements from the provided small set to this set. */ + @SuppressWarnings("unchecked") // Just for the (V) cast below + public void addAll(SmallSet other) { + for (Object v : other.elements) { + if (v == null) { + break; + } + add((V) v); + } + } + + /** Returns an array containing elements of this set. */ + public T[] getElements(Class arrayClass) { + int count = elements.length; + while (count > 0 && elements[count - 1] == null) { + count--; + } + + return Arrays.copyOf(elements, count, arrayClass); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/StringInterner.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/StringInterner.java new file mode 100644 index 00000000..38ad3099 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/StringInterner.java @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +import java.lang.ref.WeakReference; +import java.util.logging.Logger; + +/** + * Functionality for custom interning (deduplicating) strings. The main API, internString() and + * internStringArrayContents(), uses our own custom interning technique, which maintains a + * relatively small, fixed-size table of strings. Unlike with String.intern(), our goal is not to + * return the same, "canonical" String instance for every string, but rather to reduce the number of + * duplicate strings (not necessarily eliminating all of them), and at the same time avoid CPU and + * memory performance penalty as much as possible. + *

+ * To reduce CPU overhead, synchronization in our table is kept to a minimum, which means that two + * threads may end up writing strings with the same value to the table one after another. Also, we + * avoid growing the table and/or finding how to evict some strings when the table is full. That can + * be done using WeakReferences, or by always replacing the old value with the new one when there is + * a hash collision for two strings with different values. This should ultimately help to prevent + * keeping around for too long strings that are not used anywhere anymore. + */ +@SuppressWarnings("unchecked") +public class StringInterner { + private final static Logger LOGGER = Logger.getLogger("org.openjdk.jmc.joverflow.util"); //$NON-NLS-1$ + + private static final WeakReference table[]; + private static int size; + private static final int threshold; + private static volatile int numCalls, numResets; + + static { + // Set maximum intern table size such that it generally does not grow over + // more than 1 per cent of max heap size, assuming a single string in it + // takes rougly 100 bytes. + int length = (int) (Runtime.getRuntime().totalMemory() / 100 / 100); + length = Integer.highestOneBit(length); // Round down to a power of 2 + table = new WeakReference[length]; + threshold = length * 3 / 4; + } + + // The implementation based on WeakReferences, that seems to work best + public static String internString(String s) { + if (s == null) { + return null; + } + + numCalls++; // A very weak form of synchronization + + int index = s.hashCode(); + index = hash(index); + index = index & (table.length - 1); + + int gapIdx = -1; + + WeakReference entry; + while ((entry = table[index]) != null) { + String cachedValue = entry.get(); + if (cachedValue == null) { + if (gapIdx == -1) { + gapIdx = index; + } + index = (index + 1) & (table.length - 1); + } else if (cachedValue.equals(s)) { + return cachedValue; + } else { + index = (index + 1) & (table.length - 1); + } + } + + if (size > threshold && gapIdx == -1) { + for (int i = 0; i < table.length; i++) { + table[i] = null; + } + size = 0; + numResets++; + } + + if (gapIdx != -1) { + index = gapIdx; + } else { + size++; + } + table[index] = new WeakReference<>(s); + return s; + } + + public static Object internStringInObjectRef(Object obj) { + if (!(obj instanceof String)) { + return obj; + } + + return internString((String) obj); + } + + public static String[] internStringArrayContents(String[] arr) { + if (arr == null) { + return null; + } + + for (int i = 0; i < arr.length; i++) { + String result = internString(arr[i]); + if (result != null) { // Very limited protection from concurrent updates + arr[i] = result; + } + } + return arr; + } + + public static Object[] internStringsInObjectArray(Object[] arr) { + if (arr == null) { + return null; + } + + for (int i = 0; i < arr.length; i++) { + Object result = internStringInObjectRef(arr[i]); + if (result != null) { // Very limited protection from concurrent updates + arr[i] = result; + } + } + return arr; + } + + public static void printInternStats() { + int numNullEntries = 0; + synchronized (table) { + for (WeakReference tableEntry : table) { + if (tableEntry == null) { + numNullEntries++; + } + } + } + LOGGER.info("Table size: " + table.length + ", null entries: " + numNullEntries); + LOGGER.info("Num calls (may be off due to overflowing): " + numCalls); + LOGGER.info("Num resets: " + numResets); + } + + /** + * Algorithm from Thomas Wang. + */ + private static int hash(int h) { + int key = (h ^ 61) ^ (h >>> 16); + key = key + (key << 3); + key = key ^ (key >>> 4); + key = key * 0x27d4eb2d; // a prime or an odd constant + key = key ^ (key >>> 15); + return key; + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/Utils.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/Utils.java new file mode 100644 index 00000000..09c8f4ac --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/Utils.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +import java.util.ArrayList; + +/** + * Assorted static utility methods. + */ +public class Utils { + + public static String[] split(String s, char c) { + ArrayList result = new ArrayList<>(); + int prevPos = 0; + for (int i = 0; i < s.length(); i++) { + if (s.charAt(i) == c) { + result.add(s.substring(prevPos, i)); + prevPos = i + 1; + } + } + result.add(s.substring(prevPos, s.length())); + return result.toArray(new String[result.size()]); + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/ValueWitIntIdMap.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/ValueWitIntIdMap.java new file mode 100644 index 00000000..d2434bdb --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/ValueWitIntIdMap.java @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +import java.util.AbstractCollection; +import java.util.Collection; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * A highly specialized hash map implementation, that takes only objects implementing ValueWithIntId + * interface, i.e. objects that store a stable, unique int ID internally. The ID should be unique at + * least within the set of objects stored in this map, because the map does not compare the objects + * themselves - only their IDs. Since the ID is already stored in the object, there is no need to + * store it in the map itself, which saves memory. + *

+ * The internal array management depends on that the size is a power of two, since the hash code is + * calculated using shift operations. + */ +public class ValueWitIntIdMap { + // Capacity MUST be power of two + private static final int MINIMUM_CAPACITY = 4; + private static final int MAXIMUM_CAPACITY = 1 << 29; + + private V[] values; + private int size, capacity, threshold; + + volatile Collection valuesCollectionView = null; + + private long rehashTime; // Debugging + + public ValueWitIntIdMap(int expectedMaxSize) { + // Compute min capacity for expectedMaxSize given a load factor of 2/3 + int minCapacity = (3 * expectedMaxSize) / 2; + + // Compute the appropriate capacity + if (minCapacity > MAXIMUM_CAPACITY || minCapacity < 0) { + capacity = MAXIMUM_CAPACITY; + } else { + capacity = MINIMUM_CAPACITY; + while (capacity < minCapacity) { + capacity <<= 1; + } + } + createTable(); + } + + public void put(V value) { + int key = value.getId(); + int idx = hash(key); + while (values[idx] != null) { + if (values[idx].getId() == key) { + values[idx] = value; + return; + } + idx = (idx + 1) % capacity; + } + + values[idx] = value; + + size++; + if (size > threshold) { + rehash(); + } + } + + public V get(int key) { + int idx = hash(key); + while (values[idx] != null) { + if (values[idx].getId() == key) { + return values[idx]; + } + idx = (idx + 1) % capacity; + } + return null; + } + + public int size() { + return size; + } + + public int capacity() { + return capacity; + } + + public Collection values() { + Collection vs = valuesCollectionView; + return (vs != null ? vs : (valuesCollectionView = new Values())); + } + + private int hash(int v) { + return (v - (v << 7)) & (capacity - 1); + } + + private void rehash() { + if (capacity == MAXIMUM_CAPACITY) { + return; + } + + long time = System.currentTimeMillis(); + V[] oldValues = values; + + capacity <<= 1; + createTable(); + + for (int i = 0; i < oldValues.length; i++) { + if (oldValues[i] != null) { + put(oldValues[i]); + } + } + + rehashTime += System.currentTimeMillis() - time; + } + + @SuppressWarnings("unchecked") // Just for the (V[]) cast below + private void createTable() { + threshold = capacity / 4 * 3; + size = 0; + values = (V[]) (new ValueWithIntId[capacity]); + } + + public long getRehashTimeMillis() { + return rehashTime; + } + + private final class Values extends AbstractCollection { + + @Override + public Iterator iterator() { + return new ValueIterator(); + } + + @Override + public int size() { + return size; + } + + @Override + public boolean contains(Object o) { + throw new UnsupportedOperationException(); + } + + @Override + public void clear() { + throw new UnsupportedOperationException(); + } + } + + private final class ValueIterator implements Iterator { + private V next; // Next entry to return + private int index; // Current slot + + ValueIterator() { + if (size > 0) { // Advance to first entry + V[] t = values; + while (index < t.length && (next = t[index]) == null) { + index++; + } + } + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public V next() { + V v = next; + if (v == null) { + throw new NoSuchElementException(); + } + + next = null; + index++; + V[] t = values; + while (index < t.length && (next = t[index]) == null) { + index++; + } + + return v; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + } +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/ValueWithIntId.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/ValueWithIntId.java new file mode 100644 index 00000000..820d3968 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/ValueWithIntId.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +/** + * An object implementing this interface should have a stable, unique int ID. "Unique" means that no + * two objects have the same ID within a set of objects that are handled together, for example + * stored in a ValueWithIntIdMap. + */ +public interface ValueWithIntId { + + /** Returns a stable, unique ID for this object. */ + public int getId(); + +} diff --git a/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/VerboseOutputCollector.java b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/VerboseOutputCollector.java new file mode 100644 index 00000000..94b11620 --- /dev/null +++ b/cryostat-core/src/main/java/org/openjdk/jmc/joverflow/util/VerboseOutputCollector.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The contents of this file are subject to the terms of either the Universal Permissive License + * v 1.0 as shown at http://oss.oracle.com/licenses/upl + * + * or the following license: + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions + * and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.openjdk.jmc.joverflow.util; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; + +/** + * This object is used to collect various kinds of data that should be printed only in verbose mode. + * However, in normal mode we still record verbose warnings, so that at least a short summary can be + * presented to the user. + */ +public class VerboseOutputCollector { + private final HashSet warningKinds; + private final ArrayList warnings; + private final ArrayList debug; + + public VerboseOutputCollector() { + warningKinds = new HashSet<>(); + warnings = new ArrayList<>(); + debug = new ArrayList<>(); + } + + public void addWarning(String warningKind, String msg) { + warnings.add("WARNING: " + warningKind + ' ' + msg); + warningKinds.add(warningKind); + } + + public void debug(String msg) { + debug.add(msg); + } + + public List getWarnings() { + return warnings; + } + + public List getWarningKinds() { + return new ArrayList<>(warningKinds); + } + + public List getDebugInfo() { + return debug; + } +}