IcebergCacheSizeEstimator.java
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.datasource.iceberg;
import org.apache.doris.datasource.NameMapping;
import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue;
import org.apache.doris.datasource.metacache.JvmSizeUtils;
import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate;
import org.apache.doris.datasource.metacache.OwnedObjectSizeEstimator;
import org.apache.avro.generic.IndexedRecord;
import org.apache.iceberg.BlobMetadata;
import org.apache.iceberg.ContentFile;
import org.apache.iceberg.DeleteFile;
import org.apache.iceberg.HasTableOperations;
import org.apache.iceberg.HistoryEntry;
import org.apache.iceberg.ManifestFile;
import org.apache.iceberg.PartitionData;
import org.apache.iceberg.PartitionField;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.PartitionStatisticsFile;
import org.apache.iceberg.Schema;
import org.apache.iceberg.Snapshot;
import org.apache.iceberg.SnapshotRef;
import org.apache.iceberg.SortField;
import org.apache.iceberg.SortOrder;
import org.apache.iceberg.StatisticsFile;
import org.apache.iceberg.StructLike;
import org.apache.iceberg.Table;
import org.apache.iceberg.TableMetadata;
import org.apache.iceberg.TableOperations;
import org.apache.iceberg.encryption.EncryptedKey;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
/** Type-specific, full construction-time estimators for Iceberg metadata cache values. */
final class IcebergCacheSizeEstimator {
private static final long INTEGER_BYTES = JvmSizeUtils.shallowSizeOf(Integer.class);
private static final long LONG_BYTES = JvmSizeUtils.shallowSizeOf(Long.class);
private static final Set<String> SUPPORTED_TABLE_OPERATIONS = new HashSet<>(Arrays.asList(
"org.apache.iceberg.StaticTableOperations",
"org.apache.iceberg.hadoop.HadoopTableOperations",
"org.apache.iceberg.jdbc.JdbcTableOperations",
"org.apache.iceberg.rest.RESTTableOperations",
"org.apache.iceberg.hive.HiveTableOperations",
"org.apache.iceberg.aws.glue.GlueTableOperations",
"software.amazon.s3tables.iceberg.S3TablesCatalogOperations",
"org.apache.doris.datasource.iceberg.dlf.DLFTableOperations"));
private IcebergCacheSizeEstimator() {
}
static MetaCacheSizeEstimate estimateTableEntry(NameMapping key, IcebergTableCacheValue value) {
Table table = value.getRetainedIcebergTable();
MetaCacheSizeEstimate tableSupport = checkSupportedTable(table, ManifestMode.ALL);
if (!tableSupport.isComplete()) {
return tableSupport;
}
MetaCacheSizeEstimate owned = OwnedObjectSizeEstimator.estimate(key, value);
if (!owned.isComplete()) {
return owned;
}
long bytes = add(owned.getBytes(), estimateTable(table, ManifestMode.ALL));
return MetaCacheSizeEstimate.complete(add(bytes,
JvmSizeUtils.shallowSizeOf(MetaCacheSizeEstimate.class)));
}
static MetaCacheSizeEstimate estimateSnapshotEntry(
IcebergSnapshotEntryKey key, IcebergSnapshotCacheValue value) {
if (value.getIcebergTable().isPresent()) {
if (!value.isCurrentSnapshotManifestsMaterialized()) {
return MetaCacheSizeEstimate.incomplete("iceberg_snapshot_manifests_not_materialized");
}
MetaCacheSizeEstimate tableSupport = checkSupportedTable(
value.getIcebergTable().get(), ManifestMode.CURRENT);
if (!tableSupport.isComplete()) {
return tableSupport;
}
}
MetaCacheSizeEstimate owned = OwnedObjectSizeEstimator.estimate(key, value);
if (!owned.isComplete()) {
return owned;
}
long tableBytes = value.getIcebergTable()
.map(table -> estimateTable(table, ManifestMode.CURRENT)).orElse(0L);
return MetaCacheSizeEstimate.complete(add(add(owned.getBytes(), tableBytes),
JvmSizeUtils.shallowSizeOf(MetaCacheSizeEstimate.class)));
}
static MetaCacheSizeEstimate estimateManifestEntry(
IcebergManifestEntryKey key, ManifestCacheValue value) {
if (!JvmSizeUtils.isVmLayoutKnown()) {
return MetaCacheSizeEstimate.incomplete("unsupported_jvm_layout");
}
String unsupportedContentFile = findUnsupportedContentFile(value);
if (unsupportedContentFile != null) {
return MetaCacheSizeEstimate.incomplete("unsupported_iceberg_content_file:"
+ unsupportedContentFile);
}
long bytes = JvmSizeUtils.shallowSizeOf(IcebergManifestEntryKey.class);
bytes = add(bytes, JvmSizeUtils.sizeOfString(key.getManifestPath()));
bytes = add(bytes, JvmSizeUtils.shallowSizeOf(ManifestCacheValue.class));
bytes = add(bytes, estimateContentFiles(value.getDataFiles()));
bytes = add(bytes, estimateContentFiles(value.getDeleteFiles()));
return MetaCacheSizeEstimate.complete(add(bytes,
JvmSizeUtils.shallowSizeOf(MetaCacheSizeEstimate.class)));
}
private static long estimateTable(Table table, ManifestMode manifestMode) {
if (table == null) {
return 0L;
}
long bytes = JvmSizeUtils.shallowSizeOf(table.getClass());
bytes = add(bytes, JvmSizeUtils.sizeOfString(table.name()));
TableOperations operations = ((HasTableOperations) table).operations();
bytes = add(bytes, JvmSizeUtils.shallowSizeOf(operations.getClass()));
// Implementations retain catalog-specific wrappers. Their FileIO/catalog service graphs
// are shared boundaries, but reserve space for small owned path and refresh state fields.
bytes = add(bytes, 4096L);
return add(bytes, estimateTableMetadata(
operations.current(), operations.io(), manifestMode));
}
private static MetaCacheSizeEstimate checkSupportedTable(
Table table, ManifestMode manifestMode) {
if (table == null) {
return MetaCacheSizeEstimate.incomplete("missing_iceberg_table");
}
if (!(table instanceof HasTableOperations)
|| !"org.apache.iceberg.BaseTable".equals(table.getClass().getName())) {
return MetaCacheSizeEstimate.incomplete("unsupported_iceberg_table:" + table.getClass().getName());
}
TableOperations operations = ((HasTableOperations) table).operations();
TableOperations serviceOperations = IcebergSnapshotCacheValue.unwrapRetainedTableOperations(operations);
if (!SUPPORTED_TABLE_OPERATIONS.contains(serviceOperations.getClass().getName())) {
return MetaCacheSizeEstimate.incomplete(
"unsupported_iceberg_table_operations:" + serviceOperations.getClass().getName());
}
TableMetadata metadata = operations.current();
if (metadata == null) {
return MetaCacheSizeEstimate.incomplete("missing_iceberg_table_metadata");
}
for (Snapshot snapshot : metadata.snapshots()) {
if (!"org.apache.iceberg.BaseSnapshot".equals(snapshot.getClass().getName())) {
return MetaCacheSizeEstimate.incomplete(
"unsupported_iceberg_snapshot:" + snapshot.getClass().getName());
}
}
for (StatisticsFile statisticsFile : metadata.statisticsFiles()) {
if (!"org.apache.iceberg.GenericStatisticsFile".equals(
statisticsFile.getClass().getName())) {
return MetaCacheSizeEstimate.incomplete(
"unsupported_iceberg_statistics_file:" + statisticsFile.getClass().getName());
}
for (BlobMetadata blob : statisticsFile.blobMetadata()) {
if (!"org.apache.iceberg.GenericBlobMetadata".equals(blob.getClass().getName())) {
return MetaCacheSizeEstimate.incomplete(
"unsupported_iceberg_blob_metadata:" + blob.getClass().getName());
}
}
}
for (PartitionStatisticsFile statisticsFile : metadata.partitionStatisticsFiles()) {
if (!"org.apache.iceberg.ImmutableGenericPartitionStatisticsFile".equals(
statisticsFile.getClass().getName())) {
return MetaCacheSizeEstimate.incomplete(
"unsupported_iceberg_partition_statistics_file:"
+ statisticsFile.getClass().getName());
}
}
for (EncryptedKey encryptedKey : metadata.encryptionKeys()) {
if (!"org.apache.iceberg.encryption.BaseEncryptedKey".equals(
encryptedKey.getClass().getName())) {
return MetaCacheSizeEstimate.incomplete(
"unsupported_iceberg_encrypted_key:" + encryptedKey.getClass().getName());
}
}
try {
FileIO io = operations.io();
long currentSnapshotId = metadata.currentSnapshot() == null
? Long.MIN_VALUE : metadata.currentSnapshot().snapshotId();
for (Snapshot snapshot : metadata.snapshots()) {
boolean mustMaterialize = manifestMode == ManifestMode.ALL
|| snapshot.manifestListLocation() == null
|| (manifestMode == ManifestMode.CURRENT
&& snapshot.snapshotId() == currentSnapshotId);
if (!mustMaterialize) {
continue;
}
if (io == null) {
return MetaCacheSizeEstimate.incomplete("missing_iceberg_file_io");
}
for (ManifestFile manifest : snapshot.dataManifests(io)) {
if (!"org.apache.iceberg.GenericManifestFile".equals(
manifest.getClass().getName())) {
return MetaCacheSizeEstimate.incomplete(
"unsupported_iceberg_manifest_file:" + manifest.getClass().getName());
}
String unsupportedState = findUnsupportedManifestFileState(manifest);
if (unsupportedState != null) {
return MetaCacheSizeEstimate.incomplete(
"unsupported_iceberg_manifest_state:" + unsupportedState);
}
}
for (ManifestFile manifest : snapshot.deleteManifests(io)) {
if (!"org.apache.iceberg.GenericManifestFile".equals(
manifest.getClass().getName())) {
return MetaCacheSizeEstimate.incomplete(
"unsupported_iceberg_manifest_file:" + manifest.getClass().getName());
}
String unsupportedState = findUnsupportedManifestFileState(manifest);
if (unsupportedState != null) {
return MetaCacheSizeEstimate.incomplete(
"unsupported_iceberg_manifest_state:" + unsupportedState);
}
}
}
} catch (RuntimeException e) {
return MetaCacheSizeEstimate.incomplete(
"iceberg_manifest_materialization_failed:" + e.getClass().getName());
}
return MetaCacheSizeEstimate.complete(1L);
}
private static long estimateTableMetadata(TableMetadata metadata, FileIO io,
ManifestMode manifestMode) {
if (metadata == null) {
return 0L;
}
long bytes = JvmSizeUtils.shallowSizeOf(metadata.getClass());
bytes = add(bytes, JvmSizeUtils.sizeOfString(metadata.metadataFileLocation()));
bytes = add(bytes, JvmSizeUtils.sizeOfString(metadata.uuid()));
bytes = add(bytes, JvmSizeUtils.sizeOfString(metadata.location()));
bytes = add(bytes, estimateStringMap(metadata.properties()));
bytes = add(bytes, estimateListStorage(metadata.schemas()));
bytes = add(bytes, estimateMapStorage(metadata.schemasById()));
for (Schema schema : metadata.schemas()) {
bytes = add(bytes, estimateSchema(schema));
}
bytes = add(bytes, multiply(metadata.schemasById().size(), INTEGER_BYTES));
bytes = add(bytes, estimateListStorage(metadata.specs()));
bytes = add(bytes, estimateMapStorage(metadata.specsById()));
for (PartitionSpec spec : metadata.specs()) {
bytes = add(bytes, estimatePartitionSpec(spec));
}
bytes = add(bytes, multiply(metadata.specsById().size(), INTEGER_BYTES));
bytes = add(bytes, estimateListStorage(metadata.sortOrders()));
bytes = add(bytes, estimateMapStorage(metadata.sortOrdersById()));
for (SortOrder order : metadata.sortOrders()) {
bytes = add(bytes, estimateSortOrder(order));
}
bytes = add(bytes, multiply(metadata.sortOrdersById().size(), INTEGER_BYTES));
List<Snapshot> snapshots = metadata.snapshots();
bytes = add(bytes, estimateListStorage(snapshots));
Set<Object> visitedManifests = java.util.Collections.newSetFromMap(new IdentityHashMap<>());
long currentSnapshotId = metadata.currentSnapshot() == null
? Long.MIN_VALUE : metadata.currentSnapshot().snapshotId();
for (Snapshot snapshot : snapshots) {
bytes = add(bytes, estimateSnapshot(snapshot));
if (manifestMode == ManifestMode.ALL
|| snapshot.manifestListLocation() == null
|| (manifestMode == ManifestMode.CURRENT
&& snapshot.snapshotId() == currentSnapshotId)) {
bytes = add(bytes, estimateSnapshotManifestLists(snapshot, io, visitedManifests));
}
}
// TableMetadata retains a long-key lookup index in addition to the snapshot list.
bytes = add(bytes, estimateIndexedMapStorage(snapshots.size()));
bytes = add(bytes, multiply(snapshots.size(), LONG_BYTES));
bytes = add(bytes, estimateHistory(metadata.snapshotLog()));
bytes = add(bytes, estimateMetadataLog(metadata.previousFiles()));
bytes = add(bytes, estimateSnapshotRefs(metadata.refs()));
bytes = add(bytes, estimateStatistics(metadata.statisticsFiles()));
bytes = add(bytes, estimatePartitionStatistics(metadata.partitionStatisticsFiles()));
bytes = add(bytes, estimateShallowList(metadata.changes()));
bytes = add(bytes, estimateEncryptionKeys(metadata.encryptionKeys()));
// Serializable supplier and its captured snapshot list are retained even after loading.
return add(bytes, JvmSizeUtils.sizeOfObjectArray(1));
}
private static long estimateSchema(Schema schema) {
// Materialize every Schema lookup index that Doris/Iceberg may fill after publication.
// Calling one name/accessor lookup initializes the complete corresponding index.
schema.idToName();
schema.identifierFieldIds();
Map<Integer, Integer> idsToReassigned = schema.idsToReassigned();
Map<Integer, Integer> idsToOriginal = schema.idsToOriginal();
if (!schema.columns().isEmpty()) {
Types.NestedField first = schema.columns().get(0);
schema.findField(first.name());
schema.caseInsensitiveFindField(first.name());
schema.findField(first.fieldId());
schema.accessorForField(first.fieldId());
}
int fieldCount = countNestedFields(schema.asStruct());
long bytes = JvmSizeUtils.shallowSizeOf(schema.getClass());
bytes = add(bytes, estimateType(schema.asStruct()));
bytes = add(bytes, estimateMapStorage(schema.getAliases()));
bytes = add(bytes, JvmSizeUtils.intArraySize(schema.identifierFieldIds().size()));
// id->field, name->id, lower-case-name->id, id->name and id->accessor.
bytes = add(bytes, multiply(5L, estimateIndexedMapStorage(fieldCount)));
bytes = add(bytes, estimateIndexedMapStorage(
idsToReassigned == null ? 0 : idsToReassigned.size()));
bytes = add(bytes, estimateIndexedMapStorage(
idsToOriginal == null ? 0 : idsToOriginal.size()));
bytes = add(bytes, JvmSizeUtils.sizeOfConservativeMapStorage(
schema.identifierFieldIds().size()));
// Accessor instances are retained as map values and may form short nested chains.
return add(bytes, multiply(fieldCount, 128L));
}
private static int countNestedFields(Type type) {
long count = 0L;
if (type.isStructType()) {
for (Types.NestedField field : type.asStructType().fields()) {
count = add(count, 1L);
count = add(count, countNestedFields(field.type()));
}
} else if (type.isListType()) {
count = add(count, 1L);
count = add(count, countNestedFields(type.asListType().elementType()));
} else if (type.isMapType()) {
count = add(count, 2L);
count = add(count, countNestedFields(type.asMapType().keyType()));
count = add(count, countNestedFields(type.asMapType().valueType()));
}
return count >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) count;
}
private static long estimateType(Type type) {
long bytes = JvmSizeUtils.shallowSizeOf(type.getClass());
if (type.isStructType()) {
List<Types.NestedField> fields = type.asStructType().fields();
bytes = add(bytes, estimateListStorage(fields));
for (Types.NestedField field : fields) {
bytes = add(bytes, estimateNestedField(field));
}
} else if (type.isListType()) {
bytes = add(bytes, estimateNestedField(type.asListType().fields().get(0)));
} else if (type.isMapType()) {
for (Types.NestedField field : type.asMapType().fields()) {
bytes = add(bytes, estimateNestedField(field));
}
}
return bytes;
}
private static long estimateNestedField(Types.NestedField field) {
long bytes = JvmSizeUtils.shallowSizeOf(field.getClass());
bytes = add(bytes, JvmSizeUtils.sizeOfString(field.name()));
bytes = add(bytes, JvmSizeUtils.sizeOfString(field.doc()));
bytes = add(bytes, estimateType(field.type()));
bytes = add(bytes, estimateDefaultLiteral(field.initialDefaultLiteral(), field.initialDefault()));
return add(bytes, estimateDefaultLiteral(field.writeDefaultLiteral(), field.writeDefault()));
}
private static long estimateDefaultLiteral(Object literal, Object value) {
if (literal == null) {
return 0L;
}
long bytes = JvmSizeUtils.shallowSizeOf(literal.getClass());
if (value instanceof String) {
return add(bytes, JvmSizeUtils.sizeOfString((String) value));
}
if (value instanceof ByteBuffer) {
return add(bytes, estimateBuffer((ByteBuffer) value));
}
if (value instanceof byte[]) {
return add(bytes, JvmSizeUtils.byteArraySize(((byte[]) value).length));
}
if (value != null) {
bytes = add(bytes, JvmSizeUtils.shallowSizeOf(value.getClass()));
bytes = add(bytes, JvmSizeUtils.sizeOfString(value.toString()));
}
return bytes;
}
private static long estimatePartitionSpec(PartitionSpec spec) {
List<PartitionField> fields = spec.fields();
Types.StructType partitionType = spec.partitionType();
Types.StructType rawPartitionType = spec.rawPartitionType();
Class<?>[] javaClasses = spec.javaClasses();
long bytes = JvmSizeUtils.shallowSizeOf(spec.getClass());
// PartitionSpec retains both the original field array and a lazy list view.
bytes = add(bytes, JvmSizeUtils.sizeOfObjectArray(fields.size()));
bytes = add(bytes, estimateListStorage(fields));
Set<Integer> sourceIds = new java.util.HashSet<>();
for (PartitionField field : fields) {
bytes = add(bytes, JvmSizeUtils.shallowSizeOf(field.getClass()));
bytes = add(bytes, JvmSizeUtils.sizeOfString(field.name()));
bytes = add(bytes, JvmSizeUtils.shallowSizeOf(field.transform().getClass()));
sourceIds.add(field.sourceId());
spec.getFieldsBySourceId(field.sourceId());
}
bytes = add(bytes, JvmSizeUtils.sizeOfObjectArray(javaClasses.length));
bytes = add(bytes, estimateType(partitionType));
bytes = add(bytes, estimateType(rawPartitionType));
// ArrayListMultimap-style source index: one hash mapping and one list per source id.
bytes = add(bytes, estimateIndexedMapStorage(sourceIds.size()));
bytes = add(bytes, multiply(sourceIds.size(),
JvmSizeUtils.shallowSizeOf(java.util.ArrayList.class)));
bytes = add(bytes, JvmSizeUtils.sizeOfObjectArray(fields.size()));
return bytes;
}
private static long estimateSortOrder(SortOrder order) {
long bytes = JvmSizeUtils.shallowSizeOf(order.getClass());
bytes = add(bytes, estimateListStorage(order.fields()));
bytes = add(bytes, JvmSizeUtils.sizeOfObjectArray(order.fields().size()));
for (SortField field : order.fields()) {
bytes = add(bytes, JvmSizeUtils.shallowSizeOf(field.getClass()));
bytes = add(bytes, JvmSizeUtils.shallowSizeOf(field.transform().getClass()));
}
return bytes;
}
private static long estimateSnapshot(Snapshot snapshot) {
long bytes = JvmSizeUtils.shallowSizeOf(snapshot.getClass());
bytes = add(bytes, snapshot.parentId() == null ? 0L : LONG_BYTES);
bytes = add(bytes, snapshot.schemaId() == null ? 0L : INTEGER_BYTES);
bytes = add(bytes, snapshot.firstRowId() == null ? 0L : LONG_BYTES);
bytes = add(bytes, snapshot.addedRows() == null ? 0L : LONG_BYTES);
bytes = add(bytes, JvmSizeUtils.sizeOfString(snapshot.operation()));
bytes = add(bytes, JvmSizeUtils.sizeOfString(snapshot.manifestListLocation()));
bytes = add(bytes, JvmSizeUtils.sizeOfString(snapshot.keyId()));
return add(bytes, estimateStringMap(snapshot.summary()));
}
private static long estimateSnapshotManifestLists(
Snapshot snapshot, FileIO io, Set<Object> visited) {
List<ManifestFile> allManifests = snapshot.allManifests(io);
List<ManifestFile> dataManifests = snapshot.dataManifests(io);
List<ManifestFile> deleteManifests = snapshot.deleteManifests(io);
long bytes = estimateListStorage(allManifests);
bytes = add(bytes, estimateListStorage(dataManifests));
bytes = add(bytes, estimateListStorage(deleteManifests));
for (ManifestFile manifest : dataManifests) {
bytes = add(bytes, estimateManifestFile(manifest, visited));
}
for (ManifestFile manifest : deleteManifests) {
bytes = add(bytes, estimateManifestFile(manifest, visited));
}
return bytes;
}
private static long estimateManifestFile(ManifestFile manifest, Set<Object> visited) {
if (!visited.add(manifest)) {
return 0L;
}
long bytes = JvmSizeUtils.shallowSizeOf(manifest.getClass());
bytes = add(bytes, JvmSizeUtils.sizeOfString(manifest.path()));
// GenericManifestFile always retains its projection index and may lazily fill length.
bytes = add(bytes, JvmSizeUtils.intArraySize(ManifestFile.schema().columns().size()));
bytes = add(bytes, LONG_BYTES);
bytes = add(bytes, manifest.snapshotId() == null ? 0L : LONG_BYTES);
bytes = add(bytes, manifest.addedFilesCount() == null ? 0L : INTEGER_BYTES);
bytes = add(bytes, manifest.existingFilesCount() == null ? 0L : INTEGER_BYTES);
bytes = add(bytes, manifest.deletedFilesCount() == null ? 0L : INTEGER_BYTES);
bytes = add(bytes, manifest.addedRowsCount() == null ? 0L : LONG_BYTES);
bytes = add(bytes, manifest.existingRowsCount() == null ? 0L : LONG_BYTES);
bytes = add(bytes, manifest.deletedRowsCount() == null ? 0L : LONG_BYTES);
bytes = add(bytes, manifest.firstRowId() == null ? 0L : LONG_BYTES);
ByteBuffer keyMetadata = manifest.keyMetadata();
if (keyMetadata != null) {
bytes = add(bytes, estimateBuffer(keyMetadata));
}
List<ManifestFile.PartitionFieldSummary> partitions = manifest.partitions();
if (partitions != null) {
// partitions() exposes the retained array as a transient Arrays.asList view; count the
// array and summaries, not the newly allocated view wrapper.
bytes = add(bytes, JvmSizeUtils.sizeOfObjectArray(partitions.size()));
for (ManifestFile.PartitionFieldSummary summary : partitions) {
bytes = add(bytes, estimatePartitionFieldSummary(summary));
}
}
// v1 manifests may retain a small InputFile wrapper. The FileIO/configuration reachable
// from it is a shared boundary; this allowance covers only the wrapper-local state.
return add(bytes, 512L);
}
private static long estimatePartitionFieldSummary(
ManifestFile.PartitionFieldSummary summary) {
long bytes = JvmSizeUtils.shallowSizeOf(summary.getClass());
bytes = add(bytes, JvmSizeUtils.intArraySize(4));
bytes = add(bytes, summary.containsNaN() == null ? 0L
: JvmSizeUtils.shallowSizeOf(Boolean.class));
ByteBuffer lower = summary.lowerBound();
ByteBuffer upper = summary.upperBound();
bytes = add(bytes, estimateBuffer(lower));
return add(bytes, estimateBuffer(upper));
}
private static long estimateHistory(List<HistoryEntry> history) {
long bytes = estimateListStorage(history);
for (HistoryEntry entry : history) {
bytes = add(bytes, JvmSizeUtils.shallowSizeOf(entry.getClass()));
}
return bytes;
}
private static long estimateMetadataLog(List<TableMetadata.MetadataLogEntry> entries) {
long bytes = estimateListStorage(entries);
for (TableMetadata.MetadataLogEntry entry : entries) {
bytes = add(bytes, JvmSizeUtils.shallowSizeOf(entry.getClass()));
bytes = add(bytes, JvmSizeUtils.sizeOfString(entry.file()));
}
return bytes;
}
private static long estimateSnapshotRefs(Map<String, SnapshotRef> refs) {
long bytes = estimateMapStorage(refs);
for (Map.Entry<String, SnapshotRef> entry : refs.entrySet()) {
bytes = add(bytes, JvmSizeUtils.sizeOfString(entry.getKey()));
SnapshotRef ref = entry.getValue();
bytes = add(bytes, JvmSizeUtils.shallowSizeOf(ref.getClass()));
bytes = add(bytes, ref.minSnapshotsToKeep() == null ? 0L : INTEGER_BYTES);
bytes = add(bytes, ref.maxSnapshotAgeMs() == null ? 0L : LONG_BYTES);
bytes = add(bytes, ref.maxRefAgeMs() == null ? 0L : LONG_BYTES);
}
return bytes;
}
private static long estimateStatistics(List<StatisticsFile> files) {
long bytes = estimateListStorage(files);
for (StatisticsFile file : files) {
bytes = add(bytes, JvmSizeUtils.shallowSizeOf(file.getClass()));
bytes = add(bytes, JvmSizeUtils.sizeOfString(file.path()));
bytes = add(bytes, estimateListStorage(file.blobMetadata()));
for (BlobMetadata blob : file.blobMetadata()) {
bytes = add(bytes, JvmSizeUtils.shallowSizeOf(blob.getClass()));
bytes = add(bytes, JvmSizeUtils.sizeOfString(blob.type()));
bytes = add(bytes, estimateListStorage(blob.fields()));
bytes = add(bytes, multiply(blob.fields().size(), INTEGER_BYTES));
bytes = add(bytes, estimateStringMap(blob.properties()));
}
}
return bytes;
}
private static long estimatePartitionStatistics(List<PartitionStatisticsFile> files) {
long bytes = estimateListStorage(files);
for (PartitionStatisticsFile file : files) {
bytes = add(bytes, JvmSizeUtils.shallowSizeOf(file.getClass()));
bytes = add(bytes, JvmSizeUtils.sizeOfString(file.path()));
}
return bytes;
}
private static long estimateEncryptionKeys(List<EncryptedKey> keys) {
long bytes = estimateListStorage(keys);
for (EncryptedKey key : keys) {
bytes = add(bytes, JvmSizeUtils.shallowSizeOf(key.getClass()));
bytes = add(bytes, JvmSizeUtils.sizeOfString(key.keyId()));
bytes = add(bytes, JvmSizeUtils.sizeOfString(key.encryptedById()));
bytes = add(bytes, estimateBuffer(key.encryptedKeyMetadata()));
bytes = add(bytes, estimateStringMap(key.properties()));
}
return bytes;
}
private static long estimateShallowList(List<?> values) {
long bytes = estimateListStorage(values);
for (Object value : values) {
bytes = add(bytes, JvmSizeUtils.shallowSizeOf(value.getClass()));
}
return bytes;
}
private static long estimateContentFiles(List<? extends ContentFile<?>> files) {
long bytes = estimateListStorage(files);
Set<Object> visited = java.util.Collections.newSetFromMap(new IdentityHashMap<>());
for (ContentFile<?> file : files) {
bytes = add(bytes, estimateContentFile(file, visited));
}
return bytes;
}
private static String findUnsupportedContentFile(ManifestCacheValue value) {
for (ContentFile<?> file : value.getDataFiles()) {
if (!"org.apache.iceberg.GenericDataFile".equals(file.getClass().getName())) {
return file.getClass().getName();
}
String unsupportedState = findUnsupportedContentFileState(file);
if (unsupportedState != null) {
return unsupportedState;
}
}
for (ContentFile<?> file : value.getDeleteFiles()) {
if (!"org.apache.iceberg.GenericDeleteFile".equals(file.getClass().getName())) {
return file.getClass().getName();
}
String unsupportedState = findUnsupportedContentFileState(file);
if (unsupportedState != null) {
return unsupportedState;
}
}
return null;
}
private static String findUnsupportedContentFileState(ContentFile<?> file) {
StructLike partition = file.partition();
if (partition != null && !isSupportedPartitionData(partition)) {
return "partition:" + partition.getClass().getName();
}
String unsupportedBuffer = findUnsupportedBuffer(file.keyMetadata());
if (unsupportedBuffer != null) {
return unsupportedBuffer;
}
unsupportedBuffer = findUnsupportedBuffer(file.lowerBounds());
return unsupportedBuffer == null ? findUnsupportedBuffer(file.upperBounds()) : unsupportedBuffer;
}
private static String findUnsupportedManifestFileState(ManifestFile manifest) {
String unsupportedBuffer = findUnsupportedBuffer(manifest.keyMetadata());
if (unsupportedBuffer != null) {
return unsupportedBuffer;
}
List<ManifestFile.PartitionFieldSummary> partitions = manifest.partitions();
if (partitions == null) {
return null;
}
for (ManifestFile.PartitionFieldSummary summary : partitions) {
unsupportedBuffer = findUnsupportedBuffer(summary.lowerBound());
if (unsupportedBuffer == null) {
unsupportedBuffer = findUnsupportedBuffer(summary.upperBound());
}
if (unsupportedBuffer != null) {
return unsupportedBuffer;
}
}
return null;
}
private static boolean isSupportedPartitionData(StructLike partition) {
return partition.getClass() == PartitionData.class
|| (partition.size() == 0
&& "org.apache.iceberg.BaseFile$1".equals(partition.getClass().getName()));
}
private static String findUnsupportedBuffer(Map<Integer, ByteBuffer> values) {
if (values == null) {
return null;
}
for (ByteBuffer value : values.values()) {
String unsupported = findUnsupportedBuffer(value);
if (unsupported != null) {
return unsupported;
}
}
return null;
}
private static String findUnsupportedBuffer(ByteBuffer value) {
return value == null || value.hasArray() ? null : "byte_buffer:" + value.getClass().getName();
}
private static long estimateContentFile(ContentFile<?> file, Set<Object> visited) {
long bytes = JvmSizeUtils.shallowSizeOf(file.getClass());
bytes = add(bytes, estimateCharSequence(file.path(), visited));
bytes = add(bytes, estimateString(file.manifestLocation(), visited));
bytes = add(bytes, estimatePartition(file.partition(), visited));
bytes = add(bytes, estimateLongMap(file.columnSizes()));
bytes = add(bytes, estimateLongMap(file.valueCounts()));
bytes = add(bytes, estimateLongMap(file.nullValueCounts()));
bytes = add(bytes, estimateLongMap(file.nanValueCounts()));
bytes = add(bytes, estimateBufferMap(file.lowerBounds()));
bytes = add(bytes, estimateBufferMap(file.upperBounds()));
bytes = add(bytes, estimateBuffer(file.keyMetadata()));
bytes = add(bytes, file.splitOffsets() == null ? 0L
: JvmSizeUtils.longArraySize(file.splitOffsets().size()));
bytes = add(bytes, file.equalityFieldIds() == null ? 0L
: JvmSizeUtils.intArraySize(file.equalityFieldIds().size()));
bytes = add(bytes, file.pos() == null ? 0L : LONG_BYTES);
bytes = add(bytes, file.sortOrderId() == null ? 0L : INTEGER_BYTES);
bytes = add(bytes, file.dataSequenceNumber() == null ? 0L : LONG_BYTES);
bytes = add(bytes, file.fileSequenceNumber() == null ? 0L : LONG_BYTES);
bytes = add(bytes, file.firstRowId() == null ? 0L : LONG_BYTES);
if (file instanceof DeleteFile) {
DeleteFile deleteFile = (DeleteFile) file;
bytes = add(bytes, estimateString(deleteFile.referencedDataFile(), visited));
bytes = add(bytes, deleteFile.contentOffset() == null ? 0L : LONG_BYTES);
bytes = add(bytes, deleteFile.contentSizeInBytes() == null ? 0L : LONG_BYTES);
}
// Force and count the exact version-specific Avro schema instead of assuming a fixed
// number of fields. Its partition projection grows with the partition type.
return add(bytes, estimateAvroSchema(((IndexedRecord) file).getSchema(), visited));
}
private static long estimatePartition(StructLike partition, Set<Object> visited) {
if (partition == null || !visited.add(partition)) {
return 0L;
}
long bytes = add(JvmSizeUtils.shallowSizeOf(partition.getClass()),
JvmSizeUtils.sizeOfObjectArray(partition.size()));
if (!(partition instanceof PartitionData)) {
// The only other admitted implementation is Iceberg's fieldless, unpartitioned
// singleton. Its exact shallow object has already been counted above.
return bytes;
}
PartitionData partitionData = (PartitionData) partition;
Type partitionType = partitionData.getPartitionType();
if (partitionType != null && visited.add(partitionType)) {
bytes = add(bytes, estimateType(partitionType));
}
bytes = add(bytes, estimateAvroSchema(partitionData.getSchema(), visited));
for (int index = 0; index < partition.size(); index++) {
Object value = partition.get(index, Object.class);
if (value instanceof CharSequence) {
bytes = add(bytes, estimateCharSequence((CharSequence) value, visited));
} else if (value instanceof ByteBuffer && visited.add(value)) {
bytes = add(bytes, estimateBuffer((ByteBuffer) value));
} else if (value != null && visited.add(value)) {
bytes = add(bytes, JvmSizeUtils.shallowSizeOf(value.getClass()));
}
}
return bytes;
}
private static long estimateLongMap(Map<Integer, Long> values) {
if (values == null) {
return 0L;
}
long bytes = add(estimateMapStorage(values), 32L);
return add(bytes, multiply(values.size(), add(INTEGER_BYTES, LONG_BYTES)));
}
private static long estimateBufferMap(Map<Integer, ByteBuffer> values) {
if (values == null) {
return 0L;
}
long bytes = add(add(estimateMapStorage(values), 32L),
multiply(values.size(), INTEGER_BYTES));
for (ByteBuffer value : values.values()) {
bytes = add(bytes, estimateBuffer(value));
}
return bytes;
}
private static long estimateBuffer(ByteBuffer value) {
if (value == null) {
return 0L;
}
long bytes = JvmSizeUtils.shallowSizeOf(value.getClass());
return value.hasArray() ? add(bytes, JvmSizeUtils.byteArraySize(value.array().length)) : bytes;
}
private static long estimateAvroSchema(org.apache.avro.Schema schema, Set<Object> visited) {
if (schema == null || !visited.add(schema)) {
return 0L;
}
long bytes = JvmSizeUtils.shallowSizeOf(schema.getClass());
switch (schema.getType()) {
case RECORD:
bytes = add(bytes, estimateAvroNamedSchema(schema));
bytes = add(bytes, estimateListStorage(schema.getFields()));
for (org.apache.avro.Schema.Field field : schema.getFields()) {
bytes = add(bytes, JvmSizeUtils.shallowSizeOf(field.getClass()));
bytes = add(bytes, JvmSizeUtils.sizeOfString(field.name()));
bytes = add(bytes, JvmSizeUtils.sizeOfString(field.doc()));
bytes = add(bytes, estimateStringSet(field.aliases()));
bytes = add(bytes, estimateAvroSchema(field.schema(), visited));
}
break;
case ARRAY:
bytes = add(bytes, estimateAvroSchema(schema.getElementType(), visited));
break;
case MAP:
bytes = add(bytes, estimateAvroSchema(schema.getValueType(), visited));
break;
case UNION:
bytes = add(bytes, estimateListStorage(schema.getTypes()));
for (org.apache.avro.Schema child : schema.getTypes()) {
bytes = add(bytes, estimateAvroSchema(child, visited));
}
break;
case ENUM:
bytes = add(bytes, estimateAvroNamedSchema(schema));
bytes = add(bytes, estimateStringList(schema.getEnumSymbols()));
break;
case FIXED:
bytes = add(bytes, estimateAvroNamedSchema(schema));
break;
default:
break;
}
return bytes;
}
private static long estimateAvroNamedSchema(org.apache.avro.Schema schema) {
long bytes = JvmSizeUtils.sizeOfString(schema.getName());
bytes = add(bytes, JvmSizeUtils.sizeOfString(schema.getDoc()));
bytes = add(bytes, JvmSizeUtils.sizeOfString(schema.getNamespace()));
return add(bytes, estimateStringSet(schema.getAliases()));
}
private static long estimateStringSet(Set<String> values) {
if (values == null) {
return 0L;
}
long bytes = add(JvmSizeUtils.shallowSizeOf(values.getClass()),
JvmSizeUtils.sizeOfConservativeMapStorage(values.size()));
for (String value : values) {
bytes = add(bytes, JvmSizeUtils.sizeOfString(value));
}
return bytes;
}
private static long estimateCharSequence(CharSequence value, Set<Object> visited) {
if (value == null || !visited.add(value)) {
return 0L;
}
return value instanceof String
? JvmSizeUtils.sizeOfString((String) value)
: add(JvmSizeUtils.shallowSizeOf(value.getClass()),
JvmSizeUtils.sizeOfString(value.toString()));
}
private static long estimateString(String value, Set<Object> visited) {
return value == null || !visited.add(value) ? 0L : JvmSizeUtils.sizeOfString(value);
}
private static long estimateStringList(List<String> values) {
return JvmSizeUtils.sizeOfStringList(values);
}
private static long estimateStringMap(Map<String, String> values) {
return JvmSizeUtils.sizeOfStringMap(values);
}
private static long estimateListStorage(List<?> values) {
return JvmSizeUtils.sizeOfConservativeListStorage(values);
}
private static long estimateMapStorage(Map<?, ?> values) {
return JvmSizeUtils.sizeOfConservativeMap(values);
}
private static long estimateIndexedMapStorage(int size) {
return JvmSizeUtils.sizeOfConservativeIndexMap(size);
}
private static long multiply(long left, long right) {
return JvmSizeUtils.saturatedMultiply(left, right);
}
private static long add(long left, long right) {
return JvmSizeUtils.saturatedAdd(left, right);
}
private enum ManifestMode {
CURRENT,
ALL
}
}