Skip to content

Commit

Permalink
Revert "add actural size of memory of memtable"
Browse files Browse the repository at this point in the history
This reverts commit 9f974e9.
  • Loading branch information
SpriCoder committed Feb 4, 2025
1 parent a10db70 commit f94a60d
Show file tree
Hide file tree
Showing 3 changed files with 24 additions and 61 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -107,17 +107,6 @@ public void bindTo(AbstractMetricService metricService) {
Tag.LEVEL.toString(),
GlobalMemoryMetrics.LEVELS[3])
.set(config.getMemtableMemoryManager().getTotalMemorySizeInBytes());
metricService.createAutoGauge(
Metric.MEMORY_ACTUAL_SIZE.toString(),
MetricLevel.NORMAL,
config.getMemtableMemoryManager(),
MemoryManager::getUsedMemorySizeInBytes,
Tag.NAME.toString(),
STORAGE_ENGINE_WRITE_MEMTABLE,
Tag.TYPE.toString(),
GlobalMemoryMetrics.ON_HEAP,
Tag.LEVEL.toString(),
GlobalMemoryMetrics.LEVELS[3]);
metricService
.getOrCreateGauge(
Metric.MEMORY_THRESHOLD_SIZE.toString(),
Expand Down Expand Up @@ -201,26 +190,16 @@ public void unbindFrom(AbstractMetricService metricService) {
});
Arrays.asList(STORAGE_ENGINE_WRITE_MEMTABLE, STORAGE_ENGINE_WRITE_TIME_PARTITION_INFO)
.forEach(
name -> {
metricService.remove(
MetricType.GAUGE,
Metric.MEMORY_THRESHOLD_SIZE.toString(),
Tag.NAME.toString(),
name,
Tag.TYPE.toString(),
GlobalMemoryMetrics.ON_HEAP,
Tag.LEVEL.toString(),
GlobalMemoryMetrics.LEVELS[3]);
metricService.remove(
MetricType.AUTO_GAUGE,
Metric.MEMORY_ACTUAL_SIZE.toString(),
Tag.NAME.toString(),
name,
Tag.TYPE.toString(),
GlobalMemoryMetrics.ON_HEAP,
Tag.LEVEL.toString(),
GlobalMemoryMetrics.LEVELS[3]);
});
name ->
metricService.remove(
MetricType.GAUGE,
Metric.MEMORY_THRESHOLD_SIZE.toString(),
Tag.NAME.toString(),
name,
Tag.TYPE.toString(),
GlobalMemoryMetrics.ON_HEAP,
Tag.LEVEL.toString(),
GlobalMemoryMetrics.LEVELS[3]));
Arrays.asList(
STORAGE_ENGINE_WRITE_MEMTABLE_DEVICE_PATH_CACHE,
STORAGE_ENGINE_WRITE_MEMTABLE_BUFFERED_ARRAYS,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public class SystemInfo {
private long totalStorageGroupMemCost = 0L;
private volatile boolean rejected = false;

private MemoryBlock memtableMemoryBlock;
private long memorySizeForMemtable;
private long memorySizeForWalBufferQueue;
private final Map<DataRegionInfo, Long> reportedStorageGroupMemCostMap = new HashMap<>();

Expand All @@ -71,8 +71,8 @@ public class SystemInfo {

private final ExecutorService flushTaskSubmitThreadPool =
IoTDBThreadPoolFactory.newSingleThreadExecutor(ThreadName.FLUSH_TASK_SUBMIT.getName());
private double FLUSH_THRESHOLD;
private double REJECT_THRESHOLD;
private double FLUSH_THRESHOLD = memorySizeForMemtable * config.getFlushProportion();
private double REJECT_THRESHOLD = memorySizeForMemtable * config.getRejectProportion();

private volatile boolean isEncodingFasterThanIo = true;

Expand Down Expand Up @@ -122,14 +122,14 @@ public synchronized boolean reportStorageGroupStatus(
REJECT_THRESHOLD);
rejected = true;
if (chooseMemTablesToMarkFlush(tsFileProcessor)) {
if (totalStorageGroupMemCost < memtableMemoryBlock.getMaxMemorySizeInByte()) {
if (totalStorageGroupMemCost < memorySizeForMemtable) {
return true;
} else {
throw new WriteProcessRejectException(
"Total database MemCost "
+ totalStorageGroupMemCost
+ " is over than memorySizeForWriting "
+ memtableMemoryBlock.getMaxMemorySizeInByte());
+ memorySizeForMemtable);
}
} else {
return false;
Expand Down Expand Up @@ -337,15 +337,14 @@ public long getMemorySizeForCompaction() {
}

public void allocateWriteMemory() {
memtableMemoryBlock =
config.getMemtableMemoryManager().forceAllocate("Total", MemoryBlockType.FUNCTION);
memorySizeForMemtable = config.getMemtableMemoryManager().getTotalMemorySizeInBytes();
compactionMemoryBlock =
config.getCompactionMemoryManager().forceAllocate("Total", MemoryBlockType.FUNCTION);
memorySizeForWalBufferQueue = config.getWalBufferQueueManager().getTotalMemorySizeInBytes();
directBufferMemoryBlock =
config.getDirectBufferMemoryManager().forceAllocate("Total", MemoryBlockType.FUNCTION);
FLUSH_THRESHOLD = memtableMemoryBlock.getMaxMemorySizeInByte() * config.getFlushProportion();
REJECT_THRESHOLD = memtableMemoryBlock.getMaxMemorySizeInByte() * config.getRejectProportion();
FLUSH_THRESHOLD = memorySizeForMemtable * config.getFlushProportion();
REJECT_THRESHOLD = memorySizeForMemtable * config.getRejectProportion();
WritingMetrics.getInstance().recordFlushThreshold(FLUSH_THRESHOLD);
WritingMetrics.getInstance().recordRejectThreshold(REJECT_THRESHOLD);
WritingMetrics.getInstance().recordWALQueueMaxMemorySize(memorySizeForWalBufferQueue);
Expand Down Expand Up @@ -460,28 +459,18 @@ private InstanceHolder() {}
}

public synchronized void applyTemporaryMemoryForFlushing(long estimatedTemporaryMemSize) {
if (memtableMemoryBlock.allocate(estimatedTemporaryMemSize)) {
logger.error(
String.format(
"Failed to allocate %d bytes memory for flush, "
+ "total memory budget for memtable module is %d bytes, %d bytes is used",
estimatedTemporaryMemSize,
memtableMemoryBlock.getMaxMemorySizeInByte(),
memtableMemoryBlock.getMemoryUsageInBytes()));
}
FLUSH_THRESHOLD = memtableMemoryBlock.getRemainedMemoryInBytes() * config.getFlushProportion();
REJECT_THRESHOLD =
memtableMemoryBlock.getRemainedMemoryInBytes() * config.getRejectProportion();
memorySizeForMemtable -= estimatedTemporaryMemSize;
FLUSH_THRESHOLD = memorySizeForMemtable * config.getFlushProportion();
REJECT_THRESHOLD = memorySizeForMemtable * config.getRejectProportion();
WritingMetrics.getInstance().recordFlushThreshold(FLUSH_THRESHOLD);
WritingMetrics.getInstance().recordRejectThreshold(REJECT_THRESHOLD);
WritingMetrics.getInstance().recordWALQueueMaxMemorySize(memorySizeForWalBufferQueue);
}

public synchronized void releaseTemporaryMemoryForFlushing(long estimatedTemporaryMemSize) {
memtableMemoryBlock.release(estimatedTemporaryMemSize);
FLUSH_THRESHOLD = memtableMemoryBlock.getRemainedMemoryInBytes() * config.getFlushProportion();
REJECT_THRESHOLD =
memtableMemoryBlock.getRemainedMemoryInBytes() * config.getRejectProportion();
memorySizeForMemtable += estimatedTemporaryMemSize;
FLUSH_THRESHOLD = memorySizeForMemtable * config.getFlushProportion();
REJECT_THRESHOLD = memorySizeForMemtable * config.getRejectProportion();
WritingMetrics.getInstance().recordFlushThreshold(FLUSH_THRESHOLD);
WritingMetrics.getInstance().recordRejectThreshold(REJECT_THRESHOLD);
WritingMetrics.getInstance().recordWALQueueMaxMemorySize(memorySizeForWalBufferQueue);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,11 +93,6 @@ public long getMemoryUsageInBytes() {
return memoryUsageInBytes.get();
}

/** Get the remained memory in byte of this memory block */
public long getRemainedMemoryInBytes() {
return maxMemorySizeInByte - memoryUsageInBytes.get();
}

/** Get whether this memory block is released */
public boolean isReleased() {
return isReleased;
Expand Down

0 comments on commit f94a60d

Please sign in to comment.