Skip to content

Commit

Permalink
noop(cleanup) dedup w/factored-out (#1392) counter (#1393)
Browse files Browse the repository at this point in the history
this utilizes the newly factored out streaming counter, so _should_ be
noop.
  • Loading branch information
jzacsh authored Nov 20, 2024
1 parent 4cf9c5f commit 4a6b403
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 18 deletions.
1 change: 1 addition & 0 deletions portability-transfer/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ configurations {

dependencies {
compile project(':portability-api-launcher')
compile project(':portability-spi-api')
compile project(':portability-spi-service')
compile project(':portability-spi-cloud')
compile project(':portability-spi-transfer')
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package org.datatransferproject.transfer;

import java.io.IOException;
import java.io.InputStream;
import static org.datatransferproject.spi.api.transport.DiscardingStreamCounter.discardForLength;

import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
Expand All @@ -19,7 +19,9 @@ public class CallableSizeCalculator implements Callable<Map<String, Long>> {
private final Collection<? extends DownloadableItem> items;

public CallableSizeCalculator(
UUID jobId, ConnectionProvider connectionProvider, Collection<? extends DownloadableItem> items) {
UUID jobId,
ConnectionProvider connectionProvider,
Collection<? extends DownloadableItem> items) {
this.jobId = Objects.requireNonNull(jobId);
this.connectionProvider = Objects.requireNonNull(connectionProvider);
this.items = Objects.requireNonNull(items);
Expand All @@ -32,26 +34,12 @@ public Map<String, Long> call() throws Exception {
InputStreamWrapper stream = connectionProvider.getInputStreamForItem(jobId, item);
long size = stream.getBytes();
if (size <= 0) {
size = computeSize(stream);
size = discardForLength(stream.getStream());
}

result.put(item.getIdempotentId(), size);
}

return result;
}

// Reads the input stream in full
private Long computeSize(InputStreamWrapper stream) throws IOException {
long size = 0;
try (InputStream inStream = stream.getStream()) {
byte[] buffer = new byte[1024 * 1024]; // 1MB
int chunkBytesRead;
while ((chunkBytesRead = inStream.read(buffer)) != -1) {
size += chunkBytesRead;
}
}

return size;
}
}

0 comments on commit 4a6b403

Please sign in to comment.