andreacavalli
/
FileSponge
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Support up to 2PiB file size

This commit is contained in:
Andrea Cavalli 2022-06-22 12:04:38 +02:00
parent 952765e4f4
commit a607eed212
4 changed files with 31 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/main/java/org/warp/filesponge/DataBlock.java
View File

@ -18,6 +18,8 @@
package org.warp.filesponge;
import static java.lang.Math.toIntExact;
import io.netty5.buffer.api.Buffer;
import io.netty5.buffer.api.Drop;
import io.netty5.buffer.api.Owned;
@ -54,11 +56,11 @@ public final class DataBlock extends ResourceSupport<DataBlock, DataBlock> {
}
};
public static DataBlock of(int offset, int length, Send<Buffer> data) {
public static DataBlock of(long offset, int length, Send<Buffer> data) {
return new DataBlock(offset, length, data);
}
private DataBlock(int offset, int length, Send<Buffer> data) {
private DataBlock(long offset, int length, Send<Buffer> data) {
super(DROP);
try (data) {
this.offset = offset;
@ -67,7 +69,7 @@ public final class DataBlock extends ResourceSupport<DataBlock, DataBlock> {
}
}
private final int offset;
private final long offset;
private final int length;
private final Buffer data;
@ -81,10 +83,10 @@ public final class DataBlock extends ResourceSupport<DataBlock, DataBlock> {
}
public int getId() {
return offset / FileSponge.BLOCK_SIZE;
return toIntExact(offset / FileSponge.BLOCK_SIZE);
}
public int getOffset() {
public long getOffset() {
return this.offset;
}
@ -117,7 +119,8 @@ public final class DataBlock extends ResourceSupport<DataBlock, DataBlock> {
public int hashCode() {
final int PRIME = 59;
int result = 1;
result = result * PRIME + this.getOffset();
long offset = this.getOffset();
result = result * PRIME + (int) (offset ^ (offset >>> 32));
result = result * PRIME + this.getLength();
final Object $data = this.getDataUnsafe();
result = result * PRIME + ($data == null ? 43 : $data.hashCode());

9
src/main/java/org/warp/filesponge/DiskCache.java
View File

@ -18,6 +18,7 @@
package org.warp.filesponge;
import static java.lang.Math.toIntExact;
import static org.warp.filesponge.FileSponge.BLOCK_SIZE;
import io.netty5.buffer.api.Buffer;
@ -208,7 +209,7 @@ public class DiskCache implements URLsDiskHandler, URLsWriter {
// Get only downloaded blocks
.filter(Tuple2::getT2)
.flatMapSequential(blockMeta -> {
int blockId = Math.toIntExact(blockMeta.getT1());
int blockId = toIntExact(blockMeta.getT1());
boolean downloaded = blockMeta.getT2();
if (!downloaded) {
return Mono.empty();
@ -218,7 +219,7 @@ public class DiskCache implements URLsDiskHandler, URLsWriter {
.get(null, blockKeyMono)
.map(data -> {
try (data) {
int blockOffset = getBlockOffset(blockId);
long blockOffset = getBlockOffset(blockId);
int blockLength = data.readableBytes();
if (meta.size() != -1) {
if (blockOffset + blockLength >= meta.size()) {
@ -245,8 +246,8 @@ public class DiskCache implements URLsDiskHandler, URLsWriter {
}
}
private static int getBlockOffset(int blockId) {
return blockId * BLOCK_SIZE;
private static long getBlockOffset(int blockId) {
return blockId * (long) BLOCK_SIZE;
}
@Override

23
src/main/java/org/warp/filesponge/DiskMetadata.java
View File

@ -18,6 +18,8 @@
package org.warp.filesponge;
import static java.lang.Math.toIntExact;
import io.netty5.buffer.api.Buffer;
import io.netty5.buffer.api.BufferAllocator;
import io.netty5.buffer.api.Send;
@ -32,7 +34,7 @@ import org.jetbrains.annotations.NotNull;
/**
* size -1 = unknown size
*/
public record DiskMetadata(int size, BooleanArrayList downloadedBlocks) {
public record DiskMetadata(long size, BooleanArrayList downloadedBlocks) {
public boolean isDownloadedFully() {
boolean downloadedFullyVal;
@ -57,11 +59,11 @@ public record DiskMetadata(int size, BooleanArrayList downloadedBlocks) {
return expectedBlocksCount;
}
public static int getBlocksCount(int size, int blockSize) {
if (size == -1) {
public static int getBlocksCount(long size, int blockSize) {
if (size == -1L) {
return 0;
}
return (size + (blockSize - size % blockSize)) / blockSize;
return toIntExact((size + (blockSize - size % blockSize)) / blockSize);
}
public Metadata asMetadata() {
@ -81,7 +83,13 @@ public record DiskMetadata(int size, BooleanArrayList downloadedBlocks) {
@Override
public @NotNull DiskMetadata deserialize(@NotNull Buffer serialized) throws SerializationException {
var dis = new BufferDataInputShared(serialized);
int size = dis.readInt();
int legacySize = dis.readInt();
long size;
if (legacySize == -2) {
size = dis.readLong();
} else {
size = legacySize;
}
int blocksCount;
if (size == -1) {
blocksCount = dis.readShort();
@ -98,7 +106,8 @@ public record DiskMetadata(int size, BooleanArrayList downloadedBlocks) {
@Override
public void serialize(@NotNull DiskMetadata deserialized, Buffer output) throws SerializationException {
var dos = new BufferDataOutput(output);
dos.writeInt(deserialized.size);
dos.writeInt(-2);
dos.writeLong(deserialized.size);
var blocksCount = deserialized.getBlocksCount();
if (deserialized.size == -1) {
dos.writeShort(blocksCount);
@ -110,7 +119,7 @@ public record DiskMetadata(int size, BooleanArrayList downloadedBlocks) {
@Override
public int getSerializedSizeHint() {
return Integer.BYTES;
return Integer.BYTES + Long.BYTES;
}
}
}

2
src/main/java/org/warp/filesponge/Metadata.java
View File

@ -22,4 +22,4 @@ package org.warp.filesponge;
/**
* size -1 = unknown size
*/
public record Metadata(int size) {}
public record Metadata(long size) {}