Bugfixes
This commit is contained in:
parent
aeba994347
commit
f460657753
4
pom.xml
4
pom.xml
@ -72,12 +72,12 @@
|
|||||||
<dependency>
|
<dependency>
|
||||||
<groupId>io.projectreactor</groupId>
|
<groupId>io.projectreactor</groupId>
|
||||||
<artifactId>reactor-core</artifactId>
|
<artifactId>reactor-core</artifactId>
|
||||||
<version>3.4.2</version>
|
<version>3.4.3</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>io.projectreactor</groupId>
|
<groupId>io.projectreactor</groupId>
|
||||||
<artifactId>reactor-test</artifactId>
|
<artifactId>reactor-test</artifactId>
|
||||||
<version>3.4.2</version>
|
<version>3.4.3</version>
|
||||||
<scope>test</scope>
|
<scope>test</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
|
@ -72,7 +72,7 @@ public class DiskCache implements URLsDiskHandler, URLsWriter {
|
|||||||
return Optional
|
return Optional
|
||||||
.of(new DiskMetadata(
|
.of(new DiskMetadata(
|
||||||
metadata.getSize(),
|
metadata.getSize(),
|
||||||
new BooleanArrayList(DiskMetadata.getBlocksCount(metadata.getSize(), BLOCK_SIZE))
|
BooleanArrayList.wrap(new boolean[DiskMetadata.getBlocksCount(metadata.getSize(), BLOCK_SIZE)])
|
||||||
))
|
))
|
||||||
.map(diskMetadataSerializer::serialize);
|
.map(diskMetadataSerializer::serialize);
|
||||||
}
|
}
|
||||||
@ -95,7 +95,7 @@ public class DiskCache implements URLsDiskHandler, URLsWriter {
|
|||||||
.map(prevMeta -> {
|
.map(prevMeta -> {
|
||||||
if (!prevMeta.getDownloadedBlocks().getBoolean(dataBlock.getId())) {
|
if (!prevMeta.getDownloadedBlocks().getBoolean(dataBlock.getId())) {
|
||||||
BooleanArrayList bal = prevMeta.getDownloadedBlocks().clone();
|
BooleanArrayList bal = prevMeta.getDownloadedBlocks().clone();
|
||||||
bal.add(dataBlock.getId(), true);
|
bal.set(dataBlock.getId(), true);
|
||||||
return new DiskMetadata(prevMeta.getSize(), bal);
|
return new DiskMetadata(prevMeta.getSize(), bal);
|
||||||
} else {
|
} else {
|
||||||
return prevMeta;
|
return prevMeta;
|
||||||
@ -111,7 +111,7 @@ public class DiskCache implements URLsDiskHandler, URLsWriter {
|
|||||||
public Flux<DataBlock> requestContent(URL url) {
|
public Flux<DataBlock> requestContent(URL url) {
|
||||||
return requestDiskMetadata(url)
|
return requestDiskMetadata(url)
|
||||||
.filter(DiskMetadata::isDownloadedFully)
|
.filter(DiskMetadata::isDownloadedFully)
|
||||||
.flatMapMany(meta -> Flux.fromIterable(meta.getDownloadedBlocks()))
|
.flatMapMany(meta -> Flux.fromIterable(meta.getDownloadedBlocks())
|
||||||
.index()
|
.index()
|
||||||
// Get only downloaded blocks
|
// Get only downloaded blocks
|
||||||
.filter(Tuple2::getT2)
|
.filter(Tuple2::getT2)
|
||||||
@ -119,13 +119,13 @@ public class DiskCache implements URLsDiskHandler, URLsWriter {
|
|||||||
int blockId = Math.toIntExact(blockMeta.getT1());
|
int blockId = Math.toIntExact(blockMeta.getT1());
|
||||||
boolean downloaded = blockMeta.getT2();
|
boolean downloaded = blockMeta.getT2();
|
||||||
if (!downloaded) {
|
if (!downloaded) {
|
||||||
return Mono.empty();
|
return Mono.<DataBlock>empty();
|
||||||
}
|
}
|
||||||
return fileContent.get(null, getBlockKey(url, blockId)).map(data -> {
|
return fileContent.get(null, getBlockKey(url, blockId)).map(data -> {
|
||||||
int blockOffset = getBlockOffset(blockId);
|
int blockOffset = getBlockOffset(blockId);
|
||||||
int blockLength = data.length;
|
int blockLength = data.length;
|
||||||
if (blockOffset + blockLength >= blockMeta.size()) {
|
if (blockOffset + blockLength >= meta.getSize()) {
|
||||||
if (blockOffset + blockLength > blockMeta.size()) {
|
if (blockOffset + blockLength > meta.getSize()) {
|
||||||
throw new IllegalStateException("Overflowed data size");
|
throw new IllegalStateException("Overflowed data size");
|
||||||
}
|
}
|
||||||
} else if (data.length != BLOCK_SIZE) {
|
} else if (data.length != BLOCK_SIZE) {
|
||||||
@ -133,13 +133,13 @@ public class DiskCache implements URLsDiskHandler, URLsWriter {
|
|||||||
}
|
}
|
||||||
return new DataBlock(blockOffset, blockLength, ByteBuffer.wrap(data, 0, blockLength));
|
return new DataBlock(blockOffset, blockLength, ByteBuffer.wrap(data, 0, blockLength));
|
||||||
});
|
});
|
||||||
});
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
private byte[] getBlockKey(URL url, int blockId) {
|
private byte[] getBlockKey(URL url, int blockId) {
|
||||||
byte[] urlBytes = url.getSerializer().serialize(url);
|
byte[] urlBytes = url.getSerializer().serialize(url);
|
||||||
byte[] blockIdBytes = Ints.toByteArray(blockId);
|
byte[] blockIdBytes = Ints.toByteArray(blockId);
|
||||||
byte[] resultBytes = Arrays.copyOf(urlBytes, urlBytes.length);
|
byte[] resultBytes = Arrays.copyOf(urlBytes, urlBytes.length + blockIdBytes.length);
|
||||||
System.arraycopy(blockIdBytes, 0, resultBytes, urlBytes.length, blockIdBytes.length);
|
System.arraycopy(blockIdBytes, 0, resultBytes, urlBytes.length, blockIdBytes.length);
|
||||||
return resultBytes;
|
return resultBytes;
|
||||||
}
|
}
|
||||||
|
@ -53,7 +53,9 @@ public class DiskMetadata {
|
|||||||
private int getBlocksCount() {
|
private int getBlocksCount() {
|
||||||
var expectedBlocksCount = getBlocksCount(size, FileSponge.BLOCK_SIZE);
|
var expectedBlocksCount = getBlocksCount(size, FileSponge.BLOCK_SIZE);
|
||||||
if (this.getDownloadedBlocks().size() != expectedBlocksCount) {
|
if (this.getDownloadedBlocks().size() != expectedBlocksCount) {
|
||||||
throw new IllegalStateException("Blocks array length != expected blocks count");
|
throw new IllegalStateException(
|
||||||
|
"Blocks array length (" + this.getDownloadedBlocks().size()
|
||||||
|
+ ") != expected blocks count (" + expectedBlocksCount + ")");
|
||||||
}
|
}
|
||||||
return expectedBlocksCount;
|
return expectedBlocksCount;
|
||||||
}
|
}
|
||||||
@ -75,22 +77,20 @@ public class DiskMetadata {
|
|||||||
var dis = new DataInputStream(bais);
|
var dis = new DataInputStream(bais);
|
||||||
int size = dis.readInt();
|
int size = dis.readInt();
|
||||||
int blocksCount = getBlocksCount(size, FileSponge.BLOCK_SIZE);
|
int blocksCount = getBlocksCount(size, FileSponge.BLOCK_SIZE);
|
||||||
boolean[] downloadedBlocks = new boolean[blocksCount];
|
var downloadedBlocks = new BooleanArrayList(blocksCount);
|
||||||
for (int i = 0; i < blocksCount; i++) {
|
for (int i = 0; i < blocksCount; i++) {
|
||||||
downloadedBlocks[i] = dis.readBoolean();
|
downloadedBlocks.add(dis.readBoolean());
|
||||||
}
|
}
|
||||||
return new DiskMetadata(size, BooleanArrayList.wrap(downloadedBlocks, blocksCount));
|
return new DiskMetadata(size, downloadedBlocks);
|
||||||
}
|
}
|
||||||
|
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
@Override
|
@Override
|
||||||
public byte @NotNull [] serialize(@NotNull DiskMetadata deserialized) {
|
public byte @NotNull [] serialize(@NotNull DiskMetadata deserialized) {
|
||||||
try (var bos = new ByteArrayOutputStream(Integer.BYTES * 2)) {
|
try (var bos = new ByteArrayOutputStream()) {
|
||||||
try (var dos = new DataOutputStream(bos)) {
|
try (var dos = new DataOutputStream(bos)) {
|
||||||
dos.writeInt(deserialized.getSize());
|
dos.writeInt(deserialized.getSize());
|
||||||
if (deserialized.getDownloadedBlocks().size() != deserialized.getBlocksCount()) {
|
deserialized.getBlocksCount();
|
||||||
throw new IllegalStateException("Blocks array length != expected blocks count");
|
|
||||||
}
|
|
||||||
for (boolean downloadedBlock : deserialized.getDownloadedBlocks()) {
|
for (boolean downloadedBlock : deserialized.getDownloadedBlocks()) {
|
||||||
dos.writeBoolean(downloadedBlock);
|
dos.writeBoolean(downloadedBlock);
|
||||||
}
|
}
|
||||||
|
@ -20,13 +20,17 @@ package org.warp.filesponge;
|
|||||||
|
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
import reactor.core.publisher.Flux;
|
import reactor.core.publisher.Flux;
|
||||||
import reactor.core.publisher.Mono;
|
import reactor.core.publisher.Mono;
|
||||||
|
|
||||||
public class FileSponge implements URLsHandler {
|
public class FileSponge implements URLsHandler {
|
||||||
|
|
||||||
public static final int BLOCK_SIZE = 8 * 1024 * 1024; // 8 MiB
|
private static final Logger logger = LoggerFactory.getLogger(FileSponge.class);
|
||||||
public static final int MAX_BLOCKS = 256; // 2 GiB
|
|
||||||
|
public static final int BLOCK_SIZE = 1024 * 1024; // 1 MiB
|
||||||
|
|
||||||
private final Set<URLsHandler> urlsHandlers = new ConcurrentHashMap<URLsHandler, Object>().keySet(new Object());
|
private final Set<URLsHandler> urlsHandlers = new ConcurrentHashMap<URLsHandler, Object>().keySet(new Object());
|
||||||
|
|
||||||
@ -50,13 +54,20 @@ public class FileSponge implements URLsHandler {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Flux<DataBlock> requestContent(URL url) {
|
public Flux<DataBlock> requestContent(URL url) {
|
||||||
|
AtomicBoolean alreadyPrintedDebug = new AtomicBoolean(false);
|
||||||
return Flux
|
return Flux
|
||||||
.fromIterable(cacheAccess)
|
.fromIterable(cacheAccess)
|
||||||
.map(urlsHandler -> urlsHandler.requestContent(url))
|
.map(urlsHandler -> urlsHandler.requestContent(url))
|
||||||
.collectList()
|
.collectList()
|
||||||
.flatMapMany(Flux::firstWithValue)
|
.flatMapMany(monos -> FileSpongeUtils.firstWithValueFlux(monos))
|
||||||
|
.doOnNext(dataBlock -> {
|
||||||
|
if (alreadyPrintedDebug.compareAndSet(false, true)) {
|
||||||
|
logger.debug("File \"{}\" content has been found in the cache", url);
|
||||||
|
}
|
||||||
|
})
|
||||||
.switchIfEmpty(Flux
|
.switchIfEmpty(Flux
|
||||||
.fromIterable(urlsHandlers)
|
.fromIterable(urlsHandlers)
|
||||||
|
.doOnSubscribe(s -> logger.debug("Downloading file \"{}\" content", url))
|
||||||
.map(urlsHandler -> urlsHandler
|
.map(urlsHandler -> urlsHandler
|
||||||
.requestContent(url)
|
.requestContent(url)
|
||||||
.flatMapSequential(dataBlock -> Flux
|
.flatMapSequential(dataBlock -> Flux
|
||||||
@ -66,7 +77,8 @@ public class FileSponge implements URLsHandler {
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
.collectList()
|
.collectList()
|
||||||
.flatMapMany(Flux::firstWithValue)
|
.flatMapMany(monos -> FileSpongeUtils.firstWithValueFlux(monos))
|
||||||
|
.doOnComplete(() -> logger.debug("Downloaded file \"{}\" content", url))
|
||||||
)
|
)
|
||||||
.distinct(DataBlock::getId);
|
.distinct(DataBlock::getId);
|
||||||
}
|
}
|
||||||
@ -77,9 +89,15 @@ public class FileSponge implements URLsHandler {
|
|||||||
.fromIterable(cacheAccess)
|
.fromIterable(cacheAccess)
|
||||||
.map(urlsHandler -> urlsHandler.requestMetadata(url))
|
.map(urlsHandler -> urlsHandler.requestMetadata(url))
|
||||||
.collectList()
|
.collectList()
|
||||||
.flatMap(Mono::firstWithValue)
|
.flatMap(monos -> FileSpongeUtils.firstWithValueMono(monos))
|
||||||
|
.doOnSuccess(metadata -> {
|
||||||
|
if (metadata != null) {
|
||||||
|
logger.debug("File \"{}\" metadata has been found in the cache", url);
|
||||||
|
}
|
||||||
|
})
|
||||||
.switchIfEmpty(Flux
|
.switchIfEmpty(Flux
|
||||||
.fromIterable(urlsHandlers)
|
.fromIterable(urlsHandlers)
|
||||||
|
.doOnSubscribe(s -> logger.debug("Downloading file \"{}\" metadata", url))
|
||||||
.map(urlsHandler -> urlsHandler
|
.map(urlsHandler -> urlsHandler
|
||||||
.requestMetadata(url)
|
.requestMetadata(url)
|
||||||
.flatMap(dataBlock -> Flux
|
.flatMap(dataBlock -> Flux
|
||||||
@ -89,7 +107,14 @@ public class FileSponge implements URLsHandler {
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
.collectList()
|
.collectList()
|
||||||
.flatMap(Mono::firstWithValue)
|
.flatMap(monos -> FileSpongeUtils.firstWithValueMono(monos))
|
||||||
|
.doOnSuccess(s -> {
|
||||||
|
if (s != null) {
|
||||||
|
logger.debug("Downloaded file \"{}\" metadata", url);
|
||||||
|
} else {
|
||||||
|
logger.debug("File \"{}\" metadata has not been found anywhere", url);
|
||||||
|
}
|
||||||
|
})
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
75
src/main/lombok/org/warp/filesponge/FileSpongeUtils.java
Normal file
75
src/main/lombok/org/warp/filesponge/FileSpongeUtils.java
Normal file
@ -0,0 +1,75 @@
|
|||||||
|
/*
|
||||||
|
* FileSponge
|
||||||
|
* Copyright (C) 2021 Andrea Cavalli
|
||||||
|
*
|
||||||
|
* This program is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU General Public License as published by
|
||||||
|
* the Free Software Foundation, either version 3 of the License, or
|
||||||
|
* (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU General Public License
|
||||||
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.warp.filesponge;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.NoSuchElementException;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import reactor.core.Exceptions;
|
||||||
|
import reactor.core.publisher.Flux;
|
||||||
|
import reactor.core.publisher.Mono;
|
||||||
|
import reactor.core.scheduler.Schedulers;
|
||||||
|
|
||||||
|
public class FileSpongeUtils {
|
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(FileSponge.class);
|
||||||
|
|
||||||
|
public static <T> Mono<T> firstWithValueMono(List<Mono<T>> monos) {
|
||||||
|
return Mono.firstWithValue(monos).onErrorResume(FileSpongeUtils::ignoreFakeErrors);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static <T> Flux<T> firstWithValueFlux(List<Flux<T>> monos) {
|
||||||
|
return Flux.firstWithValue(monos).onErrorResume(FileSpongeUtils::ignoreFakeErrors);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static <T> Mono<T> ignoreFakeErrors(Throwable ex) {
|
||||||
|
return Mono.defer(() -> {
|
||||||
|
if (ex instanceof NoSuchElementException) {
|
||||||
|
var multiple = Exceptions.unwrapMultiple(ex.getCause());
|
||||||
|
for (Throwable throwable : multiple) {
|
||||||
|
if (!(throwable instanceof NoSuchElementException)) {
|
||||||
|
return Mono.error(ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Mono.empty();
|
||||||
|
} else {
|
||||||
|
return Mono.error(ex);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Mono<Path> deleteFileAfter(Path path, Duration delay) {
|
||||||
|
return Mono.fromCallable(() -> {
|
||||||
|
Schedulers.boundedElastic().schedule(() -> {
|
||||||
|
try {
|
||||||
|
Files.deleteIfExists(path);
|
||||||
|
} catch (IOException e) {
|
||||||
|
logger.warn("Failed to delete file \"{}\"", path, e);
|
||||||
|
}
|
||||||
|
}, delay.toMillis(), TimeUnit.MILLISECONDS);
|
||||||
|
return path;
|
||||||
|
}).subscribeOn(Schedulers.boundedElastic());
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user