Bugfixes

pom.xml

@@ -72,12 +72,12 @@
     <dependency>
       <groupId>io.projectreactor</groupId>
       <artifactId>reactor-core</artifactId>
-      <version>3.4.2</version>
+      <version>3.4.3</version>
     </dependency>
     <dependency>
       <groupId>io.projectreactor</groupId>
       <artifactId>reactor-test</artifactId>
-      <version>3.4.2</version>
+      <version>3.4.3</version>
       <scope>test</scope>
     </dependency>
     <dependency>

src/main/lombok/org/warp/filesponge/DiskCache.java

@@ -72,7 +72,7 @@ public class DiskCache implements URLsDiskHandler, URLsWriter {
 						return Optional
 								.of(new DiskMetadata(
 										metadata.getSize(),
-										new BooleanArrayList(DiskMetadata.getBlocksCount(metadata.getSize(), BLOCK_SIZE))
+										BooleanArrayList.wrap(new boolean[DiskMetadata.getBlocksCount(metadata.getSize(), BLOCK_SIZE)])
 								))
 								.map(diskMetadataSerializer::serialize);
 					}
@@ -95,7 +95,7 @@ public class DiskCache implements URLsDiskHandler, URLsWriter {
 								.map(prevMeta -> {
 									if (!prevMeta.getDownloadedBlocks().getBoolean(dataBlock.getId())) {
 										BooleanArrayList bal = prevMeta.getDownloadedBlocks().clone();
-										bal.add(dataBlock.getId(), true);
+										bal.set(dataBlock.getId(), true);
 										return new DiskMetadata(prevMeta.getSize(), bal);
 									} else {
 										return prevMeta;
@@ -111,35 +111,35 @@ public class DiskCache implements URLsDiskHandler, URLsWriter {
 	public Flux<DataBlock> requestContent(URL url) {
 		return requestDiskMetadata(url)
 				.filter(DiskMetadata::isDownloadedFully)
-				.flatMapMany(meta -> Flux.fromIterable(meta.getDownloadedBlocks()))
+				.flatMapMany(meta -> Flux.fromIterable(meta.getDownloadedBlocks())
-				.index()
+						.index()
-				// Get only downloaded blocks
+						// Get only downloaded blocks
-				.filter(Tuple2::getT2)
+						.filter(Tuple2::getT2)
-				.flatMapSequential(blockMeta -> {
+						.flatMapSequential(blockMeta -> {
-					int blockId = Math.toIntExact(blockMeta.getT1());
+							int blockId = Math.toIntExact(blockMeta.getT1());
-					boolean downloaded = blockMeta.getT2();
+							boolean downloaded = blockMeta.getT2();
-					if (!downloaded) {
+							if (!downloaded) {
-						return Mono.empty();
+								return Mono.<DataBlock>empty();
-					}
-					return fileContent.get(null, getBlockKey(url, blockId)).map(data -> {
-						int blockOffset = getBlockOffset(blockId);
-						int blockLength = data.length;
-						if (blockOffset + blockLength >= blockMeta.size()) {
-							if (blockOffset + blockLength > blockMeta.size()) {
-								throw new IllegalStateException("Overflowed data size");
 							}
-						} else if (data.length != BLOCK_SIZE) {
+							return fileContent.get(null, getBlockKey(url, blockId)).map(data -> {
-							throw new IllegalStateException("Block data length != block length");
+								int blockOffset = getBlockOffset(blockId);
-						}
+								int blockLength = data.length;
-						return new DataBlock(blockOffset, blockLength, ByteBuffer.wrap(data, 0, blockLength));
+								if (blockOffset + blockLength >= meta.getSize()) {
-					});
+									if (blockOffset + blockLength > meta.getSize()) {
-				});
+										throw new IllegalStateException("Overflowed data size");
+									}
+								} else if (data.length != BLOCK_SIZE) {
+									throw new IllegalStateException("Block data length != block length");
+								}
+								return new DataBlock(blockOffset, blockLength, ByteBuffer.wrap(data, 0, blockLength));
+							});
+						}));
 	}
 
 	private byte[] getBlockKey(URL url, int blockId) {
 		byte[] urlBytes = url.getSerializer().serialize(url);
 		byte[] blockIdBytes = Ints.toByteArray(blockId);
-		byte[] resultBytes = Arrays.copyOf(urlBytes, urlBytes.length);
+		byte[] resultBytes = Arrays.copyOf(urlBytes, urlBytes.length + blockIdBytes.length);
 		System.arraycopy(blockIdBytes, 0, resultBytes, urlBytes.length, blockIdBytes.length);
 		return resultBytes;
 	}

src/main/lombok/org/warp/filesponge/DiskMetadata.java

@@ -53,7 +53,9 @@ public class DiskMetadata {
 	private int getBlocksCount() {
 		var expectedBlocksCount = getBlocksCount(size, FileSponge.BLOCK_SIZE);
 		if (this.getDownloadedBlocks().size() != expectedBlocksCount) {
-			throw new IllegalStateException("Blocks array length != expected blocks count");
+			throw new IllegalStateException(
+					"Blocks array length (" + this.getDownloadedBlocks().size()
+							+ ") != expected blocks count (" + expectedBlocksCount + ")");
 		}
 		return expectedBlocksCount;
 	}
@@ -75,22 +77,20 @@ public class DiskMetadata {
 			var dis = new DataInputStream(bais);
 			int size = dis.readInt();
 			int blocksCount = getBlocksCount(size, FileSponge.BLOCK_SIZE);
-			boolean[] downloadedBlocks = new boolean[blocksCount];
+			var downloadedBlocks = new BooleanArrayList(blocksCount);
 			for (int i = 0; i < blocksCount; i++) {
-				downloadedBlocks[i] = dis.readBoolean();
+				downloadedBlocks.add(dis.readBoolean());
 			}
-			return new DiskMetadata(size, BooleanArrayList.wrap(downloadedBlocks, blocksCount));
+			return new DiskMetadata(size, downloadedBlocks);
 		}
 
 		@SneakyThrows
 		@Override
 		public byte @NotNull [] serialize(@NotNull DiskMetadata deserialized) {
-			try (var bos = new ByteArrayOutputStream(Integer.BYTES * 2)) {
+			try (var bos = new ByteArrayOutputStream()) {
 				try (var dos = new DataOutputStream(bos)) {
 					dos.writeInt(deserialized.getSize());
-					if (deserialized.getDownloadedBlocks().size() != deserialized.getBlocksCount()) {
+					deserialized.getBlocksCount();
-						throw new IllegalStateException("Blocks array length != expected blocks count");
-					}
 					for (boolean downloadedBlock : deserialized.getDownloadedBlocks()) {
 						dos.writeBoolean(downloadedBlock);
 					}

src/main/lombok/org/warp/filesponge/FileSponge.java

@@ -20,13 +20,17 @@ package org.warp.filesponge;
 
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
 
 public class FileSponge implements URLsHandler {
 
-	public static final int BLOCK_SIZE = 8 * 1024 * 1024; // 8 MiB
+	private static final Logger logger = LoggerFactory.getLogger(FileSponge.class);
-	public static final int MAX_BLOCKS = 256; // 2 GiB
+
+	public static final int BLOCK_SIZE = 1024 * 1024; // 1 MiB
 
 	private final Set<URLsHandler> urlsHandlers = new ConcurrentHashMap<URLsHandler, Object>().keySet(new Object());
 
@@ -50,13 +54,20 @@ public class FileSponge implements URLsHandler {
 
 	@Override
 	public Flux<DataBlock> requestContent(URL url) {
+		AtomicBoolean alreadyPrintedDebug = new AtomicBoolean(false);
 		return Flux
 				.fromIterable(cacheAccess)
 				.map(urlsHandler -> urlsHandler.requestContent(url))
 				.collectList()
-				.flatMapMany(Flux::firstWithValue)
+				.flatMapMany(monos -> FileSpongeUtils.firstWithValueFlux(monos))
+				.doOnNext(dataBlock -> {
+					if (alreadyPrintedDebug.compareAndSet(false, true)) {
+						logger.debug("File \"{}\" content has been found in the cache", url);
+					}
+				})
 				.switchIfEmpty(Flux
 						.fromIterable(urlsHandlers)
+						.doOnSubscribe(s -> logger.debug("Downloading file \"{}\" content", url))
 						.map(urlsHandler -> urlsHandler
 								.requestContent(url)
 								.flatMapSequential(dataBlock -> Flux
@@ -66,7 +77,8 @@ public class FileSponge implements URLsHandler {
 								)
 						)
 						.collectList()
-						.flatMapMany(Flux::firstWithValue)
+						.flatMapMany(monos -> FileSpongeUtils.firstWithValueFlux(monos))
+						.doOnComplete(() -> logger.debug("Downloaded file \"{}\" content", url))
 				)
 				.distinct(DataBlock::getId);
 	}
@@ -77,9 +89,15 @@ public class FileSponge implements URLsHandler {
 				.fromIterable(cacheAccess)
 				.map(urlsHandler -> urlsHandler.requestMetadata(url))
 				.collectList()
-				.flatMap(Mono::firstWithValue)
+				.flatMap(monos -> FileSpongeUtils.firstWithValueMono(monos))
+				.doOnSuccess(metadata -> {
+					if (metadata != null) {
+						logger.debug("File \"{}\" metadata has been found in the cache", url);
+					}
+				})
 				.switchIfEmpty(Flux
 						.fromIterable(urlsHandlers)
+						.doOnSubscribe(s -> logger.debug("Downloading file \"{}\" metadata", url))
 						.map(urlsHandler -> urlsHandler
 								.requestMetadata(url)
 								.flatMap(dataBlock -> Flux
@@ -89,7 +107,14 @@ public class FileSponge implements URLsHandler {
 								)
 						)
 						.collectList()
-						.flatMap(Mono::firstWithValue)
+						.flatMap(monos -> FileSpongeUtils.firstWithValueMono(monos))
+						.doOnSuccess(s -> {
+							if (s != null) {
+								logger.debug("Downloaded file \"{}\" metadata", url);
+							} else {
+								logger.debug("File \"{}\" metadata has not been found anywhere", url);
+							}
+						})
 				);
 	}
 }

src/main/lombok/org/warp/filesponge/FileSpongeUtils.java

@@ -0,0 +1,75 @@
+/*
+ *     FileSponge
+ *     Copyright (C) 2021 Andrea Cavalli
+ *
+ *     This program is free software: you can redistribute it and/or modify
+ *     it under the terms of the GNU General Public License as published by
+ *     the Free Software Foundation, either version 3 of the License, or
+ *     (at your option) any later version.
+ *
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ *     You should have received a copy of the GNU General Public License
+ *     along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.warp.filesponge;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.concurrent.TimeUnit;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import reactor.core.Exceptions;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.Mono;
+import reactor.core.scheduler.Schedulers;
+
+public class FileSpongeUtils {
+
+	private static final Logger logger = LoggerFactory.getLogger(FileSponge.class);
+
+	public static <T> Mono<T> firstWithValueMono(List<Mono<T>> monos) {
+		return Mono.firstWithValue(monos).onErrorResume(FileSpongeUtils::ignoreFakeErrors);
+	}
+
+	public static <T> Flux<T> firstWithValueFlux(List<Flux<T>> monos) {
+		return Flux.firstWithValue(monos).onErrorResume(FileSpongeUtils::ignoreFakeErrors);
+	}
+
+	private static <T> Mono<T> ignoreFakeErrors(Throwable ex) {
+		return Mono.defer(() -> {
+			if (ex instanceof NoSuchElementException) {
+				var multiple = Exceptions.unwrapMultiple(ex.getCause());
+				for (Throwable throwable : multiple) {
+					if (!(throwable instanceof NoSuchElementException)) {
+						return Mono.error(ex);
+					}
+				}
+				return Mono.empty();
+			} else {
+				return Mono.error(ex);
+			}
+		});
+	}
+
+	public static Mono<Path> deleteFileAfter(Path path, Duration delay) {
+		return Mono.fromCallable(() -> {
+			Schedulers.boundedElastic().schedule(() -> {
+				try {
+					Files.deleteIfExists(path);
+				} catch (IOException e) {
+					logger.warn("Failed to delete file \"{}\"", path, e);
+				}
+			}, delay.toMillis(), TimeUnit.MILLISECONDS);
+			return path;
+		}).subscribeOn(Schedulers.boundedElastic());
+	}
+}