andreacavalli
/
FileSponge
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Performance optimization

This commit is contained in:
Andrea Cavalli 2021-09-17 20:38:28 +02:00
parent c1747dc6dc
commit 8522e7a866
1 changed files with 54 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
src/main/java/org/warp/filesponge/FileSponge.java
View File

@ -19,6 +19,8 @@
package org.warp.filesponge; package org.warp.filesponge;
import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet; import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet;
import java.util.ArrayList;
import java.util.List;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
@ -74,28 +76,38 @@ public class FileSponge implements URLsHandler {
@Override @Override
public Flux<DataBlock> requestContent(URL url) { public Flux<DataBlock> requestContent(URL url) {
AtomicBoolean alreadyPrintedDebug = new AtomicBoolean(false); AtomicBoolean alreadyPrintedDebug = new AtomicBoolean(false);
return Flux return Mono
.defer(() -> Flux.fromIterable(cacheAccess)) .fromCallable(() -> {
.map(urlsHandler -> urlsHandler.requestContent(url)) List<Flux<DataBlock>> contentRequests = new ArrayList<>(cacheAccess.size());
.collectList() for (URLsDiskHandler urlsHandler : cacheAccess) {
contentRequests.add(urlsHandler.requestContent(url));
}
return contentRequests;
})
.flatMapMany(FileSpongeUtils::firstWithValueFlux) .flatMapMany(FileSpongeUtils::firstWithValueFlux)
.doOnNext(dataBlock -> { .doOnNext(dataBlock -> {
if (alreadyPrintedDebug.compareAndSet(false, true)) { if (alreadyPrintedDebug.compareAndSet(false, true)) {
logger.debug("File \"{}\" content has been found in the cache", url); logger.debug("File \"{}\" content has been found in the cache", url);
} }
}) })
.switchIfEmpty(Flux .switchIfEmpty(Mono
.defer(() -> Flux.fromIterable(urlsHandlers)) .fromCallable(() -> {
.doOnSubscribe(s -> logger.debug("Downloading file \"{}\" content", url)) logger.debug("Downloading file \"{}\" content", url);
.map(urlsHandler -> urlsHandler List<Flux<DataBlock>> contentRequestsAndCaching = new ArrayList<>(urlsHandlers.size());
.requestContent(url) for (URLsHandler urlsHandler : urlsHandlers) {
.flatMapSequential(dataBlock -> Flux contentRequestsAndCaching.add(urlsHandler
.defer(() -> Flux.fromIterable(cacheWrite)) .requestContent(url)
.flatMapSequential(cw -> cw.writeContentBlock(url, dataBlock)) .flatMapSequential(dataBlock -> {
.then(Mono.just(dataBlock)) List<Mono<Void>> cacheWriteActions = new ArrayList<>(cacheWrite.size());
) for (URLsWriter urlsWriter : cacheWrite) {
) cacheWriteActions.add(urlsWriter.writeContentBlock(url, dataBlock));
.collectList() }
return Mono.when(cacheWriteActions).thenReturn(dataBlock);
})
);
}
return contentRequestsAndCaching;
})
.flatMapMany(FileSpongeUtils::firstWithValueFlux) .flatMapMany(FileSpongeUtils::firstWithValueFlux)
.doOnComplete(() -> logger.debug("Downloaded file \"{}\" content", url)) .doOnComplete(() -> logger.debug("Downloaded file \"{}\" content", url))
) )
@ -106,28 +118,38 @@ public class FileSponge implements URLsHandler {
@Override @Override
public Mono<Metadata> requestMetadata(URL url) { public Mono<Metadata> requestMetadata(URL url) {
return Flux return Mono
.defer(() -> Flux.fromIterable(cacheAccess)) .fromCallable(() -> {
.map(urlsHandler -> urlsHandler.requestMetadata(url)) List<Mono<Metadata>> metadataRequests = new ArrayList<>(cacheAccess.size());
.collectList() for (URLsDiskHandler urlsHandler : cacheAccess) {
metadataRequests.add(urlsHandler.requestMetadata(url));
}
return metadataRequests;
})
.flatMap(FileSpongeUtils::firstWithValueMono) .flatMap(FileSpongeUtils::firstWithValueMono)
.doOnSuccess(metadata -> { .doOnSuccess(metadata -> {
if (metadata != null) { if (metadata != null) {
logger.debug("File \"{}\" metadata has been found in the cache", url); logger.debug("File \"{}\" metadata has been found in the cache", url);
} }
}) })
.switchIfEmpty(Flux .switchIfEmpty(Mono
.defer(() -> Flux.fromIterable(urlsHandlers)) .fromCallable(() -> {
.doOnSubscribe(s -> logger.debug("Downloading file \"{}\" metadata", url)) logger.debug("Downloading file \"{}\" metadata", url);
.map(urlsHandler -> urlsHandler List<Mono<Metadata>> metadataRequestsAndCaching = new ArrayList<>(urlsHandlers.size());
.requestMetadata(url) for (URLsHandler urlsHandler : urlsHandlers) {
.flatMap(dataBlock -> Flux metadataRequestsAndCaching.add(urlsHandler
.defer(() -> Flux.fromIterable(cacheWrite)) .requestMetadata(url)
.flatMapSequential(cw -> cw.writeMetadata(url, dataBlock)) .flatMap(meta -> {
.then(Mono.just(dataBlock)) List<Mono<Void>> cacheWriteActions = new ArrayList<>(cacheWrite.size());
) for (URLsWriter urlsWriter : cacheWrite) {
) cacheWriteActions.add(urlsWriter.writeMetadata(url, meta));
.collectList() }
return Mono.when(cacheWriteActions).thenReturn(meta);
})
);
}
return metadataRequestsAndCaching;
})
.flatMap(FileSpongeUtils::firstWithValueMono) .flatMap(FileSpongeUtils::firstWithValueMono)
.doOnSuccess(s -> { .doOnSuccess(s -> {
if (s != null) { if (s != null) {