2021-03-07 15:50:11 +01:00
|
|
|
/*
|
|
|
|
* FileSponge
|
|
|
|
* Copyright (C) 2021 Andrea Cavalli
|
|
|
|
*
|
|
|
|
* This program is free software: you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License as published by
|
|
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
|
|
* (at your option) any later version.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
|
|
*/
|
|
|
|
|
|
|
|
package org.warp.filesponge;
|
|
|
|
|
2021-03-07 15:51:54 +01:00
|
|
|
import static org.warp.filesponge.FileSponge.BLOCK_SIZE;
|
2021-03-07 15:50:11 +01:00
|
|
|
|
2021-05-03 00:33:04 +02:00
|
|
|
import io.netty.buffer.ByteBuf;
|
|
|
|
import io.netty.buffer.Unpooled;
|
2021-05-03 12:49:34 +02:00
|
|
|
import io.netty.util.ReferenceCounted;
|
2021-03-07 15:50:11 +01:00
|
|
|
import it.cavallium.dbengine.database.Column;
|
|
|
|
import it.cavallium.dbengine.database.LLDatabaseConnection;
|
|
|
|
import it.cavallium.dbengine.database.LLDictionary;
|
|
|
|
import it.cavallium.dbengine.database.LLDictionaryResultType;
|
|
|
|
import it.cavallium.dbengine.database.LLKeyValueDatabase;
|
|
|
|
import it.cavallium.dbengine.database.UpdateMode;
|
2021-05-08 03:09:49 +02:00
|
|
|
import it.cavallium.dbengine.database.UpdateReturnMode;
|
2021-07-01 21:16:34 +02:00
|
|
|
import it.cavallium.dbengine.client.DatabaseOptions;
|
2021-08-22 23:52:51 +02:00
|
|
|
import it.cavallium.dbengine.database.serialization.SerializationException;
|
2021-03-07 15:50:11 +01:00
|
|
|
import it.unimi.dsi.fastutil.booleans.BooleanArrayList;
|
|
|
|
import java.util.List;
|
2021-08-22 18:20:52 +02:00
|
|
|
import java.util.Objects;
|
2021-05-03 00:33:04 +02:00
|
|
|
import org.jetbrains.annotations.Nullable;
|
2021-03-07 15:50:11 +01:00
|
|
|
import org.warp.filesponge.DiskMetadata.DiskMetadataSerializer;
|
|
|
|
import reactor.core.publisher.Flux;
|
|
|
|
import reactor.core.publisher.Mono;
|
|
|
|
import reactor.core.scheduler.Schedulers;
|
|
|
|
import reactor.util.function.Tuple2;
|
|
|
|
import reactor.util.function.Tuples;
|
|
|
|
|
|
|
|
public class DiskCache implements URLsDiskHandler, URLsWriter {
|
|
|
|
|
2021-05-05 00:08:08 +02:00
|
|
|
private final DiskMetadataSerializer diskMetadataSerializer;
|
2021-03-07 15:50:11 +01:00
|
|
|
|
|
|
|
private final LLKeyValueDatabase db;
|
|
|
|
private final LLDictionary fileContent;
|
|
|
|
private final LLDictionary fileMetadata;
|
|
|
|
|
2021-05-05 00:08:08 +02:00
|
|
|
public DiskCache(LLKeyValueDatabase db, LLDictionary fileContent, LLDictionary fileMetadata) {
|
|
|
|
this.db = db;
|
|
|
|
this.fileContent = fileContent;
|
|
|
|
this.fileMetadata = fileMetadata;
|
|
|
|
this.diskMetadataSerializer = new DiskMetadataSerializer(db.getAllocator());
|
|
|
|
}
|
|
|
|
|
2021-06-27 17:04:48 +02:00
|
|
|
public static Mono<DiskCache> open(LLDatabaseConnection databaseConnection,
|
|
|
|
String dbName,
|
|
|
|
DatabaseOptions databaseOptions) {
|
2021-03-07 15:50:11 +01:00
|
|
|
return databaseConnection
|
2021-06-20 01:07:21 +02:00
|
|
|
.getDatabase(dbName,
|
|
|
|
List.of(Column.dictionary("file-content"), Column.dictionary("file-metadata")),
|
2021-06-27 17:04:48 +02:00
|
|
|
databaseOptions
|
2021-06-20 01:07:21 +02:00
|
|
|
)
|
2021-03-07 15:50:11 +01:00
|
|
|
.flatMap(db -> Mono.zip(
|
|
|
|
Mono.just(db).single(),
|
|
|
|
db.getDictionary("file-content", UpdateMode.ALLOW).single(),
|
|
|
|
db.getDictionary("file-metadata", UpdateMode.ALLOW).single()
|
|
|
|
))
|
|
|
|
.map(tuple -> new DiskCache(tuple.getT1(), tuple.getT2(), tuple.getT3()))
|
|
|
|
.single();
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public Mono<Void> writeMetadata(URL url, Metadata metadata) {
|
2021-08-22 18:20:52 +02:00
|
|
|
Mono<ByteBuf> keyMono = Mono.fromCallable(() -> url.getSerializer(db.getAllocator()).serialize(url));
|
|
|
|
return fileMetadata
|
|
|
|
.update(keyMono, oldValue -> Objects.requireNonNullElseGet(oldValue,
|
|
|
|
() -> diskMetadataSerializer.serialize(new DiskMetadata(metadata.size(),
|
|
|
|
BooleanArrayList.wrap(new boolean[DiskMetadata.getBlocksCount(metadata.size(), BLOCK_SIZE)])
|
|
|
|
))
|
|
|
|
), UpdateReturnMode.NOTHING)
|
2021-03-07 15:50:11 +01:00
|
|
|
.then();
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public Mono<Void> writeContentBlock(URL url, DataBlock dataBlock) {
|
2021-08-22 18:20:52 +02:00
|
|
|
Mono<ByteBuf> urlKeyMono = Mono.fromCallable(() -> url.getSerializer(db.getAllocator()).serialize(url));
|
|
|
|
Mono<ByteBuf> blockKeyMono = Mono.fromCallable(() -> getBlockKey(url, dataBlock.getId()));
|
2021-03-07 15:50:11 +01:00
|
|
|
return Mono
|
2021-05-05 17:31:50 +02:00
|
|
|
.fromCallable(dataBlock::getData)
|
2021-05-03 00:33:04 +02:00
|
|
|
.subscribeOn(Schedulers.boundedElastic())
|
2021-08-22 18:20:52 +02:00
|
|
|
.flatMap(bytes -> {
|
|
|
|
Mono<ByteBuf> bytesMono = Mono.just(bytes).map(ByteBuf::retain);
|
|
|
|
return fileContent
|
|
|
|
.put(blockKeyMono, bytesMono, LLDictionaryResultType.VOID)
|
|
|
|
.doOnNext(ReferenceCounted::release)
|
|
|
|
.then();
|
|
|
|
})
|
|
|
|
.then(fileMetadata.update(urlKeyMono, prevBytes -> {
|
|
|
|
@Nullable DiskMetadata result;
|
|
|
|
if (prevBytes != null) {
|
|
|
|
DiskMetadata prevMeta = diskMetadataSerializer.deserialize(prevBytes);
|
|
|
|
if (!prevMeta.isDownloadedBlock(dataBlock.getId())) {
|
|
|
|
BooleanArrayList bal = prevMeta.downloadedBlocks().clone();
|
|
|
|
if (prevMeta.size() == -1) {
|
|
|
|
if (bal.size() > dataBlock.getId()) {
|
|
|
|
bal.set(dataBlock.getId(), true);
|
|
|
|
} else if (bal.size() == dataBlock.getId()) {
|
|
|
|
bal.add(true);
|
2021-05-16 14:59:12 +02:00
|
|
|
} else {
|
2021-08-22 18:20:52 +02:00
|
|
|
throw new IndexOutOfBoundsException(
|
|
|
|
"Trying to write a block too much far from the last block. Previous total blocks: "
|
|
|
|
+ bal.size() + " Current block id: " + dataBlock.getId());
|
2021-05-16 14:59:12 +02:00
|
|
|
}
|
|
|
|
} else {
|
2021-08-22 18:20:52 +02:00
|
|
|
bal.set(dataBlock.getId(), true);
|
2021-05-16 14:59:12 +02:00
|
|
|
}
|
2021-08-22 18:20:52 +02:00
|
|
|
result = new DiskMetadata(prevMeta.size(), bal);
|
|
|
|
} else {
|
|
|
|
result = prevMeta;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
result = null;
|
|
|
|
}
|
|
|
|
if (result != null) {
|
|
|
|
return diskMetadataSerializer.serialize(result);
|
|
|
|
} else {
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
}, UpdateReturnMode.NOTHING)
|
2021-05-16 14:59:12 +02:00
|
|
|
)
|
2021-03-07 15:50:11 +01:00
|
|
|
.then();
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public Flux<DataBlock> requestContent(URL url) {
|
2021-08-22 18:20:52 +02:00
|
|
|
return this
|
|
|
|
.requestDiskMetadata(url)
|
2021-03-07 15:50:11 +01:00
|
|
|
.filter(DiskMetadata::isDownloadedFully)
|
2021-06-06 02:24:21 +02:00
|
|
|
.flatMapMany(meta -> Flux.fromStream(meta.downloadedBlocks()::stream)
|
2021-03-09 00:35:24 +01:00
|
|
|
.index()
|
|
|
|
// Get only downloaded blocks
|
|
|
|
.filter(Tuple2::getT2)
|
|
|
|
.flatMapSequential(blockMeta -> {
|
|
|
|
int blockId = Math.toIntExact(blockMeta.getT1());
|
|
|
|
boolean downloaded = blockMeta.getT2();
|
|
|
|
if (!downloaded) {
|
2021-08-22 18:20:52 +02:00
|
|
|
return Mono.empty();
|
2021-03-07 15:50:11 +01:00
|
|
|
}
|
2021-08-22 18:20:52 +02:00
|
|
|
Mono<ByteBuf> blockKeyMono = Mono.fromCallable(() -> getBlockKey(url, blockId));
|
|
|
|
return fileContent
|
|
|
|
.get(null, blockKeyMono)
|
2021-05-16 14:59:12 +02:00
|
|
|
.map(data -> {
|
|
|
|
try {
|
|
|
|
int blockOffset = getBlockOffset(blockId);
|
|
|
|
int blockLength = data.readableBytes();
|
2021-06-02 13:21:34 +02:00
|
|
|
if (meta.size() != -1) {
|
|
|
|
if (blockOffset + blockLength >= meta.size()) {
|
|
|
|
if (blockOffset + blockLength > meta.size()) {
|
|
|
|
throw new IllegalStateException("Overflowed data size");
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// Intermediate blocks must be of max size
|
|
|
|
assert data.readableBytes() == BLOCK_SIZE;
|
2021-05-16 14:59:12 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return new DataBlock(blockOffset, blockLength, data.retain());
|
|
|
|
} finally {
|
|
|
|
data.release();
|
2021-05-05 17:31:50 +02:00
|
|
|
}
|
2021-05-16 14:59:12 +02:00
|
|
|
});
|
2021-08-22 18:20:52 +02:00
|
|
|
})
|
|
|
|
);
|
2021-03-07 15:50:11 +01:00
|
|
|
}
|
|
|
|
|
2021-08-22 23:52:51 +02:00
|
|
|
private ByteBuf getBlockKey(URL url, int blockId) throws SerializationException {
|
2021-05-05 00:08:08 +02:00
|
|
|
ByteBuf urlBytes = url.getSerializer(db.getAllocator()).serialize(url);
|
|
|
|
ByteBuf blockIdBytes = this.db.getAllocator().directBuffer(Integer.BYTES, Integer.BYTES);
|
2021-05-03 00:33:04 +02:00
|
|
|
blockIdBytes.writeInt(blockId);
|
|
|
|
return Unpooled.wrappedBuffer(urlBytes, blockIdBytes);
|
2021-03-07 15:50:11 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
private static int getBlockOffset(int blockId) {
|
|
|
|
return blockId * BLOCK_SIZE;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public Mono<DiskMetadata> requestDiskMetadata(URL url) {
|
2021-08-22 18:20:52 +02:00
|
|
|
Mono<ByteBuf> urlKeyMono = Mono.fromCallable(() -> url.getSerializer(db.getAllocator()).serialize(url));
|
|
|
|
return fileMetadata
|
|
|
|
.get(null, urlKeyMono)
|
2021-03-07 15:50:11 +01:00
|
|
|
.map(diskMetadataSerializer::deserialize);
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public Mono<Metadata> requestMetadata(URL url) {
|
|
|
|
return requestDiskMetadata(url)
|
|
|
|
.map(DiskMetadata::asMetadata);
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public Mono<Tuple2<Metadata, Flux<DataBlock>>> request(URL url) {
|
2021-08-22 18:20:52 +02:00
|
|
|
Mono<ByteBuf> urlKeyMono = Mono.fromCallable(() -> url.getSerializer(db.getAllocator()).serialize(url));
|
2021-05-16 14:59:12 +02:00
|
|
|
return Mono
|
|
|
|
.using(
|
|
|
|
() -> url.getSerializer(db.getAllocator()).serialize(url),
|
2021-08-22 18:20:52 +02:00
|
|
|
key -> fileMetadata.get(null, urlKeyMono),
|
2021-05-16 14:59:12 +02:00
|
|
|
ReferenceCounted::release
|
|
|
|
)
|
2021-03-07 15:50:11 +01:00
|
|
|
.map(diskMetadataSerializer::deserialize)
|
|
|
|
.map(diskMeta -> {
|
|
|
|
var meta = diskMeta.asMetadata();
|
|
|
|
if (diskMeta.isDownloadedFully()) {
|
|
|
|
return Tuples.of(meta, this.requestContent(url));
|
|
|
|
} else {
|
|
|
|
return Tuples.of(meta, Flux.empty());
|
|
|
|
}
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
public Mono<Void> close() {
|
|
|
|
return db.close();
|
|
|
|
}
|
|
|
|
}
|