andreacavalli
/
strangedb
1
1
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Removed blocks logic

This commit is contained in:
Andrea Cavalli 2019-04-22 12:18:54 +02:00
parent 6d531ee1c4
commit 74e4260558
8 changed files with 53 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
src/main/java/it/cavallium/strangedb/java/database/DatabaseJava.java
View File

@ -6,12 +6,14 @@ import it.cavallium.strangedb.java.objects.EnhancedObject;
import it.cavallium.strangedb.functionalinterfaces.FunctionWithIO;
import it.cavallium.strangedb.java.objects.EnhancedObjectIndices;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.longs.LongLinkedOpenHashSet;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.UUID;
import static it.cavallium.strangedb.database.references.DatabaseReferencesMetadata.*;
import static it.cavallium.strangedb.java.database.DatabaseObjectsIO.ENHANCED_OBJECT_METADATA_CLEANER;
@ -23,8 +25,8 @@ public class DatabaseJava extends DatabaseCore implements IDatabaseTools {
private EnhancedObject loadedRootObject;
private boolean hasLoadedRootObject;
public DatabaseJava(Path dataFile, Path blocksMetaFile, Path referencesMetaFile) throws IOException {
super(dataFile, blocksMetaFile, referencesMetaFile);
public DatabaseJava(Path dataFile, Path referencesMetaFile) throws IOException {
super(dataFile, referencesMetaFile);
this.databaseTools = this;
this.objectsIO = new DatabaseObjectsIO(databaseTools, referencesIO);
this.hasLoadedRootObject = false;
@ -86,59 +88,50 @@ public class DatabaseJava extends DatabaseCore implements IDatabaseTools {
if (!this.closed) {
this.close();
}
Path newDataFile = dataFile.resolveSibling("compressed-data-file.tmp");
Path newBlocksFile = blocksMetaFile.resolveSibling("compressed-blocks-file.tmp");
Path newReferencesFile = referencesMetaFile.resolveSibling("compressed-references-file.tmp");
Path backupDataFile = dataFile.resolveSibling("backup-data.db.bak");
Path backupBlocksFile = blocksMetaFile.resolveSibling("backup-blocks.dat.bak");
Path backupReferencesFile = referencesMetaFile.resolveSibling("backup-references.dat.bak");
UUID uid = UUID.randomUUID();
Path newDataFile = dataFile.resolveSibling("compressed-data-file-" + uid + ".tmp");
Path newReferencesFile = referencesMetaFile.resolveSibling("compressed-references-file-" + uid + ".tmp");
Path backupDataFile = dataFile.resolveSibling("backup-data-" + uid + ".db.bak");
Path backupReferencesFile = referencesMetaFile.resolveSibling("backup-references-" + uid + ".dat.bak");
Files.copy(dataFile, backupDataFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES);
Files.copy(blocksMetaFile, backupBlocksFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES);
Files.copy(referencesMetaFile, backupReferencesFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES);
Files.move(dataFile, newDataFile, StandardCopyOption.REPLACE_EXISTING);
Files.move(blocksMetaFile, newBlocksFile, StandardCopyOption.REPLACE_EXISTING);
Files.move(referencesMetaFile, newReferencesFile, StandardCopyOption.REPLACE_EXISTING);
DatabaseJava databaseToClean = instantiateNewDatabase(newDataFile, newBlocksFile, newReferencesFile);
DatabaseJava newDatabase = instantiateNewDatabase(dataFile, blocksMetaFile, referencesMetaFile);
DatabaseJava databaseToClean = instantiateNewDatabase(newDataFile, newReferencesFile);
DatabaseJava newDatabase = instantiateNewDatabase(dataFile, referencesMetaFile);
long firstFreeReference = databaseToClean.referencesMetadata.getFirstFreeReference();
long referencesCount = 0;
long blocksCount = databaseToClean.blocksMetadata.getTotalBlocksCount();
long writtenReferences = 0;
long writtenBlocks = 0;
LongArrayList idsToKeep = new LongArrayList();
LongLinkedOpenHashSet idsToKeep = new LongLinkedOpenHashSet((int) (firstFreeReference / 2), 0.75f);
cleanRef(databaseToClean, idsToKeep, 0);
for (int referenceID = 0; referenceID < firstFreeReference; referenceID++) {
try {
ReferenceInfo ref = databaseToClean.referencesMetadata.getCleanReference(referenceID);
ReferenceInfo ref = databaseToClean.referencesMetadata.getReferenceInfo(referenceID);
if (!NONEXISTENT_REFERENCE_INFO.equals(ref)) {
referencesCount++;
if (idsToKeep.contains(referenceID)) {
ByteBuffer buffer = databaseToClean.referencesIO.readFromReference(referenceID);
newDatabase.referencesIO.writeToReference(referenceID, ref.getCleanerId(), buffer.limit(), buffer);
writtenReferences++;
if (buffer.limit() > 0) {
writtenBlocks++;
}
}
}
} catch (IOException ex) {
System.out.println("Error while reading reference " + referenceID + ". References written: " + writtenReferences);
}
}
System.out.println("[Java Cleaner] References written: " + writtenReferences + ". Removed " + (blocksCount - writtenBlocks) + " blocks. Removed " + (referencesCount - writtenReferences) + " references.");
System.out.println("[Java Cleaner] References written: " + writtenReferences + ". Removed " + (referencesCount - writtenReferences) + " references.");
databaseToClean.close();
newDatabase.close();
Files.deleteIfExists(newDataFile);
Files.deleteIfExists(newBlocksFile);
Files.deleteIfExists(newReferencesFile);
}
private void cleanRef(DatabaseJava db, LongArrayList idsToKeep, long ref) throws IOException {
private void cleanRef(DatabaseJava db, LongLinkedOpenHashSet idsToKeep, long ref) throws IOException {
idsToKeep.add(ref);
ReferenceInfo refInfo = db.referencesMetadata.getCleanReference(ref);
ReferenceInfo refInfo = db.referencesMetadata.getReferenceInfo(ref);
if (!NONEXISTENT_REFERENCE_INFO.equals(refInfo)) {
switch (refInfo.getCleanerId()) {
case ENHANCED_OBJECT_METADATA_CLEANER: {
@ -177,8 +170,8 @@ public class DatabaseJava extends DatabaseCore implements IDatabaseTools {
this.objectsIO.registerClass(type, id);
}
protected DatabaseJava instantiateNewDatabase(Path dataFile, Path blocksMetaFile, Path referencesMetaFile) throws IOException {
DatabaseJava newDatabaseJava = new DatabaseJava(dataFile, blocksMetaFile, referencesMetaFile);
protected DatabaseJava instantiateNewDatabase(Path dataFile, Path referencesMetaFile) throws IOException {
DatabaseJava newDatabaseJava = new DatabaseJava(dataFile, referencesMetaFile);
this.getObjectsIO().getRegisteredClasses().forEach(newDatabaseJava::registerClass);
return newDatabaseJava;
}

29
src/main/java/it/cavallium/strangedb/java/objects/lists/EnhancedObjectStrangeDbList.java
View File

@ -128,16 +128,24 @@ public class EnhancedObjectStrangeDbList<T extends EnhancedObject> extends Stran
EnhancedObjectStrangeDbList<?> dbList = ((EnhancedObjectStrangeDbList<?>) inputList);
dbList.forEachIndexParallelUnsorted((Long elementUid) -> {
EnhancedObjectIndices elementUids = dbio.loadEnhancedObjectUids(elementUid); // check if the parent object is the declared type
Class<?> declaredRootType = query.valuePointer.getRootType();
Class<?> obtainedRootType = elementUids.type;
if (isInstanceOf(obtainedRootType, declaredRootType)) {
List<Object> result = resolveItemFromDb(query.valuePointer, dbio, elementUids);
if (result.parallelStream().anyMatch(query.valueOperation::evaluate)) {
results.add(elementUids);
if (elementUids != null) {
Class<?> declaredRootType = query.valuePointer.getRootType();
Class<?> obtainedRootType = elementUids.type;
if (isInstanceOf(obtainedRootType, declaredRootType)) {
List<Object> result = resolveItemFromDb(query.valuePointer, dbio, elementUids);
if (result.size() == 1) {
if (query.valueOperation.evaluate(result.get(0))) {
results.add(elementUids);
}
} else if (result.size() > 1) {
if (result.parallelStream().anyMatch(query.valueOperation::evaluate)) {
results.add(elementUids);
}
}
} else {
//todo: use logging api
System.err.println(obtainedRootType.getSimpleName() + " is not instance of " + declaredRootType.getSimpleName());
}
} else {
//todo: use logging api
System.err.println(obtainedRootType.getSimpleName() + " is not instance of " + declaredRootType.getSimpleName());
}
});
} else if (inputList instanceof ElementsArrayList<?>) {
@ -158,7 +166,7 @@ public class EnhancedObjectStrangeDbList<T extends EnhancedObject> extends Stran
@NotNull
private static List<Object> resolveItemFromDbLoop(EnhancedObjectIndices currentElement, ValuePointer pointer, int pointerPosition, int pointerSize, IObjectsIO objectsIO, EnhancedObjectIndices element) throws IOException {
private static List<Object> resolveItemFromDbLoop(@NotNull EnhancedObjectIndices currentElement, ValuePointer pointer, int pointerPosition, int pointerSize, IObjectsIO objectsIO, EnhancedObjectIndices element) throws IOException {
if (pointerPosition > 0) {
// check if the object that we obtained is the declared type
Class<?> declaredType = pointer.resetPointerTo(pointerPosition - 1).getAdditionalData();
@ -279,6 +287,7 @@ public class EnhancedObjectStrangeDbList<T extends EnhancedObject> extends Stran
}
if (multipleCurrentElements == null) {
if (currentElement == null) return ObjectLists.emptyList();
return resolveItemFromDbLoop(currentElement, pointer, pointerPosition + 1, pointerSize, objectsIO, element);
} else {
try {

11
src/test/java/it/cavallium/strangedb/tests/EnhancedClassUpdate.java
View File

@ -15,16 +15,16 @@ import static org.junit.Assert.assertEquals;
public class EnhancedClassUpdate {
private Path path1;
private Path path2;
private Path path3;
private DatabaseJava db;
@Before
public void setUp() throws Exception {
path1 = Files.createTempFile("db-tests-data-", ".db");
path2 = Files.createTempFile("db-tests-blocks-", ".db");
path3 = Files.createTempFile("db-tests-references-", ".db");
db = new DatabaseJava(path1, path2, path3);
db = new DatabaseJava(path1, path3);
db.registerClass(V2Class.class, 0);
db.registerClass(OldClass.class, 1);
OldClass root = db.loadRoot(OldClass::new);
root.field1 = "Abc";
root.field2 = 12;
@ -34,7 +34,9 @@ public class EnhancedClassUpdate {
@Test
public void shouldUpdateClass() throws IOException {
db = new DatabaseJava(path1, path2, path3);
db = new DatabaseJava(path1, path3);
db.registerClass(V2Class.class, 0);
db.registerClass(OldClass.class, 1);
V2Class root = db.loadRoot(V2Class::new, V2Class.class);
assertEquals(root.field4, "Abc");
assertEquals(root.field2, 12);
@ -45,7 +47,6 @@ public class EnhancedClassUpdate {
@After
public void tearDown() throws Exception {
Files.deleteIfExists(path1);
Files.deleteIfExists(path2);
Files.deleteIfExists(path3);
}
}

7
src/test/java/it/cavallium/strangedb/tests/ObjectListTests.java
View File

@ -22,16 +22,14 @@ import static org.junit.Assert.assertEquals;
public class ObjectListTests {
private Path path1;
private Path path2;
private Path path3;
private DatabaseJava db;
@Before
public void setUp() throws Exception {
path1 = Files.createTempFile("db-tests-data-", ".db");
path2 = Files.createTempFile("db-tests-blocks-", ".db");
path3 = Files.createTempFile("db-tests-references-", ".db");
db = new DatabaseJava(path1, path2, path3);
db = new DatabaseJava(path1, path3);
registerClasses();
ListsRoot root = db.loadRoot(ListsRoot::new);
for (int i = 0; i < 500; i++) {
@ -66,7 +64,7 @@ public class ObjectListTests {
@Test
public void shouldUpdateClass() throws IOException {
db = new DatabaseJava(path1, path2, path3);
db = new DatabaseJava(path1, path3);
registerClasses();
ListsRoot root = db.loadRoot(ListsRoot::new);
@ -90,7 +88,6 @@ public class ObjectListTests {
@After
public void tearDown() throws Exception {
Files.deleteIfExists(path1);
Files.deleteIfExists(path2);
Files.deleteIfExists(path3);
}

6
src/test/java/it/cavallium/strangedb/tests/Performance.java
View File

@ -28,7 +28,6 @@ public class Performance {
private static Path rootDirectory;
private static Path dbDataFile;
private static Path dbBlocksFile;
private static Path dbReferencesFile;
private static DatabaseJava db;
private static boolean tempDirectory;
@ -339,13 +338,11 @@ public class Performance {
public static void generateDb() throws IOException {
dbDataFile = rootDirectory.resolve("db_data.dat");
dbBlocksFile = rootDirectory.resolve("db_blocks.dat");
dbReferencesFile = rootDirectory.resolve("db_references.dat");
deleteDbFolders();
Files.createFile(dbDataFile);
Files.createFile(dbBlocksFile);
Files.createFile(dbReferencesFile);
db = new DatabaseJava(dbDataFile, dbBlocksFile, dbReferencesFile);
db = new DatabaseJava(dbDataFile, dbReferencesFile);
int i = 0;
db.registerClass(SimpleEnhancedObject.class, i++);
db.registerClass(PreloadedListContainer.class, i++);
@ -364,7 +361,6 @@ public class Performance {
public static void deleteDbFolders() throws IOException {
Files.deleteIfExists(dbDataFile);
Files.deleteIfExists(dbBlocksFile);
Files.deleteIfExists(dbReferencesFile);
}

5
src/test/java/it/cavallium/strangedb/tests/QueryTests.java
View File

@ -22,16 +22,14 @@ public class QueryTests {
public static final String constantUsername = "is this an username?";
public static final String constantBio = "and is this a bio??? Are you mad?";
private Path path1;
private Path path2;
private Path path3;
private DatabaseJava db;
@Before
public void setUp() throws Exception {
path1 = Files.createTempFile("db-tests-data-", ".db");
path2 = Files.createTempFile("db-tests-blocks-", ".db");
path3 = Files.createTempFile("db-tests-references-", ".db");
db = new DatabaseJava(path1, path2, path3);
db = new DatabaseJava(path1, path3);
int i = 0;
db.registerClass(UserFullInfo.class, i++);
db.registerClass(User.class, i++);
@ -58,7 +56,6 @@ public class QueryTests {
@After
public void tearDown() throws Exception {
Files.deleteIfExists(path1);
Files.deleteIfExists(path2);
Files.deleteIfExists(path3);
}
}

6
src/test/java/it/cavallium/strangedb/utils/NTestUtils.java
View File

@ -4,6 +4,8 @@ import it.cavallium.strangedb.database.DatabaseCore;
import it.cavallium.strangedb.functionalinterfaces.RunnableWithIO;
import it.cavallium.strangedb.java.annotations.*;
import it.cavallium.strangedb.java.database.DatabaseJava;
import it.cavallium.strangedb.tests.Clean;
import it.cavallium.strangedb.tests.MultipleEnhancedObjects;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.cavallium.strangedb.java.objects.EnhancedObject;
import it.cavallium.strangedb.java.database.IDatabaseTools;
@ -47,11 +49,13 @@ public class NTestUtils {
}
private DatabaseJava openDatabase() throws IOException {
DatabaseJava db = new DatabaseJava(tempDir.resolve(Paths.get("data.db")), tempDir.resolve(Paths.get("blocks.dat")), tempDir.resolve(Paths.get("references.dat")));
DatabaseJava db = new DatabaseJava(tempDir.resolve(Paths.get("data.db")), tempDir.resolve(Paths.get("references.dat")));
int id = 0;
db.registerClass(RootClass.class, id++);
db.registerClass(LongArrayList.class, id++);
db.registerClass(NSimplestClass.class, id++);
db.registerClass(Clean.RootTwoClasses.class, id++);
db.registerClass(MultipleEnhancedObjects.RootTwoClasses.class, id++);
return db;
}

2
strangedb-core

@ -1 +1 @@
Subproject commit dc56da3bbb4bac11af20f2d52bd974c85c5fe925
Subproject commit 7fd7497464b5d5e92745974ed12ff4a9f51919d0