From 74e42605582742257641b77ac3d79b735dbf97a2 Mon Sep 17 00:00:00 2001 From: Andrea Cavalli Date: Mon, 22 Apr 2019 12:18:54 +0200 Subject: [PATCH] Removed blocks logic --- .../strangedb/java/database/DatabaseJava.java | 43 ++++++++----------- .../lists/EnhancedObjectStrangeDbList.java | 29 ++++++++----- .../strangedb/tests/EnhancedClassUpdate.java | 11 ++--- .../strangedb/tests/ObjectListTests.java | 7 +-- .../strangedb/tests/Performance.java | 6 +-- .../cavallium/strangedb/tests/QueryTests.java | 5 +-- .../cavallium/strangedb/utils/NTestUtils.java | 6 ++- strangedb-core | 2 +- 8 files changed, 53 insertions(+), 56 deletions(-) diff --git a/src/main/java/it/cavallium/strangedb/java/database/DatabaseJava.java b/src/main/java/it/cavallium/strangedb/java/database/DatabaseJava.java index b51cc74..a5f3b23 100644 --- a/src/main/java/it/cavallium/strangedb/java/database/DatabaseJava.java +++ b/src/main/java/it/cavallium/strangedb/java/database/DatabaseJava.java @@ -6,12 +6,14 @@ import it.cavallium.strangedb.java.objects.EnhancedObject; import it.cavallium.strangedb.functionalinterfaces.FunctionWithIO; import it.cavallium.strangedb.java.objects.EnhancedObjectIndices; import it.unimi.dsi.fastutil.longs.LongArrayList; +import it.unimi.dsi.fastutil.longs.LongLinkedOpenHashSet; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.util.UUID; import static it.cavallium.strangedb.database.references.DatabaseReferencesMetadata.*; import static it.cavallium.strangedb.java.database.DatabaseObjectsIO.ENHANCED_OBJECT_METADATA_CLEANER; @@ -23,8 +25,8 @@ public class DatabaseJava extends DatabaseCore implements IDatabaseTools { private EnhancedObject loadedRootObject; private boolean hasLoadedRootObject; - public DatabaseJava(Path dataFile, Path blocksMetaFile, Path referencesMetaFile) throws IOException { - super(dataFile, blocksMetaFile, referencesMetaFile); + public DatabaseJava(Path dataFile, Path referencesMetaFile) throws IOException { + super(dataFile, referencesMetaFile); this.databaseTools = this; this.objectsIO = new DatabaseObjectsIO(databaseTools, referencesIO); this.hasLoadedRootObject = false; @@ -86,59 +88,50 @@ public class DatabaseJava extends DatabaseCore implements IDatabaseTools { if (!this.closed) { this.close(); } - Path newDataFile = dataFile.resolveSibling("compressed-data-file.tmp"); - Path newBlocksFile = blocksMetaFile.resolveSibling("compressed-blocks-file.tmp"); - Path newReferencesFile = referencesMetaFile.resolveSibling("compressed-references-file.tmp"); - Path backupDataFile = dataFile.resolveSibling("backup-data.db.bak"); - Path backupBlocksFile = blocksMetaFile.resolveSibling("backup-blocks.dat.bak"); - Path backupReferencesFile = referencesMetaFile.resolveSibling("backup-references.dat.bak"); + UUID uid = UUID.randomUUID(); + Path newDataFile = dataFile.resolveSibling("compressed-data-file-" + uid + ".tmp"); + Path newReferencesFile = referencesMetaFile.resolveSibling("compressed-references-file-" + uid + ".tmp"); + Path backupDataFile = dataFile.resolveSibling("backup-data-" + uid + ".db.bak"); + Path backupReferencesFile = referencesMetaFile.resolveSibling("backup-references-" + uid + ".dat.bak"); Files.copy(dataFile, backupDataFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); - Files.copy(blocksMetaFile, backupBlocksFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); Files.copy(referencesMetaFile, backupReferencesFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); Files.move(dataFile, newDataFile, StandardCopyOption.REPLACE_EXISTING); - Files.move(blocksMetaFile, newBlocksFile, StandardCopyOption.REPLACE_EXISTING); Files.move(referencesMetaFile, newReferencesFile, StandardCopyOption.REPLACE_EXISTING); - DatabaseJava databaseToClean = instantiateNewDatabase(newDataFile, newBlocksFile, newReferencesFile); - DatabaseJava newDatabase = instantiateNewDatabase(dataFile, blocksMetaFile, referencesMetaFile); + DatabaseJava databaseToClean = instantiateNewDatabase(newDataFile, newReferencesFile); + DatabaseJava newDatabase = instantiateNewDatabase(dataFile, referencesMetaFile); long firstFreeReference = databaseToClean.referencesMetadata.getFirstFreeReference(); long referencesCount = 0; - long blocksCount = databaseToClean.blocksMetadata.getTotalBlocksCount(); long writtenReferences = 0; - long writtenBlocks = 0; - LongArrayList idsToKeep = new LongArrayList(); + LongLinkedOpenHashSet idsToKeep = new LongLinkedOpenHashSet((int) (firstFreeReference / 2), 0.75f); cleanRef(databaseToClean, idsToKeep, 0); for (int referenceID = 0; referenceID < firstFreeReference; referenceID++) { try { - ReferenceInfo ref = databaseToClean.referencesMetadata.getCleanReference(referenceID); + ReferenceInfo ref = databaseToClean.referencesMetadata.getReferenceInfo(referenceID); if (!NONEXISTENT_REFERENCE_INFO.equals(ref)) { referencesCount++; if (idsToKeep.contains(referenceID)) { ByteBuffer buffer = databaseToClean.referencesIO.readFromReference(referenceID); newDatabase.referencesIO.writeToReference(referenceID, ref.getCleanerId(), buffer.limit(), buffer); writtenReferences++; - if (buffer.limit() > 0) { - writtenBlocks++; - } } } } catch (IOException ex) { System.out.println("Error while reading reference " + referenceID + ". References written: " + writtenReferences); } } - System.out.println("[Java Cleaner] References written: " + writtenReferences + ". Removed " + (blocksCount - writtenBlocks) + " blocks. Removed " + (referencesCount - writtenReferences) + " references."); + System.out.println("[Java Cleaner] References written: " + writtenReferences + ". Removed " + (referencesCount - writtenReferences) + " references."); databaseToClean.close(); newDatabase.close(); Files.deleteIfExists(newDataFile); - Files.deleteIfExists(newBlocksFile); Files.deleteIfExists(newReferencesFile); } - private void cleanRef(DatabaseJava db, LongArrayList idsToKeep, long ref) throws IOException { + private void cleanRef(DatabaseJava db, LongLinkedOpenHashSet idsToKeep, long ref) throws IOException { idsToKeep.add(ref); - ReferenceInfo refInfo = db.referencesMetadata.getCleanReference(ref); + ReferenceInfo refInfo = db.referencesMetadata.getReferenceInfo(ref); if (!NONEXISTENT_REFERENCE_INFO.equals(refInfo)) { switch (refInfo.getCleanerId()) { case ENHANCED_OBJECT_METADATA_CLEANER: { @@ -177,8 +170,8 @@ public class DatabaseJava extends DatabaseCore implements IDatabaseTools { this.objectsIO.registerClass(type, id); } - protected DatabaseJava instantiateNewDatabase(Path dataFile, Path blocksMetaFile, Path referencesMetaFile) throws IOException { - DatabaseJava newDatabaseJava = new DatabaseJava(dataFile, blocksMetaFile, referencesMetaFile); + protected DatabaseJava instantiateNewDatabase(Path dataFile, Path referencesMetaFile) throws IOException { + DatabaseJava newDatabaseJava = new DatabaseJava(dataFile, referencesMetaFile); this.getObjectsIO().getRegisteredClasses().forEach(newDatabaseJava::registerClass); return newDatabaseJava; } diff --git a/src/main/java/it/cavallium/strangedb/java/objects/lists/EnhancedObjectStrangeDbList.java b/src/main/java/it/cavallium/strangedb/java/objects/lists/EnhancedObjectStrangeDbList.java index 345e01e..ebdb215 100644 --- a/src/main/java/it/cavallium/strangedb/java/objects/lists/EnhancedObjectStrangeDbList.java +++ b/src/main/java/it/cavallium/strangedb/java/objects/lists/EnhancedObjectStrangeDbList.java @@ -128,16 +128,24 @@ public class EnhancedObjectStrangeDbList extends Stran EnhancedObjectStrangeDbList dbList = ((EnhancedObjectStrangeDbList) inputList); dbList.forEachIndexParallelUnsorted((Long elementUid) -> { EnhancedObjectIndices elementUids = dbio.loadEnhancedObjectUids(elementUid); // check if the parent object is the declared type - Class declaredRootType = query.valuePointer.getRootType(); - Class obtainedRootType = elementUids.type; - if (isInstanceOf(obtainedRootType, declaredRootType)) { - List result = resolveItemFromDb(query.valuePointer, dbio, elementUids); - if (result.parallelStream().anyMatch(query.valueOperation::evaluate)) { - results.add(elementUids); + if (elementUids != null) { + Class declaredRootType = query.valuePointer.getRootType(); + Class obtainedRootType = elementUids.type; + if (isInstanceOf(obtainedRootType, declaredRootType)) { + List result = resolveItemFromDb(query.valuePointer, dbio, elementUids); + if (result.size() == 1) { + if (query.valueOperation.evaluate(result.get(0))) { + results.add(elementUids); + } + } else if (result.size() > 1) { + if (result.parallelStream().anyMatch(query.valueOperation::evaluate)) { + results.add(elementUids); + } + } + } else { + //todo: use logging api + System.err.println(obtainedRootType.getSimpleName() + " is not instance of " + declaredRootType.getSimpleName()); } - } else { - //todo: use logging api - System.err.println(obtainedRootType.getSimpleName() + " is not instance of " + declaredRootType.getSimpleName()); } }); } else if (inputList instanceof ElementsArrayList) { @@ -158,7 +166,7 @@ public class EnhancedObjectStrangeDbList extends Stran @NotNull - private static List resolveItemFromDbLoop(EnhancedObjectIndices currentElement, ValuePointer pointer, int pointerPosition, int pointerSize, IObjectsIO objectsIO, EnhancedObjectIndices element) throws IOException { + private static List resolveItemFromDbLoop(@NotNull EnhancedObjectIndices currentElement, ValuePointer pointer, int pointerPosition, int pointerSize, IObjectsIO objectsIO, EnhancedObjectIndices element) throws IOException { if (pointerPosition > 0) { // check if the object that we obtained is the declared type Class declaredType = pointer.resetPointerTo(pointerPosition - 1).getAdditionalData(); @@ -279,6 +287,7 @@ public class EnhancedObjectStrangeDbList extends Stran } if (multipleCurrentElements == null) { + if (currentElement == null) return ObjectLists.emptyList(); return resolveItemFromDbLoop(currentElement, pointer, pointerPosition + 1, pointerSize, objectsIO, element); } else { try { diff --git a/src/test/java/it/cavallium/strangedb/tests/EnhancedClassUpdate.java b/src/test/java/it/cavallium/strangedb/tests/EnhancedClassUpdate.java index 4f0cfde..32d94e9 100644 --- a/src/test/java/it/cavallium/strangedb/tests/EnhancedClassUpdate.java +++ b/src/test/java/it/cavallium/strangedb/tests/EnhancedClassUpdate.java @@ -15,16 +15,16 @@ import static org.junit.Assert.assertEquals; public class EnhancedClassUpdate { private Path path1; - private Path path2; private Path path3; private DatabaseJava db; @Before public void setUp() throws Exception { path1 = Files.createTempFile("db-tests-data-", ".db"); - path2 = Files.createTempFile("db-tests-blocks-", ".db"); path3 = Files.createTempFile("db-tests-references-", ".db"); - db = new DatabaseJava(path1, path2, path3); + db = new DatabaseJava(path1, path3); + db.registerClass(V2Class.class, 0); + db.registerClass(OldClass.class, 1); OldClass root = db.loadRoot(OldClass::new); root.field1 = "Abc"; root.field2 = 12; @@ -34,7 +34,9 @@ public class EnhancedClassUpdate { @Test public void shouldUpdateClass() throws IOException { - db = new DatabaseJava(path1, path2, path3); + db = new DatabaseJava(path1, path3); + db.registerClass(V2Class.class, 0); + db.registerClass(OldClass.class, 1); V2Class root = db.loadRoot(V2Class::new, V2Class.class); assertEquals(root.field4, "Abc"); assertEquals(root.field2, 12); @@ -45,7 +47,6 @@ public class EnhancedClassUpdate { @After public void tearDown() throws Exception { Files.deleteIfExists(path1); - Files.deleteIfExists(path2); Files.deleteIfExists(path3); } } diff --git a/src/test/java/it/cavallium/strangedb/tests/ObjectListTests.java b/src/test/java/it/cavallium/strangedb/tests/ObjectListTests.java index 6fd0f55..45fcc98 100644 --- a/src/test/java/it/cavallium/strangedb/tests/ObjectListTests.java +++ b/src/test/java/it/cavallium/strangedb/tests/ObjectListTests.java @@ -22,16 +22,14 @@ import static org.junit.Assert.assertEquals; public class ObjectListTests { private Path path1; - private Path path2; private Path path3; private DatabaseJava db; @Before public void setUp() throws Exception { path1 = Files.createTempFile("db-tests-data-", ".db"); - path2 = Files.createTempFile("db-tests-blocks-", ".db"); path3 = Files.createTempFile("db-tests-references-", ".db"); - db = new DatabaseJava(path1, path2, path3); + db = new DatabaseJava(path1, path3); registerClasses(); ListsRoot root = db.loadRoot(ListsRoot::new); for (int i = 0; i < 500; i++) { @@ -66,7 +64,7 @@ public class ObjectListTests { @Test public void shouldUpdateClass() throws IOException { - db = new DatabaseJava(path1, path2, path3); + db = new DatabaseJava(path1, path3); registerClasses(); ListsRoot root = db.loadRoot(ListsRoot::new); @@ -90,7 +88,6 @@ public class ObjectListTests { @After public void tearDown() throws Exception { Files.deleteIfExists(path1); - Files.deleteIfExists(path2); Files.deleteIfExists(path3); } diff --git a/src/test/java/it/cavallium/strangedb/tests/Performance.java b/src/test/java/it/cavallium/strangedb/tests/Performance.java index d32985d..1c41ba0 100644 --- a/src/test/java/it/cavallium/strangedb/tests/Performance.java +++ b/src/test/java/it/cavallium/strangedb/tests/Performance.java @@ -28,7 +28,6 @@ public class Performance { private static Path rootDirectory; private static Path dbDataFile; - private static Path dbBlocksFile; private static Path dbReferencesFile; private static DatabaseJava db; private static boolean tempDirectory; @@ -339,13 +338,11 @@ public class Performance { public static void generateDb() throws IOException { dbDataFile = rootDirectory.resolve("db_data.dat"); - dbBlocksFile = rootDirectory.resolve("db_blocks.dat"); dbReferencesFile = rootDirectory.resolve("db_references.dat"); deleteDbFolders(); Files.createFile(dbDataFile); - Files.createFile(dbBlocksFile); Files.createFile(dbReferencesFile); - db = new DatabaseJava(dbDataFile, dbBlocksFile, dbReferencesFile); + db = new DatabaseJava(dbDataFile, dbReferencesFile); int i = 0; db.registerClass(SimpleEnhancedObject.class, i++); db.registerClass(PreloadedListContainer.class, i++); @@ -364,7 +361,6 @@ public class Performance { public static void deleteDbFolders() throws IOException { Files.deleteIfExists(dbDataFile); - Files.deleteIfExists(dbBlocksFile); Files.deleteIfExists(dbReferencesFile); } diff --git a/src/test/java/it/cavallium/strangedb/tests/QueryTests.java b/src/test/java/it/cavallium/strangedb/tests/QueryTests.java index 01673f6..7b4fbbe 100644 --- a/src/test/java/it/cavallium/strangedb/tests/QueryTests.java +++ b/src/test/java/it/cavallium/strangedb/tests/QueryTests.java @@ -22,16 +22,14 @@ public class QueryTests { public static final String constantUsername = "is this an username?"; public static final String constantBio = "and is this a bio??? Are you mad?"; private Path path1; - private Path path2; private Path path3; private DatabaseJava db; @Before public void setUp() throws Exception { path1 = Files.createTempFile("db-tests-data-", ".db"); - path2 = Files.createTempFile("db-tests-blocks-", ".db"); path3 = Files.createTempFile("db-tests-references-", ".db"); - db = new DatabaseJava(path1, path2, path3); + db = new DatabaseJava(path1, path3); int i = 0; db.registerClass(UserFullInfo.class, i++); db.registerClass(User.class, i++); @@ -58,7 +56,6 @@ public class QueryTests { @After public void tearDown() throws Exception { Files.deleteIfExists(path1); - Files.deleteIfExists(path2); Files.deleteIfExists(path3); } } diff --git a/src/test/java/it/cavallium/strangedb/utils/NTestUtils.java b/src/test/java/it/cavallium/strangedb/utils/NTestUtils.java index 4c07e04..939eb58 100644 --- a/src/test/java/it/cavallium/strangedb/utils/NTestUtils.java +++ b/src/test/java/it/cavallium/strangedb/utils/NTestUtils.java @@ -4,6 +4,8 @@ import it.cavallium.strangedb.database.DatabaseCore; import it.cavallium.strangedb.functionalinterfaces.RunnableWithIO; import it.cavallium.strangedb.java.annotations.*; import it.cavallium.strangedb.java.database.DatabaseJava; +import it.cavallium.strangedb.tests.Clean; +import it.cavallium.strangedb.tests.MultipleEnhancedObjects; import it.unimi.dsi.fastutil.longs.LongArrayList; import it.cavallium.strangedb.java.objects.EnhancedObject; import it.cavallium.strangedb.java.database.IDatabaseTools; @@ -47,11 +49,13 @@ public class NTestUtils { } private DatabaseJava openDatabase() throws IOException { - DatabaseJava db = new DatabaseJava(tempDir.resolve(Paths.get("data.db")), tempDir.resolve(Paths.get("blocks.dat")), tempDir.resolve(Paths.get("references.dat"))); + DatabaseJava db = new DatabaseJava(tempDir.resolve(Paths.get("data.db")), tempDir.resolve(Paths.get("references.dat"))); int id = 0; db.registerClass(RootClass.class, id++); db.registerClass(LongArrayList.class, id++); db.registerClass(NSimplestClass.class, id++); + db.registerClass(Clean.RootTwoClasses.class, id++); + db.registerClass(MultipleEnhancedObjects.RootTwoClasses.class, id++); return db; } diff --git a/strangedb-core b/strangedb-core index dc56da3..7fd7497 160000 --- a/strangedb-core +++ b/strangedb-core @@ -1 +1 @@ -Subproject commit dc56da3bbb4bac11af20f2d52bd974c85c5fe925 +Subproject commit 7fd7497464b5d5e92745974ed12ff4a9f51919d0