andreacavalli
/
strangedb
1
1
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Removed blocks logic

This commit is contained in:
Andrea Cavalli 2019-04-22 12:18:54 +02:00
parent 6d531ee1c4
commit 74e4260558
8 changed files with 53 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
src/main/java/it/cavallium/strangedb/java/database/DatabaseJava.java
View File

@ -6,12 +6,14 @@ import it.cavallium.strangedb.java.objects.EnhancedObject;
import it.cavallium.strangedb.functionalinterfaces.FunctionWithIO; import it.cavallium.strangedb.functionalinterfaces.FunctionWithIO;
import it.cavallium.strangedb.java.objects.EnhancedObjectIndices; import it.cavallium.strangedb.java.objects.EnhancedObjectIndices;
import it.unimi.dsi.fastutil.longs.LongArrayList; import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.longs.LongLinkedOpenHashSet;
import java.io.IOException; import java.io.IOException;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.StandardCopyOption; import java.nio.file.StandardCopyOption;
import java.util.UUID;
import static it.cavallium.strangedb.database.references.DatabaseReferencesMetadata.*; import static it.cavallium.strangedb.database.references.DatabaseReferencesMetadata.*;
import static it.cavallium.strangedb.java.database.DatabaseObjectsIO.ENHANCED_OBJECT_METADATA_CLEANER; import static it.cavallium.strangedb.java.database.DatabaseObjectsIO.ENHANCED_OBJECT_METADATA_CLEANER;
@ -23,8 +25,8 @@ public class DatabaseJava extends DatabaseCore implements IDatabaseTools {
private EnhancedObject loadedRootObject; private EnhancedObject loadedRootObject;
private boolean hasLoadedRootObject; private boolean hasLoadedRootObject;
public DatabaseJava(Path dataFile, Path blocksMetaFile, Path referencesMetaFile) throws IOException { public DatabaseJava(Path dataFile, Path referencesMetaFile) throws IOException {
super(dataFile, blocksMetaFile, referencesMetaFile); super(dataFile, referencesMetaFile);
this.databaseTools = this; this.databaseTools = this;
this.objectsIO = new DatabaseObjectsIO(databaseTools, referencesIO); this.objectsIO = new DatabaseObjectsIO(databaseTools, referencesIO);
this.hasLoadedRootObject = false; this.hasLoadedRootObject = false;
@ -86,59 +88,50 @@ public class DatabaseJava extends DatabaseCore implements IDatabaseTools {
if (!this.closed) { if (!this.closed) {
this.close(); this.close();
} }
Path newDataFile = dataFile.resolveSibling("compressed-data-file.tmp"); UUID uid = UUID.randomUUID();
Path newBlocksFile = blocksMetaFile.resolveSibling("compressed-blocks-file.tmp"); Path newDataFile = dataFile.resolveSibling("compressed-data-file-" + uid + ".tmp");
Path newReferencesFile = referencesMetaFile.resolveSibling("compressed-references-file.tmp"); Path newReferencesFile = referencesMetaFile.resolveSibling("compressed-references-file-" + uid + ".tmp");
Path backupDataFile = dataFile.resolveSibling("backup-data.db.bak"); Path backupDataFile = dataFile.resolveSibling("backup-data-" + uid + ".db.bak");
Path backupBlocksFile = blocksMetaFile.resolveSibling("backup-blocks.dat.bak"); Path backupReferencesFile = referencesMetaFile.resolveSibling("backup-references-" + uid + ".dat.bak");
Path backupReferencesFile = referencesMetaFile.resolveSibling("backup-references.dat.bak");
Files.copy(dataFile, backupDataFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); Files.copy(dataFile, backupDataFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES);
Files.copy(blocksMetaFile, backupBlocksFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES);
Files.copy(referencesMetaFile, backupReferencesFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); Files.copy(referencesMetaFile, backupReferencesFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES);
Files.move(dataFile, newDataFile, StandardCopyOption.REPLACE_EXISTING); Files.move(dataFile, newDataFile, StandardCopyOption.REPLACE_EXISTING);
Files.move(blocksMetaFile, newBlocksFile, StandardCopyOption.REPLACE_EXISTING);
Files.move(referencesMetaFile, newReferencesFile, StandardCopyOption.REPLACE_EXISTING); Files.move(referencesMetaFile, newReferencesFile, StandardCopyOption.REPLACE_EXISTING);
DatabaseJava databaseToClean = instantiateNewDatabase(newDataFile, newBlocksFile, newReferencesFile); DatabaseJava databaseToClean = instantiateNewDatabase(newDataFile, newReferencesFile);
DatabaseJava newDatabase = instantiateNewDatabase(dataFile, blocksMetaFile, referencesMetaFile); DatabaseJava newDatabase = instantiateNewDatabase(dataFile, referencesMetaFile);
long firstFreeReference = databaseToClean.referencesMetadata.getFirstFreeReference(); long firstFreeReference = databaseToClean.referencesMetadata.getFirstFreeReference();
long referencesCount = 0; long referencesCount = 0;
long blocksCount = databaseToClean.blocksMetadata.getTotalBlocksCount();
long writtenReferences = 0; long writtenReferences = 0;
long writtenBlocks = 0;
LongArrayList idsToKeep = new LongArrayList(); LongLinkedOpenHashSet idsToKeep = new LongLinkedOpenHashSet((int) (firstFreeReference / 2), 0.75f);
cleanRef(databaseToClean, idsToKeep, 0); cleanRef(databaseToClean, idsToKeep, 0);
for (int referenceID = 0; referenceID < firstFreeReference; referenceID++) { for (int referenceID = 0; referenceID < firstFreeReference; referenceID++) {
try { try {
ReferenceInfo ref = databaseToClean.referencesMetadata.getCleanReference(referenceID); ReferenceInfo ref = databaseToClean.referencesMetadata.getReferenceInfo(referenceID);
if (!NONEXISTENT_REFERENCE_INFO.equals(ref)) { if (!NONEXISTENT_REFERENCE_INFO.equals(ref)) {
referencesCount++; referencesCount++;
if (idsToKeep.contains(referenceID)) { if (idsToKeep.contains(referenceID)) {
ByteBuffer buffer = databaseToClean.referencesIO.readFromReference(referenceID); ByteBuffer buffer = databaseToClean.referencesIO.readFromReference(referenceID);
newDatabase.referencesIO.writeToReference(referenceID, ref.getCleanerId(), buffer.limit(), buffer); newDatabase.referencesIO.writeToReference(referenceID, ref.getCleanerId(), buffer.limit(), buffer);
writtenReferences++; writtenReferences++;
if (buffer.limit() > 0) {
writtenBlocks++;
}
} }
} }
} catch (IOException ex) { } catch (IOException ex) {
System.out.println("Error while reading reference " + referenceID + ". References written: " + writtenReferences); System.out.println("Error while reading reference " + referenceID + ". References written: " + writtenReferences);
} }
} }
System.out.println("[Java Cleaner] References written: " + writtenReferences + ". Removed " + (blocksCount - writtenBlocks) + " blocks. Removed " + (referencesCount - writtenReferences) + " references."); System.out.println("[Java Cleaner] References written: " + writtenReferences + ". Removed " + (referencesCount - writtenReferences) + " references.");
databaseToClean.close(); databaseToClean.close();
newDatabase.close(); newDatabase.close();
Files.deleteIfExists(newDataFile); Files.deleteIfExists(newDataFile);
Files.deleteIfExists(newBlocksFile);
Files.deleteIfExists(newReferencesFile); Files.deleteIfExists(newReferencesFile);
} }
private void cleanRef(DatabaseJava db, LongArrayList idsToKeep, long ref) throws IOException { private void cleanRef(DatabaseJava db, LongLinkedOpenHashSet idsToKeep, long ref) throws IOException {
idsToKeep.add(ref); idsToKeep.add(ref);
ReferenceInfo refInfo = db.referencesMetadata.getCleanReference(ref); ReferenceInfo refInfo = db.referencesMetadata.getReferenceInfo(ref);
if (!NONEXISTENT_REFERENCE_INFO.equals(refInfo)) { if (!NONEXISTENT_REFERENCE_INFO.equals(refInfo)) {
switch (refInfo.getCleanerId()) { switch (refInfo.getCleanerId()) {
case ENHANCED_OBJECT_METADATA_CLEANER: { case ENHANCED_OBJECT_METADATA_CLEANER: {
@ -177,8 +170,8 @@ public class DatabaseJava extends DatabaseCore implements IDatabaseTools {
this.objectsIO.registerClass(type, id); this.objectsIO.registerClass(type, id);
} }
protected DatabaseJava instantiateNewDatabase(Path dataFile, Path blocksMetaFile, Path referencesMetaFile) throws IOException { protected DatabaseJava instantiateNewDatabase(Path dataFile, Path referencesMetaFile) throws IOException {
DatabaseJava newDatabaseJava = new DatabaseJava(dataFile, blocksMetaFile, referencesMetaFile); DatabaseJava newDatabaseJava = new DatabaseJava(dataFile, referencesMetaFile);
this.getObjectsIO().getRegisteredClasses().forEach(newDatabaseJava::registerClass); this.getObjectsIO().getRegisteredClasses().forEach(newDatabaseJava::registerClass);
return newDatabaseJava; return newDatabaseJava;
} }

29
src/main/java/it/cavallium/strangedb/java/objects/lists/EnhancedObjectStrangeDbList.java
View File

@ -128,16 +128,24 @@ public class EnhancedObjectStrangeDbList<T extends EnhancedObject> extends Stran
EnhancedObjectStrangeDbList<?> dbList = ((EnhancedObjectStrangeDbList<?>) inputList); EnhancedObjectStrangeDbList<?> dbList = ((EnhancedObjectStrangeDbList<?>) inputList);
dbList.forEachIndexParallelUnsorted((Long elementUid) -> { dbList.forEachIndexParallelUnsorted((Long elementUid) -> {
EnhancedObjectIndices elementUids = dbio.loadEnhancedObjectUids(elementUid); // check if the parent object is the declared type EnhancedObjectIndices elementUids = dbio.loadEnhancedObjectUids(elementUid); // check if the parent object is the declared type
Class<?> declaredRootType = query.valuePointer.getRootType(); if (elementUids != null) {
Class<?> obtainedRootType = elementUids.type; Class<?> declaredRootType = query.valuePointer.getRootType();
if (isInstanceOf(obtainedRootType, declaredRootType)) { Class<?> obtainedRootType = elementUids.type;
List<Object> result = resolveItemFromDb(query.valuePointer, dbio, elementUids); if (isInstanceOf(obtainedRootType, declaredRootType)) {
if (result.parallelStream().anyMatch(query.valueOperation::evaluate)) { List<Object> result = resolveItemFromDb(query.valuePointer, dbio, elementUids);
results.add(elementUids); if (result.size() == 1) {
if (query.valueOperation.evaluate(result.get(0))) {
results.add(elementUids);
}
} else if (result.size() > 1) {
if (result.parallelStream().anyMatch(query.valueOperation::evaluate)) {
results.add(elementUids);
}
}
} else {
//todo: use logging api
System.err.println(obtainedRootType.getSimpleName() + " is not instance of " + declaredRootType.getSimpleName());
} }
} else {
//todo: use logging api
System.err.println(obtainedRootType.getSimpleName() + " is not instance of " + declaredRootType.getSimpleName());
} }
}); });
} else if (inputList instanceof ElementsArrayList<?>) { } else if (inputList instanceof ElementsArrayList<?>) {
@ -158,7 +166,7 @@ public class EnhancedObjectStrangeDbList<T extends EnhancedObject> extends Stran
@NotNull @NotNull
private static List<Object> resolveItemFromDbLoop(EnhancedObjectIndices currentElement, ValuePointer pointer, int pointerPosition, int pointerSize, IObjectsIO objectsIO, EnhancedObjectIndices element) throws IOException { private static List<Object> resolveItemFromDbLoop(@NotNull EnhancedObjectIndices currentElement, ValuePointer pointer, int pointerPosition, int pointerSize, IObjectsIO objectsIO, EnhancedObjectIndices element) throws IOException {
if (pointerPosition > 0) { if (pointerPosition > 0) {
// check if the object that we obtained is the declared type // check if the object that we obtained is the declared type
Class<?> declaredType = pointer.resetPointerTo(pointerPosition - 1).getAdditionalData(); Class<?> declaredType = pointer.resetPointerTo(pointerPosition - 1).getAdditionalData();
@ -279,6 +287,7 @@ public class EnhancedObjectStrangeDbList<T extends EnhancedObject> extends Stran
} }
if (multipleCurrentElements == null) { if (multipleCurrentElements == null) {
if (currentElement == null) return ObjectLists.emptyList();
return resolveItemFromDbLoop(currentElement, pointer, pointerPosition + 1, pointerSize, objectsIO, element); return resolveItemFromDbLoop(currentElement, pointer, pointerPosition + 1, pointerSize, objectsIO, element);
} else { } else {
try { try {

11
src/test/java/it/cavallium/strangedb/tests/EnhancedClassUpdate.java
View File

@ -15,16 +15,16 @@ import static org.junit.Assert.assertEquals;
public class EnhancedClassUpdate { public class EnhancedClassUpdate {
private Path path1; private Path path1;
private Path path2;
private Path path3; private Path path3;
private DatabaseJava db; private DatabaseJava db;
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {
path1 = Files.createTempFile("db-tests-data-", ".db"); path1 = Files.createTempFile("db-tests-data-", ".db");
path2 = Files.createTempFile("db-tests-blocks-", ".db");
path3 = Files.createTempFile("db-tests-references-", ".db"); path3 = Files.createTempFile("db-tests-references-", ".db");
db = new DatabaseJava(path1, path2, path3); db = new DatabaseJava(path1, path3);
db.registerClass(V2Class.class, 0);
db.registerClass(OldClass.class, 1);
OldClass root = db.loadRoot(OldClass::new); OldClass root = db.loadRoot(OldClass::new);
root.field1 = "Abc"; root.field1 = "Abc";
root.field2 = 12; root.field2 = 12;
@ -34,7 +34,9 @@ public class EnhancedClassUpdate {
@Test @Test
public void shouldUpdateClass() throws IOException { public void shouldUpdateClass() throws IOException {
db = new DatabaseJava(path1, path2, path3); db = new DatabaseJava(path1, path3);
db.registerClass(V2Class.class, 0);
db.registerClass(OldClass.class, 1);
V2Class root = db.loadRoot(V2Class::new, V2Class.class); V2Class root = db.loadRoot(V2Class::new, V2Class.class);
assertEquals(root.field4, "Abc"); assertEquals(root.field4, "Abc");
assertEquals(root.field2, 12); assertEquals(root.field2, 12);
@ -45,7 +47,6 @@ public class EnhancedClassUpdate {
@After @After
public void tearDown() throws Exception { public void tearDown() throws Exception {
Files.deleteIfExists(path1); Files.deleteIfExists(path1);
Files.deleteIfExists(path2);
Files.deleteIfExists(path3); Files.deleteIfExists(path3);
} }
} }

7
src/test/java/it/cavallium/strangedb/tests/ObjectListTests.java
View File

@ -22,16 +22,14 @@ import static org.junit.Assert.assertEquals;
public class ObjectListTests { public class ObjectListTests {
private Path path1; private Path path1;
private Path path2;
private Path path3; private Path path3;
private DatabaseJava db; private DatabaseJava db;
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {
path1 = Files.createTempFile("db-tests-data-", ".db"); path1 = Files.createTempFile("db-tests-data-", ".db");
path2 = Files.createTempFile("db-tests-blocks-", ".db");
path3 = Files.createTempFile("db-tests-references-", ".db"); path3 = Files.createTempFile("db-tests-references-", ".db");
db = new DatabaseJava(path1, path2, path3); db = new DatabaseJava(path1, path3);
registerClasses(); registerClasses();
ListsRoot root = db.loadRoot(ListsRoot::new); ListsRoot root = db.loadRoot(ListsRoot::new);
for (int i = 0; i < 500; i++) { for (int i = 0; i < 500; i++) {
@ -66,7 +64,7 @@ public class ObjectListTests {
@Test @Test
public void shouldUpdateClass() throws IOException { public void shouldUpdateClass() throws IOException {
db = new DatabaseJava(path1, path2, path3); db = new DatabaseJava(path1, path3);
registerClasses(); registerClasses();
ListsRoot root = db.loadRoot(ListsRoot::new); ListsRoot root = db.loadRoot(ListsRoot::new);
@ -90,7 +88,6 @@ public class ObjectListTests {
@After @After
public void tearDown() throws Exception { public void tearDown() throws Exception {
Files.deleteIfExists(path1); Files.deleteIfExists(path1);
Files.deleteIfExists(path2);
Files.deleteIfExists(path3); Files.deleteIfExists(path3);
} }

6
src/test/java/it/cavallium/strangedb/tests/Performance.java
View File

@ -28,7 +28,6 @@ public class Performance {
private static Path rootDirectory; private static Path rootDirectory;
private static Path dbDataFile; private static Path dbDataFile;
private static Path dbBlocksFile;
private static Path dbReferencesFile; private static Path dbReferencesFile;
private static DatabaseJava db; private static DatabaseJava db;
private static boolean tempDirectory; private static boolean tempDirectory;
@ -339,13 +338,11 @@ public class Performance {
public static void generateDb() throws IOException { public static void generateDb() throws IOException {
dbDataFile = rootDirectory.resolve("db_data.dat"); dbDataFile = rootDirectory.resolve("db_data.dat");
dbBlocksFile = rootDirectory.resolve("db_blocks.dat");
dbReferencesFile = rootDirectory.resolve("db_references.dat"); dbReferencesFile = rootDirectory.resolve("db_references.dat");
deleteDbFolders(); deleteDbFolders();
Files.createFile(dbDataFile); Files.createFile(dbDataFile);
Files.createFile(dbBlocksFile);
Files.createFile(dbReferencesFile); Files.createFile(dbReferencesFile);
db = new DatabaseJava(dbDataFile, dbBlocksFile, dbReferencesFile); db = new DatabaseJava(dbDataFile, dbReferencesFile);
int i = 0; int i = 0;
db.registerClass(SimpleEnhancedObject.class, i++); db.registerClass(SimpleEnhancedObject.class, i++);
db.registerClass(PreloadedListContainer.class, i++); db.registerClass(PreloadedListContainer.class, i++);
@ -364,7 +361,6 @@ public class Performance {
public static void deleteDbFolders() throws IOException { public static void deleteDbFolders() throws IOException {
Files.deleteIfExists(dbDataFile); Files.deleteIfExists(dbDataFile);
Files.deleteIfExists(dbBlocksFile);
Files.deleteIfExists(dbReferencesFile); Files.deleteIfExists(dbReferencesFile);
} }

5
src/test/java/it/cavallium/strangedb/tests/QueryTests.java
View File

@ -22,16 +22,14 @@ public class QueryTests {
public static final String constantUsername = "is this an username?"; public static final String constantUsername = "is this an username?";
public static final String constantBio = "and is this a bio??? Are you mad?"; public static final String constantBio = "and is this a bio??? Are you mad?";
private Path path1; private Path path1;
private Path path2;
private Path path3; private Path path3;
private DatabaseJava db; private DatabaseJava db;
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {
path1 = Files.createTempFile("db-tests-data-", ".db"); path1 = Files.createTempFile("db-tests-data-", ".db");
path2 = Files.createTempFile("db-tests-blocks-", ".db");
path3 = Files.createTempFile("db-tests-references-", ".db"); path3 = Files.createTempFile("db-tests-references-", ".db");
db = new DatabaseJava(path1, path2, path3); db = new DatabaseJava(path1, path3);
int i = 0; int i = 0;
db.registerClass(UserFullInfo.class, i++); db.registerClass(UserFullInfo.class, i++);
db.registerClass(User.class, i++); db.registerClass(User.class, i++);
@ -58,7 +56,6 @@ public class QueryTests {
@After @After
public void tearDown() throws Exception { public void tearDown() throws Exception {
Files.deleteIfExists(path1); Files.deleteIfExists(path1);
Files.deleteIfExists(path2);
Files.deleteIfExists(path3); Files.deleteIfExists(path3);
} }
} }

6
src/test/java/it/cavallium/strangedb/utils/NTestUtils.java
View File

@ -4,6 +4,8 @@ import it.cavallium.strangedb.database.DatabaseCore;
import it.cavallium.strangedb.functionalinterfaces.RunnableWithIO; import it.cavallium.strangedb.functionalinterfaces.RunnableWithIO;
import it.cavallium.strangedb.java.annotations.*; import it.cavallium.strangedb.java.annotations.*;
import it.cavallium.strangedb.java.database.DatabaseJava; import it.cavallium.strangedb.java.database.DatabaseJava;
import it.cavallium.strangedb.tests.Clean;
import it.cavallium.strangedb.tests.MultipleEnhancedObjects;
import it.unimi.dsi.fastutil.longs.LongArrayList; import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.cavallium.strangedb.java.objects.EnhancedObject; import it.cavallium.strangedb.java.objects.EnhancedObject;
import it.cavallium.strangedb.java.database.IDatabaseTools; import it.cavallium.strangedb.java.database.IDatabaseTools;
@ -47,11 +49,13 @@ public class NTestUtils {
} }
private DatabaseJava openDatabase() throws IOException { private DatabaseJava openDatabase() throws IOException {
DatabaseJava db = new DatabaseJava(tempDir.resolve(Paths.get("data.db")), tempDir.resolve(Paths.get("blocks.dat")), tempDir.resolve(Paths.get("references.dat"))); DatabaseJava db = new DatabaseJava(tempDir.resolve(Paths.get("data.db")), tempDir.resolve(Paths.get("references.dat")));
int id = 0; int id = 0;
db.registerClass(RootClass.class, id++); db.registerClass(RootClass.class, id++);
db.registerClass(LongArrayList.class, id++); db.registerClass(LongArrayList.class, id++);
db.registerClass(NSimplestClass.class, id++); db.registerClass(NSimplestClass.class, id++);
db.registerClass(Clean.RootTwoClasses.class, id++);
db.registerClass(MultipleEnhancedObjects.RootTwoClasses.class, id++);
return db; return db;
} }

2
strangedb-core

@ -1 +1 @@
Subproject commit dc56da3bbb4bac11af20f2d52bd974c85c5fe925 Subproject commit 7fd7497464b5d5e92745974ed12ff4a9f51919d0