andreacavalli/strangedb
1
1
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Query optimizations

Browse Source
This commit is contained in:
Andrea Cavalli 2019-04-22 00:10:21 +02:00
parent 9c3cf7b023
commit 6d531ee1c4
10 changed files with 282 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
src/main/java/it/cavallium/strangedb/java/database/DatabaseObjectsIO.java
View File

@ -364,6 +364,42 @@ public class DatabaseObjectsIO implements IObjectsIO {
return arrayList;
}
@Override
public Long loadReferencesListFirstElement(long reference) throws IOException {
return loadReferencesListFirstElement_(reference);
}
private Long loadReferencesListFirstElement_(long reference) throws IOException {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return null;
}
int itemsCount = buffer.getInt();
if (itemsCount <= 0) {
return null;
} else {
return buffer.getLong();
}
}
@Override
public Long loadReferencesListLastElement(long reference) throws IOException {
return loadReferencesListLastElement_(reference);
}
private Long loadReferencesListLastElement_(long reference) throws IOException {
ByteBuffer buffer = referencesIO.readFromReferenceSizeAndLastElementOfReferencesList(reference);
if (buffer.limit() == 0) {
return null;
}
int itemsCount = buffer.getInt();
if (itemsCount <= 0) {
return null;
} else {
return buffer.getLong();
}
}
@Override
public LongList loadPrimitiveData(long reference) throws IOException {
return loadPrimitiveData_(reference);
@ -405,11 +441,12 @@ public class DatabaseObjectsIO implements IObjectsIO {
private void setReferencesList_(long reference, LongArrayList value) throws IOException {
if (value != null) {
int items = value.size();
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * items + Integer.BYTES);
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * items + Integer.BYTES * 2);
buffer.putInt(items);
for (int i = 0; i < items; i++) {
buffer.putLong(value.getLong(i));
}
buffer.putInt(items);
buffer.flip();
referencesIO.writeToReference(reference, REFERENCES_LIST_CLEANER, buffer.limit(), buffer);
} else {

4
src/main/java/it/cavallium/strangedb/java/database/IObjectsIO.java
View File

@ -60,5 +60,9 @@ public interface IObjectsIO {
IDataInitializer getDataInitializer();
Long loadReferencesListFirstElement(long reference) throws IOException;
Long loadReferencesListLastElement(long reference) throws IOException;
LongList loadPrimitiveData(long id) throws IOException;
}

8
src/main/java/it/cavallium/strangedb/java/objects/lists/ClassPosition.java
View File

@ -3,5 +3,11 @@ package it.cavallium.strangedb.java.objects.lists;
public enum ClassPosition {
PROPERTY,
FIELD,
PRIMITIVE_FIELD
PRIMITIVE_FIELD,
ENHANCED_LIST;
public static final int LIST_ALL = 0;
public static final int LIST_FIRST = 1;
public static final int LIST_LAST = 2;
public static final int LIST_POSITION = 3;
}

166
src/main/java/it/cavallium/strangedb/java/objects/lists/EnhancedObjectStrangeDbList.java
View File

@ -1,18 +1,27 @@
package it.cavallium.strangedb.java.objects.lists;
import it.cavallium.strangedb.java.database.IObjectsIO;
import it.cavallium.strangedb.java.objects.EnhancedObjectIndices;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.cavallium.strangedb.java.objects.EnhancedObject;
import it.cavallium.strangedb.java.database.IDatabaseTools;
import it.cavallium.strangedb.java.annotations.DbDataType;
import it.cavallium.strangedb.java.annotations.DbField;
import it.cavallium.strangedb.java.database.IDatabaseTools;
import it.cavallium.strangedb.java.database.IObjectsIO;
import it.cavallium.strangedb.java.objects.EnhancedObject;
import it.cavallium.strangedb.java.objects.EnhancedObjectIndices;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.objects.ObjectList;
import it.unimi.dsi.fastutil.objects.ObjectLists;
import org.jetbrains.annotations.NotNull;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.CompletionException;
import java.util.stream.Collectors;
public class EnhancedObjectStrangeDbList<T extends EnhancedObject> extends StrangeDbList<T> {
@DbField(id = 0, type = DbDataType.REFERENCES_LIST)
private static final int INDICES_REF_ID = 0;
@DbField(id = INDICES_REF_ID, type = DbDataType.REFERENCES_LIST, name = "indices")
private LongArrayList indices;
@Override
@ -96,6 +105,7 @@ public class EnhancedObjectStrangeDbList<T extends EnhancedObject> extends Stran
/**
* This method is slow
*
* @param elementsList
* @return
*/
@ -121,8 +131,8 @@ public class EnhancedObjectStrangeDbList<T extends EnhancedObject> extends Stran
Class<?> declaredRootType = query.valuePointer.getRootType();
Class<?> obtainedRootType = elementUids.type;
if (isInstanceOf(obtainedRootType, declaredRootType)) {
Object result = resolveItemFromDb(query.valuePointer, dbio, elementUids);
if (query.valueOperation.evaluate(result)) {
List<Object> result = resolveItemFromDb(query.valuePointer, dbio, elementUids);
if (result.parallelStream().anyMatch(query.valueOperation::evaluate)) {
results.add(elementUids);
}
} else {
@ -133,8 +143,8 @@ public class EnhancedObjectStrangeDbList<T extends EnhancedObject> extends Stran
} else if (inputList instanceof ElementsArrayList<?>) {
ElementsArrayList<EnhancedObjectIndices> elementsUids = ((ElementsArrayList<EnhancedObjectIndices>) inputList);
elementsUids.forEachParallelUnsorted((elementUid) -> {
Object result = resolveItemFromDb(query.valuePointer, dbio, elementUid);
if (query.valueOperation.evaluate(result)) {
List<Object> result = resolveItemFromDb(query.valuePointer, dbio, elementUid);
if (result.parallelStream().anyMatch(query.valueOperation::evaluate)) {
results.add(elementUid);
}
});
@ -142,64 +152,152 @@ public class EnhancedObjectStrangeDbList<T extends EnhancedObject> extends Stran
return results;
}
private static Object resolveItemFromDb(ValuePointer pointer, IObjectsIO objectsIO, EnhancedObjectIndices element) throws IOException {
EnhancedObjectIndices currentElement = element;
boolean isLastElement;
for (int i = 0; i < pointer.size(); i++) {
isLastElement = i >= pointer.size() - 1;
ValuePointer currentPointer = pointer.at(i);
private static List<Object> resolveItemFromDb(ValuePointer pointer, IObjectsIO objectsIO, EnhancedObjectIndices element) throws IOException {
return resolveItemFromDbLoop(element, pointer, 0, pointer.size(), objectsIO, element);
}
@NotNull
private static List<Object> resolveItemFromDbLoop(EnhancedObjectIndices currentElement, ValuePointer pointer, int pointerPosition, int pointerSize, IObjectsIO objectsIO, EnhancedObjectIndices element) throws IOException {
if (pointerPosition > 0) {
// check if the object that we obtained is the declared type
Class<?> declaredType = pointer.resetPointerTo(pointerPosition - 1).getAdditionalData();
Class<?> obtainedType = currentElement.type;
if (!isInstanceOf(obtainedType, declaredType)) {
return ObjectLists.emptyList();
}
if (pointerPosition >= pointerSize) {
throw new IOException("The pointer is empty");
}
}
boolean isLastElement = pointerPosition >= pointer.size() - 1;
ValuePointer currentPointer = pointer.resetPointerTo(pointerPosition);
int pathNumber = currentPointer.getPathNumber();
ClassPosition valueType = currentPointer.getPathType();
List<EnhancedObjectIndices> multipleCurrentElements = null;
Object value;
switch (valueType) {
case ENHANCED_LIST: {
switch (currentPointer.getPathNumber()) {
case ClassPosition.LIST_ALL: {
if (isLastElement) {
try {
return objectsIO.loadReferencesList(currentElement.fieldUids[INDICES_REF_ID]).parallelStream().map((reference) -> {
try {
return objectsIO.loadEnhancedObject(reference);
} catch (IOException e) {
throw new CompletionException(e);
}
}).filter(Objects::nonNull).collect(Collectors.toList());
} catch (CompletionException e) {
throw new IOException(e.getCause());
}
} else {
LongArrayList refList = objectsIO.loadReferencesList(currentElement.fieldUids[INDICES_REF_ID]);
try {
multipleCurrentElements = refList.parallelStream().map((reference) -> {
try {
return objectsIO.loadEnhancedObjectUids(reference);
} catch (IOException e) {
throw new CompletionException(e);
}
}).collect(Collectors.toList());
} catch (CompletionException e) {
throw new IOException(e.getCause());
}
}
break;
}
case ClassPosition.LIST_FIRST: {
try {
Long firstElementReference = objectsIO.loadReferencesListFirstElement(currentElement.fieldUids[INDICES_REF_ID]);
if (firstElementReference == null) {
return ObjectLists.emptyList();
} else {
currentElement = objectsIO.loadEnhancedObjectUids(firstElementReference);
}
} catch (CompletionException e) {
throw new IOException(e.getCause());
}
break;
}
case ClassPosition.LIST_LAST: {
try {
Long firstElementReference = objectsIO.loadReferencesListLastElement(currentElement.fieldUids[INDICES_REF_ID]);
if (firstElementReference == null) {
return ObjectLists.emptyList();
} else {
currentElement = objectsIO.loadEnhancedObjectUids(firstElementReference);
}
} catch (CompletionException e) {
throw new IOException(e.getCause());
}
break;
}
default:
throw new IOException("Unknown class position");
}
break;
}
case FIELD: {
if (isLastElement) {
//TODO: getBlock field data type. it can be an enhancedObject or an object
value = objectsIO.loadObject(currentElement.fieldUids[pathNumber]);
Object loadedField = objectsIO.loadObject(currentElement.fieldUids[pathNumber]);
if (loadedField == null) return ObjectLists.emptyList();
return ObjectLists.singleton(loadedField);
} else {
value = objectsIO.loadEnhancedObjectUids(currentElement.fieldUids[pathNumber]);
currentElement = objectsIO.loadEnhancedObjectUids(currentElement.fieldUids[pathNumber]);
}
break;
}
case PRIMITIVE_FIELD: {
if (isLastElement) {
//TODO: getBlock primitive type
value = objectsIO.loadPrimitiveData(currentElement.nativeDataUid).getLong(pathNumber);
Object loadedField = objectsIO.loadPrimitiveData(currentElement.nativeDataUid).getLong(pathNumber);
return ObjectLists.singleton(loadedField);
} else {
throw new IllegalArgumentException("You can access to a type field only in the last pointer");
}
break;
}
case PROPERTY: {
if (isLastElement) {
//TODO: getBlock field data type. it can be an enhancedObject or an object
value = objectsIO.loadObject(currentElement.fieldUids[pathNumber]);
Object loadedProperty = objectsIO.loadObject(currentElement.fieldUids[pathNumber]);
if (loadedProperty == null) return ObjectLists.emptyList();
return ObjectLists.singleton(loadedProperty);
} else {
value = objectsIO.loadEnhancedObjectUids(currentElement.propertyUids[pathNumber]);
currentElement = objectsIO.loadEnhancedObjectUids(currentElement.propertyUids[pathNumber]);
}
break;
}
default: {
throw new IllegalArgumentException("Not implemented");
}
}
if (isLastElement) {
return value;
if (multipleCurrentElements == null) {
return resolveItemFromDbLoop(currentElement, pointer, pointerPosition + 1, pointerSize, objectsIO, element);
} else {
currentElement = (EnhancedObjectIndices) value;
// check if the object that we obtained is the declared type
Class<?> declaredType = currentPointer.getAdditionalData();
Class<?> obtainedType = currentElement.type;
if (!isInstanceOf(obtainedType, declaredType)) {
return null;
try {
return multipleCurrentElements
.parallelStream()
.map((elem) -> {
try {
return resolveItemFromDbLoop(elem, pointer, pointerPosition + 1, pointerSize, objectsIO, element);
} catch (IOException e) {
throw new CompletionException(e);
}
})
.flatMap(List::stream)
.collect(Collectors.toList());
} catch (CompletionException ex) {
throw new IOException(ex.getCause());
}
}
}
throw new IOException("The pointer is empty");
}
private static boolean isInstanceOf(Class<?> clazz, Class<?> obj) {

34
src/main/java/it/cavallium/strangedb/java/objects/lists/KMP.java
View File

@ -1,14 +1,14 @@
package it.cavallium.strangedb.java.objects.lists;
public class KMP {
public static int KMP(CharSequence content, CharSequence stringToFind) {
int[] failureTable = failureTable(stringToFind);
public static int KMP(CharSequence content, CharSequence stringToFind, boolean ignoreCase) {
int[] failureTable = failureTable(ignoreCase, stringToFind);
int targetPointer = 0; // current char in target string
int searchPointer = 0; // current char in search string
while (searchPointer < content.length()) { // while there is more to search with, keep searching
if (content.charAt(searchPointer) == stringToFind.charAt(targetPointer)) { // case 1
if (charEquals(ignoreCase, content.charAt(searchPointer), stringToFind.charAt(targetPointer))) { // case 1
// found current char in targetPointer in search string
targetPointer++;
if (targetPointer == stringToFind.length()) { // found all characters
@ -17,10 +17,10 @@ public class KMP {
}
searchPointer++; // move forward if not found target string
} else if (targetPointer > 0) { // case 2
// use failureTable to use pointer pointed at nearest location of usable string prefix
// use failureTable to use pointer pointed resetPointerTo nearest location of usable string prefix
targetPointer = failureTable[targetPointer];
} else { // case 3
// targetPointer is pointing at state 0, so restart search with current searchPointer index
// targetPointer is pointing resetPointerTo state 0, so restart search with current searchPointer index
searchPointer++;
}
}
@ -30,22 +30,22 @@ public class KMP {
/**
* Returns an int[] that points to last valid string prefix, given target string
*/
public static int[] failureTable(CharSequence target) {
public static int[] failureTable(boolean ignoreCase, CharSequence target) {
int[] table = new int[target.length() + 1];
// state 0 and 1 are guarenteed be the prior
table[0] = -1;
table[1] = 0;
// the pointers pointing at last failure and current satte
// the pointers pointing resetPointerTo last failure and current satte
int left = 0;
int right = 2;
while (right < table.length) { // RIGHT NEVER MOVES RIGHT UNTIL ASSIGNED A VALID POINTER
if (target.charAt(right - 1) == target.charAt(left)) { // when both chars before left and right are equal, link both and move both forward
if (charEquals(ignoreCase, target.charAt(right - 1), target.charAt(left))) { // when both chars before left and right are equal, link both and move both forward
left++;
table[right] = left;
right++;
} else if (left > 0) { // if left isn't at the very beginning, then send left backward
} else if (left > 0) { // if left isn't resetPointerTo the very beginning, then send left backward
// by following the already set pointer to where it is pointing to
left = table[left];
} else { // left has fallen all the way back to the beginning
@ -55,4 +55,20 @@ public class KMP {
}
return table;
}
private static boolean charEquals(boolean ignoreCase, char char1, char char2) {
if (ignoreCase) {
if (char1 >= 65 && char1 <= 90) {
if (char2 >= 97 && char2 <= 122) {
return char1 == char2 - 32;
}
}
if (char2 >= 65 && char2 <= 90) {
if (char1 >= 97 && char1 <= 122) {
return char2 == char1 - 32;
}
}
}
return char1 == char2;
}
}

2
src/main/java/it/cavallium/strangedb/java/objects/lists/StrangeDbList.java
View File

@ -63,7 +63,7 @@ public abstract class StrangeDbList<T> extends EnhancedObject implements Element
protected void forEachParallelUnsorted_(ConsumerWithIO<T> action) throws IOException {
try {
int size = size();
ExecutorService executorService = ForkJoinPool.commonPool();
ExecutorService executorService = Executors.newFixedThreadPool(ForkJoinPool.getCommonPoolParallelism(), (r) -> new Thread(r, "DBList parallel foreach"));
VariableWrapper<IOException> exceptionVariableWrapper = new VariableWrapper<>(null);
for (int i = 0; i < size; i++) {
final int index = i;

35
src/main/java/it/cavallium/strangedb/java/objects/lists/ValuePointer.java
View File

@ -46,7 +46,7 @@ public class ValuePointer {
public <T extends EnhancedObject> ValuePointer prop(Class<T> parentType, String propertyName) {
ClassPosition[] newPathTypes = append(this.pathTypes, ClassPosition.FIELD);
ClassPosition[] newPathTypes = append(this.pathTypes, ClassPosition.PROPERTY);
Method[] methods = MethodUtils.getMethodsWithAnnotation(parentType, DbProperty.class);
DbProperty dbProperty = null;
@ -99,6 +99,34 @@ public class ValuePointer {
return new ValuePointer(newPathNumbers, newPathTypes, newAdditionalData, rootType);
}
public <T extends EnhancedObjectStrangeDbList<?>> ValuePointer allInList(Class<T> parentType) {
return inList(parentType, ClassPosition.LIST_ALL, -1);
}
public <T extends EnhancedObjectStrangeDbList<?>> ValuePointer firstInList(Class<T> parentType) {
return inList(parentType, ClassPosition.LIST_FIRST, -1);
}
public <T extends EnhancedObjectStrangeDbList<?>> ValuePointer lastInList(Class<T> parentType) {
return inList(parentType, ClassPosition.LIST_LAST, -1);
}
public <T extends EnhancedObjectStrangeDbList<?>> ValuePointer indexInList(Class<T> parentType, int position) {
return inList(parentType, ClassPosition.LIST_POSITION, position);
}
private <T extends EnhancedObjectStrangeDbList<?>> ValuePointer inList(Class<T> parentType, int listPosition, int itemIndex) {
ClassPosition[] newPathTypes = append(this.pathTypes, ClassPosition.ENHANCED_LIST);
int[] newPathNumbers = append(this.pathNumbers, listPosition);
Object[] newAdditionalData = append(this.additionalData, itemIndex);
if (newAdditionalData.length > 1) {
newAdditionalData[newAdditionalData.length - 2] = parentType;
}
Class<?> rootType = this.rootType == null ? parentType : this.rootType;
return new ValuePointer(newPathNumbers, newPathTypes, newAdditionalData, rootType);
}
public <T extends EnhancedObject> ValuePointer primitiveField(Class<T> parentType, String primitiveFieldName) {
ClassPosition[] newPathTypes = append(this.pathTypes, ClassPosition.PRIMITIVE_FIELD);
@ -126,7 +154,7 @@ public class ValuePointer {
return new ValuePointer(newPathNumbers, newPathTypes, newAdditionalData, rootType);
}
public ValuePointer at(int index) {
public ValuePointer resetPointerTo(int index) {
if (index >= pathNumbers.length) {
throw new ArrayIndexOutOfBoundsException();
}
@ -148,7 +176,7 @@ public class ValuePointer {
}
public ValuePointer previous() {
return at(pathNumbers.length - 2);
return resetPointerTo(pathNumbers.length - 2);
}
private static int[] append(int[] array, int value) {
@ -179,4 +207,5 @@ public class ValuePointer {
public Class<?> getRootType() {
return rootType;
}
}

2
src/main/java/it/cavallium/strangedb/java/objects/lists/operations/Contains.java
View File

@ -18,7 +18,7 @@ public class Contains<T extends CharSequence> implements ValueOperation<T> {
@Override
public boolean evaluate(Object value) {
if (value instanceof CharSequence) {
return KMP.KMP((CharSequence) value, containsValue) != -1;
return KMP.KMP((CharSequence) value, containsValue, false) != -1;
}
return false;
}

12
src/main/java/it/cavallium/strangedb/java/objects/lists/operations/ContainsIgnoreCase.java
View File

@ -5,20 +5,20 @@ import it.cavallium.strangedb.java.objects.lists.ValueOperation;
public class ContainsIgnoreCase implements ValueOperation<String> {
private final String containsValue;
private final CharSequence containsValue;
private ContainsIgnoreCase(String containsValue) {
this.containsValue = containsValue.toLowerCase();
private ContainsIgnoreCase(CharSequence containsValue) {
this.containsValue = containsValue;
}
public static ContainsIgnoreCase containsValue(String value) {
public static ContainsIgnoreCase containsValue(CharSequence value) {
return new ContainsIgnoreCase(value);
}
@Override
public boolean evaluate(Object value) {
if (value instanceof String) {
return KMP.KMP(((String) value).toLowerCase(), containsValue) != -1;
if (value instanceof CharSequence) {
return KMP.KMP( (CharSequence) value, containsValue, true) != -1;
}
return false;
}

2
src/test/java/it/cavallium/strangedb/tests/ListContainer.java
View File

@ -22,7 +22,7 @@ public class ListContainer extends EnhancedObject {
super(databaseTools);
this.usersList = new EnhancedObjectStrangeDbList<>(databaseTools, User.class);
Random random = new Random();
ExecutorService threadPool = ForkJoinPool.commonPool();
ExecutorService threadPool = Executors.newWorkStealingPool();
for (int i = 0; i < count; i++) {
threadPool.execute(() -> {
try {