Query optimizations

src/main/java/it/cavallium/strangedb/java/database/DatabaseObjectsIO.java

@@ -364,6 +364,42 @@ public class DatabaseObjectsIO implements IObjectsIO {
 		return arrayList;
 	}
 
+	@Override
+	public Long loadReferencesListFirstElement(long reference) throws IOException {
+		return loadReferencesListFirstElement_(reference);
+	}
+
+	private Long loadReferencesListFirstElement_(long reference) throws IOException {
+		ByteBuffer buffer = referencesIO.readFromReference(reference);
+		if (buffer.limit() == 0) {
+			return null;
+		}
+		int itemsCount = buffer.getInt();
+		if (itemsCount <= 0) {
+			return null;
+		} else {
+			return buffer.getLong();
+		}
+	}
+
+	@Override
+	public Long loadReferencesListLastElement(long reference) throws IOException {
+		return loadReferencesListLastElement_(reference);
+	}
+
+	private Long loadReferencesListLastElement_(long reference) throws IOException {
+		ByteBuffer buffer = referencesIO.readFromReferenceSizeAndLastElementOfReferencesList(reference);
+		if (buffer.limit() == 0) {
+			return null;
+		}
+		int itemsCount = buffer.getInt();
+		if (itemsCount <= 0) {
+			return null;
+		} else {
+			return buffer.getLong();
+		}
+	}
+
 	@Override
 	public LongList loadPrimitiveData(long reference) throws IOException {
 		return loadPrimitiveData_(reference);
@@ -405,11 +441,12 @@ public class DatabaseObjectsIO implements IObjectsIO {
 	private void setReferencesList_(long reference, LongArrayList value) throws IOException {
 		if (value != null) {
 			int items = value.size();
-			ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * items + Integer.BYTES);
+			ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * items + Integer.BYTES * 2);
 			buffer.putInt(items);
 			for (int i = 0; i < items; i++) {
 				buffer.putLong(value.getLong(i));
 			}
+			buffer.putInt(items);
 			buffer.flip();
 			referencesIO.writeToReference(reference, REFERENCES_LIST_CLEANER, buffer.limit(), buffer);
 		} else {

src/main/java/it/cavallium/strangedb/java/database/IObjectsIO.java

@@ -60,5 +60,9 @@ public interface IObjectsIO {
 
 	IDataInitializer getDataInitializer();
 
+	Long loadReferencesListFirstElement(long reference) throws IOException;
+
+	Long loadReferencesListLastElement(long reference) throws IOException;
+
 	LongList loadPrimitiveData(long id) throws IOException;
 }

src/main/java/it/cavallium/strangedb/java/objects/lists/ClassPosition.java

@@ -3,5 +3,11 @@ package it.cavallium.strangedb.java.objects.lists;
 public enum ClassPosition {
 	PROPERTY,
 	FIELD,
-	PRIMITIVE_FIELD
+	PRIMITIVE_FIELD,
+	ENHANCED_LIST;
+
+	public static final int LIST_ALL = 0;
+	public static final int LIST_FIRST = 1;
+	public static final int LIST_LAST = 2;
+	public static final int LIST_POSITION = 3;
 }

src/main/java/it/cavallium/strangedb/java/objects/lists/EnhancedObjectStrangeDbList.java

@@ -1,18 +1,27 @@
 package it.cavallium.strangedb.java.objects.lists;
 
-import it.cavallium.strangedb.java.database.IObjectsIO;
-import it.cavallium.strangedb.java.objects.EnhancedObjectIndices;
-import it.unimi.dsi.fastutil.longs.LongArrayList;
-import it.cavallium.strangedb.java.objects.EnhancedObject;
-import it.cavallium.strangedb.java.database.IDatabaseTools;
 import it.cavallium.strangedb.java.annotations.DbDataType;
 import it.cavallium.strangedb.java.annotations.DbField;
+import it.cavallium.strangedb.java.database.IDatabaseTools;
+import it.cavallium.strangedb.java.database.IObjectsIO;
+import it.cavallium.strangedb.java.objects.EnhancedObject;
+import it.cavallium.strangedb.java.objects.EnhancedObjectIndices;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import it.unimi.dsi.fastutil.objects.ObjectList;
+import it.unimi.dsi.fastutil.objects.ObjectLists;
+import org.jetbrains.annotations.NotNull;
 
 import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.CompletionException;
+import java.util.stream.Collectors;
 
 public class EnhancedObjectStrangeDbList<T extends EnhancedObject> extends StrangeDbList<T> {
 
-	@DbField(id = 0, type = DbDataType.REFERENCES_LIST)
+	private static final int INDICES_REF_ID = 0;
+
+	@DbField(id = INDICES_REF_ID, type = DbDataType.REFERENCES_LIST, name = "indices")
 	private LongArrayList indices;
 
 	@Override
@@ -96,6 +105,7 @@ public class EnhancedObjectStrangeDbList<T extends EnhancedObject> extends Stran
 
 	/**
 	 * This method is slow
+	 *
 	 * @param elementsList
 	 * @return
 	 */
@@ -121,8 +131,8 @@ public class EnhancedObjectStrangeDbList<T extends EnhancedObject> extends Stran
 				Class<?> declaredRootType = query.valuePointer.getRootType();
 				Class<?> obtainedRootType = elementUids.type;
 				if (isInstanceOf(obtainedRootType, declaredRootType)) {
-					Object result = resolveItemFromDb(query.valuePointer, dbio, elementUids);
+					List<Object> result = resolveItemFromDb(query.valuePointer, dbio, elementUids);
-					if (query.valueOperation.evaluate(result)) {
+					if (result.parallelStream().anyMatch(query.valueOperation::evaluate)) {
 						results.add(elementUids);
 					}
 				} else {
@@ -133,8 +143,8 @@ public class EnhancedObjectStrangeDbList<T extends EnhancedObject> extends Stran
 		} else if (inputList instanceof ElementsArrayList<?>) {
 			ElementsArrayList<EnhancedObjectIndices> elementsUids = ((ElementsArrayList<EnhancedObjectIndices>) inputList);
 			elementsUids.forEachParallelUnsorted((elementUid) -> {
-				Object result = resolveItemFromDb(query.valuePointer, dbio, elementUid);
+				List<Object> result = resolveItemFromDb(query.valuePointer, dbio, elementUid);
-				if (query.valueOperation.evaluate(result)) {
+				if (result.parallelStream().anyMatch(query.valueOperation::evaluate)) {
 					results.add(elementUid);
 				}
 			});
@@ -142,64 +152,152 @@ public class EnhancedObjectStrangeDbList<T extends EnhancedObject> extends Stran
 		return results;
 	}
 
-	private static Object resolveItemFromDb(ValuePointer pointer, IObjectsIO objectsIO, EnhancedObjectIndices element) throws IOException {
+	private static List<Object> resolveItemFromDb(ValuePointer pointer, IObjectsIO objectsIO, EnhancedObjectIndices element) throws IOException {
-		EnhancedObjectIndices currentElement = element;
+		return resolveItemFromDbLoop(element, pointer, 0, pointer.size(), objectsIO, element);
-		boolean isLastElement;
+	}
-		for (int i = 0; i < pointer.size(); i++) {
+
-			isLastElement = i >= pointer.size() - 1;
+
-			ValuePointer currentPointer = pointer.at(i);
+	@NotNull
+	private static List<Object> resolveItemFromDbLoop(EnhancedObjectIndices currentElement, ValuePointer pointer, int pointerPosition, int pointerSize, IObjectsIO objectsIO, EnhancedObjectIndices element) throws IOException {
+		if (pointerPosition > 0) {
+			// check if the object that we obtained is the declared type
+			Class<?> declaredType = pointer.resetPointerTo(pointerPosition - 1).getAdditionalData();
+			Class<?> obtainedType = currentElement.type;
+			if (!isInstanceOf(obtainedType, declaredType)) {
+				return ObjectLists.emptyList();
+			}
+
+			if (pointerPosition >= pointerSize) {
+				throw new IOException("The pointer is empty");
+			}
+		}
+
+		boolean isLastElement = pointerPosition >= pointer.size() - 1;
+		ValuePointer currentPointer = pointer.resetPointerTo(pointerPosition);
 
 		int pathNumber = currentPointer.getPathNumber();
 		ClassPosition valueType = currentPointer.getPathType();
+		List<EnhancedObjectIndices> multipleCurrentElements = null;
 
-			Object value;
 		switch (valueType) {
+			case ENHANCED_LIST: {
+				switch (currentPointer.getPathNumber()) {
+					case ClassPosition.LIST_ALL: {
+						if (isLastElement) {
+							try {
+								return objectsIO.loadReferencesList(currentElement.fieldUids[INDICES_REF_ID]).parallelStream().map((reference) -> {
+									try {
+										return objectsIO.loadEnhancedObject(reference);
+									} catch (IOException e) {
+										throw new CompletionException(e);
+									}
+								}).filter(Objects::nonNull).collect(Collectors.toList());
+							} catch (CompletionException e) {
+								throw new IOException(e.getCause());
+							}
+						} else {
+							LongArrayList refList = objectsIO.loadReferencesList(currentElement.fieldUids[INDICES_REF_ID]);
+							try {
+								multipleCurrentElements = refList.parallelStream().map((reference) -> {
+									try {
+										return objectsIO.loadEnhancedObjectUids(reference);
+									} catch (IOException e) {
+										throw new CompletionException(e);
+									}
+								}).collect(Collectors.toList());
+							} catch (CompletionException e) {
+								throw new IOException(e.getCause());
+							}
+						}
+						break;
+					}
+					case ClassPosition.LIST_FIRST: {
+						try {
+							Long firstElementReference = objectsIO.loadReferencesListFirstElement(currentElement.fieldUids[INDICES_REF_ID]);
+							if (firstElementReference == null) {
+								return ObjectLists.emptyList();
+							} else {
+								currentElement = objectsIO.loadEnhancedObjectUids(firstElementReference);
+							}
+						} catch (CompletionException e) {
+							throw new IOException(e.getCause());
+						}
+						break;
+					}
+					case ClassPosition.LIST_LAST: {
+						try {
+							Long firstElementReference = objectsIO.loadReferencesListLastElement(currentElement.fieldUids[INDICES_REF_ID]);
+							if (firstElementReference == null) {
+								return ObjectLists.emptyList();
+							} else {
+								currentElement = objectsIO.loadEnhancedObjectUids(firstElementReference);
+							}
+						} catch (CompletionException e) {
+							throw new IOException(e.getCause());
+						}
+						break;
+					}
+					default:
+						throw new IOException("Unknown class position");
+				}
+				break;
+			}
 			case FIELD: {
 				if (isLastElement) {
 					//TODO: getBlock field data type. it can be an enhancedObject or an object
-						value = objectsIO.loadObject(currentElement.fieldUids[pathNumber]);
+					Object loadedField = objectsIO.loadObject(currentElement.fieldUids[pathNumber]);
+					if (loadedField == null) return ObjectLists.emptyList();
+					return ObjectLists.singleton(loadedField);
 				} else {
-						value = objectsIO.loadEnhancedObjectUids(currentElement.fieldUids[pathNumber]);
+					currentElement = objectsIO.loadEnhancedObjectUids(currentElement.fieldUids[pathNumber]);
 				}
 				break;
 			}
 			case PRIMITIVE_FIELD: {
 				if (isLastElement) {
 					//TODO: getBlock primitive type
-						value = objectsIO.loadPrimitiveData(currentElement.nativeDataUid).getLong(pathNumber);
+					Object loadedField = objectsIO.loadPrimitiveData(currentElement.nativeDataUid).getLong(pathNumber);
+					return ObjectLists.singleton(loadedField);
 				} else {
 					throw new IllegalArgumentException("You can access to a type field only in the last pointer");
 				}
-					break;
 			}
 			case PROPERTY: {
 				if (isLastElement) {
 					//TODO: getBlock field data type. it can be an enhancedObject or an object
-						value = objectsIO.loadObject(currentElement.fieldUids[pathNumber]);
+					Object loadedProperty = objectsIO.loadObject(currentElement.fieldUids[pathNumber]);
+					if (loadedProperty == null) return ObjectLists.emptyList();
+					return ObjectLists.singleton(loadedProperty);
 				} else {
-						value = objectsIO.loadEnhancedObjectUids(currentElement.propertyUids[pathNumber]);
+					currentElement = objectsIO.loadEnhancedObjectUids(currentElement.propertyUids[pathNumber]);
 				}
+				break;
 			}
 			default: {
 				throw new IllegalArgumentException("Not implemented");
 			}
 		}
 
-			if (isLastElement) {
+		if (multipleCurrentElements == null) {
-				return value;
+			return resolveItemFromDbLoop(currentElement, pointer, pointerPosition + 1, pointerSize, objectsIO, element);
 		} else {
-				currentElement = (EnhancedObjectIndices) value;
+			try {
-
+				return multipleCurrentElements
-				// check if the object that we obtained is the declared type
+								.parallelStream()
-				Class<?> declaredType = currentPointer.getAdditionalData();
+								.map((elem) -> {
-				Class<?> obtainedType = currentElement.type;
+									try {
-				if (!isInstanceOf(obtainedType, declaredType)) {
+										return resolveItemFromDbLoop(elem, pointer, pointerPosition + 1, pointerSize, objectsIO, element);
-					return null;
+									} catch (IOException e) {
+										throw new CompletionException(e);
+									}
+								})
+								.flatMap(List::stream)
+								.collect(Collectors.toList());
+			} catch (CompletionException ex) {
+				throw new IOException(ex.getCause());
 			}
 		}
 	}
-		throw new IOException("The pointer is empty");
-	}
 
 
 	private static boolean isInstanceOf(Class<?> clazz, Class<?> obj) {

src/main/java/it/cavallium/strangedb/java/objects/lists/KMP.java

@@ -1,14 +1,14 @@
 package it.cavallium.strangedb.java.objects.lists;
 
 public class KMP {
-	public static int KMP(CharSequence content, CharSequence stringToFind) {
+	public static int KMP(CharSequence content, CharSequence stringToFind, boolean ignoreCase) {
-		int[] failureTable = failureTable(stringToFind);
+		int[] failureTable = failureTable(ignoreCase, stringToFind);
 
 		int targetPointer = 0; // current char in target string
 		int searchPointer = 0; // current char in search string
 
 		while (searchPointer < content.length()) { // while there is more to search with, keep searching
-			if (content.charAt(searchPointer) == stringToFind.charAt(targetPointer)) { // case 1
+			if (charEquals(ignoreCase, content.charAt(searchPointer), stringToFind.charAt(targetPointer))) { // case 1
 				// found current char in targetPointer in search string
 				targetPointer++;
 				if (targetPointer == stringToFind.length()) { // found all characters
@@ -17,10 +17,10 @@ public class KMP {
 				}
 				searchPointer++; // move forward if not found target string
 			} else if (targetPointer > 0) { // case 2
-				// use failureTable to use pointer pointed at nearest location of usable string prefix
+				// use failureTable to use pointer pointed resetPointerTo nearest location of usable string prefix
 				targetPointer = failureTable[targetPointer];
 			} else { // case 3
-				// targetPointer is pointing at state 0, so restart search with current searchPointer index
+				// targetPointer is pointing resetPointerTo state 0, so restart search with current searchPointer index
 				searchPointer++;
 			}
 		}
@@ -30,22 +30,22 @@ public class KMP {
 	/**
 	 * Returns an int[] that points to last valid string prefix, given target string
 	 */
-	public static int[] failureTable(CharSequence target) {
+	public static int[] failureTable(boolean ignoreCase, CharSequence target) {
 		int[] table = new int[target.length() + 1];
 		// state 0 and 1 are guarenteed be the prior
 		table[0] = -1;
 		table[1] = 0;
 
-		// the pointers pointing at last failure and current satte
+		// the pointers pointing resetPointerTo last failure and current satte
 		int left = 0;
 		int right = 2;
 
 		while (right < table.length) { // RIGHT NEVER MOVES RIGHT UNTIL ASSIGNED A VALID POINTER
-			if (target.charAt(right - 1) == target.charAt(left)) { // when both chars before left and right are equal, link both and move both forward
+			if (charEquals(ignoreCase, target.charAt(right - 1), target.charAt(left))) { // when both chars before left and right are equal, link both and move both forward
 				left++;
 				table[right] = left;
 				right++;
-			}  else if (left > 0) { // if left isn't at the very beginning, then send left backward
+			}  else if (left > 0) { // if left isn't resetPointerTo the very beginning, then send left backward
 				// by following the already set pointer to where it is pointing to
 				left = table[left];
 			} else { // left has fallen all the way back to the beginning
@@ -55,4 +55,20 @@ public class KMP {
 		}
 		return table;
 	}
+
+	private static boolean charEquals(boolean ignoreCase, char char1, char char2) {
+		if (ignoreCase) {
+			if (char1 >= 65 && char1 <= 90) {
+				if (char2 >= 97 && char2 <= 122) {
+					return char1 == char2 - 32;
+				}
+			}
+			if (char2 >= 65 && char2 <= 90) {
+				if (char1 >= 97 && char1 <= 122) {
+					return char2 == char1 - 32;
+				}
+			}
+		}
+		return char1 == char2;
+	}
 }

src/main/java/it/cavallium/strangedb/java/objects/lists/StrangeDbList.java

@@ -63,7 +63,7 @@ public abstract class StrangeDbList<T> extends EnhancedObject implements Element
 	protected void forEachParallelUnsorted_(ConsumerWithIO<T> action) throws IOException {
 		try {
 			int size = size();
-			ExecutorService executorService = ForkJoinPool.commonPool();
+			ExecutorService executorService = Executors.newFixedThreadPool(ForkJoinPool.getCommonPoolParallelism(), (r) -> new Thread(r, "DBList parallel foreach"));
 			VariableWrapper<IOException> exceptionVariableWrapper = new VariableWrapper<>(null);
 			for (int i = 0; i < size; i++) {
 				final int index = i;

src/main/java/it/cavallium/strangedb/java/objects/lists/ValuePointer.java

@@ -46,7 +46,7 @@ public class ValuePointer {
 
 	public <T extends EnhancedObject> ValuePointer prop(Class<T> parentType, String propertyName) {
 
-		ClassPosition[] newPathTypes = append(this.pathTypes, ClassPosition.FIELD);
+		ClassPosition[] newPathTypes = append(this.pathTypes, ClassPosition.PROPERTY);
 
 		Method[] methods = MethodUtils.getMethodsWithAnnotation(parentType, DbProperty.class);
 		DbProperty dbProperty = null;
@@ -99,6 +99,34 @@ public class ValuePointer {
 		return new ValuePointer(newPathNumbers, newPathTypes, newAdditionalData, rootType);
 	}
 
+	public <T extends EnhancedObjectStrangeDbList<?>> ValuePointer allInList(Class<T> parentType) {
+		return inList(parentType, ClassPosition.LIST_ALL, -1);
+	}
+
+	public <T extends EnhancedObjectStrangeDbList<?>> ValuePointer firstInList(Class<T> parentType) {
+		return inList(parentType, ClassPosition.LIST_FIRST, -1);
+	}
+
+	public <T extends EnhancedObjectStrangeDbList<?>> ValuePointer lastInList(Class<T> parentType) {
+		return inList(parentType, ClassPosition.LIST_LAST, -1);
+	}
+
+	public <T extends EnhancedObjectStrangeDbList<?>> ValuePointer indexInList(Class<T> parentType, int position) {
+		return inList(parentType, ClassPosition.LIST_POSITION, position);
+	}
+
+	private <T extends EnhancedObjectStrangeDbList<?>> ValuePointer inList(Class<T> parentType, int listPosition, int itemIndex) {
+		ClassPosition[] newPathTypes = append(this.pathTypes, ClassPosition.ENHANCED_LIST);
+
+		int[] newPathNumbers = append(this.pathNumbers, listPosition);
+		Object[] newAdditionalData = append(this.additionalData, itemIndex);
+		if (newAdditionalData.length > 1) {
+			newAdditionalData[newAdditionalData.length - 2] = parentType;
+		}
+		Class<?> rootType = this.rootType == null ? parentType : this.rootType;
+		return new ValuePointer(newPathNumbers, newPathTypes, newAdditionalData, rootType);
+	}
+
 	public <T extends EnhancedObject> ValuePointer primitiveField(Class<T> parentType, String primitiveFieldName) {
 		ClassPosition[] newPathTypes = append(this.pathTypes, ClassPosition.PRIMITIVE_FIELD);
 
@@ -126,7 +154,7 @@ public class ValuePointer {
 		return new ValuePointer(newPathNumbers, newPathTypes, newAdditionalData, rootType);
 	}
 
-	public ValuePointer at(int index) {
+	public ValuePointer resetPointerTo(int index) {
 		if (index >= pathNumbers.length) {
 			throw new ArrayIndexOutOfBoundsException();
 		}
@@ -148,7 +176,7 @@ public class ValuePointer {
 	}
 
 	public ValuePointer previous() {
-		return at(pathNumbers.length - 2);
+		return resetPointerTo(pathNumbers.length - 2);
 	}
 
 	private static int[] append(int[] array, int value) {
@@ -179,4 +207,5 @@ public class ValuePointer {
 	public Class<?> getRootType() {
 		return rootType;
 	}
+
 }

src/main/java/it/cavallium/strangedb/java/objects/lists/operations/Contains.java

@@ -18,7 +18,7 @@ public class Contains<T extends CharSequence> implements ValueOperation<T> {
 	@Override
 	public boolean evaluate(Object value) {
 		if (value instanceof CharSequence) {
-			return KMP.KMP((CharSequence) value, containsValue) != -1;
+			return KMP.KMP((CharSequence) value, containsValue, false) != -1;
 		}
 		return false;
 	}

src/main/java/it/cavallium/strangedb/java/objects/lists/operations/ContainsIgnoreCase.java

@@ -5,20 +5,20 @@ import it.cavallium.strangedb.java.objects.lists.ValueOperation;
 
 public class ContainsIgnoreCase implements ValueOperation<String> {
 
-	private final String containsValue;
+	private final CharSequence containsValue;
 
-	private ContainsIgnoreCase(String containsValue) {
+	private ContainsIgnoreCase(CharSequence containsValue) {
-		this.containsValue = containsValue.toLowerCase();
+		this.containsValue = containsValue;
 	}
 
-	public static ContainsIgnoreCase containsValue(String value) {
+	public static ContainsIgnoreCase containsValue(CharSequence value) {
 		return new ContainsIgnoreCase(value);
 	}
 
 	@Override
 	public boolean evaluate(Object value) {
-		if (value instanceof String) {
+		if (value instanceof CharSequence) {
-			return KMP.KMP(((String) value).toLowerCase(), containsValue) != -1;
+			return KMP.KMP( (CharSequence) value, containsValue, true) != -1;
 		}
 		return false;
 	}

src/test/java/it/cavallium/strangedb/tests/ListContainer.java

@@ -22,7 +22,7 @@ public class ListContainer extends EnhancedObject {
 		super(databaseTools);
 		this.usersList = new EnhancedObjectStrangeDbList<>(databaseTools, User.class);
 		Random random = new Random();
-		ExecutorService threadPool = ForkJoinPool.commonPool();
+		ExecutorService threadPool = Executors.newWorkStealingPool();
 		for (int i = 0; i < count; i++) {
 			threadPool.execute(() -> {
 				try {