Compare commits
493 Commits
lmdb-paral
...
master
Author | SHA1 | Date | |
---|---|---|---|
|
a14c823941 | ||
|
12669e4eb5 | ||
|
6af06ca90e | ||
|
86377a4e65 | ||
|
591963f630 | ||
|
6a68c8452b | ||
|
c859b8238f | ||
|
e1e6065036 | ||
|
ceff8f5022 | ||
|
06e754d437 | ||
|
8dbfe7a488 | ||
|
ce356e3c33 | ||
|
52863bf498 | ||
|
fe6ec9afe3 | ||
|
8bd0e7cf12 | ||
|
852a5e1d51 | ||
|
bc9d274c58 | ||
|
e8342b783a | ||
|
18191ef2fd | ||
|
6564db6c4f | ||
|
fe9370294b | ||
|
632dd41e9e | ||
|
4695f29a76 | ||
|
5888bc96b4 | ||
|
ff3cbc11b6 | ||
|
ae8e8441ac | ||
|
2792add2b9 | ||
|
89a0fa9408 | ||
|
251240996e | ||
|
024db43de8 | ||
|
a5502af24a | ||
|
fa865a654d | ||
|
361e2f04f6 | ||
|
fe0256dbf9 | ||
|
495cbdea64 | ||
|
117df4cb30 | ||
|
d0c79a57d9 | ||
|
50ce5984ac | ||
|
b3631c5513 | ||
|
0ab75623ba | ||
|
7cffc853b3 | ||
|
6fd7d249de | ||
|
a6e73b83d8 | ||
|
8b747db386 | ||
|
3df0dcf36a | ||
|
0cb1ad55a8 | ||
|
065281a4e4 | ||
|
85bfdc33e9 | ||
|
3b35c18517 | ||
|
e4ec49e9aa | ||
|
0caffbfa79 | ||
|
ce5516bd28 | ||
|
7692b21e3d | ||
|
40cd756f35 | ||
|
a4c322e96a | ||
|
fdb504d9cd | ||
|
e83270906e | ||
|
0e2b3677c4 | ||
|
faa15d64ce | ||
|
7f7f13d7f3 | ||
|
e9c765f7da | ||
|
dec229ac78 | ||
|
e66bc6ce53 | ||
|
3a6883c274 | ||
|
2a817cbc58 | ||
|
bee2fe1bf5 | ||
|
0062a36ed5 | ||
|
3e90ba3704 | ||
|
7e7e1c410b | ||
|
dc03d25fdc | ||
|
26961125c0 | ||
|
df946146a1 | ||
|
93fc28101a | ||
|
2f5c8b618f | ||
|
8499dcf89c | ||
|
0c3afa5839 | ||
|
14e00a1857 | ||
|
7c67676a25 | ||
|
2b6b447e0c | ||
|
2810571d7f | ||
|
249403016a | ||
|
7ac452d52a | ||
|
977dd472c1 | ||
|
cce49a50ee | ||
|
97df3bf725 | ||
|
09bdfed0aa | ||
|
0a325c6ef6 | ||
|
161876c1ee | ||
|
f54388efa8 | ||
|
3180b751ef | ||
|
003799b468 | ||
|
8ac067b639 | ||
|
09113207ed | ||
|
dfa1fc3ecc | ||
|
ed981581ec | ||
|
a83f1ff1a6 | ||
|
468886d154 | ||
|
1aeb0c99d3 | ||
|
af7c3dfd65 | ||
|
81f1c5643d | ||
|
404092106b | ||
|
6037a906dc | ||
|
a21c1f3cf4 | ||
|
ea1b464ddf | ||
|
011c8f839c | ||
|
024c4ee226 | ||
|
0e21c72e0a | ||
|
daa7047614 | ||
|
e0d929dbaa | ||
|
1b83c95856 | ||
|
3f88ff8f83 | ||
|
59f9f01268 | ||
|
cd15f8d23d | ||
|
a9857f7553 | ||
|
5c112484bd | ||
|
df420d4193 | ||
|
ff6922449c | ||
|
694c2d811d | ||
|
b9b420afba | ||
|
cc87c1caeb | ||
|
80710bf0c5 | ||
|
ec579d61dc | ||
|
1c8a41d133 | ||
|
447edbe103 | ||
|
855e4e50e9 | ||
|
547eb1ab13 | ||
|
10d4ac600d | ||
|
992df9b15f | ||
|
410d3563de | ||
|
17ae98750e | ||
|
204d8fa990 | ||
|
36b76d81ed | ||
|
6aa7bb6040 | ||
|
889f59772c | ||
|
548016c66e | ||
|
d28ff73ee6 | ||
|
abe18d0b66 | ||
|
d6f9398eff | ||
|
1f672a63ef | ||
|
c01485158d | ||
|
78ba98c8c4 | ||
|
b33bafec21 | ||
|
5360a7dd7c | ||
|
d004f750a0 | ||
|
32e5a100f5 | ||
|
f739f4f9f4 | ||
|
f9c2f7ca31 | ||
|
0302ad9458 | ||
|
67d3797df8 | ||
|
1c02a601bc | ||
|
ddd71d3b72 | ||
|
8b6f1dfe87 | ||
|
69c28b92d0 | ||
|
907a30dc2b | ||
|
faa2a71597 | ||
|
25a702015e | ||
|
b8835dca5b | ||
|
98444af89e | ||
|
a3ed443bad | ||
|
b1fbf39c87 | ||
|
8ff3381c72 | ||
|
200d7b5eb1 | ||
|
fd202d8dfa | ||
|
d8419a4c1b | ||
|
bca768a4b8 | ||
|
b9ffa1dd49 | ||
|
a4a8926e02 | ||
|
d896780611 | ||
|
f72a9c231b | ||
|
7b2a1677e7 | ||
|
c27f7d4e2b | ||
|
a976a4baa4 | ||
|
8d8442d55d | ||
|
ad584ff8ad | ||
|
015e6797c7 | ||
|
f2358e9bee | ||
|
0bc7b6d659 | ||
|
4ed382197b | ||
|
8f47adfc44 | ||
|
409b2985ca | ||
|
96d19c3e09 | ||
|
2a9427f0e4 | ||
|
caf55a633e | ||
|
8e50976d27 | ||
|
ab93ede348 | ||
|
831af1ef81 | ||
|
aee08f3e48 | ||
|
4a0710ed9a | ||
|
f023fc3b83 | ||
|
5483a1551d | ||
|
9e97d24955 | ||
|
8c9aca21b3 | ||
|
8083364ebf | ||
|
ea2065302a | ||
|
d2e7c56f06 | ||
|
a3d1207d76 | ||
|
73b5092785 | ||
|
d2650161fb | ||
|
367226480b | ||
|
afa159de67 | ||
|
33d8f83933 | ||
|
eb02e0f18d | ||
|
4a08a876ca | ||
|
86e48eab92 | ||
|
62692a1f9a | ||
|
17c40757ba | ||
|
0d830fbd21 | ||
|
fb0bd092a4 | ||
|
8e47c15809 | ||
|
cc6071a4de | ||
|
9db32dacae | ||
|
d4de13c2ab | ||
|
8578facfe7 | ||
|
957866ec99 | ||
|
8c2f3b89b6 | ||
|
563defb2ff | ||
|
3be0d3710c | ||
|
cc9306fbde | ||
|
956f33fb6c | ||
|
0e9b45ebf9 | ||
|
62c2dabd8c | ||
|
a1a509a6cf | ||
|
6dfc10859f | ||
|
e1bd57c837 | ||
|
23d70e55bd | ||
|
65295dbf03 | ||
|
6056eedd75 | ||
|
96de3023a0 | ||
|
fe31f9b1c7 | ||
|
bff4d87164 | ||
|
2e58189015 | ||
|
52c216c0df | ||
|
7f52339a6a | ||
|
5c4519552d | ||
|
18d5ddf6e1 | ||
|
d253111233 | ||
|
82f8e91e99 | ||
|
a720a12701 | ||
|
0e7df84c38 | ||
|
1c36ab040b | ||
|
84c9a2c3cc | ||
|
c9a12760bc | ||
|
6a5a9a3e94 | ||
|
bfc78f1465 | ||
|
fab06db239 | ||
|
99e101914d | ||
|
b5b9fc9d0d | ||
|
f89c3ff707 | ||
|
2c308f0f63 | ||
|
cc69ec3063 | ||
|
e68a4e5121 | ||
|
b30ce665a0 | ||
|
4cf9617703 | ||
|
4fc1a66e4f | ||
|
ca9a2b0794 | ||
|
02f1276181 | ||
|
a1c0e19adc | ||
|
549a3bd178 | ||
|
614d24ac8d | ||
|
19bb638f20 | ||
|
404fe03c14 | ||
|
22f0711ab8 | ||
|
3a7b1498ff | ||
|
87d000968a | ||
|
f63b70ab9d | ||
|
d9c2e8a5f9 | ||
|
c72e4d5a83 | ||
|
59e980f356 | ||
|
04623b754c | ||
|
e03afafcee | ||
|
88a1add102 | ||
|
654a62d7b8 | ||
|
e962ae6336 | ||
|
9d16ccdd9e | ||
|
e7718a8370 | ||
|
c0d4dd0c22 | ||
|
f854ce1253 | ||
|
1bd3b8a945 | ||
|
ec5bf1c5cc | ||
|
116fc88311 | ||
|
f5cb2d8895 | ||
|
735be8ecb8 | ||
|
94cdaaedef | ||
|
4b627664aa | ||
|
80ef0394b1 | ||
|
046c08e5bf | ||
|
798262d72b | ||
|
e038b41d17 | ||
|
632d575a47 | ||
|
31ea2d0e10 | ||
|
be929932aa | ||
|
7e71599893 | ||
|
0ad5a15792 | ||
|
1b150dcbaf | ||
|
d35840ec03 | ||
|
4448947cfd | ||
|
0ca6f4c2c5 | ||
|
851f73481a | ||
|
a7329925d0 | ||
|
1cac7cb0c9 | ||
|
eb5792bbe0 | ||
|
fb44c182fa | ||
|
411bf196af | ||
|
e86965efa7 | ||
|
6315175dc4 | ||
|
95d436860f | ||
|
29210cca80 | ||
|
f9fb679f9b | ||
|
5c6e6411f5 | ||
|
6c6263e1d0 | ||
|
dc69bf8e25 | ||
|
6ac9505653 | ||
|
02cd99a963 | ||
|
24cf7ea58d | ||
|
1cd5fc8eed | ||
|
1dfe0d5a77 | ||
|
a45f357bca | ||
|
c711bbc5ad | ||
|
32dc615f88 | ||
|
c4e696f359 | ||
|
c2d3f79a08 | ||
|
cd26cf61b7 | ||
|
7891b0b9e0 | ||
|
cb83c17811 | ||
|
7d0951956d | ||
|
6bd3fdb677 | ||
|
87c031fe71 | ||
|
39811dc3f3 | ||
|
81b26eed82 | ||
|
388b79c6d1 | ||
|
8e88c78ce7 | ||
|
bafe5a1fe8 | ||
|
2bed1d4d51 | ||
|
6443e75ebd | ||
|
8e6ea58823 | ||
|
372c45220c | ||
|
5f6dfac1da | ||
|
da3e7fdf33 | ||
|
b163260702 | ||
|
b1d8e3e48e | ||
|
e34e1e5852 | ||
|
de5be6564e | ||
|
e866241ff1 | ||
|
bbc77df56b | ||
|
aad7195acb | ||
|
46ac6ca481 | ||
|
28b4fdee50 | ||
|
59c37c0fc9 | ||
|
ba3765eece | ||
|
0a6a0657a3 | ||
|
8999102038 | ||
|
a20bb9c423 | ||
|
4cc8d44fd8 | ||
|
93c5251392 | ||
|
2b81006d56 | ||
|
a4df72fe46 | ||
|
9b5071c45e | ||
|
4a2d143135 | ||
|
16f6025b30 | ||
|
325457dd44 | ||
|
faa7118b8e | ||
|
35a70efec5 | ||
|
f0533a17c9 | ||
|
48f3a54e72 | ||
|
26b9de5eb0 | ||
|
090a47ae86 | ||
|
77af845a8a | ||
|
ed37a769e2 | ||
|
2022495dda | ||
|
eec46d6c50 | ||
|
53519fbc4e | ||
|
86263af6f7 | ||
|
743919b831 | ||
|
85642621df | ||
|
f5729f0b65 | ||
|
ad67aa003c | ||
|
d4bff85dc6 | ||
|
484b123199 | ||
|
6805bbd907 | ||
|
100301a645 | ||
|
8f40fd6a3f | ||
|
cf0db4be31 | ||
|
e4fa423aa3 | ||
|
6752fc8df4 | ||
|
4f52b3d542 | ||
|
f478ea97cd | ||
|
2eb4a84afa | ||
|
5155fc6c10 | ||
|
8a657b4f1d | ||
|
06613ca9e6 | ||
|
8e7f7eaf18 | ||
|
cf61636141 | ||
|
f9335d890f | ||
|
5ebc9abe43 | ||
|
83b1d29047 | ||
|
5c16a65bd3 | ||
|
79a6c3140f | ||
|
2c11b13b7a | ||
|
46e7abfd8c | ||
|
89200c2ed5 | ||
|
8e0d806d2b | ||
|
7b66259da6 | ||
|
dfe8361e19 | ||
|
58943b5e08 | ||
|
5c0434c73f | ||
|
5615c8019a | ||
|
574a35907d | ||
|
fb19a7a9f3 | ||
|
95afa6f9dd | ||
|
cdb65b31f3 | ||
|
cf53eb4f5a | ||
|
f837a1d1b2 | ||
|
ca8718780e | ||
|
adb490371f | ||
|
dfce6cd725 | ||
|
6baa05de51 | ||
|
b2216c1b2c | ||
|
119a4ffe85 | ||
|
08ef14fb76 | ||
|
4f19d2fff3 | ||
|
d2abe044cc | ||
|
ce506f22dd | ||
|
4e5e4423ff | ||
|
ecde6724e5 | ||
|
74121cf462 | ||
|
2cf743ddec | ||
|
aad377acb0 | ||
|
84dea46a35 | ||
|
e6d85d226e | ||
|
ce97f33873 | ||
|
57bcc92512 | ||
|
d6244ab18d | ||
|
64f21409f1 | ||
|
b493decd64 | ||
|
68d8b5240c | ||
|
5769bc7076 | ||
|
24493eb4ff | ||
|
7993a6210b | ||
|
dffb8eb3eb | ||
|
9309692062 | ||
|
aa04a64c34 | ||
|
582813b6c7 | ||
|
3ae3bd7944 | ||
|
2b09f5a738 | ||
|
83bc3ebb04 | ||
|
03bbf3baa9 | ||
|
a1eec93c64 | ||
|
a5666dd5b4 | ||
|
eaef75a304 | ||
|
480ab77db8 | ||
|
1de5e52ffd | ||
|
9952eaffc0 | ||
|
0f62362d21 | ||
|
638595f518 | ||
|
0e9c8c089e | ||
|
6d92ba8a68 | ||
|
1a35930909 | ||
|
1dffb55572 | ||
|
7a712722d7 | ||
|
6e312fe102 | ||
|
2e1678373c | ||
|
01099cc4d1 | ||
|
b7ca57a215 | ||
|
8ad622db0a | ||
|
c59655e844 | ||
|
d58d696ca4 | ||
|
907561d93c | ||
|
5157656a2c | ||
|
3cdafd748e | ||
|
297c249243 | ||
|
1a64d98697 | ||
|
2a5e90d667 | ||
|
18b242d746 | ||
|
6644e040dd | ||
|
b5aa8b4baa | ||
|
cc368aecc8 | ||
|
d8de969bee | ||
|
3d9247c969 | ||
|
044d189600 | ||
|
7047b512fc | ||
|
798b8a5288 | ||
|
06d98040b1 | ||
|
3810c49fa1 | ||
|
29d9aad8bf | ||
|
ed00d474d6 | ||
|
e014266b8d | ||
|
c1c57388e5 | ||
|
3d7e80b4ec | ||
|
891255e18e | ||
|
f30ad372ed | ||
|
23d5f700fb | ||
|
42c4b6e651 | ||
|
d4dae4667d |
.github/workflows
.gitignoreREADME.mdpom.xmlsrc
example/java/it.cavallium.dbengine.client
main
data-generator
java
io/net5/buffer/api/pool
it/cavallium/dbengine
SwappableLuceneSearcher.java
client
Backuppable.javaBadBlock.javaCastMapper.javaCompositeDatabase.javaCompositeDatabasePartLocation.javaCompositeSnapshot.javaCompression.javaConnectionSettings.javaCountedStream.javaDatabaseOptions.javaDbProgress.javaDefaultDatabaseOptions.javaHitEntry.javaHitKey.javaHits.javaIBackuppable.javaIndexAction.javaIndicizer.javaIndicizerAnalyzers.javaIndicizerSimilarities.javaIntOpenHashSetJsonAdapter.javaLazyHitEntry.javaLazyHitKey.javaLongProgressTracker.javaLuceneIndex.javaLuceneIndexImpl.javaLuceneOptions.javaMappedSerializer.javaMappedSerializerFixedLength.javaMemoryStats.javaNRTCachingOptions.javaNoMapper.javaSSTDumpProgress.javaSSTProgress.javaSSTVerificationProgress.javaSnapshotException.javaSort.java
query
database
ColumnProperty.javaColumnUtils.javaDatabaseOperations.javaDatabaseProperties.javaDelta.javaDiscardingCloseable.javaEnglishItalianStopFilter.javaLLDatabaseConnection.javaLLDelta.javaLLDictionary.javaLLEntry.javaLLIndexRequest.javaLLItem.javaLLKeyScore.javaLLKeyValueDatabase.javaLLKeyValueDatabaseStructure.javaLLLuceneIndex.javaLLMultiDatabaseConnection.javaLLRange.javaLLScoreMode.javaLLSearchResult.javaLLSearchResultShard.javaLLSingleton.javaLLSnapshottable.javaLLSoftUpdateDocument.javaLLTerm.javaLLTopKeys.javaLLType.javaLLUpdateDocument.javaLLUpdateFields.javaLLUtils.javaOptionalBuf.javaRocksDBLongProperty.javaRocksDBMapProperty.javaRocksDBProperty.javaRocksDBStringProperty.javaSafeCloseable.javaSerializedKey.javaSubStageEntry.javaTableWithProperties.java
collections
6
.github/workflows/maven-publish.yml
vendored
6
.github/workflows/maven-publish.yml
vendored
@ -14,7 +14,7 @@ jobs:
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- { os: ubuntu-20.04, arch: "linux/amd64" }
|
||||
- { os: ubuntu-21.04, arch: "linux/amd64" }
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
@ -27,11 +27,11 @@ jobs:
|
||||
export REVISION=${{ github.run_number }}
|
||||
|
||||
echo "REVISION=$REVISION" >> $GITHUB_ENV
|
||||
- name: Set up JDK 17
|
||||
- name: Set up JDK 18
|
||||
if: github.ref == 'refs/heads/master'
|
||||
uses: actions/setup-java@v1
|
||||
with:
|
||||
java-version: 17
|
||||
java-version: 18
|
||||
server-id: mchv-release-distribution
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
|
7
.gitignore
vendored
7
.gitignore
vendored
@ -181,3 +181,10 @@ fabric.properties
|
||||
|
||||
dbengine.iml
|
||||
/.idea/
|
||||
|
||||
.ci-friendly-pom.xml
|
||||
/.classpath
|
||||
/.project
|
||||
/.settings/
|
||||
|
||||
.flattened-pom.xml
|
||||
|
@ -4,6 +4,8 @@
|
||||
|
||||
[Reactive](https://www.reactive-streams.org/) database engine written in Java (17+) using [Project Reactor](https://github.com/reactor/reactor-core).
|
||||
|
||||
## DO NOT USE THIS PROJECT: THIS IS A PERSONAL PROJECT, THE API IS NOT STABLE, THE CODE IS NOT TESTED.
|
||||
|
||||
This library provides a basic reactive abstraction and implementation of a **key-value store** and a **search engine**.
|
||||
|
||||
Four implementations exists out-of-the-box, two for the key-value store, two for the search engine, but it's possible to add more implementations.
|
||||
|
@ -1,191 +0,0 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import it.cavallium.dbengine.client.query.QueryUtils;
|
||||
import it.cavallium.dbengine.client.query.current.data.QueryParams;
|
||||
import it.cavallium.dbengine.client.query.current.data.ScoreMode;
|
||||
import it.cavallium.dbengine.client.query.current.data.ScoreSort;
|
||||
import it.cavallium.dbengine.database.LLDocument;
|
||||
import it.cavallium.dbengine.database.LLItem;
|
||||
import it.cavallium.dbengine.database.LLLuceneIndex;
|
||||
import it.cavallium.dbengine.database.LLSignal;
|
||||
import it.cavallium.dbengine.database.LLTerm;
|
||||
import it.cavallium.dbengine.database.disk.LLLocalDatabaseConnection;
|
||||
import it.cavallium.dbengine.lucene.LuceneUtils;
|
||||
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
|
||||
import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.util.Comparator;
|
||||
import java.util.StringJoiner;
|
||||
import java.util.concurrent.CompletionException;
|
||||
import org.apache.lucene.document.Field.Store;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.core.scheduler.Schedulers;
|
||||
|
||||
public class IndicizationExample {
|
||||
|
||||
public static void main(String[] args) {
|
||||
tempIndex(true)
|
||||
.flatMap(index -> index
|
||||
.addDocument(new LLTerm("id", "123"),
|
||||
new LLDocument(new LLItem[]{
|
||||
LLItem.newStringField("id", "123", Store.YES),
|
||||
LLItem.newTextField("name", "Mario", Store.NO),
|
||||
LLItem.newStringField("surname", "Rossi", Store.NO)
|
||||
})
|
||||
)
|
||||
.then(index.refresh())
|
||||
.then(index.search(null,
|
||||
QueryParams
|
||||
.builder()
|
||||
.query(QueryUtils.exactSearch(TextFieldsAnalyzer.N4GramPartialString, "name", "Mario"))
|
||||
.limit(1)
|
||||
.sort(ScoreSort.of())
|
||||
.scoreMode(ScoreMode.of(false, true))
|
||||
.build(),
|
||||
"id"
|
||||
))
|
||||
.flatMap(results -> Mono.from(results
|
||||
.results()
|
||||
.flatMap(r -> r)
|
||||
.doOnNext(signal -> {
|
||||
if (signal.isValue()) {
|
||||
System.out.println("Value: " + signal.getValue());
|
||||
}
|
||||
})
|
||||
.filter(LLSignal::isTotalHitsCount))
|
||||
)
|
||||
.doOnNext(count -> System.out.println("Total hits: " + count))
|
||||
.doOnTerminate(() -> System.out.println("Completed"))
|
||||
.then(index.close())
|
||||
)
|
||||
.subscribeOn(Schedulers.parallel())
|
||||
.block();
|
||||
tempIndex(true)
|
||||
.flatMap(index ->
|
||||
index
|
||||
.addDocument(new LLTerm("id", "126"),
|
||||
new LLDocument(new LLItem[]{
|
||||
LLItem.newStringField("id", "126", Store.YES),
|
||||
LLItem.newTextField("name", "Marioxq", Store.NO),
|
||||
LLItem.newStringField("surname", "Rossi", Store.NO)
|
||||
})
|
||||
)
|
||||
.then(index
|
||||
.addDocument(new LLTerm("id", "123"),
|
||||
new LLDocument(new LLItem[]{
|
||||
LLItem.newStringField("id", "123", Store.YES),
|
||||
LLItem.newTextField("name", "Mario", Store.NO),
|
||||
LLItem.newStringField("surname", "Rossi", Store.NO)
|
||||
})
|
||||
))
|
||||
.then(index
|
||||
.addDocument(new LLTerm("id", "124"),
|
||||
new LLDocument(new LLItem[]{
|
||||
LLItem.newStringField("id", "124", Store.YES),
|
||||
LLItem.newTextField("name", "Mariossi", Store.NO),
|
||||
LLItem.newStringField("surname", "Rossi", Store.NO)
|
||||
})
|
||||
))
|
||||
.then(index
|
||||
.addDocument(new LLTerm("id", "125"),
|
||||
new LLDocument(new LLItem[]{
|
||||
LLItem.newStringField("id", "125", Store.YES),
|
||||
LLItem.newTextField("name", "Mario marios", Store.NO),
|
||||
LLItem.newStringField("surname", "Rossi", Store.NO)
|
||||
})
|
||||
))
|
||||
.then(index
|
||||
.addDocument(new LLTerm("id", "128"),
|
||||
new LLDocument(new LLItem[]{
|
||||
LLItem.newStringField("id", "128", Store.YES),
|
||||
LLItem.newTextField("name", "Marion", Store.NO),
|
||||
LLItem.newStringField("surname", "Rossi", Store.NO)
|
||||
})
|
||||
))
|
||||
.then(index
|
||||
.addDocument(new LLTerm("id", "127"),
|
||||
new LLDocument(new LLItem[]{
|
||||
LLItem.newStringField("id", "127", Store.YES),
|
||||
LLItem.newTextField("name", "Mariotto", Store.NO),
|
||||
LLItem.newStringField("surname", "Rossi", Store.NO)
|
||||
})
|
||||
))
|
||||
.then(index.refresh())
|
||||
.then(index.search(null,
|
||||
QueryParams
|
||||
.builder()
|
||||
.query(QueryUtils.exactSearch(TextFieldsAnalyzer.N4GramPartialString, "name", "Mario"))
|
||||
.limit(10)
|
||||
.sort(MultiSort.topScore().getQuerySort())
|
||||
.scoreMode(ScoreMode.of(false, true))
|
||||
.build(),
|
||||
"id"
|
||||
))
|
||||
.flatMap(results -> LuceneUtils.mergeSignalStreamRaw(results
|
||||
.results(), MultiSort.topScoreRaw(), 10L)
|
||||
.doOnNext(value -> System.out.println("Value: " + value))
|
||||
.then(Mono.from(results
|
||||
.results()
|
||||
.flatMap(part -> part)
|
||||
.filter(LLSignal::isTotalHitsCount)
|
||||
.map(LLSignal::getTotalHitsCount)))
|
||||
)
|
||||
.doOnNext(count -> System.out.println("Total hits: " + count))
|
||||
.doOnTerminate(() -> System.out.println("Completed"))
|
||||
.then(index.close())
|
||||
)
|
||||
.subscribeOn(Schedulers.parallel())
|
||||
.block();
|
||||
}
|
||||
|
||||
public static final class CurrentCustomType {
|
||||
|
||||
private final int number;
|
||||
|
||||
public CurrentCustomType(int number) {
|
||||
this.number = number;
|
||||
}
|
||||
|
||||
public int getNumber() {
|
||||
return number;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return new StringJoiner(", ", CurrentCustomType.class.getSimpleName() + "[", "]")
|
||||
.add("number=" + number)
|
||||
.toString();
|
||||
}
|
||||
}
|
||||
|
||||
private static <U> Mono<? extends LLLuceneIndex> tempIndex(boolean delete) {
|
||||
var wrkspcPath = Path.of("/tmp/tempdb/");
|
||||
return Mono
|
||||
.fromCallable(() -> {
|
||||
if (delete && Files.exists(wrkspcPath)) {
|
||||
Files.walk(wrkspcPath).sorted(Comparator.reverseOrder()).forEach(file -> {
|
||||
try {
|
||||
Files.delete(file);
|
||||
} catch (IOException ex) {
|
||||
throw new CompletionException(ex);
|
||||
}
|
||||
});
|
||||
}
|
||||
Files.createDirectories(wrkspcPath);
|
||||
return null;
|
||||
})
|
||||
.subscribeOn(Schedulers.boundedElastic())
|
||||
.then(new LLLocalDatabaseConnection(wrkspcPath, true).connect())
|
||||
.flatMap(conn -> conn.getLuceneIndex("testindices",
|
||||
10,
|
||||
TextFieldsAnalyzer.N4GramPartialString,
|
||||
TextFieldsSimilarity.NGramBM25Plus,
|
||||
Duration.ofSeconds(5),
|
||||
Duration.ofSeconds(5),
|
||||
false
|
||||
));
|
||||
}
|
||||
}
|
@ -2,198 +2,391 @@
|
||||
currentVersion: "0.0.0"
|
||||
interfacesData:
|
||||
Query: []
|
||||
# versions must have only numbers, lowercase letters, dots, dashes. Maximum: 99.999.9999
|
||||
superTypesData:
|
||||
Query: [
|
||||
BoxedQuery, TermQuery, IntTermQuery, IntNDTermQuery, LongTermQuery, LongNDTermQuery, FloatTermQuery,
|
||||
FloatNDTermQuery, DoubleTermQuery, DoubleNDTermQuery,
|
||||
PhraseQuery, SolrTextQuery, WildcardQuery, SynonymQuery, FuzzyQuery, MatchAllDocsQuery, MatchNoDocsQuery,
|
||||
BooleanQuery, SortedNumericDocValuesFieldSlowRangeQuery, SortedDocFieldExistsQuery,
|
||||
ConstantScoreQuery, BoostQuery, IntPointRangeQuery, IntNDPointRangeQuery, LongPointRangeQuery,
|
||||
FloatPointRangeQuery, DoublePointRangeQuery, LongNDPointRangeQuery, FloatNDPointRangeQuery,
|
||||
DoubleNDPointRangeQuery, IntPointExactQuery, IntNDPointExactQuery, LongPointExactQuery, FloatPointExactQuery,
|
||||
FloatPointExactQuery, DoublePointExactQuery, LongNDPointExactQuery, FloatNDPointExactQuery,
|
||||
DoubleNDPointExactQuery, IntPointSetQuery, LongPointSetQuery, FloatPointSetQuery, DoublePointSetQuery,
|
||||
StandardQuery, FieldExistsQuery, FilterConfigQuery, SolrFunctionQuery, MoreLikeThisQuery
|
||||
]
|
||||
Occur: [OccurMust, OccurMustNot, OccurShould, OccurFilter]
|
||||
Sort: [NoSort, NumericSort, ScoreSort, DocSort, RandomSort]
|
||||
NumberFormat: [NumberFormatDecimal]
|
||||
PointType: [PointTypeInt, PointTypeLong, PointTypeFloat, PointTypeDouble]
|
||||
customTypesData: {}
|
||||
baseTypesData:
|
||||
|
||||
# Basic data
|
||||
# ==========
|
||||
|
||||
# Wrapper for type Query
|
||||
BoxedQuery:
|
||||
data:
|
||||
query: Query
|
||||
# A term consists in a field that is exactly equal to the value string
|
||||
Term:
|
||||
data:
|
||||
field: String
|
||||
value: String
|
||||
# A Term with a position relative to the start of the query. Used internally in some specific queries
|
||||
TermPosition:
|
||||
data:
|
||||
term: Term
|
||||
position: int
|
||||
# A Term with a specified boost. Used internally in some specific queries
|
||||
TermAndBoost:
|
||||
data:
|
||||
term: Term
|
||||
boost: float
|
||||
# Occur options used for boolean queries
|
||||
OccurMust:
|
||||
data: { }
|
||||
OccurMustNot:
|
||||
data: { }
|
||||
OccurShould:
|
||||
data: { }
|
||||
OccurFilter:
|
||||
data: { }
|
||||
|
||||
# Special queries
|
||||
# ===============
|
||||
|
||||
# Raw lucene string query, parsable by lucene StandardQueryParser
|
||||
StandardQuery:
|
||||
data:
|
||||
query: String
|
||||
# Should be at least one field!
|
||||
defaultFields: String[]
|
||||
pointsConfig: PointConfig[]
|
||||
termFields: String[]
|
||||
PointConfig:
|
||||
data:
|
||||
field: String
|
||||
data: PointConfigData
|
||||
PointConfigData:
|
||||
data:
|
||||
numberFormat: NumberFormat
|
||||
type: PointType
|
||||
NumberFormatDecimal: { data: { } }
|
||||
PointTypeInt: { data: { } }
|
||||
PointTypeLong: { data: { } }
|
||||
PointTypeFloat: { data: { } }
|
||||
PointTypeDouble: { data: { } }
|
||||
|
||||
# Text queries
|
||||
# ============
|
||||
|
||||
# Query that matches a term.
|
||||
TermQuery:
|
||||
data:
|
||||
term: Term
|
||||
# Query that matches a term.
|
||||
LongTermQuery:
|
||||
data:
|
||||
field: String
|
||||
value: long
|
||||
LongNDTermQuery:
|
||||
data:
|
||||
field: String
|
||||
value: long[]
|
||||
# Query that matches a term.
|
||||
IntTermQuery:
|
||||
data:
|
||||
field: String
|
||||
value: int
|
||||
# Query that matches a term.
|
||||
IntNDTermQuery:
|
||||
data:
|
||||
field: String
|
||||
value: int[]
|
||||
# Query that matches a term.
|
||||
FloatTermQuery:
|
||||
data:
|
||||
field: String
|
||||
value: float
|
||||
# Query that matches a term.
|
||||
FloatNDTermQuery:
|
||||
data:
|
||||
field: String
|
||||
value: float[]
|
||||
# Query that matches a term.
|
||||
DoubleTermQuery:
|
||||
data:
|
||||
field: String
|
||||
value: double
|
||||
# Query that matches a term.
|
||||
DoubleNDTermQuery:
|
||||
data:
|
||||
field: String
|
||||
value: double[]
|
||||
# Query that matches the existence of a field.
|
||||
FieldExistsQuery:
|
||||
data:
|
||||
field: String
|
||||
# Query used to configure the Solr cache.
|
||||
FilterConfigQuery:
|
||||
data:
|
||||
query: Query
|
||||
cached: boolean
|
||||
# Query that represents a Solr Function Query (https://solr.apache.org/guide/solr/latest/query-guide/function-queries.html)
|
||||
SolrFunctionQuery:
|
||||
data:
|
||||
query: String
|
||||
boost: double
|
||||
MoreLikeThisQuery:
|
||||
data:
|
||||
id: String
|
||||
fieldList: String[]
|
||||
minTf: -int
|
||||
minDf: -int
|
||||
maxDf: -int
|
||||
maxDfPct: -int
|
||||
minWl: -int
|
||||
maxWl: -int
|
||||
maxQt: -int
|
||||
maxNtp: -int
|
||||
boost: -boolean
|
||||
qf: String[]
|
||||
# Query that matches a phrase.
|
||||
PhraseQuery:
|
||||
data:
|
||||
# A phrase is a collection of positioned terms, with absolute positions,
|
||||
# counted as characters from the beginning of the phrase.
|
||||
phrase: TermPosition[]
|
||||
slop: int
|
||||
# Query that matches a phrase. (Solr)
|
||||
SolrTextQuery:
|
||||
data:
|
||||
# Field name
|
||||
field: String
|
||||
# Text query
|
||||
phrase: String
|
||||
slop: int
|
||||
# Advanced query that matches text allowing asterisks in the query
|
||||
WildcardQuery:
|
||||
data:
|
||||
field: String
|
||||
pattern: String # Example: "*ing"
|
||||
# Advanced query that matches different exact values (synonyms)
|
||||
SynonymQuery:
|
||||
data:
|
||||
field: String
|
||||
parts: TermAndBoost[] # Each term has a boost. The preferred synonym has the highest boost value.
|
||||
# Advanced query. todo: document it
|
||||
FuzzyQuery:
|
||||
data:
|
||||
term: Term
|
||||
maxEdits: int
|
||||
prefixLength: int
|
||||
maxExpansions: int
|
||||
transpositions: boolean
|
||||
|
||||
|
||||
# Combination queries
|
||||
# ===================
|
||||
|
||||
# Query that matches everything
|
||||
MatchAllDocsQuery:
|
||||
data: {}
|
||||
# Query that matches nothing
|
||||
MatchNoDocsQuery:
|
||||
data: {}
|
||||
# Query that matches if the document satisfies all the required parts
|
||||
BooleanQuery:
|
||||
data:
|
||||
# Each part can be:
|
||||
# - "MUST"
|
||||
# - "MUST_NOT"
|
||||
# - "SHOULD"
|
||||
# - "FILTER" (advanced, ignore this)
|
||||
# "SHOULD" is like "MUST" but it's not necessary.
|
||||
parts: BooleanQueryPart[]
|
||||
minShouldMatch: int # If set, it specifies how many "SHOULD" parts must be matched. 0 if not set
|
||||
# Part of a boolean query
|
||||
BooleanQueryPart:
|
||||
data:
|
||||
query: Query
|
||||
occur: Occur
|
||||
|
||||
|
||||
# Number queries
|
||||
# ==============
|
||||
|
||||
# Advanced query that matches only a range of a sorted field, from "min" to "max".
|
||||
SortedNumericDocValuesFieldSlowRangeQuery:
|
||||
data:
|
||||
field: String
|
||||
min: long
|
||||
max: long
|
||||
# Query that matches if the sorted field exist in the document
|
||||
SortedDocFieldExistsQuery:
|
||||
data:
|
||||
field: String
|
||||
|
||||
|
||||
# Score modifying queries
|
||||
# ============
|
||||
|
||||
# Query that fixes the score of a query to 1
|
||||
ConstantScoreQuery:
|
||||
data:
|
||||
query: Query
|
||||
# Query that boosts the query score
|
||||
BoostQuery:
|
||||
data:
|
||||
query: Query
|
||||
scoreBoost: float
|
||||
|
||||
|
||||
# Sorted fields queries
|
||||
# =====================
|
||||
|
||||
# Query that matches an int point field, from "min", to "max"
|
||||
IntPointRangeQuery:
|
||||
data:
|
||||
field: String
|
||||
min: int
|
||||
max: int
|
||||
# Query that matches an int point field, from "min", to "max"
|
||||
IntNDPointRangeQuery:
|
||||
data:
|
||||
field: String
|
||||
min: int[]
|
||||
max: int[]
|
||||
# Query that matches a long point field, from "min", to "max"
|
||||
LongPointRangeQuery:
|
||||
data:
|
||||
field: String
|
||||
min: long
|
||||
max: long
|
||||
# Query that matches a float point field, from "min", to "max"
|
||||
FloatPointRangeQuery:
|
||||
data:
|
||||
field: String
|
||||
min: float
|
||||
max: float
|
||||
# Query that matches a double point field, from "min", to "max"
|
||||
DoublePointRangeQuery:
|
||||
data:
|
||||
field: String
|
||||
min: double
|
||||
max: double
|
||||
# Query that matches a long point field, from "min", to "max"
|
||||
LongNDPointRangeQuery:
|
||||
data:
|
||||
field: String
|
||||
min: long[]
|
||||
max: long[]
|
||||
# Query that matches a float point field, from "min", to "max"
|
||||
FloatNDPointRangeQuery:
|
||||
data:
|
||||
field: String
|
||||
min: float[]
|
||||
max: float[]
|
||||
# Query that matches a double point field, from "min", to "max"
|
||||
DoubleNDPointRangeQuery:
|
||||
data:
|
||||
field: String
|
||||
min: double[]
|
||||
max: double[]
|
||||
# Query that matches an int point field
|
||||
IntPointExactQuery:
|
||||
data:
|
||||
field: String
|
||||
value: int
|
||||
# Query that matches an int point field
|
||||
IntNDPointExactQuery:
|
||||
data:
|
||||
field: String
|
||||
value: int[]
|
||||
# Query that matches a long point field
|
||||
LongPointExactQuery:
|
||||
data:
|
||||
field: String
|
||||
value: long
|
||||
# Query that matches a float point field
|
||||
FloatPointExactQuery:
|
||||
data:
|
||||
field: String
|
||||
value: float
|
||||
# Query that matches a double point field
|
||||
DoublePointExactQuery:
|
||||
data:
|
||||
field: String
|
||||
value: double
|
||||
# Query that matches a long point field
|
||||
LongNDPointExactQuery:
|
||||
data:
|
||||
field: String
|
||||
value: long[]
|
||||
# Query that matches a float point field
|
||||
FloatNDPointExactQuery:
|
||||
data:
|
||||
field: String
|
||||
value: float[]
|
||||
# Query that matches a double point field
|
||||
DoubleNDPointExactQuery:
|
||||
data:
|
||||
field: String
|
||||
value: double[]
|
||||
# Query that matches a set of int point field
|
||||
IntPointSetQuery:
|
||||
data:
|
||||
field: String
|
||||
values: int[]
|
||||
# Query that matches a set of long point field
|
||||
LongPointSetQuery:
|
||||
data:
|
||||
field: String
|
||||
values: long[]
|
||||
# Query that matches a set of float point field
|
||||
FloatPointSetQuery:
|
||||
data:
|
||||
field: String
|
||||
values: float[]
|
||||
# Query that matches a set of double point field
|
||||
DoublePointSetQuery:
|
||||
data:
|
||||
field: String
|
||||
values: double[]
|
||||
|
||||
|
||||
# Extra data used for parameters and the client
|
||||
# =============================================
|
||||
|
||||
# Query parameters
|
||||
QueryParams:
|
||||
data:
|
||||
query: Query
|
||||
offset: long
|
||||
limit: long
|
||||
sort: Sort
|
||||
computePreciseHitsCount: boolean
|
||||
timeoutMilliseconds: long
|
||||
NoSort:
|
||||
data: { }
|
||||
NumericSort:
|
||||
data:
|
||||
field: String
|
||||
reverse: boolean
|
||||
RandomSort:
|
||||
data: { }
|
||||
ScoreSort:
|
||||
data: { }
|
||||
DocSort:
|
||||
data: { }
|
||||
TotalHitsCount:
|
||||
stringRepresenter: "it.cavallium.dbengine.client.query.QueryUtil.toHumanReadableString"
|
||||
data:
|
||||
value: long
|
||||
exact: boolean
|
||||
versions:
|
||||
0.0.0:
|
||||
details:
|
||||
changelog: "First version"
|
||||
superTypes:
|
||||
Query: [
|
||||
BoxedQuery, TermQuery, PhraseQuery, WildcardQuery, SynonymQuery, FuzzyQuery, MatchAllDocsQuery,
|
||||
MatchNoDocsQuery, BooleanQuery, SortedNumericDocValuesFieldSlowRangeQuery, SortedDocFieldExistsQuery,
|
||||
ConstantScoreQuery, BoostQuery, IntPointRangeQuery, LongPointRangeQuery, IntPointExactQuery,
|
||||
LongPointExactQuery
|
||||
]
|
||||
Occur: [OccurMust, OccurMustNot, OccurShould, OccurFilter]
|
||||
Sort: [NoSort, NumericSort, ScoreSort, DocSort, RandomSort]
|
||||
customTypes: {}
|
||||
classes:
|
||||
|
||||
# Basic data
|
||||
# ==========
|
||||
|
||||
# Wrapper for type Query
|
||||
BoxedQuery:
|
||||
data:
|
||||
query: Query
|
||||
# A term consists in a field that is exactly equal to the value string
|
||||
Term:
|
||||
data:
|
||||
field: String
|
||||
value: String
|
||||
# A Term with a position relative to the start of the query. Used internally in some specific queries
|
||||
TermPosition:
|
||||
data:
|
||||
term: Term
|
||||
position: int
|
||||
# A Term with a specified boost. Used internally in some specific queries
|
||||
TermAndBoost:
|
||||
data:
|
||||
term: Term
|
||||
boost: float
|
||||
# Occur options used for boolean queries
|
||||
OccurMust:
|
||||
data: { }
|
||||
OccurMustNot:
|
||||
data: { }
|
||||
OccurShould:
|
||||
data: { }
|
||||
OccurFilter:
|
||||
data: { }
|
||||
|
||||
# Text queries
|
||||
# ============
|
||||
|
||||
# Query that matches a term.
|
||||
TermQuery:
|
||||
data:
|
||||
term: Term
|
||||
# Query that matches a phrase.
|
||||
PhraseQuery:
|
||||
data:
|
||||
# A phrase is a collection of positioned terms, with absolute positions,
|
||||
# counted as characters from the beginning of the phrase.
|
||||
phrase: TermPosition[]
|
||||
slop: int
|
||||
# Advanced query that matches text allowing asterisks in the query
|
||||
WildcardQuery:
|
||||
data:
|
||||
field: String
|
||||
pattern: String # Example: "*ing"
|
||||
# Advanced query that matches different exact values (synonyms)
|
||||
SynonymQuery:
|
||||
data:
|
||||
field: String
|
||||
parts: TermAndBoost[] # Each term has a boost. The preferred synonym has the highest boost value.
|
||||
# Advanced query. todo: document it
|
||||
FuzzyQuery:
|
||||
data:
|
||||
term: Term
|
||||
maxEdits: int
|
||||
prefixLength: int
|
||||
maxExpansions: int
|
||||
transpositions: boolean
|
||||
|
||||
|
||||
# Combination queries
|
||||
# ===================
|
||||
|
||||
# Query that matches everything
|
||||
MatchAllDocsQuery:
|
||||
data: {}
|
||||
# Query that matches nothing
|
||||
MatchNoDocsQuery:
|
||||
data: {}
|
||||
# Query that matches if the document satisfies all the required parts
|
||||
BooleanQuery:
|
||||
data:
|
||||
# Each part can be:
|
||||
# - "MUST"
|
||||
# - "MUST_NOT"
|
||||
# - "SHOULD"
|
||||
# - "FILTER" (advanced, ignore this)
|
||||
# "SHOULD" is like "MUST" but it's not necessary.
|
||||
parts: BooleanQueryPart[]
|
||||
minShouldMatch: int # If set, it specifies how many "SHOULD" parts must be matched. 0 if not set
|
||||
# Part of a boolean query
|
||||
BooleanQueryPart:
|
||||
data:
|
||||
query: Query
|
||||
occur: Occur
|
||||
|
||||
|
||||
# Number queries
|
||||
# ==============
|
||||
|
||||
# Advanced query that matches only a range of a sorted field, from "min" to "max".
|
||||
SortedNumericDocValuesFieldSlowRangeQuery:
|
||||
data:
|
||||
field: String
|
||||
min: long
|
||||
max: long
|
||||
# Query that matches if the sorted field exist in the document
|
||||
SortedDocFieldExistsQuery:
|
||||
data:
|
||||
field: String
|
||||
|
||||
|
||||
# Score modifying queries
|
||||
# ============
|
||||
|
||||
# Query that fixes the score of a query to 1
|
||||
ConstantScoreQuery:
|
||||
data:
|
||||
query: Query
|
||||
# Query that boosts the query score
|
||||
BoostQuery:
|
||||
data:
|
||||
query: Query
|
||||
scoreBoost: float
|
||||
|
||||
|
||||
# Sorted fields queries
|
||||
# =====================
|
||||
|
||||
# Query that matches an int point field, from "min", to "max"
|
||||
IntPointRangeQuery:
|
||||
data:
|
||||
field: String
|
||||
min: int
|
||||
max: int
|
||||
# Query that matches a long point field, from "min", to "max"
|
||||
LongPointRangeQuery:
|
||||
data:
|
||||
field: String
|
||||
min: long
|
||||
max: long
|
||||
# Query that matches an int point field
|
||||
IntPointExactQuery:
|
||||
data:
|
||||
field: String
|
||||
value: int
|
||||
# Query that matches a long point field
|
||||
LongPointExactQuery:
|
||||
data:
|
||||
field: String
|
||||
value: long
|
||||
|
||||
|
||||
# Extra data used for parameters and the client
|
||||
# =============================================
|
||||
|
||||
# Query parameters
|
||||
QueryParams:
|
||||
data:
|
||||
query: Query
|
||||
offset: long
|
||||
limit: long
|
||||
minCompetitiveScore: -float
|
||||
sort: Sort
|
||||
complete: boolean
|
||||
NoSort:
|
||||
data: { }
|
||||
NumericSort:
|
||||
data:
|
||||
field: String
|
||||
reverse: boolean
|
||||
RandomSort:
|
||||
data: { }
|
||||
ScoreSort:
|
||||
data: { }
|
||||
DocSort:
|
||||
data: { }
|
||||
TotalHitsCount:
|
||||
stringRepresenter: "it.cavallium.dbengine.lucene.LuceneUtils.toHumanReadableString"
|
||||
data:
|
||||
value: long
|
||||
exact: boolean
|
||||
|
220
src/main/data-generator/quic-rpc.yaml
Normal file
220
src/main/data-generator/quic-rpc.yaml
Normal file
@ -0,0 +1,220 @@
|
||||
# A type that starts with "-" is an optional type, otherwise it can't be null
|
||||
currentVersion: "0.0.0"
|
||||
interfacesData:
|
||||
ClientBoundRequest:
|
||||
extendInterfaces: [RPCEvent]
|
||||
ClientBoundResponse:
|
||||
extendInterfaces: [RPCEvent]
|
||||
ServerBoundRequest:
|
||||
extendInterfaces: [RPCEvent]
|
||||
ServerBoundResponse:
|
||||
extendInterfaces: [RPCEvent]
|
||||
superTypesData:
|
||||
RPCEvent: [
|
||||
Empty,
|
||||
Binary,
|
||||
BinaryOptional,
|
||||
SingletonUpdateOldData,
|
||||
GeneratedEntityId,
|
||||
GetDatabase,
|
||||
Disconnect,
|
||||
GetSingleton,
|
||||
SingletonGet,
|
||||
SingletonSet,
|
||||
SingletonUpdateInit,
|
||||
SingletonUpdateEnd,
|
||||
RPCCrash,
|
||||
CloseDatabase
|
||||
]
|
||||
ServerBoundRequest: [
|
||||
GetDatabase,
|
||||
Disconnect,
|
||||
GetSingleton,
|
||||
SingletonGet,
|
||||
SingletonSet,
|
||||
SingletonUpdateInit,
|
||||
CloseDatabase
|
||||
]
|
||||
ClientBoundResponse: [
|
||||
Empty,
|
||||
GeneratedEntityId,
|
||||
Binary,
|
||||
BinaryOptional,
|
||||
RPCCrash
|
||||
]
|
||||
ClientBoundRequest: [
|
||||
SingletonUpdateOldData
|
||||
]
|
||||
ServerBoundResponse: [
|
||||
Empty,
|
||||
SingletonUpdateEnd
|
||||
]
|
||||
Filter: [
|
||||
NoFilter,
|
||||
BloomFilter
|
||||
]
|
||||
customTypesData:
|
||||
Path:
|
||||
javaClass: java.nio.file.Path
|
||||
serializer: it.cavallium.dbengine.database.remote.PathSerializer
|
||||
Compression:
|
||||
javaClass: it.cavallium.dbengine.client.Compression
|
||||
serializer: it.cavallium.dbengine.database.remote.CompressionSerializer
|
||||
Duration:
|
||||
javaClass: java.time.Duration
|
||||
serializer: it.cavallium.dbengine.database.remote.DurationSerializer
|
||||
RocksDB:
|
||||
javaClass: org.rocksdb.RocksDB
|
||||
serializer: it.cavallium.dbengine.database.remote.RocksDBSerializer
|
||||
ColumnFamilyHandle:
|
||||
javaClass: org.rocksdb.ColumnFamilyHandle
|
||||
serializer: it.cavallium.dbengine.database.remote.ColumnFamilyHandleSerializer
|
||||
UpdateReturnMode:
|
||||
javaClass: it.cavallium.dbengine.database.UpdateReturnMode
|
||||
serializer: it.cavallium.dbengine.database.remote.UpdateReturnModeSerializer
|
||||
LLSnapshot:
|
||||
javaClass: it.cavallium.dbengine.database.LLSnapshot
|
||||
serializer: it.cavallium.dbengine.database.remote.LLSnapshotSerializer
|
||||
|
||||
Bytes:
|
||||
javaClass: it.cavallium.buffer.Buf
|
||||
serializer: it.cavallium.dbengine.database.remote.BufSerializer
|
||||
StringMap:
|
||||
javaClass: java.util.Map<java.lang.String, java.lang.String>
|
||||
serializer: it.cavallium.dbengine.database.remote.StringMapSerializer
|
||||
String2ColumnFamilyHandleMap:
|
||||
javaClass: java.util.Map<java.lang.String, org.rocksdb.ColumnFamilyHandle>
|
||||
serializer: it.cavallium.dbengine.database.remote.String2ColumnFamilyHandleMapSerializer
|
||||
baseTypesData:
|
||||
BoxedRPCEvent:
|
||||
data:
|
||||
val: RPCEvent
|
||||
|
||||
# Server-bound requests
|
||||
|
||||
GetDatabase:
|
||||
data:
|
||||
name: String
|
||||
columns: Column[]
|
||||
databaseOptions: DatabaseOptions
|
||||
Disconnect: { data: { } }
|
||||
GetSingleton:
|
||||
data:
|
||||
databaseId: long
|
||||
singletonListColumnName: byte[]
|
||||
name: byte[]
|
||||
defaultValue: -Bytes
|
||||
SingletonGet:
|
||||
data:
|
||||
singletonId: long
|
||||
snapshot: -LLSnapshot
|
||||
SingletonSet:
|
||||
data:
|
||||
singletonId: long
|
||||
value: -Bytes
|
||||
SingletonUpdateInit:
|
||||
data:
|
||||
singletonId: long
|
||||
updateReturnMode: UpdateReturnMode
|
||||
SingletonUpdateEnd:
|
||||
data:
|
||||
exist: boolean
|
||||
value: byte[]
|
||||
CloseDatabase:
|
||||
data:
|
||||
databaseId: long
|
||||
|
||||
# Client-bound responses
|
||||
|
||||
GeneratedEntityId:
|
||||
data:
|
||||
id: long
|
||||
RPCCrash:
|
||||
data:
|
||||
code: int
|
||||
message: -String
|
||||
|
||||
# Client-bound requests
|
||||
|
||||
SingletonUpdateOldData:
|
||||
data:
|
||||
exist: boolean
|
||||
oldValue: byte[]
|
||||
|
||||
# Server-bound responses
|
||||
|
||||
|
||||
# Data
|
||||
|
||||
BinaryOptional:
|
||||
data:
|
||||
val: -Binary
|
||||
Binary:
|
||||
data:
|
||||
val: byte[]
|
||||
Empty: { data: { } }
|
||||
Column:
|
||||
data:
|
||||
name: String
|
||||
DatabaseOptions:
|
||||
data:
|
||||
volumes: DatabaseVolume[]
|
||||
extraFlags: StringMap
|
||||
absoluteConsistency: boolean
|
||||
lowMemory: boolean
|
||||
useDirectIO: boolean
|
||||
allowMemoryMapping: boolean
|
||||
optimistic: boolean
|
||||
maxOpenFiles: -int
|
||||
blockCache: -long
|
||||
persistentCaches: PersistentCache[]
|
||||
writeBufferManager: -long
|
||||
spinning: boolean
|
||||
defaultColumnOptions: ColumnOptions
|
||||
columnOptions: NamedColumnOptions[]
|
||||
logPath: -String
|
||||
walPath: -String
|
||||
openAsSecondary: boolean
|
||||
secondaryDirectoryName: -String
|
||||
ColumnOptions:
|
||||
data:
|
||||
levels: DatabaseLevel[]
|
||||
memtableMemoryBudgetBytes: -long
|
||||
cacheIndexAndFilterBlocks: -boolean
|
||||
partitionFilters: -boolean
|
||||
filter: -Filter
|
||||
blockSize: -int
|
||||
persistentCacheId: -String
|
||||
writeBufferSize: -long
|
||||
blobFiles: boolean
|
||||
minBlobSize: -long
|
||||
blobFileSize: -long
|
||||
blobCompressionType: -Compression
|
||||
NamedColumnOptions:
|
||||
data:
|
||||
name: String
|
||||
options: ColumnOptions
|
||||
NoFilter:
|
||||
data: {}
|
||||
BloomFilter:
|
||||
data:
|
||||
bitsPerKey: int
|
||||
optimizeForHits: -boolean
|
||||
PersistentCache:
|
||||
data:
|
||||
id: String
|
||||
path: String
|
||||
size: long
|
||||
optimizeForNvm: boolean
|
||||
DatabaseVolume:
|
||||
data:
|
||||
volumePath: Path
|
||||
targetSizeBytes: long
|
||||
DatabaseLevel:
|
||||
data:
|
||||
maxDictBytes: int
|
||||
compression: Compression
|
||||
versions:
|
||||
0.0.0:
|
||||
details:
|
||||
changelog: "First version"
|
@ -1,46 +0,0 @@
|
||||
package io.net5.buffer.api.pool;
|
||||
|
||||
import java.lang.invoke.MethodHandle;
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.lang.invoke.MethodType;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Netty5 hides some metrics. This utility class can read them.
|
||||
*/
|
||||
public class MetricUtils {
|
||||
|
||||
private static final MethodHandle GET_ARENA_METRICS;
|
||||
|
||||
static {
|
||||
var lookup = MethodHandles.lookup();
|
||||
|
||||
// Get the method handle that returns the metrics of each pool arena
|
||||
MethodHandle handle = null;
|
||||
try {
|
||||
// Find the class
|
||||
var pooledBufferClass = Class.forName("io.net5.buffer.api.pool.PooledBufferAllocatorMetric");
|
||||
// Find the handle of the method
|
||||
handle = lookup.findVirtual(pooledBufferClass, "arenaMetrics", MethodType.methodType(List.class));
|
||||
} catch (NoSuchMethodException | IllegalAccessException | ClassNotFoundException ignored) {
|
||||
|
||||
}
|
||||
GET_ARENA_METRICS = handle;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the metrics of each pool arena of a pooled allocator
|
||||
* @param allocator Pooled allocator
|
||||
* @return A list of {@link PoolArenaMetric}
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
public static List<PoolArenaMetric> getPoolArenaMetrics(PooledBufferAllocator allocator) {
|
||||
var metric = allocator.metric();
|
||||
try {
|
||||
// Invoke the method to get the metrics
|
||||
return (List<PoolArenaMetric>) GET_ARENA_METRICS.invoke(metric);
|
||||
} catch (Throwable e) {
|
||||
return List.of();
|
||||
}
|
||||
}
|
||||
}
|
@ -1,60 +0,0 @@
|
||||
package io.net5.buffer.api.pool;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class PooledBufferAllocatorMetricUtils implements BufferAllocatorMetric {
|
||||
|
||||
private final PooledBufferAllocator allocator;
|
||||
|
||||
@SuppressWarnings("RedundantThrows")
|
||||
public PooledBufferAllocatorMetricUtils(PooledBufferAllocator allocator) throws Throwable {
|
||||
this.allocator = allocator;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the number of arenas.
|
||||
*/
|
||||
public int numArenas() {
|
||||
return allocator.numArenas();
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a {@link List} of all {@link PoolArenaMetric}s that are provided by this pool.
|
||||
*/
|
||||
public List<PoolArenaMetric> arenaMetrics() {
|
||||
return allocator.arenaMetrics();
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the number of thread local caches used by this {@link PooledBufferAllocator}.
|
||||
*/
|
||||
public int numThreadLocalCaches() {
|
||||
return allocator.numThreadLocalCaches();
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the size of the small cache.
|
||||
*/
|
||||
public int smallCacheSize() {
|
||||
return allocator.smallCacheSize();
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the size of the normal cache.
|
||||
*/
|
||||
public int normalCacheSize() {
|
||||
return allocator.normalCacheSize();
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the chunk size for an arena.
|
||||
*/
|
||||
public int chunkSize() {
|
||||
return allocator.chunkSize();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long usedMemory() {
|
||||
return allocator.usedMemory();
|
||||
}
|
||||
}
|
@ -1,79 +0,0 @@
|
||||
package it.cavallium.dbengine;
|
||||
|
||||
import static java.util.Objects.requireNonNull;
|
||||
import static java.util.Objects.requireNonNullElseGet;
|
||||
|
||||
import io.net5.buffer.api.Send;
|
||||
import it.cavallium.dbengine.database.disk.LLIndexSearcher;
|
||||
import it.cavallium.dbengine.database.disk.LLIndexSearchers;
|
||||
import it.cavallium.dbengine.lucene.searcher.LLSearchTransformer;
|
||||
import it.cavallium.dbengine.lucene.searcher.LocalQueryParams;
|
||||
import it.cavallium.dbengine.lucene.searcher.LocalSearcher;
|
||||
import it.cavallium.dbengine.lucene.searcher.MultiSearcher;
|
||||
import it.cavallium.dbengine.lucene.searcher.LuceneSearchResult;
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
public class SwappableLuceneSearcher implements LocalSearcher, MultiSearcher, Closeable {
|
||||
|
||||
private final AtomicReference<LocalSearcher> single = new AtomicReference<>(null);
|
||||
private final AtomicReference<MultiSearcher> multi = new AtomicReference<>(null);
|
||||
|
||||
public SwappableLuceneSearcher() {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<LuceneSearchResult> collect(Mono<Send<LLIndexSearcher>> indexSearcherMono,
|
||||
LocalQueryParams queryParams,
|
||||
String keyFieldName,
|
||||
LLSearchTransformer transformer) {
|
||||
var single = requireNonNullElseGet(this.single.get(), this.multi::get);
|
||||
requireNonNull(single, "LuceneLocalSearcher not set");
|
||||
return single.collect(indexSearcherMono, queryParams, keyFieldName, transformer);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
var single = this.single.get();
|
||||
var multi = this.multi.get();
|
||||
if (single == multi) {
|
||||
if (single == null) {
|
||||
return "swappable";
|
||||
} else {
|
||||
return single.getName();
|
||||
}
|
||||
} else {
|
||||
return "swappable[single=" + single.getName() + ",multi=" + multi.getName() + "]";
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<LuceneSearchResult> collectMulti(Mono<Send<LLIndexSearchers>> indexSearchersMono,
|
||||
LocalQueryParams queryParams,
|
||||
String keyFieldName,
|
||||
LLSearchTransformer transformer) {
|
||||
var multi = requireNonNull(this.multi.get(), "LuceneMultiSearcher not set");
|
||||
return multi.collectMulti(indexSearchersMono, queryParams, keyFieldName, transformer);
|
||||
}
|
||||
|
||||
public void setSingle(LocalSearcher single) {
|
||||
this.single.set(single);
|
||||
}
|
||||
|
||||
public void setMulti(MultiSearcher multi) {
|
||||
this.multi.set(multi);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
if (this.single.get() instanceof Closeable closeable) {
|
||||
closeable.close();
|
||||
}
|
||||
if (this.multi.get() instanceof Closeable closeable) {
|
||||
closeable.close();
|
||||
}
|
||||
}
|
||||
}
|
56
src/main/java/it/cavallium/dbengine/client/Backuppable.java
Normal file
56
src/main/java/it/cavallium/dbengine/client/Backuppable.java
Normal file
@ -0,0 +1,56 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
public abstract class Backuppable implements IBackuppable {
|
||||
|
||||
public enum State {
|
||||
RUNNING, PAUSING, PAUSED, RESUMING, STOPPED
|
||||
}
|
||||
|
||||
private final AtomicInteger state = new AtomicInteger();
|
||||
|
||||
@Override
|
||||
public final void pauseForBackup() {
|
||||
if (state.compareAndSet(State.RUNNING.ordinal(), State.PAUSING.ordinal())) {
|
||||
try {
|
||||
onPauseForBackup();
|
||||
state.compareAndSet(State.PAUSING.ordinal(), State.PAUSED.ordinal());
|
||||
} catch (Throwable ex) {
|
||||
state.compareAndSet(State.PAUSING.ordinal(), State.RUNNING.ordinal());
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public final void resumeAfterBackup() {
|
||||
if (state.compareAndSet(State.PAUSED.ordinal(), State.RESUMING.ordinal())) {
|
||||
try {
|
||||
onResumeAfterBackup();
|
||||
state.compareAndSet(State.RESUMING.ordinal(), State.RUNNING.ordinal());
|
||||
} catch (Throwable ex) {
|
||||
state.compareAndSet(State.RESUMING.ordinal(), State.PAUSED.ordinal());
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public final boolean isPaused() {
|
||||
return state.get() == State.PAUSED.ordinal();
|
||||
}
|
||||
|
||||
public final State getState() {
|
||||
return State.values()[state.get()];
|
||||
}
|
||||
|
||||
protected abstract void onPauseForBackup();
|
||||
|
||||
protected abstract void onResumeAfterBackup();
|
||||
|
||||
public final void setStopped() {
|
||||
state.set(State.STOPPED.ordinal());
|
||||
}
|
||||
}
|
@ -1,8 +0,0 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import it.cavallium.dbengine.database.Column;
|
||||
import it.unimi.dsi.fastutil.bytes.ByteList;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
public record BadBlock(String databaseName, @Nullable Column column, @Nullable ByteList rawKey,
|
||||
@Nullable Throwable ex) {}
|
@ -1,14 +1,14 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import it.cavallium.dbengine.client.Mapper;
|
||||
|
||||
public class CastMapper<T, U> implements Mapper<T, U> {
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public U map(T key) {
|
||||
return (U) key;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public T unmap(U key) {
|
||||
return (T) key;
|
||||
|
@ -1,32 +1,32 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import io.micrometer.core.instrument.MeterRegistry;
|
||||
import io.net5.buffer.api.BufferAllocator;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
import it.cavallium.dbengine.database.DatabaseOperations;
|
||||
import it.cavallium.dbengine.database.DatabaseProperties;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
public interface CompositeDatabase {
|
||||
public interface CompositeDatabase extends DatabaseProperties, DatabaseOperations {
|
||||
|
||||
Mono<Void> close();
|
||||
void preClose();
|
||||
|
||||
void close();
|
||||
|
||||
/**
|
||||
* Can return SnapshotException
|
||||
*/
|
||||
Mono<CompositeSnapshot> takeSnapshot();
|
||||
CompositeSnapshot takeSnapshot();
|
||||
|
||||
/**
|
||||
* Can return SnapshotException
|
||||
*/
|
||||
Mono<Void> releaseSnapshot(CompositeSnapshot snapshot);
|
||||
|
||||
BufferAllocator getAllocator();
|
||||
void releaseSnapshot(CompositeSnapshot snapshot);
|
||||
|
||||
MeterRegistry getMeterRegistry();
|
||||
|
||||
/**
|
||||
* Find corrupted items
|
||||
*/
|
||||
Flux<BadBlock> badBlocks();
|
||||
Stream<DbProgress<SSTVerificationProgress>> verify();
|
||||
|
||||
Mono<Void> verifyChecksum();
|
||||
void verifyChecksum();
|
||||
}
|
||||
|
@ -18,8 +18,7 @@ public class CompositeDatabasePartLocation {
|
||||
}
|
||||
|
||||
public enum CompositeDatabasePartType {
|
||||
KV_DATABASE,
|
||||
LUCENE_INDEX
|
||||
KV_DATABASE
|
||||
}
|
||||
|
||||
public CompositeDatabasePartType getPartType() {
|
||||
|
@ -2,7 +2,6 @@ package it.cavallium.dbengine.client;
|
||||
|
||||
import it.cavallium.dbengine.client.CompositeDatabasePartLocation.CompositeDatabasePartType;
|
||||
import it.cavallium.dbengine.database.LLKeyValueDatabaseStructure;
|
||||
import it.cavallium.dbengine.database.LLLuceneIndex;
|
||||
import it.cavallium.dbengine.database.LLSnapshot;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
@ -20,12 +19,6 @@ public class CompositeSnapshot {
|
||||
)), () -> "No snapshot for database with name \"" + database.getDatabaseName() + "\"");
|
||||
}
|
||||
|
||||
public LLSnapshot getSnapshot(LLLuceneIndex luceneIndex) {
|
||||
return Objects.requireNonNull(snapshots.get(CompositeDatabasePartLocation.of(CompositeDatabasePartType.LUCENE_INDEX,
|
||||
luceneIndex.getLuceneIndexName()
|
||||
)), () -> "No snapshot for lucene index with name \"" + luceneIndex.getLuceneIndexName() + "\"");
|
||||
}
|
||||
|
||||
public Map<CompositeDatabasePartLocation, LLSnapshot> getAllSnapshots() {
|
||||
return snapshots;
|
||||
}
|
||||
|
23
src/main/java/it/cavallium/dbengine/client/Compression.java
Normal file
23
src/main/java/it/cavallium/dbengine/client/Compression.java
Normal file
@ -0,0 +1,23 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import org.rocksdb.CompressionType;
|
||||
|
||||
public enum Compression {
|
||||
PLAIN(CompressionType.NO_COMPRESSION),
|
||||
SNAPPY(CompressionType.SNAPPY_COMPRESSION),
|
||||
LZ4(CompressionType.LZ4_COMPRESSION),
|
||||
LZ4_HC(CompressionType.LZ4HC_COMPRESSION),
|
||||
ZSTD(CompressionType.ZSTD_COMPRESSION),
|
||||
ZLIB(CompressionType.ZLIB_COMPRESSION),
|
||||
BZLIB2(CompressionType.BZLIB2_COMPRESSION);
|
||||
|
||||
private final CompressionType type;
|
||||
|
||||
Compression(CompressionType compressionType) {
|
||||
this.type = compressionType;
|
||||
}
|
||||
|
||||
public CompressionType getType() {
|
||||
return type;
|
||||
}
|
||||
}
|
@ -0,0 +1,34 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import com.google.common.collect.Multimap;
|
||||
import com.google.common.collect.Multimaps;
|
||||
import java.net.SocketAddress;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Map;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
public sealed interface ConnectionSettings {
|
||||
|
||||
sealed interface PrimaryConnectionSettings extends ConnectionSettings {}
|
||||
|
||||
sealed interface SubConnectionSettings extends ConnectionSettings {}
|
||||
|
||||
record MemoryConnectionSettings() implements PrimaryConnectionSettings, SubConnectionSettings {}
|
||||
|
||||
record LocalConnectionSettings(Path dataPath) implements PrimaryConnectionSettings, SubConnectionSettings {}
|
||||
|
||||
record MultiConnectionSettings(Map<ConnectionPart, SubConnectionSettings> parts) implements
|
||||
PrimaryConnectionSettings {
|
||||
|
||||
public Multimap<SubConnectionSettings, ConnectionPart> getConnections() {
|
||||
Multimap<SubConnectionSettings, ConnectionPart> result = com.google.common.collect.HashMultimap.create();
|
||||
parts.forEach((connectionPart, subConnectionSettings) -> result.put(subConnectionSettings,connectionPart));
|
||||
return Multimaps.unmodifiableMultimap(result);
|
||||
}
|
||||
}
|
||||
|
||||
sealed interface ConnectionPart {
|
||||
|
||||
record ConnectionPartRocksDB(@Nullable String name) implements ConnectionPart {}
|
||||
}
|
||||
}
|
@ -1,47 +0,0 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
public class CountedStream<T> {
|
||||
|
||||
private final Flux<T> stream;
|
||||
private final long count;
|
||||
|
||||
public CountedStream(Flux<T> stream, long count) {
|
||||
this.stream = stream;
|
||||
this.count = count;
|
||||
}
|
||||
|
||||
public Flux<T> getStream() {
|
||||
return stream;
|
||||
}
|
||||
|
||||
public long getCount() {
|
||||
return count;
|
||||
}
|
||||
|
||||
@SafeVarargs
|
||||
public static <T> CountedStream<T> merge(CountedStream<T>... stream) {
|
||||
return merge(List.of(stream));
|
||||
}
|
||||
|
||||
public static <T> CountedStream<T> merge(Collection<CountedStream<T>> stream) {
|
||||
return stream
|
||||
.stream()
|
||||
.reduce((a, b) -> new CountedStream<>(Flux.merge(a.getStream(), b.getStream()), a.getCount() + b.getCount()))
|
||||
.orElseGet(() -> new CountedStream<>(Flux.empty(), 0));
|
||||
}
|
||||
|
||||
public static <T> Mono<CountedStream<T>> merge(Flux<CountedStream<T>> stream) {
|
||||
return stream
|
||||
.reduce((a, b) -> new CountedStream<>(Flux.merge(a.getStream(), b.getStream()), a.getCount() + b.getCount()))
|
||||
.switchIfEmpty(Mono.fromSupplier(() -> new CountedStream<>(Flux.empty(), 0)));
|
||||
}
|
||||
|
||||
public Mono<List<T>> collectList() {
|
||||
return stream.collectList();
|
||||
}
|
||||
}
|
@ -1,17 +0,0 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import io.soabase.recordbuilder.core.RecordBuilder;
|
||||
import it.cavallium.dbengine.database.Column;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@RecordBuilder
|
||||
public record DatabaseOptions(Map<String, String> extraFlags,
|
||||
boolean absoluteConsistency,
|
||||
boolean lowMemory,
|
||||
boolean inMemory,
|
||||
boolean useDirectIO,
|
||||
boolean allowMemoryMapping,
|
||||
boolean allowNettyDirect,
|
||||
int maxOpenFiles) {
|
||||
}
|
45
src/main/java/it/cavallium/dbengine/client/DbProgress.java
Normal file
45
src/main/java/it/cavallium/dbengine/client/DbProgress.java
Normal file
@ -0,0 +1,45 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import it.cavallium.dbengine.client.SSTProgress.SSTProgressReport;
|
||||
import it.cavallium.dbengine.client.SSTProgress.SSTStart;
|
||||
import it.cavallium.dbengine.rpc.current.data.Column;
|
||||
import java.nio.file.Path;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.stream.Stream;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
public interface DbProgress<T extends SSTProgress> {
|
||||
|
||||
String databaseName();
|
||||
|
||||
record DbSSTProgress<T extends SSTProgress>(String databaseName, Column column, @Nullable Path file, long scanned,
|
||||
long total, T sstProgress) implements DbProgress<T> {
|
||||
|
||||
public double getProgress() {
|
||||
if (total == 0) {
|
||||
return 0d;
|
||||
}
|
||||
return scanned / (double) total;
|
||||
}
|
||||
|
||||
public String fileString() {
|
||||
return file != null ? file.normalize().toString() : null;
|
||||
}
|
||||
}
|
||||
|
||||
static <T extends SSTProgress> Stream<DbProgress<T>> toDbProgress(String dbName,
|
||||
String columnName,
|
||||
LongProgressTracker totalTracker,
|
||||
Stream<T> stream) {
|
||||
Column column = Column.of(columnName);
|
||||
AtomicReference<Path> filePath = new AtomicReference<>();
|
||||
return stream.map(state -> {
|
||||
switch (state) {
|
||||
case SSTStart start -> filePath.set(start.metadata().filePath());
|
||||
case SSTProgressReport progress -> totalTracker.incrementAndGet();
|
||||
default -> {}
|
||||
}
|
||||
return new DbSSTProgress<>(dbName, column, filePath.get(), totalTracker.getCurrent(), totalTracker.getTotal(), state);
|
||||
});
|
||||
}
|
||||
}
|
@ -0,0 +1,74 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import it.cavallium.datagen.nativedata.NullableString;
|
||||
import it.cavallium.datagen.nativedata.Nullableboolean;
|
||||
import it.cavallium.datagen.nativedata.Nullableint;
|
||||
import it.cavallium.datagen.nativedata.Nullablelong;
|
||||
import it.cavallium.dbengine.rpc.current.data.ColumnOptions;
|
||||
import it.cavallium.dbengine.rpc.current.data.ColumnOptionsBuilder;
|
||||
import it.cavallium.dbengine.rpc.current.data.DatabaseOptions;
|
||||
import it.cavallium.dbengine.rpc.current.data.DatabaseOptionsBuilder;
|
||||
import it.cavallium.dbengine.rpc.current.data.NamedColumnOptions;
|
||||
import it.cavallium.dbengine.rpc.current.data.NamedColumnOptionsBuilder;
|
||||
import it.cavallium.dbengine.rpc.current.data.nullables.NullableCompression;
|
||||
import it.cavallium.dbengine.rpc.current.data.nullables.NullableFilter;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.rocksdb.RocksDB;
|
||||
|
||||
public class DefaultDatabaseOptions {
|
||||
|
||||
public static ColumnOptions DEFAULT_DEFAULT_COLUMN_OPTIONS = new ColumnOptions(
|
||||
Collections.emptyList(),
|
||||
Nullablelong.empty(),
|
||||
Nullableboolean.empty(),
|
||||
Nullableboolean.empty(),
|
||||
NullableFilter.empty(),
|
||||
Nullableint.empty(),
|
||||
NullableString.empty(),
|
||||
Nullablelong.empty(),
|
||||
false,
|
||||
Nullablelong.empty(),
|
||||
Nullablelong.empty(),
|
||||
NullableCompression.empty()
|
||||
);
|
||||
|
||||
public static NamedColumnOptions DEFAULT_NAMED_COLUMN_OPTIONS = new NamedColumnOptions(
|
||||
new String(RocksDB.DEFAULT_COLUMN_FAMILY, StandardCharsets.UTF_8),
|
||||
DEFAULT_DEFAULT_COLUMN_OPTIONS
|
||||
);
|
||||
|
||||
public static DatabaseOptions DEFAULT_DATABASE_OPTIONS = new DatabaseOptions(List.of(),
|
||||
Map.of(),
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
Nullableint.empty(),
|
||||
Nullablelong.empty(),
|
||||
Collections.emptyList(),
|
||||
Nullablelong.empty(),
|
||||
false,
|
||||
DEFAULT_DEFAULT_COLUMN_OPTIONS,
|
||||
List.of(),
|
||||
NullableString.empty(),
|
||||
NullableString.empty(),
|
||||
false,
|
||||
NullableString.empty()
|
||||
);
|
||||
|
||||
public static DatabaseOptionsBuilder builder() {
|
||||
return DatabaseOptionsBuilder.builder(DEFAULT_DATABASE_OPTIONS);
|
||||
}
|
||||
|
||||
public static ColumnOptionsBuilder defaultColumnOptionsBuilder() {
|
||||
return ColumnOptionsBuilder.builder(DEFAULT_DEFAULT_COLUMN_OPTIONS);
|
||||
}
|
||||
|
||||
public static NamedColumnOptionsBuilder namedColumnOptionsBuilder() {
|
||||
return NamedColumnOptionsBuilder.builder(DEFAULT_NAMED_COLUMN_OPTIONS);
|
||||
}
|
||||
}
|
@ -1,13 +1,26 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import org.jetbrains.annotations.Contract;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import reactor.core.publisher.Mono;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import org.jetbrains.annotations.Unmodifiable;
|
||||
|
||||
public record HitEntry<T, U>(T key, U value, float score)
|
||||
public record HitEntry<T, U>(T key, @Nullable U value, float score)
|
||||
implements Comparable<HitEntry<T, U>> {
|
||||
|
||||
@Override
|
||||
public int compareTo(@NotNull HitEntry<T, U> o) {
|
||||
return Float.compare(o.score, this.score);
|
||||
}
|
||||
|
||||
@Contract(pure = true)
|
||||
public @Nullable @Unmodifiable Entry<T, U> toEntry() {
|
||||
if (value != null) {
|
||||
return Map.entry(key, value);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -1,14 +1,21 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import java.util.Comparator;
|
||||
import it.cavallium.dbengine.database.collections.DatabaseEmpty.Nothing;
|
||||
import java.util.function.Function;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
public record HitKey<T>(T key, float score) implements Comparable<HitKey<T>> {
|
||||
|
||||
public <U> Mono<HitEntry<T, U>> withValue(Function<T, Mono<U>> valueGetter) {
|
||||
return valueGetter.apply(key).map(value -> new HitEntry<>(key, value, score));
|
||||
public <U> HitEntry<T, U> withValue(Function<T, U> valueGetter) {
|
||||
return new HitEntry<>(key, valueGetter.apply(key), score);
|
||||
}
|
||||
|
||||
public <U> HitEntry<T, U> withNullValue() {
|
||||
return new HitEntry<>(key, null, score);
|
||||
}
|
||||
|
||||
public HitEntry<T, Nothing> withNothingValue() {
|
||||
return new HitEntry<>(key, Nothing.INSTANCE, score);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -1,107 +1,47 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import io.net5.buffer.api.Drop;
|
||||
import io.net5.buffer.api.Owned;
|
||||
import io.net5.buffer.api.Send;
|
||||
import com.google.common.collect.Lists;
|
||||
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
|
||||
import io.net5.buffer.api.internal.ResourceSupport;
|
||||
import it.cavallium.dbengine.database.DiscardingCloseable;
|
||||
import it.cavallium.dbengine.database.LLUtils;
|
||||
import it.cavallium.dbengine.database.SafeCloseable;
|
||||
import it.cavallium.dbengine.database.collections.ValueGetter;
|
||||
import it.cavallium.dbengine.database.collections.ValueTransformer;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Optional;
|
||||
import it.cavallium.dbengine.utils.SimpleResource;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.function.Function;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.util.function.Tuples;
|
||||
import java.util.stream.Stream;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
public final class Hits<T> extends ResourceSupport<Hits<T>, Hits<T>> {
|
||||
public class Hits<T> {
|
||||
|
||||
private static final Drop<Hits<?>> DROP = new Drop<>() {
|
||||
@Override
|
||||
public void drop(Hits<?> obj) {
|
||||
if (obj.onClose != null) {
|
||||
obj.onClose.run();
|
||||
}
|
||||
}
|
||||
private static final Logger LOG = LogManager.getLogger(Hits.class);
|
||||
private static final Hits<?> EMPTY_HITS = new Hits<>(List.of(), TotalHitsCount.of(0, true));
|
||||
private final List<T> results;
|
||||
private final TotalHitsCount totalHitsCount;
|
||||
|
||||
@Override
|
||||
public Drop<Hits<?>> fork() {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void attach(Hits<?> obj) {
|
||||
|
||||
}
|
||||
};
|
||||
|
||||
private Flux<T> results;
|
||||
private TotalHitsCount totalHitsCount;
|
||||
private Runnable onClose;
|
||||
|
||||
@SuppressWarnings({"unchecked", "rawtypes"})
|
||||
public Hits(Flux<T> results, TotalHitsCount totalHitsCount, Runnable onClose) {
|
||||
super((Drop<Hits<T>>) (Drop) DROP);
|
||||
public Hits(List<T> results, TotalHitsCount totalHitsCount) {
|
||||
this.results = results;
|
||||
this.totalHitsCount = totalHitsCount;
|
||||
this.onClose = onClose;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public static <T> Hits<T> empty() {
|
||||
return new Hits<>(Flux.empty(), TotalHitsCount.of(0, true), null);
|
||||
return (Hits<T>) EMPTY_HITS;
|
||||
}
|
||||
|
||||
public static <K, V> Hits<LazyHitEntry<K, V>> withValuesLazy(Hits<LazyHitKey<K>> hits,
|
||||
ValueGetter<K, V> valuesGetter) {
|
||||
var hitsEntry = hits.results().map(hitKey -> hitKey.withValue(valuesGetter::get));
|
||||
|
||||
return new Hits<>(hitsEntry, hits.totalHitsCount, hits::close);
|
||||
}
|
||||
|
||||
public static <K, V> Hits<HitEntry<K, V>> withValues(Hits<HitKey<K>> hits, ValueGetter<K, V> valuesGetter) {
|
||||
var hitsEntry = hits.results().flatMap(hitKey -> hitKey.withValue(valuesGetter::get));
|
||||
|
||||
return new Hits<>(hitsEntry, hits.totalHitsCount, hits::close);
|
||||
}
|
||||
|
||||
public static <T, U> Function<Hits<HitKey<T>>, Hits<LazyHitEntry<T, U>>> generateMapper(
|
||||
public static <T, U> Function<Hits<HitKey<T>>, Hits<HitEntry<T, U>>> generateMapper(
|
||||
ValueGetter<T, U> valueGetter) {
|
||||
return result -> {
|
||||
var hitsToTransform = result.results()
|
||||
.map(hit -> new LazyHitEntry<>(Mono.just(hit.key()), valueGetter.get(hit.key()), hit.score()));
|
||||
return new Hits<>(hitsToTransform, result.totalHitsCount(), result::close);
|
||||
List<HitEntry<T, U>> hitsToTransform = LLUtils.mapList(result.results,
|
||||
hit -> new HitEntry<>(hit.key(), valueGetter.get(hit.key()), hit.score())
|
||||
);
|
||||
return new Hits<>(hitsToTransform, result.totalHitsCount());
|
||||
};
|
||||
}
|
||||
|
||||
public static <T, U> Function<Hits<HitKey<T>>, Hits<LazyHitEntry<T, U>>> generateMapper(
|
||||
ValueTransformer<T, U> valueTransformer) {
|
||||
return result -> {
|
||||
try {
|
||||
var sharedHitsFlux = result.results().publish().refCount(3);
|
||||
var scoresFlux = sharedHitsFlux.map(HitKey::score);
|
||||
var keysFlux = sharedHitsFlux.map(HitKey::key);
|
||||
|
||||
var valuesFlux = valueTransformer.transform(keysFlux);
|
||||
|
||||
var transformedFlux = Flux.zip((Object[] data) -> {
|
||||
//noinspection unchecked
|
||||
var keyMono = Mono.just((T) data[0]);
|
||||
//noinspection unchecked
|
||||
var val = (Entry<T, Optional<U>>) data[1];
|
||||
var valMono = Mono.justOrEmpty(val.getValue());
|
||||
var score = (Float) data[2];
|
||||
return new LazyHitEntry<>(keyMono, valMono, score);
|
||||
}, keysFlux, valuesFlux, scoresFlux);
|
||||
|
||||
return new Hits<>(transformedFlux, result.totalHitsCount(), result::close);
|
||||
} catch (Throwable t) {
|
||||
result.close();
|
||||
throw t;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public Flux<T> results() {
|
||||
public List<T> results() {
|
||||
return results;
|
||||
}
|
||||
|
||||
@ -113,27 +53,4 @@ public final class Hits<T> extends ResourceSupport<Hits<T>, Hits<T>> {
|
||||
public String toString() {
|
||||
return "Hits[" + "results=" + results + ", " + "totalHitsCount=" + totalHitsCount + ']';
|
||||
}
|
||||
|
||||
@Override
|
||||
protected RuntimeException createResourceClosedException() {
|
||||
return new IllegalStateException("Closed");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Owned<Hits<T>> prepareSend() {
|
||||
var results = this.results;
|
||||
var totalHitsCount = this.totalHitsCount;
|
||||
var onClose = this.onClose;
|
||||
return drop -> {
|
||||
var instance = new Hits<>(results, totalHitsCount, onClose);
|
||||
drop.attach(instance);
|
||||
return instance;
|
||||
};
|
||||
}
|
||||
|
||||
protected void makeInaccessible() {
|
||||
this.results = null;
|
||||
this.totalHitsCount = null;
|
||||
this.onClose = null;
|
||||
}
|
||||
}
|
||||
|
10
src/main/java/it/cavallium/dbengine/client/IBackuppable.java
Normal file
10
src/main/java/it/cavallium/dbengine/client/IBackuppable.java
Normal file
@ -0,0 +1,10 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
public interface IBackuppable {
|
||||
|
||||
void pauseForBackup();
|
||||
|
||||
void resumeAfterBackup();
|
||||
|
||||
boolean isPaused();
|
||||
}
|
@ -1,128 +0,0 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import it.cavallium.dbengine.client.IndexAction.Add;
|
||||
import it.cavallium.dbengine.client.IndexAction.AddMulti;
|
||||
import it.cavallium.dbengine.client.IndexAction.Update;
|
||||
import it.cavallium.dbengine.client.IndexAction.UpdateMulti;
|
||||
import it.cavallium.dbengine.client.IndexAction.Delete;
|
||||
import it.cavallium.dbengine.client.IndexAction.DeleteAll;
|
||||
import it.cavallium.dbengine.client.IndexAction.TakeSnapshot;
|
||||
import it.cavallium.dbengine.client.IndexAction.ReleaseSnapshot;
|
||||
import it.cavallium.dbengine.client.IndexAction.Flush;
|
||||
import it.cavallium.dbengine.client.IndexAction.Refresh;
|
||||
import it.cavallium.dbengine.client.IndexAction.Close;
|
||||
import it.cavallium.dbengine.database.LLUpdateDocument;
|
||||
import it.cavallium.dbengine.database.LLSnapshot;
|
||||
import it.cavallium.dbengine.database.LLTerm;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.MonoSink;
|
||||
|
||||
sealed interface IndexAction permits Add, AddMulti, Update, UpdateMulti, Delete, DeleteAll, TakeSnapshot,
|
||||
ReleaseSnapshot, Flush, Refresh, Close {
|
||||
|
||||
IndexActionType getType();
|
||||
|
||||
final record Add(LLTerm key, LLUpdateDocument doc, MonoSink<Void> addedFuture) implements IndexAction {
|
||||
|
||||
@Override
|
||||
public IndexActionType getType() {
|
||||
return IndexActionType.ADD;
|
||||
}
|
||||
}
|
||||
|
||||
final record AddMulti(Flux<Entry<LLTerm, LLUpdateDocument>> docsFlux, MonoSink<Void> addedMultiFuture) implements IndexAction {
|
||||
|
||||
@Override
|
||||
public IndexActionType getType() {
|
||||
return IndexActionType.ADD_MULTI;
|
||||
}
|
||||
}
|
||||
|
||||
final record Update(LLTerm key, LLUpdateDocument doc, MonoSink<Void> updatedFuture) implements IndexAction {
|
||||
|
||||
@Override
|
||||
public IndexActionType getType() {
|
||||
return IndexActionType.UPDATE;
|
||||
}
|
||||
}
|
||||
|
||||
final record UpdateMulti(Map<LLTerm, LLUpdateDocument> docs, MonoSink<Void> updatedMultiFuture) implements IndexAction {
|
||||
|
||||
@Override
|
||||
public IndexActionType getType() {
|
||||
return IndexActionType.UPDATE_MULTI;
|
||||
}
|
||||
}
|
||||
|
||||
final record Delete(LLTerm key, MonoSink<Void> deletedFuture) implements IndexAction {
|
||||
|
||||
@Override
|
||||
public IndexActionType getType() {
|
||||
return IndexActionType.DELETE;
|
||||
}
|
||||
}
|
||||
|
||||
final record DeleteAll(MonoSink<Void> deletedAllFuture) implements IndexAction {
|
||||
|
||||
@Override
|
||||
public IndexActionType getType() {
|
||||
return IndexActionType.DELETE_ALL;
|
||||
}
|
||||
}
|
||||
|
||||
final record TakeSnapshot(MonoSink<LLSnapshot> snapshotFuture) implements IndexAction {
|
||||
|
||||
@Override
|
||||
public IndexActionType getType() {
|
||||
return IndexActionType.TAKE_SNAPSHOT;
|
||||
}
|
||||
}
|
||||
|
||||
final record ReleaseSnapshot(LLSnapshot snapshot, MonoSink<Void> releasedFuture) implements IndexAction {
|
||||
|
||||
@Override
|
||||
public IndexActionType getType() {
|
||||
return IndexActionType.RELEASE_SNAPSHOT;
|
||||
}
|
||||
}
|
||||
|
||||
final record Flush(MonoSink<Void> flushFuture) implements IndexAction {
|
||||
|
||||
@Override
|
||||
public IndexActionType getType() {
|
||||
return IndexActionType.FLUSH;
|
||||
}
|
||||
}
|
||||
|
||||
final record Refresh(boolean force, MonoSink<Void> refreshFuture) implements IndexAction {
|
||||
|
||||
@Override
|
||||
public IndexActionType getType() {
|
||||
return IndexActionType.REFRESH;
|
||||
}
|
||||
}
|
||||
|
||||
final record Close(MonoSink<Void> closeFuture) implements IndexAction {
|
||||
|
||||
@Override
|
||||
public IndexActionType getType() {
|
||||
return IndexActionType.CLOSE;
|
||||
}
|
||||
}
|
||||
|
||||
enum IndexActionType {
|
||||
ADD,
|
||||
ADD_MULTI,
|
||||
UPDATE,
|
||||
UPDATE_MULTI,
|
||||
DELETE,
|
||||
DELETE_ALL,
|
||||
TAKE_SNAPSHOT,
|
||||
RELEASE_SNAPSHOT,
|
||||
FLUSH,
|
||||
REFRESH,
|
||||
CLOSE
|
||||
}
|
||||
}
|
@ -1,49 +0,0 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import it.cavallium.dbengine.database.LLIndexRequest;
|
||||
import it.cavallium.dbengine.database.LLSoftUpdateDocument;
|
||||
import it.cavallium.dbengine.database.LLUpdateDocument;
|
||||
import it.cavallium.dbengine.database.LLTerm;
|
||||
import it.cavallium.dbengine.database.LLUpdateFields;
|
||||
import it.cavallium.dbengine.database.LLUtils;
|
||||
import java.util.Set;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.util.function.Tuple2;
|
||||
|
||||
public abstract class Indicizer<T, U> {
|
||||
|
||||
/**
|
||||
* Transform a value to an IndexRequest.
|
||||
*/
|
||||
public abstract @NotNull Mono<? extends LLIndexRequest> toIndexRequest(@NotNull T key, @NotNull U value);
|
||||
|
||||
public final @NotNull Mono<LLUpdateDocument> toDocument(@NotNull T key, @NotNull U value) {
|
||||
return toIndexRequest(key, value).map(req -> {
|
||||
if (req instanceof LLUpdateFields updateFields) {
|
||||
return new LLUpdateDocument(updateFields.items());
|
||||
} else if (req instanceof LLUpdateDocument updateDocument) {
|
||||
return updateDocument;
|
||||
} else if (req instanceof LLSoftUpdateDocument softUpdateDocument) {
|
||||
return new LLUpdateDocument(softUpdateDocument.items());
|
||||
} else {
|
||||
throw new UnsupportedOperationException("Unexpected request type: " + req);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public abstract @NotNull LLTerm toIndex(@NotNull T key);
|
||||
|
||||
public abstract @NotNull String getKeyFieldName();
|
||||
|
||||
public abstract @NotNull T getKey(String key);
|
||||
|
||||
public abstract IndicizerAnalyzers getPerFieldAnalyzer();
|
||||
|
||||
public abstract IndicizerSimilarities getPerFieldSimilarity();
|
||||
|
||||
public Flux<Tuple2<String, Set<String>>> getMoreLikeThisDocumentFields(T key, U value) {
|
||||
return Flux.empty();
|
||||
}
|
||||
}
|
@ -1,19 +0,0 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
|
||||
import java.util.Map;
|
||||
|
||||
public record IndicizerAnalyzers(TextFieldsAnalyzer defaultAnalyzer, Map<String, TextFieldsAnalyzer> fieldAnalyzer) {
|
||||
|
||||
public static IndicizerAnalyzers of() {
|
||||
return of(TextFieldsAnalyzer.FullText);
|
||||
}
|
||||
|
||||
public static IndicizerAnalyzers of(TextFieldsAnalyzer defaultAnalyzer) {
|
||||
return of(defaultAnalyzer, Map.of());
|
||||
}
|
||||
|
||||
public static IndicizerAnalyzers of(TextFieldsAnalyzer defaultAnalyzer, Map<String, TextFieldsAnalyzer> fieldAnalyzer) {
|
||||
return new IndicizerAnalyzers(defaultAnalyzer, fieldAnalyzer);
|
||||
}
|
||||
}
|
@ -1,20 +0,0 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
|
||||
import it.cavallium.dbengine.lucene.analyzer.TextFieldsSimilarity;
|
||||
import java.util.Map;
|
||||
|
||||
public record IndicizerSimilarities(TextFieldsSimilarity defaultSimilarity, Map<String, TextFieldsSimilarity> fieldSimilarity) {
|
||||
|
||||
public static IndicizerSimilarities of() {
|
||||
return of(TextFieldsSimilarity.BM25Plus);
|
||||
}
|
||||
|
||||
public static IndicizerSimilarities of(TextFieldsSimilarity defaultSimilarity) {
|
||||
return of(defaultSimilarity, Map.of());
|
||||
}
|
||||
|
||||
public static IndicizerSimilarities of(TextFieldsSimilarity defaultSimilarity, Map<String, TextFieldsSimilarity> fieldSimilarity) {
|
||||
return new IndicizerSimilarities(defaultSimilarity, fieldSimilarity);
|
||||
}
|
||||
}
|
@ -2,7 +2,6 @@ package it.cavallium.dbengine.client;
|
||||
|
||||
import com.squareup.moshi.JsonReader;
|
||||
import com.squareup.moshi.JsonWriter;
|
||||
import it.cavallium.data.generator.nativedata.Int52;
|
||||
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
|
||||
import java.io.IOException;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
|
@ -1,11 +0,0 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
public record LazyHitEntry<T, U>(Mono<T> key, Mono<U> value, float score) {
|
||||
|
||||
public Mono<HitEntry<T, U>> resolve() {
|
||||
return Mono.zip(key, value, (k, v) -> new HitEntry<>(k, v, score));
|
||||
}
|
||||
}
|
@ -1,19 +0,0 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import java.util.function.Function;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
public record LazyHitKey<T>(Mono<T> key, float score) {
|
||||
|
||||
public <U> LazyHitEntry<T, U> withValue(Function<T, Mono<U>> valueGetter) {
|
||||
return new LazyHitEntry<>(key, key.flatMap(valueGetter), score);
|
||||
}
|
||||
|
||||
public Mono<HitKey<T>> resolve() {
|
||||
return key.map(k -> new HitKey<>(k, score));
|
||||
}
|
||||
|
||||
public <U> Mono<HitEntry<T, U>> resolveWithValue(Function<T, Mono<U>> valueGetter) {
|
||||
return resolve().flatMap(key -> key.withValue(valueGetter));
|
||||
}
|
||||
}
|
@ -0,0 +1,42 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
public class LongProgressTracker {
|
||||
|
||||
private final AtomicLong current = new AtomicLong();
|
||||
private final AtomicLong total = new AtomicLong();
|
||||
|
||||
public LongProgressTracker(long size) {
|
||||
setTotal(size);
|
||||
}
|
||||
|
||||
public LongProgressTracker() {
|
||||
|
||||
}
|
||||
|
||||
public LongProgressTracker setTotal(long estimate) {
|
||||
total.set(estimate);
|
||||
return this;
|
||||
}
|
||||
|
||||
public long getCurrent() {
|
||||
return current.get();
|
||||
}
|
||||
|
||||
public long incrementAndGet() {
|
||||
return current.incrementAndGet();
|
||||
}
|
||||
|
||||
public long getAndIncrement() {
|
||||
return current.getAndIncrement();
|
||||
}
|
||||
|
||||
public long getTotal() {
|
||||
return Math.max(current.get(), total.get());
|
||||
}
|
||||
|
||||
public double progress() {
|
||||
return getCurrent() / (double) Math.max(1L, getTotal());
|
||||
}
|
||||
}
|
@ -1,65 +0,0 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import io.net5.buffer.api.Send;
|
||||
import it.cavallium.dbengine.client.query.ClientQueryParams;
|
||||
import it.cavallium.dbengine.client.query.current.data.Query;
|
||||
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
|
||||
import it.cavallium.dbengine.database.Delta;
|
||||
import it.cavallium.dbengine.database.LLSnapshottable;
|
||||
import it.cavallium.dbengine.database.collections.ValueGetter;
|
||||
import it.cavallium.dbengine.database.collections.ValueTransformer;
|
||||
import java.util.Map.Entry;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
public interface LuceneIndex<T, U> extends LLSnapshottable {
|
||||
|
||||
Mono<Void> addDocument(T key, U value);
|
||||
|
||||
Mono<Void> addDocuments(Flux<Entry<T, U>> entries);
|
||||
|
||||
Mono<Void> deleteDocument(T key);
|
||||
|
||||
Mono<Void> updateDocument(T key, @NotNull U value);
|
||||
|
||||
Mono<Void> updateDocuments(Flux<Entry<T, U>> entries);
|
||||
|
||||
default Mono<Void> updateOrDeleteDocument(T key, @Nullable U value) {
|
||||
if (value == null) {
|
||||
return deleteDocument(key);
|
||||
} else {
|
||||
return updateDocument(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
default Mono<Void> updateOrDeleteDocumentIfModified(T key, @NotNull Delta<U> delta) {
|
||||
return updateOrDeleteDocumentIfModified(key, delta.current(), delta.isModified());
|
||||
}
|
||||
|
||||
default Mono<Void> updateOrDeleteDocumentIfModified(T key, @Nullable U currentValue, boolean modified) {
|
||||
if (modified) {
|
||||
return updateOrDeleteDocument(key, currentValue);
|
||||
} else {
|
||||
return Mono.empty();
|
||||
}
|
||||
}
|
||||
|
||||
Mono<Void> deleteAll();
|
||||
|
||||
Mono<Hits<HitKey<T>>> moreLikeThis(ClientQueryParams queryParams, T key,
|
||||
U mltDocumentValue);
|
||||
|
||||
Mono<Hits<HitKey<T>>> search(ClientQueryParams queryParams);
|
||||
|
||||
Mono<TotalHitsCount> count(@Nullable CompositeSnapshot snapshot, Query query);
|
||||
|
||||
boolean isLowMemoryMode();
|
||||
|
||||
Mono<Void> close();
|
||||
|
||||
Mono<Void> flush();
|
||||
|
||||
Mono<Void> refresh(boolean force);
|
||||
}
|
@ -1,167 +0,0 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import io.net5.buffer.api.Send;
|
||||
import it.cavallium.dbengine.client.query.ClientQueryParams;
|
||||
import it.cavallium.dbengine.client.query.current.data.Query;
|
||||
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
|
||||
import it.cavallium.dbengine.database.LLLuceneIndex;
|
||||
import it.cavallium.dbengine.database.LLSearchResultShard;
|
||||
import it.cavallium.dbengine.database.LLSnapshot;
|
||||
import it.cavallium.dbengine.database.LLTerm;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Set;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.util.function.Tuple2;
|
||||
|
||||
public class LuceneIndexImpl<T, U> implements LuceneIndex<T, U> {
|
||||
|
||||
private final LLLuceneIndex luceneIndex;
|
||||
private final Indicizer<T,U> indicizer;
|
||||
|
||||
public LuceneIndexImpl(LLLuceneIndex luceneIndex, Indicizer<T, U> indicizer) {
|
||||
this.luceneIndex = luceneIndex;
|
||||
this.indicizer = indicizer;
|
||||
}
|
||||
|
||||
private LLSnapshot resolveSnapshot(CompositeSnapshot snapshot) {
|
||||
if (snapshot == null) {
|
||||
return null;
|
||||
} else {
|
||||
return snapshot.getSnapshot(luceneIndex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Void> addDocument(T key, U value) {
|
||||
return indicizer
|
||||
.toDocument(key, value)
|
||||
.flatMap(doc -> luceneIndex.addDocument(indicizer.toIndex(key), doc));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Void> addDocuments(Flux<Entry<T, U>> entries) {
|
||||
return luceneIndex
|
||||
.addDocuments(entries
|
||||
.flatMap(entry -> indicizer
|
||||
.toDocument(entry.getKey(), entry.getValue())
|
||||
.map(doc -> Map.entry(indicizer.toIndex(entry.getKey()), doc)))
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Void> deleteDocument(T key) {
|
||||
LLTerm id = indicizer.toIndex(key);
|
||||
return luceneIndex.deleteDocument(id);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Void> updateDocument(T key, @NotNull U value) {
|
||||
return indicizer
|
||||
.toIndexRequest(key, value)
|
||||
.flatMap(doc -> luceneIndex.update(indicizer.toIndex(key), doc));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Void> updateDocuments(Flux<Entry<T, U>> entries) {
|
||||
return luceneIndex
|
||||
.updateDocuments(entries
|
||||
.flatMap(entry -> indicizer
|
||||
.toDocument(entry.getKey(), entry.getValue())
|
||||
.map(doc -> Map.entry(indicizer.toIndex(entry.getKey()), doc)))
|
||||
.collectMap(Entry::getKey, Entry::getValue)
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Void> deleteAll() {
|
||||
return luceneIndex.deleteAll();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Hits<HitKey<T>>> moreLikeThis(ClientQueryParams queryParams,
|
||||
T key,
|
||||
U mltDocumentValue) {
|
||||
Flux<Tuple2<String, Set<String>>> mltDocumentFields
|
||||
= indicizer.getMoreLikeThisDocumentFields(key, mltDocumentValue);
|
||||
|
||||
return luceneIndex
|
||||
.moreLikeThis(resolveSnapshot(queryParams.snapshot()),
|
||||
queryParams.toQueryParams(),
|
||||
indicizer.getKeyFieldName(),
|
||||
mltDocumentFields
|
||||
)
|
||||
.map(this::mapResults)
|
||||
.single();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Hits<HitKey<T>>> search(ClientQueryParams queryParams) {
|
||||
return luceneIndex
|
||||
.search(resolveSnapshot(queryParams.snapshot()),
|
||||
queryParams.toQueryParams(),
|
||||
indicizer.getKeyFieldName()
|
||||
)
|
||||
.map(this::mapResults)
|
||||
.single();
|
||||
}
|
||||
|
||||
private Hits<HitKey<T>> mapResults(LLSearchResultShard llSearchResult) {
|
||||
var scoresWithKeysFlux = llSearchResult
|
||||
.results()
|
||||
.map(hit -> new HitKey<>(indicizer.getKey(hit.key()), hit.score()));
|
||||
|
||||
return new Hits<>(scoresWithKeysFlux, llSearchResult.totalHitsCount(), llSearchResult::close);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<TotalHitsCount> count(@Nullable CompositeSnapshot snapshot, Query query) {
|
||||
return this
|
||||
.search(ClientQueryParams.builder().snapshot(snapshot).query(query).limit(0).build())
|
||||
.single()
|
||||
.map(searchResultKeys -> {
|
||||
try (searchResultKeys) {
|
||||
return searchResultKeys.totalHitsCount();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isLowMemoryMode() {
|
||||
return luceneIndex.isLowMemoryMode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Void> close() {
|
||||
return luceneIndex.close();
|
||||
}
|
||||
|
||||
/**
|
||||
* Flush writes to disk
|
||||
*/
|
||||
@Override
|
||||
public Mono<Void> flush() {
|
||||
return luceneIndex.flush();
|
||||
}
|
||||
|
||||
/**
|
||||
* Refresh index searcher
|
||||
*/
|
||||
@Override
|
||||
public Mono<Void> refresh(boolean force) {
|
||||
return luceneIndex.refresh(force);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<LLSnapshot> takeSnapshot() {
|
||||
return luceneIndex.takeSnapshot();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Void> releaseSnapshot(LLSnapshot snapshot) {
|
||||
return luceneIndex.releaseSnapshot(snapshot);
|
||||
}
|
||||
}
|
@ -1,20 +0,0 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import io.soabase.recordbuilder.core.RecordBuilder;
|
||||
import java.time.Duration;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
@RecordBuilder
|
||||
public record LuceneOptions(Map<String, String> extraFlags,
|
||||
Duration queryRefreshDebounceTime,
|
||||
Duration commitDebounceTime,
|
||||
boolean lowMemory,
|
||||
boolean inMemory,
|
||||
Optional<DirectIOOptions> directIOOptions,
|
||||
boolean allowMemoryMapping,
|
||||
Optional<NRTCachingOptions> nrtCachingOptions,
|
||||
int indexWriterBufferSize,
|
||||
boolean applyAllDeletes,
|
||||
boolean writeAllDeletes) {}
|
@ -1,11 +1,12 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import io.net5.buffer.api.Buffer;
|
||||
import io.net5.buffer.api.Send;
|
||||
import it.cavallium.buffer.Buf;
|
||||
import it.cavallium.buffer.BufDataInput;
|
||||
import it.cavallium.buffer.BufDataOutput;
|
||||
import it.cavallium.dbengine.database.serialization.SerializationException;
|
||||
import it.cavallium.dbengine.database.serialization.Serializer;
|
||||
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
public class MappedSerializer<A, B> implements Serializer<B> {
|
||||
|
||||
@ -18,14 +19,24 @@ public class MappedSerializer<A, B> implements Serializer<B> {
|
||||
this.keyMapper = keyMapper;
|
||||
}
|
||||
|
||||
@Override
|
||||
public @NotNull B deserialize(@NotNull Buffer serialized) throws SerializationException {
|
||||
return keyMapper.map(serializer.deserialize(serialized));
|
||||
public static <A, B> Serializer<B> of(Serializer<A> ser,
|
||||
Mapper<A, B> keyMapper) {
|
||||
if (keyMapper.getClass() == NoMapper.class) {
|
||||
//noinspection unchecked
|
||||
return (Serializer<B>) ser;
|
||||
} else {
|
||||
return new MappedSerializer<>(ser, keyMapper);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void serialize(@NotNull B deserialized, Buffer output) throws SerializationException {
|
||||
serializer.serialize(keyMapper.unmap(deserialized), output);
|
||||
public @NotNull B deserialize(@NotNull BufDataInput in) throws SerializationException {
|
||||
return keyMapper.map(serializer.deserialize(in));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void serialize(@NotNull B deserialized, BufDataOutput out) throws SerializationException {
|
||||
serializer.serialize(keyMapper.unmap(deserialized), out);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -1,11 +1,11 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import io.net5.buffer.api.Buffer;
|
||||
import io.net5.buffer.api.Send;
|
||||
import it.cavallium.buffer.Buf;
|
||||
import it.cavallium.buffer.BufDataInput;
|
||||
import it.cavallium.buffer.BufDataOutput;
|
||||
import it.cavallium.dbengine.database.serialization.SerializationException;
|
||||
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
public class MappedSerializerFixedLength<A, B> implements SerializerFixedBinaryLength<B> {
|
||||
|
||||
@ -18,14 +18,24 @@ public class MappedSerializerFixedLength<A, B> implements SerializerFixedBinaryL
|
||||
this.keyMapper = keyMapper;
|
||||
}
|
||||
|
||||
@Override
|
||||
public @NotNull B deserialize(@NotNull Buffer serialized) throws SerializationException {
|
||||
return keyMapper.map(fixedLengthSerializer.deserialize(serialized));
|
||||
public static <A, B> SerializerFixedBinaryLength<B> of(SerializerFixedBinaryLength<A> fixedLengthSerializer,
|
||||
Mapper<A, B> keyMapper) {
|
||||
if (keyMapper.getClass() == NoMapper.class) {
|
||||
//noinspection unchecked
|
||||
return (SerializerFixedBinaryLength<B>) fixedLengthSerializer;
|
||||
} else {
|
||||
return new MappedSerializerFixedLength<>(fixedLengthSerializer, keyMapper);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void serialize(@NotNull B deserialized, Buffer output) throws SerializationException {
|
||||
fixedLengthSerializer.serialize(keyMapper.unmap(deserialized), output);
|
||||
public @NotNull B deserialize(@NotNull BufDataInput in) throws SerializationException {
|
||||
return keyMapper.map(fixedLengthSerializer.deserialize(in));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void serialize(@NotNull B deserialized, BufDataOutput out) throws SerializationException {
|
||||
fixedLengthSerializer.serialize(keyMapper.unmap(deserialized), out);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -0,0 +1,5 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
public record MemoryStats(long estimateTableReadersMem, long sizeAllMemTables,
|
||||
long curSizeAllMemTables, long estimateNumKeys, long blockCacheCapacity,
|
||||
long blockCacheUsage, long blockCachePinnedUsage, long liveVersions) {}
|
@ -1,6 +0,0 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import io.soabase.recordbuilder.core.RecordBuilder;
|
||||
|
||||
@RecordBuilder
|
||||
public record NRTCachingOptions(double maxMergeSizeMB, double maxCachedMB) {}
|
@ -1,7 +1,5 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import it.cavallium.dbengine.client.Mapper;
|
||||
|
||||
public class NoMapper<T> implements Mapper<T, T> {
|
||||
|
||||
@Override
|
||||
|
@ -0,0 +1,17 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import it.cavallium.buffer.Buf;
|
||||
import it.cavallium.dbengine.client.SSTDumpProgress.SSTBlockFail;
|
||||
import it.cavallium.dbengine.client.SSTDumpProgress.SSTBlockKeyValue;
|
||||
import it.cavallium.dbengine.client.SSTProgress.SSTOk;
|
||||
import it.cavallium.dbengine.client.SSTProgress.SSTProgressReport;
|
||||
import it.cavallium.dbengine.client.SSTProgress.SSTStart;
|
||||
import org.rocksdb.RocksDBException;
|
||||
|
||||
public sealed interface SSTDumpProgress extends SSTProgress permits SSTBlockFail, SSTBlockKeyValue, SSTOk,
|
||||
SSTProgressReport, SSTStart {
|
||||
|
||||
record SSTBlockKeyValue(Buf rawKey, Buf rawValue) implements SSTDumpProgress {}
|
||||
|
||||
record SSTBlockFail(RocksDBException ex) implements SSTDumpProgress {}
|
||||
}
|
21
src/main/java/it/cavallium/dbengine/client/SSTProgress.java
Normal file
21
src/main/java/it/cavallium/dbengine/client/SSTProgress.java
Normal file
@ -0,0 +1,21 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import it.cavallium.dbengine.database.disk.RocksDBFile.IterationMetadata;
|
||||
import it.cavallium.dbengine.rpc.current.data.Column;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
public interface SSTProgress {
|
||||
|
||||
record SSTStart(IterationMetadata metadata) implements SSTProgress, SSTVerificationProgress, SSTDumpProgress {}
|
||||
|
||||
record SSTOk(long scannedCount) implements SSTProgress, SSTVerificationProgress, SSTDumpProgress {}
|
||||
|
||||
record SSTProgressReport(long fileScanned, long fileTotal) implements SSTProgress, SSTVerificationProgress,
|
||||
SSTDumpProgress {
|
||||
|
||||
public double getFileProgress() {
|
||||
if (fileTotal == 0) return 0d;
|
||||
return fileScanned / (double) fileTotal;
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,17 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import it.cavallium.buffer.Buf;
|
||||
import it.cavallium.dbengine.client.DbProgress.DbSSTProgress;
|
||||
import it.cavallium.dbengine.client.SSTProgress.SSTOk;
|
||||
import it.cavallium.dbengine.client.SSTProgress.SSTProgressReport;
|
||||
import it.cavallium.dbengine.client.SSTProgress.SSTStart;
|
||||
import it.cavallium.dbengine.client.SSTVerificationProgress.SSTBlockBad;
|
||||
import it.cavallium.dbengine.rpc.current.data.Column;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
public sealed interface SSTVerificationProgress extends SSTProgress permits SSTOk, SSTProgressReport, SSTStart,
|
||||
SSTBlockBad {
|
||||
|
||||
record SSTBlockBad(Buf rawKey, Throwable ex) implements SSTVerificationProgress {}
|
||||
}
|
@ -1,6 +1,6 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
public class SnapshotException extends RuntimeException {
|
||||
public class SnapshotException extends IllegalStateException {
|
||||
|
||||
public SnapshotException(Throwable ex) {
|
||||
super(ex);
|
||||
|
@ -1,6 +1,6 @@
|
||||
package it.cavallium.dbengine.client;
|
||||
|
||||
import it.cavallium.dbengine.client.query.BasicType;
|
||||
import it.cavallium.dbengine.client.query.BaseType;
|
||||
import it.cavallium.dbengine.client.query.current.data.DocSort;
|
||||
import it.cavallium.dbengine.client.query.current.data.NoSort;
|
||||
import it.cavallium.dbengine.client.query.current.data.NumericSort;
|
||||
@ -11,7 +11,7 @@ import org.jetbrains.annotations.NotNull;
|
||||
public record Sort(@NotNull it.cavallium.dbengine.client.query.current.data.Sort querySort) {
|
||||
|
||||
public boolean isSorted() {
|
||||
return querySort.getBasicType$() != BasicType.NoSort;
|
||||
return querySort.getBaseType$() != BaseType.NoSort;
|
||||
}
|
||||
|
||||
public static Sort random() {
|
||||
|
@ -1,49 +0,0 @@
|
||||
package it.cavallium.dbengine.client.query;
|
||||
|
||||
import io.soabase.recordbuilder.core.RecordBuilder;
|
||||
import it.cavallium.data.generator.nativedata.Nullablefloat;
|
||||
import it.cavallium.dbengine.client.CompositeSnapshot;
|
||||
import it.cavallium.dbengine.client.Sort;
|
||||
import it.cavallium.dbengine.client.query.current.data.NoSort;
|
||||
import it.cavallium.dbengine.client.query.current.data.Query;
|
||||
import it.cavallium.dbengine.client.query.current.data.QueryParams;
|
||||
import it.cavallium.dbengine.client.query.current.data.QueryParamsBuilder;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
@RecordBuilder
|
||||
public final record ClientQueryParams(@Nullable CompositeSnapshot snapshot,
|
||||
@NotNull Query query,
|
||||
long offset,
|
||||
long limit,
|
||||
@Nullable Float minCompetitiveScore,
|
||||
@Nullable Sort sort,
|
||||
boolean complete) {
|
||||
|
||||
public static ClientQueryParamsBuilder builder() {
|
||||
return ClientQueryParamsBuilder
|
||||
.builder()
|
||||
.snapshot(null)
|
||||
.offset(0)
|
||||
.limit(Long.MAX_VALUE)
|
||||
.minCompetitiveScore(null)
|
||||
.sort(null)
|
||||
.complete(true);
|
||||
}
|
||||
|
||||
public boolean isSorted() {
|
||||
return sort != null && sort.isSorted();
|
||||
}
|
||||
|
||||
public QueryParams toQueryParams() {
|
||||
return QueryParamsBuilder
|
||||
.builder()
|
||||
.query(query())
|
||||
.sort(sort != null ? sort.querySort() : new NoSort())
|
||||
.minCompetitiveScore(Nullablefloat.ofNullable(minCompetitiveScore()))
|
||||
.offset(offset())
|
||||
.limit(limit())
|
||||
.complete(complete())
|
||||
.build();
|
||||
}
|
||||
}
|
@ -1,87 +0,0 @@
|
||||
package it.cavallium.dbengine.client.query;
|
||||
|
||||
import com.squareup.moshi.JsonAdapter;
|
||||
import it.cavallium.dbengine.client.IntOpenHashSetJsonAdapter;
|
||||
import it.cavallium.dbengine.client.query.current.CurrentVersion;
|
||||
import it.cavallium.dbengine.client.query.current.data.IBasicType;
|
||||
import it.cavallium.dbengine.client.query.current.data.IType;
|
||||
import it.unimi.dsi.fastutil.booleans.BooleanList;
|
||||
import it.unimi.dsi.fastutil.bytes.ByteList;
|
||||
import it.unimi.dsi.fastutil.chars.CharList;
|
||||
import it.unimi.dsi.fastutil.ints.IntList;
|
||||
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
|
||||
import it.unimi.dsi.fastutil.longs.LongList;
|
||||
import it.unimi.dsi.fastutil.shorts.ShortList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import org.warp.commonutils.moshi.BooleanListJsonAdapter;
|
||||
import org.warp.commonutils.moshi.ByteListJsonAdapter;
|
||||
import org.warp.commonutils.moshi.CharListJsonAdapter;
|
||||
import org.warp.commonutils.moshi.IntListJsonAdapter;
|
||||
import org.warp.commonutils.moshi.LongListJsonAdapter;
|
||||
import org.warp.commonutils.moshi.MoshiPolymorphic;
|
||||
import org.warp.commonutils.moshi.ShortListJsonAdapter;
|
||||
|
||||
public class QueryMoshi extends MoshiPolymorphic<IType> {
|
||||
|
||||
private final Set<Class<IType>> abstractClasses;
|
||||
private final Set<Class<IType>> concreteClasses;
|
||||
private final Map<Class<?>, JsonAdapter<?>> extraAdapters;
|
||||
|
||||
@SuppressWarnings({"unchecked", "RedundantCast", "rawtypes"})
|
||||
public QueryMoshi() {
|
||||
super(true, GetterStyle.RECORDS_GETTERS);
|
||||
HashSet<Class<IType>> abstractClasses = new HashSet<>();
|
||||
HashSet<Class<IType>> concreteClasses = new HashSet<>();
|
||||
|
||||
// Add all super types with their implementations
|
||||
for (var superTypeClass : CurrentVersion.getSuperTypeClasses()) {
|
||||
for (Class<? extends IBasicType> superTypeSubtypesClass : CurrentVersion.getSuperTypeSubtypesClasses(
|
||||
superTypeClass)) {
|
||||
concreteClasses.add((Class<IType>) (Class) superTypeSubtypesClass);
|
||||
}
|
||||
abstractClasses.add((Class<IType>) (Class) superTypeClass);
|
||||
}
|
||||
|
||||
// Add IBasicType with all basic types
|
||||
abstractClasses.add((Class<IType>) (Class) IBasicType.class);
|
||||
for (BasicType basicType : BasicType.values()) {
|
||||
concreteClasses.add((Class<IType>) (Class) CurrentVersion.VERSION.getClass(basicType));
|
||||
}
|
||||
|
||||
this.abstractClasses = abstractClasses;
|
||||
this.concreteClasses = concreteClasses;
|
||||
Map<Class<?>, JsonAdapter<?>> extraAdapters = new HashMap<>();
|
||||
extraAdapters.put(BooleanList.class, new BooleanListJsonAdapter());
|
||||
extraAdapters.put(ByteList.class, new ByteListJsonAdapter());
|
||||
extraAdapters.put(ShortList.class, new ShortListJsonAdapter());
|
||||
extraAdapters.put(CharList.class, new CharListJsonAdapter());
|
||||
extraAdapters.put(IntList.class, new IntListJsonAdapter());
|
||||
extraAdapters.put(LongList.class, new LongListJsonAdapter());
|
||||
extraAdapters.put(IntOpenHashSet.class, new IntOpenHashSetJsonAdapter());
|
||||
this.extraAdapters = Collections.unmodifiableMap(extraAdapters);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<Class<?>, JsonAdapter<?>> getExtraAdapters() {
|
||||
return extraAdapters;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Set<Class<IType>> getAbstractClasses() {
|
||||
return abstractClasses;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Set<Class<IType>> getConcreteClasses() {
|
||||
return concreteClasses;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean shouldIgnoreField(String fieldName) {
|
||||
return fieldName.contains("$");
|
||||
}
|
||||
}
|
@ -1,156 +1,454 @@
|
||||
package it.cavallium.dbengine.client.query;
|
||||
|
||||
import com.google.common.xml.XmlEscapers;
|
||||
import it.cavallium.dbengine.client.query.current.data.BooleanQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.BooleanQueryPart;
|
||||
import it.cavallium.dbengine.client.query.current.data.BoostQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.BoxedQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.ConstantScoreQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.DoubleNDPointExactQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.DoubleNDPointRangeQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.DoubleNDTermQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.DoublePointExactQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.DoublePointRangeQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.DoublePointSetQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.DoubleTermQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.FieldExistsQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.FloatNDPointExactQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.FloatNDPointRangeQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.FloatNDTermQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.FloatPointExactQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.FloatPointRangeQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.FloatPointSetQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.FloatTermQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.IntNDPointRangeQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.IntNDTermQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.IntPointExactQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.IntPointRangeQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.IntPointSetQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.IntTermQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.LongNDPointExactQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.LongNDPointRangeQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.LongNDTermQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.LongPointExactQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.LongPointRangeQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.NumericSort;
|
||||
import it.cavallium.dbengine.client.query.current.data.LongPointSetQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.LongTermQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.OccurShould;
|
||||
import it.cavallium.dbengine.client.query.current.data.PhraseQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.QueryParams;
|
||||
import it.cavallium.dbengine.client.query.current.data.SolrTextQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.SortedDocFieldExistsQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.SortedNumericDocValuesFieldSlowRangeQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.SynonymQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.TermAndBoost;
|
||||
import it.cavallium.dbengine.client.query.current.data.TermPosition;
|
||||
import it.cavallium.dbengine.client.query.current.data.TermQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.WildcardQuery;
|
||||
import it.cavallium.dbengine.lucene.RandomSortField;
|
||||
import org.apache.lucene.document.IntPoint;
|
||||
import org.apache.lucene.document.LongPoint;
|
||||
import org.apache.lucene.document.SortedNumericDocValuesField;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.search.BooleanClause.Occur;
|
||||
import org.apache.lucene.search.BooleanQuery.Builder;
|
||||
import org.apache.lucene.search.DocValuesFieldExistsQuery;
|
||||
import org.apache.lucene.search.FuzzyQuery;
|
||||
import org.apache.lucene.search.MatchAllDocsQuery;
|
||||
import org.apache.lucene.search.MatchNoDocsQuery;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.ScoreMode;
|
||||
import org.apache.lucene.search.Sort;
|
||||
import org.apache.lucene.search.SortField;
|
||||
import org.apache.lucene.search.SortField.Type;
|
||||
import org.apache.lucene.search.SortedNumericSortField;
|
||||
import java.text.BreakIterator;
|
||||
import java.util.Comparator;
|
||||
import java.util.Locale;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
public class QueryParser {
|
||||
public static Query toQuery(it.cavallium.dbengine.client.query.current.data.Query query) {
|
||||
if (query == null) return null;
|
||||
switch (query.getBasicType$()) {
|
||||
case BooleanQuery:
|
||||
|
||||
private static final String[] QUERY_STRING_FIND = {"\\", "\""};
|
||||
private static final String[] QUERY_STRING_REPLACE = {"\\\\", "\\\""};
|
||||
|
||||
public static void toQueryXML(StringBuilder out,
|
||||
it.cavallium.dbengine.client.query.current.data.Query query,
|
||||
@Nullable Float boost) {
|
||||
if (query == null) {
|
||||
return;
|
||||
}
|
||||
switch (query.getBaseType$()) {
|
||||
case StandardQuery -> {
|
||||
var standardQuery = (it.cavallium.dbengine.client.query.current.data.StandardQuery) query;
|
||||
|
||||
out.append("<UserQuery");
|
||||
if (standardQuery.defaultFields().size() > 1) {
|
||||
throw new UnsupportedOperationException("Maximum supported default fields count: 1");
|
||||
}
|
||||
if (boost != null) {
|
||||
out.append(" boost=\"").append(boost).append("\"");
|
||||
}
|
||||
if (standardQuery.defaultFields().size() == 1) {
|
||||
out
|
||||
.append(" fieldName=\"")
|
||||
.append(XmlEscapers.xmlAttributeEscaper().escape(standardQuery.defaultFields().get(0)))
|
||||
.append("\"");
|
||||
}
|
||||
if (!standardQuery.termFields().isEmpty()) {
|
||||
throw new UnsupportedOperationException("Term fields unsupported");
|
||||
}
|
||||
if (!standardQuery.pointsConfig().isEmpty()) {
|
||||
throw new UnsupportedOperationException("Points config unsupported");
|
||||
}
|
||||
out.append(">");
|
||||
out.append(XmlEscapers.xmlContentEscaper().escape(standardQuery.query()));
|
||||
out.append("</UserQuery>\n");
|
||||
}
|
||||
case BooleanQuery -> {
|
||||
var booleanQuery = (it.cavallium.dbengine.client.query.current.data.BooleanQuery) query;
|
||||
var bq = new Builder();
|
||||
for (BooleanQueryPart part : booleanQuery.parts()) {
|
||||
Occur occur = switch (part.occur().getBasicType$()) {
|
||||
case OccurFilter -> Occur.FILTER;
|
||||
case OccurMust -> Occur.MUST;
|
||||
case OccurShould -> Occur.SHOULD;
|
||||
case OccurMustNot -> Occur.MUST_NOT;
|
||||
default -> throw new IllegalStateException("Unexpected value: " + part.occur().getBasicType$());
|
||||
};
|
||||
bq.add(toQuery(part.query()), occur);
|
||||
if (booleanQuery.parts().size() == 1
|
||||
&& booleanQuery.parts().get(0).occur().getBaseType$() == BaseType.OccurMust) {
|
||||
toQueryXML(out, booleanQuery.parts().get(0).query(), boost);
|
||||
} else {
|
||||
out.append("<BooleanQuery");
|
||||
if (boost != null) {
|
||||
out.append(" boost=\"").append(boost).append("\"");
|
||||
}
|
||||
out.append(" minimumNumberShouldMatch=\"").append(booleanQuery.minShouldMatch()).append("\"");
|
||||
out.append(">\n");
|
||||
|
||||
for (BooleanQueryPart part : booleanQuery.parts()) {
|
||||
out.append("<Clause");
|
||||
out.append(" occurs=\"").append(switch (part.occur().getBaseType$()) {
|
||||
case OccurFilter -> "filter";
|
||||
case OccurMust -> "must";
|
||||
case OccurShould -> "should";
|
||||
case OccurMustNot -> "mustNot";
|
||||
default -> throw new IllegalStateException("Unexpected value: " + part.occur().getBaseType$());
|
||||
}).append("\"");
|
||||
out.append(">\n");
|
||||
toQueryXML(out, part.query(), null);
|
||||
out.append("</Clause>\n");
|
||||
}
|
||||
out.append("</BooleanQuery>\n");
|
||||
}
|
||||
bq.setMinimumNumberShouldMatch(booleanQuery.minShouldMatch());
|
||||
return bq.build();
|
||||
case IntPointExactQuery:
|
||||
}
|
||||
case IntPointExactQuery -> {
|
||||
var intPointExactQuery = (IntPointExactQuery) query;
|
||||
return IntPoint.newExactQuery(intPointExactQuery.field(), intPointExactQuery.value());
|
||||
case LongPointExactQuery:
|
||||
out.append("<PointRangeQuery type=\"int\"");
|
||||
if (boost != null) {
|
||||
out.append(" boost=\"").append(boost).append("\"");
|
||||
}
|
||||
out.append(" fieldName=\"").append(XmlEscapers.xmlAttributeEscaper().escape(intPointExactQuery.field())).append("\"");
|
||||
out.append(" lowerTerm=\"").append(intPointExactQuery.value()).append("\"");
|
||||
out.append(" upperTerm=\"").append(intPointExactQuery.value()).append("\"");
|
||||
out.append(" />\n");
|
||||
}
|
||||
case IntNDPointExactQuery -> {
|
||||
var intPointExactQuery = (IntPointExactQuery) query;
|
||||
throw new UnsupportedOperationException("N-dimensional point queries are not supported");
|
||||
}
|
||||
case LongPointExactQuery -> {
|
||||
var longPointExactQuery = (LongPointExactQuery) query;
|
||||
return LongPoint.newExactQuery(longPointExactQuery.field(), longPointExactQuery.value());
|
||||
case TermQuery:
|
||||
out.append("<PointRangeQuery type=\"long\"");
|
||||
if (boost != null) {
|
||||
out.append(" boost=\"").append(boost).append("\"");
|
||||
}
|
||||
out.append(" fieldName=\"").append(XmlEscapers.xmlAttributeEscaper().escape(longPointExactQuery.field())).append("\"");
|
||||
out.append(" lowerTerm=\"").append(longPointExactQuery.value()).append("\"");
|
||||
out.append(" upperTerm=\"").append(longPointExactQuery.value()).append("\"");
|
||||
out.append(" />\n");
|
||||
}
|
||||
case FloatPointExactQuery -> {
|
||||
var floatPointExactQuery = (FloatPointExactQuery) query;
|
||||
out.append("<PointRangeQuery type=\"float\"");
|
||||
if (boost != null) {
|
||||
out.append(" boost=\"").append(boost).append("\"");
|
||||
}
|
||||
out.append(" fieldName=\"").append(XmlEscapers.xmlAttributeEscaper().escape(floatPointExactQuery.field())).append("\"");
|
||||
out.append(" lowerTerm=\"").append(floatPointExactQuery.value()).append("\"");
|
||||
out.append(" upperTerm=\"").append(floatPointExactQuery.value()).append("\"");
|
||||
out.append(" />\n");
|
||||
}
|
||||
case DoublePointExactQuery -> {
|
||||
var doublePointExactQuery = (DoublePointExactQuery) query;
|
||||
out.append("<PointRangeQuery type=\"double\"");
|
||||
if (boost != null) {
|
||||
out.append(" boost=\"").append(boost).append("\"");
|
||||
}
|
||||
out.append(" fieldName=\"").append(XmlEscapers.xmlAttributeEscaper().escape(doublePointExactQuery.field())).append("\"");
|
||||
out.append(" lowerTerm=\"").append(doublePointExactQuery.value()).append("\"");
|
||||
out.append(" upperTerm=\"").append(doublePointExactQuery.value()).append("\"");
|
||||
out.append(" />\n");
|
||||
}
|
||||
case LongNDPointExactQuery -> {
|
||||
var longndPointExactQuery = (LongNDPointExactQuery) query;
|
||||
throw new UnsupportedOperationException("N-dimensional point queries are not supported");
|
||||
}
|
||||
case FloatNDPointExactQuery -> {
|
||||
var floatndPointExactQuery = (FloatNDPointExactQuery) query;
|
||||
throw new UnsupportedOperationException("N-dimensional point queries are not supported");
|
||||
}
|
||||
case DoubleNDPointExactQuery -> {
|
||||
var doublendPointExactQuery = (DoubleNDPointExactQuery) query;
|
||||
throw new UnsupportedOperationException("N-dimensional point queries are not supported");
|
||||
}
|
||||
case IntPointSetQuery -> {
|
||||
var intPointSetQuery = (IntPointSetQuery) query;
|
||||
// Polyfill
|
||||
toQueryXML(out, BooleanQuery.of(intPointSetQuery.values().intStream()
|
||||
.mapToObj(val -> IntPointExactQuery.of(intPointSetQuery.field(), val))
|
||||
.map(q -> BooleanQueryPart.of(q, OccurShould.of()))
|
||||
.toList(), 1), boost);
|
||||
}
|
||||
case LongPointSetQuery -> {
|
||||
var longPointSetQuery = (LongPointSetQuery) query;
|
||||
// Polyfill
|
||||
toQueryXML(out, BooleanQuery.of(longPointSetQuery.values().longStream()
|
||||
.mapToObj(val -> LongPointExactQuery.of(longPointSetQuery.field(), val))
|
||||
.map(q -> BooleanQueryPart.of(q, OccurShould.of()))
|
||||
.toList(), 1), boost);
|
||||
}
|
||||
case FloatPointSetQuery -> {
|
||||
var floatPointSetQuery = (FloatPointSetQuery) query;
|
||||
// Polyfill
|
||||
toQueryXML(out, BooleanQuery.of(floatPointSetQuery.values().stream()
|
||||
.map(val -> FloatPointExactQuery.of(floatPointSetQuery.field(), val))
|
||||
.map(q -> BooleanQueryPart.of(q, OccurShould.of()))
|
||||
.toList(), 1), boost);
|
||||
}
|
||||
case DoublePointSetQuery -> {
|
||||
var doublePointSetQuery = (DoublePointSetQuery) query;
|
||||
// Polyfill
|
||||
toQueryXML(out, BooleanQuery.of(doublePointSetQuery.values().doubleStream()
|
||||
.mapToObj(val -> DoublePointExactQuery.of(doublePointSetQuery.field(), val))
|
||||
.map(q -> BooleanQueryPart.of(q, OccurShould.of()))
|
||||
.toList(), 1), boost);
|
||||
}
|
||||
case TermQuery -> {
|
||||
var termQuery = (TermQuery) query;
|
||||
return new org.apache.lucene.search.TermQuery(toTerm(termQuery.term()));
|
||||
case BoostQuery:
|
||||
out
|
||||
.append("<TermQuery");
|
||||
if (boost != null) {
|
||||
out.append(" boost=\"").append(boost).append("\"");
|
||||
}
|
||||
out
|
||||
.append(" fieldName=\"")
|
||||
.append(XmlEscapers.xmlAttributeEscaper().escape(termQuery.term().field()))
|
||||
.append("\"");
|
||||
out.append(">");
|
||||
out.append(XmlEscapers.xmlContentEscaper().escape(termQuery.term().value()));
|
||||
out.append("</TermQuery>\n");
|
||||
}
|
||||
case IntTermQuery -> {
|
||||
var intTermQuery = (IntTermQuery) query;
|
||||
throw new UnsupportedOperationException("Non-string term fields are not supported");
|
||||
}
|
||||
case IntNDTermQuery -> {
|
||||
var intNDTermQuery = (IntNDTermQuery) query;
|
||||
throw new UnsupportedOperationException("Non-string term fields are not supported");
|
||||
}
|
||||
case LongTermQuery -> {
|
||||
var longTermQuery = (LongTermQuery) query;
|
||||
throw new UnsupportedOperationException("Non-string term fields are not supported");
|
||||
}
|
||||
case LongNDTermQuery -> {
|
||||
var longNDTermQuery = (LongNDTermQuery) query;
|
||||
throw new UnsupportedOperationException("Non-string term fields are not supported");
|
||||
}
|
||||
case FloatTermQuery -> {
|
||||
var floatTermQuery = (FloatTermQuery) query;
|
||||
throw new UnsupportedOperationException("Non-string term fields are not supported");
|
||||
}
|
||||
case FloatNDTermQuery -> {
|
||||
var floatNDTermQuery = (FloatNDTermQuery) query;
|
||||
throw new UnsupportedOperationException("Non-string term fields are not supported");
|
||||
}
|
||||
case DoubleTermQuery -> {
|
||||
var doubleTermQuery = (DoubleTermQuery) query;
|
||||
throw new UnsupportedOperationException("Non-string term fields are not supported");
|
||||
}
|
||||
case DoubleNDTermQuery -> {
|
||||
var doubleNDTermQuery = (DoubleNDTermQuery) query;
|
||||
throw new UnsupportedOperationException("Non-string term fields are not supported");
|
||||
}
|
||||
case FieldExistsQuery -> {
|
||||
var fieldExistQuery = (FieldExistsQuery) query;
|
||||
out.append("<UserQuery");
|
||||
if (boost != null) {
|
||||
out.append(" boost=\"").append(boost).append("\"");
|
||||
}
|
||||
out.append(">");
|
||||
ensureValidField(fieldExistQuery.field());
|
||||
out.append(fieldExistQuery.field());
|
||||
out.append(":[* TO *]");
|
||||
out.append("</UserQuery>\n");
|
||||
}
|
||||
case SolrTextQuery -> {
|
||||
var solrTextQuery = (SolrTextQuery) query;
|
||||
out.append("<UserQuery");
|
||||
if (boost != null) {
|
||||
out.append(" boost=\"").append(boost).append("\"");
|
||||
}
|
||||
out.append(">");
|
||||
ensureValidField(solrTextQuery.field());
|
||||
out.append(solrTextQuery.field());
|
||||
out.append(":");
|
||||
out.append("\"").append(XmlEscapers.xmlContentEscaper().escape(escapeQueryStringValue(solrTextQuery.phrase()))).append("\"");
|
||||
if (solrTextQuery.slop() > 0 && hasMoreThanOneWord(solrTextQuery.phrase())) {
|
||||
out.append("~").append(solrTextQuery.slop());
|
||||
}
|
||||
out.append("</UserQuery>\n");
|
||||
}
|
||||
case BoostQuery -> {
|
||||
var boostQuery = (BoostQuery) query;
|
||||
return new org.apache.lucene.search.BoostQuery(toQuery(boostQuery.query()), boostQuery.scoreBoost());
|
||||
case ConstantScoreQuery:
|
||||
toQueryXML(out, boostQuery.query(), boostQuery.scoreBoost());
|
||||
}
|
||||
case ConstantScoreQuery -> {
|
||||
var constantScoreQuery = (ConstantScoreQuery) query;
|
||||
return new org.apache.lucene.search.ConstantScoreQuery(toQuery(constantScoreQuery.query()));
|
||||
case BoxedQuery:
|
||||
return toQuery(((BoxedQuery) query).query());
|
||||
case FuzzyQuery:
|
||||
var fuzzyQuery = (it.cavallium.dbengine.client.query.current.data.FuzzyQuery) query;
|
||||
return new FuzzyQuery(toTerm(fuzzyQuery.term()),
|
||||
fuzzyQuery.maxEdits(),
|
||||
fuzzyQuery.prefixLength(),
|
||||
fuzzyQuery.maxExpansions(),
|
||||
fuzzyQuery.transpositions()
|
||||
);
|
||||
case IntPointRangeQuery:
|
||||
out.append("<ConstantScoreQuery");
|
||||
if (boost != null) {
|
||||
out.append(" boost=\"").append(boost).append("\"");
|
||||
}
|
||||
out.append(">\n");
|
||||
toQueryXML(out, query, null);
|
||||
out.append("</ConstantScoreQuery>\n");
|
||||
}
|
||||
case BoxedQuery -> {
|
||||
toQueryXML(out, ((BoxedQuery) query).query(), boost);
|
||||
}
|
||||
case FuzzyQuery -> {
|
||||
throw new UnsupportedOperationException("Fuzzy query is not supported, use span queries");
|
||||
}
|
||||
case IntPointRangeQuery -> {
|
||||
var intPointRangeQuery = (IntPointRangeQuery) query;
|
||||
return IntPoint.newRangeQuery(intPointRangeQuery.field(),
|
||||
intPointRangeQuery.min(),
|
||||
intPointRangeQuery.max()
|
||||
);
|
||||
case LongPointRangeQuery:
|
||||
out.append("<PointRangeQuery type=\"int\"");
|
||||
if (boost != null) {
|
||||
out.append(" boost=\"").append(boost).append("\"");
|
||||
}
|
||||
out.append(" fieldName=\"").append(XmlEscapers.xmlAttributeEscaper().escape(intPointRangeQuery.field())).append("\"");
|
||||
out.append(" lowerTerm=\"").append(intPointRangeQuery.min()).append("\"");
|
||||
out.append(" upperTerm=\"").append(intPointRangeQuery.max()).append("\"");
|
||||
out.append(" />\n");
|
||||
}
|
||||
case IntNDPointRangeQuery -> {
|
||||
var intndPointRangeQuery = (IntNDPointRangeQuery) query;
|
||||
throw new UnsupportedOperationException("N-dimensional point queries are not supported");
|
||||
}
|
||||
case LongPointRangeQuery -> {
|
||||
var longPointRangeQuery = (LongPointRangeQuery) query;
|
||||
return LongPoint.newRangeQuery(longPointRangeQuery.field(),
|
||||
longPointRangeQuery.min(),
|
||||
longPointRangeQuery.max()
|
||||
);
|
||||
case MatchAllDocsQuery:
|
||||
return new MatchAllDocsQuery();
|
||||
case MatchNoDocsQuery:
|
||||
return new MatchNoDocsQuery();
|
||||
case PhraseQuery:
|
||||
out.append("<PointRangeQuery type=\"long\"");
|
||||
if (boost != null) {
|
||||
out.append(" boost=\"").append(boost).append("\"");
|
||||
}
|
||||
out.append(" fieldName=\"").append(XmlEscapers.xmlAttributeEscaper().escape(longPointRangeQuery.field())).append("\"");
|
||||
out.append(" lowerTerm=\"").append(longPointRangeQuery.min()).append("\"");
|
||||
out.append(" upperTerm=\"").append(longPointRangeQuery.max()).append("\"");
|
||||
out.append(" />\n");
|
||||
}
|
||||
case FloatPointRangeQuery -> {
|
||||
var floatPointRangeQuery = (FloatPointRangeQuery) query;
|
||||
out.append("<PointRangeQuery type=\"float\"");
|
||||
if (boost != null) {
|
||||
out.append(" boost=\"").append(boost).append("\"");
|
||||
}
|
||||
out.append(" fieldName=\"").append(XmlEscapers.xmlAttributeEscaper().escape(floatPointRangeQuery.field())).append("\"");
|
||||
out.append(" lowerTerm=\"").append(floatPointRangeQuery.min()).append("\"");
|
||||
out.append(" upperTerm=\"").append(floatPointRangeQuery.max()).append("\"");
|
||||
out.append(" />\n");
|
||||
}
|
||||
case DoublePointRangeQuery -> {
|
||||
var doublePointRangeQuery = (DoublePointRangeQuery) query;
|
||||
out.append("<PointRangeQuery type=\"double\"");
|
||||
if (boost != null) {
|
||||
out.append(" boost=\"").append(boost).append("\"");
|
||||
}
|
||||
out.append(" fieldName=\"").append(XmlEscapers.xmlAttributeEscaper().escape(doublePointRangeQuery.field())).append("\"");
|
||||
out.append(" lowerTerm=\"").append(doublePointRangeQuery.min()).append("\"");
|
||||
out.append(" upperTerm=\"").append(doublePointRangeQuery.max()).append("\"");
|
||||
out.append(" />\n");
|
||||
}
|
||||
case LongNDPointRangeQuery -> {
|
||||
var longndPointRangeQuery = (LongNDPointRangeQuery) query;
|
||||
throw new UnsupportedOperationException("N-dimensional point queries are not supported");
|
||||
}
|
||||
case FloatNDPointRangeQuery -> {
|
||||
var floatndPointRangeQuery = (FloatNDPointRangeQuery) query;
|
||||
throw new UnsupportedOperationException("N-dimensional point queries are not supported");
|
||||
}
|
||||
case DoubleNDPointRangeQuery -> {
|
||||
var doublendPointRangeQuery = (DoubleNDPointRangeQuery) query;
|
||||
throw new UnsupportedOperationException("N-dimensional point queries are not supported");
|
||||
}
|
||||
case MatchAllDocsQuery -> {
|
||||
out.append("<UserQuery");
|
||||
if (boost != null) {
|
||||
out.append(" boost=\"").append(boost).append("\"");
|
||||
}
|
||||
out.append(">");
|
||||
out.append("*:*");
|
||||
out.append("</UserQuery>\n");
|
||||
}
|
||||
case MatchNoDocsQuery -> {
|
||||
out.append("<UserQuery");
|
||||
if (boost != null) {
|
||||
out.append(" boost=\"").append(boost).append("\"");
|
||||
}
|
||||
out.append(">");
|
||||
//todo: check if it's correct
|
||||
out.append("!*:*");
|
||||
out.append("</UserQuery>\n");
|
||||
}
|
||||
case PhraseQuery -> {
|
||||
//todo: check if it's correct
|
||||
|
||||
var phraseQuery = (PhraseQuery) query;
|
||||
var pqb = new org.apache.lucene.search.PhraseQuery.Builder();
|
||||
for (TermPosition phrase : phraseQuery.phrase()) {
|
||||
pqb.add(toTerm(phrase.term()), phrase.position());
|
||||
out.append("<SpanNear");
|
||||
if (boost != null) {
|
||||
out.append(" boost=\"").append(boost).append("\"");
|
||||
}
|
||||
pqb.setSlop(phraseQuery.slop());
|
||||
return pqb.build();
|
||||
case SortedDocFieldExistsQuery:
|
||||
out.append(" inOrder=\"true\"");
|
||||
out.append(">\n");
|
||||
phraseQuery.phrase().stream().sorted(Comparator.comparingInt(TermPosition::position)).forEach(term -> {
|
||||
out
|
||||
.append("<SpanTerm fieldName=\"")
|
||||
.append(XmlEscapers.xmlAttributeEscaper().escape(term.term().field()))
|
||||
.append("\">")
|
||||
.append(XmlEscapers.xmlContentEscaper().escape(term.term().value()))
|
||||
.append("</SpanTerm>\n");
|
||||
});
|
||||
out.append("</SpanNear>\n");
|
||||
}
|
||||
case SortedDocFieldExistsQuery -> {
|
||||
var sortedDocFieldExistsQuery = (SortedDocFieldExistsQuery) query;
|
||||
return new DocValuesFieldExistsQuery(sortedDocFieldExistsQuery.field());
|
||||
case SynonymQuery:
|
||||
throw new UnsupportedOperationException("Field existence query is not supported");
|
||||
}
|
||||
case SynonymQuery -> {
|
||||
var synonymQuery = (SynonymQuery) query;
|
||||
var sqb = new org.apache.lucene.search.SynonymQuery.Builder(synonymQuery.field());
|
||||
for (TermAndBoost part : synonymQuery.parts()) {
|
||||
sqb.addTerm(toTerm(part.term()), part.boost());
|
||||
}
|
||||
return sqb.build();
|
||||
case SortedNumericDocValuesFieldSlowRangeQuery:
|
||||
var sortedNumericDocValuesFieldSlowRangeQuery = (SortedNumericDocValuesFieldSlowRangeQuery) query;
|
||||
return SortedNumericDocValuesField.newSlowRangeQuery(sortedNumericDocValuesFieldSlowRangeQuery.field(),
|
||||
sortedNumericDocValuesFieldSlowRangeQuery.min(),
|
||||
sortedNumericDocValuesFieldSlowRangeQuery.max()
|
||||
);
|
||||
case WildcardQuery:
|
||||
throw new UnsupportedOperationException("Synonym query is not supported");
|
||||
}
|
||||
case SortedNumericDocValuesFieldSlowRangeQuery -> {
|
||||
throw new UnsupportedOperationException("Slow range query is not supported");
|
||||
}
|
||||
case WildcardQuery -> {
|
||||
var wildcardQuery = (WildcardQuery) query;
|
||||
return new org.apache.lucene.search.WildcardQuery(new Term(wildcardQuery.field(), wildcardQuery.pattern()));
|
||||
default:
|
||||
throw new IllegalStateException("Unexpected value: " + query.getBasicType$());
|
||||
throw new UnsupportedOperationException("Wildcard query is not supported");
|
||||
}
|
||||
default -> throw new IllegalStateException("Unexpected value: " + query.getBaseType$());
|
||||
}
|
||||
}
|
||||
|
||||
private static Term toTerm(it.cavallium.dbengine.client.query.current.data.Term term) {
|
||||
return new Term(term.field(), term.value());
|
||||
}
|
||||
private static boolean hasMoreThanOneWord(String sentence) {
|
||||
BreakIterator iterator = BreakIterator.getWordInstance(Locale.ENGLISH);
|
||||
iterator.setText(sentence);
|
||||
|
||||
public static Sort toSort(it.cavallium.dbengine.client.query.current.data.Sort sort) {
|
||||
switch (sort.getBasicType$()) {
|
||||
case NoSort:
|
||||
return null;
|
||||
case ScoreSort:
|
||||
return new Sort(SortField.FIELD_SCORE);
|
||||
case DocSort:
|
||||
return new Sort(SortField.FIELD_DOC);
|
||||
case NumericSort:
|
||||
NumericSort numericSort = (NumericSort) sort;
|
||||
return new Sort(new SortedNumericSortField(numericSort.field(), Type.LONG, numericSort.reverse()));
|
||||
case RandomSort:
|
||||
return new Sort(new RandomSortField());
|
||||
default:
|
||||
throw new IllegalStateException("Unexpected value: " + sort.getBasicType$());
|
||||
boolean firstWord = false;
|
||||
iterator.first();
|
||||
int end = iterator.next();
|
||||
while (end != BreakIterator.DONE) {
|
||||
if (!firstWord) {
|
||||
firstWord = true;
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
end = iterator.next();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public static it.cavallium.dbengine.client.query.current.data.Term toQueryTerm(Term term) {
|
||||
return it.cavallium.dbengine.client.query.current.data.Term.of(term.field(), term.text());
|
||||
private static String escapeQueryStringValue(String text) {
|
||||
return StringUtils.replaceEach(text, QUERY_STRING_FIND, QUERY_STRING_REPLACE);
|
||||
}
|
||||
|
||||
private static void ensureValidField(String field) {
|
||||
field.codePoints().forEach(codePoint -> {
|
||||
if (!Character.isLetterOrDigit(codePoint) && codePoint != '_') {
|
||||
throw new UnsupportedOperationException(
|
||||
"Invalid character \"" + codePoint + "\" in field name \"" + field + "\"");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -0,0 +1,16 @@
|
||||
package it.cavallium.dbengine.client.query;
|
||||
|
||||
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
|
||||
|
||||
public class QueryUtil {
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
public static String toHumanReadableString(TotalHitsCount totalHitsCount) {
|
||||
if (totalHitsCount.exact()) {
|
||||
return Long.toString(totalHitsCount.value());
|
||||
} else {
|
||||
return totalHitsCount.value() + "+";
|
||||
}
|
||||
}
|
||||
|
||||
}
|
@ -1,97 +0,0 @@
|
||||
package it.cavallium.dbengine.client.query;
|
||||
|
||||
import it.cavallium.dbengine.client.query.current.data.BooleanQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.BooleanQueryPart;
|
||||
import it.cavallium.dbengine.client.query.current.data.Occur;
|
||||
import it.cavallium.dbengine.client.query.current.data.OccurFilter;
|
||||
import it.cavallium.dbengine.client.query.current.data.OccurMust;
|
||||
import it.cavallium.dbengine.client.query.current.data.OccurMustNot;
|
||||
import it.cavallium.dbengine.client.query.current.data.OccurShould;
|
||||
import it.cavallium.dbengine.client.query.current.data.PhraseQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.Query;
|
||||
import it.cavallium.dbengine.client.query.current.data.SynonymQuery;
|
||||
import it.cavallium.dbengine.client.query.current.data.TermAndBoost;
|
||||
import it.cavallium.dbengine.client.query.current.data.TermPosition;
|
||||
import it.cavallium.dbengine.client.query.current.data.TermQuery;
|
||||
import it.cavallium.dbengine.lucene.LuceneUtils;
|
||||
import it.cavallium.dbengine.lucene.analyzer.TextFieldsAnalyzer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
import org.apache.lucene.search.BooleanClause;
|
||||
import org.apache.lucene.util.QueryBuilder;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
public class QueryUtils {
|
||||
|
||||
public static Query approximateSearch(TextFieldsAnalyzer preferredAnalyzer, String field, String text) {
|
||||
var qb = new QueryBuilder(LuceneUtils.getAnalyzer(preferredAnalyzer));
|
||||
var luceneQuery = qb.createMinShouldMatchQuery(field, text, 0.75f);
|
||||
return transformQuery(field, luceneQuery);
|
||||
}
|
||||
|
||||
public static Query exactSearch(TextFieldsAnalyzer preferredAnalyzer, String field, String text) {
|
||||
var qb = new QueryBuilder(LuceneUtils.getAnalyzer(preferredAnalyzer));
|
||||
var luceneQuery = qb.createPhraseQuery(field, text);
|
||||
return transformQuery(field, luceneQuery);
|
||||
}
|
||||
|
||||
@NotNull
|
||||
private static Query transformQuery(String field, org.apache.lucene.search.Query luceneQuery) {
|
||||
if (luceneQuery == null) {
|
||||
return TermQuery.of(it.cavallium.dbengine.client.query.current.data.Term.of(field, ""));
|
||||
}
|
||||
if (luceneQuery instanceof org.apache.lucene.search.TermQuery) {
|
||||
return TermQuery.of(QueryParser.toQueryTerm(((org.apache.lucene.search.TermQuery) luceneQuery).getTerm()));
|
||||
}
|
||||
if (luceneQuery instanceof org.apache.lucene.search.BooleanQuery) {
|
||||
var booleanQuery = (org.apache.lucene.search.BooleanQuery) luceneQuery;
|
||||
var queryParts = new ArrayList<BooleanQueryPart>();
|
||||
for (BooleanClause booleanClause : booleanQuery) {
|
||||
org.apache.lucene.search.Query queryPartQuery = booleanClause.getQuery();
|
||||
|
||||
Occur occur;
|
||||
switch (booleanClause.getOccur()) {
|
||||
case MUST:
|
||||
occur = OccurMust.of();
|
||||
break;
|
||||
case FILTER:
|
||||
occur = OccurFilter.of();
|
||||
break;
|
||||
case SHOULD:
|
||||
occur = OccurShould.of();
|
||||
break;
|
||||
case MUST_NOT:
|
||||
occur = OccurMustNot.of();
|
||||
break;
|
||||
default:
|
||||
throw new IllegalArgumentException();
|
||||
}
|
||||
queryParts.add(BooleanQueryPart.of(transformQuery(field, queryPartQuery), occur));
|
||||
}
|
||||
return BooleanQuery.of(List.copyOf(queryParts), booleanQuery.getMinimumNumberShouldMatch());
|
||||
}
|
||||
if (luceneQuery instanceof org.apache.lucene.search.PhraseQuery) {
|
||||
var phraseQuery = (org.apache.lucene.search.PhraseQuery) luceneQuery;
|
||||
int slop = phraseQuery.getSlop();
|
||||
var terms = phraseQuery.getTerms();
|
||||
var positions = phraseQuery.getPositions();
|
||||
TermPosition[] termPositions = new TermPosition[terms.length];
|
||||
for (int i = 0; i < terms.length; i++) {
|
||||
var term = terms[i];
|
||||
var position = positions[i];
|
||||
termPositions[i] = TermPosition.of(QueryParser.toQueryTerm(term), position);
|
||||
}
|
||||
return PhraseQuery.of(List.of(termPositions), slop);
|
||||
}
|
||||
org.apache.lucene.search.SynonymQuery synonymQuery = (org.apache.lucene.search.SynonymQuery) luceneQuery;
|
||||
return SynonymQuery.of(field,
|
||||
synonymQuery
|
||||
.getTerms()
|
||||
.stream()
|
||||
.map(term -> TermAndBoost.of(QueryParser.toQueryTerm(term), 1))
|
||||
.toList()
|
||||
);
|
||||
}
|
||||
}
|
@ -0,0 +1,3 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
public record ColumnProperty<T>(String columnName, String propertyName, T value) {}
|
@ -1,10 +1,13 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import it.cavallium.dbengine.rpc.current.data.Column;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Objects;
|
||||
import java.util.StringJoiner;
|
||||
|
||||
public record Column(String name) {
|
||||
public class ColumnUtils {
|
||||
|
||||
private ColumnUtils() {
|
||||
|
||||
}
|
||||
|
||||
public static Column dictionary(String name) {
|
||||
return new Column("hash_map_" + name);
|
@ -0,0 +1,10 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import it.cavallium.dbengine.rpc.current.data.Column;
|
||||
import java.nio.file.Path;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
public interface DatabaseOperations {
|
||||
|
||||
void ingestSST(Column column, Stream<Path> files, boolean replaceExisting);
|
||||
}
|
@ -0,0 +1,31 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import it.cavallium.dbengine.client.MemoryStats;
|
||||
import it.cavallium.dbengine.rpc.current.data.Column;
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Stream;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
public interface DatabaseProperties {
|
||||
|
||||
MemoryStats getMemoryStats();
|
||||
|
||||
String getRocksDBStats();
|
||||
|
||||
Map<String, String> getMapProperty(@Nullable Column column, RocksDBMapProperty property);
|
||||
|
||||
Stream<ColumnProperty<Map<String, String>>> getMapColumnProperties(RocksDBMapProperty property);
|
||||
|
||||
String getStringProperty(@Nullable Column column, RocksDBStringProperty property);
|
||||
|
||||
Stream<ColumnProperty<String>> getStringColumnProperties(RocksDBStringProperty property);
|
||||
|
||||
Long getLongProperty(@Nullable Column column, RocksDBLongProperty property);
|
||||
|
||||
Stream<ColumnProperty<Long>> getLongColumnProperties(RocksDBLongProperty property);
|
||||
|
||||
Long getAggregatedLongProperty(RocksDBLongProperty property);
|
||||
|
||||
Stream<TableWithProperties> getTableProperties();
|
||||
}
|
@ -5,6 +5,7 @@ import org.jetbrains.annotations.Nullable;
|
||||
|
||||
public class Delta<T> {
|
||||
|
||||
private static final Delta<?> EMPTY = new Delta<>(null, null);
|
||||
private final @Nullable T previous;
|
||||
private final @Nullable T current;
|
||||
|
||||
@ -25,6 +26,11 @@ public class Delta<T> {
|
||||
return current;
|
||||
}
|
||||
|
||||
public static <X> Delta<X> empty() {
|
||||
//noinspection unchecked
|
||||
return (Delta<X>) EMPTY;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (obj == this)
|
||||
|
@ -0,0 +1,6 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
/**
|
||||
* Closeable resource that can be closed if discarded
|
||||
*/
|
||||
public interface DiscardingCloseable extends SafeCloseable {}
|
File diff suppressed because it is too large
Load Diff
@ -1,35 +1,20 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import io.micrometer.core.instrument.MeterRegistry;
|
||||
import io.net5.buffer.api.BufferAllocator;
|
||||
import it.cavallium.dbengine.client.DatabaseOptions;
|
||||
import it.cavallium.dbengine.client.IndicizerAnalyzers;
|
||||
import it.cavallium.dbengine.client.IndicizerSimilarities;
|
||||
import it.cavallium.dbengine.client.LuceneOptions;
|
||||
import it.cavallium.dbengine.lucene.LuceneHacks;
|
||||
import it.cavallium.dbengine.rpc.current.data.Column;
|
||||
import it.cavallium.dbengine.rpc.current.data.DatabaseOptions;
|
||||
import java.util.List;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
@SuppressWarnings("UnusedReturnValue")
|
||||
public interface LLDatabaseConnection {
|
||||
|
||||
BufferAllocator getAllocator();
|
||||
|
||||
MeterRegistry getMeterRegistry();
|
||||
|
||||
Mono<? extends LLDatabaseConnection> connect();
|
||||
LLDatabaseConnection connect();
|
||||
|
||||
Mono<? extends LLKeyValueDatabase> getDatabase(String name,
|
||||
LLKeyValueDatabase getDatabase(String name,
|
||||
List<Column> columns,
|
||||
DatabaseOptions databaseOptions);
|
||||
|
||||
Mono<? extends LLLuceneIndex> getLuceneIndex(String name,
|
||||
int instancesCount,
|
||||
IndicizerAnalyzers indicizerAnalyzers,
|
||||
IndicizerSimilarities indicizerSimilarities,
|
||||
LuceneOptions luceneOptions,
|
||||
@Nullable LuceneHacks luceneHacks);
|
||||
|
||||
Mono<Void> disconnect();
|
||||
void disconnect();
|
||||
}
|
||||
|
@ -1,109 +1,41 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import io.net5.buffer.api.Buffer;
|
||||
import io.net5.buffer.api.Drop;
|
||||
import io.net5.buffer.api.Owned;
|
||||
import io.net5.buffer.api.Send;
|
||||
import io.net5.buffer.api.internal.ResourceSupport;
|
||||
import static it.cavallium.dbengine.database.LLUtils.unmodifiableBytes;
|
||||
|
||||
import it.cavallium.buffer.Buf;
|
||||
import java.util.StringJoiner;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import org.warp.commonutils.log.Logger;
|
||||
import org.warp.commonutils.log.LoggerFactory;
|
||||
|
||||
public class LLDelta extends ResourceSupport<LLDelta, LLDelta> {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(LLDelta.class);
|
||||
|
||||
private static final Drop<LLDelta> DROP = new Drop<>() {
|
||||
@Override
|
||||
public void drop(LLDelta obj) {
|
||||
try {
|
||||
if (obj.previous != null) {
|
||||
obj.previous.close();
|
||||
}
|
||||
} catch (Throwable ex) {
|
||||
logger.error("Failed to close previous", ex);
|
||||
}
|
||||
try {
|
||||
if (obj.current != null) {
|
||||
obj.current.close();
|
||||
}
|
||||
} catch (Throwable ex) {
|
||||
logger.error("Failed to close current", ex);
|
||||
}
|
||||
try {
|
||||
if (obj.onClose != null) {
|
||||
obj.onClose.run();
|
||||
}
|
||||
} catch (Throwable ex) {
|
||||
logger.error("Failed to close onDrop", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Drop<LLDelta> fork() {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void attach(LLDelta obj) {
|
||||
|
||||
}
|
||||
};
|
||||
public class LLDelta {
|
||||
|
||||
@Nullable
|
||||
private Buffer previous;
|
||||
private final Buf previous;
|
||||
@Nullable
|
||||
private Buffer current;
|
||||
@Nullable
|
||||
private Runnable onClose;
|
||||
private final Buf current;
|
||||
|
||||
private LLDelta(@Nullable Send<Buffer> previous, @Nullable Send<Buffer> current, @Nullable Runnable onClose) {
|
||||
super(DROP);
|
||||
assert isAllAccessible();
|
||||
this.previous = previous != null ? previous.receive().makeReadOnly() : null;
|
||||
this.current = current != null ? current.receive().makeReadOnly() : null;
|
||||
this.onClose = onClose;
|
||||
private LLDelta(@Nullable Buf previous, @Nullable Buf current) {
|
||||
super();
|
||||
this.previous = unmodifiableBytes(previous);
|
||||
this.current = unmodifiableBytes(current);
|
||||
}
|
||||
|
||||
private boolean isAllAccessible() {
|
||||
assert previous == null || previous.isAccessible();
|
||||
assert current == null || current.isAccessible();
|
||||
assert this.isAccessible();
|
||||
assert this.isOwned();
|
||||
return true;
|
||||
}
|
||||
|
||||
public static LLDelta of(Send<Buffer> previous, Send<Buffer> current) {
|
||||
public static LLDelta of(Buf previous, Buf current) {
|
||||
assert (previous == null && current == null) || (previous != current);
|
||||
return new LLDelta(previous, current, null);
|
||||
return new LLDelta(previous, current);
|
||||
}
|
||||
|
||||
public Send<Buffer> previous() {
|
||||
ensureOwned();
|
||||
return previous != null ? previous.copy().send() : null;
|
||||
public Buf previous() {
|
||||
return previous;
|
||||
}
|
||||
|
||||
public Send<Buffer> current() {
|
||||
ensureOwned();
|
||||
return current != null ? current.copy().send() : null;
|
||||
public Buf current() {
|
||||
return current;
|
||||
}
|
||||
|
||||
public boolean isModified() {
|
||||
return !LLUtils.equals(previous, current);
|
||||
}
|
||||
|
||||
private void ensureOwned() {
|
||||
assert isAllAccessible();
|
||||
if (!isOwned()) {
|
||||
if (!isAccessible()) {
|
||||
throw this.createResourceClosedException();
|
||||
} else {
|
||||
throw new IllegalStateException("Resource not owned");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
@ -131,28 +63,4 @@ public class LLDelta extends ResourceSupport<LLDelta, LLDelta> {
|
||||
.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected RuntimeException createResourceClosedException() {
|
||||
return new IllegalStateException("Closed");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void makeInaccessible() {
|
||||
this.current = null;
|
||||
this.previous = null;
|
||||
this.onClose = null;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Owned<LLDelta> prepareSend() {
|
||||
Send<Buffer> minSend = this.previous != null ? this.previous.send() : null;
|
||||
Send<Buffer> maxSend = this.current != null ? this.current.send() : null;
|
||||
Runnable onClose = this.onClose;
|
||||
return drop -> {
|
||||
var instance = new LLDelta(minSend, maxSend, onClose);
|
||||
drop.attach(instance);
|
||||
return instance;
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -1,140 +1,93 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import io.net5.buffer.api.Buffer;
|
||||
import io.net5.buffer.api.BufferAllocator;
|
||||
import io.net5.buffer.api.Send;
|
||||
import it.cavallium.dbengine.client.BadBlock;
|
||||
import it.cavallium.buffer.Buf;
|
||||
import it.cavallium.dbengine.client.DbProgress;
|
||||
import it.cavallium.dbengine.client.SSTVerificationProgress;
|
||||
import it.cavallium.dbengine.database.serialization.KVSerializationFunction;
|
||||
import it.cavallium.dbengine.database.serialization.SerializationFunction;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ForkJoinPool;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Stream;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import org.warp.commonutils.concurrency.atomicity.NotAtomic;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@NotAtomic
|
||||
public interface LLDictionary extends LLKeyValueDatabaseStructure {
|
||||
|
||||
String getColumnName();
|
||||
|
||||
BufferAllocator getAllocator();
|
||||
Buf get(@Nullable LLSnapshot snapshot, Buf key);
|
||||
|
||||
Mono<Send<Buffer>> get(@Nullable LLSnapshot snapshot, Mono<Send<Buffer>> key, boolean existsAlmostCertainly);
|
||||
Buf put(Buf key, Buf value, LLDictionaryResultType resultType);
|
||||
|
||||
default Mono<Send<Buffer>> get(@Nullable LLSnapshot snapshot, Mono<Send<Buffer>> key) {
|
||||
return get(snapshot, key, false);
|
||||
UpdateMode getUpdateMode();
|
||||
|
||||
default Buf update(Buf key, SerializationFunction<@Nullable Buf, @Nullable Buf> updater, UpdateReturnMode updateReturnMode) {
|
||||
LLDelta prev = this.updateAndGetDelta(key, updater);
|
||||
return LLUtils.resolveLLDelta(prev, updateReturnMode);
|
||||
}
|
||||
|
||||
Mono<Send<Buffer>> put(Mono<Send<Buffer>> key, Mono<Send<Buffer>> value, LLDictionaryResultType resultType);
|
||||
LLDelta updateAndGetDelta(Buf key, SerializationFunction<@Nullable Buf, @Nullable Buf> updater);
|
||||
|
||||
Mono<UpdateMode> getUpdateMode();
|
||||
void clear();
|
||||
|
||||
default Mono<Send<Buffer>> update(Mono<Send<Buffer>> key,
|
||||
SerializationFunction<@Nullable Send<Buffer>, @Nullable Buffer> updater,
|
||||
UpdateReturnMode updateReturnMode,
|
||||
boolean existsAlmostCertainly) {
|
||||
return this
|
||||
.updateAndGetDelta(key, updater, existsAlmostCertainly)
|
||||
.transform(prev -> LLUtils.resolveLLDelta(prev, updateReturnMode));
|
||||
}
|
||||
Buf remove(Buf key, LLDictionaryResultType resultType);
|
||||
|
||||
default Mono<Send<Buffer>> update(Mono<Send<Buffer>> key,
|
||||
SerializationFunction<@Nullable Send<Buffer>, @Nullable Buffer> updater,
|
||||
UpdateReturnMode returnMode) {
|
||||
return update(key, updater, returnMode, false);
|
||||
}
|
||||
Stream<OptionalBuf> getMulti(@Nullable LLSnapshot snapshot, Stream<Buf> keys);
|
||||
|
||||
Mono<Send<LLDelta>> updateAndGetDelta(Mono<Send<Buffer>> key,
|
||||
SerializationFunction<@Nullable Send<Buffer>, @Nullable Buffer> updater,
|
||||
boolean existsAlmostCertainly);
|
||||
void putMulti(Stream<LLEntry> entries);
|
||||
|
||||
default Mono<Send<LLDelta>> updateAndGetDelta(Mono<Send<Buffer>> key,
|
||||
SerializationFunction<@Nullable Send<Buffer>, @Nullable Buffer> updater) {
|
||||
return updateAndGetDelta(key, updater, false);
|
||||
}
|
||||
<K> Stream<Boolean> updateMulti(Stream<SerializedKey<K>> keys,
|
||||
KVSerializationFunction<K, @Nullable Buf, @Nullable Buf> updateFunction);
|
||||
|
||||
Mono<Void> clear();
|
||||
Stream<LLEntry> getRange(@Nullable LLSnapshot snapshot,
|
||||
LLRange range,
|
||||
boolean reverse,
|
||||
boolean smallRange);
|
||||
|
||||
Mono<Send<Buffer>> remove(Mono<Send<Buffer>> key, LLDictionaryResultType resultType);
|
||||
|
||||
Flux<Optional<Buffer>> getMulti(@Nullable LLSnapshot snapshot,
|
||||
Flux<Send<Buffer>> keys,
|
||||
boolean existsAlmostCertainly);
|
||||
|
||||
default Flux<Optional<Buffer>> getMulti(@Nullable LLSnapshot snapshot,
|
||||
Flux<Send<Buffer>> keys) {
|
||||
return getMulti(snapshot, keys, false);
|
||||
}
|
||||
|
||||
Flux<Send<LLEntry>> putMulti(Flux<Send<LLEntry>> entries, boolean getOldValues);
|
||||
|
||||
<K> Flux<Boolean> updateMulti(Flux<K> keys, Flux<Send<Buffer>> serializedKeys,
|
||||
KVSerializationFunction<K, @Nullable Send<Buffer>, @Nullable Buffer> updateFunction);
|
||||
|
||||
Flux<Send<LLEntry>> getRange(@Nullable LLSnapshot snapshot, Mono<Send<LLRange>> range, boolean existsAlmostCertainly);
|
||||
|
||||
default Flux<Send<LLEntry>> getRange(@Nullable LLSnapshot snapshot, Mono<Send<LLRange>> range) {
|
||||
return getRange(snapshot, range, false);
|
||||
}
|
||||
|
||||
Flux<List<Send<LLEntry>>> getRangeGrouped(@Nullable LLSnapshot snapshot,
|
||||
Mono<Send<LLRange>> range,
|
||||
Stream<List<LLEntry>> getRangeGrouped(@Nullable LLSnapshot snapshot,
|
||||
LLRange range,
|
||||
int prefixLength,
|
||||
boolean existsAlmostCertainly);
|
||||
boolean smallRange);
|
||||
|
||||
default Flux<List<Send<LLEntry>>> getRangeGrouped(@Nullable LLSnapshot snapshot,
|
||||
Mono<Send<LLRange>> range,
|
||||
int prefixLength) {
|
||||
return getRangeGrouped(snapshot, range, prefixLength, false);
|
||||
}
|
||||
Stream<Buf> getRangeKeys(@Nullable LLSnapshot snapshot,
|
||||
LLRange range,
|
||||
boolean reverse,
|
||||
boolean smallRange);
|
||||
|
||||
Flux<Send<Buffer>> getRangeKeys(@Nullable LLSnapshot snapshot, Mono<Send<LLRange>> range);
|
||||
Stream<List<Buf>> getRangeKeysGrouped(@Nullable LLSnapshot snapshot,
|
||||
LLRange range,
|
||||
int prefixLength,
|
||||
boolean smallRange);
|
||||
|
||||
Flux<List<Send<Buffer>>> getRangeKeysGrouped(@Nullable LLSnapshot snapshot, Mono<Send<LLRange>> range, int prefixLength);
|
||||
Stream<Buf> getRangeKeyPrefixes(@Nullable LLSnapshot snapshot,
|
||||
LLRange range,
|
||||
int prefixLength,
|
||||
boolean smallRange);
|
||||
|
||||
Flux<Send<Buffer>> getRangeKeyPrefixes(@Nullable LLSnapshot snapshot, Mono<Send<LLRange>> range, int prefixLength);
|
||||
Stream<DbProgress<SSTVerificationProgress>> verifyChecksum(LLRange range);
|
||||
|
||||
Flux<BadBlock> badBlocks(Mono<Send<LLRange>> range);
|
||||
void setRange(LLRange range, Stream<LLEntry> entries, boolean smallRange);
|
||||
|
||||
Mono<Void> setRange(Mono<Send<LLRange>> range, Flux<Send<LLEntry>> entries);
|
||||
|
||||
default Mono<Void> replaceRange(Mono<Send<LLRange>> range,
|
||||
default void replaceRange(LLRange range,
|
||||
boolean canKeysChange,
|
||||
Function<Send<LLEntry>, Mono<Send<LLEntry>>> entriesReplacer,
|
||||
boolean existsAlmostCertainly) {
|
||||
return Mono.defer(() -> {
|
||||
if (canKeysChange) {
|
||||
return this
|
||||
.setRange(range, this
|
||||
.getRange(null, range, existsAlmostCertainly)
|
||||
.flatMap(entriesReplacer)
|
||||
);
|
||||
} else {
|
||||
return this
|
||||
.putMulti(this
|
||||
.getRange(null, range, existsAlmostCertainly)
|
||||
.flatMap(entriesReplacer), false)
|
||||
.then();
|
||||
}
|
||||
});
|
||||
Function<@NotNull LLEntry, @NotNull LLEntry> entriesReplacer,
|
||||
boolean smallRange) {
|
||||
if (canKeysChange) {
|
||||
this.setRange(range, this.getRange(null, range, false, smallRange).map(entriesReplacer), smallRange);
|
||||
} else {
|
||||
this.putMulti(this.getRange(null, range, false, smallRange).map(entriesReplacer));
|
||||
}
|
||||
}
|
||||
|
||||
default Mono<Void> replaceRange(Mono<Send<LLRange>> range,
|
||||
boolean canKeysChange,
|
||||
Function<Send<LLEntry>, Mono<Send<LLEntry>>> entriesReplacer) {
|
||||
return replaceRange(range, canKeysChange, entriesReplacer, false);
|
||||
}
|
||||
boolean isRangeEmpty(@Nullable LLSnapshot snapshot, LLRange range, boolean fillCache);
|
||||
|
||||
Mono<Boolean> isRangeEmpty(@Nullable LLSnapshot snapshot, Mono<Send<LLRange>> range);
|
||||
long sizeRange(@Nullable LLSnapshot snapshot, LLRange range, boolean fast);
|
||||
|
||||
Mono<Long> sizeRange(@Nullable LLSnapshot snapshot, Mono<Send<LLRange>> range, boolean fast);
|
||||
LLEntry getOne(@Nullable LLSnapshot snapshot, LLRange range);
|
||||
|
||||
Mono<Send<LLEntry>> getOne(@Nullable LLSnapshot snapshot, Mono<Send<LLRange>> range);
|
||||
Buf getOneKey(@Nullable LLSnapshot snapshot, LLRange range);
|
||||
|
||||
Mono<Send<Buffer>> getOneKey(@Nullable LLSnapshot snapshot, Mono<Send<LLRange>> range);
|
||||
|
||||
Mono<Send<LLEntry>> removeOne(Mono<Send<LLRange>> range);
|
||||
LLEntry removeOne(LLRange range);
|
||||
}
|
||||
|
@ -1,120 +1,37 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import io.net5.buffer.api.Buffer;
|
||||
import io.net5.buffer.api.Drop;
|
||||
import io.net5.buffer.api.Owned;
|
||||
import io.net5.buffer.api.Resource;
|
||||
import io.net5.buffer.api.Send;
|
||||
import io.net5.buffer.api.internal.ResourceSupport;
|
||||
import it.cavallium.buffer.Buf;
|
||||
import java.util.Objects;
|
||||
import java.util.StringJoiner;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import org.warp.commonutils.log.Logger;
|
||||
import org.warp.commonutils.log.LoggerFactory;
|
||||
|
||||
public class LLEntry extends ResourceSupport<LLEntry, LLEntry> {
|
||||
public class LLEntry {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(LLEntry.class);
|
||||
private static final Logger logger = LogManager.getLogger(LLEntry.class);
|
||||
private final Buf key;
|
||||
private final Buf value;
|
||||
|
||||
private static final Drop<LLEntry> DROP = new Drop<>() {
|
||||
@Override
|
||||
public void drop(LLEntry obj) {
|
||||
try {
|
||||
if (obj.key != null) {
|
||||
obj.key.close();
|
||||
}
|
||||
} catch (Throwable ex) {
|
||||
logger.error("Failed to close key", ex);
|
||||
}
|
||||
try {
|
||||
if (obj.value != null) {
|
||||
obj.value.close();
|
||||
}
|
||||
} catch (Throwable ex) {
|
||||
logger.error("Failed to close value", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Drop<LLEntry> fork() {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void attach(LLEntry obj) {
|
||||
|
||||
}
|
||||
};
|
||||
|
||||
@Nullable
|
||||
private Buffer key;
|
||||
@Nullable
|
||||
private Buffer value;
|
||||
|
||||
private LLEntry(@NotNull Send<Buffer> key, @NotNull Send<Buffer> value) {
|
||||
super(DROP);
|
||||
this.key = key.receive().makeReadOnly();
|
||||
this.value = value.receive().makeReadOnly();
|
||||
assert isAllAccessible();
|
||||
}
|
||||
private LLEntry(@NotNull Buffer key, @NotNull Buffer value) {
|
||||
super(DROP);
|
||||
this.key = key.makeReadOnly();
|
||||
this.value = value.makeReadOnly();
|
||||
assert isAllAccessible();
|
||||
private LLEntry(@NotNull Buf key, @NotNull Buf value) {
|
||||
this.key = key;
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
private boolean isAllAccessible() {
|
||||
assert key != null && key.isAccessible();
|
||||
assert value != null && value.isAccessible();
|
||||
assert this.isAccessible();
|
||||
assert this.isOwned();
|
||||
return true;
|
||||
}
|
||||
|
||||
public static LLEntry of(@NotNull Send<Buffer> key, @NotNull Send<Buffer> value) {
|
||||
public static LLEntry of(@NotNull Buf key, @NotNull Buf value) {
|
||||
return new LLEntry(key, value);
|
||||
}
|
||||
|
||||
public static LLEntry of(@NotNull Buffer key, @NotNull Buffer value) {
|
||||
return new LLEntry(key, value);
|
||||
public static LLEntry copyOf(Buf keyView, Buf valueView) {
|
||||
return new LLEntry(keyView.copy(), valueView.copy());
|
||||
}
|
||||
|
||||
public Send<Buffer> getKey() {
|
||||
ensureOwned();
|
||||
return Objects.requireNonNull(key).copy().send();
|
||||
public Buf getKey() {
|
||||
return Objects.requireNonNull(key);
|
||||
}
|
||||
|
||||
public Buffer getKeyUnsafe() {
|
||||
return key;
|
||||
}
|
||||
|
||||
public Send<Buffer> getValue() {
|
||||
ensureOwned();
|
||||
return Objects.requireNonNull(value).copy().send();
|
||||
}
|
||||
|
||||
|
||||
public Buffer getValueUnsafe() {
|
||||
return value;
|
||||
}
|
||||
|
||||
private void ensureOwned() {
|
||||
assert isAllAccessible();
|
||||
if (!isOwned()) {
|
||||
if (!isAccessible()) {
|
||||
throw this.createResourceClosedException();
|
||||
} else {
|
||||
throw new IllegalStateException("Resource not owned");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void makeInaccessible() {
|
||||
this.key = null;
|
||||
this.value = null;
|
||||
public Buf getValue() {
|
||||
return Objects.requireNonNull(value);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -143,22 +60,4 @@ public class LLEntry extends ResourceSupport<LLEntry, LLEntry> {
|
||||
.add("value=" + LLUtils.toString(value))
|
||||
.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected RuntimeException createResourceClosedException() {
|
||||
return new IllegalStateException("Closed");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Owned<LLEntry> prepareSend() {
|
||||
Send<Buffer> keySend;
|
||||
Send<Buffer> valueSend;
|
||||
keySend = Objects.requireNonNull(this.key).send();
|
||||
valueSend = Objects.requireNonNull(this.value).send();
|
||||
return drop -> {
|
||||
var instance = new LLEntry(keySend, valueSend);
|
||||
drop.attach(instance);
|
||||
return instance;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
@ -1,3 +0,0 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
public sealed interface LLIndexRequest permits LLSoftUpdateDocument, LLUpdateDocument, LLUpdateFields {}
|
@ -1,127 +0,0 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import com.google.common.primitives.Ints;
|
||||
import com.google.common.primitives.Longs;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
import java.util.Objects;
|
||||
import java.util.StringJoiner;
|
||||
import org.apache.lucene.document.Field;
|
||||
|
||||
public class LLItem {
|
||||
|
||||
private final LLType type;
|
||||
private final String name;
|
||||
private final byte[] data;
|
||||
|
||||
public LLItem(LLType type, String name, byte[] data) {
|
||||
this.type = type;
|
||||
this.name = name;
|
||||
this.data = data;
|
||||
}
|
||||
|
||||
private LLItem(LLType type, String name, String data) {
|
||||
this.type = type;
|
||||
this.name = name;
|
||||
this.data = data.getBytes(StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
private LLItem(LLType type, String name, int data) {
|
||||
this.type = type;
|
||||
this.name = name;
|
||||
this.data = Ints.toByteArray(data);
|
||||
}
|
||||
|
||||
private LLItem(LLType type, String name, float data) {
|
||||
this.type = type;
|
||||
this.name = name;
|
||||
this.data = ByteBuffer.allocate(4).putFloat(data).array();
|
||||
}
|
||||
|
||||
private LLItem(LLType type, String name, long data) {
|
||||
this.type = type;
|
||||
this.name = name;
|
||||
this.data = Longs.toByteArray(data);
|
||||
}
|
||||
|
||||
public static LLItem newIntPoint(String name, int data) {
|
||||
return new LLItem(LLType.IntPoint, name, data);
|
||||
}
|
||||
|
||||
public static LLItem newLongPoint(String name, long data) {
|
||||
return new LLItem(LLType.LongPoint, name, data);
|
||||
}
|
||||
|
||||
public static LLItem newFloatPoint(String name, float data) {
|
||||
return new LLItem(LLType.FloatPoint, name, data);
|
||||
}
|
||||
|
||||
public static LLItem newTextField(String name, String data, Field.Store store) {
|
||||
if (store == Field.Store.YES) {
|
||||
return new LLItem(LLType.TextFieldStored, name, data);
|
||||
} else {
|
||||
return new LLItem(LLType.TextField, name, data);
|
||||
}
|
||||
}
|
||||
|
||||
public static LLItem newStringField(String name, String data, Field.Store store) {
|
||||
if (store == Field.Store.YES) {
|
||||
return new LLItem(LLType.StringFieldStored, name, data);
|
||||
} else {
|
||||
return new LLItem(LLType.StringField, name, data);
|
||||
}
|
||||
}
|
||||
|
||||
public static LLItem newSortedNumericDocValuesField(String name, long data) {
|
||||
return new LLItem(LLType.SortedNumericDocValuesField, name, data);
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public LLType getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
public byte[] getData() {
|
||||
return data;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
LLItem llItem = (LLItem) o;
|
||||
return type == llItem.type &&
|
||||
Objects.equals(name, llItem.name) &&
|
||||
Arrays.equals(data, llItem.data);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
int result = Objects.hash(type, name);
|
||||
result = 31 * result + Arrays.hashCode(data);
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
var sj = new StringJoiner(", ", "[", "]")
|
||||
.add("type=" + type)
|
||||
.add("name='" + name + "'");
|
||||
if (data != null && data.length > 0) {
|
||||
sj.add("data=" + new String(data));
|
||||
}
|
||||
return sj.toString();
|
||||
}
|
||||
|
||||
public String stringValue() {
|
||||
return new String(data, StandardCharsets.UTF_8);
|
||||
}
|
||||
}
|
@ -1,8 +0,0 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import java.util.Objects;
|
||||
import java.util.StringJoiner;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
public record LLKeyScore(int docId, float score, @Nullable String key) {}
|
@ -3,52 +3,67 @@ package it.cavallium.dbengine.database;
|
||||
import com.google.common.primitives.Ints;
|
||||
import com.google.common.primitives.Longs;
|
||||
import io.micrometer.core.instrument.MeterRegistry;
|
||||
import io.net5.buffer.api.BufferAllocator;
|
||||
import it.cavallium.dbengine.client.IBackuppable;
|
||||
import it.cavallium.dbengine.database.collections.DatabaseInt;
|
||||
import it.cavallium.dbengine.database.collections.DatabaseLong;
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import reactor.core.publisher.Mono;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import org.rocksdb.RocksDBException;
|
||||
|
||||
public interface LLKeyValueDatabase extends LLSnapshottable, LLKeyValueDatabaseStructure {
|
||||
public interface LLKeyValueDatabase extends LLSnapshottable, LLKeyValueDatabaseStructure, DatabaseProperties,
|
||||
IBackuppable, DatabaseOperations, Closeable {
|
||||
|
||||
Mono<? extends LLSingleton> getSingleton(byte[] singletonListColumnName, byte[] name, byte[] defaultValue);
|
||||
LLSingleton getSingleton(byte[] singletonListColumnName, byte[] name, byte @Nullable [] defaultValue);
|
||||
|
||||
Mono<? extends LLDictionary> getDictionary(byte[] columnName, UpdateMode updateMode);
|
||||
LLDictionary getDictionary(byte[] columnName, UpdateMode updateMode);
|
||||
|
||||
@Deprecated
|
||||
default Mono<? extends LLDictionary> getDeprecatedSet(String name, UpdateMode updateMode) {
|
||||
return getDictionary(Column.deprecatedSet(name).name().getBytes(StandardCharsets.US_ASCII), updateMode);
|
||||
default LLDictionary getDeprecatedSet(String name, UpdateMode updateMode) {
|
||||
return getDictionary(ColumnUtils.deprecatedSet(name).name().getBytes(StandardCharsets.US_ASCII), updateMode);
|
||||
}
|
||||
|
||||
default Mono<? extends LLDictionary> getDictionary(String name, UpdateMode updateMode) {
|
||||
return getDictionary(Column.dictionary(name).name().getBytes(StandardCharsets.US_ASCII), updateMode);
|
||||
default LLDictionary getDictionary(String name, UpdateMode updateMode) {
|
||||
return getDictionary(ColumnUtils.dictionary(name).name().getBytes(StandardCharsets.US_ASCII), updateMode);
|
||||
}
|
||||
|
||||
default Mono<DatabaseInt> getInteger(String singletonListName, String name, int defaultValue) {
|
||||
return this
|
||||
.getSingleton(Column.special(singletonListName).name().getBytes(StandardCharsets.US_ASCII),
|
||||
name.getBytes(StandardCharsets.US_ASCII),
|
||||
Ints.toByteArray(defaultValue)
|
||||
)
|
||||
.map(DatabaseInt::new);
|
||||
default LLSingleton getSingleton(String singletonListName, String name) {
|
||||
return getSingleton(ColumnUtils.special(singletonListName).name().getBytes(StandardCharsets.US_ASCII),
|
||||
name.getBytes(StandardCharsets.US_ASCII),
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
default Mono<DatabaseLong> getLong(String singletonListName, String name, long defaultValue) {
|
||||
return this
|
||||
.getSingleton(Column.special(singletonListName).name().getBytes(StandardCharsets.US_ASCII),
|
||||
name.getBytes(StandardCharsets.US_ASCII),
|
||||
Longs.toByteArray(defaultValue)
|
||||
)
|
||||
.map(DatabaseLong::new);
|
||||
default DatabaseInt getInteger(String singletonListName, String name, int defaultValue) {
|
||||
return new DatabaseInt(this.getSingleton(ColumnUtils
|
||||
.special(singletonListName)
|
||||
.name()
|
||||
.getBytes(StandardCharsets.US_ASCII),
|
||||
name.getBytes(StandardCharsets.US_ASCII),
|
||||
Ints.toByteArray(defaultValue)
|
||||
));
|
||||
}
|
||||
|
||||
Mono<Long> getProperty(String propertyName);
|
||||
default DatabaseLong getLong(String singletonListName, String name, long defaultValue) {
|
||||
return new DatabaseLong(this.getSingleton(ColumnUtils
|
||||
.special(singletonListName)
|
||||
.name()
|
||||
.getBytes(StandardCharsets.US_ASCII),
|
||||
name.getBytes(StandardCharsets.US_ASCII),
|
||||
Longs.toByteArray(defaultValue)
|
||||
));
|
||||
}
|
||||
|
||||
Mono<Void> verifyChecksum();
|
||||
void verifyChecksum();
|
||||
|
||||
BufferAllocator getAllocator();
|
||||
void compact();
|
||||
|
||||
void flush();
|
||||
|
||||
MeterRegistry getMeterRegistry();
|
||||
|
||||
Mono<Void> close();
|
||||
void preClose();
|
||||
|
||||
void close();
|
||||
}
|
||||
|
@ -1,6 +1,12 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import java.util.concurrent.ForkJoinPool;
|
||||
|
||||
public interface LLKeyValueDatabaseStructure {
|
||||
|
||||
String getDatabaseName();
|
||||
|
||||
ForkJoinPool getDbReadPool();
|
||||
|
||||
ForkJoinPool getDbWritePool();
|
||||
}
|
||||
|
@ -1,83 +0,0 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import io.net5.buffer.api.Resource;
|
||||
import io.net5.buffer.api.Send;
|
||||
import it.cavallium.data.generator.nativedata.Nullablefloat;
|
||||
import it.cavallium.dbengine.client.query.current.data.NoSort;
|
||||
import it.cavallium.dbengine.client.query.current.data.Query;
|
||||
import it.cavallium.dbengine.client.query.current.data.QueryParams;
|
||||
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Set;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.util.function.Tuple2;
|
||||
|
||||
public interface LLLuceneIndex extends LLSnapshottable {
|
||||
|
||||
String getLuceneIndexName();
|
||||
|
||||
Mono<Void> addDocument(LLTerm id, LLUpdateDocument doc);
|
||||
|
||||
/**
|
||||
* WARNING! This operation is atomic!
|
||||
* Please don't send infinite or huge documents fluxes, because they will
|
||||
* be kept in ram all at once.
|
||||
*/
|
||||
Mono<Void> addDocuments(Flux<Entry<LLTerm, LLUpdateDocument>> documents);
|
||||
|
||||
Mono<Void> deleteDocument(LLTerm id);
|
||||
|
||||
Mono<Void> update(LLTerm id, LLIndexRequest request);
|
||||
|
||||
Mono<Void> updateDocuments(Mono<Map<LLTerm, LLUpdateDocument>> documents);
|
||||
|
||||
Mono<Void> deleteAll();
|
||||
|
||||
/**
|
||||
* @param queryParams the limit is valid for each lucene instance. If you have 15 instances, the number of elements
|
||||
* returned can be at most <code>limit * 15</code>.
|
||||
* <p>
|
||||
* The additional query will be used with the moreLikeThis query: "mltQuery AND additionalQuery"
|
||||
* @return the collection has one or more flux
|
||||
*/
|
||||
Mono<LLSearchResultShard> moreLikeThis(@Nullable LLSnapshot snapshot,
|
||||
QueryParams queryParams,
|
||||
String keyFieldName,
|
||||
Flux<Tuple2<String, Set<String>>> mltDocumentFields);
|
||||
|
||||
/**
|
||||
* @param queryParams the limit is valid for each lucene instance. If you have 15 instances, the number of elements
|
||||
* returned can be at most <code>limit * 15</code>
|
||||
* @return the collection has one or more flux
|
||||
*/
|
||||
Mono<LLSearchResultShard> search(@Nullable LLSnapshot snapshot, QueryParams queryParams, String keyFieldName);
|
||||
|
||||
default Mono<TotalHitsCount> count(@Nullable LLSnapshot snapshot, Query query) {
|
||||
QueryParams params = QueryParams.of(query, 0, 0, Nullablefloat.empty(), NoSort.of(), false);
|
||||
return Mono.from(this.search(snapshot, params, null)
|
||||
.map(llSearchResultShard -> {
|
||||
try (llSearchResultShard) {
|
||||
return llSearchResultShard.totalHitsCount();
|
||||
}
|
||||
})
|
||||
.defaultIfEmpty(TotalHitsCount.of(0, true))
|
||||
).doOnDiscard(Send.class, Send::close).doOnDiscard(Resource.class, Resource::close);
|
||||
}
|
||||
|
||||
boolean isLowMemoryMode();
|
||||
|
||||
Mono<Void> close();
|
||||
|
||||
/**
|
||||
* Flush writes to disk
|
||||
*/
|
||||
Mono<Void> flush();
|
||||
|
||||
/**
|
||||
* Refresh index searcher
|
||||
*/
|
||||
Mono<Void> refresh(boolean force);
|
||||
}
|
@ -0,0 +1,104 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import static it.cavallium.dbengine.utils.StreamUtils.collect;
|
||||
import static it.cavallium.dbengine.utils.StreamUtils.executing;
|
||||
|
||||
import com.google.common.collect.Multimap;
|
||||
import io.micrometer.core.instrument.MeterRegistry;
|
||||
import it.cavallium.dbengine.client.ConnectionSettings.ConnectionPart;
|
||||
import it.cavallium.dbengine.client.ConnectionSettings.ConnectionPart.ConnectionPartRocksDB;
|
||||
import it.cavallium.dbengine.rpc.current.data.Column;
|
||||
import it.cavallium.dbengine.rpc.current.data.DatabaseOptions;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.StringJoiner;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
public class LLMultiDatabaseConnection implements LLDatabaseConnection {
|
||||
|
||||
private static final Logger LOG = LogManager.getLogger(LLMultiDatabaseConnection.class);
|
||||
private final Map<String, LLDatabaseConnection> databaseShardConnections = new HashMap<>();
|
||||
private final Set<LLDatabaseConnection> allConnections = new HashSet<>();
|
||||
private final LLDatabaseConnection defaultDatabaseConnection;
|
||||
private final LLDatabaseConnection anyConnection;
|
||||
|
||||
public LLMultiDatabaseConnection(Multimap<LLDatabaseConnection, ConnectionPart> subConnections) {
|
||||
LLDatabaseConnection defaultDatabaseConnection = null;
|
||||
for (Entry<LLDatabaseConnection, ConnectionPart> entry : subConnections.entries()) {
|
||||
var subConnectionSettings = entry.getKey();
|
||||
var connectionPart = entry.getValue();
|
||||
if (connectionPart instanceof ConnectionPartRocksDB connectionPartRocksDB) {
|
||||
if (connectionPartRocksDB.name() == null) {
|
||||
defaultDatabaseConnection = subConnectionSettings;
|
||||
} else {
|
||||
databaseShardConnections.put(connectionPartRocksDB.name(), subConnectionSettings);
|
||||
}
|
||||
} else {
|
||||
throw new IllegalArgumentException("Unsupported connection part: " + connectionPart);
|
||||
}
|
||||
}
|
||||
this.defaultDatabaseConnection = defaultDatabaseConnection;
|
||||
if (defaultDatabaseConnection != null) {
|
||||
anyConnection = defaultDatabaseConnection;
|
||||
} else {
|
||||
anyConnection = subConnections.keySet().stream().findAny().orElse(null);
|
||||
}
|
||||
if (defaultDatabaseConnection != null) {
|
||||
allConnections.add(defaultDatabaseConnection);
|
||||
}
|
||||
allConnections.addAll(databaseShardConnections.values());
|
||||
}
|
||||
|
||||
@Override
|
||||
public MeterRegistry getMeterRegistry() {
|
||||
return anyConnection.getMeterRegistry();
|
||||
}
|
||||
|
||||
@Override
|
||||
public LLDatabaseConnection connect() {
|
||||
collect(allConnections.stream(), executing(connection -> {
|
||||
try {
|
||||
connection.connect();
|
||||
} catch (Exception ex) {
|
||||
LOG.error("Failed to open connection", ex);
|
||||
}
|
||||
}));
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public LLKeyValueDatabase getDatabase(String name,
|
||||
List<Column> columns,
|
||||
DatabaseOptions databaseOptions) {
|
||||
var conn = databaseShardConnections.getOrDefault(name, defaultDatabaseConnection);
|
||||
Objects.requireNonNull(conn, "Null connection");
|
||||
return conn.getDatabase(name, columns, databaseOptions);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void disconnect() {
|
||||
collect(allConnections.stream(), executing(connection -> {
|
||||
try {
|
||||
connection.disconnect();
|
||||
} catch (Exception ex) {
|
||||
LOG.error("Failed to close connection", ex);
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return new StringJoiner(", ", LLMultiDatabaseConnection.class.getSimpleName() + "[", "]")
|
||||
.add("databaseShardConnections=" + databaseShardConnections)
|
||||
.add("allConnections=" + allConnections)
|
||||
.add("defaultDatabaseConnection=" + defaultDatabaseConnection)
|
||||
.add("anyConnection=" + anyConnection)
|
||||
.toString();
|
||||
}
|
||||
}
|
@ -1,205 +1,145 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import static io.net5.buffer.Unpooled.wrappedBuffer;
|
||||
|
||||
import io.net5.buffer.api.Buffer;
|
||||
import io.net5.buffer.api.Drop;
|
||||
import io.net5.buffer.api.Owned;
|
||||
import io.net5.buffer.api.Send;
|
||||
import io.net5.buffer.api.internal.ResourceSupport;
|
||||
import java.util.StringJoiner;
|
||||
import it.cavallium.buffer.Buf;
|
||||
import java.util.Objects;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import org.warp.commonutils.log.Logger;
|
||||
import org.warp.commonutils.log.LoggerFactory;
|
||||
|
||||
/**
|
||||
* Range of data, from min (inclusive),to max (exclusive)
|
||||
* Range of data, from min (inclusive), to max (exclusive)
|
||||
*/
|
||||
public class LLRange extends ResourceSupport<LLRange, LLRange> {
|
||||
public class LLRange {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(LLRange.class);
|
||||
|
||||
private static final Drop<LLRange> DROP = new Drop<>() {
|
||||
@Override
|
||||
public void drop(LLRange obj) {
|
||||
try {
|
||||
if (obj.min != null) {
|
||||
obj.min.close();
|
||||
}
|
||||
} catch (Throwable ex) {
|
||||
logger.error("Failed to close min", ex);
|
||||
}
|
||||
try {
|
||||
if (obj.max != null) {
|
||||
obj.max.close();
|
||||
}
|
||||
} catch (Throwable ex) {
|
||||
logger.error("Failed to close max", ex);
|
||||
}
|
||||
try {
|
||||
if (obj.single != null) {
|
||||
obj.single.close();
|
||||
}
|
||||
} catch (Throwable ex) {
|
||||
logger.error("Failed to close single", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Drop<LLRange> fork() {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void attach(LLRange obj) {
|
||||
|
||||
}
|
||||
};
|
||||
|
||||
private static final LLRange RANGE_ALL = new LLRange((Buffer) null, (Buffer) null, (Buffer) null);
|
||||
private static final LLRange RANGE_ALL = new LLRange( null, null, (Buf) null);
|
||||
@Nullable
|
||||
private Buffer min;
|
||||
private final Buf min;
|
||||
@Nullable
|
||||
private Buffer max;
|
||||
private final Buf max;
|
||||
@Nullable
|
||||
private Buffer single;
|
||||
private final Buf single;
|
||||
|
||||
private LLRange(Send<Buffer> min, Send<Buffer> max, Send<Buffer> single) {
|
||||
super(DROP);
|
||||
assert isAllAccessible();
|
||||
private LLRange(@Nullable Buf min, @Nullable Buf max, @Nullable Buf single) {
|
||||
assert single == null || (min == null && max == null);
|
||||
this.min = min != null ? min.receive().makeReadOnly() : null;
|
||||
this.max = max != null ? max.receive().makeReadOnly() : null;
|
||||
this.single = single != null ? single.receive().makeReadOnly() : null;
|
||||
}
|
||||
|
||||
private LLRange(Buffer min, Buffer max, Buffer single) {
|
||||
super(DROP);
|
||||
assert isAllAccessible();
|
||||
assert single == null || (min == null && max == null);
|
||||
this.min = min != null ? min.makeReadOnly() : null;
|
||||
this.max = max != null ? max.makeReadOnly() : null;
|
||||
this.single = single != null ? single.makeReadOnly() : null;
|
||||
}
|
||||
|
||||
private boolean isAllAccessible() {
|
||||
assert min == null || min.isAccessible();
|
||||
assert max == null || max.isAccessible();
|
||||
assert single == null || single.isAccessible();
|
||||
assert this.isAccessible();
|
||||
assert this.isOwned();
|
||||
return true;
|
||||
assert min == null || max == null || min.compareTo(max) <= 0
|
||||
: "Minimum buffer is bigger than maximum buffer: " + min + " > " + max;
|
||||
this.min = min;
|
||||
this.max = max;
|
||||
this.single = single;
|
||||
}
|
||||
|
||||
public static LLRange all() {
|
||||
return RANGE_ALL.copy();
|
||||
return RANGE_ALL;
|
||||
}
|
||||
|
||||
public static LLRange from(Send<Buffer> min) {
|
||||
public static LLRange from(Buf min) {
|
||||
return new LLRange(min, null, null);
|
||||
}
|
||||
|
||||
public static LLRange to(Send<Buffer> max) {
|
||||
public static LLRange to(Buf max) {
|
||||
return new LLRange(null, max, null);
|
||||
}
|
||||
|
||||
public static LLRange single(Send<Buffer> single) {
|
||||
public static LLRange single(Buf single) {
|
||||
return new LLRange(null, null, single);
|
||||
}
|
||||
|
||||
public static LLRange of(Send<Buffer> min, Send<Buffer> max) {
|
||||
public static LLRange of(Buf min, Buf max) {
|
||||
return new LLRange(min, max, null);
|
||||
}
|
||||
|
||||
public static LLRange ofUnsafe(Buffer min, Buffer max) {
|
||||
return new LLRange(min, max, null);
|
||||
public static boolean isInside(LLRange rangeSub, LLRange rangeParent) {
|
||||
if (rangeParent.isAll()) {
|
||||
return true;
|
||||
} else if (rangeParent.isSingle()) {
|
||||
return Objects.equals(rangeSub, rangeParent);
|
||||
} else {
|
||||
return ((!rangeParent.hasMin() || (rangeSub.hasMin() && rangeParent.getMin().compareTo(rangeSub.getMin()) <= 0)))
|
||||
&& ((!rangeParent.hasMax() || (rangeSub.hasMax() && rangeParent.getMax().compareTo(rangeSub.getMax()) >= 0)));
|
||||
}
|
||||
}
|
||||
|
||||
@Nullable
|
||||
public static LLRange intersect(LLRange rangeA, LLRange rangeB) {
|
||||
boolean aEndInclusive = rangeA.isSingle();
|
||||
boolean bEndInclusive = rangeB.isSingle();
|
||||
Buf min = rangeA.isAll()
|
||||
? rangeB.getMin()
|
||||
: (rangeB.isAll()
|
||||
? rangeA.getMin()
|
||||
: (rangeA.getMin().compareTo(rangeB.getMin()) <= 0 ? rangeB.getMin() : rangeA.getMin()));
|
||||
int aComparedToB;
|
||||
Buf max;
|
||||
boolean maxInclusive;
|
||||
if (rangeA.isAll()) {
|
||||
max = rangeB.getMax();
|
||||
maxInclusive = bEndInclusive;
|
||||
} else if (rangeB.isAll()) {
|
||||
max = rangeA.getMax();
|
||||
maxInclusive = aEndInclusive;
|
||||
} else if ((aComparedToB = rangeA.getMax().compareTo(rangeB.getMax())) >= 0) {
|
||||
max = rangeB.getMax();
|
||||
if (aComparedToB == 0) {
|
||||
maxInclusive = bEndInclusive && aEndInclusive;
|
||||
} else {
|
||||
maxInclusive = bEndInclusive;
|
||||
}
|
||||
} else {
|
||||
max = rangeA.getMax();
|
||||
maxInclusive = aEndInclusive;
|
||||
}
|
||||
if (min != null && max != null && min.compareTo(max) >= (maxInclusive ? 1 : 0)) {
|
||||
return null;
|
||||
} else {
|
||||
if (min != null && min.equals(max)) {
|
||||
return LLRange.single(min);
|
||||
} else {
|
||||
return LLRange.of(min, max);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isAll() {
|
||||
ensureOwned();
|
||||
return min == null && max == null && single == null;
|
||||
}
|
||||
|
||||
public boolean isSingle() {
|
||||
ensureOwned();
|
||||
return single != null;
|
||||
}
|
||||
|
||||
public boolean hasMin() {
|
||||
ensureOwned();
|
||||
return min != null || single != null;
|
||||
}
|
||||
|
||||
public Send<Buffer> getMin() {
|
||||
ensureOwned();
|
||||
if (min != null) {
|
||||
return min.copy().send();
|
||||
} else if (single != null) {
|
||||
return single.copy().send();
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public Buffer getMinUnsafe() {
|
||||
ensureOwned();
|
||||
public Buf getMin() {
|
||||
// todo: use a read-only copy
|
||||
if (min != null) {
|
||||
return min;
|
||||
} else if (single != null) {
|
||||
return single;
|
||||
} else {
|
||||
return null;
|
||||
return single;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean hasMax() {
|
||||
ensureOwned();
|
||||
return max != null || single != null;
|
||||
}
|
||||
|
||||
public Send<Buffer> getMax() {
|
||||
ensureOwned();
|
||||
if (max != null) {
|
||||
return max.copy().send();
|
||||
} else if (single != null) {
|
||||
return single.copy().send();
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public Buffer getMaxUnsafe() {
|
||||
ensureOwned();
|
||||
public Buf getMax() {
|
||||
// todo: use a read-only copy
|
||||
if (max != null) {
|
||||
return max;
|
||||
} else if (single != null) {
|
||||
return single;
|
||||
} else {
|
||||
return null;
|
||||
return single;
|
||||
}
|
||||
}
|
||||
|
||||
public Send<Buffer> getSingle() {
|
||||
ensureOwned();
|
||||
assert isSingle();
|
||||
return single != null ? single.copy().send() : null;
|
||||
}
|
||||
|
||||
public Buffer getSingleUnsafe() {
|
||||
ensureOwned();
|
||||
public Buf getSingle() {
|
||||
assert isSingle();
|
||||
// todo: use a read-only copy
|
||||
return single;
|
||||
}
|
||||
|
||||
private void ensureOwned() {
|
||||
assert isAllAccessible();
|
||||
if (!isOwned()) {
|
||||
if (!isAccessible()) {
|
||||
throw this.createResourceClosedException();
|
||||
} else {
|
||||
throw new IllegalStateException("Resource not owned");
|
||||
}
|
||||
}
|
||||
public Buf getSingleUnsafe() {
|
||||
assert isSingle();
|
||||
return single;
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -221,45 +161,24 @@ public class LLRange extends ResourceSupport<LLRange, LLRange> {
|
||||
return result;
|
||||
}
|
||||
|
||||
@SuppressWarnings("UnnecessaryUnicodeEscape")
|
||||
@Override
|
||||
public String toString() {
|
||||
return new StringJoiner(", ", LLRange.class.getSimpleName() + "[", "]")
|
||||
.add("min=" + LLUtils.toString(min))
|
||||
.add("max=" + LLUtils.toString(max))
|
||||
.toString();
|
||||
if (single != null) {
|
||||
return "[" + single + "]";
|
||||
} else if (min != null && max != null) {
|
||||
return "[" + LLUtils.toString(min) + "," + LLUtils.toString(max) + ")";
|
||||
} else if (min != null) {
|
||||
return "[" + min + ",\u221E)";
|
||||
} else if (max != null) {
|
||||
return "[\u2205," + max + ")";
|
||||
} else {
|
||||
return "[\u221E)";
|
||||
}
|
||||
}
|
||||
|
||||
public LLRange copy() {
|
||||
ensureOwned();
|
||||
return new LLRange(min != null ? min.copy().send() : null,
|
||||
max != null ? max.copy().send() : null,
|
||||
single != null ? single.copy().send(): null
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected RuntimeException createResourceClosedException() {
|
||||
return new IllegalStateException("Closed");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Owned<LLRange> prepareSend() {
|
||||
Send<Buffer> minSend;
|
||||
Send<Buffer> maxSend;
|
||||
Send<Buffer> singleSend;
|
||||
minSend = this.min != null ? this.min.send() : null;
|
||||
maxSend = this.max != null ? this.max.send() : null;
|
||||
singleSend = this.single != null ? this.single.send() : null;
|
||||
return drop -> {
|
||||
var instance = new LLRange(minSend, maxSend, singleSend);
|
||||
drop.attach(instance);
|
||||
return instance;
|
||||
};
|
||||
}
|
||||
|
||||
protected void makeInaccessible() {
|
||||
this.min = null;
|
||||
this.max = null;
|
||||
this.single = null;
|
||||
// todo: use a read-only copy
|
||||
return new LLRange(min, max, single);
|
||||
}
|
||||
}
|
||||
|
@ -1,7 +1,5 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import org.apache.lucene.search.Scorer;
|
||||
|
||||
public enum LLScoreMode {
|
||||
/**
|
||||
* Produced scorers will allow visiting all matches and get their score.
|
||||
@ -15,7 +13,7 @@ public enum LLScoreMode {
|
||||
COMPLETE_NO_SCORES,
|
||||
/**
|
||||
* Produced scorers will optionally allow skipping over non-competitive
|
||||
* hits using the {@link Scorer#setMinCompetitiveScore(float)} API.
|
||||
* hits using the Scorer#setMinCompetitiveScore(float) API.
|
||||
* This can reduce time if using setMinCompetitiveScore.
|
||||
*/
|
||||
TOP_SCORES,
|
||||
|
@ -1,13 +0,0 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import java.util.function.BiFunction;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
public record LLSearchResult(Flux<LLSearchResultShard> results) {
|
||||
|
||||
@NotNull
|
||||
public static BiFunction<LLSearchResult, LLSearchResult, LLSearchResult> accumulator() {
|
||||
return (a, b) -> new LLSearchResult(Flux.merge(a.results, b.results));
|
||||
}
|
||||
}
|
@ -1,102 +0,0 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import io.net5.buffer.api.Drop;
|
||||
import io.net5.buffer.api.Owned;
|
||||
import io.net5.buffer.api.internal.ResourceSupport;
|
||||
import it.cavallium.dbengine.client.query.current.data.TotalHitsCount;
|
||||
import java.util.Objects;
|
||||
import org.warp.commonutils.log.Logger;
|
||||
import org.warp.commonutils.log.LoggerFactory;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
public final class LLSearchResultShard extends ResourceSupport<LLSearchResultShard, LLSearchResultShard> {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(LLSearchResultShard.class);
|
||||
|
||||
private static final Drop<LLSearchResultShard> DROP = new Drop<>() {
|
||||
@Override
|
||||
public void drop(LLSearchResultShard obj) {
|
||||
try {
|
||||
if (obj.onClose != null) {
|
||||
obj.onClose.run();
|
||||
}
|
||||
} catch (Throwable ex) {
|
||||
logger.error("Failed to close onClose", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Drop<LLSearchResultShard> fork() {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void attach(LLSearchResultShard obj) {
|
||||
|
||||
}
|
||||
};
|
||||
|
||||
private Flux<LLKeyScore> results;
|
||||
private TotalHitsCount totalHitsCount;
|
||||
private Runnable onClose;
|
||||
|
||||
public LLSearchResultShard(Flux<LLKeyScore> results, TotalHitsCount totalHitsCount, Runnable onClose) {
|
||||
super(DROP);
|
||||
this.results = results;
|
||||
this.totalHitsCount = totalHitsCount;
|
||||
this.onClose = onClose;
|
||||
}
|
||||
|
||||
public Flux<LLKeyScore> results() {
|
||||
if (!isOwned()) {
|
||||
throw attachTrace(new IllegalStateException("LLSearchResultShard must be owned to be used"));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
public TotalHitsCount totalHitsCount() {
|
||||
if (!isOwned()) {
|
||||
throw attachTrace(new IllegalStateException("LLSearchResultShard must be owned to be used"));
|
||||
}
|
||||
return totalHitsCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (obj == this)
|
||||
return true;
|
||||
if (obj == null || obj.getClass() != this.getClass())
|
||||
return false;
|
||||
var that = (LLSearchResultShard) obj;
|
||||
return Objects.equals(this.results, that.results) && Objects.equals(this.totalHitsCount, that.totalHitsCount);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(results, totalHitsCount);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "LLSearchResultShard[" + "results=" + results + ", " + "totalHitsCount=" + totalHitsCount + ']';
|
||||
}
|
||||
|
||||
@Override
|
||||
protected RuntimeException createResourceClosedException() {
|
||||
return new IllegalStateException("Closed");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Owned<LLSearchResultShard> prepareSend() {
|
||||
var results = this.results;
|
||||
var totalHitsCount = this.totalHitsCount;
|
||||
var onClose = this.onClose;
|
||||
return drop -> new LLSearchResultShard(results, totalHitsCount, onClose);
|
||||
}
|
||||
|
||||
protected void makeInaccessible() {
|
||||
this.results = null;
|
||||
this.totalHitsCount = null;
|
||||
this.onClose = null;
|
||||
}
|
||||
}
|
@ -1,11 +1,24 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import it.cavallium.buffer.Buf;
|
||||
import it.cavallium.dbengine.database.serialization.SerializationFunction;
|
||||
import java.io.IOException;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
public interface LLSingleton extends LLKeyValueDatabaseStructure {
|
||||
|
||||
Mono<byte[]> get(@Nullable LLSnapshot snapshot);
|
||||
Buf get(@Nullable LLSnapshot snapshot);
|
||||
|
||||
Mono<Void> set(byte[] value);
|
||||
void set(Buf value);
|
||||
|
||||
default Buf update(SerializationFunction<@Nullable Buf, @Nullable Buf> updater, UpdateReturnMode updateReturnMode) {
|
||||
var prev = this.updateAndGetDelta(updater);
|
||||
return LLUtils.resolveLLDelta(prev, updateReturnMode);
|
||||
}
|
||||
|
||||
LLDelta updateAndGetDelta(SerializationFunction<@Nullable Buf, @Nullable Buf> updater);
|
||||
|
||||
String getColumnName();
|
||||
|
||||
String getName();
|
||||
}
|
||||
|
@ -1,10 +1,10 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import reactor.core.publisher.Mono;
|
||||
import java.io.IOException;
|
||||
|
||||
public interface LLSnapshottable {
|
||||
|
||||
Mono<LLSnapshot> takeSnapshot();
|
||||
LLSnapshot takeSnapshot();
|
||||
|
||||
Mono<Void> releaseSnapshot(LLSnapshot snapshot);
|
||||
void releaseSnapshot(LLSnapshot snapshot);
|
||||
}
|
||||
|
@ -1,3 +0,0 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
public record LLSoftUpdateDocument(LLItem[] items, LLItem[] softDeleteItems) implements LLIndexRequest {}
|
@ -1,48 +0,0 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public class LLTerm {
|
||||
|
||||
private final String key;
|
||||
private final String value;
|
||||
|
||||
public LLTerm(String key, String value) {
|
||||
this.key = key;
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
public String getKey() {
|
||||
return key;
|
||||
}
|
||||
|
||||
public String getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "LLTerm{" +
|
||||
"key='" + key + '\'' +
|
||||
", value='" + value + '\'' +
|
||||
'}';
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
LLTerm llTerm = (LLTerm) o;
|
||||
return Objects.equals(key, llTerm.key) &&
|
||||
Objects.equals(value, llTerm.value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(key, value);
|
||||
}
|
||||
}
|
@ -1,52 +0,0 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Objects;
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
public class LLTopKeys {
|
||||
|
||||
private final long totalHitsCount;
|
||||
private final LLKeyScore[] hits;
|
||||
|
||||
public LLTopKeys(long totalHitsCount, LLKeyScore[] hits) {
|
||||
this.totalHitsCount = totalHitsCount;
|
||||
this.hits = hits;
|
||||
}
|
||||
|
||||
public long getTotalHitsCount() {
|
||||
return totalHitsCount;
|
||||
}
|
||||
|
||||
public LLKeyScore[] getHits() {
|
||||
return hits;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
LLTopKeys llTopKeys = (LLTopKeys) o;
|
||||
return totalHitsCount == llTopKeys.totalHitsCount &&
|
||||
Arrays.equals(hits, llTopKeys.hits);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
int result = Objects.hash(totalHitsCount);
|
||||
result = 31 * result + Arrays.hashCode(hits);
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "LLTopKeys{" +
|
||||
"totalHitsCount=" + totalHitsCount +
|
||||
", hits=" + Arrays.toString(hits) +
|
||||
'}';
|
||||
}
|
||||
}
|
@ -1,11 +1,22 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
/**
|
||||
* <a href="https://lucene.apache.org/core/8_0_0/core/org/apache/lucene/document/Field.html">Field.html</a>
|
||||
*/
|
||||
public enum LLType {
|
||||
StringField,
|
||||
StringFieldStored,
|
||||
IntPoint,
|
||||
LongPoint,
|
||||
FloatPoint,
|
||||
DoublePoint,
|
||||
IntPointND,
|
||||
LongPointND,
|
||||
FloatPointND,
|
||||
DoublePointND,
|
||||
LongStoredField,
|
||||
BytesStoredField,
|
||||
NumericDocValuesField,
|
||||
SortedNumericDocValuesField,
|
||||
TextField,
|
||||
TextFieldStored
|
||||
|
@ -1,3 +0,0 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
public record LLUpdateDocument(LLItem[] items) implements LLIndexRequest {}
|
@ -1,3 +0,0 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
public record LLUpdateFields(LLItem[] items) implements LLIndexRequest {}
|
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,97 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import it.cavallium.buffer.Buf;
|
||||
import java.util.NoSuchElementException;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Function;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
public final class OptionalBuf {
|
||||
|
||||
private static final OptionalBuf EMPTY = new OptionalBuf(null);
|
||||
private final Buf buffer;
|
||||
|
||||
private OptionalBuf(@Nullable Buf buffer) {
|
||||
this.buffer = buffer;
|
||||
}
|
||||
|
||||
public static OptionalBuf ofNullable(@Nullable Buf buffer) {
|
||||
return new OptionalBuf(buffer);
|
||||
}
|
||||
|
||||
public static OptionalBuf of(@NotNull Buf buffer) {
|
||||
Objects.requireNonNull(buffer);
|
||||
return new OptionalBuf(buffer);
|
||||
}
|
||||
|
||||
public static OptionalBuf empty() {
|
||||
return EMPTY;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
if (buffer != null) {
|
||||
return buffer.toString();
|
||||
} else {
|
||||
return "(empty)";
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
OptionalBuf that = (OptionalBuf) o;
|
||||
|
||||
return Objects.equals(buffer, that.buffer);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return buffer != null ? buffer.hashCode() : 0;
|
||||
}
|
||||
|
||||
public Buf get() {
|
||||
if (buffer == null) {
|
||||
throw new NoSuchElementException();
|
||||
}
|
||||
return buffer;
|
||||
}
|
||||
|
||||
public Buf orElse(Buf alternative) {
|
||||
if (buffer == null) {
|
||||
return alternative;
|
||||
}
|
||||
return buffer;
|
||||
}
|
||||
|
||||
public void ifPresent(Consumer<Buf> consumer) {
|
||||
if (buffer != null) {
|
||||
consumer.accept(buffer);
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isPresent() {
|
||||
return buffer != null;
|
||||
}
|
||||
|
||||
public boolean isEmpty() {
|
||||
return buffer == null;
|
||||
}
|
||||
|
||||
public <U> Optional<U> map(Function<Buf, U> mapper) {
|
||||
if (buffer != null) {
|
||||
return Optional.of(mapper.apply(buffer));
|
||||
} else {
|
||||
return Optional.empty();
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,107 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
public enum RocksDBLongProperty implements RocksDBProperty {
|
||||
NUM_FILES_AT_LEVEL_0("num-files-at-level0"),
|
||||
NUM_FILES_AT_LEVEL_1("num-files-at-level1"),
|
||||
NUM_FILES_AT_LEVEL_2("num-files-at-level2"),
|
||||
NUM_FILES_AT_LEVEL_3("num-files-at-level3"),
|
||||
NUM_FILES_AT_LEVEL_4("num-files-at-level4"),
|
||||
NUM_FILES_AT_LEVEL_5("num-files-at-level5"),
|
||||
NUM_FILES_AT_LEVEL_6("num-files-at-level6"),
|
||||
NUM_FILES_AT_LEVEL_7("num-files-at-level7"),
|
||||
NUM_FILES_AT_LEVEL_8("num-files-at-level8"),
|
||||
NUM_FILES_AT_LEVEL_9("num-files-at-level9"),
|
||||
COMPRESSION_RATIO_AT_LEVEL_0("compression-ratio-at-level0"),
|
||||
COMPRESSION_RATIO_AT_LEVEL_1("compression-ratio-at-level1"),
|
||||
COMPRESSION_RATIO_AT_LEVEL_2("compression-ratio-at-level2"),
|
||||
COMPRESSION_RATIO_AT_LEVEL_3("compression-ratio-at-level3"),
|
||||
COMPRESSION_RATIO_AT_LEVEL_4("compression-ratio-at-level4"),
|
||||
COMPRESSION_RATIO_AT_LEVEL_5("compression-ratio-at-level5"),
|
||||
COMPRESSION_RATIO_AT_LEVEL_6("compression-ratio-at-level6"),
|
||||
COMPRESSION_RATIO_AT_LEVEL_7("compression-ratio-at-level7"),
|
||||
COMPRESSION_RATIO_AT_LEVEL_8("compression-ratio-at-level8"),
|
||||
COMPRESSION_RATIO_AT_LEVEL_9("compression-ratio-at-level9"),
|
||||
NUM_IMMUTABLE_MEM_TABLE("num-immutable-mem-table"),
|
||||
NUM_IMMUTABLE_MEM_TABLE_FLUSHED("num-immutable-mem-table-flushed"),
|
||||
MEM_TABLE_FLUSH_PENDING("mem-table-flush-pending"),
|
||||
NUM_RUNNING_FLUSHES("num-running-flushes"),
|
||||
COMPACTION_PENDING("compaction-pending"),
|
||||
NUM_RUNNING_COMPACTIONS("num-running-compactions"),
|
||||
BACKGROUND_ERRORS("background-errors"),
|
||||
CUR_SIZE_ACTIVE_MEM_TABLE("cur-size-active-mem-table"),
|
||||
CUR_SIZE_ALL_MEM_TABLES("cur-size-all-mem-tables"),
|
||||
SIZE_ALL_MEM_TABLES("size-all-mem-tables"),
|
||||
NUM_ENTRIES_ACTIVE_MEM_TABLE("num-entries-active-mem-table"),
|
||||
NUM_ENTRIES_IMMUTABLE_MEM_TABLES("num-entries-imm-mem-tables"),
|
||||
NUM_DELETES_ACTIVE_MEM_TABLE("num-deletes-active-mem-table"),
|
||||
NUM_DELETES_IMMUTABLE_MEM_TABLES("num-deletes-imm-mem-tables"),
|
||||
ESTIMATE_NUM_KEYS("estimate-num-keys"),
|
||||
ESTIMATE_TABLE_READERS_MEM("estimate-table-readers-mem"),
|
||||
IS_FILE_DELETIONS_ENABLED("is-file-deletions-enabled"),
|
||||
NUM_SNAPSHOTS("num-snapshots"),
|
||||
OLDEST_SNAPSHOT_TIME("oldest-snapshot-time"),
|
||||
OLDEST_SNAPSHOT_SEQUENCE("oldest-snapshot-sequence"),
|
||||
NUM_LIVE_VERSIONS("num-live-versions"),
|
||||
CURRENT_SUPER_VERSION_NUMBER("current-super-version-number"),
|
||||
ESTIMATE_LIVE_DATA_SIZE("estimate-live-data-size"),
|
||||
MIN_LOG_NUMBER_TO_KEEP("min-log-number-to-keep"),
|
||||
MIN_OBSOLETE_SST_NUMBER_TO_KEEP("min-obsolete-sst-number-to-keep"),
|
||||
TOTAL_SST_FILES_SIZE("total-sst-files-size"),
|
||||
LIVE_SST_FILES_SIZE("live-sst-files-size"),
|
||||
LIVE_SST_FILES_SIZE_AT_TEMPERATURE("live-sst-files-size-at-temperature"),
|
||||
BASE_LEVEL("base-level"),
|
||||
ESTIMATE_PENDING_COMPACTION_BYTES("estimate-pending-compaction-bytes"),
|
||||
ACTUAL_DELAYED_WRITE_RATE("actual-delayed-write-rate"),
|
||||
IS_WRITE_STOPPED("is-write-stopped"),
|
||||
ESTIMATE_OLDEST_KEY_TIME("estimate-oldest-key-time"),
|
||||
BLOCK_CACHE_CAPACITY("block-cache-capacity", false),
|
||||
BLOCK_CACHE_USAGE("block-cache-usage", false),
|
||||
BLOCK_CACHE_PINNED_USAGE("block-cache-pinned-usage", false),
|
||||
NUM_BLOB_FILES("num-blob-files"),
|
||||
TOTAL_BLOB_FILE_SIZE("total-blob-file-size"),
|
||||
LIVE_BLOB_FILE_SIZE("live-blob-file-size"),
|
||||
LIVE_BLOB_FILE_GARBAGE_SIZE("live-blob-file-garbage-size"),
|
||||
FILE_READ_DB_OPEN_MICROS("file.read.db.open.micros")
|
||||
;
|
||||
|
||||
private final String name;
|
||||
private final boolean dividedByColumnFamily;
|
||||
|
||||
RocksDBLongProperty(String name) {
|
||||
this(name, true);
|
||||
}
|
||||
|
||||
RocksDBLongProperty(String name, boolean dividedByColumnFamily) {
|
||||
this.name = name;
|
||||
this.dividedByColumnFamily = dividedByColumnFamily;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "rocksdb." + name;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "rocksdb." + name;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isNumeric() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMap() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isString() {
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean isDividedByColumnFamily() {
|
||||
return dividedByColumnFamily;
|
||||
}
|
||||
}
|
@ -0,0 +1,40 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
public enum RocksDBMapProperty implements RocksDBProperty {
|
||||
CFSTATS("cfstats"),
|
||||
DBSTATS("dbstats"),
|
||||
BLOCK_CACHE_ENTRY_STATS("block-cache-entry-stats"),
|
||||
AGGREGATED_TABLE_PROPERTIES("aggregated-table-properties"),
|
||||
;
|
||||
|
||||
private final String name;
|
||||
|
||||
RocksDBMapProperty(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "rocksdb." + name;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "rocksdb." + name;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isNumeric() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMap() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isString() {
|
||||
return false;
|
||||
}
|
||||
}
|
@ -0,0 +1,16 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
public interface RocksDBProperty {
|
||||
|
||||
/**
|
||||
* Get rocksdb property name
|
||||
* @return name, with the "rocksdb." prefix included
|
||||
*/
|
||||
String getName();
|
||||
|
||||
boolean isNumeric();
|
||||
|
||||
boolean isMap();
|
||||
|
||||
boolean isString();
|
||||
}
|
@ -0,0 +1,53 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
public enum RocksDBStringProperty implements RocksDBProperty {
|
||||
STATS("stats"),
|
||||
SSTABLES("sstables"),
|
||||
CFSTATS_NO_FILE_HISTOGRAM("cfstats-no-file-histogram"),
|
||||
CF_FILE_HISTOGRAM("cf-file-histogram"),
|
||||
LEVELSTATS("levelstats"),
|
||||
AGGREGATED_TABLE_PROPERTIES_AT_LEVEL_0("aggregated-table-properties-at-level0"),
|
||||
AGGREGATED_TABLE_PROPERTIES_AT_LEVEL_1("aggregated-table-properties-at-level1"),
|
||||
AGGREGATED_TABLE_PROPERTIES_AT_LEVEL_2("aggregated-table-properties-at-level2"),
|
||||
AGGREGATED_TABLE_PROPERTIES_AT_LEVEL_3("aggregated-table-properties-at-level3"),
|
||||
AGGREGATED_TABLE_PROPERTIES_AT_LEVEL_4("aggregated-table-properties-at-level4"),
|
||||
AGGREGATED_TABLE_PROPERTIES_AT_LEVEL_5("aggregated-table-properties-at-level5"),
|
||||
AGGREGATED_TABLE_PROPERTIES_AT_LEVEL_6("aggregated-table-properties-at-level6"),
|
||||
AGGREGATED_TABLE_PROPERTIES_AT_LEVEL_7("aggregated-table-properties-at-level7"),
|
||||
AGGREGATED_TABLE_PROPERTIES_AT_LEVEL_8("aggregated-table-properties-at-level8"),
|
||||
AGGREGATED_TABLE_PROPERTIES_AT_LEVEL_9("aggregated-table-properties-at-level9"),
|
||||
OPTIONS_STATISTICS("options-statistics"),
|
||||
BLOB_STATS("blob-stats")
|
||||
;
|
||||
|
||||
private final String name;
|
||||
|
||||
RocksDBStringProperty(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "rocksdb." + name;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "rocksdb." + name;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isNumeric() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMap() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isString() {
|
||||
return true;
|
||||
}
|
||||
}
|
@ -2,6 +2,5 @@ package it.cavallium.dbengine.database;
|
||||
|
||||
public interface SafeCloseable extends AutoCloseable {
|
||||
|
||||
@Override
|
||||
void close();
|
||||
}
|
||||
|
@ -0,0 +1,5 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import it.cavallium.buffer.Buf;
|
||||
|
||||
public record SerializedKey<T>(T key, Buf serialized) {}
|
@ -0,0 +1,53 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import it.cavallium.dbengine.database.collections.DatabaseStage;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Objects;
|
||||
|
||||
public final class SubStageEntry<T, U extends DatabaseStage<?>> implements Entry<T, U> {
|
||||
|
||||
private final T key;
|
||||
private final U value;
|
||||
|
||||
public SubStageEntry(T key, U value) {
|
||||
this.key = key;
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
@Override
|
||||
public T getKey() {
|
||||
return key;
|
||||
}
|
||||
|
||||
@Override
|
||||
public U getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
@Override
|
||||
public U setValue(U value) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (obj == this)
|
||||
return true;
|
||||
if (obj == null || obj.getClass() != this.getClass())
|
||||
return false;
|
||||
//noinspection rawtypes
|
||||
var that = (SubStageEntry) obj;
|
||||
return Objects.equals(this.key, that.key) && Objects.equals(this.value, that.value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(key, value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "SubStageEntry[" + "key=" + key + ", " + "value=" + value + ']';
|
||||
}
|
||||
|
||||
}
|
@ -0,0 +1,5 @@
|
||||
package it.cavallium.dbengine.database;
|
||||
|
||||
import org.rocksdb.TableProperties;
|
||||
|
||||
public record TableWithProperties(String column, String table, TableProperties properties) {}
|
@ -1,31 +1,28 @@
|
||||
package it.cavallium.dbengine.database.collections;
|
||||
|
||||
import io.net5.buffer.api.Buffer;
|
||||
import io.net5.buffer.api.BufferAllocator;
|
||||
import io.net5.buffer.api.Drop;
|
||||
import io.net5.buffer.api.Send;
|
||||
import it.cavallium.buffer.Buf;
|
||||
import it.cavallium.buffer.BufDataInput;
|
||||
import it.cavallium.buffer.BufDataOutput;
|
||||
import it.cavallium.dbengine.database.LLDictionary;
|
||||
import it.cavallium.dbengine.database.LLUtils;
|
||||
import it.cavallium.dbengine.database.serialization.SerializationException;
|
||||
import it.cavallium.dbengine.database.serialization.Serializer;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
public class DatabaseEmpty {
|
||||
|
||||
@SuppressWarnings({"unused", "InstantiationOfUtilityClass"})
|
||||
public static final Nothing NOTHING = new Nothing();
|
||||
|
||||
public static Serializer<Nothing> nothingSerializer(BufferAllocator bufferAllocator) {
|
||||
public static Serializer<Nothing> nothingSerializer() {
|
||||
return new Serializer<>() {
|
||||
|
||||
@Override
|
||||
public @NotNull Nothing deserialize(@NotNull Buffer serialized) {
|
||||
public @NotNull Nothing deserialize(@NotNull BufDataInput in) throws SerializationException {
|
||||
return NOTHING;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void serialize(@NotNull Nothing deserialized, Buffer output) {
|
||||
public void serialize(@NotNull Nothing deserialized, BufDataOutput out) throws SerializationException {
|
||||
|
||||
}
|
||||
|
||||
@ -39,8 +36,8 @@ public class DatabaseEmpty {
|
||||
private DatabaseEmpty() {
|
||||
}
|
||||
|
||||
public static DatabaseStageEntry<Nothing> create(LLDictionary dictionary, Buffer key, Runnable onClose) {
|
||||
return new DatabaseSingle<>(dictionary, key, nothingSerializer(dictionary.getAllocator()), onClose);
|
||||
public static DatabaseStageEntry<Nothing> create(LLDictionary dictionary, Buf key) {
|
||||
return new DatabaseMapSingle<>(dictionary, key, nothingSerializer());
|
||||
}
|
||||
|
||||
public static final class Nothing {
|
||||
|
@ -1,30 +1,47 @@
|
||||
package it.cavallium.dbengine.database.collections;
|
||||
|
||||
import com.google.common.primitives.Ints;
|
||||
import it.cavallium.buffer.BufDataInput;
|
||||
import it.cavallium.buffer.BufDataOutput;
|
||||
import it.cavallium.dbengine.database.LLKeyValueDatabaseStructure;
|
||||
import it.cavallium.dbengine.database.LLSingleton;
|
||||
import it.cavallium.dbengine.database.LLSnapshot;
|
||||
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
|
||||
import java.util.concurrent.ForkJoinPool;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
public class DatabaseInt implements LLKeyValueDatabaseStructure {
|
||||
|
||||
private final LLSingleton singleton;
|
||||
private final SerializerFixedBinaryLength<Integer> serializer;
|
||||
|
||||
public DatabaseInt(LLSingleton singleton) {
|
||||
this.singleton = singleton;
|
||||
this.serializer = SerializerFixedBinaryLength.intSerializer();
|
||||
}
|
||||
|
||||
public Mono<Integer> get(@Nullable LLSnapshot snapshot) {
|
||||
return singleton.get(snapshot).map(Ints::fromByteArray);
|
||||
public Integer get(@Nullable LLSnapshot snapshot) {
|
||||
var result = singleton.get(snapshot);
|
||||
return serializer.deserialize(BufDataInput.create(result));
|
||||
}
|
||||
|
||||
public Mono<Void> set(int value) {
|
||||
return singleton.set(Ints.toByteArray(value));
|
||||
public void set(int value) {
|
||||
var buf = BufDataOutput.createLimited(Integer.BYTES);
|
||||
serializer.serialize(value, buf);
|
||||
singleton.set(buf.asList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDatabaseName() {
|
||||
return singleton.getDatabaseName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ForkJoinPool getDbReadPool() {
|
||||
return singleton.getDbReadPool();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ForkJoinPool getDbWritePool() {
|
||||
return singleton.getDbWritePool();
|
||||
}
|
||||
}
|
||||
|
@ -1,37 +1,95 @@
|
||||
package it.cavallium.dbengine.database.collections;
|
||||
|
||||
import com.google.common.primitives.Ints;
|
||||
import com.google.common.primitives.Longs;
|
||||
import it.cavallium.buffer.Buf;
|
||||
import it.cavallium.buffer.BufDataInput;
|
||||
import it.cavallium.buffer.BufDataOutput;
|
||||
import it.cavallium.dbengine.database.LLKeyValueDatabaseStructure;
|
||||
import it.cavallium.dbengine.database.LLSingleton;
|
||||
import it.cavallium.dbengine.database.LLSnapshot;
|
||||
import it.cavallium.dbengine.database.UpdateReturnMode;
|
||||
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
|
||||
import java.util.concurrent.ForkJoinPool;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
public class DatabaseLong implements LLKeyValueDatabaseStructure {
|
||||
|
||||
private final LLSingleton singleton;
|
||||
private final SerializerFixedBinaryLength<Long> serializer;
|
||||
private final SerializerFixedBinaryLength<Integer> bugSerializer;
|
||||
|
||||
public DatabaseLong(LLSingleton singleton) {
|
||||
this.singleton = singleton;
|
||||
this.serializer = SerializerFixedBinaryLength.longSerializer();
|
||||
this.bugSerializer = SerializerFixedBinaryLength.intSerializer();
|
||||
}
|
||||
|
||||
public Mono<Long> get(@Nullable LLSnapshot snapshot) {
|
||||
return singleton.get(snapshot).map(array -> {
|
||||
if (array.length == 4) {
|
||||
return (long) Ints.fromByteArray(array);
|
||||
public Long get(@Nullable LLSnapshot snapshot) {
|
||||
var result = BufDataInput.create(singleton.get(snapshot));
|
||||
if (result.available() == 4) {
|
||||
return (long) (int) bugSerializer.deserialize(result);
|
||||
} else {
|
||||
return serializer.deserialize(result);
|
||||
}
|
||||
}
|
||||
|
||||
public Long incrementAndGet() {
|
||||
return addAnd(1, UpdateReturnMode.GET_NEW_VALUE);
|
||||
}
|
||||
|
||||
public Long getAndIncrement() {
|
||||
return addAnd(1, UpdateReturnMode.GET_OLD_VALUE);
|
||||
}
|
||||
|
||||
public Long decrementAndGet() {
|
||||
return addAnd(-1, UpdateReturnMode.GET_NEW_VALUE);
|
||||
}
|
||||
|
||||
public Long getAndDecrement() {
|
||||
return addAnd(-1, UpdateReturnMode.GET_OLD_VALUE);
|
||||
}
|
||||
|
||||
public Long addAndGet(long count) {
|
||||
return addAnd(count, UpdateReturnMode.GET_NEW_VALUE);
|
||||
}
|
||||
|
||||
public Long getAndAdd(long count) {
|
||||
return addAnd(count, UpdateReturnMode.GET_OLD_VALUE);
|
||||
}
|
||||
|
||||
private Long addAnd(long count, UpdateReturnMode updateReturnMode) {
|
||||
var result = singleton.update(prev -> {
|
||||
if (prev != null) {
|
||||
var prevLong = prev.getLong(0);
|
||||
var buf = Buf.createZeroes(Long.BYTES);
|
||||
buf.setLong(0, prevLong + count);
|
||||
return buf;
|
||||
} else {
|
||||
return Longs.fromByteArray(array);
|
||||
var buf = Buf.createZeroes(Long.BYTES);
|
||||
buf.setLong(0, count);
|
||||
return buf;
|
||||
}
|
||||
});
|
||||
}, updateReturnMode);
|
||||
return result.getLong(0);
|
||||
}
|
||||
|
||||
public Mono<Void> set(long value) {
|
||||
return singleton.set(Longs.toByteArray(value));
|
||||
public void set(long value) {
|
||||
var buf = BufDataOutput.createLimited(Long.BYTES);
|
||||
serializer.serialize(value, buf);
|
||||
singleton.set(buf.asList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDatabaseName() {
|
||||
return singleton.getDatabaseName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ForkJoinPool getDbReadPool() {
|
||||
return singleton.getDbReadPool();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ForkJoinPool getDbWritePool() {
|
||||
return singleton.getDbWritePool();
|
||||
}
|
||||
}
|
||||
|
@ -1,507 +1,618 @@
|
||||
package it.cavallium.dbengine.database.collections;
|
||||
|
||||
import io.net5.buffer.api.Buffer;
|
||||
import io.net5.buffer.api.Resource;
|
||||
import io.net5.buffer.api.Send;
|
||||
import io.net5.buffer.api.internal.ResourceSupport;
|
||||
import static it.cavallium.dbengine.utils.StreamUtils.resourceStream;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import it.cavallium.buffer.Buf;
|
||||
import it.cavallium.buffer.BufDataInput;
|
||||
import it.cavallium.buffer.BufDataOutput;
|
||||
import it.cavallium.dbengine.client.CompositeSnapshot;
|
||||
import it.cavallium.dbengine.database.Delta;
|
||||
import it.cavallium.dbengine.database.LLDictionary;
|
||||
import it.cavallium.dbengine.database.LLDictionaryResultType;
|
||||
import it.cavallium.dbengine.database.LLEntry;
|
||||
import it.cavallium.dbengine.database.LLRange;
|
||||
import it.cavallium.dbengine.database.LLUtils;
|
||||
import it.cavallium.dbengine.database.SerializedKey;
|
||||
import it.cavallium.dbengine.database.SubStageEntry;
|
||||
import it.cavallium.dbengine.database.UpdateMode;
|
||||
import it.cavallium.dbengine.database.UpdateReturnMode;
|
||||
import it.cavallium.dbengine.database.disk.CachedSerializationFunction;
|
||||
import it.cavallium.dbengine.database.disk.LLLocalDictionary;
|
||||
import it.cavallium.dbengine.database.disk.RocksDBFile.RocksDBFileIterationKeyState.RocksDBFileIterationStateKeyError;
|
||||
import it.cavallium.dbengine.database.disk.RocksDBFile.RocksDBFileIterationKeyState.RocksDBFileIterationStateKeyOk;
|
||||
import it.cavallium.dbengine.database.disk.RocksDBFile.RocksDBFileIterationState.RocksDBFileIterationStateBegin;
|
||||
import it.cavallium.dbengine.database.disk.RocksDBFile.RocksDBFileIterationState.RocksDBFileIterationStateEnd;
|
||||
import it.cavallium.dbengine.database.disk.RocksDBFile.RocksDBFileIterationState.RocksDBFileIterationStateKey;
|
||||
import it.cavallium.dbengine.database.disk.SSTRange.SSTRangeFull;
|
||||
import it.cavallium.dbengine.database.serialization.KVSerializationFunction;
|
||||
import it.cavallium.dbengine.database.serialization.SerializationException;
|
||||
import it.cavallium.dbengine.database.serialization.SerializationFunction;
|
||||
import it.cavallium.dbengine.database.serialization.Serializer;
|
||||
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import it.cavallium.dbengine.utils.StreamUtils;
|
||||
import it.unimi.dsi.fastutil.objects.Object2ObjectLinkedOpenHashMap;
|
||||
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
|
||||
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMaps;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.function.BiConsumer;
|
||||
import java.util.function.BiFunction;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.core.publisher.SynchronousSink;
|
||||
import reactor.util.function.Tuple2;
|
||||
import reactor.util.function.Tuples;
|
||||
import org.rocksdb.RocksDBException;
|
||||
|
||||
/**
|
||||
* Optimized implementation of "DatabaseMapDictionary with SubStageGetterSingle"
|
||||
*/
|
||||
public class DatabaseMapDictionary<T, U> extends DatabaseMapDictionaryDeep<T, U, DatabaseStageEntry<U>> {
|
||||
|
||||
private static final Logger LOG = LogManager.getLogger(DatabaseMapDictionary.class);
|
||||
|
||||
private final AtomicLong totalZeroBytesErrors = new AtomicLong();
|
||||
private final Serializer<U> valueSerializer;
|
||||
|
||||
protected DatabaseMapDictionary(LLDictionary dictionary,
|
||||
@Nullable Buffer prefixKey,
|
||||
@Nullable Buf prefixKey,
|
||||
SerializerFixedBinaryLength<T> keySuffixSerializer,
|
||||
Serializer<U> valueSerializer,
|
||||
Runnable onClose) {
|
||||
Serializer<U> valueSerializer) {
|
||||
// Do not retain or release or use the prefixKey here
|
||||
super(dictionary, prefixKey, keySuffixSerializer, new SubStageGetterSingle<>(valueSerializer), 0, onClose);
|
||||
super(dictionary, prefixKey, keySuffixSerializer, new SubStageGetterSingle<>(valueSerializer), 0);
|
||||
this.valueSerializer = valueSerializer;
|
||||
}
|
||||
|
||||
public static <T, U> DatabaseMapDictionary<T, U> simple(LLDictionary dictionary,
|
||||
SerializerFixedBinaryLength<T> keySerializer,
|
||||
Serializer<U> valueSerializer,
|
||||
Runnable onClose) {
|
||||
return new DatabaseMapDictionary<>(dictionary, null, keySerializer,
|
||||
valueSerializer, onClose);
|
||||
Serializer<U> valueSerializer) {
|
||||
return new DatabaseMapDictionary<>(dictionary, null, keySerializer, valueSerializer);
|
||||
}
|
||||
|
||||
public static <T, U> DatabaseMapDictionary<T, U> tail(LLDictionary dictionary,
|
||||
@Nullable Buffer prefixKey,
|
||||
@Nullable Buf prefixKey,
|
||||
SerializerFixedBinaryLength<T> keySuffixSerializer,
|
||||
Serializer<U> valueSerializer,
|
||||
Runnable onClose) {
|
||||
return new DatabaseMapDictionary<>(dictionary, prefixKey, keySuffixSerializer, valueSerializer, onClose);
|
||||
Serializer<U> valueSerializer) {
|
||||
return new DatabaseMapDictionary<>(dictionary, prefixKey, keySuffixSerializer, valueSerializer);
|
||||
}
|
||||
|
||||
private void deserializeValue(Send<Buffer> valueToReceive, SynchronousSink<U> sink) {
|
||||
try (var value = valueToReceive.receive()) {
|
||||
sink.next(valueSerializer.deserialize(value));
|
||||
} catch (Throwable ex) {
|
||||
sink.error(ex);
|
||||
public static <K, V> Stream<Entry<K, V>> getLeavesFrom(DatabaseMapDictionary<K, V> databaseMapDictionary,
|
||||
CompositeSnapshot snapshot,
|
||||
@Nullable K keyMin,
|
||||
@Nullable K keyMax,
|
||||
boolean reverse,
|
||||
boolean smallRange) {
|
||||
|
||||
if (keyMin != null || keyMax != null) {
|
||||
return databaseMapDictionary.getAllEntries(snapshot,
|
||||
keyMin,
|
||||
keyMax,
|
||||
reverse,
|
||||
smallRange,
|
||||
Map::entry
|
||||
);
|
||||
} else {
|
||||
return databaseMapDictionary.getAllEntries(snapshot, smallRange, Map::entry);
|
||||
}
|
||||
}
|
||||
|
||||
private Buffer serializeValue(U value) throws SerializationException {
|
||||
public static <K> Stream<K> getKeyLeavesFrom(DatabaseMapDictionary<K, ?> databaseMapDictionary,
|
||||
CompositeSnapshot snapshot,
|
||||
@Nullable K keyMin,
|
||||
@Nullable K keyMax,
|
||||
boolean reverse,
|
||||
boolean smallRange) {
|
||||
|
||||
Stream<? extends Entry<K, ? extends DatabaseStageEntry<?>>> stagesFlux;
|
||||
if (keyMin != null || keyMax != null) {
|
||||
stagesFlux = databaseMapDictionary.getAllStages(snapshot, keyMin, keyMax, reverse, smallRange);
|
||||
} else {
|
||||
stagesFlux = databaseMapDictionary.getAllStages(snapshot, smallRange);
|
||||
}
|
||||
return stagesFlux.map(Entry::getKey);
|
||||
}
|
||||
|
||||
private U deserializeValue(Buf value) {
|
||||
return valueSerializer.deserialize(BufDataInput.create(value));
|
||||
}
|
||||
|
||||
private @Nullable U deserializeValue(T keySuffix, BufDataInput value) {
|
||||
try {
|
||||
return valueSerializer.deserialize(value);
|
||||
} catch (IndexOutOfBoundsException ex) {
|
||||
var exMessage = ex.getMessage();
|
||||
if (exMessage != null && exMessage.contains("read 0 to 0, write 0 to ")) {
|
||||
var totalZeroBytesErrors = this.totalZeroBytesErrors.incrementAndGet();
|
||||
if (totalZeroBytesErrors < 512 || totalZeroBytesErrors % 10000 == 0) {
|
||||
var keySuffixBytes = serializeKeySuffixToKey(keySuffix);
|
||||
try {
|
||||
LOG.error(
|
||||
"Unexpected zero-bytes value at " + dictionary.getDatabaseName() + ":" + dictionary.getColumnName()
|
||||
+ ":" + LLUtils.toStringSafe(keyPrefix) + ":" + keySuffix + "(" + LLUtils.toStringSafe(
|
||||
keySuffixBytes) + ") total=" + totalZeroBytesErrors);
|
||||
} catch (SerializationException e) {
|
||||
LOG.error(
|
||||
"Unexpected zero-bytes value at " + dictionary.getDatabaseName() + ":" + dictionary.getColumnName()
|
||||
+ ":" + LLUtils.toStringSafe(keyPrefix) + ":" + keySuffix + "(?) total=" + totalZeroBytesErrors);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
} else {
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Buf serializeValue(U value) throws SerializationException {
|
||||
var valSizeHint = valueSerializer.getSerializedSizeHint();
|
||||
if (valSizeHint == -1) valSizeHint = 128;
|
||||
var valBuf = dictionary.getAllocator().allocate(valSizeHint);
|
||||
var valBuf = BufDataOutput.create(valSizeHint);
|
||||
try {
|
||||
valueSerializer.serialize(value, valBuf);
|
||||
return valBuf;
|
||||
} catch (Throwable t) {
|
||||
valBuf.close();
|
||||
throw t;
|
||||
} catch (SerializationException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
throw new SerializationException("Failed to serialize value", ex);
|
||||
}
|
||||
return valBuf.asList();
|
||||
}
|
||||
|
||||
private Buffer serializeKeySuffixToKey(T keySuffix) throws SerializationException {
|
||||
Buffer keyBuf;
|
||||
private Buf serializeKeySuffixToKey(T keySuffix) throws SerializationException {
|
||||
BufDataOutput keyBuf = BufDataOutput.createLimited(keyPrefixLength + keySuffixLength + keyExtLength);
|
||||
if (keyPrefix != null) {
|
||||
keyBuf = keyPrefix.copy();
|
||||
} else {
|
||||
keyBuf = this.dictionary.getAllocator().allocate(keyPrefixLength + keySuffixLength + keyExtLength);
|
||||
}
|
||||
try {
|
||||
assert keyBuf.readableBytes() == keyPrefixLength;
|
||||
keyBuf.ensureWritable(keySuffixLength + keyExtLength);
|
||||
serializeSuffix(keySuffix, keyBuf);
|
||||
assert keyBuf.readableBytes() == keyPrefixLength + keySuffixLength + keyExtLength;
|
||||
return keyBuf;
|
||||
} catch (Throwable t) {
|
||||
keyBuf.close();
|
||||
throw t;
|
||||
keyBuf.writeBytes(keyPrefix);
|
||||
}
|
||||
assert keyBuf.size() == keyPrefixLength;
|
||||
serializeSuffixTo(keySuffix, keyBuf);
|
||||
assert keyBuf.size() == keyPrefixLength + keySuffixLength + keyExtLength;
|
||||
return keyBuf.asList();
|
||||
}
|
||||
|
||||
private Buffer toKey(Buffer suffixKey) {
|
||||
assert suffixKeyLengthConsistency(suffixKey.readableBytes());
|
||||
if (keyPrefix != null && keyPrefix.readableBytes() > 0) {
|
||||
var result = LLUtils.compositeBuffer(dictionary.getAllocator(),
|
||||
LLUtils.copy(dictionary.getAllocator(), keyPrefix),
|
||||
suffixKey.send()
|
||||
);
|
||||
try {
|
||||
assert result.readableBytes() == keyPrefixLength + keySuffixLength + keyExtLength;
|
||||
return result;
|
||||
} catch (Throwable t) {
|
||||
result.close();
|
||||
throw t;
|
||||
}
|
||||
private Buf toKey(Buf suffixKey) {
|
||||
assert suffixKeyLengthConsistency(suffixKey.size());
|
||||
if (keyPrefix != null) {
|
||||
var result = keyPrefix.copy();
|
||||
result.addAll(suffixKey);
|
||||
assert result.size() == keyPrefixLength + keySuffixLength + keyExtLength;
|
||||
return result;
|
||||
} else {
|
||||
assert suffixKey.readableBytes() == keyPrefixLength + keySuffixLength + keyExtLength;
|
||||
assert suffixKey.size() == keyPrefixLength + keySuffixLength + keyExtLength;
|
||||
return suffixKey;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Map<T, U>> get(@Nullable CompositeSnapshot snapshot, boolean existsAlmostCertainly) {
|
||||
return dictionary
|
||||
.getRange(resolveSnapshot(snapshot), rangeMono, existsAlmostCertainly)
|
||||
.<Entry<T, U>>handle((entrySend, sink) -> {
|
||||
public Object2ObjectSortedMap<T, U> get(@Nullable CompositeSnapshot snapshot) {
|
||||
Stream<Entry<T, U>> stream = dictionary
|
||||
.getRange(resolveSnapshot(snapshot), range, false, true)
|
||||
.map(entry -> {
|
||||
Entry<T, U> deserializedEntry;
|
||||
try {
|
||||
try (var entry = entrySend.receive()) {
|
||||
T key;
|
||||
try (var serializedKey = entry.getKey().receive()) {
|
||||
splitPrefix(serializedKey).close();
|
||||
suffixKeyLengthConsistency(serializedKey.readableBytes());
|
||||
key = deserializeSuffix(serializedKey);
|
||||
}
|
||||
U value;
|
||||
try (var valueBuf = entry.getValue().receive()) {
|
||||
value = valueSerializer.deserialize(valueBuf);
|
||||
}
|
||||
deserializedEntry = Map.entry(key, value);
|
||||
}
|
||||
sink.next(deserializedEntry);
|
||||
} catch (Throwable ex) {
|
||||
sink.error(ex);
|
||||
}
|
||||
})
|
||||
.collectMap(Entry::getKey, Entry::getValue, HashMap::new)
|
||||
.filter(map -> !map.isEmpty());
|
||||
T key;
|
||||
// serializedKey
|
||||
var buf1 = BufDataInput.create(entry.getKey());
|
||||
var serializedValue = BufDataInput.create(entry.getValue());
|
||||
// after this, it becomes serializedSuffixAndExt
|
||||
buf1.skipNBytes(keyPrefixLength);
|
||||
suffixAndExtKeyConsistency(buf1.available());
|
||||
|
||||
key = deserializeSuffix(buf1);
|
||||
U value = valueSerializer.deserialize(serializedValue);
|
||||
deserializedEntry = Map.entry(key, value);
|
||||
return deserializedEntry;
|
||||
});
|
||||
// serializedKey
|
||||
// after this, it becomes serializedSuffixAndExt
|
||||
var map = StreamUtils.collect(stream,
|
||||
Collectors.toMap(Entry::getKey, Entry::getValue, (a, b) -> a, Object2ObjectLinkedOpenHashMap::new)
|
||||
);
|
||||
return map == null || map.isEmpty() ? null : map;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Map<T, U>> setAndGetPrevious(Map<T, U> value) {
|
||||
return this
|
||||
.get(null, false)
|
||||
.concatWith(dictionary.setRange(rangeMono, Flux
|
||||
.fromIterable(Collections.unmodifiableMap(value).entrySet())
|
||||
.handle(this::serializeEntrySink)
|
||||
).then(Mono.empty()))
|
||||
.singleOrEmpty()
|
||||
.transform(LLUtils::handleDiscard);
|
||||
public Object2ObjectSortedMap<T, U> setAndGetPrevious(Object2ObjectSortedMap<T, U> value) {
|
||||
Object2ObjectSortedMap<T, U> prev = this.get(null);
|
||||
if (value == null || value.isEmpty()) {
|
||||
dictionary.clear();
|
||||
} else {
|
||||
dictionary.setRange(range, value.entrySet().stream().map(this::serializeEntry), true);
|
||||
}
|
||||
return prev != null && prev.isEmpty() ? null : prev;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Map<T, U>> clearAndGetPrevious() {
|
||||
return this
|
||||
.setAndGetPrevious(Map.of());
|
||||
public Object2ObjectSortedMap<T, U> clearAndGetPrevious() {
|
||||
return this.setAndGetPrevious(Object2ObjectSortedMaps.emptyMap());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Long> leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
|
||||
return dictionary.sizeRange(resolveSnapshot(snapshot), rangeMono, fast);
|
||||
public long leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
|
||||
return dictionary.sizeRange(resolveSnapshot(snapshot), range, fast);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Boolean> isEmpty(@Nullable CompositeSnapshot snapshot) {
|
||||
return dictionary.isRangeEmpty(resolveSnapshot(snapshot), rangeMono);
|
||||
public boolean isEmpty(@Nullable CompositeSnapshot snapshot) {
|
||||
return dictionary.isRangeEmpty(resolveSnapshot(snapshot), range, false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<DatabaseStageEntry<U>> at(@Nullable CompositeSnapshot snapshot, T keySuffix) {
|
||||
return Mono.fromCallable(() ->
|
||||
new DatabaseSingle<>(dictionary, serializeKeySuffixToKey(keySuffix), valueSerializer, null));
|
||||
public @NotNull DatabaseStageEntry<U> at(@Nullable CompositeSnapshot snapshot, T keySuffix) {
|
||||
return new DatabaseMapSingle<>(dictionary, serializeKeySuffixToKey(keySuffix), valueSerializer);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<U> getValue(@Nullable CompositeSnapshot snapshot, T keySuffix, boolean existsAlmostCertainly) {
|
||||
return dictionary
|
||||
.get(resolveSnapshot(snapshot),
|
||||
Mono.fromCallable(() -> serializeKeySuffixToKey(keySuffix).send()),
|
||||
existsAlmostCertainly
|
||||
)
|
||||
.handle(this::deserializeValue);
|
||||
public boolean containsKey(@Nullable CompositeSnapshot snapshot, T keySuffix) {
|
||||
return !dictionary.isRangeEmpty(resolveSnapshot(snapshot),
|
||||
LLRange.single(serializeKeySuffixToKey(keySuffix)), true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Void> putValue(T keySuffix, U value) {
|
||||
var keyMono = Mono.fromCallable(() -> serializeKeySuffixToKey(keySuffix).send()).single();
|
||||
var valueMono = Mono.fromCallable(() -> serializeValue(value).send()).single();
|
||||
return dictionary
|
||||
.put(keyMono, valueMono, LLDictionaryResultType.VOID)
|
||||
.doOnNext(Send::close)
|
||||
.then();
|
||||
public U getValue(@Nullable CompositeSnapshot snapshot, T keySuffix) {
|
||||
var keySuffixBuf = serializeKeySuffixToKey(keySuffix);
|
||||
Buf value = dictionary.get(resolveSnapshot(snapshot), keySuffixBuf);
|
||||
return value != null ? deserializeValue(keySuffix, BufDataInput.create(value)) : null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<UpdateMode> getUpdateMode() {
|
||||
public void putValue(T keySuffix, U value) {
|
||||
var keyMono = serializeKeySuffixToKey(keySuffix);
|
||||
var valueMono = serializeValue(value);
|
||||
dictionary.put(keyMono, valueMono, LLDictionaryResultType.VOID);
|
||||
}
|
||||
|
||||
@Override
|
||||
public UpdateMode getUpdateMode() {
|
||||
return dictionary.getUpdateMode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<U> updateValue(T keySuffix,
|
||||
public U updateValue(T keySuffix,
|
||||
UpdateReturnMode updateReturnMode,
|
||||
boolean existsAlmostCertainly,
|
||||
SerializationFunction<@Nullable U, @Nullable U> updater) {
|
||||
var keyMono = Mono.fromCallable(() -> serializeKeySuffixToKey(keySuffix).send());
|
||||
return dictionary
|
||||
.update(keyMono, getSerializedUpdater(updater), updateReturnMode, existsAlmostCertainly)
|
||||
.handle(this::deserializeValue);
|
||||
var keyMono = serializeKeySuffixToKey(keySuffix);
|
||||
var serializedUpdater = getSerializedUpdater(updater);
|
||||
dictionary.update(keyMono, serializedUpdater, UpdateReturnMode.NOTHING);
|
||||
return serializedUpdater.getResult(updateReturnMode);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Delta<U>> updateValueAndGetDelta(T keySuffix,
|
||||
boolean existsAlmostCertainly,
|
||||
SerializationFunction<@Nullable U, @Nullable U> updater) {
|
||||
var keyMono = Mono.fromCallable(() -> serializeKeySuffixToKey(keySuffix).send());
|
||||
return dictionary
|
||||
.updateAndGetDelta(keyMono, getSerializedUpdater(updater), existsAlmostCertainly)
|
||||
.transform(mono -> LLUtils.mapLLDelta(mono, serializedToReceive -> {
|
||||
try (var serialized = serializedToReceive.receive()) {
|
||||
return valueSerializer.deserialize(serialized);
|
||||
}
|
||||
}));
|
||||
public Delta<U> updateValueAndGetDelta(T keySuffix, SerializationFunction<@Nullable U, @Nullable U> updater) {
|
||||
var keyMono = serializeKeySuffixToKey(keySuffix);
|
||||
var serializedUpdater = getSerializedUpdater(updater);
|
||||
dictionary.update(keyMono, serializedUpdater, UpdateReturnMode.NOTHING);
|
||||
return serializedUpdater.getDelta();
|
||||
}
|
||||
|
||||
public SerializationFunction<@Nullable Send<Buffer>, @Nullable Buffer> getSerializedUpdater(
|
||||
SerializationFunction<@Nullable U, @Nullable U> updater) {
|
||||
return oldSerialized -> {
|
||||
try (oldSerialized) {
|
||||
U result;
|
||||
if (oldSerialized == null) {
|
||||
result = updater.apply(null);
|
||||
} else {
|
||||
try (var oldSerializedReceived = oldSerialized.receive()) {
|
||||
result = updater.apply(valueSerializer.deserialize(oldSerializedReceived));
|
||||
}
|
||||
}
|
||||
if (result == null) {
|
||||
return null;
|
||||
} else {
|
||||
return serializeValue(result);
|
||||
}
|
||||
}
|
||||
};
|
||||
public CachedSerializationFunction<U, Buf, Buf> getSerializedUpdater(SerializationFunction<@Nullable U, @Nullable U> updater) {
|
||||
return new CachedSerializationFunction<>(updater, this::serializeValue, this::deserializeValue);
|
||||
}
|
||||
|
||||
public KVSerializationFunction<@NotNull T, @Nullable Send<Buffer>, @Nullable Buffer> getSerializedUpdater(
|
||||
public KVSerializationFunction<@NotNull T, @Nullable Buf, @Nullable Buf> getSerializedUpdater(
|
||||
KVSerializationFunction<@NotNull T, @Nullable U, @Nullable U> updater) {
|
||||
return (key, oldSerialized) -> {
|
||||
try (oldSerialized) {
|
||||
U result;
|
||||
if (oldSerialized == null) {
|
||||
result = updater.apply(key, null);
|
||||
} else {
|
||||
try (var oldSerializedReceived = oldSerialized.receive()) {
|
||||
result = updater.apply(key, valueSerializer.deserialize(oldSerializedReceived));
|
||||
}
|
||||
}
|
||||
if (result == null) {
|
||||
return null;
|
||||
} else {
|
||||
return serializeValue(result);
|
||||
}
|
||||
U result;
|
||||
if (oldSerialized == null) {
|
||||
result = updater.apply(key, null);
|
||||
} else {
|
||||
result = updater.apply(key, valueSerializer.deserialize(BufDataInput.create(oldSerialized)));
|
||||
}
|
||||
if (result == null) {
|
||||
return null;
|
||||
} else {
|
||||
return serializeValue(result);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<U> putValueAndGetPrevious(T keySuffix, U value) {
|
||||
var keyMono = Mono.fromCallable(() -> serializeKeySuffixToKey(keySuffix).send());
|
||||
var valueMono = Mono.fromCallable(() -> serializeValue(value).send());
|
||||
return dictionary.put(keyMono, valueMono, LLDictionaryResultType.PREVIOUS_VALUE).handle(this::deserializeValue);
|
||||
public U putValueAndGetPrevious(T keySuffix, U value) {
|
||||
var keyMono = serializeKeySuffixToKey(keySuffix);
|
||||
var valueMono = serializeValue(value);
|
||||
var valueBuf = dictionary.put(keyMono, valueMono, LLDictionaryResultType.PREVIOUS_VALUE);
|
||||
if (valueBuf == null) {
|
||||
return null;
|
||||
}
|
||||
return deserializeValue(keySuffix, BufDataInput.create(valueBuf));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Boolean> putValueAndGetChanged(T keySuffix, U value) {
|
||||
var keyMono = Mono.fromCallable(() -> serializeKeySuffixToKey(keySuffix).send());
|
||||
var valueMono = Mono.fromCallable(() -> serializeValue(value).send());
|
||||
public boolean putValueAndGetChanged(T keySuffix, U value) {
|
||||
var keyMono = serializeKeySuffixToKey(keySuffix);
|
||||
var valueMono = serializeValue(value);
|
||||
var oldValueBuf = dictionary.put(keyMono, valueMono, LLDictionaryResultType.PREVIOUS_VALUE);
|
||||
var oldValue = oldValueBuf != null ? deserializeValue(keySuffix, BufDataInput.create(oldValueBuf)) : null;
|
||||
if (oldValue == null) {
|
||||
return value != null;
|
||||
} else {
|
||||
return !Objects.equals(oldValue, value);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove(T keySuffix) {
|
||||
var keyMono = serializeKeySuffixToKey(keySuffix);
|
||||
dictionary.remove(keyMono, LLDictionaryResultType.VOID);
|
||||
}
|
||||
|
||||
@Override
|
||||
public U removeAndGetPrevious(T keySuffix) {
|
||||
var keyMono = serializeKeySuffixToKey(keySuffix);
|
||||
var valueBuf = dictionary.remove(keyMono, LLDictionaryResultType.PREVIOUS_VALUE);
|
||||
return valueBuf != null ? deserializeValue(keySuffix, BufDataInput.create(valueBuf)) : null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean removeAndGetStatus(T keySuffix) {
|
||||
var keyMono = serializeKeySuffixToKey(keySuffix);
|
||||
return LLUtils.responseToBoolean(dictionary.remove(keyMono, LLDictionaryResultType.PREVIOUS_VALUE_EXISTENCE));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Stream<Optional<U>> getMulti(@Nullable CompositeSnapshot snapshot, Stream<T> keys) {
|
||||
var mappedKeys = keys.map(keySuffix -> serializeKeySuffixToKey(keySuffix));
|
||||
return dictionary
|
||||
.put(keyMono, valueMono, LLDictionaryResultType.PREVIOUS_VALUE)
|
||||
.handle(this::deserializeValue)
|
||||
.map(oldValue -> !Objects.equals(oldValue, value))
|
||||
.defaultIfEmpty(value != null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Void> remove(T keySuffix) {
|
||||
var keyMono = Mono.fromCallable(() -> serializeKeySuffixToKey(keySuffix).send());
|
||||
return dictionary
|
||||
.remove(keyMono, LLDictionaryResultType.VOID)
|
||||
.doOnNext(Send::close)
|
||||
.then();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<U> removeAndGetPrevious(T keySuffix) {
|
||||
var keyMono = Mono.fromCallable(() -> serializeKeySuffixToKey(keySuffix).send());
|
||||
return dictionary.remove(keyMono, LLDictionaryResultType.PREVIOUS_VALUE).handle(this::deserializeValue);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Boolean> removeAndGetStatus(T keySuffix) {
|
||||
var keyMono = Mono.fromCallable(() -> serializeKeySuffixToKey(keySuffix).send());
|
||||
return dictionary
|
||||
.remove(keyMono, LLDictionaryResultType.PREVIOUS_VALUE_EXISTENCE)
|
||||
.map(LLUtils::responseToBoolean);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<Optional<U>> getMulti(@Nullable CompositeSnapshot snapshot, Flux<T> keys, boolean existsAlmostCertainly) {
|
||||
var mappedKeys = keys
|
||||
.<Send<Buffer>>handle((keySuffix, sink) -> {
|
||||
try {
|
||||
sink.next(serializeKeySuffixToKey(keySuffix).send());
|
||||
} catch (Throwable ex) {
|
||||
sink.error(ex);
|
||||
.getMulti(resolveSnapshot(snapshot), mappedKeys)
|
||||
.map(valueBufOpt -> {
|
||||
if (valueBufOpt.isPresent()) {
|
||||
return Optional.of(valueSerializer.deserialize(BufDataInput.create(valueBufOpt.get())));
|
||||
} else {
|
||||
return Optional.empty();
|
||||
}
|
||||
});
|
||||
return dictionary
|
||||
.getMulti(resolveSnapshot(snapshot), mappedKeys, existsAlmostCertainly)
|
||||
.<Optional<U>>handle((valueBufOpt, sink) -> {
|
||||
try {
|
||||
Optional<U> valueOpt;
|
||||
if (valueBufOpt.isPresent()) {
|
||||
valueOpt = Optional.of(valueSerializer.deserialize(valueBufOpt.get()));
|
||||
} else {
|
||||
valueOpt = Optional.empty();
|
||||
}
|
||||
sink.next(valueOpt);
|
||||
} catch (Throwable ex) {
|
||||
sink.error(ex);
|
||||
} finally {
|
||||
valueBufOpt.ifPresent(Resource::close);
|
||||
}
|
||||
})
|
||||
.transform(LLUtils::handleDiscard);
|
||||
}
|
||||
|
||||
private LLEntry serializeEntry(T keySuffix, U value) throws SerializationException {
|
||||
var key = serializeKeySuffixToKey(keySuffix);
|
||||
try {
|
||||
var serializedValue = serializeValue(value);
|
||||
return LLEntry.of(key, serializedValue);
|
||||
} catch (Throwable t) {
|
||||
key.close();
|
||||
throw t;
|
||||
}
|
||||
var serializedValue = serializeValue(value);
|
||||
return LLEntry.of(key, serializedValue);
|
||||
}
|
||||
|
||||
private void serializeEntrySink(Entry<T,U> entry, SynchronousSink<Send<LLEntry>> sink) {
|
||||
try {
|
||||
sink.next(serializeEntry(entry.getKey(), entry.getValue()).send());
|
||||
} catch (Throwable e) {
|
||||
sink.error(e);
|
||||
private LLEntry serializeEntry(Entry<T, U> entry) throws SerializationException {
|
||||
return serializeEntry(entry.getKey(), entry.getValue());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void putMulti(Stream<Entry<T, U>> entries) {
|
||||
try (var serializedEntries = entries.map(entry -> serializeEntry(entry))) {
|
||||
dictionary.putMulti(serializedEntries);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Void> putMulti(Flux<Entry<T, U>> entries) {
|
||||
var serializedEntries = entries
|
||||
.<Send<LLEntry>>handle((entry, sink) -> {
|
||||
try {
|
||||
sink.next(serializeEntry(entry.getKey(), entry.getValue()).send());
|
||||
} catch (Throwable e) {
|
||||
sink.error(e);
|
||||
}
|
||||
})
|
||||
.doOnDiscard(Send.class, Send::close)
|
||||
.doOnDiscard(Resource.class, Resource::close);
|
||||
return dictionary
|
||||
.putMulti(serializedEntries, false)
|
||||
.then()
|
||||
.doOnDiscard(Send.class, Send::close)
|
||||
.doOnDiscard(Resource.class, Resource::close)
|
||||
.doOnDiscard(LLEntry.class, ResourceSupport::close)
|
||||
.doOnDiscard(List.class, list -> {
|
||||
for (Object o : list) {
|
||||
if (o instanceof Send send) {
|
||||
send.close();
|
||||
} else if (o instanceof Buffer buf) {
|
||||
buf.close();
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<Boolean> updateMulti(Flux<T> keys,
|
||||
public Stream<Boolean> updateMulti(Stream<T> keys,
|
||||
KVSerializationFunction<T, @Nullable U, @Nullable U> updater) {
|
||||
var sharedKeys = keys.publish().refCount(2);
|
||||
var serializedKeys = sharedKeys
|
||||
.<Send<Buffer>>handle((key, sink) -> {
|
||||
try {
|
||||
Send<Buffer> serializedKey = serializeKeySuffixToKey(key).send();
|
||||
sink.next(serializedKey);
|
||||
} catch (Throwable ex) {
|
||||
sink.error(ex);
|
||||
}
|
||||
})
|
||||
.doOnDiscard(Tuple2.class, uncastedEntry -> {
|
||||
if (uncastedEntry.getT1() instanceof Buffer byteBuf) {
|
||||
byteBuf.close();
|
||||
}
|
||||
if (uncastedEntry.getT2() instanceof Buffer byteBuf) {
|
||||
byteBuf.close();
|
||||
}
|
||||
});
|
||||
var serializedKeys = keys.map(keySuffix -> new SerializedKey<>(keySuffix, serializeKeySuffixToKey(keySuffix)));
|
||||
var serializedUpdater = getSerializedUpdater(updater);
|
||||
return dictionary.updateMulti(sharedKeys, serializedKeys, serializedUpdater);
|
||||
return dictionary.updateMulti(serializedKeys, serializedUpdater);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<Entry<T, DatabaseStageEntry<U>>> getAllStages(@Nullable CompositeSnapshot snapshot) {
|
||||
public Stream<SubStageEntry<T, DatabaseStageEntry<U>>> getAllStages(@Nullable CompositeSnapshot snapshot, boolean smallRange) {
|
||||
return getAllStages(snapshot, range, false, smallRange);
|
||||
}
|
||||
|
||||
private LLRange getPatchedRange(@NotNull LLRange range, @Nullable T keyMin, @Nullable T keyMax)
|
||||
throws SerializationException {
|
||||
Buf keyMinBuf = serializeSuffixForRange(keyMin);
|
||||
if (keyMinBuf == null) {
|
||||
keyMinBuf = range.getMin();
|
||||
}
|
||||
Buf keyMaxBuf = serializeSuffixForRange(keyMax);
|
||||
if (keyMaxBuf == null) {
|
||||
keyMaxBuf = range.getMax();
|
||||
}
|
||||
return LLRange.of(keyMinBuf, keyMaxBuf);
|
||||
}
|
||||
|
||||
private Buf serializeSuffixForRange(@Nullable T key) throws SerializationException {
|
||||
if (key == null) {
|
||||
return null;
|
||||
}
|
||||
var keyWithoutExtBuf = BufDataOutput.createLimited(keyPrefixLength + keySuffixLength);
|
||||
if (keyPrefix != null) {
|
||||
keyWithoutExtBuf.writeBytes(keyPrefix);
|
||||
}
|
||||
serializeSuffixTo(key, keyWithoutExtBuf);
|
||||
return keyWithoutExtBuf.asList();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all stages
|
||||
* @param reverse if true, the results will go backwards from the specified key (inclusive)
|
||||
*/
|
||||
public Stream<SubStageEntry<T, DatabaseStageEntry<U>>> getAllStages(@Nullable CompositeSnapshot snapshot,
|
||||
@Nullable T keyMin,
|
||||
@Nullable T keyMax,
|
||||
boolean reverse,
|
||||
boolean smallRange) {
|
||||
if (keyMin == null && keyMax == null) {
|
||||
return getAllStages(snapshot, smallRange);
|
||||
} else {
|
||||
LLRange boundedRange = getPatchedRange(range, keyMin, keyMax);
|
||||
return getAllStages(snapshot, boundedRange, reverse, smallRange);
|
||||
}
|
||||
}
|
||||
|
||||
private Stream<SubStageEntry<T, DatabaseStageEntry<U>>> getAllStages(@Nullable CompositeSnapshot snapshot,
|
||||
LLRange sliceRange, boolean reverse, boolean smallRange) {
|
||||
return dictionary
|
||||
.getRangeKeys(resolveSnapshot(snapshot), rangeMono)
|
||||
.handle((keyBufToReceive, sink) -> {
|
||||
var keyBuf = keyBufToReceive.receive();
|
||||
try {
|
||||
assert keyBuf.readableBytes() == keyPrefixLength + keySuffixLength + keyExtLength;
|
||||
// Remove prefix. Keep only the suffix and the ext
|
||||
splitPrefix(keyBuf).close();
|
||||
suffixKeyLengthConsistency(keyBuf.readableBytes());
|
||||
T keySuffix;
|
||||
try (var keyBufCopy = keyBuf.copy()) {
|
||||
keySuffix = deserializeSuffix(keyBufCopy);
|
||||
}
|
||||
var subStage = new DatabaseSingle<>(dictionary, toKey(keyBuf), valueSerializer, null);
|
||||
sink.next(Map.entry(keySuffix, subStage));
|
||||
} catch (Throwable ex) {
|
||||
keyBuf.close();
|
||||
sink.error(ex);
|
||||
}
|
||||
.getRangeKeys(resolveSnapshot(snapshot), sliceRange, reverse, smallRange)
|
||||
.map(keyBuf -> {
|
||||
assert keyBuf.size() == keyPrefixLength + keySuffixLength + keyExtLength;
|
||||
// Remove prefix. Keep only the suffix and the ext
|
||||
var suffixAndExtIn = BufDataInput.create(keyBuf);
|
||||
suffixAndExtIn.skipBytes(keyPrefixLength);
|
||||
|
||||
suffixKeyLengthConsistency(suffixAndExtIn.available());
|
||||
T keySuffix = deserializeSuffix(suffixAndExtIn);
|
||||
var subStage = new DatabaseMapSingle<>(dictionary, keyBuf, valueSerializer);
|
||||
return new SubStageEntry<>(keySuffix, subStage);
|
||||
});
|
||||
}
|
||||
|
||||
private Stream<T> getAllKeys(@Nullable CompositeSnapshot snapshot,
|
||||
LLRange sliceRange, boolean reverse, boolean smallRange) {
|
||||
return dictionary
|
||||
.getRangeKeys(resolveSnapshot(snapshot), sliceRange, reverse, smallRange)
|
||||
.map(keyBuf -> {
|
||||
assert keyBuf.size() == keyPrefixLength + keySuffixLength + keyExtLength;
|
||||
// Remove prefix. Keep only the suffix and the ext
|
||||
var suffixAndExtIn = BufDataInput.create(keyBuf);
|
||||
suffixAndExtIn.skipBytes(keyPrefixLength);
|
||||
|
||||
suffixKeyLengthConsistency(suffixAndExtIn.available());
|
||||
return deserializeSuffix(suffixAndExtIn);
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<Entry<T, U>> getAllValues(@Nullable CompositeSnapshot snapshot) {
|
||||
return dictionary
|
||||
.getRange(resolveSnapshot(snapshot), rangeMono)
|
||||
.<Entry<T, U>>handle((serializedEntryToReceive, sink) -> {
|
||||
try {
|
||||
Entry<T, U> entry;
|
||||
try (var serializedEntry = serializedEntryToReceive.receive()) {
|
||||
var keyBuf = serializedEntry.getKeyUnsafe();
|
||||
assert keyBuf != null;
|
||||
assert keyBuf.readableBytes() == keyPrefixLength + keySuffixLength + keyExtLength;
|
||||
// Remove prefix. Keep only the suffix and the ext
|
||||
splitPrefix(keyBuf).close();
|
||||
suffixKeyLengthConsistency(keyBuf.readableBytes());
|
||||
T keySuffix = deserializeSuffix(keyBuf);
|
||||
public Stream<Entry<T, U>> getAllEntries(@Nullable CompositeSnapshot snapshot, boolean smallRange) {
|
||||
return getAllEntries(snapshot, smallRange, Map::entry);
|
||||
}
|
||||
|
||||
assert serializedEntry.getValueUnsafe() != null;
|
||||
U value = valueSerializer.deserialize(serializedEntry.getValueUnsafe());
|
||||
entry = Map.entry(keySuffix, value);
|
||||
}
|
||||
sink.next(entry);
|
||||
} catch (Throwable e) {
|
||||
sink.error(e);
|
||||
}
|
||||
})
|
||||
.doOnDiscard(Entry.class, uncastedEntry -> {
|
||||
if (uncastedEntry.getKey() instanceof Buffer byteBuf) {
|
||||
byteBuf.close();
|
||||
}
|
||||
if (uncastedEntry.getValue() instanceof Buffer byteBuf) {
|
||||
byteBuf.close();
|
||||
}
|
||||
@Override
|
||||
public Stream<U> getAllValues(@Nullable CompositeSnapshot snapshot, boolean smallRange) {
|
||||
return getAllEntries(snapshot, range, false, smallRange, (k, v) -> v);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Stream<T> getAllKeys(@Nullable CompositeSnapshot snapshot, boolean smallRange) {
|
||||
return getAllKeys(snapshot, range, false, smallRange);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all values
|
||||
* @param reverse if true, the results will go backwards from the specified key (inclusive)
|
||||
*/
|
||||
public Stream<Entry<T, U>> getAllEntries(@Nullable CompositeSnapshot snapshot,
|
||||
@Nullable T keyMin,
|
||||
@Nullable T keyMax,
|
||||
boolean reverse,
|
||||
boolean smallRange) {
|
||||
return getAllEntries(snapshot, keyMin, keyMax, reverse, smallRange, Map::entry);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all values
|
||||
* @param reverse if true, the results will go backwards from the specified key (inclusive)
|
||||
*/
|
||||
public <X> Stream<X> getAllEntries(@Nullable CompositeSnapshot snapshot,
|
||||
@Nullable T keyMin,
|
||||
@Nullable T keyMax,
|
||||
boolean reverse,
|
||||
boolean smallRange,
|
||||
BiFunction<T, U, X> mapper) {
|
||||
if (keyMin == null && keyMax == null) {
|
||||
return getAllEntries(snapshot, smallRange, mapper);
|
||||
} else {
|
||||
LLRange boundedRange = getPatchedRange(range, keyMin, keyMax);
|
||||
return getAllEntries(snapshot, boundedRange, reverse, smallRange, mapper);
|
||||
}
|
||||
}
|
||||
|
||||
private <X> Stream<X> getAllEntries(@Nullable CompositeSnapshot snapshot, boolean smallRange, BiFunction<T, U, X> mapper) {
|
||||
return getAllEntries(snapshot, range, false, smallRange, mapper);
|
||||
}
|
||||
|
||||
private <X> Stream<X> getAllEntries(@Nullable CompositeSnapshot snapshot,
|
||||
LLRange sliceRangeMono,
|
||||
boolean reverse,
|
||||
boolean smallRange,
|
||||
BiFunction<T, U, X> mapper) {
|
||||
return dictionary
|
||||
.getRange(resolveSnapshot(snapshot), sliceRangeMono, reverse, smallRange)
|
||||
.map((serializedEntry) -> {
|
||||
X entry;
|
||||
var keyBuf = serializedEntry.getKey();
|
||||
assert keyBuf != null;
|
||||
assert keyBuf.size() == keyPrefixLength + keySuffixLength + keyExtLength;
|
||||
|
||||
// Remove prefix. Keep only the suffix and the ext
|
||||
var suffixAndExtIn = BufDataInput.create(keyBuf);
|
||||
suffixAndExtIn.skipBytes(keyPrefixLength);
|
||||
|
||||
assert suffixKeyLengthConsistency(suffixAndExtIn.available());
|
||||
T keySuffix = deserializeSuffix(suffixAndExtIn);
|
||||
|
||||
assert serializedEntry.getValue() != null;
|
||||
U value = valueSerializer.deserialize(BufDataInput.create(serializedEntry.getValue()));
|
||||
entry = mapper.apply(keySuffix, value);
|
||||
return entry;
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<Entry<T, U>> setAllValuesAndGetPrevious(Flux<Entry<T, U>> entries) {
|
||||
return Flux.concat(
|
||||
this.getAllValues(null),
|
||||
dictionary.setRange(rangeMono, entries.handle(this::serializeEntrySink)).then(Mono.empty())
|
||||
public Stream<Entry<T, U>> setAllEntriesAndGetPrevious(Stream<Entry<T, U>> entries) {
|
||||
return resourceStream(
|
||||
() -> getAllEntries(null, false),
|
||||
() -> dictionary.setRange(range, entries.map(entry -> serializeEntry(entry)), false)
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Void> clear() {
|
||||
public void clear() {
|
||||
if (range.isAll()) {
|
||||
return dictionary.clear();
|
||||
dictionary.clear();
|
||||
} else if (range.isSingle()) {
|
||||
return dictionary
|
||||
.remove(Mono.fromCallable(range::getSingle), LLDictionaryResultType.VOID)
|
||||
.doOnNext(Send::close)
|
||||
.then();
|
||||
dictionary.remove(range.getSingleUnsafe(), LLDictionaryResultType.VOID);
|
||||
} else {
|
||||
return dictionary.setRange(rangeMono, Flux.empty());
|
||||
dictionary.setRange(range, Stream.empty(), false);
|
||||
}
|
||||
}
|
||||
|
||||
public static <T, U> List<Stream<UnsafeSSTEntry<T, U>>> getAllEntriesFastUnsafe(DatabaseMapDictionary<T, U> dict,
|
||||
boolean disableRocksdbChecks,
|
||||
BiConsumer<UnsafeRawSSTEntry<T, U>, Throwable> deserializationErrorHandler) {
|
||||
try {
|
||||
var liveFiles = ((LLLocalDictionary) dict.dictionary).getAllLiveFiles();
|
||||
return Lists.transform(liveFiles, file -> file.iterate(new SSTRangeFull(), disableRocksdbChecks)
|
||||
.map(state -> switch (state) {
|
||||
case RocksDBFileIterationStateBegin rocksDBFileIterationStateBegin:
|
||||
yield null;
|
||||
case RocksDBFileIterationStateEnd rocksDBFileIterationStateEnd:
|
||||
yield null;
|
||||
case RocksDBFileIterationStateKey rocksDBFileIterationStateKey:
|
||||
yield switch (rocksDBFileIterationStateKey.state()) {
|
||||
case RocksDBFileIterationStateKeyError e -> null;
|
||||
case RocksDBFileIterationStateKeyOk rocksDBFileIterationStateKeyOk -> {
|
||||
var key = rocksDBFileIterationStateKey.key();
|
||||
var value = rocksDBFileIterationStateKeyOk.value();
|
||||
try {
|
||||
var deserializedKey = dict.deserializeSuffix(BufDataInput.create(key));
|
||||
var deserializedValue = dict.deserializeValue(value);
|
||||
yield new UnsafeSSTEntry<>(file,
|
||||
deserializedKey,
|
||||
deserializedValue,
|
||||
key,
|
||||
value,
|
||||
k -> dict.deserializeSuffix(BufDataInput.create(k)),
|
||||
dict::deserializeValue
|
||||
);
|
||||
} catch (Throwable t) {
|
||||
if (deserializationErrorHandler != null) {
|
||||
deserializationErrorHandler.accept(new UnsafeRawSSTEntry<>(file,
|
||||
key,
|
||||
value,
|
||||
k -> dict.deserializeSuffix(BufDataInput.create(k)),
|
||||
dict::deserializeValue
|
||||
), t);
|
||||
yield null;
|
||||
} else {
|
||||
throw t;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
};
|
||||
})
|
||||
.filter(Objects::nonNull));
|
||||
} catch (RocksDBException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1,139 +1,118 @@
|
||||
package it.cavallium.dbengine.database.collections;
|
||||
|
||||
import io.net5.buffer.api.Buffer;
|
||||
import io.net5.buffer.api.BufferAllocator;
|
||||
import io.net5.buffer.api.Drop;
|
||||
import io.net5.buffer.api.Owned;
|
||||
import io.net5.buffer.api.Resource;
|
||||
import io.net5.buffer.api.Send;
|
||||
import it.cavallium.dbengine.client.BadBlock;
|
||||
import static it.cavallium.dbengine.utils.StreamUtils.resourceStream;
|
||||
|
||||
import it.cavallium.buffer.Buf;
|
||||
import it.cavallium.buffer.BufDataInput;
|
||||
import it.cavallium.buffer.BufDataOutput;
|
||||
import it.cavallium.dbengine.client.DbProgress;
|
||||
import it.cavallium.dbengine.client.CompositeSnapshot;
|
||||
import it.cavallium.dbengine.client.SSTVerificationProgress;
|
||||
import it.cavallium.dbengine.database.LLDictionary;
|
||||
import it.cavallium.dbengine.database.LLDictionaryResultType;
|
||||
import it.cavallium.dbengine.database.LLRange;
|
||||
import it.cavallium.dbengine.database.LLSnapshot;
|
||||
import it.cavallium.dbengine.database.LLUtils;
|
||||
import io.net5.buffer.api.internal.ResourceSupport;
|
||||
import it.cavallium.dbengine.database.SubStageEntry;
|
||||
import it.cavallium.dbengine.database.UpdateMode;
|
||||
import it.cavallium.dbengine.database.collections.DatabaseEmpty.Nothing;
|
||||
import it.cavallium.dbengine.database.serialization.SerializationException;
|
||||
import it.cavallium.dbengine.database.serialization.Serializer;
|
||||
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
|
||||
import java.util.Map;
|
||||
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CompletionException;
|
||||
import java.util.concurrent.ForkJoinPool;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.stream.Stream;
|
||||
import org.apache.commons.lang3.function.TriFunction;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import org.warp.commonutils.log.Logger;
|
||||
import org.warp.commonutils.log.LoggerFactory;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
import org.jetbrains.annotations.VisibleForTesting;
|
||||
|
||||
// todo: implement optimized methods (which?)
|
||||
public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extends ResourceSupport<DatabaseStage<Map<T, U>>, DatabaseMapDictionaryDeep<T, U, US>>
|
||||
implements DatabaseStageMap<T, U, US> {
|
||||
public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> implements DatabaseStageMap<T, U, US> {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(DatabaseMapDictionaryDeep.class);
|
||||
|
||||
private static final Drop<DatabaseMapDictionaryDeep<?, ?, ?>> DROP = new Drop<>() {
|
||||
@Override
|
||||
public void drop(DatabaseMapDictionaryDeep<?, ?, ?> obj) {
|
||||
try {
|
||||
if (obj.range != null) {
|
||||
obj.range.close();
|
||||
}
|
||||
} catch (Throwable ex) {
|
||||
logger.error("Failed to close range", ex);
|
||||
}
|
||||
try {
|
||||
if (obj.keyPrefix != null) {
|
||||
obj.keyPrefix.close();
|
||||
}
|
||||
} catch (Throwable ex) {
|
||||
logger.error("Failed to close keyPrefix", ex);
|
||||
}
|
||||
try {
|
||||
if (obj.onClose != null) {
|
||||
obj.onClose.run();
|
||||
}
|
||||
} catch (Throwable ex) {
|
||||
logger.error("Failed to close onClose", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Drop<DatabaseMapDictionaryDeep<?, ?, ?>> fork() {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void attach(DatabaseMapDictionaryDeep<?, ?, ?> obj) {
|
||||
|
||||
}
|
||||
};
|
||||
private static final Logger LOG = LogManager.getLogger(DatabaseMapDictionaryDeep.class);
|
||||
|
||||
protected final LLDictionary dictionary;
|
||||
private final BufferAllocator alloc;
|
||||
private final AtomicLong totalZeroBytesErrors = new AtomicLong();
|
||||
protected final SubStageGetter<U, US> subStageGetter;
|
||||
protected final SerializerFixedBinaryLength<T> keySuffixSerializer;
|
||||
protected final int keyPrefixLength;
|
||||
protected final int keySuffixLength;
|
||||
protected final int keyExtLength;
|
||||
protected final Mono<Send<LLRange>> rangeMono;
|
||||
protected final LLRange range;
|
||||
|
||||
protected LLRange range;
|
||||
protected Buffer keyPrefix;
|
||||
protected Runnable onClose;
|
||||
protected Buf keyPrefix;
|
||||
|
||||
private static void incrementPrefix(Buffer prefix, int prefixLength) {
|
||||
assert prefix.readableBytes() >= prefixLength;
|
||||
assert prefix.readerOffset() == 0;
|
||||
final var originalKeyLength = prefix.readableBytes();
|
||||
private static void incrementPrefix(Buf modifiablePrefix, int prefixLength) {
|
||||
assert modifiablePrefix.size() >= prefixLength;
|
||||
final var originalKeyLength = modifiablePrefix.size();
|
||||
boolean overflowed = true;
|
||||
final int ff = 0xFF;
|
||||
int writtenBytes = 0;
|
||||
for (int i = prefixLength - 1; i >= 0; i--) {
|
||||
int iByte = prefix.getUnsignedByte(i);
|
||||
int iByte = Byte.toUnsignedInt(modifiablePrefix.getByte(i));
|
||||
if (iByte != ff) {
|
||||
prefix.setUnsignedByte(i, iByte + 1);
|
||||
modifiablePrefix.set(i, (byte) (iByte + 1));
|
||||
writtenBytes++;
|
||||
overflowed = false;
|
||||
break;
|
||||
} else {
|
||||
prefix.setUnsignedByte(i, 0x00);
|
||||
modifiablePrefix.set(i, (byte) 0x00);
|
||||
writtenBytes++;
|
||||
}
|
||||
}
|
||||
assert prefixLength - writtenBytes >= 0;
|
||||
|
||||
if (overflowed) {
|
||||
assert prefix.writerOffset() == originalKeyLength;
|
||||
prefix.ensureWritable(1, 1, true);
|
||||
prefix.writerOffset(originalKeyLength + 1);
|
||||
modifiablePrefix.add((byte) 0);
|
||||
for (int i = 0; i < originalKeyLength; i++) {
|
||||
prefix.setUnsignedByte(i, 0xFF);
|
||||
modifiablePrefix.set(i, (byte) 0xFF);
|
||||
}
|
||||
prefix.setUnsignedByte(originalKeyLength, (byte) 0x00);
|
||||
modifiablePrefix.set(originalKeyLength, (byte) 0x00);
|
||||
}
|
||||
}
|
||||
|
||||
static void firstRangeKey(Buffer prefixKey, int prefixLength, int suffixLength, int extLength) {
|
||||
zeroFillKeySuffixAndExt(prefixKey, prefixLength, suffixLength, extLength);
|
||||
@VisibleForTesting
|
||||
public static Buf firstRangeKey(Buf prefixKey, int prefixLength, Buf suffixAndExtZeroes) {
|
||||
return createFullKeyWithEmptySuffixAndExt(prefixKey, prefixLength, suffixAndExtZeroes);
|
||||
}
|
||||
|
||||
static void nextRangeKey(Buffer prefixKey, int prefixLength, int suffixLength, int extLength) {
|
||||
zeroFillKeySuffixAndExt(prefixKey, prefixLength, suffixLength, extLength);
|
||||
incrementPrefix(prefixKey, prefixLength);
|
||||
@VisibleForTesting
|
||||
public static Buf nextRangeKey(Buf prefixKey, int prefixLength, Buf suffixAndExtZeroes) {
|
||||
Buf modifiablePrefixKey = createFullKeyWithEmptySuffixAndExt(prefixKey, prefixLength, suffixAndExtZeroes);
|
||||
incrementPrefix(modifiablePrefixKey, prefixLength);
|
||||
return modifiablePrefixKey;
|
||||
}
|
||||
|
||||
protected static void zeroFillKeySuffixAndExt(@NotNull Buffer prefixKey,
|
||||
int prefixLength, int suffixLength, int extLength) {
|
||||
private static Buf createFullKeyWithEmptySuffixAndExt(Buf prefixKey, int prefixLength, Buf suffixAndExtZeroes) {
|
||||
var modifiablePrefixKey = Buf.create(prefixLength + suffixAndExtZeroes.size());
|
||||
if (prefixKey != null) {
|
||||
modifiablePrefixKey.addAll(prefixKey);
|
||||
}
|
||||
assert prefixKey != null || prefixLength == 0 : "Prefix length is " + prefixLength + " but the prefix key is null";
|
||||
zeroFillKeySuffixAndExt(modifiablePrefixKey, prefixLength, suffixAndExtZeroes);
|
||||
return modifiablePrefixKey;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param modifiablePrefixKey This field content will be modified
|
||||
*/
|
||||
protected static void zeroFillKeySuffixAndExt(@NotNull Buf modifiablePrefixKey, int prefixLength, Buf suffixAndExtZeroes) {
|
||||
//noinspection UnnecessaryLocalVariable
|
||||
var result = prefixKey;
|
||||
assert result.readableBytes() == prefixLength;
|
||||
assert suffixLength > 0;
|
||||
assert extLength >= 0;
|
||||
result.ensureWritable(suffixLength + extLength, suffixLength + extLength, true);
|
||||
for (int i = 0; i < suffixLength + extLength; i++) {
|
||||
result.writeByte((byte) 0x0);
|
||||
}
|
||||
var result = modifiablePrefixKey;
|
||||
var suffixLengthAndExtLength = suffixAndExtZeroes.size();
|
||||
assert result.size() == prefixLength;
|
||||
assert suffixLengthAndExtLength > 0 : "Suffix length + ext length is < 0: " + suffixLengthAndExtLength;
|
||||
result.size(prefixLength);
|
||||
modifiablePrefixKey.addAll(suffixAndExtZeroes);
|
||||
assert modifiablePrefixKey.size() == prefixLength + suffixAndExtZeroes.size() : "Result buffer size is wrong";
|
||||
}
|
||||
|
||||
/**
|
||||
@ -141,107 +120,74 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extend
|
||||
*/
|
||||
@Deprecated
|
||||
public static <T, U> DatabaseMapDictionaryDeep<T, U, DatabaseStageEntry<U>> simple(LLDictionary dictionary,
|
||||
SerializerFixedBinaryLength<T> keySerializer, SubStageGetterSingle<U> subStageGetter,
|
||||
Runnable onClose) {
|
||||
return new DatabaseMapDictionaryDeep<>(dictionary, null, keySerializer,
|
||||
subStageGetter, 0, onClose);
|
||||
SerializerFixedBinaryLength<T> keySerializer, SubStageGetterSingle<U> subStageGetter) {
|
||||
return new DatabaseMapDictionaryDeep<>(dictionary, null, keySerializer, subStageGetter, 0);
|
||||
}
|
||||
|
||||
public static <T, U, US extends DatabaseStage<U>> DatabaseMapDictionaryDeep<T, U, US> deepTail(
|
||||
LLDictionary dictionary, SerializerFixedBinaryLength<T> keySerializer, int keyExtLength,
|
||||
SubStageGetter<U, US> subStageGetter, Runnable onClose) {
|
||||
return new DatabaseMapDictionaryDeep<>(dictionary, null, keySerializer,
|
||||
subStageGetter, keyExtLength, onClose);
|
||||
SubStageGetter<U, US> subStageGetter) {
|
||||
return new DatabaseMapDictionaryDeep<>(dictionary, null, keySerializer, subStageGetter, keyExtLength);
|
||||
}
|
||||
|
||||
public static <T, U, US extends DatabaseStage<U>> DatabaseMapDictionaryDeep<T, U, US> deepIntermediate(
|
||||
LLDictionary dictionary, Buffer prefixKey, SerializerFixedBinaryLength<T> keySuffixSerializer,
|
||||
SubStageGetter<U, US> subStageGetter, int keyExtLength, Runnable onClose) {
|
||||
return new DatabaseMapDictionaryDeep<>(dictionary, prefixKey, keySuffixSerializer, subStageGetter,
|
||||
keyExtLength, onClose);
|
||||
LLDictionary dictionary, Buf prefixKey, SerializerFixedBinaryLength<T> keySuffixSerializer,
|
||||
SubStageGetter<U, US> subStageGetter, int keyExtLength) {
|
||||
return new DatabaseMapDictionaryDeep<>(dictionary, prefixKey, keySuffixSerializer, subStageGetter, keyExtLength);
|
||||
}
|
||||
|
||||
@SuppressWarnings({"unchecked", "rawtypes"})
|
||||
protected DatabaseMapDictionaryDeep(LLDictionary dictionary, @Nullable Buffer prefixKey,
|
||||
SerializerFixedBinaryLength<T> keySuffixSerializer, SubStageGetter<U, US> subStageGetter, int keyExtLength,
|
||||
Runnable onClose) {
|
||||
super((Drop<DatabaseMapDictionaryDeep<T, U, US>>) (Drop) DROP);
|
||||
try {
|
||||
this.dictionary = dictionary;
|
||||
this.alloc = dictionary.getAllocator();
|
||||
this.subStageGetter = subStageGetter;
|
||||
this.keySuffixSerializer = keySuffixSerializer;
|
||||
assert prefixKey == null || prefixKey.isAccessible();
|
||||
this.keyPrefixLength = prefixKey == null ? 0 : prefixKey.readableBytes();
|
||||
this.keySuffixLength = keySuffixSerializer.getSerializedBinaryLength();
|
||||
this.keyExtLength = keyExtLength;
|
||||
var firstKey = prefixKey == null ? alloc.allocate(keyPrefixLength + keySuffixLength + keyExtLength)
|
||||
: prefixKey.copy();
|
||||
try {
|
||||
firstRangeKey(firstKey, keyPrefixLength, keySuffixLength, keyExtLength);
|
||||
var nextRangeKey = prefixKey == null ? alloc.allocate(keyPrefixLength + keySuffixLength + keyExtLength)
|
||||
: prefixKey.copy();
|
||||
try {
|
||||
nextRangeKey(nextRangeKey, keyPrefixLength, keySuffixLength, keyExtLength);
|
||||
assert prefixKey == null || prefixKey.isAccessible();
|
||||
assert keyPrefixLength == 0 || !LLUtils.equals(firstKey, nextRangeKey);
|
||||
if (keyPrefixLength == 0) {
|
||||
this.range = LLRange.all();
|
||||
firstKey.close();
|
||||
nextRangeKey.close();
|
||||
} else {
|
||||
this.range = LLRange.ofUnsafe(firstKey, nextRangeKey);
|
||||
}
|
||||
this.rangeMono = LLUtils.lazyRetainRange(this.range);
|
||||
assert subStageKeysConsistency(keyPrefixLength + keySuffixLength + keyExtLength);
|
||||
} catch (Throwable t) {
|
||||
nextRangeKey.close();
|
||||
throw t;
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
firstKey.close();
|
||||
throw t;
|
||||
}
|
||||
|
||||
this.keyPrefix = prefixKey;
|
||||
this.onClose = onClose;
|
||||
} catch (Throwable t) {
|
||||
if (prefixKey != null && prefixKey.isAccessible()) {
|
||||
prefixKey.close();
|
||||
}
|
||||
throw t;
|
||||
protected DatabaseMapDictionaryDeep(LLDictionary dictionary, @Nullable Buf prefixKey,
|
||||
SerializerFixedBinaryLength<T> keySuffixSerializer, SubStageGetter<U, US> subStageGetter, int keyExtLength) {
|
||||
this.dictionary = dictionary;
|
||||
this.subStageGetter = subStageGetter;
|
||||
this.keySuffixSerializer = keySuffixSerializer;
|
||||
this.keyPrefixLength = prefixKey != null ? prefixKey.size() : 0;
|
||||
this.keySuffixLength = keySuffixSerializer.getSerializedBinaryLength();
|
||||
this.keyExtLength = keyExtLength;
|
||||
var keySuffixAndExtZeroBuffer = Buf.createZeroes(keySuffixLength + keyExtLength);
|
||||
assert keySuffixAndExtZeroBuffer.size() == keySuffixLength + keyExtLength :
|
||||
"Key suffix and ext zero buffer readable length is not equal"
|
||||
+ " to the key suffix length + key ext length. keySuffixAndExtZeroBuffer="
|
||||
+ keySuffixAndExtZeroBuffer.size() + " keySuffixLength=" + keySuffixLength + " keyExtLength="
|
||||
+ keyExtLength;
|
||||
assert keySuffixAndExtZeroBuffer.size() > 0;
|
||||
var firstKey = firstRangeKey(prefixKey, keyPrefixLength, keySuffixAndExtZeroBuffer);
|
||||
var nextRangeKey = nextRangeKey(prefixKey, keyPrefixLength, keySuffixAndExtZeroBuffer);
|
||||
assert keyPrefixLength == 0 || !LLUtils.equals(firstKey, nextRangeKey);
|
||||
if (keyPrefixLength == 0) {
|
||||
this.range = LLRange.all();
|
||||
} else {
|
||||
this.range = LLRange.of(firstKey, nextRangeKey);
|
||||
}
|
||||
}
|
||||
assert subStageKeysConsistency(keyPrefixLength + keySuffixLength + keyExtLength);
|
||||
|
||||
@SuppressWarnings({"unchecked", "rawtypes"})
|
||||
this.keyPrefix = prefixKey;
|
||||
}
|
||||
private DatabaseMapDictionaryDeep(LLDictionary dictionary,
|
||||
BufferAllocator alloc,
|
||||
SubStageGetter<U, US> subStageGetter,
|
||||
SerializerFixedBinaryLength<T> keySuffixSerializer,
|
||||
int keyPrefixLength,
|
||||
int keySuffixLength,
|
||||
int keyExtLength,
|
||||
Mono<Send<LLRange>> rangeMono,
|
||||
Send<LLRange> range,
|
||||
Send<Buffer> keyPrefix,
|
||||
Runnable onClose) {
|
||||
super((Drop<DatabaseMapDictionaryDeep<T,U,US>>) (Drop) DROP);
|
||||
LLRange range,
|
||||
Buf keyPrefix) {
|
||||
this.dictionary = dictionary;
|
||||
this.alloc = alloc;
|
||||
this.subStageGetter = subStageGetter;
|
||||
this.keySuffixSerializer = keySuffixSerializer;
|
||||
this.keyPrefixLength = keyPrefixLength;
|
||||
this.keySuffixLength = keySuffixLength;
|
||||
this.keyExtLength = keyExtLength;
|
||||
this.rangeMono = rangeMono;
|
||||
this.range = range;
|
||||
|
||||
this.range = range.receive();
|
||||
this.keyPrefix = keyPrefix.receive();
|
||||
this.onClose = onClose;
|
||||
this.keyPrefix = keyPrefix;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
protected boolean suffixKeyLengthConsistency(int keySuffixLength) {
|
||||
assert
|
||||
this.keySuffixLength == keySuffixLength :
|
||||
"Key suffix length is " + keySuffixLength + ", but it should be " + this.keySuffixLength + " bytes long";
|
||||
//noinspection ConstantValue
|
||||
return this.keySuffixLength == keySuffixLength;
|
||||
}
|
||||
|
||||
@ -256,16 +202,39 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extend
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes the prefix from the key
|
||||
* @return the prefix
|
||||
*/
|
||||
protected Buffer splitPrefix(Buffer key) {
|
||||
assert key.readableBytes() == keyPrefixLength + keySuffixLength + keyExtLength
|
||||
|| key.readableBytes() == keyPrefixLength + keySuffixLength;
|
||||
var prefix = key.readSplit(this.keyPrefixLength);
|
||||
assert key.readableBytes() == keySuffixLength + keyExtLength
|
||||
|| key.readableBytes() == keySuffixLength;
|
||||
return prefix;
|
||||
protected Buf prefixSubList(Buf key) {
|
||||
assert key.size() == keyPrefixLength + keySuffixLength + keyExtLength
|
||||
|| key.size() == keyPrefixLength + keySuffixLength;
|
||||
return key.subList(0, this.keyPrefixLength);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the suffix
|
||||
*/
|
||||
protected Buf suffixSubList(Buf key) {
|
||||
assert key.size() == keyPrefixLength + keySuffixLength + keyExtLength
|
||||
|| key.size() == keyPrefixLength + keySuffixLength;
|
||||
return key.subList(this.keyPrefixLength, keyPrefixLength + keySuffixLength);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the suffix
|
||||
*/
|
||||
protected Buf suffixAndExtSubList(Buf key) {
|
||||
assert key.size() == keyPrefixLength + keySuffixLength + keyExtLength
|
||||
|| key.size() == keyPrefixLength + keySuffixLength;
|
||||
return key.subList(this.keyPrefixLength, key.size());
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the ext
|
||||
*/
|
||||
protected Buf extSubList(Buf key) {
|
||||
assert key.size() == keyPrefixLength + keySuffixLength + keyExtLength
|
||||
|| key.size() == keyPrefixLength + keySuffixLength;
|
||||
return key.subList(this.keyPrefixLength + this.keySuffixLength, key.size());
|
||||
}
|
||||
|
||||
protected LLSnapshot resolveSnapshot(@Nullable CompositeSnapshot snapshot) {
|
||||
@ -277,75 +246,46 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extend
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Long> leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
|
||||
return dictionary.sizeRange(resolveSnapshot(snapshot), rangeMono, fast);
|
||||
public long leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
|
||||
return dictionary.sizeRange(resolveSnapshot(snapshot), range, fast);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Boolean> isEmpty(@Nullable CompositeSnapshot snapshot) {
|
||||
return dictionary.isRangeEmpty(resolveSnapshot(snapshot), rangeMono);
|
||||
public boolean isEmpty(@Nullable CompositeSnapshot snapshot) {
|
||||
return dictionary.isRangeEmpty(resolveSnapshot(snapshot), range, false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<US> at(@Nullable CompositeSnapshot snapshot, T keySuffix) {
|
||||
var suffixKeyWithoutExt = Mono.fromCallable(() -> {
|
||||
try (var keyWithoutExtBuf = keyPrefix == null
|
||||
? alloc.allocate(keySuffixLength + keyExtLength) : keyPrefix.copy()) {
|
||||
keyWithoutExtBuf.ensureWritable(keySuffixLength + keyExtLength);
|
||||
serializeSuffix(keySuffix, keyWithoutExtBuf);
|
||||
return keyWithoutExtBuf.send();
|
||||
}
|
||||
});
|
||||
return this.subStageGetter
|
||||
.subStage(dictionary, snapshot, suffixKeyWithoutExt)
|
||||
.transform(LLUtils::handleDiscard)
|
||||
.doOnDiscard(DatabaseStage.class, DatabaseStage::close);
|
||||
public @NotNull US at(@Nullable CompositeSnapshot snapshot, T keySuffix) {
|
||||
BufDataOutput bufOutput = BufDataOutput.createLimited(keyPrefixLength + keySuffixLength + keyExtLength);
|
||||
if (keyPrefix != null) {
|
||||
bufOutput.writeBytes(keyPrefix);
|
||||
}
|
||||
serializeSuffixTo(keySuffix, bufOutput);
|
||||
return this.subStageGetter.subStage(dictionary, snapshot, bufOutput.asList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<UpdateMode> getUpdateMode() {
|
||||
public UpdateMode getUpdateMode() {
|
||||
return dictionary.getUpdateMode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<BadBlock> badBlocks() {
|
||||
return dictionary.badBlocks(rangeMono);
|
||||
public Stream<DbProgress<SSTVerificationProgress>> verifyChecksum() {
|
||||
return dictionary.verifyChecksum(range);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<Entry<T, US>> getAllStages(@Nullable CompositeSnapshot snapshot) {
|
||||
public Stream<SubStageEntry<T, US>> getAllStages(@Nullable CompositeSnapshot snapshot, boolean smallRange) {
|
||||
return dictionary
|
||||
.getRangeKeyPrefixes(resolveSnapshot(snapshot), rangeMono, keyPrefixLength + keySuffixLength)
|
||||
.flatMapSequential(groupKeyWithoutExtSend_ -> Mono.using(
|
||||
groupKeyWithoutExtSend_::receive,
|
||||
groupKeyWithoutExtSend -> this.subStageGetter
|
||||
.subStage(dictionary, snapshot, Mono.fromCallable(() -> groupKeyWithoutExtSend.copy().send()))
|
||||
.<Entry<T, US>>handle((us, sink) -> {
|
||||
T deserializedSuffix;
|
||||
try {
|
||||
deserializedSuffix = this.deserializeSuffix(splitGroupSuffix(groupKeyWithoutExtSend));
|
||||
sink.next(Map.entry(deserializedSuffix, us));
|
||||
} catch (SerializationException ex) {
|
||||
sink.error(ex);
|
||||
}
|
||||
}),
|
||||
Resource::close
|
||||
))
|
||||
.transform(LLUtils::handleDiscard);
|
||||
}
|
||||
|
||||
/**
|
||||
* Split the input. The input will become the ext, the returned data will be the group suffix
|
||||
* @param groupKey group key, will become ext
|
||||
* @return group suffix
|
||||
*/
|
||||
private Buffer splitGroupSuffix(@NotNull Buffer groupKey) {
|
||||
assert subStageKeysConsistency(groupKey.readableBytes())
|
||||
|| subStageKeysConsistency(groupKey.readableBytes() + keyExtLength);
|
||||
this.splitPrefix(groupKey).close();
|
||||
assert subStageKeysConsistency(keyPrefixLength + groupKey.readableBytes())
|
||||
|| subStageKeysConsistency(keyPrefixLength + groupKey.readableBytes() + keyExtLength);
|
||||
return groupKey.readSplit(keySuffixLength);
|
||||
.getRangeKeyPrefixes(resolveSnapshot(snapshot), range, keyPrefixLength + keySuffixLength, smallRange)
|
||||
.map(groupKeyWithoutExt -> {
|
||||
T deserializedSuffix;
|
||||
var splittedGroupSuffix = suffixSubList(groupKeyWithoutExt);
|
||||
deserializedSuffix = this.deserializeSuffix(BufDataInput.create(splittedGroupSuffix));
|
||||
return new SubStageEntry<>(deserializedSuffix,
|
||||
this.subStageGetter.subStage(dictionary, snapshot, groupKeyWithoutExt));
|
||||
});
|
||||
}
|
||||
|
||||
private boolean subStageKeysConsistency(int totalKeyLength) {
|
||||
@ -361,83 +301,151 @@ public class DatabaseMapDictionaryDeep<T, U, US extends DatabaseStage<U>> extend
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<Entry<T, U>> setAllValuesAndGetPrevious(Flux<Entry<T, U>> entries) {
|
||||
return this
|
||||
.getAllValues(null)
|
||||
.concatWith(this
|
||||
.clear()
|
||||
.then(this.putMulti(entries))
|
||||
.then(Mono.empty())
|
||||
);
|
||||
public void setAllEntries(Stream<Entry<T, U>> entries) {
|
||||
this.clear();
|
||||
this.putMulti(entries);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Void> clear() {
|
||||
return Mono
|
||||
.defer(() -> {
|
||||
if (range.isAll()) {
|
||||
return dictionary.clear();
|
||||
} else if (range.isSingle()) {
|
||||
return dictionary
|
||||
.remove(Mono.fromCallable(range::getSingle), LLDictionaryResultType.VOID)
|
||||
.doOnNext(Send::close)
|
||||
.then();
|
||||
} else {
|
||||
return dictionary.setRange(rangeMono, Flux.empty());
|
||||
public Stream<Entry<T, U>> setAllEntriesAndGetPrevious(Stream<Entry<T, U>> entries) {
|
||||
return resourceStream(() -> this.getAllEntries(null, false), () -> setAllEntries(entries));
|
||||
}
|
||||
|
||||
@Override
|
||||
public ForkJoinPool getDbReadPool() {
|
||||
return dictionary.getDbReadPool();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ForkJoinPool getDbWritePool() {
|
||||
return dictionary.getDbWritePool();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clear() {
|
||||
if (range.isAll()) {
|
||||
dictionary.clear();
|
||||
} else if (range.isSingle()) {
|
||||
dictionary.remove(range.getSingleUnsafe(), LLDictionaryResultType.VOID);
|
||||
} else {
|
||||
dictionary.setRange(range, Stream.empty(), false);
|
||||
}
|
||||
}
|
||||
|
||||
protected T deserializeSuffix(@NotNull BufDataInput keySuffix) throws SerializationException {
|
||||
assert suffixKeyLengthConsistency(keySuffix.available());
|
||||
return keySuffixSerializer.deserialize(keySuffix);
|
||||
}
|
||||
|
||||
protected void serializeSuffixTo(T keySuffix, BufDataOutput output) throws SerializationException {
|
||||
var beforeWriterOffset = output.size();
|
||||
assert beforeWriterOffset == keyPrefixLength;
|
||||
assert keySuffixSerializer.getSerializedBinaryLength() == keySuffixLength
|
||||
: "Invalid key suffix serializer length: " + keySuffixSerializer.getSerializedBinaryLength()
|
||||
+ ". Expected: " + keySuffixLength;
|
||||
keySuffixSerializer.serialize(keySuffix, output);
|
||||
var afterWriterOffset = output.size();
|
||||
assert suffixKeyLengthConsistency(afterWriterOffset - beforeWriterOffset)
|
||||
: "Invalid key suffix length: " + (afterWriterOffset - beforeWriterOffset) + ". Expected: " + keySuffixLength;
|
||||
}
|
||||
|
||||
public static <K1, K2, V, R> Stream<R> getAllLeaves2(DatabaseMapDictionaryDeep<K1, Object2ObjectSortedMap<K2, V>, ? extends DatabaseStageMap<K2, V, DatabaseStageEntry<V>>> deepMap,
|
||||
CompositeSnapshot snapshot,
|
||||
TriFunction<K1, K2, V, R> merger,
|
||||
@Nullable K1 savedProgressKey1) {
|
||||
var keySuffix1Serializer = deepMap.keySuffixSerializer;
|
||||
SerializerFixedBinaryLength<?> keySuffix2Serializer;
|
||||
Serializer<?> valueSerializer;
|
||||
boolean isHashed;
|
||||
boolean isHashedSet;
|
||||
if (deepMap.subStageGetter instanceof SubStageGetterMap subStageGetterMap) {
|
||||
isHashed = false;
|
||||
isHashedSet = false;
|
||||
keySuffix2Serializer = subStageGetterMap.keySerializer;
|
||||
valueSerializer = subStageGetterMap.valueSerializer;
|
||||
} else if (deepMap.subStageGetter instanceof SubStageGetterHashMap subStageGetterHashMap) {
|
||||
isHashed = true;
|
||||
isHashedSet = false;
|
||||
keySuffix2Serializer = subStageGetterHashMap.keyHashSerializer;
|
||||
|
||||
//noinspection unchecked
|
||||
ValueWithHashSerializer<K2, V> valueWithHashSerializer = new ValueWithHashSerializer<>(
|
||||
(Serializer<K2>) subStageGetterHashMap.keySerializer,
|
||||
(Serializer<V>) subStageGetterHashMap.valueSerializer
|
||||
);
|
||||
valueSerializer = new ValuesSetSerializer<>(valueWithHashSerializer);
|
||||
} else if (deepMap.subStageGetter instanceof SubStageGetterHashSet subStageGetterHashSet) {
|
||||
isHashed = true;
|
||||
isHashedSet = true;
|
||||
keySuffix2Serializer = subStageGetterHashSet.keyHashSerializer;
|
||||
|
||||
//noinspection unchecked
|
||||
valueSerializer = new ValuesSetSerializer<K2>(subStageGetterHashSet.keySerializer);
|
||||
} else {
|
||||
throw new IllegalArgumentException();
|
||||
}
|
||||
|
||||
var firstKey = Optional.ofNullable(savedProgressKey1);
|
||||
var fullRange = deepMap.range;
|
||||
|
||||
|
||||
LLRange range;
|
||||
if (firstKey.isPresent()) {
|
||||
var key1Buf = BufDataOutput.create(keySuffix1Serializer.getSerializedBinaryLength());
|
||||
keySuffix1Serializer.serialize(firstKey.get(), key1Buf);
|
||||
range = LLRange.of(key1Buf.asList(), fullRange.getMax());
|
||||
} else {
|
||||
range = fullRange;
|
||||
}
|
||||
|
||||
return deepMap.dictionary.getRange(deepMap.resolveSnapshot(snapshot), range, false, false)
|
||||
.flatMap(entry -> {
|
||||
K1 key1 = null;
|
||||
Object key2 = null;
|
||||
try {
|
||||
var keyBuf = entry.getKey();
|
||||
var valueBuf = entry.getValue();
|
||||
try {
|
||||
assert keyBuf != null;
|
||||
var suffix1And2 = BufDataInput.create(keyBuf.subList(deepMap.keyPrefixLength, deepMap.keyPrefixLength + deepMap.keySuffixLength + deepMap.keyExtLength));
|
||||
key1 = keySuffix1Serializer.deserialize(suffix1And2);
|
||||
key2 = keySuffix2Serializer.deserialize(suffix1And2);
|
||||
assert valueBuf != null;
|
||||
Object value = valueSerializer.deserialize(BufDataInput.create(valueBuf));
|
||||
if (isHashedSet) {
|
||||
//noinspection unchecked
|
||||
Set<K2> set = (Set<K2>) value;
|
||||
K1 finalKey1 = key1;
|
||||
//noinspection unchecked
|
||||
return set.stream().map(e -> merger.apply(finalKey1, e, (V) Nothing.INSTANCE));
|
||||
} else if (isHashed) {
|
||||
//noinspection unchecked
|
||||
Set<Entry<K2, V>> set = (Set<Entry<K2, V>>) value;
|
||||
K1 finalKey1 = key1;
|
||||
return set.stream().map(e -> merger.apply(finalKey1, e.getKey(), e.getValue()));
|
||||
} else {
|
||||
//noinspection unchecked
|
||||
return Stream.of(merger.apply(key1, (K2) key2, (V) value));
|
||||
}
|
||||
} catch (IndexOutOfBoundsException ex) {
|
||||
var exMessage = ex.getMessage();
|
||||
if (exMessage != null && exMessage.contains("read 0 to 0, write 0 to ")) {
|
||||
var totalZeroBytesErrors = deepMap.totalZeroBytesErrors.incrementAndGet();
|
||||
if (totalZeroBytesErrors < 512 || totalZeroBytesErrors % 10000 == 0) {
|
||||
LOG.error("Unexpected zero-bytes value at " + deepMap.dictionary.getDatabaseName()
|
||||
+ ":" + deepMap.dictionary.getColumnName()
|
||||
+ ":[" + key1
|
||||
+ ":" + key2
|
||||
+ "](" + LLUtils.toStringSafe(keyBuf) + ") total=" + totalZeroBytesErrors);
|
||||
}
|
||||
return Stream.empty();
|
||||
} else {
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
} catch (SerializationException ex) {
|
||||
throw new CompletionException(ex);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
//todo: temporary wrapper. convert the whole class to buffers
|
||||
protected T deserializeSuffix(@NotNull Buffer keySuffix) throws SerializationException {
|
||||
assert suffixKeyLengthConsistency(keySuffix.readableBytes());
|
||||
var result = keySuffixSerializer.deserialize(keySuffix);
|
||||
assert keyPrefix == null || keyPrefix.isAccessible();
|
||||
return result;
|
||||
}
|
||||
|
||||
//todo: temporary wrapper. convert the whole class to buffers
|
||||
protected void serializeSuffix(T keySuffix, Buffer output) throws SerializationException {
|
||||
output.ensureWritable(keySuffixLength);
|
||||
var beforeWriterOffset = output.writerOffset();
|
||||
keySuffixSerializer.serialize(keySuffix, output);
|
||||
var afterWriterOffset = output.writerOffset();
|
||||
assert suffixKeyLengthConsistency(afterWriterOffset - beforeWriterOffset);
|
||||
assert keyPrefix == null || keyPrefix.isAccessible();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected RuntimeException createResourceClosedException() {
|
||||
throw new IllegalStateException("Closed");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Owned<DatabaseMapDictionaryDeep<T, U, US>> prepareSend() {
|
||||
var keyPrefix = this.keyPrefix == null ? null : this.keyPrefix.send();
|
||||
var range = this.range.send();
|
||||
var onClose = this.onClose;
|
||||
return drop -> {
|
||||
var instance = new DatabaseMapDictionaryDeep<>(dictionary,
|
||||
alloc,
|
||||
subStageGetter,
|
||||
keySuffixSerializer,
|
||||
keyPrefixLength,
|
||||
keySuffixLength,
|
||||
keyExtLength,
|
||||
rangeMono,
|
||||
range,
|
||||
keyPrefix,
|
||||
onClose
|
||||
);
|
||||
drop.attach(instance);
|
||||
return instance;
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void makeInaccessible() {
|
||||
this.keyPrefix = null;
|
||||
this.range = null;
|
||||
this.onClose = null;
|
||||
}
|
||||
}
|
||||
|
@ -1,138 +1,99 @@
|
||||
package it.cavallium.dbengine.database.collections;
|
||||
|
||||
import io.net5.buffer.api.Buffer;
|
||||
import io.net5.buffer.api.BufferAllocator;
|
||||
import io.net5.buffer.api.Drop;
|
||||
import io.net5.buffer.api.Owned;
|
||||
import io.net5.buffer.api.Resource;
|
||||
import io.net5.buffer.api.Send;
|
||||
import it.cavallium.dbengine.client.BadBlock;
|
||||
import it.cavallium.buffer.Buf;
|
||||
import it.cavallium.dbengine.client.CompositeSnapshot;
|
||||
import it.cavallium.dbengine.client.DbProgress;
|
||||
import it.cavallium.dbengine.client.SSTVerificationProgress;
|
||||
import it.cavallium.dbengine.database.LLDictionary;
|
||||
import it.cavallium.dbengine.database.LLUtils;
|
||||
import io.net5.buffer.api.internal.ResourceSupport;
|
||||
import it.cavallium.dbengine.database.SubStageEntry;
|
||||
import it.cavallium.dbengine.database.UpdateMode;
|
||||
import it.cavallium.dbengine.database.serialization.Serializer;
|
||||
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
|
||||
import it.unimi.dsi.fastutil.objects.Object2ObjectLinkedOpenHashMap;
|
||||
import it.unimi.dsi.fastutil.objects.Object2ObjectSortedMap;
|
||||
import it.unimi.dsi.fastutil.objects.ObjectArraySet;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ForkJoinPool;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Stream;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import org.warp.commonutils.log.Logger;
|
||||
import org.warp.commonutils.log.LoggerFactory;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
public class DatabaseMapDictionaryHashed<T, U, TH> extends ResourceSupport<DatabaseStage<Map<T, U>>, DatabaseMapDictionaryHashed<T, U, TH>>
|
||||
implements DatabaseStageMap<T, U, DatabaseStageEntry<U>> {
|
||||
public class DatabaseMapDictionaryHashed<T, U, TH> implements DatabaseStageMap<T, U, DatabaseStageEntry<U>> {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(DatabaseMapDictionaryHashed.class);
|
||||
private static final Logger logger = LogManager.getLogger(DatabaseMapDictionaryHashed.class);
|
||||
|
||||
private static final Drop<DatabaseMapDictionaryHashed<?, ?, ?>> DROP = new Drop<>() {
|
||||
@Override
|
||||
public void drop(DatabaseMapDictionaryHashed<?, ?, ?> obj) {
|
||||
try {
|
||||
if (obj.subDictionary != null) {
|
||||
obj.subDictionary.close();
|
||||
}
|
||||
} catch (Throwable ex) {
|
||||
logger.error("Failed to close subDictionary", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Drop<DatabaseMapDictionaryHashed<?, ?, ?>> fork() {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void attach(DatabaseMapDictionaryHashed<?, ?, ?> obj) {
|
||||
|
||||
}
|
||||
};
|
||||
|
||||
private final BufferAllocator alloc;
|
||||
private final Function<T, TH> keySuffixHashFunction;
|
||||
|
||||
private DatabaseMapDictionary<TH, ObjectArraySet<Entry<T, U>>> subDictionary;
|
||||
private final DatabaseMapDictionary<TH, ObjectArraySet<Entry<T, U>>> subDictionary;
|
||||
|
||||
@SuppressWarnings({"unchecked", "rawtypes"})
|
||||
protected DatabaseMapDictionaryHashed(LLDictionary dictionary,
|
||||
@Nullable Buffer prefixKey,
|
||||
@Nullable Buf prefixKeySupplier,
|
||||
Serializer<T> keySuffixSerializer,
|
||||
Serializer<U> valueSerializer,
|
||||
Function<T, TH> keySuffixHashFunction,
|
||||
SerializerFixedBinaryLength<TH> keySuffixHashSerializer,
|
||||
Runnable onClose) {
|
||||
super((Drop<DatabaseMapDictionaryHashed<T, U, TH>>) (Drop) DROP);
|
||||
if (dictionary.getUpdateMode().block() != UpdateMode.ALLOW) {
|
||||
SerializerFixedBinaryLength<TH> keySuffixHashSerializer) {
|
||||
var updateMode = dictionary.getUpdateMode();
|
||||
if (updateMode != UpdateMode.ALLOW) {
|
||||
throw new IllegalArgumentException("Hashed maps only works when UpdateMode is ALLOW");
|
||||
}
|
||||
this.alloc = dictionary.getAllocator();
|
||||
ValueWithHashSerializer<T, U> valueWithHashSerializer
|
||||
= new ValueWithHashSerializer<>(keySuffixSerializer, valueSerializer);
|
||||
ValuesSetSerializer<Entry<T, U>> valuesSetSerializer
|
||||
= new ValuesSetSerializer<>(valueWithHashSerializer);
|
||||
this.subDictionary = DatabaseMapDictionary.tail(dictionary, prefixKey, keySuffixHashSerializer,
|
||||
valuesSetSerializer, onClose);
|
||||
this.subDictionary = DatabaseMapDictionary.tail(dictionary, prefixKeySupplier, keySuffixHashSerializer,
|
||||
valuesSetSerializer);
|
||||
this.keySuffixHashFunction = keySuffixHashFunction;
|
||||
}
|
||||
|
||||
@SuppressWarnings({"unchecked", "rawtypes"})
|
||||
private DatabaseMapDictionaryHashed(BufferAllocator alloc,
|
||||
Function<T, TH> keySuffixHashFunction,
|
||||
Send<DatabaseStage<Map<TH, ObjectArraySet<Entry<T, U>>>>> subDictionary,
|
||||
Drop<DatabaseMapDictionaryHashed<T, U, TH>> drop) {
|
||||
super((Drop<DatabaseMapDictionaryHashed<T, U, TH>>) (Drop) DROP);
|
||||
this.alloc = alloc;
|
||||
private DatabaseMapDictionaryHashed(Function<T, TH> keySuffixHashFunction,
|
||||
DatabaseStage<Object2ObjectSortedMap<TH, ObjectArraySet<Entry<T, U>>>> subDictionary) {
|
||||
this.keySuffixHashFunction = keySuffixHashFunction;
|
||||
|
||||
this.subDictionary = (DatabaseMapDictionary<TH, ObjectArraySet<Entry<T, U>>>) subDictionary.receive();
|
||||
this.subDictionary = (DatabaseMapDictionary<TH, ObjectArraySet<Entry<T, U>>>) subDictionary;
|
||||
}
|
||||
|
||||
public static <T, U, UH> DatabaseMapDictionaryHashed<T, U, UH> simple(LLDictionary dictionary,
|
||||
Serializer<T> keySerializer,
|
||||
Serializer<U> valueSerializer,
|
||||
Function<T, UH> keyHashFunction,
|
||||
SerializerFixedBinaryLength<UH> keyHashSerializer,
|
||||
Runnable onClose) {
|
||||
SerializerFixedBinaryLength<UH> keyHashSerializer) {
|
||||
return new DatabaseMapDictionaryHashed<>(
|
||||
dictionary,
|
||||
null,
|
||||
keySerializer,
|
||||
valueSerializer,
|
||||
keyHashFunction,
|
||||
keyHashSerializer,
|
||||
onClose
|
||||
keyHashSerializer
|
||||
);
|
||||
}
|
||||
|
||||
public static <T, U, UH> DatabaseMapDictionaryHashed<T, U, UH> tail(LLDictionary dictionary,
|
||||
@Nullable Buffer prefixKey,
|
||||
@Nullable Buf prefixKeySupplier,
|
||||
Serializer<T> keySuffixSerializer,
|
||||
Serializer<U> valueSerializer,
|
||||
Function<T, UH> keySuffixHashFunction,
|
||||
SerializerFixedBinaryLength<UH> keySuffixHashSerializer,
|
||||
Runnable onClose) {
|
||||
SerializerFixedBinaryLength<UH> keySuffixHashSerializer) {
|
||||
return new DatabaseMapDictionaryHashed<>(dictionary,
|
||||
prefixKey,
|
||||
prefixKeySupplier,
|
||||
keySuffixSerializer,
|
||||
valueSerializer,
|
||||
keySuffixHashFunction,
|
||||
keySuffixHashSerializer,
|
||||
onClose
|
||||
keySuffixHashSerializer
|
||||
);
|
||||
}
|
||||
|
||||
private Map<TH, ObjectArraySet<Entry<T, U>>> serializeMap(Map<T, U> map) {
|
||||
var newMap = new HashMap<TH, ObjectArraySet<Entry<T, U>>>(map.size());
|
||||
private Object2ObjectSortedMap<TH, ObjectArraySet<Entry<T, U>>> serializeMap(Object2ObjectSortedMap<T, U> map) {
|
||||
var newMap = new Object2ObjectLinkedOpenHashMap<TH, ObjectArraySet<Entry<T, U>>>(map.size());
|
||||
map.forEach((key, value) -> newMap.compute(keySuffixHashFunction.apply(key), (hash, prev) -> {
|
||||
if (prev == null) {
|
||||
prev = new ObjectArraySet<>();
|
||||
@ -143,133 +104,144 @@ public class DatabaseMapDictionaryHashed<T, U, TH> extends ResourceSupport<Datab
|
||||
return newMap;
|
||||
}
|
||||
|
||||
private Map<T, U> deserializeMap(Map<TH, ObjectArraySet<Entry<T, U>>> map) {
|
||||
var newMap = new HashMap<T, U>(map.size());
|
||||
private Object2ObjectSortedMap<T, U> deserializeMap(Object2ObjectSortedMap<TH, ObjectArraySet<Entry<T, U>>> map) {
|
||||
var newMap = new Object2ObjectLinkedOpenHashMap<T, U>(map.size());
|
||||
map.forEach((hash, set) -> set.forEach(entry -> newMap.put(entry.getKey(), entry.getValue())));
|
||||
return newMap;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Map<T, U>> get(@Nullable CompositeSnapshot snapshot) {
|
||||
return subDictionary.get(snapshot).map(this::deserializeMap);
|
||||
public ForkJoinPool getDbReadPool() {
|
||||
return subDictionary.getDbReadPool();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Map<T, U>> getOrDefault(@Nullable CompositeSnapshot snapshot, Mono<Map<T, U>> defaultValue) {
|
||||
return this.get(snapshot).switchIfEmpty(defaultValue);
|
||||
public ForkJoinPool getDbWritePool() {
|
||||
return subDictionary.getDbWritePool();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Void> set(Map<T, U> map) {
|
||||
return Mono.fromSupplier(() -> this.serializeMap(map)).flatMap(subDictionary::set);
|
||||
public Object2ObjectSortedMap<T, U> get(@Nullable CompositeSnapshot snapshot) {
|
||||
var v = subDictionary.get(snapshot);
|
||||
var result = v != null ? deserializeMap(v) : null;
|
||||
return result != null && result.isEmpty() ? null : result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Boolean> setAndGetChanged(Map<T, U> map) {
|
||||
return Mono.fromSupplier(() -> this.serializeMap(map)).flatMap(subDictionary::setAndGetChanged).single();
|
||||
public Object2ObjectSortedMap<T, U> getOrDefault(@Nullable CompositeSnapshot snapshot,
|
||||
Object2ObjectSortedMap<T, U> defaultValue) {
|
||||
return Objects.requireNonNullElse(this.get(snapshot), defaultValue);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Boolean> clearAndGetStatus() {
|
||||
public void set(Object2ObjectSortedMap<T, U> map) {
|
||||
var value = this.serializeMap(map);
|
||||
subDictionary.set(value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean setAndGetChanged(Object2ObjectSortedMap<T, U> map) {
|
||||
return subDictionary.setAndGetChanged(this.serializeMap(map));
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean clearAndGetStatus() {
|
||||
return subDictionary.clearAndGetStatus();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Boolean> isEmpty(@Nullable CompositeSnapshot snapshot) {
|
||||
public boolean isEmpty(@Nullable CompositeSnapshot snapshot) {
|
||||
return subDictionary.isEmpty(snapshot);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatabaseStageEntry<Map<T, U>> entry() {
|
||||
public DatabaseStageEntry<Object2ObjectSortedMap<T, U>> entry() {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<BadBlock> badBlocks() {
|
||||
return this.subDictionary.badBlocks();
|
||||
public Stream<DbProgress<SSTVerificationProgress>> verifyChecksum() {
|
||||
return this.subDictionary.verifyChecksum();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<DatabaseStageEntry<U>> at(@Nullable CompositeSnapshot snapshot, T key) {
|
||||
return this
|
||||
.atPrivate(snapshot, key, keySuffixHashFunction.apply(key))
|
||||
.map(cast -> (DatabaseStageEntry<U>) cast)
|
||||
.doOnDiscard(Resource.class, Resource::close);
|
||||
public @NotNull DatabaseStageEntry<U> at(@Nullable CompositeSnapshot snapshot, T key) {
|
||||
return this.atPrivate(snapshot, key, keySuffixHashFunction.apply(key));
|
||||
}
|
||||
|
||||
private Mono<DatabaseSingleBucket<T, U, TH>> atPrivate(@Nullable CompositeSnapshot snapshot, T key, TH hash) {
|
||||
return subDictionary
|
||||
.at(snapshot, hash)
|
||||
.map(entry -> new DatabaseSingleBucket<T, U, TH>(entry, key, null))
|
||||
.doOnDiscard(Resource.class, Resource::close);
|
||||
private DatabaseSingleBucket<T, U, TH> atPrivate(@Nullable CompositeSnapshot snapshot, T key, TH hash) {
|
||||
return new DatabaseSingleBucket<T, U, TH>(subDictionary.at(snapshot, hash), key);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<UpdateMode> getUpdateMode() {
|
||||
public UpdateMode getUpdateMode() {
|
||||
return subDictionary.getUpdateMode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<Entry<T, DatabaseStageEntry<U>>> getAllStages(@Nullable CompositeSnapshot snapshot) {
|
||||
public Stream<SubStageEntry<T, DatabaseStageEntry<U>>> getAllStages(@Nullable CompositeSnapshot snapshot,
|
||||
boolean smallRange) {
|
||||
return subDictionary
|
||||
.getAllValues(snapshot)
|
||||
.getAllEntries(snapshot, smallRange)
|
||||
.map(Entry::getValue)
|
||||
.map(Collections::unmodifiableSet)
|
||||
.flatMap(bucket -> Flux
|
||||
.fromIterable(bucket)
|
||||
.flatMap(bucket -> bucket.stream()
|
||||
.map(Entry::getKey)
|
||||
.flatMap(key -> this.at(snapshot, key).map(stage -> Map.entry(key, stage)))
|
||||
);
|
||||
.map(key -> new SubStageEntry<>(key, this.at(snapshot, key))));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<Entry<T, U>> getAllValues(@Nullable CompositeSnapshot snapshot) {
|
||||
public Stream<Entry<T, U>> getAllEntries(@Nullable CompositeSnapshot snapshot, boolean smallRange) {
|
||||
return subDictionary
|
||||
.getAllValues(snapshot)
|
||||
.getAllEntries(snapshot, smallRange)
|
||||
.map(Entry::getValue)
|
||||
.map(Collections::unmodifiableSet)
|
||||
.concatMapIterable(list -> list);
|
||||
.flatMap(Collection::stream);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<Entry<T, U>> setAllValuesAndGetPrevious(Flux<Entry<T, U>> entries) {
|
||||
return entries
|
||||
.flatMap(entry -> LLUtils.usingResource(this.at(null, entry.getKey()),
|
||||
stage -> stage
|
||||
.setAndGetPrevious(entry.getValue())
|
||||
.map(prev -> Map.entry(entry.getKey(), prev)), true)
|
||||
);
|
||||
public Stream<T> getAllKeys(@Nullable CompositeSnapshot snapshot, boolean smallRange) {
|
||||
return getAllEntries(snapshot, smallRange).map(Entry::getKey);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Void> clear() {
|
||||
return subDictionary.clear();
|
||||
public Stream<U> getAllValues(@Nullable CompositeSnapshot snapshot, boolean smallRange) {
|
||||
return getAllEntries(snapshot, smallRange).map(Entry::getValue);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Map<T, U>> setAndGetPrevious(Map<T, U> value) {
|
||||
return Mono
|
||||
.fromSupplier(() -> this.serializeMap(value))
|
||||
.flatMap(subDictionary::setAndGetPrevious)
|
||||
.map(this::deserializeMap);
|
||||
public Stream<Entry<T, U>> setAllEntriesAndGetPrevious(Stream<Entry<T, U>> entries) {
|
||||
List<Entry<T, U>> prevList = entries.map(entry -> {
|
||||
var prev = this.at(null, entry.getKey()).setAndGetPrevious(entry.getValue());
|
||||
if (prev != null) {
|
||||
return Map.entry(entry.getKey(), prev);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}).filter(Objects::nonNull).toList();
|
||||
return prevList.stream();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Map<T, U>> clearAndGetPrevious() {
|
||||
return subDictionary
|
||||
.clearAndGetPrevious()
|
||||
.map(this::deserializeMap);
|
||||
public void clear() {
|
||||
subDictionary.clear();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Map<T, U>> get(@Nullable CompositeSnapshot snapshot, boolean existsAlmostCertainly) {
|
||||
return subDictionary
|
||||
.get(snapshot, existsAlmostCertainly)
|
||||
.map(this::deserializeMap);
|
||||
public Object2ObjectSortedMap<T, U> setAndGetPrevious(Object2ObjectSortedMap<T, U> value) {
|
||||
var v = subDictionary.setAndGetPrevious(this.serializeMap(value));
|
||||
var result = v != null ? deserializeMap(v) : null;
|
||||
return result != null && result.isEmpty() ? null : result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<Long> leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
|
||||
public Object2ObjectSortedMap<T, U> clearAndGetPrevious() {
|
||||
var v = subDictionary.clearAndGetPrevious();
|
||||
return v != null ? deserializeMap(v) : null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
|
||||
return subDictionary.leavesCount(snapshot, fast);
|
||||
}
|
||||
|
||||
@ -282,13 +254,14 @@ public class DatabaseMapDictionaryHashed<T, U, TH> extends ResourceSupport<Datab
|
||||
@Override
|
||||
public ValueGetter<T, U> getAsyncDbValueGetter(@Nullable CompositeSnapshot snapshot) {
|
||||
ValueGetter<TH, ObjectArraySet<Entry<T, U>>> getter = subDictionary.getAsyncDbValueGetter(snapshot);
|
||||
return key -> getter
|
||||
.get(keySuffixHashFunction.apply(key))
|
||||
.flatMap(set -> this.extractValueTransformation(set, key));
|
||||
}
|
||||
|
||||
private Mono<U> extractValueTransformation(ObjectArraySet<Entry<T, U>> entries, T key) {
|
||||
return Mono.fromCallable(() -> extractValue(entries, key));
|
||||
return key -> {
|
||||
ObjectArraySet<Entry<T, U>> set = getter.get(keySuffixHashFunction.apply(key));
|
||||
if (set != null) {
|
||||
return this.extractValue(set, key);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@ -336,20 +309,4 @@ public class DatabaseMapDictionaryHashed<T, U, TH> extends ResourceSupport<Datab
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected RuntimeException createResourceClosedException() {
|
||||
throw new IllegalStateException("Closed");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Owned<DatabaseMapDictionaryHashed<T, U, TH>> prepareSend() {
|
||||
var subDictionary = this.subDictionary.send();
|
||||
return drop -> new DatabaseMapDictionaryHashed<>(alloc, keySuffixHashFunction, subDictionary, drop);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void makeInaccessible() {
|
||||
this.subDictionary = null;
|
||||
}
|
||||
}
|
||||
|
@ -0,0 +1,140 @@
|
||||
package it.cavallium.dbengine.database.collections;
|
||||
|
||||
import it.cavallium.buffer.Buf;
|
||||
import it.cavallium.buffer.BufDataInput;
|
||||
import it.cavallium.buffer.BufDataOutput;
|
||||
import it.cavallium.dbengine.client.CompositeSnapshot;
|
||||
import it.cavallium.dbengine.client.DbProgress;
|
||||
import it.cavallium.dbengine.client.SSTVerificationProgress;
|
||||
import it.cavallium.dbengine.database.Delta;
|
||||
import it.cavallium.dbengine.database.LLDictionary;
|
||||
import it.cavallium.dbengine.database.LLDictionaryResultType;
|
||||
import it.cavallium.dbengine.database.LLRange;
|
||||
import it.cavallium.dbengine.database.LLSnapshot;
|
||||
import it.cavallium.dbengine.database.LLUtils;
|
||||
import it.cavallium.dbengine.database.UpdateReturnMode;
|
||||
import it.cavallium.dbengine.database.disk.CachedSerializationFunction;
|
||||
import it.cavallium.dbengine.database.serialization.SerializationException;
|
||||
import it.cavallium.dbengine.database.serialization.SerializationFunction;
|
||||
import it.cavallium.dbengine.database.serialization.Serializer;
|
||||
import java.util.concurrent.ForkJoinPool;
|
||||
import java.util.stream.Stream;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
public final class DatabaseMapSingle<U> implements DatabaseStageEntry<U> {
|
||||
|
||||
private static final Logger LOG = LogManager.getLogger(DatabaseMapSingle.class);
|
||||
|
||||
private final LLDictionary dictionary;
|
||||
private final Buf key;
|
||||
private final Serializer<U> serializer;
|
||||
|
||||
public DatabaseMapSingle(LLDictionary dictionary, Buf key, Serializer<U> serializer) {
|
||||
this.dictionary = dictionary;
|
||||
this.key = key;
|
||||
this.serializer = serializer;
|
||||
}
|
||||
|
||||
private LLSnapshot resolveSnapshot(@Nullable CompositeSnapshot snapshot) {
|
||||
if (snapshot == null) {
|
||||
return null;
|
||||
} else {
|
||||
return snapshot.getSnapshot(dictionary);
|
||||
}
|
||||
}
|
||||
|
||||
private U deserializeValue(Buf value) {
|
||||
try {
|
||||
return serializer.deserialize(BufDataInput.create(value));
|
||||
} catch (IndexOutOfBoundsException ex) {
|
||||
var exMessage = ex.getMessage();
|
||||
if (exMessage != null && exMessage.contains("read 0 to 0, write 0 to ")) {
|
||||
LOG.error("Unexpected zero-bytes value at %s:%s:%s".formatted(dictionary.getDatabaseName(),
|
||||
dictionary.getColumnName(),
|
||||
LLUtils.toStringSafe(key)
|
||||
));
|
||||
return null;
|
||||
} else {
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Buf serializeValue(U value) throws SerializationException {
|
||||
BufDataOutput valBuf = BufDataOutput.create(serializer.getSerializedSizeHint());
|
||||
serializer.serialize(value, valBuf);
|
||||
return valBuf.asList();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ForkJoinPool getDbReadPool() {
|
||||
return dictionary.getDbReadPool();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ForkJoinPool getDbWritePool() {
|
||||
return dictionary.getDbWritePool();
|
||||
}
|
||||
|
||||
@Override
|
||||
public U get(@Nullable CompositeSnapshot snapshot) {
|
||||
var result = dictionary.get(resolveSnapshot(snapshot), key);
|
||||
if (result != null) {
|
||||
return deserializeValue(result);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public U setAndGetPrevious(U value) {
|
||||
var serializedKey = value != null ? serializeValue(value) : null;
|
||||
var result = dictionary.put(key, serializedKey, LLDictionaryResultType.PREVIOUS_VALUE);
|
||||
if (result != null) {
|
||||
return deserializeValue(result);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public U update(SerializationFunction<@Nullable U, @Nullable U> updater, UpdateReturnMode updateReturnMode) {
|
||||
var serializedUpdater = createUpdater(updater);
|
||||
dictionary.update(key, serializedUpdater, UpdateReturnMode.NOTHING);
|
||||
return serializedUpdater.getResult(updateReturnMode);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Delta<U> updateAndGetDelta(SerializationFunction<@Nullable U, @Nullable U> updater) {
|
||||
var serializedUpdater = createUpdater(updater);
|
||||
dictionary.update(key, serializedUpdater, UpdateReturnMode.NOTHING);
|
||||
return serializedUpdater.getDelta();
|
||||
}
|
||||
|
||||
private CachedSerializationFunction<U, Buf, Buf> createUpdater(SerializationFunction<U, U> updater) {
|
||||
return new CachedSerializationFunction<>(updater, this::serializeValue, this::deserializeValue);
|
||||
}
|
||||
|
||||
@Override
|
||||
public U clearAndGetPrevious() {
|
||||
return deserializeValue(dictionary.remove(key, LLDictionaryResultType.PREVIOUS_VALUE));
|
||||
}
|
||||
|
||||
@Override
|
||||
public long leavesCount(@Nullable CompositeSnapshot snapshot, boolean fast) {
|
||||
return dictionary.isRangeEmpty(resolveSnapshot(snapshot), LLRange.single(key), false) ? 0L : 1L;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isEmpty(@Nullable CompositeSnapshot snapshot) {
|
||||
return dictionary.isRangeEmpty(resolveSnapshot(snapshot), LLRange.single(key), true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Stream<DbProgress<SSTVerificationProgress>> verifyChecksum() {
|
||||
return dictionary.verifyChecksum(LLRange.single(key));
|
||||
}
|
||||
|
||||
}
|
@ -1,55 +1,50 @@
|
||||
package it.cavallium.dbengine.database.collections;
|
||||
|
||||
import io.net5.buffer.api.Buffer;
|
||||
import io.net5.buffer.api.Drop;
|
||||
import io.net5.buffer.api.Send;
|
||||
import it.cavallium.buffer.Buf;
|
||||
import it.cavallium.dbengine.client.CompositeSnapshot;
|
||||
import it.cavallium.dbengine.database.LLDictionary;
|
||||
import it.cavallium.dbengine.database.LLUtils;
|
||||
import it.cavallium.dbengine.database.collections.DatabaseEmpty.Nothing;
|
||||
import it.cavallium.dbengine.database.serialization.SerializerFixedBinaryLength;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import it.unimi.dsi.fastutil.objects.Object2ObjectLinkedOpenHashMap;
|
||||
import java.util.Set;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
public class DatabaseSetDictionary<T> extends DatabaseMapDictionary<T, Nothing> {
|
||||
|
||||
protected DatabaseSetDictionary(LLDictionary dictionary,
|
||||
Buffer prefixKey,
|
||||
SerializerFixedBinaryLength<T> keySuffixSerializer,
|
||||
Runnable onClose) {
|
||||
super(dictionary, prefixKey, keySuffixSerializer, DatabaseEmpty.nothingSerializer(dictionary.getAllocator()), onClose);
|
||||
Buf prefixKeySupplier,
|
||||
SerializerFixedBinaryLength<T> keySuffixSerializer) {
|
||||
super(dictionary, prefixKeySupplier, keySuffixSerializer, DatabaseEmpty.nothingSerializer());
|
||||
}
|
||||
|
||||
public static <T> DatabaseSetDictionary<T> simple(LLDictionary dictionary,
|
||||
SerializerFixedBinaryLength<T> keySerializer,
|
||||
Runnable onClose) {
|
||||
return new DatabaseSetDictionary<>(dictionary, null, keySerializer, onClose);
|
||||
SerializerFixedBinaryLength<T> keySerializer) {
|
||||
return new DatabaseSetDictionary<>(dictionary, null, keySerializer);
|
||||
}
|
||||
|
||||
public static <T> DatabaseSetDictionary<T> tail(LLDictionary dictionary,
|
||||
Buffer prefixKey,
|
||||
SerializerFixedBinaryLength<T> keySuffixSerializer,
|
||||
Runnable onClose) {
|
||||
return new DatabaseSetDictionary<>(dictionary, prefixKey, keySuffixSerializer, onClose);
|
||||
Buf prefixKeySupplier,
|
||||
SerializerFixedBinaryLength<T> keySuffixSerializer) {
|
||||
return new DatabaseSetDictionary<>(dictionary, prefixKeySupplier, keySuffixSerializer);
|
||||
}
|
||||
|
||||
public Mono<Set<T>> getKeySet(@Nullable CompositeSnapshot snapshot) {
|
||||
return get(snapshot).map(Map::keySet);
|
||||
public Set<T> getKeySet(@Nullable CompositeSnapshot snapshot) {
|
||||
var v = get(snapshot);
|
||||
return v != null ? v.keySet() : null;
|
||||
}
|
||||
|
||||
public Mono<Set<T>> setAndGetPreviousKeySet(Set<T> value) {
|
||||
var hm = new HashMap<T, Nothing>();
|
||||
public Set<T> setAndGetPreviousKeySet(Set<T> value) {
|
||||
var hm = new Object2ObjectLinkedOpenHashMap<T, Nothing>();
|
||||
for (T t : value) {
|
||||
hm.put(t, DatabaseEmpty.NOTHING);
|
||||
}
|
||||
return setAndGetPrevious(hm).map(Map::keySet);
|
||||
var v = setAndGetPrevious(hm);
|
||||
return v != null ? v.keySet() : null;
|
||||
}
|
||||
|
||||
public Mono<Set<T>> clearAndGetPreviousKeySet() {
|
||||
return clearAndGetPrevious().map(Map::keySet);
|
||||
public Set<T> clearAndGetPreviousKeySet() {
|
||||
var v = clearAndGetPrevious();
|
||||
return v != null ? v.keySet() : null;
|
||||
}
|
||||
}
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user