From 4f444cb1a7d66ab740ae4baa528066378aa6715d Mon Sep 17 00:00:00 2001 From: Florin Braghis Date: Mon, 26 May 2025 13:16:33 +0200 Subject: [PATCH 01/14] Counted Hash Map implementation --- deps.edn | 2 +- src/xitdb/hash_map.clj | 9 ++++++++- src/xitdb/hash_set.clj | 12 +++++++++-- src/xitdb/util/conversion.clj | 15 +++++++------- src/xitdb/util/operations.clj | 38 ++++++++--------------------------- src/xitdb/xitdb_types.clj | 33 +++++++++++++++--------------- test/xitdb/database_test.clj | 5 +++++ test/xitdb/map_test.clj | 10 +++++++++ 8 files changed, 66 insertions(+), 58 deletions(-) create mode 100644 test/xitdb/map_test.clj diff --git a/deps.edn b/deps.edn index 32b0f18..3845236 100644 --- a/deps.edn +++ b/deps.edn @@ -1,6 +1,6 @@ {:paths ["src" "test"] :deps {org.clojure/clojure {:mvn/version "1.12.0"} - io.github.radarroark/xitdb {:mvn/version "0.16.0"}} + io.github.radarroark/xitdb {:mvn/version "0.19.0"}} :aliases {:test diff --git a/src/xitdb/hash_map.clj b/src/xitdb/hash_map.clj index ac41c56..ab30bb1 100644 --- a/src/xitdb/hash_map.clj +++ b/src/xitdb/hash_map.clj @@ -4,7 +4,7 @@ [xitdb.util.conversion :as conversion] [xitdb.util.operations :as operations]) (:import - [io.github.radarroark.xitdb ReadCursor ReadHashMap WriteCursor WriteHashMap])) + [io.github.radarroark.xitdb ReadCountedHashMap ReadCursor ReadHashMap WriteCountedHashMap WriteCursor WriteHashMap])) (defn map-seq [rhm] @@ -185,4 +185,11 @@ (defn xhash-map [^ReadCursor read-cursor] (->XITDBHashMap (ReadHashMap. read-cursor))) +(defn xwrite-hash-map-counted [^WriteCursor write-cursor] + (->XITDBWriteHashMap (WriteCountedHashMap. write-cursor))) + +(defn xhash-map-counted [^ReadCursor read-cursor] + (->XITDBHashMap (ReadCountedHashMap. read-cursor))) + + diff --git a/src/xitdb/hash_set.clj b/src/xitdb/hash_set.clj index e9cc51c..7b5acc3 100644 --- a/src/xitdb/hash_set.clj +++ b/src/xitdb/hash_set.clj @@ -4,7 +4,7 @@ [xitdb.util.conversion :as conversion] [xitdb.util.operations :as operations]) (:import - [io.github.radarroark.xitdb ReadHashMap WriteCursor WriteHashMap])) + [io.github.radarroark.xitdb ReadCountedHashSet ReadCursor ReadHashMap WriteCursor WriteHashMap])) (defn set-seq [rhm] @@ -144,5 +144,13 @@ (let [whm (operations/init-hash-set! write-cursor)] (->XITDBWriteHashSet whm))) -(defn xhash-set [^ReadHashMap read-cursor] +(defn xhash-set [^ReadCursor read-cursor] (->XITDBHashSet (ReadHashMap. read-cursor))) + +(defn xwrite-hash-set-counted [^WriteCursor write-cursor] + ;; TODO: Use WriteHashSetCounted + (let [whm (operations/init-hash-set! write-cursor)] + (->XITDBWriteHashSet whm))) + +(defn xhash-set-counted [^ReadCursor cursor] + (->XITDBHashSet (ReadCountedHashSet. cursor))) \ No newline at end of file diff --git a/src/xitdb/util/conversion.clj b/src/xitdb/util/conversion.clj index 371163d..e51e509 100644 --- a/src/xitdb/util/conversion.clj +++ b/src/xitdb/util/conversion.clj @@ -2,7 +2,11 @@ (:require [xitdb.util.validation :as validation]) (:import - [io.github.radarroark.xitdb Database$Float Database$Bytes Database$Int Database$Uint ReadArrayList ReadCursor ReadHashMap ReadLinkedArrayList Slot WriteArrayList WriteCursor WriteHashMap Tag WriteLinkedArrayList])) + [io.github.radarroark.xitdb + Database$Bytes Database$Float Database$Int + ReadArrayList ReadCursor ReadHashMap ReadCountedHashMap + Slot Tag WriteArrayList WriteCursor WriteCountedHashMap + WriteHashMap WriteLinkedArrayList])) (defn xit-tag->keyword "Converts a XitDB Tag enum to a corresponding Clojure keyword." @@ -26,7 +30,7 @@ {:keyword "kw" :boolean "bl" :key-integer "ki" - :nil "nl" + :nil "nl" ;; TODO: Could use Tag/NONE instead :inst "in" :date "da"}) @@ -228,16 +232,11 @@ (.append write-list (primitive-for v)))) (.-cursor write-list))) -;; Forward declarations for mutual dependencies -(declare map-assoc-value!) -(declare init-hash-set!) -(declare set-assoc-value!) - (defn ^WriteCursor map->WriteHashMapCursor! "Writes a Clojure map to a XitDB WriteHashMap. Returns the cursor of the created WriteHashMap." [^WriteCursor cursor m] - (let [whm (WriteHashMap. cursor)] + (let [whm (WriteCountedHashMap. cursor)] (doseq [[k v] m] (let [cursor (.putCursor whm (db-key k))] (.write cursor (v->slot! cursor v)))) diff --git a/src/xitdb/util/operations.clj b/src/xitdb/util/operations.clj index 62206bd..d2c03d5 100644 --- a/src/xitdb/util/operations.clj +++ b/src/xitdb/util/operations.clj @@ -3,7 +3,7 @@ [xitdb.util.conversion :as conversion] [xitdb.util.validation :as validation]) (:import - [io.github.radarroark.xitdb ReadArrayList ReadHashMap ReadLinkedArrayList Tag WriteArrayList WriteCursor WriteHashMap WriteLinkedArrayList])) + [io.github.radarroark.xitdb ReadArrayList ReadCountedHashMap ReadHashMap ReadLinkedArrayList Tag WriteArrayList WriteCursor WriteHashMap WriteLinkedArrayList])) (def internal-keys "Map of logical internal key names to their actual storage keys in XitDB. @@ -99,24 +99,6 @@ ;; Map Operations ;; ============================================================================ -(defn- update-map-item-count! - "Update the internal key `:count` by applying `f` to the current value. - If the key `:count` does not exist, it is created." - [^WriteHashMap whm f] - (when *enable-map-fast-count?* - (let [count-cursor (.putCursor whm (conversion/db-key (internal-keys :count))) - value (try - (.readInt count-cursor) - (catch Exception _ 0)) - new-value (conversion/primitive-for (f (or value 0)))] - (.write count-cursor new-value)))) - -(defn- map-item-count-stored - "Returns the value of the internal key `:count`." - [^ReadHashMap rhm] - (let [count-cursor (.getCursor rhm (conversion/db-key (internal-keys :count)))] - (.readInt count-cursor))) - (defn map-assoc-value! "Associates a key-value pair in a WriteHashMap. @@ -133,11 +115,8 @@ (when (contains? hidden-keys k) (throw (IllegalArgumentException. (str "Cannot assoc key. " k ". It is reserved for internal use.")))) - (let [cursor (.putCursor whm (conversion/db-key k)) - new? (= (-> cursor .slot .tag) Tag/NONE)] + (let [cursor (.putCursor whm (conversion/db-key k))] (.write cursor (conversion/v->slot! cursor v)) - (when new? - (update-map-item-count! whm inc)) whm)) (defn map-dissoc-key! @@ -148,8 +127,8 @@ (when (contains? hidden-keys k) (throw (IllegalArgumentException. (str "Cannot dissoc key. " k ". It is reserved for internal use.")))) - (when (.remove whm (conversion/db-key k)) - (update-map-item-count! whm dec))) + (.remove whm (conversion/db-key k)) + whm) (defn ^WriteHashMap map-empty! "Empties a WriteHashMap by replacing its contents with an empty map. @@ -180,8 +159,8 @@ (defn map-item-count "Returns the number of key/vals in the map." [^ReadHashMap rhm] - (if *enable-map-fast-count?* - (map-item-count-stored rhm) + (if (instance? ReadCountedHashMap rhm) + (.count ^ReadCountedHashMap rhm) (map-item-count-iterated rhm))) (defn map-read-cursor @@ -211,8 +190,7 @@ new? (= (-> cursor .slot .tag) Tag/NONE)] (when new? ;; Only write value when the hashCode key doesn't exist - (.write cursor (conversion/v->slot! cursor v)) - (update-map-item-count! whm inc)) + (.write cursor (conversion/v->slot! cursor v))) whm))) (defn ^WriteHashMap mark-as-set! @@ -250,7 +228,7 @@ (defn map-seq "Return a lazy seq of key-value MapEntry pairs, skipping hidden keys." - [^ReadHashMap rhm read-from-cursor] + [^ReadCountedHashMap rhm read-from-cursor] (let [it (.iterator rhm)] (letfn [(step [] (lazy-seq diff --git a/src/xitdb/xitdb_types.clj b/src/xitdb/xitdb_types.clj index 529570d..0d6a1f1 100644 --- a/src/xitdb/xitdb_types.clj +++ b/src/xitdb/xitdb_types.clj @@ -7,20 +7,7 @@ [xitdb.hash-set :as xhash-set] [xitdb.util.conversion :as conversion]) (:import - (io.github.radarroark.xitdb ReadCursor ReadHashMap Slot Tag WriteCursor))) - -(defn xhash-map-or-set [^ReadCursor cursor] - (let [hm (ReadHashMap. cursor)] - (if (.getCursor hm (conversion/db-key :%xitdb_set)) - (xhash-set/xhash-set cursor) - (xhash-map/xhash-map cursor)))) - -(defn x-write-map-or-set [^ReadCursor cursor] - (let [hm (ReadHashMap. cursor)] - (if (.getCursor hm (conversion/db-key :%xitdb_set)) - (xhash-set/xwrite-hash-set cursor) - (xhash-map/xwrite-hash-map cursor)))) - + (io.github.radarroark.xitdb ReadCountedHashMap ReadCursor ReadHashMap Slot Tag WriteCursor WriteHashMap))) (defn read-from-cursor [^ReadCursor cursor for-writing?] (let [value-tag (some-> cursor .slot .tag)] @@ -39,10 +26,24 @@ (.readFloat cursor) (= value-tag Tag/HASH_MAP) + (if for-writing? + (xhash-map/xwrite-hash-map cursor) + (xhash-map/xhash-map cursor)) + + (= value-tag Tag/COUNTED_HASH_MAP) + (if for-writing? + (xhash-map/xwrite-hash-map-counted cursor) + (xhash-map/xhash-map-counted cursor)) + + (= value-tag Tag/HASH_SET) + (if for-writing? + (xhash-set/xwrite-hash-set cursor) + (xhash-set/xhash-set cursor)) + (= value-tag Tag/COUNTED_HASH_SET) (if for-writing? - (x-write-map-or-set cursor) - (xhash-map-or-set cursor)) + (xhash-set/xwrite-hash-set-counted cursor) + (xhash-set/xhash-set-counted cursor)) (= value-tag Tag/ARRAY_LIST) (if for-writing? diff --git a/test/xitdb/database_test.clj b/test/xitdb/database_test.clj index 7bc787a..93ca41c 100644 --- a/test/xitdb/database_test.clj +++ b/test/xitdb/database_test.clj @@ -3,6 +3,11 @@ [clojure.test :refer :all] [xitdb.test-utils :as tu :refer [with-db]])) +(comment + (let [db (tu/test-memory-db-raw)] + (reset! db {:foo :bar}) + @db)) + (deftest DatabaseTest (with-db [db (tu/test-db)] (testing "Resetting to map" diff --git a/test/xitdb/map_test.clj b/test/xitdb/map_test.clj new file mode 100644 index 0000000..c6062da --- /dev/null +++ b/test/xitdb/map_test.clj @@ -0,0 +1,10 @@ +(ns xitdb.map-test + (:require + [clojure.test :refer :all] + [xitdb.test-utils :as tu :refer [with-db]])) + +(deftest map-with-complex-keys + (with-db [db (tu/test-db)] + (reset! db {:foo {{:bar :baz} 42}}) + #_(reset! db {:foo {[1 :bar] 31 + [2 :baz] 42}}))) \ No newline at end of file From e12307dc4e2a5a0f5be01a21079848b00c6290ed Mon Sep 17 00:00:00 2001 From: Florin Braghis Date: Mon, 26 May 2025 16:26:45 +0200 Subject: [PATCH 02/14] Counted set, hash the keys, support composite keys --- deps.edn | 2 +- src/xitdb/hash_map.clj | 4 +- src/xitdb/hash_set.clj | 47 +++++++------- src/xitdb/util/conversion.clj | 58 ++++++++++++------ src/xitdb/util/operations.clj | 111 +++++++++++++++++++--------------- test/xitdb/set_test.clj | 23 ++++++- 6 files changed, 148 insertions(+), 97 deletions(-) diff --git a/deps.edn b/deps.edn index 3845236..896586e 100644 --- a/deps.edn +++ b/deps.edn @@ -1,6 +1,6 @@ {:paths ["src" "test"] :deps {org.clojure/clojure {:mvn/version "1.12.0"} - io.github.radarroark/xitdb {:mvn/version "0.19.0"}} + io.github.radarroark/xitdb {:mvn/version "0.20.0"}} :aliases {:test diff --git a/src/xitdb/hash_map.clj b/src/xitdb/hash_map.clj index ab30bb1..6eaa139 100644 --- a/src/xitdb/hash_map.clj +++ b/src/xitdb/hash_map.clj @@ -18,14 +18,14 @@ (.valAt this key nil)) (valAt [this key not-found] - (let [cursor (.getCursor rhm (conversion/db-key key))] + (let [cursor (operations/map-read-cursor rhm key)] (if (nil? cursor) not-found (common/-read-from-cursor cursor)))) clojure.lang.Associative (containsKey [this key] - (not (nil? (.getCursor rhm (conversion/db-key key))))) + (operations/map-contains-key? rhm key)) (entryAt [this key] (let [v (.valAt this key nil)] diff --git a/src/xitdb/hash_set.clj b/src/xitdb/hash_set.clj index 7b5acc3..3acf5f8 100644 --- a/src/xitdb/hash_set.clj +++ b/src/xitdb/hash_set.clj @@ -4,20 +4,20 @@ [xitdb.util.conversion :as conversion] [xitdb.util.operations :as operations]) (:import - [io.github.radarroark.xitdb ReadCountedHashSet ReadCursor ReadHashMap WriteCursor WriteHashMap])) + [io.github.radarroark.xitdb ReadCountedHashSet ReadCursor ReadHashMap ReadHashSet WriteCountedHashSet WriteCursor WriteHashMap WriteHashSet])) (defn set-seq [rhm] "The cursors used must implement the IReadFromCursor protocol." - (map val (operations/map-seq rhm #(common/-read-from-cursor %)))) + (operations/set-seq rhm common/-read-from-cursor)) -(deftype XITDBHashSet [^ReadHashMap rhm] +(deftype XITDBHashSet [^ReadHashSet rhs] clojure.lang.IPersistentSet (disjoin [_ k] (throw (UnsupportedOperationException. "XITDBHashSet is read-only"))) (contains [this k] - (not (nil? (.getCursor rhm (conversion/db-key (if (nil? k) 0 (.hashCode k))))))) + (not (nil? (.getCursor rhs (conversion/hash-value (-> rhs .cursor .db) key))))) (get [this k] (when (.contains this k) @@ -36,11 +36,11 @@ (every? #(.contains this %) other))) (count [_] - (operations/map-item-count rhm)) + (operations/set-item-count rhs)) clojure.lang.Seqable (seq [_] - (set-seq rhm)) + (set-seq rhs)) clojure.lang.ILookup (valAt [this k] @@ -68,7 +68,7 @@ common/IUnwrap (-unwrap [_] - rhm) + rhs) Object (toString [this] @@ -84,26 +84,26 @@ (into #{} (map common/materialize (seq this))))) ;; Writable version of the set -(deftype XITDBWriteHashSet [^WriteHashMap whm] +(deftype XITDBWriteHashSet [^WriteHashSet whs] clojure.lang.IPersistentSet - (disjoin [this k] - (operations/map-dissoc-key! whm (.hashCode k)) + (disjoin [this v] + (operations/set-disj-value! whs (common/unwrap v)) this) - (contains [this k] - (operations/map-contains-key? whm (.hashCode k))) + (contains [this v] + (operations/set-contains? whs (common/unwrap v))) (get [this k] - (when (.contains this k) + (when (.contains this (common/unwrap k)) k)) clojure.lang.IPersistentCollection (cons [this o] - (operations/set-assoc-value! whm (common/unwrap o)) + (operations/set-assoc-value! whs (common/unwrap o)) this) (empty [this] - (operations/set-empty! whm) + (operations/set-empty! whs) this) (equiv [this other] @@ -112,11 +112,11 @@ (every? #(.contains this %) other))) (count [_] - (operations/map-item-count whm)) + (operations/set-item-count whs)) clojure.lang.Seqable (seq [_] - (set-seq whm)) + (set-seq whs)) clojure.lang.ILookup (valAt [this k] @@ -129,11 +129,11 @@ common/ISlot (-slot [_] - (-> whm .cursor .slot)) + (-> whs .cursor .slot)) common/IUnwrap (-unwrap [_] - whm) + whs) Object (toString [_] @@ -141,16 +141,13 @@ ;; Constructor functions (defn xwrite-hash-set [^WriteCursor write-cursor] - (let [whm (operations/init-hash-set! write-cursor)] - (->XITDBWriteHashSet whm))) + (->XITDBWriteHashSet (WriteHashSet. write-cursor))) (defn xhash-set [^ReadCursor read-cursor] - (->XITDBHashSet (ReadHashMap. read-cursor))) + (->XITDBHashSet (ReadHashSet. read-cursor))) (defn xwrite-hash-set-counted [^WriteCursor write-cursor] - ;; TODO: Use WriteHashSetCounted - (let [whm (operations/init-hash-set! write-cursor)] - (->XITDBWriteHashSet whm))) + (->XITDBWriteHashSet (WriteCountedHashSet. write-cursor))) (defn xhash-set-counted [^ReadCursor cursor] (->XITDBHashSet (ReadCountedHashSet. cursor))) \ No newline at end of file diff --git a/src/xitdb/util/conversion.clj b/src/xitdb/util/conversion.clj index e51e509..e18d897 100644 --- a/src/xitdb/util/conversion.clj +++ b/src/xitdb/util/conversion.clj @@ -3,10 +3,10 @@ [xitdb.util.validation :as validation]) (:import [io.github.radarroark.xitdb - Database$Bytes Database$Float Database$Int - ReadArrayList ReadCursor ReadHashMap ReadCountedHashMap - Slot Tag WriteArrayList WriteCursor WriteCountedHashMap - WriteHashMap WriteLinkedArrayList])) + Database Database$Bytes Database$Float Database$Int + ReadArrayList ReadCountedHashSet ReadCursor ReadHashMap ReadCountedHashMap + ReadHashSet Slot Tag WriteArrayList WriteCountedHashSet WriteCursor WriteCountedHashMap + WriteHashMap WriteHashSet WriteLinkedArrayList])) (defn xit-tag->keyword "Converts a XitDB Tag enum to a corresponding Clojure keyword." @@ -50,6 +50,13 @@ (name key)) key)) +(defn hash-value ^bytes [^Database jdb v] + (let [hash-code (if (nil? v) 0 (.hashCode v)) + buffer (java.nio.ByteBuffer/allocate Integer/BYTES) + _ (.putInt buffer hash-code) + bytes (.array buffer)] + (.digest (.md jdb) bytes))) + (defn ^Slot primitive-for "Converts a Clojure primitive value to its corresponding XitDB representation. Handles strings, keywords, integers, booleans, and floats. @@ -109,13 +116,30 @@ (instance? WriteHashMap v) (-> ^WriteHashMap v .cursor .slot) - ;;TODO: Confirm that it is correct to return the Read slots (instance? ReadHashMap v) (-> ^ReadHashMap v .cursor .slot) + (instance? ReadCountedHashMap v) + (-> ^ReadCountedHashMap v .cursor .slot) + + (instance? WriteCountedHashMap v) + (-> ^WriteCountedHashMap v .cursor .slot) + (instance? ReadArrayList v) (-> ^ReadArrayList v .cursor .slot) + (instance? ReadHashSet v) + (-> ^ReadHashSet v .cursor .slot) + + (instance? ReadCountedHashSet v) + (-> ^ReadCountedHashSet v .cursor .slot) + + (instance? WriteHashSet v) + (-> ^WriteHashSet v .cursor .slot) + + (instance? WriteCountedHashSet v) + (-> ^WriteCountedHashSet v .cursor .slot) + (map? v) (do (.write cursor nil) @@ -238,23 +262,21 @@ [^WriteCursor cursor m] (let [whm (WriteCountedHashMap. cursor)] (doseq [[k v] m] - (let [cursor (.putCursor whm (db-key k))] + (let [hash-value (hash-value (-> cursor .db) k) + key-cursor (.putKeyCursor whm hash-value) + cursor (.putCursor whm hash-value)] + (.writeIfEmpty key-cursor (v->slot! key-cursor k)) (.write cursor (v->slot! cursor v)))) (.-cursor whm))) (defn ^WriteCursor set->WriteCursor! - "Creates a hash-map and associates the internal key :is-set? to 1. - Map is keyed by the .hashCode of the value, valued by the value :)" + "Writes a Clojure set `s` to a XitDB WriteHashSet. + Returns the cursor of the created WriteHashSet." [^WriteCursor cursor s] - (let [whm (WriteHashMap. cursor) - ;; Mark as set - is-set-key (db-key :%xitdb_set)] - (-> whm - (.putCursor is-set-key) - (.write (primitive-for 1))) - ;; Add values + (let [whm (WriteCountedHashSet. cursor) + db (-> cursor .db)] (doseq [v s] - (let [hash-code (if v (.hashCode v) 0) - cursor (.putCursor whm (db-key hash-code))] - (.write cursor (v->slot! cursor v)))) + (let [hash-code (hash-value db v) + cursor (.putCursor whm hash-code)] + (.writeIfEmpty cursor (v->slot! cursor v)))) (.-cursor whm))) \ No newline at end of file diff --git a/src/xitdb/util/operations.clj b/src/xitdb/util/operations.clj index d2c03d5..6d63426 100644 --- a/src/xitdb/util/operations.clj +++ b/src/xitdb/util/operations.clj @@ -3,7 +3,7 @@ [xitdb.util.conversion :as conversion] [xitdb.util.validation :as validation]) (:import - [io.github.radarroark.xitdb ReadArrayList ReadCountedHashMap ReadHashMap ReadLinkedArrayList Tag WriteArrayList WriteCursor WriteHashMap WriteLinkedArrayList])) + [io.github.radarroark.xitdb Database ReadArrayList ReadCountedHashMap ReadCountedHashSet ReadHashMap ReadHashSet ReadLinkedArrayList Tag WriteArrayList WriteCursor WriteHashMap WriteHashSet WriteLinkedArrayList])) (def internal-keys "Map of logical internal key names to their actual storage keys in XitDB. @@ -112,12 +112,11 @@ Throws IllegalArgumentException if attempting to associate an internal key. Updates the internal count if fast counting is enabled." [^WriteHashMap whm k v] - (when (contains? hidden-keys k) - (throw (IllegalArgumentException. (str "Cannot assoc key. " k ". It is reserved for internal use.")))) - - (let [cursor (.putCursor whm (conversion/db-key k))] - (.write cursor (conversion/v->slot! cursor v)) - whm)) + (let [hash-value (conversion/hash-value (-> whm .cursor .db) k) + key-cursor (.putKeyCursor whm hash-value) + cursor (.putCursor whm hash-value)] + (.writeIfEmpty key-cursor (conversion/v->slot! key-cursor k)) + (.write cursor (conversion/v->slot! cursor v)))) (defn map-dissoc-key! "Removes a key-value pair from a WriteHashMap. @@ -127,7 +126,8 @@ (when (contains? hidden-keys k) (throw (IllegalArgumentException. (str "Cannot dissoc key. " k ". It is reserved for internal use.")))) - (.remove whm (conversion/db-key k)) + (let [hash-value (conversion/hash-value (-> whm .cursor .db) k)] + (.remove whm hash-value)) whm) (defn ^WriteHashMap map-empty! @@ -141,13 +141,14 @@ (defn map-contains-key? "Checks if a WriteHashMap contains the specified key. Returns true if the key exists, false otherwise." - [^WriteHashMap whm key] - (not (nil? (.getCursor whm (conversion/db-key key))))) + [^ReadHashMap whm key] + (let [hash-value (conversion/hash-value (-> whm .cursor .db) key)] + (not (nil? (.getKeyCursor whm hash-value))))) (defn map-item-count-iterated "Returns the number of keys in the map by iterating. The count includes internal keys if any." - [^ReadHashMap rhm] + [^Iterable rhm] (let [it (.iterator rhm)] (loop [cnt 0] (if (.hasNext it) @@ -167,60 +168,57 @@ "Gets a read cursor for the specified key in a ReadHashMap. Returns the cursor if the key exists, nil otherwise." [^ReadHashMap rhm key] - (.getCursor rhm (conversion/db-key key))) + (let [hash-value (conversion/hash-value (-> rhm .cursor .db) key)] + (.getCursor rhm hash-value))) + (defn map-write-cursor "Gets a write cursor for the specified key in a WriteHashMap. Creates the key if it doesn't exist." [^WriteHashMap whm key] - (.putCursor whm (conversion/db-key key))) + (let [hash-value (conversion/hash-value (-> whm .cursor .db) key)] + (.putCursor whm hash-value))) ;; ============================================================================ ;; Set Operations ;; ============================================================================ +(defn set-item-count + "Returns the number of key/vals in the map." + [^ReadHashSet rhs] + (if (instance? ReadCountedHashSet rhs) + (.count ^ReadCountedHashSet rhs) + (map-item-count-iterated rhs))) + (defn set-assoc-value! "Adds a value to a set (implemented as a WriteHashMap). Uses the value's hashCode as the key and the value itself as the value. Only adds the value if it doesn't already exist (based on hashCode). Returns the modified WriteHashMap." - [^WriteHashMap whm v] - (let [hash-code (if v (.hashCode v) 0)] - (let [cursor (.putCursor whm (conversion/db-key hash-code)) - new? (= (-> cursor .slot .tag) Tag/NONE)] - (when new? - ;; Only write value when the hashCode key doesn't exist - (.write cursor (conversion/v->slot! cursor v))) - whm))) - -(defn ^WriteHashMap mark-as-set! - "Marks a WriteHashMap as being a set by adding an internal marker. - This allows the system to distinguish between maps and sets. - Returns the modified WriteHashMap." - [^WriteHashMap whm] - (let [is-set-key (conversion/db-key (internal-keys :is-set?))] - (-> whm - (.putCursor is-set-key) - (.write (conversion/primitive-for 1))) - whm)) - -(defn ^WriteHashMap init-hash-set! - "Initializes a new WriteHashMap as a set. - Creates a WriteHashMap and marks it as a set using the internal marker. - Returns the newly created WriteHashMap configured as a set." - [^WriteCursor cursor] - (let [whm (WriteHashMap. cursor)] - (mark-as-set! whm) - whm)) + [^WriteHashSet whs v] + (let [hash-code (conversion/hash-value (-> whs .cursor .db) v) + cursor (.putCursor whs hash-code)] + (.writeIfEmpty cursor (conversion/v->slot! cursor v)) + whs)) + +(defn set-disj-value! + [^WriteHashSet whs v] + (let [hash-code (conversion/hash-value (-> whs .cursor .db) v)] + (.remove whs hash-code) + whs)) + +(defn set-contains? + [rhs v] + (let [hash-code (conversion/hash-value (-> rhs .-cursor .-db) v) + cursor (.getCursor rhs hash-code)] + (some? cursor))) (defn ^WriteHashMap set-empty! - "Empties a set (WriteHashMap) and re-initializes it as an empty set. - Clears all values and re-adds the internal set marker. - Returns the emptied and re-initialized WriteHashMap." - [^WriteHashMap whm] - (map-empty! whm) - (init-hash-set! (.cursor whm)) - whm) + "" + [^WriteHashSet whs] + (let [empty-set (conversion/v->slot! (.cursor whs) #{})] + (.write ^WriteCursor (.cursor whs) empty-set)) + whs) ;; ============================================================================ ;; Sequence Operations @@ -228,20 +226,33 @@ (defn map-seq "Return a lazy seq of key-value MapEntry pairs, skipping hidden keys." - [^ReadCountedHashMap rhm read-from-cursor] + [^ReadHashMap rhm read-from-cursor] (let [it (.iterator rhm)] (letfn [(step [] (lazy-seq (when (.hasNext it) (let [cursor (.next it) kv (.readKeyValuePair cursor) - k (conversion/read-bytes-with-format-tag (.-keyCursor kv))] + k (read-from-cursor (.-keyCursor kv))] (if (contains? hidden-keys k) (step) (let [v (read-from-cursor (.-valueCursor kv))] (cons (clojure.lang.MapEntry. k v) (step))))))))] (step)))) +(defn set-seq + "Return a lazy seq values from the set." + [rhm read-from-cursor] + (let [it (.iterator rhm)] + (letfn [(step [] + (lazy-seq + (when (.hasNext it) + (let [cursor (.next it) + kv (.readKeyValuePair cursor) + v (read-from-cursor (.-keyCursor kv))] + (cons v (step))))))] + (step)))) + (defn array-seq "Creates a lazy sequence from a ReadArrayList. Uses the provided read-from-cursor function to convert cursors to values. diff --git a/test/xitdb/set_test.clj b/test/xitdb/set_test.clj index 59c12fd..686708f 100644 --- a/test/xitdb/set_test.clj +++ b/test/xitdb/set_test.clj @@ -10,7 +10,6 @@ (testing "Set works" (with-db [db (tu/test-db)] (reset! db #{1 2 3 4 5}) - (swap! db conj 6) (swap! db disj 2 3) @@ -89,4 +88,26 @@ (is (true? (contains? sweets nil))))))) +(defn -intersection [s1 s2] + (if (< (count s2) (count s1)) + (recur s2 s1) + (reduce (fn [result item] + (println "acc:" result "s2:" s2 "item:" item + (type item) "contains?" (contains? s2 item) + (type result) + "result:" (if (contains? s2 item) + result + (disj result item))) + (if (contains? s2 item) + result + (disj result item))) + s1 s1))) +(comment + (with-db [db (tu/test-db)] + (reset! db #{1 2 3 4 5}) + + (swap! db set/intersection #{4 5 8}) + + #_(is (= #{4 5} @db)) + #_(is (tu/db-equal-to-atom? db)))) From 2c4390fa38bb25d409a2bd1f520aecc4047c06f4 Mon Sep 17 00:00:00 2001 From: Florin Braghis Date: Mon, 26 May 2025 16:39:22 +0200 Subject: [PATCH 03/14] Get rid of hidden keys, handle nil keys --- src/xitdb/util/conversion.clj | 12 ++++++---- src/xitdb/util/operations.clj | 43 ++++++++--------------------------- test/xitdb/database_test.clj | 4 +--- 3 files changed, 18 insertions(+), 41 deletions(-) diff --git a/src/xitdb/util/conversion.clj b/src/xitdb/util/conversion.clj index e18d897..ec4d64d 100644 --- a/src/xitdb/util/conversion.clj +++ b/src/xitdb/util/conversion.clj @@ -51,11 +51,13 @@ key)) (defn hash-value ^bytes [^Database jdb v] - (let [hash-code (if (nil? v) 0 (.hashCode v)) - buffer (java.nio.ByteBuffer/allocate Integer/BYTES) - _ (.putInt buffer hash-code) - bytes (.array buffer)] - (.digest (.md jdb) bytes))) + (if (nil? v) + (byte-array (-> jdb .-header .hashSize)) + (let [hash-code (.hashCode v) + buffer (java.nio.ByteBuffer/allocate Integer/BYTES) + _ (.putInt buffer hash-code) + bytes (.array buffer)] + (.digest (.md jdb) bytes)))) (defn ^Slot primitive-for "Converts a Clojure primitive value to its corresponding XitDB representation. diff --git a/src/xitdb/util/operations.clj b/src/xitdb/util/operations.clj index 6d63426..df7196f 100644 --- a/src/xitdb/util/operations.clj +++ b/src/xitdb/util/operations.clj @@ -5,23 +5,6 @@ (:import [io.github.radarroark.xitdb Database ReadArrayList ReadCountedHashMap ReadCountedHashSet ReadHashMap ReadHashSet ReadLinkedArrayList Tag WriteArrayList WriteCursor WriteHashMap WriteHashSet WriteLinkedArrayList])) -(def internal-keys - "Map of logical internal key names to their actual storage keys in XitDB. - These keys are used internally by the system and should not be exposed to users." - {:count :%xitdb__count - :is-set? :%xitdb_set}) - -(def hidden-keys - "Set of keys that are used internally and should be hidden from user operations. - Operations like seq, reduce, and count will skip these keys." - (set (vals internal-keys))) - -(def ^:dynamic *enable-map-fast-count?* - "When true, maps store their item count in an internal key for O(1) count operations. - When false, count operations require iteration over all entries (O(n)). - Default is false to minimize storage overhead." - false) - ;; ============================================================================ ;; Array List Operations ;; ============================================================================ @@ -123,9 +106,6 @@ Throws IllegalArgumentException if attempting to remove an internal key. Updates the internal count if fast counting is enabled." [^WriteHashMap whm k] - (when (contains? hidden-keys k) - (throw (IllegalArgumentException. (str "Cannot dissoc key. " k ". It is reserved for internal use.")))) - (let [hash-value (conversion/hash-value (-> whm .cursor .db) k)] (.remove whm hash-value)) whm) @@ -225,7 +205,7 @@ ;; ============================================================================ (defn map-seq - "Return a lazy seq of key-value MapEntry pairs, skipping hidden keys." + "Return a lazy seq of key-value MapEntry pairs." [^ReadHashMap rhm read-from-cursor] (let [it (.iterator rhm)] (letfn [(step [] @@ -234,10 +214,8 @@ (let [cursor (.next it) kv (.readKeyValuePair cursor) k (read-from-cursor (.-keyCursor kv))] - (if (contains? hidden-keys k) - (step) - (let [v (read-from-cursor (.-valueCursor kv))] - (cons (clojure.lang.MapEntry. k v) (step))))))))] + (let [v (read-from-cursor (.-valueCursor kv))] + (cons (clojure.lang.MapEntry. k v) (step)))))))] (step)))) (defn set-seq @@ -287,14 +265,13 @@ (if (.hasNext it) (let [cursor (.next it) kv (.readKeyValuePair cursor) - k (conversion/read-bytes-with-format-tag (.-keyCursor kv))] - (if (contains? hidden-keys k) - (recur result) - (let [v (read-from-cursor (.-valueCursor kv)) - new-result (f result k v)] - (if (reduced? new-result) - @new-result - (recur new-result))))) + k (read-from-cursor (.-keyCursor kv))] + (let [v (read-from-cursor (.-valueCursor kv)) + new-result (f result k v)] + (if (reduced? new-result) + @new-result + (recur new-result)))) + result)))) (defn array-kv-reduce diff --git a/test/xitdb/database_test.clj b/test/xitdb/database_test.clj index 93ca41c..fb5cd26 100644 --- a/test/xitdb/database_test.clj +++ b/test/xitdb/database_test.clj @@ -376,11 +376,9 @@ (is (= 0 (count @db))) (is (empty? @db)) - (is (thrown? IllegalArgumentException (swap! db assoc :%xitdb__count -3))) - (is (thrown? IllegalArgumentException (swap! db dissoc :%xitdb__count))) - (is (tu/db-equal-to-atom? db)))) + (deftest NilTest (testing "nil values" (with-db [db (tu/test-db)] From 76a7c90962565677b5cbf6e812d232e0b2100eb5 Mon Sep 17 00:00:00 2001 From: Florin Braghis Date: Mon, 26 May 2025 18:38:38 +0200 Subject: [PATCH 04/14] Use `hash` instead of .hashCode, fix writing the slot to the db --- src/xitdb/array_list.clj | 1 - src/xitdb/db.clj | 11 +++++++++-- src/xitdb/util/conversion.clj | 2 +- test/xitdb/database_test.clj | 7 +++++++ test/xitdb/set_test.clj | 26 +++----------------------- 5 files changed, 20 insertions(+), 27 deletions(-) diff --git a/src/xitdb/array_list.clj b/src/xitdb/array_list.clj index eb4e9da..6d65d48 100644 --- a/src/xitdb/array_list.clj +++ b/src/xitdb/array_list.clj @@ -132,7 +132,6 @@ (.count wal)) (cons [this o] - ;;TODO: Figure out if it is correct to append to the end (operations/array-list-append-value! wal (common/unwrap o)) this) diff --git a/src/xitdb/db.clj b/src/xitdb/db.clj index 0b624bd..6e7112d 100644 --- a/src/xitdb/db.clj +++ b/src/xitdb/db.clj @@ -1,11 +1,12 @@ (ns xitdb.db (:require + [xitdb.common :as common] [xitdb.util.conversion :as conversion] [xitdb.xitdb-types :as xtypes]) (:import [io.github.radarroark.xitdb CoreBufferedFile CoreFile CoreMemory Database Database$ContextFunction Hasher - RandomAccessBufferedFile RandomAccessMemory ReadArrayList WriteArrayList WriteCursor] + RandomAccessBufferedFile RandomAccessMemory ReadArrayList ReadCursor WriteArrayList WriteCursor] [java.io File RandomAccessFile] [java.security MessageDigest] [java.util.concurrent.locks ReentrantLock])) @@ -47,6 +48,12 @@ hasher (Hasher. (MessageDigest/getInstance "SHA-1"))] (Database. core hasher))) +(defn v->slot! + [^WriteCursor cursor v] + (if (satisfies? common/ISlot v) + (common/-slot v) + (conversion/v->slot! cursor v))) + (defn xitdb-swap! "Returns history index." [db f & args] @@ -58,7 +65,7 @@ (fn [^WriteCursor cursor] (let [obj (xtypes/read-from-cursor cursor true)] (let [retval (apply f (into [obj] args))] - (.write cursor (conversion/v->slot! cursor retval)))))))) + (.write cursor (v->slot! cursor retval)))))))) (defn xitdb-swap-with-lock! "Performs the 'swap!' operation while locking `db.lock`. diff --git a/src/xitdb/util/conversion.clj b/src/xitdb/util/conversion.clj index ec4d64d..56a6a36 100644 --- a/src/xitdb/util/conversion.clj +++ b/src/xitdb/util/conversion.clj @@ -53,7 +53,7 @@ (defn hash-value ^bytes [^Database jdb v] (if (nil? v) (byte-array (-> jdb .-header .hashSize)) - (let [hash-code (.hashCode v) + (let [hash-code (hash v) buffer (java.nio.ByteBuffer/allocate Integer/BYTES) _ (.putInt buffer hash-code) bytes (.array buffer)] diff --git a/test/xitdb/database_test.clj b/test/xitdb/database_test.clj index fb5cd26..351559f 100644 --- a/test/xitdb/database_test.clj +++ b/test/xitdb/database_test.clj @@ -378,6 +378,12 @@ (is (tu/db-equal-to-atom? db)))) +(comment + (with-db [db (tu/test-memory-db-raw)] + (reset! db {:raw-songs-swap [1 2 3 4]}) + (swap! db update :raw-songs-swap into [5 6 7]) + nil)) + (deftest NilTest (testing "nil values" @@ -450,3 +456,4 @@ + diff --git a/test/xitdb/set_test.clj b/test/xitdb/set_test.clj index 686708f..37327fa 100644 --- a/test/xitdb/set_test.clj +++ b/test/xitdb/set_test.clj @@ -87,27 +87,7 @@ (let [sweets (:sweets @db)] (is (true? (contains? sweets nil))))))) - -(defn -intersection [s1 s2] - (if (< (count s2) (count s1)) - (recur s2 s1) - (reduce (fn [result item] - (println "acc:" result "s2:" s2 "item:" item - (type item) "contains?" (contains? s2 item) - (type result) - "result:" (if (contains? s2 item) - result - (disj result item))) - (if (contains? s2 item) - result - (disj result item))) - s1 s1))) -(comment +(deftest HashCodeTest (with-db [db (tu/test-db)] - (reset! db #{1 2 3 4 5}) - - (swap! db set/intersection #{4 5 8}) - - #_(is (= #{4 5} @db)) - #_(is (tu/db-equal-to-atom? db)))) - + (reset! db #{:one 1 []}) + (is (= #{:one 1 []} @db)))) From 1a4d28195549ead9a271b78933fb4ec4ae3ade97 Mon Sep 17 00:00:00 2001 From: Florin Braghis Date: Mon, 26 May 2025 18:46:11 +0200 Subject: [PATCH 05/14] Rename to db-key-hash, add docstring --- src/xitdb/hash_set.clj | 2 +- src/xitdb/util/conversion.clj | 14 +++++++++----- src/xitdb/util/operations.clj | 16 ++++++++-------- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/xitdb/hash_set.clj b/src/xitdb/hash_set.clj index 3acf5f8..e49c49e 100644 --- a/src/xitdb/hash_set.clj +++ b/src/xitdb/hash_set.clj @@ -17,7 +17,7 @@ (throw (UnsupportedOperationException. "XITDBHashSet is read-only"))) (contains [this k] - (not (nil? (.getCursor rhs (conversion/hash-value (-> rhs .cursor .db) key))))) + (not (nil? (.getCursor rhs (conversion/db-key-hash (-> rhs .cursor .db) key))))) (get [this k] (when (.contains this k) diff --git a/src/xitdb/util/conversion.clj b/src/xitdb/util/conversion.clj index 56a6a36..1f88c61 100644 --- a/src/xitdb/util/conversion.clj +++ b/src/xitdb/util/conversion.clj @@ -6,7 +6,8 @@ Database Database$Bytes Database$Float Database$Int ReadArrayList ReadCountedHashSet ReadCursor ReadHashMap ReadCountedHashMap ReadHashSet Slot Tag WriteArrayList WriteCountedHashSet WriteCursor WriteCountedHashMap - WriteHashMap WriteHashSet WriteLinkedArrayList])) + WriteHashMap WriteHashSet WriteLinkedArrayList] + [java.nio ByteBuffer])) (defn xit-tag->keyword "Converts a XitDB Tag enum to a corresponding Clojure keyword." @@ -50,11 +51,14 @@ (name key)) key)) -(defn hash-value ^bytes [^Database jdb v] +(defn db-key-hash + "Returns a byte array representing the stable hash digest of (Clojure) value `v`. + Uses the MessageDigest from the database." + ^bytes [^Database jdb v] (if (nil? v) (byte-array (-> jdb .-header .hashSize)) (let [hash-code (hash v) - buffer (java.nio.ByteBuffer/allocate Integer/BYTES) + buffer (ByteBuffer/allocate Integer/BYTES) _ (.putInt buffer hash-code) bytes (.array buffer)] (.digest (.md jdb) bytes)))) @@ -264,7 +268,7 @@ [^WriteCursor cursor m] (let [whm (WriteCountedHashMap. cursor)] (doseq [[k v] m] - (let [hash-value (hash-value (-> cursor .db) k) + (let [hash-value (db-key-hash (-> cursor .db) k) key-cursor (.putKeyCursor whm hash-value) cursor (.putCursor whm hash-value)] (.writeIfEmpty key-cursor (v->slot! key-cursor k)) @@ -278,7 +282,7 @@ (let [whm (WriteCountedHashSet. cursor) db (-> cursor .db)] (doseq [v s] - (let [hash-code (hash-value db v) + (let [hash-code (db-key-hash db v) cursor (.putCursor whm hash-code)] (.writeIfEmpty cursor (v->slot! cursor v)))) (.-cursor whm))) \ No newline at end of file diff --git a/src/xitdb/util/operations.clj b/src/xitdb/util/operations.clj index df7196f..a2a0e2f 100644 --- a/src/xitdb/util/operations.clj +++ b/src/xitdb/util/operations.clj @@ -95,7 +95,7 @@ Throws IllegalArgumentException if attempting to associate an internal key. Updates the internal count if fast counting is enabled." [^WriteHashMap whm k v] - (let [hash-value (conversion/hash-value (-> whm .cursor .db) k) + (let [hash-value (conversion/db-key-hash (-> whm .cursor .db) k) key-cursor (.putKeyCursor whm hash-value) cursor (.putCursor whm hash-value)] (.writeIfEmpty key-cursor (conversion/v->slot! key-cursor k)) @@ -106,7 +106,7 @@ Throws IllegalArgumentException if attempting to remove an internal key. Updates the internal count if fast counting is enabled." [^WriteHashMap whm k] - (let [hash-value (conversion/hash-value (-> whm .cursor .db) k)] + (let [hash-value (conversion/db-key-hash (-> whm .cursor .db) k)] (.remove whm hash-value)) whm) @@ -122,7 +122,7 @@ "Checks if a WriteHashMap contains the specified key. Returns true if the key exists, false otherwise." [^ReadHashMap whm key] - (let [hash-value (conversion/hash-value (-> whm .cursor .db) key)] + (let [hash-value (conversion/db-key-hash (-> whm .cursor .db) key)] (not (nil? (.getKeyCursor whm hash-value))))) (defn map-item-count-iterated @@ -148,7 +148,7 @@ "Gets a read cursor for the specified key in a ReadHashMap. Returns the cursor if the key exists, nil otherwise." [^ReadHashMap rhm key] - (let [hash-value (conversion/hash-value (-> rhm .cursor .db) key)] + (let [hash-value (conversion/db-key-hash (-> rhm .cursor .db) key)] (.getCursor rhm hash-value))) @@ -156,7 +156,7 @@ "Gets a write cursor for the specified key in a WriteHashMap. Creates the key if it doesn't exist." [^WriteHashMap whm key] - (let [hash-value (conversion/hash-value (-> whm .cursor .db) key)] + (let [hash-value (conversion/db-key-hash (-> whm .cursor .db) key)] (.putCursor whm hash-value))) ;; ============================================================================ @@ -176,20 +176,20 @@ Only adds the value if it doesn't already exist (based on hashCode). Returns the modified WriteHashMap." [^WriteHashSet whs v] - (let [hash-code (conversion/hash-value (-> whs .cursor .db) v) + (let [hash-code (conversion/db-key-hash (-> whs .cursor .db) v) cursor (.putCursor whs hash-code)] (.writeIfEmpty cursor (conversion/v->slot! cursor v)) whs)) (defn set-disj-value! [^WriteHashSet whs v] - (let [hash-code (conversion/hash-value (-> whs .cursor .db) v)] + (let [hash-code (conversion/db-key-hash (-> whs .cursor .db) v)] (.remove whs hash-code) whs)) (defn set-contains? [rhs v] - (let [hash-code (conversion/hash-value (-> rhs .-cursor .-db) v) + (let [hash-code (conversion/db-key-hash (-> rhs .-cursor .-db) v) cursor (.getCursor rhs hash-code)] (some? cursor))) From 2112f390ebe8580542d09e7e58d09a97c963183b Mon Sep 17 00:00:00 2001 From: Florin Braghis Date: Mon, 26 May 2025 18:54:06 +0200 Subject: [PATCH 06/14] Clean up --- src/xitdb/db.clj | 5 ++++- src/xitdb/hash_map.clj | 5 +++-- src/xitdb/hash_set.clj | 6 ++++-- src/xitdb/util/operations.clj | 2 +- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/xitdb/db.clj b/src/xitdb/db.clj index 6e7112d..a84f141 100644 --- a/src/xitdb/db.clj +++ b/src/xitdb/db.clj @@ -6,7 +6,7 @@ (:import [io.github.radarroark.xitdb CoreBufferedFile CoreFile CoreMemory Database Database$ContextFunction Hasher - RandomAccessBufferedFile RandomAccessMemory ReadArrayList ReadCursor WriteArrayList WriteCursor] + RandomAccessBufferedFile RandomAccessMemory ReadArrayList WriteArrayList WriteCursor] [java.io File RandomAccessFile] [java.security MessageDigest] [java.util.concurrent.locks ReentrantLock])) @@ -49,6 +49,9 @@ (Database. core hasher))) (defn v->slot! + "Converts a value to a slot which can be written to a cursor. + For XITDB* types (which support ISlot), will return `-slot`, + for all other types `conversion/v->slot!`" [^WriteCursor cursor v] (if (satisfies? common/ISlot v) (common/-slot v) diff --git a/src/xitdb/hash_map.clj b/src/xitdb/hash_map.clj index 6eaa139..cf30a84 100644 --- a/src/xitdb/hash_map.clj +++ b/src/xitdb/hash_map.clj @@ -1,10 +1,11 @@ (ns xitdb.hash-map (:require [xitdb.common :as common] - [xitdb.util.conversion :as conversion] [xitdb.util.operations :as operations]) (:import - [io.github.radarroark.xitdb ReadCountedHashMap ReadCursor ReadHashMap WriteCountedHashMap WriteCursor WriteHashMap])) + [io.github.radarroark.xitdb + ReadCountedHashMap ReadCursor ReadHashMap + WriteCountedHashMap WriteCursor WriteHashMap])) (defn map-seq [rhm] diff --git a/src/xitdb/hash_set.clj b/src/xitdb/hash_set.clj index e49c49e..6d2da8b 100644 --- a/src/xitdb/hash_set.clj +++ b/src/xitdb/hash_set.clj @@ -4,7 +4,9 @@ [xitdb.util.conversion :as conversion] [xitdb.util.operations :as operations]) (:import - [io.github.radarroark.xitdb ReadCountedHashSet ReadCursor ReadHashMap ReadHashSet WriteCountedHashSet WriteCursor WriteHashMap WriteHashSet])) + [io.github.radarroark.xitdb + ReadCountedHashSet ReadCursor ReadHashSet + WriteCountedHashSet WriteCursor WriteHashSet])) (defn set-seq [rhm] @@ -17,7 +19,7 @@ (throw (UnsupportedOperationException. "XITDBHashSet is read-only"))) (contains [this k] - (not (nil? (.getCursor rhs (conversion/db-key-hash (-> rhs .cursor .db) key))))) + (operations/set-contains? rhs k)) (get [this k] (when (.contains this k) diff --git a/src/xitdb/util/operations.clj b/src/xitdb/util/operations.clj index a2a0e2f..07e238e 100644 --- a/src/xitdb/util/operations.clj +++ b/src/xitdb/util/operations.clj @@ -3,7 +3,7 @@ [xitdb.util.conversion :as conversion] [xitdb.util.validation :as validation]) (:import - [io.github.radarroark.xitdb Database ReadArrayList ReadCountedHashMap ReadCountedHashSet ReadHashMap ReadHashSet ReadLinkedArrayList Tag WriteArrayList WriteCursor WriteHashMap WriteHashSet WriteLinkedArrayList])) + [io.github.radarroark.xitdb ReadArrayList ReadCountedHashMap ReadCountedHashSet ReadHashMap ReadHashSet ReadLinkedArrayList Tag WriteArrayList WriteCursor WriteHashMap WriteHashSet WriteLinkedArrayList])) ;; ============================================================================ ;; Array List Operations From f3152e2f4fd8ab3359ce8123ae0584439c4a8f22 Mon Sep 17 00:00:00 2001 From: Florin Braghis Date: Mon, 26 May 2025 18:55:56 +0200 Subject: [PATCH 07/14] Remove old utils, no longer relevant --- src/xitdb/xitdb_util.clj | 503 --------------------------------------- 1 file changed, 503 deletions(-) delete mode 100644 src/xitdb/xitdb_util.clj diff --git a/src/xitdb/xitdb_util.clj b/src/xitdb/xitdb_util.clj deleted file mode 100644 index 67d9def..0000000 --- a/src/xitdb/xitdb_util.clj +++ /dev/null @@ -1,503 +0,0 @@ -(ns xitdb.xitdb-util - (:import - [io.github.radarroark.xitdb Database$Float Database$Bytes Database$Int Database$Uint ReadArrayList ReadCursor ReadHashMap ReadLinkedArrayList Slot WriteArrayList WriteCursor WriteHashMap Tag WriteLinkedArrayList])) - -(defn xit-tag->keyword - "Converts a XitDB Tag enum to a corresponding Clojure keyword." - [tag] - (cond - (= tag Tag/NONE) :none - (= tag Tag/INDEX) :index - (= tag Tag/ARRAY_LIST) :array-list - (= tag Tag/LINKED_ARRAY_LIST) :linked-array-list - (= tag Tag/HASH_MAP) :hash-map - (= tag Tag/KV_PAIR) :kv-pair - (= tag Tag/BYTES) :bytes - (= tag Tag/SHORT_BYTES) :short-bytes - (= tag Tag/UINT) :uint - (= tag Tag/INT) :int - (= tag Tag/FLOAT) :float - :else :unknown)) - -;; map of logical tag -> string used as formatTag in the Bytes record. -(def fmt-tag-value - {:keyword "kw" - :boolean "bl" - :key-integer "ki" - :nil "nl" - :inst "in" - :date "da"}) - -(def true-str "#t") -(def false-str "#f") - -;; map of logical key -> key stored in the HashMap -(def internal-keys - {:count :%xitdb__count - :is-set? :%xitdb_set}) - -;; HashMap keys which are used internally and should be hidden from user -(def hidden-keys (set (vals internal-keys))) - -(declare ^WriteCursor map->WriteHashMapCursor!) -(declare ^WriteCursor coll->ArrayListCursor!) -(declare ^WriteCursor list->LinkedArrayListCursor!) -(declare ^WriteCursor set->WriteCursor!) - -(def ^:dynamic *debug?* false) - -(defn lazy-seq? [v] - (instance? clojure.lang.LazySeq v)) - -(defn vector-or-chunked? [v] - (or (vector? v) (chunked-seq? v))) - -(defn list-or-cons? [v] - (or (list? v) (instance? clojure.lang.Cons v))) - -(defn ^String keyname [key] - (if (keyword? key) - (if (namespace key) - (str (namespace key) "/" (name key)) - (name key)) - key)) - - -(defn ^Database$Bytes database-bytes - ([^String s] - (Database$Bytes. s)) - ([^String s ^String tag] - (Database$Bytes. s tag))) - - -(defn ^Slot primitive-for - "Converts a Clojure primitive value to its corresponding XitDB representation. - Handles strings, keywords, integers, booleans, and floats. - Throws an IllegalArgumentException for unsupported types." - [v] - (cond - - (lazy-seq? v) - (throw (IllegalArgumentException. "Lazy sequences can be infinite and not allowed!")) - - (string? v) - (database-bytes v) - - (keyword? v) - (database-bytes (keyname v) (fmt-tag-value :keyword)) - - (integer? v) - (Database$Int. v) - - (boolean? v) - (database-bytes (if v true-str false-str) (fmt-tag-value :boolean)) - - (double? v) - (Database$Float. v) - - (nil? v) - (database-bytes "" (fmt-tag-value :nil)) - - (instance? java.time.Instant v) - (database-bytes (str v) (fmt-tag-value :inst)) - - (instance? java.util.Date v) - (database-bytes (str (.toInstant ^java.util.Date v)) (fmt-tag-value :date)) - - :else - (throw (IllegalArgumentException. (str "Unsupported type: " (type v) v))))) - -(defn ^Slot v->slot! - "Converts a value to a XitDB slot. - Handles WriteArrayList and WriteHashMap instances directly. - Recursively processes Clojure maps and collections. - Falls back to primitive conversion for other types." - [^WriteCursor cursor v] - (cond - - (instance? WriteArrayList v) - (-> ^WriteArrayList v .cursor .slot) - - (instance? WriteLinkedArrayList v) - (-> ^WriteLinkedArrayList v .cursor .slot) - - (instance? WriteHashMap v) - (-> ^WriteHashMap v .cursor .slot) - - ;;TODO: Confirm that it is correct to return the Read slots - (instance? ReadHashMap v) - (-> ^ReadHashMap v .cursor .slot) - - (instance? ReadArrayList v) - (-> ^ReadArrayList v .cursor .slot) - - (map? v) - (do - (.write cursor nil) - (.slot (map->WriteHashMapCursor! cursor v))) - - (list-or-cons? v) - (do - (.write cursor nil) - (.slot (list->LinkedArrayListCursor! cursor v))) - - (vector-or-chunked? v) - (do - (.write cursor nil) - (.slot (coll->ArrayListCursor! cursor v))) - - (set? v) - (do - (.write cursor nil) - (.slot (set->WriteCursor! cursor v))) - - :else - (primitive-for v))) - -(defn ^WriteArrayList array-list-append-value! - "Appends a value to a WriteArrayList. - Converts the value to an appropriate XitDB representation using v->slot!." - [^WriteArrayList wal v] - (let [cursor (.appendCursor wal)] - (.write cursor (v->slot! cursor v)) - wal)) - -(defn ^WriteArrayList array-list-assoc-value! - "Associates a value at index i in a WriteArrayList. - Appends the value if the index equals the current count. - Replaces the value at the specified index otherwise. - Throws an IllegalArgumentException if the index is out of bounds." - [^WriteArrayList wal i v] - - (assert (= Tag/ARRAY_LIST (-> wal .cursor .slot .tag))) - (assert (number? i)) - - (when (> i (.count wal)) - (throw (IllegalArgumentException. "Index out of bounds. "))) - - (let [cursor (if (= i (.count wal)) - (.appendCursor wal) - (.putCursor wal i))] - (.write cursor (v->slot! cursor v))) - wal) - -(defn array-list-pop! [^WriteArrayList wal] - (when (zero? (.count wal)) - (throw (IllegalStateException. "Can't pop empty array"))) - - (.slice wal (dec (.count wal)))) - -(defn array-list-empty! [^WriteArrayList wal] - (let [^WriteCursor cursor (-> wal .cursor)] - (.write cursor (v->slot! cursor [])))) - -(defn linked-array-list-append-value! - "Appends a value to a WriteLinkedArrayList. - Converts the value to an appropriate XitDB representation using v->slot!." - [^WriteLinkedArrayList wlal v] - (let [cursor (.appendCursor wlal)] - (.write cursor (v->slot! cursor v)) - nil)) - -(defn linked-array-list-insert-value! - "Appends a value to a WriteLinkedArrayList. - Converts the value to an appropriate XitDB representation using v->slot!." - [^WriteLinkedArrayList wlal pos v] - (let [cursor (-> wlal .cursor)] - (.insert wlal pos (v->slot! cursor v))) - nil) - -(defn linked-array-list-pop! - [^WriteLinkedArrayList wlal] - (.remove wlal 0) - nil) - -(defn ^Database$Bytes db-key - "Converts k from a Clojure type to a Database$Bytes representation to be used in - cursor functions." - [k] - (cond - (integer? k) - (database-bytes (str k) "ki") ;integer keys are stored as strings with 'ki' format tag - :else - (primitive-for k))) - -;; Enable storing the count of items in the hashmap under an internal key :count -(def ^:dynamic *enable-map-fast-count?* false) - -(defn- update-map-item-count! - "Update the internal key `:count` by applying `f` to the current value. - If the key `:count` does not exist, it is created." - [^WriteHashMap whm f] - (when *enable-map-fast-count?* - (let [count-cursor (.putCursor whm (db-key (internal-keys :count))) - value (try - (.readInt count-cursor) - (catch Exception _ 0)) - new-value (primitive-for (f (or value 0)))] - (.write count-cursor new-value)))) - -(defn- map-item-count-stored - "Returns the value of the internal key `:count`." - [^ReadHashMap rhm] - (let [count-cursor (.getCursor rhm (db-key (internal-keys :count)))] - (.readInt count-cursor))) - -(defn map-assoc-value! - "Associates a key-value pair in a WriteHashMap. - Converts the key to a string and the value to an appropriate XitDB representation. - throws when trying to associate a internal key." - [^WriteHashMap whm k v] - (when (contains? hidden-keys k) - (throw (IllegalArgumentException. (str "Cannot assoc key. " k ". It is reserved for internal use.")))) - - (let [cursor (.putCursor whm (db-key k)) - new? (= (-> cursor .slot .tag) Tag/NONE)] - (.write cursor (v->slot! cursor v)) - (when new? - (update-map-item-count! whm inc)) - whm)) - -(defn map-dissoc-key! - [^WriteHashMap whm k] - (when (contains? hidden-keys k) - (throw (IllegalArgumentException. (str "Cannot dissoc key. " k ". It is reserved for internal use.")))) - - (when (.remove whm (db-key k)) - (update-map-item-count! whm dec))) - -(defn ^WriteHashMap map-empty! [^WriteHashMap whm] - (let [^WriteCursor cursor (-> whm .cursor)] - (.write cursor (v->slot! cursor {})) - whm)) - -(defn map-contains-key? [^WriteHashMap whm key] - (not (nil? (.getCursor whm (db-key key))))) - -(defn map-item-count-iterated - "Returns the number of keys in the map by iterating. - The count includes internal keys if any." - [^ReadHashMap rhm] - (let [it (.iterator rhm)] - (loop [cnt 0] - (if (.hasNext it) - (do - (.next it) - (recur (inc cnt))) - cnt)))) - -(defn map-item-count - "Returns the number of key/vals in the map." - [^ReadHashMap rhm] - (if *enable-map-fast-count?* - (map-item-count-stored rhm) - (map-item-count-iterated rhm))) - -(defn map-read-cursor [^ReadHashMap rhm key] - (.getCursor rhm (db-key key))) - -(defn map-write-cursor [^WriteHashMap whm key] - (.putCursor whm (db-key key))) - -(defn ^WriteCursor coll->ArrayListCursor! - "Converts a Clojure collection to a XitDB ArrayList cursor. - Handles nested maps and collections recursively. - Returns the cursor of the created WriteArrayList." - [^WriteCursor cursor coll] - (when *debug?* (println "Write array" (type coll))) - (let [write-array (WriteArrayList. cursor)] - (doseq [v coll] - (cond - (map? v) - (let [v-cursor (.appendCursor write-array)] - (map->WriteHashMapCursor! v-cursor v)) - - (list-or-cons? v) - (let [v-cursor (.appendCursor write-array)] - (list->LinkedArrayListCursor! v-cursor v)) - - (vector-or-chunked? v) - (let [v-cursor (.appendCursor write-array)] - (coll->ArrayListCursor! v-cursor v)) - - :else - (.append write-array (primitive-for v)))) - (.-cursor write-array))) - -(defn ^WriteCursor list->LinkedArrayListCursor! - "Converts a Clojure list or seq-like collection to a XitDB LinkedArrayList cursor. - Optimized for sequential access collections rather than random access ones." - [^WriteCursor cursor coll] - (when *debug?* (println "Write list" (type coll))) - (let [write-list (WriteLinkedArrayList. cursor)] - (doseq [v coll] - (when *debug?* (println "v=" v)) - (cond - (map? v) - (let [v-cursor (.appendCursor write-list)] - (map->WriteHashMapCursor! v-cursor v)) - - (lazy-seq? v) - (throw (IllegalArgumentException. "Lazy sequences can be infinite and not allowed !")) - - (list-or-cons? v) - (let [v-cursor (.appendCursor write-list)] - (list->LinkedArrayListCursor! v-cursor v)) - - (vector-or-chunked? v) - (let [v-cursor (.appendCursor write-list)] - (coll->ArrayListCursor! v-cursor v)) - - :else - (.append write-list (primitive-for v)))) - (.-cursor write-list))) - -;; ---------- - -(defn set-assoc-value! - [^WriteHashMap whm v] - (let [hash-code (if v (.hashCode v) 0)] - (let [cursor (.putCursor whm (db-key hash-code)) - new? (= (-> cursor .slot .tag) Tag/NONE)] - (when new? - ;; Only write value when the hashCode key doesn't exist - (.write cursor (v->slot! cursor v)) - (update-map-item-count! whm inc)) - whm))) - - - -(defn ^WriteHashMap mark-as-set! [^WriteHashMap whm] - (let [is-set-key (db-key (internal-keys :is-set?))] - (-> whm - (.putCursor is-set-key) - (.write (primitive-for 1))) - whm)) - -(defn ^WriteHashMap init-hash-set! [^WriteCursor cursor] - (let [whm (WriteHashMap. cursor)] - (mark-as-set! whm) - whm)) - -(defn ^WriteHashMap set-empty! [^WriteHashMap whm] - (map-empty! whm) - (init-hash-set! (.cursor whm)) - whm) - -(defn ^WriteCursor set->WriteCursor! - "Creates a hash-map and associates the internal key :is-set? to 1. - Map is keyed by the .hashCode of the value, valued by the value :)" - [^WriteCursor cursor s] - (let [whm (init-hash-set! cursor)] - (doseq [v s] - (set-assoc-value! whm v)) - (.-cursor whm))) - -(defn ^WriteCursor map->WriteHashMapCursor! - "Writes a Clojure map to a XitDB WriteHashMap. - Returns the cursor of the created WriteHashMap." - [^WriteCursor cursor m] - (let [whm (WriteHashMap. cursor)] - (doseq [[k v] m] - (map-assoc-value! whm k v)) - (.-cursor whm))) - -(defn read-bytes-with-format-tag [^ReadCursor cursor] - (let [bytes-obj (.readBytesObject cursor nil) - str (String. (.value bytes-obj)) - fmt-tag (some-> bytes-obj .formatTag String.)] - (cond - - (= fmt-tag (fmt-tag-value :keyword)) - (keyword str) - - (= fmt-tag (fmt-tag-value :boolean)) - (= str true-str) - - (= fmt-tag (fmt-tag-value :key-integer)) - (Integer/parseInt str) - - (= fmt-tag (fmt-tag-value :inst)) - (java.time.Instant/parse str) - - - (= fmt-tag (fmt-tag-value :date)) - (java.util.Date/from - (java.time.Instant/parse str)) - - - (= fmt-tag (fmt-tag-value :nil)) - nil - - :else - str))) - -(defn map-seq - "Return a lazy seq of key-value MapEntry pairs, skipping hidden keys." - [^ReadHashMap rhm read-from-cursor] - (let [it (.iterator rhm)] - (letfn [(step [] - (lazy-seq - (when (.hasNext it) - (let [cursor (.next it) - kv (.readKeyValuePair cursor) - k (read-bytes-with-format-tag (.-keyCursor kv))] - (if (contains? hidden-keys k) - (step) - (let [v (read-from-cursor (.-valueCursor kv))] - (cons (clojure.lang.MapEntry. k v) (step))))))))] - (step)))) - -(defn array-seq [^ReadArrayList ral read-from-cursor] - (let [iter (.iterator ral) - lazy-iter (fn lazy-iter [] - (when (.hasNext iter) - (let [cursor (.next iter) - value (read-from-cursor cursor)] - (lazy-seq (cons value (lazy-iter))))))] - (lazy-iter))) - -;;Same as above, but different type hints -(defn linked-array-seq [^ReadLinkedArrayList rlal read-from-cursor] - (let [iter (.iterator rlal) - lazy-iter (fn lazy-iter [] - (when (.hasNext iter) - (let [cursor (.next iter) - value (read-from-cursor cursor)] - (lazy-seq (cons value (lazy-iter))))))] - (lazy-iter))) - -(defn map-kv-reduce - "Efficiently reduces over key-value pairs in a ReadHashMap, skipping hidden keys." - [^ReadHashMap rhm read-from-cursor f init] - (let [it (.iterator rhm)] - (loop [result init] - (if (.hasNext it) - (let [cursor (.next it) - kv (.readKeyValuePair cursor) - k (read-bytes-with-format-tag (.-keyCursor kv))] - (if (contains? hidden-keys k) - (recur result) - (let [v (read-from-cursor (.-valueCursor kv)) - new-result (f result k v)] - (if (reduced? new-result) - @new-result - (recur new-result))))) - result)))) - -(defn array-kv-reduce - "Efficiently reduces over index-value pairs in a ReadArrayList." - [^ReadArrayList ral read-from-cursor f init] - (let [count (.count ral)] - (loop [i 0 - result init] - (if (< i count) - (let [cursor (.getCursor ral i) - v (read-from-cursor cursor) - new-result (f result i v)] - (if (reduced? new-result) - @new-result - (recur (inc i) new-result))) - result)))) - - From 909e138785c76b3f06bddf20e5ff95e10feee51a Mon Sep 17 00:00:00 2001 From: Florin Braghis Date: Mon, 26 May 2025 18:59:06 +0200 Subject: [PATCH 08/14] Fix docstring --- src/xitdb/util/operations.clj | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/xitdb/util/operations.clj b/src/xitdb/util/operations.clj index 07e238e..b459d24 100644 --- a/src/xitdb/util/operations.clj +++ b/src/xitdb/util/operations.clj @@ -164,17 +164,14 @@ ;; ============================================================================ (defn set-item-count - "Returns the number of key/vals in the map." + "Returns the number of values in the set." [^ReadHashSet rhs] (if (instance? ReadCountedHashSet rhs) (.count ^ReadCountedHashSet rhs) (map-item-count-iterated rhs))) (defn set-assoc-value! - "Adds a value to a set (implemented as a WriteHashMap). - Uses the value's hashCode as the key and the value itself as the value. - Only adds the value if it doesn't already exist (based on hashCode). - Returns the modified WriteHashMap." + "Adds a value to a set." [^WriteHashSet whs v] (let [hash-code (conversion/db-key-hash (-> whs .cursor .db) v) cursor (.putCursor whs hash-code)] @@ -182,19 +179,21 @@ whs)) (defn set-disj-value! + "Removes a value from a set" [^WriteHashSet whs v] (let [hash-code (conversion/db-key-hash (-> whs .cursor .db) v)] (.remove whs hash-code) whs)) (defn set-contains? + "Returns true if `v` is in the set." [rhs v] (let [hash-code (conversion/db-key-hash (-> rhs .-cursor .-db) v) cursor (.getCursor rhs hash-code)] (some? cursor))) (defn ^WriteHashMap set-empty! - "" + "Replaces the whs value with an empty set." [^WriteHashSet whs] (let [empty-set (conversion/v->slot! (.cursor whs) #{})] (.write ^WriteCursor (.cursor whs) empty-set)) From 53eae3e04efa69c0975e34c4baaf96bc949896a1 Mon Sep 17 00:00:00 2001 From: Florin Braghis Date: Mon, 26 May 2025 19:01:53 +0200 Subject: [PATCH 09/14] Use key-hash instead of hash-value --- src/xitdb/util/operations.clj | 42 +++++++++++++++++------------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/xitdb/util/operations.clj b/src/xitdb/util/operations.clj index b459d24..aac62e3 100644 --- a/src/xitdb/util/operations.clj +++ b/src/xitdb/util/operations.clj @@ -95,9 +95,9 @@ Throws IllegalArgumentException if attempting to associate an internal key. Updates the internal count if fast counting is enabled." [^WriteHashMap whm k v] - (let [hash-value (conversion/db-key-hash (-> whm .cursor .db) k) - key-cursor (.putKeyCursor whm hash-value) - cursor (.putCursor whm hash-value)] + (let [key-hash (conversion/db-key-hash (-> whm .cursor .db) k) + key-cursor (.putKeyCursor whm key-hash) + cursor (.putCursor whm key-hash)] (.writeIfEmpty key-cursor (conversion/v->slot! key-cursor k)) (.write cursor (conversion/v->slot! cursor v)))) @@ -106,8 +106,8 @@ Throws IllegalArgumentException if attempting to remove an internal key. Updates the internal count if fast counting is enabled." [^WriteHashMap whm k] - (let [hash-value (conversion/db-key-hash (-> whm .cursor .db) k)] - (.remove whm hash-value)) + (let [key-hash (conversion/db-key-hash (-> whm .cursor .db) k)] + (.remove whm key-hash)) whm) (defn ^WriteHashMap map-empty! @@ -122,8 +122,8 @@ "Checks if a WriteHashMap contains the specified key. Returns true if the key exists, false otherwise." [^ReadHashMap whm key] - (let [hash-value (conversion/db-key-hash (-> whm .cursor .db) key)] - (not (nil? (.getKeyCursor whm hash-value))))) + (let [key-hash (conversion/db-key-hash (-> whm .cursor .db) key)] + (not (nil? (.getKeyCursor whm key-hash))))) (defn map-item-count-iterated "Returns the number of keys in the map by iterating. @@ -148,16 +148,16 @@ "Gets a read cursor for the specified key in a ReadHashMap. Returns the cursor if the key exists, nil otherwise." [^ReadHashMap rhm key] - (let [hash-value (conversion/db-key-hash (-> rhm .cursor .db) key)] - (.getCursor rhm hash-value))) + (let [key-hash (conversion/db-key-hash (-> rhm .cursor .db) key)] + (.getCursor rhm key-hash))) (defn map-write-cursor "Gets a write cursor for the specified key in a WriteHashMap. Creates the key if it doesn't exist." [^WriteHashMap whm key] - (let [hash-value (conversion/db-key-hash (-> whm .cursor .db) key)] - (.putCursor whm hash-value))) + (let [key-hash (conversion/db-key-hash (-> whm .cursor .db) key)] + (.putCursor whm key-hash))) ;; ============================================================================ ;; Set Operations @@ -174,7 +174,7 @@ "Adds a value to a set." [^WriteHashSet whs v] (let [hash-code (conversion/db-key-hash (-> whs .cursor .db) v) - cursor (.putCursor whs hash-code)] + cursor (.putCursor whs hash-code)] (.writeIfEmpty cursor (conversion/v->slot! cursor v)) whs)) @@ -189,7 +189,7 @@ "Returns true if `v` is in the set." [rhs v] (let [hash-code (conversion/db-key-hash (-> rhs .-cursor .-db) v) - cursor (.getCursor rhs hash-code)] + cursor (.getCursor rhs hash-code)] (some? cursor))) (defn ^WriteHashMap set-empty! @@ -235,11 +235,11 @@ Uses the provided read-from-cursor function to convert cursors to values. Returns a lazy sequence of the array elements." [^ReadArrayList ral read-from-cursor] - (let [iter (.iterator ral) + (let [iter (.iterator ral) lazy-iter (fn lazy-iter [] (when (.hasNext iter) (let [cursor (.next iter) - value (read-from-cursor cursor)] + value (read-from-cursor cursor)] (lazy-seq (cons value (lazy-iter))))))] (lazy-iter))) @@ -248,11 +248,11 @@ Uses the provided read-from-cursor function to convert cursors to values. Returns a lazy sequence of the linked array elements." [^ReadLinkedArrayList rlal read-from-cursor] - (let [iter (.iterator rlal) + (let [iter (.iterator rlal) lazy-iter (fn lazy-iter [] (when (.hasNext iter) (let [cursor (.next iter) - value (read-from-cursor cursor)] + value (read-from-cursor cursor)] (lazy-seq (cons value (lazy-iter))))))] (lazy-iter))) @@ -265,7 +265,7 @@ (let [cursor (.next it) kv (.readKeyValuePair cursor) k (read-from-cursor (.-keyCursor kv))] - (let [v (read-from-cursor (.-valueCursor kv)) + (let [v (read-from-cursor (.-valueCursor kv)) new-result (f result k v)] (if (reduced? new-result) @new-result @@ -277,11 +277,11 @@ "Efficiently reduces over index-value pairs in a ReadArrayList." [^ReadArrayList ral read-from-cursor f init] (let [count (.count ral)] - (loop [i 0 + (loop [i 0 result init] (if (< i count) - (let [cursor (.getCursor ral i) - v (read-from-cursor cursor) + (let [cursor (.getCursor ral i) + v (read-from-cursor cursor) new-result (f result i v)] (if (reduced? new-result) @new-result From 4de1ef332e4b302b5d6ecc301dc3dc02f23fc2f8 Mon Sep 17 00:00:00 2001 From: Florin Braghis Date: Mon, 26 May 2025 19:02:57 +0200 Subject: [PATCH 10/14] Reformat --- src/xitdb/util/conversion.clj | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/xitdb/util/conversion.clj b/src/xitdb/util/conversion.clj index 1f88c61..96973e7 100644 --- a/src/xitdb/util/conversion.clj +++ b/src/xitdb/util/conversion.clj @@ -31,7 +31,7 @@ {:keyword "kw" :boolean "bl" :key-integer "ki" - :nil "nl" ;; TODO: Could use Tag/NONE instead + :nil "nl" ;; TODO: Could use Tag/NONE instead :inst "in" :date "da"}) @@ -175,14 +175,14 @@ [k] (cond (integer? k) - (database-bytes (str k) "ki") ;integer keys are stored as strings with 'ki' format tag + (database-bytes (str k) "ki") ;integer keys are stored as strings with 'ki' format tag :else (primitive-for k))) (defn read-bytes-with-format-tag [^ReadCursor cursor] (let [bytes-obj (.readBytesObject cursor nil) - str (String. (.value bytes-obj)) - fmt-tag (some-> bytes-obj .formatTag String.)] + str (String. (.value bytes-obj)) + fmt-tag (some-> bytes-obj .formatTag String.)] (cond (= fmt-tag (fmt-tag-value :keyword)) @@ -270,7 +270,7 @@ (doseq [[k v] m] (let [hash-value (db-key-hash (-> cursor .db) k) key-cursor (.putKeyCursor whm hash-value) - cursor (.putCursor whm hash-value)] + cursor (.putCursor whm hash-value)] (.writeIfEmpty key-cursor (v->slot! key-cursor k)) (.write cursor (v->slot! cursor v)))) (.-cursor whm))) @@ -280,9 +280,9 @@ Returns the cursor of the created WriteHashSet." [^WriteCursor cursor s] (let [whm (WriteCountedHashSet. cursor) - db (-> cursor .db)] + db (-> cursor .db)] (doseq [v s] (let [hash-code (db-key-hash db v) - cursor (.putCursor whm hash-code)] + cursor (.putCursor whm hash-code)] (.writeIfEmpty cursor (v->slot! cursor v)))) (.-cursor whm))) \ No newline at end of file From 06f26adf7941a1d13b76be62c0777b97d2405026 Mon Sep 17 00:00:00 2001 From: Florin Braghis Date: Mon, 26 May 2025 19:07:38 +0200 Subject: [PATCH 11/14] Remove comment --- test/xitdb/database_test.clj | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/xitdb/database_test.clj b/test/xitdb/database_test.clj index 351559f..65a0355 100644 --- a/test/xitdb/database_test.clj +++ b/test/xitdb/database_test.clj @@ -3,11 +3,6 @@ [clojure.test :refer :all] [xitdb.test-utils :as tu :refer [with-db]])) -(comment - (let [db (tu/test-memory-db-raw)] - (reset! db {:foo :bar}) - @db)) - (deftest DatabaseTest (with-db [db (tu/test-db)] (testing "Resetting to map" From 87988052542a18259f3b0d7c10e166b888a58e4f Mon Sep 17 00:00:00 2001 From: Florin Braghis Date: Mon, 26 May 2025 19:07:59 +0200 Subject: [PATCH 12/14] Remove another comment --- test/xitdb/database_test.clj | 7 ------- 1 file changed, 7 deletions(-) diff --git a/test/xitdb/database_test.clj b/test/xitdb/database_test.clj index 65a0355..544b99c 100644 --- a/test/xitdb/database_test.clj +++ b/test/xitdb/database_test.clj @@ -373,13 +373,6 @@ (is (tu/db-equal-to-atom? db)))) -(comment - (with-db [db (tu/test-memory-db-raw)] - (reset! db {:raw-songs-swap [1 2 3 4]}) - (swap! db update :raw-songs-swap into [5 6 7]) - nil)) - - (deftest NilTest (testing "nil values" (with-db [db (tu/test-db)] From cb74f19d828b1a30ece3784198c9b16a547b94fe Mon Sep 17 00:00:00 2001 From: Florin Braghis Date: Mon, 26 May 2025 19:10:12 +0200 Subject: [PATCH 13/14] Remove redundant let --- src/xitdb/util/operations.clj | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/xitdb/util/operations.clj b/src/xitdb/util/operations.clj index aac62e3..8b83086 100644 --- a/src/xitdb/util/operations.clj +++ b/src/xitdb/util/operations.clj @@ -262,15 +262,14 @@ (let [it (.iterator rhm)] (loop [result init] (if (.hasNext it) - (let [cursor (.next it) - kv (.readKeyValuePair cursor) - k (read-from-cursor (.-keyCursor kv))] - (let [v (read-from-cursor (.-valueCursor kv)) - new-result (f result k v)] - (if (reduced? new-result) - @new-result - (recur new-result)))) - + (let [cursor (.next it) + kv (.readKeyValuePair cursor) + k (read-from-cursor (.-keyCursor kv)) + v (read-from-cursor (.-valueCursor kv)) + new-result (f result k v)] + (if (reduced? new-result) + @new-result + (recur new-result))) result)))) (defn array-kv-reduce From ee169ff8c16de66675254d442b3924e501ee3a91 Mon Sep 17 00:00:00 2001 From: Florin Braghis Date: Mon, 26 May 2025 19:13:51 +0200 Subject: [PATCH 14/14] Finish the test --- test/xitdb/map_test.clj | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/test/xitdb/map_test.clj b/test/xitdb/map_test.clj index c6062da..976690f 100644 --- a/test/xitdb/map_test.clj +++ b/test/xitdb/map_test.clj @@ -5,6 +5,18 @@ (deftest map-with-complex-keys (with-db [db (tu/test-db)] - (reset! db {:foo {{:bar :baz} 42}}) - #_(reset! db {:foo {[1 :bar] 31 - [2 :baz] 42}}))) \ No newline at end of file + (testing "Composite values as keys" + (reset! db {:foo {{:bar :baz} 42}}) + (is (= {:foo {{:bar :baz} 42}} + (tu/materialize @db))) + + (reset! db {:foo {[1 :bar] 31 + [2 :baz] 42}}) + (is (= {:foo {[1 :bar] 31 + [2 :baz] 42}} + (tu/materialize @db))) + + (swap! db update :foo dissoc [2 :baz]) + + (is (= {:foo {[1 :bar] 31}} + (tu/materialize @db)))))) \ No newline at end of file