Release 7.020

techascent · Oct 29, 2023 · 2baeeff · 2baeeff
1 parent 80d8b09
commit 2baeeff
Show file tree

Hide file tree

Showing 40 changed files with 88 additions and 81 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,11 @@
 # Changelog
+# 7.020
+ * hamf perf upgrades.
+ * big perf upgrade for parsing sequences of maps.
+
+ # 7.019
+ * hamf perf upgrades.
+
 # 7.018
  * hamf perf upgrades.
 

diff --git a/deps.edn b/deps.edn
@@ -1,5 +1,5 @@
 {:paths ["src" "resources" "target/classes"]
- :deps {cnuernber/dtype-next {:mvn/version "10.106"}
+ :deps {cnuernber/dtype-next {:mvn/version "10.107"}
         techascent/tech.io {:mvn/version "4.31"
                             :exclusions [org.apache.commons/commons-compress]}
         org.apache.datasketches/datasketches-java {:mvn/version "4.2.0"}}
@@ -12,7 +12,7 @@
    :exec-fn codox.main/-main
    :exec-args {:group-id "techascent"
                :artifact-id "tech.ml.dataset"
-               :version "7.019"
+               :version "7.020"
                :name "TMD"
                :description "A Clojure high performance data processing system"
                :metadata {:doc/format :markdown}

diff --git a/docs/000-getting-started.html b/docs/000-getting-started.html
diff --git a/docs/100-walkthrough.html b/docs/100-walkthrough.html
diff --git a/docs/200-quick-reference.html b/docs/200-quick-reference.html
diff --git a/docs/columns-readers-and-datatypes.html b/docs/columns-readers-and-datatypes.html
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/nippy-serialization-rocks.html b/docs/nippy-serialization-rocks.html
diff --git a/docs/supported-datatypes.html b/docs/supported-datatypes.html
diff --git a/docs/tech.v3.dataset.categorical.html b/docs/tech.v3.dataset.categorical.html
diff --git a/docs/tech.v3.dataset.clipboard.html b/docs/tech.v3.dataset.clipboard.html
diff --git a/docs/tech.v3.dataset.column-filters.html b/docs/tech.v3.dataset.column-filters.html
diff --git a/docs/tech.v3.dataset.column.html b/docs/tech.v3.dataset.column.html
diff --git a/docs/tech.v3.dataset.html b/docs/tech.v3.dataset.html
diff --git a/docs/tech.v3.dataset.io.csv.html b/docs/tech.v3.dataset.io.csv.html
diff --git a/docs/tech.v3.dataset.io.datetime.html b/docs/tech.v3.dataset.io.datetime.html
diff --git a/docs/tech.v3.dataset.io.string-row-parser.html b/docs/tech.v3.dataset.io.string-row-parser.html
diff --git a/docs/tech.v3.dataset.io.univocity.html b/docs/tech.v3.dataset.io.univocity.html
diff --git a/docs/tech.v3.dataset.join.html b/docs/tech.v3.dataset.join.html
diff --git a/docs/tech.v3.dataset.math.html b/docs/tech.v3.dataset.math.html
diff --git a/docs/tech.v3.dataset.metamorph.html b/docs/tech.v3.dataset.metamorph.html
diff --git a/docs/tech.v3.dataset.modelling.html b/docs/tech.v3.dataset.modelling.html
diff --git a/docs/tech.v3.dataset.neanderthal.html b/docs/tech.v3.dataset.neanderthal.html
diff --git a/docs/tech.v3.dataset.print.html b/docs/tech.v3.dataset.print.html
diff --git a/docs/tech.v3.dataset.reductions.apache-data-sketch.html b/docs/tech.v3.dataset.reductions.apache-data-sketch.html
diff --git a/docs/tech.v3.dataset.reductions.html b/docs/tech.v3.dataset.reductions.html
diff --git a/docs/tech.v3.dataset.rolling.html b/docs/tech.v3.dataset.rolling.html
diff --git a/docs/tech.v3.dataset.set.html b/docs/tech.v3.dataset.set.html
diff --git a/docs/tech.v3.dataset.tensor.html b/docs/tech.v3.dataset.tensor.html
diff --git a/docs/tech.v3.dataset.zip.html b/docs/tech.v3.dataset.zip.html
diff --git a/docs/tech.v3.libs.arrow.html b/docs/tech.v3.libs.arrow.html
diff --git a/docs/tech.v3.libs.fastexcel.html b/docs/tech.v3.libs.fastexcel.html
diff --git a/docs/tech.v3.libs.guava.cache.html b/docs/tech.v3.libs.guava.cache.html
diff --git a/docs/tech.v3.libs.parquet.html b/docs/tech.v3.libs.parquet.html
diff --git a/docs/tech.v3.libs.poi.html b/docs/tech.v3.libs.poi.html
diff --git a/docs/tech.v3.libs.smile.data.html b/docs/tech.v3.libs.smile.data.html
diff --git a/docs/tech.v3.libs.tribuo.html b/docs/tech.v3.libs.tribuo.html
diff --git a/src/tech/v3/dataset/impl/column_data_process.clj b/src/tech/v3/dataset/impl/column_data_process.clj
@@ -114,11 +114,11 @@
          #:tech.v3.dataset{:force-datatype? true})))
     (ds-proto/is-column? obj-data)
     (let [cm (meta obj-data)]
-      #:tech.v3.datatype{:name (:name cm)
-                         :missing (ds-proto/missing obj-data)
-                         :force-datatype? true
-                         :data (ds-proto/column-buffer obj-data)
-                         :metadata cm})
+      #:tech.v3.dataset{:name (:name cm)
+                        :missing (ds-proto/missing obj-data)
+                        :force-datatype? true
+                        :data (ds-proto/column-buffer obj-data)
+                        :metadata cm})
     :else
     (scan-data obj-data nil)))
 

diff --git a/src/tech/v3/dataset/io/column_parsers.clj b/src/tech/v3/dataset/io/column_parsers.clj
@@ -478,6 +478,7 @@
                                   ^{:unsynchronized-mutable true} missing-value
                                   ^RoaringBitmap missing
                                   column-name
+                                  ^:unsynchronized-mutable ^long last-idx
                                   ^:unsynchronized-mutable ^long max-idx
                                   options]
   dtype-proto/PECount
@@ -492,44 +493,42 @@
         (.get container idx))))
   PParser
   (addValue [_p idx value]
-    (set! max-idx (max idx max-idx))
+    (set! max-idx idx)
     (when-not (missing-value? value)
-      (let [org-datatype (dtype/datatype value)
-            ;;Avoid the pack call if possible
-            packed-dtype (if (identical? container-dtype org-datatype)
-                           org-datatype
-                           (packing/pack-datatype org-datatype))
-            container-ecount (- (.size container) (.getCardinality missing))]
-        (if (or (== 0 container-ecount)
-                (identical? container-dtype packed-dtype))
-          (do
-            (when (== 0 container-ecount)
-              (set! container (column-base/make-container packed-dtype options))
-              (set! container-dtype packed-dtype)
-              (set! missing-value (column-base/datatype->missing-value packed-dtype)))
-            (when-not (== container-ecount idx)
-              (add-missing-values! container missing missing-value idx))
-            (.add container value))
-          ;;boolean present a problem here.  We generally want to keep them as booleans
-          ;;and not promote them to full numbers.
-          (let [widest-datatype (if (identical? org-datatype :boolean)
-                                  (if (identical? container-dtype :boolean)
-                                    :boolean
-                                    :object)
-                                  (casting/widest-datatype
-                                   (packing/unpack-datatype container-dtype)
-                                   org-datatype))]
-            (when-not (= widest-datatype container-dtype)
-              (let [new-container (promote-container container
-                                                     missing widest-datatype
-                                                     options)]
-                (set! container new-container)
-                (set! container-dtype widest-datatype)
-                (set! missing-value (column-base/datatype->missing-value
-                                     widest-datatype))))
-            (when-not (== container-ecount idx)
-              (add-missing-values! container missing missing-value idx))
-            (.add container value))))))
+      (let [val-dtype (fast-dtype value)]
+        ;;setup container for new data
+        (when-not (identical? container-dtype val-dtype)
+          (let [;;Avoid the pack call if possible
+                packed-dtype (packing/pack-datatype val-dtype)
+                container-ecount (.size container)
+                logical-ecount (- container-ecount (.getCardinality missing))]
+            ;;Setup container
+            (if (== 0 logical-ecount)
+              (do
+                (set! container (column-base/make-container packed-dtype options))
+                (set! container-dtype val-dtype)
+                (set! missing-value (column-base/datatype->missing-value packed-dtype)))
+              ;;boolean present a problem here.  We generally want to keep them as booleans
+              ;;and not promote them to full numbers.
+              (let [widest-datatype (if (identical? val-dtype :boolean)
+                                      (if (identical? container-dtype :boolean)
+                                        :boolean
+                                        :object)
+                                      (casting/widest-datatype
+                                       (packing/unpack-datatype container-dtype)
+                                       val-dtype))]
+                (when-not (= widest-datatype container-dtype)
+                  (let [new-container (promote-container container
+                                                         missing widest-datatype
+                                                         options)]
+                    (set! container new-container)
+                    (set! container-dtype widest-datatype)
+                    (set! missing-value (column-base/datatype->missing-value
+                                         widest-datatype))))))))
+        (when (> (- idx last-idx) 1)
+          (add-missing-values! container missing missing-value idx))
+        (set! last-idx idx)
+        (.add container value))))
   (finalize [_p rowcount]
     (finalize-parser-data! container missing nil nil
                            missing-value rowcount)))
@@ -543,4 +542,5 @@
                             (bitmap/->bitmap)
                             column-name
                             -1
+                            -1
                             options))
diff --git a/src/tech/v3/dataset/string_table.clj b/src/tech/v3/dataset/string_table.clj
@@ -62,7 +62,7 @@
   (add [this value]
     (errors/when-not-errorf
      (or (nil? value) (instance? String value))
-     "Value added to string table is not a string: %s" value)
+     "Value added to string table is not a string: %s" (type value))
     (let [value (or value "")
           item-idx (int (.computeIfAbsent
                          str->int