Column frames: rid and list-of-number columns travel in binary kinds, not JSON

native/src/main/java/org/humemai/nativelib/ArcadeNative.java

@@ -1,9 +1,10 @@
 /*
  * SPDX-License-Identifier: Apache-2.0
  *
- * Phase 2: the C entry points behind the Python native backend (arcadedb/_native). Every function returns a
+ * Phase 2: the C entry points behind the Python native backend (arcadedb/_native). Most functions return a
  * malloc'd UTF-8 JSON text that the caller releases with arcadedb_free: {"ok":true,...} or
  * {"ok":false,"error":msg,"cls":java class,"chain":[[class,msg],..],"retry":"need_retry"|"duplicated_key"|null,"retry_ms":n}.
+ * arcadedb_n_cmd and arcadedb_n_tx return a long (-1 for an error, whose text in that form arcadedb_n_last_error returns).
  * Databases, result sets and servers are long handles. Any thread attached to the isolate may call any function; a
  * transaction belongs to the (Java) thread that began it, as in the embedded API.
  */
@@ -184,9 +185,15 @@ public static CCharPointer info(final IsolateThread thread, final long h) {
     }
   }
 
-  /** op 0 begin, 1 commit, 2 rollback, 3 mark. Every reply carries {"active":b,"depth":n,"commits":n}. */
+  /**
+   * op 0 begin, 1 commit, 2 rollback, 3 mark. Returns the transaction state after the call as one non-negative long: bit 0 is
+   * isTransactionActive, bits 1 to 31 the nesting depth, bits 32 to 62 the low 31 bits of the commit count of the thread's
+   * transaction context (callers compare two counts for equality). -1 on error: the error text is then read with
+   * arcadedb_n_last_error on the same thread, as for arcadedb_n_cmd. No reply text, so no malloc, no arcadedb_free and no JSON
+   * decode in the caller.
+   */
   @CEntryPoint(name = "arcadedb_n_tx")
-  public static CCharPointer tx(final IsolateThread thread, final long h, final int op) {
+  public static long tx(final IsolateThread thread, final long h, final int op) {
     try {
       final Database db = db(h);
       switch (op) {
@@ -196,14 +203,15 @@ public static CCharPointer tx(final IsolateThread thread, final long h, final in
       default -> {
       }
       }
-      return out(txState(db));
+      return txState(db);
     } catch (final Throwable t) {
-      return err(t);
+      LAST_ERROR.set(errorJson(t));
+      return -1;
     }
   }
 
-  private static String txState(final Database db) {
-    int depth = 0;
+  private static long txState(final Database db) {
+    long depth = 0;
     long commits = 0;
     if (db instanceof LocalDatabase local)
       depth = local.getNestedTransactions();
@@ -212,7 +220,7 @@ private static String txState(final Database db) {
       if (ctx != null)
         commits = ctx.getCommitCount();
     }
-    return "{\"ok\":true,\"active\":" + db.isTransactionActive() + ",\"depth\":" + depth + ",\"commits\":" + commits + "}";
+    return (db.isTransactionActive() ? 1L : 0L) | ((depth & 0x7FFFFFFFL) << 1) | ((commits & 0x7FFFFFFFL) << 32);
   }
 
   // ---- statements

native/src/main/java/org/humemai/nativelib/ColumnFrames.java

@@ -13,15 +13,22 @@
  *   b1   one byte per row                           dt   int64 epoch milliseconds, dtu: epoch microseconds (see micros)
  *   f4v  dim float32 LE per row (float[])           f8v  dim float64 LE per row (double[])
  *   str  int32 LE offsets (n + 1) then UTF-8 bytes  dec  a BigDecimal as text in the str layout
+ *   rid  two int64 LE per row: the bucket and the position of a record link (the text "#bucket:position")
+ *   li8  a list of int64 per row (Long, Integer, Short, Byte or null elements) and lf8 a list of float64 (Double, Float or null
+ *        elements; NaN and the infinities are values): int32 offsets (n + 1), a bitmap of the null elements (one bit per element
+ *        of the flat array), then the elements of every list one after the other. A null row is an empty list under the row's
+ *        null bit; an empty list and a null element are what they were in the json form
  *   json one JSON array of the column in the tagged encoding of NativeCodec: what no fixed layout carries without changing a value
- *        (nested lists and maps, records and RIDs, BigInteger, a column that mixes kinds, a DATE or a time-zone-aware temporal next to
- *        datetimes with micros, a ragged vector, a Long beyond 2**53 next to a floating value)
+ *        (maps, nested lists, lists of strings or of RIDs, lists that mix ints and floats, a list of nothing but nulls, embedded
+ *        records, other record values, a RID that is not valid, BigInteger, a column that mixes kinds, a DATE or a time-zone-aware
+ *        temporal next to datetimes with micros, a ragged vector, a Long beyond 2**53 next to a floating value)
  *
  * A frame holds at most {@code max} rows and stops once the rows read add up to about FRAME_BYTES, so the working set of a read is a frame
  * and not the table.
  */
 package org.humemai.nativelib;
 
+import com.arcadedb.database.RID;
 import com.arcadedb.query.sql.executor.Result;
 import com.arcadedb.query.sql.executor.ResultSet;
 
@@ -96,6 +103,55 @@ private static boolean yearFits(final LocalDateTime ldt) {
     return ldt.getYear() >= 1 && ldt.getYear() <= 9999;
   }
 
+  /**
+   * The column kind of one list: "li8" (every element a Long, Integer, Short or Byte, or null), "lf8" (every element a Double or Float, or
+   * null), "l0" (no element that is not null: it takes the kind of the other lists of the column) or "json" (a string, a boolean, a
+   * nested value, ints next to floats: what no fixed layout carries without changing a value).
+   */
+  private static String listKind(final List<?> list) {
+    String kind = "l0";
+    for (final Object e : list) {
+      if (e == null)
+        continue;
+      final String k;
+      if (e instanceof Long || e instanceof Integer || e instanceof Short || e instanceof Byte)
+        k = "li8";
+      else if (e instanceof Double || e instanceof Float)
+        k = "lf8";
+      else
+        return "json";
+      if (kind.equals("l0"))
+        kind = k;
+      else if (!kind.equals(k))
+        return "json";
+    }
+    return kind;
+  }
+
+  private static boolean isListKind(final String kind) {
+    return kind.equals("li8") || kind.equals("lf8") || kind.equals("l0");
+  }
+
+  /** The kind of a column that holds lists of kind {@code a} and of kind {@code b}, or null when no list layout carries both. */
+  private static String mergeListKinds(final String a, final String b) {
+    if (!isListKind(a) || !isListKind(b))
+      return null;
+    if (a.equals(b) || b.equals("l0"))
+      return a;
+    if (a.equals("l0"))
+      return b;
+    return null; // ints next to floats: each element keeps its own type in a json column
+  }
+
+  /** The elements of the lists of a column of list kind (the size of the flat buffer), or -1 when the buffer would not fit an int32 offset. */
+  private static long listElements(final List<Object[]> rows, final int c, final int count) {
+    long total = 0;
+    for (int r = 0; r < count; r++)
+      if (rows.get(r)[c] instanceof List<?> l)
+        total += l.size();
+    return total <= (1 << 27) ? total : -1;
+  }
+
   /** About how many bytes the values of a row take in a frame (a stop rule, not an exact size). */
   private static long estimate(final Result row) {
     long n = 0;
@@ -218,12 +274,20 @@ else if (v instanceof float[])
           t = "f4v";
         else if (v instanceof double[])
           t = "f8v";
+        else if (v instanceof RID rid)
+          // a record link: bucket and position, which write the text of the RID ("#b:p") for a valid RID
+          t = rid.getBucketId() >= 0 && rid.getPosition() >= 0 ? "rid" : "json";
+        else if (v instanceof List<?> list)
+          t = listKind(list);
         else
           t = "json";
         if (type == null)
           type = t;
         else if (!type.equals(t)) {
-          if ((type.equals("i8") && t.equals("f8")) || (type.equals("f8") && t.equals("i8")))
+          final String merged = mergeListKinds(type, t);
+          if (merged != null)
+            type = merged;
+          else if ((type.equals("i8") && t.equals("f8")) || (type.equals("f8") && t.equals("i8")))
             type = "f8";
           else {
             type = "json";
@@ -233,6 +297,8 @@ else if (!type.equals(t)) {
       }
       if (type == null)
         type = "str"; // an all-null column
+      if (type.equals("l0"))
+        type = "json"; // lists with no element that is not null say nothing about the element type: Arrow types them list<null>
       // an integer beyond 2**53 next to a floating value has no exact double: the values stay what they are, in a json column
       if (type.equals("f8") && sawInt && sawFloating && bigInt)
         type = "json";
@@ -255,6 +321,13 @@ else if (vdim == -1)
           vdim = 0;
       }
 
+      long listTotal = 0;
+      if (type.equals("li8") || type.equals("lf8")) {
+        listTotal = listElements(rows, c, count);
+        if (listTotal < 0)
+          type = "json";
+      }
+
       final byte[] colBuf;
       if (type.equals("i8") || type.equals("dt") || type.equals("dtu")) {
         final ByteBuffer bb = ByteBuffer.allocate(count * 8).order(ByteOrder.LITTLE_ENDIAN);
@@ -319,6 +392,55 @@ else if (type.equals("dtu"))
               bb.putDouble(x);
         }
         colBuf = bb.array();
+      } else if (type.equals("rid")) {
+        // bucket and position as two int64 per row; a null row is 0, 0
+        final ByteBuffer bb = ByteBuffer.allocate(count * 16).order(ByteOrder.LITTLE_ENDIAN);
+        for (int r = 0; r < count; r++) {
+          final Object v = rows.get(r)[c];
+          if (v == null) {
+            nulls[r >> 3] |= (1 << (r & 7));
+            bb.putLong(0);
+            bb.putLong(0);
+          } else {
+            bb.putLong(((RID) v).getBucketId());
+            bb.putLong(((RID) v).getPosition());
+          }
+        }
+        colBuf = bb.array();
+      } else if (type.equals("li8") || type.equals("lf8")) {
+        // int32 offsets (count + 1), a bitmap of the null elements (bit j of byte j/8 set for a null element, one bit per element of
+        // the flat array), then the elements of every list one after the other, int64 or float64 LE. A null row is an empty list.
+        final boolean ints = type.equals("li8");
+        final ByteBuffer offs = ByteBuffer.allocate((count + 1) * 4).order(ByteOrder.LITTLE_ENDIAN);
+        final ByteBuffer vals = ByteBuffer.allocate((int) listTotal * 8).order(ByteOrder.LITTLE_ENDIAN);
+        final byte[] elemNulls = new byte[(int) ((listTotal + 7) / 8)];
+        int pos = 0;
+        offs.putInt(0);
+        for (int r = 0; r < count; r++) {
+          final Object v = rows.get(r)[c];
+          if (v == null)
+            nulls[r >> 3] |= (1 << (r & 7));
+          else
+            for (final Object e : (List<?>) v) {
+              if (e == null) {
+                elemNulls[pos >> 3] |= (1 << (pos & 7));
+                if (ints)
+                  vals.putLong(0);
+                else
+                  vals.putDouble(Double.NaN);
+              } else if (ints)
+                vals.putLong(((Number) e).longValue());
+              else
+                vals.putDouble(((Number) e).doubleValue());
+              pos++;
+            }
+          offs.putInt(pos);
+        }
+        final ByteBuffer bb = ByteBuffer.allocate(offs.capacity() + elemNulls.length + vals.capacity());
+        bb.put(offs.array());
+        bb.put(elemNulls);
+        bb.put(vals.array());
+        colBuf = bb.array();
       } else if (type.equals("json")) {
         final StringBuilder arr = new StringBuilder();
         arr.append('[');

src/arcadedb/_columnar.py

@@ -5,8 +5,11 @@
 vector, ``dim``), then for each column its null bitmap (bit i of byte i/8 set means row i is null) and its buffer. A column type is
 ``i8`` (int64), ``f8`` (float64), ``b1`` (one byte per bool), ``dt`` and ``dtu`` (int64 epoch milliseconds and microseconds),
 ``f4v`` and ``f8v`` (``dim`` floats per row), ``str`` (int32 offsets, count + 1 of them, then the UTF-8 bytes: Arrow's own string
-layout), ``dec`` (a BigDecimal as text in the string layout) and ``json`` (a JSON array of the column, for values no fixed layout
-carries: nested lists and maps, RIDs, a DATE next to datetimes, a mixed column).
+layout), ``dec`` (a BigDecimal as text in the string layout), ``rid`` (two int64 per row: the bucket and the position of a record
+link), ``li8`` and ``lf8`` (a list of int64 or of float64 per row: int32 offsets, count + 1 of them, a bitmap of the null elements, one
+bit per element of the flat array, then the elements of every list one after the other) and ``json`` (a JSON array of the column, for
+values no fixed layout carries: maps, nested lists, lists of anything else, embedded records, a DATE next to datetimes, a mixed
+column). A writer that does not know ``rid``, ``li8`` and ``lf8`` writes ``json`` for those columns, and the decoder reads both.
 
 The interface and the format are shared, the plumbing is per wheel: what differs is how the bytes get here (JPype or ctypes) and the
 text of a ``json`` column (the JVM wheel's tagged JSON with a side array of Java objects, the native library's tagged JSON), so the
@@ -66,6 +69,34 @@ def _decode_decimals(data, count: int, mask) -> list:
     ]
 
 
+def rid_texts(pairs) -> list:
+    """The RID texts (``"#bucket:position"``) of the ``(count, 2)`` int64 array of a ``rid`` column."""
+    return [f"#{b}:{p}" for b, p in pairs.tolist()]
+
+
+def _decode_list_column(data, count: int, ctype: str):
+    """``(offsets, flat, null_elements)`` of a ``li8`` or ``lf8`` column buffer: int32 offsets (count + 1), the bitmap of the null
+    elements, then the flat int64 or float64 array of every list's elements. ``null_elements`` is a bool array (True = null) or None
+    when no element is null."""
+    import numpy as np
+
+    offs = np.frombuffer(data[: (count + 1) * 4], dtype="<i4")
+    total = int(offs[count])
+    start = (count + 1) * 4
+    bitmap = (total + 7) // 8
+    bits = np.frombuffer(data[start : start + bitmap], dtype=np.uint8)
+    emask = (
+        np.unpackbits(bits, bitorder="little")[:total].astype(bool)
+        if bits.any()
+        else None
+    )
+    start += bitmap
+    flat = np.frombuffer(
+        data[start : start + total * 8], dtype="<i8" if ctype == "li8" else "<f8"
+    )
+    return offs, flat, emask
+
+
 def _missing_column_part(count: int, template):
     """A batch of ``count`` rows that lack a column some other batch has: nulls in
     the shape ``to_columns`` uses for that column (NaN for numbers, NaT for
@@ -235,14 +266,18 @@ def read_header(buf) -> Tuple[Dict[str, Any], int]:
 
 
 def decode_columns_frame(
-    buf, side, json_values: Callable, micros: bool = False
+    buf, side, json_values: Callable, micros: bool = False, rid_values: Callable = None
 ) -> Tuple[int, Dict[str, Any], Dict[str, Any]]:
     """``(count, parts, header)`` of one frame as ``to_columns`` columns: ``parts`` maps a column name to a numpy array
     (int64, float64, bool, datetime64) or a Python list (text, JSON-typed values, a bool column with nulls); a count of 0 has
     no parts. A vector column is a (count, dim) matrix, a null row of it NaN.
 
     ``micros`` is the ``to_dataframe`` form: a naive datetime column is ``datetime64[us]`` (``dtu``), and a vector column is a
     list with one numpy array per cell and None for a null (a NaN row would read as a vector of NaN, not as a missing value).
+
+    A ``rid`` column is a list of what ``rid_values(pairs, side, col)`` makes of the ``(count, 2)`` int64 array of its cells (the
+    wheel's own RID object; the null rows are set to None here), or of the RID texts without it. A ``li8`` or ``lf8`` column is a list
+    of Python lists (ints or floats, None for a null element), None for a null row.
     """
     import numpy as np
 
@@ -301,6 +336,27 @@ def decode_columns_frame(
                 values = [
                     None if has_nulls and mask[i] else values[i] for i in range(count)
                 ]
+        elif ctype == "rid":
+            pairs = np.frombuffer(data, dtype="<i8").reshape(count, 2)
+            values = (
+                rid_texts(pairs)
+                if rid_values is None
+                else list(rid_values(pairs, side, col))
+            )
+            if has_nulls:
+                for i in np.flatnonzero(mask).tolist():
+                    values[i] = None
+        elif ctype in ("li8", "lf8"):
+            offs, flat, emask = _decode_list_column(data, count, ctype)
+            items = flat.tolist()
+            if emask is not None:
+                for j in np.flatnonzero(emask).tolist():
+                    items[j] = None
+            bounds = offs.tolist()
+            values = [
+                None if has_nulls and mask[i] else items[bounds[i] : bounds[i + 1]]
+                for i in range(count)
+            ]
         elif ctype == "json":
             values = json_values(data, side, count)
         elif ctype == "dec":
@@ -463,6 +519,26 @@ def decode_arrow_frame(
                 arr = pa.FixedSizeListArray.from_arrays(child, dim)
                 if mask is not None:
                     arr = pa.array(arr.to_pylist(), type=arr.type, mask=mask)
+        elif ctype == "rid":
+            # text, as the column of RID objects of the json form degrades to (pyarrow has no type for a RID object)
+            texts = rid_texts(np.frombuffer(data, dtype="<i8").reshape(count, 2))
+            if has_nulls:
+                for i in np.flatnonzero(mask).tolist():
+                    texts[i] = None
+            arr = pa.array(texts, type=pa.string())
+        elif ctype in ("li8", "lf8"):
+            # a list<int64> or list<double> over the frame's own offsets and flat array, with a validity for null rows and elements
+            offs, flat, emask = _decode_list_column(data, count, ctype)
+            child = pa.array(flat, mask=emask)
+            validity = (
+                None if not has_nulls else pa.py_buffer(np.invert(null_bits).tobytes())
+            )
+            arr = pa.ListArray.from_buffers(
+                pa.list_(child.type),
+                count,
+                [validity, pa.py_buffer(offs.tobytes())],
+                children=[child],
+            )
         elif ctype == "json":
             values = json_values(data, side, count)
             try: