Native: a reply is read with no peek at its tail marker, and the unused arcadedb_command entry point is removed (refs #388)

native/src/main/java/org/humemai/nativelib/ArcadeLib.java

@@ -10,10 +10,6 @@
 import com.arcadedb.ContextConfiguration;
 import com.arcadedb.database.Database;
 import com.arcadedb.database.DatabaseFactory;
-import com.arcadedb.query.sql.executor.Result;
-import com.arcadedb.query.sql.executor.ResultSet;
-import com.arcadedb.serializer.json.JSONArray;
-import com.arcadedb.serializer.json.JSONObject;
 import com.arcadedb.server.ArcadeDBServer;
 import org.graalvm.nativeimage.IsolateThread;
 import org.graalvm.nativeimage.UnmanagedMemory;
@@ -58,29 +54,6 @@ public static CCharPointer lastError(final IsolateThread thread) {
     return toC(lastError);
   }
 
-  /** Runs one command (any language: sql, sqlscript, opencypher...) and returns its rows as JSON. */
-  @CEntryPoint(name = "arcadedb_command")
-  public static CCharPointer command(final IsolateThread thread, final long handle, final CCharPointer language,
-      final CCharPointer text, final CCharPointer paramsJson) {
-    try {
-      final Database db = DATABASES.get(handle);
-      if (db == null)
-        throw new IllegalArgumentException("unknown handle " + handle);
-      final String paramsText = CTypeConversion.toJavaString(paramsJson);
-      final Map<String, Object> params = paramsText == null || paramsText.isEmpty() ? Map.of() : new JSONObject(paramsText).toMap();
-      final JSONArray rows = new JSONArray();
-      try (final ResultSet rs = db.command(CTypeConversion.toJavaString(language), CTypeConversion.toJavaString(text), params)) {
-        while (rs.hasNext()) {
-          final Result r = rs.next();
-          rows.put(r.toJSON());
-        }
-      }
-      return toC(new JSONObject().put("ok", true).put("result", rows).toString());
-    } catch (final Throwable t) {
-      return toC(new JSONObject().put("ok", false).put("error", describe(t)).toString());
-    }
-  }
-
   @CEntryPoint(name = "arcadedb_close")
   public static int close(final IsolateThread thread, final long handle) {
     try {

native/src/main/java/org/humemai/nativelib/ArcadeNative.java

@@ -1,9 +1,10 @@
 /*
  * SPDX-License-Identifier: Apache-2.0
  *
- * Phase 2: the C entry points behind the Python native backend (arcadedb/_native). Every function returns a
+ * Phase 2: the C entry points behind the Python native backend (arcadedb/_native). Most functions return a
  * malloc'd UTF-8 JSON text that the caller releases with arcadedb_free: {"ok":true,...} or
  * {"ok":false,"error":msg,"cls":java class,"chain":[[class,msg],..],"retry":"need_retry"|"duplicated_key"|null,"retry_ms":n}.
+ * arcadedb_n_cmd and arcadedb_n_tx return a long (-1 for an error, whose text in that form arcadedb_n_last_error returns).
  * Databases, result sets and servers are long handles. Any thread attached to the isolate may call any function; a
  * transaction belongs to the (Java) thread that began it, as in the embedded API.
  */
@@ -184,9 +185,15 @@ public static CCharPointer info(final IsolateThread thread, final long h) {
     }
   }
 
-  /** op 0 begin, 1 commit, 2 rollback, 3 mark. Every reply carries {"active":b,"depth":n,"commits":n}. */
+  /**
+   * op 0 begin, 1 commit, 2 rollback, 3 mark. Returns the transaction state after the call as one non-negative long: bit 0 is
+   * isTransactionActive, bits 1 to 31 the nesting depth, bits 32 to 62 the low 31 bits of the commit count of the thread's
+   * transaction context (callers compare two counts for equality). -1 on error: the error text is then read with
+   * arcadedb_n_last_error on the same thread, as for arcadedb_n_cmd. No reply text, so no malloc, no arcadedb_free and no JSON
+   * decode in the caller.
+   */
   @CEntryPoint(name = "arcadedb_n_tx")
-  public static CCharPointer tx(final IsolateThread thread, final long h, final int op) {
+  public static long tx(final IsolateThread thread, final long h, final int op) {
     try {
       final Database db = db(h);
       switch (op) {
@@ -196,14 +203,15 @@ public static CCharPointer tx(final IsolateThread thread, final long h, final in
       default -> {
       }
       }
-      return out(txState(db));
+      return txState(db);
     } catch (final Throwable t) {
-      return err(t);
+      LAST_ERROR.set(errorJson(t));
+      return -1;
     }
   }
 
-  private static String txState(final Database db) {
-    int depth = 0;
+  private static long txState(final Database db) {
+    long depth = 0;
     long commits = 0;
     if (db instanceof LocalDatabase local)
       depth = local.getNestedTransactions();
@@ -212,7 +220,7 @@ private static String txState(final Database db) {
       if (ctx != null)
         commits = ctx.getCommitCount();
     }
-    return "{\"ok\":true,\"active\":" + db.isTransactionActive() + ",\"depth\":" + depth + ",\"commits\":" + commits + "}";
+    return (db.isTransactionActive() ? 1L : 0L) | ((depth & 0x7FFFFFFFL) << 1) | ((commits & 0x7FFFFFFFL) << 32);
   }
 
   // ---- statements

src/arcadedb/_native/database.py

@@ -74,12 +74,10 @@ def _call(prefix: str, name: str, *args) -> dict:
     return reply
 
 
-# Every successful ``arcadedb_n_tx`` reply is ``{"ok":true,"active":<bool>,"depth":<n>,"commits":<n>}`` (``ArcadeNative.txState``). A caller
-# that reads none of the state (begin, commit, rollback), or only ``active``, tells success and that bool from the text it already holds
-# instead of decoding a JSON object. An error reply is ``{"ok":false,...}``: it fails the prefix test and is decoded, as before.
-_TX_OK = '{"ok":true,'
-_TX_ACTIVE = '{"ok":true,"active":true,'
-_TX_IDLE = '{"ok":true,"active":false,'
+# ``arcadedb_n_tx`` answers with the transaction state after the call as one int (``ArcadeNative.tx``, no reply text to copy, free or
+# decode): bit 0 is ``active``, bits 1 to 31 the nesting ``depth``, bits 32 to 62 the ``commits`` of the transaction context (its low 31
+# bits: callers compare two of them for equality). Callers read the fields inline: ``state & 1``, ``(state >> 1) & 0x7FFFFFFF``,
+# ``state >> 32``. -1 is an error: its text, the ``{"ok":false,...}`` reply of every other entry point, is ``lib.last_error()``.
 
 
 def _first_row(database, reply: dict):
@@ -793,73 +791,63 @@ def _command_first_row(self, language: str, command: str, *args) -> NativeResult
 
     # -- transactions
 
-    def _tx_text(self, op: int, prefix: str) -> str:
-        """One ``arcadedb_n_tx`` crossing: the reply text of a success (not decoded), an ArcadeDBError for a failure."""
+    def _tx_state(self, op: int, prefix: str) -> int:
+        """One ``arcadedb_n_tx`` crossing, one foreign call: the transaction state after the operation as an int (its layout is
+        described at the top of this module), an ArcadeDBError for a failure.
+        """
         if self._closed:
             raise ArcadeDBError("Database is closed")
         if (op == 1 or op == 2) and lib.CLOSE_BATCH:
             lib.flush_closes()
-        text = lib.call("arcadedb_n_tx", self._h, op)
-        if text.startswith(_TX_OK):
-            return text
-        reply = json.loads(text)
-        if not reply.get("ok"):
-            raise _error(prefix, reply)
-        return text
-
-    def _tx(self, op: int, prefix: str) -> dict:
-        """The decoded state reply: for a caller that reads ``depth`` or ``commits``."""
-        return json.loads(self._tx_text(op, prefix))
+        state = lib.tx(self._h, op)
+        if state < 0:
+            raise _error(prefix, json.loads(lib.last_error()))
+        return state
 
     def begin(self):
-        self._tx_text(0, "Failed to begin transaction")
+        self._tx_state(0, "Failed to begin transaction")
 
-    def _begin_state(self) -> dict:
-        """begin(), and the state right after it: the reply of ``arcadedb_n_tx`` carries ``active``, ``depth`` and ``commits``."""
-        return self._tx(0, "Failed to begin transaction")
+    def _begin_state(self) -> int:
+        """begin(), and the state right after it: the int of ``arcadedb_n_tx`` carries ``active``, ``depth`` and ``commits``."""
+        return self._tx_state(0, "Failed to begin transaction")
 
     def commit(self):
-        self._tx_text(1, "Failed to commit transaction")
+        self._tx_state(1, "Failed to commit transaction")
 
     def rollback(self):
-        self._tx_text(2, "Failed to rollback transaction")
+        self._tx_state(2, "Failed to rollback transaction")
 
-    def _rollback_state(self) -> dict:
-        """rollback(), and the state right after it (the reply carries it, as begin's does): the unwind reads it instead of asking."""
-        return self._tx(2, "Failed to rollback transaction")
+    def _rollback_state(self) -> int:
+        """rollback(), and the state right after it (the int carries it, as begin's does): the unwind reads it instead of asking."""
+        return self._tx_state(2, "Failed to rollback transaction")
 
     def transaction(self) -> TransactionContext:
         return TransactionContext(self)
 
     def is_transaction_active(self) -> bool:
-        text = self._tx_text(3, "Failed to check transaction status")
-        if text.startswith(_TX_ACTIVE):
-            return True
-        if text.startswith(_TX_IDLE):
-            return False
-        return json.loads(text)["active"]
+        return bool(self._tx_state(3, "Failed to check transaction status") & 1)
 
     def _transaction_stack_mark(self):
-        state = self._tx(3, "Failed to read the transaction state")
-        return state["active"], state["depth"]
+        state = self._tx_state(3, "Failed to read the transaction state")
+        return bool(state & 1), (state >> 1) & 0x7FFFFFFF
 
     def _transaction_commit_mark(self):
         """The commit count of the current transaction (the witness that a BATCH boundary published part of its work)."""
-        return self._tx(3, "Failed to read the transaction state")["commits"]
+        return self._tx_state(3, "Failed to read the transaction state") >> 32
 
     def _unwind_opened_since(self, mark):
         """Roll back every level opened since ``mark``, and no other; returns ``(state, first_error)``.
 
-        ``state`` is the transaction state after the last crossing (every rollback reply carries it, so no read sits between two
+        ``state`` is the transaction state int after the last crossing (every rollback carries it, so no read sits between two
         steps), or None when that crossing failed. A level the caller held at the mark is theirs, as in the JPype ``Database``.
         """
         was_active, depth_before = mark
         first_error = None
-        state = self._tx(3, "Failed to read the transaction state")
-        for _ in range(state["depth"] - depth_before):
+        state = self._tx_state(3, "Failed to read the transaction state")
+        for _ in range(((state >> 1) & 0x7FFFFFFF) - depth_before):
             if state is None:
-                state = self._tx(3, "Failed to check transaction status")
-            if not state["active"]:
+                state = self._tx_state(3, "Failed to check transaction status")
+            if not state & 1:
                 break
             try:
                 state = self._rollback_state()
@@ -868,8 +856,8 @@ def _unwind_opened_since(self, mark):
                 first_error = first_error or e
         if not was_active:
             if state is None:
-                state = self._tx(3, "Failed to check transaction status")
-            if state["active"]:
+                state = self._tx_state(3, "Failed to check transaction status")
+            if state & 1:
                 try:
                     state = self._rollback_state()
                 except Exception as e:
@@ -893,7 +881,7 @@ def run_in_transaction(self, fn, retries: int = 12, backoff_s: float = 0.005):
             mark = self._transaction_stack_mark()
             commits_at_start = None
             try:
-                commits_at_start = self._begin_state()["commits"]
+                commits_at_start = self._begin_state() >> 32
                 result = fn()
                 self.commit()
                 return result
@@ -919,7 +907,7 @@ def run_in_transaction(self, fn, retries: int = 12, backoff_s: float = 0.005):
                     # retry, as before); the raise sits outside that try, or its own except would swallow it.
                     try:
                         commits_now = (
-                            state["commits"]
+                            state >> 32
                             if state is not None
                             else self._transaction_commit_mark()
                         )

src/arcadedb/_native/lib.py

@@ -259,6 +259,9 @@ class _Params(ctypes.Structure):
             "arcadedb_n_set_upcall": [P, P],
             "arcadedb_free": [P, P],
         }
+        # the entry points that answer with a number, not with the address of a reply text: a cursor id (arcadedb_n_cmd) and the
+        # transaction state (arcadedb_n_tx); -1 is an error for both, its text is read with arcadedb_n_last_error
+        returns_long = ("arcadedb_n_cmd", "arcadedb_n_tx")
         for name, argtypes in sigs.items():
             fn = getattr(lib, name, None)
             if fn is None:  # a library built before this entry point existed
@@ -272,7 +275,7 @@ class _Params(ctypes.Structure):
                     "arcadedb_n_rs_close_many",
                     "arcadedb_n_set_upcall",
                 )
-                else ctypes.c_longlong if name == "arcadedb_n_cmd" else ctypes.c_void_p
+                else ctypes.c_longlong if name in returns_long else ctypes.c_void_p
             )
         global BINARY_PARAMS, CMD_PATH, CLOSE_BATCH, INSERT_ROWS, EXEC_MANY, NEXT_BIN, GB_BIN, TS_BIN
         if not hasattr(
@@ -341,38 +344,36 @@ def _thread_handle():
         ctypes.cdll.msvcrt if sys.platform == "win32" else ctypes.CDLL(None),
     )
 )
-_TAIL_KEY = b',"bin":'
+_TAIL_KEY = ',"bin":'
 _TAIL_WINDOW = 40
 
 
 def _take(lib, th, ptr, tail: bool):
     """Read the reply at ``ptr`` as ``(text, floats)`` and free it: ``floats`` is None unless ``tail`` and the reply ends with
     ``,"bin":N}`` (then N float32 follow the text's NUL, 4-byte aligned, and the text is returned without the ``,"bin":N`` part).
+
+    The marker is looked for in the decoded text, in its last ``_TAIL_WINDOW`` characters. A reply with no marker, nearly every
+    reply, costs the foreign calls of the read itself (``strlen``, the decode and the free) and no peek at its last bytes.
     """
     if not ptr:
         raise ArcadeDBError("The native library returned no reply")
     try:
         end = _strlen(ptr)
-        floats = None
-        if tail:
-            # only the last bytes can hold the marker: peek at them (at most 40 bytes), never at the whole reply
-            start = end - _TAIL_WINDOW if end > _TAIL_WINDOW else 0
-            peek = ctypes.string_at(ptr + start, end - start)
-            if peek.endswith(b"}"):
-                i = peek.rfind(_TAIL_KEY)
-                if i >= 0:
-                    n = int(peek[i + 7 : -1])
-                    import numpy as np
-
-                    off = (end + 1 + 3) & ~3
-                    floats = np.frombuffer(
-                        ctypes.string_at(ptr + off, 4 * n), dtype=np.float32
-                    )
-                    # the text ends at the marker, closed by the "}" that the JSON object needs: written over the marker's comma
-                    # in the reply's own memory (it is freed below), so the decode reads the JSON as it stands
-                    ctypes.c_char.from_address(ptr + start + i).value = b"}"
-                    end = start + i + 1
-        return _str_from_utf8(ptr, end), floats
+        text = _str_from_utf8(ptr, end)
+        if not tail or not text.endswith("}"):
+            return text, None
+        i = text.rfind(_TAIL_KEY, max(0, len(text) - _TAIL_WINDOW))
+        if i < 0:
+            return text, None
+        n = int(text[i + 7 : -1])
+        import numpy as np
+
+        off = (end + 1 + 3) & ~3
+        floats = np.frombuffer(ctypes.string_at(ptr + off, 4 * n), dtype=np.float32)
+        # the text ends at the marker, closed by the "}" that the JSON object needs
+        text = text[:i]
+        text += "}"
+        return text, floats
     finally:
         lib.arcadedb_free(th, ptr)
 
@@ -488,7 +489,19 @@ def cmd(handle, flags, language, command, params, buf, count, first) -> int:
     )
 
 
+def tx(handle, op) -> int:
+    """``arcadedb_n_tx``: op 0 begin, 1 commit, 2 rollback, 3 mark. Returns the transaction state after the call as one int (bit 0
+    ``active``, bits 1 to 31 ``depth``, bits 32 to 62 ``commits``, as ``ArcadeNative.tx`` writes it), or -1 for an error (read it
+    with :func:`last_error`). One foreign call: no reply text to copy, free or decode.
+    """
+    lib = _lib or _load()
+    t = _local.__dict__.get("thread")
+    th = t.handle if t is not None else _thread_handle()
+    return lib.arcadedb_n_tx(th, handle, op)
+
+
 def last_error() -> str:
+    """The error of the last failed :func:`cmd` or :func:`tx` on this thread, as the JSON text of an error reply."""
     return call("arcadedb_n_last_error")
 
 

src/arcadedb/_native/records.py

@@ -359,8 +359,11 @@ def _save_update(self, db) -> None:
 
         Like every write (and like the JVM build), it needs an active transaction while auto-transaction is off, the engine default:
         outside one it raises ArcadeDBError ("Transaction not begun") before it reads or writes anything. This method opens no
-        transaction of its own."""
-        if not db.is_transaction_active() and not db._auto_transaction:
+        transaction of its own. With auto-transaction on the check cannot refuse, so the state is not asked for (one crossing less).
+        The engine's own refusal cannot stand in for the check with it off: the engine runs the before-update callbacks and the
+        validation before it looks for a transaction, and an UPDATE of a record that is gone raises nothing.
+        """
+        if not db._auto_transaction and not db.is_transaction_active():
             raise ArcadeDBError(
                 "Failed to save record: com.arcadedb.exception.TransactionException: Transaction not begun "
                 "(wrap the write in `with db.transaction():` or call begin() first)"

src/arcadedb/core.py

@@ -241,7 +241,8 @@ def _column_to_java(name, values):
     into a ``long[]``, ``double[]``, or ``boolean[]``; a string or object array
     and any other sequence of mixed values convert per element into an ``Object[]`` (a ``str``
     reuses one Java String per distinct value, ``None`` is a null); a column of nothing but ``str`` and ``None`` crosses instead as one
-    ``StringColumn`` (``_string_column``). A pandas
+    ``StringColumn`` (``_string_column``). A bridge ``TemporalColumn``, which ``insert_arrow`` and ``insert_dataframe`` build from a
+    timestamp or date column (``dataframe._temporal_column``), crosses as it is. A pandas
     ``Series`` is read through ``to_numpy``, with a nullable dtype's ``<NA>``
     as ``None``. A 2-D ``float32`` array is a column of vectors: it crosses as
     one ``float[]`` and each document gets one row of it. Kinds that do not cross natively raise ``TypeError`` rather
@@ -313,6 +314,8 @@ def _column_to_java(name, values):
         # a string, bytes, or object array: its elements are Python objects (str, None, ...), converted one by one below
         values = list(values.astype(object, copy=False))
     elif not isinstance(values, (list, tuple)):
+        if isinstance(values, _java_class("com.arcadedb.python.TemporalColumn")):
+            return values, int(values.rows())
         values = list(values)
     column = _string_column(values)
     if column is not None: