attempt 3

BUILD.md

@@ -0,0 +1,114 @@
+# Building qroissant wheels
+
+This project ships as a [maturin](https://www.maturin.rs/) / PyO3 mixed
+Rust+Python package. The native extension uses `#![feature(portable_simd)]` so
+**nightly Rust is required**, pinned via `rust-toolchain.toml`.
+
+All builds run inside a Docker image (`scripts/Dockerfile.build`) so the host
+needs only Docker and Python 3 — no Rust, no MSVC SDK, no mingw to install.
+
+## Prerequisites
+
+- Docker (tested with 29.x)
+- Python 3.11+ on the host, **only** if you want to install/test the produced
+  Linux wheel locally via `scripts/build.sh check`.
+
+## One-time setup
+
+```sh
+scripts/build.sh image
+```
+
+Builds `qroissant-build:latest`. Pulls `ghcr.io/rust-cross/cargo-xwin`,
+installs the nightly toolchain + `x86_64-pc-windows-msvc` target + maturin.
+~5 min on a cold cache, then it's a no-op.
+
+## Build commands
+
+```sh
+scripts/build.sh linux      # -> dist-linux/qroissant-*-manylinux_*_x86_64.whl
+scripts/build.sh windows    # -> dist-windows/qroissant-*-win_amd64.whl
+scripts/build.sh all        # image + linux + windows
+scripts/build.sh check      # install latest linux wheel into .venv, import-smoke,
+                            # then `zipfile -l` the windows wheel
+scripts/build.sh clean      # rm dist-linux dist-windows
+scripts/build.sh clean-cache  # also drop the cargo/target/xwin Docker volumes
+```
+
+The wheels target **abi3-py311**, so a single artifact per platform covers
+CPython 3.11, 3.12, 3.13, and onward.
+
+## What lives where
+
+| Artifact | Path |
+| --- | --- |
+| Build image definition | `scripts/Dockerfile.build` |
+| Wrapper script | `scripts/build.sh` |
+| Toolchain pin | `rust-toolchain.toml` |
+| Linux wheels | `dist-linux/` (gitignored) |
+| Windows wheels | `dist-windows/` (gitignored) |
+| Cargo registry cache | Docker volume `qroissant-cargo-registry` |
+| Cargo target dir | Docker volume `qroissant-target` |
+| cargo-xwin MSVC cache | Docker volume `qroissant-xwin` |
+
+The three Docker volumes persist between runs so reruns only rebuild changed
+crates; nuke them with `scripts/build.sh clean-cache` if anything looks stale.
+
+## Debug symbols on Windows
+
+The release profile sets `debug = "line-tables-only"`, so a `_native.pdb` is
+produced alongside the wheel. `scripts/build.sh windows` copies it into
+`dist-windows/`. To get resolved frames in a Rust panic backtrace on a Windows
+machine:
+
+1. `pip install qroissant-*-win_amd64.whl`
+2. Locate the installed extension, e.g.
+   `Lib\site-packages\qroissant\_native.pyd`
+3. Drop `_native.pdb` next to it (same directory, same basename).
+
+Symbol info is line-tables-only, so frames resolve to `file:line` but local
+variables and types aren't included. That's enough for backtraces and is
+~30 MB; full debug info would be much larger.
+
+## How Windows cross-compile works
+
+The Windows path uses [cargo-xwin](https://github.com/rust-cross/cargo-xwin),
+which downloads the Microsoft CRT + Windows SDK headers (cached in the xwin
+volume; license auto-accepted via `XWIN_ACCEPT_LICENSE=1`). PyO3's
+`generate-import-lib` feature — enabled in
+`crates/qroissant-python/Cargo.toml` — lets us produce the `python3.dll`
+import library at build time, so no Windows Python install is required on the
+host.
+
+## Quick wheel sanity checks
+
+```sh
+# Listing
+python3 -m zipfile -l dist-windows/qroissant-*-win_amd64.whl
+
+# Validate the dist-info
+pipx run twine check dist-linux/*.whl dist-windows/*.whl
+```
+
+## Iterating on Python code
+
+For Python-side changes (no Rust touched), the fastest loop is still
+`maturin develop` against a local rustup install. The Docker flow above is
+optimized for producing wheels, not for inner-loop iteration. If you have a
+nightly rustup toolchain on the host, you can do:
+
+```sh
+python3 -m venv .venv && .venv/bin/pip install 'maturin>=1.8,<2.0'
+.venv/bin/maturin develop --release
+.venv/bin/python -c 'import qroissant'
+```
+
+## Troubleshooting
+
+- **"manylinux_2_34" wheel won't install on old distros** — the image base is
+  Debian 13 (glibc 2.38). For broader compatibility, build inside a
+  manylinux2014 image or run `auditwheel repair` against the produced wheel.
+- **"failed to generate python3.dll import library"** — means `llvm-dlltool`
+  is missing inside the image. Rebuild it with `scripts/build.sh image`.
+- **xwin download is slow / fails** — Microsoft's CDN occasionally rate-limits.
+  Drop the cache (`docker volume rm qroissant-xwin`) and retry.

Cargo.lock

@@ -868,6 +868,7 @@ version = "0.28.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8bf94ee265674bf76c09fa430b0e99c26e319c945d96ca0d5a8215f31bf81cf7"
 dependencies = [
+ "python3-dll-a",
  "target-lexicon",
 ]
 
@@ -906,9 +907,18 @@ dependencies = [
  "syn 2.0.117",
 ]
 
+[[package]]
+name = "python3-dll-a"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d80ba7540edb18890d444c5aa8e1f1f99b1bdf26fb26ae383135325f4a36042b"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "qroissant-arrow"
-version = "0.3.0"
+version = "0.3.1"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -926,7 +936,7 @@ dependencies = [
 
 [[package]]
 name = "qroissant-core"
-version = "0.3.0"
+version = "0.3.1"
 dependencies = [
  "bytemuck",
  "bytes",
@@ -938,11 +948,11 @@ dependencies = [
 
 [[package]]
 name = "qroissant-kernels"
-version = "0.3.0"
+version = "0.3.1"
 
 [[package]]
 name = "qroissant-python"
-version = "0.3.0"
+version = "0.3.1"
 dependencies = [
  "bb8",
  "bytes",
@@ -962,7 +972,7 @@ dependencies = [
 
 [[package]]
 name = "qroissant-transport"
-version = "0.3.0"
+version = "0.3.1"
 dependencies = [
  "bytes",
  "futures",

Cargo.toml

@@ -3,7 +3,7 @@ members = ["crates/qroissant-core", "crates/qroissant-kernels", "crates/qroissan
 resolver = "3"
 
 [workspace.package]
-version = "0.3.0"
+version = "0.3.1"
 edition = "2024"
 license = "Apache-2.0"
 repository = "https://github.com/qroissant/qroissant"
@@ -12,6 +12,11 @@ repository = "https://github.com/qroissant/qroissant"
 lto = "fat"
 codegen-units = 1
 opt-level = 3
+# Line tables only: keeps optimisations identical to a normal release build
+# but emits enough info for Rust panic backtraces to resolve to file:line.
+# On windows-msvc this produces a PDB next to _native.pyd (see BUILD.md).
+debug = "line-tables-only"
+strip = "none"
 
 [workspace.dependencies]
 pyo3 = "0.28.2"

crates/qroissant-arrow/src/projection.rs

@@ -1,6 +1,5 @@
 //! Arrow projection: converts decoded q `Value` trees into Arrow arrays.
 
-use std::mem::size_of;
 use std::ptr::NonNull;
 use std::sync::Arc;
 
@@ -601,9 +600,9 @@ fn project_vector(vector: &Vector, opts: &ProjectionOptions) -> ProjectionResult
                 validity = merge_infinity_i64(src, Q_INF_TIMESTAMP, Q_NINF_TIMESTAMP, validity);
             }
             let nulls = to_null_buffer(validity);
-            let mut bytes_buf = vec![0u8; src.len() * size_of::<i64>()];
+            let mut typed_buf = vec![0i64; src.len()];
-            copy_and_offset_timestamps(src, bytemuck::cast_slice_mut(&mut bytes_buf));
+            copy_and_offset_timestamps(src, &mut typed_buf);
-            let buf = Buffer::from_vec(bytes_buf);
+            let buf = Buffer::from_vec(typed_buf);
             let arr: ArrayRef = Arc::new(TimestampNanosecondArray::new(
                 ScalarBuffer::new(buf, 0, src.len()),
                 nulls,
@@ -625,9 +624,9 @@ fn project_vector(vector: &Vector, opts: &ProjectionOptions) -> ProjectionResult
                 validity = merge_infinity_i32(src, Q_INF_DATE, Q_NINF_DATE, validity);
             }
             let nulls = to_null_buffer(validity);
-            let mut bytes_buf = vec![0u8; src.len() * size_of::<i32>()];
+            let mut typed_buf = vec![0i32; src.len()];
-            copy_and_offset_dates(src, bytemuck::cast_slice_mut(&mut bytes_buf));
+            copy_and_offset_dates(src, &mut typed_buf);
-            let buf = Buffer::from_vec(bytes_buf);
+            let buf = Buffer::from_vec(typed_buf);
             let arr: ArrayRef =
                 Arc::new(Date32Array::new(ScalarBuffer::new(buf, 0, src.len()), nulls));
             (arr, DataType::Date32)
@@ -667,9 +666,9 @@ fn project_vector(vector: &Vector, opts: &ProjectionOptions) -> ProjectionResult
                 validity = merge_infinity_i32(src, Q_INF_MINUTE, Q_NINF_MINUTE, validity);
             }
             let nulls = to_null_buffer(validity);
-            let mut bytes_buf = vec![0u8; src.len() * size_of::<i32>()];
+            let mut typed_buf = vec![0i32; src.len()];
-            copy_and_minutes_to_seconds(src, bytemuck::cast_slice_mut(&mut bytes_buf));
+            copy_and_minutes_to_seconds(src, &mut typed_buf);
-            let buf = Buffer::from_vec(bytes_buf);
+            let buf = Buffer::from_vec(typed_buf);
             let arr: ArrayRef = Arc::new(Time32SecondArray::new(
                 ScalarBuffer::new(buf, 0, src.len()),
                 nulls,

crates/qroissant-python/Cargo.toml

@@ -14,7 +14,7 @@ path = "src/lib.rs"
 bb8 = "0.9.0"
 bytes = "1.11.1"
 chrono = "0.4.44"
-pyo3 = { workspace = true, features = ["extension-module"] }
+pyo3 = { workspace = true, features = ["extension-module", "abi3-py311", "generate-import-lib"] }
 pyo3-arrow = { version = "0.17.0", default-features = false }
 pyo3-async-runtimes = { version = "0.28.0", features = ["tokio-runtime"] }
 qroissant-arrow = { path = "../qroissant-arrow" }

crates/qroissant-python/src/values.rs

@@ -15,6 +15,7 @@ use pyo3::types::PyTuple;
 use pyo3_arrow::ffi::ArrayIterator;
 use pyo3_arrow::ffi::to_array_pycapsules;
 use pyo3_arrow::ffi::to_stream_pycapsule;
+use qroissant_arrow::HeterogeneousListMode;
 use qroissant_arrow::IngestionError;
 use qroissant_arrow::ProjectionOptions;
 use qroissant_arrow::ingest_array;
@@ -210,7 +211,7 @@ impl Atom {
         let schema_capsule: Option<Bound<'_, PyCapsule>> = requested_schema
             .map(|s| s.downcast_into::<PyCapsule>())
             .transpose()?;
-        let opts = slf.as_super().projection_opts().clone();
+        let opts = effective_export_opts(py, slf.as_super().projection_opts());
         let export = project(slf.as_super().inner(), &opts)
             .map_err(|e| PyNotImplementedError::new_err(e.to_string()))?;
         let capsules =
@@ -286,7 +287,7 @@ impl Vector {
         let schema_capsule: Option<Bound<'_, PyCapsule>> = requested_schema
             .map(|s| s.downcast_into::<PyCapsule>())
             .transpose()?;
-        let opts = slf.as_super().projection_opts().clone();
+        let opts = effective_export_opts(py, slf.as_super().projection_opts());
         let export = project(slf.as_super().inner(), &opts)
             .map_err(|e| PyNotImplementedError::new_err(e.to_string()))?;
         let capsules =
@@ -366,7 +367,7 @@ impl List {
         let schema_capsule: Option<Bound<'_, PyCapsule>> = requested_schema
             .map(|s| s.downcast_into::<PyCapsule>())
             .transpose()?;
-        let opts = slf.as_super().projection_opts().clone();
+        let opts = effective_export_opts(py, slf.as_super().projection_opts());
         let export = project(slf.as_super().inner(), &opts)
             .map_err(|e| PyNotImplementedError::new_err(e.to_string()))?;
         let capsules =
@@ -438,7 +439,7 @@ impl Dictionary {
         let schema_capsule: Option<Bound<'_, PyCapsule>> = requested_schema
             .map(|s| s.downcast_into::<PyCapsule>())
             .transpose()?;
-        let opts = slf.as_super().projection_opts().clone();
+        let opts = effective_export_opts(py, slf.as_super().projection_opts());
         let export = project(slf.as_super().inner(), &opts)
             .map_err(|e| PyNotImplementedError::new_err(e.to_string()))?;
         let capsules =
@@ -535,7 +536,7 @@ impl Table {
             qroissant_core::Value::Table(t) => t.clone(),
             _ => unreachable!("Table instances always hold q tables"),
         };
-        let opts = slf.as_super().projection_opts().clone();
+        let opts = effective_export_opts(py, slf.as_super().projection_opts());
         let export = py
             .detach(|| project_table(&table, &opts).map_err(|e| e.to_string()))
             .map_err(|e| PyNotImplementedError::new_err(e))?;
@@ -829,6 +830,34 @@ fn map_ingestion_error(e: IngestionError) -> PyErr {
     PyValueError::new_err(e.to_string())
 }
 
+/// If Polars is loaded in this Python process and the caller has not opted out
+/// of the default heterogeneous-list mode (still `Union`), transparently switch
+/// to `CoalesceTemporals` so mixed-precision temporal columns surface as a flat
+/// `Timestamp(ns)` instead of an Arrow `Union` (which Polars rejects with a
+/// hard panic).
+///
+/// Heuristic: presence of `polars` in `sys.modules` strongly correlates with a
+/// Polars consumer for the Arrow PyCapsule export. Users who want faithful
+/// `Union` despite Polars being imported should set the mode explicitly via
+/// `DecodeOptions.with_coalesce_temporals(False)` — left as a documented
+/// limitation, since explicit `Union` and the default are indistinguishable.
+fn effective_export_opts(py: Python<'_>, opts: &Arc<ProjectionOptions>) -> Arc<ProjectionOptions> {
+    if opts.heterogeneous_list_mode != HeterogeneousListMode::Union {
+        return opts.clone();
+    }
+    let polars_loaded = py
+        .import("sys")
+        .and_then(|sys| sys.getattr("modules"))
+        .and_then(|m| m.contains("polars"))
+        .unwrap_or(false);
+    if !polars_loaded {
+        return opts.clone();
+    }
+    let mut next = (**opts).clone();
+    next.heterogeneous_list_mode = HeterogeneousListMode::CoalesceTemporals;
+    Arc::new(next)
+}
+
 pub fn python_to_core_value(value: &Bound<'_, PyAny>) -> PyResult<CoreValue> {
     // Try qroissant Value first (it also implements Arrow protocols, so must come first).
     if let Ok(q_value) = value.extract::<PyRef<'_, Value>>() {

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "qroissant"
-version = "0.3.0"
+version = "0.3.1"
 description = "q/kdb+ IPC client library with Arrow-native Python interoperability"
 readme = "README.md"
 requires-python = ">=3.10"

rust-toolchain.toml

@@ -0,0 +1,4 @@
+[toolchain]
+channel = "nightly"
+components = ["rustfmt", "clippy", "rust-src"]
+targets = ["x86_64-unknown-linux-gnu"]

scripts/Dockerfile.build

@@ -0,0 +1,25 @@
+# Build environment for qroissant.
+#
+# Base: ghcr.io/rust-cross/cargo-xwin — Debian 13, ships with cargo-xwin,
+# llvm-dlltool, and the x86_64-pc-windows-msvc rustup target preinstalled.
+# We layer on nightly Rust (project uses #![feature(portable_simd)]),
+# lld (so cargo-xwin can invoke lld-link), and maturin.
+FROM ghcr.io/rust-cross/cargo-xwin:latest
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends lld python3-pip python3-venv \
+ && rm -rf /var/lib/apt/lists/* \
+ && ln -sf /usr/bin/lld /usr/bin/lld-link \
+ && ln -sf /usr/bin/clang /usr/bin/clang-cl
+
+RUN rustup toolchain install nightly --profile minimal \
+        --component rustfmt --component clippy --component rust-src \
+        --component llvm-tools-preview \
+ && rustup target add --toolchain nightly x86_64-pc-windows-msvc \
+ && rustup default nightly
+
+RUN pip3 install --no-cache-dir --break-system-packages 'maturin>=1.8,<2.0'
+
+ENV PATH="/usr/local/cargo/bin:${PATH}"

scripts/build.sh

@@ -0,0 +1,158 @@
+#!/usr/bin/env bash
+# Build qroissant wheels via Docker. Both Linux and Windows builds run inside
+# the qroissant-build:latest image (defined in scripts/Dockerfile.build) so the
+# host needs only Docker + Python.
+#
+# Usage:
+#   scripts/build.sh image            Build the qroissant-build Docker image.
+#   scripts/build.sh linux            Build a manylinux x86_64 wheel -> dist-linux/
+#   scripts/build.sh windows          Build a win_amd64 wheel        -> dist-windows/
+#   scripts/build.sh all              image + linux + windows.
+#   scripts/build.sh check            Install the latest linux wheel into .venv
+#                                     and import qroissant as a smoke test.
+#   scripts/build.sh clean            Remove dist-*/ output dirs (volumes kept).
+#   scripts/build.sh clean-cache      Also drop the Docker volumes (cargo
+#                                     registry, target dir, xwin SDK cache).
+
+set -euo pipefail
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+ROOT_DIR=$(cd -- "$SCRIPT_DIR/.." &>/dev/null && pwd)
+cd "$ROOT_DIR"
+
+IMAGE=qroissant-build:latest
+VOL_CARGO=qroissant-cargo-registry
+VOL_TARGET=qroissant-target
+VOL_XWIN=qroissant-xwin
+
+need_docker() {
+  command -v docker >/dev/null 2>&1 || {
+    echo "error: docker is required but not on PATH" >&2
+    exit 1
+  }
+}
+
+ensure_volumes() {
+  local uid gid
+  uid=$(id -u); gid=$(id -g)
+  for v in "$VOL_CARGO" "$VOL_TARGET" "$VOL_XWIN"; do
+    if ! docker volume inspect "$v" >/dev/null 2>&1; then
+      docker volume create "$v" >/dev/null
+    fi
+    # Re-chown the volume to the invoking user every run; cheap and idempotent.
+    docker run --rm -v "$v":/v --entrypoint=sh "$IMAGE" -c "chown -R $uid:$gid /v" >/dev/null
+  done
+}
+
+ensure_image() {
+  if ! docker image inspect "$IMAGE" >/dev/null 2>&1; then
+    echo ">>> building $IMAGE"
+    docker build -t "$IMAGE" -f "$SCRIPT_DIR/Dockerfile.build" "$SCRIPT_DIR"
+  fi
+}
+
+run_in_image() {
+  # Run as the invoking user so produced wheels are owned by them, not root.
+  # CARGO_HOME / XDG_CACHE_HOME are redirected into the persistent volumes,
+  # which must therefore be owned by the same uid (handled by ensure_volumes).
+  docker run --rm \
+    --user "$(id -u):$(id -g)" \
+    -e CARGO_HOME=/cargo \
+    -e XDG_CACHE_HOME=/xdg-cache \
+    -e HOME=/tmp \
+    -e XWIN_ACCEPT_LICENSE=1 \
+    -v "$ROOT_DIR":/io -w /io \
+    -v "$VOL_CARGO":/cargo \
+    -v "$VOL_TARGET":/io/target \
+    -v "$VOL_XWIN":/xdg-cache/cargo-xwin \
+    --entrypoint=sh \
+    "$IMAGE" -c "$1"
+}
+
+cmd_image() {
+  need_docker
+  echo ">>> (re)building $IMAGE"
+  docker build -t "$IMAGE" -f "$SCRIPT_DIR/Dockerfile.build" "$SCRIPT_DIR"
+}
+
+cmd_linux() {
+  need_docker
+  ensure_image
+  ensure_volumes
+  echo ">>> building Linux wheel -> dist-linux/"
+  run_in_image 'maturin build --release --out /io/dist-linux'
+  ls -1 dist-linux/*.whl
+}
+
+cmd_windows() {
+  need_docker
+  ensure_image
+  ensure_volumes
+  echo ">>> building Windows wheel -> dist-windows/"
+  run_in_image '
+    maturin build --release --target x86_64-pc-windows-msvc --out /io/dist-windows
+    # Copy the PDB next to the wheel when one was produced (release profile
+    # has debug = "line-tables-only"). Place it next to _native.pyd at
+    # runtime on Windows to get resolved panic backtraces.
+    pdb=$(find /io/target/x86_64-pc-windows-msvc/release -maxdepth 2 -name "_native.pdb" 2>/dev/null | head -1)
+    if [ -n "$pdb" ]; then
+      cp "$pdb" /io/dist-windows/
+      echo "+ copied $(basename "$pdb") -> dist-windows/"
+    fi
+  '
+  ls -1 dist-windows/
+}
+
+cmd_check() {
+  local venv="$ROOT_DIR/.venv"
+  if [[ ! -x "$venv/bin/python" ]]; then
+    echo ">>> creating venv at $venv"
+    python3 -m venv "$venv"
+    "$venv/bin/pip" install --quiet --upgrade pip
+  fi
+  local wheel
+  wheel=$(ls -t dist-linux/qroissant-*-linux*.whl dist-linux/qroissant-*-manylinux*.whl 2>/dev/null | head -1 || true)
+  if [[ -z "$wheel" ]]; then
+    echo "error: no Linux wheel in dist-linux/; run '$0 linux' first" >&2
+    exit 1
+  fi
+  echo ">>> installing $wheel"
+  "$venv/bin/pip" install --quiet --force-reinstall "$wheel"
+  "$venv/bin/python" -c 'import qroissant; print("ok: qroissant", qroissant.__version__ if hasattr(qroissant, "__version__") else "(no __version__)")'
+  if [[ -f dist-windows/qroissant-*-win_amd64.whl ]] 2>/dev/null || ls dist-windows/qroissant-*-win_amd64.whl >/dev/null 2>&1; then
+    echo ">>> inspecting Windows wheel"
+    for w in dist-windows/qroissant-*-win_amd64.whl; do
+      python3 -m zipfile -l "$w" | grep -E '_native\.pyd|WHEEL' || true
+    done
+  fi
+}
+
+cmd_clean() {
+  rm -rf dist-linux dist-windows
+  echo ">>> removed dist-linux/ dist-windows/"
+}
+
+cmd_clean_cache() {
+  cmd_clean
+  for v in "$VOL_CARGO" "$VOL_TARGET" "$VOL_XWIN"; do
+    docker volume rm "$v" >/dev/null 2>&1 && echo ">>> removed volume $v" || true
+  done
+}
+
+case "${1:-}" in
+  image)        cmd_image ;;
+  linux)        cmd_linux ;;
+  windows)      cmd_windows ;;
+  all)          cmd_image; cmd_linux; cmd_windows ;;
+  check)        cmd_check ;;
+  clean)        cmd_clean ;;
+  clean-cache)  cmd_clean_cache ;;
+  ""|-h|--help)
+    sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' | sed '/^set -euo/d'
+    ;;
+  *)
+    echo "error: unknown command: $1" >&2
+    echo "run '$0 --help' for usage" >&2
+    exit 2
+    ;;
+esac