From 7a3bf53115df6faccb8ef9523b21443cdc89675e Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Mon, 6 Jul 2026 21:20:59 +0200 Subject: [PATCH 1/8] test: add Raincloud real-world conformance harness (#205) Cross-implementation conformance against the Raincloud corpus: 247 public datasets whose Vortex files are written by the Python bindings (vortex-data 0.69.0), each with a Parquet sibling built from the same Arrow table. - expected-status.csv enumerates every vortex-enabled slug with a triaged status: ok (must match the Parquet oracle exactly), gap: (must still fail, so a fix forces the flip in the same change), untriaged (runs and reports as aborted without failing the build). - RaincloudConformanceIntegrationTest discovers whatever the hydrate manifest lists; the oracle reads the Parquet sibling through hardwood and writes through the same fastcsv writer as CsvExporter, so a zero-diff proves every value survived the Python-write/Java-read boundary. Oracle-side limitations (nested parquet columns) abort the slug instead of failing it. - scripts/hydrate-raincloud-corpus.sh resolves slugs through the raincloud loader (cache -> mirror -> local build) pinned at v0.2.1, with a per-slug size budget and tolerant per-slug failures. - raincloud-conformance.yml runs a size-capped sweep weekly; off the PR path because upstream dataset URLs rot independently of our code. First triage of 10 hydrated slugs: 4 ok (value-for-value exact), 6 gaps filed as #206 (lazy dict U8/U16 values), #207 (nested struct columns), Co-Authored-By: Claude Opus 4.8 --- .github/workflows/raincloud-conformance.yml | 64 +++++ CHANGELOG.md | 4 + CLAUDE.md | 2 + docs/compatibility.md | 25 ++ .../RaincloudConformanceIntegrationTest.java | 229 ++++++++++++++++ .../resources/raincloud/expected-status.csv | 256 ++++++++++++++++++ scripts/hydrate-raincloud-corpus.sh | 87 ++++++ 7 files changed, 667 insertions(+) create mode 100644 .github/workflows/raincloud-conformance.yml create mode 100644 integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java create mode 100644 integration/src/test/resources/raincloud/expected-status.csv create mode 100755 scripts/hydrate-raincloud-corpus.sh diff --git a/.github/workflows/raincloud-conformance.yml b/.github/workflows/raincloud-conformance.yml new file mode 100644 index 00000000..1acdea37 --- /dev/null +++ b/.github/workflows/raincloud-conformance.yml @@ -0,0 +1,64 @@ +name: Raincloud conformance + +# Real-world cross-implementation conformance (issue #205): hydrates the Raincloud +# corpus (Vortex files written by the Python bindings + Parquet oracles) and runs +# RaincloudConformanceIntegrationTest against everything hydrated. Scheduled, not on +# the PR path: hydration depends on upstream dataset URLs that rot independently of +# our code, so a failure here is a finding to triage, not a broken build. + +on: + schedule: + - cron: '0 5 * * 1' # Mondays 05:00 UTC + workflow_dispatch: + inputs: + max-mb: + description: 'Per-slug artifact size cap in MB (0 = uncapped)' + default: '200' + +jobs: + conformance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Set up Azul Zulu JDK 25 + uses: actions/setup-java@v5 + with: + distribution: zulu + java-version: '25' + + - name: Cache Maven repository + uses: actions/cache@v6 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + + # keyed on the matrix + pinned tag: a corpus change rebuilds, otherwise + # weekly runs reuse artifacts instead of re-fetching upstream sources + - name: Cache Raincloud corpus + uses: actions/cache@v6 + with: + path: ~/.cache/raincloud + key: raincloud-corpus-${{ hashFiles('scripts/hydrate-raincloud-corpus.sh', 'integration/src/test/resources/raincloud/expected-status.csv') }} + + - name: Hydrate corpus + run: scripts/hydrate-raincloud-corpus.sh --max-mb "${{ github.event.inputs.max-mb || '200' }}" + + - name: Run conformance suite + run: ./mvnw verify -pl integration -am -Dit.test=RaincloudConformanceIntegrationTest + + - name: Publish per-slug results to job summary + if: always() + run: | + { + echo '## Raincloud conformance' + echo '```' + grep -hE "Tests run|slug" integration/target/failsafe-reports/*.txt || true + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/CHANGELOG.md b/CHANGELOG.md index 6368bea8..4d9bfce8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - CSV export renders unsigned integer columns (U8–U64) with their unsigned values — high-half values previously printed as two's-complement negatives (uci-wine `magnesium` U8 132 exported as -124), silent corruption found by the Raincloud conformance suite. ([#208](https://github.com/dfa1/vortex-java/issues/208)) - `fastlanes.rle` decodes F64/F32 value pools (`LazyRleDoubleArray`, `LazyRleFloatArray`) — files from the Python bindings RLE-encode double columns with long constant runs, which previously failed to scan with `unsupported ptype F64`. ([#209](https://github.com/dfa1/vortex-java/issues/209)) +### Added + +- Real-world conformance suite against the [Raincloud](https://github.com/spiraldb/raincloud) corpus: 247 public datasets whose Vortex files are written by the Python bindings, each validated value-for-value against its Parquet sibling. `scripts/hydrate-raincloud-corpus.sh` hydrates any subset (cache → mirror → local build), `RaincloudConformanceIntegrationTest` tests whatever is hydrated against the checked-in per-dataset status matrix, and a weekly workflow runs a size-capped sweep. First triage found four reader gaps on real data, including one silent-corruption bug (unsigned integers rendered as signed). ([#205](https://github.com/dfa1/vortex-java/issues/205)) + ## [0.12.0] — 2026-07-04 Identity gets **types**: encoding ids, layout ids, and column names are now validated domain diff --git a/CLAUDE.md b/CLAUDE.md index 5ef60e2c..1f9ae7f2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -63,6 +63,8 @@ Trunk-based. PRs fine but always squash or rebase — no merge commits. Keep com ./mvnw verify -pl integration -am # integration (failsafe, NOT surefire) ./mvnw verify -pl integration -am -Dit.test="RustWritesJavaReadsIntegrationTest#method" ./bench JavaVsJniReadBenchmark.javaReadVolume # benchmark — always ClassName.methodName filter +scripts/hydrate-raincloud-corpus.sh --max-mb 200 # hydrate real-world conformance corpus (#205), + # then verify -Dit.test=RaincloudConformanceIntegrationTest ``` Regenerate after editing `.fbs`/`.proto` (both generators are in-house, no external tools): diff --git a/docs/compatibility.md b/docs/compatibility.md index ee13e7c1..8e6eb26c 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -43,6 +43,31 @@ resolves only the standalone decoders in `reader`; no encoder class is loaded. | Duplicate struct field names | Rust writer rejects ("StructLayout must have unique field names"); Rust reader tolerates foreign files (first-match access) | ⚠️ Deliberate divergence on read: Java rejects such files with `VortexException("duplicate field name in file schema")` instead of tolerating them — the name-keyed `Chunk` API cannot represent both columns, and silent column loss is worse than a loud failure on a file the reference writer refuses to produce. Java's writer mirrors the Rust writer's rejection. | | Blank / control-character field names | Wire-legal; the Rust writer produces `""` and whitespace-only names. NUL (`U+0000`) additionally aborts the Rust toolchain: Arrow FFI schema export hits a panic-cannot-unwind in `arrow-rs` (`ffi_stream::get_schema`) and SIGABRTs the process (measured against vortex-jni 0.75.0) | ⚠️ Deliberate strictness BOTH ways: vortex-java's writer refuses blank and control-character field names (`IllegalArgumentException`), and its reader rejects files carrying them (`VortexException` naming the producing pipeline as the likely bug) — the JSON-`""`-key principle: wire-legal is a floor, not a policy. Printable names of any shape (`$`-runs, spaces inside, emoji) are legal and round-trip intact both directions (measured; pinned by `ColumnNameEdgeCasesIntegrationTest`). | +## Real-world conformance: the Raincloud corpus + +Cross-implementation conformance against [Raincloud](https://github.com/spiraldb/raincloud) +(tracked in [#205](https://github.com/dfa1/vortex-java/issues/205)): 247 curated public +datasets whose `.vortex` files are written by the Vortex **Python** bindings +(`vortex-data 0.69.0`, pinned via Raincloud `v0.2.1`) with a Parquet sibling built from the +same Arrow table. `RaincloudConformanceIntegrationTest` reads each hydrated `.vortex` with +vortex-java, exports CSV, and diffs it against the Parquet sibling read by hardwood (the +oracle) — a zero-diff proves every value survived the Python-write / Java-read boundary. + +```bash +scripts/hydrate-raincloud-corpus.sh --max-mb 200 # cache → mirror → local build +./mvnw verify -pl integration -am -Dit.test=RaincloudConformanceIntegrationTest +``` + +Per-slug status lives in `integration/src/test/resources/raincloud/expected-status.csv` +(`ok` must pass; `gap:` must still fail, so a fix flips the entry in the same change; +`untriaged` runs and reports without failing the build). A scheduled workflow +(`raincloud-conformance.yml`) hydrates a size-capped subset weekly. Current triage — +4 `ok`, 6 known gaps: lazy dict U8/U16 values +([#206](https://github.com/dfa1/vortex-java/issues/206)), nested struct columns in scan +([#207](https://github.com/dfa1/vortex-java/issues/207)), unsigned integers rendered signed +([#208](https://github.com/dfa1/vortex-java/issues/208) — silent corruption), RLE over F64 +([#209](https://github.com/dfa1/vortex-java/issues/209)); 237 slugs untriaged. + ## Encodings | Encoding ID | Decoder | Encoder | Decode | Encode | Notes | diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java new file mode 100644 index 00000000..21b89613 --- /dev/null +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java @@ -0,0 +1,229 @@ +package io.github.dfa1.vortex.integration; + +import de.siegmar.fastcsv.writer.CsvWriter; +import dev.hardwood.InputFile; +import dev.hardwood.metadata.RepetitionType; +import dev.hardwood.reader.ParquetFileReader; +import dev.hardwood.reader.RowReader; +import dev.hardwood.schema.ColumnSchema; +import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.csv.CsvExporter; +import io.github.dfa1.vortex.csv.ExportOptions; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; +import org.opentest4j.TestAbortedException; + +import java.io.IOException; +import java.io.StringWriter; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/// Real-world cross-implementation conformance over the Raincloud corpus +/// ([issue #205](https://github.com/dfa1/vortex-java/issues/205)). +/// +/// Each corpus dataset is a `.vortex` file written by the Vortex Python bindings with +/// its `.parquet` sibling built from the same Arrow table. The parquet file is the +/// oracle: hardwood reads it row-by-row and formats cells with the exact rules of +/// `CsvExporter.cellValue` (null → empty field, `Double.toString`, …), writing through +/// the same fastcsv writer so quoting is identical. A zero-diff against vortex-java's +/// CSV export proves every value survived the Python-write / Java-read boundary. +/// +/// Skipped (visibly, via assumption) when the corpus is not hydrated — run +/// `scripts/hydrate-raincloud-corpus.sh` first, or point `RAINCLOUD_CORPUS_MANIFEST` +/// at a manifest TSV (`slugvortex-pathparquet-path` per line). +/// +/// Expected per-slug status lives in `src/test/resources/raincloud/expected-status.csv`. +/// Known gaps assert that the failure still occurs, so fixing the reader forces the +/// matrix entry to flip to `ok` in the same change. +class RaincloudConformanceIntegrationTest { + + private static final Path DEFAULT_MANIFEST = + Path.of(System.getProperty("user.home"), ".cache", "raincloud", "corpus-manifest.tsv"); + + @Test + void corpusIsHydrated() { + // Given / When / Then — visible skip marker when the corpus is absent; the + // factory below yields zero tests in that case, which would otherwise pass silently + assumeTrue(Files.exists(manifestPath()), + "raincloud corpus not hydrated — run scripts/hydrate-raincloud-corpus.sh"); + } + + @TestFactory + Stream conformancePerSlug() throws IOException { + Path manifest = manifestPath(); + if (!Files.exists(manifest)) { + return Stream.empty(); + } + Map expected = readExpectedStatus(); + return Files.readAllLines(manifest).stream() + .filter(line -> !line.isBlank()) + .map(line -> line.split("\t")) + .map(parts -> DynamicTest.dynamicTest(parts[0], () -> { + String slug = parts[0]; + Path vortex = Path.of(parts[1]); + Path parquet = Path.of(parts[2]); + // slugs missing from the matrix are treated as untriaged, not failing + String status = expected.getOrDefault(slug, "untriaged"); + switch (status) { + case "ok" -> assertMatchesParquetOracle(vortex, parquet); + case "untriaged" -> reportUntriaged(vortex, parquet); + default -> assertStillFails(vortex, parquet, status); + } + })); + } + + private static void assertMatchesParquetOracle(Path vortex, Path parquet) throws IOException { + // Given + List oracleLines = oracleLines(parquet); + + // When + List result = exportVortex(vortex); + + // Then + assertLinesMatch(result, oracleLines); + } + + /// Runs the full conformance check but reports the outcome as an aborted test + /// either way: only triaged matrix entries may count as green or red. The abort + /// message says which way to flip the entry. + private static void reportUntriaged(Path vortex, Path parquet) { + try { + assertMatchesParquetOracle(vortex, parquet); + } catch (TestAbortedException e) { + throw e; // oracle limitation, not a conformance outcome + } catch (AssertionError | Exception e) { + throw new TestAbortedException( + "untriaged slug fails — classify as gap: in expected-status.csv: " + e); + } + throw new TestAbortedException("untriaged slug passes — flip its matrix entry to ok"); + } + + private static void assertStillFails(Path vortex, Path parquet, String status) { + // Given / When — a decode gap still reproduces when the export itself throws; + // no oracle needed (the oracle may not even read this parquet, e.g. nested columns) + List result; + try { + result = exportVortex(vortex); + } catch (VortexException e) { + return; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + // Then — the export now succeeds, so the gap must still be a silent-corruption + // one (e.g. #208): values must still mismatch the oracle. A clean pass means the + // gap was fixed: flip the expected-status.csv entry to ok in the same change + List oracleLines = oracleLines(parquet); + assertThatThrownBy(() -> assertLinesMatch(result, oracleLines)) + .as("known gap %s no longer reproduces — flip its matrix entry to ok", status) + .isInstanceOf(AssertionError.class); + } + + private static void assertLinesMatch(List result, List oracleLines) { + assertThat(result).hasSameSizeAs(oracleLines); + for (int i = 0; i < oracleLines.size(); i++) { + assertThat(result.get(i)).as("line %d", i + 1).isEqualTo(oracleLines.get(i)); + } + } + + private static List exportVortex(Path vortex) throws IOException { + StringWriter out = new StringWriter(); + CsvExporter.exportCsv(vortex, out, ExportOptions.defaults()); + return out.toString().lines().toList(); + } + + /// Reads the parquet sibling through hardwood; an oracle-side failure (nested + /// columns, unsupported physical type) aborts the slug rather than failing it — + /// it says nothing about vortex-java. + private static List oracleLines(Path parquet) { + StringWriter out = new StringWriter(); + try { + writeOracleCsv(parquet, out); + } catch (TestAbortedException e) { + throw e; + } catch (Exception e) { + throw new TestAbortedException("oracle cannot read the parquet sibling: " + e); + } + return out.toString().lines().toList(); + } + + /// Oracle: hardwood reads the parquet sibling and emits CSV through the same + /// fastcsv writer configuration as `CsvExporter`, using its exact cell rules. + private static void writeOracleCsv(Path parquet, StringWriter out) throws IOException { + try (ParquetFileReader pfr = ParquetFileReader.open(InputFile.of(parquet)); + RowReader rows = pfr.rowReader(); + CsvWriter csv = CsvWriter.builder().fieldSeparator(',').build(out)) { + + List cols = pfr.getFileSchema().getColumns(); + csv.writeRecord(cols.stream().map(ColumnSchema::name).toList()); + + String[] row = new String[cols.size()]; + while (rows.hasNext()) { + rows.next(); + for (int c = 0; c < cols.size(); c++) { + row[c] = oracleCell(cols.get(c), rows); + } + csv.writeRecord(row); + } + } + } + + /// Formats a parquet cell with the exact rules of `CsvExporter.cellValue`: + /// null rows export as an empty field, valid rows use the JDK canonical + /// `toString` of the value. + /// + /// @param col the column schema + /// @param rows the row reader positioned at the current row + /// @return the formatted cell string + private static String oracleCell(ColumnSchema col, RowReader rows) { + String name = col.name(); + if (col.repetitionType() == RepetitionType.OPTIONAL && rows.isNull(name)) { + return ""; + } + return switch (col.type()) { + case INT32 -> Integer.toString(rows.getInt(name)); + case INT64 -> Long.toString(rows.getLong(name)); + case FLOAT -> Float.toString(rows.getFloat(name)); + case DOUBLE -> Double.toString(rows.getDouble(name)); + case BOOLEAN -> Boolean.toString(rows.getBoolean(name)); + case BYTE_ARRAY -> rows.getString(name); + // aborts (not fails) the slug: the oracle can't format this physical type + // yet, which is an oracle limitation rather than a vortex-java gap + default -> throw new TestAbortedException( + "oracle cannot format parquet type " + col.type() + " (column: " + name + ")"); + }; + } + + private static Path manifestPath() { + String env = System.getenv("RAINCLOUD_CORPUS_MANIFEST"); + return env != null ? Path.of(env) : DEFAULT_MANIFEST; + } + + private static Map readExpectedStatus() throws IOException { + try (var in = RaincloudConformanceIntegrationTest.class + .getResourceAsStream("/raincloud/expected-status.csv")) { + if (in == null) { + throw new UncheckedIOException(new IOException("expected-status.csv not on test classpath")); + } + Map statuses = new HashMap<>(); + for (String line : new String(in.readAllBytes()).lines().toList()) { + if (line.isBlank() || line.startsWith("#")) { + continue; + } + String[] parts = line.split(",", 2); + statuses.put(parts[0].trim(), parts[1].trim()); + } + return statuses; + } + } +} diff --git a/integration/src/test/resources/raincloud/expected-status.csv b/integration/src/test/resources/raincloud/expected-status.csv new file mode 100644 index 00000000..4937e3ad --- /dev/null +++ b/integration/src/test/resources/raincloud/expected-status.csv @@ -0,0 +1,256 @@ +# Raincloud conformance matrix — one line per vortex-enabled corpus slug: , +# status: ok — vortex-java must read the file and match the parquet oracle exactly +# gap: — known reader gap tracked by issue #; the test asserts the failure +# still occurs, so a fix forces flipping the entry to ok in the same change +# untriaged — not yet run; the test executes it and reports the outcome as an +# aborted (skipped) test instead of failing the build — flip to ok/gap +# as hydration coverage grows +# Corpus: all slugs with a Vortex artifact in the Raincloud catalog (spiraldb/raincloud), +# written by vortex-data 0.69.0. Hydrate any subset with scripts/hydrate-raincloud-corpus.sh. +120-years-of-olympic-history-athletes-and-results,untriaged +ai2-arc,untriaged +airbnb-prices-in-european-cities,untriaged +airbnbopendata,untriaged +amazon-reviews-2023-subscription-boxes,untriaged +ampds-whe,untriaged +anthropic-economic-index,untriaged +anthropic-hh-rlhf-helpful-base,untriaged +anthropic-interviewer,untriaged +aya-collection-templated,untriaged +aya-dataset,untriaged +bank-account-fraud-dataset-neurips-2022,untriaged +basketball,untriaged +behavioral-risk-factor-surveillance-system,untriaged +beir-msmarco,untriaged +bi-arade,untriaged +bi-bimbo,untriaged +bi-citymaxcapita,untriaged +bi-cmsprovider,untriaged +bi-commongovernment,untriaged +bi-corporations,untriaged +bi-eixo,untriaged +bi-euro2016,untriaged +bi-food,untriaged +bi-generico,untriaged +bi-hashtags,untriaged +bi-hatred,untriaged +bi-iglocations1,untriaged +bi-iglocations2,untriaged +bi-iublibrary,untriaged +bi-medicare1,untriaged +bi-medicare2,untriaged +bi-medicare3,untriaged +bi-medpayment1,untriaged +bi-medpayment2,untriaged +bi-mlb,untriaged +bi-motos,untriaged +bi-mulheresmil,untriaged +bi-nyc,untriaged +bi-pancreactomy1,untriaged +bi-pancreactomy2,untriaged +bi-physicians,untriaged +bi-provider,untriaged +bi-realestate1,untriaged +bi-realestate2,untriaged +bi-redfin1,untriaged +bi-redfin2,untriaged +bi-redfin3,untriaged +bi-redfin4,untriaged +bi-rentabilidad,untriaged +bi-romance,untriaged +bi-salariesfrance,untriaged +bi-tablerosistemapenal,untriaged +bi-taxpayer,untriaged +bi-telco,untriaged +bi-trainsuk1,untriaged +bi-trainsuk2,untriaged +bi-uberlandia,untriaged +bi-uscensus,untriaged +bi-wins,untriaged +bi-yalelanguages,untriaged +bigscience-p3-eval-subset,untriaged +building-permit-applications-data,untriaged +c4-en-validation,untriaged +california-housing-prices,untriaged +cardiovascular-diseases-risk-prediction-dataset,untriaged +chess,untriaged +clickbench-hits,untriaged +cnn-dailymail,untriaged +codeparrot-clean-valid,untriaged +cohere-wikipedia-simple-embed,untriaged +coig,untriaged +coronahack-chest-xraydataset,untriaged +cosmopedia-stanford,untriaged +countries-of-the-world,gap:207 +covid-world-vaccination-progress,untriaged +covid19-data-from-john-hopkins-university,untriaged +crimes-in-boston,untriaged +databricks-dolly-15k,untriaged +dbpedia-embeddings,untriaged +diabetes-health-indicators-dataset,untriaged +disease-symptom-description-dataset,untriaged +docmatix-zero-shot,untriaged +electric-motor-temperature,untriaged +emotions-dataset-for-nlp,untriaged +fhv_tripdata_2025,untriaged +fhvhv_tripdata_2025,untriaged +finemath-4plus,untriaged +finepdfs-en-test,untriaged +finetranslations-swedish,untriaged +fineweb-2-swedish,untriaged +fineweb-sample-10bt,untriaged +fitbit,untriaged +formula-1-world-championship-1950-2020,untriaged +frames-benchmark,untriaged +ghcn-daily,untriaged +glass,untriaged +global-fossil-co2-emissions-by-country-2002-2022,untriaged +glove-6b-100d,untriaged +glove-6b-200d,untriaged +glove-6b-50d,untriaged +goodbooks-10k,untriaged +google-cluster-trace-2011-machine-events,untriaged +green_tripdata_2025,untriaged +gsm8k,untriaged +gtsrb-german-traffic-sign,untriaged +hacker-news,untriaged +heart-disease-health-indicators-dataset,untriaged +hellaswag,untriaged +helpsteer2,untriaged +hotel-booking-demand,untriaged +hotpotqa-fullwiki,untriaged +housing-prices-dataset,untriaged +humaneval,untriaged +insurance,untriaged +international-football-results-from-1872-to-2017,untriaged +iowa-liquor-sales,untriaged +kepler-exoplanet-search-results,gap:206 +kepler-labelled-time-series-data,untriaged +laion-400m,untriaged +librispeech-test-clean,untriaged +loan-default-dataset,untriaged +lung-cancer,untriaged +marketing-data,untriaged +mbpp,untriaged +medmcqa,untriaged +mlcourse,untriaged +mmlu,untriaged +mmlu-pro,untriaged +mmmlu,untriaged +mmmu,untriaged +mnist,untriaged +movies,untriaged +new-york-city-airbnb-open-data,untriaged +news-headlines-dataset-for-sarcasm-detection,untriaged +nips-papers,untriaged +no-robots,untriaged +nyc-311,untriaged +nyc-parking-tickets,untriaged +nyc-property-sales,untriaged +nypd-complaints,untriaged +oasst1,untriaged +open-food-facts,untriaged +openlibrary-authors,untriaged +openlibrary-editions,untriaged +openlibrary-works,untriaged +openorca,untriaged +openpowerlifting,untriaged +osm-germany-nodes,untriaged +osm-germany-relations,untriaged +osm-germany-ways,untriaged +osmi-mental-health-in-tech-2016,untriaged +osmi-mental-health-in-tech-2017,untriaged +osmi-mental-health-in-tech-2018,untriaged +osmi-mental-health-in-tech-2019,untriaged +osmi-mental-health-in-tech-2020,untriaged +osmi-mental-health-in-tech-2021,untriaged +osmi-mental-health-in-tech-2022,untriaged +osmi-mental-health-in-tech-2023,untriaged +palmer-archipelago-antarctica-penguin-data,gap:206 +peoples-speech-clean-validation,untriaged +personal-key-indicators-of-heart-disease,untriaged +pleias-synth,untriaged +pubmedqa-labeled,untriaged +sf-salaries,untriaged +slimpajama-6b,untriaged +squad-v2,untriaged +stack-overflow-2018-developer-survey,untriaged +stackoverflow-badges,untriaged +stackoverflow-postlinks,untriaged +stackoverflow-posts,untriaged +stackoverflow-tags,untriaged +stackoverflow-users,untriaged +synthetic-text-to-sql,untriaged +temperature-change,untriaged +tinystories,untriaged +titanic-dataset,untriaged +truthfulqa-mc,untriaged +uber-pickups-in-new-york-city,untriaged +uci-abalone,untriaged +uci-adult,untriaged +uci-ai4i-2020-predictive-maintenance,untriaged +uci-air-quality,untriaged +uci-auto-mpg,untriaged +uci-automobile,untriaged +uci-bank-marketing,untriaged +uci-beijing-multi-site-air-quality,untriaged +uci-bike-sharing-dataset,gap:206 +uci-breast-cancer,untriaged +uci-breast-cancer-wisconsin-diagnostic,untriaged +uci-breast-cancer-wisconsin-original,untriaged +uci-car-evaluation,untriaged +uci-cdc-diabetes-health-indicators,untriaged +uci-census-income,untriaged +uci-chronic-kidney-disease,untriaged +uci-concrete-compressive-strength,untriaged +uci-credit-approval,untriaged +uci-default-of-credit-card-clients,untriaged +uci-diabetes,untriaged +uci-diabetes-130-us-hospitals,untriaged +uci-dry-bean-dataset,untriaged +uci-electricityloaddiagrams20112014,untriaged +uci-energy-efficiency,untriaged +uci-estimation-of-obesity-levels,ok +uci-forest-fires,untriaged +uci-heart-disease,untriaged +uci-heart-failure-clinical-records,untriaged +uci-human-activity-recognition-using-smartphones,untriaged +uci-individual-household-electric-power-consumption,untriaged +uci-iris,ok +uci-magic-gamma-telescope,untriaged +uci-mushroom,untriaged +uci-online-retail,untriaged +uci-online-retail-ii,untriaged +uci-online-shoppers-purchasing-intention,untriaged +uci-optical-recognition-of-handwritten-digits,untriaged +uci-parkinsons,untriaged +uci-phishing-websites,untriaged +uci-predict-students-dropout-and-academic-success,untriaged +uci-real-estate-valuation-data-set,untriaged +uci-seeds,ok +uci-seoul-bike-sharing-demand,gap:209 +uci-sms-spam-collection,untriaged +uci-spambase,untriaged +uci-statlog-german-credit-data,untriaged +uci-student-performance,untriaged +uci-thyroid-disease,untriaged +uci-wholesale-customers,untriaged +uci-wine,gap:208 +uci-wine-quality,ok +ufcdata,untriaged +uk-price-paid,untriaged +uk-road-safety-accidents-and-vehicles,untriaged +ultrachat-200k,untriaged +ultrafeedback-binarized,untriaged +us-accidents,untriaged +us-airbnb-open-data,untriaged +us-drought-meteorological-data,untriaged +walmart-dataset,untriaged +waxal-dagbani-asr-test,untriaged +wdi,untriaged +websight-v01,untriaged +wikipedia-en,untriaged +world-energy-consumption,untriaged +yellow_tripdata_2025,untriaged +youtube-commons-sample,untriaged +zoo-animal-classification,untriaged diff --git a/scripts/hydrate-raincloud-corpus.sh b/scripts/hydrate-raincloud-corpus.sh new file mode 100755 index 00000000..29cdf61a --- /dev/null +++ b/scripts/hydrate-raincloud-corpus.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Hydrates the Raincloud conformance corpus (issue #205) for +# RaincloudConformanceIntegrationTest. +# +# Installs the pinned Raincloud release into a private venv, resolves each corpus +# slug via the raincloud loader (cache -> mirror -> local build), and writes the +# manifest TSV the integration test discovers: +# \t\t +# +# Usage: +# scripts/hydrate-raincloud-corpus.sh [--max-mb N] [slug ...] +# +# With no slugs, hydrates every entry of the conformance matrix +# (integration/src/test/resources/raincloud/expected-status.csv) whose combined +# artifact size fits --max-mb (default 200; use --max-mb 0 for no size cap — the +# full corpus is hundreds of GB and includes multi-hour builds). +# +# Per-slug failures (rotted upstream URL, missing Kaggle/HF credentials, size cap) +# skip the slug and keep going: the test only sees what hydrated. Set +# RAINCLOUD_MIRROR to turn builds into downloads. +set -euo pipefail + +RAINCLOUD_TAG="v0.2.1" +CACHE_ROOT="${RAINCLOUD_HOME:-$HOME/.cache/raincloud}" +MANIFEST="${RAINCLOUD_CORPUS_MANIFEST:-$CACHE_ROOT/corpus-manifest.tsv}" +VENV="$CACHE_ROOT/conformance-venv-$RAINCLOUD_TAG" +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +MATRIX="$REPO_ROOT/integration/src/test/resources/raincloud/expected-status.csv" + +MAX_MB=200 +if [[ "${1:-}" == "--max-mb" ]]; then + MAX_MB="$2" + shift 2 +fi + +if [[ ! -d "$VENV" ]]; then + echo "creating venv for raincloud@$RAINCLOUD_TAG" + python3 -m venv "$VENV" + "$VENV/bin/pip" --quiet install "raincloud[build] @ git+https://github.com/spiraldb/raincloud@$RAINCLOUD_TAG" +fi + +if [[ $# -gt 0 ]]; then + SLUGS=("$@") +else + # not mapfile: macOS ships bash 3.2 + SLUGS=() + while IFS= read -r slug; do + SLUGS+=("$slug") + done < <(grep -v '^#' "$MATRIX" | cut -d, -f1) +fi + +mkdir -p "$(dirname "$MANIFEST")" +printf '%s\n' "${SLUGS[@]}" | MAX_MB="$MAX_MB" MANIFEST="$MANIFEST" "$VENV/bin/python" - <<'PY' +import json +import os +import sys +from importlib import resources + +import raincloud + +max_bytes = int(os.environ["MAX_MB"]) * 1024 * 1024 +snapshot = json.loads(resources.files("raincloud").joinpath("_data/snapshot.json").read_text()) +sizes = { + slug: entry.get("parquet_bytes", 0) + entry.get("vortex_bytes", 0) + for slug, entry in snapshot["slugs"].items() +} + +hydrated, skipped = 0, 0 +with open(os.environ["MANIFEST"], "w") as manifest: + for slug in (line.strip() for line in sys.stdin if line.strip()): + size = sizes.get(slug, 0) + if max_bytes and size > max_bytes: + print(f"skip {slug}: {size / 1e6:.0f} MB exceeds --max-mb") + skipped += 1 + continue + try: + vortex = raincloud.load(slug, format="vortex").path() + parquet = raincloud.load(slug, format="parquet").path() + except Exception as e: # rotted URL, missing credentials, build failure + print(f"skip {slug}: {type(e).__name__}: {e}") + skipped += 1 + continue + manifest.write(f"{slug}\t{vortex}\t{parquet}\n") + hydrated += 1 + print(f"ok {slug}") +print(f"\nhydrated={hydrated} skipped={skipped} manifest={os.environ['MANIFEST']}") +PY From 27194b908551350440a6689543c86b6c9d1b6808 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Mon, 6 Jul 2026 21:21:18 +0200 Subject: [PATCH 2/8] fix: lazy dict decode for I8/U8/I16/U16 value columns (#206) Files from the Python bindings (vortex-data 0.69.0) dict-encode narrow-integer columns; the dict values land as U8/U16 primitives and buildLazyDictPrimitive threw "unsupported ptype for lazy dict". Add DictByteArray and DictShortArray mirroring the existing lazy dict records: raw-signed storage semantics (unsigned widening stays in the ByteArray/ShortArray getInt defaults, keyed on dtype), hoisted code-width switches in materialize/forEach/fold, and the same validateCodes construction guard. F16 remains the only unsupported value ptype. On the Raincloud corpus this flips uci-bike-sharing-dataset to a full value-for-value pass and unblocks the kepler/penguins scans far enough to expose two pre-existing bugs, reclassified in the conformance matrix: nullable ALP columns dropping row validity (#210) and CSV export lacking a NullArray arm (#211). Closes #206 Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 1 + docs/compatibility.md | 9 +- .../resources/raincloud/expected-status.csv | 6 +- .../vortex/reader/array/DictByteArray.java | 150 +++++++++++ .../vortex/reader/array/DictShortArray.java | 150 +++++++++++ .../reader/layout/DictLayoutDecoder.java | 7 + .../reader/array/DictRecordSmokeTest.java | 242 ++++++++++++++++++ 7 files changed, 559 insertions(+), 6 deletions(-) create mode 100644 reader/src/main/java/io/github/dfa1/vortex/reader/array/DictByteArray.java create mode 100644 reader/src/main/java/io/github/dfa1/vortex/reader/array/DictShortArray.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d9bfce8..1ce8491b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - CSV export renders unsigned integer columns (U8–U64) with their unsigned values — high-half values previously printed as two's-complement negatives (uci-wine `magnesium` U8 132 exported as -124), silent corruption found by the Raincloud conformance suite. ([#208](https://github.com/dfa1/vortex-java/issues/208)) - `fastlanes.rle` decodes F64/F32 value pools (`LazyRleDoubleArray`, `LazyRleFloatArray`) — files from the Python bindings RLE-encode double columns with long constant runs, which previously failed to scan with `unsupported ptype F64`. ([#209](https://github.com/dfa1/vortex-java/issues/209)) +- Lazy dict decode now covers I8/U8/I16/U16 value columns (`DictByteArray`, `DictShortArray`) — files from the Python bindings dict-encode narrow-integer columns, which previously failed to scan with `unsupported ptype for lazy dict`. ([#206](https://github.com/dfa1/vortex-java/issues/206)) ### Added diff --git a/docs/compatibility.md b/docs/compatibility.md index 8e6eb26c..a7cb3a6a 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -62,11 +62,14 @@ Per-slug status lives in `integration/src/test/resources/raincloud/expected-stat (`ok` must pass; `gap:` must still fail, so a fix flips the entry in the same change; `untriaged` runs and reports without failing the build). A scheduled workflow (`raincloud-conformance.yml`) hydrates a size-capped subset weekly. Current triage — -4 `ok`, 6 known gaps: lazy dict U8/U16 values -([#206](https://github.com/dfa1/vortex-java/issues/206)), nested struct columns in scan +5 `ok`, 5 known gaps: nested struct columns in scan ([#207](https://github.com/dfa1/vortex-java/issues/207)), unsigned integers rendered signed ([#208](https://github.com/dfa1/vortex-java/issues/208) — silent corruption), RLE over F64 -([#209](https://github.com/dfa1/vortex-java/issues/209)); 237 slugs untriaged. +([#209](https://github.com/dfa1/vortex-java/issues/209)), nullable ALP validity dropped +([#210](https://github.com/dfa1/vortex-java/issues/210) — silent corruption), all-null +columns in CSV export ([#211](https://github.com/dfa1/vortex-java/issues/211)); +237 slugs untriaged. Fixed so far by this suite: lazy dict U8/U16 values +([#206](https://github.com/dfa1/vortex-java/issues/206)). ## Encodings diff --git a/integration/src/test/resources/raincloud/expected-status.csv b/integration/src/test/resources/raincloud/expected-status.csv index 4937e3ad..b25d4729 100644 --- a/integration/src/test/resources/raincloud/expected-status.csv +++ b/integration/src/test/resources/raincloud/expected-status.csv @@ -124,7 +124,7 @@ humaneval,untriaged insurance,untriaged international-football-results-from-1872-to-2017,untriaged iowa-liquor-sales,untriaged -kepler-exoplanet-search-results,gap:206 +kepler-exoplanet-search-results,gap:211 kepler-labelled-time-series-data,untriaged laion-400m,untriaged librispeech-test-clean,untriaged @@ -166,7 +166,7 @@ osmi-mental-health-in-tech-2020,untriaged osmi-mental-health-in-tech-2021,untriaged osmi-mental-health-in-tech-2022,untriaged osmi-mental-health-in-tech-2023,untriaged -palmer-archipelago-antarctica-penguin-data,gap:206 +palmer-archipelago-antarctica-penguin-data,gap:210 peoples-speech-clean-validation,untriaged personal-key-indicators-of-heart-disease,untriaged pleias-synth,untriaged @@ -194,7 +194,7 @@ uci-auto-mpg,untriaged uci-automobile,untriaged uci-bank-marketing,untriaged uci-beijing-multi-site-air-quality,untriaged -uci-bike-sharing-dataset,gap:206 +uci-bike-sharing-dataset,ok uci-breast-cancer,untriaged uci-breast-cancer-wisconsin-diagnostic,untriaged uci-breast-cancer-wisconsin-original,untriaged diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictByteArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictByteArray.java new file mode 100644 index 00000000..aec09337 --- /dev/null +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictByteArray.java @@ -0,0 +1,150 @@ +package io.github.dfa1.vortex.reader.array; + +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.error.VortexException; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SegmentAllocator; +import java.lang.foreign.ValueLayout; +import java.util.function.LongBinaryOperator; + +/// Dict-encoded [ByteArray] view. ADR 0012 shape. +/// +/// Stores `values` (the dictionary pool) and `codes` (one index per +/// row into `values`). Scalar access resolves on demand: +/// `getByte(i) = values.getByte(codes.getCode(i))`. Per ADR 0012, this +/// preserves zero-copy on dict-encoded categorical columns. +/// +/// The `codes` array is typed as [Array] because the codes ptype +/// varies with dictionary size — U8/U16/U32/U64 backed by +/// [ByteArray]/[ShortArray]/[IntArray]/[LongArray]. +/// +/// @param dtype logical element type (matches `values.dtype()`) +/// @param length total logical row count (matches `codes.length()`) +/// @param values dictionary pool — element at code `c` is `values.getByte(c)` +/// @param codes per-row index into `values`; must be one of +/// [ByteArray], [ShortArray], [IntArray], [LongArray] +public record DictByteArray(DType dtype, long length, ByteArray values, Array codes) implements ByteArray { + + /// Builds a [DictByteArray], validating that `codes` is one of the + /// four narrow-int code array types and that its length matches `length`. + /// + /// @param dtype logical element type + /// @param length total logical row count + /// @param values dictionary pool + /// @param codes per-row code array (must be [ByteArray], [ShortArray], + /// [IntArray], or [LongArray]) + /// @return a new [DictByteArray] + /// @throws VortexException if `codes` is not a supported code-array type or + /// its length does not equal `length` + public static DictByteArray of(DType dtype, long length, ByteArray values, Array codes) { + DictArrays.validateCodes(codes, length); + return new DictByteArray(dtype, length, values, codes); + } + + @Override + public byte getByte(long i) { + return values.getByte(DictArrays.readCode(codes, i)); + } + + /// Materializes by gathering one dictionary value per code into a fresh + /// one-byte-per-element segment. The codes switch is hoisted outside the loop so + /// each branch is a uniform gather over a single code width. + /// + /// @param arena allocator for the output segment + /// @return a read-only segment of `length()` gathered bytes + /// @throws VortexException if `codes` is not a supported code-array type + @Override + public MemorySegment materialize(SegmentAllocator arena) { + long n = length; + MemorySegment dst = arena.allocate(n); + ByteArray vals = values; + switch (codes) { + case ByteArray ba -> { + for (long i = 0; i < n; i++) { + dst.set(ValueLayout.JAVA_BYTE, i, vals.getByte(Byte.toUnsignedLong(ba.getByte(i)))); + } + } + case ShortArray sa -> { + for (long i = 0; i < n; i++) { + dst.set(ValueLayout.JAVA_BYTE, i, vals.getByte(Short.toUnsignedLong(sa.getShort(i)))); + } + } + case IntArray ia -> { + for (long i = 0; i < n; i++) { + dst.set(ValueLayout.JAVA_BYTE, i, vals.getByte(Integer.toUnsignedLong(ia.getInt(i)))); + } + } + case LongArray la -> { + for (long i = 0; i < n; i++) { + dst.set(ValueLayout.JAVA_BYTE, i, vals.getByte(la.getLong(i))); + } + } + default -> throw new VortexException("DictByteArray: invalid codes type: " + + codes.getClass().getSimpleName()); + } + return dst.asReadOnly(); + } + + @Override + public void forEachByte(ByteConsumer cons) { + long n = length; + ByteArray vals = values; + switch (codes) { + case ByteArray ba -> { + for (long i = 0; i < n; i++) { + cons.accept(vals.getByte(Byte.toUnsignedLong(ba.getByte(i)))); + } + } + case ShortArray sa -> { + for (long i = 0; i < n; i++) { + cons.accept(vals.getByte(Short.toUnsignedLong(sa.getShort(i)))); + } + } + case IntArray ia -> { + for (long i = 0; i < n; i++) { + cons.accept(vals.getByte(Integer.toUnsignedLong(ia.getInt(i)))); + } + } + case LongArray la -> { + for (long i = 0; i < n; i++) { + cons.accept(vals.getByte(la.getLong(i))); + } + } + default -> throw new VortexException("DictByteArray: invalid codes type: " + + codes.getClass().getSimpleName()); + } + } + + @Override + public long fold(long identity, LongBinaryOperator op) { + long n = length; + ByteArray vals = values; + long result = identity; + switch (codes) { + case ByteArray ba -> { + for (long i = 0; i < n; i++) { + result = op.applyAsLong(result, vals.getByte(Byte.toUnsignedLong(ba.getByte(i)))); + } + } + case ShortArray sa -> { + for (long i = 0; i < n; i++) { + result = op.applyAsLong(result, vals.getByte(Short.toUnsignedLong(sa.getShort(i)))); + } + } + case IntArray ia -> { + for (long i = 0; i < n; i++) { + result = op.applyAsLong(result, vals.getByte(Integer.toUnsignedLong(ia.getInt(i)))); + } + } + case LongArray la -> { + for (long i = 0; i < n; i++) { + result = op.applyAsLong(result, vals.getByte(la.getLong(i))); + } + } + default -> throw new VortexException("DictByteArray: invalid codes type: " + + codes.getClass().getSimpleName()); + } + return result; + } +} diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictShortArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictShortArray.java new file mode 100644 index 00000000..84846e90 --- /dev/null +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictShortArray.java @@ -0,0 +1,150 @@ +package io.github.dfa1.vortex.reader.array; + +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.core.io.VortexFormat; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SegmentAllocator; +import java.util.function.LongBinaryOperator; + +/// Dict-encoded [ShortArray] view. ADR 0012 shape. +/// +/// Stores `values` (the dictionary pool) and `codes` (one index per +/// row into `values`). Scalar access resolves on demand: +/// `getShort(i) = values.getShort(codes.getCode(i))`. Per ADR 0012, this +/// preserves zero-copy on dict-encoded categorical columns. +/// +/// The `codes` array is typed as [Array] because the codes ptype +/// varies with dictionary size — U8/U16/U32/U64 backed by +/// [ByteArray]/[ShortArray]/[IntArray]/[LongArray]. +/// +/// @param dtype logical element type (matches `values.dtype()`) +/// @param length total logical row count (matches `codes.length()`) +/// @param values dictionary pool — element at code `c` is `values.getShort(c)` +/// @param codes per-row index into `values`; must be one of +/// [ByteArray], [ShortArray], [IntArray], [LongArray] +public record DictShortArray(DType dtype, long length, ShortArray values, Array codes) implements ShortArray { + + /// Builds a [DictShortArray], validating that `codes` is one of the + /// four narrow-int code array types and that its length matches `length`. + /// + /// @param dtype logical element type + /// @param length total logical row count + /// @param values dictionary pool + /// @param codes per-row code array (must be [ByteArray], [ShortArray], + /// [IntArray], or [LongArray]) + /// @return a new [DictShortArray] + /// @throws VortexException if `codes` is not a supported code-array type or + /// its length does not equal `length` + public static DictShortArray of(DType dtype, long length, ShortArray values, Array codes) { + DictArrays.validateCodes(codes, length); + return new DictShortArray(dtype, length, values, codes); + } + + @Override + public short getShort(long i) { + return values.getShort(DictArrays.readCode(codes, i)); + } + + /// Materializes by gathering one dictionary value per code into a fresh + /// little-endian `i16` segment. The codes switch is hoisted outside the loop so + /// each branch is a uniform gather over a single code width. + /// + /// @param arena allocator for the output segment + /// @return a read-only little-endian `i16` segment of gathered values + /// @throws VortexException if `codes` is not a supported code-array type + @Override + public MemorySegment materialize(SegmentAllocator arena) { + long n = length; + MemorySegment dst = arena.allocate(n * 2L, 2); + ShortArray vals = values; + switch (codes) { + case ByteArray ba -> { + for (long i = 0; i < n; i++) { + dst.setAtIndex(VortexFormat.LE_SHORT, i, vals.getShort(Byte.toUnsignedLong(ba.getByte(i)))); + } + } + case ShortArray sa -> { + for (long i = 0; i < n; i++) { + dst.setAtIndex(VortexFormat.LE_SHORT, i, vals.getShort(Short.toUnsignedLong(sa.getShort(i)))); + } + } + case IntArray ia -> { + for (long i = 0; i < n; i++) { + dst.setAtIndex(VortexFormat.LE_SHORT, i, vals.getShort(Integer.toUnsignedLong(ia.getInt(i)))); + } + } + case LongArray la -> { + for (long i = 0; i < n; i++) { + dst.setAtIndex(VortexFormat.LE_SHORT, i, vals.getShort(la.getLong(i))); + } + } + default -> throw new VortexException("DictShortArray: invalid codes type: " + + codes.getClass().getSimpleName()); + } + return dst.asReadOnly(); + } + + @Override + public void forEachShort(ShortConsumer cons) { + long n = length; + ShortArray vals = values; + switch (codes) { + case ByteArray ba -> { + for (long i = 0; i < n; i++) { + cons.accept(vals.getShort(Byte.toUnsignedLong(ba.getByte(i)))); + } + } + case ShortArray sa -> { + for (long i = 0; i < n; i++) { + cons.accept(vals.getShort(Short.toUnsignedLong(sa.getShort(i)))); + } + } + case IntArray ia -> { + for (long i = 0; i < n; i++) { + cons.accept(vals.getShort(Integer.toUnsignedLong(ia.getInt(i)))); + } + } + case LongArray la -> { + for (long i = 0; i < n; i++) { + cons.accept(vals.getShort(la.getLong(i))); + } + } + default -> throw new VortexException("DictShortArray: invalid codes type: " + + codes.getClass().getSimpleName()); + } + } + + @Override + public long fold(long identity, LongBinaryOperator op) { + long n = length; + ShortArray vals = values; + long result = identity; + switch (codes) { + case ByteArray ba -> { + for (long i = 0; i < n; i++) { + result = op.applyAsLong(result, vals.getShort(Byte.toUnsignedLong(ba.getByte(i)))); + } + } + case ShortArray sa -> { + for (long i = 0; i < n; i++) { + result = op.applyAsLong(result, vals.getShort(Short.toUnsignedLong(sa.getShort(i)))); + } + } + case IntArray ia -> { + for (long i = 0; i < n; i++) { + result = op.applyAsLong(result, vals.getShort(Integer.toUnsignedLong(ia.getInt(i)))); + } + } + case LongArray la -> { + for (long i = 0; i < n; i++) { + result = op.applyAsLong(result, vals.getShort(la.getLong(i))); + } + } + default -> throw new VortexException("DictShortArray: invalid codes type: " + + codes.getClass().getSimpleName()); + } + return result; + } +} diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java index 4c2608f7..5131f87c 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java @@ -10,14 +10,18 @@ import io.github.dfa1.vortex.core.model.LayoutId; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.ByteArray; +import io.github.dfa1.vortex.reader.array.DictByteArray; import io.github.dfa1.vortex.reader.array.DictDoubleArray; import io.github.dfa1.vortex.reader.array.DictFloatArray; import io.github.dfa1.vortex.reader.array.DictIntArray; import io.github.dfa1.vortex.reader.array.DictLongArray; +import io.github.dfa1.vortex.reader.array.DictShortArray; import io.github.dfa1.vortex.reader.array.DoubleArray; import io.github.dfa1.vortex.reader.array.FloatArray; import io.github.dfa1.vortex.reader.array.IntArray; import io.github.dfa1.vortex.reader.array.LongArray; +import io.github.dfa1.vortex.reader.array.ShortArray; import io.github.dfa1.vortex.reader.array.MaskedArray; import io.github.dfa1.vortex.reader.array.VarBinArray; @@ -132,8 +136,11 @@ private static Array buildLazyDictPrimitive(DType.Primitive dtype, long n, Array return switch (ptype) { case I64, U64 -> DictLongArray.of(dtype, n, (LongArray) valuesData, codesData); case I32, U32 -> DictIntArray.of(dtype, n, (IntArray) valuesData, codesData); + case I16, U16 -> DictShortArray.of(dtype, n, (ShortArray) valuesData, codesData); + case I8, U8 -> DictByteArray.of(dtype, n, (ByteArray) valuesData, codesData); case F64 -> DictDoubleArray.of(dtype, n, (DoubleArray) valuesData, codesData); case F32 -> DictFloatArray.of(dtype, n, (FloatArray) valuesData, codesData); + // F16 has no Array subtype yet default -> throw new VortexException(EncodingId.VORTEX_DICT, "layout: unsupported ptype for lazy dict: " + ptype); }; diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/DictRecordSmokeTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/DictRecordSmokeTest.java index 70561fc1..c1ab3650 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/DictRecordSmokeTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/DictRecordSmokeTest.java @@ -455,6 +455,248 @@ void bulkOps_invalidCodesType_throw() { } } + @Nested + class DictByte { + + @Test + void u8CodesDispatch() { + try (Arena arena = Arena.ofConfined()) { + // Given a byte pool and U8 codes selecting out of order + ByteArray values = byteArray(arena, (byte) 10, (byte) 20, (byte) 30); + ByteArray codes = byteArray(arena, (byte) 0, (byte) 2); + DictByteArray sut = DictByteArray.of(U8, 2, values, codes); + + // When/Then + assertThat(sut.getByte(0)).isEqualTo((byte) 10); + assertThat(sut.getByte(1)).isEqualTo((byte) 30); + } + } + + @Test + void u8DtypeGetIntWidensUnsigned() { + try (Arena arena = Arena.ofConfined()) { + // Given a U8 pool holding 132 (0x84 — negative as a signed byte). This is the + // real-world shape from the Raincloud corpus (uci-wine magnesium): dict-encoded + // U8 values above 127 must widen unsigned through the ByteArray.getInt default. + ByteArray values = byteArray(arena, (byte) 132, (byte) 7); + ByteArray codes = byteArray(arena, (byte) 0, (byte) 1); + DictByteArray sut = DictByteArray.of(U8, 2, values, codes); + + // When + int result = sut.getInt(0); + + // Then — unsigned 132, not -124; the raw storage byte stays signed + assertThat(result).isEqualTo(132); + assertThat(sut.getByte(0)).isEqualTo((byte) -124); + } + } + + @Test + void forEachByteMaterializesInOrder() { + try (Arena arena = Arena.ofConfined()) { + // Given + ByteArray values = byteArray(arena, (byte) 10, (byte) 20); + ByteArray codes = byteArray(arena, (byte) 1, (byte) 0, (byte) 1); + DictByteArray sut = DictByteArray.of(U8, 3, values, codes); + + // When + var result = new ArrayList(); + sut.forEachByte(result::add); + + // Then + assertThat(result).containsExactly((byte) 20, (byte) 10, (byte) 20); + } + } + + @Test + void foldThroughCodes() { + try (Arena arena = Arena.ofConfined()) { + // Given codes = [0, 1, 1] -> values = [1, 5, 5] -> sum 11 + ByteArray values = byteArray(arena, (byte) 1, (byte) 5); + ByteArray codes = byteArray(arena, (byte) 0, (byte) 1, (byte) 1); + DictByteArray sut = DictByteArray.of(U8, 3, values, codes); + + // When + long result = sut.fold(0L, Long::sum); + + // Then + assertThat(result).isEqualTo(11L); + } + } + + @Test + void allCodeTypes_getForEachFoldMaterialize() { + try (Arena arena = Arena.ofConfined()) { + // Given — same selection [2,0,1] expressed as U8/U16/U32/U64 codes + ByteArray values = byteArray(arena, (byte) 100, (byte) 101, (byte) 102); + byte[] expected = {102, 100, 101}; + List codeVariants = List.of( + byteArray(arena, (byte) 2, (byte) 0, (byte) 1), + shortArray(arena, U16, (short) 2, (short) 0, (short) 1), + intArray(arena, U32, 2, 0, 1), + longArray(arena, U64, 2L, 0L, 1L)); + + for (Array codes : codeVariants) { + DictByteArray sut = DictByteArray.of(U8, 3, values, codes); + String label = codes.getClass().getSimpleName(); + + // getter + for (int i = 0; i < 3; i++) { + assertThat(sut.getByte(i)).as(label).isEqualTo(expected[i]); + } + // forEach + var seen = new ArrayList(); + sut.forEachByte(seen::add); + assertThat(seen).as(label).containsExactly((byte) 102, (byte) 100, (byte) 101); + // fold + assertThat(sut.fold(0L, Long::sum)).as(label).isEqualTo(303L); + // materialize + MemorySegment m = sut.materialize(arena); + for (int i = 0; i < 3; i++) { + assertThat(m.get(ValueLayout.JAVA_BYTE, i)).as(label).isEqualTo(expected[i]); + } + } + } + } + + @Test + void bulkOps_invalidCodesType_throw() { + // Given — a record built directly (bypassing of()) with a non-int codes array + try (Arena arena = Arena.ofConfined()) { + ByteArray values = byteArray(arena, (byte) 1); + DictByteArray sut = new DictByteArray(U8, 1, values, floatArray(arena, 0f)); + + // When / Then — every bulk path hits the defensive default arm + assertThatThrownBy(() -> sut.materialize(arena)) + .isInstanceOf(VortexException.class).hasMessageContaining("invalid codes"); + assertThatThrownBy(() -> sut.forEachByte(v -> { })) + .isInstanceOf(VortexException.class); + assertThatThrownBy(() -> sut.fold(0L, Long::sum)) + .isInstanceOf(VortexException.class); + } + } + } + + @Nested + class DictShort { + + @Test + void u8CodesDispatch() { + try (Arena arena = Arena.ofConfined()) { + // Given + ShortArray values = shortArray(arena, U16, (short) 100, (short) 200, (short) 300); + ByteArray codes = byteArray(arena, (byte) 0, (byte) 2); + DictShortArray sut = DictShortArray.of(U16, 2, values, codes); + + // When/Then + assertThat(sut.getShort(0)).isEqualTo((short) 100); + assertThat(sut.getShort(1)).isEqualTo((short) 300); + } + } + + @Test + void u16DtypeGetIntWidensUnsigned() { + try (Arena arena = Arena.ofConfined()) { + // Given a U16 pool holding 40000 (negative as a signed short) — the U16 + // sibling of the uci-wine U8 corner: values above 32767 must widen unsigned. + ShortArray values = shortArray(arena, U16, (short) 40000, (short) 7); + ByteArray codes = byteArray(arena, (byte) 0, (byte) 1); + DictShortArray sut = DictShortArray.of(U16, 2, values, codes); + + // When + int result = sut.getInt(0); + + // Then + assertThat(result).isEqualTo(40000); + } + } + + @Test + void forEachShortMaterializesInOrder() { + try (Arena arena = Arena.ofConfined()) { + // Given + ShortArray values = shortArray(arena, U16, (short) 10, (short) 20); + ByteArray codes = byteArray(arena, (byte) 1, (byte) 0, (byte) 1); + DictShortArray sut = DictShortArray.of(U16, 3, values, codes); + + // When + var result = new ArrayList(); + sut.forEachShort(result::add); + + // Then + assertThat(result).containsExactly((short) 20, (short) 10, (short) 20); + } + } + + @Test + void foldThroughCodes() { + try (Arena arena = Arena.ofConfined()) { + // Given codes = [0, 1, 1] -> values = [1, 5, 5] -> sum 11 + ShortArray values = shortArray(arena, U16, (short) 1, (short) 5); + ByteArray codes = byteArray(arena, (byte) 0, (byte) 1, (byte) 1); + DictShortArray sut = DictShortArray.of(U16, 3, values, codes); + + // When + long result = sut.fold(0L, Long::sum); + + // Then + assertThat(result).isEqualTo(11L); + } + } + + @Test + void allCodeTypes_getForEachFoldMaterialize() { + try (Arena arena = Arena.ofConfined()) { + // Given — same selection [2,0,1] expressed as U8/U16/U32/U64 codes + ShortArray values = shortArray(arena, U16, (short) 100, (short) 200, (short) 300); + short[] expected = {300, 100, 200}; + List codeVariants = List.of( + byteArray(arena, (byte) 2, (byte) 0, (byte) 1), + shortArray(arena, U16, (short) 2, (short) 0, (short) 1), + intArray(arena, U32, 2, 0, 1), + longArray(arena, U64, 2L, 0L, 1L)); + + for (Array codes : codeVariants) { + DictShortArray sut = DictShortArray.of(U16, 3, values, codes); + String label = codes.getClass().getSimpleName(); + + // getter + for (int i = 0; i < 3; i++) { + assertThat(sut.getShort(i)).as(label).isEqualTo(expected[i]); + } + // forEach + var seen = new ArrayList(); + sut.forEachShort(seen::add); + assertThat(seen).as(label).containsExactly((short) 300, (short) 100, (short) 200); + // fold + assertThat(sut.fold(0L, Long::sum)).as(label).isEqualTo(600L); + // materialize + MemorySegment m = sut.materialize(arena); + for (int i = 0; i < 3; i++) { + assertThat(m.getAtIndex(VortexFormat.LE_SHORT, i)).as(label).isEqualTo(expected[i]); + } + } + } + } + + @Test + void bulkOps_invalidCodesType_throw() { + // Given — a record built directly (bypassing of()) with a non-int codes array + try (Arena arena = Arena.ofConfined()) { + ShortArray values = shortArray(arena, U16, (short) 1); + DictShortArray sut = new DictShortArray(U16, 1, values, floatArray(arena, 0f)); + + // When / Then — every bulk path hits the defensive default arm + assertThatThrownBy(() -> sut.materialize(arena)) + .isInstanceOf(VortexException.class).hasMessageContaining("invalid codes"); + assertThatThrownBy(() -> sut.forEachShort(v -> { })) + .isInstanceOf(VortexException.class); + assertThatThrownBy(() -> sut.fold(0L, Long::sum)) + .isInstanceOf(VortexException.class); + } + } + } + @Nested class MaskedValues { From 7bccef29254474aa3f669cad9e47c9aefec13978 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Mon, 6 Jul 2026 21:46:05 +0200 Subject: [PATCH 3/8] fix: propagate row validity through wrapper decoders; NullArray CSV export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Files from the Python bindings (vortex-data 0.69.0) carry row validity deep in the encoding tree, in three representations our decoders flattened away — every one silent corruption (nulls exported as invented values), found by the Raincloud conformance suite (#205): - a trailing bool validity child on fastlanes.bitpacked, reached through vortex.alp / vortex.zigzag / fastlanes.for, which per the Rust ValidityChild contract delegate their validity to the encoded child. AlpEncodingDecoder and ZigZagEncodingDecoder now decode that child as an Array (nullability inherited from the parent dtype), split off a MaskedArray, and re-wrap their result — the pattern FrameOfReferenceEncodingDecoder already used. Bitpacked decodes the trailing validity child at the index its patch shape dictates (0 / 2 / 3, mirroring the Rust vtable deserialize). - dict pools with invalid slots: null rows are codes pointing at an invalid pool entry; slot validity is now gathered through the codes into per-row validity. - dict codes with their own validity child: propagated directly, and combined with pool validity when both are present. Applied to the eager vortex.dict decoder (primitive + utf8 paths) and the lazy dict layout path. Also: CSV export renders all-null (DType.Null) columns as empty fields instead of throwing (kepler has fully-empty columns). Corpus evidence: penguins bill_length/bill_depth and 20k+ kepler cells exported values where the file holds nulls (verified against both the Parquet oracle and the Python bindings reading the same file). After the fix kepler passes value-for-value on all 153 columns and flips to ok in the conformance matrix; penguins' remaining failure is #208 only. Closes #210 Closes #211 Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 2 + .../github/dfa1/vortex/csv/CsvExporter.java | 4 + .../dfa1/vortex/csv/CsvExporterTest.java | 28 +++++ docs/compatibility.md | 13 +- .../resources/raincloud/expected-status.csv | 4 +- .../reader/decode/AlpEncodingDecoder.java | 33 +++-- .../decode/BitpackedEncodingDecoder.java | 41 ++++++- .../reader/decode/DictEncodingDecoder.java | 114 ++++++++++++++++-- .../reader/decode/ZigZagEncodingDecoder.java | 19 ++- .../reader/layout/DictLayoutDecoder.java | 64 ++++++++-- 10 files changed, 290 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ce8491b..914ee95e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - CSV export renders unsigned integer columns (U8–U64) with their unsigned values — high-half values previously printed as two's-complement negatives (uci-wine `magnesium` U8 132 exported as -124), silent corruption found by the Raincloud conformance suite. ([#208](https://github.com/dfa1/vortex-java/issues/208)) - `fastlanes.rle` decodes F64/F32 value pools (`LazyRleDoubleArray`, `LazyRleFloatArray`) — files from the Python bindings RLE-encode double columns with long constant runs, which previously failed to scan with `unsupported ptype F64`. ([#209](https://github.com/dfa1/vortex-java/issues/209)) - Lazy dict decode now covers I8/U8/I16/U16 value columns (`DictByteArray`, `DictShortArray`) — files from the Python bindings dict-encode narrow-integer columns, which previously failed to scan with `unsupported ptype for lazy dict`. ([#206](https://github.com/dfa1/vortex-java/issues/206)) +- CSV export handles all-null (`DType.Null`) columns as empty fields instead of throwing `unsupported array type: NullArray`. ([#211](https://github.com/dfa1/vortex-java/issues/211)) +- Null rows no longer silently decode as values: wrapper decoders now propagate the row validity that files from the Python bindings carry deep in the encoding tree. Three representations were dropped — a trailing validity child on `fastlanes.bitpacked` (reached through `vortex.alp`/`vortex.zigzag`/`fastlanes.for`, which delegate validity to their encoded child per the Rust `ValidityChild` contract), dict pools with invalid slots, and dict codes with their own validity — across both the eager `vortex.dict` decoder and the lazy dict layout path. Found on real data: penguins and kepler exported invented values (`32.1`, `0.0`) for thousands of null cells. ([#210](https://github.com/dfa1/vortex-java/issues/210)) ### Added diff --git a/csv/src/main/java/io/github/dfa1/vortex/csv/CsvExporter.java b/csv/src/main/java/io/github/dfa1/vortex/csv/CsvExporter.java index c366728b..6a48e88b 100644 --- a/csv/src/main/java/io/github/dfa1/vortex/csv/CsvExporter.java +++ b/csv/src/main/java/io/github/dfa1/vortex/csv/CsvExporter.java @@ -13,6 +13,7 @@ import io.github.dfa1.vortex.reader.array.LongArray; import io.github.dfa1.vortex.reader.array.ShortArray; import io.github.dfa1.vortex.reader.array.MaskedArray; +import io.github.dfa1.vortex.reader.array.NullArray; import io.github.dfa1.vortex.reader.array.VarBinArray; import io.github.dfa1.vortex.reader.VortexReader; import io.github.dfa1.vortex.reader.ScanIterator; @@ -159,6 +160,9 @@ private static String cellValue(Array arr, long rowIdx) { // Nullable columns decode as MaskedArray: null rows export as an empty field, valid // rows defer to the inner values array. case MaskedArray ma -> ma.isValid(rowIdx) ? cellValue(ma.inner(), rowIdx) : ""; + // All-null columns (DType.Null) hold only a row count: every cell is an empty + // field, same rule as a MaskedArray null row. + case NullArray ignored -> ""; default -> throw new VortexException( "unsupported array type for CSV export: " + arr.getClass().getSimpleName()); }; diff --git a/csv/src/test/java/io/github/dfa1/vortex/csv/CsvExporterTest.java b/csv/src/test/java/io/github/dfa1/vortex/csv/CsvExporterTest.java index b5672a56..fc7bd650 100644 --- a/csv/src/test/java/io/github/dfa1/vortex/csv/CsvExporterTest.java +++ b/csv/src/test/java/io/github/dfa1/vortex/csv/CsvExporterTest.java @@ -5,7 +5,9 @@ import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.writer.VortexWriter; import io.github.dfa1.vortex.writer.WriteOptions; +import io.github.dfa1.vortex.writer.encode.NullEncodingEncoder; import io.github.dfa1.vortex.writer.encode.NullableData; +import io.github.dfa1.vortex.writer.encode.PrimitiveEncodingEncoder; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -71,6 +73,32 @@ void exportsToWriter(@TempDir Path tmp) throws Exception { assertThat(lines[2]).isEqualTo("2.7"); } + @Test + void exportsAllNullColumnAsEmptyFields(@TempDir Path tmp) throws Exception { + // Given a file mixing a value column with an all-null (DType.Null) column — + // the shape that made real-world exports throw (#211); the long[] carrier for + // the null column only supplies the row count + Path vortex = tmp.resolve("data.vortex"); + DType.Struct schema = new DType.Struct( + List.of(ColumnName.of("id"), ColumnName.of("empty")), + List.of(DType.I64, DType.NULL), + false); + try (FileChannel ch = FileChannel.open(vortex, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + // explicit encoder list: the default cascading set has no DType.Null codec + VortexWriter writer = VortexWriter.create(ch, schema, WriteOptions.defaults(), + List.of(new NullEncodingEncoder(), new PrimitiveEncodingEncoder()))) { + writer.writeChunk(Map.of(ColumnName.of("id"), new long[]{1L, 2L}, ColumnName.of("empty"), new long[2])); + } + Path csv = tmp.resolve("out.csv"); + + // When + CsvExporter.exportCsv(vortex, csv); + + // Then — null cells are empty fields, same as MaskedArray null rows + List result = Files.readAllLines(csv); + assertThat(result).containsExactly("id,empty", "1,", "2,"); + } + @Test void suppressesHeaderWhenConfigured(@TempDir Path tmp) throws Exception { // Given diff --git a/docs/compatibility.md b/docs/compatibility.md index a7cb3a6a..17a2ec05 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -62,14 +62,15 @@ Per-slug status lives in `integration/src/test/resources/raincloud/expected-stat (`ok` must pass; `gap:` must still fail, so a fix flips the entry in the same change; `untriaged` runs and reports without failing the build). A scheduled workflow (`raincloud-conformance.yml`) hydrates a size-capped subset weekly. Current triage — -5 `ok`, 5 known gaps: nested struct columns in scan +6 `ok`, 4 known gaps: nested struct columns in scan ([#207](https://github.com/dfa1/vortex-java/issues/207)), unsigned integers rendered signed ([#208](https://github.com/dfa1/vortex-java/issues/208) — silent corruption), RLE over F64 -([#209](https://github.com/dfa1/vortex-java/issues/209)), nullable ALP validity dropped -([#210](https://github.com/dfa1/vortex-java/issues/210) — silent corruption), all-null -columns in CSV export ([#211](https://github.com/dfa1/vortex-java/issues/211)); -237 slugs untriaged. Fixed so far by this suite: lazy dict U8/U16 values -([#206](https://github.com/dfa1/vortex-java/issues/206)). +([#209](https://github.com/dfa1/vortex-java/issues/209)); 237 slugs untriaged. Fixed so +far by this suite: lazy dict U8/U16 values +([#206](https://github.com/dfa1/vortex-java/issues/206)), row validity dropped by wrapper +decoders ([#210](https://github.com/dfa1/vortex-java/issues/210) — silent corruption), +all-null columns in CSV export +([#211](https://github.com/dfa1/vortex-java/issues/211)). ## Encodings diff --git a/integration/src/test/resources/raincloud/expected-status.csv b/integration/src/test/resources/raincloud/expected-status.csv index b25d4729..ff733401 100644 --- a/integration/src/test/resources/raincloud/expected-status.csv +++ b/integration/src/test/resources/raincloud/expected-status.csv @@ -124,7 +124,7 @@ humaneval,untriaged insurance,untriaged international-football-results-from-1872-to-2017,untriaged iowa-liquor-sales,untriaged -kepler-exoplanet-search-results,gap:211 +kepler-exoplanet-search-results,ok kepler-labelled-time-series-data,untriaged laion-400m,untriaged librispeech-test-clean,untriaged @@ -166,7 +166,7 @@ osmi-mental-health-in-tech-2020,untriaged osmi-mental-health-in-tech-2021,untriaged osmi-mental-health-in-tech-2022,untriaged osmi-mental-health-in-tech-2023,untriaged -palmer-archipelago-antarctica-penguin-data,gap:210 +palmer-archipelago-antarctica-penguin-data,gap:208 peoples-speech-clean-validation,untriaged personal-key-indicators-of-heart-disease,untriaged pleias-synth,untriaged diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java index fbc02523..4a53a282 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java @@ -8,10 +8,12 @@ import io.github.dfa1.vortex.core.proto.ProtoALPMetadata; import io.github.dfa1.vortex.core.proto.ProtoPatchesMetadata; import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.BoolArray; import io.github.dfa1.vortex.reader.array.LazyAlpDoubleArray; import io.github.dfa1.vortex.reader.array.LazyAlpFloatArray; import io.github.dfa1.vortex.reader.array.LazyConstantDoubleArray; import io.github.dfa1.vortex.reader.array.LazyConstantFloatArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; import io.github.dfa1.vortex.reader.array.MaterializedDoubleArray; import io.github.dfa1.vortex.reader.array.MaterializedFloatArray; @@ -55,21 +57,38 @@ public Array decode(DecodeContext ctx) { PType ptype = p.ptype(); long n = ctx.rowCount(); - return switch (ptype) { - case F64 -> decodeF64(ctx, meta, expE, expF, n); - case F32 -> decodeF32(ctx, meta, expE, expF, n); + // Validity mirrors the Rust reference (`ValidityChild`): an ALP array's + // validity IS its encoded child's validity, and the encoded child's dtype + // inherits the ALP dtype's nullability. Decode the child as an Array so a + // nullable primitive surfaces its MaskedArray instead of being flattened to a + // raw segment (which silently dropped nulls — #210). + DType.Primitive encodedDtype = new DType.Primitive( + ptype == PType.F64 ? PType.I64 : PType.I32, p.nullable()); + Array encoded = ctx.decodeChild(0, encodedDtype, n); + BoolArray validity = null; + Array rawEncoded = encoded; + if (encoded instanceof MaskedArray masked) { + rawEncoded = masked.inner(); + validity = masked.validity(); + } + MemorySegment src = ctx.materialize(rawEncoded); + + Array result = switch (ptype) { + case F64 -> decodeF64(ctx, meta, expE, expF, n, src); + case F32 -> decodeF32(ctx, meta, expE, expF, n, src); default -> throw new VortexException(EncodingId.VORTEX_ALP, "unsupported dtype " + ptype); }; + return validity != null ? new MaskedArray(result, validity) : result; } - private static Array decodeF64(DecodeContext ctx, ProtoALPMetadata meta, int expE, int expF, long n) { + private static Array decodeF64(DecodeContext ctx, ProtoALPMetadata meta, int expE, int expF, long n, + MemorySegment src) { // Decode formula mirrors the Rust reference (`ALPFloat::decode_single`): two-step // `encoded * F10[f] * IF10[e]`. A pre-multiplied `scale = F10[f] * IF10[e]` // gives different IEEE rounding for non-trivial `expF`, breaking round-trip with // the encoder's verify step. double df = F10_F64[expF]; double de = IF10_F64[expE]; - MemorySegment src = ctx.decodeChildSegment(0, DType.I64, n); long srcCap = SegmentBroadcast.capacity(src, 8); if (meta.patches() == null) { @@ -96,10 +115,10 @@ private static Array decodeF64(DecodeContext ctx, ProtoALPMetadata meta, int exp return new MaterializedDoubleArray(ctx.dtype(), n, buf.asReadOnly()); } - private static Array decodeF32(DecodeContext ctx, ProtoALPMetadata meta, int expE, int expF, long n) { + private static Array decodeF32(DecodeContext ctx, ProtoALPMetadata meta, int expE, int expF, long n, + MemorySegment src) { float df = F10_F32[expF]; float de = IF10_F32[expE]; - MemorySegment src = ctx.decodeChildSegment(0, DType.I32, n); long srcCap = SegmentBroadcast.capacity(src, 4); if (meta.patches() == null) { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java index 26f495ce..6907ac3d 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java @@ -8,6 +8,8 @@ import io.github.dfa1.vortex.core.proto.ProtoBitPackedMetadata; import io.github.dfa1.vortex.core.proto.ProtoPatchesMetadata; import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.BoolArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; import io.github.dfa1.vortex.reader.array.MaterializedByteArray; import io.github.dfa1.vortex.reader.array.MaterializedIntArray; import io.github.dfa1.vortex.reader.array.MaterializedLongArray; @@ -59,13 +61,50 @@ public Array decode(DecodeContext ctx) { applyPatches(ctx, meta.patches(), output, ptype.byteSize()); } - return switch (ptype) { + Array values = switch (ptype) { case I64, U64 -> new MaterializedLongArray(ctx.dtype(), rowCount, output); case I32, U32 -> new MaterializedIntArray(ctx.dtype(), rowCount, output); case I16, U16 -> new MaterializedShortArray(ctx.dtype(), rowCount, output); case I8, U8 -> new MaterializedByteArray(ctx.dtype(), rowCount, output); default -> throw new VortexException(EncodingId.FASTLANES_BITPACKED, "unsupported ptype " + ptype); }; + return wrapValidity(ctx, meta, values, rowCount); + } + + /// Row validity mirrors the Rust reference (`BitPacked` vtable `deserialize`): the + /// children are `[patch_indices, patch_values, patch_chunk_offsets?]` (present only + /// with patches) followed by an optional trailing bool validity child. Encodings + /// stacked above bitpacked (ALP, FoR, …) delegate their validity here + /// (`ValidityChild`), so dropping this child silently un-nulls rows (#210). + /// + /// @param ctx decode context + /// @param meta bitpacked metadata (patch shape determines the validity index) + /// @param values the unpacked (and patched) values array + /// @param rowCount logical row count + /// @return `values`, wrapped in a [MaskedArray] when a validity child is present + private static Array wrapValidity(DecodeContext ctx, ProtoBitPackedMetadata meta, Array values, long rowCount) { + int validityIdx = validityChildIndex(meta); + int childCount = ctx.node().children().length; + if (childCount == validityIdx) { + return values; + } + if (childCount != validityIdx + 1) { + throw new VortexException(EncodingId.FASTLANES_BITPACKED, + "expected " + validityIdx + " or " + (validityIdx + 1) + " children, got " + childCount); + } + Array va = ctx.decodeChild(validityIdx, DType.BOOL, rowCount); + if (!(va instanceof BoolArray validity)) { + throw new VortexException(EncodingId.FASTLANES_BITPACKED, + "validity child decoded to unexpected type: " + va.getClass().getSimpleName()); + } + return new MaskedArray(values, validity); + } + + private static int validityChildIndex(ProtoBitPackedMetadata meta) { + if (meta.patches() == null) { + return 0; + } + return meta.patches().chunk_offsets_ptype() != null ? 3 : 2; } private static void fastlanesUnpackToSeg( diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java index 7dedf576..10137cac 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java @@ -7,6 +7,9 @@ import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoDictMetadata; import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.BoolArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; +import io.github.dfa1.vortex.reader.array.MaterializedBoolArray; import io.github.dfa1.vortex.reader.array.MaterializedByteArray; import io.github.dfa1.vortex.reader.array.MaterializedDoubleArray; import io.github.dfa1.vortex.reader.array.MaterializedFloatArray; @@ -101,9 +104,28 @@ private static Array decodeRustProto(DecodeContext ctx, MemorySegment metaBuf) { PType valPType = ((DType.Primitive) ctx.dtype()).ptype(); int elemSize = valPType.byteSize(); - DType codesDtype = new DType.Primitive(codePType, false); - MemorySegment codesBuf = ctx.decodeChildSegment(0, codesDtype, rowCount); - MemorySegment valuesBuf = ctx.decodeChildSegment(1, ctx.dtype(), valuesLen); + // Row validity mirrors the Rust reference: a DictArray row is null when its CODE + // is null (codes-side validity child) or when the code points at an invalid pool + // slot (pool-null representation). Both arrive as MaskedArray children and must + // be propagated per row, not flattened away (#210). + DType codesDtype = new DType.Primitive(codePType, ctx.dtype().nullable()); + Array codesArr = ctx.decodeChild(0, codesDtype, rowCount); + BoolArray codesValidity = null; + Array rawCodes = codesArr; + if (codesArr instanceof MaskedArray masked) { + rawCodes = masked.inner(); + codesValidity = masked.validity(); + } + MemorySegment codesBuf = ctx.materialize(rawCodes); + + Array valuesArr = ctx.decodeChild(1, ctx.dtype(), valuesLen); + BoolArray poolValidity = null; + Array rawValues = valuesArr; + if (valuesArr instanceof MaskedArray masked) { + rawValues = masked.inner(); + poolValidity = masked.validity(); + } + MemorySegment valuesBuf = ctx.materialize(rawValues); MemorySegment out = ctx.arena().allocate(rowCount * elemSize); switch (codePType) { @@ -112,7 +134,66 @@ private static Array decodeRustProto(DecodeContext ctx, MemorySegment metaBuf) { case U32 -> expandU32(codesBuf, valuesBuf, out, rowCount, elemSize); default -> throw new VortexException(EncodingId.VORTEX_DICT, "unexpected code type: " + codePType); } - return typedArray(ctx.dtype(), valPType, rowCount, out.asReadOnly()); + Array values = typedArray(ctx.dtype(), valPType, rowCount, out.asReadOnly()); + BoolArray rowValidity = rowValidity(ctx, codesBuf, codePType, codesValidity, poolValidity, rowCount); + return rowValidity == null ? values : new MaskedArray(values, rowValidity); + } + + /// Combines codes-side and pool-side validity into per-row validity: row `i` is + /// valid iff its code is valid and the pool slot the code references is valid. + /// Returns `null` when neither side carries validity (all rows valid), and the + /// codes-side mask unchanged when the pool is all-valid (it is already per-row). + /// Output is bit-packed LSB-first ([MaterializedBoolArray] layout). The broadcast + /// branch mirrors the `expandXxx` loops (undersized codes buffer = ConstantEncoding + /// fan-out) and is split out of the fast path. + /// + /// @param ctx decode context (allocation arena) + /// @param codesBuf decoded raw codes buffer + /// @param codePType unsigned code ptype (U8/U16/U32) + /// @param codesValidity per-row validity from the codes child, or `null` + /// @param poolValidity validity of the values pool, or `null` + /// @param rowCount logical row count + /// @return a bit-packed row validity array of `rowCount` bits, or `null` when all valid + private static BoolArray rowValidity(DecodeContext ctx, MemorySegment codesBuf, PType codePType, + BoolArray codesValidity, BoolArray poolValidity, long rowCount) { + if (poolValidity == null) { + return codesValidity; + } + MemorySegment bits = ctx.arena().allocate((rowCount + 7) >>> 3); + long codesCap = SegmentBroadcast.capacity(codesBuf, codePType.byteSize()); + if (codesCap >= rowCount) { + for (long i = 0; i < rowCount; i++) { + boolean valid = (codesValidity == null || codesValidity.getBoolean(i)) + && poolValidity.getBoolean(readCode(codesBuf, i, codePType)); + if (valid) { + setBit(bits, i); + } + } + } else { + for (long i = 0; i < rowCount; i++) { + boolean valid = (codesValidity == null || codesValidity.getBoolean(i)) + && poolValidity.getBoolean(readCode(codesBuf, i % codesCap, codePType)); + if (valid) { + setBit(bits, i); + } + } + } + return new MaterializedBoolArray(DType.BOOL, rowCount, bits.asReadOnly()); + } + + private static long readCode(MemorySegment codes, long i, PType codePType) { + return switch (codePType) { + case U8 -> Byte.toUnsignedLong(codes.get(ValueLayout.JAVA_BYTE, i)); + case U16 -> Short.toUnsignedLong(codes.getAtIndex(VortexFormat.LE_SHORT, i)); + case U32 -> Integer.toUnsignedLong(codes.getAtIndex(VortexFormat.LE_INT, i)); + default -> throw new VortexException(EncodingId.VORTEX_DICT, "unexpected code type: " + codePType); + }; + } + + private static void setBit(MemorySegment bits, long i) { + long byteIdx = i >>> 3; + byte cur = bits.get(ValueLayout.JAVA_BYTE, byteIdx); + bits.set(ValueLayout.JAVA_BYTE, byteIdx, (byte) (cur | (1 << (i & 7)))); } private static Array decodeUtf8DictLegacy(DecodeContext ctx, MemorySegment meta) { @@ -140,15 +221,32 @@ private static Array decodeUtf8DictProto(DecodeContext ctx, MemorySegment metaBu long dictSize = meta.values_len(); long n = ctx.rowCount(); - DType codesDtype = new DType.Primitive(codePType, false); - MemorySegment codesBuf = ctx.decodeChildSegment(0, codesDtype, n); + // Same two null representations as the primitive path (#210): codes-side + // validity and/or an invalid pool slot referenced by null rows. + DType codesDtype = new DType.Primitive(codePType, ctx.dtype().nullable()); + Array codesArr = ctx.decodeChild(0, codesDtype, n); + BoolArray codesValidity = null; + Array rawCodes = codesArr; + if (codesArr instanceof MaskedArray masked) { + rawCodes = masked.inner(); + codesValidity = masked.validity(); + } + MemorySegment codesBuf = ctx.materialize(rawCodes); - VarBinArray valuesArr = (VarBinArray) ctx.decodeChild(1, ctx.dtype(), dictSize); + Array valuesDecoded = ctx.decodeChild(1, ctx.dtype(), dictSize); + BoolArray poolValidity = null; + if (valuesDecoded instanceof MaskedArray masked) { + valuesDecoded = masked.inner(); + poolValidity = masked.validity(); + } + VarBinArray valuesArr = (VarBinArray) valuesDecoded; VarBinArray.OffsetMode dictValues = VarBinArray.toOffsetMode(valuesArr, ctx.arena()); - return VarBinArray.ofDict(ctx.dtype(), n, + BoolArray rowValidity = rowValidity(ctx, codesBuf, codePType, codesValidity, poolValidity, n); + Array dict = VarBinArray.ofDict(ctx.dtype(), n, dictValues.bytesSegment(), dictValues.offsetsSegment(), PType.I64, codesBuf, codePType); + return rowValidity == null ? dict : new MaskedArray(dict, rowValidity); } static void expandU8(MemorySegment codes, MemorySegment values, MemorySegment out, long rowCount, int elemSize) { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoder.java index fe05b257..f39fbaab 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoder.java @@ -6,6 +6,7 @@ import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.BoolArray; import io.github.dfa1.vortex.reader.array.LazyConstantByteArray; import io.github.dfa1.vortex.reader.array.LazyConstantIntArray; import io.github.dfa1.vortex.reader.array.LazyConstantLongArray; @@ -14,6 +15,7 @@ import io.github.dfa1.vortex.reader.array.LazyZigZagIntArray; import io.github.dfa1.vortex.reader.array.LazyZigZagLongArray; import io.github.dfa1.vortex.reader.array.LazyZigZagShortArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; @@ -35,10 +37,25 @@ public Array decode(DecodeContext ctx) { PType unsigned = toUnsigned(signed); long n = ctx.rowCount(); - MemorySegment src = ctx.decodeChildSegment(0, new DType.Primitive(unsigned, false), n); + // Validity mirrors the Rust reference (`ValidityChild`): a zigzag array's + // validity IS its encoded child's, and the child dtype inherits the nullability. + // Decode as an Array so a masked child is split rather than flattened (#210). + Array encoded = ctx.decodeChild(0, new DType.Primitive(unsigned, p.nullable()), n); + BoolArray validity = null; + Array rawEncoded = encoded; + if (encoded instanceof MaskedArray masked) { + rawEncoded = masked.inner(); + validity = masked.validity(); + } + MemorySegment src = ctx.materialize(rawEncoded); int elemBytes = signed.byteSize(); long srcCap = SegmentBroadcast.capacity(src, elemBytes); + Array result = decodeUnwrapped(ctx, signed, n, src, srcCap); + return validity != null ? new MaskedArray(result, validity) : result; + } + + private static Array decodeUnwrapped(DecodeContext ctx, PType signed, long n, MemorySegment src, long srcCap) { if (srcCap < n) { // Broadcast: single encoded value maps to all n rows — decode once, return constant return switch (signed) { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java index 5131f87c..a454e251 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java @@ -10,6 +10,7 @@ import io.github.dfa1.vortex.core.model.LayoutId; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.BoolArray; import io.github.dfa1.vortex.reader.array.ByteArray; import io.github.dfa1.vortex.reader.array.DictByteArray; import io.github.dfa1.vortex.reader.array.DictDoubleArray; @@ -23,6 +24,7 @@ import io.github.dfa1.vortex.reader.array.LongArray; import io.github.dfa1.vortex.reader.array.ShortArray; import io.github.dfa1.vortex.reader.array.MaskedArray; +import io.github.dfa1.vortex.reader.array.MaterializedBoolArray; import io.github.dfa1.vortex.reader.array.VarBinArray; import java.lang.foreign.MemorySegment; @@ -87,7 +89,7 @@ public Array decode(LayoutDecodeContext ctx, Layout dictLayout, DType dtype) { // mmap-bounded. Validate by inspecting the underlying segment without forcing // materialization of non-segment-backed codes (lazy variants). validateDictCodesCapacity(codes, codesPType, n); - return buildLazyDictPrimitive(pDtype, n, values, codes); + return buildLazyDictPrimitive(pDtype, n, values, codes, arena); } // Non-Utf8, non-Primitive dict — e.g. extension types backed by VarBin. Fall through // to the existing string expansion for compatibility. @@ -120,20 +122,28 @@ private static void validateDictCodesCapacity(Array codes, PType codesPType, lon } } - /// Builds the matching `DictXxxArray` for a primitive dictionary, unwrapping - /// any [MaskedArray] layer on either side — dictionary lookups are keyed by code - /// so value-side validity is meaningless at this layer. + /// Builds the matching `DictXxxArray` for a primitive dictionary. + /// + /// Row validity mirrors the Rust reference (#210): a dict row is null when its CODE + /// is null (codes child arrives as a [MaskedArray]) or when the code points at an + /// invalid pool slot (nullable values pool arrives as a [MaskedArray]). Both masks + /// must be propagated to per-row validity — dropping either silently un-nulls rows. /// /// @param dtype primitive logical type of dict values /// @param n total logical row count /// @param values dictionary values /// @param codes per-row codes into `values` - /// @return a lazy `DictXxxArray` matching the value ptype - private static Array buildLazyDictPrimitive(DType.Primitive dtype, long n, Array values, Array codes) { + /// @param arena allocator for the gathered row-validity bitmap + /// @return a lazy `DictXxxArray` matching the value ptype, wrapped in a + /// [MaskedArray] when either side carries validity + private static Array buildLazyDictPrimitive(DType.Primitive dtype, long n, Array values, Array codes, + SegmentAllocator arena) { + BoolArray poolValidity = values instanceof MaskedArray mv ? mv.validity() : null; Array valuesData = values instanceof MaskedArray mv ? mv.inner() : values; + BoolArray codesValidity = codes instanceof MaskedArray mc ? mc.validity() : null; Array codesData = codes instanceof MaskedArray mc ? mc.inner() : codes; PType ptype = dtype.ptype(); - return switch (ptype) { + Array dict = switch (ptype) { case I64, U64 -> DictLongArray.of(dtype, n, (LongArray) valuesData, codesData); case I32, U32 -> DictIntArray.of(dtype, n, (IntArray) valuesData, codesData); case I16, U16 -> DictShortArray.of(dtype, n, (ShortArray) valuesData, codesData); @@ -144,6 +154,46 @@ private static Array buildLazyDictPrimitive(DType.Primitive dtype, long n, Array default -> throw new VortexException(EncodingId.VORTEX_DICT, "layout: unsupported ptype for lazy dict: " + ptype); }; + if (poolValidity == null && codesValidity == null) { + return dict; + } + if (poolValidity == null) { + return new MaskedArray(dict, codesValidity); + } + return new MaskedArray(dict, gatherRowValidity(codesData, codesValidity, poolValidity, n, arena)); + } + + /// Combines codes-side and pool-side validity per row: row `i` is valid iff its code + /// is valid and the pool slot the code references is valid. Bit-packed LSB-first + /// ([MaterializedBoolArray] layout). + /// + /// @param codes raw per-row codes + /// @param codesValidity per-row validity from the codes side, or `null` + /// @param poolValidity validity of the values pool + /// @param n logical row count + /// @param arena allocator for the bitmap + /// @return a bit-packed row validity array of `n` bits + private static BoolArray gatherRowValidity(Array codes, BoolArray codesValidity, BoolArray poolValidity, + long n, SegmentAllocator arena) { + MemorySegment bits = arena.allocate((n + 7) >>> 3); + for (long i = 0; i < n; i++) { + long code = switch (codes) { + case ByteArray ba -> Byte.toUnsignedLong(ba.getByte(i)); + case ShortArray sa -> Short.toUnsignedLong(sa.getShort(i)); + case IntArray ia -> Integer.toUnsignedLong(ia.getInt(i)); + case LongArray la -> la.getLong(i); + default -> throw new VortexException(EncodingId.VORTEX_DICT, + "layout: invalid codes type: " + codes.getClass().getSimpleName()); + }; + boolean valid = (codesValidity == null || codesValidity.getBoolean(i)) + && poolValidity.getBoolean(code); + if (valid) { + long byteIdx = i >>> 3; + byte cur = bits.get(ValueLayout.JAVA_BYTE, byteIdx); + bits.set(ValueLayout.JAVA_BYTE, byteIdx, (byte) (cur | (1 << (i & 7)))); + } + } + return new MaterializedBoolArray(DType.BOOL, n, bits.asReadOnly()); } private static PType readDictLayoutCodesPType(MemorySegment rawMeta) { From 7640a823be1c01ebbf7fdd6f69fd707d23556e3a Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Mon, 6 Jul 2026 21:59:07 +0200 Subject: [PATCH 4/8] =?UTF-8?q?test:=20triage=20round=202=20=E2=80=94=2010?= =?UTF-8?q?=20more=20corpus=20slugs=20(13=20ok=20total);=20gap=20catch=20w?= =?UTF-8?q?idened?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Classifications from the round-2 sweep (all verified against the parquet oracle with the fixes on this branch): uci-adult, uci-abalone, uci-mushroom, uci-forest-fires, uci-dry-bean-dataset, insurance, international-football ok; uci-auto-mpg + uci-heart-disease gap:208 (unsigned U8 columns); uci-magic-gamma-telescope gap:215 (new finding: narrow dict-offset ptypes IOOBE in VarBinArray.DictMode). assertStillFails now accepts any RuntimeException as a reproducing gap: a gap may surface as IndexOutOfBoundsException before its decoder gains proper VortexException bounds guards (#215 does exactly that). Co-Authored-By: Claude Opus 4.8 --- docs/compatibility.md | 9 +++++---- .../RaincloudConformanceIntegrationTest.java | 9 +++++---- .../resources/raincloud/expected-status.csv | 20 +++++++++---------- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/docs/compatibility.md b/docs/compatibility.md index 17a2ec05..ad1a780b 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -62,11 +62,12 @@ Per-slug status lives in `integration/src/test/resources/raincloud/expected-stat (`ok` must pass; `gap:` must still fail, so a fix flips the entry in the same change; `untriaged` runs and reports without failing the build). A scheduled workflow (`raincloud-conformance.yml`) hydrates a size-capped subset weekly. Current triage — -6 `ok`, 4 known gaps: nested struct columns in scan +13 `ok`, 7 known gaps: nested struct columns in scan ([#207](https://github.com/dfa1/vortex-java/issues/207)), unsigned integers rendered signed -([#208](https://github.com/dfa1/vortex-java/issues/208) — silent corruption), RLE over F64 -([#209](https://github.com/dfa1/vortex-java/issues/209)); 237 slugs untriaged. Fixed so -far by this suite: lazy dict U8/U16 values +([#208](https://github.com/dfa1/vortex-java/issues/208) — silent corruption, 4 datasets), +RLE over F64 ([#209](https://github.com/dfa1/vortex-java/issues/209)), narrow dict-offset +ptypes in string dicts ([#215](https://github.com/dfa1/vortex-java/issues/215)); +227 slugs untriaged. Fixed so far by this suite: lazy dict U8/U16 values ([#206](https://github.com/dfa1/vortex-java/issues/206)), row validity dropped by wrapper decoders ([#210](https://github.com/dfa1/vortex-java/issues/210) — silent corruption), all-null columns in CSV export diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java index 21b89613..87374ceb 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java @@ -6,7 +6,6 @@ import dev.hardwood.reader.ParquetFileReader; import dev.hardwood.reader.RowReader; import dev.hardwood.schema.ColumnSchema; -import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.csv.CsvExporter; import io.github.dfa1.vortex.csv.ExportOptions; import org.junit.jupiter.api.DynamicTest; @@ -109,12 +108,14 @@ private static void reportUntriaged(Path vortex, Path parquet) { } private static void assertStillFails(Path vortex, Path parquet, String status) { - // Given / When — a decode gap still reproduces when the export itself throws; - // no oracle needed (the oracle may not even read this parquet, e.g. nested columns) + // Given / When — a decode gap still reproduces when the export itself throws; any + // RuntimeException counts (a gap may surface as IndexOutOfBoundsException before its + // decoder gains proper VortexException bounds guards, e.g. #215). No oracle needed + // (the oracle may not even read this parquet, e.g. nested columns). List result; try { result = exportVortex(vortex); - } catch (VortexException e) { + } catch (RuntimeException e) { return; } catch (IOException e) { throw new UncheckedIOException(e); diff --git a/integration/src/test/resources/raincloud/expected-status.csv b/integration/src/test/resources/raincloud/expected-status.csv index ff733401..3831ac5b 100644 --- a/integration/src/test/resources/raincloud/expected-status.csv +++ b/integration/src/test/resources/raincloud/expected-status.csv @@ -121,8 +121,8 @@ hotel-booking-demand,untriaged hotpotqa-fullwiki,untriaged housing-prices-dataset,untriaged humaneval,untriaged -insurance,untriaged -international-football-results-from-1872-to-2017,untriaged +insurance,ok +international-football-results-from-1872-to-2017,ok iowa-liquor-sales,untriaged kepler-exoplanet-search-results,ok kepler-labelled-time-series-data,untriaged @@ -186,11 +186,11 @@ tinystories,untriaged titanic-dataset,untriaged truthfulqa-mc,untriaged uber-pickups-in-new-york-city,untriaged -uci-abalone,untriaged -uci-adult,untriaged +uci-abalone,ok +uci-adult,ok uci-ai4i-2020-predictive-maintenance,untriaged uci-air-quality,untriaged -uci-auto-mpg,untriaged +uci-auto-mpg,gap:208 uci-automobile,untriaged uci-bank-marketing,untriaged uci-beijing-multi-site-air-quality,untriaged @@ -207,18 +207,18 @@ uci-credit-approval,untriaged uci-default-of-credit-card-clients,untriaged uci-diabetes,untriaged uci-diabetes-130-us-hospitals,untriaged -uci-dry-bean-dataset,untriaged +uci-dry-bean-dataset,ok uci-electricityloaddiagrams20112014,untriaged uci-energy-efficiency,untriaged uci-estimation-of-obesity-levels,ok -uci-forest-fires,untriaged -uci-heart-disease,untriaged +uci-forest-fires,ok +uci-heart-disease,gap:208 uci-heart-failure-clinical-records,untriaged uci-human-activity-recognition-using-smartphones,untriaged uci-individual-household-electric-power-consumption,untriaged uci-iris,ok -uci-magic-gamma-telescope,untriaged -uci-mushroom,untriaged +uci-magic-gamma-telescope,gap:215 +uci-mushroom,ok uci-online-retail,untriaged uci-online-retail-ii,untriaged uci-online-shoppers-purchasing-intention,untriaged From 804a0c8c564c5eb0cf67daf15745eac7efd790a1 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Mon, 6 Jul 2026 22:07:08 +0200 Subject: [PATCH 5/8] test: flip four #208-blocked slugs to ok after rebase onto its merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit uci-wine, uci-auto-mpg, uci-heart-disease, and the penguins dataset now pass value-for-value with #208 merged beneath this branch — the suite's gap assertions forced the flips (21/21 green over 20 hydrated slugs; matrix now 17 ok / 3 gaps / 227 untriaged). Co-Authored-By: Claude Opus 4.8 --- .../src/test/resources/raincloud/expected-status.csv | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration/src/test/resources/raincloud/expected-status.csv b/integration/src/test/resources/raincloud/expected-status.csv index 3831ac5b..ea472707 100644 --- a/integration/src/test/resources/raincloud/expected-status.csv +++ b/integration/src/test/resources/raincloud/expected-status.csv @@ -166,7 +166,7 @@ osmi-mental-health-in-tech-2020,untriaged osmi-mental-health-in-tech-2021,untriaged osmi-mental-health-in-tech-2022,untriaged osmi-mental-health-in-tech-2023,untriaged -palmer-archipelago-antarctica-penguin-data,gap:208 +palmer-archipelago-antarctica-penguin-data,ok peoples-speech-clean-validation,untriaged personal-key-indicators-of-heart-disease,untriaged pleias-synth,untriaged @@ -190,7 +190,7 @@ uci-abalone,ok uci-adult,ok uci-ai4i-2020-predictive-maintenance,untriaged uci-air-quality,untriaged -uci-auto-mpg,gap:208 +uci-auto-mpg,ok uci-automobile,untriaged uci-bank-marketing,untriaged uci-beijing-multi-site-air-quality,untriaged @@ -212,7 +212,7 @@ uci-electricityloaddiagrams20112014,untriaged uci-energy-efficiency,untriaged uci-estimation-of-obesity-levels,ok uci-forest-fires,ok -uci-heart-disease,gap:208 +uci-heart-disease,ok uci-heart-failure-clinical-records,untriaged uci-human-activity-recognition-using-smartphones,untriaged uci-individual-household-electric-power-consumption,untriaged @@ -235,7 +235,7 @@ uci-statlog-german-credit-data,untriaged uci-student-performance,untriaged uci-thyroid-disease,untriaged uci-wholesale-customers,untriaged -uci-wine,gap:208 +uci-wine,ok uci-wine-quality,ok ufcdata,untriaged uk-price-paid,untriaged From 3a21468450bb7310046bc75e65e51fb59e615000 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Mon, 6 Jul 2026 22:09:51 +0200 Subject: [PATCH 6/8] review: narrow gap catch, guard untrusted pool codes, harden hydrate script Applied from the PR review: assertStillFails now catches only VortexException plus (narrowly, tied to #215) IndexOutOfBoundsException instead of blanket RuntimeException; pool-validity lookups guard out-of-range codes with VortexException per the untrusted-input rule; the hydrate script rejects a valueless --max-mb, misplaced flags, and an empty slug list (bash 3.2 + set -u would otherwise die cryptically); oracle-abort asymmetry documented on oracleLines. Co-Authored-By: Claude Opus 4.8 --- .../RaincloudConformanceIntegrationTest.java | 15 ++++++++++----- .../reader/decode/DictEncodingDecoder.java | 15 +++++++++++++-- .../vortex/reader/layout/DictLayoutDecoder.java | 7 +++++++ scripts/hydrate-raincloud-corpus.sh | 17 +++++++++++++++++ 4 files changed, 47 insertions(+), 7 deletions(-) diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java index 87374ceb..29c7ce5a 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java @@ -6,6 +6,7 @@ import dev.hardwood.reader.ParquetFileReader; import dev.hardwood.reader.RowReader; import dev.hardwood.schema.ColumnSchema; +import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.csv.CsvExporter; import io.github.dfa1.vortex.csv.ExportOptions; import org.junit.jupiter.api.DynamicTest; @@ -108,14 +109,16 @@ private static void reportUntriaged(Path vortex, Path parquet) { } private static void assertStillFails(Path vortex, Path parquet, String status) { - // Given / When — a decode gap still reproduces when the export itself throws; any - // RuntimeException counts (a gap may surface as IndexOutOfBoundsException before its - // decoder gains proper VortexException bounds guards, e.g. #215). No oracle needed + // Given / When — a decode gap still reproduces when the export itself throws. + // Only VortexException (the contractual untrusted-input failure) and, narrowly, + // IndexOutOfBoundsException count: the latter is itself a bounds-guard bug (#215 + // throws it today) and this arm dies with that fix — a blanket RuntimeException + // catch would green unrelated regressions (NPEs, ...). No oracle needed here // (the oracle may not even read this parquet, e.g. nested columns). List result; try { result = exportVortex(vortex); - } catch (RuntimeException e) { + } catch (VortexException | IndexOutOfBoundsException e) { return; } catch (IOException e) { throw new UncheckedIOException(e); @@ -145,7 +148,9 @@ private static List exportVortex(Path vortex) throws IOException { /// Reads the parquet sibling through hardwood; an oracle-side failure (nested /// columns, unsupported physical type) aborts the slug rather than failing it — - /// it says nothing about vortex-java. + /// it says nothing about vortex-java. Deliberate asymmetry: an `ok` slug whose + /// parquet the oracle cannot read stops being verified (visibly, as skipped) — + /// widen the oracle rather than let unverifiable entries fail the build. private static List oracleLines(Path parquet) { StringWriter out = new StringWriter(); try { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java index 10137cac..7789eb82 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java @@ -164,7 +164,7 @@ private static BoolArray rowValidity(DecodeContext ctx, MemorySegment codesBuf, if (codesCap >= rowCount) { for (long i = 0; i < rowCount; i++) { boolean valid = (codesValidity == null || codesValidity.getBoolean(i)) - && poolValidity.getBoolean(readCode(codesBuf, i, codePType)); + && poolValid(poolValidity, readCode(codesBuf, i, codePType)); if (valid) { setBit(bits, i); } @@ -172,7 +172,7 @@ private static BoolArray rowValidity(DecodeContext ctx, MemorySegment codesBuf, } else { for (long i = 0; i < rowCount; i++) { boolean valid = (codesValidity == null || codesValidity.getBoolean(i)) - && poolValidity.getBoolean(readCode(codesBuf, i % codesCap, codePType)); + && poolValid(poolValidity, readCode(codesBuf, i % codesCap, codePType)); if (valid) { setBit(bits, i); } @@ -181,6 +181,17 @@ private static BoolArray rowValidity(DecodeContext ctx, MemorySegment codesBuf, return new MaterializedBoolArray(DType.BOOL, rowCount, bits.asReadOnly()); } + /// Untrusted-input guard: a malformed file may carry codes outside the pool, and the + /// validity bitmap lookup must fail as [VortexException], never as a raw JDK + /// IndexOutOfBoundsException. + private static boolean poolValid(BoolArray poolValidity, long code) { + if (code >= poolValidity.length()) { + throw new VortexException(EncodingId.VORTEX_DICT, + "code " + code + " out of range for pool validity of length " + poolValidity.length()); + } + return poolValidity.getBoolean(code); + } + private static long readCode(MemorySegment codes, long i, PType codePType) { return switch (codePType) { case U8 -> Byte.toUnsignedLong(codes.get(ValueLayout.JAVA_BYTE, i)); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java index a454e251..c9b13390 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java @@ -185,6 +185,13 @@ private static BoolArray gatherRowValidity(Array codes, BoolArray codesValidity, default -> throw new VortexException(EncodingId.VORTEX_DICT, "layout: invalid codes type: " + codes.getClass().getSimpleName()); }; + // Untrusted-input guard: codes outside the pool must fail as VortexException, + // never as a raw JDK IndexOutOfBoundsException. + if (code >= poolValidity.length()) { + throw new VortexException(EncodingId.VORTEX_DICT, + "layout: code " + code + " out of range for pool validity of length " + + poolValidity.length()); + } boolean valid = (codesValidity == null || codesValidity.getBoolean(i)) && poolValidity.getBoolean(code); if (valid) { diff --git a/scripts/hydrate-raincloud-corpus.sh b/scripts/hydrate-raincloud-corpus.sh index 29cdf61a..5b11e76c 100755 --- a/scripts/hydrate-raincloud-corpus.sh +++ b/scripts/hydrate-raincloud-corpus.sh @@ -29,9 +29,20 @@ MATRIX="$REPO_ROOT/integration/src/test/resources/raincloud/expected-status.csv" MAX_MB=200 if [[ "${1:-}" == "--max-mb" ]]; then + if [[ -z "${2:-}" ]]; then + echo "error: --max-mb needs a value" >&2 + exit 2 + fi MAX_MB="$2" shift 2 fi +# after the flag: any remaining --max-mb is misplaced, not a slug +for arg in "$@"; do + if [[ "$arg" == --* ]]; then + echo "error: flags must come before slugs: $arg" >&2 + exit 2 + fi +done if [[ ! -d "$VENV" ]]; then echo "creating venv for raincloud@$RAINCLOUD_TAG" @@ -49,6 +60,12 @@ else done < <(grep -v '^#' "$MATRIX" | cut -d, -f1) fi +# bash 3.2 + set -u: expanding an empty array is an unbound-variable error +if [ ${#SLUGS[@]} -eq 0 ]; then + echo "error: no slugs to hydrate (empty matrix?)" >&2 + exit 2 +fi + mkdir -p "$(dirname "$MANIFEST")" printf '%s\n' "${SLUGS[@]}" | MAX_MB="$MAX_MB" MANIFEST="$MANIFEST" "$VENV/bin/python" - <<'PY' import json From bd1639e4949b2db2f361150b743ae1ad3267b5cc Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Mon, 6 Jul 2026 22:18:01 +0200 Subject: [PATCH 7/8] test: flip seoul to ok after #209 merge; refresh triage counts Co-Authored-By: Claude Opus 4.8 --- docs/compatibility.md | 12 ++++++------ .../src/test/resources/raincloud/expected-status.csv | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/compatibility.md b/docs/compatibility.md index ad1a780b..783e1136 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -62,13 +62,13 @@ Per-slug status lives in `integration/src/test/resources/raincloud/expected-stat (`ok` must pass; `gap:` must still fail, so a fix flips the entry in the same change; `untriaged` runs and reports without failing the build). A scheduled workflow (`raincloud-conformance.yml`) hydrates a size-capped subset weekly. Current triage — -13 `ok`, 7 known gaps: nested struct columns in scan -([#207](https://github.com/dfa1/vortex-java/issues/207)), unsigned integers rendered signed -([#208](https://github.com/dfa1/vortex-java/issues/208) — silent corruption, 4 datasets), -RLE over F64 ([#209](https://github.com/dfa1/vortex-java/issues/209)), narrow dict-offset -ptypes in string dicts ([#215](https://github.com/dfa1/vortex-java/issues/215)); +18 `ok`, 2 known gaps: nested struct columns in scan +([#207](https://github.com/dfa1/vortex-java/issues/207)), narrow dict-offset ptypes in +string dicts ([#215](https://github.com/dfa1/vortex-java/issues/215)); 227 slugs untriaged. Fixed so far by this suite: lazy dict U8/U16 values -([#206](https://github.com/dfa1/vortex-java/issues/206)), row validity dropped by wrapper +([#206](https://github.com/dfa1/vortex-java/issues/206)), unsigned integers rendered signed +([#208](https://github.com/dfa1/vortex-java/issues/208) — silent corruption), RLE over F64 +([#209](https://github.com/dfa1/vortex-java/issues/209)), row validity dropped by wrapper decoders ([#210](https://github.com/dfa1/vortex-java/issues/210) — silent corruption), all-null columns in CSV export ([#211](https://github.com/dfa1/vortex-java/issues/211)). diff --git a/integration/src/test/resources/raincloud/expected-status.csv b/integration/src/test/resources/raincloud/expected-status.csv index ea472707..337115a9 100644 --- a/integration/src/test/resources/raincloud/expected-status.csv +++ b/integration/src/test/resources/raincloud/expected-status.csv @@ -228,7 +228,7 @@ uci-phishing-websites,untriaged uci-predict-students-dropout-and-academic-success,untriaged uci-real-estate-valuation-data-set,untriaged uci-seeds,ok -uci-seoul-bike-sharing-demand,gap:209 +uci-seoul-bike-sharing-demand,ok uci-sms-spam-collection,untriaged uci-spambase,untriaged uci-statlog-german-credit-data,untriaged From d4aeed45024a9a914ce946a4d4330ee34f860ee5 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Mon, 6 Jul 2026 22:23:55 +0200 Subject: [PATCH 8/8] test: deterministic coverage for validity propagation (#210 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The silent-corruption fix shipped with only the hydration-gated conformance suite covering it. These CI-runnable unit tests pin each validity-propagation seam directly through the decoder harness. - BitpackedEncodingDecoderTest (new): pins validityChildIndex — trailing bool validity at index 0 (no patches), 2 (patches, no chunk offsets), 3 (patches with chunk offsets); no-validity plain array; unexpected child count throws with the expected counts. bit_width=0 constant-residual shape isolates index selection from the packed unpack. - DictEncodingDecoderTest.RowValidity: decodeRustProto's three null representations — pool-null (koi_gmag), codes-null (koi_smet_err2), both (AND semantics), and an out-of-range code guarded as VortexException. - Alp/ZigZag MaskedChildPropagation: masked encoded child surfaces as a MaskedArray with nulls preserved and values decoded (ALP also over the LazyConstant broadcast arm); plain child stays unmasked. - DictLayoutDecoderTest (new): buildLazyDictPrimitive pool/codes gather driven through the public decode() seam with a stub LayoutDecodeContext. Co-Authored-By: Claude Opus 4.8 --- .../reader/decode/AlpEncodingDecoderTest.java | 92 +++++++- .../decode/BitpackedEncodingDecoderTest.java | 211 ++++++++++++++++++ .../decode/DictEncodingDecoderTest.java | 142 +++++++++++- .../decode/ZigZagEncodingDecoderTest.java | 63 +++++- .../reader/layout/DictLayoutDecoderTest.java | 187 ++++++++++++++++ 5 files changed, 692 insertions(+), 3 deletions(-) create mode 100644 reader/src/test/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoderTest.java create mode 100644 reader/src/test/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoderTest.java diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java index 7419d15a..9237af35 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java @@ -5,8 +5,11 @@ import io.github.dfa1.vortex.core.proto.ProtoALPMetadata; import io.github.dfa1.vortex.core.proto.ProtoPatchesMetadata; import io.github.dfa1.vortex.reader.ReadRegistry; +import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.DoubleArray; import io.github.dfa1.vortex.reader.array.FloatArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import java.lang.foreign.Arena; @@ -21,7 +24,8 @@ class AlpEncodingDecoderTest { private static final AlpEncodingDecoder SUT = new AlpEncodingDecoder(); - private static final ReadRegistry REGISTRY = TestRegistry.ofDecoders(SUT, new PrimitiveEncodingDecoder()); + private static final ReadRegistry REGISTRY = TestRegistry.ofDecoders( + SUT, new PrimitiveEncodingDecoder(), new BoolEncodingDecoder()); private static final DType F64 = DType.F64; private static final DType F32 = DType.F32; @@ -161,4 +165,90 @@ void decode_patches_nonUnsignedIndexPtype_throws() { // When / Then assertThatThrownBy(() -> SUT.decode(ctx)).hasMessageContaining("non-unsigned patch index ptype"); } + + /// An ALP array's validity IS its encoded child's (`ValidityChild`, #210): a nullable + /// encoded primitive child surfaces as a [MaskedArray], and that mask must ride through both the + /// per-row lazy path and the single-value constant-broadcast path rather than being flattened. + @Nested + class MaskedChildPropagation { + + @Test + void maskedEncodedChild_lazyPath_propagatesNullsAndDecodesValues() { + // Given — encoded I64 [100,200,300] with row 1 null; exp_e=2 scales by 0.01 + ArrayNode validityNode = new ArrayNode(EncodingId.VORTEX_BOOL, null, new ArrayNode[0], new int[]{1}); + ArrayNode enc = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, + new ArrayNode[]{validityNode}, new int[]{0}); + byte[] meta = new ProtoALPMetadata(2, 0, null).encode(); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ALP, MemorySegment.ofArray(meta), + new ArrayNode[]{enc}, new int[0]); + MemorySegment[] segs = {leLongs(100L, 200L, 300L), boolBitmap(true, false, true)}; + DecodeContext ctx = new DecodeContext(node, F64, 3, segs, REGISTRY, Arena.ofAuto()); + + // When + Array result = SUT.decode(ctx); + + // Then — nulls survive and valid rows decode through the lazy ALP path + MaskedArray masked = (MaskedArray) result; + assertThat(masked.isValid(0)).isTrue(); + assertThat(masked.isValid(1)).isFalse(); + assertThat(masked.isValid(2)).isTrue(); + DoubleArray inner = (DoubleArray) masked.inner(); + assertThat(inner.getDouble(0)).isCloseTo(1.0, within(1e-9)); + assertThat(inner.getDouble(2)).isCloseTo(3.0, within(1e-9)); + } + + @Test + void maskedEncodedChild_constantBroadcast_preservesValidity() { + // Given — a single encoded value broadcast to 3 rows (capacity < n, no patches) routes + // through the LazyConstant arm; the validity mask must survive that path too. + ArrayNode validityNode = new ArrayNode(EncodingId.VORTEX_BOOL, null, new ArrayNode[0], new int[]{1}); + ArrayNode enc = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, + new ArrayNode[]{validityNode}, new int[]{0}); + byte[] meta = new ProtoALPMetadata(2, 0, null).encode(); // *0.01 + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ALP, MemorySegment.ofArray(meta), + new ArrayNode[]{enc}, new int[0]); + MemorySegment[] segs = {leLongs(100L), boolBitmap(true, false, true)}; + DecodeContext ctx = new DecodeContext(node, F64, 3, segs, REGISTRY, Arena.ofAuto()); + + // When + Array result = SUT.decode(ctx); + + // Then — every row is the broadcast constant 1.0, and row 1 stays null + MaskedArray masked = (MaskedArray) result; + assertThat(masked.isValid(0)).isTrue(); + assertThat(masked.isValid(1)).isFalse(); + assertThat(masked.isValid(2)).isTrue(); + DoubleArray inner = (DoubleArray) masked.inner(); + for (int i = 0; i < 3; i++) { + assertThat(inner.getDouble(i)).as("row %d", i).isCloseTo(1.0, within(1e-9)); + } + } + + @Test + void plainEncodedChild_returnsPlainArray() { + // Given — a non-nullable encoded child (no validity): no-regression path must NOT mask + ArrayNode enc = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); + byte[] meta = new ProtoALPMetadata(2, 0, null).encode(); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ALP, MemorySegment.ofArray(meta), + new ArrayNode[]{enc}, new int[0]); + DecodeContext ctx = new DecodeContext(node, F64, 2, + new MemorySegment[]{leLongs(100L, 200L)}, REGISTRY, Arena.ofAuto()); + + // When + Array result = SUT.decode(ctx); + + // Then + assertThat(result).isInstanceOf(DoubleArray.class).isNotInstanceOf(MaskedArray.class); + } + + private MemorySegment boolBitmap(boolean... valid) { + byte[] bytes = new byte[(valid.length + 7) / 8]; + for (int i = 0; i < valid.length; i++) { + if (valid[i]) { + bytes[i >>> 3] |= (byte) (1 << (i & 7)); + } + } + return MemorySegment.ofArray(bytes); + } + } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoderTest.java new file mode 100644 index 00000000..e29a5e82 --- /dev/null +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoderTest.java @@ -0,0 +1,211 @@ +package io.github.dfa1.vortex.reader.decode; + +import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.core.proto.ProtoBitPackedMetadata; +import io.github.dfa1.vortex.core.proto.ProtoPType; +import io.github.dfa1.vortex.core.proto.ProtoPatchesMetadata; +import io.github.dfa1.vortex.encoding.TestSegments; +import io.github.dfa1.vortex.reader.ReadRegistry; +import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.IntArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/// Pins the validity-child propagation that [BitpackedEncodingDecoder] performs for the +/// `ValidityChild` shape (#210): a trailing bool validity child that stacked +/// encodings (ALP, FoR) delegate down to bitpacked, and whose position in the child vector +/// depends on the patch shape. Dropping it silently un-nulls rows. +/// +/// Every fixture uses `bit_width=0`, the real constant-residual shape seen when bitpacked is +/// nested under FoR/RLE (penguins `bill_length` decodes alp -> for -> bitpacked with the +/// residual collapsed to zero width). That makes the unpacked base deterministically all-zero, +/// so the tests isolate validity-index selection and mask wrapping from the packed-bit unpack. +/// Patch children then write real non-zero values, proving the patch path and validity index +/// coexist. +class BitpackedEncodingDecoderTest { + + private static final BitpackedEncodingDecoder SUT = new BitpackedEncodingDecoder(); + private static final ReadRegistry REGISTRY = TestRegistry.ofDecoders( + SUT, new PrimitiveEncodingDecoder(), new BoolEncodingDecoder()); + + private static final EncodingId BITPACKED = EncodingId.FASTLANES_BITPACKED; + private static final EncodingId PRIMITIVE = EncodingId.VORTEX_PRIMITIVE; + private static final EncodingId BOOL = EncodingId.VORTEX_BOOL; + private static final DType I32 = new DType.Primitive(io.github.dfa1.vortex.core.model.PType.I32, true); + + @Test + void encodingId_isFastlanesBitpacked() { + // Given / When / Then + assertThat(SUT.encodingId()).isEqualTo(EncodingId.FASTLANES_BITPACKED); + } + + @Nested + class ValidityChildIndex { + + @Test + void noPatches_trailingValidityAtIndexZero_masksRows() { + // Given — no patches, so the validity child is the first (index 0) child. This is the + // penguins bill_length shape: bitpacked residual (bit_width 0 -> all zero) carrying only + // a validity child. Rows 0,2,4 valid; 1,3,5 null. + byte[] meta = new ProtoBitPackedMetadata(0, 0, null).encode(); + MemorySegment validity = boolBitmap(true, false, true, false, true, false); + ArrayNode validityNode = new ArrayNode(BOOL, null, new ArrayNode[0], new int[]{1}); + ArrayNode node = new ArrayNode(BITPACKED, MemorySegment.ofArray(meta), + new ArrayNode[]{validityNode}, new int[]{0}); + DecodeContext ctx = new DecodeContext(node, I32, 6, + new MemorySegment[]{empty(), validity}, REGISTRY, Arena.ofAuto()); + + // When + Array result = SUT.decode(ctx); + + // Then — a MaskedArray whose validity mirrors the child; base values all zero + MaskedArray masked = assertMasked(result); + assertValidity(masked, true, false, true, false, true, false); + IntArray inner = (IntArray) masked.inner(); + for (int i = 0; i < 6; i++) { + assertThat(inner.getInt(i)).as("row %d", i).isZero(); + } + } + + @Test + void patchesWithoutChunkOffsets_validityAtIndexTwo_masksRows() { + // Given — patches present but no chunk offsets: children are [indices, values, validity], + // so validity sits at index 2. One patch overwrites row 2 with 700; rows 1,4 null. + ProtoPatchesMetadata pm = new ProtoPatchesMetadata(1L, 0L, ProtoPType.U8, null, null, null); + byte[] meta = new ProtoBitPackedMetadata(0, 0, pm).encode(); + + ArrayNode indices = new ArrayNode(PRIMITIVE, null, new ArrayNode[0], new int[]{1}); + ArrayNode values = new ArrayNode(PRIMITIVE, null, new ArrayNode[0], new int[]{2}); + ArrayNode validityNode = new ArrayNode(BOOL, null, new ArrayNode[0], new int[]{3}); + ArrayNode node = new ArrayNode(BITPACKED, MemorySegment.ofArray(meta), + new ArrayNode[]{indices, values, validityNode}, new int[]{0}); + + MemorySegment[] segs = { + empty(), // 0: packed (unread, bit_width 0) + MemorySegment.ofArray(new byte[]{2}), // 1: patch index -> row 2 + TestSegments.leInts(700), // 2: patch value + boolBitmap(true, false, true, true, false) // 3: validity + }; + DecodeContext ctx = new DecodeContext(node, I32, 5, segs, REGISTRY, Arena.ofAuto()); + + // When + Array result = SUT.decode(ctx); + + // Then — validity at index 2 survives; the patch wrote row 2 + MaskedArray masked = assertMasked(result); + assertValidity(masked, true, false, true, true, false); + IntArray inner = (IntArray) masked.inner(); + assertThat(inner.getInt(2)).isEqualTo(700); + assertThat(inner.getInt(0)).isZero(); + assertThat(inner.getInt(4)).isZero(); + } + + @Test + void patchesWithChunkOffsets_validityAtIndexThree_masksRows() { + // Given — patches carrying chunk offsets: children are [indices, values, chunkOffsets, + // validity], pushing validity to index 3. The chunk-offsets child is never decoded by the + // patch path, so it is a placeholder. One patch overwrites row 1 with 900; rows 3,4 null. + ProtoPatchesMetadata pm = new ProtoPatchesMetadata(1L, 0L, ProtoPType.U8, 1L, ProtoPType.U32, 0L); + byte[] meta = new ProtoBitPackedMetadata(0, 0, pm).encode(); + + ArrayNode indices = new ArrayNode(PRIMITIVE, null, new ArrayNode[0], new int[]{1}); + ArrayNode values = new ArrayNode(PRIMITIVE, null, new ArrayNode[0], new int[]{2}); + ArrayNode chunkOffsets = new ArrayNode(PRIMITIVE, null, new ArrayNode[0], new int[]{0}); // placeholder + ArrayNode validityNode = new ArrayNode(BOOL, null, new ArrayNode[0], new int[]{3}); + ArrayNode node = new ArrayNode(BITPACKED, MemorySegment.ofArray(meta), + new ArrayNode[]{indices, values, chunkOffsets, validityNode}, new int[]{0}); + + MemorySegment[] segs = { + empty(), + MemorySegment.ofArray(new byte[]{1}), // patch index -> row 1 + TestSegments.leInts(900), // patch value + boolBitmap(true, true, true, false, false) + }; + DecodeContext ctx = new DecodeContext(node, I32, 5, segs, REGISTRY, Arena.ofAuto()); + + // When + Array result = SUT.decode(ctx); + + // Then — validity at index 3 survives; the patch wrote row 1 + MaskedArray masked = assertMasked(result); + assertValidity(masked, true, true, true, false, false); + IntArray inner = (IntArray) masked.inner(); + assertThat(inner.getInt(1)).isEqualTo(900); + assertThat(inner.getInt(0)).isZero(); + } + + @Test + void noValidityChild_returnsPlainArray() { + // Given — no patches and zero children: nothing to mask, so the decoder returns the bare + // values array, never a MaskedArray (the no-regression path for non-nullable columns). + byte[] meta = new ProtoBitPackedMetadata(0, 0, null).encode(); + ArrayNode node = new ArrayNode(BITPACKED, MemorySegment.ofArray(meta), + new ArrayNode[0], new int[]{0}); + DecodeContext ctx = new DecodeContext(node, I32, 4, + new MemorySegment[]{empty()}, REGISTRY, Arena.ofAuto()); + + // When + Array result = SUT.decode(ctx); + + // Then + assertThat(result).isInstanceOf(IntArray.class).isNotInstanceOf(MaskedArray.class); + } + + @Test + void unexpectedChildCount_throwsWithExpectedCounts() { + // Given — no patches (expected 0 or 1 children) but two children are present: an + // ambiguous shape a trusted encoder never emits, so decode fails loudly. + byte[] meta = new ProtoBitPackedMetadata(0, 0, null).encode(); + ArrayNode a = new ArrayNode(BOOL, null, new ArrayNode[0], new int[]{1}); + ArrayNode b = new ArrayNode(BOOL, null, new ArrayNode[0], new int[]{1}); + ArrayNode node = new ArrayNode(BITPACKED, MemorySegment.ofArray(meta), + new ArrayNode[]{a, b}, new int[]{0}); + DecodeContext ctx = new DecodeContext(node, I32, 4, + new MemorySegment[]{empty(), boolBitmap(true, true, true, true)}, REGISTRY, Arena.ofAuto()); + + // When / Then + assertThatThrownBy(() -> SUT.decode(ctx)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("expected 0 or 1 children") + .hasMessageContaining("got 2"); + } + } + + // ── helpers ───────────────────────────────────────────────────────────────── + + private static MaskedArray assertMasked(Array result) { + assertThat(result).isInstanceOf(MaskedArray.class); + return (MaskedArray) result; + } + + private static void assertValidity(MaskedArray masked, boolean... expected) { + for (int i = 0; i < expected.length; i++) { + assertThat(masked.isValid(i)).as("valid row %d", i).isEqualTo(expected[i]); + } + } + + /// Builds an LSB-first packed validity bitmap (`MaterializedBoolArray` layout) where bit `i` + /// is set when `valid[i]` is true. + private static MemorySegment boolBitmap(boolean... valid) { + byte[] bytes = new byte[(valid.length + 7) / 8]; + for (int i = 0; i < valid.length; i++) { + if (valid[i]) { + bytes[i >>> 3] |= (byte) (1 << (i & 7)); + } + } + return MemorySegment.ofArray(bytes); + } + + private static MemorySegment empty() { + return MemorySegment.ofArray(new byte[0]); + } +} diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java index bdab8a31..85096013 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java @@ -14,6 +14,7 @@ import io.github.dfa1.vortex.reader.array.FloatArray; import io.github.dfa1.vortex.reader.array.IntArray; import io.github.dfa1.vortex.reader.array.LongArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; import io.github.dfa1.vortex.reader.array.ShortArray; import io.github.dfa1.vortex.reader.array.VarBinArray; import org.junit.jupiter.api.Nested; @@ -35,7 +36,7 @@ class DictEncodingDecoderTest { private static final DictEncodingDecoder SUT = new DictEncodingDecoder(); private static final ReadRegistry REGISTRY = TestRegistry.ofDecoders( - SUT, new PrimitiveEncodingDecoder(), new VarBinEncodingDecoder()); + SUT, new PrimitiveEncodingDecoder(), new VarBinEncodingDecoder(), new BoolEncodingDecoder()); @Test void encodingId_isVortexDict() { @@ -416,6 +417,145 @@ void protoLayout_emptyMetadata_throws() { } } + /// Per-row validity for the Rust proto primitive path (`decodeRustProto`, #210). A DictArray + /// row is null when its CODE is null (codes-side validity child) or when the code points at an + /// invalid pool slot (pool-null representation); both arrive as [MaskedArray] children and must + /// combine per row. Shapes mirror real Kepler columns: `koi_gmag` is pool-null, `koi_smet_err2` + /// is codes-side. + @Nested + class RowValidity { + + @Test + void poolNull_masksRowsPointingAtInvalidSlot() { + // Given — pool [10,20,30] with slot 1 invalid (koi_gmag shape); codes point rows 1 and 3 + // at that dead slot, so those rows are null while their expanded value is still present. + ArrayNode codesNode = primitiveNode(0); // plain codes: no codes-side nulls + ArrayNode valuesNode = maskedPrimitiveNode(1, 2); // masked pool: slot validity + MemorySegment[] segs = { + u8Codes(0, 1, 2, 1), + TestSegments.leInts(10, 20, 30), + boolBitmap(true, false, true) // slot 1 invalid + }; + + // When + Array result = decodeDict(DType.I32, PType.U8, 3, 4, segs, codesNode, valuesNode); + + // Then + MaskedArray masked = assertMasked(result); + assertValidity(masked, true, false, true, false); + assertIntValues(masked, 10, 20, 30, 20); + } + + @Test + void codesNull_masksRowsWithNullCode() { + // Given — pool [100,200] all valid; the codes child carries its own validity + // (koi_smet_err2 shape). Row nulls must mirror the codes mask exactly. + ArrayNode codesNode = maskedPrimitiveNode(0, 1); // masked codes: rows 1,3 null + ArrayNode valuesNode = primitiveNode(2); // plain pool: all valid + MemorySegment[] segs = { + u8Codes(0, 1, 0, 1), + boolBitmap(true, false, true, false), + TestSegments.leInts(100, 200) + }; + + // When + Array result = decodeDict(DType.I32, PType.U8, 2, 4, segs, codesNode, valuesNode); + + // Then + MaskedArray masked = assertMasked(result); + assertValidity(masked, true, false, true, false); + assertIntValues(masked, 100, 200, 100, 200); + } + + @Test + void bothSides_combineWithAndSemantics() { + // Given — codes-side nulls AND a pool-null slot: row i is valid iff its code is valid and + // the pool slot it references is valid. Row 2 is null via its code; rows 1,3 via the pool. + ArrayNode codesNode = maskedPrimitiveNode(0, 1); // codes: row 2 null + ArrayNode valuesNode = maskedPrimitiveNode(2, 3); // pool: slot 1 invalid + MemorySegment[] segs = { + u8Codes(0, 1, 2, 1), + boolBitmap(true, true, false, true), // codes valid except row 2 + TestSegments.leInts(10, 20, 30), + boolBitmap(true, false, true) // pool slot 1 invalid + }; + + // When + Array result = decodeDict(DType.I32, PType.U8, 3, 4, segs, codesNode, valuesNode); + + // Then — only row 0 survives both masks + MaskedArray masked = assertMasked(result); + assertValidity(masked, true, false, false, false); + assertIntValues(masked, 10, 20, 30, 20); + } + + @Test + void codeOutOfRangeWithPoolValidity_throwsVortexException() { + // Given — a single-element pool (so expandU8 broadcasts, never reading out of bounds) with + // pool validity present, and an untrusted code 5 that overruns the validity bitmap. The + // poolValid guard must convert the overrun into a VortexException, not an IOOBE. + ArrayNode codesNode = primitiveNode(0); + ArrayNode valuesNode = maskedPrimitiveNode(1, 2); + MemorySegment[] segs = { + u8Codes(0, 5), + TestSegments.leInts(10), + boolBitmap(true) // pool length 1 + }; + + // When / Then + assertThatThrownBy(() -> decodeDict(DType.I32, PType.U8, 1, 2, segs, codesNode, valuesNode)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("out of range for pool validity"); + } + + private Array decodeDict(DType dtype, PType codePType, int valuesLen, long rowCount, + MemorySegment[] segs, ArrayNode codesNode, ArrayNode valuesNode) { + MemorySegment meta = MemorySegment.ofArray( + new ProtoDictMetadata(valuesLen, protoPType(codePType), null, null).encode()); + ArrayNode dictNode = new ArrayNode(EncodingId.VORTEX_DICT, meta, + new ArrayNode[]{codesNode, valuesNode}, new int[]{}); + DecodeContext ctx = new DecodeContext(dictNode, dtype, rowCount, segs, REGISTRY, Arena.ofAuto()); + return SUT.decode(ctx); + } + + private MaskedArray assertMasked(Array result) { + assertThat(result).isInstanceOf(MaskedArray.class); + return (MaskedArray) result; + } + + private void assertValidity(MaskedArray masked, boolean... expected) { + for (int i = 0; i < expected.length; i++) { + assertThat(masked.isValid(i)).as("valid row %d", i).isEqualTo(expected[i]); + } + } + + private void assertIntValues(MaskedArray masked, int... expected) { + IntArray inner = (IntArray) masked.inner(); + for (int i = 0; i < expected.length; i++) { + assertThat(inner.getInt(i)).as("row %d", i).isEqualTo(expected[i]); + } + } + + /// A primitive node whose single validity child (a `vortex.bool` bitmap) makes + /// [PrimitiveEncodingDecoder] surface it as a [MaskedArray]. + private ArrayNode maskedPrimitiveNode(int dataBufIndex, int validityBufIndex) { + ArrayNode validity = new ArrayNode(EncodingId.VORTEX_BOOL, null, new ArrayNode[0], + new int[]{validityBufIndex}); + return new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[]{validity}, + new int[]{dataBufIndex}); + } + + private MemorySegment boolBitmap(boolean... valid) { + byte[] bytes = new byte[(valid.length + 7) / 8]; + for (int i = 0; i < valid.length; i++) { + if (valid[i]) { + bytes[i >>> 3] |= (byte) (1 << (i & 7)); + } + } + return MemorySegment.ofArray(bytes); + } + } + // ── parameter sources ────────────────────────────────────────────────────── static Stream codeAndValueTypes() { diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoderTest.java index af4b5da5..cce4dd3a 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoderTest.java @@ -10,7 +10,9 @@ import io.github.dfa1.vortex.reader.array.ByteArray; import io.github.dfa1.vortex.reader.array.IntArray; import io.github.dfa1.vortex.reader.array.LongArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; import io.github.dfa1.vortex.reader.array.ShortArray; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import java.lang.foreign.Arena; @@ -23,7 +25,8 @@ class ZigZagEncodingDecoderTest { private static final ZigZagEncodingDecoder SUT = new ZigZagEncodingDecoder(); - private static final ReadRegistry REGISTRY = TestRegistry.ofDecoders(SUT, new PrimitiveEncodingDecoder()); + private static final ReadRegistry REGISTRY = TestRegistry.ofDecoders( + SUT, new PrimitiveEncodingDecoder(), new BoolEncodingDecoder()); // --- zigzag encode helpers (mirror of the decoder's (u >>> 1) ^ -(u & 1)) --- @@ -211,4 +214,62 @@ void decode_nonPrimitiveDtype_throws() { .isInstanceOf(VortexException.class) .hasMessageContaining("expected primitive dtype"); } + + /// A zigzag array's validity IS its encoded child's (`ValidityChild`, #210): when the + /// encoded primitive child is nullable it arrives as a [MaskedArray], and that mask must ride + /// through to the decoded result rather than being flattened to a raw segment. + @Nested + class MaskedChildPropagation { + + @Test + void maskedEncodedChild_propagatesNullsAndDecodesValues() { + // Given — encoded I32 child [5,-3,7] with a validity child marking row 1 null + MemorySegment encoded = encodedInts(5, -3, 7); + MemorySegment validity = boolBitmap(true, false, true); + + // When + Array result = decodeMasked(PType.I32, 3, encoded, validity); + + // Then — nulls survive and valid rows decode correctly + MaskedArray masked = (MaskedArray) result; + assertThat(masked.isValid(0)).isTrue(); + assertThat(masked.isValid(1)).isFalse(); + assertThat(masked.isValid(2)).isTrue(); + IntArray inner = (IntArray) masked.inner(); + assertThat(inner.getInt(0)).isEqualTo(5); + assertThat(inner.getInt(1)).isEqualTo(-3); + assertThat(inner.getInt(2)).isEqualTo(7); + } + + @Test + void plainEncodedChild_returnsPlainArray() { + // Given — a non-nullable encoded child (no validity): the no-regression path must NOT + // wrap the result in a MaskedArray. + Array result = decode(PType.I32, 3, encodedInts(5, -3, 7)); + + // Then + assertThat(result).isInstanceOf(IntArray.class).isNotInstanceOf(MaskedArray.class); + } + + private Array decodeMasked(PType signed, long n, MemorySegment encoded, MemorySegment validity) { + DType dtype = new DType.Primitive(signed, true); + ArrayNode validityNode = new ArrayNode(EncodingId.VORTEX_BOOL, null, new ArrayNode[0], new int[]{1}); + ArrayNode child = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, + new ArrayNode[]{validityNode}, new int[]{0}); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ZIGZAG, null, new ArrayNode[]{child}, new int[]{}); + DecodeContext ctx = new DecodeContext(node, dtype, n, + new MemorySegment[]{encoded, validity}, REGISTRY, Arena.ofAuto()); + return SUT.decode(ctx); + } + + private MemorySegment boolBitmap(boolean... valid) { + byte[] bytes = new byte[(valid.length + 7) / 8]; + for (int i = 0; i < valid.length; i++) { + if (valid[i]) { + bytes[i >>> 3] |= (byte) (1 << (i & 7)); + } + } + return MemorySegment.ofArray(bytes); + } + } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoderTest.java new file mode 100644 index 00000000..2cd117f4 --- /dev/null +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoderTest.java @@ -0,0 +1,187 @@ +package io.github.dfa1.vortex.reader.layout; + +import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.LayoutId; +import io.github.dfa1.vortex.core.model.PType; +import io.github.dfa1.vortex.encoding.TestSegments; +import io.github.dfa1.vortex.reader.SegmentSpec; +import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.BoolArray; +import io.github.dfa1.vortex.reader.array.IntArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; +import io.github.dfa1.vortex.reader.array.MaterializedBoolArray; +import io.github.dfa1.vortex.reader.array.MaterializedByteArray; +import io.github.dfa1.vortex.reader.array.MaterializedIntArray; +import org.junit.jupiter.api.Test; + +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SegmentAllocator; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/// Pins the pool/codes row-validity gather in [DictLayoutDecoder.buildLazyDictPrimitive] (#210). +/// +/// Driven through the public [DictLayoutDecoder#decode(LayoutDecodeContext, Layout, DType)] seam +/// with a stub [LayoutDecodeContext] that hands the decoder pre-built values and codes arrays — the +/// closest constructible seam, since the private gather takes already-decoded children and building +/// a real on-disk dict layout fixture would be disproportionate. Shapes mirror real Kepler columns: +/// `koi_gmag` (pool-null) and `koi_smet_err2` (codes-side). +class DictLayoutDecoderTest { + + private static final DType I32 = new DType.Primitive(PType.I32, true); + + @Test + void poolNull_masksRowsPointingAtInvalidSlot() { + // Given — pool [10,20,30] with slot 1 invalid; codes route rows 1,3 at the dead slot + Array values = new MaskedArray(intPool(10, 20, 30), boolArray(true, false, true)); + Array codes = byteCodes(0, 1, 2, 1); + + // When + Array result = decode(4, I32, values, codes); + + // Then + MaskedArray masked = assertMasked(result); + assertValidity(masked, true, false, true, false); + assertIntValues(masked, 10, 20, 30, 20); + } + + @Test + void codesNull_masksRowsWithNullCode() { + // Given — an all-valid pool but a codes child carrying its own validity (rows 1,3 null) + Array values = intPool(100, 200); + Array codes = new MaskedArray(byteCodes(0, 1, 0, 1), boolArray(true, false, true, false)); + + // When + Array result = decode(4, I32, values, codes); + + // Then — the codes mask passes straight through unchanged + MaskedArray masked = assertMasked(result); + assertValidity(masked, true, false, true, false); + assertIntValues(masked, 100, 200, 100, 200); + } + + @Test + void bothSides_combineWithAndSemantics() { + // Given — codes-side null on row 2 AND a pool-null slot 1: a row is valid iff both hold + Array values = new MaskedArray(intPool(10, 20, 30), boolArray(true, false, true)); + Array codes = new MaskedArray(byteCodes(0, 1, 2, 1), boolArray(true, true, false, true)); + + // When + Array result = decode(4, I32, values, codes); + + // Then — only row 0 survives both masks + MaskedArray masked = assertMasked(result); + assertValidity(masked, true, false, false, false); + assertIntValues(masked, 10, 20, 30, 20); + } + + @Test + void codeOutOfRangeWithPoolValidity_throwsVortexException() { + // Given — pool validity of length 1 and an untrusted code 5 overrunning it; the gather guard + // must raise a VortexException, not a raw JDK IndexOutOfBoundsException + Array values = new MaskedArray(intPool(10), boolArray(true)); + Array codes = byteCodes(0, 5); + + // When / Then + assertThatThrownBy(() -> decode(2, I32, values, codes)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("out of range for pool validity"); + } + + @Test + void noValidityEitherSide_returnsPlainDictArray() { + // Given — all-valid pool and plain codes: nothing to mask (no-regression path) + Array values = intPool(10, 20, 30); + Array codes = byteCodes(0, 1, 2); + + // When + Array result = decode(3, I32, values, codes); + + // Then + assertThat(result).isInstanceOf(IntArray.class).isNotInstanceOf(MaskedArray.class); + } + + // ── harness ───────────────────────────────────────────────────────────────── + + private static Array decode(long n, DType dtype, Array values, Array codes) { + // metadata null -> codesPType defaults to U8, matching the byte codes built below + Layout valuesLayout = new Layout(LayoutId.FLAT, values.length(), null, List.of(), List.of()); + Layout codesLayout = new Layout(LayoutId.FLAT, n, null, List.of(), List.of()); + Layout dictLayout = new Layout(LayoutId.DICT, n, null, + List.of(valuesLayout, codesLayout), List.of()); + LayoutDecodeContext ctx = new StubContext(valuesLayout, values, codesLayout, codes); + return new DictLayoutDecoder().decode(ctx, dictLayout, dtype); + } + + private static IntArray intPool(int... values) { + return new MaterializedIntArray(I32, values.length, TestSegments.leInts(values)); + } + + private static Array byteCodes(int... codes) { + byte[] bytes = new byte[codes.length]; + for (int i = 0; i < codes.length; i++) { + bytes[i] = (byte) codes[i]; + } + return new MaterializedByteArray(new DType.Primitive(PType.U8, false), codes.length, + MemorySegment.ofArray(bytes)); + } + + private static BoolArray boolArray(boolean... valid) { + byte[] bytes = new byte[(valid.length + 7) / 8]; + for (int i = 0; i < valid.length; i++) { + if (valid[i]) { + bytes[i >>> 3] |= (byte) (1 << (i & 7)); + } + } + return new MaterializedBoolArray(DType.BOOL, valid.length, MemorySegment.ofArray(bytes)); + } + + private static MaskedArray assertMasked(Array result) { + assertThat(result).isInstanceOf(MaskedArray.class); + return (MaskedArray) result; + } + + private static void assertValidity(MaskedArray masked, boolean... expected) { + for (int i = 0; i < expected.length; i++) { + assertThat(masked.isValid(i)).as("valid row %d", i).isEqualTo(expected[i]); + } + } + + private static void assertIntValues(MaskedArray masked, int... expected) { + IntArray inner = (IntArray) masked.inner(); + for (int i = 0; i < expected.length; i++) { + assertThat(inner.getInt(i)).as("row %d", i).isEqualTo(expected[i]); + } + } + + /// Stub context that hands the decoder pre-built children keyed by layout identity, and + /// allocates the gathered validity bitmap from a real arena. Segment I/O is never reached on + /// the primitive dict path, so those methods are unsupported. + private record StubContext(Layout valuesLayout, Array values, Layout codesLayout, Array codes) + implements LayoutDecodeContext { + + @Override + public Array decodeChild(Layout child, DType dtype) { + return child == valuesLayout ? values : codes; + } + + @Override + public Array decodeSegment(SegmentSpec spec, DType dtype, long rowCount) { + throw new UnsupportedOperationException("not used on the primitive dict path"); + } + + @Override + public SegmentSpec segmentSpec(int index) { + throw new UnsupportedOperationException("not used on the primitive dict path"); + } + + @Override + public SegmentAllocator arena() { + return Arena.ofAuto(); + } + } +}