Expose metadata reading and writing in vortex-jni

java/vortex-jni/src/main/java/dev/vortex/api/VortexWriter.java

@@ -8,6 +8,7 @@
 import dev.vortex.io.NativeWritable;
 import dev.vortex.jni.NativeWriter;
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -19,6 +20,9 @@
 /**
  * Writer for Vortex files.
  *
+ * <p>Open one with {@link #builder(Session, String, Schema, BufferAllocator)} to write to a URI, or
+ * {@link #builder(Session, NativeWritable, Schema, BufferAllocator)} to write into a caller-provided byte sink.
+ *
  * <p>Batches are accepted via the Arrow C Data Interface: callers export an Arrow record batch to an {@code ArrowArray}
  * / {@code ArrowSchema} pair and pass the addresses to {@link #writeBatch(long, long)}. The writer accepts up to four
  * in-flight batches on the session's runtime thread before back-pressuring the caller.
@@ -44,53 +48,133 @@ private VortexWriter(long pointer) {
     }
 
     /**
-     * Create a writer that streams records into the file at {@code uri}. The path may be a full URI or a plain local
-     * filesystem path. The Arrow schema describes the exact layout of every batch written.
+     * Start configuring a writer that streams records into the file at {@code uri} through a native storage client. The
+     * path may be a full URI or a plain local filesystem path.
+     *
+     * @param arrowSchema describes the exact layout of every batch written
      */
-    public static VortexWriter create(
-            Session session, String uri, Schema arrowSchema, Map<String, String> options, BufferAllocator allocator)
-            throws IOException {
-        Objects.requireNonNull(session, "session");
-        Objects.requireNonNull(uri, "uri");
-        Objects.requireNonNull(arrowSchema, "arrowSchema");
-        Objects.requireNonNull(allocator, "allocator");
-        ArrowSchema ffi = ArrowSchema.allocateNew(allocator);
-        try {
-            Data.exportSchema(allocator, arrowSchema, null, ffi);
-            long ptr = NativeWriter.create(session.nativePointer(), uri, ffi.memoryAddress(), options);
-            if (ptr <= 0) {
-                throw new IOException("failed to create writer for uri " + uri + " (ptr=" + ptr + ")");
-            }
-            return new VortexWriter(ptr);
-        } finally {
-            ffi.close();
-        }
+    public static Builder builder(Session session, String uri, Schema arrowSchema, BufferAllocator allocator) {
+        return new Builder(session, Objects.requireNonNull(uri, "uri"), null, arrowSchema, allocator);
     }
 
     /**
-     * Create a writer that streams the file into a caller-provided byte sink instead of a native storage client. This
-     * is the integration point for external I/O abstractions (for example Iceberg's {@code PositionOutputStream}).
+     * Start configuring a writer that streams the file into a caller-provided byte sink instead of a native storage
+     * client. This is the integration point for external I/O abstractions (for example Iceberg's
+     * {@code PositionOutputStream}).
      *
      * <p>The native side writes and flushes the sink but never closes it: after {@link #close()} returns, all bytes
      * have been written and flushed, and the caller must close the sink to finalize the file.
+     *
+     * @param arrowSchema describes the exact layout of every batch written
      */
-    public static VortexWriter create(
-            Session session, NativeWritable writable, Schema arrowSchema, BufferAllocator allocator)
-            throws IOException {
-        Objects.requireNonNull(session, "session");
-        Objects.requireNonNull(writable, "writable");
-        Objects.requireNonNull(arrowSchema, "arrowSchema");
-        Objects.requireNonNull(allocator, "allocator");
-        ArrowSchema ffi = ArrowSchema.allocateNew(allocator);
-        try {
-            Data.exportSchema(allocator, arrowSchema, null, ffi);
-            long ptr = NativeWriter.createStream(session.nativePointer(), writable, ffi.memoryAddress());
-            if (ptr <= 0) {
-                throw new IOException("failed to create stream writer (ptr=" + ptr + ")");
+    public static Builder builder(
+            Session session, NativeWritable writable, Schema arrowSchema, BufferAllocator allocator) {
+        return new Builder(session, null, Objects.requireNonNull(writable, "writable"), arrowSchema, allocator);
+    }
+
+    /**
+     * Configures and opens a {@link VortexWriter}. Everything required to open one is fixed by
+     * {@link VortexWriter#builder}; the setters here are all optional.
+     *
+     * <p>Not thread-safe, and not reusable: each {@link #build()} opens one file.
+     */
+    public static final class Builder {
+        private final Session session;
+        private final String uri;
+        private final NativeWritable writable;
+        private final Schema arrowSchema;
+        private final BufferAllocator allocator;
+        private final Map<String, String> options = new HashMap<>();
+        private final Map<String, byte[]> metadata = new HashMap<>();
+
+        private Builder(
+                Session session, String uri, NativeWritable writable, Schema arrowSchema, BufferAllocator allocator) {
+            this.session = Objects.requireNonNull(session, "session");
+            this.uri = uri;
+            this.writable = writable;
+            this.arrowSchema = Objects.requireNonNull(arrowSchema, "arrowSchema");
+            this.allocator = Objects.requireNonNull(allocator, "allocator");
+        }
+
+        /**
+         * Object-store credentials and options, replacing any set so far. Only the URI destination reaches a storage
+         * client, so this is rejected on a writer built over a {@link NativeWritable}.
+         */
+        public Builder options(Map<String, String> newOptions) {
+            checkOptionsApply();
+            Objects.requireNonNull(newOptions, "options");
+            options.clear();
+            newOptions.forEach(this::putOption);
+            return this;
+        }
+
+        /** Add a single object-store option. See {@link #options(Map)}. */
+        public Builder putOption(String key, String value) {
+            checkOptionsApply();
+            Objects.requireNonNull(key, "key");
+            Objects.requireNonNull(value, "value");
+            options.put(key, value);
+            return this;
+        }
+
+        /**
+         * User-defined metadata segments to store in the file footer, replacing any set so far.
+         *
+         * <p>Values are opaque bytes, returned verbatim by {@code NativeFiles.readMetadata}. The native writer caps
+         * both the number of segments per file and the length of each key; the limits are enforced (and named in the
+         * error) by {@link #build()}, rather than when the file is finalized.
+         */
+        public Builder metadata(Map<String, byte[]> newMetadata) {
+            Objects.requireNonNull(newMetadata, "metadata");
+            metadata.clear();
+            newMetadata.forEach(this::putMetadata);
+            return this;
+        }
+
+        /** Add a single metadata segment. See {@link #metadata(Map)}. */
+        public Builder putMetadata(String key, byte[] value) {
+            Objects.requireNonNull(key, "key");
+            Objects.requireNonNull(value, "value");
+            metadata.put(key, value);
+            return this;
+        }
+
+        /**
+         * Open the writer.
+         *
+         * @throws IOException if the writer cannot be opened, including when the native side rejects the configured
+         *     metadata
+         */
+        public VortexWriter build() throws IOException {
+            ArrowSchema ffi = ArrowSchema.allocateNew(allocator);
+            try {
+                Data.exportSchema(allocator, arrowSchema, null, ffi);
+                final long pointer;
+                try {
+                    pointer = uri != null
+                            ? NativeWriter.create(session.nativePointer(), uri, ffi.memoryAddress(), options, metadata)
+                            : NativeWriter.createStream(
+                                    session.nativePointer(), writable, ffi.memoryAddress(), metadata);
+                } catch (RuntimeException e) {
+                    // Native failures arrive as RuntimeException. Wrap them so callers can handle every
+                    // failure to open a writer as an IOException, as writeBatch and finish already do.
+                    throw new IOException("failed to create writer for " + destination(), e);
+                }
+                if (pointer <= 0) {
+                    throw new IOException("failed to create writer for " + destination() + " (ptr=" + pointer + ")");
+                }
+                return new VortexWriter(pointer);
+            } finally {
+                ffi.close();
             }
-            return new VortexWriter(ptr);
-        } finally {
-            ffi.close();
+        }
+
+        private String destination() {
+            return uri != null ? "uri " + uri : "stream";
+        }
+
+        private void checkOptionsApply() {
+            Preconditions.checkState(uri != null, "options apply to uri destinations only");
         }
     }
 

java/vortex-jni/src/main/java/dev/vortex/jni/NativeFiles.java

@@ -3,13 +3,16 @@
 
 package dev.vortex.jni;
 
+import com.google.common.base.Preconditions;
 import dev.vortex.api.Session;
+import dev.vortex.io.NativeReadable;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 /**
- * Static utilities for discovering and deleting Vortex files on an object store. The caller supplies a {@link Session};
- * its runtime handle is forwarded to the underlying object store.
+ * Static utilities for discovering, deleting, and inspecting the metadata of Vortex files on an object store. The
+ * caller supplies a {@link Session}; its runtime handle is forwarded to the underlying object store.
  */
 public final class NativeFiles {
     static {
@@ -28,7 +31,42 @@ public static void delete(Session session, String[] uris, Map<String, String> op
         delete(session.nativePointer(), uris, options);
     }
 
+    /**
+     * Read the user-defined metadata segments written into the Vortex file at {@code uri}, as opaque bytes keyed by the
+     * names the {@code VortexWriter} builder was given. Returns an empty map for a file that carries no metadata.
+     *
+     * <p>This opens the file on its own. Metadata lives in dedicated segments that opening a file does not read by
+     * default, so a segment that falls outside the file-tail read costs an extra round trip.
+     */
+    public static Map<String, byte[]> readMetadata(Session session, String uri, Map<String, String> options) {
+        return readMetadata(session.nativePointer(), uri, options);
+    }
+
+    /**
+     * Read the user-defined metadata segments of a Vortex file through a caller-provided {@link NativeReadable},
+     * instead of a native storage client. See {@link #readMetadata(Session, String, Map)}.
+     *
+     * <p>The readable must stay open for the duration of the call; native code never closes it.
+     *
+     * <p>{@link NativeReadable#name()} keys the file in the session's footer cache, exactly as it does when the
+     * readable is scanned, so reading metadata and then scanning the same file share one footer read.
+     */
+    public static Map<String, byte[]> readMetadata(Session session, NativeReadable readable) {
+        Objects.requireNonNull(readable, "readable");
+        String name = readable.name();
+        Preconditions.checkArgument(name != null && !name.isEmpty(), "readable reported an empty name");
+        long length = readable.length();
+        Preconditions.checkArgument(length >= 0, "readable for %s reported negative length", name);
+        return readMetadataFromReadable(session.nativePointer(), readable, name, length);
+    }
+
     private static native List<String> listFiles(long sessionPointer, String uri, Map<String, String> options);
 
     private static native void delete(long sessionPointer, String[] uris, Map<String, String> options);
+
+    private static native Map<String, byte[]> readMetadata(
+            long sessionPointer, String uri, Map<String, String> options);
+
+    private static native Map<String, byte[]> readMetadataFromReadable(
+            long sessionPointer, Object readable, String name, long length);
 }

java/vortex-jni/src/main/java/dev/vortex/jni/NativeWriter.java

@@ -14,15 +14,26 @@ public final class NativeWriter {
 
     private NativeWriter() {}
 
-    /** Open a writer at {@code uri} that accepts batches matching the Arrow schema at {@code arrowSchemaAddress}. */
+    /**
+     * Open a writer at {@code uri} that accepts batches matching the Arrow schema at {@code arrowSchemaAddress}.
+     *
+     * @param metadata user-defined metadata segments to store in the file footer (may be null)
+     */
     public static native long create(
-            long sessionPointer, String uri, long arrowSchemaAddress, Map<String, String> options);
+            long sessionPointer,
+            String uri,
+            long arrowSchemaAddress,
+            Map<String, String> options,
+            Map<String, byte[]> metadata);
 
     /**
      * Open a writer that streams the file into a caller-provided {@link dev.vortex.io.NativeWritable}. The native side
      * writes and flushes but never closes the writable; the caller must close it after {@link #close}.
+     *
+     * @param metadata user-defined metadata segments to store in the file footer (may be null)
      */
-    public static native long createStream(long sessionPointer, Object writable, long arrowSchemaAddress);
+    public static native long createStream(
+            long sessionPointer, Object writable, long arrowSchemaAddress, Map<String, byte[]> metadata);
 
     /**
      * Write a batch directly from Arrow C Data Interface addresses.

java/vortex-jni/src/test/java/dev/vortex/api/GeoTypesTest.java

@@ -15,7 +15,6 @@
 import java.nio.ByteOrder;
 import java.nio.file.Path;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import org.apache.arrow.c.ArrowArray;
@@ -68,7 +67,8 @@ static void setup() throws IOException {
         BufferAllocator allocator = ArrowAllocation.rootAllocator();
         Schema schema = new Schema(List.of(wkbField("geom")));
         Session session = Session.create();
-        try (VortexWriter writer = VortexWriter.create(session, writePath, schema, new HashMap<>(), allocator);
+        try (VortexWriter writer = VortexWriter.builder(session, writePath, schema, allocator)
+                        .build();
                 VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
             VarBinaryVector geomVec = (VarBinaryVector) root.getVector("geom");
             geomVec.allocateNew(WKB_POINTS.size());

java/vortex-jni/src/test/java/dev/vortex/api/TestMinimal.java

@@ -14,7 +14,6 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Objects;
 import org.apache.arrow.c.ArrowArray;
@@ -86,7 +85,8 @@ static void setup() throws IOException {
                 Field.notNullable("Salary", ArrowType.Decimal.createDecimal(9, 2, 128)),
                 Field.nullable("State", ArrowType.Utf8View.INSTANCE)));
         Session session = Session.create();
-        try (VortexWriter writer = VortexWriter.create(session, writePath, schema, new HashMap<>(), allocator);
+        try (VortexWriter writer = VortexWriter.builder(session, writePath, schema, allocator)
+                        .build();
                 VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
             ViewVarCharVector nameVec = (ViewVarCharVector) root.getVector("Name");
             DecimalVector salaryVec = (DecimalVector) root.getVector("Salary");