[SPARK-60101][SQL] Support native data sources with a frozen C core and versioned messages

.github/workflows/build_and_test.yml

@@ -952,6 +952,15 @@ jobs:
         export MAVEN_OPTS="-Xss64m -Xmx4g -Xms4g -XX:ReservedCodeCacheSize=128m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
         export MAVEN_CLI_OPTS="--no-transfer-progress"
         ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Pcredential-aws clean install
+    # Native data sources are called with the FFM API, which requires Java 22 or later, so the
+    # other jobs, on Java 17, skip the tests that load a library. The tests build their library
+    # with the C++ compiler of the runner.
+    - name: Test native data sources
+      run: |
+        export MAVEN_OPTS="-Xss64m -Xmx4g -Xms4g -XX:ReservedCodeCacheSize=128m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
+        export MAVEN_CLI_OPTS="--no-transfer-progress"
+        ./build/mvn $MAVEN_CLI_OPTS -pl sql/core -Dtest=none -DwildcardSuites=org.apache.spark.sql.execution.datasources.v2.ffi.NativeDataSourceSuite,org.apache.spark.sql.execution.datasources.v2.ffi.NativeDataSourceClusterSuite test
+        ./build/mvn $MAVEN_CLI_OPTS -pl sql/connect/server -Dtest=none -DwildcardSuites=org.apache.spark.sql.connect.NativeDataSourceConnectSuite test
 
   # Documentation build
   docs:

LICENSE-binary

@@ -301,6 +301,7 @@ joda-time:joda-time
 net.sf.opencsv:opencsv
 net.sf.supercsv:super-csv
 net.sf.jpam:jpam
+org.apache.arrow:arrow-c-data
 org.apache.arrow:arrow-compression
 org.apache.arrow:arrow-format
 org.apache.arrow:arrow-memory-core

common/utils/src/main/resources/error/error-conditions.json

@@ -4792,6 +4792,57 @@
     ],
     "sqlState" : "42000"
   },
+  "INVALID_NATIVE_DATA_SOURCE_LIBRARY" : {
+    "message" : [
+      "Invalid native data source library <path>:"
+    ],
+    "subClass" : {
+      "CANNOT_LOAD" : {
+        "message" : [
+          "<reason>"
+        ]
+      },
+      "REPLACED" : {
+        "message" : [
+          "The library was replaced after Spark loaded it: Spark loaded the build with the SHA-256 checksum <loadedChecksum>, but the file now has the checksum <checksum>. A JVM cannot load another build of a library from the same path, so restart the Spark application to use the new build."
+        ]
+      },
+      "UNSUPPORTED_ABI_VERSION" : {
+        "message" : [
+          "It requires version <version> of the native data source interface, but Spark supports version <supported>."
+        ]
+      }
+    },
+    "sqlState" : "22023"
+  },
+  "INVALID_NATIVE_DATA_SOURCE_PACKAGE" : {
+    "message" : [
+      "Invalid native data source package <path>:"
+    ],
+    "subClass" : {
+      "INVALID_LIBRARY" : {
+        "message" : [
+          "Cannot load the library <library>: <reason>"
+        ]
+      },
+      "INVALID_MANIFEST" : {
+        "message" : [
+          "<reason>"
+        ]
+      },
+      "UNSUPPORTED_ABI_VERSION" : {
+        "message" : [
+          "It requires version <version> of the native data source interface, but Spark supports version <supported>."
+        ]
+      },
+      "UNSUPPORTED_PLATFORM" : {
+        "message" : [
+          "It has no library for the platform <platform>. It has libraries for: <platforms>."
+        ]
+      }
+    },
+    "sqlState" : "22023"
+  },
   "INVALID_NON_DETERMINISTIC_EXPRESSIONS" : {
     "message" : [
       "The operator expects a deterministic expression, but the actual expression is <sqlExprs>."
@@ -6262,6 +6313,36 @@
     ],
     "sqlState" : "0A000"
   },
+  "NATIVE_DATA_SOURCE_ERROR" : {
+    "message" : [
+      "Failed to <action> native data source <name>: <msg>"
+    ],
+    "sqlState" : "38000"
+  },
+  "NATIVE_DATA_SOURCE_LIBRARY_NOT_FOUND" : {
+    "message" : [
+      "Cannot find the native data source library <fileName> with the SHA-256 checksum <checksum>, which is installed at <path> on the driver. Install the same build of the library on every node, at the same path or in the native library path."
+    ],
+    "sqlState" : "42K02"
+  },
+  "NATIVE_DATA_SOURCE_PACKAGE_CONFLICT" : {
+    "message" : [
+      "More than one native data source package provides the data source <name>: <packages>. Remove all but one of them."
+    ],
+    "sqlState" : "42710"
+  },
+  "NATIVE_DATA_SOURCE_PACKAGE_NOT_FOUND" : {
+    "message" : [
+      "Cannot find the native data source package <fileName> with the SHA-256 checksum <checksum>. Add the package with `spark.addArtifact`, or install the same package under `spark.sql.dataSource.native.paths` on every node."
+    ],
+    "sqlState" : "42K02"
+  },
+  "NATIVE_DATA_SOURCE_UNSUPPORTED_JAVA_VERSION" : {
+    "message" : [
+      "Native data sources require Java <required> or later, but Spark runs on Java <current>."
+    ],
+    "sqlState" : "0A000"
+  },
   "NEAREST_BY_JOIN" : {
     "message" : [
       "Invalid nearest-by join."

dev/deps/spark-deps-hadoop-3-hive-2.3

@@ -21,6 +21,7 @@ antlr4-runtime/4.13.1//antlr4-runtime-4.13.1.jar
 aopalliance-repackaged/3.0.6//aopalliance-repackaged-3.0.6.jar
 arpack/3.2.0//arpack-3.2.0.jar
 arpack_combined_all/0.1//arpack_combined_all-0.1.jar
+arrow-c-data/19.0.0//arrow-c-data-19.0.0.jar
 arrow-compression/19.0.0//arrow-compression-19.0.0.jar
 arrow-format/19.0.0//arrow-format-19.0.0.jar
 arrow-memory-core/19.0.0//arrow-memory-core-19.0.0.jar

dev/make-distribution.sh

@@ -223,6 +223,13 @@ if [ -f "$DISTDIR"/jars/datasketches-memory-3.0.2.jar ]; then
   zip -d "$DISTDIR"/jars/datasketches-memory-3.0.2.jar org/apache/datasketches/memory/internal/ResourceImpl.class
 fi
 
+# Copy the C header that the libraries of native data sources implement, and its messages
+mkdir -p "$DISTDIR/include"
+cp "$SPARK_HOME/sql/core/src/main/resources/org/apache/spark/sql/datasource/spark_datasource.h" \
+  "$DISTDIR/include/"
+cp "$SPARK_HOME/sql/core/src/main/protobuf/org/apache/spark/sql/datasource/spark_datasource.proto" \
+  "$DISTDIR/include/"
+
 # Only create the yarn directory if the yarn artifacts were built.
 if [ -f "$SPARK_HOME"/common/network-yarn/target/scala*/spark-*-yarn-shuffle.jar ]; then
   mkdir "$DISTDIR/yarn"

docs/_data/menu-sql.yaml

@@ -75,6 +75,19 @@
       url: sql-v2-data-sources.html#streaming
     - text: Further Reading
       url: sql-v2-data-sources.html#further-reading
+- text: Native Data Sources
+  url: sql-native-data-sources.html
+  subitems:
+    - text: Overview
+      url: sql-native-data-sources.html#overview
+    - text: How It Works
+      url: sql-native-data-sources.html#how-it-works
+    - text: Packages
+      url: sql-native-data-sources.html#packages
+    - text: Example in Rust
+      url: sql-native-data-sources.html#example-in-rust
+    - text: Example in C++
+      url: sql-native-data-sources.html#example-in-c
 - text: Performance Tuning
   url: sql-performance-tuning.html
   subitems: