bundle SQLean extensions

A common complain with libSQL is how to run extensions. The main
mechanism, with a .so, has a lot of issues around how those .so are
distributed.

The most common extensions are the ones in the sqlean package. We can
improve this experience by bundling them in our sqlite build.

Not all SQLean extensions are kosher: some of them, like fileio, use
the vfs. Others, are deemed too complex.

The extensions included here are a subset that we deem important enough,
and low risk enough, to just be a part of the main bundle.

Author: glommer

Cargo.lock

@@ -16,6 +16,7 @@ libsql-wasmtime-bindings = { version = "0.2.1", optional = true }
 [build-dependencies]
 bindgen = "0.66.1"
 cc = "1.0"
+glob = "0.3"
 
 [features]
 session = []
@@ -28,3 +29,17 @@ wasm32-wasi-vfs = []
 unlock_notify = []
 preupdate_hook = []
 sqlcipher = []
+sqlean-extension-uuid = []
+sqlean-extension-crypto = []
+sqlean-extension-fuzzy = []
+sqlean-extension-math = []
+sqlean-extension-stats = []
+sqlean-extension-text = []
+sqlean-extensions = [
+    "sqlean-extension-uuid",
+    "sqlean-extension-crypto",
+    "sqlean-extension-fuzzy",
+    "sqlean-extension-math",
+    "sqlean-extension-stats",
+    "sqlean-extension-text"
+]

libsql-ffi/Cargo.toml

@@ -1,3 +1,4 @@
+use glob::glob;
 use std::env;
 use std::ffi::OsString;
 use std::fs::{self, OpenOptions};
@@ -130,6 +131,57 @@ fn make_amalgamation() {
     .unwrap();
 }
 
+fn generate_sqlean(enabled_extensions: &[&str], output_path: &Path) -> io::Result<()> {
+    let mut content = String::from(
+        r#"// Generated by build.rs
+
+
+#include "sqlite3.c"
+SQLITE_EXTENSION_INIT1
+
+"#,
+    );
+
+    for ext in enabled_extensions {
+        content.push_str(&format!("#include \"{}/extension.h\"\n", ext));
+    }
+
+    content.push_str(
+        r#"
+#include "sqlean.h"
+
+static void sqlean_version(sqlite3_context* context, int argc, sqlite3_value** argv) {
+    sqlite3_result_text(context, SQLEAN_VERSION, -1, SQLITE_STATIC);
+}
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_sqlean_init(sqlite3* db, char** errmsg_ptr, const sqlite3_api_routines* api) {
+    (void)errmsg_ptr;
+    SQLITE_EXTENSION_INIT2(api);
+    static const int flags = SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC;
+    sqlite3_create_function(db, "sqlean_version", 0, flags, 0, sqlean_version, 0, 0);
+"#,
+    );
+
+    for ext in enabled_extensions {
+        content.push_str(&format!("    {}_init(db);\n", ext));
+    }
+
+    content.push_str(
+        r#"    return SQLITE_OK;
+}
+
+int core_init(const char* dummy) {
+    return sqlite3_auto_extension((void*)sqlite3_sqlean_init);
+}
+"#,
+    );
+
+    std::fs::write(output_path, content)
+}
+
 pub fn build_bundled(out_dir: &str, out_path: &Path) {
     let bindgen_rs_path = if cfg!(feature = "session") {
         "bundled/bindings/session_bindgen.rs"
@@ -146,8 +198,7 @@ pub fn build_bundled(out_dir: &str, out_path: &Path) {
     std::fs::copy(format!("{dir}/{bindgen_rs_path}"), out_path).unwrap();
 
     let mut cfg = cc::Build::new();
-    cfg.file(format!("{BUNDLED_DIR}/src/sqlite3.c"))
-        .flag("-std=c11")
+    cfg.flag("-std=c11")
         .flag("-DSQLITE_CORE")
         .flag("-DSQLITE_DEFAULT_FOREIGN_KEYS=1")
         .flag("-DSQLITE_ENABLE_API_ARMOR")
@@ -169,6 +220,62 @@ pub fn build_bundled(out_dir: &str, out_path: &Path) {
         .flag("-D_POSIX_THREAD_SAFE_FUNCTIONS") // cross compile with MinGW
         .warnings(false);
 
+    let mut sqlean_patterns = vec![];
+    let mut enabled_extensions = Vec::new();
+
+    if cfg!(feature = "sqlean-extension-crypto") {
+        enabled_extensions.push("crypto");
+        sqlean_patterns.push("crypto/*.c");
+    }
+
+    if cfg!(feature = "sqlean-extension-fuzzy") {
+        enabled_extensions.push("fuzzy");
+        sqlean_patterns.push("fuzzy/*.c");
+    }
+
+    if cfg!(feature = "sqlean-extension-math") {
+        enabled_extensions.push("math");
+        sqlean_patterns.push("math/*.c");
+    }
+
+    if cfg!(feature = "sqlean-extension-stats") {
+        enabled_extensions.push("stats");
+        sqlean_patterns.push("stats/*.c");
+    }
+
+    if cfg!(feature = "sqlean-extension-text") {
+        enabled_extensions.push("text");
+        sqlean_patterns.push("text/*.c");
+        sqlean_patterns.push("text/*/*.c");
+    }
+
+    if cfg!(feature = "sqlean-extension-uuid") {
+        enabled_extensions.push("uuid");
+        sqlean_patterns.push("uuid/*.c");
+    }
+
+    if sqlean_patterns.is_empty() {
+        cfg.file(format!("{BUNDLED_DIR}/src/sqlite3.c"));
+    } else {
+        cfg.flag("-DSQLITE_EXTRA_INIT=core_init");
+
+        let mut sqlean_sources = Vec::new();
+        for pattern in sqlean_patterns {
+            let full_pattern = format!("{BUNDLED_DIR}/sqlean/{}", pattern);
+            sqlean_sources.extend(glob(&full_pattern).unwrap().filter_map(Result::ok));
+        }
+
+        cfg.files(sqlean_sources);
+
+        let sqlean = Path::new(BUNDLED_DIR)
+            .join("src")
+            .join("sqlite3-sqlean-generated.c");
+        generate_sqlean(&enabled_extensions, &sqlean).unwrap();
+        cfg.file(&sqlean);
+
+        cfg.include(format!("{BUNDLED_DIR}/sqlean/"));
+    }
+
     if cfg!(feature = "wasmtime-bindings") {
         cfg.flag("-DLIBSQL_ENABLE_WASM_RUNTIME=1");
     }

libsql-ffi/build.rs

@@ -0,0 +1,82 @@
+// Copyright (c) 2023 Anton Zhiyanov, MIT License
+// https://github.com/nalgeon/sqlean
+
+// Base32 encoding/decoding (RFC 4648)
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+static const char base32_chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
+
+uint8_t* base32_encode(const uint8_t* src, size_t len, size_t* out_len) {
+    *out_len = ((len + 4) / 5) * 8;
+    uint8_t* encoded = malloc(*out_len + 1);
+    if (encoded == NULL) {
+        *out_len = 0;
+        return NULL;
+    }
+
+    for (size_t i = 0, j = 0; i < len;) {
+        uint32_t octet0 = i < len ? src[i++] : 0;
+        uint32_t octet1 = i < len ? src[i++] : 0;
+        uint32_t octet2 = i < len ? src[i++] : 0;
+        uint32_t octet3 = i < len ? src[i++] : 0;
+        uint32_t octet4 = i < len ? src[i++] : 0;
+
+        encoded[j++] = base32_chars[octet0 >> 3];
+        encoded[j++] = base32_chars[((octet0 & 0x07) << 2) | (octet1 >> 6)];
+        encoded[j++] = base32_chars[(octet1 >> 1) & 0x1F];
+        encoded[j++] = base32_chars[((octet1 & 0x01) << 4) | (octet2 >> 4)];
+        encoded[j++] = base32_chars[((octet2 & 0x0F) << 1) | (octet3 >> 7)];
+        encoded[j++] = base32_chars[(octet3 >> 2) & 0x1F];
+        encoded[j++] = base32_chars[((octet3 & 0x03) << 3) | (octet4 >> 5)];
+        encoded[j++] = base32_chars[octet4 & 0x1F];
+    }
+
+    if (len % 5 != 0) {
+        size_t padding = 7 - (len % 5) * 8 / 5;
+        for (size_t i = 0; i < padding; i++) {
+            encoded[*out_len - padding + i] = '=';
+        }
+    }
+
+    encoded[*out_len] = '\0';
+    return encoded;
+}
+
+uint8_t* base32_decode(const uint8_t* src, size_t len, size_t* out_len) {
+    while (len > 0 && src[len - 1] == '=') {
+        len--;
+    }
+    *out_len = len * 5 / 8;
+    uint8_t* decoded = malloc(*out_len);
+    if (decoded == NULL) {
+        *out_len = 0;
+        return NULL;
+    }
+
+    size_t bits = 0, value = 0, count = 0;
+    for (size_t i = 0; i < len; i++) {
+        uint8_t c = src[i];
+        if (c >= 'A' && c <= 'Z') {
+            c -= 'A';
+        } else if (c >= '2' && c <= '7') {
+            c -= '2' - 26;
+        } else {
+            continue;
+        }
+        value = (value << 5) | c;
+        bits += 5;
+        if (bits >= 8) {
+            decoded[count++] = (uint8_t)(value >> (bits - 8));
+            bits -= 8;
+        }
+    }
+    if (bits >= 5 || (value & ((1 << bits) - 1)) != 0) {
+        free(decoded);
+        return NULL;
+    }
+    *out_len = count;
+    return decoded;
+}

libsql-ffi/bundled/sqlean/crypto/base32.c

@@ -0,0 +1,14 @@
+// Copyright (c) 2023 Anton Zhiyanov, MIT License
+// https://github.com/nalgeon/sqlean
+
+// Base32 encoding/decoding (RFC 4648)
+
+#ifndef _BASE32_H_
+#define _BASE32_H_
+
+#include <stdint.h>
+
+uint8_t* base32_encode(const uint8_t* src, size_t len, size_t* out_len);
+uint8_t* base32_decode(const uint8_t* src, size_t len, size_t* out_len);
+
+#endif /* _BASE32_H_ */

libsql-ffi/bundled/sqlean/crypto/base32.h

@@ -0,0 +1,103 @@
+// Copyright (c) 2023 Anton Zhiyanov, MIT License
+// https://github.com/nalgeon/sqlean
+
+// Base64 encoding/decoding (RFC 4648)
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+static const char base64_chars[] =
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+uint8_t* base64_encode(const uint8_t* src, size_t len, size_t* out_len) {
+    uint8_t* encoded = NULL;
+    size_t i, j;
+    uint32_t octets;
+
+    *out_len = ((len + 2) / 3) * 4;
+    encoded = malloc(*out_len + 1);
+    if (encoded == NULL) {
+        *out_len = 0;
+        return NULL;
+    }
+
+    for (i = 0, j = 0; i < len; i += 3, j += 4) {
+        octets =
+            (src[i] << 16) | ((i + 1 < len ? src[i + 1] : 0) << 8) | (i + 2 < len ? src[i + 2] : 0);
+        encoded[j] = base64_chars[(octets >> 18) & 0x3f];
+        encoded[j + 1] = base64_chars[(octets >> 12) & 0x3f];
+        encoded[j + 2] = base64_chars[(octets >> 6) & 0x3f];
+        encoded[j + 3] = base64_chars[octets & 0x3f];
+    }
+
+    if (len % 3 == 1) {
+        encoded[*out_len - 1] = '=';
+        encoded[*out_len - 2] = '=';
+    } else if (len % 3 == 2) {
+        encoded[*out_len - 1] = '=';
+    }
+
+    encoded[*out_len] = '\0';
+    return encoded;
+}
+
+static const uint8_t base64_table[] = {
+    // Map base64 characters to their corresponding values
+    ['A'] = 0,  ['B'] = 1,  ['C'] = 2,  ['D'] = 3,  ['E'] = 4,  ['F'] = 5,  ['G'] = 6,  ['H'] = 7,
+    ['I'] = 8,  ['J'] = 9,  ['K'] = 10, ['L'] = 11, ['M'] = 12, ['N'] = 13, ['O'] = 14, ['P'] = 15,
+    ['Q'] = 16, ['R'] = 17, ['S'] = 18, ['T'] = 19, ['U'] = 20, ['V'] = 21, ['W'] = 22, ['X'] = 23,
+    ['Y'] = 24, ['Z'] = 25, ['a'] = 26, ['b'] = 27, ['c'] = 28, ['d'] = 29, ['e'] = 30, ['f'] = 31,
+    ['g'] = 32, ['h'] = 33, ['i'] = 34, ['j'] = 35, ['k'] = 36, ['l'] = 37, ['m'] = 38, ['n'] = 39,
+    ['o'] = 40, ['p'] = 41, ['q'] = 42, ['r'] = 43, ['s'] = 44, ['t'] = 45, ['u'] = 46, ['v'] = 47,
+    ['w'] = 48, ['x'] = 49, ['y'] = 50, ['z'] = 51, ['0'] = 52, ['1'] = 53, ['2'] = 54, ['3'] = 55,
+    ['4'] = 56, ['5'] = 57, ['6'] = 58, ['7'] = 59, ['8'] = 60, ['9'] = 61, ['+'] = 62, ['/'] = 63,
+};
+
+uint8_t* base64_decode(const uint8_t* src, size_t len, size_t* out_len) {
+    if (len % 4 != 0) {
+        return NULL;
+    }
+
+    size_t padding = 0;
+    if (src[len - 1] == '=') {
+        padding++;
+    }
+    if (src[len - 2] == '=') {
+        padding++;
+    }
+
+    *out_len = (len / 4) * 3 - padding;
+    uint8_t* decoded = malloc(*out_len);
+    if (decoded == NULL) {
+        *out_len = 0;
+        return NULL;
+    }
+
+    for (size_t i = 0, j = 0; i < len; i += 4, j += 3) {
+        uint32_t block = 0;
+        for (size_t k = 0; k < 4; k++) {
+            block <<= 6;
+            if (src[i + k] == '=') {
+                padding--;
+            } else {
+                uint8_t index = base64_table[src[i + k]];
+                if (index == 0 && src[i + k] != 'A') {
+                    free(decoded);
+                    return NULL;
+                }
+                block |= index;
+            }
+        }
+
+        decoded[j] = (block >> 16) & 0xFF;
+        if (j + 1 < *out_len) {
+            decoded[j + 1] = (block >> 8) & 0xFF;
+        }
+        if (j + 2 < *out_len) {
+            decoded[j + 2] = block & 0xFF;
+        }
+    }
+
+    return decoded;
+}

libsql-ffi/bundled/sqlean/crypto/base64.c

@@ -0,0 +1,15 @@
+// Copyright (c) 2023 Anton Zhiyanov, MIT License
+// https://github.com/nalgeon/sqlean
+
+// Base64 encoding/decoding (RFC 4648)
+
+#ifndef BASE64_H
+#define BASE64_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+uint8_t* base64_encode(const uint8_t* src, size_t len, size_t* out_len);
+uint8_t* base64_decode(const uint8_t* src, size_t len, size_t* out_len);
+
+#endif /* BASE64_H */