feat: add native database support for e2e tests without Docker (sqlc-dev#4236)

Author: kyleconroy

internal/endtoend/CLAUDE.md

@@ -0,0 +1,117 @@
+# End-to-End Tests - Native Database Setup
+
+This document describes how to set up MySQL and PostgreSQL for running end-to-end tests in environments without Docker, particularly when using an HTTP proxy.
+
+## Overview
+
+The end-to-end tests support three methods for connecting to databases:
+
+1. **Environment Variables**: Set `POSTGRESQL_SERVER_URI` and `MYSQL_SERVER_URI` directly
+2. **Docker**: Automatically starts containers via the docker package
+3. **Native Installation**: Starts existing database services on Linux
+
+## Installing Databases with HTTP Proxy
+
+In environments where DNS doesn't work directly but an HTTP proxy is available (e.g., some CI environments), you need to configure apt to use the proxy before installing packages.
+
+### Configure apt Proxy
+
+```bash
+# Check if HTTP_PROXY is set
+echo $HTTP_PROXY
+
+# Configure apt to use the proxy
+sudo tee /etc/apt/apt.conf.d/99proxy << EOF
+Acquire::http::Proxy "$HTTP_PROXY";
+Acquire::https::Proxy "$HTTPS_PROXY";
+EOF
+
+# Update package lists
+sudo apt-get update -qq
+```
+
+### Install PostgreSQL
+
+```bash
+# Install PostgreSQL
+sudo DEBIAN_FRONTEND=noninteractive apt-get install -y postgresql postgresql-contrib
+
+# Start the service
+sudo service postgresql start
+
+# Set password for postgres user
+sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';"
+
+# Configure pg_hba.conf for password authentication
+# Find the hba_file location:
+sudo -u postgres psql -t -c "SHOW hba_file;"
+
+# Add md5 authentication for localhost (add to the beginning of pg_hba.conf):
+# host    all             all             127.0.0.1/32            md5
+
+# Reload PostgreSQL
+sudo service postgresql reload
+```
+
+### Install MySQL
+
+```bash
+# Pre-configure MySQL root password
+echo "mysql-server mysql-server/root_password password mysecretpassword" | sudo debconf-set-selections
+echo "mysql-server mysql-server/root_password_again password mysecretpassword" | sudo debconf-set-selections
+
+# Install MySQL
+sudo DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server
+
+# Start the service
+sudo service mysql start
+
+# Verify connection
+mysql -uroot -pmysecretpassword -e "SELECT 1;"
+```
+
+## Expected Database Credentials
+
+The native database support expects the following credentials:
+
+### PostgreSQL
+- **URI**: `postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable`
+- **User**: `postgres`
+- **Password**: `postgres`
+- **Port**: `5432`
+
+### MySQL
+- **URI**: `root:mysecretpassword@tcp(localhost:3306)/mysql?multiStatements=true&parseTime=true`
+- **User**: `root`
+- **Password**: `mysecretpassword`
+- **Port**: `3306`
+
+## Running Tests
+
+```bash
+# Run end-to-end tests
+go test -v -run TestReplay -timeout 20m ./internal/endtoend/...
+
+# With verbose logging
+go test -v -run TestReplay -timeout 20m ./internal/endtoend/... 2>&1 | tee test.log
+```
+
+## Troubleshooting
+
+### apt-get times out or fails
+- Ensure HTTP proxy is configured in `/etc/apt/apt.conf.d/99proxy`
+- Check that the proxy URL is correct: `echo $HTTP_PROXY`
+- Try running `sudo apt-get update` first to verify connectivity
+
+### MySQL connection refused
+- Check if MySQL is running: `sudo service mysql status`
+- Verify the password: `mysql -uroot -pmysecretpassword -e "SELECT 1;"`
+- Check if MySQL is listening on TCP: `netstat -tlnp | grep 3306`
+
+### PostgreSQL authentication failed
+- Verify pg_hba.conf has md5 authentication for localhost
+- Check password: `PGPASSWORD=postgres psql -h localhost -U postgres -c "SELECT 1;"`
+- Reload PostgreSQL after config changes: `sudo service postgresql reload`
+
+### DNS resolution fails
+This is expected in some environments. Configure apt proxy as shown above.

internal/endtoend/endtoend_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/sqlc-dev/sqlc/internal/config"
 	"github.com/sqlc-dev/sqlc/internal/opts"
 	"github.com/sqlc-dev/sqlc/internal/sqltest/docker"
+	"github.com/sqlc-dev/sqlc/internal/sqltest/native"
 )
 
 func lineEndings() cmp.Option {
@@ -113,23 +114,63 @@ func TestReplay(t *testing.T) {
 	ctx := context.Background()
 
 	var mysqlURI, postgresURI string
-	if err := docker.Installed(); err == nil {
-		{
-			host, err := docker.StartPostgreSQLServer(ctx)
-			if err != nil {
-				t.Fatalf("starting postgresql failed: %s", err)
+
+	// First, check environment variables
+	if uri := os.Getenv("POSTGRESQL_SERVER_URI"); uri != "" {
+		postgresURI = uri
+	}
+	if uri := os.Getenv("MYSQL_SERVER_URI"); uri != "" {
+		mysqlURI = uri
+	}
+
+	// Try Docker for any missing databases
+	if postgresURI == "" || mysqlURI == "" {
+		if err := docker.Installed(); err == nil {
+			if postgresURI == "" {
+				host, err := docker.StartPostgreSQLServer(ctx)
+				if err != nil {
+					t.Logf("docker postgresql startup failed: %s", err)
+				} else {
+					postgresURI = host
+				}
+			}
+			if mysqlURI == "" {
+				host, err := docker.StartMySQLServer(ctx)
+				if err != nil {
+					t.Logf("docker mysql startup failed: %s", err)
+				} else {
+					mysqlURI = host
+				}
 			}
-			postgresURI = host
 		}
-		{
-			host, err := docker.StartMySQLServer(ctx)
-			if err != nil {
-				t.Fatalf("starting mysql failed: %s", err)
+	}
+
+	// Try native installation for any missing databases (Linux only)
+	if postgresURI == "" || mysqlURI == "" {
+		if err := native.Supported(); err == nil {
+			if postgresURI == "" {
+				host, err := native.StartPostgreSQLServer(ctx)
+				if err != nil {
+					t.Logf("native postgresql startup failed: %s", err)
+				} else {
+					postgresURI = host
+				}
+			}
+			if mysqlURI == "" {
+				host, err := native.StartMySQLServer(ctx)
+				if err != nil {
+					t.Logf("native mysql startup failed: %s", err)
+				} else {
+					mysqlURI = host
+				}
 			}
-			mysqlURI = host
 		}
 	}
 
+	// Log which databases are available
+	t.Logf("PostgreSQL available: %v (URI: %s)", postgresURI != "", postgresURI)
+	t.Logf("MySQL available: %v (URI: %s)", mysqlURI != "", mysqlURI)
+
 	contexts := map[string]textContext{
 		"base": {
 			Mutate:  func(t *testing.T, path string) func(*config.Config) { return func(c *config.Config) {} },
@@ -138,19 +179,20 @@ func TestReplay(t *testing.T) {
 		"managed-db": {
 			Mutate: func(t *testing.T, path string) func(*config.Config) {
 				return func(c *config.Config) {
+					// Add all servers - tests will fail if database isn't available
 					c.Servers = []config.Server{
 						{
 							Name:   "postgres",
 							Engine: config.EnginePostgreSQL,
 							URI:    postgresURI,
 						},
-
 						{
 							Name:   "mysql",
 							Engine: config.EngineMySQL,
 							URI:    mysqlURI,
 						},
 					}
+
 					for i := range c.SQL {
 						switch c.SQL[i].Engine {
 						case config.EnginePostgreSQL:
@@ -172,8 +214,8 @@ func TestReplay(t *testing.T) {
 				}
 			},
 			Enabled: func() bool {
-				err := docker.Installed()
-				return err == nil
+				// Enabled if at least one database URI is available
+				return postgresURI != "" || mysqlURI != ""
 			},
 		},
 	}

internal/sqltest/local/mysql.go

@@ -14,6 +14,7 @@ import (
 	migrate "github.com/sqlc-dev/sqlc/internal/migrations"
 	"github.com/sqlc-dev/sqlc/internal/sql/sqlpath"
 	"github.com/sqlc-dev/sqlc/internal/sqltest/docker"
+	"github.com/sqlc-dev/sqlc/internal/sqltest/native"
 )
 
 var mysqlSync sync.Once
@@ -31,8 +32,15 @@ func MySQL(t *testing.T, migrations []string) string {
 				t.Fatal(err)
 			}
 			dburi = u
+		} else if ierr := native.Supported(); ierr == nil {
+			// Fall back to native installation when Docker is not available
+			u, err := native.StartMySQLServer(ctx)
+			if err != nil {
+				t.Fatal(err)
+			}
+			dburi = u
 		} else {
-			t.Skip("MYSQL_SERVER_URI is empty")
+			t.Skip("MYSQL_SERVER_URI is empty and neither Docker nor native installation is available")
 		}
 	}
 

internal/sqltest/local/postgres.go

@@ -16,6 +16,7 @@ import (
 	"github.com/sqlc-dev/sqlc/internal/pgx/poolcache"
 	"github.com/sqlc-dev/sqlc/internal/sql/sqlpath"
 	"github.com/sqlc-dev/sqlc/internal/sqltest/docker"
+	"github.com/sqlc-dev/sqlc/internal/sqltest/native"
 )
 
 var flight singleflight.Group
@@ -41,8 +42,15 @@ func postgreSQL(t *testing.T, migrations []string, rw bool) string {
 				t.Fatal(err)
 			}
 			dburi = u
+		} else if ierr := native.Supported(); ierr == nil {
+			// Fall back to native installation when Docker is not available
+			u, err := native.StartPostgreSQLServer(ctx)
+			if err != nil {
+				t.Fatal(err)
+			}
+			dburi = u
 		} else {
-			t.Skip("POSTGRESQL_SERVER_URI is empty")
+			t.Skip("POSTGRESQL_SERVER_URI is empty and neither Docker nor native installation is available")
 		}
 	}
 

internal/sqltest/native/enabled.go

@@ -0,0 +1,20 @@
+package native
+
+import (
+	"fmt"
+	"os/exec"
+	"runtime"
+)
+
+// Supported returns nil if native database installation is supported on this platform.
+// Currently only Linux (Ubuntu/Debian) is supported.
+func Supported() error {
+	if runtime.GOOS != "linux" {
+		return fmt.Errorf("native database installation only supported on linux, got %s", runtime.GOOS)
+	}
+	// Check if apt-get is available (Debian/Ubuntu)
+	if _, err := exec.LookPath("apt-get"); err != nil {
+		return fmt.Errorf("apt-get not found: %w", err)
+	}
+	return nil
+}