fix(migrations): robustly strip psql meta commands without breaking SQL

Replace naive PostgreSQL schema preprocessing with a single-pass state machine that correctly distinguishes top-level psql meta-commands from backslashes in valid SQL code, literals, identifiers, comments, and dollar-quoted bodies.

The previous implementation would either leave pg_dump-style backslash directives in place for some schema-loading paths or strip too aggressively, breaking valid SQL containing:
- Backslashes in string literals, including `E'...'` escapes and simple `standard_conforming_strings` variants
- Meta-command text in comments or documentation
- Dollar-quoted function bodies, including Unicode-tagged bodies
- Double-quoted identifiers and identifiers containing `$`

Changes:
- Add engine-aware preprocessing helpers so rollback removal always applies while PostgreSQL preprocessing preserves server-side SQL, including PL/pgSQL bodies and extension/language DDL, before stripping psql meta-commands.
- Replace the line-based PostgreSQL filter with a single-pass state machine that tracks single quotes, double quotes, dollar quotes, line comments, and nested block comments.
- Handle escape-string prefixes, simple `standard_conforming_strings` value variants, Unicode dollar-quote tags, identifier-boundary checks, the documented `\\` separator command, and broader top-level unknown backslash directives.
- Preserve SQL that follows a valid whitespace-delimited inline `\\` separator on the same psql meta-command line, matching tested `psql 17.9` behavior.
- Strip semantic psql commands such as `\connect`, `\copy`, `\gexec`, `\i`, and `\ir` with warnings in parse/codegen paths, but reject them in schema-application paths where sqlc cannot reproduce their effects safely.
- Reject psql conditionals (`\if`, `\elif`, `\else`, `\endif`) during preprocessing instead of flattening branches and changing SQL semantics.
- Treat `standard_conforming_strings` and transaction-scoped script behavior as best-effort parsing aids rather than full psql emulation, and surface an explicit warning when preprocessing has to approximate those semantics.
- Apply shared schema preprocessing and warning propagation in compiler parsing, `createdb`, `verify`, managed `vet`, and PostgreSQL sqltest seeding paths, including stable hashing of preprocessed read-only PostgreSQL fixtures.
- Re-enable the `pg_dump` end-to-end fixture now that its schema parses and seeds successfully.
- Add targeted regression coverage covering:
  * All documented psql meta-commands plus broader top-level unknown backslash directives
  * String literals with backslashes, escape-string prefixes, and simple `standard_conforming_strings` variants
  * Unicode dollar-quote tags, identifier-boundary cases, inline `\\` separator behavior, and rejected conditional directives
  * Semantic-command warnings and apply-mode rejections, `\copy ... from stdin`, line comments, nested block comments, quoted identifiers, helper edge cases, and schema preprocessing rollout
  * Bare CR normalization, CRLF preservation, and managed/PostgreSQL preprocessing behavior

Performance improvements:
- Pre-allocate output buffers with `strings.Builder.Grow()`
- Keep parsing single-pass rather than rescanning line slices
- Consolidate schema preprocessing into engine-aware helpers reused across schema-loading paths

Author: ignatremizov

internal/cmd/createdb.go

@@ -85,7 +85,14 @@ func CreateDB(ctx context.Context, dir, filename, querySetName string, o *Option
 		if err != nil {
 			return fmt.Errorf("read file: %w", err)
 		}
-		ddl = append(ddl, migrations.RemoveRollbackStatements(string(contents)))
+		ddlText, warnings, err := migrations.PreprocessSchemaForApply(string(contents), string(queryset.Engine))
+		if err != nil {
+			return err
+		}
+		for _, warning := range warnings {
+			fmt.Fprintln(o.Stderr, warning)
+		}
+		ddl = append(ddl, ddlText)
 	}
 
 	now := time.Now().UTC().UnixNano()

internal/cmd/generate.go

@@ -306,6 +306,9 @@ func parse(ctx context.Context, name, dir string, sql config.SQL, combo config.C
 		return nil, true
 	}
 	if err := c.ParseCatalog(sql.Schema); err != nil {
+		for _, warning := range c.Warnings() {
+			fmt.Fprintln(stderr, warning)
+		}
 		fmt.Fprintf(stderr, "# package %s\n", name)
 		if parserErr, ok := err.(*multierr.Error); ok {
 			for _, fileErr := range parserErr.Errs() {
@@ -316,6 +319,9 @@ func parse(ctx context.Context, name, dir string, sql config.SQL, combo config.C
 		}
 		return nil, true
 	}
+	for _, warning := range c.Warnings() {
+		fmt.Fprintln(stderr, warning)
+	}
 	if parserOpts.Debug.DumpCatalog {
 		debug.Dump(c.Catalog())
 	}

internal/cmd/process.go

@@ -122,12 +122,12 @@ func processQuerySets(ctx context.Context, rp ResultProcessor, conf *config.Conf
 	if err := grp.Wait(); err != nil {
 		return err
 	}
-	if errored {
-		for i, _ := range stderrs {
-			if _, err := io.Copy(stderr, &stderrs[i]); err != nil {
-				return err
-			}
+	for i := range stderrs {
+		if _, err := io.Copy(stderr, &stderrs[i]); err != nil {
+			return err
 		}
+	}
+	if errored {
 		return fmt.Errorf("errored")
 	}
 	return nil

internal/cmd/verify.go

@@ -102,7 +102,14 @@ func Verify(ctx context.Context, dir, filename string, opts *Options) error {
 				if err != nil {
 					return fmt.Errorf("read file: %w", err)
 				}
-				ddl = append(ddl, migrations.RemoveRollbackStatements(string(contents)))
+				ddlText, warnings, err := migrations.PreprocessSchemaForApply(string(contents), string(current.Engine))
+				if err != nil {
+					return err
+				}
+				for _, warning := range warnings {
+					fmt.Fprintln(stderr, warning)
+				}
+				ddl = append(ddl, ddlText)
 			}
 
 			var codegen plugin.GenerateRequest

internal/cmd/vet.go

@@ -432,7 +432,14 @@ func (c *checker) fetchDatabaseUri(ctx context.Context, s config.SQL) (string, f
 		if err != nil {
 			return "", cleanup, fmt.Errorf("read file: %w", err)
 		}
-		ddl = append(ddl, migrations.RemoveRollbackStatements(string(contents)))
+		ddlText, warnings, err := migrations.PreprocessSchemaForApply(string(contents), string(s.Engine))
+		if err != nil {
+			return "", cleanup, err
+		}
+		for _, warning := range warnings {
+			fmt.Fprintln(c.Stderr, warning)
+		}
+		ddl = append(ddl, ddlText)
 	}
 
 	resp, err := c.Client.CreateDatabase(ctx, &dbmanager.CreateDatabaseRequest{
@@ -549,7 +556,13 @@ func (c *checker) checkSQL(ctx context.Context, s config.SQL) error {
 					if err != nil {
 						return fmt.Errorf("read schema file: %w", err)
 					}
-					ddl := migrations.RemoveRollbackStatements(string(contents))
+					ddl, warnings, err := migrations.PreprocessSchemaForApply(string(contents), string(s.Engine))
+					if err != nil {
+						return err
+					}
+					for _, warning := range warnings {
+						fmt.Fprintln(c.Stderr, warning)
+					}
 					if _, err := db.ExecContext(ctx, ddl); err != nil {
 						return fmt.Errorf("apply schema %s: %w", schema, err)
 					}

internal/compiler/compile.go

@@ -38,8 +38,12 @@ func (c *Compiler) parseCatalog(schemas []string) error {
 			merr.Add(filename, "", 0, err)
 			continue
 		}
-		contents := migrations.RemoveRollbackStatements(string(blob))
-		contents = migrations.RemovePsqlMetaCommands(contents)
+		contents, warnings, err := migrations.PreprocessSchema(string(blob), string(c.conf.Engine))
+		if err != nil {
+			merr.Add(filename, string(blob), 0, err)
+			continue
+		}
+		c.warns = append(c.warns, warnings...)
 		c.schema = append(c.schema, contents)
 
 		// In database-only mode, we parse the schema to validate syntax

internal/compiler/engine.go

@@ -28,6 +28,7 @@ type Compiler struct {
 	selector selector
 
 	schema []string
+	warns  []string
 
 	// databaseOnlyMode indicates that the compiler should use database-only analysis
 	// and skip building the internal catalog from schema files (analyzer.database: only)
@@ -138,6 +139,12 @@ func (c *Compiler) Result() *Result {
 	return c.result
 }
 
+// Warnings returns a copy of any non-fatal schema preprocessing warnings
+// collected while parsing the catalog.
+func (c *Compiler) Warnings() []string {
+	return append([]string(nil), c.warns...)
+}
+
 func (c *Compiler) Close(ctx context.Context) {
 	if c.analyzer != nil {
 		c.analyzer.Close(ctx)

internal/endtoend/testdata/pg_dump/exec.json

@@ -1,6 +1,3 @@
 {
-    "contexts": ["base"],
-    "meta": {
-        "invalid_schema": true
-    }
+    "contexts": ["base"]
 }