diff --git a/docs/user-guide/ddl-reference.md b/docs/user-guide/ddl-reference.md index 871a3daa..d67a3599 100644 --- a/docs/user-guide/ddl-reference.md +++ b/docs/user-guide/ddl-reference.md @@ -50,6 +50,34 @@ remains imperative and explicit. Detection of *incompatible* metadata changes `CREATE` and `CREATE OR REPLACE` apply the new definition regardless. Use caution when changing schemas of in-flight pipelines. +## Dry-run: validate mode + +`mode=validate` is a dry-run. Every statement — `CREATE`, `DROP`, and the +trigger verbs (`FIRE`/`PAUSE`/`RESUME`) — is fully parsed, planned, and +validated (including deployer-level validation), but **no real object is ever +created, updated, or deleted.** A statement that would succeed reports +`(0 rows modified)`; a statement that would fail validation raises the same +error a real run would. Use it to check a script against a live environment +without touching anything: + +``` +jdbc:hoptimator://...;mode=validate +``` + +With respect to `OR REPLACE`, `validate` behaves like `apply`: an +already-existing resource is treated as an in-place update, so a plain +`CREATE` over an existing object validates cleanly instead of erroring. + +Crucially, a dry-run still evolves the **in-memory** schema as it goes, so a +series of statements is validated against each other: a dry-run `DROP VIEW` +followed by a query against that view fails validation, even though no real +`View` object was deleted. Only the deployment is skipped, never the +in-memory bookkeeping. (This is what distinguishes `validate` from the +one-shot `!specify` preview, which renders a single statement's YAML and then +restores the schema.) `validate`, `apply`, and `create` are mutually +exclusive; pick one per connection. + + ## CREATE VIEW ``` diff --git a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlExecutor.java b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlExecutor.java index 1c2d871d..251083f2 100644 --- a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlExecutor.java +++ b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlExecutor.java @@ -126,10 +126,11 @@ public void execute(SqlCreateView create, CalcitePrepare.Context context) { HoptimatorDdlUtils.DdlMode mode = HoptimatorDdlUtils.effectiveMode(create.getReplace(), connection); for (Function function : schemaPlus.getFunctions(pair.right)) { if (function.getParameters().isEmpty()) { - if (mode == HoptimatorDdlUtils.DdlMode.CREATE) { + if (mode.failsIfResourceExists()) { throw new DdlException(create, "View " + pair.right + " already exists. Use CREATE OR REPLACE to update."); } + // Replace the existing in-memory definition. (!specify never reaches plain CREATE VIEW.) pair.left.removeFunction(pair.right); } } @@ -152,14 +153,20 @@ public void execute(SqlCreateView create, CalcitePrepare.Context context) { deployers = DeploymentService.deployers(view, connection); ValidationService.validateOrThrow(deployers, connection); logger.info("Validated view {}", viewName); + // CREATE→create, UPDATE→update, VALIDATE (dry-run)→deploy nothing. The in-memory view is + // registered in every case (VALIDATE keeps it without deploying), so a later statement in + // the same script sees it. if (mode == HoptimatorDdlUtils.DdlMode.UPDATE) { logger.info("Deploying update view {}", viewName); - DeploymentService.update(deployers); - } else { + } else if (mode == HoptimatorDdlUtils.DdlMode.CREATE) { logger.info("Deploying create view {}", viewName); - DeploymentService.create(deployers); + } else { + logger.info("Validating (dry-run) view {}; skipping deployment", viewName); + } + mode.executeDeployers(deployers, connection); + if (mode.shouldDeploy()) { + logger.info("Deployed view {}", viewName); } - logger.info("Deployed view {}", viewName); schemaPlus.add(viewName, viewTable); logger.info("Added view {} to schema {}", viewName, schemaPlus.getName()); } catch (SQLException | RuntimeException e) { @@ -243,12 +250,17 @@ public void execute(SqlCreateTrigger create, CalcitePrepare.Context context) { HoptimatorDdlUtils.DdlMode mode = HoptimatorDdlUtils.effectiveMode(create.getReplace(), connection); if (mode == HoptimatorDdlUtils.DdlMode.UPDATE) { logger.info("Updating trigger {}", name); - DeploymentService.update(deployers); - } else { + } else if (mode == HoptimatorDdlUtils.DdlMode.CREATE) { logger.info("Creating trigger {}", name); - DeploymentService.create(deployers); + } else { + logger.info("Validating (dry-run) trigger {}; skipping deployment", name); + } + // CREATE→create, UPDATE→update, VALIDATE (dry-run)→no-op. Triggers are not part of the + // Calcite schema, so there is no in-memory state to persist. + mode.executeDeployers(deployers, connection); + if (mode.shouldDeploy()) { + logger.info("Deployed trigger {}", name); } - logger.info("Deployed trigger {}", name); logger.info("CREATE TRIGGER {} completed", name); } catch (Exception e) { if (deployers != null) { @@ -336,7 +348,12 @@ public void execute(SqlFireTrigger fire, CalcitePrepare.Context context) { try { logger.info("Firing trigger {} with {} option(s)", name, options.size() - 1); deployers = DeploymentService.deployers(trigger, connection); - DeploymentService.update(deployers); + // No OR REPLACE on FIRE; shouldDeploy() is false only in validate (dry-run) mode. + if (HoptimatorDdlUtils.effectiveMode(false, connection).shouldDeploy()) { + DeploymentService.update(deployers); + } else { + logger.info("Validated FIRE TRIGGER (dry-run) {}; skipping deployment", name); + } logger.info("FIRE TRIGGER {} completed", name); } catch (Exception e) { if (deployers != null) { @@ -366,8 +383,13 @@ public void execute(SqlDropTrigger drop, CalcitePrepare.Context context) { try { logger.info("Deleting trigger {}", name); deployers = DeploymentService.deployers(trigger, connection); - DeploymentService.delete(deployers); - logger.info("Deleted trigger {}", name); + // No OR REPLACE on DROP; shouldDeploy() is false only in validate (dry-run) mode. + if (HoptimatorDdlUtils.effectiveMode(false, connection).shouldDeploy()) { + DeploymentService.delete(deployers); + logger.info("Deleted trigger {}", name); + } else { + logger.info("Validated DROP TRIGGER (dry-run) {}; skipping deletion", name); + } logger.info("DROP TRIGGER {} completed", name); } catch (Exception e) { if (deployers != null) { @@ -403,7 +425,12 @@ private void updateTriggerPausedState(SqlNode sqlNode, SqlIdentifier triggerName try { logger.info("Updating trigger {} with paused state: {}", name, paused); deployers = DeploymentService.deployers(trigger, connection); - DeploymentService.update(deployers); + // No OR REPLACE on PAUSE/RESUME; shouldDeploy() is false only in validate (dry-run) mode. + if (HoptimatorDdlUtils.effectiveMode(false, connection).shouldDeploy()) { + DeploymentService.update(deployers); + } else { + logger.info("Validated paused-state update (dry-run) for trigger {}; skipping deployment", name); + } logger.info("Successfully updated trigger {} with paused state: {}", name, paused); } catch (Exception e) { if (deployers != null) { @@ -450,6 +477,9 @@ public void execute(SqlDropObject drop, CalcitePrepare.Context context) { tablePath.add(tableName); Collection deployers = null; + // No OR REPLACE on DROP; shouldDeploy() is false only in validate (dry-run) mode, where we + // skip the real deletion but still evolve the in-memory schema below. + final boolean shouldDeploy = HoptimatorDdlUtils.effectiveMode(false, connection).shouldDeploy(); try { if (table instanceof MaterializedViewTable) { if (!(drop instanceof SqlDropMaterializedView)) { @@ -459,8 +489,14 @@ public void execute(SqlDropObject drop, CalcitePrepare.Context context) { MaterializedViewTable materializedViewTable = (MaterializedViewTable) table; View view = new View(tablePath, materializedViewTable.viewSql()); deployers = DeploymentService.deployers(view, connection); - logger.info("Deleting materialized view {}", tableName); - DeploymentService.delete(deployers); + if (shouldDeploy) { + logger.info("Deleting materialized view {}", tableName); + DeploymentService.delete(deployers); + } else { + logger.info("Validated DROP (dry-run) for materialized view {}; skipping deletion", tableName); + } + // Always evolve the in-memory schema — even in a dry-run, dropping the view must make a + // subsequent reference to it fail validation, regardless of whether a real object existed. schemaPlus.removeTable(tableName); logger.info("Removed materialized table {} from schema {}", tableName, schemaPlus.getName()); } else if (table instanceof ViewTable) { @@ -471,8 +507,12 @@ public void execute(SqlDropObject drop, CalcitePrepare.Context context) { ViewTable viewTable = (ViewTable) table; View view = new View(tablePath, viewTable.getViewSql()); deployers = DeploymentService.deployers(view, connection); - logger.info("Deleting view {}", tableName); - DeploymentService.delete(deployers); + if (shouldDeploy) { + logger.info("Deleting view {}", tableName); + DeploymentService.delete(deployers); + } else { + logger.info("Validated DROP (dry-run) for view {}; skipping deletion", tableName); + } schemaPlus.removeTable(tableName); logger.info("Removed view {} from schema {}", tableName, schemaPlus.getName()); } else if (table instanceof HoptimatorJdbcTable || table instanceof TemporaryTable) { @@ -491,11 +531,15 @@ public void execute(SqlDropObject drop, CalcitePrepare.Context context) { } // Pre-delete dependency guard. PendingDelete is the explicit "delete intent" signal // — only validators that key off it (the K8s dep checker) fire here. The check throws - // before any deployer-level state change. + // before any deployer-level state change. This validation runs in dry-run too. ValidationService.validateOrThrow(new PendingDelete<>(source), connection); deployers = DeploymentService.deployers(source, connection); - logger.info("Deleting table {}", tableName); - DeploymentService.delete(deployers); + if (shouldDeploy) { + logger.info("Deleting table {}", tableName); + DeploymentService.delete(deployers); + } else { + logger.info("Validated DROP (dry-run) for table {}; skipping deletion", tableName); + } schemaPlus.removeTable(tableName); logger.info("Removed table {} from schema {}", tableName, schemaPlus.getName()); } else { diff --git a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlUtils.java b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlUtils.java index 7ef9bd1b..1ddc9a3d 100644 --- a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlUtils.java +++ b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlUtils.java @@ -98,6 +98,12 @@ private HoptimatorDdlUtils() { *
  • {@code apply} — declarative, K8s-style reconciliation. Both {@code CREATE} and * {@code CREATE OR REPLACE} converge the resource to the declared definition, so * running the same script twice is a no-op. Idempotent by design.
    • + *
  • {@code validate} — dry-run. Resolves to {@link DdlMode#VALIDATE}: every statement is + * fully validated (deployers resolved and validated) but no real object is created, + * updated, or deleted. Unlike {@code !specify}, in-memory schema changes are kept, so a + * series of statements validates against each other. With respect to {@code OR REPLACE} + * it behaves like {@code apply}: an already-existing resource converges, it does not + * error.
    • * * *

    Set per connection on the JDBC URL, e.g. {@code jdbc:hoptimator://...;mode=apply} @@ -111,15 +117,23 @@ private HoptimatorDdlUtils() { /** Apply-mode value of {@link #MODE_PROPERTY}. */ public static final String MODE_APPLY = "apply"; + /** Validate-only (dry-run) value of {@link #MODE_PROPERTY}. Resolves to {@link DdlMode#VALIDATE}. */ + public static final String MODE_VALIDATE = "validate"; + /** * Resolves the effective {@link DdlMode} for a {@code CREATE} statement, combining the * statement's {@code OR REPLACE} flag with the connection's {@link #MODE_PROPERTY}. * *

    In {@code create} mode (the default): {@code CREATE} → {@link DdlMode#CREATE} and * {@code CREATE OR REPLACE} → {@link DdlMode#UPDATE}. In {@code apply} mode: both forms - * resolve to {@link DdlMode#UPDATE}, making CREATE idempotent. + * resolve to {@link DdlMode#UPDATE}, making CREATE idempotent. In {@code validate} mode: + * both forms resolve to {@link DdlMode#VALIDATE}, the non-deploying dry-run (which, like + * apply, never errors on an already-existing resource). */ static DdlMode effectiveMode(boolean orReplace, HoptimatorConnection conn) { + if (isValidateMode(conn)) { + return DdlMode.VALIDATE; + } if (isApplyMode(conn)) { return DdlMode.UPDATE; } @@ -128,12 +142,21 @@ static DdlMode effectiveMode(boolean orReplace, HoptimatorConnection conn) { /** Whether the connection is configured for apply-mode DDL. */ static boolean isApplyMode(HoptimatorConnection conn) { + return modeIs(conn, MODE_APPLY); + } + + /** Whether the connection is configured for validate-only (dry-run) DDL. */ + static boolean isValidateMode(HoptimatorConnection conn) { + return modeIs(conn, MODE_VALIDATE); + } + + private static boolean modeIs(HoptimatorConnection conn, String expected) { Properties props = conn.connectionProperties(); if (props == null) { return false; } String mode = props.getProperty(MODE_PROPERTY, MODE_CREATE); - return MODE_APPLY.equalsIgnoreCase(mode); + return expected.equalsIgnoreCase(mode); } /** @@ -160,8 +183,44 @@ public static final class SpecifyResult { } /** - * Controls whether a DDL operation performs a real deployment (CREATE or UPDATE) - * or a dry-run (SPECIFY). + * The four ways a {@code CREATE}/{@code DROP} statement can be carried out. Rather than have + * callers test {@code mode == X} or reason about "mutable" schemas, each mode answers a few + * intent questions that read directly at the call site: + * + *

    + * + *

    {@code VALIDATE} (dry-run) is {@code shouldDeploy()==false} yet {@code shouldRestoreSchema() + * ==false}: it deploys nothing but still evolves the in-memory schema, so e.g. a dry-run + * {@code DROP VIEW} followed by a query against that view fails validation, even though no real + * {@code View} object was deleted. + * + *

    Why two non-deploying modes (VALIDATE keeps schema changes, SPECIFY restores) instead of + * one? It is tempting to unify them and have the {@code !specify} caller own the restore — + * e.g. by snapshotting the schema and discarding changes, or by running against a throwaway + * mutable root. Calcite does not make that clean: + *

    + * The robust mechanism is the targeted per-statement snapshot the {@code processCreate*} + * methods already compute (they know the exact table/view/sink added). So we keep the snapshot + * there and let {@link #shouldRestoreSchema()} decide whether to discard it. */ enum DdlMode { CREATE { @@ -172,7 +231,17 @@ List executeDeployers(Collection deployers, Connection conn) t } @Override - boolean mutable() { + boolean shouldDeploy() { + return true; + } + + @Override + boolean shouldRestoreSchema() { + return false; + } + + @Override + boolean failsIfResourceExists() { return true; } }, @@ -184,9 +253,19 @@ List executeDeployers(Collection deployers, Connection conn) t } @Override - boolean mutable() { + boolean shouldDeploy() { return true; } + + @Override + boolean shouldRestoreSchema() { + return false; + } + + @Override + boolean failsIfResourceExists() { + return false; + } }, SPECIFY { @Override @@ -199,14 +278,56 @@ List executeDeployers(Collection deployers, Connection conn) t } @Override - boolean mutable() { + boolean shouldDeploy() { + return false; + } + + @Override + boolean shouldRestoreSchema() { + return true; + } + + @Override + boolean failsIfResourceExists() { + return false; + } + }, + VALIDATE { + @Override + List executeDeployers(Collection deployers, Connection conn) { + // Dry-run: validation has already happened by the time we get here. Deploy nothing and + // render nothing. The in-memory schema changes are kept (shouldRestoreSchema()==false) + // so a series of statements validates against each other. + return Collections.emptyList(); + } + + @Override + boolean shouldDeploy() { + return false; + } + + @Override + boolean shouldRestoreSchema() { + return false; + } + + @Override + boolean failsIfResourceExists() { return false; } }; + /** Runs the deployers per this mode: create, update, render specs, or nothing. */ abstract List executeDeployers(Collection deployers, Connection conn) throws SQLException; - abstract boolean mutable(); + /** Whether this mode makes real changes to deployed resources (create/update/delete). */ + abstract boolean shouldDeploy(); + + /** Whether in-memory schema changes should be discarded after the statement (the {@code !specify} render). */ + abstract boolean shouldRestoreSchema(); + + /** Whether a plain {@code CREATE} over an already-existing resource is an error (strict create). */ + abstract boolean failsIfResourceExists(); } // N.B. copy-pasted from Apache Calcite @@ -354,8 +475,9 @@ static SpecifyResult processCreateMaterializedView(CalcitePrepare.Context ctx, RelRoot root = prepare.convert(ctx, sql).root; RelDataType sinkRowType = root.rel.getRowType(); - // Navigate to the schema (mutable only when actually deploying). - final Pair pair = schema(ctx, mode.mutable(), create.name); + // Navigate to the schema. !specify (SPECIFY) works against the read-only root and restores; + // every other mode keeps its changes, so it uses the mutable root. + final Pair pair = schema(ctx, !mode.shouldRestoreSchema(), create.name); if (pair.left == null) { throw new SQLException("Schema for " + create.name + " not found."); } @@ -367,25 +489,17 @@ static SpecifyResult processCreateMaterializedView(CalcitePrepare.Context ctx, throw new SQLException( "Cannot overwrite physical table " + pair.right + " with a view."); } - // A view already exists. The executor pre-resolved apply-mode into DdlMode.UPDATE, - // so UPDATE always means "converge to this definition". Strict CREATE is the only - // path that errors. SPECIFY (dry-run) keeps the original syntax-driven preview so - // it accurately reflects what a real run would do. + // Strict CREATE is the only mode that errors on an existing resource; UPDATE (apply mode / + // OR REPLACE), VALIDATE (dry-run), and SPECIFY (render) all converge on it. boolean replaceExisting; - if (mode == DdlMode.UPDATE) { - replaceExisting = true; - } else if (mode == DdlMode.CREATE) { + if (mode.failsIfResourceExists()) { if (!create.ifNotExists) { throw new SQLException( "View " + pair.right + " already exists. Use CREATE OR REPLACE to update."); } replaceExisting = false; - } else { // SPECIFY - if (!create.ifNotExists && !create.getReplace()) { - throw new SQLException( - "View " + pair.right + " already exists. Use CREATE OR REPLACE to update."); - } - replaceExisting = create.getReplace(); + } else { + replaceExisting = true; } if (replaceExisting) { schemaPlus.removeTable(pair.right); @@ -442,20 +556,22 @@ static SpecifyResult processCreateMaterializedView(CalcitePrepare.Context ctx, ValidationService.validateOrThrow(deployers, conn); logger.info("Validated materialized view {}", viewName); - // Execute (create/update) or collect specs (specify). + // Execute (create/update), render specs (specify), or do nothing (validate dry-run). if (mode == DdlMode.UPDATE) { logger.info("Deploying update materialized view {}", viewName); } else if (mode == DdlMode.CREATE) { logger.info("Deploying create materialized view {}", viewName); + } else if (mode == DdlMode.VALIDATE) { + logger.info("Validating (dry-run) materialized view {}", viewName); } else { logger.info("Specifying materialized view {}", viewName); } List specs = mode.executeDeployers(deployers, conn); - if (mode.mutable()) { - logger.info("Deployed materialized view {}", viewName); - } else { - // SPECIFY (dry-run): roll back any side effects made by deployers during specify(). + if (mode.shouldRestoreSchema()) { + // Render-only (!specify): roll back any side effects deployers made while producing specs. DeploymentService.restore(deployers); + } else if (mode.shouldDeploy()) { + logger.info("Deployed materialized view {}", viewName); } success = true; return new SpecifyResult(specs, sinkRowType, viewPath); @@ -467,12 +583,10 @@ static SpecifyResult processCreateMaterializedView(CalcitePrepare.Context ctx, } throw e; } finally { - // Restore the schema snapshot. - // For SPECIFY (dry-run): always restore — the view was temporarily added to the schema - // for planning purposes and must be removed afterward. - // For CREATE/UPDATE on failure: restore to undo the partial schema mutation. - // For CREATE/UPDATE on success: do NOT restore — the view should remain in the schema. - if (!success || !mode.mutable()) { + // Restore the in-memory schema only when the mode wants it discarded (SPECIFY render) or the + // statement failed (undo the partial mutation). On success, CREATE/UPDATE/VALIDATE all keep + // the view in-memory so subsequent statements see it — VALIDATE does so without deploying. + if (!success || mode.shouldRestoreSchema()) { if (schemaSnapshot != null) { if (schemaSnapshot.right != null) { schemaSnapshot.left.add(viewName, schemaSnapshot.right); @@ -515,11 +629,12 @@ static SpecifyResult processCreateTable(CalcitePrepare.Context ctx, HoptimatorCo } boolean isNewSchema = false; - Pair pair = schema(ctx, mode.mutable(), create.name); + // !specify (SPECIFY) navigates the read-only root and restores; other modes keep changes. + Pair pair = schema(ctx, !mode.shouldRestoreSchema(), create.name); if (pair.left == null) { // If the schema is not found, it might be because it's a 3-level path (CATALOG.SCHEMA.TABLE) if (create.name.names.size() > 2) { - pair = catalog(ctx, mode.mutable(), create.name); + pair = catalog(ctx, !mode.shouldRestoreSchema(), create.name); isNewSchema = true; if (pair.left == null) { throw new SQLException("Catalog for " + create.name + " not found."); @@ -541,17 +656,9 @@ static SpecifyResult processCreateTable(CalcitePrepare.Context ctx, HoptimatorCo } if (!isNewSchema && schemaPlus.tables().get(tableName) != null) { - // Strict CREATE without IF NOT EXISTS is the only path that errors. UPDATE - // (apply mode or explicit OR REPLACE) targets the existing table; SPECIFY - // (dry-run) preserves its syntax-driven semantics. - boolean wouldFail; - if (mode == DdlMode.UPDATE) { - wouldFail = false; - } else if (mode == DdlMode.CREATE) { - wouldFail = !create.ifNotExists; - } else { // SPECIFY - wouldFail = !create.ifNotExists && !create.getReplace(); - } + // Strict CREATE is the only mode that errors on an existing table; UPDATE (apply mode / + // OR REPLACE), VALIDATE (dry-run) and SPECIFY (render) all converge on it. + boolean wouldFail = mode.failsIfResourceExists() && !create.ifNotExists; if (wouldFail) { throw new SQLException( "Table " + tableName + " already exists. Use CREATE OR REPLACE to update."); @@ -654,15 +761,17 @@ public RexNode newColumnDefaultValue(RelOptTable table, int iColumn, logger.info("Deploying update table {}", source); } else if (mode == DdlMode.CREATE) { logger.info("Deploying create table {}", source); + } else if (mode == DdlMode.VALIDATE) { + logger.info("Validating (dry-run) table {}", source); } else { logger.info("Specifying table {}", source); } List specs = mode.executeDeployers(deployers, conn); - if (mode.mutable()) { - logger.info("Deployed table {}", source); - } else { - // SPECIFY (dry-run): roll back any side effects made by deployers during specify() + if (mode.shouldRestoreSchema()) { + // Render-only (!specify): roll back any side effects deployers made while producing specs. DeploymentService.restore(deployers); + } else if (mode.shouldDeploy()) { + logger.info("Deployed table {}", source); } success = true; return new SpecifyResult(specs, rowType, tablePath); @@ -674,10 +783,10 @@ public RexNode newColumnDefaultValue(RelOptTable table, int iColumn, } throw e; } finally { - // For SPECIFY (dry-run): always restore schema. - // For CREATE/UPDATE on success: do NOT restore. - // For CREATE/UPDATE on failure: restore. - if (!success || !mode.mutable()) { + // Restore the in-memory schema only when the mode wants it discarded (SPECIFY render) or the + // statement failed. On success, CREATE/UPDATE/VALIDATE keep the table in-memory so subsequent + // statements validate against it — VALIDATE does so without deploying. + if (!success || mode.shouldRestoreSchema()) { if (schemaSnapshot != null) { if (schemaSnapshot.right == null) { schemaSnapshot.left.removeTable(tableName); @@ -728,10 +837,11 @@ static SpecifyResult processCreateDatabase(HoptimatorConnection conn, ValidationService.validateOrThrow(deployers, conn); List specs = mode.executeDeployers(deployers, conn); - if (mode.mutable()) { - logger.info("Deployed database {}", name); - } else { + if (mode.shouldRestoreSchema()) { + // Render-only (!specify): roll back any side effects deployers made while producing specs. DeploymentService.restore(deployers); + } else if (mode.shouldDeploy()) { + logger.info("Deployed database {}", name); } return new SpecifyResult(specs, null, Collections.singletonList(name)); } catch (SQLException | RuntimeException e) { diff --git a/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlExecutorTest.java b/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlExecutorTest.java index a902cccf..3c7c245a 100644 --- a/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlExecutorTest.java +++ b/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlExecutorTest.java @@ -49,6 +49,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.when; @@ -1169,6 +1170,137 @@ void testResumeTriggerValidationFailure() { assertTrue(ex.getMessage().contains("validation failed")); } + // --------------------------------------------------------------------------- + // Validate (dry-run) mode + // --------------------------------------------------------------------------- + + /** Switches the shared connection into validate (dry-run) mode. */ + private void enableValidateMode() { + connection.connectionProperties() + .setProperty(HoptimatorDdlUtils.MODE_PROPERTY, HoptimatorDdlUtils.MODE_VALIDATE); + } + + @Test + void testValidateModeCreateViewPersistsInMemoryWithoutDeploying() { + enableValidateMode(); + HoptimatorDdlExecutor executor = new HoptimatorDdlExecutor(connection); + CalcitePrepare.Context context = connection.createPrepareContext(); + SqlNode node = HoptimatorDriver.parseQuery(connection, + "CREATE VIEW \"dryRunView\" AS SELECT 1 AS \"col1\""); + + assertDoesNotThrow(() -> executor.executeDdl(context, node)); + + // No real deployment happened... + mockDeploymentService.verify(() -> DeploymentService.create(any()), never()); + mockDeploymentService.verify(() -> DeploymentService.update(any()), never()); + + // ...but the view IS registered in-memory, so a subsequent statement sees it: dropping it + // resolves (and likewise performs no real deletion). + SqlNode dropNode = HoptimatorDriver.parseQuery(connection, "DROP VIEW \"dryRunView\""); + assertDoesNotThrow(() -> executor.executeDdl(context, dropNode)); + mockDeploymentService.verify(() -> DeploymentService.delete(any()), never()); + } + + @Test + void testValidateModeCreateTableSkipsDeployment() { + enableValidateMode(); + HoptimatorDdlExecutor executor = new HoptimatorDdlExecutor(connection); + CalcitePrepare.Context context = connection.createPrepareContext(); + SqlNode node = HoptimatorDriver.parseQuery(connection, + "CREATE TABLE \"DEFAULT\".\"dryRunTable\" (\"col1\" VARCHAR)"); + + assertDoesNotThrow(() -> executor.executeDdl(context, node)); + + mockDeploymentService.verify(() -> DeploymentService.create(any()), never()); + mockDeploymentService.verify(() -> DeploymentService.update(any()), never()); + } + + @Test + void testValidateModeCreateViewOnExistingDoesNotErrorOrDeploy() { + HoptimatorDdlExecutor executor = new HoptimatorDdlExecutor(connection); + CalcitePrepare.Context context = connection.createPrepareContext(); + // A view already exists. + addViewTableToDefaultSchema(context, "existingView", "SELECT 1 AS \"col1\""); + + enableValidateMode(); + // Plain CREATE (no OR REPLACE) over an existing view: in validate mode this is apply-like, + // so it validates without error and without deploying — unlike strict create mode. + SqlNode node = HoptimatorDriver.parseQuery(connection, + "CREATE VIEW \"existingView\" AS SELECT 1 AS \"col1\""); + + assertDoesNotThrow(() -> executor.executeDdl(context, node)); + mockDeploymentService.verify(() -> DeploymentService.create(any()), never()); + mockDeploymentService.verify(() -> DeploymentService.update(any()), never()); + } + + @Test + void testValidateModeDropViewRemovesInMemoryButSkipsDeletion() { + HoptimatorDdlExecutor executor = new HoptimatorDdlExecutor(connection); + CalcitePrepare.Context context = connection.createPrepareContext(); + addViewTableToDefaultSchema(context, "dryRunDropView", "SELECT 1 AS \"col1\""); + + enableValidateMode(); + SqlNode dropNode = HoptimatorDriver.parseQuery(connection, "DROP VIEW \"dryRunDropView\""); + assertDoesNotThrow(() -> executor.executeDdl(context, dropNode)); + + // No real deletion happened... + mockDeploymentService.verify(() -> DeploymentService.delete(any()), never()); + + // ...but the in-memory drop took effect: dropping it again can't find it. This is the key + // property — validating a script that references a dropped view must fail. + SqlNode dropAgain = HoptimatorDriver.parseQuery(connection, "DROP VIEW \"dryRunDropView\""); + HoptimatorDdlExecutor.DdlException ex = assertThrows( + HoptimatorDdlExecutor.DdlException.class, + () -> executor.executeDdl(context, dropAgain)); + assertTrue(ex.getMessage().contains("not found")); + } + + @Test + void testValidateModeCreateTriggerSkipsDeployment() { + enableValidateMode(); + HoptimatorDdlExecutor executor = new HoptimatorDdlExecutor(connection); + CalcitePrepare.Context context = connection.createPrepareContext(); + SqlNode node = HoptimatorDriver.parseQuery(connection, + "CREATE TRIGGER \"dryRunTrigger\" ON \"UTIL\".\"PRINT\" AS 'myJob'"); + + assertDoesNotThrow(() -> executor.executeDdl(context, node)); + mockDeploymentService.verify(() -> DeploymentService.create(any()), never()); + mockDeploymentService.verify(() -> DeploymentService.update(any()), never()); + } + + @Test + void testValidateModeDropTriggerSkipsDeletion() { + enableValidateMode(); + HoptimatorDdlExecutor executor = new HoptimatorDdlExecutor(connection); + CalcitePrepare.Context context = connection.createPrepareContext(); + SqlNode node = HoptimatorDriver.parseQuery(connection, "DROP TRIGGER \"dryRunTrigger\""); + + assertDoesNotThrow(() -> executor.executeDdl(context, node)); + mockDeploymentService.verify(() -> DeploymentService.delete(any()), never()); + } + + @Test + void testValidateModeFireTriggerSkipsDeployment() { + enableValidateMode(); + HoptimatorDdlExecutor executor = new HoptimatorDdlExecutor(connection); + CalcitePrepare.Context context = connection.createPrepareContext(); + SqlNode node = HoptimatorDriver.parseQuery(connection, "FIRE TRIGGER \"dryRunTrigger\""); + + assertDoesNotThrow(() -> executor.executeDdl(context, node)); + mockDeploymentService.verify(() -> DeploymentService.update(any()), never()); + } + + @Test + void testValidateModePauseTriggerSkipsDeployment() { + enableValidateMode(); + HoptimatorDdlExecutor executor = new HoptimatorDdlExecutor(connection); + CalcitePrepare.Context context = connection.createPrepareContext(); + SqlNode node = HoptimatorDriver.parseQuery(connection, "PAUSE TRIGGER \"dryRunTrigger\""); + + assertDoesNotThrow(() -> executor.executeDdl(context, node)); + mockDeploymentService.verify(() -> DeploymentService.update(any()), never()); + } + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- diff --git a/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlUtilsTest.java b/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlUtilsTest.java index a0c51dc3..37e978cf 100644 --- a/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlUtilsTest.java +++ b/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlUtilsTest.java @@ -60,6 +60,7 @@ import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; @@ -873,18 +874,35 @@ void ddlModeSpecifyExecuteDeployersWithNoDeployersReturnsEmptyList() throws SQLE } @Test - void ddlModeSpecifyMutableReturnsFalse() { - assertFalse(HoptimatorDdlUtils.DdlMode.SPECIFY.mutable()); - } + void ddlModePredicatesDescribeEachMode() { + // CREATE / UPDATE deploy for real and keep their schema changes; CREATE is strict. + assertTrue(HoptimatorDdlUtils.DdlMode.CREATE.shouldDeploy()); + assertFalse(HoptimatorDdlUtils.DdlMode.CREATE.shouldRestoreSchema()); + assertTrue(HoptimatorDdlUtils.DdlMode.CREATE.failsIfResourceExists()); - @Test - void ddlModeCreateMutableReturnsTrue() { - assertTrue(HoptimatorDdlUtils.DdlMode.CREATE.mutable()); + assertTrue(HoptimatorDdlUtils.DdlMode.UPDATE.shouldDeploy()); + assertFalse(HoptimatorDdlUtils.DdlMode.UPDATE.shouldRestoreSchema()); + assertFalse(HoptimatorDdlUtils.DdlMode.UPDATE.failsIfResourceExists()); + + // SPECIFY (!specify render) deploys nothing and discards its schema changes. + assertFalse(HoptimatorDdlUtils.DdlMode.SPECIFY.shouldDeploy()); + assertTrue(HoptimatorDdlUtils.DdlMode.SPECIFY.shouldRestoreSchema()); + assertFalse(HoptimatorDdlUtils.DdlMode.SPECIFY.failsIfResourceExists()); } @Test - void ddlModeUpdateMutableReturnsTrue() { - assertTrue(HoptimatorDdlUtils.DdlMode.UPDATE.mutable()); + void ddlModeValidateKeepsSchemaButDeploysNothing() throws SQLException { + // The dry-run contract: VALIDATE deploys nothing, keeps in-memory schema changes (so a series + // of statements validates against each other), and never errors on an existing resource. + assertFalse(HoptimatorDdlUtils.DdlMode.VALIDATE.shouldDeploy()); + assertFalse(HoptimatorDdlUtils.DdlMode.VALIDATE.shouldRestoreSchema()); + assertFalse(HoptimatorDdlUtils.DdlMode.VALIDATE.failsIfResourceExists()); + + Deployer deployer = mock(Deployer.class); + List result = HoptimatorDdlUtils.DdlMode.VALIDATE.executeDeployers( + List.of(deployer), null); + assertTrue(result.isEmpty()); + verifyNoInteractions(deployer); } // ── ColumnDef tests ────────────────────────────────────────────────────────── @@ -1784,4 +1802,41 @@ void testIsApplyMode() { assertFalse(HoptimatorDdlUtils.isApplyMode(connectionWith(new Properties()))); } + + @Test + void testEffectiveModeValidateMapsBothCreateFormsToValidate() { + Properties props = new Properties(); + props.setProperty(HoptimatorDdlUtils.MODE_PROPERTY, HoptimatorDdlUtils.MODE_VALIDATE); + HoptimatorConnection conn = connectionWith(props); + + // validate (dry-run) resolves to the non-deploying VALIDATE mode, and is apply-like w.r.t. + // OR REPLACE: both plain CREATE and CREATE OR REPLACE resolve to VALIDATE. + assertEquals(HoptimatorDdlUtils.DdlMode.VALIDATE, + HoptimatorDdlUtils.effectiveMode(false, conn)); + assertEquals(HoptimatorDdlUtils.DdlMode.VALIDATE, + HoptimatorDdlUtils.effectiveMode(true, conn)); + } + + @Test + void testEffectiveModeValidateIsCaseInsensitive() { + Properties props = new Properties(); + props.setProperty(HoptimatorDdlUtils.MODE_PROPERTY, "VALIDATE"); + HoptimatorConnection conn = connectionWith(props); + + assertEquals(HoptimatorDdlUtils.DdlMode.VALIDATE, + HoptimatorDdlUtils.effectiveMode(false, conn)); + } + + @Test + void testIsValidateMode() { + Properties validateProps = new Properties(); + validateProps.setProperty(HoptimatorDdlUtils.MODE_PROPERTY, HoptimatorDdlUtils.MODE_VALIDATE); + assertTrue(HoptimatorDdlUtils.isValidateMode(connectionWith(validateProps))); + + Properties applyProps = new Properties(); + applyProps.setProperty(HoptimatorDdlUtils.MODE_PROPERTY, HoptimatorDdlUtils.MODE_APPLY); + assertFalse(HoptimatorDdlUtils.isValidateMode(connectionWith(applyProps))); + + assertFalse(HoptimatorDdlUtils.isValidateMode(connectionWith(new Properties()))); + } } diff --git a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/TestSqlScripts.java b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/TestSqlScripts.java index 186964b9..6c250697 100644 --- a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/TestSqlScripts.java +++ b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/TestSqlScripts.java @@ -29,6 +29,11 @@ public void k8sApplyMode() throws Exception { run("k8s-apply-mode.id", "mode=apply"); } + @Test + public void k8sValidateMode() throws Exception { + run("k8s-validate-mode.id", "mode=validate"); + } + @Test public void k8sMetadataTables() throws Exception { run("k8s-metadata.id"); diff --git a/hoptimator-k8s/src/test/resources/k8s-validate-mode.id b/hoptimator-k8s/src/test/resources/k8s-validate-mode.id new file mode 100644 index 00000000..973e94d4 --- /dev/null +++ b/hoptimator-k8s/src/test/resources/k8s-validate-mode.id @@ -0,0 +1,38 @@ +!set outputformat mysql +!use k8s + +-- In validate (dry-run) mode, statements are validated and the in-memory schema evolves across +-- the script, but nothing is ever deployed. A statement that would succeed reports +-- (0 rows modified). + +create materialized view ads.audience as select first_name, last_name from ads.page_views natural join profile.members; +(0 rows modified) + +!update + +-- Validation still enforces real contracts. Overwriting a physical table with a view fails. + +create or replace materialized view ads.page_views as select first_name, last_name from profile.members; +Cannot overwrite physical table PAGE_VIEWS with a view. +!error + +-- An unknown schema is rejected during validation. + +create or replace materialized view invalid.page_views as select first_name, last_name from ads.page_views natural join profile.members; +Schema for INVALID.PAGE_VIEWS not found. +!error + +-- The dry-run CREATE evolved the in-memory schema, so the view now exists and can be dropped +-- (again without deleting any real object). + +drop materialized view ads.audience; +(0 rows modified) + +!update + +-- The in-memory DROP took effect: dropping it again fails, even though no real View was ever +-- created or deleted. This is what lets a multi-statement script validate correctly. + +drop materialized view ads.audience; +Element AUDIENCE not found +!error