Automatically turn off DEFENSIVE mode in the shell tool when executing scripts generated by the ".dump" command against an empty database. Add a warning to the top of generated ".dump" scripts that populate virtual tables.

FossilOrigin-Name: cd016f26bb61549a304f2148035e050f76a8f4a35cdb7131bba2f5fc5d09f49e

Author: dan

manifest

@@ -1,5 +1,5 @@
-C Minor\schange\sto\sos_unix.c\sto\sfacilitate\s100%\sMC/DC\stesting.
-D 2024-01-08T15:23:45.210
+C Automatically\sturn\soff\sDEFENSIVE\smode\sin\sthe\sshell\stool\swhen\sexecuting\sscripts\sgenerated\sby\sthe\s".dump"\scommand\sagainst\san\sempty\sdatabase.\sAdd\sa\swarning\sto\sthe\stop\sof\sgenerated\s".dump"\sscripts\sthat\spopulate\svirtual\stables.
+D 2024-01-08T19:55:40.348
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -738,7 +738,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c e25f51a473a5f30a0d978e4df2aaa98aeec84eac29ecae1ad4708a6c3e669345
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
 F src/select.c f1a81ff4f8e9e76c224e2ab3a4baa799add0db22158c7fcede65d8cc4a6fa2da
-F src/shell.c.in 85f8d52fa4f7773823736dd39d0a268fd739207fcae95883c9ec8ce4af59f7df
+F src/shell.c.in 3d19abd924ed1cec9c9908d5a10cb1580b8ca30df24c26bfe80efa0c00f664d8
 F src/sqlite.h.in 61a60b4ea04db8ead15e1579b20b64cb56e9f55d52c5f9f9694de630110593a3
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54
@@ -1588,6 +1588,7 @@ F test/shell5.test c8b6c54f26ec537f8558273d7ed293ca3725ef42e6b12b8f151718628bd14
 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3
 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f
 F test/shell8.test 3fd093d481aaa94dc77fb73f1044c1f19c7efe3477a395cc4f7450133bc54915
+F test/shell9.test fe41f890a89f57bdde7351d7a828a1d4b503ebe239d413137369f20faf7f8865
 F test/shmlock.test 3dbf017d34ab0c60abe6a44e447d3552154bd0c87b41eaf5ceacd408dd13fda5
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
@@ -1665,7 +1666,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30
 F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d1631311a16
 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637
 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc
-F test/tester.tcl 68454ef88508c196d19e8694daa27bff7107a91857799eaa12f417188ae53ede
+F test/tester.tcl 6f6e53981b4bdd42ef088f52e23236bc1ba0ca41ed395cbd7f33cbcff7d74d3c
 F test/testrunner.tcl 8e2a5c7550b78d3283eee6103104ae2bcf56aa1df892dbd1608f27b93ebf4de8
 F test/testrunner_data.tcl 7ffd951527bbc614e723fd8d123b6834321878530696adecfdf6035100bac64e
 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899
@@ -2156,8 +2157,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a8e9af1356f5fb2ec460f932dfbe89283bb4e3cf9fa677d1acdbe77ffa11dd04
-R 37733634dd468dc6ea6d1e6aa3b50fba
-U drh
-Z 78db706fe3fe0ce3a2f3eeab5f6bda34
+P 0dfa7b4da134db281c3c4eddb4569c53a450f955f0af2f410e13db801aff4ea2 c82da712113d5dcd63b764dbc68842026989627abc840acb4a33f3a4972b832a
+R 546d683a554613be241caf7c62f4af75
+T +closed c82da712113d5dcd63b764dbc68842026989627abc840acb4a33f3a4972b832a
+U dan
+Z 4d6951c4af3d99936a8684cb8259f448
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-0dfa7b4da134db281c3c4eddb4569c53a450f955f0af2f410e13db801aff4ea2
+cd016f26bb61549a304f2148035e050f76a8f4a35cdb7131bba2f5fc5d09f49e

src/shell.c.in

@@ -1292,6 +1292,7 @@ struct ShellState {
   u8 eTraceType;         /* SHELL_TRACE_* value for type of trace */
   u8 bSafeMode;          /* True to prohibit unsafe operations */
   u8 bSafeModePersist;   /* The long-term value of bSafeMode */
+  u8 eRestoreState;      /* See comments above doAutoDetectRestore() */
   ColModeOpts cmOpts;    /* Option values affecting columnar mode output */
   unsigned statsOn;      /* True to display memory stats before each finalize */
   unsigned mEqpLines;    /* Mask of vertical lines in the EQP output graph */
@@ -6731,7 +6732,6 @@ static int lintDotCommand(
   return SQLITE_ERROR;
 }
 
-#if !defined SQLITE_OMIT_VIRTUALTABLE
 static void shellPrepare(
   sqlite3 *db,
   int *pRc,
@@ -6750,12 +6750,8 @@ static void shellPrepare(
 
 /*
 ** Create a prepared statement using printf-style arguments for the SQL.
-**
-** This routine is could be marked "static".  But it is not always used,
-** depending on compile-time options.  By omitting the "static", we avoid
-** nuisance compiler warnings about "defined but not used".
 */
-void shellPreparePrintf(
+static void shellPreparePrintf(
   sqlite3 *db,
   int *pRc,
   sqlite3_stmt **ppStmt,
@@ -6778,13 +6774,10 @@ void shellPreparePrintf(
   }
 }
 
-/* Finalize the prepared statement created using shellPreparePrintf().
-**
-** This routine is could be marked "static".  But it is not always used,
-** depending on compile-time options.  By omitting the "static", we avoid
-** nuisance compiler warnings about "defined but not used".
+/* 
+** Finalize the prepared statement created using shellPreparePrintf().
 */
-void shellFinalize(
+static void shellFinalize(
   int *pRc,
   sqlite3_stmt *pStmt
 ){
@@ -6800,6 +6793,7 @@ void shellFinalize(
   }
 }
 
+#if !defined SQLITE_OMIT_VIRTUALTABLE
 /* Reset the prepared statement created using shellPreparePrintf().
 **
 ** This routine is could be marked "static".  But it is not always used,
@@ -7866,6 +7860,30 @@ FROM (\
   }
 }
 
+/*
+** Check if the sqlite_schema table contains one or more virtual tables. If
+** parameter zLike is not NULL, then it is an SQL expression that the
+** sqlite_schema row must also match. If one or more such rows are found,
+** print the following warning to the output:
+**
+** WARNING: Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled
+*/
+static int outputDumpWarning(ShellState *p, const char *zLike){
+  int rc = SQLITE_OK;
+  sqlite3_stmt *pStmt = 0;
+  shellPreparePrintf(p->db, &rc, &pStmt,
+    "SELECT 1 FROM sqlite_schema o WHERE "
+    "sql LIKE 'CREATE VIRTUAL TABLE%%' AND %s", zLike ? zLike : "true"
+  );
+  if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
+    oputz("/* WARNING: "
+          "Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled */\n"
+    );
+  }
+  shellFinalize(&rc, pStmt);
+  return rc;
+}
+
 /*
 ** If an input line begins with "." then invoke this routine to
 ** process that line.
@@ -8328,6 +8346,7 @@ static int do_meta_command(char *zLine, ShellState *p){
 
     open_db(p, 0);
 
+    outputDumpWarning(p, zLike);
     if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){
       /* When playing back a "dump", the content might appear in an order
       ** which causes immediate foreign key constraints to be violated.
@@ -11391,6 +11410,80 @@ static int line_is_complete(char *zSql, int nSql){
   return rc;
 }
 
+/*
+** This function is called after processing each line of SQL in the
+** runOneSqlLine() function. Its purpose is to detect scenarios where
+** defensive mode should be automatically turned off. Specifically, when
+**
+**   1. The first line of input is "PRAGMA foreign_keys=OFF;",
+**   2. The second line of input is "BEGIN TRANSACTION;",
+**   3. The database is empty, and
+**   4. The shell is not running in --safe mode.
+** 
+** The implementation uses the ShellState.eRestoreState to maintain state:
+**
+**    0: Have not seen any SQL.
+**    1: Have seen "PRAGMA foreign_keys=OFF;".
+**    2: Currently assuming we are parsing ".dump" restore, defensive mode
+**       should be disabled following the current transaction.
+**    3: Nothing left to do.
+*/
+static int doAutoDetectRestore(ShellState *p, const char *zSql){
+  int rc = SQLITE_OK;
+
+  switch( p->eRestoreState ){
+    case 0: {
+      int bDefense = 0;           /* True if in defensive mode */ 
+      const char *zExpect = "PRAGMA foreign_keys=OFF;";
+      assert( strlen(zExpect)==24 );
+      sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, -1, &bDefense);
+      if( p->bSafeMode==0 && bDefense && memcmp(zSql, zExpect, 25)==0 ){
+        p->eRestoreState = 1;
+      }else{
+        p->eRestoreState = 3;
+      }
+      break;
+    };
+
+    case 1: {
+      const char *zExpect = "BEGIN TRANSACTION;";
+      assert( strlen(zExpect)==18 );
+      if( memcmp(zSql, zExpect, 19)==0 ){
+        /* Now check if the database is empty. */
+        const char *zQuery = "SELECT 1 FROM sqlite_schema LIMIT 1";
+        sqlite3_stmt *pStmt = 0;
+        int bEmpty = 1;
+
+        shellPrepare(p->db, &rc, zQuery, &pStmt);
+        if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
+          bEmpty = 0;
+        }
+        shellFinalize(&rc, pStmt);
+        if( bEmpty && rc==SQLITE_OK ){
+          sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, 0, 0);
+        }else{
+          p->eRestoreState = 3;
+        }
+      }
+      break;
+    }
+
+    case 2: {
+      if( sqlite3_get_autocommit(p->db) ){
+        sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, 0, 0);
+        p->eRestoreState = 3;
+      }
+      break;
+    }
+
+    default: /* Nothing to do */
+      assert( p->eRestoreState==3 );
+      break;
+  }
+
+  return rc;
+}
+
 /*
 ** Run a single line of SQL.  Return the number of errors.
 */
@@ -11438,6 +11531,8 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){
             sqlite3_changes64(p->db), sqlite3_total_changes64(p->db));
     oputf("%s\n", zLineBuf);
   }
+
+  if( doAutoDetectRestore(p, zSql) ) return 1;
   return 0;
 }
 

test/shell9.test

@@ -0,0 +1,127 @@
+# 2024 Jan 8
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The focus of this file is testing the CLI shell tool. Specifically, 
+# testing that it is possible to run a ".dump" script that creates
+# virtual tables without explicitly disabling defensive mode.
+#
+
+# Test plan:
+#
+#   shell1-1.*: Basic command line option handling.
+#   shell1-2.*: Basic "dot" command token parsing.
+#   shell1-3.*: Basic test that "dot" command can be called.
+#   shell1-{4-8}.*: Test various "dot" commands's functionality.
+#   shell1-9.*: Basic test that "dot" commands and SQL intermix ok.
+#
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set CLI [test_cli_invocation]
+
+set ::testprefix shell9
+
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+#----------------------------------------------------------------------------
+# Test cases shell9-1.* verify that scripts output by .dump may be parsed
+# by the shell tool without explicitly disabling DEFENSIVE mode, unless
+# the shell is in safe mode.
+#
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
+  INSERT INTO t1 VALUES('one', 'two', 'three');
+}
+db close
+
+# Create .dump file in "testdump.txt".
+#
+set out [open testdump.txt w]
+puts $out [lindex [catchcmd test.db .dump] 1]
+close $out
+
+# Check testdump.txt can be processed if the initial db is empty.
+#
+do_test 1.1.1 {
+  forcedelete test.db
+  catchcmd test.db ".read testdump.txt"
+} {0 {}}
+sqlite3 db test.db
+do_execsql_test 1.1.2 {
+  SELECT * FROM t1;
+} {one two three}
+
+# Check testdump.txt cannot be processed if the initial db is not empty.
+#
+reset_db
+do_execsql_test 1.2.1 {
+  CREATE TABLE t4(hello);
+}
+db close
+do_test 1.2.2 {
+  catchcmd test.db ".read testdump.txt"
+} {1 {Parse error near line 5: table sqlite_master may not be modified}}
+
+# Check testdump.txt cannot be processed if the db is in safe mode
+#
+do_test 1.3.1 {
+  forcedelete test.db
+  catchsafecmd test.db ".read testdump.txt"
+} {1 {line 1: cannot run .read in safe mode}}
+do_test 1.3.2 {
+  set fd [open testdump.txt]
+  set script [read $fd]
+  close $fd
+  forcedelete test.db
+  catchsafecmd test.db $script
+} {1 {Parse error near line 5: table sqlite_master may not be modified}}
+do_test 1.3.3 {
+  # Quick check that the above would have worked but for safe mode.
+  forcedelete test.db
+  catchcmd test.db $script
+} {0 {}}
+
+#----------------------------------------------------------------------------
+# Test cases shell9-2.* verify that a warning is printed at the top of
+# .dump scripts that contain virtual tables.
+#
+proc contains_warning {text} {
+  return [string match "*WARNING: Script requires that*" $text]
+}
+
+reset_db
+do_execsql_test 2.0.1 {
+  CREATE TABLE t1(x);
+  CREATE TABLE t2(y);
+  INSERT INTO t1 VALUES('one');
+  INSERT INTO t2 VALUES('two');
+}
+do_test 2.0.2 {
+  contains_warning [catchcmd test.db .dump]
+} 0
+
+do_execsql_test 2.1.1 {
+  CREATE virtual TABLE r1 USING fts5(x);
+}
+do_test 2.1.2 {
+  contains_warning [catchcmd test.db .dump]
+} 1
+
+do_test 2.2.1 {
+  contains_warning [catchcmd test.db ".dump t1"]
+} 0
+do_test 2.2.2 {
+  contains_warning [catchcmd test.db ".dump r1"]
+} 1
+
+finish_test

test/tester.tcl

@@ -884,6 +884,15 @@ proc catchcmd {db {cmd ""}} {
   set rc [catch { eval $line } msg]
   list $rc $msg
 }
+proc catchsafecmd {db {cmd ""}} {
+  global CLI
+  set out [open cmds.txt w]
+  puts $out $cmd
+  close $out
+  set line "exec $CLI -safe $db < cmds.txt"
+  set rc [catch { eval $line } msg]
+  list $rc $msg
+}
 
 proc catchcmdex {db {cmd ""}} {
   global CLI