From b03c75772ea4922638622fd90b5f85dc9917b9c9 Mon Sep 17 00:00:00 2001 From: TheWitness Date: Tue, 20 Oct 2026 14:02:31 -0400 Subject: [PATCH 01/37] fix: Wrap checkboxes in spans for 1.2.32 --- includes/functions.php | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/includes/functions.php b/includes/functions.php index bbe9f3f..224b188 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -944,12 +944,16 @@ function flowview_display_filter(): void { '> - > - + + > + + '> - > - + + > + + @@ -1037,20 +1041,28 @@ function flowview_display_filter(): void { - > - + + > + + - > - + + > + + - > - + + > + + - > - + + > + + From aec53322ddaab30a5f44415422bfd2b43d128098 Mon Sep 17 00:00:00 2001 From: TheWitness Date: Tue, 20 Oct 2026 14:09:58 -0400 Subject: [PATCH 02/37] fix: Define time shift handlers so the backward/forward icons work The filter's timeshiftBackward/timeshiftForward handlers called timeshiftFilterLeft()/timeshiftFilterRight(), which neither the plugin nor Cacti core defines, causing an uncaught ReferenceError. Add a timeshift action that shifts date1/date2 by the selected interval and a plugin-owned timeshiftFilter() that applies it and reloads the report. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 2 ++ flowview.php | 4 +++ includes/functions.php | 55 ++++++++++++++++++++++++++++++-- tests/Unit/CoreFunctionsTest.php | 37 +++++++++++++++++++++ 4 files changed, 96 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fde7b01..f513bac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ --- develop --- +* bug: Fix "Uncaught ReferenceError: timeshiftFilterLeft is not defined" when using the Shift Time Backward/Forward icons on the flow report filter; the plugin relied on undefined global functions, so the icons now call a plugin-owned timeshiftFilter() that shifts the From/To range by the selected interval through a new `timeshift` action and reloads the report + * fix: Make bundled Net_DNS2 constructor nullability explicit for PHP 8.4 without changing accepted arguments * issue: Shorten the graph drilldown button title from 'View NetFlow Traffic In Range' to 'NetFlow' so it reads cleanly as a label in Cacti's graph action menu diff --git a/flowview.php b/flowview.php index 0a2a903..c780ecc 100644 --- a/flowview.php +++ b/flowview.php @@ -81,6 +81,10 @@ flowview_request_vars(); flowview_gettimespan(); + break; + case 'timeshift': + flowview_timeshift(); + break; case 'query': load_session_for_filter(); diff --git a/includes/functions.php b/includes/functions.php index 224b188..9e26f3d 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -674,6 +674,38 @@ function flowview_gettimespan(): void { print json_encode($span); } +/** + * Shifts the current date1/date2 range backward or forward by the interval + * selected in the predefined_timeshift dropdown and returns the new range as + * JSON. Uses the same interval strings as Cacti's graph time shifting. + * + * @return void + */ +function flowview_timeshift(): void { + global $graph_timeshifts_vals; + + $direction = get_nfilter_request_var('direction') == 'left' ? '-' : '+'; + $shift = get_filter_request_var('predefined_timeshift'); + $date1 = strtotime(get_nfilter_request_var('date1')); + $date2 = strtotime(get_nfilter_request_var('date2')); + $span = []; + + if (!isset($graph_timeshifts_vals[$shift])) { + $shift = GTS_1_DAY; + } + + if ($date1 !== false && $date2 !== false) { + $interval = $graph_timeshifts_vals[$shift]; + + $span['begin_now'] = strtotime($direction . $interval, $date1); + $span['end_now'] = strtotime($direction . $interval, $date2); + $span['current_value_date1'] = date('Y-m-d H:i', $span['begin_now']); + $span['current_value_date2'] = date('Y-m-d H:i', $span['end_now']); + } + + print json_encode($span); +} + /** * @param mixed $data * @@ -1191,11 +1223,11 @@ function flowview_display_filter(): void { }); $('.timeshiftBackward').off('click').on('click', function() { - timeshiftFilterLeft(); + timeshiftFilter('left'); }); $('.timeshiftForward').off('click').on('click', function() { - timeshiftFilterRight(); + timeshiftFilter('right'); }); $('#flowview_filter').on('submit', function(event) { @@ -1820,6 +1852,25 @@ function applyTimespan() { } } + function timeshiftFilter(direction) { + $.getJSON(urlPath + 'plugins/flowview/flowview.php' + + '?action=timeshift' + + '&direction=' + direction + + '&predefined_timeshift=' + $('#predefined_timeshift').val() + + '&date1=' + encodeURIComponent($('#date1').val()) + + '&date2=' + encodeURIComponent($('#date2').val()), function(data) { + if (data['current_value_date1'] !== undefined) { + $('#date1').val(data['current_value_date1']); + $('#date2').val(data['current_value_date2']); + $('#predefined_timespan').val('0'); + + $('#predefined_timespan').selectmenu('refresh'); + + applyFilter(); + } + }); + } + function changeQuery() { loadPageNoHeader(urlPath+'plugins/flowview/flowview.php' + '?action=query' + diff --git a/tests/Unit/CoreFunctionsTest.php b/tests/Unit/CoreFunctionsTest.php index f8860e7..25adf65 100644 --- a/tests/Unit/CoreFunctionsTest.php +++ b/tests/Unit/CoreFunctionsTest.php @@ -146,3 +146,40 @@ expect($sql)->toBe('(`start_time` BETWEEN ? AND ?)') ->and($params)->toBe(array('2024-01-01 00:00:00', '2024-01-02 00:00:00')); }); + +it('shifts the date range backward and forward by the selected interval', function ($direction, $shift, $date1, $date2, $expected1, $expected2) { + if (!defined('GTS_1_DAY')) { + define('GTS_1_DAY', 7); + } + + $GLOBALS['graph_timeshifts_vals'] = array(7 => '1 Day', 11 => '1 Week', 13 => '1 Month'); + + $_REQUEST['direction'] = $direction; + $_REQUEST['predefined_timeshift'] = $shift; + $_REQUEST['date1'] = $date1; + $_REQUEST['date2'] = $date2; + + ob_start(); + flowview_timeshift(); + $span = json_decode(ob_get_clean(), true); + + expect($span['current_value_date1'])->toBe($expected1) + ->and($span['current_value_date2'])->toBe($expected2); +})->with(array( + 'left one week' => array('left', 11, '2026-10-13 12:02', '2026-10-20 12:02', '2026-10-06 12:02', '2026-10-13 12:02'), + 'right one week' => array('right', 11, '2026-10-13 12:02', '2026-10-20 12:02', '2026-10-20 12:02', '2026-10-27 12:02'), + 'right one month' => array('right', 13, '2026-09-01 00:00', '2026-09-30 23:59', '2026-10-01 00:00', '2026-10-30 23:59'), + 'unknown interval is 1day' => array('left', 99, '2026-10-13 12:02', '2026-10-20 12:02', '2026-10-12 12:02', '2026-10-19 12:02'), +)); + +it('returns an empty range when the time shift dates cannot be parsed', function () { + $_REQUEST['direction'] = 'left'; + $_REQUEST['predefined_timeshift'] = 7; + $_REQUEST['date1'] = 'not a date'; + $_REQUEST['date2'] = ''; + + ob_start(); + flowview_timeshift(); + + expect(json_decode(ob_get_clean(), true))->toBe(array()); +}); From 079b27497a35d7a63969311417f9b37cc916eacd Mon Sep 17 00:00:00 2001 From: TheWitness Date: Tue, 20 Oct 2026 14:32:37 -0400 Subject: [PATCH 03/37] fix: Switch to Custom timespan and reload after a time shift The time shift callback called selectmenu('refresh') unguarded, which throws when the timespan select is not an initialized selectmenu, aborting before applyFilter() ran. Use a guarded setCustomTimespan() helper (instance check, as core does) for both the time shift and the date change handler. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- includes/functions.php | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/includes/functions.php b/includes/functions.php index 9e26f3d..90340e0 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -1852,6 +1852,14 @@ function applyTimespan() { } } + function setCustomTimespan() { + $('#predefined_timespan').val('0'); + + if ($('#predefined_timespan').selectmenu('instance') !== undefined) { + $('#predefined_timespan').selectmenu('refresh'); + } + } + function timeshiftFilter(direction) { $.getJSON(urlPath + 'plugins/flowview/flowview.php' + '?action=timeshift' + @@ -1862,10 +1870,7 @@ function timeshiftFilter(direction) { if (data['current_value_date1'] !== undefined) { $('#date1').val(data['current_value_date1']); $('#date2').val(data['current_value_date2']); - $('#predefined_timespan').val('0'); - - $('#predefined_timespan').selectmenu('refresh'); - + setCustomTimespan(); applyFilter(); } }); @@ -1925,10 +1930,7 @@ function clearFilter() { } $('#date1, #date2').on('change', function() { - $('#predefined_timespan').val('0'); - - $('#predefined_timespan').selectmenu('refresh'); - + setCustomTimespan(); }); From 834c65738b9f590721e3e6cc9b050d2daa45b310 Mon Sep 17 00:00:00 2001 From: TheWitness Date: Tue, 20 Oct 2026 14:49:57 -0400 Subject: [PATCH 04/37] feature: Add cli/materialize_queries.php and move CLI scripts into cli/ Add a CLI script that runs every saved query through the parallel query for the entire range of flow partitions with NAT on and off. It registers a process with an 8 hour limit, reuses cached partitions that still match their data, and rebuilds stale or dropped ones. Run each query in its own process so reaching the run limit does not end the rest. Move fixprivate.php, import_flows.php and repair_tables.php into cli/, tombstone the old paths in manifest.json, and locate the Cacti root relative to each script. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/copilot-instructions.md | 6 +- CHANGELOG.md | 4 + README.md | 21 +- fixprivate.php => cli/fixprivate.php | 2 +- import_flows.php => cli/import_flows.php | 2 +- cli/materialize_queries.php | 345 +++++++++++++++++++++ repair_tables.php => cli/repair_tables.php | 2 +- includes/functions.php | 139 +++++++++ manifest.json | 7 +- tests/Unit/CoreFunctionsTest.php | 111 +++++++ tests/bin/patch-coverage.php | 7 +- 11 files changed, 635 insertions(+), 11 deletions(-) rename fixprivate.php => cli/fixprivate.php (99%) rename import_flows.php => cli/import_flows.php (98%) create mode 100644 cli/materialize_queries.php rename repair_tables.php => cli/repair_tables.php (98%) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index c836c8d..d854645 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -44,7 +44,11 @@ flowview/ # Repository root (install to plugins/fl ├── flowview_upgrade.php # Background schema upgrade runner ├── flow_collector.php # NetFlow v5/v9/v10 collector daemon ├── functions.php / functions-pre13.php # Core UI + protocol decode helpers -├── import_flows.php # CLI flow file importer +├── cli/ # CLI utilities (importer, repair, materializer) +│ ├── fixprivate.php # CLI: repair domains of private-range addresses +│ ├── import_flows.php # CLI flow file importer +│ ├── materialize_queries.php # CLI: pre-build each query's partition cache (NAT on/off) +│ └── repair_tables.php # CLI: widen the raw partition table columns ├── run_schedule.php # CLI scheduled-report executor ├── INFO # Plugin metadata (name, version, compat) ├── README.md diff --git a/CHANGELOG.md b/CHANGELOG.md index f513bac..b7bd54c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ --- develop --- +* feature: Add cli/materialize_queries.php, which runs every saved FlowView query through the parallel query for the entire range of flow partitions, once with NAT on and once with NAT off (--query-id, --nat, --dry-run and --force are optional), so the partition results are already cached when a report is opened; it registers with register_process_start() for 8 hours so only one run is active at a time, reuses cached partitions that still match their partition's data bounds, rebuilds the ones that have since gained flows or been dropped, and runs each query and NAT mode in its own process so one that reaches the Maximum Run Time does not end the rest + +* dev: Move fixprivate.php, import_flows.php and repair_tables.php into the plugin's cli/ directory (listed as tombstones in manifest.json so upgrades remove the old copies); they now locate the Cacti root relative to their own location + * bug: Fix "Uncaught ReferenceError: timeshiftFilterLeft is not defined" when using the Shift Time Backward/Forward icons on the flow report filter; the plugin relied on undefined global functions, so the icons now call a plugin-owned timeshiftFilter() that shifts the From/To range by the selected interval through a new `timeshift` action and reloads the report * fix: Make bundled Net_DNS2 constructor nullability explicit for PHP 8.4 without changing accepted arguments diff --git a/README.md b/README.md index 8de558c..ac38e6c 100644 --- a/README.md +++ b/README.md @@ -194,11 +194,30 @@ this migration, ensure that your Cacti system, has enough space to handle all the flow data. You should check the size of your existing flows, and then verify that you have enough space to handle the data inside of a MySQL database. -Once you have done this, simply run the 'import_flows.php' script and all your +Once you have done this, simply run the 'cli/import_flows.php' script and all your legacy flow data will be imported. Remember, it's important that you define your partitioning scheme ahead of time, especially if you have large quantities of flow data streaming into the Cacti server. +## Pre-Building Query Results + +The Parallel Query caches the result of each fully covered partition, so +reports over older data open quickly once a partition has been scanned. To +build that cache ahead of time, run the following from the plugin's `cli` +directory with the Parallel Query enabled (Max Concurrent Threads of 2 or +more under the FlowView Settings): + +``` +php cli/materialize_queries.php +``` + +The script runs every saved query over the entire range of flow partitions, +once with NAT on and once with NAT off. Use `--query-id=ID` or `--nat=on|off` +to narrow the run, and `--dry-run` to see what would be run. Partitions that +are already cached and still match their data are skipped, while partitions +that have received more flows since, or have been dropped, are rebuilt. Only +one run may be active at a time, for up to 8 hours, unless `--force` is used. + # Possible Bugs? If you figure out this problem, goto GitHub and create a pulls request or open diff --git a/fixprivate.php b/cli/fixprivate.php similarity index 99% rename from fixprivate.php rename to cli/fixprivate.php index 0c6bdb1..35d87a8 100644 --- a/fixprivate.php +++ b/cli/fixprivate.php @@ -23,7 +23,7 @@ +-------------------------------------------------------------------------+ */ -chdir('../../'); +chdir(__DIR__ . '/../../../'); require('./include/cli_check.php'); global $config; diff --git a/import_flows.php b/cli/import_flows.php similarity index 98% rename from import_flows.php rename to cli/import_flows.php index 6bc7e9f..84d23b6 100644 --- a/import_flows.php +++ b/cli/import_flows.php @@ -23,7 +23,7 @@ +-------------------------------------------------------------------------+ */ -chdir('../../'); +chdir(__DIR__ . '/../../../'); require('./include/cli_check.php'); global $config; diff --git a/cli/materialize_queries.php b/cli/materialize_queries.php new file mode 100644 index 0000000..53a4c20 --- /dev/null +++ b/cli/materialize_queries.php @@ -0,0 +1,345 @@ +#!/usr/bin/env php + $value) { + $value = is_array($value) ? end($value) : $value; + + switch ($arg) { + case 'query-id': + if (!ctype_digit((string) $value) || $value < 1) { + print 'ERROR: --query-id must be a positive integer' . PHP_EOL; + exit(1); + } + + $query_id = (int) $value; + + break; + case 'nat': + $nat = strtolower((string) $value); + + if ($nat != 'on' && $nat != 'off') { + print 'ERROR: --nat must be either on or off' . PHP_EOL; + exit(1); + } + + break; + case 'start': + $start = ctype_digit((string) $value) ? (int) $value : false; + + break; + case 'end': + $end = ctype_digit((string) $value) ? (int) $value : false; + + break; + case 'debug': + $debug = true; + + break; + case 'force': + $force = true; + + break; + case 'dry-run': + $dry_run = true; + + break; + case 'worker': + $worker = true; + + break; + case 'V': + case 'v': + case 'version': + display_version(); + exit(0); + case 'H': + case 'h': + case 'help': + display_help(); + exit(0); + default: + print 'ERROR: Invalid option ' . $arg . PHP_EOL; + display_help(); + exit(1); + } +} + +/* a worker materializes exactly one query and NAT mode for the range its parent chose */ +if ($worker) { + if ($query_id === false || $nat === false || $start === false || $end === false) { + print 'ERROR: --worker requires --query-id, --nat, --start and --end' . PHP_EOL; + exit(1); + } + + exit(materialize_run_worker($query_id, $nat, $start, $end)); +} + +if (!$force && !$dry_run) { + if (!register_process_start('flowview', 'materialize', 0, MATERIALIZE_RUNTIME_LIMIT)) { + print 'NOTE: A materialization is already running. To override use --force' . PHP_EOL; + exit(0); + } + + register_shutdown_function(function () { + unregister_process('flowview', 'materialize', 0); + }); +} + +$threads = read_config_option('flowview_parallel_threads'); + +if (empty($threads) || $threads == 1) { + print 'ERROR: Materialization requires the Parallel Query. Set Max Concurrent Threads to 2 or more under the FlowView Settings.' . PHP_EOL; + exit(1); +} + +$tables = flowview_db_fetch_assoc('SELECT TABLE_NAME AS `table` + FROM information_schema.TABLES + WHERE TABLE_NAME LIKE "plugin_flowview_raw_%"'); + +$range = flowview_get_materialize_range(array_column($tables, 'table')); + +if ($range === false) { + print 'NOTE: There are no flow partitions to materialize.' . PHP_EOL; + exit(0); +} + +list($start, $end) = $range; + +if ($query_id !== false) { + $queries = flowview_db_fetch_assoc_prepared('SELECT id, name + FROM plugin_flowview_queries + WHERE id = ?', + [$query_id]); +} else { + $queries = flowview_db_fetch_assoc('SELECT id, name + FROM plugin_flowview_queries + ORDER BY id'); +} + +if (!cacti_sizeof($queries)) { + print 'NOTE: There are no FlowView queries to materialize.' . PHP_EOL; + exit(0); +} + +$modes = $nat !== false ? [$nat] : ['on', 'off']; + +print sprintf('FlowView Materialization: Partitions:%d From:%s To:%s Queries:%d NAT:%s', + cacti_sizeof($tables), + date('Y-m-d H:i:s', $start + 1), + date('Y-m-d H:i:s', $end - 1), + cacti_sizeof($queries), + implode('/', $modes)) . PHP_EOL; + +/* partitions that are still accurate keep their cache, everything else is rebuilt below */ +if (!$dry_run) { + $cache = flowview_invalidate_stale_shard_cache(); + + print sprintf('Cached Partitions Still Accurate:%d, Stale Partitions Removed:%d', $cache['kept'], $cache['removed']) . PHP_EOL; +} + +$began = microtime(true); +$total = 0; +$failed = 0; + +foreach ($queries as $query) { + foreach ($modes as $mode) { + $total++; + + if ($dry_run) { + print sprintf('DRY RUN: Would materialize Query:%d [%s] NAT:%s', $query['id'], $query['name'], $mode) . PHP_EOL; + + continue; + } + + /* a separate process keeps one query hitting the run limit from ending the rest */ + $command = escapeshellarg(PHP_BINARY) . ' ' . escapeshellarg(__FILE__) . + ' --worker' . + ' --query-id=' . (int) $query['id'] . + ' --nat=' . $mode . + ' --start=' . $start . + ' --end=' . $end . + ($debug ? ' --debug' : ''); + + passthru($command, $status); + + if ($status != 0) { + $failed++; + + print sprintf('ERROR: Query:%d [%s] NAT:%s failed to materialize', $query['id'], $query['name'], $mode) . PHP_EOL; + } + } +} + +$time = microtime(true) - $began; + +print sprintf('FlowView Materialization Complete: Time:%0.2f Runs:%d Failed:%d', $time, $total, $failed) . PHP_EOL; + +if (!$dry_run) { + cacti_log(sprintf('FLOWVIEW MATERIALIZE STATS: Time:%0.2f Runs:%d Failed:%d', $time, $total, $failed), false, 'SYSTEM'); +} + +exit($failed > 0 ? 1 : 0); + +/** + * Materializes one query for one NAT mode across the given range. The query + * runs through the normal parallel query path, so each partition that is + * already cached and still accurate is reused instead of being scanned again. + * + * @param int $query_id The FlowView query to run + * @param string $nat Either on or off + * @param int $start The Unix start time of the range + * @param int $end The Unix end time of the range + * + * @return int The exit status, 0 on success + */ +function materialize_run_worker(int $query_id, string $nat, int $start, int $end): int { + $finished = false; + + /* the run limit ends the process without returning, so report it as a failure */ + register_shutdown_function(function () use (&$finished) { + if (!$finished) { + print 'ERROR: The materialization did not finish. Check the Maximum Run Time under the FlowView Settings.' . PHP_EOL; + exit(1); + } + }); + + set_request_var('usenat', $nat == 'on' ? 'true' : 'false'); + + $last_request = (int) flowview_db_fetch_cell('SELECT COALESCE(MAX(id), 0) + FROM parallel_database_query'); + + $began = microtime(true); + + $data = load_data_for_filter($query_id, $start, $end); + + $finished = true; + + if ($data === false) { + print sprintf('ERROR: Query:%d NAT:%s did not return a report. See the Cacti log for details.', $query_id, $nat) . PHP_EOL; + + return 1; + } + + $stats = flowview_db_fetch_row_prepared('SELECT COUNT(*) AS requests, + COALESCE(SUM(total_shards), 0) AS shards, + COALESCE(SUM(cached_shards), 0) AS cached + FROM parallel_database_query + WHERE user_id = 0 + AND id > ?', + [$last_request]); + + if (cacti_sizeof($stats) && $stats['requests'] > 0) { + print sprintf('Query:%d NAT:%s Partitions:%d AlreadyMaterialized:%d Materialized:%d Time:%0.2f', + $query_id, + $nat, + $stats['shards'], + $stats['cached'], + $stats['shards'] - $stats['cached'], + microtime(true) - $began) . PHP_EOL; + } else { + print sprintf('Query:%d NAT:%s Already Materialized and Accurate Time:%0.2f', $query_id, $nat, microtime(true) - $began) . PHP_EOL; + } + + return 0; +} + +/** + * Prints this script's version and copyright banner. Called from + * display_help() and when invoked with '--version'. + * + * @return void + */ +function display_version(): void { + $version = get_cacti_cli_version(); + print "Cacti FlowView Query Materializer, Version $version, " . COPYRIGHT_YEARS . PHP_EOL; +} + +/** + * Prints the version banner followed by this script's command-line + * usage summary. Called when invoked with '--help' or with invalid + * arguments. + * + * @return void + */ +function display_help(): void { + display_version(); + + print PHP_EOL . 'usage: materialize_queries.php [--query-id=ID] [--nat=on|off] [--dry-run] [--force] [--debug]' . PHP_EOL . PHP_EOL; + + print 'Materializes each FlowView query into the parallel query cache for the' . PHP_EOL; + print 'entire range of flow partitions, once with NAT on and once with NAT off.' . PHP_EOL; + print 'Partitions that are already materialized and still accurate are skipped.' . PHP_EOL; + print 'Only one run may be active at a time, for up to 8 hours.' . PHP_EOL . PHP_EOL; + + print 'Options:' . PHP_EOL; + print ' --query-id=ID Only materialize this query.' . PHP_EOL; + print ' --nat=on|off Only materialize this NAT mode.' . PHP_EOL; + print ' --dry-run List what would be materialized and exit.' . PHP_EOL; + print ' --force Run even if another materialization is running.' . PHP_EOL; + print ' --debug Provide some debug output during the run.' . PHP_EOL . PHP_EOL; +} diff --git a/repair_tables.php b/cli/repair_tables.php similarity index 98% rename from repair_tables.php rename to cli/repair_tables.php index 86b1301..1976fe1 100644 --- a/repair_tables.php +++ b/cli/repair_tables.php @@ -23,7 +23,7 @@ +-------------------------------------------------------------------------+ */ -chdir('../../'); +chdir(__DIR__ . '/../../../'); require('./include/cli_check.php'); global $config, $flowviewdb_default; diff --git a/includes/functions.php b/includes/functions.php index 90340e0..d5bbc50 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -3998,6 +3998,145 @@ function flowview_convert_yeardayhour_to_date($range, $year, $day, int $hour = 0 'end_time' => $end_time ]; } + +/** + * flowview_get_materialize_range - Given a list of raw partition table + * names, return the Unix start and end timestamps spanning every partition. + * Names that are not partition tables are ignored, and false is returned + * when none qualify. + * + * Both bounds are padded by one second. A shard is only a full scan, and + * therefore only cached in parallel_database_query_shard_cache, when the + * requested range lies strictly outside the data in the partition. + * + * @param array $tables + * + * @return array|false + */ +function flowview_get_materialize_range(array $tables): array|false { + $start = false; + $end = false; + + foreach ($tables as $table) { + $table_start = flowview_table_name_to_time($table, 'start_time'); + $table_end = flowview_table_name_to_time($table, 'end_time'); + + if ($table_start === false || $table_end === false) { + continue; + } + + if ($start === false || $table_start < $start) { + $start = $table_start; + } + + if ($end === false || $table_end > $end) { + $end = $table_end; + } + } + + if ($start === false || $end === false) { + return false; + } + + return [$start - 1, $end + 1]; +} + +/** + * flowview_shard_cache_is_current - Decide whether a cached partition shard + * still reflects its partition. The cache stores the partition's + * MIN(start_time) and MAX(end_time) from when it was built, so it is current + * only while the partition still reports the same bounds. + * + * The raw tables keep microseconds but the cache keeps whole seconds, which + * the database either rounds or truncates, so a difference of one second is + * treated as equal. + * + * @param array $cached The min_date and max_date stored with the cached shard + * @param array $current The min_date and max_date the partition reports now + * + * @return bool + */ +function flowview_shard_cache_is_current(array $cached, array $current): bool { + foreach (['min_date', 'max_date'] as $column) { + if (empty($cached[$column]) || empty($current[$column])) { + return false; + } + + $cached_time = strtotime($cached[$column]); + $current_time = strtotime($current[$column]); + + if ($cached_time === false || $current_time === false || abs($cached_time - $current_time) > 1) { + return false; + } + } + + return true; +} + +/** + * flowview_invalidate_stale_shard_cache - Remove cached partition shards that + * no longer match their partition, either because more flows arrived or + * because the partition table is gone. Shards that are still current are + * left alone so they are never materialized a second time. + * + * When anything is removed, the finished CLI requests (user 0) are dropped + * as well so their results are rebuilt from the refreshed shards. + * + * @return array The number of shards kept and removed + */ +function flowview_invalidate_stale_shard_cache(): array { + $kept = 0; + $removed = 0; + $current = []; + + $cache = flowview_db_fetch_assoc('SELECT md5sum, map_table, map_partition, min_date, max_date + FROM parallel_database_query_shard_cache + ORDER BY map_table'); + + if (cacti_sizeof($cache)) { + foreach ($cache as $entry) { + $table = $entry['map_table']; + + if (!isset($current[$table])) { + if (preg_match('/^plugin_flowview_raw_[0-9]+$/', $table) && flowview_db_table_exists($table)) { + $current[$table] = flowview_db_fetch_row("SELECT MIN(start_time) AS min_date, MAX(end_time) AS max_date + FROM $table"); + } else { + $current[$table] = []; + } + } + + if (cacti_sizeof($current[$table]) && flowview_shard_cache_is_current($entry, $current[$table])) { + $kept++; + + continue; + } + + flowview_db_execute_prepared('DELETE FROM parallel_database_query_shard_cache + WHERE md5sum = ? + AND map_table = ? + AND map_partition = ?', + [$entry['md5sum'], $entry['map_table'], $entry['map_partition']]); + + $removed++; + } + } + + if ($removed > 0) { + $requests = flowview_db_fetch_assoc_prepared('SELECT id + FROM parallel_database_query + WHERE user_id = 0 + AND status = ?', + ['complete']); + + foreach ($requests as $request) { + parallel_database_query_cleanup($request['id'], true); + } + } + + return ['kept' => $kept, 'removed' => $removed]; +} + /** * parallel_database_query_request - Given a series of tables and * and inner (map phase) and outer (reduce phase) SQL queries, diff --git a/manifest.json b/manifest.json index 00e7668..0db1888 100644 --- a/manifest.json +++ b/manifest.json @@ -2,9 +2,12 @@ "tombstones": [ "arrays.php", "database.php", + "fixprivate.php", "flowview_security.php", "functions-pre13.php", "functions.php", + "import_flows.php", + "repair_tables.php", "themes/" ], "expected": [ @@ -13,9 +16,9 @@ "LICENSE", "Net/", "README.md", + "cli/", "config.php.dist", "css/", - "fixprivate.php", "flow_collector.php", "flowview.php", "flowview_bulkarin.php", @@ -30,7 +33,6 @@ "flowview_upgrade.php", "flowview_upgrade_nat_columns.php", "images/", - "import_flows.php", "includes/", "index.php", "irr_tables.php", @@ -38,7 +40,6 @@ "manifest.json", "maxscale/", "plugin_flowview_ports.sql", - "repair_tables.php", "run_schedule.php", "service/", "setup.php" diff --git a/tests/Unit/CoreFunctionsTest.php b/tests/Unit/CoreFunctionsTest.php index 25adf65..af36d43 100644 --- a/tests/Unit/CoreFunctionsTest.php +++ b/tests/Unit/CoreFunctionsTest.php @@ -183,3 +183,114 @@ expect(json_decode(ob_get_clean(), true))->toBe(array()); }); + +it('spans every partition when materializing and pads the bounds so shards are full scans', function () { + $range = flowview_get_materialize_range(array( + 'plugin_flowview_raw_2024003', + 'not_a_partition_table', + 'plugin_flowview_raw_2024001', + 'plugin_flowview_raw_202400105', + )); + + expect($range)->toBe(array( + strtotime('2024-01-02 00:00:00 UTC') - 1, + strtotime('2024-01-05 00:00:00 UTC') + 1, + )); +}); + +it('has no materialization range without a valid partition table', function ($tables) { + expect(flowview_get_materialize_range($tables))->toBeFalse(); +})->with(array( + 'no tables' => array(array()), + 'not partitions' => array(array('plugin_flowview_queries', 'plugin_flowview_raw_abc')), +)); + +it('treats a cached partition shard as current only while the partition bounds match', function ($cached, $current, $expected) { + expect(flowview_shard_cache_is_current($cached, $current))->toBe($expected); +})->with(array( + 'identical bounds' => array( + array('min_date' => '2026-10-20 12:00:00', 'max_date' => '2026-10-20 12:59:59'), + array('min_date' => '2026-10-20 12:00:00', 'max_date' => '2026-10-20 12:59:59'), + true, + ), + 'microsecond precision' => array( + array('min_date' => '2026-10-20 12:00:00', 'max_date' => '2026-10-20 12:59:59'), + array('min_date' => '2026-10-20 12:00:00.400000', 'max_date' => '2026-10-20 12:59:59.900000'), + true, + ), + 'partition has more data' => array( + array('min_date' => '2026-10-20 12:00:00', 'max_date' => '2026-10-20 12:30:00'), + array('min_date' => '2026-10-20 12:00:00', 'max_date' => '2026-10-20 12:45:00'), + false, + ), + 'partition starts earlier' => array( + array('min_date' => '2026-10-20 12:05:00', 'max_date' => '2026-10-20 12:30:00'), + array('min_date' => '2026-10-20 12:00:00', 'max_date' => '2026-10-20 12:30:00'), + false, + ), + 'empty partition' => array( + array('min_date' => '2026-10-20 12:00:00', 'max_date' => '2026-10-20 12:30:00'), + array('min_date' => null, 'max_date' => null), + false, + ), + 'unset cached bounds' => array( + array('min_date' => '0000-00-00', 'max_date' => '0000-00-00'), + array('min_date' => '2026-10-20 12:00:00', 'max_date' => '2026-10-20 12:30:00'), + false, + ), +)); + +it('keeps accurate cached partitions and removes ones that changed or were dropped', function () { + plugin_test_queue_db_result('db_fetch_assoc', array( + array('md5sum' => 'aaa', 'map_table' => 'plugin_flowview_raw_2024001', 'map_partition' => '', 'min_date' => '2024-01-02 00:00:00', 'max_date' => '2024-01-02 23:00:00'), + array('md5sum' => 'bbb', 'map_table' => 'plugin_flowview_raw_2024001', 'map_partition' => '', 'min_date' => '2024-01-02 00:00:00', 'max_date' => '2024-01-02 23:00:00'), + array('md5sum' => 'aaa', 'map_table' => 'plugin_flowview_raw_2024002', 'map_partition' => '', 'min_date' => '2024-01-03 00:00:00', 'max_date' => '2024-01-03 12:00:00'), + array('md5sum' => 'aaa', 'map_table' => 'plugin_flowview_raw_2024009', 'map_partition' => '', 'min_date' => '2024-01-10 00:00:00', 'max_date' => '2024-01-10 23:00:00'), + )); + + // Each distinct partition table is checked once: it exists, then its current bounds. + plugin_test_queue_db_result('db_fetch_cell', 1); + plugin_test_queue_db_result('db_fetch_row', array('min_date' => '2024-01-02 00:00:00.120000', 'max_date' => '2024-01-02 23:00:00.480000')); + plugin_test_queue_db_result('db_fetch_cell', 1); + plugin_test_queue_db_result('db_fetch_row', array('min_date' => '2024-01-03 00:00:00', 'max_date' => '2024-01-03 20:00:00')); + plugin_test_queue_db_result('db_fetch_cell', 0); + + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('id' => 9))); + + expect(flowview_invalidate_stale_shard_cache())->toBe(array('kept' => 2, 'removed' => 2)); + + $deleted = array(); + foreach ($GLOBALS['__test_db_calls'] as $call) { + if ($call['fn'] == 'db_execute_prepared' && strpos($call['sql'], 'DELETE FROM parallel_database_query_shard_cache') === 0) { + $deleted[] = $call['params']; + } + } + + expect($deleted)->toBe(array( + array('aaa', 'plugin_flowview_raw_2024002', ''), + array('aaa', 'plugin_flowview_raw_2024009', ''), + )); + + $requests = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute_prepared' && preg_match('/^DELETE FROM parallel_database_query\s+WHERE id = \?/', $call['sql']) === 1; + })); + + expect($requests)->toHaveCount(1) + ->and($requests[0]['params'])->toBe(array(9)); +}); + +it('leaves finished requests alone when every cached partition is still accurate', function () { + plugin_test_queue_db_result('db_fetch_assoc', array( + array('md5sum' => 'aaa', 'map_table' => 'plugin_flowview_raw_2024001', 'map_partition' => '', 'min_date' => '2024-01-02 00:00:00', 'max_date' => '2024-01-02 23:00:00'), + )); + plugin_test_queue_db_result('db_fetch_cell', 1); + plugin_test_queue_db_result('db_fetch_row', array('min_date' => '2024-01-02 00:00:00', 'max_date' => '2024-01-02 23:00:00')); + + expect(flowview_invalidate_stale_shard_cache())->toBe(array('kept' => 1, 'removed' => 0)); + + $writes = array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['fn'], 'db_execute') === 0; + }); + + expect($writes)->toBeEmpty(); +}); diff --git a/tests/bin/patch-coverage.php b/tests/bin/patch-coverage.php index ecbcd77..030facc 100644 --- a/tests/bin/patch-coverage.php +++ b/tests/bin/patch-coverage.php @@ -167,7 +167,10 @@ function changed_lines($base_ref) { // Web/CLI entry points (chdir + require auth.php or top-level execution) // that cannot load in the isolated unit process. Changed here only by the // includes/ library relocation + include_once -> require_once sweep. - 'fixprivate.php', + 'cli/fixprivate.php', + 'cli/import_flows.php', + 'cli/materialize_queries.php', + 'cli/repair_tables.php', 'flow_collector.php', 'flowview.php', 'flowview_bulkarin.php', @@ -181,8 +184,6 @@ function changed_lines($base_ref) { 'flowview_schedules.php', 'flowview_upgrade.php', 'flowview_upgrade_nat_columns.php', - 'import_flows.php', - 'repair_tables.php', 'run_schedule.php', ]; $unmeasured = array_values(array_diff(array_keys($changed), array_keys($measured))); From c04294b40dd034bccca43c47d05ce3c80b23f75d Mon Sep 17 00:00:00 2001 From: TheWitness Date: Tue, 20 Oct 2026 15:10:06 -0400 Subject: [PATCH 05/37] feature: Add Materialize View action, re-materialization advice and cached shard counts Add a Materialize View action to the Filters page that, after confirmation, starts cli/materialize_queries.php in the background for the selected filters; --query-id now accepts a comma separated list. Recommend a forced re-materialization when a filter is saved with a change that alters what it materializes, from both the filter form and the report page. Show the number of cached partitions with NAT and without for each filter next to its ID. run_flow_query() gains a plan-only mode that returns the cache keys without running the query, and the map query hashing is shared with the parallel query request. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 6 + README.md | 20 +- cli/materialize_queries.php | 33 ++- flowview_filters.php | 31 ++- includes/functions.php | 420 ++++++++++++++++++++++++------- tests/Unit/CoreFunctionsTest.php | 208 +++++++++++++++ 6 files changed, 603 insertions(+), 115 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7bd54c..db2b404 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ --- develop --- +* feature: Show the number of partitions each Filter has stored in the Parallel Query cache, with NAT and without, as the NAT Shards and Non-NAT Shards columns next to the ID on the Filters page; the cache keys are worked out from the Filter without running it (run_flow_query() gained a plan-only mode, and the map query hashing is shared with the parallel query request) + +* feature: Add a 'Materialize View' action to the Filters page that, after confirmation, starts cli/materialize_queries.php in the background for the selected Filters (--query-id now accepts a comma separated list), and tells you when a materialization is already running + +* feature: Recommend a forced re-materialization when a Filter is saved, from either the Filter form or the report page, with a change that alters what it materializes (report type, device, exporter, template, or any address, port, interface, AS, protocol, TCP flag or TOS restriction, plus the sort field of printed reports); display settings, the time range, cutoffs and NAT mode do not count + * feature: Add cli/materialize_queries.php, which runs every saved FlowView query through the parallel query for the entire range of flow partitions, once with NAT on and once with NAT off (--query-id, --nat, --dry-run and --force are optional), so the partition results are already cached when a report is opened; it registers with register_process_start() for 8 hours so only one run is active at a time, reuses cached partitions that still match their partition's data bounds, rebuilds the ones that have since gained flows or been dropped, and runs each query and NAT mode in its own process so one that reaches the Maximum Run Time does not end the rest * dev: Move fixprivate.php, import_flows.php and repair_tables.php into the plugin's cli/ directory (listed as tombstones in manifest.json so upgrades remove the old copies); they now locate the Cacti root relative to their own location diff --git a/README.md b/README.md index ac38e6c..990708f 100644 --- a/README.md +++ b/README.md @@ -212,11 +212,21 @@ php cli/materialize_queries.php ``` The script runs every saved query over the entire range of flow partitions, -once with NAT on and once with NAT off. Use `--query-id=ID` or `--nat=on|off` -to narrow the run, and `--dry-run` to see what would be run. Partitions that -are already cached and still match their data are skipped, while partitions -that have received more flows since, or have been dropped, are rebuilt. Only -one run may be active at a time, for up to 8 hours, unless `--force` is used. +once with NAT on and once with NAT off. Use `--query-id=ID[,ID...]` or +`--nat=on|off` to narrow the run, and `--dry-run` to see what would be run. +Partitions that are already cached and still match their data are skipped, +while partitions that have received more flows since, or have been dropped, +are rebuilt. Only one run may be active at a time, for up to 8 hours, unless +`--force` is used. + +You can also start it from the Filters page by selecting one or more Filters +and choosing the 'Materialize View' action. It runs in the background after +you confirm, and the result is written to the Cacti log. The 'NAT Shards' and +'Non-NAT Shards' columns on that page show how many partitions of each Filter +are already cached, with NAT on and with NAT off. When you save a +Filter in a way that changes the data it materializes, such as its report, +device, exporter, or any address, port, interface, AS, protocol, flag or TOS +restriction, you are advised to re-materialize it. # Possible Bugs? diff --git a/cli/materialize_queries.php b/cli/materialize_queries.php index 53a4c20..97a385e 100644 --- a/cli/materialize_queries.php +++ b/cli/materialize_queries.php @@ -45,7 +45,7 @@ $force = false; $dry_run = false; $worker = false; -$query_id = false; +$query_ids = []; $nat = false; $start = false; $end = false; @@ -71,12 +71,18 @@ switch ($arg) { case 'query-id': - if (!ctype_digit((string) $value) || $value < 1) { - print 'ERROR: --query-id must be a positive integer' . PHP_EOL; - exit(1); + $query_ids = []; + + foreach (explode(',', (string) $value) as $id) { + if (!ctype_digit($id) || $id < 1) { + print 'ERROR: --query-id must be a positive integer or a comma separated list of them' . PHP_EOL; + exit(1); + } + + $query_ids[(int) $id] = (int) $id; } - $query_id = (int) $value; + $query_ids = array_values($query_ids); break; case 'nat': @@ -131,12 +137,12 @@ /* a worker materializes exactly one query and NAT mode for the range its parent chose */ if ($worker) { - if ($query_id === false || $nat === false || $start === false || $end === false) { - print 'ERROR: --worker requires --query-id, --nat, --start and --end' . PHP_EOL; + if (cacti_sizeof($query_ids) != 1 || $nat === false || $start === false || $end === false) { + print 'ERROR: --worker requires one --query-id, --nat, --start and --end' . PHP_EOL; exit(1); } - exit(materialize_run_worker($query_id, $nat, $start, $end)); + exit(materialize_run_worker($query_ids[0], $nat, $start, $end)); } if (!$force && !$dry_run) { @@ -170,11 +176,12 @@ list($start, $end) = $range; -if ($query_id !== false) { +if (cacti_sizeof($query_ids)) { $queries = flowview_db_fetch_assoc_prepared('SELECT id, name FROM plugin_flowview_queries - WHERE id = ?', - [$query_id]); + WHERE id IN (' . implode(', ', array_fill(0, cacti_sizeof($query_ids), '?')) . ') + ORDER BY id', + $query_ids); } else { $queries = flowview_db_fetch_assoc('SELECT id, name FROM plugin_flowview_queries @@ -329,7 +336,7 @@ function display_version(): void { function display_help(): void { display_version(); - print PHP_EOL . 'usage: materialize_queries.php [--query-id=ID] [--nat=on|off] [--dry-run] [--force] [--debug]' . PHP_EOL . PHP_EOL; + print PHP_EOL . 'usage: materialize_queries.php [--query-id=ID[,ID...]] [--nat=on|off] [--dry-run] [--force] [--debug]' . PHP_EOL . PHP_EOL; print 'Materializes each FlowView query into the parallel query cache for the' . PHP_EOL; print 'entire range of flow partitions, once with NAT on and once with NAT off.' . PHP_EOL; @@ -337,7 +344,7 @@ function display_help(): void { print 'Only one run may be active at a time, for up to 8 hours.' . PHP_EOL . PHP_EOL; print 'Options:' . PHP_EOL; - print ' --query-id=ID Only materialize this query.' . PHP_EOL; + print ' --query-id=ID Only materialize this query, or a comma separated list of them.' . PHP_EOL; print ' --nat=on|off Only materialize this NAT mode.' . PHP_EOL; print ' --dry-run List what would be materialized and exit.' . PHP_EOL; print ' --force Run even if another materialization is running.' . PHP_EOL; diff --git a/flowview_filters.php b/flowview_filters.php index ca758a5..2edd7f2 100644 --- a/flowview_filters.php +++ b/flowview_filters.php @@ -41,7 +41,8 @@ /* 2 => __('Send Now', 'flowview'), */ 1 => __('Delete', 'flowview'), 3 => __('Disable', 'flowview'), - 4 => __('Enable', 'flowview') + 4 => __('Enable', 'flowview'), + 5 => __('Materialize View', 'flowview') ); switch (get_request_var('action')) { @@ -110,6 +111,12 @@ function actions_filters(): void { for ($i=0; $i + +

" . __('Click \'Continue\' to Materialize the following Filter(s) in the background, with NAT on and off, for the entire range of flow data. Partitions that are already materialized and accurate are skipped. This can take a long time on large data sets, and only one Materialization can run at a time.', 'flowview') . "

+
    $filter_list
+ + "; } if (!isset($filter_array)) { @@ -377,6 +391,16 @@ function clearFilter() { 'align' => 'right', 'sort' => 'ASC' ), + 'nosort2' => array( + 'display' => __('NAT Shards', 'flowview'), + 'align' => 'right', + 'tip' => __('The number of partitions of this Filter that are stored in the Parallel Query cache with NAT on.', 'flowview') + ), + 'nosort3' => array( + 'display' => __('Non-NAT Shards', 'flowview'), + 'align' => 'right', + 'tip' => __('The number of partitions of this Filter that are stored in the Parallel Query cache with NAT off.', 'flowview') + ), 'sortfield' => array( 'display' => __('Sort Field', 'flowview'), 'align' => 'right', @@ -420,6 +444,11 @@ function clearFilter() { form_selectable_cell($type, $filter['id']); form_selectable_cell($filter['id'], $filter['id'], '', 'right'); + + $shards = flowview_count_cached_shards((int) $filter['id']); + + form_selectable_cell($shards['nat'], $filter['id'], '', 'right'); + form_selectable_cell($shards['non_nat'], $filter['id'], '', 'right'); form_selectable_cell($sort, $filter['id'], '', 'right'); form_selectable_cell($filter['resolve'], $filter['id'], '', 'right'); form_checkbox_cell($filter['name'], $filter['id']); diff --git a/includes/functions.php b/includes/functions.php index d5bbc50..03dc761 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -451,6 +451,82 @@ function changeRType() { 0 && array_key_exists('sortfield', $new)) { + return trim((string) ($old['sortfield'] ?? '')) !== trim((string) $new['sortfield']); + } + + return false; +} + +/** + * flowview_rematerialize_recommendation - The text recommending that a + * filter be materialized again after a material change. + * + * @return string + */ +function flowview_rematerialize_recommendation(): string { + return __('The changes to this Filter affect its cached results. A forced re-materialization of this Filter is recommended. To do so, select the Filter on the Filters page and choose the \'Materialize View\' action.', 'flowview'); +} + /** * @return void */ @@ -481,6 +557,21 @@ function save_filter_form(): void { $statistical = 0; } + $old = flowview_db_fetch_row_prepared('SELECT * + FROM plugin_flowview_queries + WHERE id = ?', + array(get_request_var('query'))); + + $material = flowview_filter_change_is_material( + is_array($old) ? $old : [], + [ + 'device_id' => get_request_var('device_id'), + 'statistics' => $statistical, + 'printed' => $printed, + 'sortfield' => get_nfilter_request_var('sortfield'), + ] + ); + flowview_db_execute_prepared('UPDATE plugin_flowview_queries SET timespan = ?, device_id = ?, @@ -515,6 +606,8 @@ function save_filter_form(): void { get_request_var('query') ) ); + + print json_encode(['rematerialize' => $material]); } /** @@ -587,6 +680,15 @@ function save_filter(): void { $save['panel_table'] = 'on'; } + $old = []; + + if (!empty($save['id'])) { + $old = flowview_db_fetch_row_prepared('SELECT * + FROM plugin_flowview_queries + WHERE id = ?', + array($save['id'])); + } + $id = flowview_sql_save($save, 'plugin_flowview_queries', 'id', true); if (is_error_message()) { @@ -603,6 +705,10 @@ function save_filter(): void { raise_message(1); + if (flowview_filter_change_is_material(is_array($old) ? $old : [], $save)) { + raise_message('flowview_rematerialize', flowview_rematerialize_recommendation(), MESSAGE_LEVEL_WARN); + } + flowview_purge_query_caches($id); if (!isset_request_var('return') || get_request_var('return') == '') { @@ -1791,7 +1897,7 @@ function numFormatter(num) { } function saveFilter() { - $.get(urlPath + 'plugins/flowview/flowview.php' + + $.getJSON(urlPath + 'plugins/flowview/flowview.php' + '?action=savefilter' + '&query=' + $('#query').val() + '&domains=' + $('#domains').is(':checked') + @@ -1809,9 +1915,14 @@ function saveFilter() { '&flows=' + $('#flows').is(':checked') + '&exclude=' + $('#exclude').val() + '&graph_type=' + $('#graph_type').val() + - '&graph_height=' + $('#graph_height').val(), function() { + '&graph_height=' + $('#graph_height').val(), function(data) { Pace.stop(); - $('#text').show().text('[ ]').fadeOut(2000); + + if (data['rematerialize']) { + $('#text').show().text('[ ]').fadeOut(10000); + } else { + $('#text').show().text('[ ]').fadeOut(2000); + } }); } @@ -2780,14 +2891,21 @@ function get_category_columns($statistics, $domain) { * filter table * an error. * - * @param mixed $session - * @param mixed $query_id - * @param mixed $start - * @param mixed $end + * When $plan_only is true nothing is run. The function returns the + * md5sum_tables of each map query the parallel query would cache + * against, which is how the cached shards of the query are found. + * $force_nat then decides the NAT mode instead of the request variables. + * + * @param mixed $session + * @param mixed $query_id + * @param mixed $start + * @param mixed $end + * @param bool $plan_only + * @param bool|null $force_nat * * @return mixed */ -function run_flow_query($session, $query_id, $start, $end) { +function run_flow_query($session, $query_id, $start, $end, bool $plan_only = false, ?bool $force_nat = null) { global $stat_columns_array; global $config, $graph_timespans; @@ -2809,7 +2927,9 @@ function run_flow_query($session, $query_id, $start, $end) { } /* close session to allow offpage navigation */ - cacti_session_close(); + if (!$plan_only) { + cacti_session_close(); + } require($config['base_path'] . '/plugins/flowview/includes/arrays.php'); @@ -2818,6 +2938,10 @@ function run_flow_query($session, $query_id, $start, $end) { WHERE id = ?', [$query_id]); + if ($plan_only && !cacti_sizeof($data)) { + return false; + } + $title = flowview_db_fetch_cell_prepared('SELECT name FROM plugin_flowview_queries WHERE id = ?', @@ -3413,11 +3537,18 @@ function run_flow_query($session, $query_id, $start, $end) { (preg_match('/^(bytes|packets|flows)$/i', $data['sortfield']) ? ' DESC':' ASC'); } } else { - cacti_log(sprintf('WARNING: The Flowview Query %s [%s] does not include either a Statistical or Printed Report.', $data['name'], $data['id']), false, 'FLOWVIEW'); + if (!$plan_only) { + cacti_log(sprintf('WARNING: The Flowview Query %s [%s] does not include either a Statistical or Printed Report.', $data['name'], $data['id']), false, 'FLOWVIEW'); + } + return false; } if ($sql_outer == '' || ($sql_inner == '' && $sql_inner1 == '')) { + if ($plan_only) { + return false; + } + if ($data['statistics'] > 0) { $message = __('The Statistical Report %s is not known. Please open a bug on GitHub', $data['statistics'], 'flowview'); } elseif($data['printed'] > 0) { @@ -3434,7 +3565,9 @@ function run_flow_query($session, $query_id, $start, $end) { } /* use post-NAT data instead of the original src/dst values - issue#110 */ - if (isset_request_var('usenat')) { + if ($force_nat !== null) { + $use_nat = $force_nat; + } elseif (isset_request_var('usenat')) { $use_nat = (get_request_var('usenat') == 'true'); } else { $use_nat = (isset($data['usenat']) && $data['usenat'] == 'on'); @@ -3474,6 +3607,83 @@ function run_flow_query($session, $query_id, $start, $end) { $sql_inner_groupby2 = flowview_apply_nat_toggle($sql_inner_groupby2, $use_nat, false); } + /** + * The map (inner) structures cache their shards by an md5 of the + * query without the time range, so they can be built before the + * tables are known. + */ + if ($sql_inner1 != '') { + $stru_outer = [ + 'sql_query' => $sql_outer, + 'sql_where' => '', + 'sql_groupby' => $sql_groupby, + 'sql_having' => $sql_having, + 'sql_order' => $sql_order, + 'sql_limit' => $sql_limit, + 'sql_params' => [] + ]; + + $stru_inners = []; + + foreach ([[$sql_inner1, $sql_inner_groupby1], [$sql_inner2, $sql_inner_groupby2]] as $inner) { + $stru_inner = [ + 'sql_query' => $inner[0], + 'sql_where' => $sql_where, + 'sql_range' => $sql_range, + 'sql_groupby' => $inner[1], + 'sql_having' => '', + 'sql_order' => '', + 'sql_limit' => '', + 'sql_params' => $sql_params, + 'sql_range_params' => $sql_range_params, + 'sql_start_time' => $start, + 'sql_end_time' => $end + ]; + + if ($request_type == 'printed') { + $stru_inner['sql_limit'] = 'LIMIT 100'; + $stru_inner['sql_order'] = $sql_order; + } + + $stru_inners[] = $stru_inner; + } + } else { + $stru_inner = [ + 'sql_query' => $sql_inner, + 'sql_where' => $sql_where, + 'sql_range' => $sql_range, + 'sql_having' => '', + 'sql_order' => '', + 'sql_limit' => '', + 'sql_groupby' => $sql_inner_groupby, + 'sql_params' => $sql_params, + 'sql_range_params' => $sql_range_params, + 'sql_start_time' => $start, + 'sql_end_time' => $end + ]; + + if ($request_type == 'printed') { + $stru_inner['sql_limit'] = 'LIMIT 100'; + $stru_inner['sql_order'] = $sql_order; + } + + $stru_outer = [ + 'sql_query' => $sql_outer, + 'sql_where' => '', + 'sql_having' => $sql_having, + 'sql_groupby' => $sql_groupby, + 'sql_order' => $sql_order, + 'sql_limit' => $sql_limit, + 'sql_params' => [] + ]; + + $stru_inners = [$stru_inner]; + } + + if ($plan_only) { + return array_map('parallel_database_query_table_md5', $stru_inners); + } + $tables = get_tables_for_query($start, $end); $sql = ''; $all_params = []; @@ -3541,84 +3751,7 @@ function run_flow_query($session, $query_id, $start, $end) { cacti_log(sprintf('PARALLEL STATS: Time:%0.3f Threads:%d Shards:%d', $end - $start, $threads, $shards), false, 'FLOWVIEW'); } else { - if ($sql_inner1 != '') { - $stru_inner1 = [ - 'sql_query' => $sql_inner1, - 'sql_where' => $sql_where, - 'sql_range' => $sql_range, - 'sql_groupby' => $sql_inner_groupby1, - 'sql_having' => '', - 'sql_order' => '', - 'sql_limit' => '', - 'sql_params' => $sql_params, - 'sql_range_params' => $sql_range_params, - 'sql_start_time' => $start, - 'sql_end_time' => $end - ]; - - $stru_inner2 = [ - 'sql_query' => $sql_inner2, - 'sql_where' => $sql_where, - 'sql_range' => $sql_range, - 'sql_groupby' => $sql_inner_groupby2, - 'sql_having' => '', - 'sql_order' => '', - 'sql_limit' => '', - 'sql_params' => $sql_params, - 'sql_range_params' => $sql_range_params, - 'sql_start_time' => $start, - 'sql_end_time' => $end - ]; - - $stru_outer = [ - 'sql_query' => $sql_outer, - 'sql_where' => '', - 'sql_groupby' => $sql_groupby, - 'sql_having' => $sql_having, - 'sql_order' => $sql_order, - 'sql_limit' => $sql_limit, - 'sql_params' => [] - ]; - - if ($request_type == 'printed') { - $stru_inner1['sql_limit'] = 'LIMIT 100'; - $stru_inner1['sql_order'] = $sql_order; - $stru_inner2['sql_limit'] = 'LIMIT 100'; - $stru_inner2['sql_order'] = $sql_order; - } - - $requests[] = parallel_database_query_request($tables, $stru_inner1, $stru_outer); - $requests[] = parallel_database_query_request($tables, $stru_inner2, $stru_outer); - } else { - $stru_inner = [ - 'sql_query' => $sql_inner, - 'sql_where' => $sql_where, - 'sql_range' => $sql_range, - 'sql_having' => '', - 'sql_order' => '', - 'sql_limit' => '', - 'sql_groupby' => $sql_inner_groupby, - 'sql_params' => $sql_params, - 'sql_range_params' => $sql_range_params, - 'sql_start_time' => $start, - 'sql_end_time' => $end - ]; - - if ($request_type == 'printed') { - $stru_inner['sql_limit'] = 'LIMIT 100'; - $stru_inner['sql_order'] = $sql_order; - } - - $stru_outer = [ - 'sql_query' => $sql_outer, - 'sql_where' => '', - 'sql_having' => $sql_having, - 'sql_groupby' => $sql_groupby, - 'sql_order' => $sql_order, - 'sql_limit' => $sql_limit, - 'sql_params' => [] - ]; - + foreach ($stru_inners as $stru_inner) { $requests[] = parallel_database_query_request($tables, $stru_inner, $stru_outer); } @@ -4137,6 +4270,107 @@ function flowview_invalidate_stale_shard_cache(): array { return ['kept' => $kept, 'removed' => $removed]; } +/** + * flowview_launch_materialize - Start cli/materialize_queries.php in the + * background for the given FlowView queries. Nothing is started when a + * materialization is already registered as running, since only one may be + * active at a time. + * + * @param array $query_ids The FlowView query ids to materialize + * + * @return bool false when a materialization is already running + */ +function flowview_launch_materialize(array $query_ids): bool { + global $config; + + $running = db_fetch_cell_prepared('SELECT COUNT(*) + FROM processes + WHERE tasktype = ? + AND taskname = ? + AND taskid = ?', + ['flowview', 'materialize', 0]); + + if ($running > 0) { + return false; + } + + $script = $config['base_path'] . '/plugins/flowview/cli/materialize_queries.php'; + + exec_background( + read_config_option('path_php_binary'), + cacti_escapeshellarg($script) . ' --query-id=' . implode(',', array_map('intval', $query_ids)) + ); + + return true; +} + +/** + * parallel_database_query_map_query - The map query of a request as the JSON + * that is stored and hashed. The time range is left out, so the same query + * over a different range shares the same cached partition shards. + * + * @param array $stru_inner The map structure of a parallel query request + * + * @return string + */ +function parallel_database_query_map_query(array $stru_inner): string { + unset($stru_inner['sql_range']); + unset($stru_inner['sql_range_params']); + unset($stru_inner['sql_start_time']); + unset($stru_inner['sql_end_time']); + + return (string) json_encode($stru_inner); +} + +/** + * parallel_database_query_table_md5 - The md5sum_tables of a map structure, + * which is the key the partition shards are cached under. + * + * @param array $stru_inner The map structure of a parallel query request + * + * @return string + */ +function parallel_database_query_table_md5(array $stru_inner): string { + return md5(parallel_database_query_map_query($stru_inner)); +} + +/** + * flowview_count_cached_shards - Count the partitions of a filter that are + * stored in the parallel query cache, with NAT and without. Some reports + * use two map queries, and a partition only counts once every one of them is + * cached. Nothing is run, the cache keys are worked out from the filter. + * + * @param int $query_id The FlowView query to count the cached shards of + * + * @return array The partition counts as nat and non_nat + */ +function flowview_count_cached_shards(int $query_id): array { + $counts = ['nat' => 0, 'non_nat' => 0]; + $now = time(); + + foreach (['nat' => true, 'non_nat' => false] as $mode => $use_nat) { + $md5sums = run_flow_query(false, $query_id, $now, $now, true, $use_nat); + + if (!is_array($md5sums) || !cacti_sizeof($md5sums)) { + continue; + } + + $md5sums = array_values(array_unique($md5sums)); + + $counts[$mode] = (int) flowview_db_fetch_cell_prepared('SELECT COUNT(*) + FROM ( + SELECT map_table + FROM parallel_database_query_shard_cache + WHERE md5sum IN (' . implode(', ', array_fill(0, cacti_sizeof($md5sums), '?')) . ') + GROUP BY map_table, map_partition + HAVING COUNT(DISTINCT md5sum) = ? + ) AS cached', + array_merge($md5sums, [cacti_sizeof($md5sums)])); + } + + return $counts; +} + /** * parallel_database_query_request - Given a series of tables and * and inner (map phase) and outer (reduce phase) SQL queries, @@ -4202,19 +4436,13 @@ function parallel_database_query_request($tables, $stru_inner, $stru_outer) { */ $map_range = $stru_inner['sql_range']; $map_range_params = $stru_inner['sql_range_params']; - $map_query = $stru_inner; $sql_start_time = $stru_inner['sql_start_time']; $sql_end_time = $stru_inner['sql_end_time']; $md5_array = [$stru_inner, $stru_outer]; $query_md5 = md5((string) json_encode($md5_array)); - unset($map_query['sql_range']); - unset($map_query['sql_range_params']); - unset($map_query['sql_start_time']); - unset($map_query['sql_end_time']); - $reduce_query = json_encode($stru_outer); - $map_query = (string) json_encode($map_query); + $map_query = parallel_database_query_map_query($stru_inner); $table_md5 = md5($map_query); diff --git a/tests/Unit/CoreFunctionsTest.php b/tests/Unit/CoreFunctionsTest.php index af36d43..e80e45f 100644 --- a/tests/Unit/CoreFunctionsTest.php +++ b/tests/Unit/CoreFunctionsTest.php @@ -294,3 +294,211 @@ expect($writes)->toBeEmpty(); }); + +it('flags only changes that alter what a filter materializes', function ($old, $new, $expected) { + expect(flowview_filter_change_is_material($old, $new))->toBe($expected); +})->with(array( + 'new filter' => array( + array(), + array('sourceip' => '10.0.0.1', 'statistics' => 10), + false, + ), + 'nothing changed' => array( + array('device_id' => '3', 'sourceip' => '10.0.0.1', 'statistics' => '10', 'printed' => '0'), + array('device_id' => '3', 'sourceip' => '10.0.0.1', 'statistics' => '10', 'printed' => '0'), + false, + ), + 'display and time settings' => array( + array('name' => 'Top Talkers', 'timespan' => '3', 'graph_type' => 'bar', 'cutofflines' => '20', 'usenat' => '', 'statistics' => '10'), + array('name' => 'Renamed', 'timespan' => '5', 'graph_type' => 'pie', 'cutofflines' => '50', 'usenat' => 'on', 'statistics' => '10'), + false, + ), + 'source address changed' => array( + array('sourceip' => '10.0.0.1'), + array('sourceip' => '10.0.0.2'), + true, + ), + 'protocol added' => array( + array('protocols' => ''), + array('protocols' => '6, 17'), + true, + ), + 'report type changed' => array( + array('statistics' => '10', 'printed' => '0'), + array('statistics' => '11', 'printed' => '0'), + true, + ), + 'device restricted' => array( + array('device_id' => '0'), + array('device_id' => '3'), + true, + ), + 'unrestricted saved differently' => array( + array('device_id' => '0', 'ex_addr' => '-1', 'protocols' => '0', 'template_id' => ''), + array('device_id' => '', 'ex_addr' => '', 'protocols' => '', 'template_id' => '0'), + false, + ), + 'sort field on printed report' => array( + array('printed' => '2', 'statistics' => '0', 'sortfield' => 'bytes'), + array('printed' => '2', 'statistics' => '0', 'sortfield' => 'packets'), + true, + ), + 'sort field on statistics report' => array( + array('printed' => '0', 'statistics' => '10', 'sortfield' => 'bytes'), + array('printed' => '0', 'statistics' => '10', 'sortfield' => 'packets'), + false, + ), + 'partial save only compares what it carries' => array( + array('device_id' => '3', 'sourceip' => '10.0.0.1', 'statistics' => '10'), + array('device_id' => '3'), + false, + ), +)); + +it('recommends a forced re-materialization pointing at the Materialize View action', function () { + expect(flowview_rematerialize_recommendation())->toContain('re-materialization') + ->and(flowview_rematerialize_recommendation())->toContain('Materialize View'); +}); + +it('starts the materialization in the background for the given queries', function () { + if (!function_exists('exec_background')) { + function exec_background($filename, $args = '', $redirect_args = '') { + $GLOBALS['__test_exec'][] = array($filename, $args); + } + } + + if (!function_exists('cacti_escapeshellarg')) { + function cacti_escapeshellarg($string) { + return escapeshellarg($string); + } + } + + $GLOBALS['__test_exec'] = array(); + $original_base_path = $GLOBALS['config']['base_path']; + $GLOBALS['config']['base_path'] = '/var/www/cacti'; + $GLOBALS['__test_config']['path_php_binary'] = '/usr/bin/php'; + + plugin_test_queue_db_result('db_fetch_cell_prepared', 0); + + try { + $started = flowview_launch_materialize(array('3', 5, '7; rm -rf /')); + } finally { + $GLOBALS['config']['base_path'] = $original_base_path; + } + + expect($started)->toBeTrue() + ->and($GLOBALS['__test_exec'])->toBe(array( + array('/usr/bin/php', "'/var/www/cacti/plugins/flowview/cli/materialize_queries.php' --query-id=3,5,7"), + )) + ->and($GLOBALS['__test_db_calls'][0]['params'])->toBe(array('flowview', 'materialize', 0)); +}); + +it('does not start another materialization while one is registered', function () { + $GLOBALS['__test_exec'] = array(); + + plugin_test_queue_db_result('db_fetch_cell_prepared', 1); + + expect(flowview_launch_materialize(array(3)))->toBeFalse() + ->and($GLOBALS['__test_exec'])->toBe(array()); +}); + +it('keys the cached shards on the map query without the time range', function () { + $inner = array( + 'sql_query' => 'SELECT src_addr, SUM(bytes) AS bytes', + 'sql_where' => 'WHERE protocol IN (?)', + 'sql_range' => '(`start_time` BETWEEN ? AND ?)', + 'sql_groupby' => 'GROUP BY src_addr', + 'sql_params' => array(6), + 'sql_range_params' => array('2024-01-01 00:00:00', '2024-01-02 00:00:00'), + 'sql_start_time' => 1704067200, + 'sql_end_time' => 1704153600, + ); + + $later = $inner; + $later['sql_range_params'] = array('2024-02-01 00:00:00', '2024-02-02 00:00:00'); + $later['sql_start_time'] = 1706745600; + $later['sql_end_time'] = 1706832000; + + $other = $inner; + $other['sql_where'] = 'WHERE protocol IN (?, ?)'; + + expect(parallel_database_query_table_md5($inner))->toBe(md5(parallel_database_query_map_query($inner))) + ->and(parallel_database_query_table_md5($later))->toBe(parallel_database_query_table_md5($inner)) + ->and(parallel_database_query_table_md5($other))->not->toBe(parallel_database_query_table_md5($inner)) + ->and(parallel_database_query_map_query($inner))->not->toContain('sql_range'); +}); + +it('plans the cache keys of a filter for each NAT mode without running it', function () { + if (!function_exists('array_rekey')) { + function array_rekey($array, $key, $value) { + return array(); + } + } + + $filter = array( + 'id' => 4, 'name' => 'Top Talkers', 'device_id' => 0, 'ex_addr' => '', 'template_id' => -1, + 'statistics' => 10, 'printed' => 0, 'sortfield' => 'bytes', 'cutofflines' => 20, 'cutoffoctets' => 0, + 'includeif' => 0, 'usenat' => '', 'protocols' => '', 'tosfields' => '', 'tcpflags' => '', + ); + + $plan = function ($nat) use ($filter) { + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + + return run_flow_query(false, 4, 1704067200, 1704153600, true, $nat); + }; + + $nat = $plan(true); + $non_nat = $plan(false); + + expect($nat)->toBeArray()->toHaveCount(1) + ->and($nat[0])->toMatch('/^[0-9a-f]{32}$/') + ->and($non_nat[0])->toMatch('/^[0-9a-f]{32}$/') + ->and($nat[0])->not->toBe($non_nat[0]) + ->and($plan(true))->toBe($nat); + + $writes = array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['fn'], 'db_execute') === 0; + }); + + expect($writes)->toBeEmpty(); +}); + +it('counts the partitions of a filter that are cached with and without NAT', function () { + if (!function_exists('array_rekey')) { + function array_rekey($array, $key, $value) { + return array(); + } + } + + $filter = array( + 'id' => 4, 'name' => 'Top Talkers', 'device_id' => 0, 'ex_addr' => '', 'template_id' => -1, + 'statistics' => 10, 'printed' => 0, 'sortfield' => 'bytes', 'cutofflines' => 20, 'cutoffoctets' => 0, + 'includeif' => 0, 'usenat' => '', 'protocols' => '', 'tosfields' => '', 'tcpflags' => '', + ); + + // Each mode plans the filter (a row and the name) and then counts its cache rows. + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + plugin_test_queue_db_result('db_fetch_cell_prepared', 'Top Talkers'); + plugin_test_queue_db_result('db_fetch_cell_prepared', '12'); + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + plugin_test_queue_db_result('db_fetch_cell_prepared', 'Top Talkers'); + plugin_test_queue_db_result('db_fetch_cell_prepared', '7'); + + expect(flowview_count_cached_shards(4))->toBe(array('nat' => 12, 'non_nat' => 7)); + + $counts = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['sql'] ?? '', 'FROM parallel_database_query_shard_cache') !== false; + })); + + expect($counts)->toHaveCount(2) + ->and($counts[0]['params'])->toHaveCount(2) + ->and($counts[0]['params'][1])->toBe(1) + ->and($counts[0]['params'][0])->not->toBe($counts[1]['params'][0]); +}); + +it('counts nothing for a filter that cannot be planned', function () { + plugin_test_queue_db_result('db_fetch_row_prepared', array()); + plugin_test_queue_db_result('db_fetch_row_prepared', array()); + + expect(flowview_count_cached_shards(99))->toBe(array('nat' => 0, 'non_nat' => 0)); +}); From 391364a659bbe6fff807182886222d54552b11bd Mon Sep 17 00:00:00 2001 From: TheWitness Date: Tue, 20 Oct 2026 15:12:09 -0400 Subject: [PATCH 06/37] feature: Show the total partition count on the Filters page Add a Partitions column after the NAT and Non-NAT shard columns so the progress of a materialization can be followed. Count only genuine raw partition tables, and use the same helper in cli/materialize_queries.php so its partition count matches the page. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 2 +- README.md | 3 ++- cli/materialize_queries.php | 6 ++---- flowview_filters.php | 8 ++++++++ includes/functions.php | 23 +++++++++++++++++++++++ tests/Unit/CoreFunctionsTest.php | 16 ++++++++++++++++ 6 files changed, 52 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db2b404..754c998 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ --- develop --- -* feature: Show the number of partitions each Filter has stored in the Parallel Query cache, with NAT and without, as the NAT Shards and Non-NAT Shards columns next to the ID on the Filters page; the cache keys are worked out from the Filter without running it (run_flow_query() gained a plan-only mode, and the map query hashing is shared with the parallel query request) +* feature: Show the number of partitions each Filter has stored in the Parallel Query cache, with NAT and without, as the NAT Shards and Non-NAT Shards columns next to the ID on the Filters page, along with a Partitions column holding the total number of flow partitions so the progress of a materialization can be followed; the cache keys are worked out from the Filter without running it (run_flow_query() gained a plan-only mode, and the map query hashing is shared with the parallel query request) * feature: Add a 'Materialize View' action to the Filters page that, after confirmation, starts cli/materialize_queries.php in the background for the selected Filters (--query-id now accepts a comma separated list), and tells you when a materialization is already running diff --git a/README.md b/README.md index 990708f..b69d217 100644 --- a/README.md +++ b/README.md @@ -223,7 +223,8 @@ You can also start it from the Filters page by selecting one or more Filters and choosing the 'Materialize View' action. It runs in the background after you confirm, and the result is written to the Cacti log. The 'NAT Shards' and 'Non-NAT Shards' columns on that page show how many partitions of each Filter -are already cached, with NAT on and with NAT off. When you save a +are already cached, with NAT on and with NAT off, out of the total shown in the +'Partitions' column. When you save a Filter in a way that changes the data it materializes, such as its report, device, exporter, or any address, port, interface, AS, protocol, flag or TOS restriction, you are advised to re-materialize it. diff --git a/cli/materialize_queries.php b/cli/materialize_queries.php index 97a385e..839c847 100644 --- a/cli/materialize_queries.php +++ b/cli/materialize_queries.php @@ -163,11 +163,9 @@ exit(1); } -$tables = flowview_db_fetch_assoc('SELECT TABLE_NAME AS `table` - FROM information_schema.TABLES - WHERE TABLE_NAME LIKE "plugin_flowview_raw_%"'); +$tables = flowview_get_partition_tables(); -$range = flowview_get_materialize_range(array_column($tables, 'table')); +$range = flowview_get_materialize_range($tables); if ($range === false) { print 'NOTE: There are no flow partitions to materialize.' . PHP_EOL; diff --git a/flowview_filters.php b/flowview_filters.php index 2edd7f2..2d86c8f 100644 --- a/flowview_filters.php +++ b/flowview_filters.php @@ -401,6 +401,11 @@ function clearFilter() { 'align' => 'right', 'tip' => __('The number of partitions of this Filter that are stored in the Parallel Query cache with NAT off.', 'flowview') ), + 'nosort4' => array( + 'display' => __('Partitions', 'flowview'), + 'align' => 'right', + 'tip' => __('The total number of flow partitions, which is the most shards a Filter can have cached with or without NAT.', 'flowview') + ), 'sortfield' => array( 'display' => __('Sort Field', 'flowview'), 'align' => 'right', @@ -424,6 +429,8 @@ function clearFilter() { html_header_sort_checkbox($display_array, get_request_var('sort_column'), get_request_var('sort_direction'), false); if (cacti_sizeof($filters)) { + $partitions = cacti_sizeof(flowview_get_partition_tables()); + foreach ($filters as $filter) { form_alternate_row('line' . $filter['id'], true); form_selectable_cell('' . html_escape($filter['name']) . '', $filter['id']); @@ -449,6 +456,7 @@ function clearFilter() { form_selectable_cell($shards['nat'], $filter['id'], '', 'right'); form_selectable_cell($shards['non_nat'], $filter['id'], '', 'right'); + form_selectable_cell($partitions, $filter['id'], '', 'right'); form_selectable_cell($sort, $filter['id'], '', 'right'); form_selectable_cell($filter['resolve'], $filter['id'], '', 'right'); form_checkbox_cell($filter['name'], $filter['id']); diff --git a/includes/functions.php b/includes/functions.php index 03dc761..1716f00 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -4334,6 +4334,29 @@ function parallel_database_query_table_md5(array $stru_inner): string { return md5(parallel_database_query_map_query($stru_inner)); } +/** + * flowview_get_partition_tables - The names of the raw flow partition tables + * that exist. Anything that merely looks like a partition table, such as + * plugin_flowview_raw_abc, is left out. + * + * @return array + */ +function flowview_get_partition_tables(): array { + $tables = flowview_db_fetch_assoc('SELECT TABLE_NAME AS `table` + FROM information_schema.TABLES + WHERE TABLE_NAME LIKE "plugin_flowview_raw_%"'); + + $partitions = []; + + foreach ($tables as $table) { + if (preg_match('/^plugin_flowview_raw_[0-9]{7}([0-9]{2})?$/', $table['table'])) { + $partitions[] = $table['table']; + } + } + + return $partitions; +} + /** * flowview_count_cached_shards - Count the partitions of a filter that are * stored in the parallel query cache, with NAT and without. Some reports diff --git a/tests/Unit/CoreFunctionsTest.php b/tests/Unit/CoreFunctionsTest.php index e80e45f..a96ca0d 100644 --- a/tests/Unit/CoreFunctionsTest.php +++ b/tests/Unit/CoreFunctionsTest.php @@ -502,3 +502,19 @@ function array_rekey($array, $key, $value) { expect(flowview_count_cached_shards(99))->toBe(array('nat' => 0, 'non_nat' => 0)); }); + +it('lists only genuine raw partition tables', function () { + plugin_test_queue_db_result('db_fetch_assoc', array( + array('table' => 'plugin_flowview_raw_2024001'), + array('table' => 'plugin_flowview_raw_202400105'), + array('table' => 'plugin_flowview_raw_abc'), + array('table' => 'plugin_flowview_rawX2024001'), + array('table' => 'plugin_flowview_raw_2024001_bak'), + )); + + expect(flowview_get_partition_tables())->toBe(array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_202400105')); +}); + +it('lists no partition tables when none exist', function () { + expect(flowview_get_partition_tables())->toBe(array()); +}); From 9e7e05d6b4a2fa09527171a333e0127b9bdfd027 Mon Sep 17 00:00:00 2001 From: TheWitness Date: Tue, 20 Oct 2026 15:13:29 -0400 Subject: [PATCH 07/37] fix: Format the shard and partition counts with number_format_i18n Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- flowview_filters.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flowview_filters.php b/flowview_filters.php index 2d86c8f..eb6708e 100644 --- a/flowview_filters.php +++ b/flowview_filters.php @@ -454,9 +454,9 @@ function clearFilter() { $shards = flowview_count_cached_shards((int) $filter['id']); - form_selectable_cell($shards['nat'], $filter['id'], '', 'right'); - form_selectable_cell($shards['non_nat'], $filter['id'], '', 'right'); - form_selectable_cell($partitions, $filter['id'], '', 'right'); + form_selectable_cell(number_format_i18n($shards['nat'], 0), $filter['id'], '', 'right'); + form_selectable_cell(number_format_i18n($shards['non_nat'], 0), $filter['id'], '', 'right'); + form_selectable_cell(number_format_i18n($partitions, 0), $filter['id'], '', 'right'); form_selectable_cell($sort, $filter['id'], '', 'right'); form_selectable_cell($filter['resolve'], $filter['id'], '', 'right'); form_checkbox_cell($filter['name'], $filter['id']); From f6be4f7e3f42106e17b437ef6266fda60c36327f Mon Sep 17 00:00:00 2001 From: TheWitness Date: Tue, 20 Oct 2026 15:27:06 -0400 Subject: [PATCH 08/37] feature: Add materialization status pills and automatic materialization of rotated partitions Add a Status column to the Filters page showing whether the automatic materialization of each filter is required, running or up to date. Each poller cycle now records the newest partition and starts cli/materialize_queries.php in the background for the partitions that went cold since the previous cycle, for every filter with NAT on and off. The script gains a --partitions option, and the cache check can be limited to those partitions. The cached shard lookup now returns the partitions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 4 + README.md | 13 +- cli/materialize_queries.php | 52 ++++-- flowview_filters.php | 20 ++- flowview_process.php | 3 + includes/functions.php | 267 ++++++++++++++++++++++++++----- tests/Unit/CoreFunctionsTest.php | 251 +++++++++++++++++++++++++++-- 7 files changed, 538 insertions(+), 72 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 754c998..d5c1aa8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ --- develop --- +* feature: Materialize the partitions that have gone cold automatically. Each poller cycle flowview_process.php records the newest partition, which is the one still receiving flows, and starts cli/materialize_queries.php in the background for the partitions that have rotated since the previous cycle, for every Filter with NAT on and off. Partitions that are already materialized and accurate are skipped. When a materialization is already running, or the Parallel Query is not enabled, the record is left alone so a later cycle catches up. The first cycle only records the newest partition, so older partitions are still materialized with cli/materialize_queries.php. The script gained a --partitions option for this + +* feature: Add a Status column, to the right of the Filter Name on the Filters page, showing the automatic materialization status as a pill: Required (down) when rotated partitions are not materialized yet, Running (recovering) while a materialization is in progress, and Up to date (up) once every cold partition is materialized with NAT on and off + * feature: Show the number of partitions each Filter has stored in the Parallel Query cache, with NAT and without, as the NAT Shards and Non-NAT Shards columns next to the ID on the Filters page, along with a Partitions column holding the total number of flow partitions so the progress of a materialization can be followed; the cache keys are worked out from the Filter without running it (run_flow_query() gained a plan-only mode, and the map query hashing is shared with the parallel query request) * feature: Add a 'Materialize View' action to the Filters page that, after confirmation, starts cli/materialize_queries.php in the background for the selected Filters (--query-id now accepts a comma separated list), and tells you when a materialization is already running diff --git a/README.md b/README.md index b69d217..7a3f80e 100644 --- a/README.md +++ b/README.md @@ -224,7 +224,18 @@ and choosing the 'Materialize View' action. It runs in the background after you confirm, and the result is written to the Cacti log. The 'NAT Shards' and 'Non-NAT Shards' columns on that page show how many partitions of each Filter are already cached, with NAT on and with NAT off, out of the total shown in the -'Partitions' column. When you save a +'Partitions' column. + +New partitions are materialized automatically. Each poller cycle records the +newest partition, which is the one still receiving flows, and materializes the +partitions that have rotated since the previous cycle for every Filter, with +NAT on and off, in the background. The first cycle only records the newest +partition, so run the script once to materialize the existing ones. The +'Status' column on the Filters page shows the result for each Filter, as +Required, Running or Up to date. The partition still receiving flows is not +materialized automatically, so it does not count. + +When you save a Filter in a way that changes the data it materializes, such as its report, device, exporter, or any address, port, interface, AS, protocol, flag or TOS restriction, you are advised to re-materialize it. diff --git a/cli/materialize_queries.php b/cli/materialize_queries.php index 839c847..6f7cdb9 100644 --- a/cli/materialize_queries.php +++ b/cli/materialize_queries.php @@ -41,18 +41,20 @@ /* another materialization may not start until this one ends or this many seconds pass */ const MATERIALIZE_RUNTIME_LIMIT = 28800; -$debug = false; -$force = false; -$dry_run = false; -$worker = false; +$debug = false; +$force = false; +$dry_run = false; +$worker = false; $query_ids = []; -$nat = false; -$start = false; -$end = false; +$partitions = []; +$nat = false; +$start = false; +$end = false; $shortopts = 'VvHh'; $longopts = [ 'query-id:', + 'partitions:', 'nat:', 'start:', 'end:', @@ -84,6 +86,21 @@ $query_ids = array_values($query_ids); + break; + case 'partitions': + $partitions = []; + + foreach (explode(',', (string) $value) as $table) { + if (flowview_partition_sort_key($table) === '') { + print 'ERROR: --partitions must be a comma separated list of raw partition tables' . PHP_EOL; + exit(1); + } + + $partitions[$table] = $table; + } + + $partitions = array_values($partitions); + break; case 'nat': $nat = strtolower((string) $value); @@ -165,6 +182,11 @@ $tables = flowview_get_partition_tables(); +/* only the partitions asked for, that still exist */ +if (cacti_sizeof($partitions)) { + $tables = array_values(array_intersect($tables, $partitions)); +} + $range = flowview_get_materialize_range($tables); if ($range === false) { @@ -202,7 +224,7 @@ /* partitions that are still accurate keep their cache, everything else is rebuilt below */ if (!$dry_run) { - $cache = flowview_invalidate_stale_shard_cache(); + $cache = flowview_invalidate_stale_shard_cache(cacti_sizeof($partitions) ? $tables : []); print sprintf('Cached Partitions Still Accurate:%d, Stale Partitions Removed:%d', $cache['kept'], $cache['removed']) . PHP_EOL; } @@ -334,7 +356,7 @@ function display_version(): void { function display_help(): void { display_version(); - print PHP_EOL . 'usage: materialize_queries.php [--query-id=ID[,ID...]] [--nat=on|off] [--dry-run] [--force] [--debug]' . PHP_EOL . PHP_EOL; + print PHP_EOL . 'usage: materialize_queries.php [--query-id=ID[,ID...]] [--partitions=TABLE[,TABLE...]] [--nat=on|off] [--dry-run] [--force] [--debug]' . PHP_EOL . PHP_EOL; print 'Materializes each FlowView query into the parallel query cache for the' . PHP_EOL; print 'entire range of flow partitions, once with NAT on and once with NAT off.' . PHP_EOL; @@ -342,9 +364,11 @@ function display_help(): void { print 'Only one run may be active at a time, for up to 8 hours.' . PHP_EOL . PHP_EOL; print 'Options:' . PHP_EOL; - print ' --query-id=ID Only materialize this query, or a comma separated list of them.' . PHP_EOL; - print ' --nat=on|off Only materialize this NAT mode.' . PHP_EOL; - print ' --dry-run List what would be materialized and exit.' . PHP_EOL; - print ' --force Run even if another materialization is running.' . PHP_EOL; - print ' --debug Provide some debug output during the run.' . PHP_EOL . PHP_EOL; + print ' --query-id=ID Only materialize this query, or a comma separated list of them.' . PHP_EOL; + print ' --partitions=TABLE Only materialize these partition tables, as the poller does for the' . PHP_EOL; + print ' partitions that have rotated.' . PHP_EOL; + print ' --nat=on|off Only materialize this NAT mode.' . PHP_EOL; + print ' --dry-run List what would be materialized and exit.' . PHP_EOL; + print ' --force Run even if another materialization is running.' . PHP_EOL; + print ' --debug Provide some debug output during the run.' . PHP_EOL . PHP_EOL; } diff --git a/flowview_filters.php b/flowview_filters.php index eb6708e..01d2503 100644 --- a/flowview_filters.php +++ b/flowview_filters.php @@ -376,6 +376,11 @@ function clearFilter() { 'align' => 'left', 'sort' => 'ASC' ), + 'nosort5' => array( + 'display' => __('Status', 'flowview'), + 'align' => 'left', + 'tip' => __('The automatic materialization status of the Filter. Required means partitions that have rotated are not materialized yet, Running means a materialization is in progress, and Up to date means every rotated partition is materialized with NAT on and off. The partition still receiving flows is not materialized automatically.', 'flowview') + ), 'device' => array( 'display' => __('Listener', 'flowview'), 'align' => 'left', @@ -429,11 +434,18 @@ function clearFilter() { html_header_sort_checkbox($display_array, get_request_var('sort_column'), get_request_var('sort_direction'), false); if (cacti_sizeof($filters)) { - $partitions = cacti_sizeof(flowview_get_partition_tables()); + $tables = flowview_get_partition_tables(); + $cold = flowview_get_cold_partitions($tables); + $partitions = cacti_sizeof($tables); + $running = flowview_materialization_running(); foreach ($filters as $filter) { + $cached = flowview_get_cached_shard_tables((int) $filter['id']); + $status = flowview_materialization_status($cached['nat'], $cached['non_nat'], $cold, $running); + form_alternate_row('line' . $filter['id'], true); form_selectable_cell('' . html_escape($filter['name']) . '', $filter['id']); + form_selectable_cell(flowview_get_colored_materialization_status($status), $filter['id']); if (empty($filter['device'])) { form_selectable_cell(__('All', 'flowview'), $filter['id']); @@ -452,10 +464,8 @@ function clearFilter() { form_selectable_cell($type, $filter['id']); form_selectable_cell($filter['id'], $filter['id'], '', 'right'); - $shards = flowview_count_cached_shards((int) $filter['id']); - - form_selectable_cell(number_format_i18n($shards['nat'], 0), $filter['id'], '', 'right'); - form_selectable_cell(number_format_i18n($shards['non_nat'], 0), $filter['id'], '', 'right'); + form_selectable_cell(number_format_i18n(cacti_sizeof($cached['nat']), 0), $filter['id'], '', 'right'); + form_selectable_cell(number_format_i18n(cacti_sizeof($cached['non_nat']), 0), $filter['id'], '', 'right'); form_selectable_cell(number_format_i18n($partitions, 0), $filter['id'], '', 'right'); form_selectable_cell($sort, $filter['id'], '', 'right'); form_selectable_cell($filter['resolve'], $filter['id'], '', 'right'); diff --git a/flowview_process.php b/flowview_process.php index 7b3f235..501bb18 100644 --- a/flowview_process.php +++ b/flowview_process.php @@ -196,6 +196,9 @@ /* prune expired query results */ parallel_database_query_expire(); +/* partitions that went cold this cycle are materialized for every filter */ +flowview_materialize_rotated_partitions(); + if ($maint) { flowview_debug('Performing Table Maintenance'); diff --git a/includes/functions.php b/includes/functions.php index 1716f00..4502835 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -4215,16 +4215,27 @@ function flowview_shard_cache_is_current(array $cached, array $current): bool { * When anything is removed, the finished CLI requests (user 0) are dropped * as well so their results are rebuilt from the refreshed shards. * + * @param array $only Only check the cache of these partition tables, or all + * of it when empty + * * @return array The number of shards kept and removed */ -function flowview_invalidate_stale_shard_cache(): array { +function flowview_invalidate_stale_shard_cache(array $only = []): array { $kept = 0; $removed = 0; $current = []; - $cache = flowview_db_fetch_assoc('SELECT md5sum, map_table, map_partition, min_date, max_date - FROM parallel_database_query_shard_cache - ORDER BY map_table'); + if (cacti_sizeof($only)) { + $cache = flowview_db_fetch_assoc_prepared('SELECT md5sum, map_table, map_partition, min_date, max_date + FROM parallel_database_query_shard_cache + WHERE map_table IN (' . implode(', ', array_fill(0, cacti_sizeof($only), '?')) . ') + ORDER BY map_table', + array_values($only)); + } else { + $cache = flowview_db_fetch_assoc('SELECT md5sum, map_table, map_partition, min_date, max_date + FROM parallel_database_query_shard_cache + ORDER BY map_table'); + } if (cacti_sizeof($cache)) { foreach ($cache as $entry) { @@ -4272,34 +4283,38 @@ function flowview_invalidate_stale_shard_cache(): array { /** * flowview_launch_materialize - Start cli/materialize_queries.php in the - * background for the given FlowView queries. Nothing is started when a + * background, for the given FlowView queries and the given partitions, or + * all of them when none are given. Nothing is started when a * materialization is already registered as running, since only one may be * active at a time. * - * @param array $query_ids The FlowView query ids to materialize + * @param array $query_ids The FlowView query ids to materialize + * @param array $partitions The partition tables to materialize * * @return bool false when a materialization is already running */ -function flowview_launch_materialize(array $query_ids): bool { +function flowview_launch_materialize(array $query_ids = [], array $partitions = []): bool { global $config; - $running = db_fetch_cell_prepared('SELECT COUNT(*) - FROM processes - WHERE tasktype = ? - AND taskname = ? - AND taskid = ?', - ['flowview', 'materialize', 0]); - - if ($running > 0) { + if (flowview_materialization_running()) { return false; } - $script = $config['base_path'] . '/plugins/flowview/cli/materialize_queries.php'; + $arguments = cacti_escapeshellarg($config['base_path'] . '/plugins/flowview/cli/materialize_queries.php'); - exec_background( - read_config_option('path_php_binary'), - cacti_escapeshellarg($script) . ' --query-id=' . implode(',', array_map('intval', $query_ids)) - ); + if (cacti_sizeof($query_ids)) { + $arguments .= ' --query-id=' . implode(',', array_map('intval', $query_ids)); + } + + $partitions = array_filter($partitions, function ($table) { + return flowview_partition_sort_key($table) !== ''; + }); + + if (cacti_sizeof($partitions)) { + $arguments .= ' --partitions=' . implode(',', $partitions); + } + + exec_background(read_config_option('path_php_binary'), $arguments); return true; } @@ -4334,10 +4349,28 @@ function parallel_database_query_table_md5(array $stru_inner): string { return md5(parallel_database_query_map_query($stru_inner)); } +/** + * flowview_partition_sort_key - The partition of a raw table name as a + * fixed length key that sorts in time order, whether the partitions are + * daily or hourly. An empty key is returned for anything that is not a + * raw partition table. + * + * @param string $table + * + * @return string + */ +function flowview_partition_sort_key(string $table): string { + if (!preg_match('/^plugin_flowview_raw_([0-9]{7})([0-9]{2})?$/', $table, $matches)) { + return ''; + } + + return $matches[1] . ($matches[2] ?? '00'); +} + /** * flowview_get_partition_tables - The names of the raw flow partition tables - * that exist. Anything that merely looks like a partition table, such as - * plugin_flowview_raw_abc, is left out. + * that exist, oldest first. Anything that merely looks like a partition + * table, such as plugin_flowview_raw_abc, is left out. * * @return array */ @@ -4349,26 +4382,187 @@ function flowview_get_partition_tables(): array { $partitions = []; foreach ($tables as $table) { - if (preg_match('/^plugin_flowview_raw_[0-9]{7}([0-9]{2})?$/', $table['table'])) { + if (flowview_partition_sort_key($table['table']) !== '') { $partitions[] = $table['table']; } } + usort($partitions, function ($a, $b) { + return strcmp(flowview_partition_sort_key($a), flowview_partition_sort_key($b)); + }); + return $partitions; } /** - * flowview_count_cached_shards - Count the partitions of a filter that are + * flowview_get_cold_partitions - The partitions that have rotated. The + * newest partition is the one still receiving flows, every older one is cold. + * + * @param array $tables The partition tables + * + * @return array + */ +function flowview_get_cold_partitions(array $tables): array { + $keys = array_map('flowview_partition_sort_key', $tables); + + if (!cacti_sizeof($keys)) { + return []; + } + + $newest = max($keys); + + return array_values(array_filter($tables, function ($table) use ($newest) { + return flowview_partition_sort_key($table) < $newest; + })); +} + +/** + * flowview_get_rotated_partitions - The partitions that went cold since the + * previous newest partition was recorded. That partition was still hot then, + * so it is the first of them, and the newest partition is still hot now. + * + * @param array $tables The partition tables + * @param string $previous The newest partition recorded by the previous cycle + * + * @return array + */ +function flowview_get_rotated_partitions(array $tables, string $previous): array { + $previous_key = flowview_partition_sort_key($previous); + $keys = array_map('flowview_partition_sort_key', $tables); + + if ($previous_key === '' || !cacti_sizeof($keys)) { + return []; + } + + $newest = max($keys); + + return array_values(array_filter($tables, function ($table) use ($previous_key, $newest) { + $key = flowview_partition_sort_key($table); + + return $key >= $previous_key && $key < $newest; + })); +} + +/** + * flowview_materialization_running - Whether a materialization is registered + * as running. Only one may be active at a time. + * + * @return bool + */ +function flowview_materialization_running(): bool { + return db_fetch_cell_prepared('SELECT COUNT(*) + FROM processes + WHERE tasktype = ? + AND taskname = ? + AND taskid = ?', + ['flowview', 'materialize', 0]) > 0; +} + +/** + * flowview_materialization_status - The automatic materialization status of + * a filter. It is current once every cold partition is cached with NAT and + * without. The hot partition is not materialized automatically, so it does + * not count. Otherwise it is running while a materialization is active, and + * required when none is. + * + * @param array $cached_nat The partitions cached for the filter with NAT + * @param array $cached_non_nat The partitions cached for the filter without NAT + * @param array $cold The cold partitions + * @param bool $running Whether a materialization is running + * + * @return string current, running or required + */ +function flowview_materialization_status(array $cached_nat, array $cached_non_nat, array $cold, bool $running): string { + if (!cacti_sizeof(array_diff($cold, $cached_nat)) && !cacti_sizeof(array_diff($cold, $cached_non_nat))) { + return 'current'; + } + + return $running ? 'running' : 'required'; +} + +/** + * flowview_get_colored_materialization_status - The pill for the automatic + * materialization status of a filter, using the same colors as the device + * status in Cacti: down when it is required, recovering while it is running + * and up once it is up to date. + * + * @param string $status current, running or required + * + * @return string + */ +function flowview_get_colored_materialization_status(string $status): string { + switch ($status) { + case 'current': + return '' . __('Up to date', 'flowview') . ''; + case 'running': + return '' . __('Running', 'flowview') . ''; + default: + return '' . __('Required', 'flowview') . ''; + } +} + +/** + * flowview_materialize_rotated_partitions - Called every poller cycle to + * materialize the partitions that went cold since the previous cycle, for + * every filter with NAT and without. Partitions that are already + * materialized and still accurate are skipped by the materializer itself. + * + * The newest partition is recorded each cycle. The record is left alone + * while nothing can be started, so a later cycle picks up what was missed. + * The first cycle only records it, as the older partitions are materialized + * with cli/materialize_queries.php. + * + * @return array The partitions a materialization was started for + */ +function flowview_materialize_rotated_partitions(): array { + $tables = flowview_get_partition_tables(); + + if (!cacti_sizeof($tables)) { + return []; + } + + $newest = end($tables); + $previous = (string) read_config_option('flowview_last_hot_partition'); + + if ($previous == '') { + set_config_option('flowview_last_hot_partition', $newest); + + return []; + } + + $rotated = flowview_get_rotated_partitions($tables, $previous); + + if (!cacti_sizeof($rotated)) { + if ($previous != $newest) { + set_config_option('flowview_last_hot_partition', $newest); + } + + return []; + } + + $threads = read_config_option('flowview_parallel_threads'); + + if (empty($threads) || $threads == 1 || !flowview_launch_materialize([], $rotated)) { + return []; + } + + set_config_option('flowview_last_hot_partition', $newest); + + return $rotated; +} + +/** + * flowview_get_cached_shard_tables - The partitions of a filter that are * stored in the parallel query cache, with NAT and without. Some reports * use two map queries, and a partition only counts once every one of them is * cached. Nothing is run, the cache keys are worked out from the filter. * - * @param int $query_id The FlowView query to count the cached shards of + * @param int $query_id The FlowView query to find the cached shards of * - * @return array The partition counts as nat and non_nat + * @return array The partition tables cached as nat and non_nat */ -function flowview_count_cached_shards(int $query_id): array { - $counts = ['nat' => 0, 'non_nat' => 0]; +function flowview_get_cached_shard_tables(int $query_id): array { + $cached = ['nat' => [], 'non_nat' => []]; $now = time(); foreach (['nat' => true, 'non_nat' => false] as $mode => $use_nat) { @@ -4380,18 +4574,17 @@ function flowview_count_cached_shards(int $query_id): array { $md5sums = array_values(array_unique($md5sums)); - $counts[$mode] = (int) flowview_db_fetch_cell_prepared('SELECT COUNT(*) - FROM ( - SELECT map_table - FROM parallel_database_query_shard_cache - WHERE md5sum IN (' . implode(', ', array_fill(0, cacti_sizeof($md5sums), '?')) . ') - GROUP BY map_table, map_partition - HAVING COUNT(DISTINCT md5sum) = ? - ) AS cached', + $tables = flowview_db_fetch_assoc_prepared('SELECT map_table + FROM parallel_database_query_shard_cache + WHERE md5sum IN (' . implode(', ', array_fill(0, cacti_sizeof($md5sums), '?')) . ') + GROUP BY map_table, map_partition + HAVING COUNT(DISTINCT md5sum) = ?', array_merge($md5sums, [cacti_sizeof($md5sums)])); + + $cached[$mode] = array_column($tables, 'map_table'); } - return $counts; + return $cached; } /** diff --git a/tests/Unit/CoreFunctionsTest.php b/tests/Unit/CoreFunctionsTest.php index a96ca0d..e092033 100644 --- a/tests/Unit/CoreFunctionsTest.php +++ b/tests/Unit/CoreFunctionsTest.php @@ -463,7 +463,7 @@ function array_rekey($array, $key, $value) { expect($writes)->toBeEmpty(); }); -it('counts the partitions of a filter that are cached with and without NAT', function () { +it('finds the partitions of a filter that are cached with and without NAT', function () { if (!function_exists('array_rekey')) { function array_rekey($array, $key, $value) { return array(); @@ -476,45 +476,266 @@ function array_rekey($array, $key, $value) { 'includeif' => 0, 'usenat' => '', 'protocols' => '', 'tosfields' => '', 'tcpflags' => '', ); - // Each mode plans the filter (a row and the name) and then counts its cache rows. + // Each mode plans the filter (a row and the name) and then looks up its cache rows. plugin_test_queue_db_result('db_fetch_row_prepared', $filter); plugin_test_queue_db_result('db_fetch_cell_prepared', 'Top Talkers'); - plugin_test_queue_db_result('db_fetch_cell_prepared', '12'); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array( + array('map_table' => 'plugin_flowview_raw_2024001'), + array('map_table' => 'plugin_flowview_raw_2024002'), + )); plugin_test_queue_db_result('db_fetch_row_prepared', $filter); plugin_test_queue_db_result('db_fetch_cell_prepared', 'Top Talkers'); - plugin_test_queue_db_result('db_fetch_cell_prepared', '7'); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array( + array('map_table' => 'plugin_flowview_raw_2024001'), + )); - expect(flowview_count_cached_shards(4))->toBe(array('nat' => 12, 'non_nat' => 7)); + expect(flowview_get_cached_shard_tables(4))->toBe(array( + 'nat' => array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002'), + 'non_nat' => array('plugin_flowview_raw_2024001'), + )); - $counts = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + $lookups = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { return strpos($call['sql'] ?? '', 'FROM parallel_database_query_shard_cache') !== false; })); - expect($counts)->toHaveCount(2) - ->and($counts[0]['params'])->toHaveCount(2) - ->and($counts[0]['params'][1])->toBe(1) - ->and($counts[0]['params'][0])->not->toBe($counts[1]['params'][0]); + expect($lookups)->toHaveCount(2) + ->and($lookups[0]['params'])->toHaveCount(2) + ->and($lookups[0]['params'][1])->toBe(1) + ->and($lookups[0]['params'][0])->not->toBe($lookups[1]['params'][0]); }); -it('counts nothing for a filter that cannot be planned', function () { +it('finds nothing cached for a filter that cannot be planned', function () { plugin_test_queue_db_result('db_fetch_row_prepared', array()); plugin_test_queue_db_result('db_fetch_row_prepared', array()); - expect(flowview_count_cached_shards(99))->toBe(array('nat' => 0, 'non_nat' => 0)); + expect(flowview_get_cached_shard_tables(99))->toBe(array('nat' => array(), 'non_nat' => array())); }); -it('lists only genuine raw partition tables', function () { +it('lists only genuine raw partition tables, oldest first', function () { plugin_test_queue_db_result('db_fetch_assoc', array( - array('table' => 'plugin_flowview_raw_2024001'), array('table' => 'plugin_flowview_raw_202400105'), + array('table' => 'plugin_flowview_raw_2024001'), array('table' => 'plugin_flowview_raw_abc'), array('table' => 'plugin_flowview_rawX2024001'), array('table' => 'plugin_flowview_raw_2024001_bak'), + array('table' => 'plugin_flowview_raw_2023364'), )); - expect(flowview_get_partition_tables())->toBe(array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_202400105')); + expect(flowview_get_partition_tables())->toBe(array( + 'plugin_flowview_raw_2023364', + 'plugin_flowview_raw_2024001', + 'plugin_flowview_raw_202400105', + )); }); it('lists no partition tables when none exist', function () { expect(flowview_get_partition_tables())->toBe(array()); }); + +it('keys daily and hourly partitions so they sort in time order', function () { + expect(flowview_partition_sort_key('plugin_flowview_raw_2024001'))->toBe('202400100') + ->and(flowview_partition_sort_key('plugin_flowview_raw_202400105'))->toBe('202400105') + ->and(flowview_partition_sort_key('plugin_flowview_raw_abc'))->toBe('') + ->and(flowview_partition_sort_key('plugin_flowview_queries'))->toBe(''); +}); + +it('treats every partition but the newest as cold', function ($tables, $expected) { + expect(flowview_get_cold_partitions($tables))->toBe($expected); +})->with(array( + 'none' => array(array(), array()), + 'only the hot one' => array(array('plugin_flowview_raw_2024003'), array()), + 'several' => array( + array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002', 'plugin_flowview_raw_2024003'), + array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002'), + ), + 'hourly' => array( + array('plugin_flowview_raw_202400105', 'plugin_flowview_raw_202400106'), + array('plugin_flowview_raw_202400105'), + ), +)); + +it('finds the partitions that rotated since the previous newest one', function ($previous, $expected) { + $tables = array( + 'plugin_flowview_raw_2024001', + 'plugin_flowview_raw_2024002', + 'plugin_flowview_raw_2024003', + 'plugin_flowview_raw_2024004', + ); + + expect(flowview_get_rotated_partitions($tables, $previous))->toBe($expected); +})->with(array( + 'nothing recorded' => array('', array()), + 'not a partition' => array('whatever', array()), + 'newest is unchanged' => array('plugin_flowview_raw_2024004', array()), + 'one rotated' => array('plugin_flowview_raw_2024003', array('plugin_flowview_raw_2024003')), + 'several rotated' => array('plugin_flowview_raw_2024002', array('plugin_flowview_raw_2024002', 'plugin_flowview_raw_2024003')), + 'previous is gone' => array('plugin_flowview_raw_2024002_old', array()), + 'recorded is ahead' => array('plugin_flowview_raw_2024009', array()), +)); + +it('reports the automatic materialization status of a filter', function ($nat, $non_nat, $running, $expected) { + $cold = array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002'); + + expect(flowview_materialization_status($nat, $non_nat, $cold, $running))->toBe($expected); +})->with(array( + 'both complete' => array(array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002'), array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002'), false, 'current'), + 'complete while running' => array(array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002'), array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002'), true, 'current'), + 'hot one cached too' => array(array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002', 'plugin_flowview_raw_2024003'), array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002'), false, 'current'), + 'nat missing one' => array(array('plugin_flowview_raw_2024001'), array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002'), false, 'required'), + 'non nat missing' => array(array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002'), array(), false, 'required'), + 'incomplete and running' => array(array(), array(), true, 'running'), + 'nothing cached' => array(array(), array(), false, 'required'), +)); + +it('is up to date when there are no cold partitions to materialize', function () { + expect(flowview_materialization_status(array(), array(), array(), false))->toBe('current'); +}); + +it('shows the status pills in the colors of the device status', function ($status, $class, $label) { + expect(flowview_get_colored_materialization_status($status))->toBe('' . $label . ''); +})->with(array( + 'current' => array('current', 'deviceUp', 'Up to date'), + 'running' => array('running', 'deviceRecovering', 'Running'), + 'required' => array('required', 'deviceDown', 'Required'), + 'unknown' => array('anything', 'deviceDown', 'Required'), +)); + +it('starts a materialization for only the partitions it is given', function () { + if (!function_exists('exec_background')) { + function exec_background($filename, $args = '', $redirect_args = '') { + $GLOBALS['__test_exec'][] = array($filename, $args); + } + } + + if (!function_exists('cacti_escapeshellarg')) { + function cacti_escapeshellarg($string) { + return escapeshellarg($string); + } + } + + $GLOBALS['__test_exec'] = array(); + $original_base_path = $GLOBALS['config']['base_path']; + $GLOBALS['config']['base_path'] = '/var/www/cacti'; + $GLOBALS['__test_config']['path_php_binary'] = '/usr/bin/php'; + + plugin_test_queue_db_result('db_fetch_cell_prepared', 0); + + try { + $started = flowview_launch_materialize(array(), array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024001; rm', 'plugin_flowview_raw_2024002')); + } finally { + $GLOBALS['config']['base_path'] = $original_base_path; + } + + expect($started)->toBeTrue() + ->and($GLOBALS['__test_exec'])->toBe(array( + array('/usr/bin/php', "'/var/www/cacti/plugins/flowview/cli/materialize_queries.php' --partitions=plugin_flowview_raw_2024001,plugin_flowview_raw_2024002"), + )); +}); + +it('only checks the cache of the partitions it is limited to', function () { + plugin_test_queue_db_result('db_fetch_assoc_prepared', array( + array('md5sum' => 'aaa', 'map_table' => 'plugin_flowview_raw_2024001', 'map_partition' => '', 'min_date' => '2024-01-02 00:00:00', 'max_date' => '2024-01-02 23:00:00'), + )); + plugin_test_queue_db_result('db_fetch_cell', 1); + plugin_test_queue_db_result('db_fetch_row', array('min_date' => '2024-01-02 00:00:00', 'max_date' => '2024-01-02 23:00:00')); + + expect(flowview_invalidate_stale_shard_cache(array('plugin_flowview_raw_2024001')))->toBe(array('kept' => 1, 'removed' => 0)); + + $lookups = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_fetch_assoc_prepared' && strpos($call['sql'], 'map_table IN (?)') !== false; + })); + + expect($lookups)->toHaveCount(1) + ->and($lookups[0]['params'])->toBe(array('plugin_flowview_raw_2024001')); +}); + +describe('materializing the partitions that rotated', function () { + beforeEach(function () { + if (!function_exists('exec_background')) { + function exec_background($filename, $args = '', $redirect_args = '') { + $GLOBALS['__test_exec'][] = array($filename, $args); + } + } + + if (!function_exists('cacti_escapeshellarg')) { + function cacti_escapeshellarg($string) { + return escapeshellarg($string); + } + } + + $GLOBALS['__test_exec'] = array(); + $GLOBALS['__test_config']['flowview_parallel_threads'] = 4; + $GLOBALS['__test_config']['path_php_binary'] = '/usr/bin/php'; + + $this->tables = array( + array('table' => 'plugin_flowview_raw_2024001'), + array('table' => 'plugin_flowview_raw_2024002'), + array('table' => 'plugin_flowview_raw_2024003'), + ); + }); + + it('only records the newest partition on the first cycle', function () { + plugin_test_queue_db_result('db_fetch_assoc', $this->tables); + + expect(flowview_materialize_rotated_partitions())->toBe(array()) + ->and($GLOBALS['__test_config']['flowview_last_hot_partition'])->toBe('plugin_flowview_raw_2024003') + ->and($GLOBALS['__test_exec'])->toBe(array()); + }); + + it('does nothing when no partition has rotated', function () { + $GLOBALS['__test_config']['flowview_last_hot_partition'] = 'plugin_flowview_raw_2024003'; + plugin_test_queue_db_result('db_fetch_assoc', $this->tables); + + expect(flowview_materialize_rotated_partitions())->toBe(array()) + ->and($GLOBALS['__test_config']['flowview_last_hot_partition'])->toBe('plugin_flowview_raw_2024003') + ->and($GLOBALS['__test_exec'])->toBe(array()); + }); + + it('starts a materialization for the partitions that rotated and records the newest', function () { + $GLOBALS['__test_config']['flowview_last_hot_partition'] = 'plugin_flowview_raw_2024002'; + plugin_test_queue_db_result('db_fetch_assoc', $this->tables); + plugin_test_queue_db_result('db_fetch_cell_prepared', 0); + + $original_base_path = $GLOBALS['config']['base_path']; + $GLOBALS['config']['base_path'] = '/var/www/cacti'; + + try { + $started = flowview_materialize_rotated_partitions(); + } finally { + $GLOBALS['config']['base_path'] = $original_base_path; + } + + expect($started)->toBe(array('plugin_flowview_raw_2024002')) + ->and($GLOBALS['__test_config']['flowview_last_hot_partition'])->toBe('plugin_flowview_raw_2024003') + ->and($GLOBALS['__test_exec'])->toBe(array( + array('/usr/bin/php', "'/var/www/cacti/plugins/flowview/cli/materialize_queries.php' --partitions=plugin_flowview_raw_2024002"), + )); + }); + + it('leaves the record alone while a materialization is running so it is caught up later', function () { + $GLOBALS['__test_config']['flowview_last_hot_partition'] = 'plugin_flowview_raw_2024001'; + plugin_test_queue_db_result('db_fetch_assoc', $this->tables); + plugin_test_queue_db_result('db_fetch_cell_prepared', 1); + + expect(flowview_materialize_rotated_partitions())->toBe(array()) + ->and($GLOBALS['__test_config']['flowview_last_hot_partition'])->toBe('plugin_flowview_raw_2024001') + ->and($GLOBALS['__test_exec'])->toBe(array()); + }); + + it('leaves the record alone when the parallel query is not enabled', function () { + $GLOBALS['__test_config']['flowview_parallel_threads'] = 1; + $GLOBALS['__test_config']['flowview_last_hot_partition'] = 'plugin_flowview_raw_2024001'; + plugin_test_queue_db_result('db_fetch_assoc', $this->tables); + + expect(flowview_materialize_rotated_partitions())->toBe(array()) + ->and($GLOBALS['__test_config']['flowview_last_hot_partition'])->toBe('plugin_flowview_raw_2024001') + ->and($GLOBALS['__test_exec'])->toBe(array()); + }); + + it('does nothing without any partitions', function () { + plugin_test_queue_db_result('db_fetch_assoc', array()); + + expect(flowview_materialize_rotated_partitions())->toBe(array()) + ->and($GLOBALS['__test_config'])->not->toHaveKey('flowview_last_hot_partition'); + }); +}); From f954bfeffd26a155f0d69044dbe895cc04d086ad Mon Sep 17 00:00:00 2001 From: TheWitness Date: Tue, 20 Oct 2026 15:37:32 -0400 Subject: [PATCH 09/37] fix: Prepare partitions up front, stop repeated engine conversions and show status pills Have cli/materialize_queries.php wait for any ALTER TABLE already running on the partitions it needs and do the alters still due before the parallel query starts, checking each partition again just before altering it. What a partition needs altered is decided by flowview_get_partition_alter(), which the schema upgrade now also uses. Skip the poller's engine conversion while an ALTER TABLE is already running on the partition, so each cycle no longer queues another rebuild of the same table. Print a parallel query timeout in words when run from a script instead of the web page's JavaScript. Render the Filters page status as a pill in the form of the device status. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 8 +- README.md | 10 ++ cli/materialize_queries.php | 118 +++++++++++++- flowview_filters.php | 4 +- flowview_process.php | 5 +- flowview_upgrade.php | 65 ++------ includes/functions.php | 261 ++++++++++++++++++++++++++++++- tests/Unit/CoreFunctionsTest.php | 212 ++++++++++++++++++++++++- 8 files changed, 621 insertions(+), 62 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5c1aa8..4f93afa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,15 @@ --- develop --- +* bug: Stop the poller from starting the same ALTER TABLE ENGINE conversion on a partition every cycle. Until a long conversion finishes the partition still reports its old engine, so each cycle started another that waited for the first and then rebuilt the table again, which also held up every query scanning that partition. The conversion is skipped while an ALTER TABLE is already running on the partition, and cli/materialize_queries.php checks a partition again just before altering it, so it is never rebuilt twice + +* bug: Make a Parallel Query that reaches the Maximum Run Time report it in words when run from a script, such as cli/materialize_queries.php, instead of printing the JavaScript meant for the web page; the message is also written to the Cacti log, a script now exits with a failure, and the web page is unchanged + +* feature: Have cli/materialize_queries.php get the partitions ready before it starts the Parallel Query. It waits for an ALTER TABLE that is already running on a partition it needs, such as from the schema upgrade, since a query waits for it and then ends at the Maximum Run Time, and then does the alters that are still due itself, one partition at a time with progress shown. The partition still receiving flows is left alone. It also notes the partitions that have no NAT columns, as they have no NAT data. What a partition needs altered is now decided in one place, flowview_get_partition_alter(), which the schema upgrade also uses, so the ALTER it issues is unchanged + * feature: Materialize the partitions that have gone cold automatically. Each poller cycle flowview_process.php records the newest partition, which is the one still receiving flows, and starts cli/materialize_queries.php in the background for the partitions that have rotated since the previous cycle, for every Filter with NAT on and off. Partitions that are already materialized and accurate are skipped. When a materialization is already running, or the Parallel Query is not enabled, the record is left alone so a later cycle catches up. The first cycle only records the newest partition, so older partitions are still materialized with cli/materialize_queries.php. The script gained a --partitions option for this -* feature: Add a Status column, to the right of the Filter Name on the Filters page, showing the automatic materialization status as a pill: Required (down) when rotated partitions are not materialized yet, Running (recovering) while a materialization is in progress, and Up to date (up) once every cold partition is materialized with NAT on and off +* feature: Add a Status column, to the right of the Filter Name on the Filters page, showing the automatic materialization status as a pill in the form of the device status: Required (down) when rotated partitions are not materialized yet, Running (recovering) while a materialization is in progress, and Up to date (up) once every cold partition is materialized with NAT on and off * feature: Show the number of partitions each Filter has stored in the Parallel Query cache, with NAT and without, as the NAT Shards and Non-NAT Shards columns next to the ID on the Filters page, along with a Partitions column holding the total number of flow partitions so the progress of a materialization can be followed; the cache keys are worked out from the Filter without running it (run_flow_query() gained a plan-only mode, and the map query hashing is shared with the parallel query request) diff --git a/README.md b/README.md index 7a3f80e..12b40d2 100644 --- a/README.md +++ b/README.md @@ -219,6 +219,16 @@ while partitions that have received more flows since, or have been dropped, are rebuilt. Only one run may be active at a time, for up to 8 hours, unless `--force` is used. +Before it scans anything the script gets the partitions ready. A query waits +for an ALTER TABLE that is running on the partition it scans, and a long one +ends the query at the Maximum Run Time. So the script waits for an alter that +is already running on a partition it needs, for instance from the schema +upgrade, and then does the alters that are still due itself, one partition at a +time. The partition still receiving flows is left alone. The post-NAT columns +are not added, as that is a deliberate choice for large installs. The script +notes the partitions that do not have them, and they have no NAT data until +`flowview_upgrade_nat_columns.php` is run. + You can also start it from the Filters page by selecting one or more Filters and choosing the 'Materialize View' action. It runs in the background after you confirm, and the result is written to the Cacti log. The 'NAT Shards' and diff --git a/cli/materialize_queries.php b/cli/materialize_queries.php index 6f7cdb9..dc22c20 100644 --- a/cli/materialize_queries.php +++ b/cli/materialize_queries.php @@ -182,11 +182,16 @@ $tables = flowview_get_partition_tables(); +/* the newest partition is still receiving flows, so it is left as it is */ +$cold = flowview_get_cold_partitions($tables); + /* only the partitions asked for, that still exist */ if (cacti_sizeof($partitions)) { $tables = array_values(array_intersect($tables, $partitions)); } +$cold = array_values(array_intersect($tables, $cold)); + $range = flowview_get_materialize_range($tables); if ($range === false) { @@ -222,6 +227,13 @@ cacti_sizeof($queries), implode('/', $modes)) . PHP_EOL; +/** + * Any alter that is due has to be done before the parallel query starts. A + * query waits for an alter running on the partition it scans, and a long one + * ends the query at the Maximum Run Time. + */ +materialize_prepare_partitions($cold, $modes, $dry_run); + /* partitions that are still accurate keep their cache, everything else is rebuilt below */ if (!$dry_run) { $cache = flowview_invalidate_stale_shard_cache(cacti_sizeof($partitions) ? $tables : []); @@ -266,12 +278,111 @@ print sprintf('FlowView Materialization Complete: Time:%0.2f Runs:%d Failed:%d', $time, $total, $failed) . PHP_EOL; +if ($failed > 0) { + print 'NOTE: Partitions that finished materializing are kept, so run the materializer again to continue.' . PHP_EOL; +} + if (!$dry_run) { cacti_log(sprintf('FLOWVIEW MATERIALIZE STATS: Time:%0.2f Runs:%d Failed:%d', $time, $total, $failed), false, 'SYSTEM'); } exit($failed > 0 ? 1 : 0); +/** + * Waits until no ALTER TABLE is running on any of the partitions. A query + * waits for an alter on the partition it scans, so the parallel query must + * not start while one is running. + * + * @param array $tables The partitions to wait for + * + * @return void + */ +function materialize_wait_for_alters(array $tables): void { + $last_report = 0; + + while (cacti_sizeof($running = flowview_get_running_partition_alters($tables))) { + if (time() - $last_report >= 30) { + $last_report = time(); + + arsort($running); + + print sprintf('Waiting for an ALTER TABLE to finish on %d partition(s), the longest %s for %d seconds', cacti_sizeof($running), array_key_first($running), reset($running)) . PHP_EOL; + } + + sleep(5); + } +} + +/** + * Gets the partitions ready to be scanned. It waits for an alter that is + * already running on one of them, for instance from the schema upgrade, and + * then does the alters that are still due itself, so nothing is altered while + * the parallel query runs. The partition still receiving flows is left alone. + * + * A partition is looked at again just before it is altered. If another + * process started an alter on it in the meantime, this waits for it and then + * only alters what is still due, so a partition is never rebuilt twice. + * + * @param array $cold The partitions to prepare + * @param array $modes The NAT modes being materialized, on and off + * @param bool $dry_run Only say what would be done + * + * @return void + */ +function materialize_prepare_partitions(array $cold, array $modes, bool $dry_run): void { + if (!cacti_sizeof($cold)) { + return; + } + + $raw_engine = get_set_default_fast_engine(); + + if (!$dry_run) { + materialize_wait_for_alters($cold); + } + + $pending = flowview_get_pending_partition_alters($cold, $raw_engine); + + if (cacti_sizeof($pending)) { + print sprintf('%s %d partition(s) that still need an ALTER TABLE', $dry_run ? 'DRY RUN: Would alter' : 'Altering', cacti_sizeof($pending)) . PHP_EOL; + + $i = 0; + + foreach ($pending as $table => $plan) { + $i++; + + if (!$dry_run && cacti_sizeof(flowview_get_running_partition_alters([$table]))) { + materialize_wait_for_alters([$table]); + + $again = flowview_get_pending_partition_alters([$table], $raw_engine); + + if (!isset($again[$table])) { + print sprintf('Table %s (%d of %d) was altered by another process', $table, $i, cacti_sizeof($pending)) . PHP_EOL; + + continue; + } + + $plan = $again[$table]; + } + + print sprintf('%s Table %s (%d of %d): %s', $dry_run ? 'DRY RUN: Would alter' : 'Altering', $table, $i, cacti_sizeof($pending), trim($plan['alter'])) . PHP_EOL; + + if (!$dry_run) { + $began = microtime(true); + + flowview_apply_partition_alter($table, $plan); + + print sprintf('Altered Table %s in %0.2f seconds', $table, microtime(true) - $began) . PHP_EOL; + } + } + } + + $no_nat = in_array('on', $modes, true) ? flowview_get_partitions_without_nat_columns($cold) : []; + + if (cacti_sizeof($no_nat)) { + print sprintf('NOTE: %d partition(s) have no NAT columns, so they have no NAT data. Run flowview_upgrade_nat_columns.php before materializing if they should.', cacti_sizeof($no_nat)) . PHP_EOL; + } +} + /** * Materializes one query for one NAT mode across the given range. The query * runs through the normal parallel query path, so each partition that is @@ -287,10 +398,13 @@ function materialize_run_worker(int $query_id, string $nat, int $start, int $end): int { $finished = false; - /* the run limit ends the process without returning, so report it as a failure */ + /* the parallel query ends the process on a timeout, after it has said why */ register_shutdown_function(function () use (&$finished) { if (!$finished) { - print 'ERROR: The materialization did not finish. Check the Maximum Run Time under the FlowView Settings.' . PHP_EOL; + if (empty($GLOBALS['flowview_error_reported'])) { + print 'ERROR: The materialization ended before it finished. See the Cacti log for details.' . PHP_EOL; + } + exit(1); } }); diff --git a/flowview_filters.php b/flowview_filters.php index 01d2503..3871398 100644 --- a/flowview_filters.php +++ b/flowview_filters.php @@ -378,7 +378,7 @@ function clearFilter() { ), 'nosort5' => array( 'display' => __('Status', 'flowview'), - 'align' => 'left', + 'align' => 'center', 'tip' => __('The automatic materialization status of the Filter. Required means partitions that have rotated are not materialized yet, Running means a materialization is in progress, and Up to date means every rotated partition is materialized with NAT on and off. The partition still receiving flows is not materialized automatically.', 'flowview') ), 'device' => array( @@ -445,7 +445,7 @@ function clearFilter() { form_alternate_row('line' . $filter['id'], true); form_selectable_cell('' . html_escape($filter['name']) . '', $filter['id']); - form_selectable_cell(flowview_get_colored_materialization_status($status), $filter['id']); + form_selectable_cell(flowview_get_colored_materialization_status($status), $filter['id'], '', 'center'); if (empty($filter['device'])) { form_selectable_cell(__('All', 'flowview'), $filter['id']); diff --git a/flowview_process.php b/flowview_process.php index 501bb18..a3d6f45 100644 --- a/flowview_process.php +++ b/flowview_process.php @@ -185,7 +185,10 @@ if (cacti_sizeof($last_tables)) { foreach($last_tables as $table) { if ($table['ENGINE'] != $raw_engine) { - flowview_db_execute("ALTER TABLE {$table['TABLE_NAME']} ENGINE=$raw_engine"); + /* a conversion from an earlier cycle can still be running, and must not be started again */ + if (!flowview_convert_partition_engine($table['TABLE_NAME'], $raw_engine)) { + flowview_debug("An ALTER TABLE is already running on {$table['TABLE_NAME']}, skipping the engine conversion"); + } } } } diff --git a/flowview_upgrade.php b/flowview_upgrade.php index a0e8224..869bacc 100644 --- a/flowview_upgrade.php +++ b/flowview_upgrade.php @@ -535,64 +535,31 @@ function flowview_upgrade($current, $old): void { $raw_engine = get_set_default_fast_engine(); foreach($raw_tables as $t) { - $alter = ''; - - if ($t['TABLE_COLLATION'] != 'latin1_swedish_ci') { - $alter = 'CONVERT TO CHARACTER SET latin1'; - } - - if (flowview_db_column_exists($t['TABLE_NAME'], 'keycol', false)) { - $alter .= ($alter != '' ? ', ':'') . 'DROP KEY keycol'; - } - - if (!flowview_db_column_exists($t['TABLE_NAME'], 'template_id', false)) { - $alter .= ($alter != '' ? ', ':'') . 'ADD COLUMN template_id int(11) unsigned NOT NULL default "0" AFTER listener_id'; - } - - if (!flowview_db_index_exists($t['TABLE_NAME'], 'template_id')) { - $alter .= ($alter != '' ? ', ':'') . 'ADD INDEX template_id (template_id)'; - } - - if (!flowview_db_index_exists($t['TABLE_NAME'], 'start_time')) { - $alter .= ($alter != '' ? ', ':'') . 'ADD INDEX start_time (start_time)'; - } - - if (!flowview_db_index_exists($t['TABLE_NAME'], 'end_time')) { - $alter .= ($alter != '' ? ', ':'') . 'ADD INDEX end_time (end_time)'; - } - $columns = array_rekey( flowview_db_fetch_assoc('SHOW COLUMNS FROM ' . $t['TABLE_NAME']), 'Field', ['Type', 'Null', 'Key', 'Default', 'Extra'] ); - if (isset($columns['ex_addr']) && stripos($columns['ex_addr']['Type'], 'VARCHAR') === false) { - $alter .= ($alter != '' ? ', ':'') . 'MODIFY COLUMN ex_addr VARCHAR(46) NOT NULL default ""'; - $ex_change = true; - } else { - $ex_change = false; - } + $indexes = []; - if (!flowview_db_index_exists($t['TABLE_NAME'], 'ex_addr')) { - $alter .= ($alter != '' ? ', ':'') . 'ADD INDEX ex_addr (ex_addr)'; - } - - if ($t['ENGINE'] != $raw_engine) { - if ($raw_engine == 'Aria') { - $alter .= ($alter != '' ? ',':'') . " ENGINE=$raw_engine ROW_FORMAT=page"; - } else { - $alter .= ($alter != '' ? ',':'') . " ENGINE=$raw_engine ROW_FORMAT=Dynamic"; + foreach (['template_id', 'start_time', 'end_time', 'ex_addr'] as $index) { + if (flowview_db_index_exists($t['TABLE_NAME'], $index)) { + $indexes[] = $index; } } - if ($alter != '') { - cacti_log("Altering Table {$t['TABLE_NAME']} Using this alter $alter.", true, 'FLOWVIEW'); - - flowview_db_execute('ALTER TABLE ' . $t['TABLE_NAME'] . ' ' . $alter); - - if ($ex_change) { - flowview_db_execute('UPDATE ' . $t['TABLE_NAME'] . ' SET ex_addr = SUBSTRING_INDEX(ex_addr, ":", 1)'); - } + /* the same plan the materializer uses to prepare the partitions it scans */ + $plan = flowview_get_partition_alter([ + 'collation' => $t['TABLE_COLLATION'], + 'engine' => $t['ENGINE'], + 'has_keycol' => flowview_db_column_exists($t['TABLE_NAME'], 'keycol', false), + 'has_template_id' => flowview_db_column_exists($t['TABLE_NAME'], 'template_id', false), + 'indexes' => $indexes, + 'ex_addr_type' => isset($columns['ex_addr']) ? $columns['ex_addr']['Type'] : null, + ], $raw_engine); + + if ($plan['alter'] != '') { + flowview_apply_partition_alter($t['TABLE_NAME'], $plan); } flowview_db_execute('ALTER TABLE plugin_flowview_arin_information diff --git a/includes/functions.php b/includes/functions.php index 4502835..b4ec701 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -4443,6 +4443,211 @@ function flowview_get_rotated_partitions(array $tables, string $previous): array })); } +/** + * flowview_get_partition_alter - What a raw partition table needs altered to + * match the current layout: the character set, the keys and columns the + * queries use, the ex_addr type, and the storage engine. This is the single + * definition used by the schema upgrade and by the materializer, so a + * partition the upgrade has not reached yet is prepared the same way. + * + * Keys of $info: collation and engine of the table, has_keycol and + * has_template_id for those columns, indexes as the names of the indexes + * it has, and ex_addr_type as the type of the ex_addr column, or null. + * + * @param array $info What the partition table currently is + * @param string $raw_engine The engine a partition table that has rotated uses + * + * @return array The alter as a string, which is empty when none is needed, + * and whether ex_addr has to be rewritten afterwards + */ +function flowview_get_partition_alter(array $info, string $raw_engine): array { + $alter = ''; + $ex_change = false; + $indexes = $info['indexes'] ?? []; + + if (($info['collation'] ?? '') != 'latin1_swedish_ci') { + $alter = 'CONVERT TO CHARACTER SET latin1'; + } + + if (!empty($info['has_keycol'])) { + $alter .= ($alter != '' ? ', ':'') . 'DROP KEY keycol'; + } + + if (empty($info['has_template_id'])) { + $alter .= ($alter != '' ? ', ':'') . 'ADD COLUMN template_id int(11) unsigned NOT NULL default "0" AFTER listener_id'; + } + + foreach (['template_id', 'start_time', 'end_time'] as $index) { + if (!in_array($index, $indexes, true)) { + $alter .= ($alter != '' ? ', ':'') . "ADD INDEX $index ($index)"; + } + } + + if (isset($info['ex_addr_type']) && stripos($info['ex_addr_type'], 'VARCHAR') === false) { + $alter .= ($alter != '' ? ', ':'') . 'MODIFY COLUMN ex_addr VARCHAR(46) NOT NULL default ""'; + $ex_change = true; + } + + if (!in_array('ex_addr', $indexes, true)) { + $alter .= ($alter != '' ? ', ':'') . 'ADD INDEX ex_addr (ex_addr)'; + } + + if (($info['engine'] ?? '') != $raw_engine) { + if ($raw_engine == 'Aria') { + $alter .= ($alter != '' ? ',':'') . " ENGINE=$raw_engine ROW_FORMAT=page"; + } else { + $alter .= ($alter != '' ? ',':'') . " ENGINE=$raw_engine ROW_FORMAT=Dynamic"; + } + } + + return ['alter' => $alter, 'ex_change' => $ex_change]; +} + +/** + * flowview_apply_partition_alter - Alter a raw partition table as planned by + * flowview_get_partition_alter(). + * + * @param string $table The raw partition table + * @param array $plan The plan from flowview_get_partition_alter() + * + * @return void + */ +function flowview_apply_partition_alter(string $table, array $plan): void { + cacti_log("Altering Table $table Using this alter {$plan['alter']}.", true, 'FLOWVIEW'); + + flowview_db_execute('ALTER TABLE ' . $table . ' ' . $plan['alter']); + + if ($plan['ex_change']) { + flowview_db_execute('UPDATE ' . $table . ' SET ex_addr = SUBSTRING_INDEX(ex_addr, ":", 1)'); + } +} + +/** + * flowview_get_pending_partition_alters - The partitions that still need + * altering, looked at all at once so that a long list of partitions does not + * mean a long list of queries. + * + * @param array $tables The raw partition tables to look at + * @param string $raw_engine The engine a partition table that has rotated uses + * + * @return array The plan from flowview_get_partition_alter() of each + * partition that needs one, by table + */ +function flowview_get_pending_partition_alters(array $tables, string $raw_engine): array { + if (!cacti_sizeof($tables)) { + return []; + } + + $info = []; + + foreach (flowview_db_fetch_assoc('SELECT TABLE_NAME, TABLE_COLLATION, ENGINE + FROM information_schema.TABLES + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME LIKE "plugin_flowview_raw_%"') as $row) { + $info[$row['TABLE_NAME']] = [ + 'collation' => $row['TABLE_COLLATION'], + 'engine' => $row['ENGINE'], + 'has_keycol' => false, + 'has_template_id' => false, + 'indexes' => [], + 'ex_addr_type' => null, + ]; + } + + foreach (flowview_db_fetch_assoc('SELECT TABLE_NAME, COLUMN_NAME, COLUMN_TYPE + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME LIKE "plugin_flowview_raw_%" + AND COLUMN_NAME IN ("keycol", "template_id", "ex_addr")') as $row) { + if (!isset($info[$row['TABLE_NAME']])) { + continue; + } + + if ($row['COLUMN_NAME'] == 'keycol') { + $info[$row['TABLE_NAME']]['has_keycol'] = true; + } elseif ($row['COLUMN_NAME'] == 'template_id') { + $info[$row['TABLE_NAME']]['has_template_id'] = true; + } else { + $info[$row['TABLE_NAME']]['ex_addr_type'] = $row['COLUMN_TYPE']; + } + } + + foreach (flowview_db_fetch_assoc('SELECT DISTINCT TABLE_NAME, INDEX_NAME + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME LIKE "plugin_flowview_raw_%" + AND INDEX_NAME IN ("template_id", "start_time", "end_time", "ex_addr")') as $row) { + if (isset($info[$row['TABLE_NAME']])) { + $info[$row['TABLE_NAME']]['indexes'][] = $row['INDEX_NAME']; + } + } + + $pending = []; + + foreach ($tables as $table) { + if (!isset($info[$table])) { + continue; + } + + $plan = flowview_get_partition_alter($info[$table], $raw_engine); + + if ($plan['alter'] != '') { + $pending[$table] = $plan; + } + } + + return $pending; +} + +/** + * flowview_get_running_partition_alters - The raw partition tables that an + * ALTER TABLE is running on right now, other than from this connection. A + * query against such a table waits for the alter to finish. + * + * @param array $tables The raw partition tables to look at + * + * @return array How many seconds each alter has been running, by table + */ +function flowview_get_running_partition_alters(array $tables): array { + $running = []; + + $processes = flowview_db_fetch_assoc('SELECT TIME, INFO + FROM information_schema.PROCESSLIST + WHERE ID != CONNECTION_ID() + AND INFO LIKE "ALTER TABLE%plugin_flowview_raw_%"'); + + foreach ($processes as $process) { + if (!preg_match('/^\s*ALTER\s+TABLE\s+(?:`?[A-Za-z0-9_]+`?\.)?`?(plugin_flowview_raw_[0-9]+)`?/i', (string) $process['INFO'], $matches)) { + continue; + } + + if (in_array($matches[1], $tables, true)) { + $running[$matches[1]] = max($running[$matches[1]] ?? 0, (int) $process['TIME']); + } + } + + return $running; +} + +/** + * flowview_get_partitions_without_nat_columns - The raw partition tables that + * pre-date the post-NAT columns. Queries against them still work, but have + * no NAT data to show. + * + * @param array $tables The raw partition tables to look at + * + * @return array + */ +function flowview_get_partitions_without_nat_columns(array $tables): array { + $with = array_column(flowview_db_fetch_assoc('SELECT TABLE_NAME + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME LIKE "plugin_flowview_raw_%" + AND COLUMN_NAME = "post_nat_src_addr"'), 'TABLE_NAME'); + + return array_values(array_diff($tables, $with)); +} + /** * flowview_materialization_running - Whether a materialization is registered * as running. Only one may be active at a time. @@ -4493,12 +4698,34 @@ function flowview_materialization_status(array $cached_nat, array $cached_non_na function flowview_get_colored_materialization_status(string $status): string { switch ($status) { case 'current': - return '' . __('Up to date', 'flowview') . ''; + return "" . __('Up to date', 'flowview') . ''; case 'running': - return '' . __('Running', 'flowview') . ''; + return "" . __('Running', 'flowview') . ''; default: - return '' . __('Required', 'flowview') . ''; + return "" . __('Required', 'flowview') . ''; + } +} + +/** + * flowview_convert_partition_engine - Change a raw partition table that has + * rotated to the storage engine they use. Nothing is done while an ALTER + * TABLE is already running on it. Until one finishes the table still shows + * its old engine, so without that check every poller cycle would start + * another that waits for the first and then rebuilds the table again. + * + * @param string $table The raw partition table + * @param string $raw_engine The engine a partition table that has rotated uses + * + * @return bool false when an alter is already running on it + */ +function flowview_convert_partition_engine(string $table, string $raw_engine): bool { + if (cacti_sizeof(flowview_get_running_partition_alters([$table]))) { + return false; } + + flowview_db_execute("ALTER TABLE $table ENGINE=$raw_engine"); + + return true; } /** @@ -4779,6 +5006,19 @@ function parallel_database_query_request($tables, $stru_inner, $stru_outer) { return $request_id; } +/** + * flowview_parallel_timeout_message - The message for a parallel query that + * did not finish within the Maximum Run Time, worded for the Cacti log and + * for a script. + * + * @param int $seconds The Maximum Run Time in seconds + * + * @return string + */ +function flowview_parallel_timeout_message(int $seconds): string { + return sprintf(__('ERROR: The Parallel Query did not finish within the Maximum Run Time of %d seconds. Check the Maximum Run Time under the FlowView Settings, and that the database is not busy with another operation such as an ALTER TABLE.', 'flowview'), $seconds); +} + /** * parallel_database_query_is_running - Checks to see if an existing parallel * database query is currently running and return a true of false. This prevents @@ -4865,13 +5105,24 @@ function parallel_database_query_run($requests) { } if ($total_time >= $max_time) { - raise_message_javascript('parallel_query_timeout', __('The Parallel Query Timed Out. Please contact your administrator', 'flowview'), MESSAGE_LEVEL_ERROR); + $timeout_message = flowview_parallel_timeout_message((int) $max_time); + + cacti_log($timeout_message, false, 'FLOWVIEW'); + + /* the web page shows the message in the browser, a script has no browser to run it */ + if (defined('CACTI_WEB') && CACTI_WEB) { + raise_message_javascript('parallel_query_timeout', __('The Parallel Query Timed Out. Please contact your administrator', 'flowview'), MESSAGE_LEVEL_ERROR); + } else { + print $timeout_message . PHP_EOL; + + $GLOBALS['flowview_error_reported'] = true; + } foreach($requests as $request_id) { parallel_database_query_cleanup($request_id); } - exit; + exit(defined('CACTI_WEB') && CACTI_WEB ? 0 : 1); } if (cacti_sizeof($requests) == 1) { diff --git a/tests/Unit/CoreFunctionsTest.php b/tests/Unit/CoreFunctionsTest.php index e092033..dbbd8e7 100644 --- a/tests/Unit/CoreFunctionsTest.php +++ b/tests/Unit/CoreFunctionsTest.php @@ -591,8 +591,8 @@ function array_rekey($array, $key, $value) { expect(flowview_materialization_status(array(), array(), array(), false))->toBe('current'); }); -it('shows the status pills in the colors of the device status', function ($status, $class, $label) { - expect(flowview_get_colored_materialization_status($status))->toBe('' . $label . ''); +it('shows the status pills in the form of the device status', function ($status, $class, $label) { + expect(flowview_get_colored_materialization_status($status))->toBe("" . $label . ''); })->with(array( 'current' => array('current', 'deviceUp', 'Up to date'), 'running' => array('running', 'deviceRecovering', 'Running'), @@ -600,6 +600,16 @@ function array_rekey($array, $key, $value) { 'unknown' => array('anything', 'deviceDown', 'Required'), )); +it('words the parallel query timeout for the log and a script, not the browser', function () { + $message = flowview_parallel_timeout_message(300); + + expect($message)->toStartWith('ERROR: ') + ->and($message)->toContain('300 seconds') + ->and($message)->toContain('Maximum Run Time') + ->and($message)->not->toContain('and($message)->not->toContain('DOMPurify'); +}); + it('starts a materialization for only the partitions it is given', function () { if (!function_exists('exec_background')) { function exec_background($filename, $args = '', $redirect_args = '') { @@ -739,3 +749,201 @@ function cacti_escapeshellarg($string) { ->and($GLOBALS['__test_config'])->not->toHaveKey('flowview_last_hot_partition'); }); }); + +describe('preparing the partitions before they are scanned', function () { + $current = array( + 'collation' => 'latin1_swedish_ci', + 'engine' => 'Aria', + 'has_keycol' => false, + 'has_template_id' => true, + 'indexes' => array('template_id', 'start_time', 'end_time', 'ex_addr'), + 'ex_addr_type' => 'varchar(46)', + ); + + it('needs no alter for a partition that is already current', function () use ($current) { + expect(flowview_get_partition_alter($current, 'Aria'))->toBe(array('alter' => '', 'ex_change' => false)); + }); + + it('alters only the engine of a partition that has rotated but kept the live engine', function ($raw_engine, $expected) use ($current) { + $info = $current; + $info['engine'] = 'InnoDB'; + + expect(flowview_get_partition_alter($info, $raw_engine))->toBe(array('alter' => $expected, 'ex_change' => false)); + })->with(array( + 'aria' => array('Aria', ' ENGINE=Aria ROW_FORMAT=page'), + 'myisam' => array('MyISAM', ' ENGINE=MyISAM ROW_FORMAT=Dynamic'), + )); + + it('plans every alter a partition from an old version needs, as the schema upgrade always did', function () { + $info = array( + 'collation' => 'utf8mb4_general_ci', + 'engine' => 'InnoDB', + 'has_keycol' => true, + 'has_template_id' => false, + 'indexes' => array(), + 'ex_addr_type' => 'varbinary(16)', + ); + + expect(flowview_get_partition_alter($info, 'Aria'))->toBe(array( + 'alter' => 'CONVERT TO CHARACTER SET latin1, DROP KEY keycol, ADD COLUMN template_id int(11) unsigned NOT NULL default "0" AFTER listener_id, ' . + 'ADD INDEX template_id (template_id), ADD INDEX start_time (start_time), ADD INDEX end_time (end_time), ' . + 'MODIFY COLUMN ex_addr VARCHAR(46) NOT NULL default "", ADD INDEX ex_addr (ex_addr), ENGINE=Aria ROW_FORMAT=page', + 'ex_change' => true, + )); + }); + + it('leaves ex_addr alone when it is a varchar in any case or is missing', function ($type) use ($current) { + $info = $current; + $info['ex_addr_type'] = $type; + + expect(flowview_get_partition_alter($info, 'Aria'))->toBe(array('alter' => '', 'ex_change' => false)); + })->with(array('VARCHAR(46)', 'varchar(20)', null)); + + it('adds only the index a partition is missing', function () use ($current) { + $info = $current; + $info['indexes'] = array('template_id', 'start_time', 'ex_addr'); + + expect(flowview_get_partition_alter($info, 'Aria')['alter'])->toBe('ADD INDEX end_time (end_time)'); + }); + + it('alters a partition and rewrites ex_addr afterwards when it was converted', function () { + flowview_apply_partition_alter('plugin_flowview_raw_2024001', array('alter' => 'MODIFY COLUMN ex_addr VARCHAR(46) NOT NULL default ""', 'ex_change' => true)); + + $sql = array_column(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute'; + }), 'sql'); + + expect(array_values($sql))->toBe(array( + 'ALTER TABLE plugin_flowview_raw_2024001 MODIFY COLUMN ex_addr VARCHAR(46) NOT NULL default ""', + 'UPDATE plugin_flowview_raw_2024001 SET ex_addr = SUBSTRING_INDEX(ex_addr, ":", 1)', + )); + }); + + it('does not rewrite ex_addr when it was not converted', function () { + flowview_apply_partition_alter('plugin_flowview_raw_2024001', array('alter' => 'ADD INDEX end_time (end_time)', 'ex_change' => false)); + + $sql = array_column(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute'; + }), 'sql'); + + expect(array_values($sql))->toBe(array('ALTER TABLE plugin_flowview_raw_2024001 ADD INDEX end_time (end_time)')); + }); + + it('finds the partitions that still need an alter in a single pass', function () { + $table = function ($name, $engine) { + return array('TABLE_NAME' => $name, 'TABLE_COLLATION' => 'latin1_swedish_ci', 'ENGINE' => $engine); + }; + + $columns = function ($name) { + return array( + array('TABLE_NAME' => $name, 'COLUMN_NAME' => 'template_id', 'COLUMN_TYPE' => 'int(11) unsigned'), + array('TABLE_NAME' => $name, 'COLUMN_NAME' => 'ex_addr', 'COLUMN_TYPE' => 'varchar(46)'), + ); + }; + + $indexes = function ($name, $names) { + return array_map(function ($index) use ($name) { + return array('TABLE_NAME' => $name, 'INDEX_NAME' => $index); + }, $names); + }; + + $all = array('template_id', 'start_time', 'end_time', 'ex_addr'); + + plugin_test_queue_db_result('db_fetch_assoc', array( + $table('plugin_flowview_raw_2024001', 'Aria'), + $table('plugin_flowview_raw_2024002', 'InnoDB'), + $table('plugin_flowview_raw_2024003', 'Aria'), + )); + plugin_test_queue_db_result('db_fetch_assoc', array_merge( + $columns('plugin_flowview_raw_2024001'), + $columns('plugin_flowview_raw_2024002'), + $columns('plugin_flowview_raw_2024003') + )); + plugin_test_queue_db_result('db_fetch_assoc', array_merge( + $indexes('plugin_flowview_raw_2024001', $all), + $indexes('plugin_flowview_raw_2024002', $all), + $indexes('plugin_flowview_raw_2024003', array('template_id', 'start_time', 'ex_addr')) + )); + + $pending = flowview_get_pending_partition_alters( + array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002', 'plugin_flowview_raw_2024003', 'plugin_flowview_raw_2024009'), + 'Aria' + ); + + expect(array_keys($pending))->toBe(array('plugin_flowview_raw_2024002', 'plugin_flowview_raw_2024003')) + ->and($pending['plugin_flowview_raw_2024002']['alter'])->toBe(' ENGINE=Aria ROW_FORMAT=page') + ->and($pending['plugin_flowview_raw_2024003']['alter'])->toBe('ADD INDEX end_time (end_time)') + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_fetch_assoc'; + }))->toHaveCount(3); + }); + + it('looks at nothing when there are no partitions', function () { + expect(flowview_get_pending_partition_alters(array(), 'Aria'))->toBe(array()) + ->and($GLOBALS['__test_db_calls'])->toBe(array()); + }); + + it('finds the alters running on the partitions it needs', function () { + plugin_test_queue_db_result('db_fetch_assoc', array( + array('TIME' => 120, 'INFO' => 'ALTER TABLE plugin_flowview_raw_2024001 ADD INDEX end_time (end_time)'), + array('TIME' => 5, 'INFO' => 'alter table `plugin_flowview_raw_2024002` ENGINE=Aria'), + array('TIME' => 300, 'INFO' => 'ALTER TABLE `flowview`.`plugin_flowview_raw_2024001` ENGINE=Aria'), + array('TIME' => 9, 'INFO' => 'ALTER TABLE plugin_flowview_raw_2024099 ENGINE=Aria'), + array('TIME' => 3, 'INFO' => 'ALTER TABLE plugin_flowview_arin_information MODIFY COLUMN comments VARCHAR(512)'), + array('TIME' => 1, 'INFO' => null), + )); + + expect(flowview_get_running_partition_alters(array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002')))->toBe(array( + 'plugin_flowview_raw_2024001' => 300, + 'plugin_flowview_raw_2024002' => 5, + )); + }); + + it('finds no running alters when there are none', function () { + expect(flowview_get_running_partition_alters(array('plugin_flowview_raw_2024001')))->toBe(array()); + }); + + it('converts the engine of a partition when no alter is running on it', function () { + plugin_test_queue_db_result('db_fetch_assoc', array()); + + expect(flowview_convert_partition_engine('plugin_flowview_raw_202629214', 'Aria'))->toBeTrue(); + + $sql = array_column(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute'; + }), 'sql'); + + expect(array_values($sql))->toBe(array('ALTER TABLE plugin_flowview_raw_202629214 ENGINE=Aria')); + }); + + it('does not start the same engine conversion again while one is running', function () { + plugin_test_queue_db_result('db_fetch_assoc', array( + array('TIME' => 1605, 'INFO' => 'ALTER TABLE plugin_flowview_raw_202629214 ENGINE=Aria'), + )); + + expect(flowview_convert_partition_engine('plugin_flowview_raw_202629214', 'Aria'))->toBeFalse(); + + $writes = array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['fn'], 'db_execute') === 0; + }); + + expect($writes)->toBeEmpty(); + }); + + it('still converts a partition when the alter running is on another one', function () { + plugin_test_queue_db_result('db_fetch_assoc', array( + array('TIME' => 12, 'INFO' => 'ALTER TABLE plugin_flowview_raw_202629213 ENGINE=Aria'), + )); + + expect(flowview_convert_partition_engine('plugin_flowview_raw_202629214', 'Aria'))->toBeTrue(); + }); + + it('finds the partitions that have no NAT columns', function () { + plugin_test_queue_db_result('db_fetch_assoc', array( + array('TABLE_NAME' => 'plugin_flowview_raw_2024001'), + array('TABLE_NAME' => 'plugin_flowview_raw_2024003'), + )); + + expect(flowview_get_partitions_without_nat_columns(array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002', 'plugin_flowview_raw_2024003'))) + ->toBe(array('plugin_flowview_raw_2024002')); + }); +}); From ec096b65e999787c64b485dde95362f64a03f60e Mon Sep 17 00:00:00 2001 From: TheWitness Date: Tue, 20 Oct 2026 15:48:10 -0400 Subject: [PATCH 10/37] fix: Show running materializations and the number of running queries per filter The Running status only looked at the materializer's own registration, so it disappeared when the script ended, even though the parallel query it started kept working. Look for the runner and shard worker processes of requests made by a script too, ignoring rows left behind by processes that are gone, and do not start a new materialization while one is running. Make the parallel query cleanup and the runner's termination handler use the names the processes register, db_query_ and db_shard_, so a timeout actually stops them and a terminated one removes its row. Show the number of running queries of each filter in a pill beside the status pill, matched to the filter by its cache key. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 6 ++ flowview_filters.php | 8 +- flowview_runner.php | 4 +- includes/functions.php | 154 ++++++++++++++++++++++++--- tests/Unit/CoreFunctionsTest.php | 177 +++++++++++++++++++++++++++++++ 5 files changed, 328 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f93afa..b3b33e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ --- develop --- +* feature: Show the number of queries of each Filter that are running now in a second pill, in the style of a disabled device and reading Running: N, beside the status pill on the Filters page. A running query is matched to a Filter by the cache key its map query is stored under, which is worked out from the Filter, so the number is for that Filter and for NAT and non-NAT together, and there is no pill when none are running + +* bug: Show the Running status on the Filters page while the parallel query started by cli/materialize_queries.php is still working, not only while the script itself is registered. The query keeps going after the script has ended, for instance when it timed out, and the status now looks for the runner and shard worker processes of the requests a script made, ignoring rows left behind by processes that are gone. A new materialization is not started while one is running + +* bug: Make the Parallel Query timeout actually stop the runner and shard workers. They register as db_query_ and db_shard_, but the cleanup, and the runner's own termination handler, looked for db_query and db_shard, so they were never found: a timed out query left its processes running and a terminated one left its row in the process table + * bug: Stop the poller from starting the same ALTER TABLE ENGINE conversion on a partition every cycle. Until a long conversion finishes the partition still reports its old engine, so each cycle started another that waited for the first and then rebuilt the table again, which also held up every query scanning that partition. The conversion is skipped while an ALTER TABLE is already running on the partition, and cli/materialize_queries.php checks a partition again just before altering it, so it is never rebuilt twice * bug: Make a Parallel Query that reaches the Maximum Run Time report it in words when run from a script, such as cli/materialize_queries.php, instead of printing the JavaScript meant for the web page; the message is also written to the Cacti log, a script now exits with a failure, and the web page is unchanged diff --git a/flowview_filters.php b/flowview_filters.php index 3871398..330a356 100644 --- a/flowview_filters.php +++ b/flowview_filters.php @@ -438,14 +438,16 @@ function clearFilter() { $cold = flowview_get_cold_partitions($tables); $partitions = cacti_sizeof($tables); $running = flowview_materialization_running(); + $requests = flowview_get_running_materialization_requests(); foreach ($filters as $filter) { - $cached = flowview_get_cached_shard_tables((int) $filter['id']); - $status = flowview_materialization_status($cached['nat'], $cached['non_nat'], $cold, $running); + $md5sums = flowview_get_filter_md5sums((int) $filter['id']); + $cached = flowview_get_cached_shard_tables((int) $filter['id'], $md5sums); + $status = flowview_materialization_status($cached['nat'], $cached['non_nat'], $cold, $running); form_alternate_row('line' . $filter['id'], true); form_selectable_cell('' . html_escape($filter['name']) . '', $filter['id']); - form_selectable_cell(flowview_get_colored_materialization_status($status), $filter['id'], '', 'center'); + form_selectable_cell(trim(flowview_get_colored_materialization_status($status) . ' ' . flowview_get_colored_running_queries(flowview_count_running_queries($md5sums, $requests))), $filter['id'], '', 'center'); if (empty($filter['device'])) { form_selectable_cell(__('All', 'flowview'), $filter['id']); diff --git a/flowview_runner.php b/flowview_runner.php index 0867050..c2970bc 100644 --- a/flowview_runner.php +++ b/flowview_runner.php @@ -205,9 +205,9 @@ function sig_handler($signo): void { cacti_log('WARNING: Flowview Database Parallel Poller terminated by user', false, 'BOOST'); if ($shard_id) { - unregister_process('flowview', "db_shard{$query_id}", $shard_id, (int) getmypid()); + unregister_process('flowview', "db_shard_{$query_id}", $shard_id, (int) getmypid()); } else { - unregister_process('flowview', "db_query{$query_id}", 0, (int) getmypid()); + unregister_process('flowview', "db_query_{$query_id}", 0, (int) getmypid()); } exit; diff --git a/includes/functions.php b/includes/functions.php index b4ec701..d33b7a1 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -4649,18 +4649,77 @@ function flowview_get_partitions_without_nat_columns(array $tables): array { } /** - * flowview_materialization_running - Whether a materialization is registered - * as running. Only one may be active at a time. + * flowview_get_running_materialization_requests - The parallel query requests + * made by a script, as the materializer makes them, that still have a runner + * or shard worker process alive. They keep going after the script that + * started them has ended, for instance when it timed out. A request made + * from a web page belongs to its user, so it is not one of these. + * + * @return array The cache key (md5sum_tables) of each request, by request id + */ +function flowview_get_running_materialization_requests(): array { + $processes = db_fetch_assoc_prepared('SELECT pid, taskname + FROM processes + WHERE tasktype = ? + AND (taskname LIKE ? OR taskname LIKE ?)', + ['flowview', 'db\_query\_%', 'db\_shard\_%']); + + $pids = []; + + foreach ($processes as $process) { + if (preg_match('/^db_(?:query|shard)_([0-9]+)$/', $process['taskname'], $matches)) { + $pids[(int) $matches[1]][] = (int) $process['pid']; + } + } + + if (!cacti_sizeof($pids)) { + return []; + } + + $ids = array_keys($pids); + + $requests = flowview_db_fetch_assoc_prepared('SELECT id, md5sum_tables + FROM parallel_database_query + WHERE user_id = 0 + AND id IN (' . implode(', ', array_fill(0, cacti_sizeof($ids), '?')) . ')', + $ids); + + $running = []; + + foreach ($requests as $request) { + foreach ($pids[(int) $request['id']] as $pid) { + /* a process that was killed can leave its row behind */ + if (!function_exists('cacti_process_still_running') || cacti_process_still_running($pid)) { + $running[(int) $request['id']] = (string) $request['md5sum_tables']; + + break; + } + } + } + + return $running; +} + +/** + * flowview_materialization_running - Whether a materialization is running. + * That is when one is registered as running, since only one may be active at + * a time, or when the parallel query it started still has processes alive. * * @return bool */ function flowview_materialization_running(): bool { - return db_fetch_cell_prepared('SELECT COUNT(*) + $registered = db_fetch_cell_prepared('SELECT COUNT(*) FROM processes WHERE tasktype = ? AND taskname = ? AND taskid = ?', - ['flowview', 'materialize', 0]) > 0; + ['flowview', 'materialize', 0]); + + if ($registered > 0) { + return true; + } + + return cacti_sizeof(flowview_get_running_materialization_requests()) > 0; } /** @@ -4706,6 +4765,23 @@ function flowview_get_colored_materialization_status(string $status): string { } } +/** + * flowview_get_colored_running_queries - The pill for the number of queries + * of a filter that are running now, in the gray of a disabled device. There + * is none when nothing is running. + * + * @param int $count The number of queries running + * + * @return string + */ +function flowview_get_colored_running_queries(int $count): string { + if ($count < 1) { + return ''; + } + + return "" . sprintf(__('Running: %s', 'flowview'), number_format_i18n($count, 0)) . ''; +} + /** * flowview_convert_partition_engine - Change a raw partition table that has * rotated to the storage engine they use. Nothing is done while an ALTER @@ -4778,35 +4854,80 @@ function flowview_materialize_rotated_partitions(): array { return $rotated; } +/** + * flowview_get_filter_md5sums - The cache keys (md5sum_tables) the parallel + * query stores the partitions of a filter under, with NAT and without. Some + * reports use two map queries and so have two. Nothing is run, they are + * worked out from the filter. + * + * @param int $query_id The FlowView query to work the cache keys out for + * + * @return array The cache keys as nat and non_nat + */ +function flowview_get_filter_md5sums(int $query_id): array { + $md5sums = ['nat' => [], 'non_nat' => []]; + $now = time(); + + foreach (['nat' => true, 'non_nat' => false] as $mode => $use_nat) { + $planned = run_flow_query(false, $query_id, $now, $now, true, $use_nat); + + if (is_array($planned) && cacti_sizeof($planned)) { + $md5sums[$mode] = array_values(array_unique($planned)); + } + } + + return $md5sums; +} + +/** + * flowview_count_running_queries - How many of the running requests are + * queries of a filter, with NAT or without. + * + * @param array $md5sums The cache keys of the filter from flowview_get_filter_md5sums() + * @param array $running The running requests from flowview_get_running_materialization_requests() + * + * @return int + */ +function flowview_count_running_queries(array $md5sums, array $running): int { + $keys = array_merge($md5sums['nat'] ?? [], $md5sums['non_nat'] ?? []); + + return cacti_sizeof(array_filter($running, function ($md5sum) use ($keys) { + return in_array($md5sum, $keys, true); + })); +} + /** * flowview_get_cached_shard_tables - The partitions of a filter that are * stored in the parallel query cache, with NAT and without. Some reports * use two map queries, and a partition only counts once every one of them is * cached. Nothing is run, the cache keys are worked out from the filter. * - * @param int $query_id The FlowView query to find the cached shards of + * @param int $query_id The FlowView query to find the cached shards of + * @param array|null $md5sums The cache keys of the filter when they are + * already known, otherwise they are worked out * * @return array The partition tables cached as nat and non_nat */ -function flowview_get_cached_shard_tables(int $query_id): array { +function flowview_get_cached_shard_tables(int $query_id, ?array $md5sums = null): array { $cached = ['nat' => [], 'non_nat' => []]; - $now = time(); - foreach (['nat' => true, 'non_nat' => false] as $mode => $use_nat) { - $md5sums = run_flow_query(false, $query_id, $now, $now, true, $use_nat); + if ($md5sums === null) { + $md5sums = flowview_get_filter_md5sums($query_id); + } - if (!is_array($md5sums) || !cacti_sizeof($md5sums)) { + foreach (['nat', 'non_nat'] as $mode) { + $keys = $md5sums[$mode] ?? []; + + if (!cacti_sizeof($keys)) { continue; } - $md5sums = array_values(array_unique($md5sums)); - $tables = flowview_db_fetch_assoc_prepared('SELECT map_table FROM parallel_database_query_shard_cache - WHERE md5sum IN (' . implode(', ', array_fill(0, cacti_sizeof($md5sums), '?')) . ') + WHERE md5sum IN (' . implode(', ', array_fill(0, cacti_sizeof($keys), '?')) . ') GROUP BY map_table, map_partition HAVING COUNT(DISTINCT md5sum) = ?', - array_merge($md5sums, [cacti_sizeof($md5sums)])); + array_merge($keys, [cacti_sizeof($keys)])); $cached[$mode] = array_column($tables, 'map_table'); } @@ -5365,10 +5486,11 @@ function parallel_database_query_cleanup($request_id, bool $remove = false): voi WHERE query_id = ?', [$request_id]); + /* the runner and its shard workers register as db_query_ and db_shard_ */ $processes = db_fetch_assoc_prepared('SELECT * FROM processes WHERE tasktype = ? - AND taskname LIKE ?', - ['flowview', "db_query{$request_id}"]); + AND taskname IN (?, ?)', + ['flowview', "db_query_{$request_id}", "db_shard_{$request_id}"]); if (cacti_sizeof($processes)) { foreach($processes as $p) { diff --git a/tests/Unit/CoreFunctionsTest.php b/tests/Unit/CoreFunctionsTest.php index dbbd8e7..2481d6c 100644 --- a/tests/Unit/CoreFunctionsTest.php +++ b/tests/Unit/CoreFunctionsTest.php @@ -947,3 +947,180 @@ function cacti_escapeshellarg($string) { ->toBe(array('plugin_flowview_raw_2024002')); }); }); + +describe('finding a materialization that is still running', function () { + beforeEach(function () { + if (!function_exists('cacti_process_still_running')) { + function cacti_process_still_running($pid) { + return in_array($pid, $GLOBALS['__test_live_pids'], true); + } + } + + $GLOBALS['__test_live_pids'] = array(); + }); + + it('finds the runner and shard workers of a request by the names they register', function () { + if (!function_exists('posix_kill')) { + $this->markTestSkipped('The posix extension is not available.'); + } + + // A pid that cannot be a live process, so nothing is signalled. + plugin_test_queue_db_result('db_fetch_assoc_prepared', array()); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array( + array('pid' => 4999999, 'taskname' => 'db_query_991'), + array('pid' => 4999998, 'taskname' => 'db_shard_991'), + )); + + parallel_database_query_cleanup(991); + + $lookup = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_fetch_assoc_prepared' && strpos($call['sql'], 'FROM processes') !== false; + })); + + $removed = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute_prepared' && strpos($call['sql'], 'DELETE FROM processes') === 0; + })); + + expect($lookup)->toHaveCount(1) + ->and($lookup[0]['params'])->toBe(array('flowview', 'db_query_991', 'db_shard_991')) + ->and(array_column($removed, 'params'))->toBe(array(array(4999999), array(4999998))); + }); + + it('finds a request made by a script that still has a live process', function () { + $GLOBALS['__test_live_pids'] = array(222); + + plugin_test_queue_db_result('db_fetch_assoc_prepared', array( + array('pid' => 111, 'taskname' => 'db_query_991'), + array('pid' => 222, 'taskname' => 'db_shard_991'), + array('pid' => 333, 'taskname' => 'db_shard_992'), + array('pid' => 444, 'taskname' => 'db_shard_abc'), + )); + // 992 belongs to a web user, so it is not one of these. + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('id' => 991, 'md5sum_tables' => 'aaaa'))); + + expect(flowview_get_running_materialization_requests())->toBe(array(991 => 'aaaa')); + }); + + it('ignores the rows left behind by processes that are gone', function () { + plugin_test_queue_db_result('db_fetch_assoc_prepared', array( + array('pid' => 111, 'taskname' => 'db_query_991'), + array('pid' => 222, 'taskname' => 'db_shard_991'), + )); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('id' => 991, 'md5sum_tables' => 'aaaa'))); + + expect(flowview_get_running_materialization_requests())->toBe(array()); + }); + + it('finds nothing without any runner processes', function () { + expect(flowview_get_running_materialization_requests())->toBe(array()) + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['sql'], 'parallel_database_query') !== false; + }))->toBeEmpty(); + }); + + it('is running while a materialization is registered', function () { + plugin_test_queue_db_result('db_fetch_cell_prepared', 1); + + expect(flowview_materialization_running())->toBeTrue() + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['sql'], 'db\_query\_') !== false || ($call['params'][1] ?? '') === 'db\_query\_%'; + }))->toBeEmpty(); + }); + + it('is still running after the script that started it has ended', function () { + $GLOBALS['__test_live_pids'] = array(222); + + plugin_test_queue_db_result('db_fetch_cell_prepared', 0); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('pid' => 222, 'taskname' => 'db_shard_991'))); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('id' => 991, 'md5sum_tables' => 'aaaa'))); + + expect(flowview_materialization_running())->toBeTrue(); + }); + + it('is not running when nothing is registered and no runner is alive', function () { + plugin_test_queue_db_result('db_fetch_cell_prepared', 0); + + expect(flowview_materialization_running())->toBeFalse(); + }); +}); + +describe('showing the queries of a filter that are running', function () { + $filter = array( + 'id' => 4, 'name' => 'Top Talkers', 'device_id' => 0, 'ex_addr' => '', 'template_id' => -1, + 'statistics' => 10, 'printed' => 0, 'sortfield' => 'bytes', 'cutofflines' => 20, 'cutoffoctets' => 0, + 'includeif' => 0, 'usenat' => '', 'protocols' => '', 'tosfields' => '', 'tcpflags' => '', + ); + + beforeEach(function () { + if (!function_exists('array_rekey')) { + function array_rekey($array, $key, $value) { + return array(); + } + } + + if (!function_exists('number_format_i18n')) { + function number_format_i18n($number, $decimals = 0) { + return number_format($number, $decimals); + } + } + }); + + it('works out the cache keys of a filter for each NAT mode', function () use ($filter) { + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + + $md5sums = flowview_get_filter_md5sums(4); + + expect($md5sums['nat'])->toHaveCount(1) + ->and($md5sums['non_nat'])->toHaveCount(1) + ->and($md5sums['nat'][0])->toMatch('/^[0-9a-f]{32}$/') + ->and($md5sums['nat'][0])->not->toBe($md5sums['non_nat'][0]); + }); + + it('has no cache keys for a filter that cannot be planned', function () { + plugin_test_queue_db_result('db_fetch_row_prepared', array()); + plugin_test_queue_db_result('db_fetch_row_prepared', array()); + + expect(flowview_get_filter_md5sums(99))->toBe(array('nat' => array(), 'non_nat' => array())); + }); + + it('uses the cache keys it is given instead of planning the filter again', function () { + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('map_table' => 'plugin_flowview_raw_2024001'))); + + $cached = flowview_get_cached_shard_tables(4, array('nat' => array('aaaa'), 'non_nat' => array())); + + expect($cached)->toBe(array('nat' => array('plugin_flowview_raw_2024001'), 'non_nat' => array())) + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_fetch_row_prepared'; + }))->toBeEmpty(); + }); + + it('counts the running requests that are queries of a filter', function ($running, $expected) { + $md5sums = array('nat' => array('aaaa'), 'non_nat' => array('bbbb', 'cccc')); + + expect(flowview_count_running_queries($md5sums, $running))->toBe($expected); + })->with(array( + 'none running' => array(array(), 0), + 'one with nat' => array(array(991 => 'aaaa'), 1), + 'both modes' => array(array(991 => 'aaaa', 992 => 'bbbb'), 2), + 'two of the report' => array(array(991 => 'bbbb', 992 => 'cccc'), 2), + 'someone elses' => array(array(991 => 'zzzz'), 0), + 'mixed' => array(array(991 => 'aaaa', 992 => 'zzzz', 993 => 'cccc'), 2), + )); + + it('counts nothing for a filter that has no cache keys', function () { + expect(flowview_count_running_queries(array('nat' => array(), 'non_nat' => array()), array(991 => 'aaaa')))->toBe(0); + }); + + it('shows the number of running queries in a pill like a disabled device', function () { + $pill = flowview_get_colored_running_queries(1234); + + expect($pill)->toStartWith("and($pill)->toContain('Running: 1,234') + ->and($pill)->toContain('title='); + }); + + it('shows no pill when nothing is running', function ($count) { + expect(flowview_get_colored_running_queries($count))->toBe(''); + })->with(array(0, -1)); +}); From 7b89d91094f52bd63d566801797936981fd42e2a Mon Sep 17 00:00:00 2001 From: TheWitness Date: Tue, 20 Oct 2026 15:59:58 -0400 Subject: [PATCH 11/37] fix: Find running queries from the process table and say which filter they are for Take the running parallel queries from the db_query_ and db_shard_ registrations in the process table, whoever started them, counting a registration while it is recent or its process is alive. This no longer depends on the request belonging to user 0. Match each running request to its filter by cache key and show the number of running queries per filter, with the NAT and non-NAT split in the tip. Add --status to cli/materialize_queries.php to list the running requests, the filter each is for and their progress. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 6 +- README.md | 3 +- cli/materialize_queries.php | 64 +++++++++++++++- flowview_filters.php | 5 +- includes/functions.php | 122 ++++++++++++++++++++++--------- tests/Unit/CoreFunctionsTest.php | 98 ++++++++++++++++++++----- 6 files changed, 240 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3b33e4..c7ae6c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,11 @@ --- develop --- -* feature: Show the number of queries of each Filter that are running now in a second pill, in the style of a disabled device and reading Running: N, beside the status pill on the Filters page. A running query is matched to a Filter by the cache key its map query is stored under, which is worked out from the Filter, so the number is for that Filter and for NAT and non-NAT together, and there is no pill when none are running +* feature: Show the number of queries of each Filter that are running now in a second pill, in the style of a disabled device and reading Running: N, beside the status pill on the Filters page, with how many are with NAT and without in its tip. The process table only has the id of a running request, so a running query is matched to a Filter by the cache key its map query is stored under, which is worked out from the Filter, and there is no pill when none are running -* bug: Show the Running status on the Filters page while the parallel query started by cli/materialize_queries.php is still working, not only while the script itself is registered. The query keeps going after the script has ended, for instance when it timed out, and the status now looks for the runner and shard worker processes of the requests a script made, ignoring rows left behind by processes that are gone. A new materialization is not started while one is running +* feature: Add --status to cli/materialize_queries.php to list the parallel queries that are running and the Filter each one is for, with its progress + +* bug: Show the Running status on the Filters page while a parallel query is still working, not only while cli/materialize_queries.php itself is registered. The query keeps going after the script has ended, for instance when it timed out, so the status now comes from the runner and shard worker (db_query_ and db_shard_) registrations in the process table, whoever started them. A registration counts while it is recent, or when its process is still alive, so a row left behind by a killed process does not. A new materialization is not started while one is running * bug: Make the Parallel Query timeout actually stop the runner and shard workers. They register as db_query_ and db_shard_, but the cleanup, and the runner's own termination handler, looked for db_query and db_shard, so they were never found: a timed out query left its processes running and a terminated one left its row in the process table diff --git a/README.md b/README.md index 12b40d2..c81a08b 100644 --- a/README.md +++ b/README.md @@ -217,7 +217,8 @@ once with NAT on and once with NAT off. Use `--query-id=ID[,ID...]` or Partitions that are already cached and still match their data are skipped, while partitions that have received more flows since, or have been dropped, are rebuilt. Only one run may be active at a time, for up to 8 hours, unless -`--force` is used. +`--force` is used. `--status` lists the parallel queries that are running now +and the Filter each one is for, with its progress. Before it scans anything the script gets the partitions ready. A query waits for an ALTER TABLE that is running on the partition it scans, and a long one diff --git a/cli/materialize_queries.php b/cli/materialize_queries.php index dc22c20..3d1d029 100644 --- a/cli/materialize_queries.php +++ b/cli/materialize_queries.php @@ -44,6 +44,7 @@ $debug = false; $force = false; $dry_run = false; +$status = false; $worker = false; $query_ids = []; $partitions = []; @@ -61,6 +62,7 @@ 'debug', 'force', 'dry-run', + 'status', 'worker', 'version', 'help', @@ -130,6 +132,10 @@ case 'dry-run': $dry_run = true; + break; + case 'status': + $status = true; + break; case 'worker': $worker = true; @@ -152,6 +158,12 @@ } } +if ($status) { + materialize_show_status(); + + exit(0); +} + /* a worker materializes exactly one query and NAT mode for the range its parent chose */ if ($worker) { if (cacti_sizeof($query_ids) != 1 || $nat === false || $start === false || $end === false) { @@ -288,6 +300,55 @@ exit($failed > 0 ? 1 : 0); +/** + * Shows the parallel queries that are running and the Filter each one is for. + * The process table only has the id of the request, so the Filter is found + * from the cache key of the request. + * + * @return void + */ +function materialize_show_status(): void { + $running = flowview_get_running_requests(); + + if (!cacti_sizeof($running)) { + print 'No parallel queries are running.' . PHP_EOL; + + return; + } + + $names = []; + $md5sums = []; + + foreach (flowview_db_fetch_assoc('SELECT id, name FROM plugin_flowview_queries ORDER BY id') as $filter) { + $names[(int) $filter['id']] = $filter['name']; + $md5sums[(int) $filter['id']] = flowview_get_filter_md5sums((int) $filter['id']); + } + + $requests = array_column(flowview_db_fetch_assoc_prepared('SELECT id, status, total_shards, finished_shards, cached_shards + FROM parallel_database_query + WHERE id IN (' . implode(', ', array_fill(0, cacti_sizeof($running), '?')) . ')', + array_keys($running)), null, 'id'); + + print sprintf('%d parallel queries are running:', cacti_sizeof($running)) . PHP_EOL; + + foreach ($running as $id => $md5sum) { + $for = []; + + foreach (flowview_find_request_filters($md5sums, $md5sum) as $match) { + $mode = $match['nat'] && $match['non_nat'] ? 'on/off' : ($match['nat'] ? 'on' : 'off'); + $for[] = sprintf('Query:%d [%s] NAT:%s', $match['id'], $names[$match['id']], $mode); + } + + if (isset($requests[$id])) { + $progress = sprintf('Shards:%d of %d, Already Materialized:%d, State:%s', $requests[$id]['finished_shards'], $requests[$id]['total_shards'], $requests[$id]['cached_shards'], $requests[$id]['status']); + } else { + $progress = 'The request no longer exists, its processes are still registered'; + } + + print sprintf('Request:%d %s %s', $id, cacti_sizeof($for) ? implode(', ', $for) : 'No Filter matches it, it may be an ad hoc report or a Filter that has changed', $progress) . PHP_EOL; + } +} + /** * Waits until no ALTER TABLE is running on any of the partitions. A query * waits for an alter on the partition it scans, so the parallel query must @@ -470,7 +531,7 @@ function display_version(): void { function display_help(): void { display_version(); - print PHP_EOL . 'usage: materialize_queries.php [--query-id=ID[,ID...]] [--partitions=TABLE[,TABLE...]] [--nat=on|off] [--dry-run] [--force] [--debug]' . PHP_EOL . PHP_EOL; + print PHP_EOL . 'usage: materialize_queries.php [--query-id=ID[,ID...]] [--partitions=TABLE[,TABLE...]] [--nat=on|off] [--dry-run] [--status] [--force] [--debug]' . PHP_EOL . PHP_EOL; print 'Materializes each FlowView query into the parallel query cache for the' . PHP_EOL; print 'entire range of flow partitions, once with NAT on and once with NAT off.' . PHP_EOL; @@ -483,6 +544,7 @@ function display_help(): void { print ' partitions that have rotated.' . PHP_EOL; print ' --nat=on|off Only materialize this NAT mode.' . PHP_EOL; print ' --dry-run List what would be materialized and exit.' . PHP_EOL; + print ' --status List the parallel queries that are running and the Filter each is for.' . PHP_EOL; print ' --force Run even if another materialization is running.' . PHP_EOL; print ' --debug Provide some debug output during the run.' . PHP_EOL . PHP_EOL; } diff --git a/flowview_filters.php b/flowview_filters.php index 330a356..6f9fc4d 100644 --- a/flowview_filters.php +++ b/flowview_filters.php @@ -438,16 +438,17 @@ function clearFilter() { $cold = flowview_get_cold_partitions($tables); $partitions = cacti_sizeof($tables); $running = flowview_materialization_running(); - $requests = flowview_get_running_materialization_requests(); + $requests = flowview_get_running_requests(); foreach ($filters as $filter) { $md5sums = flowview_get_filter_md5sums((int) $filter['id']); $cached = flowview_get_cached_shard_tables((int) $filter['id'], $md5sums); $status = flowview_materialization_status($cached['nat'], $cached['non_nat'], $cold, $running); + $queries = flowview_count_running_queries($md5sums, $requests); form_alternate_row('line' . $filter['id'], true); form_selectable_cell('' . html_escape($filter['name']) . '', $filter['id']); - form_selectable_cell(trim(flowview_get_colored_materialization_status($status) . ' ' . flowview_get_colored_running_queries(flowview_count_running_queries($md5sums, $requests))), $filter['id'], '', 'center'); + form_selectable_cell(trim(flowview_get_colored_materialization_status($status) . ' ' . flowview_get_colored_running_queries($queries, flowview_count_running_queries_by_mode($md5sums, $requests))), $filter['id'], '', 'center'); if (empty($filter['device'])) { form_selectable_cell(__('All', 'flowview'), $filter['id']); diff --git a/includes/functions.php b/includes/functions.php index d33b7a1..8e8e422 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -4649,52 +4649,53 @@ function flowview_get_partitions_without_nat_columns(array $tables): array { } /** - * flowview_get_running_materialization_requests - The parallel query requests - * made by a script, as the materializer makes them, that still have a runner - * or shard worker process alive. They keep going after the script that - * started them has ended, for instance when it timed out. A request made - * from a web page belongs to its user, so it is not one of these. - * - * @return array The cache key (md5sum_tables) of each request, by request id + * flowview_get_running_requests - The parallel query requests that have a + * runner or shard worker registered in the process table, as db_query_ or + * db_shard_. They keep going after the script that started them has + * ended, for instance when it timed out. + * + * A registration counts when it is recent, or when its process is still + * alive. A recent one is trusted as the web server may not be able to see + * the process of another user, and an old one is checked, as a process that + * was killed can leave its row behind. + * + * @return array The cache key (md5sum_tables) of each request, by request id, + * which is empty for a request that no longer exists */ -function flowview_get_running_materialization_requests(): array { - $processes = db_fetch_assoc_prepared('SELECT pid, taskname +function flowview_get_running_requests(): array { + $recent = max(60, (int) read_config_option('flowview_parallel_runlimit')); + + $processes = db_fetch_assoc_prepared('SELECT pid, taskname, UNIX_TIMESTAMP() - UNIX_TIMESTAMP(started) AS age FROM processes WHERE tasktype = ? AND (taskname LIKE ? OR taskname LIKE ?)', ['flowview', 'db\_query\_%', 'db\_shard\_%']); - $pids = []; + $ids = []; foreach ($processes as $process) { - if (preg_match('/^db_(?:query|shard)_([0-9]+)$/', $process['taskname'], $matches)) { - $pids[(int) $matches[1]][] = (int) $process['pid']; + if (!preg_match('/^db_(?:query|shard)_([0-9]+)$/', $process['taskname'], $matches)) { + continue; + } + + if ((int) $process['age'] <= $recent || !function_exists('cacti_process_still_running') || cacti_process_still_running((int) $process['pid'])) { + $ids[(int) $matches[1]] = true; } } - if (!cacti_sizeof($pids)) { + if (!cacti_sizeof($ids)) { return []; } - $ids = array_keys($pids); - $requests = flowview_db_fetch_assoc_prepared('SELECT id, md5sum_tables FROM parallel_database_query - WHERE user_id = 0 - AND id IN (' . implode(', ', array_fill(0, cacti_sizeof($ids), '?')) . ')', - $ids); + WHERE id IN (' . implode(', ', array_fill(0, cacti_sizeof($ids), '?')) . ')', + array_keys($ids)); - $running = []; + $running = array_fill_keys(array_keys($ids), ''); foreach ($requests as $request) { - foreach ($pids[(int) $request['id']] as $pid) { - /* a process that was killed can leave its row behind */ - if (!function_exists('cacti_process_still_running') || cacti_process_still_running($pid)) { - $running[(int) $request['id']] = (string) $request['md5sum_tables']; - - break; - } - } + $running[(int) $request['id']] = (string) $request['md5sum_tables']; } return $running; @@ -4703,7 +4704,7 @@ function flowview_get_running_materialization_requests(): array { /** * flowview_materialization_running - Whether a materialization is running. * That is when one is registered as running, since only one may be active at - * a time, or when the parallel query it started still has processes alive. + * a time, or when a parallel query still has processes registered. * * @return bool */ @@ -4719,7 +4720,7 @@ function flowview_materialization_running(): bool { return true; } - return cacti_sizeof(flowview_get_running_materialization_requests()) > 0; + return cacti_sizeof(flowview_get_running_requests()) > 0; } /** @@ -4770,16 +4771,24 @@ function flowview_get_colored_materialization_status(string $status): string { * of a filter that are running now, in the gray of a disabled device. There * is none when nothing is running. * - * @param int $count The number of queries running + * @param int $count The number of queries running + * @param array|null $by_mode How many of them are with NAT and without, as + * nat and non_nat, to say so in the tip * * @return string */ -function flowview_get_colored_running_queries(int $count): string { +function flowview_get_colored_running_queries(int $count, ?array $by_mode = null): string { if ($count < 1) { return ''; } - return "" . sprintf(__('Running: %s', 'flowview'), number_format_i18n($count, 0)) . ''; + $tip = __esc('The number of queries of this Filter that are running now.', 'flowview'); + + if ($by_mode !== null) { + $tip .= ' ' . sprintf(__esc('With NAT: %d, without NAT: %d.', 'flowview'), $by_mode['nat'] ?? 0, $by_mode['non_nat'] ?? 0); + } + + return "" . sprintf(__('Running: %s', 'flowview'), number_format_i18n($count, 0)) . ''; } /** @@ -4884,7 +4893,7 @@ function flowview_get_filter_md5sums(int $query_id): array { * queries of a filter, with NAT or without. * * @param array $md5sums The cache keys of the filter from flowview_get_filter_md5sums() - * @param array $running The running requests from flowview_get_running_materialization_requests() + * @param array $running The running requests from flowview_get_running_requests() * * @return int */ @@ -4896,6 +4905,53 @@ function flowview_count_running_queries(array $md5sums, array $running): int { })); } +/** + * flowview_count_running_queries_by_mode - How many of the running requests + * are queries of a filter with NAT, and how many without. A request whose + * cache key is the same in both, as for a report that has no address to + * translate, is counted in both. + * + * @param array $md5sums The cache keys of the filter from flowview_get_filter_md5sums() + * @param array $running The running requests from flowview_get_running_requests() + * + * @return array The counts as nat and non_nat + */ +function flowview_count_running_queries_by_mode(array $md5sums, array $running): array { + return [ + 'nat' => flowview_count_running_queries(['nat' => $md5sums['nat'] ?? []], $running), + 'non_nat' => flowview_count_running_queries(['non_nat' => $md5sums['non_nat'] ?? []], $running), + ]; +} + +/** + * flowview_find_request_filters - The filters a running request belongs to, + * and whether it is with NAT or without. The process table only knows the id + * of the request, so it is the cache key of the request that finds them. + * + * @param array $filter_md5sums The cache keys from flowview_get_filter_md5sums(), by filter id + * @param string $md5sum The cache key of the request + * + * @return array The id of each filter, with whether it matches as nat and non_nat + */ +function flowview_find_request_filters(array $filter_md5sums, string $md5sum): array { + $found = []; + + if ($md5sum === '') { + return $found; + } + + foreach ($filter_md5sums as $id => $md5sums) { + $nat = in_array($md5sum, $md5sums['nat'] ?? [], true); + $non_nat = in_array($md5sum, $md5sums['non_nat'] ?? [], true); + + if ($nat || $non_nat) { + $found[] = ['id' => (int) $id, 'nat' => $nat, 'non_nat' => $non_nat]; + } + } + + return $found; +} + /** * flowview_get_cached_shard_tables - The partitions of a filter that are * stored in the parallel query cache, with NAT and without. Some reports diff --git a/tests/Unit/CoreFunctionsTest.php b/tests/Unit/CoreFunctionsTest.php index 2481d6c..d710e69 100644 --- a/tests/Unit/CoreFunctionsTest.php +++ b/tests/Unit/CoreFunctionsTest.php @@ -986,58 +986,90 @@ function cacti_process_still_running($pid) { ->and(array_column($removed, 'params'))->toBe(array(array(4999999), array(4999998))); }); - it('finds a request made by a script that still has a live process', function () { + it('finds the running requests from the process table', function () { $GLOBALS['__test_live_pids'] = array(222); plugin_test_queue_db_result('db_fetch_assoc_prepared', array( - array('pid' => 111, 'taskname' => 'db_query_991'), - array('pid' => 222, 'taskname' => 'db_shard_991'), - array('pid' => 333, 'taskname' => 'db_shard_992'), - array('pid' => 444, 'taskname' => 'db_shard_abc'), + // old, and the process is gone + array('pid' => 111, 'taskname' => 'db_query_990', 'age' => 5000), + // old, but the process is alive + array('pid' => 222, 'taskname' => 'db_shard_991', 'age' => 5000), + array('pid' => 223, 'taskname' => 'db_shard_991', 'age' => 5000), + // recent, so it is trusted even though the process cannot be seen + array('pid' => 333, 'taskname' => 'db_shard_992', 'age' => 10), + array('pid' => 444, 'taskname' => 'db_shard_abc', 'age' => 10), + )); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array( + array('id' => 991, 'md5sum_tables' => 'aaaa'), + array('id' => 992, 'md5sum_tables' => 'bbbb'), )); - // 992 belongs to a web user, so it is not one of these. - plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('id' => 991, 'md5sum_tables' => 'aaaa'))); - expect(flowview_get_running_materialization_requests())->toBe(array(991 => 'aaaa')); + expect(flowview_get_running_requests())->toBe(array(991 => 'aaaa', 992 => 'bbbb')); + + $lookup = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['sql'], 'FROM parallel_database_query') !== false; + })); + + // it is not limited to the requests of one user + expect($lookup[0]['sql'])->not->toContain('user_id') + ->and($lookup[0]['params'])->toBe(array(991, 992)); + }); + + it('keeps a running request whose row is gone, without a cache key', function () { + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('pid' => 111, 'taskname' => 'db_query_991', 'age' => 5))); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array()); + + expect(flowview_get_running_requests())->toBe(array(991 => '')); }); it('ignores the rows left behind by processes that are gone', function () { plugin_test_queue_db_result('db_fetch_assoc_prepared', array( - array('pid' => 111, 'taskname' => 'db_query_991'), - array('pid' => 222, 'taskname' => 'db_shard_991'), + array('pid' => 111, 'taskname' => 'db_query_991', 'age' => 9000), + array('pid' => 222, 'taskname' => 'db_shard_991', 'age' => 9000), )); - plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('id' => 991, 'md5sum_tables' => 'aaaa'))); - expect(flowview_get_running_materialization_requests())->toBe(array()); + expect(flowview_get_running_requests())->toBe(array()) + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['sql'], 'FROM parallel_database_query') !== false; + }))->toBeEmpty(); }); it('finds nothing without any runner processes', function () { - expect(flowview_get_running_materialization_requests())->toBe(array()) + expect(flowview_get_running_requests())->toBe(array()) ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { - return strpos($call['sql'], 'parallel_database_query') !== false; + return strpos($call['sql'], 'FROM parallel_database_query') !== false; }))->toBeEmpty(); }); + it('looks for the runner and shard worker names in the process table', function () { + flowview_get_running_requests(); + + $lookup = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['sql'], 'FROM processes') !== false; + })); + + expect($lookup)->toHaveCount(1) + ->and($lookup[0]['params'])->toBe(array('flowview', 'db\_query\_%', 'db\_shard\_%')); + }); + it('is running while a materialization is registered', function () { plugin_test_queue_db_result('db_fetch_cell_prepared', 1); expect(flowview_materialization_running())->toBeTrue() ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { - return strpos($call['sql'], 'db\_query\_') !== false || ($call['params'][1] ?? '') === 'db\_query\_%'; + return strpos($call['sql'], 'FROM processes') !== false && ($call['params'][1] ?? '') === 'db\_query\_%'; }))->toBeEmpty(); }); it('is still running after the script that started it has ended', function () { - $GLOBALS['__test_live_pids'] = array(222); - plugin_test_queue_db_result('db_fetch_cell_prepared', 0); - plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('pid' => 222, 'taskname' => 'db_shard_991'))); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('pid' => 222, 'taskname' => 'db_shard_991', 'age' => 12))); plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('id' => 991, 'md5sum_tables' => 'aaaa'))); expect(flowview_materialization_running())->toBeTrue(); }); - it('is not running when nothing is registered and no runner is alive', function () { + it('is not running when nothing is registered and no runner is registered', function () { plugin_test_queue_db_result('db_fetch_cell_prepared', 0); expect(flowview_materialization_running())->toBeFalse(); @@ -1112,6 +1144,34 @@ function number_format_i18n($number, $decimals = 0) { expect(flowview_count_running_queries(array('nat' => array(), 'non_nat' => array()), array(991 => 'aaaa')))->toBe(0); }); + it('counts the running queries of a filter with NAT and without', function () { + $md5sums = array('nat' => array('aaaa'), 'non_nat' => array('bbbb')); + + expect(flowview_count_running_queries_by_mode($md5sums, array(991 => 'aaaa', 992 => 'bbbb', 993 => 'bbbb', 994 => 'zzzz'))) + ->toBe(array('nat' => 1, 'non_nat' => 2)); + }); + + it('finds the filter a running request belongs to, by its cache key', function () { + $filters = array( + 1 => array('nat' => array('aaaa'), 'non_nat' => array('bbbb')), + 15 => array('nat' => array('cccc'), 'non_nat' => array('cccc', 'dddd')), + 16 => array('nat' => array(), 'non_nat' => array()), + ); + + expect(flowview_find_request_filters($filters, 'aaaa'))->toBe(array(array('id' => 1, 'nat' => true, 'non_nat' => false))) + ->and(flowview_find_request_filters($filters, 'bbbb'))->toBe(array(array('id' => 1, 'nat' => false, 'non_nat' => true))) + ->and(flowview_find_request_filters($filters, 'cccc'))->toBe(array(array('id' => 15, 'nat' => true, 'non_nat' => true))) + ->and(flowview_find_request_filters($filters, 'zzzz'))->toBe(array()) + ->and(flowview_find_request_filters($filters, ''))->toBe(array()); + }); + + it('says how many running queries are with NAT and without in the tip', function () { + $pill = flowview_get_colored_running_queries(3, array('nat' => 1, 'non_nat' => 2)); + + expect($pill)->toContain('Running: 3') + ->and($pill)->toContain('With NAT: 1, without NAT: 2.'); + }); + it('shows the number of running queries in a pill like a disabled device', function () { $pill = flowview_get_colored_running_queries(1234); From 7f04d8e345b69f2142970805e525df9cc4d58089 Mon Sep 17 00:00:00 2001 From: TheWitness Date: Tue, 20 Oct 2026 16:05:49 -0400 Subject: [PATCH 12/37] fix: Report each partition alter once and rename the Filters status column Stop cli/materialize_queries.php from showing each ALTER TABLE twice: the log message was also printed, as well as the script's own progress line. The alter is still logged, and the schema upgrade still prints it. Rename the Filters page Status column to Materialization and explain in its tip what materialization is and what the pills and the Running count mean. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 4 +++- README.md | 4 ++-- cli/materialize_queries.php | 2 +- flowview_filters.php | 4 ++-- includes/functions.php | 6 ++++-- tests/Unit/CoreFunctionsTest.php | 20 ++++++++++++++++++++ tests/bootstrap-unit.php | 1 + 7 files changed, 33 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7ae6c3..43afdfc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ --- develop --- +* bug: Stop cli/materialize_queries.php from reporting each ALTER TABLE twice, once from its own progress line and once from the log message that is also printed. The alter is still written to the Cacti log, and the schema upgrade, which has no other report, still shows it. The logged text no longer has the extra space in it + * feature: Show the number of queries of each Filter that are running now in a second pill, in the style of a disabled device and reading Running: N, beside the status pill on the Filters page, with how many are with NAT and without in its tip. The process table only has the id of a running request, so a running query is matched to a Filter by the cache key its map query is stored under, which is worked out from the Filter, and there is no pill when none are running * feature: Add --status to cli/materialize_queries.php to list the parallel queries that are running and the Filter each one is for, with its progress @@ -18,7 +20,7 @@ * feature: Materialize the partitions that have gone cold automatically. Each poller cycle flowview_process.php records the newest partition, which is the one still receiving flows, and starts cli/materialize_queries.php in the background for the partitions that have rotated since the previous cycle, for every Filter with NAT on and off. Partitions that are already materialized and accurate are skipped. When a materialization is already running, or the Parallel Query is not enabled, the record is left alone so a later cycle catches up. The first cycle only records the newest partition, so older partitions are still materialized with cli/materialize_queries.php. The script gained a --partitions option for this -* feature: Add a Status column, to the right of the Filter Name on the Filters page, showing the automatic materialization status as a pill in the form of the device status: Required (down) when rotated partitions are not materialized yet, Running (recovering) while a materialization is in progress, and Up to date (up) once every cold partition is materialized with NAT on and off +* feature: Add a Materialization column, to the right of the Filter Name on the Filters page, with a tip explaining what materialization is, showing the automatic materialization status as a pill in the form of the device status: Required (down) when rotated partitions are not materialized yet, Running (recovering) while a materialization is in progress, and Up to date (up) once every cold partition is materialized with NAT on and off * feature: Show the number of partitions each Filter has stored in the Parallel Query cache, with NAT and without, as the NAT Shards and Non-NAT Shards columns next to the ID on the Filters page, along with a Partitions column holding the total number of flow partitions so the progress of a materialization can be followed; the cache keys are worked out from the Filter without running it (run_flow_query() gained a plan-only mode, and the map query hashing is shared with the parallel query request) diff --git a/README.md b/README.md index c81a08b..b58e39e 100644 --- a/README.md +++ b/README.md @@ -242,8 +242,8 @@ newest partition, which is the one still receiving flows, and materializes the partitions that have rotated since the previous cycle for every Filter, with NAT on and off, in the background. The first cycle only records the newest partition, so run the script once to materialize the existing ones. The -'Status' column on the Filters page shows the result for each Filter, as -Required, Running or Up to date. The partition still receiving flows is not +'Materialization' column on the Filters page shows the result for each Filter, +as Required, Running or Up to date. The partition still receiving flows is not materialized automatically, so it does not count. When you save a diff --git a/cli/materialize_queries.php b/cli/materialize_queries.php index 3d1d029..aa39446 100644 --- a/cli/materialize_queries.php +++ b/cli/materialize_queries.php @@ -430,7 +430,7 @@ function materialize_prepare_partitions(array $cold, array $modes, bool $dry_run if (!$dry_run) { $began = microtime(true); - flowview_apply_partition_alter($table, $plan); + flowview_apply_partition_alter($table, $plan, false); print sprintf('Altered Table %s in %0.2f seconds', $table, microtime(true) - $began) . PHP_EOL; } diff --git a/flowview_filters.php b/flowview_filters.php index 6f9fc4d..ef04d61 100644 --- a/flowview_filters.php +++ b/flowview_filters.php @@ -377,9 +377,9 @@ function clearFilter() { 'sort' => 'ASC' ), 'nosort5' => array( - 'display' => __('Status', 'flowview'), + 'display' => __('Materialization', 'flowview'), 'align' => 'center', - 'tip' => __('The automatic materialization status of the Filter. Required means partitions that have rotated are not materialized yet, Running means a materialization is in progress, and Up to date means every rotated partition is materialized with NAT on and off. The partition still receiving flows is not materialized automatically.', 'flowview') + 'tip' => __('Materialization pre-computes the results of a Filter for each flow partition, with NAT on and off, and stores them in the Parallel Query cache, so reports over older data open quickly. New partitions are materialized automatically once they stop receiving flows. Required means partitions that have rotated are not materialized yet, Running means a materialization is in progress, and Up to date means every rotated partition is materialized. Running: N is how many queries of the Filter are working now. The partition still receiving flows is not materialized, so it does not count.', 'flowview') ), 'device' => array( 'display' => __('Listener', 'flowview'), diff --git a/includes/functions.php b/includes/functions.php index 8e8e422..ae4656d 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -4509,11 +4509,13 @@ function flowview_get_partition_alter(array $info, string $raw_engine): array { * * @param string $table The raw partition table * @param array $plan The plan from flowview_get_partition_alter() + * @param bool $print Also show what is being altered, as well as logging it. + * A script that reports it itself passes false. * * @return void */ -function flowview_apply_partition_alter(string $table, array $plan): void { - cacti_log("Altering Table $table Using this alter {$plan['alter']}.", true, 'FLOWVIEW'); +function flowview_apply_partition_alter(string $table, array $plan, bool $print = true): void { + cacti_log("Altering Table $table Using this alter " . trim($plan['alter']) . '.', $print, 'FLOWVIEW'); flowview_db_execute('ALTER TABLE ' . $table . ' ' . $plan['alter']); diff --git a/tests/Unit/CoreFunctionsTest.php b/tests/Unit/CoreFunctionsTest.php index d710e69..e3e5875 100644 --- a/tests/Unit/CoreFunctionsTest.php +++ b/tests/Unit/CoreFunctionsTest.php @@ -829,6 +829,26 @@ function cacti_escapeshellarg($string) { expect(array_values($sql))->toBe(array('ALTER TABLE plugin_flowview_raw_2024001 ADD INDEX end_time (end_time)')); }); + it('logs an alter and shows it too, as the schema upgrade always did', function () { + $GLOBALS['__test_cacti_log'] = array(); + $GLOBALS['__test_cacti_log_print'] = array(); + + flowview_apply_partition_alter('plugin_flowview_raw_2024001', array('alter' => 'ADD INDEX end_time (end_time)', 'ex_change' => false)); + + expect($GLOBALS['__test_cacti_log'])->toBe(array('Altering Table plugin_flowview_raw_2024001 Using this alter ADD INDEX end_time (end_time).')) + ->and($GLOBALS['__test_cacti_log_print'])->toBe(array(true)); + }); + + it('only logs an alter when the caller reports it itself, so it is not said twice', function () { + $GLOBALS['__test_cacti_log'] = array(); + $GLOBALS['__test_cacti_log_print'] = array(); + + flowview_apply_partition_alter('plugin_flowview_raw_2024001', array('alter' => ' ENGINE=Aria ROW_FORMAT=page', 'ex_change' => false), false); + + expect($GLOBALS['__test_cacti_log'])->toBe(array('Altering Table plugin_flowview_raw_2024001 Using this alter ENGINE=Aria ROW_FORMAT=page.')) + ->and($GLOBALS['__test_cacti_log_print'])->toBe(array(false)); + }); + it('finds the partitions that still need an alter in a single pass', function () { $table = function ($name, $engine) { return array('TABLE_NAME' => $name, 'TABLE_COLLATION' => 'latin1_swedish_ci', 'ENGINE' => $engine); diff --git a/tests/bootstrap-unit.php b/tests/bootstrap-unit.php index c83ff97..10b9c66 100644 --- a/tests/bootstrap-unit.php +++ b/tests/bootstrap-unit.php @@ -288,6 +288,7 @@ function __esc($text, ...$args) { function cacti_log($message, $also_print = false, $log_type = '', $level = 0) { $args = func_get_args(); $GLOBALS["__test_cacti_log"][] = isset($args[0]) ? (string) $args[0] : ""; + $GLOBALS["__test_cacti_log_print"][] = (bool) $also_print; } } From a18e47b79b190a50b0eb614a674a79824f22991b Mon Sep 17 00:00:00 2001 From: TheWitness Date: Tue, 20 Oct 2026 16:06:42 -0400 Subject: [PATCH 13/37] fix: Put the ID column right after the Filter Name Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 2 +- flowview_filters.php | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43afdfc..a5a31dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ * feature: Materialize the partitions that have gone cold automatically. Each poller cycle flowview_process.php records the newest partition, which is the one still receiving flows, and starts cli/materialize_queries.php in the background for the partitions that have rotated since the previous cycle, for every Filter with NAT on and off. Partitions that are already materialized and accurate are skipped. When a materialization is already running, or the Parallel Query is not enabled, the record is left alone so a later cycle catches up. The first cycle only records the newest partition, so older partitions are still materialized with cli/materialize_queries.php. The script gained a --partitions option for this -* feature: Add a Materialization column, to the right of the Filter Name on the Filters page, with a tip explaining what materialization is, showing the automatic materialization status as a pill in the form of the device status: Required (down) when rotated partitions are not materialized yet, Running (recovering) while a materialization is in progress, and Up to date (up) once every cold partition is materialized with NAT on and off +* feature: Add a Materialization column, to the right of the Filter Name and its ID on the Filters page, with a tip explaining what materialization is, showing the automatic materialization status as a pill in the form of the device status: Required (down) when rotated partitions are not materialized yet, Running (recovering) while a materialization is in progress, and Up to date (up) once every cold partition is materialized with NAT on and off. The ID column now follows the Filter Name * feature: Show the number of partitions each Filter has stored in the Parallel Query cache, with NAT and without, as the NAT Shards and Non-NAT Shards columns next to the ID on the Filters page, along with a Partitions column holding the total number of flow partitions so the progress of a materialization can be followed; the cache keys are worked out from the Filter without running it (run_flow_query() gained a plan-only mode, and the map query hashing is shared with the parallel query request) diff --git a/flowview_filters.php b/flowview_filters.php index ef04d61..70943fa 100644 --- a/flowview_filters.php +++ b/flowview_filters.php @@ -376,6 +376,11 @@ function clearFilter() { 'align' => 'left', 'sort' => 'ASC' ), + 'id' => array( + 'display' => __('ID', 'flowview'), + 'align' => 'right', + 'sort' => 'ASC' + ), 'nosort5' => array( 'display' => __('Materialization', 'flowview'), 'align' => 'center', @@ -391,11 +396,6 @@ function clearFilter() { 'align' => 'left', 'sort' => 'ASC' ), - 'id' => array( - 'display' => __('ID', 'flowview'), - 'align' => 'right', - 'sort' => 'ASC' - ), 'nosort2' => array( 'display' => __('NAT Shards', 'flowview'), 'align' => 'right', @@ -448,6 +448,7 @@ function clearFilter() { form_alternate_row('line' . $filter['id'], true); form_selectable_cell('' . html_escape($filter['name']) . '', $filter['id']); + form_selectable_cell($filter['id'], $filter['id'], '', 'right'); form_selectable_cell(trim(flowview_get_colored_materialization_status($status) . ' ' . flowview_get_colored_running_queries($queries, flowview_count_running_queries_by_mode($md5sums, $requests))), $filter['id'], '', 'center'); if (empty($filter['device'])) { @@ -465,7 +466,6 @@ function clearFilter() { } form_selectable_cell($type, $filter['id']); - form_selectable_cell($filter['id'], $filter['id'], '', 'right'); form_selectable_cell(number_format_i18n(cacti_sizeof($cached['nat']), 0), $filter['id'], '', 'right'); form_selectable_cell(number_format_i18n(cacti_sizeof($cached['non_nat']), 0), $filter['id'], '', 'right'); From 79c15dd8073b70ade6506f5c58a4c71650c2a6e1 Mon Sep 17 00:00:00 2001 From: TheWitness Date: Tue, 20 Oct 2026 16:27:31 -0400 Subject: [PATCH 14/37] fix: Pass the values to __() before the text domain in the running pill and timeout The Running: N pill tip passed only the text domain to __esc() for a text with placeholders, which Cacti rejects with an ArgumentCountError and broke the Filters page whenever a query was running. The pill label and the Parallel Query timeout message had the same mistake and would have shown the domain and 0 seconds. Pass the values first, as the rest of the plugin does. Make the unit test __() and __esc() stubs treat every argument as a sprintf argument like Cacti, so a mistake like this fails a test. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 2 ++ includes/functions.php | 6 +++--- tests/bootstrap-unit.php | 18 +++++++++--------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5a31dd..f494a70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ --- develop --- +* bug: Fix the Filters page failing with "3 arguments are required, 2 given" once a query was running. The Running: N pill and its tip, and the Parallel Query timeout message, passed the text domain where Cacti's __() expects the values to put in the text, so the tip crashed, the pill would have read "Running: flowview", and the timeout would have said 0 seconds. The values now come before the domain, as elsewhere in the plugin + * bug: Stop cli/materialize_queries.php from reporting each ALTER TABLE twice, once from its own progress line and once from the log message that is also printed. The alter is still written to the Cacti log, and the schema upgrade, which has no other report, still shows it. The logged text no longer has the extra space in it * feature: Show the number of queries of each Filter that are running now in a second pill, in the style of a disabled device and reading Running: N, beside the status pill on the Filters page, with how many are with NAT and without in its tip. The process table only has the id of a running request, so a running query is matched to a Filter by the cache key its map query is stored under, which is worked out from the Filter, and there is no pill when none are running diff --git a/includes/functions.php b/includes/functions.php index ae4656d..9541dd6 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -4787,10 +4787,10 @@ function flowview_get_colored_running_queries(int $count, ?array $by_mode = null $tip = __esc('The number of queries of this Filter that are running now.', 'flowview'); if ($by_mode !== null) { - $tip .= ' ' . sprintf(__esc('With NAT: %d, without NAT: %d.', 'flowview'), $by_mode['nat'] ?? 0, $by_mode['non_nat'] ?? 0); + $tip .= ' ' . __esc('With NAT: %d, without NAT: %d.', $by_mode['nat'] ?? 0, $by_mode['non_nat'] ?? 0, 'flowview'); } - return "" . sprintf(__('Running: %s', 'flowview'), number_format_i18n($count, 0)) . ''; + return "" . __('Running: %s', number_format_i18n($count, 0), 'flowview') . ''; } /** @@ -5195,7 +5195,7 @@ function parallel_database_query_request($tables, $stru_inner, $stru_outer) { * @return string */ function flowview_parallel_timeout_message(int $seconds): string { - return sprintf(__('ERROR: The Parallel Query did not finish within the Maximum Run Time of %d seconds. Check the Maximum Run Time under the FlowView Settings, and that the database is not busy with another operation such as an ALTER TABLE.', 'flowview'), $seconds); + return __('ERROR: The Parallel Query did not finish within the Maximum Run Time of %d seconds. Check the Maximum Run Time under the FlowView Settings, and that the database is not busy with another operation such as an ALTER TABLE.', $seconds, 'flowview'); } /** diff --git a/tests/bootstrap-unit.php b/tests/bootstrap-unit.php index 10b9c66..5d48cff 100644 --- a/tests/bootstrap-unit.php +++ b/tests/bootstrap-unit.php @@ -266,21 +266,21 @@ function html_escape($string) { } } +/* + * Like Cacti's __(), every argument after the text is a sprintf() argument + * unless the last one is a registered text domain. The plugin's domain is + * not, so a text with placeholders and only a domain fails in Cacti, and the + * stub must fail the same way. + */ if (!function_exists('__')) { - function __($text, $domain = '') { - return $text; + function __($text, ...$args) { + return empty($args) ? $text : vsprintf($text, $args); } } if (!function_exists('__esc')) { function __esc($text, ...$args) { - if (!empty($args)) { - array_pop($args); - } - if (!empty($args)) { - $text = vsprintf($text, $args); - } - return htmlspecialchars($text, ENT_QUOTES | ENT_HTML5, 'UTF-8'); + return htmlspecialchars(__($text, ...$args), ENT_QUOTES | ENT_HTML5, 'UTF-8'); } } From 4eb0d56a8e21c7c3bb5129cb003032091c869ee2 Mon Sep 17 00:00:00 2001 From: TheWitness Date: Tue, 20 Oct 2026 16:36:06 -0400 Subject: [PATCH 15/37] fix: Cache the earliest and rotated partitions and say which are missing A shard is only cached when the range starts before the earliest flow in the partition. Flows can start before the window of their partition, but the range was padded by only a second, so the oldest partition, or a rotated one materialized on its own, was never cached and its filter stayed Required. Pad the start by a day, and report the first and last partition in the summary. Say which partitions are not materialized, with NAT and without, in the tip of the Required and Running pill. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 4 +++ cli/materialize_queries.php | 4 +-- flowview_filters.php | 3 +- includes/functions.php | 60 ++++++++++++++++++++++++++++---- tests/Unit/CoreFunctionsTest.php | 44 ++++++++++++++++++++++- 5 files changed, 104 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f494a70..8bfb638 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ --- develop --- +* bug: Make the earliest partition, and the partitions that rotate, get cached by a materialization. A shard is only cached when the range starts before the earliest flow in the partition, and a flow can start before the window of the partition it was written to, but the range was padded by only a second, so the oldest partition, or a rotated one on its own, was never cached and its Filter stayed Required. The start is now padded by a day, which only adds partitions that already exist and are cached, and the summary now shows the first and last partition instead of the padded range. Run the materializer again to cache what was missed + +* feature: Say which partitions a Filter is not materialized for, with NAT and without, in the tip of its Required and Running pill + * bug: Fix the Filters page failing with "3 arguments are required, 2 given" once a query was running. The Running: N pill and its tip, and the Parallel Query timeout message, passed the text domain where Cacti's __() expects the values to put in the text, so the tip crashed, the pill would have read "Running: flowview", and the timeout would have said 0 seconds. The values now come before the domain, as elsewhere in the plugin * bug: Stop cli/materialize_queries.php from reporting each ALTER TABLE twice, once from its own progress line and once from the log message that is also printed. The alter is still written to the Cacti log, and the schema upgrade, which has no other report, still shows it. The logged text no longer has the extra space in it diff --git a/cli/materialize_queries.php b/cli/materialize_queries.php index aa39446..ea6a3b3 100644 --- a/cli/materialize_queries.php +++ b/cli/materialize_queries.php @@ -234,8 +234,8 @@ print sprintf('FlowView Materialization: Partitions:%d From:%s To:%s Queries:%d NAT:%s', cacti_sizeof($tables), - date('Y-m-d H:i:s', $start + 1), - date('Y-m-d H:i:s', $end - 1), + date('Y-m-d H:i:s', flowview_table_name_to_time(reset($tables), 'start_time')), + date('Y-m-d H:i:s', flowview_table_name_to_time(end($tables), 'end_time')), cacti_sizeof($queries), implode('/', $modes)) . PHP_EOL; diff --git a/flowview_filters.php b/flowview_filters.php index 70943fa..32a7daf 100644 --- a/flowview_filters.php +++ b/flowview_filters.php @@ -445,11 +445,12 @@ function clearFilter() { $cached = flowview_get_cached_shard_tables((int) $filter['id'], $md5sums); $status = flowview_materialization_status($cached['nat'], $cached['non_nat'], $cold, $running); $queries = flowview_count_running_queries($md5sums, $requests); + $missing = flowview_get_missing_partitions($cached['nat'], $cached['non_nat'], $cold); form_alternate_row('line' . $filter['id'], true); form_selectable_cell('' . html_escape($filter['name']) . '', $filter['id']); form_selectable_cell($filter['id'], $filter['id'], '', 'right'); - form_selectable_cell(trim(flowview_get_colored_materialization_status($status) . ' ' . flowview_get_colored_running_queries($queries, flowview_count_running_queries_by_mode($md5sums, $requests))), $filter['id'], '', 'center'); + form_selectable_cell(trim(flowview_get_colored_materialization_status($status, $missing) . ' ' . flowview_get_colored_running_queries($queries, flowview_count_running_queries_by_mode($md5sums, $requests))), $filter['id'], '', 'center'); if (empty($filter['device'])) { form_selectable_cell(__('All', 'flowview'), $filter['id']); diff --git a/includes/functions.php b/includes/functions.php index 9541dd6..a931d67 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -4138,9 +4138,14 @@ function flowview_convert_yeardayhour_to_date($range, $year, $day, int $hour = 0 * Names that are not partition tables are ignored, and false is returned * when none qualify. * - * Both bounds are padded by one second. A shard is only a full scan, and - * therefore only cached in parallel_database_query_shard_cache, when the - * requested range lies strictly outside the data in the partition. + * A shard is only a full scan, and therefore only cached in + * parallel_database_query_shard_cache, when the requested range lies strictly + * outside the data in the partition. The end is padded by a second, as a + * flow ends no later than it was captured. The start is padded by a day, + * because a flow can start well before the window of the partition it was + * written to, and with less the earliest partition, or one that rotated, is + * never treated as a full scan and so is never cached. A partition that does + * not exist in the extra day is simply not there to scan. * * @param array $tables * @@ -4171,7 +4176,7 @@ function flowview_get_materialize_range(array $tables): array|false { return false; } - return [$start - 1, $end + 1]; + return [$start - 86400, $end + 1]; } /** @@ -4757,17 +4762,58 @@ function flowview_materialization_status(array $cached_nat, array $cached_non_na * * @return string */ -function flowview_get_colored_materialization_status(string $status): string { +function flowview_get_colored_materialization_status(string $status, ?array $missing = null): string { + $tip = ''; + + /* say which partitions are not materialized, when it is not up to date */ + if ($missing !== null && $status != 'current') { + $tip = " title='" . __esc('Not materialized yet, with NAT: %s, without NAT: %s.', flowview_describe_partitions($missing['nat'] ?? []), flowview_describe_partitions($missing['non_nat'] ?? []), 'flowview') . "'"; + } + switch ($status) { case 'current': return "" . __('Up to date', 'flowview') . ''; case 'running': - return "" . __('Running', 'flowview') . ''; + return "" . __('Running', 'flowview') . ''; default: - return "" . __('Required', 'flowview') . ''; + return "" . __('Required', 'flowview') . ''; } } +/** + * flowview_get_missing_partitions - The cold partitions that a filter is not + * materialized for, with NAT and without. + * + * @param array $cached_nat The partitions cached for the filter with NAT + * @param array $cached_non_nat The partitions cached for the filter without NAT + * @param array $cold The cold partitions + * + * @return array The partitions as nat and non_nat + */ +function flowview_get_missing_partitions(array $cached_nat, array $cached_non_nat, array $cold): array { + return [ + 'nat' => array_values(array_diff($cold, $cached_nat)), + 'non_nat' => array_values(array_diff($cold, $cached_non_nat)), + ]; +} + +/** + * flowview_describe_partitions - How many partitions there are and the first + * few of them by name, for a tip. + * + * @param array $tables The partition tables + * @param int $limit How many to name + * + * @return string + */ +function flowview_describe_partitions(array $tables, int $limit = 3): string { + if (!cacti_sizeof($tables)) { + return '0'; + } + + return cacti_sizeof($tables) . ' (' . implode(', ', array_slice($tables, 0, $limit)) . (cacti_sizeof($tables) > $limit ? ', ...' : '') . ')'; +} + /** * flowview_get_colored_running_queries - The pill for the number of queries * of a filter that are running now, in the gray of a disabled device. There diff --git a/tests/Unit/CoreFunctionsTest.php b/tests/Unit/CoreFunctionsTest.php index e3e5875..948e749 100644 --- a/tests/Unit/CoreFunctionsTest.php +++ b/tests/Unit/CoreFunctionsTest.php @@ -192,8 +192,9 @@ 'plugin_flowview_raw_202400105', )); + // A flow can start before the window of its partition, so the start has a day of room. expect($range)->toBe(array( - strtotime('2024-01-02 00:00:00 UTC') - 1, + strtotime('2024-01-02 00:00:00 UTC') - 86400, strtotime('2024-01-05 00:00:00 UTC') + 1, )); }); @@ -591,6 +592,47 @@ function array_rekey($array, $key, $value) { expect(flowview_materialization_status(array(), array(), array(), false))->toBe('current'); }); +it('is required when the live partition is cached but an older one is not, and names it', function () { + $tables = array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002', 'plugin_flowview_raw_2024003', 'plugin_flowview_raw_2024004'); + $cold = flowview_get_cold_partitions($tables); + + // every partition but the oldest is cached, and that includes the live one + $cached = array('plugin_flowview_raw_2024002', 'plugin_flowview_raw_2024003', 'plugin_flowview_raw_2024004'); + + expect(flowview_materialization_status($cached, $cached, $cold, false))->toBe('required') + ->and(flowview_get_missing_partitions($cached, $cached, $cold))->toBe(array( + 'nat' => array('plugin_flowview_raw_2024001'), + 'non_nat' => array('plugin_flowview_raw_2024001'), + )); +}); + +it('is up to date when every older partition is cached and only the live one is not', function () { + $tables = array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002', 'plugin_flowview_raw_2024003'); + $cold = flowview_get_cold_partitions($tables); + $cached = array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024002'); + + expect(flowview_materialization_status($cached, $cached, $cold, false))->toBe('current') + ->and(flowview_get_missing_partitions($cached, $cached, $cold))->toBe(array('nat' => array(), 'non_nat' => array())); +}); + +it('describes the partitions that are missing in a tip', function ($tables, $expected) { + expect(flowview_describe_partitions($tables))->toBe($expected); +})->with(array( + 'none' => array(array(), '0'), + 'one' => array(array('plugin_flowview_raw_2024001'), '1 (plugin_flowview_raw_2024001)'), + 'three' => array(array('a', 'b', 'c'), '3 (a, b, c)'), + 'too many' => array(array('a', 'b', 'c', 'd', 'e'), '5 (a, b, c, ...)'), +)); + +it('says which partitions are missing in the tip of a pill that is not up to date', function () { + $missing = array('nat' => array('plugin_flowview_raw_2024001'), 'non_nat' => array()); + + expect(flowview_get_colored_materialization_status('required', $missing)) + ->toContain("deviceDown' title='Not materialized yet, with NAT: 1 (plugin_flowview_raw_2024001), without NAT: 0.'") + ->and(flowview_get_colored_materialization_status('running', $missing))->toContain('deviceRecovering\' title=') + ->and(flowview_get_colored_materialization_status('current', $missing))->not->toContain('title='); +}); + it('shows the status pills in the form of the device status', function ($status, $class, $label) { expect(flowview_get_colored_materialization_status($status))->toBe("" . $label . ''); })->with(array( From 8b4af55d5b1d4279594563f80fd252782628010b Mon Sep 17 00:00:00 2001 From: TheWitness Date: Tue, 20 Oct 2026 16:42:25 -0400 Subject: [PATCH 16/37] fix: Never cache the live partition The partition still receiving flows was scanned as a full scan by a full run and its result cached, which is wrong as soon as more flows arrive, and it was counted as materialized. Decide whether a shard is a full scan in flowview_is_full_scan(), which refuses the newest partition, so no query can cache it. cli/materialize_queries.php now only materializes partitions that have rotated, and removes anything cached for the live partition before. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 2 + README.md | 6 ++- cli/materialize_queries.php | 21 ++++++---- includes/functions.php | 69 +++++++++++++++++++++++++++++--- tests/Unit/CoreFunctionsTest.php | 63 +++++++++++++++++++++++++++++ 5 files changed, 146 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bfb638..82b5ff1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ --- develop --- +* bug: Never cache the partition that is still receiving flows. A full run included it, and as its flows ended before the end of the range it was scanned as a full scan and its result was cached, which was wrong as soon as more flows arrived, and counted it as materialized. Whether a shard is a full scan is now decided in flowview_is_full_scan(), which refuses the newest partition, so a report that covers it cannot cache it either, and it is scanned for the range as it is asked for. cli/materialize_queries.php only materializes the partitions that have rotated, and removes what was cached for the live partition before, so NAT Shards and Non-NAT Shards can no longer equal the number of Partitions + * bug: Make the earliest partition, and the partitions that rotate, get cached by a materialization. A shard is only cached when the range starts before the earliest flow in the partition, and a flow can start before the window of the partition it was written to, but the range was padded by only a second, so the oldest partition, or a rotated one on its own, was never cached and its Filter stayed Required. The start is now padded by a day, which only adds partitions that already exist and are cached, and the summary now shows the first and last partition instead of the padded range. Run the materializer again to cache what was missed * feature: Say which partitions a Filter is not materialized for, with NAT and without, in the tip of its Required and Running pill diff --git a/README.md b/README.md index b58e39e..3733bce 100644 --- a/README.md +++ b/README.md @@ -211,8 +211,10 @@ more under the FlowView Settings): php cli/materialize_queries.php ``` -The script runs every saved query over the entire range of flow partitions, -once with NAT on and once with NAT off. Use `--query-id=ID[,ID...]` or +The script runs every saved query over every flow partition that has rotated, +once with NAT on and once with NAT off. The partition still receiving flows is +never cached, as a result cached for it would be wrong as soon as more flows +arrive. Use `--query-id=ID[,ID...]` or `--nat=on|off` to narrow the run, and `--dry-run` to see what would be run. Partitions that are already cached and still match their data are skipped, while partitions that have received more flows since, or have been dropped, diff --git a/cli/materialize_queries.php b/cli/materialize_queries.php index ea6a3b3..a464512 100644 --- a/cli/materialize_queries.php +++ b/cli/materialize_queries.php @@ -192,17 +192,15 @@ exit(1); } -$tables = flowview_get_partition_tables(); - -/* the newest partition is still receiving flows, so it is left as it is */ -$cold = flowview_get_cold_partitions($tables); +/* the live partition is still receiving flows, so it is never materialized */ +$tables = flowview_get_cold_partitions(flowview_get_partition_tables()); /* only the partitions asked for, that still exist */ if (cacti_sizeof($partitions)) { $tables = array_values(array_intersect($tables, $partitions)); } -$cold = array_values(array_intersect($tables, $cold)); +$cold = $tables; $range = flowview_get_materialize_range($tables); @@ -248,6 +246,12 @@ /* partitions that are still accurate keep their cache, everything else is rebuilt below */ if (!$dry_run) { + $live = flowview_remove_live_partition_cache(); + + if ($live > 0) { + print sprintf('Removed %d cached result(s) of the live partition, as it is still receiving flows', $live) . PHP_EOL; + } + $cache = flowview_invalidate_stale_shard_cache(cacti_sizeof($partitions) ? $tables : []); print sprintf('Cached Partitions Still Accurate:%d, Stale Partitions Removed:%d', $cache['kept'], $cache['removed']) . PHP_EOL; @@ -533,9 +537,10 @@ function display_help(): void { print PHP_EOL . 'usage: materialize_queries.php [--query-id=ID[,ID...]] [--partitions=TABLE[,TABLE...]] [--nat=on|off] [--dry-run] [--status] [--force] [--debug]' . PHP_EOL . PHP_EOL; - print 'Materializes each FlowView query into the parallel query cache for the' . PHP_EOL; - print 'entire range of flow partitions, once with NAT on and once with NAT off.' . PHP_EOL; - print 'Partitions that are already materialized and still accurate are skipped.' . PHP_EOL; + print 'Materializes each FlowView query into the parallel query cache for every' . PHP_EOL; + print 'flow partition that has rotated, once with NAT on and once with NAT off.' . PHP_EOL; + print 'The partition still receiving flows is never cached. Partitions that are' . PHP_EOL; + print 'already materialized and still accurate are skipped.' . PHP_EOL; print 'Only one run may be active at a time, for up to 8 hours.' . PHP_EOL . PHP_EOL; print 'Options:' . PHP_EOL; diff --git a/includes/functions.php b/includes/functions.php index a931d67..d97972c 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -4399,6 +4399,68 @@ function flowview_get_partition_tables(): array { return $partitions; } +/** + * flowview_get_live_partition - The raw partition table that is still + * receiving flows, which is the newest one. + * + * @return string The table, or an empty string when there are no partitions + */ +function flowview_get_live_partition(): string { + $tables = flowview_get_partition_tables(); + + return cacti_sizeof($tables) ? (string) end($tables) : ''; +} + +/** + * flowview_is_full_scan - Whether a shard scans the whole of its partition, + * which is when the range starts before the earliest flow in it and ends + * after the last. Such a shard does not depend on the range, so its result + * is cached and reused by any query that covers the partition. + * + * The live partition is never a full scan. It is still receiving flows, so + * a result cached for it would be wrong as soon as more arrive. + * + * @param int $start The start of the range + * @param int $end The end of the range + * @param int $partition_start The time of the earliest flow in the partition + * @param int $partition_end The time of the latest flow in the partition + * @param bool $live Whether it is the live partition + * + * @return bool + */ +function flowview_is_full_scan(int $start, int $end, int $partition_start, int $partition_end, bool $live): bool { + return !$live && $start < $partition_start && $end > $partition_end; +} + +/** + * flowview_remove_live_partition_cache - Remove anything cached for the live + * partition, from before it was kept out of the cache. It would never be + * used, as the live partition is no longer scanned as a full scan, but it + * would be counted as materialized. + * + * @return int How many cached results were removed + */ +function flowview_remove_live_partition_cache(): int { + $live = flowview_get_live_partition(); + + if ($live == '') { + return 0; + } + + $cached = (int) flowview_db_fetch_cell_prepared('SELECT COUNT(*) + FROM parallel_database_query_shard_cache + WHERE map_table = ?', + [$live]); + + if ($cached > 0) { + flowview_db_execute_prepared('DELETE FROM parallel_database_query_shard_cache + WHERE map_table = ?', + [$live]); + } + + return $cached; +} + /** * flowview_get_cold_partitions - The partitions that have rotated. The * newest partition is the one still receiving flows, every older one is cold. @@ -5164,6 +5226,7 @@ function parallel_database_query_request($tables, $stru_inner, $stru_outer) { $start = $sql_start_time; $end = $sql_end_time; $index = 0; + $live = flowview_get_live_partition(); foreach($tables as $table => $details) { if (!empty($details['min_date'])) { @@ -5181,11 +5244,7 @@ function parallel_database_query_request($tables, $stru_inner, $stru_outer) { $fsql_where = ''; $fsql_params = []; - if ($start < $start_time && $end > $end_time) { - $full_scan = true; - } else { - $full_scan = false; - } + $full_scan = flowview_is_full_scan($start, $end, $start_time, $end_time, $table == $live); if (!$full_scan) { $fsql_where = $stru_inner['sql_where'] . ($stru_inner['sql_where'] != '' ? ' AND ':'WHERE ') . $stru_inner['sql_range']; diff --git a/tests/Unit/CoreFunctionsTest.php b/tests/Unit/CoreFunctionsTest.php index 948e749..db059a6 100644 --- a/tests/Unit/CoreFunctionsTest.php +++ b/tests/Unit/CoreFunctionsTest.php @@ -1246,3 +1246,66 @@ function number_format_i18n($number, $decimals = 0) { expect(flowview_get_colored_running_queries($count))->toBe(''); })->with(array(0, -1)); }); + +describe('keeping the live partition out of the cache', function () { + it('scans the whole of a partition only when the range covers all of it', function ($start, $end, $live, $expected) { + // the partition holds flows from 1000 to 2000 + expect(flowview_is_full_scan($start, $end, 1000, 2000, $live))->toBe($expected); + })->with(array( + 'covers it' => array(500, 3000, false, true), + 'starts after its first flow' => array(1500, 3000, false, false), + 'ends before its last flow' => array(500, 1500, false, false), + 'starts at its first flow' => array(1000, 3000, false, false), + 'ends at its last flow' => array(500, 2000, false, false), + 'covers it but is live' => array(500, 3000, true, false), + )); + + it('finds the live partition as the newest one', function () { + plugin_test_queue_db_result('db_fetch_assoc', array( + array('table' => 'plugin_flowview_raw_202400105'), + array('table' => 'plugin_flowview_raw_202400107'), + array('table' => 'plugin_flowview_raw_202400106'), + array('table' => 'plugin_flowview_raw_abc'), + )); + + expect(flowview_get_live_partition())->toBe('plugin_flowview_raw_202400107'); + }); + + it('has no live partition when there are no partitions', function () { + expect(flowview_get_live_partition())->toBe(''); + }); + + it('removes what was cached for the live partition', function () { + plugin_test_queue_db_result('db_fetch_assoc', array( + array('table' => 'plugin_flowview_raw_2024001'), + array('table' => 'plugin_flowview_raw_2024002'), + )); + plugin_test_queue_db_result('db_fetch_cell_prepared', 3); + + expect(flowview_remove_live_partition_cache())->toBe(3); + + $deleted = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute_prepared' && strpos($call['sql'], 'DELETE FROM parallel_database_query_shard_cache') === 0; + })); + + expect($deleted)->toHaveCount(1) + ->and($deleted[0]['params'])->toBe(array('plugin_flowview_raw_2024002')); + }); + + it('removes nothing when the live partition has nothing cached', function () { + plugin_test_queue_db_result('db_fetch_assoc', array(array('table' => 'plugin_flowview_raw_2024001'))); + plugin_test_queue_db_result('db_fetch_cell_prepared', 0); + + expect(flowview_remove_live_partition_cache())->toBe(0) + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['fn'], 'db_execute') === 0; + }))->toBeEmpty(); + }); + + it('does nothing without any partitions', function () { + expect(flowview_remove_live_partition_cache())->toBe(0) + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['sql'] ?? '', 'parallel_database_query_shard_cache') !== false; + }))->toBeEmpty(); + }); +}); From bbc3fb24b9164ef13ff03c5aa9208a611b7d2c12 Mon Sep 17 00:00:00 2001 From: TheWitness Date: Wed, 21 Oct 2026 08:54:49 -0400 Subject: [PATCH 17/37] fix: Scan the live partition once per report, not once per panel The live partition is no longer cached, so the table and every chart of a report scanned it again with the same map query. Workers running the same map query and range against it now take a named lock and the first keeps its result for 60 seconds for the others, under map_partition 'live' so it is never counted as materialized. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 2 + README.md | 3 +- includes/functions.php | 145 +++++++++++++++++++++++++++++-- tests/Unit/CoreFunctionsTest.php | 115 ++++++++++++++++++++++++ 4 files changed, 259 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82b5ff1..92c61cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ --- develop --- +* bug: Scan the partition that is still receiving flows once per report, not once per panel. Since it is no longer cached, the table and each chart of a report page scanned it again with the same query. The workers that run the same map query, with the same range, against it now take a named lock, and the first one keeps its result for 60 seconds for the others. It is kept in parallel_database_query_shard_cache under map_partition 'live', never as a full scan, so it is never counted as materialized, is not checked by the stale cache cleanup, and is purged by the poller once it is a minute old. A report can therefore show live data up to 60 seconds old + * bug: Never cache the partition that is still receiving flows. A full run included it, and as its flows ended before the end of the range it was scanned as a full scan and its result was cached, which was wrong as soon as more flows arrived, and counted it as materialized. Whether a shard is a full scan is now decided in flowview_is_full_scan(), which refuses the newest partition, so a report that covers it cannot cache it either, and it is scanned for the range as it is asked for. cli/materialize_queries.php only materializes the partitions that have rotated, and removes what was cached for the live partition before, so NAT Shards and Non-NAT Shards can no longer equal the number of Partitions * bug: Make the earliest partition, and the partitions that rotate, get cached by a materialization. A shard is only cached when the range starts before the earliest flow in the partition, and a flow can start before the window of the partition it was written to, but the range was padded by only a second, so the oldest partition, or a rotated one on its own, was never cached and its Filter stayed Required. The start is now padded by a day, which only adds partitions that already exist and are cached, and the summary now shows the first and last partition instead of the padded range. Run the materializer again to cache what was missed diff --git a/README.md b/README.md index 3733bce..5b22ce3 100644 --- a/README.md +++ b/README.md @@ -214,7 +214,8 @@ php cli/materialize_queries.php The script runs every saved query over every flow partition that has rotated, once with NAT on and once with NAT off. The partition still receiving flows is never cached, as a result cached for it would be wrong as soon as more flows -arrive. Use `--query-id=ID[,ID...]` or +arrive. Instead, the table and charts of one report share a single scan of it, +whose result is reused for 60 seconds and then discarded. Use `--query-id=ID[,ID...]` or `--nat=on|off` to narrow the run, and `--dry-run` to see what would be run. Partitions that are already cached and still match their data are skipped, while partitions that have received more flows since, or have been dropped, diff --git a/includes/functions.php b/includes/functions.php index d97972c..709a933 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -23,6 +23,16 @@ +-------------------------------------------------------------------------+ */ +// The live partition is never materialized. The workers of one report page +// share a single scan of it for a short time, kept in the shard cache table +// under this marker in map_partition, behind a named lock. +if (!defined('FLOWVIEW_LIVE_SHARD_MARKER')) { + define('FLOWVIEW_LIVE_SHARD_MARKER', 'live'); + define('FLOWVIEW_LIVE_SHARD_TTL', 60); + define('FLOWVIEW_LIVE_SHARD_LOCK', 'flowview_live_'); + define('FLOWVIEW_LIVE_SHARD_LOCK_WAIT', 120); +} + // reports_run()/reports_queue()/reports_log_and_notify() became native in // Cacti 1.3. Whether core will declare them is fixed by the Cacti version at // bootstrap, so gate the pre-1.3 shims on that rather than on function_exists(): @@ -4234,11 +4244,13 @@ function flowview_invalidate_stale_shard_cache(array $only = []): array { $cache = flowview_db_fetch_assoc_prepared('SELECT md5sum, map_table, map_partition, min_date, max_date FROM parallel_database_query_shard_cache WHERE map_table IN (' . implode(', ', array_fill(0, cacti_sizeof($only), '?')) . ') + AND map_partition != \'' . FLOWVIEW_LIVE_SHARD_MARKER . '\' ORDER BY map_table', array_values($only)); } else { $cache = flowview_db_fetch_assoc('SELECT md5sum, map_table, map_partition, min_date, max_date FROM parallel_database_query_shard_cache + WHERE map_partition != \'' . FLOWVIEW_LIVE_SHARD_MARKER . '\' ORDER BY map_table'); } @@ -4449,18 +4461,118 @@ function flowview_remove_live_partition_cache(): int { $cached = (int) flowview_db_fetch_cell_prepared('SELECT COUNT(*) FROM parallel_database_query_shard_cache - WHERE map_table = ?', + WHERE map_table = ? + AND map_partition != \'' . FLOWVIEW_LIVE_SHARD_MARKER . '\'', [$live]); if ($cached > 0) { flowview_db_execute_prepared('DELETE FROM parallel_database_query_shard_cache - WHERE map_table = ?', + WHERE map_table = ? + AND map_partition != \'' . FLOWVIEW_LIVE_SHARD_MARKER . '\'', [$live]); } return $cached; } +/** + * flowview_live_shard_key - The key a live partition shard result is shared + * under. Requests that run the same map query, with the same range, against + * the same partition get the same key, such as the table and the charts of a + * single report page. It is derived from the map query, so it never matches + * the key of a materialized shard. + * + * @param array $query The parallel query request + * @param array $shard The shard of the live partition + * + * @return string + */ +function flowview_live_shard_key(array $query, array $shard): string { + return md5($query['md5sum_tables'] . '|' . $shard['map_table'] . '|' . $shard['map_params']); +} + +/** + * flowview_live_shard_lock - Take the lock that makes the workers of one + * live shard key run the map query one at a time, so the rest can reuse the + * result of the first. + * + * @param string $key The live shard key + * + * @return bool Whether the lock was taken within the wait time + */ +function flowview_live_shard_lock(string $key): bool { + return (int) flowview_db_fetch_cell_prepared('SELECT GET_LOCK(?, ?)', + [FLOWVIEW_LIVE_SHARD_LOCK . $key, FLOWVIEW_LIVE_SHARD_LOCK_WAIT]) === 1; +} + +/** + * flowview_live_shard_unlock - Release the lock of flowview_live_shard_lock(). + * + * @param string $key The live shard key + * + * @return void + */ +function flowview_live_shard_unlock(string $key): void { + flowview_db_fetch_cell_prepared('SELECT RELEASE_LOCK(?)', [FLOWVIEW_LIVE_SHARD_LOCK . $key]); +} + +/** + * flowview_live_shard_get - The result of a live shard that was run in the + * last FLOWVIEW_LIVE_SHARD_TTL seconds. + * + * @param string $key The live shard key + * @param string $table The live partition table + * + * @return array|null null when there is no recent result + */ +function flowview_live_shard_get(string $key, string $table): ?array { + $row = flowview_db_fetch_row_prepared('SELECT results + FROM parallel_database_query_shard_cache + WHERE md5sum = ? + AND map_table = ? + AND map_partition = ? + AND date_created >= DATE_SUB(NOW(), INTERVAL ? SECOND)', + [$key, $table, FLOWVIEW_LIVE_SHARD_MARKER, FLOWVIEW_LIVE_SHARD_TTL]); + + if (!cacti_sizeof($row)) { + return null; + } + + $results = json_decode($row['results'], true); + + return is_array($results) ? $results : null; +} + +/** + * flowview_live_shard_save - Keep the result of a live shard for + * FLOWVIEW_LIVE_SHARD_TTL seconds. + * + * @param string $key The live shard key + * @param string $table The live partition table + * @param array $results The result of the map query + * + * @return void + */ +function flowview_live_shard_save(string $key, string $table, array $results): void { + flowview_db_execute_prepared('REPLACE INTO parallel_database_query_shard_cache + (md5sum, map_table, map_partition, min_date, max_date, results, date_created) + VALUES (?, ?, ?, NOW(), NOW(), ?, NOW())', + [$key, $table, FLOWVIEW_LIVE_SHARD_MARKER, json_encode($results)]); +} + +/** + * flowview_live_shard_expire - Remove the live shard results that are no + * longer reusable. + * + * @return void + */ +function flowview_live_shard_expire(): void { + flowview_db_execute_prepared('DELETE FROM parallel_database_query_shard_cache + WHERE map_partition = ? + AND date_created < DATE_SUB(NOW(), INTERVAL ? SECOND)', + [FLOWVIEW_LIVE_SHARD_MARKER, FLOWVIEW_LIVE_SHARD_TTL]); +} + /** * flowview_get_cold_partitions - The partitions that have rotated. The * newest partition is the one still receiving flows, every older one is cold. @@ -5277,7 +5389,7 @@ function parallel_database_query_request($tables, $stru_inner, $stru_outer) { $index, $full_scan ? '1':'0', $details['table_name'], - $details['table_partition'], + ($table == $live && !$full_scan) ? FLOWVIEW_LIVE_SHARD_MARKER : $details['table_partition'], $map_query, json_encode($fsql_params) ) @@ -5605,6 +5717,8 @@ function parallel_database_query_create_reduce_table($request_id, $sql_query, $t function parallel_database_query_expire(): void { $ttl = read_config_option('flowview_parallel_time_to_live'); + flowview_live_shard_expire(); + if (empty($ttl)) { set_config_option('flowview_parallel_time_to_live', '21600'); $ttl = 21600; @@ -5928,9 +6042,26 @@ function parallel_database_child_runner($query_id, $shard_id): void { $exists = []; } - if (!cacti_sizeof($exists)) { + $live_key = ''; + $live_locked = false; + $live_reused = false; + + if (!cacti_sizeof($exists) && $shard['map_partition'] == FLOWVIEW_LIVE_SHARD_MARKER) { + $live_key = flowview_live_shard_key($query, $shard); + $live_locked = flowview_live_shard_lock($live_key); + $results = flowview_live_shard_get($live_key, $shard['map_table']); + $live_reused = $results !== null; + } + + if (!cacti_sizeof($exists) && !$live_reused) { $results = flowview_db_fetch_assoc_prepared("{$shard['map_query']}", json_decode($shard['map_params']), false, $max_cnn); - } else { + + if ($live_key != '') { + if (is_array($results)) { + flowview_live_shard_save($live_key, $shard['map_table'], $results); + } + } + } elseif (cacti_sizeof($exists)) { flowview_db_execute_prepared('UPDATE parallel_database_query SET cached_shards = cached_shards + 1 WHERE id = ?', @@ -5939,6 +6070,10 @@ function parallel_database_child_runner($query_id, $shard_id): void { $results = json_decode($exists['results'], true); } + if ($live_locked) { + flowview_live_shard_unlock($live_key); + } + if ($shard['full_scan'] == 1 && !cacti_sizeof($exists)) { $details = flowview_db_fetch_row("SELECT MIN(start_time) AS min_date, MAX(end_time) AS max_date FROM {$shard['map_table']}"); diff --git a/tests/Unit/CoreFunctionsTest.php b/tests/Unit/CoreFunctionsTest.php index db059a6..58e9e0c 100644 --- a/tests/Unit/CoreFunctionsTest.php +++ b/tests/Unit/CoreFunctionsTest.php @@ -1308,4 +1308,119 @@ function number_format_i18n($number, $decimals = 0) { return strpos($call['sql'] ?? '', 'parallel_database_query_shard_cache') !== false; }))->toBeEmpty(); }); + + it('keeps the short lived live shard results out of the removal and the staleness checks', function () { + plugin_test_queue_db_result('db_fetch_assoc', array(array('table' => 'plugin_flowview_raw_2024001'))); + plugin_test_queue_db_result('db_fetch_cell_prepared', 1); + + flowview_remove_live_partition_cache(); + flowview_invalidate_stale_shard_cache(); + flowview_invalidate_stale_shard_cache(array('plugin_flowview_raw_2024001')); + + $selects = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['sql'] ?? '', 'FROM parallel_database_query_shard_cache') !== false; + })); + + expect($selects)->not->toBeEmpty(); + + foreach ($selects as $call) { + expect($call['sql'])->toContain("map_partition != 'live'"); + } + }); +}); + +describe('sharing one scan of the live partition', function () { + beforeEach(function () { + plugin_test_reset(); + }); + + $query = array('md5sum_tables' => 'abc'); + $shard = array('map_table' => 'plugin_flowview_raw_2024001', 'map_params' => '["x"]'); + + it('derives one key for the same map query, table and range', function () use ($query, $shard) { + $key = flowview_live_shard_key($query, $shard); + + expect($key)->toHaveLength(32) + ->and(flowview_live_shard_key($query, $shard))->toBe($key) + ->and(flowview_live_shard_key(array('md5sum_tables' => 'def'), $shard))->not->toBe($key) + ->and(flowview_live_shard_key($query, array_merge($shard, array('map_params' => '["y"]'))))->not->toBe($key) + ->and(flowview_live_shard_key($query, array_merge($shard, array('map_table' => 'plugin_flowview_raw_2024002'))))->not->toBe($key) + ->and($key)->not->toBe('abc'); + }); + + it('takes and releases a named lock on that key', function () { + plugin_test_queue_db_result('db_fetch_cell_prepared', '1'); + plugin_test_queue_db_result('db_fetch_cell_prepared', '1'); + + expect(flowview_live_shard_lock('k1'))->toBeTrue(); + + flowview_live_shard_unlock('k1'); + + $calls = array_values($GLOBALS['__test_db_calls']); + + expect($calls[0]['sql'])->toContain('GET_LOCK') + ->and($calls[0]['params'])->toBe(array('flowview_live_k1', FLOWVIEW_LIVE_SHARD_LOCK_WAIT)) + ->and($calls[1]['sql'])->toContain('RELEASE_LOCK') + ->and($calls[1]['params'])->toBe(array('flowview_live_k1')); + }); + + it('reports a lock that was not obtained in time', function () { + plugin_test_queue_db_result('db_fetch_cell_prepared', '0'); + + expect(flowview_live_shard_lock('k1'))->toBeFalse(); + }); + + it('reuses a recent live result', function () { + plugin_test_queue_db_result('db_fetch_row_prepared', array('results' => json_encode(array(array('bytes' => 5))))); + + expect(flowview_live_shard_get('k1', 'plugin_flowview_raw_2024001'))->toBe(array(array('bytes' => 5))); + + $call = $GLOBALS['__test_db_calls'][0]; + + expect($call['params'])->toBe(array('k1', 'plugin_flowview_raw_2024001', 'live', FLOWVIEW_LIVE_SHARD_TTL)) + ->and($call['sql'])->toContain('date_created >= DATE_SUB(NOW(), INTERVAL ? SECOND)'); + }); + + it('reuses an empty live result', function () { + plugin_test_queue_db_result('db_fetch_row_prepared', array('results' => '[]')); + + expect(flowview_live_shard_get('k1', 'plugin_flowview_raw_2024001'))->toBe(array()); + }); + + it('has nothing to reuse without a recent live result', function () { + expect(flowview_live_shard_get('k1', 'plugin_flowview_raw_2024001'))->toBeNull(); + }); + + it('has nothing to reuse when the stored result is unreadable', function () { + plugin_test_queue_db_result('db_fetch_row_prepared', array('results' => 'not json')); + + expect(flowview_live_shard_get('k1', 'plugin_flowview_raw_2024001'))->toBeNull(); + }); + + it('keeps a live result under the live marker', function () { + flowview_live_shard_save('k1', 'plugin_flowview_raw_2024001', array(array('bytes' => 5))); + + $call = $GLOBALS['__test_db_calls'][0]; + + expect($call['sql'])->toStartWith('REPLACE INTO parallel_database_query_shard_cache') + ->and($call['params'])->toBe(array('k1', 'plugin_flowview_raw_2024001', 'live', json_encode(array(array('bytes' => 5))))); + }); + + it('removes only expired live results', function () { + flowview_live_shard_expire(); + + $call = $GLOBALS['__test_db_calls'][0]; + + expect($call['sql'])->toContain('map_partition = ?') + ->and($call['sql'])->toContain('date_created < DATE_SUB(NOW(), INTERVAL ? SECOND)') + ->and($call['params'])->toBe(array('live', FLOWVIEW_LIVE_SHARD_TTL)); + }); + + it('expires them with the poller clean up of parallel queries', function () { + parallel_database_query_expire(); + + $sql = array_column($GLOBALS['__test_db_calls'], 'sql'); + + expect(array_filter($sql, fn ($s) => strpos($s, "DELETE FROM parallel_database_query_shard_cache") === 0))->toHaveCount(1); + }); }); From a38abb01cbe300b981fa4c9e38a3d7ae06a95662 Mon Sep 17 00:00:00 2001 From: TheWitness Date: Wed, 21 Oct 2026 09:09:41 -0400 Subject: [PATCH 18/37] fix: Keep cached partitions until dropped, scope Running to the filters materialized, convert every cold partition The Go query service aged out every cached partition after the Cached Data Time to Live, emptying the NAT and non-NAT shards overnight, so it only expires the short lived live shard results now. The Filters page showed every filter as Running while any materialization ran, so it uses the filters the run is for. The poller only converted the 2nd to 4th newest partitions, so any it missed stayed InnoDB, so it converts every cold partition, three per cycle and one converter at a time. flowview_runner.php now loads setup.php itself instead of relying on the plugin hooks having loaded it. The bug entries of this session are removed from the CHANGELOG as nothing has been released. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 20 --- cli/materialize_queries.php | 4 + flowview_filters.php | 7 +- flowview_process.php | 22 +--- flowview_runner.php | 2 + includes/functions.php | 103 +++++++++++++++ service/flowview-query/README.md | 14 +- service/flowview-query/scheduler.go | 24 ++-- service/flowview-query/scheduler_test.go | 20 ++- tests/Unit/CoreFunctionsTest.php | 155 +++++++++++++++++++++++ 10 files changed, 312 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92c61cc..6bb30c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,30 +2,12 @@ --- develop --- -* bug: Scan the partition that is still receiving flows once per report, not once per panel. Since it is no longer cached, the table and each chart of a report page scanned it again with the same query. The workers that run the same map query, with the same range, against it now take a named lock, and the first one keeps its result for 60 seconds for the others. It is kept in parallel_database_query_shard_cache under map_partition 'live', never as a full scan, so it is never counted as materialized, is not checked by the stale cache cleanup, and is purged by the poller once it is a minute old. A report can therefore show live data up to 60 seconds old - -* bug: Never cache the partition that is still receiving flows. A full run included it, and as its flows ended before the end of the range it was scanned as a full scan and its result was cached, which was wrong as soon as more flows arrived, and counted it as materialized. Whether a shard is a full scan is now decided in flowview_is_full_scan(), which refuses the newest partition, so a report that covers it cannot cache it either, and it is scanned for the range as it is asked for. cli/materialize_queries.php only materializes the partitions that have rotated, and removes what was cached for the live partition before, so NAT Shards and Non-NAT Shards can no longer equal the number of Partitions - -* bug: Make the earliest partition, and the partitions that rotate, get cached by a materialization. A shard is only cached when the range starts before the earliest flow in the partition, and a flow can start before the window of the partition it was written to, but the range was padded by only a second, so the oldest partition, or a rotated one on its own, was never cached and its Filter stayed Required. The start is now padded by a day, which only adds partitions that already exist and are cached, and the summary now shows the first and last partition instead of the padded range. Run the materializer again to cache what was missed - * feature: Say which partitions a Filter is not materialized for, with NAT and without, in the tip of its Required and Running pill -* bug: Fix the Filters page failing with "3 arguments are required, 2 given" once a query was running. The Running: N pill and its tip, and the Parallel Query timeout message, passed the text domain where Cacti's __() expects the values to put in the text, so the tip crashed, the pill would have read "Running: flowview", and the timeout would have said 0 seconds. The values now come before the domain, as elsewhere in the plugin - -* bug: Stop cli/materialize_queries.php from reporting each ALTER TABLE twice, once from its own progress line and once from the log message that is also printed. The alter is still written to the Cacti log, and the schema upgrade, which has no other report, still shows it. The logged text no longer has the extra space in it - * feature: Show the number of queries of each Filter that are running now in a second pill, in the style of a disabled device and reading Running: N, beside the status pill on the Filters page, with how many are with NAT and without in its tip. The process table only has the id of a running request, so a running query is matched to a Filter by the cache key its map query is stored under, which is worked out from the Filter, and there is no pill when none are running * feature: Add --status to cli/materialize_queries.php to list the parallel queries that are running and the Filter each one is for, with its progress -* bug: Show the Running status on the Filters page while a parallel query is still working, not only while cli/materialize_queries.php itself is registered. The query keeps going after the script has ended, for instance when it timed out, so the status now comes from the runner and shard worker (db_query_ and db_shard_) registrations in the process table, whoever started them. A registration counts while it is recent, or when its process is still alive, so a row left behind by a killed process does not. A new materialization is not started while one is running - -* bug: Make the Parallel Query timeout actually stop the runner and shard workers. They register as db_query_ and db_shard_, but the cleanup, and the runner's own termination handler, looked for db_query and db_shard, so they were never found: a timed out query left its processes running and a terminated one left its row in the process table - -* bug: Stop the poller from starting the same ALTER TABLE ENGINE conversion on a partition every cycle. Until a long conversion finishes the partition still reports its old engine, so each cycle started another that waited for the first and then rebuilt the table again, which also held up every query scanning that partition. The conversion is skipped while an ALTER TABLE is already running on the partition, and cli/materialize_queries.php checks a partition again just before altering it, so it is never rebuilt twice - -* bug: Make a Parallel Query that reaches the Maximum Run Time report it in words when run from a script, such as cli/materialize_queries.php, instead of printing the JavaScript meant for the web page; the message is also written to the Cacti log, a script now exits with a failure, and the web page is unchanged - * feature: Have cli/materialize_queries.php get the partitions ready before it starts the Parallel Query. It waits for an ALTER TABLE that is already running on a partition it needs, such as from the schema upgrade, since a query waits for it and then ends at the Maximum Run Time, and then does the alters that are still due itself, one partition at a time with progress shown. The partition still receiving flows is left alone. It also notes the partitions that have no NAT columns, as they have no NAT data. What a partition needs altered is now decided in one place, flowview_get_partition_alter(), which the schema upgrade also uses, so the ALTER it issues is unchanged * feature: Materialize the partitions that have gone cold automatically. Each poller cycle flowview_process.php records the newest partition, which is the one still receiving flows, and starts cli/materialize_queries.php in the background for the partitions that have rotated since the previous cycle, for every Filter with NAT on and off. Partitions that are already materialized and accurate are skipped. When a materialization is already running, or the Parallel Query is not enabled, the record is left alone so a later cycle catches up. The first cycle only records the newest partition, so older partitions are still materialized with cli/materialize_queries.php. The script gained a --partitions option for this @@ -42,8 +24,6 @@ * dev: Move fixprivate.php, import_flows.php and repair_tables.php into the plugin's cli/ directory (listed as tombstones in manifest.json so upgrades remove the old copies); they now locate the Cacti root relative to their own location -* bug: Fix "Uncaught ReferenceError: timeshiftFilterLeft is not defined" when using the Shift Time Backward/Forward icons on the flow report filter; the plugin relied on undefined global functions, so the icons now call a plugin-owned timeshiftFilter() that shifts the From/To range by the selected interval through a new `timeshift` action and reloads the report - * fix: Make bundled Net_DNS2 constructor nullability explicit for PHP 8.4 without changing accepted arguments * issue: Shorten the graph drilldown button title from 'View NetFlow Traffic In Range' to 'NetFlow' so it reads cleanly as a label in Cacti's graph action menu diff --git a/cli/materialize_queries.php b/cli/materialize_queries.php index a464512..4cec5d2 100644 --- a/cli/materialize_queries.php +++ b/cli/materialize_queries.php @@ -181,8 +181,12 @@ } register_shutdown_function(function () { + flowview_set_materialization_scope([]); unregister_process('flowview', 'materialize', 0); }); + + /* the Filters page shows only these filters as running */ + flowview_set_materialization_scope($query_ids); } $threads = read_config_option('flowview_parallel_threads'); diff --git a/flowview_filters.php b/flowview_filters.php index 32a7daf..813ac54 100644 --- a/flowview_filters.php +++ b/flowview_filters.php @@ -437,14 +437,17 @@ function clearFilter() { $tables = flowview_get_partition_tables(); $cold = flowview_get_cold_partitions($tables); $partitions = cacti_sizeof($tables); - $running = flowview_materialization_running(); + $scope = flowview_get_materialization_scope(); $requests = flowview_get_running_requests(); foreach ($filters as $filter) { $md5sums = flowview_get_filter_md5sums((int) $filter['id']); $cached = flowview_get_cached_shard_tables((int) $filter['id'], $md5sums); - $status = flowview_materialization_status($cached['nat'], $cached['non_nat'], $cold, $running); $queries = flowview_count_running_queries($md5sums, $requests); + + /* only the filters being materialized, or with a query running, are running */ + $running = flowview_filter_is_materializing((int) $filter['id'], $scope) || $queries > 0; + $status = flowview_materialization_status($cached['nat'], $cached['non_nat'], $cold, $running); $missing = flowview_get_missing_partitions($cached['nat'], $cached['non_nat'], $cold); form_alternate_row('line' . $filter['id'], true); diff --git a/flowview_process.php b/flowview_process.php index a3d6f45..9ef0863 100644 --- a/flowview_process.php +++ b/flowview_process.php @@ -174,23 +174,11 @@ } } -$raw_engine = get_set_default_fast_engine(); - -$last_tables = flowview_db_fetch_assoc('SELECT TABLE_NAME, ENGINE - FROM information_schema.TABLES - WHERE TABLE_NAME LIKE "plugin_flowview_raw_%" - ORDER BY TABLE_NAME DESC - LIMIT 1, 3'); - -if (cacti_sizeof($last_tables)) { - foreach($last_tables as $table) { - if ($table['ENGINE'] != $raw_engine) { - /* a conversion from an earlier cycle can still be running, and must not be started again */ - if (!flowview_convert_partition_engine($table['TABLE_NAME'], $raw_engine)) { - flowview_debug("An ALTER TABLE is already running on {$table['TABLE_NAME']}, skipping the engine conversion"); - } - } - } +$raw_engine = get_set_default_fast_engine(); + +/* every partition that has rotated, so ones missed while the poller was not running are caught up */ +foreach (flowview_convert_cold_partitions($raw_engine) as $table) { + flowview_debug("Converted partition {$table} to {$raw_engine}"); } set_config_option('flowview_last_sequence', $sequence); diff --git a/flowview_runner.php b/flowview_runner.php index c2970bc..083f049 100644 --- a/flowview_runner.php +++ b/flowview_runner.php @@ -37,6 +37,8 @@ global $config; require_once($config['base_path'] . '/lib/poller.php'); +require_once($config['base_path'] . '/plugins/flowview/setup.php'); +require_once($config['base_path'] . '/plugins/flowview/includes/functions.php'); flowview_connect(); diff --git a/includes/functions.php b/includes/functions.php index 709a933..2d4fb07 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -4904,6 +4904,58 @@ function flowview_materialization_running(): bool { return cacti_sizeof(flowview_get_running_requests()) > 0; } +/** + * flowview_set_materialization_scope - Record which filters the + * materialization that is starting is for, so the Filters page only shows + * those as running. It only counts while the materialization is registered + * as running. + * + * @param array $query_ids The FlowView query ids, or none for all of them + * + * @return void + */ +function flowview_set_materialization_scope(array $query_ids): void { + set_config_option('flowview_materialize_queries', implode(',', array_map('intval', $query_ids))); +} + +/** + * flowview_get_materialization_scope - Which filters the registered + * materialization is for. + * + * @return array|null null when no materialization is registered, an empty + * array when it is for every filter, otherwise the ids of + * the filters it is for + */ +function flowview_get_materialization_scope(): ?array { + $registered = db_fetch_cell_prepared('SELECT COUNT(*) + FROM processes + WHERE tasktype = ? + AND taskname = ? + AND taskid = ?', + ['flowview', 'materialize', 0]); + + if ($registered < 1) { + return null; + } + + $ids = array_filter(array_map('intval', explode(',', (string) read_config_option('flowview_materialize_queries', true)))); + + return array_values(array_unique($ids)); +} + +/** + * flowview_filter_is_materializing - Whether the registered materialization + * is working on a filter. + * + * @param int $query_id The FlowView query + * @param array|null $scope From flowview_get_materialization_scope() + * + * @return bool + */ +function flowview_filter_is_materializing(int $query_id, ?array $scope): bool { + return $scope !== null && (!cacti_sizeof($scope) || in_array($query_id, $scope, true)); +} + /** * flowview_materialization_status - The automatic materialization status of * a filter. It is current once every cold partition is cached with NAT and @@ -5035,6 +5087,57 @@ function flowview_convert_partition_engine(string $table, string $raw_engine): b return true; } +/** + * flowview_convert_cold_partitions - Change the raw partition tables that + * have rotated, and are not yet using the fast engine, to it. Every + * partition that has rotated is looked at, not just the last few, so + * partitions missed while the poller was not running, or while the plugin was + * disabled, are converted as well, newest first. + * + * Only $limit are converted per call, as each ALTER TABLE is run in turn and + * takes a while. Nothing is converted while an ALTER TABLE is running on any + * of them, so the conversions of one poller cycle never pile up on those of + * the previous one. + * + * @param string $raw_engine The engine a partition table that has rotated uses + * @param int $limit The most partitions to convert + * + * @return array The partitions that were converted + */ +function flowview_convert_cold_partitions(string $raw_engine, int $limit = 3): array { + $cold = flowview_get_cold_partitions(flowview_get_partition_tables()); + + if (!cacti_sizeof($cold) || cacti_sizeof(flowview_get_running_partition_alters($cold))) { + return []; + } + + $engines = []; + + foreach (flowview_db_fetch_assoc('SELECT TABLE_NAME, ENGINE + FROM information_schema.TABLES + WHERE TABLE_NAME LIKE "plugin_flowview_raw_%"') as $row) { + $engines[$row['TABLE_NAME']] = $row['ENGINE']; + } + + $converted = []; + + foreach (array_reverse($cold) as $table) { + if (cacti_sizeof($converted) >= $limit) { + break; + } + + if (!isset($engines[$table]) || $engines[$table] == $raw_engine) { + continue; + } + + if (flowview_convert_partition_engine($table, $raw_engine)) { + $converted[] = $table; + } + } + + return $converted; +} + /** * flowview_materialize_rotated_partitions - Called every poller cycle to * materialize the partitions that went cold since the previous cycle, for diff --git a/service/flowview-query/README.md b/service/flowview-query/README.md index f56ad76..2eed6fd 100644 --- a/service/flowview-query/README.md +++ b/service/flowview-query/README.md @@ -87,8 +87,10 @@ spawning processes and polling tables can be seconds. pool; when `flowview_use_maxscale` is on, map (read) queries are routed through the MaxScale read/write-split port so shards round-robin across backends. - **Maintenance off the critical path.** A scheduler goroutine expires queries - past their `time_to_live`, ages out cached partitions, and drops cache rows for - partitions that have been rolled off — the work PHP does inline today. + past their `time_to_live`, drops cache rows for partitions that have been rolled + off, and expires the short lived live-partition results; cached results of a + cold partition are kept until the partition is dropped, the work PHP does + inline today. ### Map-reduce reduction (unchanged contract) @@ -261,9 +263,11 @@ A goroutine runs every `scheduler_interval` (and once at startup): - expires `parallel_database_query` rows past `time_to_live` and drops their intermediary map tables; -- ages out `parallel_database_query_shard_cache` rows older than - `flowview_parallel_time_to_live`; -- drops cache rows whose partition table no longer exists (rolled-off data). +- drops cache rows whose partition table no longer exists (rolled-off data); +- expires the short lived (60 second) live-partition results that the PHP + workers store with `map_partition = 'live'`. Cached results of a cold + partition are never aged out, as the materialization of a filter is built + from them. Deleting a saved query or filter should call `POST /cache/invalidate` with that signature so its cache does not linger until TTL. diff --git a/service/flowview-query/scheduler.go b/service/flowview-query/scheduler.go index e1a68ae..7c54980 100644 --- a/service/flowview-query/scheduler.go +++ b/service/flowview-query/scheduler.go @@ -103,22 +103,28 @@ func (s *Scheduler) expireQueries(ctx context.Context) error { return nil } -// expireShardCache ages out cached partition results older than the configured -// time-to-live so hot partitions stay cached while cold data is reclaimed. -func (s *Scheduler) expireShardCache(ctx context.Context) error { - ttl := int64(s.eng.settings.TimeToLive().Seconds()) - if ttl <= 0 { - return nil - } +// Live partition shard results (map_partition = 'live') are written by the PHP +// workers so the panels of one report share a single scan of the partition that +// is still receiving flows. They are only good for a short time. +const ( + liveShardMarker = "live" + liveShardTTLSeconds = 60 +) +// expireShardCache removes the short lived results of the live partition. +// Cached results of a cold partition are never aged out: they stay until the +// partition is dropped (see dropOrphanCache) or no longer matches its data, +// as the materialization of a filter is built from them. +func (s *Scheduler) expireShardCache(ctx context.Context) error { res, err := s.eng.flow.ExecContext(ctx, `DELETE FROM parallel_database_query_shard_cache - WHERE date_created < (NOW() - INTERVAL ? SECOND)`, ttl) + WHERE map_partition = ? + AND date_created < (NOW() - INTERVAL ? SECOND)`, liveShardMarker, liveShardTTLSeconds) if err != nil { return err } if n, _ := res.RowsAffected(); n > 0 { - logf("scheduler: expired %d cached shards", n) + logf("scheduler: expired %d live shard results", n) } return nil } diff --git a/service/flowview-query/scheduler_test.go b/service/flowview-query/scheduler_test.go index cef34bc..d7dc127 100644 --- a/service/flowview-query/scheduler_test.go +++ b/service/flowview-query/scheduler_test.go @@ -49,30 +49,38 @@ func TestExpireQueriesKeepsRowOnDropFailure(t *testing.T) { } } -func TestExpireShardCache(t *testing.T) { +func TestExpireShardCacheOnlyExpiresLiveResults(t *testing.T) { flow, mock, cleanup := newMock(t) defer cleanup() eng := NewEngine(flow, nil, nil, defaultSettings()) s := &Scheduler{eng: eng} - mock.ExpectExec("DELETE FROM parallel_database_query_shard_cache"). - WithArgs(int64(eng.settings.TimeToLive().Seconds())). + // Only the short lived live partition results go, whatever the time to + // live is: a cold partition stays cached until it is dropped. + mock.ExpectExec("DELETE FROM parallel_database_query_shard_cache\\s+WHERE map_partition = \\?"). + WithArgs("live", liveShardTTLSeconds). WillReturnResult(sqlmock.NewResult(0, 3)) if err := s.expireShardCache(context.Background()); err != nil { t.Fatalf("expireShardCache: %v", err) } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } } -func TestExpireShardCacheSkippedWhenTTLZero(t *testing.T) { - flow, _, cleanup := newMock(t) +func TestExpireShardCacheIgnoresTimeToLive(t *testing.T) { + flow, mock, cleanup := newMock(t) defer cleanup() settings := defaultSettings() settings.timeToLive = 0 eng := NewEngine(flow, nil, nil, settings) s := &Scheduler{eng: eng} - // No expectations: a zero TTL must skip the delete entirely. + mock.ExpectExec("DELETE FROM parallel_database_query_shard_cache"). + WithArgs("live", liveShardTTLSeconds). + WillReturnResult(sqlmock.NewResult(0, 0)) + if err := s.expireShardCache(context.Background()); err != nil { t.Fatalf("expireShardCache: %v", err) } diff --git a/tests/Unit/CoreFunctionsTest.php b/tests/Unit/CoreFunctionsTest.php index 58e9e0c..2775547 100644 --- a/tests/Unit/CoreFunctionsTest.php +++ b/tests/Unit/CoreFunctionsTest.php @@ -1424,3 +1424,158 @@ function number_format_i18n($number, $decimals = 0) { expect(array_filter($sql, fn ($s) => strpos($s, "DELETE FROM parallel_database_query_shard_cache") === 0))->toHaveCount(1); }); }); + +describe('converting the partitions that have rotated', function () { + beforeEach(function () { + plugin_test_reset(); + + $this->partitions = function (array $engines) { + plugin_test_queue_db_result('db_fetch_assoc', array_map(function ($table) { + return array('table' => $table); + }, array_keys($engines))); + }; + + $this->alters = function () { + return array_values(array_column(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute'; + }), 'sql')); + }; + }); + + it('converts every cold partition that is not on the fast engine, newest first, and never the live one', function () { + $engines = array( + 'plugin_flowview_raw_202629214' => 'InnoDB', + 'plugin_flowview_raw_202629215' => 'Aria', + 'plugin_flowview_raw_202629216' => 'InnoDB', + 'plugin_flowview_raw_202629217' => 'InnoDB', + ); + + ($this->partitions)($engines); + plugin_test_queue_db_result('db_fetch_assoc', array()); + plugin_test_queue_db_result('db_fetch_assoc', array_map(function ($table, $engine) { + return array('TABLE_NAME' => $table, 'ENGINE' => $engine); + }, array_keys($engines), $engines)); + + expect(flowview_convert_cold_partitions('Aria'))->toBe(array('plugin_flowview_raw_202629216', 'plugin_flowview_raw_202629214')) + ->and(($this->alters)())->toBe(array( + 'ALTER TABLE plugin_flowview_raw_202629216 ENGINE=Aria', + 'ALTER TABLE plugin_flowview_raw_202629214 ENGINE=Aria', + )); + }); + + it('catches up on partitions that were missed for more than a few hours', function () { + $engines = array(); + + foreach (range(10, 20) as $hour) { + $engines['plugin_flowview_raw_2026292' . $hour] = 'InnoDB'; + } + + ($this->partitions)($engines); + plugin_test_queue_db_result('db_fetch_assoc', array()); + plugin_test_queue_db_result('db_fetch_assoc', array_map(function ($table, $engine) { + return array('TABLE_NAME' => $table, 'ENGINE' => $engine); + }, array_keys($engines), $engines)); + + expect(flowview_convert_cold_partitions('Aria', 100))->toHaveCount(10); + }); + + it('converts only as many as it is allowed to in one go', function () { + $engines = array( + 'plugin_flowview_raw_202629214' => 'InnoDB', + 'plugin_flowview_raw_202629215' => 'InnoDB', + 'plugin_flowview_raw_202629216' => 'InnoDB', + 'plugin_flowview_raw_202629217' => 'InnoDB', + 'plugin_flowview_raw_202629218' => 'InnoDB', + ); + + ($this->partitions)($engines); + plugin_test_queue_db_result('db_fetch_assoc', array()); + plugin_test_queue_db_result('db_fetch_assoc', array_map(function ($table, $engine) { + return array('TABLE_NAME' => $table, 'ENGINE' => $engine); + }, array_keys($engines), $engines)); + + expect(flowview_convert_cold_partitions('Aria', 2))->toBe(array('plugin_flowview_raw_202629217', 'plugin_flowview_raw_202629216')); + }); + + it('does nothing while an alter is running on any cold partition', function () { + ($this->partitions)(array( + 'plugin_flowview_raw_202629214' => 'InnoDB', + 'plugin_flowview_raw_202629215' => 'InnoDB', + 'plugin_flowview_raw_202629216' => 'InnoDB', + )); + plugin_test_queue_db_result('db_fetch_assoc', array( + array('TIME' => 14, 'INFO' => 'ALTER TABLE plugin_flowview_raw_202629214 ENGINE=Aria ROW_FORMAT=page'), + )); + + expect(flowview_convert_cold_partitions('Aria'))->toBe(array()) + ->and(($this->alters)())->toBe(array()); + }); + + it('does nothing when the cold partitions are all on the fast engine', function () { + $engines = array( + 'plugin_flowview_raw_202629214' => 'Aria', + 'plugin_flowview_raw_202629215' => 'Aria', + 'plugin_flowview_raw_202629216' => 'InnoDB', + ); + + ($this->partitions)($engines); + plugin_test_queue_db_result('db_fetch_assoc', array()); + plugin_test_queue_db_result('db_fetch_assoc', array_map(function ($table, $engine) { + return array('TABLE_NAME' => $table, 'ENGINE' => $engine); + }, array_keys($engines), $engines)); + + expect(flowview_convert_cold_partitions('Aria'))->toBe(array()) + ->and(($this->alters)())->toBe(array()); + }); + + it('does nothing when there is no cold partition', function () { + ($this->partitions)(array('plugin_flowview_raw_202629216' => 'InnoDB')); + + expect(flowview_convert_cold_partitions('Aria'))->toBe(array()); + }); +}); + +describe('showing only the filters being materialized as running', function () { + beforeEach(function () { + plugin_test_reset(); + }); + + it('records the filters a materialization is for, or all of them', function () { + flowview_set_materialization_scope(array(3, '7')); + + expect(read_config_option('flowview_materialize_queries'))->toBe('3,7'); + + flowview_set_materialization_scope(array()); + + expect(read_config_option('flowview_materialize_queries'))->toBe(''); + }); + + it('has no scope when no materialization is registered', function () { + set_config_option('flowview_materialize_queries', '3'); + plugin_test_queue_db_result('db_fetch_cell_prepared', 0); + + expect(flowview_get_materialization_scope())->toBeNull(); + }); + + it('is for the filters that were chosen while it is registered', function () { + set_config_option('flowview_materialize_queries', '3,7,3'); + plugin_test_queue_db_result('db_fetch_cell_prepared', 1); + + expect(flowview_get_materialization_scope())->toBe(array(3, 7)); + }); + + it('is for every filter when none were chosen', function () { + set_config_option('flowview_materialize_queries', ''); + plugin_test_queue_db_result('db_fetch_cell_prepared', 1); + + expect(flowview_get_materialization_scope())->toBe(array()); + }); + + it('marks only the chosen filters as being materialized', function () { + expect(flowview_filter_is_materializing(3, array(3, 7)))->toBeTrue() + ->and(flowview_filter_is_materializing(7, array(3, 7)))->toBeTrue() + ->and(flowview_filter_is_materializing(1, array(3, 7)))->toBeFalse() + ->and(flowview_filter_is_materializing(1, array()))->toBeTrue() + ->and(flowview_filter_is_materializing(1, null))->toBeFalse(); + }); +}); From bb3eee2368db79dfcfc259c2da98d4d23950bb80 Mon Sep 17 00:00:00 2001 From: TheWitness Date: Wed, 21 Oct 2026 09:28:23 -0400 Subject: [PATCH 19/37] feature: Fix the report of a filter on the report page and add Prune for cached shards that match no filter The report is part of the cache key, so the Report selector is disabled on the report page for a filter that has one, and the server ignores a report sent for it. The Filters page tells the cached shards that match a filter from the others by md5, and a Prune button removes the others after confirmation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 12 +++ flowview.php | 20 +++- flowview_filters.php | 101 ++++++++++++++++++- includes/functions.php | 149 +++++++++++++++++++++++++--- tests/Unit/CoreFunctionsTest.php | 162 +++++++++++++++++++++++++++++++ 5 files changed, 427 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 5b22ce3..66a243d 100644 --- a/README.md +++ b/README.md @@ -254,6 +254,18 @@ Filter in a way that changes the data it materializes, such as its report, device, exporter, or any address, port, interface, AS, protocol, flag or TOS restriction, you are advised to re-materialize it. +The report of a Filter is part of what is cached for it, so it can not be +changed on the report page: the Report selector is disabled once the Filter has +one. To change it, Edit the Filter and then materialize it again. Saving a +Filter does not remove anything from the cache. The results of the old settings +simply stop matching the Filter. The line under the Filters table says how many +cached shards match a Filter, which is told by their md5 value, and how many +match none, such as those of a Filter before it was changed, of a deleted +Filter, or of a report with its settings changed on the report page. The +'Prune' button on the Filters page removes the ones that match no Filter, after +you confirm. It keeps everything that matches a Filter, and does nothing when +the cache keys of a Filter that has a report can not be worked out. + # Possible Bugs? If you figure out this problem, goto GitHub and create a pulls request or open diff --git a/flowview.php b/flowview.php index c780ecc..7437708 100644 --- a/flowview.php +++ b/flowview.php @@ -307,6 +307,21 @@ function load_session_for_filter() { * settings first, and then overlay others only if they are set */ if (isset_request_var('query') && get_filter_request_var('query') > 0) { + $query = flowview_db_fetch_row_prepared('SELECT * + FROM plugin_flowview_queries + WHERE id = ?', + array(get_request_var('query'))); + + /** + * The report of a filter can not be changed here, as it is part of + * what is cached for it. It is changed by editing the filter. + */ + $report = cacti_sizeof($query) ? flowview_filter_report_key($query) : ''; + + if ($report != '') { + set_request_var('report', $report); + } + // Handle Report Column if (isset_request_var('report')) { if (get_nfilter_request_var('report') != '0' && trim(get_nfilter_request_var('report'), 'sp') != '0') { @@ -320,11 +335,6 @@ function load_session_for_filter() { } } - $query = flowview_db_fetch_row_prepared('SELECT * - FROM plugin_flowview_queries - WHERE id = ?', - array(get_request_var('query'))); - if (cacti_sizeof($query)) { foreach($query as $column => $value) { switch($column) { diff --git a/flowview_filters.php b/flowview_filters.php index 813ac54..6695c29 100644 --- a/flowview_filters.php +++ b/flowview_filters.php @@ -55,7 +55,14 @@ case 'sort_filter': sort_filter(); break; - case 'edit': + case 'prune': + top_header(); + confirm_prune_shards(); + bottom_footer(); + break; + case 'prune_shards': + prune_shards(); + break; if (!isset_request_var('embed')) { top_header(); } @@ -74,6 +81,69 @@ break; } +/** + * Asks for confirmation to prune the cached shards that match no Filter, + * saying how many there are and how much space they take, and posts to + * prune_shards(). Called from this script's main request-dispatch switch + * when action=prune. + * + * @return void + */ +function confirm_prune_shards(): void { + $summary = flowview_get_shard_cache_summary(); + + form_start('flowview_filters.php'); + + html_start_box(__('Prune Cached Shards', 'flowview'), '60%', false, 3, 'center', ''); + + if ($summary === null) { + print "" . __('The cache keys of the Filters could not be worked out, so nothing can be pruned right now. Check that each Filter has a valid Report.', 'flowview') . ''; + $save_html = ''; + } elseif ($summary['other']['shards'] == 0) { + print "

" . __('Every cached shard matches a Filter. There is nothing to prune.', 'flowview') . '

'; + $save_html = ''; + } else { + print " + +

" . __('Click \'Continue\' to remove %s cached shard(s) that do not match any Filter. They come from Filters that were changed or deleted, and from reports with settings changed on the report page. The %s cached shard(s) of the Filters are kept.', number_format_i18n($summary['other']['shards'], 0), number_format_i18n($summary['matching']['shards'], 0), 'flowview') . "

+ + "; + $save_html = "'; + } + + print " + + + + $save_html + + "; + + html_end_box(); + + form_end(); +} + +/** + * Removes the cached shards that match no Filter and returns to the Filters + * page. Called from this script's main request-dispatch switch when + * action=prune_shards. + * + * @return void + */ +function prune_shards(): void { + $pruned = flowview_prune_shard_cache(); + + if ($pruned === null) { + raise_message('flowview_prune', __('The cache keys of the Filters could not be worked out, so nothing was pruned.', 'flowview'), MESSAGE_LEVEL_ERROR); + } else { + raise_message('flowview_prune', __('Pruned %s cached shard(s) that did not match any Filter.', number_format_i18n($pruned, 0), 'flowview'), MESSAGE_LEVEL_INFO); + } + + header('Location: flowview_filters.php?header=false'); + exit; +} + /** * Handles the bulk-action confirmation page/submission for the saved * filters list: on first display, renders a confirmation box listing @@ -307,6 +377,7 @@ function show_filters(): void { + @@ -330,6 +401,10 @@ function clearFilter() { clearFilter(); }); + $('#prune').on('click', function() { + loadPageNoHeader('flowview_filters.php?action=prune&header=false'); + }); + $('#rows').on('change', function() { applyFilter(); }); @@ -491,5 +566,29 @@ function clearFilter() { draw_actions_dropdown($sched_actions); form_end(); + + show_shard_cache_summary(); +} + +/** + * Shows how much of the Parallel Query cache matches a Filter and how much + * does not, which is what the Prune button removes. + * + * @return void + */ +function show_shard_cache_summary(): void { + $summary = flowview_get_shard_cache_summary(); + + html_start_box('', '100%', false, 3, 'center', ''); + + if ($summary === null) { + $text = __('The cached shards of the Filters could not be told apart from the others, as the cache keys of a Filter could not be worked out.', 'flowview'); + } else { + $text = __('Cached Shards: %s matching a Filter. %s not matching any Filter, which Prune removes.', number_format_i18n($summary['matching']['shards'], 0), number_format_i18n($summary['other']['shards'], 0), 'flowview'); + } + + print "" . html_escape($text) . ''; + + html_end_box(); } diff --git a/includes/functions.php b/includes/functions.php index 2d4fb07..d0a641d 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -846,6 +846,8 @@ function flowview_display_filter(): void { $title = __esc('Undefined Filter [ Select Filter to get Details ]', 'flowview'); + $report_fixed = false; + if (get_filter_request_var('query') > 0) { $row = flowview_db_fetch_row_prepared('SELECT name, statistics, printed FROM plugin_flowview_queries @@ -853,6 +855,7 @@ function flowview_display_filter(): void { array(get_request_var('query'))); if (cacti_sizeof($row)) { + $report_fixed = flowview_filter_report_key($row) != ''; if ($row['statistics'] > 0) { $title = __esc('Statistical Report: %s [ Including overrides as specified below ]', $stat_report_array[$row['statistics']]); } elseif ($row['printed'] > 0) { @@ -955,11 +958,11 @@ function flowview_display_filter(): void { - "; @@ -512,7 +517,7 @@ function clearFilter() { $tables = flowview_get_partition_tables(); $cold = flowview_get_cold_partitions($tables); $partitions = cacti_sizeof($tables); - $scope = flowview_get_materialization_scope(); + $materializing = flowview_get_materializing_filters(); $requests = flowview_get_running_requests(); foreach ($filters as $filter) { @@ -521,7 +526,7 @@ function clearFilter() { $queries = flowview_count_running_queries($md5sums, $requests); /* only the filters being materialized, or with a query running, are running */ - $running = flowview_filter_is_materializing((int) $filter['id'], $scope) || $queries > 0; + $running = in_array((int) $filter['id'], $materializing, true) || $queries > 0; $status = flowview_materialization_status($cached['nat'], $cached['non_nat'], $cold, $running); $missing = flowview_get_missing_partitions($cached['nat'], $cached['non_nat'], $cold); diff --git a/flowview_process.php b/flowview_process.php index 9ef0863..d9cad79 100644 --- a/flowview_process.php +++ b/flowview_process.php @@ -176,9 +176,17 @@ $raw_engine = get_set_default_fast_engine(); -/* every partition that has rotated, so ones missed while the poller was not running are caught up */ -foreach (flowview_convert_cold_partitions($raw_engine) as $table) { - flowview_debug("Converted partition {$table} to {$raw_engine}"); +/* + * A materialization gets the partitions ready itself, and waits for any alter + * that is running on them, so it is left to do that, rather than starting more. + */ +flowview_purge_dead_materializations(); + +if (!cacti_sizeof(flowview_get_materializing_filters())) { + /* every partition that has rotated, so ones missed while the poller was not running are caught up */ + foreach (flowview_convert_cold_partitions($raw_engine) as $table) { + flowview_debug("Converted partition {$table} to {$raw_engine}"); + } } set_config_option('flowview_last_sequence', $sequence); diff --git a/includes/functions.php b/includes/functions.php index 19245c6..df9925d 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -4305,29 +4305,38 @@ function flowview_invalidate_stale_shard_cache(array $only = []): array { /** * flowview_launch_materialize - Start cli/materialize_queries.php in the - * background, for the given FlowView queries and the given partitions, or - * all of them when none are given. Nothing is started when a - * materialization is already registered as running, since only one may be - * active at a time. + * background, for the given FlowView queries, or all of them when none are + * given, and for the given partitions, or all of them when none are given. + * A filter that is already being materialized is left to the run that has + * it, so the filters can be materialized separately. * * @param array $query_ids The FlowView query ids to materialize * @param array $partitions The partition tables to materialize * - * @return bool false when a materialization is already running + * @return array The filters a materialization was started for */ -function flowview_launch_materialize(array $query_ids = [], array $partitions = []): bool { +function flowview_launch_materialize(array $query_ids = [], array $partitions = []): array { global $config; - if (flowview_materialization_running()) { - return false; + if (cacti_sizeof($query_ids)) { + $targets = array_map('intval', $query_ids); + } else { + $targets = array_map('intval', array_column(flowview_db_fetch_assoc('SELECT id + FROM plugin_flowview_queries + ORDER BY id'), 'id')); } - $arguments = cacti_escapeshellarg($config['base_path'] . '/plugins/flowview/cli/materialize_queries.php'); + $targets = array_values(array_diff(array_unique(array_filter($targets, function ($id) { + return $id > 0; + })), flowview_get_materializing_filters())); - if (cacti_sizeof($query_ids)) { - $arguments .= ' --query-id=' . implode(',', array_map('intval', $query_ids)); + if (!cacti_sizeof($targets)) { + return []; } + $arguments = cacti_escapeshellarg($config['base_path'] . '/plugins/flowview/cli/materialize_queries.php') . + ' --query-id=' . implode(',', $targets); + $partitions = array_filter($partitions, function ($table) { return flowview_partition_sort_key($table) !== ''; }); @@ -4338,7 +4347,7 @@ function flowview_launch_materialize(array $query_ids = [], array $partitions = exec_background(read_config_option('path_php_binary'), $arguments); - return true; + return $targets; } /** @@ -4895,14 +4904,7 @@ function flowview_get_running_requests(): array { * @return bool */ function flowview_materialization_running(): bool { - $registered = db_fetch_cell_prepared('SELECT COUNT(*) - FROM processes - WHERE tasktype = ? - AND taskname = ? - AND taskid = ?', - ['flowview', 'materialize', 0]); - - if ($registered > 0) { + if (cacti_sizeof(flowview_get_materializing_filters())) { return true; } @@ -4910,55 +4912,59 @@ function flowview_materialization_running(): bool { } /** - * flowview_set_materialization_scope - Record which filters the - * materialization that is starting is for, so the Filters page only shows - * those as running. It only counts while the materialization is registered - * as running. - * - * @param array $query_ids The FlowView query ids, or none for all of them + * flowview_get_materializing_filters - The filters that are being + * materialized. Each filter a materialization works on is registered in the + * process table as materialize, with the id of the filter as the task id, so + * the filters can be materialized separately and what is being done is + * visible from the process table. * - * @return void + * @return array The filter ids */ -function flowview_set_materialization_scope(array $query_ids): void { - set_config_option('flowview_materialize_queries', implode(',', array_map('intval', $query_ids))); +function flowview_get_materializing_filters(): array { + $rows = db_fetch_assoc_prepared('SELECT taskid + FROM processes + WHERE tasktype = ? + AND taskname = ? + AND taskid > 0 + ORDER BY taskid', + ['flowview', 'materialize']); + + return array_values(array_unique(array_map('intval', array_column($rows, 'taskid')))); } /** - * flowview_get_materialization_scope - Which filters the registered - * materialization is for. + * flowview_purge_dead_materializations - Remove the registrations of a + * materialization whose process is gone, such as one that was killed and could + * not clean up. Without this the filter would show as running, and could not + * be materialized, until the registration timed out. Meant for a script, as the + * web server may not be able to see the process. * - * @return array|null null when no materialization is registered, an empty - * array when it is for every filter, otherwise the ids of - * the filters it is for + * @return int The registrations removed */ -function flowview_get_materialization_scope(): ?array { - $registered = db_fetch_cell_prepared('SELECT COUNT(*) +function flowview_purge_dead_materializations(): int { + if (!function_exists('cacti_process_still_running')) { + return 0; + } + + $removed = 0; + + $rows = db_fetch_assoc_prepared('SELECT id, pid FROM processes WHERE tasktype = ? - AND taskname = ? - AND taskid = ?', - ['flowview', 'materialize', 0]); - - if ($registered < 1) { - return null; - } + AND taskname = ?', + ['flowview', 'materialize']); - $ids = array_filter(array_map('intval', explode(',', (string) read_config_option('flowview_materialize_queries', true)))); + foreach ($rows as $row) { + if (!cacti_process_still_running((int) $row['pid'])) { + db_execute_prepared('DELETE FROM processes + WHERE id = ?', + [$row['id']]); - return array_values(array_unique($ids)); -} + $removed++; + } + } -/** - * flowview_filter_is_materializing - Whether the registered materialization - * is working on a filter. - * - * @param int $query_id The FlowView query - * @param array|null $scope From flowview_get_materialization_scope() - * - * @return bool - */ -function flowview_filter_is_materializing(int $query_id, ?array $scope): bool { - return $scope !== null && (!cacti_sizeof($scope) || in_array($query_id, $scope, true)); + return $removed; } /** @@ -5184,7 +5190,12 @@ function flowview_materialize_rotated_partitions(): array { $threads = read_config_option('flowview_parallel_threads'); - if (empty($threads) || $threads == 1 || !flowview_launch_materialize([], $rotated)) { + /* a filter is left alone while it is being materialized, so this waits for a quiet moment to catch them all up */ + if (empty($threads) || $threads == 1 || flowview_materialization_running()) { + return []; + } + + if (!cacti_sizeof(flowview_launch_materialize([], $rotated))) { return []; } @@ -5645,6 +5656,7 @@ function parallel_database_query_request($tables, $stru_inner, $stru_outer) { */ function flowview_track_request(int $request_id): void { $GLOBALS['flowview_requests_in_flight'][$request_id] = $request_id; + $GLOBALS['flowview_requests_made'][$request_id] = $request_id; } /** diff --git a/tests/Unit/CoreFunctionsTest.php b/tests/Unit/CoreFunctionsTest.php index f4576f9..bcd0b57 100644 --- a/tests/Unit/CoreFunctionsTest.php +++ b/tests/Unit/CoreFunctionsTest.php @@ -379,30 +379,51 @@ function cacti_escapeshellarg($string) { $GLOBALS['config']['base_path'] = '/var/www/cacti'; $GLOBALS['__test_config']['path_php_binary'] = '/usr/bin/php'; - plugin_test_queue_db_result('db_fetch_cell_prepared', 0); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('taskid' => 4))); try { - $started = flowview_launch_materialize(array('3', 5, '7; rm -rf /')); + $started = flowview_launch_materialize(array('3', 5, '7; rm -rf /', 4, 0, 5)); } finally { $GLOBALS['config']['base_path'] = $original_base_path; } - expect($started)->toBeTrue() + expect($started)->toBe(array(3, 5, 7)) ->and($GLOBALS['__test_exec'])->toBe(array( array('/usr/bin/php', "'/var/www/cacti/plugins/flowview/cli/materialize_queries.php' --query-id=3,5,7"), )) - ->and($GLOBALS['__test_db_calls'][0]['params'])->toBe(array('flowview', 'materialize', 0)); + ->and($GLOBALS['__test_db_calls'][0]['params'])->toBe(array('flowview', 'materialize')); }); -it('does not start another materialization while one is registered', function () { +it('does not start a materialization for filters that are already being materialized', function () { $GLOBALS['__test_exec'] = array(); - plugin_test_queue_db_result('db_fetch_cell_prepared', 1); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('taskid' => 3))); - expect(flowview_launch_materialize(array(3)))->toBeFalse() + expect(flowview_launch_materialize(array(3)))->toBe(array()) ->and($GLOBALS['__test_exec'])->toBe(array()); }); +it('materializes every filter that is not already being materialized when none are given', function () { + $GLOBALS['__test_exec'] = array(); + $original_base_path = $GLOBALS['config']['base_path']; + $GLOBALS['config']['base_path'] = '/var/www/cacti'; + $GLOBALS['__test_config']['path_php_binary'] = '/usr/bin/php'; + + plugin_test_queue_db_result('db_fetch_assoc', array(array('id' => 1), array('id' => 4), array('id' => 9))); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('taskid' => 4))); + + try { + $started = flowview_launch_materialize(); + } finally { + $GLOBALS['config']['base_path'] = $original_base_path; + } + + expect($started)->toBe(array(1, 9)) + ->and($GLOBALS['__test_exec'])->toBe(array( + array('/usr/bin/php', "'/var/www/cacti/plugins/flowview/cli/materialize_queries.php' --query-id=1,9"), + )); +}); + it('keys the cached shards on the map query without the time range', function () { $inner = array( 'sql_query' => 'SELECT src_addr, SUM(bytes) AS bytes', @@ -670,7 +691,7 @@ function cacti_escapeshellarg($string) { $GLOBALS['config']['base_path'] = '/var/www/cacti'; $GLOBALS['__test_config']['path_php_binary'] = '/usr/bin/php'; - plugin_test_queue_db_result('db_fetch_cell_prepared', 0); + plugin_test_queue_db_result('db_fetch_assoc', array(array('id' => 1), array('id' => 2))); try { $started = flowview_launch_materialize(array(), array('plugin_flowview_raw_2024001', 'plugin_flowview_raw_2024001; rm', 'plugin_flowview_raw_2024002')); @@ -678,9 +699,9 @@ function cacti_escapeshellarg($string) { $GLOBALS['config']['base_path'] = $original_base_path; } - expect($started)->toBeTrue() + expect($started)->toBe(array(1, 2)) ->and($GLOBALS['__test_exec'])->toBe(array( - array('/usr/bin/php', "'/var/www/cacti/plugins/flowview/cli/materialize_queries.php' --partitions=plugin_flowview_raw_2024001,plugin_flowview_raw_2024002"), + array('/usr/bin/php', "'/var/www/cacti/plugins/flowview/cli/materialize_queries.php' --query-id=1,2 --partitions=plugin_flowview_raw_2024001,plugin_flowview_raw_2024002"), )); }); @@ -746,7 +767,7 @@ function cacti_escapeshellarg($string) { it('starts a materialization for the partitions that rotated and records the newest', function () { $GLOBALS['__test_config']['flowview_last_hot_partition'] = 'plugin_flowview_raw_2024002'; plugin_test_queue_db_result('db_fetch_assoc', $this->tables); - plugin_test_queue_db_result('db_fetch_cell_prepared', 0); + plugin_test_queue_db_result('db_fetch_assoc', array(array('id' => 1), array('id' => 2))); $original_base_path = $GLOBALS['config']['base_path']; $GLOBALS['config']['base_path'] = '/var/www/cacti'; @@ -760,14 +781,14 @@ function cacti_escapeshellarg($string) { expect($started)->toBe(array('plugin_flowview_raw_2024002')) ->and($GLOBALS['__test_config']['flowview_last_hot_partition'])->toBe('plugin_flowview_raw_2024003') ->and($GLOBALS['__test_exec'])->toBe(array( - array('/usr/bin/php', "'/var/www/cacti/plugins/flowview/cli/materialize_queries.php' --partitions=plugin_flowview_raw_2024002"), + array('/usr/bin/php', "'/var/www/cacti/plugins/flowview/cli/materialize_queries.php' --query-id=1,2 --partitions=plugin_flowview_raw_2024002"), )); }); it('leaves the record alone while a materialization is running so it is caught up later', function () { $GLOBALS['__test_config']['flowview_last_hot_partition'] = 'plugin_flowview_raw_2024001'; plugin_test_queue_db_result('db_fetch_assoc', $this->tables); - plugin_test_queue_db_result('db_fetch_cell_prepared', 1); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('taskid' => 3))); expect(flowview_materialize_rotated_partitions())->toBe(array()) ->and($GLOBALS['__test_config']['flowview_last_hot_partition'])->toBe('plugin_flowview_raw_2024001') @@ -1114,8 +1135,8 @@ function cacti_process_still_running($pid) { ->and($lookup[0]['params'])->toBe(array('flowview', 'db\_query\_%', 'db\_shard\_%')); }); - it('is running while a materialization is registered', function () { - plugin_test_queue_db_result('db_fetch_cell_prepared', 1); + it('is running while a filter is registered as being materialized', function () { + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('taskid' => 3))); expect(flowview_materialization_running())->toBeTrue() ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { @@ -1124,7 +1145,7 @@ function cacti_process_still_running($pid) { }); it('is still running after the script that started it has ended', function () { - plugin_test_queue_db_result('db_fetch_cell_prepared', 0); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array()); plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('pid' => 222, 'taskname' => 'db_shard_991', 'age' => 12))); plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('id' => 991, 'md5sum_tables' => 'aaaa'))); @@ -1132,8 +1153,6 @@ function cacti_process_still_running($pid) { }); it('is not running when nothing is registered and no runner is registered', function () { - plugin_test_queue_db_result('db_fetch_cell_prepared', 0); - expect(flowview_materialization_running())->toBeFalse(); }); }); @@ -1538,45 +1557,57 @@ function number_format_i18n($number, $decimals = 0) { describe('showing only the filters being materialized as running', function () { beforeEach(function () { plugin_test_reset(); + + if (!function_exists('cacti_process_still_running')) { + function cacti_process_still_running($pid) { + return in_array($pid, $GLOBALS['__test_live_pids'], true); + } + } + + $GLOBALS['__test_live_pids'] = array(); }); - it('records the filters a materialization is for, or all of them', function () { - flowview_set_materialization_scope(array(3, '7')); + it('finds the filters being materialized from their task ids', function () { + plugin_test_queue_db_result('db_fetch_assoc_prepared', array( + array('taskid' => 3), + array('taskid' => '7'), + array('taskid' => 3), + )); - expect(read_config_option('flowview_materialize_queries'))->toBe('3,7'); + expect(flowview_get_materializing_filters())->toBe(array(3, 7)); - flowview_set_materialization_scope(array()); + $lookup = $GLOBALS['__test_db_calls'][0]; - expect(read_config_option('flowview_materialize_queries'))->toBe(''); + expect($lookup['params'])->toBe(array('flowview', 'materialize')) + ->and($lookup['sql'])->toContain('taskid > 0'); }); - it('has no scope when no materialization is registered', function () { - set_config_option('flowview_materialize_queries', '3'); - plugin_test_queue_db_result('db_fetch_cell_prepared', 0); - - expect(flowview_get_materialization_scope())->toBeNull(); + it('finds no filter being materialized when none is registered', function () { + expect(flowview_get_materializing_filters())->toBe(array()); }); - it('is for the filters that were chosen while it is registered', function () { - set_config_option('flowview_materialize_queries', '3,7,3'); - plugin_test_queue_db_result('db_fetch_cell_prepared', 1); + it('removes the registrations of a materialization whose process is gone', function () { + $GLOBALS['__test_live_pids'] = array(222); - expect(flowview_get_materialization_scope())->toBe(array(3, 7)); - }); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array( + array('id' => 10, 'pid' => 111), + array('id' => 11, 'pid' => 222), + array('id' => 12, 'pid' => '333'), + )); - it('is for every filter when none were chosen', function () { - set_config_option('flowview_materialize_queries', ''); - plugin_test_queue_db_result('db_fetch_cell_prepared', 1); + expect(flowview_purge_dead_materializations())->toBe(2); - expect(flowview_get_materialization_scope())->toBe(array()); + $deleted = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute_prepared' && strpos($call['sql'], 'DELETE FROM processes') === 0; + })); + + expect(array_column($deleted, 'params'))->toBe(array(array(10), array(12))); }); - it('marks only the chosen filters as being materialized', function () { - expect(flowview_filter_is_materializing(3, array(3, 7)))->toBeTrue() - ->and(flowview_filter_is_materializing(7, array(3, 7)))->toBeTrue() - ->and(flowview_filter_is_materializing(1, array(3, 7)))->toBeFalse() - ->and(flowview_filter_is_materializing(1, array()))->toBeTrue() - ->and(flowview_filter_is_materializing(1, null))->toBeFalse(); + it('is not running for a filter that is not registered, while another is', function () { + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('taskid' => 3))); + + expect(in_array(1, flowview_get_materializing_filters(), true))->toBeFalse(); }); }); From 8c32d207cc8600ffa3f8674a2b5154ca45cad7fe Mon Sep 17 00:00:00 2001 From: TheWitness Date: Wed, 21 Oct 2026 09:57:03 -0400 Subject: [PATCH 22/37] fix: Run the shards of a materialization instead of reusing a finished request, and add Prune to the Filters actions A parallel query request is reused for the same query and range, and a finished one only hands back its result, so a materialization after the shard cache was lost finished at once and cached nothing. The materializer now discards a finished request of the same query first. Prune Cached Shards is also an action in the Filters page dropdown, as well as the Prune button. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 2 +- cli/materialize_queries.php | 4 +++- flowview_filters.php | 14 +++++++++++++- includes/functions.php | 32 +++++++++++++++++++++++++++++++ tests/Unit/CoreFunctionsTest.php | 33 ++++++++++++++++++++++++++++++++ 5 files changed, 82 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index bf84f7a..1771503 100644 --- a/README.md +++ b/README.md @@ -269,7 +269,7 @@ simply stop matching the Filter. The line under the Filters table says how many cached shards match a Filter, which is told by their md5 value, and how many match none, such as those of a Filter before it was changed, of a deleted Filter, or of a report with its settings changed on the report page. The -'Prune' button on the Filters page removes the ones that match no Filter, after +'Prune' button on the Filters page, or the 'Prune Cached Shards' action, removes the ones that match no Filter, after you confirm. It keeps everything that matches a Filter, and does nothing when the cache keys of a Filter that has a report can not be worked out. diff --git a/cli/materialize_queries.php b/cli/materialize_queries.php index 1b2d0c3..c40ae12 100644 --- a/cli/materialize_queries.php +++ b/cli/materialize_queries.php @@ -689,7 +689,9 @@ function materialize_run_worker(int $query_id, string $nat, int $start, int $end set_request_var('usenat', $nat == 'on' ? 'true' : 'false'); - $GLOBALS['flowview_requests_made'] = []; + /* run the shards, even if the same query was run before, as it is the cached shards that are wanted */ + $GLOBALS['flowview_fresh_requests'] = true; + $GLOBALS['flowview_requests_made'] = []; materialize_say(sprintf('Materializing Query:%d NAT:%s', $query_id, $nat)); diff --git a/flowview_filters.php b/flowview_filters.php index 1a30af9..8da3644 100644 --- a/flowview_filters.php +++ b/flowview_filters.php @@ -42,7 +42,8 @@ 1 => __('Delete', 'flowview'), 3 => __('Disable', 'flowview'), 4 => __('Enable', 'flowview'), - 5 => __('Materialize View', 'flowview') + 5 => __('Materialize View', 'flowview'), + 6 => __('Prune Cached Shards', 'flowview') ); switch (get_request_var('action')) { @@ -165,6 +166,17 @@ function actions_filters(): void { get_filter_request_var('drp_action'); /* ==================================================== */ + /* pruning is for the whole cache, so it does not depend on which Filters are selected */ + if (get_nfilter_request_var('drp_action') == '6') { + general_header(); + + confirm_prune_shards(); + + bottom_footer(); + + return; + } + if (isset_request_var('selected_items')) { $selected_items = sanitize_unserialize_selected_items(get_nfilter_request_var('selected_items')); diff --git a/includes/functions.php b/includes/functions.php index df9925d..c4dd8a4 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -5540,6 +5540,10 @@ function parallel_database_query_request($tables, $stru_inner, $stru_outer) { AND user_id = ?', [$query_md5, $user_id]); + /* a materialization has to run the shards, not be given the result of an earlier request */ + if (!empty($request_id) && !empty($GLOBALS['flowview_fresh_requests']) && flowview_discard_completed_request((int) $request_id)) { + $request_id = 0; + } $time_to_live = read_config_option('flowview_parallel_time_to_live'); if (empty($time_to_live)) { @@ -5645,6 +5649,34 @@ function parallel_database_query_request($tables, $stru_inner, $stru_outer) { return $request_id; } +/** + * flowview_discard_completed_request - Remove a parallel query request that + * has finished, so the same query is run again. A request is reused for the + * same query and range until it expires, and a finished one is never run + * again, it only hands back its result. That result does not mean the cached + * shards are there: they may have been pruned, or removed because the + * partition changed. A request that is still running is left alone, as + * another process is using it. + * + * @param int $request_id + * + * @return bool Whether it was finished and was removed + */ +function flowview_discard_completed_request(int $request_id): bool { + $status = flowview_db_fetch_cell_prepared('SELECT status + FROM parallel_database_query + WHERE id = ?', + [$request_id]); + + if ($status != 'complete') { + return false; + } + + parallel_database_query_cleanup($request_id, true); + + return true; +} + /** * flowview_track_request - Remember a parallel query request this process * made and has not cleaned up, so a script that is told to stop can cancel diff --git a/tests/Unit/CoreFunctionsTest.php b/tests/Unit/CoreFunctionsTest.php index bcd0b57..f7f9d7e 100644 --- a/tests/Unit/CoreFunctionsTest.php +++ b/tests/Unit/CoreFunctionsTest.php @@ -1828,4 +1828,37 @@ function cacti_process_still_running($pid) { it('cancels nothing when no request is in flight', function () { expect(flowview_cancel_requests_in_flight())->toBe(array()); }); + + it('removes a finished request so the same query is run again', function () { + plugin_test_queue_db_result('db_fetch_cell_prepared', 'complete'); + + expect(flowview_discard_completed_request(991))->toBeTrue(); + + $removed = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute_prepared' && preg_match('/^DELETE FROM parallel_database_query\s+WHERE id = \?/', $call['sql']) === 1; + })); + + expect(array_column($removed, 'params'))->toBe(array(array(991))); + }); + + it('leaves a request that is still being run, as another process is using it', function ($status) { + plugin_test_queue_db_result('db_fetch_cell_prepared', $status); + + expect(flowview_discard_completed_request(991))->toBeFalse() + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['fn'], 'db_execute') === 0; + }))->toBeEmpty(); + })->with(array('pending', 'scheduled', 'running', '')); + + it('offers pruning in the actions of the Filters page, for the whole cache', function () { + $source = plugin_test_read_source('flowview_filters.php'); + + expect($source)->toContain("6 => __('Prune Cached Shards'") + ->and($source)->toMatch("/drp_action'\) == '6'\) \{[^}]*confirm_prune_shards\(\)/s"); + }); + + it('runs the shards of a materialization instead of reusing an earlier request', function () { + expect(plugin_test_read_source('cli/materialize_queries.php'))->toContain("\$GLOBALS['flowview_fresh_requests'] = true") + ->and(plugin_test_read_source('includes/functions.php'))->toContain("!empty(\$GLOBALS['flowview_fresh_requests']) && flowview_discard_completed_request"); + }); }); From ab2263a87d522879bed1748f44f5101e66d5960f Mon Sep 17 00:00:00 2001 From: TheWitness Date: Wed, 21 Oct 2026 09:58:49 -0400 Subject: [PATCH 23/37] fix: Hide the Report on the report page behind a hidden input The Report selector is no longer shown, only a hidden input id=report that carries the value the requests use, unless the selected filter has no report and needs one. The title of a printed report named the report by the statistical index. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- includes/functions.php | 15 ++++++-- tests/Unit/FlowviewFilterControlsTest.php | 47 +++++++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/includes/functions.php b/includes/functions.php index c4dd8a4..a918ee0 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -859,7 +859,7 @@ function flowview_display_filter(): void { if ($row['statistics'] > 0) { $title = __esc('Statistical Report: %s [ Including overrides as specified below ]', $stat_report_array[$row['statistics']]); } elseif ($row['printed'] > 0) { - $title = __esc('Printed Report: %s [ Including overrides as specified below ]', $print_report_array[$row['statistics']]); + $title = __esc('Printed Report: %s [ Including overrides as specified below ]', $print_report_array[$row['printed']]); } } } @@ -958,11 +958,19 @@ function flowview_display_filter(): void {
+ > - > for a statistical report and + * p for a printed one. The report is part of the cache key of the filter. + * + * @param array $filter The filter, with its statistics and printed columns + * + * @return string The report, or an empty string when the filter has none + */ +function flowview_filter_report_key(array $filter): string { + if ((int) ($filter['statistics'] ?? 0) > 0) { + return 's' . (int) $filter['statistics']; + } + + if ((int) ($filter['printed'] ?? 0) > 0) { + return 'p' . (int) $filter['printed']; + } + + return ''; +} + +/** + * flowview_get_filter_shard_keys - Every cache key (md5sum_tables) that a + * filter, with NAT or without, stores its partitions under. + * + * @return array|null The keys as the array keys, or null when the keys of a + * filter that has a report could not be worked out, as + * what belongs to which filter is not known then + */ +function flowview_get_filter_shard_keys(): ?array { + $keys = []; + + foreach (flowview_db_fetch_assoc('SELECT id, statistics, printed + FROM plugin_flowview_queries') as $filter) { + $md5sums = flowview_get_filter_md5sums((int) $filter['id']); + + foreach (['nat', 'non_nat'] as $mode) { + if (!cacti_sizeof($md5sums[$mode])) { + /* a filter with no report has nothing cached */ + if (flowview_filter_report_key($filter) != '') { + return null; + } + + continue; + } + + foreach ($md5sums[$mode] as $key) { + $keys[$key] = true; + } + } + } + + return $keys; +} + +/** + * flowview_get_shard_cache_summary - What is in the shard cache, told apart + * by whether it matches a filter. A cached partition matches when its + * md5sum is the cache key of a filter, with NAT or without. The rest comes + * from queries that no filter has, such as a filter before it was changed, a + * deleted filter, or a report with settings changed on the report page. The + * short lived results of the live partition are not counted. + * + * @return array|null matching and other, each with the number of cached + * shards, and the md5sums of the other; null when the + * keys of the filters are not known + */ +function flowview_get_shard_cache_summary(): ?array { + /* the size of the results is not looked at, as that reads every cached result */ + $cached = flowview_db_fetch_assoc('SELECT md5sum, COUNT(*) AS shards + FROM parallel_database_query_shard_cache + WHERE map_partition != \'' . FLOWVIEW_LIVE_SHARD_MARKER . '\' + GROUP BY md5sum'); + + $keys = flowview_get_filter_shard_keys(); + + if ($keys === null) { + return null; + } + + $summary = [ + 'matching' => ['shards' => 0], + 'other' => ['shards' => 0, 'md5sums' => []], + ]; + + foreach ($cached as $row) { + if (isset($keys[$row['md5sum']])) { + $summary['matching']['shards'] += (int) $row['shards']; + } else { + $summary['other']['shards'] += (int) $row['shards']; + $summary['other']['md5sums'][] = $row['md5sum']; + } + } + + return $summary; +} + +/** + * flowview_prune_shard_cache - Remove the cached shards that match no filter. + * Whatever matches a filter is kept, even when the filter is not materialized + * for every partition. + * + * @return int|null The number of shards removed, or null when nothing was + * removed because the keys of the filters are not known + */ +function flowview_prune_shard_cache(): ?int { + $summary = flowview_get_shard_cache_summary(); + + if ($summary === null) { + return null; + } + + foreach ($summary['other']['md5sums'] as $md5sum) { + flowview_db_execute_prepared('DELETE FROM parallel_database_query_shard_cache + WHERE md5sum = ? + AND map_partition != \'' . FLOWVIEW_LIVE_SHARD_MARKER . '\'', + [$md5sum]); + } + + return $summary['other']['shards']; +} + /** * flowview_count_running_queries - How many of the running requests are * queries of a filter, with NAT or without. diff --git a/tests/Unit/CoreFunctionsTest.php b/tests/Unit/CoreFunctionsTest.php index 2775547..1a3dfb9 100644 --- a/tests/Unit/CoreFunctionsTest.php +++ b/tests/Unit/CoreFunctionsTest.php @@ -1579,3 +1579,165 @@ function number_format_i18n($number, $decimals = 0) { ->and(flowview_filter_is_materializing(1, null))->toBeFalse(); }); }); + +describe('keeping the report of a filter fixed and pruning what no filter caches', function () { + $filter = array( + 'id' => 4, 'name' => 'Top Talkers', 'device_id' => 0, 'ex_addr' => '', 'template_id' => -1, + 'statistics' => 10, 'printed' => 0, 'sortfield' => 'bytes', 'cutofflines' => 20, 'cutoffoctets' => 0, + 'includeif' => 0, 'usenat' => '', 'protocols' => '', 'tosfields' => '', 'tcpflags' => '', + ); + + beforeEach(function () { + plugin_test_reset(); + }); + + /* queue the lookups of one filter for each NAT mode, and return its cache keys */ + $keys_of = function (array $filter) { + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + + $keys = flowview_get_filter_md5sums((int) $filter['id']); + + plugin_test_reset(); + + return $keys; + }; + + it('names the report of a filter the way the report selector does', function ($filter, $expected) { + expect(flowview_filter_report_key($filter))->toBe($expected); + })->with(array( + 'statistical' => array(array('statistics' => 10, 'printed' => 0), 's10'), + 'printed' => array(array('statistics' => '0', 'printed' => '2'), 'p2'), + 'none' => array(array('statistics' => 0, 'printed' => 0), ''), + 'missing' => array(array(), ''), + )); + + it('does not let the request change the report of a filter', function () use ($filter) { + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + $plain = run_flow_query(false, 4, 1704067200, 1704153600, true, true); + + $_REQUEST['report'] = 's11'; + + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + + expect(run_flow_query(false, 4, 1704067200, 1704153600, true, true))->toBe($plain); + }); + + it('still lets the request pick the report of a filter that has none', function () use ($filter) { + $none = array_merge($filter, array('statistics' => 0, 'printed' => 0)); + + plugin_test_queue_db_result('db_fetch_row_prepared', $none); + + expect(run_flow_query(false, 4, 1704067200, 1704153600, true, true))->toBeFalse(); + + $_REQUEST['report'] = 's10'; + + plugin_test_queue_db_result('db_fetch_row_prepared', $none); + + expect(run_flow_query(false, 4, 1704067200, 1704153600, true, true))->toBeArray()->toHaveCount(1); + }); + + it('knows the cache keys of every filter', function () use ($filter, $keys_of) { + $keys = $keys_of($filter); + + plugin_test_queue_db_result('db_fetch_assoc', array( + array('id' => 4, 'statistics' => 10, 'printed' => 0), + array('id' => 5, 'statistics' => 10, 'printed' => 0), + )); + + foreach (range(1, 4) as $unused) { + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + } + + expect(flowview_get_filter_shard_keys())->toBe(array( + $keys['nat'][0] => true, + $keys['non_nat'][0] => true, + )); + }); + + it('does not know the cache keys when a filter that has a report cannot be planned', function () { + plugin_test_queue_db_result('db_fetch_assoc', array(array('id' => 4, 'statistics' => 10, 'printed' => 0))); + + expect(flowview_get_filter_shard_keys())->toBeNull(); + }); + + it('knows the cache keys when only a filter that has no report cannot be planned', function () { + plugin_test_queue_db_result('db_fetch_assoc', array(array('id' => 4, 'statistics' => 0, 'printed' => 0))); + + expect(flowview_get_filter_shard_keys())->toBe(array()); + }); + + it('tells the cached shards of the filters from the others', function () use ($filter, $keys_of) { + $keys = $keys_of($filter); + + plugin_test_queue_db_result('db_fetch_assoc', array( + array('md5sum' => $keys['nat'][0], 'shards' => 40), + array('md5sum' => $keys['non_nat'][0], 'shards' => 38), + array('md5sum' => 'old1', 'shards' => 12), + array('md5sum' => 'old2', 'shards' => 3), + )); + plugin_test_queue_db_result('db_fetch_assoc', array(array('id' => 4, 'statistics' => 10, 'printed' => 0))); + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + + expect(flowview_get_shard_cache_summary())->toBe(array( + 'matching' => array('shards' => 78), + 'other' => array('shards' => 15, 'md5sums' => array('old1', 'old2')), + )); + + $lookup = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['sql'], 'FROM parallel_database_query_shard_cache') !== false; + })); + + expect($lookup[0]['sql'])->toContain("map_partition != 'live'"); + }); + + it('removes only the cached shards that match no filter', function () use ($filter, $keys_of) { + $keys = $keys_of($filter); + + plugin_test_queue_db_result('db_fetch_assoc', array( + array('md5sum' => $keys['nat'][0], 'shards' => 40), + array('md5sum' => 'old1', 'shards' => 12), + array('md5sum' => 'old2', 'shards' => 3), + )); + plugin_test_queue_db_result('db_fetch_assoc', array(array('id' => 4, 'statistics' => 10, 'printed' => 0))); + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + + expect(flowview_prune_shard_cache())->toBe(15); + + $deleted = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute_prepared'; + })); + + expect($deleted)->toHaveCount(2) + ->and($deleted[0]['params'])->toBe(array('old1')) + ->and($deleted[1]['params'])->toBe(array('old2')) + ->and($deleted[0]['sql'])->toContain('DELETE FROM parallel_database_query_shard_cache') + ->and($deleted[0]['sql'])->toContain("map_partition != 'live'"); + }); + + it('removes nothing when there is nothing to prune', function () use ($filter, $keys_of) { + $keys = $keys_of($filter); + + plugin_test_queue_db_result('db_fetch_assoc', array(array('md5sum' => $keys['nat'][0], 'shards' => 40))); + plugin_test_queue_db_result('db_fetch_assoc', array(array('id' => 4, 'statistics' => 10, 'printed' => 0))); + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + plugin_test_queue_db_result('db_fetch_row_prepared', $filter); + + expect(flowview_prune_shard_cache())->toBe(0) + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['fn'], 'db_execute') === 0; + }))->toBeEmpty(); + }); + + it('removes nothing when the cache keys of the filters are not known', function () { + plugin_test_queue_db_result('db_fetch_assoc', array(array('md5sum' => 'old1', 'shards' => 12))); + plugin_test_queue_db_result('db_fetch_assoc', array(array('id' => 4, 'statistics' => 10, 'printed' => 0))); + + expect(flowview_prune_shard_cache())->toBeNull() + ->and(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return strpos($call['fn'], 'db_execute') === 0; + }))->toBeEmpty(); + }); +}); From 1de91b3bfc4ab7bd6bf6842d12130bd8609121c5 Mon Sep 17 00:00:00 2001 From: TheWitness Date: Wed, 21 Oct 2026 09:40:33 -0400 Subject: [PATCH 20/37] fix: Unregister the materializer on SIGTERM/SIGINT and stop using pcntl constants from the web server cli/materialize_queries.php now handles SIGTERM and SIGINT like the flow collector: the script stops its worker and removes its process registration, and a worker cancels its parallel queries, which stops their runner and shard workers and removes their registrations. Requests are tracked per process for that. parallel_database_query_cleanup() used SIGTERM, which does not exist under the web server without pcntl, so a timed out report was a fatal error that disabled the plugin; signals are now sent through flowview_signal_process(). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cli/materialize_queries.php | 165 ++++++++++++++++++++++++++++++- includes/functions.php | 79 ++++++++++++++- tests/Unit/CoreFunctionsTest.php | 57 +++++++++++ 3 files changed, 292 insertions(+), 9 deletions(-) diff --git a/cli/materialize_queries.php b/cli/materialize_queries.php index 4cec5d2..e898d3b 100644 --- a/cli/materialize_queries.php +++ b/cli/materialize_queries.php @@ -36,6 +36,11 @@ ini_set('max_execution_time', 0); ini_set('memory_limit', '-1'); +/* signals are handled as they arrive, not when the script next ticks */ +if (function_exists('pcntl_async_signals')) { + pcntl_async_signals(true); +} + flowview_connect(); /* another materialization may not start until this one ends or this many seconds pass */ @@ -46,6 +51,10 @@ $dry_run = false; $status = false; $worker = false; + +/* what the signal handler has to clean up: the process registration, and the worker being run */ +$materialize_registered = false; +$materialize_child = null; $query_ids = []; $partitions = []; $nat = false; @@ -171,6 +180,8 @@ exit(1); } + materialize_install_signal_handlers(); + exit(materialize_run_worker($query_ids[0], $nat, $start, $end)); } @@ -180,15 +191,16 @@ exit(0); } - register_shutdown_function(function () { - flowview_set_materialization_scope([]); - unregister_process('flowview', 'materialize', 0); - }); + $materialize_registered = true; + + register_shutdown_function('materialize_release'); /* the Filters page shows only these filters as running */ flowview_set_materialization_scope($query_ids); } +materialize_install_signal_handlers(); + $threads = read_config_option('flowview_parallel_threads'); if (empty($threads) || $threads == 1) { @@ -284,7 +296,7 @@ ' --end=' . $end . ($debug ? ' --debug' : ''); - passthru($command, $status); + $status = materialize_run_command($command); if ($status != 0) { $failed++; @@ -452,6 +464,149 @@ function materialize_prepare_partitions(array $cold, array $modes, bool $dry_run } } +/** + * Makes SIGTERM and SIGINT end the script through materialize_sig_handler(), + * so what it registered is cleaned up, as the flow collector does. + * + * @return void + */ +function materialize_install_signal_handlers(): void { + if (function_exists('pcntl_signal')) { + pcntl_signal(SIGTERM, 'materialize_sig_handler'); + pcntl_signal(SIGINT, 'materialize_sig_handler'); + } +} + +/** + * Cleans up on SIGTERM and SIGINT and exits. The script that runs the + * materialization stops the worker it started, and removes its registration + * from the process table. A worker cancels the parallel queries it started, + * which stops their runner and shard workers and removes their registrations. + * + * @param int $signo + * + * @return void + */ +function materialize_sig_handler($signo): void { + global $worker; + + static $handling = false; + + switch ($signo) { + case SIGTERM: + case SIGINT: + if ($handling) { + return; + } + + $handling = true; + + if ($worker) { + cacti_log('WARNING: FlowView Materialization worker is shutting down by signal!', false, 'FLOWVIEW'); + + flowview_cancel_requests_in_flight(); + + /* the shutdown function must not report this as a failure to finish */ + $GLOBALS['flowview_error_reported'] = true; + } else { + cacti_log('WARNING: FlowView Materialization is shutting down by signal!', false, 'FLOWVIEW'); + + materialize_stop_child(); + materialize_release(); + } + + exit(1); + default: + /* ignore all other signals */ + } +} + +/** + * Removes the registration of this materialization from the process table, + * and which Filters it was for. It does nothing the second time it is called. + * + * @return void + */ +function materialize_release(): void { + global $materialize_registered; + + if ($materialize_registered) { + $materialize_registered = false; + + flowview_set_materialization_scope([]); + unregister_process('flowview', 'materialize', 0, (int) getmypid()); + } +} + +/** + * Runs a worker and waits for it, with its output shown as it is written. + * The worker is kept so a signal can stop it. + * + * @param string $command The command to run + * + * @return int The exit status, 0 on success + */ +function materialize_run_command(string $command): int { + global $materialize_child; + + $process = proc_open($command, [0 => STDIN, 1 => STDOUT, 2 => STDERR], $pipes); + + if (!is_resource($process)) { + return 1; + } + + $materialize_child = $process; + $status = 1; + + while (true) { + $info = proc_get_status($process); + + if (!$info['running']) { + /* the exit status can only be read the first time it is asked for */ + $status = $info['exitcode'] >= 0 ? $info['exitcode'] : 1; + + break; + } + + usleep(100000); + } + + $materialize_child = null; + + proc_close($process); + + return $status; +} + +/** + * Stops the worker being run, giving it time to cancel its queries first. + * + * @return void + */ +function materialize_stop_child(): void { + global $materialize_child; + + if (!is_resource($materialize_child)) { + return; + } + + proc_terminate($materialize_child, SIGTERM); + + $deadline = microtime(true) + 15; + + while (microtime(true) < $deadline) { + $info = proc_get_status($materialize_child); + + if (!$info['running']) { + return; + } + + usleep(100000); + } + + proc_terminate($materialize_child, SIGKILL); +} + /** * Materializes one query for one NAT mode across the given range. The query * runs through the normal parallel query path, so each partition that is diff --git a/includes/functions.php b/includes/functions.php index d0a641d..19245c6 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -5629,9 +5629,42 @@ function parallel_database_query_request($tables, $stru_inner, $stru_outer) { } } + flowview_track_request((int) $request_id); + return $request_id; } +/** + * flowview_track_request - Remember a parallel query request this process + * made and has not cleaned up, so a script that is told to stop can cancel + * it. It is forgotten by parallel_database_query_cleanup(). + * + * @param int $request_id + * + * @return void + */ +function flowview_track_request(int $request_id): void { + $GLOBALS['flowview_requests_in_flight'][$request_id] = $request_id; +} + +/** + * flowview_cancel_requests_in_flight - Cancel the parallel query requests + * this process made that are not finished. The runner and shard workers are + * told to stop, which unregisters them from the process table, and the request + * and its tables are removed. Meant for a signal handler. + * + * @return array The requests that were cancelled + */ +function flowview_cancel_requests_in_flight(): array { + $requests = array_values($GLOBALS['flowview_requests_in_flight'] ?? []); + + foreach ($requests as $request_id) { + parallel_database_query_cleanup($request_id, true); + } + + return $requests; +} + /** * flowview_parallel_timeout_message - The message for a parallel query that * did not finish within the Maximum Run Time, worded for the Cacti log and @@ -5975,6 +6008,8 @@ function parallel_database_query_expire(): void { * @return void */ function parallel_database_query_cleanup($request_id, bool $remove = false): void { + unset($GLOBALS['flowview_requests_in_flight'][(int) $request_id]); + $table = flowview_db_fetch_cell_prepared('SELECT map_table FROM parallel_database_query WHERE id = ?', @@ -6001,9 +6036,7 @@ function parallel_database_query_cleanup($request_id, bool $remove = false): voi if (cacti_sizeof($processes)) { foreach($processes as $p) { - if (posix_kill($p['pid'], 0)) { - posix_kill($p['pid'], SIGTERM); - } + flowview_signal_process((int) $p['pid'], 'SIGTERM'); db_execute_prepared('DELETE FROM processes WHERE pid = ?', @@ -6018,6 +6051,44 @@ function parallel_database_query_cleanup($request_id, bool $remove = false): voi } } +/** + * flowview_signal_number - The number of a signal. The SIG constants only + * exist when the pcntl extension is loaded, which it is not for the web + * server, so using one there is a fatal error. The numbers are those of + * Linux. + * + * @param string $signal SIGTERM, SIGKILL or SIGINT + * + * @return int + */ +function flowview_signal_number(string $signal): int { + if (defined($signal)) { + return (int) constant($signal); + } + + $numbers = ['SIGINT' => 2, 'SIGKILL' => 9, 'SIGTERM' => 15]; + + return $numbers[$signal] ?? 15; +} + +/** + * flowview_signal_process - Send a signal to a process, from a script or from + * the web server. A process of another user can not be signalled from the web + * server, which is not an error here. + * + * @param int $pid The process + * @param string $signal SIGTERM, SIGKILL or SIGINT + * + * @return bool Whether the signal was sent + */ +function flowview_signal_process(int $pid, string $signal = 'SIGTERM'): bool { + if ($pid <= 0 || !function_exists('posix_kill')) { + return false; + } + + return posix_kill($pid, flowview_signal_number($signal)); +} + /** * @param mixed $query_id * @@ -6035,7 +6106,7 @@ function parallel_database_query_cancel($query_id): void { foreach($processes as $p) { cacti_log("WARNING: Killing FlowView Query Process with Task Name:{$p['taskname']} and PID:{$p['pid']}", false, 'FLOWVIEW'); - posix_kill($p['pid'], SIGKILL); + flowview_signal_process((int) $p['pid'], 'SIGKILL'); db_execute_prepared('DELETE FROM processes WHERE id = ?', [$p['id']]); } diff --git a/tests/Unit/CoreFunctionsTest.php b/tests/Unit/CoreFunctionsTest.php index 1a3dfb9..f4576f9 100644 --- a/tests/Unit/CoreFunctionsTest.php +++ b/tests/Unit/CoreFunctionsTest.php @@ -1741,3 +1741,60 @@ function number_format_i18n($number, $decimals = 0) { }))->toBeEmpty(); }); }); + +describe('stopping the processes of a parallel query', function () { + beforeEach(function () { + plugin_test_reset(); + $GLOBALS['flowview_requests_in_flight'] = array(); + }); + + it('names signals without needing the pcntl constants', function () { + expect(flowview_signal_number('SIGTERM'))->toBe(15) + ->and(flowview_signal_number('SIGKILL'))->toBe(9) + ->and(flowview_signal_number('SIGINT'))->toBe(2) + ->and(flowview_signal_number('SIGNOTREAL'))->toBe(15); + }); + + it('does not signal something that is not a process', function () { + if (!function_exists('posix_kill')) { + $this->markTestSkipped('The posix extension is not available.'); + } + + expect(flowview_signal_process(0))->toBeFalse() + ->and(flowview_signal_process(-5))->toBeFalse() + ->and(flowview_signal_process(4999999))->toBeFalse(); + }); + + it('does not use a signal constant that the web server does not have in the clean up', function () { + $source = plugin_test_read_source('includes/functions.php'); + + expect($source)->not->toMatch('/posix_kill\([^)]*\bSIG[A-Z]+\b/'); + }); + + it('remembers a request until it is cleaned up', function () { + flowview_track_request(991); + flowview_track_request(992); + + parallel_database_query_cleanup(991); + + expect($GLOBALS['flowview_requests_in_flight'])->toBe(array(992 => 992)); + }); + + it('cancels the requests that are not finished, removing their rows', function () { + flowview_track_request(991); + flowview_track_request(992); + + expect(flowview_cancel_requests_in_flight())->toBe(array(991, 992)) + ->and($GLOBALS['flowview_requests_in_flight'])->toBe(array()); + + $removed = array_values(array_filter($GLOBALS['__test_db_calls'], function ($call) { + return $call['fn'] == 'db_execute_prepared' && preg_match('/^DELETE FROM parallel_database_query\s+WHERE id = \?/', $call['sql']) === 1; + })); + + expect(array_column($removed, 'params'))->toBe(array(array(991), array(992))); + }); + + it('cancels nothing when no request is in flight', function () { + expect(flowview_cancel_requests_in_flight())->toBe(array()); + }); +}); From 54fe466772f312ccba034c00f313768cd3d46641 Mon Sep 17 00:00:00 2001 From: TheWitness Date: Wed, 21 Oct 2026 09:47:48 -0400 Subject: [PATCH 21/37] fix: Register a materialization per filter, log its progress, and stop the poller converting under it Each filter a materialization works on is registered in the process table as materialize with the filter id as the task id, so filters can be materialized separately and the Filters page reads which are running from the process table instead of a setting. The poller no longer starts ALTER TABLE conversions while a filter is being materialized, as the materializer waits for any alter on the partitions it needs and the poller kept one running, and the materializer logs its progress to the Cacti log when it has no terminal. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 13 ++- cli/materialize_queries.php | 135 ++++++++++++++++++++++--------- flowview_filters.php | 17 ++-- flowview_process.php | 14 +++- includes/functions.php | 128 ++++++++++++++++------------- tests/Unit/CoreFunctionsTest.php | 117 +++++++++++++++++---------- 6 files changed, 273 insertions(+), 151 deletions(-) diff --git a/README.md b/README.md index 66a243d..bf84f7a 100644 --- a/README.md +++ b/README.md @@ -219,9 +219,16 @@ whose result is reused for 60 seconds and then discarded. Use `--query-id=ID[,I `--nat=on|off` to narrow the run, and `--dry-run` to see what would be run. Partitions that are already cached and still match their data are skipped, while partitions that have received more flows since, or have been dropped, -are rebuilt. Only one run may be active at a time, for up to 8 hours, unless -`--force` is used. `--status` lists the parallel queries that are running now -and the Filter each one is for, with its progress. +are rebuilt. Each Filter being materialized is registered in the Cacti process +table as `materialize`, with the id of the Filter as the task id, for up to 8 +hours, so Filters can be materialized separately and at the same time. A Filter +that is already being materialized is skipped, unless `--force` is used. The +Filters page shows only those Filters as Running. `--status` lists the Filters +being materialized, and the parallel queries that are running now with the +Filter each one is for, with its progress. When it is not run from a terminal, +for instance from the Filters page or by the poller, its progress is written to +the Cacti log. Sending it SIGTERM or SIGINT stops its workers and removes +its registrations from the process table. Before it scans anything the script gets the partitions ready. A query waits for an ALTER TABLE that is running on the partition it scans, and a long one diff --git a/cli/materialize_queries.php b/cli/materialize_queries.php index e898d3b..1b2d0c3 100644 --- a/cli/materialize_queries.php +++ b/cli/materialize_queries.php @@ -52,8 +52,8 @@ $status = false; $worker = false; -/* what the signal handler has to clean up: the process registration, and the worker being run */ -$materialize_registered = false; +/* what the signal handler has to clean up: the filters registered, and the worker being run */ +$materialize_registered = []; $materialize_child = null; $query_ids = []; $partitions = []; @@ -185,18 +185,48 @@ exit(materialize_run_worker($query_ids[0], $nat, $start, $end)); } +/* the filters to materialize, all of them unless some were asked for */ +if (!cacti_sizeof($query_ids)) { + $query_ids = array_map('intval', array_column(flowview_db_fetch_assoc('SELECT id + FROM plugin_flowview_queries + ORDER BY id'), 'id')); +} + +if (!cacti_sizeof($query_ids)) { + print 'NOTE: There are no FlowView queries to materialize.' . PHP_EOL; + exit(0); +} + +/** + * Each filter is registered in the process table as its own materialize + * process, with the filter id as the task id, so the filters can be + * materialized separately and what is being done can be seen there. A filter + * that already has one is left to it. + */ if (!$force && !$dry_run) { - if (!register_process_start('flowview', 'materialize', 0, MATERIALIZE_RUNTIME_LIMIT)) { - print 'NOTE: A materialization is already running. To override use --force' . PHP_EOL; - exit(0); + flowview_purge_dead_materializations(); + + $skipped = []; + + foreach ($query_ids as $query_id) { + if (register_process_start('flowview', 'materialize', $query_id, MATERIALIZE_RUNTIME_LIMIT)) { + $materialize_registered[$query_id] = $query_id; + } else { + $skipped[] = $query_id; + } } - $materialize_registered = true; + if (cacti_sizeof($skipped)) { + materialize_say(sprintf('NOTE: Query(s) %s are already being materialized, so they are skipped. To override use --force', implode(', ', $skipped))); + } + + if (!cacti_sizeof($materialize_registered)) { + exit(0); + } register_shutdown_function('materialize_release'); - /* the Filters page shows only these filters as running */ - flowview_set_materialization_scope($query_ids); + $query_ids = array_values($materialize_registered); } materialize_install_signal_handlers(); @@ -246,12 +276,12 @@ $modes = $nat !== false ? [$nat] : ['on', 'off']; -print sprintf('FlowView Materialization: Partitions:%d From:%s To:%s Queries:%d NAT:%s', +materialize_say(sprintf('FlowView Materialization: Partitions:%d From:%s To:%s Queries:%d NAT:%s', cacti_sizeof($tables), date('Y-m-d H:i:s', flowview_table_name_to_time(reset($tables), 'start_time')), date('Y-m-d H:i:s', flowview_table_name_to_time(end($tables), 'end_time')), cacti_sizeof($queries), - implode('/', $modes)) . PHP_EOL; + implode('/', $modes))); /** * Any alter that is due has to be done before the parallel query starts. A @@ -265,12 +295,12 @@ $live = flowview_remove_live_partition_cache(); if ($live > 0) { - print sprintf('Removed %d cached result(s) of the live partition, as it is still receiving flows', $live) . PHP_EOL; + materialize_say(sprintf('Removed %d cached result(s) of the live partition, as it is still receiving flows', $live)); } $cache = flowview_invalidate_stale_shard_cache(cacti_sizeof($partitions) ? $tables : []); - print sprintf('Cached Partitions Still Accurate:%d, Stale Partitions Removed:%d', $cache['kept'], $cache['removed']) . PHP_EOL; + materialize_say(sprintf('Cached Partitions Still Accurate:%d, Stale Partitions Removed:%d', $cache['kept'], $cache['removed'])); } $began = microtime(true); @@ -301,14 +331,14 @@ if ($status != 0) { $failed++; - print sprintf('ERROR: Query:%d [%s] NAT:%s failed to materialize', $query['id'], $query['name'], $mode) . PHP_EOL; + materialize_say(sprintf('ERROR: Query:%d [%s] NAT:%s failed to materialize', $query['id'], $query['name'], $mode)); } } } $time = microtime(true) - $began; -print sprintf('FlowView Materialization Complete: Time:%0.2f Runs:%d Failed:%d', $time, $total, $failed) . PHP_EOL; +materialize_say(sprintf('FlowView Materialization Complete: Time:%0.2f Runs:%d Failed:%d', $time, $total, $failed)); if ($failed > 0) { print 'NOTE: Partitions that finished materializing are kept, so run the materializer again to continue.' . PHP_EOL; @@ -328,6 +358,14 @@ * @return void */ function materialize_show_status(): void { + $materializing = flowview_get_materializing_filters(); + + if (cacti_sizeof($materializing)) { + print sprintf('Materializing Query(s): %s', implode(', ', $materializing)) . PHP_EOL; + } else { + print 'No Query is being materialized.' . PHP_EOL; + } + $running = flowview_get_running_requests(); if (!cacti_sizeof($running)) { @@ -387,7 +425,7 @@ function materialize_wait_for_alters(array $tables): void { arsort($running); - print sprintf('Waiting for an ALTER TABLE to finish on %d partition(s), the longest %s for %d seconds', cacti_sizeof($running), array_key_first($running), reset($running)) . PHP_EOL; + materialize_say(sprintf('Waiting for an ALTER TABLE to finish on %d partition(s), the longest %s for %d seconds', cacti_sizeof($running), array_key_first($running), reset($running))); } sleep(5); @@ -424,7 +462,7 @@ function materialize_prepare_partitions(array $cold, array $modes, bool $dry_run $pending = flowview_get_pending_partition_alters($cold, $raw_engine); if (cacti_sizeof($pending)) { - print sprintf('%s %d partition(s) that still need an ALTER TABLE', $dry_run ? 'DRY RUN: Would alter' : 'Altering', cacti_sizeof($pending)) . PHP_EOL; + materialize_say(sprintf('%s %d partition(s) that still need an ALTER TABLE', $dry_run ? 'DRY RUN: Would alter' : 'Altering', cacti_sizeof($pending))); $i = 0; @@ -437,7 +475,7 @@ function materialize_prepare_partitions(array $cold, array $modes, bool $dry_run $again = flowview_get_pending_partition_alters([$table], $raw_engine); if (!isset($again[$table])) { - print sprintf('Table %s (%d of %d) was altered by another process', $table, $i, cacti_sizeof($pending)) . PHP_EOL; + materialize_say(sprintf('Table %s (%d of %d) was altered by another process', $table, $i, cacti_sizeof($pending))); continue; } @@ -445,14 +483,14 @@ function materialize_prepare_partitions(array $cold, array $modes, bool $dry_run $plan = $again[$table]; } - print sprintf('%s Table %s (%d of %d): %s', $dry_run ? 'DRY RUN: Would alter' : 'Altering', $table, $i, cacti_sizeof($pending), trim($plan['alter'])) . PHP_EOL; + materialize_say(sprintf('%s Table %s (%d of %d): %s', $dry_run ? 'DRY RUN: Would alter' : 'Altering', $table, $i, cacti_sizeof($pending), trim($plan['alter']))); if (!$dry_run) { $began = microtime(true); flowview_apply_partition_alter($table, $plan, false); - print sprintf('Altered Table %s in %0.2f seconds', $table, microtime(true) - $began) . PHP_EOL; + materialize_say(sprintf('Altered Table %s in %0.2f seconds', $table, microtime(true) - $began)); } } } @@ -460,7 +498,7 @@ function materialize_prepare_partitions(array $cold, array $modes, bool $dry_run $no_nat = in_array('on', $modes, true) ? flowview_get_partitions_without_nat_columns($cold) : []; if (cacti_sizeof($no_nat)) { - print sprintf('NOTE: %d partition(s) have no NAT columns, so they have no NAT data. Run flowview_upgrade_nat_columns.php before materializing if they should.', cacti_sizeof($no_nat)) . PHP_EOL; + materialize_say(sprintf('NOTE: %d partition(s) have no NAT columns, so they have no NAT data. Run flowview_upgrade_nat_columns.php before materializing if they should.', cacti_sizeof($no_nat))); } } @@ -522,19 +560,35 @@ function materialize_sig_handler($signo): void { } /** - * Removes the registration of this materialization from the process table, - * and which Filters it was for. It does nothing the second time it is called. + * Removes the registration of each filter of this materialization from the + * process table. It does nothing the second time it is called. * * @return void */ function materialize_release(): void { global $materialize_registered; - if ($materialize_registered) { - $materialize_registered = false; + foreach ($materialize_registered as $query_id) { + unregister_process('flowview', 'materialize', $query_id, (int) getmypid()); + } + + $materialize_registered = []; +} + +/** + * Says something, on the screen, and also in the Cacti log when the output is + * not a terminal. A materialization started from the Filters page, or by the + * poller, has nowhere to show it otherwise. + * + * @param string $text + * + * @return void + */ +function materialize_say(string $text): void { + print $text . PHP_EOL; - flowview_set_materialization_scope([]); - unregister_process('flowview', 'materialize', 0, (int) getmypid()); + if (!function_exists('posix_isatty') || !posix_isatty(STDOUT)) { + cacti_log($text, false, 'FLOWVIEW'); } } @@ -635,8 +689,9 @@ function materialize_run_worker(int $query_id, string $nat, int $start, int $end set_request_var('usenat', $nat == 'on' ? 'true' : 'false'); - $last_request = (int) flowview_db_fetch_cell('SELECT COALESCE(MAX(id), 0) - FROM parallel_database_query'); + $GLOBALS['flowview_requests_made'] = []; + + materialize_say(sprintf('Materializing Query:%d NAT:%s', $query_id, $nat)); $began = microtime(true); @@ -645,29 +700,31 @@ function materialize_run_worker(int $query_id, string $nat, int $start, int $end $finished = true; if ($data === false) { - print sprintf('ERROR: Query:%d NAT:%s did not return a report. See the Cacti log for details.', $query_id, $nat) . PHP_EOL; + materialize_say(sprintf('ERROR: Query:%d NAT:%s did not return a report. See the Cacti log for details.', $query_id, $nat)); return 1; } - $stats = flowview_db_fetch_row_prepared('SELECT COUNT(*) AS requests, + /* only the requests of this worker, as the workers of other filters can run at the same time */ + $made = array_map('intval', array_values($GLOBALS['flowview_requests_made'])); + + $stats = cacti_sizeof($made) ? flowview_db_fetch_row_prepared('SELECT COUNT(*) AS requests, COALESCE(SUM(total_shards), 0) AS shards, COALESCE(SUM(cached_shards), 0) AS cached FROM parallel_database_query - WHERE user_id = 0 - AND id > ?', - [$last_request]); + WHERE id IN (' . implode(', ', array_fill(0, cacti_sizeof($made), '?')) . ')', + $made) : []; if (cacti_sizeof($stats) && $stats['requests'] > 0) { - print sprintf('Query:%d NAT:%s Partitions:%d AlreadyMaterialized:%d Materialized:%d Time:%0.2f', + materialize_say(sprintf('Query:%d NAT:%s Partitions:%d AlreadyMaterialized:%d Materialized:%d Time:%0.2f', $query_id, $nat, $stats['shards'], $stats['cached'], $stats['shards'] - $stats['cached'], - microtime(true) - $began) . PHP_EOL; + microtime(true) - $began)); } else { - print sprintf('Query:%d NAT:%s Already Materialized and Accurate Time:%0.2f', $query_id, $nat, microtime(true) - $began) . PHP_EOL; + materialize_say(sprintf('Query:%d NAT:%s Already Materialized and Accurate Time:%0.2f', $query_id, $nat, microtime(true) - $began)); } return 0; @@ -700,7 +757,9 @@ function display_help(): void { print 'flow partition that has rotated, once with NAT on and once with NAT off.' . PHP_EOL; print 'The partition still receiving flows is never cached. Partitions that are' . PHP_EOL; print 'already materialized and still accurate are skipped.' . PHP_EOL; - print 'Only one run may be active at a time, for up to 8 hours.' . PHP_EOL . PHP_EOL; + print 'Each Filter is registered in the process table as materialize, with its id as the task id,' . PHP_EOL; + print 'so Filters can be materialized separately. A Filter that is already being materialized' . PHP_EOL; + print 'is skipped. Registrations last up to 8 hours.' . PHP_EOL . PHP_EOL; print 'Options:' . PHP_EOL; print ' --query-id=ID Only materialize this query, or a comma separated list of them.' . PHP_EOL; @@ -709,6 +768,6 @@ function display_help(): void { print ' --nat=on|off Only materialize this NAT mode.' . PHP_EOL; print ' --dry-run List what would be materialized and exit.' . PHP_EOL; print ' --status List the parallel queries that are running and the Filter each is for.' . PHP_EOL; - print ' --force Run even if another materialization is running.' . PHP_EOL; + print ' --force Run even if a Filter is already being materialized.' . PHP_EOL; print ' --debug Provide some debug output during the run.' . PHP_EOL . PHP_EOL; } diff --git a/flowview_filters.php b/flowview_filters.php index 6695c29..1a30af9 100644 --- a/flowview_filters.php +++ b/flowview_filters.php @@ -182,10 +182,15 @@ function actions_filters(): void { flowview_db_execute("UPDATE plugin_flowview_queries SET enabled='on' WHERE id = " . $selected_items[$i]); } } elseif (get_nfilter_request_var('drp_action') == '5') { - if (flowview_launch_materialize($selected_items)) { - raise_message('flowview_materialize', __('Materialization of %d Filter(s) was started in the background. Its results are reported in the Cacti log.', cacti_count($selected_items), 'flowview'), MESSAGE_LEVEL_INFO); + $started = flowview_launch_materialize($selected_items); + $skipped = cacti_count($selected_items) - cacti_sizeof($started); + + if (cacti_sizeof($started) && $skipped > 0) { + raise_message('flowview_materialize', __('Materialization of %d Filter(s) was started in the background. %d Filter(s) are already being materialized. Its results are reported in the Cacti log.', cacti_sizeof($started), $skipped, 'flowview'), MESSAGE_LEVEL_INFO); + } elseif (cacti_sizeof($started)) { + raise_message('flowview_materialize', __('Materialization of %d Filter(s) was started in the background. Its results are reported in the Cacti log.', cacti_sizeof($started), 'flowview'), MESSAGE_LEVEL_INFO); } else { - raise_message('flowview_materialize', __('A Materialization is already running. Please try again once it has finished.', 'flowview'), MESSAGE_LEVEL_WARN); + raise_message('flowview_materialize', __('The selected Filter(s) are already being materialized. Please try again once that has finished.', 'flowview'), MESSAGE_LEVEL_WARN); } /* } elseif (get_nfilter_request_var('drp_action') == '2') { for ($i=0; $i -

" . __('Click \'Continue\' to Materialize the following Filter(s) in the background, with NAT on and off, for the entire range of flow data. Partitions that are already materialized and accurate are skipped. This can take a long time on large data sets, and only one Materialization can run at a time.', 'flowview') . "

+

" . __('Click \'Continue\' to Materialize the following Filter(s) in the background, with NAT on and off, for the entire range of flow data. Partitions that are already materialized and accurate are skipped. This can take a long time on large data sets. A Filter that is already being materialized is skipped.', 'flowview') . "

    $filter_list
- > + 0 && !$report_fixed; + + if (!$report_shown) { + print ""; + } else { + ?> + + diff --git a/tests/Unit/FlowviewFilterControlsTest.php b/tests/Unit/FlowviewFilterControlsTest.php index 7513659..1b08e0c 100644 --- a/tests/Unit/FlowviewFilterControlsTest.php +++ b/tests/Unit/FlowviewFilterControlsTest.php @@ -61,3 +61,50 @@ function array_rekey($array, $key, $value) { expect($output)->toContain('timeshiftForward'); expect($output)->not->toMatch('/\son(?:click|change)\s*=/i'); }); + +it('does not show the Report on the report page, but keeps its value for the requests', function ($query, $row) { + plugin_test_reset(); + + $_REQUEST['query'] = $query; + $_REQUEST['report'] = 's10'; + + plugin_test_queue_db_result('db_fetch_row_prepared', $row); + + ob_start(); + + try { + flowview_display_filter(); + } catch (\Throwable $e) { + // as above, the part of the form that is checked is emitted first + } + + $output = ob_get_clean(); + + expect($output)->toContain("") + ->and($output)->not->toContain("") + ->and($output)->not->toContain("type='hidden' id='report'"); +}); From 5db13c6f051c244eda103c1fc1b075ba5fea8e53 Mon Sep 17 00:00:00 2001 From: TheWitness Date: Wed, 21 Oct 2026 10:03:02 -0400 Subject: [PATCH 24/37] fix: Take the Report off the report page, the server knows it from the filter No Report control or field is on the page for a filter that has a report. The requests no longer send it, the server takes it from the filter for the page, the charts and the session, a printed report has its panel checkboxes disabled by the server, and Save no longer changes the report of a filter. A filter with no report still shows the selector as it needs one. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- flowview.php | 20 ++++++ includes/functions.php | 86 +++++++++++++++-------- tests/Unit/FlowviewFilterControlsTest.php | 34 +++++++-- 3 files changed, 104 insertions(+), 36 deletions(-) diff --git a/flowview.php b/flowview.php index 7437708..c3e8448 100644 --- a/flowview.php +++ b/flowview.php @@ -320,6 +320,14 @@ function load_session_for_filter() { if ($report != '') { set_request_var('report', $report); + + /* a printed report is a table, there is nothing to chart */ + if (substr($report, 0, 1) == 'p') { + set_request_var('table', 'true'); + set_request_var('bytes', 'false'); + set_request_var('packets', 'false'); + set_request_var('flows', 'false'); + } } // Handle Report Column @@ -503,6 +511,18 @@ function flowview_request_vars(): void { } } + /* the report is not sent for a Filter that has one, as it is part of the Filter */ + if (isset_request_var('query') && get_filter_request_var('query') > 0) { + $filter_report = flowview_filter_report_key(flowview_db_fetch_row_prepared('SELECT statistics, printed + FROM plugin_flowview_queries + WHERE id = ?', + array(get_request_var('query'))) ?: []); + + if ($filter_report != '') { + set_request_var('report', $filter_report); + } + } + /* ================= input validation and session storage ================= */ $filters = array( 'device_id' => [ diff --git a/includes/functions.php b/includes/functions.php index a918ee0..09f163c 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -572,6 +572,11 @@ function save_filter_form(): void { WHERE id = ?', array(get_request_var('query'))); + /* the report of a Filter is not on the page, so it is not changed from here */ + if (is_array($old) && flowview_filter_report_key($old) != '') { + $statistical = (int) $old['statistics']; + $printed = (int) $old['printed']; + } $material = flowview_filter_change_is_material( is_array($old) ? $old : [], [ @@ -846,7 +851,8 @@ function flowview_display_filter(): void { $title = __esc('Undefined Filter [ Select Filter to get Details ]', 'flowview'); - $report_fixed = false; + $report_fixed = false; + $report_printed = substr((string) get_nfilter_request_var('report'), 0, 1) == 'p'; if (get_filter_request_var('query') > 0) { $row = flowview_db_fetch_row_prepared('SELECT name, statistics, printed @@ -959,12 +965,10 @@ function flowview_display_filter(): void {
-
0 && !$report_fixed; - if (!$report_shown) { - print ""; - } else { + if ($report_shown) { ?> @@ -1643,14 +1647,16 @@ function exportData() { initTimespan(); } - if ($('#report').val().indexOf('p') >= 0) { - $('#bytes').prop('disabled', true); - $('#flows').prop('disabled', true); - $('#packets').prop('disabled', true); - } else { - $('#bytes').prop('disabled', false); - $('#flows').prop('disabled', false); - $('#packets').prop('disabled', false); + if ($('#report').length) { + if ($('#report').val().indexOf('p') >= 0) { + $('#bytes').prop('disabled', true); + $('#flows').prop('disabled', true); + $('#packets').prop('disabled', true); + } else { + $('#bytes').prop('disabled', false); + $('#flows').prop('disabled', false); + $('#packets').prop('disabled', false); + } } // Setup the charts @@ -1693,12 +1699,17 @@ function exportData() { }); }); + /* the Report is not on the page when the Filter has one, as the server knows it from the Filter */ + function reportParam() { + return $('#report').length ? '&report=' + $('#report').val() : ''; + } + function renderBarChart(type, bindto, title, label, height, width) { $.getJSON('flowview.php?action=chartdata&type=' + type + '&domains=' + $('#domains').is(':checked') + '&usenat=' + $('#usenat').is(':checked') + '&query=' + $('#query').val() + - '&report=' + $('#report').val() + + reportParam() + '&device_id=' + $('#device_id').val() + '&ex_addr=' + ($('#ex_addr').val() != null ? $('#ex_addr').val():'') + '&sortfield=' + ($('#sortfield').val() != null ? $('#sortfield').val():'') + @@ -1763,7 +1774,7 @@ function renderTreemapChart(type, bindto, title, label, height, width) { '&domains=' + $('#domains').is(':checked') + '&usenat=' + $('#usenat').is(':checked') + '&query=' + $('#query').val() + - '&report=' + $('#report').val() + + reportParam() + '&device_id=' + $('#device_id').val() + '&ex_addr=' + ($('#ex_addr').val() != null ? $('#ex_addr').val():'') + '&sortfield=' + ($('#sortfield').val() != null ? $('#sortfield').val():'') + @@ -1829,7 +1840,7 @@ function renderPieChart(type, bindto, title, label, height, width) { '&domains=' + $('#domains').is(':checked') + '&usenat=' + $('#usenat').is(':checked') + '&query=' + $('#query').val() + - '&report=' + $('#report').val() + + reportParam() + '&device_id=' + $('#device_id').val() + '&ex_addr=' + ($('#ex_addr').val() != null ? $('#ex_addr').val():'') + '&sortfield=' + ($('#sortfield').val() != null ? $('#sortfield').val():'') + @@ -1925,7 +1936,7 @@ function saveFilter() { '&domains=' + $('#domains').is(':checked') + '&usenat=' + $('#usenat').is(':checked') + '×pan=' + $('#predefined_timespan').val() + - '&report=' + $('#report').val() + + reportParam() + '&device_id=' + $('#device_id').val() + '&ex_addr=' + ($('#ex_addr').val() != null ? $('#ex_addr').val():'') + '&sortfield=' + ($('#sortfield').val() != null ? $('#sortfield').val():'') + @@ -2024,18 +2035,21 @@ function changeQuery() { } function applyFilter(reset) { + var report = ''; + var extra = ''; + if (reset) { - var report = 0; + report = '&report=0'; var device_id = 0; } else { - var report = $('#report').val(); - var device_id = $('#device_id').val(); - } + /* only a Filter that has no Report has one on the page */ + report = reportParam(); - if (report.indexOf('p') >= 0) { - var extra='&table=true&bytes=false&packets=false&flows=false'; - } else { - var extra=''; + if ($('#report').length && $('#report').val().indexOf('p') >= 0) { + extra = '&table=true&bytes=false&packets=false&flows=false'; + } + + var device_id = $('#device_id').val(); } loadPageNoHeader(urlPath+'plugins/flowview/flowview.php' + @@ -2044,7 +2058,7 @@ function applyFilter(reset) { '&usenat=' + $('#usenat').is(':checked') + '&query=' + $('#query').val() + '&predefined_timespan=' + $('#predefined_timespan').val() + - '&report=' + report + + report + '&device_id=' + device_id + '&ex_addr=' + ($('#ex_addr').val() != null ? $('#ex_addr').val():'') + '&sortfield=' + ($('#sortfield').val() != null ? $('#sortfield').val():'') + @@ -2713,8 +2727,20 @@ function flowview_get_chartdata(): void { $start = strtotime(get_nfilter_request_var('date1')); $end = strtotime(get_nfilter_request_var('date2')); + /* the report is not sent for a Filter that has one, as it is part of the Filter */ + if ($query_id > 0) { + $filter_report = flowview_filter_report_key(flowview_db_fetch_row_prepared('SELECT statistics, printed + FROM plugin_flowview_queries + WHERE id = ?', + [$query_id]) ?: []); + + if ($filter_report != '') { + $report = $filter_report; + } + } + if (!empty($report) && substr($report, 0, 1) == 's') { - $report = trim(get_nfilter_request_var('report'), 'sp'); + $report = trim($report, 'sp'); $output = run_flow_query(true, $query_id, $start, $end); diff --git a/tests/Unit/FlowviewFilterControlsTest.php b/tests/Unit/FlowviewFilterControlsTest.php index 1b08e0c..d3b64b2 100644 --- a/tests/Unit/FlowviewFilterControlsTest.php +++ b/tests/Unit/FlowviewFilterControlsTest.php @@ -62,7 +62,7 @@ function array_rekey($array, $key, $value) { expect($output)->not->toMatch('/\son(?:click|change)\s*=/i'); }); -it('does not show the Report on the report page, but keeps its value for the requests', function ($query, $row) { +it('does not put the Report on the report page for a filter that has one', function ($query, $row) { plugin_test_reset(); $_REQUEST['query'] = $query; @@ -80,14 +80,37 @@ function array_rekey($array, $key, $value) { $output = ob_get_clean(); - expect($output)->toContain("") - ->and($output)->not->toContain("") - ->and($output)->not->toContain("type='hidden' id='report'"); + expect($output)->toContain("'; - $save_html = ''; - } elseif ($summary['other']['shards'] == 0) { - print "'; - $save_html = ''; - } else { - print " - - "; - $save_html = "'; - } - - print " - - "; - - html_end_box(); - - form_end(); -} - /** * Removes the cached shards that match no Filter and returns to the Filters - * page. Called from this script's main request-dispatch switch when - * action=prune_shards. + * page. It is posted by the confirmation dialog of the Prune button. Called + * from this script's main request-dispatch switch when action=prune_shards. * * @return void */ function prune_shards(): void { + /* it removes data, so it is only done when posted from the dialog */ + if (($_SERVER['REQUEST_METHOD'] ?? 'GET') !== 'POST') { + header('Location: flowview_filters.php?header=false'); + exit; + } + $pruned = flowview_prune_shard_cache(); if ($pruned === null) { @@ -166,17 +124,6 @@ function actions_filters(): void { get_filter_request_var('drp_action'); /* ==================================================== */ - /* pruning is for the whole cache, so it does not depend on which Filters are selected */ - if (get_nfilter_request_var('drp_action') == '6') { - general_header(); - - confirm_prune_shards(); - - bottom_footer(); - - return; - } - if (isset_request_var('selected_items')) { $selected_items = sanitize_unserialize_selected_items(get_nfilter_request_var('selected_items')); @@ -418,10 +365,6 @@ function clearFilter() { clearFilter(); }); - $('#prune').on('click', function() { - loadPageNoHeader('flowview_filters.php?action=prune&header=false'); - }); - $('#rows').on('change', function() { applyFilter(); }); @@ -595,17 +538,77 @@ function clearFilter() { */ function show_shard_cache_summary(): void { $summary = flowview_get_shard_cache_summary(); + $prune = false; html_start_box('', '100%', false, 3, 'center', ''); if ($summary === null) { - $text = __('The cached shards of the Filters could not be told apart from the others, as the cache keys of a Filter could not be worked out.', 'flowview'); + $text = __('The cached shards of the Filters could not be told apart from the others, as the cache keys of a Filter could not be worked out.', 'flowview'); + $dialog = __('The cache keys of the Filters could not be worked out, so nothing can be pruned right now. Check that each Filter has a valid Report.', 'flowview'); } else { $text = __('Cached Shards: %s matching a Filter. %s not matching any Filter, which Prune removes.', number_format_i18n($summary['matching']['shards'], 0), number_format_i18n($summary['other']['shards'], 0), 'flowview'); + + if ($summary['other']['shards'] > 0) { + $prune = true; + $dialog = __('This removes %s cached shard(s) that do not match any Filter, for every Filter and every user. They come from Filters that were changed or deleted, and from reports with settings changed on the report page. The %s cached shard(s) of the Filters are kept. This can not be undone.', number_format_i18n($summary['other']['shards'], 0), number_format_i18n($summary['matching']['shards'], 0), 'flowview'); + } else { + $dialog = __('Every cached shard matches a Filter. There is nothing to prune.', 'flowview'); + } } print "'; html_end_box(); + + $js = JSON_HEX_TAG | JSON_HEX_APOS | JSON_HEX_QUOT | JSON_HEX_AMP; + + print "'; + + ?> + + toBeEmpty(); })->with(array('pending', 'scheduled', 'running', '')); - it('offers pruning in the actions of the Filters page, for the whole cache', function () { + it('prunes from a button with a confirmation dialog and a post, not from the actions', function () { $source = plugin_test_read_source('flowview_filters.php'); - expect($source)->toContain("6 => __('Prune Cached Shards'") - ->and($source)->toMatch("/drp_action'\) == '6'\) \{[^}]*confirm_prune_shards\(\)/s"); + expect($source)->not->toContain("6 => __('Prune Cached Shards'") + ->and($source)->not->toContain("drp_action') == '6'") + ->and($source)->not->toContain('function confirm_prune_shards') + ->and($source)->toContain("id='prune_dialog'") + ->and($source)->toContain("\$('#prune_dialog').dialog(") + ->and($source)->toContain("\$.post('flowview_filters.php', data)") + ->and($source)->toMatch("/function prune_shards\(\): void \{.*?REQUEST_METHOD'\] \?\? 'GET'\) !== 'POST'/s") + ->and($source)->toMatch("/id='clear'[^\n]*\n\s*
@@ -1208,19 +1212,19 @@ function flowview_display_filter(): void { - > + > - > + > - > + >
" . __('The cache keys of the Filters could not be worked out, so nothing can be pruned right now. Check that each Filter has a valid Report.', 'flowview') . '

" . __('Every cached shard matches a Filter. There is nothing to prune.', 'flowview') . '

-

" . __('Click \'Continue\' to remove %s cached shard(s) that do not match any Filter. They come from Filters that were changed or deleted, and from reports with settings changed on the report page. The %s cached shard(s) of the Filters are kept.', number_format_i18n($summary['other']['shards'], 0), number_format_i18n($summary['matching']['shards'], 0), 'flowview') . "

-
- - - $save_html -
" . html_escape($text) . '
'); + $none = array(); + + ob_start(); + flowview_show_summary($report); + flowview_show_summary($none); + + expect(ob_get_clean())->toBe('
'); + }); +}); + +describe('a parallel query that does not finish in time', function () { + it('says so in words from a script, cleans up, and ends with a failure', function () { + $GLOBALS['__test_config']['flowview_parallel_runlimit'] = 0; + $GLOBALS['flowview_requests_in_flight'] = array(5 => 5); + + ob_start(); + + try { + parallel_database_query_run(array(5)); + $status = null; + } catch (FlowviewTestExit $e) { + $status = $e->getCode(); + } + + $printed = ob_get_clean(); + + expect($status)->toBe(1) + ->and($printed)->toContain('did not finish within the Maximum Run Time') + ->and($printed)->not->toContain('and($GLOBALS['flowview_error_reported'])->toBeTrue() + ->and($GLOBALS['flowview_requests_in_flight'])->toBe(array()); + + unset($GLOBALS['flowview_error_reported']); + }); +}); + +describe('cancelling a parallel query', function () { + it('kills its runner and shard workers and removes them', function () { + if (!function_exists('posix_kill')) { + $this->markTestSkipped('The posix extension is not available.'); + } + + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('id' => 9, 'pid' => 4999999, 'taskname' => 'db_shard_5'))); + + parallel_database_query_cancel(5); + + expect(plugin_test_db_calls('db_execute_prepared', 'DELETE FROM processes'))->toHaveCount(1) + ->and(implode("\n", $GLOBALS['__test_cacti_log']))->toContain('Killing FlowView Query Process with Task Name:db_shard_5'); + }); +}); + +describe('a shard worker on the live partition', function () { + $query = array('id' => 5, 'map_table' => 'parallel_database_query_map_5', 'md5sum_tables' => 'abc'); + $shard = array('shard_id' => 2, 'full_scan' => 0, 'map_table' => 'plugin_flowview_raw_2024002', 'map_partition' => 'live', 'map_query' => 'SELECT 1', 'map_params' => '[]'); + + it('runs the scan once and keeps it for the other panels', function () use ($query, $shard) { + plugin_test_queue_db_result('db_fetch_row_prepared', $query); + plugin_test_queue_db_result('db_fetch_row_prepared', $shard); + plugin_test_queue_db_result('db_fetch_cell_prepared', '1'); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('src_addr' => '10.0.0.1', 'bytes' => 5))); + + parallel_database_child_runner(5, 2); + + expect(plugin_test_db_calls('db_execute_prepared', 'REPLACE INTO parallel_database_query_shard_cache'))->toHaveCount(1) + ->and(plugin_test_db_calls('db_fetch_cell_prepared', 'RELEASE_LOCK'))->toHaveCount(1) + ->and(plugin_test_db_calls('db_execute', 'INSERT INTO parallel_database_query_map_5'))->toHaveCount(1) + ->and($GLOBALS['__test_unregistered'])->toBe(array(array('flowview', 'db_shard_5', 2))); + }); + + it('reuses the scan another worker just ran', function () use ($query, $shard) { + plugin_test_queue_db_result('db_fetch_row_prepared', $query); + plugin_test_queue_db_result('db_fetch_row_prepared', $shard); + plugin_test_queue_db_result('db_fetch_cell_prepared', '1'); + plugin_test_queue_db_result('db_fetch_row_prepared', array('results' => json_encode(array(array('src_addr' => '10.0.0.1', 'bytes' => 5))))); + + parallel_database_child_runner(5, 2); + + expect(plugin_test_db_calls('db_fetch_assoc_prepared', 'SELECT 1'))->toBeEmpty() + ->and(plugin_test_db_calls('db_execute_prepared', 'REPLACE INTO parallel_database_query_shard_cache'))->toBeEmpty() + ->and(plugin_test_db_calls('db_execute', 'INSERT INTO parallel_database_query_map_5'))->toHaveCount(1); + }); + + it('serves a cold partition from its cached shard', function () use ($query, $shard) { + plugin_test_queue_db_result('db_fetch_row_prepared', $query); + plugin_test_queue_db_result('db_fetch_row_prepared', array_merge($shard, array('full_scan' => 1, 'map_partition' => ''))); + plugin_test_queue_db_result('db_fetch_row_prepared', array('results' => json_encode(array(array('src_addr' => '10.0.0.1', 'bytes' => 5))))); + + parallel_database_child_runner(5, 2); + + expect(plugin_test_db_calls('db_execute_prepared', 'SET cached_shards = cached_shards + 1'))->toHaveCount(1) + ->and(plugin_test_db_calls('db_fetch_cell_prepared', 'GET_LOCK'))->toBeEmpty(); + }); + + it('scans a whole cold partition and caches it for the next report', function () use ($query, $shard) { + plugin_test_queue_db_result('db_fetch_row_prepared', $query); + plugin_test_queue_db_result('db_fetch_row_prepared', array_merge($shard, array('full_scan' => 1, 'map_partition' => ''))); + plugin_test_queue_db_result('db_fetch_row_prepared', array()); + plugin_test_queue_db_result('db_fetch_assoc_prepared', array(array('src_addr' => '10.0.0.1', 'bytes' => 5))); + + parallel_database_child_runner(5, 2); + + $cached = plugin_test_db_calls('db_execute_prepared', 'INSERT INTO parallel_database_query_shard_cache'); + + expect($cached)->toHaveCount(1) + ->and($cached[0]['params'][3])->toBe(date('Y-m-d 00:00:00')) + ->and(plugin_test_db_calls('db_fetch_assoc_prepared', 'SELECT 1'))->toHaveCount(1); + }); +}); + +describe('the rest of the partition bookkeeping', function () { + it('reads every partition when it is asked about many, and notes the old key column', function () { + $tables = array_map(fn ($i) => sprintf('plugin_flowview_raw_2024%03d', $i), range(1, 101)); + + plugin_test_queue_db_result('db_fetch_assoc', array(array('TABLE_NAME' => $tables[0], 'TABLE_COLLATION' => 'latin1_swedish_ci', 'ENGINE' => 'Aria'))); + plugin_test_queue_db_result('db_fetch_assoc', array( + array('TABLE_NAME' => $tables[0], 'COLUMN_NAME' => 'keycol', 'COLUMN_TYPE' => 'int(11)'), + array('TABLE_NAME' => 'plugin_flowview_raw_2099001', 'COLUMN_NAME' => 'template_id', 'COLUMN_TYPE' => 'int(11)'), + )); + + $pending = flowview_get_pending_partition_alters($tables, 'Aria'); + + expect($GLOBALS['__test_db_calls'][0]['sql'])->toContain('TABLE_NAME LIKE "plugin_flowview_raw_%"') + ->and($pending[$tables[0]]['alter'])->toContain('DROP KEY keycol'); + }); + + it('records the newest partition when nothing rotated but it is not the one recorded', function () { + $GLOBALS['__test_config']['flowview_last_hot_partition'] = 'plugin_flowview_raw_2024009'; + + plugin_test_queue_db_result('db_fetch_assoc', array(array('table' => 'plugin_flowview_raw_2024001'), array('table' => 'plugin_flowview_raw_2024003'))); + + expect(flowview_materialize_rotated_partitions())->toBe(array()) + ->and($GLOBALS['__test_config']['flowview_last_hot_partition'])->toBe('plugin_flowview_raw_2024003'); + }); + + it('keeps the record when there is no Filter to materialize the rotated partitions for', function () { + $GLOBALS['__test_config']['flowview_parallel_threads'] = 4; + $GLOBALS['__test_config']['flowview_last_hot_partition'] = 'plugin_flowview_raw_2024001'; + + plugin_test_queue_db_result('db_fetch_assoc', array(array('table' => 'plugin_flowview_raw_2024001'), array('table' => 'plugin_flowview_raw_2024003'))); + + expect(flowview_materialize_rotated_partitions())->toBe(array()) + ->and($GLOBALS['__test_config']['flowview_last_hot_partition'])->toBe('plugin_flowview_raw_2024001'); + }); +}); diff --git a/tests/bootstrap-unit.php b/tests/bootstrap-unit.php index 5d48cff..8b28464 100644 --- a/tests/bootstrap-unit.php +++ b/tests/bootstrap-unit.php @@ -389,7 +389,7 @@ function form_input_validate($value, $name, $regex, $optional, $error) { if (!function_exists('is_error_message')) { function is_error_message() { - return false; + return !empty($GLOBALS['__test_error_message']); } } @@ -399,6 +399,18 @@ function sql_save($array, $table, $key = 'id') { } } +if (!class_exists('FlowviewTestExit')) { + /* thrown by the flowview_exit() stand in, so a test sees where a request ends */ + class FlowviewTestExit extends RuntimeException { + } +} + +if (!function_exists('flowview_exit')) { + function flowview_exit(int $status = 0): void { + throw new FlowviewTestExit('flowview_exit', $status); + } +} + if (!defined('CACTI_PATH_BASE')) { define('CACTI_PATH_BASE', $GLOBALS['config']['base_path']); } From 42aa70734ff1def72d36b49bda94b4bac7909584 Mon Sep 17 00:00:00 2001 From: TheWitness Date: Thu, 22 Oct 2026 09:25:55 -0400 Subject: [PATCH 37/37] tests: Cover both ways a parallel query timeout is reported The web page branch of the timeout report depends on CACTI_WEB, a constant a unit test cannot change, so the report moves into flowview_report_parallel_timeout() with whether it is a web request as an argument, and both ways are tested. The cleanup of a request's map table and db_debug() are covered too, as the change moves the diff onto them. The test stub of form_start() records its arguments instead of printing them, which SonarCloud took for a reflected XSS. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- includes/functions.php | 39 ++++++++++++++++------- tests/Unit/FilterEditAndSaveTest.php | 3 +- tests/Unit/ParallelQueryPathTest.php | 47 +++++++++++++++++++++++++++- 3 files changed, 75 insertions(+), 14 deletions(-) diff --git a/includes/functions.php b/includes/functions.php index 17b7fc2..668717b 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -6114,6 +6114,30 @@ function flowview_parallel_timeout_message(int $seconds): string { return __('ERROR: The Parallel Query did not finish within the Maximum Run Time of %d seconds. Check the Maximum Run Time under the FlowView Settings, and that the database is not busy with another operation such as an ALTER TABLE.', $seconds, 'flowview'); } +/** + * flowview_report_parallel_timeout - Log that a parallel query did not finish + * within the Maximum Run Time, and say so. The web page shows the message in + * the browser, a script has no browser to run it, so it is printed. + * + * @param int $seconds The Maximum Run Time + * @param bool $web Whether this is a web request + * + * @return void + */ +function flowview_report_parallel_timeout(int $seconds, bool $web): void { + $timeout_message = flowview_parallel_timeout_message($seconds); + + cacti_log($timeout_message, false, 'FLOWVIEW'); + + if ($web) { + raise_message_javascript('parallel_query_timeout', __('The Parallel Query Timed Out. Please contact your administrator', 'flowview'), MESSAGE_LEVEL_ERROR); + } else { + print $timeout_message . PHP_EOL; + + $GLOBALS['flowview_error_reported'] = true; + } +} + /** * parallel_database_query_is_running - Checks to see if an existing parallel * database query is currently running and return a true of false. This prevents @@ -6200,24 +6224,15 @@ function parallel_database_query_run($requests) { } if ($total_time >= $max_time) { - $timeout_message = flowview_parallel_timeout_message((int) $max_time); + $web = defined('CACTI_WEB') && CACTI_WEB; - cacti_log($timeout_message, false, 'FLOWVIEW'); - - /* the web page shows the message in the browser, a script has no browser to run it */ - if (defined('CACTI_WEB') && CACTI_WEB) { - raise_message_javascript('parallel_query_timeout', __('The Parallel Query Timed Out. Please contact your administrator', 'flowview'), MESSAGE_LEVEL_ERROR); - } else { - print $timeout_message . PHP_EOL; - - $GLOBALS['flowview_error_reported'] = true; - } + flowview_report_parallel_timeout((int) $max_time, $web); foreach($requests as $request_id) { parallel_database_query_cleanup($request_id); } - flowview_exit(defined('CACTI_WEB') && CACTI_WEB ? 0 : 1); + flowview_exit($web ? 0 : 1); } if (cacti_sizeof($requests) == 1) { diff --git a/tests/Unit/FilterEditAndSaveTest.php b/tests/Unit/FilterEditAndSaveTest.php index 1700f3b..5a2fa5d 100644 --- a/tests/Unit/FilterEditAndSaveTest.php +++ b/tests/Unit/FilterEditAndSaveTest.php @@ -17,7 +17,7 @@ require_once __DIR__ . '/../../includes/functions.php'; if (!function_exists('isempty_request_var')) { function isempty_request_var($name) { return empty($_REQUEST[$name]); } } - if (!function_exists('form_start')) { function form_start($action, $id = '') { print "
"; } } + if (!function_exists('form_start')) { function form_start($action, $id = '') { $GLOBALS['__test_form_start'] = array($action, $id); } } if (!function_exists('html_start_box')) { function html_start_box(...$args) {} } if (!function_exists('html_end_box')) { function html_end_box(...$args) {} } if (!function_exists('read_user_setting')) { function read_user_setting($name, $default = false) { return $default; } } @@ -87,6 +87,7 @@ function plugin_test_stored_filter(array $override = array()): array { ->and($output)->toContain("\$('#protocols').multiselect();") ->and($output)->toContain("id='flowview_saved_dialog'") ->and(isset($_SESSION['flowview_rematerialize_saved']))->toBeFalse() + ->and($GLOBALS['__test_form_start'][1])->toBe('chk') ->and($GLOBALS['__test_edit_form']['fields']['protocols']['select_all_text'])->toBe('All Protocols'); }); diff --git a/tests/Unit/ParallelQueryPathTest.php b/tests/Unit/ParallelQueryPathTest.php index c1ff49a..8b021b0 100644 --- a/tests/Unit/ParallelQueryPathTest.php +++ b/tests/Unit/ParallelQueryPathTest.php @@ -21,6 +21,12 @@ if (!function_exists('db_qstr')) { function db_qstr($value) { return "'" . addslashes((string) $value) . "'"; } } if (!function_exists('number_format_i18n')) { function number_format_i18n($number, $decimals = 0) { return number_format($number, $decimals); } } + if (!function_exists('raise_message_javascript')) { + function raise_message_javascript($title, $message, $level) { + $GLOBALS['__test_javascript_messages'][] = array($title, $message, $level); + } + } + if (!function_exists('unregister_process')) { function unregister_process($tasktype, $taskname, $taskid = 0, $pid = 0) { $GLOBALS['__test_unregistered'][] = array($tasktype, $taskname, $taskid); @@ -45,7 +51,7 @@ function array_rekey($array, $key, $value) { $GLOBALS['__test_cacti_log'] = array(); $GLOBALS['__test_unregistered'] = array(); $GLOBALS['flowview_requests_in_flight'] = array(); - unset($GLOBALS['flowview_fresh_requests'], $GLOBALS['flowview_partition_allowlist']); + unset($GLOBALS['flowview_fresh_requests'], $GLOBALS['flowview_partition_allowlist'], $GLOBALS['flowview_error_reported']); }); function plugin_test_query_filter(array $override = array()): array { @@ -320,6 +326,31 @@ function plugin_test_run_packet_summary(int $packets): array { ->and(plugin_test_db_calls('db_fetch_assoc', 'SELECT map_table, map_create, results'))->toHaveCount(1); }); + it('drops the table a request reduced its shards into when it is cleaned up', function () { + plugin_test_queue_db_result('db_fetch_cell_prepared', 'parallel_database_query_map_5'); + + parallel_database_query_cleanup(5, true); + + expect(plugin_test_db_calls('db_execute_prepared', 'DROP TABLE IF EXISTS parallel_database_query_map_5'))->toHaveCount(1); + }); + + it('logs the progress of a shard only when debugging', function () { + global $debug; + + $saved = $debug; + + $debug = false; + db_debug('Query 5 and Shard 2 Complete'); + + $debug = true; + db_debug('Query 6 and Shard 1 Complete'); + + $debug = $saved; + + expect($GLOBALS['__test_cacti_log'])->toHaveCount(1) + ->and(implode("\n", $GLOBALS['__test_cacti_log']))->toContain('Query 6 and Shard 1 Complete'); + }); + it('removes the requests that have expired', function () { $GLOBALS['__test_config']['flowview_parallel_time_to_live'] = 21600; @@ -346,6 +377,20 @@ function plugin_test_run_packet_summary(int $packets): array { }); describe('a parallel query that does not finish in time', function () { + it('says so in the browser of a web page, and logs it', function () { + $GLOBALS['__test_javascript_messages'] = array(); + + ob_start(); + flowview_report_parallel_timeout(30, true); + $printed = ob_get_clean(); + + expect($printed)->toBe('') + ->and($GLOBALS['__test_javascript_messages'])->toHaveCount(1) + ->and($GLOBALS['__test_javascript_messages'][0][0])->toBe('parallel_query_timeout') + ->and($GLOBALS['__test_javascript_messages'][0][2])->toBe(MESSAGE_LEVEL_ERROR) + ->and(isset($GLOBALS['flowview_error_reported']))->toBeFalse() + ->and(implode("\n", $GLOBALS['__test_cacti_log']))->toContain('Maximum Run Time of 30 seconds'); + }); it('says so in words from a script, cleans up, and ends with a failure', function () { $GLOBALS['__test_config']['flowview_parallel_runlimit'] = 0; $GLOBALS['flowview_requests_in_flight'] = array(5 => 5);